Blame view
LDA/05-lts_scoring.py
2.69 KB
2af8e57f4
change all
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
import sys
import shelve
import pickle
from utils import *
import sys
import os
import json
import glob
import tempfile
import pandas
import subprocess
from subprocess import CalledProcessError
import shutil
import numpy
in_dir = sys.argv[1]
json_conf =json.load(open(sys.argv[2]))
name = json_conf["name"]
ae_m = shelve.open("{}/{}/ae_model.shelve".format(in_dir,name))
y_train=numpy.argmax(ae_m["LABEL"]["ASR"]["TRAIN"],axis=1)
_,ytr_path=tempfile.mkstemp()
ytr_open= open(ytr_path,"w")
for el in y_train:
print >>ytr_open, el
ytr_open.close()
y_dev=numpy.argmax(ae_m["LABEL"]["ASR"]["DEV"],axis=1)
_,yd_path=tempfile.mkstemp()
yd_open = open(yd_path,"w")
for el in y_dev:
print >>yd_open, el
yd_open.close()
y_test=numpy.argmax(ae_m["LABEL"]["ASR"]["TEST"],axis=1)
_,yte_path=tempfile.mkstemp()
yte_open=open(yte_path,"w")
for el in y_test:
print >>yte_open, el
yte_open.close()
hdfs_files=glob.glob("{}/{}/*.hdf".format(in_dir,name))
temp_dir=tempfile.mkdtemp()
out_file=open("{}/{}/malaha_res.txt".format(in_dir,name),"a")
for hdf in hdfs_files:
print >>out_file, "Start ---------------------------------------------------"
print >>out_file, hdf
x_train = pandas.read_hdf(hdf,"TRAIN")
x_train.to_csv("{}/xtrain.dat".format(temp_dir),sep=" ",header=False,index=False, index_label=False)
x_train = pandas.read_hdf(hdf,"DEV")
x_train.to_csv("{}/xdev.dat".format(temp_dir),sep=" ",header=False,index=False, index_label=False)
x_train = pandas.read_hdf(hdf,"TEST")
x_train.to_csv("{}/xtest.dat".format(temp_dir),sep=" ",header=False,index=False, index_label=False)
try :
resdev=subprocess.check_output(['Rscript',
'/home/laboinfo/janod/WorkingDir/erreur_traduction/Author_Topic_Decoda/estimate.R',
"{}/xtrain.dat".format(temp_dir),
"{}/xdev.dat".format(temp_dir),
ytr_path,yd_path])
restest=subprocess.check_output(['Rscript',
'/home/laboinfo/janod/WorkingDir/erreur_traduction/Author_Topic_Decoda/estimate.R',
"{}/xtrain.dat".format(temp_dir),
"{}/xtest.dat".format(temp_dir),
ytr_path,yte_path])
print >>out_file, resdev
print >>out_file, hdf
print >>out_file, restest
except CalledProcessError:
print >>out_file, "FAILED"
print >>out_file, hdf
print >>out_file, "End ---------------------------------------------------"
shutil.rmtree(temp_dir)
os.remove(ytr_path)
os.remove(yd_path)
os.remove(yte_path)
|