Killian / decodopr

Download as
Browse Code ยป

Commit e5108393c82cba7f29adcc24d27a17368a398e7e

Authored by Killian
1 parent dcac9f70ba
Exists in master

replace du mlp.py et ajout de la mmf_experience

Showing 13 changed files with 490 additions and 289 deletions Inline Diff

LDA/00-mmf_make_features.py
Diff comments   View file @ e510839
1 import sys 1 import sys
2 import os 2 import os
3 3
4 import pandas 4 import pandas
5 import numpy 5 import numpy
6 import shelve 6 import shelve
7 7
8 from sklearn.preprocessing import LabelBinarizer 8 from sklearn.preprocessing import LabelBinarizer
9 9
10 from utils import select_mmf as select 10 from utils import select_mmf as select
11 11
12 input_dir = sys.argv[1] # Dossier de premire niveau contient ASR et TRS 12 input_dir = sys.argv[1] # Dossier de premire niveau contient ASR et TRS
13 level = sys.argv[2] # taille de LDA ( -5) voulu 13 level = sys.argv[2] # taille de LDA ( -5) voulu
14 output_dir = sys.argv[3]
14 15
15 lb=LabelBinarizer() 16 lb=LabelBinarizer()
16 #y_train=lb.fit_transform([utils.select(ligneid) for ligneid in origin_corps["LABEL"]["TRAIN"]]) 17 #y_train=lb.fit_transform([utils.select(ligneid) for ligneid in origin_corps["LABEL"]["TRAIN"]])
17 18
18 19
19 data = shelve.open("{}/mmf_{}.shelve".format(input_dir,level)) 20 data = shelve.open("{}/mmf_{}.shelve".format(output_dir,level),writeback=True)
20 data["LABEL"]= {"LDA":{}} 21 data["LABEL"]= {}
21 for mod in ["ASR", "TRS" ] 22 data["LDA"] = {"ASR":{},"TRS":{}}
23 for mod in ["ASR", "TRS" ]:
22 train = pandas.read_table("{}/{}/train_{}.ssv".format(input_dir, mod, level), sep=" ", header=None ) 24 train = pandas.read_table("{}/{}/train_{}.ssv".format(input_dir, mod, level), sep=" ", header=None )
23 dev = pandas.read_table("{}/{}/dev_{}.ssv".format(input_dir, mod, level), sep=" ", header=None ) 25 dev = pandas.read_table("{}/{}/dev_{}.ssv".format(input_dir, mod, level), sep=" ", header=None )
24 test = pandas.read_table("{}/{}/test_{}.ssv".format(input_dir, mod, level), sep=" ", header=None ) 26 test = pandas.read_table("{}/{}/test_{}.ssv".format(input_dir, mod, level), sep=" ", header=None )
25 27
26 y_train = train.iloc[:,0].apply(select) 28 y_train = train.iloc[:,0].apply(select)
27 y_dev = dev.iloc[:,0].apply(select) 29 y_dev = dev.iloc[:,0].apply(select)
28 y_test = test.iloc[:,0].apply(select) 30 y_test = test.iloc[:,0].apply(select)
29 lb.fit(y_train) 31 lb.fit(y_train)
30 data["LABEL"][mod]={"TRAIN":lb.transform(y_train),"DEV":lb.transform(y_dev), "TEST": lb.transform(y_test)} 32 data["LABEL"][mod]={"TRAIN":lb.transform(y_train),"DEV":lb.transform(y_dev), "TEST": lb.transform(y_test)}
31 33
32 data["LDA"][mod]={} 34 # data["LDA"][mod]={'ASR':[]}
33 data["LDA"][mod]["TRAIN"]=train.iloc[:,1:].values 35 print data["LDA"][mod]
34 data["LDA"][mod]["DEV"]=dev.iloc[:,1:].values 36 print train.values
35 data["LDA"][mod]["TEST"]=test.iloc[:,1:].values 37 data["LDA"][mod]["TRAIN"]=train.iloc[:,1:-1].values
38 data["LDA"][mod]["DEV"]=dev.iloc[:,1:-1].values
39 data["LDA"][mod]["TEST"]=test.iloc[:,1:-1].values
36 40
37 data.sync() 41 data.sync()
38 data.close() 42 data.close()
39 43
1 1
2 # coding: utf-8 2 # coding: utf-8
3 3
4 # In[29]: 4 # In[29]:
5 5
6 # Import 6 # Import
7 import itertools 7 import itertools
8 import shelve 8 import shelve
9 import pickle 9 import pickle
10 import numpy 10 import numpy
11 import scipy 11 import scipy
12 from scipy import sparse 12 from scipy import sparse
13 import scipy.sparse 13 import scipy.sparse
14 import scipy.io 14 import scipy.io
15 from mlp import * 15 from mlp import *
16 import mlp 16 import mlp
17 import sys 17 import sys
18 import utils 18 import utils
19 import dill 19 import dill
20 from collections import Counter 20 from collections import Counter
21 from gensim.models import LdaModel 21 from gensim.models import LdaModel
22 22
23 23
24 24
25 # In[3]: 25 # In[3]:
26 26
27 #30_50_50_150_0.0001 27 #30_50_50_150_0.0001
28 28
29 # In[4]: 29 # In[4]:
30 30
31 #db=shelve.open("SPELIKE_MLP_DB.shelve",writeback=True) 31 #db=shelve.open("SPELIKE_MLP_DB.shelve",writeback=True)
32 origin_corps=shelve.open("{}".format(sys.argv[2])) 32 origin_corps=shelve.open("{}".format(sys.argv[2]))
33 in_dir = sys.argv[1] 33 in_dir = sys.argv[1]
34 34
35 35
36 out_db=shelve.open("{}/mlp_scores.shelve".format(in_dir),writeback=True) 36 out_db=shelve.open("{}/mlp_scores.shelve".format(in_dir),writeback=True)
37 37
38 mlp_h = [ 250, 250 ] 38 mlp_h = [ 250, 250 ]
39 mlp_loss = "categorical_crossentropy" 39 mlp_loss = "categorical_crossentropy"
40 mlp_dropouts = [0.25]* len(mlp_h) 40 mlp_dropouts = [0.25]* len(mlp_h)
41 mlp_sgd = Adam(lr=0.0001) 41 mlp_sgd = Adam(lr=0.0001)
42 mlp_epochs = 3000 42 mlp_epochs = 3000
43 mlp_batch_size = 1 43 mlp_batch_size = 1
44 mlp_input_activation = "relu" 44 mlp_input_activation = "relu"
45 mlp_output_activation="softmax" 45 mlp_output_activation="softmax"
46 46
47 ress = [] 47 ress = []
48 for key in ["TRS", "ASR"] : 48 for key in ["TRS", "ASR"] :
49 49
50 res=mlp.train_mlp(origin_corps["LDA"][key]["TRAIN"],origin_corps["LABEL"][key]["TRAIN"], 50 res=mlp.train_mlp(origin_corps["LDA"][key]["TRAIN"],origin_corps["LABEL"][key]["TRAIN"],
51 origin_corps["LDA"][key]["DEV"],origin_corps["LABEL"][key]["DEV"], 51 origin_corps["LDA"][key]["DEV"],origin_corps["LABEL"][key]["DEV"],
52 origin_corps["LDA"][key]["TEST"],origin_corps["LABEL"][key]["TEST"], 52 origin_corps["LDA"][key]["TEST"],origin_corps["LABEL"][key]["TEST"],
53 mlp_h,dropouts=mlp_dropouts,sgd=mlp_sgd, 53 mlp_h,dropouts=mlp_dropouts,sgd=mlp_sgd,
54 epochs=mlp_epochs, 54 epochs=mlp_epochs,
55 batch_size=mlp_batch_size, 55 batch_size=mlp_batch_size,
56 save_pred=False,keep_histo=False, 56 save_pred=False,keep_histo=False,
57 loss="categorical_crossentropy",fit_verbose=0) 57 loss="categorical_crossentropy",fit_verbose=0)
58 arg_best=[] 58 arg_best=[]
59 dev_best=[] 59 dev_best=[]
60 arg_best.append(numpy.argmax(res[1])) 60 arg_best.append(numpy.argmax(res[1]))
61 dev_best.append(res[1][arg_best[-1]]) 61 dev_best.append(res[1][arg_best[-1]])
62 res[1][arg_best[-1]]=0 62 res[1][arg_best[-1]]=0
63 arg_best.append(numpy.argmax(res[1])) 63 arg_best.append(numpy.argmax(res[1]))
64 dev_best.append(res[1][arg_best[-1]]) 64 dev_best.append(res[1][arg_best[-1]])
65 res[1][arg_best[-1]]=0 65 res[1][arg_best[-1]]=0
66 arg_best.append(numpy.argmax(res[1])) 66 arg_best.append(numpy.argmax(res[1]))
67 dev_best.append(res[1][arg_best[-1]]) 67 dev_best.append(res[1][arg_best[-1]])
68 res[1][arg_best[-1]]=0 68 res[1][arg_best[-1]]=0
69 arg_best.append(numpy.argmax(res[1])) 69 arg_best.append(numpy.argmax(res[1]))
70 dev_best.append(res[1][arg_best[-1]]) 70 dev_best.append(res[1][arg_best[-1]])
71 res[1][arg_best[-1]]=0 71 res[1][arg_best[-1]]=0
72 arg_best.append(numpy.argmax(res[1])) 72 arg_best.append(numpy.argmax(res[1]))
73 dev_best.append(res[1][arg_best[-1]]) 73 dev_best.append(res[1][arg_best[-1]])
74 res[1][arg_best[-1]]=0 74 res[1][arg_best[-1]]=0
75 arg_best.append(numpy.argmax(res[1])) 75 arg_best.append(numpy.argmax(res[1]))
76 dev_best.append(res[1][arg_best[-1]]) 76 dev_best.append(res[1][arg_best[-1]])
77 res[1][arg_best[-1]]=0 77 res[1][arg_best[-1]]=0
78 arg_best.append(numpy.argmax(res[1])) 78 arg_best.append(numpy.argmax(res[1]))
79 dev_best.append(res[1][arg_best[-1]]) 79 dev_best.append(res[1][arg_best[-1]])
80 res[1][arg_best[-1]]=0 80 res[1][arg_best[-1]]=0
81 arg_best.append(numpy.argmax(res[1])) 81 arg_best.append(numpy.argmax(res[1]))
82 dev_best.append(res[1][arg_best[-1]]) 82 dev_best.append(res[1][arg_best[-1]])
83 res[1][arg_best[-1]]=0 83 res[1][arg_best[-1]]=0
84 arg_best.append(numpy.argmax(res[1])) 84 arg_best.append(numpy.argmax(res[1]))
85 dev_best.append(res[1][arg_best[-1]]) 85 dev_best.append(res[1][arg_best[-1]])
86 res[1][arg_best[-1]]=0 86 res[1][arg_best[-1]]=0
87 arg_best.append(numpy.argmax(res[1])) 87 arg_best.append(numpy.argmax(res[1]))
88 dev_best.append(res[1][arg_best[-1]]) 88 dev_best.append(res[1][arg_best[-1]])
89 res[1][arg_best[-1]]=0 89 res[1][arg_best[-1]]=0
90 arg_best.append(numpy.argmax(res[1])) 90 arg_best.append(numpy.argmax(res[1]))
91 dev_best.append(res[1][arg_best[-1]]) 91 dev_best.append(res[1][arg_best[-1]])
92 res[1][arg_best[-1]]=0 92 res[1][arg_best[-1]]=0
93 arg_best.append(numpy.argmax(res[1])) 93 arg_best.append(numpy.argmax(res[1]))
94 dev_best.append(res[1][arg_best[-1]]) 94 dev_best.append(res[1][arg_best[-1]])
95 res[1][arg_best[-1]]=0 95 res[1][arg_best[-1]]=0
96 96
97 97
98 98
99 99
100 test_best =[ res[2][x] for x in arg_best ] 100 test_best =[ res[2][x] for x in arg_best ]
101 test_max = numpy.max(res[2]) 101 test_max = numpy.max(res[2])
102 out_db[key]=(res,(dev_best,test_best,test_max)) 102 out_db[key]=(res,(dev_best,test_best,test_max))
103 ress.append((key,dev_best,test_best,test_max)) 103 ress.append((key,dev_best,test_best,test_max))
104 104 print sys.argv[2]
105 for el in ress : 105 for el in ress :
106 print el 106 print el
107 out_db.close() 107 out_db.close()
108 origin_corps.close() 108 origin_corps.close()
109 109
LDA/04b-mmf_mini_ae.py
Diff comments   View file @ e510839
1 1
2 # coding: utf-8 2 # coding: utf-8
3 3
4 # In[2]: 4 # In[2]:
5 5
6 # Import 6 # Import
7 import gensim 7 import gensim
8 from scipy import sparse 8 from scipy import sparse
9 import itertools 9 import itertools
10 from sklearn import preprocessing 10 from sklearn import preprocessing
11 from keras.models import Sequential 11 from keras.models import Sequential
12 from keras.optimizers import SGD,Adam 12 from keras.optimizers import SGD,Adam
13 from mlp import * 13 from mlp import *
14 import sklearn.metrics 14 import sklearn.metrics
15 import shelve 15 import shelve
16 import pickle 16 import pickle
17 from utils import * 17 from utils import *
18 import sys 18 import sys
19 import os 19 import os
20 import json 20 import json
21 # In[4]: 21 # In[4]:
22 22
23 infer_model=shelve.open("{}".format(sys.argv[2])) 23 infer_model=shelve.open("{}".format(sys.argv[2]))
24 in_dir = sys.argv[1] 24 in_dir = sys.argv[1]
25 #['ASR', 'TRS', 'LABEL'] 25 #['ASR', 'TRS', 'LABEL']
26 # In[6]: 26 # In[6]:
27 27
28 json_conf =json.load(open(sys.argv[3]))
29 ae_conf = json_conf["ae"]
28 30
29 hidden_size=[ 100 , 50, 100 ] 31 hidden_size= ae_conf["hidden_size"]
30 input_activation="tanh" 32 input_activation=ae_conf["input_activation"]
31 output_activation="tanh" 33 output_activation=ae_conf["output_activation"]
32 loss="mse" 34 loss=ae_conf["loss"]
33 epochs=1000 35 epochs=ae_conf["epochs"]
34 batch=1 36 batch=ae_conf["batch"]
35 patience=60 37 patience=ae_conf["patience"]
36 do_do=[False] 38 do_do=ae_conf["do"]
37 sgd = Adam(lr=0.000001)#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True) 39 try:
40 k = ae_conf["sgd"]
41 if ae_conf["sgd"]["name"] == "adam":
42 sgd = Adam(lr=ae_conf["sgd"]["lr"])#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
43 elif ae_conf["sgd"]["name"] == "sgd":
44 sgd = SGD(lr=ae_conf["sgd"]["lr"])
45 except:
46 sgd = ae_conf["sgd"]
38 47
48 mlp_conf = json_conf["mlp"]
49 mlp_h = mlp_conf["hidden_size"]
50 mlp_loss = mlp_conf["loss"]
51 mlp_dropouts = mlp_conf["do"]
52 mlp_epochs = mlp_conf["epochs"]
53 mlp_batch_size = mlp_conf["batch"]
54 mlp_input_activation=mlp_conf["input_activation"]
55 mlp_output_activation=mlp_conf["output_activation"]
39 56
57 try:
58 k = mlp_conf["sgd"]
59 if mlp_conf["sgd"]["name"] == "adam":
60 mlp_sgd = Adam(lr=mlp_conf["sgd"]["lr"])#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
61 elif mlp_conf["sgd"]["name"] == "sgd":
62 mlp_sgd = SGD(lr=mlp_conf["sgd"]["lr"])
63 except:
64 mlp_sgd = mlp_conf["sgd"]
40 65
41 mlp_h = [ 150 ,150 ,150 ]
42 mlp_loss = "categorical_crossentropy"
43 mlp_dropouts = []
44 mlp_sgd = Adam(lr=0.0001)
45 mlp_epochs = 2000
46 mlp_batch_size = 8
47 mlp_output_activation="softmax"
48 66
49 try : 67 name = json_conf["name"]
50 sgd_repr=sgd.get_config()["name"]
51 except AttributeError :
52 sgd_repr=sgd
53
54 try :
55 mlp_sgd_repr=mlp_sgd.get_config()["name"]
56 except AttributeError :
57 mlp_sgd_repr=mlp_sgd
58
59
60 params={ "h1" : "_".join([ str(x) for x in hidden_size ]),
61 "inside_activation" : input_activation,
62 "output_activation" : output_activation,
63 "do_dropout": "_".join([str(x) for x in do_do]),
64 "loss" : loss,
65 "epochs" : epochs ,
66 "batch_size" : batch,
67 "patience" : patience,
68 "sgd" : sgd_repr,
69 "mlp_h ": "_".join([str(x) for x in mlp_h]),
70 "mlp_loss ": mlp_loss,
71 "mlp_dropouts ": "_".join([str(x) for x in mlp_dropouts]),
72 "mlp_sgd ": mlp_sgd_repr,
73 "mlp_epochs ": mlp_epochs,
74 "mlp_batch_size ": mlp_batch_size,
75 "mlp_output" : mlp_output_activation
76 }
77 name = "_".join([ str(x) for x in params.values()])
78 try: 68 try:
79 os.mkdir("{}/{}".format(in_dir,name)) 69 os.mkdir("{}/{}".format(in_dir,name))
80 except: 70 except:
81 pass 71 pass
82 db = shelve.open("{}/{}/ae_model.shelve".format(in_dir,name),writeback=True) 72 db = shelve.open("{}/{}/ae_model.shelve".format(in_dir,name),writeback=True)
83 db["params"] = params
84 db["LABEL"]=infer_model["LABEL"] 73 db["LABEL"]=infer_model["LABEL"]
85 # 74 #
86 json.dump(params,
87 open("{}/{}/ae_model.json".format(in_dir,name),"w"),
88 indent=4)
89
90 keys = ["ASR","TRS"] 75 keys = ["ASR","TRS"]
91 76
92 db["AE"] = {} 77 db["AE"] = {}
93 db["LDA"] = {} 78 db["LDA"] = {}
94 for mod in keys : 79 for mod in keys :
95 print mod
96 db["LDA"][mod] = train_mlp(infer_model["LDA"][mod]["TRAIN"],infer_model["LABEL"][mod]["TRAIN"], 80 db["LDA"][mod] = train_mlp(infer_model["LDA"][mod]["TRAIN"],infer_model["LABEL"][mod]["TRAIN"],
97 infer_model["LDA"][mod]["DEV"],infer_model["LABEL"][mod]["DEV"], 81 infer_model["LDA"][mod]["DEV"],infer_model["LABEL"][mod]["DEV"],
98 infer_model["LDA"][mod]["TEST"],infer_model["LABEL"][mod]["TEST"], 82 infer_model["LDA"][mod]["TEST"],infer_model["LABEL"][mod]["TEST"],
99 mlp_h ,sgd=mlp_sgd, 83 mlp_h ,sgd=mlp_sgd,
100 epochs=mlp_epochs, 84 epochs=mlp_epochs,
101 batch_size=mlp_batch_size, 85 batch_size=mlp_batch_size,
102 input_activation=input_activation, 86 input_activation=mlp_input_activation,
103 output_activation=mlp_output_activation, 87 output_activation=mlp_output_activation,
104 dropouts=mlp_dropouts, 88 dropouts=mlp_dropouts,
105 fit_verbose=0) 89 fit_verbose=0)
106 90
107 res=train_ae(infer_model["LDA"][mod]["TRAIN"],infer_model["LDA"][mod]["DEV"],infer_model["LDA"][mod]["TEST"], 91 res=train_ae(infer_model["LDA"][mod]["TRAIN"],infer_model["LDA"][mod]["DEV"],infer_model["LDA"][mod]["TEST"],
108 hidden_size,patience = params["patience"],sgd=sgd, 92 hidden_size,patience = patience,sgd=sgd,
109 dropouts=do_do,input_activation=input_activation,output_activation=output_activation, 93 dropouts=do_do,input_activation=input_activation,output_activation=output_activation,
110 loss=loss,epochs=epochs,batch_size=batch,verbose=0) 94 loss=loss,epochs=epochs,batch_size=batch,verbose=0)
111 mlp_res_list=[] 95 mlp_res_list=[]
112 for layer in res : 96 for layer in res :
113 mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"], 97 mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"],
114 layer[1],infer_model["LABEL"][mod]["DEV"], 98 layer[1],infer_model["LABEL"][mod]["DEV"],
115 layer[2],infer_model["LABEL"][mod]["TEST"], 99 layer[2],infer_model["LABEL"][mod]["TEST"],
116 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts,sgd=mlp_sgd,epochs=mlp_epochs, 100 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts,sgd=mlp_sgd,epochs=mlp_epochs,
117 output_activation=mlp_output_activation, 101 output_activation=mlp_output_activation,
118 input_activation=input_activation, 102 input_activation=mlp_input_activation,
119 batch_size=mlp_batch_size,fit_verbose=0)) 103 batch_size=mlp_batch_size,fit_verbose=0))
120 db["AE"][mod]=mlp_res_list 104 db["AE"][mod]=mlp_res_list
121 105
122 mod = "ASR" 106 mod = "ASR"
123 mod2= "TRS" 107 mod2= "TRS"
124 mlp_res_list=[] 108 mlp_res_list=[]
1 1
2 # coding: utf-8 2 # coding: utf-8
3 3
4 # In[2]: 4 # In[2]:
5 5
6 # Import 6 # Import
7 import gensim 7 import gensim
8 from scipy import sparse 8 from scipy import sparse
9 import itertools 9 import itertools
10 from sklearn import preprocessing 10 from sklearn import preprocessing
11 from keras.models import Sequential 11 from keras.models import Sequential
12 from keras.optimizers import SGD,Adam 12 from keras.optimizers import SGD,Adam
13 from mlp import * 13 from mlp import *
14 import mlp 14 import mlp
15 import sklearn.metrics 15 import sklearn.metrics
16 import shelve 16 import shelve
17 import pickle 17 import pickle
18 from utils import * 18 from utils import *
19 import sys 19 import sys
20 import os 20 import os
21 import json 21 import json
22 # In[4]: 22 # In[4]:
23 23
24 infer_model=shelve.open("{}".format(sys.argv[2])) 24 infer_model=shelve.open("{}".format(sys.argv[2]))
25 in_dir = sys.argv[1] 25 in_dir = sys.argv[1]
26 #['ASR', 'TRS', 'LABEL'] 26 #['ASR', 'TRS', 'LABEL']
27 # In[6]: 27 # In[6]:
28 json_conf =json.load(open(sys.argv[3]))
29 sae_conf = json_conf["sae"]
28 30
31 hidden_size= sae_conf["hidden_size"]
32 input_activation=sae_conf["input_activation"]
33 output_activation=sae_conf["output_activation"]
34 loss=sae_conf["loss"]
35 epochs=sae_conf["epochs"]
36 batch=sae_conf["batch"]
37 patience=sae_conf["patience"]
38 do_do=sae_conf["do"]
29 39
30 hidden_size=[ 100, 80, 50 , 20 ] 40 try:
31 input_activation="relu" 41 k = sae_conf["sgd"]
32 output_activation="relu" 42 if sae_conf["sgd"]["name"] == "adam":
33 loss="mse" 43 sgd = Adam(lr=sae_conf["sgd"]["lr"])
34 epochs=3000 44 elif sae_conf["sgd"]["name"] == "sgd":
35 batch=1 45 sgd = SGD(lr=sae_conf["sgd"]["lr"])
36 patience=20 46 except :
37 do_do=[ 0 ] * len(hidden_size) 47 sgd = sae_conf["sgd"]
38 sgd = Adam(lr=0.0001)#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
39 try :
40 sgd_repr=sgd.get_config()["name"]
41 except AttributeError :
42 sgd_repr=sgd
43 48
44 params={ "h1" : "_".join([str(x) for x in hidden_size]), 49 name = json_conf["name"]
45 "inside_activation" : input_activation,
46 "out_activation" : output_activation,
47 "do_dropout": "_".join([str(x) for x in do_do]),
48 "loss" : loss,
49 "epochs" : epochs ,
50 "batch_size" : batch,
51 "patience" : patience,
52 "sgd" : sgd_repr}
53 name = "_".join([ str(x) for x in params.values()])
54 try: 50 try:
55 os.mkdir("{}/SAE_{}".format(in_dir,name)) 51 os.mkdir("{}/{}".format(in_dir,name))
56 except: 52 except:
57 pass 53 pass
58 db = shelve.open("{}/SAE_{}/ae_model.shelve".format(in_dir,name),writeback=True) 54 db = shelve.open("{}/{}/ae_model.shelve".format(in_dir,name),writeback=True)
59 # 55 #
60 json.dump(params,
61 open("{}/SAE_{}/ae_model.json".format(in_dir,name),"w"),
62 indent=4)
63
64 keys = ["ASR","TRS"] 56 keys = ["ASR","TRS"]
57 mlp_conf = json_conf["mlp"]
58 mlp_h = mlp_conf["hidden_size"]
59 mlp_loss = mlp_conf["loss"]
60 mlp_dropouts = mlp_conf["do"]
61 mlp_epochs = mlp_conf["epochs"]
62 mlp_batch_size = mlp_conf["batch"]
63 mlp_input_activation=mlp_conf["input_activation"]
64 mlp_output_activation=mlp_conf["output_activation"]
65 65
66 mlp_h = [ 150 , 300 ] 66 try:
67 mlp_loss ="categorical_crossentropy" 67 k = mlp_conf["sgd"]
68 mlp_dropouts = [0,0,0,0] 68 if mlp_conf["sgd"]["name"] == "adam":
69 mlp_sgd = Adam(0.001) 69 mlp_sgd = Adam(lr=mlp_conf["sgd"]["lr"])
70 mlp_epochs = 2000 70 elif mlp_conf["sgd"]["name"] == "sgd" :
71 mlp_batch_size = 8 71 mlp_sgd = SGD(lr=mlp_conf["sgd"]["lr"])
72 except :
73 mlp_sgd = mlp_conf["sgd"]
72 74
75
73 db["SAE"] = {} 76 db["SAE"] = {}
74 77
75 db["SAEFT"] = {} 78 db["SAEFT"] = {}
76 for mod in keys : 79 for mod in keys :
77 print "MODE ", mod
78 res_tuple=train_sae(infer_model["LDA"][mod]["TRAIN"],infer_model["LDA"][mod]["DEV"], 80 res_tuple=train_sae(infer_model["LDA"][mod]["TRAIN"],infer_model["LDA"][mod]["DEV"],
79 infer_model["LDA"][mod]["TEST"], 81 infer_model["LDA"][mod]["TEST"],
80 hidden_size,dropouts=do_do, 82 hidden_size,dropouts=do_do,
81 patience = params["patience"],sgd=sgd,input_activation="tanh", 83 patience = "patience",sgd=sgd,input_activation="tanh",
82 output_activation="tanh",loss=loss,epochs=epochs, 84 output_activation="tanh",loss=loss,epochs=epochs,
83 batch_size=batch,verbose=0) 85 batch_size=batch,verbose=0)
84 #print len(res), [len(x) for x in res[0]], [ len(x) for x in res[1]] 86 #print len(res), [len(x) for x in res[0]], [ len(x) for x in res[1]]
85 for name , levels in zip(["SAE","SAEFT"],res_tuple): 87 for name , levels in zip(["SAE","SAEFT"],res_tuple):
86 print "NAME", name
87 mlp_res_by_level = [] 88 mlp_res_by_level = []
88 for res in levels: 89 for res in levels:
89 mlp_res_list=[] 90 mlp_res_list=[]
90 for nb,layer in enumerate(res) : 91 for nb,layer in enumerate(res) :
91 print "layer NB",nb
92 mlp_res_list.append(train_mlp(layer[0],infer_model["LABEL"][mod]["TRAIN"], 92 mlp_res_list.append(train_mlp(layer[0],infer_model["LABEL"][mod]["TRAIN"],
93 layer[1],infer_model["LABEL"][mod]["DEV"], 93 layer[1],infer_model["LABEL"][mod]["DEV"],
94 layer[2],infer_model["LABEL"][mod]["TEST"], 94 layer[2],infer_model["LABEL"][mod]["TEST"],
95 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts, 95 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts,
96 sgd=mlp_sgd,epochs=mlp_epochs,batch_size=mlp_batch_size, 96 sgd=mlp_sgd,epochs=mlp_epochs,batch_size=mlp_batch_size,
97 fit_verbose=0)) 97 fit_verbose=0))
98 mlp_res_by_level.append(mlp_res_list) 98 mlp_res_by_level.append(mlp_res_list)
99 db[name][mod]=mlp_res_by_level 99 db[name][mod]=mlp_res_by_level
100 100
101 mod = "ASR" 101 mod = "ASR"
102 mod2= "TRS" 102 mod2= "TRS"
103 print "mode SPE "
104 res_tuple = train_sae(infer_model["LDA"][mod]["TRAIN"], 103 res_tuple = train_sae(infer_model["LDA"][mod]["TRAIN"],
105 infer_model["LDA"][mod]["DEV"], 104 infer_model["LDA"][mod]["DEV"],
106 infer_model["LDA"][mod]["TEST"], 105 infer_model["LDA"][mod]["TEST"],
107 hidden_size,dropouts=[0],patience=params["patience"], 106 hidden_size,dropouts=[0],patience="patience",
108 sgd=sgd,input_activation=input_activation,output_activation=input_activation, 107 sgd=sgd,input_activation=input_activation,output_activation=input_activation,
1 1
2 # coding: utf-8 2 # coding: utf-8
3 3
4 # In[2]: 4 # In[2]:
5 5
6 # Import 6 # Import
7 import gensim 7 import gensim
8 from scipy import sparse 8 from scipy import sparse
9 import itertools 9 import itertools
10 from sklearn import preprocessing 10 from sklearn import preprocessing
11 from keras.models import Sequential 11 from keras.models import Sequential
12 from keras.optimizers import SGD,Adam 12 from keras.optimizers import SGD,Adam
13 from mlp import * 13 from mlp import *
14 import mlp 14 import mlp
15 import sklearn.metrics 15 import sklearn.metrics
16 import shelve 16 import shelve
17 import pickle 17 import pickle
18
18 from utils import * 19 from utils import *
19 import sys 20 import sys
20 import os 21 import os
21 import json 22 import json
22 # In[4]: 23 # In[4]:
23 24
24 infer_model=shelve.open("{}".format(sys.argv[2])) 25 infer_model=shelve.open("{}".format(sys.argv[2]))
25 in_dir = sys.argv[1] 26 in_dir = sys.argv[1]
26 #['ASR', 'TRS', 'LABEL'] 27 #['ASR', 'TRS', 'LABEL']
27 # In[6]: 28 # In[6]:
28 29
29 # AE params 30 json_conf =json.load(open(sys.argv[3]))
30 hidden_size=[ 100, 100 ]
31 input_activation="relu"
32 output_activation="relu"
33 loss="mse"
34 epochs= 1000
35 batch_size=1
36 patience=20
37 do_do=[ 0.25 ] * len(hidden_size)
38 sgd = Adam(lr=0.00001)#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
39 try :
40 sgd_repr=sgd.get_config()["name"]
41 except AttributeError :
42 sgd_repr=sgd
43 31
44 # Transforme : 32 dsae_conf = json_conf["dsae"]
45 trans_hidden_size=[ 300 , 300 ]
46 trans_input_activation="relu"
47 trans_output_activation="relu"
48 trans_loss="mse"
49 trans_epochs=1000
50 trans_batch_size=8
51 trans_patience=20
52 trans_do=[ 0.25 ] * len(trans_hidden_size)
53 trans_sgd = Adam(lr=0.0001)#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
54 try :
55 trans_sgd_repr=trans_sgd.get_config()["name"]
56 except AttributeError :
57 trans_sgd_repr=trans_sgd
58 33
34 hidden_size= dsae_conf["hidden_size"]
35 input_activation=dsae_conf["input_activation"]
36 output_activation=dsae_conf["output_activation"]
37 loss=dsae_conf["loss"]
38 epochs=dsae_conf["epochs"]
39 batch_size=dsae_conf["batch"]
40 patience=dsae_conf["patience"]
41 do_do=dsae_conf["do"]
42 try:
43 k = dsae_conf["sgd"]
44 if dsae_conf["sgd"]["name"] == "adam":
45 sgd = Adam(lr=dsae_conf["sgd"]["lr"])#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
46 elif dsae_conf["sgd"]["name"] == "sgd":
47 sgd = SGD(lr=dsae_conf["sgd"]["lr"])
48 except:
49 sgd = dsae_conf["sgd"]
59 50
51 trans_conf = json_conf["dsae"]["transform"]
52 trans_hidden_size=trans_conf["hidden_size"]
53 trans_input_activation=trans_conf["input_activation"]
54 trans_output_activation=trans_conf["output_activation"]
55 trans_loss=trans_conf["loss"]
56 trans_epochs=trans_conf["epochs"]
57 trans_batch_size=trans_conf["batch"]
58 trans_patience=trans_conf["patience"]
59 trans_do=trans_conf["do"]
60 try:
61 k = trans_conf["sgd"]
62 if trans_conf["sgd"]["name"] == "adam":
63 trans_sgd = Adam(lr=trans_conf["sgd"]["lr"])#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
64 elif trans_conf["sgd"]["name"] == "sgd":
65 trans_sgd = SGD(lr=trans_conf["sgd"]["lr"])
66 except e :
67 trans_sgd = trans_conf["sgd"]
60 68
61 ae={ "h1" : "_".join([str(x) for x in hidden_size]),
62 "inside_activation" : input_activation,
63 "out_activation" : output_activation,
64 "do_dropout": "_".join([str(x) for x in do_do]),
65 "loss" : loss,
66 "epochs" : epochs ,
67 "batch_size" : batch_size,
68 "patience" : patience,
69 "sgd" : sgd_repr}
70 name = "_".join([ str(x) for x in ae.values()])
71 69
72 trans={ "h1" : "_".join([str(x) for x in trans_hidden_size]), 70 mlp_conf = json_conf["mlp"]
73 "inside_activation" : trans_input_activation, 71 mlp_h = mlp_conf["hidden_size"]
74 "out_activation" : trans_output_activation, 72 mlp_loss = mlp_conf["loss"]
75 "do_dropout": "_".join([str(x) for x in trans_do]), 73 mlp_dropouts = mlp_conf["do"]
76 "loss" : trans_loss, 74 mlp_epochs = mlp_conf["epochs"]
77 "epochs" : trans_epochs , 75 mlp_batch_size = mlp_conf["batch"]
78 "batch_size" : trans_batch_size, 76 mlp_input_activation=mlp_conf["input_activation"]
79 "patience" : trans_patience, 77 mlp_output_activation=mlp_conf["output_activation"]
80 "sgd" : trans_sgd_repr} 78 try:
79 k = mlp_conf["sgd"]
80 if mlp_conf["sgd"]["name"] == "adam":
81 mlp_sgd = Adam(lr=mlp_conf["sgd"]["lr"])#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
82 elif mlp_conf["sgd"]["name"] == "sgd":
83 mlp_sgd = SGD(lr=mlp_conf["sgd"]["lr"])
84 except:
85 mlp_sgd = mlp_conf["sgd"]
81 86
82 mlp_h = [ 300 , 300 ]
83 mlp_loss ="categorical_crossentropy"
84 mlp_dropouts = [0,0,0,0]
85 mlp_sgd = Adam(0.0001)
86 mlp_epochs = 1000
87 mlp_batch_size = 8
88 mlp_input_activation = "relu"
89 mlp_output_activation = "softmax"
90 87
91 try : 88 name = json_conf["name"]
92 mlp_sgd_repr=mlp_sgd.get_config()["name"]
93 except AttributeError :
94 mlp_sgd_repr=mlp_sgd
95
96
97
98 mlp={ "h1" : "_".join([str(x) for x in mlp_h ]),
99 "inside_activation" : mlp_input_activation,
100 "out_activation" : mlp_output_activation,
101 "do_dropout": "_".join([str(x) for x in mlp_dropouts]),
102 "loss" : mlp_loss,
103 "epochs" : mlp_epochs ,
104 "batch_size" : mlp_batch_size,
105 "sgd" : mlp_sgd_repr}
106
107 params = { "ae":ae, "trans":trans, "mlp":mlp}
108 try: 89 try:
109 os.mkdir("{}/DSAE_{}".format(in_dir,name)) 90 os.mkdir("{}/{}".format(in_dir,name))
110 except: 91 except:
111 pass 92 pass
112 db = shelve.open("{}/DSAE_{}/ae_model.shelve".format(in_dir,name),writeback=True) 93 db = shelve.open("{}/{}/ae_model.shelve".format(in_dir,name),writeback=True)
113 # 94 #
114 json.dump(params,
115 open("{}/DSAE_{}/ae_model.json".format(in_dir,name),"w"),
116 indent=4)
117 95
118 keys = ["ASR","TRS"] 96 keys = ["ASR","TRS"]
119 97
120 98
121 99
122 db["DSAE"] = {} 100 db["DSAE"] = {}
123 101
124 db["DSAEFT"] = {} 102 db["DSAEFT"] = {}
125 mod = "ASR" 103 mod = "ASR"
126 res_tuple_ASR = train_ae(infer_model["LDA"][mod]["TRAIN"], 104 res_tuple_ASR = train_ae(infer_model["LDA"][mod]["TRAIN"],
127 infer_model["LDA"][mod]["DEV"], 105 infer_model["LDA"][mod]["DEV"],
128 infer_model["LDA"][mod]["TEST"], 106 infer_model["LDA"][mod]["TEST"],
129 hidden_size,dropouts=do_do, 107 hidden_size,dropouts=do_do,
130 patience = patience,sgd=sgd, 108 patience = patience,sgd=sgd,
131 input_activation=input_activation, 109 input_activation=input_activation,
132 output_activation=output_activation,loss=loss,epochs=epochs, 110 output_activation=output_activation,loss=loss,epochs=epochs,
133 batch_size=batch_size,verbose=0,get_weights=True) 111 batch_size=batch_size,verbose=0,get_weights=True)
134 mlp_res_list = [] 112 mlp_res_list = []
135 for layer in res_tuple_ASR[0]: 113 for layer in res_tuple_ASR[0]:
136 mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"], 114 mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"],
137 layer[1],infer_model["LABEL"][mod]["DEV"], 115 layer[1],infer_model["LABEL"][mod]["DEV"],
138 layer[2],infer_model["LABEL"][mod]["TEST"], 116 layer[2],infer_model["LABEL"][mod]["TEST"],
139 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts, 117 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts,
140 sgd=mlp_sgd,epochs=mlp_epochs, 118 sgd=mlp_sgd,epochs=mlp_epochs,
141 output_activation=mlp_output_activation, 119 output_activation=mlp_output_activation,
142 input_activation=mlp_input_activation, 120 input_activation=mlp_input_activation,
143 batch_size=mlp_batch_size,fit_verbose=0)) 121 batch_size=mlp_batch_size,fit_verbose=0))
144 122
145 db["DSAE"][mod] = mlp_res_list 123 db["DSAE"][mod] = mlp_res_list
146 mod = "TRS" 124 mod = "TRS"
147 print hidden_size
148 res_tuple_TRS = train_ae(infer_model["LDA"][mod]["TRAIN"], 125 res_tuple_TRS = train_ae(infer_model["LDA"][mod]["TRAIN"],
149 infer_model["LDA"][mod]["DEV"], 126 infer_model["LDA"][mod]["DEV"],
150 infer_model["LDA"][mod]["TEST"], 127 infer_model["LDA"][mod]["TEST"],
151 hidden_size,dropouts=do_do, 128 hidden_size,dropouts=do_do,
152 sgd=sgd,input_activation=input_activation, 129 sgd=sgd,input_activation=input_activation,
153 output_activation=output_activation,loss=loss,epochs=epochs, 130 output_activation=output_activation,loss=loss,epochs=epochs,
154 batch_size=batch_size,patience=patience, 131 batch_size=batch_size,patience=patience,
155 verbose=0,get_weights=True) 132 verbose=0,get_weights=True)
156 133
157 mlp_res_list = [] 134 mlp_res_list = []
158 for layer in res_tuple_TRS[0]: 135 for layer in res_tuple_TRS[0]:
159 mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"], 136 mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"],
160 layer[1],infer_model["LABEL"][mod]["DEV"], 137 layer[1],infer_model["LABEL"][mod]["DEV"],
161 layer[2],infer_model["LABEL"][mod]["TEST"], 138 layer[2],infer_model["LABEL"][mod]["TEST"],
162 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts, 139 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts,
163 sgd=mlp_sgd,epochs=mlp_epochs, 140 sgd=mlp_sgd,epochs=mlp_epochs,
164 output_activation=mlp_output_activation, 141 output_activation=mlp_output_activation,
165 input_activation=mlp_input_activation, 142 input_activation=mlp_input_activation,
166 batch_size=mlp_batch_size,fit_verbose=0)) 143 batch_size=mlp_batch_size,fit_verbose=0))
167 144
168 db["DSAE"][mod] = mlp_res_list 145 db["DSAE"][mod] = mlp_res_list
169 146
170 147
171 148
172 transfert = [] 149 transfert = []
173 150
174 print " get weight trans" 151 print " get weight trans"
175 152
176 for asr_pred, trs_pred in zip(res_tuple_ASR[0], res_tuple_TRS[0]): 153 #for asr_pred, trs_pred in zip(res_tuple_ASR[0], res_tuple_TRS[0]):
177 print "ASR", [ x.shape for x in asr_pred] 154 # print "ASR", [ x.shape for x in asr_pred]
178 155
179 print "TRS", [ x.shape for x in trs_pred] 156 # print "TRS", [ x.shape for x in trs_pred]
180 print
181 157
182 for asr_pred, trs_pred in zip(res_tuple_ASR[0], res_tuple_TRS[0]): 158 for asr_pred, trs_pred in zip(res_tuple_ASR[0], res_tuple_TRS[0]):
183 print "ASR", [ x.shape for x in asr_pred] 159 # print "ASR", [ x.shape for x in asr_pred]
184 160
185 print "TRS", [ x.shape for x in trs_pred] 161 # print "TRS", [ x.shape for x in trs_pred]
162 # print " TRANS SGD", trans_sgd
186 transfert.append( train_ae(asr_pred[0], 163 transfert.append( train_ae(asr_pred[0],
187 asr_pred[1], 164 asr_pred[1],
188 asr_pred[2], 165 asr_pred[2],
189 trans_hidden_size, 166 trans_hidden_size,
190 dropouts=trans_do, 167 dropouts=trans_do,
191 y_train = trs_pred[0], 168 y_train = trs_pred[0],
192 y_dev=trs_pred[1], 169 y_dev=trs_pred[1],
193 y_test = trs_pred[2], 170 y_test = trs_pred[2],
194 patience = trans_patience,sgd=trans_sgd, 171 patience = trans_patience,sgd=trans_sgd,
195 input_activation=trans_input_activation, 172 input_activation=trans_input_activation,
196 output_activation=trans_output_activation, 173 output_activation=trans_output_activation,
197 loss=trans_loss, 174 loss=trans_loss,
198 epochs=trans_epochs, 175 epochs=trans_epochs,
199 batch_size=trans_batch_size,verbose=0,get_weights=True) ) 176 batch_size=trans_batch_size,verbose=0,get_weights=True) )
200 mod = "ASR" 177 mod = "ASR"
201 mlp_res_bylvl = [] 178 mlp_res_bylvl = []
202 print " MLP on transfert " 179 print " MLP on transfert "
203 for level, w in transfert : 180 for level, w in transfert :
204 mlp_res_list = [] 181 mlp_res_list = []
205 for layer in level : 182 for layer in level :
206 mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"], 183 mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"],
207 layer[1],infer_model["LABEL"][mod]["DEV"], 184 layer[1],infer_model["LABEL"][mod]["DEV"],
208 layer[2],infer_model["LABEL"][mod]["TEST"], 185 layer[2],infer_model["LABEL"][mod]["TEST"],
209 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts, 186 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts,
210 sgd=mlp_sgd,epochs=mlp_epochs, 187 sgd=mlp_sgd,epochs=mlp_epochs,
211 output_activation=mlp_output_activation, 188 output_activation=mlp_output_activation,
212 input_activation=mlp_input_activation, 189 input_activation=mlp_input_activation,
213 batch_size=mlp_batch_size,fit_verbose=0)) 190 batch_size=mlp_batch_size,fit_verbose=0))
214 mlp_res_bylvl.append(mlp_res_list) 191 mlp_res_bylvl.append(mlp_res_list)
215 db["DSAE"]["transfert"] = mlp_res_bylvl 192 db["DSAE"]["transfert"] = mlp_res_bylvl
216 193
217 194
218 print " FT " 195 print " FT "
219 WA = res_tuple_ASR[1] 196 WA = res_tuple_ASR[1]
220 print "WA", len(WA), [ len(x) for x in WA] 197 #print "WA", len(WA), [ len(x) for x in WA]
221 WT = res_tuple_TRS[1] 198 WT = res_tuple_TRS[1]
222 199
223 print "WT", len(WT), [ len(x) for x in WT] 200 #print "WT", len(WT), [ len(x) for x in WT]
224 Wtr = [ x[1] for x in transfert] 201 Wtr = [ x[1] for x in transfert]
1 1
2 # coding: utf-8 2 # coding: utf-8
3
4 # In[2]:
5
6 # Import
7 import gensim 3 import gensim
8 from scipy import sparse 4 from scipy import sparse
9 import itertools 5 import itertools
10 from sklearn import preprocessing 6 from sklearn import preprocessing
11 from keras.models import Sequential 7 from keras.models import Sequential
12 from keras.optimizers import SGD,Adam 8 from keras.optimizers import SGD,Adam
13 from mlp import * 9 from mlp import *
14 from vae import * 10 from vae import *
15 import sklearn.metrics 11 import sklearn.metrics
16 import shelve 12 import shelve
17 import pickle 13 import pickle
18 from utils import * 14 from utils import *
19 import sys 15 import sys
20 import os 16 import os
21 import json 17 import json
22 # In[4]: 18 # In[4]:
23 19
24 infer_model=shelve.open("{}".format(sys.argv[2])) 20 infer_model=shelve.open("{}".format(sys.argv[2]))
25 in_dir = sys.argv[1] 21 in_dir = sys.argv[1]
26 #['ASR', 'TRS', 'LABEL'] 22 #['ASR', 'TRS', 'LABEL']
27 # In[6]: 23 # In[6]:
28 24
25 json_conf =json.load(open(sys.argv[3]))
26 vae_conf = json_conf["vae"]
29 27
30 hidden_size= [60] 28 hidden_size= vae_conf["hidden_size"]
31 input_activation="tanh" 29 input_activation=vae_conf["input_activation"]
32 output_activation="sigmoid" 30 output_activation=vae_conf["output_activation"]
33 epochs=300 31 epochs=vae_conf["epochs"]
34 batch=1 32 batch=vae_conf["batch"]
35 patience=60 33 patience=vae_conf["patience"]
36 sgd = Adam(lr=0.0001)#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True) 34 latent_dim = vae_conf["latent"]
37 latent_dim = 30 35 try:
36 k = vae_conf["sgd"]
37 if vae_conf["sgd"]["name"] == "adam":
38 sgd = Adam(lr=vae_conf["sgd"]["lr"])#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
39 elif vae_conf["sgd"]["name"] == "sgd":
40 sgd = SGD(lr=vae_conf["sgd"]["lr"])
41 except:
42 sgd = vae_conf["sgd"]
38 43
44 mlp_conf = json_conf["mlp"]
45 mlp_h = mlp_conf["hidden_size"]
46 mlp_loss = mlp_conf["loss"]
47 mlp_dropouts = mlp_conf["do"]
48 mlp_epochs = mlp_conf["epochs"]
49 mlp_batch_size = mlp_conf["batch"]
50 mlp_input_activation=mlp_conf["input_activation"]
51 mlp_output_activation=mlp_conf["output_activation"]
39 52
40 53
41 mlp_h = [ 256 ] 54 try:
42 mlp_loss = "categorical_crossentropy" 55 k = mlp_conf["sgd"]
43 mlp_dropouts = [] 56 if mlp_conf["sgd"]["name"] == "adam":
44 mlp_sgd = Adam(lr=0.001) 57 mlp_sgd = Adam(lr=mlp_conf["sgd"]["lr"])#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
45 mlp_epochs = 1000 58 elif mlp_conf["sgd"]["name"] == "sgd":
46 mlp_batch_size = 16 59 mlp_sgd = SGD(lr=mlp_conf["sgd"]["lr"])
47 mlp_output_activation="softmax" 60 except:
61 mlp_sgd = mlp_conf["sgd"]
48 62
49 try :
50 sgd_repr=sgd.get_config()["name"]
51 except AttributeError :
52 sgd_repr=sgd
53 63
54 try : 64 name = json_conf["name"]
55 mlp_sgd_repr=mlp_sgd.get_config()["name"]
56 except AttributeError :
57 mlp_sgd_repr=mlp_sgd
58 65
59 66
60 params={ "h1" : "_".join([ str(x) for x in hidden_size ]),
61 "inside_activation" : input_activation,
62 "output_activation" : output_activation,
63 "epochs" : epochs ,
64 "batch_size" : batch,
65 "patience" : patience,
66 "sgd" : sgd_repr,
67 "mlp_h ": "_".join([str(x) for x in mlp_h]),
68 "mlp_loss ": mlp_loss,
69 "mlp_dropouts ": "_".join([str(x) for x in mlp_dropouts]),
70 "mlp_sgd ": mlp_sgd_repr,
71 "mlp_epochs ": mlp_epochs,
72 "mlp_batch_size ": mlp_batch_size,
73 "mlp_output" : mlp_output_activation
74 }
75 name = "_".join([ str(x) for x in params.values()])
76 try: 67 try:
77 os.mkdir("{}/VAE_{}".format(in_dir,name)) 68 os.mkdir("{}/{}".format(in_dir,name))
78 except: 69 except:
79 pass 70 pass
80 db = shelve.open("{}/VAE_{}/ae_model.shelve".format(in_dir,name),writeback=True) 71
81 db["params"] = params 72
73 db = shelve.open("{}/{}/ae_model.shelve".format(in_dir,name),writeback=True)
82 db["LABEL"]=infer_model["LABEL"] 74 db["LABEL"]=infer_model["LABEL"]
83 # 75 #
84 json.dump(params,
85 open("{}/VAE_{}/ae_model.json".format(in_dir,name),"w"),
86 indent=4)
87 76
88 keys = ["ASR","TRS"] 77 keys = ["ASR","TRS"]
89 78
90 db["VAE"] = {} 79 db["VAE"] = {}
91 db["LDA"] = {} 80 db["LDA"] = {}
92 for mod in keys : 81 for mod in keys :
93 print mod 82 #print mod
94 db["LDA"][mod] = train_mlp(infer_model["LDA"][mod]["TRAIN"],infer_model["LABEL"][mod]["TRAIN"], 83 db["LDA"][mod] = train_mlp(infer_model["LDA"][mod]["TRAIN"],infer_model["LABEL"][mod]["TRAIN"],
95 infer_model["LDA"][mod]["DEV"],infer_model["LABEL"][mod]["DEV"], 84 infer_model["LDA"][mod]["DEV"],infer_model["LABEL"][mod]["DEV"],
96 infer_model["LDA"][mod]["TEST"],infer_model["LABEL"][mod]["TEST"], 85 infer_model["LDA"][mod]["TEST"],infer_model["LABEL"][mod]["TEST"],
97 mlp_h ,sgd=mlp_sgd, 86 mlp_h ,sgd=mlp_sgd,
98 epochs=mlp_epochs, 87 epochs=mlp_epochs,
99 batch_size=mlp_batch_size, 88 batch_size=mlp_batch_size,
100 input_activation=input_activation, 89 input_activation=input_activation,
101 output_activation=mlp_output_activation, 90 output_activation=mlp_output_activation,
102 dropouts=mlp_dropouts, 91 dropouts=mlp_dropouts,
103 fit_verbose=0) 92 fit_verbose=0)
104 93
105 res=train_vae(infer_model["LDA"][mod]["TRAIN"],infer_model["LDA"][mod]["DEV"],infer_model["LDA"][mod]["TEST"], 94 res=train_vae(infer_model["LDA"][mod]["TRAIN"],infer_model["LDA"][mod]["DEV"],infer_model["LDA"][mod]["TEST"],
106 hidden_size=hidden_size[0], 95 hidden_size=hidden_size[0],
107 latent_dim=latent_dim,sgd=sgd, 96 latent_dim=latent_dim,sgd=sgd,
108 input_activation=input_activation,output_activation=output_activation, 97 input_activation=input_activation,output_activation=output_activation,
109 nb_epochs=epochs,batch_size=batch) 98 nb_epochs=epochs,batch_size=batch)
110 mlp_res_list=[] 99 mlp_res_list=[]
111 for layer in res : 100 for layer in res :
112 mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"], 101 mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"],
113 layer[1],infer_model["LABEL"][mod]["DEV"], 102 layer[1],infer_model["LABEL"][mod]["DEV"],
114 layer[2],infer_model["LABEL"][mod]["TEST"], 103 layer[2],infer_model["LABEL"][mod]["TEST"],
115 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts,sgd=mlp_sgd,epochs=mlp_epochs, 104 mlp_h,loss=mlp_loss,dropouts=mlp_dropouts,sgd=mlp_sgd,epochs=mlp_epochs,
116 output_activation=mlp_output_activation, 105 output_activation=mlp_output_activation,
117 input_activation=input_activation, 106 input_activation=input_activation,
118 batch_size=mlp_batch_size,fit_verbose=0)) 107 batch_size=mlp_batch_size,fit_verbose=0))
119 db["VAE"][mod]=mlp_res_list 108 db["VAE"][mod]=mlp_res_list
120 109
121 mod = "ASR" 110 mod = "ASR"
122 mod2= "TRS" 111 mod2= "TRS"
123 mlp_res_list=[] 112 mlp_res_list=[]
124 113
125 res = train_vae(infer_model["LDA"][mod]["TRAIN"], 114 res = train_vae(infer_model["LDA"][mod]["TRAIN"],
126 infer_model["LDA"][mod]["DEV"], 115 infer_model["LDA"][mod]["DEV"],
127 infer_model["LDA"][mod]["TEST"], 116 infer_model["LDA"][mod]["TEST"],
128 hidden_size=hidden_size[0], 117 hidden_size=hidden_size[0],
129 sgd=sgd,input_activation=input_activation,output_activation=output_activation, 118 sgd=sgd,input_activation=input_activation,output_activation=output_activation,
130 latent_dim=latent_dim, 119 latent_dim=latent_dim,
131 nb_epochs=epochs, 120 nb_epochs=epochs,
1 ../VARIATIONAL/Variational-Autoencoder/mlp.py File was deleted
1 # -*- coding: utf-8 -*-
2 import keras
3 import numpy as np
4 #from keras.layers.core import Dense, Dropout, Activation
5 from keras.optimizers import SGD,Adam
6 from keras.models import Sequential
7 from keras.layers import Input, Dense, Dropout
8 from keras.models import Model
9 from keras.utils.layer_utils import layer_from_config
10 from itertools import izip_longest
11
12 import pandas
13 from collections import namedtuple
14 from sklearn.metrics import accuracy_score as perf
15 save_tuple= namedtuple("save_tuple",["pred_train","pred_dev","pred_test"])
16
17
18 def ft_dsae(train,dev,test,
19 y_train=None,y_dev=None,y_test=None,
20 ae_hidden=[20],transfer_hidden=[20],
21 start_weights=None,transfer_weights=None,end_weights=None,
22 input_activation="tanh", output_activation="tanh",
23 init="glorot_uniform",
24 ae_dropouts=[None], transfer_do=[None],
25 sgd="sgd", loss="mse", patience=5, verbose=0, epochs=5, batch_size=8):
26
27 if not start_weights :
28 start_weights = [ None ] * len(ae_hidden)
29 if not transfer_weights :
30 transfer_weights = [None ] * len(transfer_hidden)
31 if not end_weights :
32 end_weights = [ None ] * len(end_weights)
33 if not transfer_do :
34 transfer_do = [0] * len(transfer_hidden)
35 predict_y = True
36 if y_train is None or y_dev is None or y_test is None :
37 y_train = train
38 y_dev = dev
39 y_test = test
40 predict_y = False
41 param_predict = [ train, dev, test ]
42 if predict_y :
43 param_predict += [ y_train, y_dev ,y_test ]
44
45 pred_by_level = [] # Contient les prediction par niveaux de transfert
46 layers = [Input(shape=(train.shape[1],))]
47 #for w in transfer_weights:
48 #print "TW",[ [ y.shape for y in x ] for x in w]
49 #print "SW",[ [ y.shape for y in x] for x in start_weights]
50 #print "EW",[ [ y.shape for y in x ] for x in end_weights]
51 for cpt in range(1,len(ae_hidden)):
52 #print ae_hidden,cpt
53 #print cpt, "before"
54 #print "before2", [ [ x.shape for x in y] for y in start_weights[:cpt] ]
55 #print "before3", [ [ x.shape for x in y] for y in transfer_weights[cpt]]
56 #print "before4", [ [ x.shape for x in y] for y in end_weights[cpt:]]
57 sizes = ae_hidden[:cpt] + transfer_hidden + ae_hidden[cpt:]
58 weights = start_weights[:cpt] + transfer_weights[(cpt-1)] + end_weights[cpt:]
59 #print "SIZES", sizes
60 #print "AW",[ [ y.shape for y in x ] for x in weights]
61 #print "WEI", len(weights) , [ len(x) for x in weights ]
62 if len(ae_dropouts) == len(ae_hidden):
63 do = ae_dropouts[:cpt] + transfer_do + ae_dropouts[cpt:]
64 else :
65 do = [ 0 ] * (len(ae_hidden) + len(transfer_hidden))
66 for w in weights[:-1]:
67 #print "STEP", size
68 layers.append(Dense(w[1].shape[0],activation=input_activation,init=init,weights=w)(layers[-1]))
69 if do :
70 d = do.pop(0)
71 if d > 0 :
72 layers.append(Dropout(d)(layers[-1]))
73
74 layers.append(Dense(y_train.shape[1],activation=output_activation)(layers[-1]))
75 models = [Model(input=layers[0] , output=x) for x in layers[1:]]
76 models[-1].compile(optimizer=sgd,loss=loss)
77 models[-1].fit(train,y_train,nb_epoch=epochs,batch_size=batch_size,callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=0)],validation_data=(dev,dev),verbose=verbose)
78 predictions = [ [x.predict(y) for y in param_predict ] for x in models ]
79 pred_by_level.append(predictions)
80
81 return pred_by_level
82
83 def train_mlp(x_train,y_train,x_dev,y_dev,x_test,y_test,hidden_size,input_activation="relu",hidden_activation="relu",output_activation="softmax",loss="mse",init="glorot_uniform",dropouts=None,sgd=None,epochs=1200,batch_size=16,fit_verbose=1,test_verbose=0,save_pred=False,keep_histo=False):
84
85
86 layers = [Input(shape=(x_train.shape[1],))]
87
88 for h in hidden_size:
89 if dropouts:
90 d = dropouts.pop(0)
91 if d > 0 :
92 layers.append(Dropout(d)(layers[-1]))
93
94 layers.append(Dense(h,init=init,activation=input_activation)(layers[-1]))
95 #if dropouts:
96 # drop_prob=dropouts.pop(0)
97 # if drop_prob > 0:
98 # model.add(Dropout(drop_prob))
99
100 #if dropouts:
101 # drop_prob=dropouts.pop(0)
102 # if drop_prob > 0:
103 # model.add(Dropout(drop_prob))
104
105 #if dropouts:
106 # model.add(Dropout(dropouts.pop(0)))
107 if dropouts:
108 d = dropouts.pop(0)
109 if d > 0 :
110 layers.append(Dropout(d)(layers[-1]))
111
112 layers.append(Dense( y_train.shape[1],activation=output_activation,init=init)(layers[-1]))
113
114 model = Model(layers[0] , layers[-1])
115 if not sgd:
116 sgd = SGD(lr=0.01, decay=0, momentum=0.9)
117
118 model.compile(loss=loss, optimizer=sgd,metrics=['accuracy'])
119
120 scores_dev=[]
121 scores_test=[]
122 scores_train=[]
123 save=None
124 for i in range(epochs):
125 hist=model.fit(x_train, y_train, nb_epoch=1, batch_size=batch_size,verbose=fit_verbose,validation_data=(x_dev,y_dev))
126 pred_train=model.predict(x_train)
127 pred_dev=model.predict(x_dev)
128 pred_test=model.predict(x_test)
129
130 scores_train.append(perf(np.argmax(y_train,axis=1),np.argmax(pred_train,axis=1)))
131 scores_dev.append(perf(np.argmax(y_dev,axis=1),np.argmax(pred_dev,axis=1)))
132 scores_test.append(perf(np.argmax(y_test,axis=1),np.argmax(pred_test,axis=1)))
133 if fit_verbose :
134 print "{} {} {} {}".format(i,scores_train[-1],scores_dev[-1],scores_test[-1])
135 if save is None or (len(scores_dev)>2 and scores_dev[-1] > scores_dev[-2]):
136 save=save_tuple(pred_train,pred_dev,pred_test)
137 arg_dev = np.argmax(scores_dev)
138 best_dev=scores_dev[arg_dev]
139 best_test=scores_test[arg_dev]
140 max_test=np.max(scores_test)
141 if fit_verbose:
142 print " res : {} {} {}".format(best_dev,best_test,max_test)
143
144 res=[scores_train,scores_dev,scores_test]
145 if save_pred:
146 res.append(save)
147 if keep_histo:
148 res.append(hist)
149 return res
150
151 def train_ae(train,dev,test,hidden_sizes,y_train=None,y_dev=None,y_test=None,dropouts=None,input_activation="tanh",output_activation="tanh",loss="mse",sgd=None,epochs=500,batch_size=8,verbose=1,patience=20,get_weights=False,set_weights=[]):
152
153 input_vect = Input(shape=(train.shape[1],))
154
155 previous = [input_vect]
156
157 if dropouts is None:
158 dropouts = [ 0 ] * (len(hidden_sizes) +1)
159 if sgd is None :
160 sgd = SGD(lr=0.01, decay=0, momentum=0.9)
161 did_do = False
162 if dropouts :
163 d = dropouts.pop(0)
164 if d :
165 previous.append(Dropout(d)(previous[-1]))
166 did_do = True
167
168 for h_layer,weight_layer in izip_longest(hidden_sizes,set_weights,fillvalue=None) :
169 # ,weights=w
170 if weight_layer :
171 w = weight_layer[0]
172 else :
173 w = None
174 #print "ADD SIZE" , h_layer
175 if did_do :
176 p = previous.pop()
177 did_do = False
178 else :
179 p = previous[-1]
180 previous.append(Dense(h_layer,activation=input_activation,weights=w)(previous[-1]))
181 if dropouts:
182 d = dropouts.pop(0)
183 if d :
184 previous.append(Dropout(d)(previous[-1]))
185 did_do = True
186
187 predict_y = True
188 if y_train is None or y_dev is None or y_test is None :
189 y_train = train
190 y_dev = dev
191 y_test = test
192 predict_y = False
193 previous.append(Dense(y_train.shape[1],activation=output_activation)(previous[-1]))
194 models = [Model(input=previous[0] , output=x) for x in previous[1:]]
195 print "MLP", sgd, loss
196 models[-1].compile(optimizer=sgd,loss=loss)
197 models[-1].fit(train,y_train,nb_epoch=epochs,batch_size=batch_size,callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=0)],validation_data=(dev,dev),verbose=verbose)
198 param_predict = [ train, dev, test ]
199 if predict_y :
200 param_predict += [ y_train, y_dev ,y_test ]
201 predictions = [ [x.predict(y) for y in param_predict ] for x in models ]
202 if get_weights :
203 weights = [ x.get_weights() for x in models[-1].layers if x.get_weights() ]
204 return ( predictions , weights )
205 else :
206 return predictions
207
208 def train_sae(train,dev,test,hidden_sizes,y_train=None,y_dev=None,y_test=None,dropouts=None,input_activation="tanh",output_activation="tanh",loss="mse",sgd=None,epochs=500,batch_size=8,verbose=1,patience=20):
209
210 weights = []
211 predictions = [[(train,dev,test),()]]
212 ft_pred = []
213 past_sizes = []
214
215
216 for size in hidden_sizes :
217 #print "DO size " , size , "FROM" , hidden_sizes
218 res_pred, res_wght = train_ae(predictions[-1][-2][0], predictions[-1][-2][1],predictions[-1][-2][2],[size],
219 dropouts=dropouts, input_activation=input_activation,
220 output_activation=output_activation, loss=loss, sgd=sgd,
221 epochs=epochs, batch_size=batch_size, verbose=verbose,
222 patience=patience,get_weights=True)
223 past_sizes.append(size)
224 weights.append(res_wght)
225 predictions.append(res_pred)
226 #print "FINE TUNE "
227 res_ftpred = train_ae(train,dev,test,past_sizes,y_train=y_train,y_dev=y_dev,y_test=y_test,
228 dropouts=dropouts,
229 input_activation=input_activation,
230 output_activation=output_activation,
231 loss=loss,sgd=sgd,epochs=epochs,
232 batch_size=batch_size,verbose=verbose,patience=patience,
233 set_weights=weights)
234 ft_pred.append(res_ftpred)
235
236 return ( predictions[1:] , ft_pred)
237
238
File was created 1 # -*- coding: utf-8 -*-
2 import keras
3 import numpy as np
4 #from keras.layers.core import Dense, Dropout, Activation
5 from keras.optimizers import SGD,Adam
6 from keras.models import Sequential
7 from keras.layers import Input, Dense, Dropout
8 from keras.models import Model
9 from keras.utils.layer_utils import layer_from_config
10 from itertools import izip_longest
11
12 import pandas
13 from collections import namedtuple
14 from sklearn.metrics import accuracy_score as perf
15 save_tuple= namedtuple("save_tuple",["pred_train","pred_dev","pred_test"])
16
17
18 def ft_dsae(train,dev,test,
19 y_train=None,y_dev=None,y_test=None,
20 ae_hidden=[20],transfer_hidden=[20],
21 start_weights=None,transfer_weights=None,end_weights=None,
22 input_activation="tanh", output_activation="tanh",
23 init="glorot_uniform",
24 ae_dropouts=[None], transfer_do=[None],
25 sgd="sgd", loss="mse", patience=5, verbose=0, epochs=5, batch_size=8):
26
27 if not start_weights :
28 start_weights = [ None ] * len(ae_hidden)
29 if not transfer_weights :
30 transfer_weights = [None ] * len(transfer_hidden)
31 if not end_weights :
32 end_weights = [ None ] * len(end_weights)
33 if not transfer_do :
34 transfer_do = [0] * len(transfer_hidden)
35 predict_y = True
36 if y_train is None or y_dev is None or y_test is None :
37 y_train = train
38 y_dev = dev
39 y_test = test
40 predict_y = False
41 param_predict = [ train, dev, test ]
42 if predict_y :
43 param_predict += [ y_train, y_dev ,y_test ]
44
45 pred_by_level = [] # Contient les prediction par niveaux de transfert
46 layers = [Input(shape=(train.shape[1],))]
47 #for w in transfer_weights:
48 #print "TW",[ [ y.shape for y in x ] for x in w]
49 #print "SW",[ [ y.shape for y in x] for x in start_weights]
50 #print "EW",[ [ y.shape for y in x ] for x in end_weights]
51 for cpt in range(1,len(ae_hidden)):
52 #print ae_hidden,cpt
53 #print cpt, "before"
54 #print "before2", [ [ x.shape for x in y] for y in start_weights[:cpt] ]
55 #print "before3", [ [ x.shape for x in y] for y in transfer_weights[cpt]]
56 #print "before4", [ [ x.shape for x in y] for y in end_weights[cpt:]]
57 sizes = ae_hidden[:cpt] + transfer_hidden + ae_hidden[cpt:]
58 weights = start_weights[:cpt] + transfer_weights[(cpt-1)] + end_weights[cpt:]
59 #print "SIZES", sizes
60 #print "AW",[ [ y.shape for y in x ] for x in weights]
61 #print "WEI", len(weights) , [ len(x) for x in weights ]
62 if len(ae_dropouts) == len(ae_hidden):
63 do = ae_dropouts[:cpt] + transfer_do + ae_dropouts[cpt:]
64 else :
65 do = [ 0 ] * (len(ae_hidden) + len(transfer_hidden))
66 for w in weights[:-1]:
67 #print "STEP", size
68 layers.append(Dense(w[1].shape[0],activation=input_activation,init=init,weights=w)(layers[-1]))
69 if do :
70 d = do.pop(0)
71 if d > 0 :
72 layers.append(Dropout(d)(layers[-1]))
73
74 layers.append(Dense(y_train.shape[1],activation=output_activation)(layers[-1]))
75 models = [Model(input=layers[0] , output=x) for x in layers[1:]]
76 models[-1].compile(optimizer=sgd,loss=loss)
77 models[-1].fit(train,y_train,nb_epoch=epochs,batch_size=batch_size,callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=0)],validation_data=(dev,dev),verbose=verbose)
78 predictions = [ [x.predict(y) for y in param_predict ] for x in models ]
79 pred_by_level.append(predictions)
80
81 return pred_by_level
82
83 def train_mlp(x_train,y_train,x_dev,y_dev,x_test,y_test,hidden_size,input_activation="relu",hidden_activation="relu",output_activation="softmax",loss="mse",init="glorot_uniform",dropouts=None,sgd=None,epochs=1200,batch_size=16,fit_verbose=1,test_verbose=0,save_pred=False,keep_histo=False):
84
85
86 layers = [Input(shape=(x_train.shape[1],))]
87
88 for h in hidden_size:
89 if dropouts:
90 d = dropouts.pop(0)
91 if d > 0 :
92 layers.append(Dropout(d)(layers[-1]))
93
94 layers.append(Dense(h,init=init,activation=input_activation)(layers[-1]))
95 #if dropouts:
96 # drop_prob=dropouts.pop(0)
97 # if drop_prob > 0:
98 # model.add(Dropout(drop_prob))
99
100 #if dropouts:
101 # drop_prob=dropouts.pop(0)
102 # if drop_prob > 0:
103 # model.add(Dropout(drop_prob))
104
105 #if dropouts:
106 # model.add(Dropout(dropouts.pop(0)))
107 if dropouts:
108 d = dropouts.pop(0)
109 if d > 0 :
110 layers.append(Dropout(d)(layers[-1]))
111
112 layers.append(Dense( y_train.shape[1],activation=output_activation,init=init)(layers[-1]))
113
114 model = Model(layers[0] , layers[-1])
115 if not sgd:
116 sgd = SGD(lr=0.01, decay=0, momentum=0.9)
117
118 model.compile(loss=loss, optimizer=sgd,metrics=['accuracy'])
119
120 scores_dev=[]
121 scores_test=[]
122 scores_train=[]
123 save=None
124 for i in range(epochs):
125 hist=model.fit(x_train, y_train, nb_epoch=1, batch_size=batch_size,verbose=fit_verbose,validation_data=(x_dev,y_dev))
126 pred_train=model.predict(x_train)
127 pred_dev=model.predict(x_dev)
128 pred_test=model.predict(x_test)
129
130 scores_train.append(perf(np.argmax(y_train,axis=1),np.argmax(pred_train,axis=1)))
131 scores_dev.append(perf(np.argmax(y_dev,axis=1),np.argmax(pred_dev,axis=1)))
132 scores_test.append(perf(np.argmax(y_test,axis=1),np.argmax(pred_test,axis=1)))
133 if fit_verbose :
134 print "{} {} {} {}".format(i,scores_train[-1],scores_dev[-1],scores_test[-1])
135 if save is None or (len(scores_dev)>2 and scores_dev[-1] > scores_dev[-2]):
136 save=save_tuple(pred_train,pred_dev,pred_test)
137 arg_dev = np.argmax(scores_dev)
138 best_dev=scores_dev[arg_dev]
139 best_test=scores_test[arg_dev]
140 max_test=np.max(scores_test)
141 if fit_verbose:
142 print " res : {} {} {}".format(best_dev,best_test,max_test)
143
144 res=[scores_train,scores_dev,scores_test]
145 if save_pred:
146 res.append(save)
147 if keep_histo:
148 res.append(hist)
149 return res
150
151 def train_ae(train,dev,test,hidden_sizes,y_train=None,y_dev=None,y_test=None,dropouts=None,input_activation="tanh",output_activation="tanh",loss="mse",sgd=None,epochs=500,batch_size=8,verbose=1,patience=20,get_weights=False,set_weights=[]):
152
153 input_vect = Input(shape=(train.shape[1],))
154
155 previous = [input_vect]
156
157 if dropouts is None:
158 dropouts = [ 0 ] * (len(hidden_sizes) +1)
159 if sgd is None :
160 sgd = SGD(lr=0.01, decay=0, momentum=0.9)
161 did_do = False
162 if dropouts :
163 d = dropouts.pop(0)
164 if d :
165 previous.append(Dropout(d)(previous[-1]))
166 did_do = True
167
168 for h_layer,weight_layer in izip_longest(hidden_sizes,set_weights,fillvalue=None) :
169 # ,weights=w
170 if weight_layer :
171 w = weight_layer[0]
172 else :
173 w = None
174 #print "ADD SIZE" , h_layer
175 if did_do :
176 p = previous.pop()
177 did_do = False
178 else :
179 p = previous[-1]
180 previous.append(Dense(h_layer,activation=input_activation,weights=w)(previous[-1]))
181 if dropouts:
182 d = dropouts.pop(0)
183 if d :
184 previous.append(Dropout(d)(previous[-1]))
185 did_do = True
186
187 predict_y = True
188 if y_train is None or y_dev is None or y_test is None :
189 y_train = train
190 y_dev = dev
191 y_test = test
192 predict_y = False
193 previous.append(Dense(y_train.shape[1],activation=output_activation)(previous[-1]))
194 models = [Model(input=previous[0] , output=x) for x in previous[1:]]
195 print "MLP", sgd, loss
196 models[-1].compile(optimizer=sgd,loss=loss)
197 models[-1].fit(train,y_train,nb_epoch=epochs,batch_size=batch_size,callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=0)],validation_data=(dev,dev),verbose=verbose)
198 param_predict = [ train, dev, test ]
199 if predict_y :
200 param_predict += [ y_train, y_dev ,y_test ]
201 predictions = [ [x.predict(y) for y in param_predict ] for x in models ]
202 if get_weights :
203 weights = [ x.get_weights() for x in models[-1].layers if x.get_weights() ]
204 return ( predictions , weights )
205 else :
206 return predictions
207
208 def train_sae(train,dev,test,hidden_sizes,y_train=None,y_dev=None,y_test=None,dropouts=None,input_activation="tanh",output_activation="tanh",loss="mse",sgd=None,epochs=500,batch_size=8,verbose=1,patience=20):
209
210 weights = []
211 predictions = [[(train,dev,test),()]]
212 ft_pred = []
213 past_sizes = []
214
215
216 for size in hidden_sizes :
217 #print "DO size " , size , "FROM" , hidden_sizes
218 res_pred, res_wght = train_ae(predictions[-1][-2][0], predictions[-1][-2][1],predictions[-1][-2][2],[size],
219 dropouts=dropouts, input_activation=input_activation,
220 output_activation=output_activation, loss=loss, sgd=sgd,
221 epochs=epochs, batch_size=batch_size, verbose=verbose,
222 patience=patience,get_weights=True)
223 past_sizes.append(size)
224 weights.append(res_wght)
225 predictions.append(res_pred)
226 #print "FINE TUNE "
227 res_ftpred = train_ae(train,dev,test,past_sizes,y_train=y_train,y_dev=y_dev,y_test=y_test,
228 dropouts=dropouts,
229 input_activation=input_activation,
230 output_activation=output_activation,
231 loss=loss,sgd=sgd,epochs=epochs,
232 batch_size=batch_size,verbose=verbose,patience=patience,
233 set_weights=weights)
234 ft_pred.append(res_ftpred)
235
236 return ( predictions[1:] , ft_pred)
1 ../VARIATIONAL/Variational-Autoencoder/mlp.py 237
238
239
1 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/193/ MM_features/data_w20/mmf_193.shelve >> output_v7/recap.txt 1 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/193/ MM_features/data_w99/mmf_193.shelve >> output_v8/recap.txt
2 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/194/ MM_features/data_w20/mmf_194.shelve >> output_v7/recap.txt 2 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/194/ MM_features/data_w99/mmf_194.shelve >> output_v8/recap.txt
3 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/195/ MM_features/data_w20/mmf_195.shelve >> output_v7/recap.txt 3 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/195/ MM_features/data_w99/mmf_195.shelve >> output_v8/recap.txt
4 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/196/ MM_features/data_w20/mmf_196.shelve >> output_v7/recap.txt 4 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/196/ MM_features/data_w99/mmf_196.shelve >> output_v8/recap.txt
5 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/197/ MM_features/data_w20/mmf_197.shelve >> output_v7/recap.txt 5 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/197/ MM_features/data_w99/mmf_197.shelve >> output_v8/recap.txt
6 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/198/ MM_features/data_w20/mmf_198.shelve >> output_v7/recap.txt 6 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/198/ MM_features/data_w99/mmf_198.shelve >> output_v8/recap.txt
7 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/393/ MM_features/data_w20/mmf_393.shelve >> output_v7/recap.txt 7 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/393/ MM_features/data_w99/mmf_393.shelve >> output_v8/recap.txt
8 8
1 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/394/ MM_features/data_w20/mmf_394.shelve >> output_v7/recap.txt 1 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/394/ MM_features/data_w99/mmf_394.shelve >> output_v8/recap.txt
2 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/395/ MM_features/data_w20/mmf_395.shelve >> output_v7/recap.txt 2 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/395/ MM_features/data_w99/mmf_395.shelve >> output_v8/recap.txt
3 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/396/ MM_features/data_w20/mmf_396.shelve >> output_v7/recap.txt 3 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/396/ MM_features/data_w99/mmf_396.shelve >> output_v8/recap.txt
4 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/397/ MM_features/data_w20/mmf_397.shelve >> output_v7/recap.txt 4 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/397/ MM_features/data_w99/mmf_397.shelve >> output_v8/recap.txt
5 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/43/ MM_features/data_w20/mmf_43.shelve >> output_v7/recap.txt 5 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/43/ MM_features/data_w99/mmf_43.shelve >> output_v8/recap.txt
6 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/44/ MM_features/data_w20/mmf_44.shelve >> output_v7/recap.txt 6 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/44/ MM_features/data_w99/mmf_44.shelve >> output_v8/recap.txt
7 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/45/ MM_features/data_w20/mmf_45.shelve >> output_v7/recap.txt 7 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/45/ MM_features/data_w99/mmf_45.shelve >> output_v8/recap.txt
8 8
1 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/46/ MM_features/data_w20/mmf_46.shelve >> output_v7/recap.txt 1 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/46/ MM_features/data_w99/mmf_46.shelve >> output_v8/recap.txt
2 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/47/ MM_features/data_w20/mmf_47.shelve >> output_v7/recap.txt 2 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/47/ MM_features/data_w99/mmf_47.shelve >> output_v8/recap.txt
3 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/48/ MM_features/data_w20/mmf_48.shelve >> output_v7/recap.txt 3 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/48/ MM_features/data_w99/mmf_48.shelve >> output_v8/recap.txt
4 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/493/ MM_features/data_w20/mmf_493.shelve >> output_v7/recap.txt 4 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/493/ MM_features/data_w99/mmf_493.shelve >> output_v8/recap.txt
5 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/494/ MM_features/data_w20/mmf_494.shelve >> output_v7/recap.txt 5 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/494/ MM_features/data_w99/mmf_494.shelve >> output_v8/recap.txt
6 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/495/ MM_features/data_w20/mmf_495.shelve >> output_v7/recap.txt 6 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/495/ MM_features/data_w99/mmf_495.shelve >> output_v8/recap.txt
7 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v7/496/ MM_features/data_w20/mmf_496.shelve >> output_v7/recap.txt 7 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04a-mmdf.py output_v8/496/ MM_features/data_w99/mmf_496.shelve >> output_v8/recap.txt
8 8
1 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/497/ MM_features/data_w20/mmf_497.shelve >> output_v7/recap.txt 1 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/497/ MM_features/data_w99/mmf_497.shelve >> output_v8/recap.txt
2 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/50/ MM_features/data_w20/mmf_50.shelve >> output_v7/recap.txt 2 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/50/ MM_features/data_w99/mmf_50.shelve >> output_v8/recap.txt
3 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/93/ MM_features/data_w20/mmf_93.shelve >> output_v7/recap.txt 3 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/93/ MM_features/data_w99/mmf_93.shelve >> output_v8/recap.txt
4 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/94/ MM_features/data_w20/mmf_94.shelve >> output_v7/recap.txt 4 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/94/ MM_features/data_w99/mmf_94.shelve >> output_v8/recap.txt
5 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/95/ MM_features/data_w20/mmf_95.shelve >> output_v7/recap.txt 5 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/95/ MM_features/data_w99/mmf_95.shelve >> output_v8/recap.txt
6 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/96/ MM_features/data_w20/mmf_96.shelve >> output_v7/recap.txt 6 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/96/ MM_features/data_w99/mmf_96.shelve >> output_v8/recap.txt
7 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/97/ MM_features/data_w20/mmf_97.shelve >> output_v7/recap.txt 7 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/97/ MM_features/data_w99/mmf_97.shelve >> output_v8/recap.txt
8 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v7/98/ MM_features/data_w20/mmf_98.shelve >> output_v7/recap.txt 8 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04a-mmdf.py output_v8/98/ MM_features/data_w99/mmf_98.shelve >> output_v8/recap.txt
9 9
File was created 1 output_dir=$1
2 features=$2
3 json_conf=$3
4 time (
5 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04b-mmf_mini_ae.py $output_dir $features $json_conf >> $output_dir/miniae.log &
6 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04c-mmf_sae.py $output_dir $features $json_conf >> $output_dir/sae.log &
7 THEANO_FLAGS=mode=FAST_RUN,device=gpu0,floatX=float32 python 04d-mmf_dsae.py $output_dir $features $json_conf >> $output_dir/dsae.log &
8 THEANO_FLAGS=mode=FAST_RUN,device=gpu1,floatX=float32 python 04e-mm_vae.py $output_dir $features $json_conf >> $output_dir/vae.log &
9 wait
10 )
11