Killian / decodopr

Blame view

LDA/04e-mm_vae.py 6.4 KB
edit raw normal view history
7db73861f   Killian   add vae et mmf 
1
2
 
  
  # coding: utf-8
7db73861f   Killian   add vae et mmf 
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
  import gensim
  from scipy import sparse
  import itertools
  from sklearn import preprocessing
  from keras.models import Sequential
  from keras.optimizers import SGD,Adam
  from mlp import *
  from vae import *
  import sklearn.metrics
  import shelve
  import pickle
  from utils import *
  import sys
  import os
  import json
  # In[4]:
  
  infer_model=shelve.open("{}".format(sys.argv[2]))
  in_dir = sys.argv[1]
  #['ASR', 'TRS', 'LABEL']
  # In[6]:
2af8e57f4   Killian   change all 
24
25
26
27
 
  if len(sys.argv) > 4 :
      features_key = sys.argv[4]
  else :
      features_key = "LDA"
7db73861f   Killian   add vae et mmf 
28
 








2af8e57f4
 
 Killian
 
change all




29




  save_projection = True









e5108393c
 
 Killian
 
replace du  mlp.p...




30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69




  json_conf =json.load(open(sys.argv[3]))
  vae_conf = json_conf["vae"]
  
  hidden_size= vae_conf["hidden_size"]
  input_activation=vae_conf["input_activation"]
  output_activation=vae_conf["output_activation"]
  epochs=vae_conf["epochs"]
  batch=vae_conf["batch"]
  patience=vae_conf["patience"]
  latent_dim = vae_conf["latent"]
  try:
      k = vae_conf["sgd"]
      if vae_conf["sgd"]["name"] == "adam":
          sgd = Adam(lr=vae_conf["sgd"]["lr"])#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
      elif vae_conf["sgd"]["name"] == "sgd":
          sgd = SGD(lr=vae_conf["sgd"]["lr"])
  except: 
      sgd = vae_conf["sgd"]
  
  mlp_conf = json_conf["mlp"]
  mlp_h = mlp_conf["hidden_size"]
  mlp_loss = mlp_conf["loss"]
  mlp_dropouts = mlp_conf["do"]
  mlp_epochs = mlp_conf["epochs"]
  mlp_batch_size = mlp_conf["batch"]
  mlp_input_activation=mlp_conf["input_activation"]
  mlp_output_activation=mlp_conf["output_activation"]
  
  
  try:
      k = mlp_conf["sgd"]
      if mlp_conf["sgd"]["name"] == "adam":
          mlp_sgd = Adam(lr=mlp_conf["sgd"]["lr"])#SGD(lr=0.00001,nesterov=False) #'rmsprop'# Adam(lr=0.00001)#SGD(lr=0.001, momentum=0.9, nesterov=True)
      elif mlp_conf["sgd"]["name"] == "sgd":
          mlp_sgd = SGD(lr=mlp_conf["sgd"]["lr"])
  except: 
      mlp_sgd = mlp_conf["sgd"]
  
  
  name = json_conf["name"]









2af8e57f4
 
 Killian
 
change all




70
71




  try :
      print "make folder "









e5108393c
 
 Killian
 
replace du  mlp.p...




72




      os.mkdir("{}/{}".format(in_dir,name))









7db73861f
 
 Killian
 
add vae et mmf




73




  except:









2af8e57f4
 
 Killian
 
change all




74




      print "folder not maked"









7db73861f
 
 Killian
 
add vae et mmf




75




      pass









e5108393c
 
 Killian
 
replace du  mlp.p...




76
77
78




  
  
  db = shelve.open("{}/{}/ae_model.shelve".format(in_dir,name),writeback=True)









7db73861f
 
 Killian
 
add vae et mmf




79
80




  db["LABEL"]=infer_model["LABEL"]
  #









7db73861f
 
 Killian
 
add vae et mmf




81












2af8e57f4
 
 Killian
 
change all




82
83




  
  keys = infer_model[features_key].keys()









7db73861f
 
 Killian
 
add vae et mmf




84
85




  
  db["VAE"] = {}









2af8e57f4
 
 Killian
 
change all




86




  db[features_key] = {}









7db73861f
 
 Killian
 
add vae et mmf




87




  for mod in keys : 









e5108393c
 
 Killian
 
replace du  mlp.p...




88




      #print mod









2af8e57f4
 
 Killian
 
change all




89
90
91




      db[features_key][mod] = train_mlp(infer_model[features_key][mod]["TRAIN"],infer_model["LABEL"][mod]["TRAIN"],
                              infer_model[features_key][mod]["DEV"],infer_model["LABEL"][mod]["DEV"],
                              infer_model[features_key][mod]["TEST"],infer_model["LABEL"][mod]["TEST"],









7db73861f
 
 Killian
 
add vae et mmf




92
93
94
95
96
97
98




                              mlp_h ,sgd=mlp_sgd,
                              epochs=mlp_epochs,
                              batch_size=mlp_batch_size,
                              input_activation=input_activation,
                              output_activation=mlp_output_activation,
                              dropouts=mlp_dropouts,
                              fit_verbose=0)









2af8e57f4
 
 Killian
 
change all




99




      res=train_vae(infer_model[features_key][mod]["TRAIN"],infer_model[features_key][mod]["DEV"],infer_model[features_key][mod]["TEST"],









7db73861f
 
 Killian
 
add vae et mmf




100
101
102
103
104




                   hidden_size=hidden_size[0],
                   latent_dim=latent_dim,sgd=sgd,
                   input_activation=input_activation,output_activation=output_activation,
                   nb_epochs=epochs,batch_size=batch)
      mlp_res_list=[]









2af8e57f4
 
 Killian
 
change all




105
106
107
108




      for nb,layer in enumerate(res) :
          if save_projection:
              pd = pandas.DataFrame(layer[0])
              col_count = (pd.sum(axis=0) != 0)









d1012a7a1
 
 Killian
 
update LDA/.py




109




              pd = pd.loc[:,col_count]









2af8e57f4
 
 Killian
 
change all




110
111
112
113
114
115
116
117




              pd.to_hdf("{}/{}/VAE_{}_{}_df.hdf".format(in_dir,name,nb,mod),"TRAIN")
              pd = pandas.DataFrame(layer[1])
              pd = pd.loc[:,col_count]
              pd.to_hdf("{}/{}/VAE_{}_{}_df.hdf".format(in_dir,name,nb,mod),"DEV")
              pd = pandas.DataFrame(layer[2])
              pd = pd.loc[:,col_count]
              pd.to_hdf("{}/{}/VAE_{}_{}_df.hdf".format(in_dir,name,nb,mod),"TEST")
              del pd









7db73861f
 
 Killian
 
add vae et mmf




118
119
120
121
122
123
124
125




          mlp_res_list.append(train_mlp(layer[0],infer_model['LABEL'][mod]["TRAIN"],
                                        layer[1],infer_model["LABEL"][mod]["DEV"],
                                        layer[2],infer_model["LABEL"][mod]["TEST"],
                                        mlp_h,loss=mlp_loss,dropouts=mlp_dropouts,sgd=mlp_sgd,epochs=mlp_epochs,
                                        output_activation=mlp_output_activation,
                                        input_activation=input_activation,
                                        batch_size=mlp_batch_size,fit_verbose=0))
      db["VAE"][mod]=mlp_res_list









2af8e57f4
 
 Killian
 
change all




126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165




  if "ASR" in keys and "TRS" in keys :
      mod = "ASR"
      mod2= "TRS"
      mlp_res_list=[]
  
      res = train_vae(infer_model[features_key][mod]["TRAIN"],
                      infer_model[features_key][mod]["DEV"],
                      infer_model[features_key][mod]["TEST"],
                      hidden_size=hidden_size[0],
                      sgd=sgd,input_activation=input_activation,output_activation=output_activation,
                      latent_dim=latent_dim,
                      nb_epochs=epochs,
                      batch_size=batch,
                      y_train=infer_model[features_key][mod2]["TRAIN"],
                      y_dev=infer_model[features_key][mod2]["DEV"],
                      y_test=infer_model[features_key][mod2]["TEST"])
  
      for nb,layer in enumerate(res) :
          if save_projection:
              pd = pandas.DataFrame(layer[0])
              col_count = (pd.sum(axis=0) != 0)
              pd = pd.loc[:,col_count]
              pd.to_hdf("{}/{}/VAE_{}_{}_df.hdf".format(in_dir,name,nb,"SPE"),"TRAIN")
              pd = pandas.DataFrame(layer[1])
              pd = pd.loc[:,col_count]
              pd.to_hdf("{}/{}/VAE_{}_{}_df.hdf".format(in_dir,name,nb,"SPE"),"DEV")
              pd = pandas.DataFrame(layer[2])
              pd = pd.loc[:,col_count]
              pd.to_hdf("{}/{}/VAE_{}_{}_df.hdf".format(in_dir,name,nb,"SPE"),"TEST")
              del pd
  
          mlp_res_list.append(train_mlp(layer[0],infer_model["LABEL"][mod]["TRAIN"],
                                        layer[1],infer_model["LABEL"][mod]["DEV"],
                                        layer[2],infer_model["LABEL"][mod]["TEST"],
                                        mlp_h,loss=mlp_loss,dropouts=mlp_dropouts,sgd=mlp_sgd,epochs=mlp_epochs,
                                        output_activation=mlp_output_activation,
                                        input_activation=input_activation,
                                        batch_size=mlp_batch_size,fit_verbose=0))
  
      db["VAE"]["SPE"] = mlp_res_list









7db73861f
 
 Killian
 
add vae et mmf




166
167
168




  
  db.sync()
  db.close()