diff --git a/LDA/mlp.py b/LDA/mlp.py index c83db76..ff80f14 100755 --- a/LDA/mlp.py +++ b/LDA/mlp.py @@ -6,13 +6,15 @@ from keras.optimizers import SGD,Adam from keras.models import Sequential from keras.layers import Input, Dense, Dropout from keras.models import Model +from keras.callbacks import ModelCheckpoint, EarlyStopping from keras.utils.layer_utils import layer_from_config from itertools import izip_longest - +import tempfile +import shutil import pandas from collections import namedtuple from sklearn.metrics import accuracy_score as perf -save_tuple= namedtuple("save_tuple",["pred_train","pred_dev","pred_test"]) +save_tuple = namedtuple("save_tuple",["pred_train","pred_dev","pred_test"]) def ft_dsae(train,dev,test, @@ -74,12 +76,114 @@ def ft_dsae(train,dev,test, layers.append(Dense(y_train.shape[1],activation=output_activation)(layers[-1])) models = [Model(input=layers[0] , output=x) for x in layers[1:]] models[-1].compile(optimizer=sgd,loss=loss) - models[-1].fit(train,y_train,nb_epoch=epochs,batch_size=batch_size,callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=0)],validation_data=(dev,dev),verbose=verbose) + models[-1].fit(train,y_train,nb_epoch=epochs,batch_size=batch_size,callbacks=[EarlyStopping(monitor='val_loss', patience=patience, verbose=0)],validation_data=(dev,dev),verbose=verbose) predictions = [ [x.predict(y) for y in param_predict ] for x in models ] pred_by_level.append(predictions) return pred_by_level +def train_mlp_proj(x_train,y_train,x_dev,y_dev,x_test,y_test,hidden_size,input_activation="relu",hidden_activation="relu",output_activation="softmax",loss="mse",init="glorot_uniform",dropouts=None,sgd=None,epochs=1200,batch_size=16,fit_verbose=1,patience=20,test_verbose=0): + + #model_tempfile=tempfile.mkstemp() + tempfold = tempfile.mkdtemp() + model_tempfile= tempfold+"/model.hdf" + + layers = [Input(shape=(x_train.shape[1],))] + + for h in hidden_size: + print h + if dropouts: + d = dropouts.pop(0) + if d > 0 : + ldo = Dropout(d)(layers[-1]) + print 'append' + layers.append(Dense(h,init=init,activation=input_activation)(ldo)) + else : + print " append" + layers.append(Dense(h,init=init,activation=input_activation)(layers[-1])) + + + if dropouts: + d = dropouts.pop(0) + if d > 0 : + ldo =Dropout(d)(layers[-1]) + print "end" + layers.append(Dense( y_train.shape[1],activation=output_activation,init=init)(ldo)) + else: + print "end" + layers.append(Dense( y_train.shape[1],activation=output_activation,init=init)(layers[-1])) + + models = [] + for l in layers[1:] : + models.append(Model(layers[0] , l)) + print "nb models : ", len(models), "h :",hidden_size , "layer", len(layers) + if not sgd: + sgd = SGD(lr=0.01, decay=0, momentum=0.9) + + models[-1].compile(loss=loss, optimizer=sgd,metrics=['accuracy']) + callbacks = [ModelCheckpoint(model_tempfile, monitor='val_acc', verbose=test_verbose, save_best_only=True, save_weights_only=True, mode='auto'), + EarlyStopping(monitor='val_acc', patience=patience, verbose=test_verbose) ] # On pourrai essayer avec la loss aussi + print models[-1].summary() + hist=models[-1].fit(x_train, y_train, nb_epoch=epochs, batch_size=batch_size,verbose=fit_verbose,validation_data=(x_dev,y_dev),callbacks=callbacks) + models[-1].load_weights(model_tempfile, by_name=False) + proj = [] + for layer,model in enumerate(models): + proj.append((model.predict(x_train),model.predict(x_dev),model.predict(x_test))) + + shutil.rmtree(tempfold) + return models[-1].summary(),proj + + + + + +def train_mlp_pred(x_train,y_train,x_dev,y_dev,x_test,y_test,hidden_size,input_activation="relu",hidden_activation="relu",output_activation="softmax",loss="mse",init="glorot_uniform",dropouts=None,sgd=None,epochs=1200,batch_size=16,fit_verbose=1,patience=20,test_verbose=0): + + #model_tempfile=tempfile.mkstemp() + tempfold = tempfile.mkdtemp() + model_tempfile= tempfold+"/model.hdf" + + layers = [Input(shape=(x_train.shape[1],))] + + for h in hidden_size: + if dropouts: + d = dropouts.pop(0) + if d > 0 : + ldo = Dropout(d)(layers[-1]) + layers.append(Dense(h,init=init,activation=input_activation)(ldo)) + else : + layers.append(Dense(h,init=init,activation=input_activation)(layers[-1])) + + + if dropouts: + d = dropouts.pop(0) + if d > 0 : + ldo =Dropout(d)(layers[-1]) + layers.append(Dense( y_train.shape[1],activation=output_activation,init=init)(ldo)) + else: + layers.append(Dense( y_train.shape[1],activation=output_activation,init=init)(layers[-1])) + + model=Model(layers[0] , layers[-1]) + if not sgd: + sgd = SGD(lr=0.01, decay=0, momentum=0.9) + + model.compile(loss=loss, optimizer=sgd,metrics=['accuracy']) + callbacks = [ModelCheckpoint(model_tempfile, monitor='val_acc', verbose=test_verbose, save_best_only=True, save_weights_only=True, mode='auto'), + EarlyStopping(monitor='val_acc', patience=patience, verbose=test_verbose) ] # On pourrai essayer avec la loss aussi + print model.summary() + hist=model.fit(x_train, y_train, nb_epoch=epochs, batch_size=batch_size,verbose=fit_verbose,validation_data=(x_dev,y_dev),callbacks=callbacks) + model.load_weights(model_tempfile, by_name=False) + pred=(model.predict(x_train),model.predict(x_dev),model.predict(x_test)) + + shutil.rmtree(tempfold) + return pred,hist + + + + + + + def train_mlp(x_train,y_train,x_dev,y_dev,x_test,y_test,hidden_size,input_activation="relu",hidden_activation="relu",output_activation="softmax",loss="mse",init="glorot_uniform",dropouts=None,sgd=None,epochs=1200,batch_size=16,fit_verbose=1,test_verbose=0,save_pred=False,keep_histo=False): layers = [Input(shape=(x_train.shape[1],))] @@ -107,7 +211,7 @@ def train_mlp(x_train,y_train,x_dev,y_dev,x_test,y_test,hidden_size,input_activa d = dropouts.pop(0) if d > 0 : layers.append(Dropout(d)(layers[-1])) - + print y_train[2:10] layers.append(Dense( y_train.shape[1],activation=output_activation,init=init)(layers[-1])) model = Model(layers[0] , layers[-1]) @@ -147,7 +251,7 @@ def train_mlp(x_train,y_train,x_dev,y_dev,x_test,y_test,hidden_size,input_activa res.append(hist) return res -def train_ae(train,dev,test,hidden_sizes,y_train=None,y_dev=None,y_test=None,dropouts=None,input_activation="tanh",output_activation="tanh",loss="mse",sgd=None,epochs=500,batch_size=8,verbose=1,patience=20,get_weights=False,set_weights=[]): +def train_ae(train,dev,test,hidden_sizes,y_train=None,y_dev=None,y_test=None,dropouts=None,input_activation="tanh",output_activation="tanh",loss="mse",sgd=None,epochs=500,batch_size=8,test_verbose=0,verbose=1,patience=20,get_weights=False,set_weights=[],best_mod=False): input_vect = Input(shape=(train.shape[1],)) @@ -193,7 +297,17 @@ def train_ae(train,dev,test,hidden_sizes,y_train=None,y_dev=None,y_test=None,dro models = [Model(input=previous[0] , output=x) for x in previous[1:]] print "MLP", sgd, loss models[-1].compile(optimizer=sgd,loss=loss) - models[-1].fit(train,y_train,nb_epoch=epochs,batch_size=batch_size,callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=0)],validation_data=(dev,dev),verbose=verbose) + cb = [EarlyStopping(monitor='val_loss', patience=patience, verbose=0)] + if best_mod: + tempfold = tempfile.mkdtemp() + model_tempfile= tempfold+"/model.hdf" + cb.append( ModelCheckpoint(model_tempfile, monitor='val_loss', verbose=test_verbose, save_best_only=True, save_weights_only=True, mode='auto') ) + + models[-1].summary() + models[-1].fit(train,y_train,nb_epoch=epochs,batch_size=batch_size,callbacks=cb,validation_data=(dev,dev),verbose=verbose) + if best_mod: + models[-1].load_weights(model_tempfile) + shutil.rmtree(tempfold) param_predict = [ train, dev, test ] if predict_y : param_predict += [ y_train, y_dev ,y_test ]