04a-mmdf.py 2.85 KB
# coding: utf-8

# In[29]:

# Import
import itertools
import shelve
import pickle
import numpy
import scipy
from scipy import sparse
import scipy.sparse
import scipy.io
from mlp import *
import mlp
import sys
import utils
import dill
from collections import Counter
from gensim.models import LdaModel



# In[3]:

#30_50_50_150_0.0001

# In[4]:

#db=shelve.open("SPELIKE_MLP_DB.shelve",writeback=True)
origin_corps=shelve.open("{}".format(sys.argv[2]))
in_dir = sys.argv[1]


out_db=shelve.open("{}/mlp_scores.shelve".format(in_dir),writeback=True)

mlp_h = [ 250, 250  ]
mlp_loss = "categorical_crossentropy"
mlp_dropouts = [0.25]* len(mlp_h)
mlp_sgd = Adam(lr=0.0001)
mlp_epochs = 3000
mlp_batch_size = 1
mlp_input_activation = "relu"
mlp_output_activation="softmax"

ress = []
for key in ["TRS", "ASR"] :

    res=mlp.train_mlp(origin_corps["LDA"][key]["TRAIN"],origin_corps["LABEL"][key]["TRAIN"],
            origin_corps["LDA"][key]["DEV"],origin_corps["LABEL"][key]["DEV"],
            origin_corps["LDA"][key]["TEST"],origin_corps["LABEL"][key]["TEST"],
            mlp_h,dropouts=mlp_dropouts,sgd=mlp_sgd,
            epochs=mlp_epochs,
            batch_size=mlp_batch_size,
            save_pred=False,keep_histo=False,
            loss="categorical_crossentropy",fit_verbose=0)
    arg_best=[]
    dev_best=[]
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0
    arg_best.append(numpy.argmax(res[1]))
    dev_best.append(res[1][arg_best[-1]])
    res[1][arg_best[-1]]=0




    test_best =[  res[2][x] for x in arg_best ]
    test_max = numpy.max(res[2])
    out_db[key]=(res,(dev_best,test_best,test_max))
    ress.append((key,dev_best,test_best,test_max))
    
for el in ress :
    print el
out_db.close()
origin_corps.close()