00-mmf_make_features.py 1.4 KB
import sys 
import os 

import pandas 
import numpy 
import shelve

from sklearn.preprocessing import LabelBinarizer

from utils import select_mmf as select

input_dir = sys.argv[1] # Dossier de premire niveau contient ASR et TRS
level = sys.argv[2] # taille de LDA ( -5) voulu 
output_dir = sys.argv[3]

lb=LabelBinarizer()
#y_train=lb.fit_transform([utils.select(ligneid) for ligneid in origin_corps["LABEL"]["TRAIN"]])


data = shelve.open("{}/mmf_{}.shelve".format(output_dir,level),writeback=True)
data["LABEL"]= {}
data["LDA"] = {"ASR":{},"TRS":{}}
for mod in ["ASR", "TRS" ]:
    train = pandas.read_table("{}/{}/train_{}.tab".format(input_dir, mod, level), sep=" ", header=None )
    dev = pandas.read_table("{}/{}/dev_{}.tab".format(input_dir, mod, level), sep=" ", header=None )
    test = pandas.read_table("{}/{}/test_{}.tab".format(input_dir, mod, level), sep=" ", header=None )

    y_train = train.iloc[:,0].apply(select)
    y_dev = dev.iloc[:,0].apply(select)
    y_test = test.iloc[:,0].apply(select)
    lb.fit(y_train)
    data["LABEL"][mod]={"TRAIN":lb.transform(y_train),"DEV":lb.transform(y_dev), "TEST": lb.transform(y_test)}

   # data["LDA"][mod]={'ASR':[]}
    print train.values
    data["LDA"][mod]["TRAIN"]=train.iloc[:,1:-1].values
    data["LDA"][mod]["DEV"]=dev.iloc[:,1:-1].values
    data["LDA"][mod]["TEST"]=test.iloc[:,1:-1].values

    print data["LDA"][mod]["TRAIN"].shape
data.sync()
data.close()