import sys 
  import os 
  
  import pandas 
  import numpy 
  import shelve
  
  from sklearn.preprocessing import LabelBinarizer
  
  from utils import select_mmf as select
  
  input_dir = sys.argv[1] # Dossier de premire niveau contient ASR et TRS
  level = sys.argv[2] # taille de LDA ( -5) voulu 
  
  lb=LabelBinarizer()
  #y_train=lb.fit_transform([utils.select(ligneid) for ligneid in origin_corps["LABEL"]["TRAIN"]])
  
  
  data = shelve.open("{}/mmf_{}.shelve".format(input_dir,level))
  data["LABEL"]= {"LDA":{}}
  for mod in ["ASR", "TRS" ]
      train = pandas.read_table("{}/{}/train_{}.ssv".format(input_dir, mod, level), sep=" ", header=None )
      dev = pandas.read_table("{}/{}/dev_{}.ssv".format(input_dir, mod, level), sep=" ", header=None )
      test = pandas.read_table("{}/{}/test_{}.ssv".format(input_dir, mod, level), sep=" ", header=None )
  
      y_train = train.iloc[:,0].apply(select)
      y_dev = dev.iloc[:,0].apply(select)
      y_test = test.iloc[:,0].apply(select)
      lb.fit(y_train)
      data["LABEL"][mod]={"TRAIN":lb.transform(y_train),"DEV":lb.transform(y_dev), "TEST": lb.transform(y_test)}
  
      data["LDA"][mod]={}
      data["LDA"][mod]["TRAIN"]=train.iloc[:,1:].values
      data["LDA"][mod]["DEV"]=dev.iloc[:,1:].values
      data["LDA"][mod]["TEST"]=test.iloc[:,1:].values
  
  data.sync()
  data.close()