Quillot Mathias / volia

Compare View

switch
from
...
to
 
Commits (8)

Changes

Showing 7 changed files Side-by-side Diff

... ... @@ -7,4 +7,6 @@ data/
7 7 __pycache__
8 8 exp/
9 9 volia/*.pyc
10   -out.pdf
11 10 \ No newline at end of file
  11 +out.pdf
  12 +*.ipynb
  13 +*.pdf
12 14 \ No newline at end of file
... ... @@ -64,7 +64,7 @@ def read_features(file_path: str) -> KeyToFeatures:
64 64  
65 65 Returns a dictionary with id as key and a list of values as associated values
66 66 '''
67   - return read_id_values(file_path, float)
  67 + return read_id_values(file_path, np.float64)
68 68  
69 69  
70 70 def read_labels(file_path: str) -> KeyToLabels:
volia/core/persistency.py
Diff comments   View file @ 3b5a487
... ... @@ -0,0 +1,90 @@
  1 +import json
  2 +
  3 +"""
  4 +This module aims to manage environement variables local to the application.
  5 +These variables are persitent and located in a json file.
  6 +"""
  7 +
  8 +class PersistencyData(object):
  9 + """Initialize with an persistent file (json)
  10 +
  11 + You can access to environment variables using the brackets operators.
  12 + You can also check with the 'in' operator if a key is present among
  13 + the variables.
  14 +
  15 + example code:
  16 + ```
  17 + env = Env("myfile.json")
  18 +
  19 + if "MyVariable" in env:
  20 + print(env["MyVariable"])
  21 + else:
  22 + env["MyVariable"] = "New value"
  23 + ```
  24 +
  25 + """
  26 + def __init__(self, file: str):
  27 + self.__variables = {}
  28 + self.file = file
  29 + self.load()
  30 +
  31 + @property
  32 + def variables(self):
  33 + return self.__variables
  34 +
  35 + @property
  36 + def file(self):
  37 + return self.__file
  38 +
  39 + @file.setter
  40 + def file(self, file):
  41 + self.__file = file
  42 +
  43 +
  44 + def set(self, key: str, value: str):
  45 + """Modify an environment variable
  46 +
  47 + Args:
  48 + key (str): [description]
  49 + value ([type]): [description]
  50 + """
  51 + self.__variables[str]
  52 + self.save()
  53 +
  54 +
  55 + def get(self, key: str) -> str:
  56 + """Get an environment variable.
  57 +
  58 + Args:
  59 + key (str): [description]
  60 +
  61 + Returns:
  62 + [str]: [description]
  63 + """
  64 + return self.variables[key]
  65 +
  66 + def __getitem__(self, key):
  67 + print(key)
  68 + return self.variables[key]
  69 +
  70 + def __setitem__(self, key, value):
  71 + self.variables[key] = value
  72 +
  73 + def __delitem__(self, key):
  74 + del self.variables[key]
  75 +
  76 + def __contains__(self, key):
  77 + return key in self.variables
  78 +
  79 + def load(self):
  80 + """Load the variables from the json file.
  81 + """
  82 + with open(self.file, "r") as f:
  83 + self.__variables = json.load(f)
  84 +
  85 +
  86 + def save(self):
  87 + """Save the variables into the json file.
  88 + """
  89 + with open(self.file, "w") as f:
  90 + json.dump(self.variables, f)
... ... @@ -4,34 +4,36 @@ from utils import SubCommandRunner
4 4 #from decouple import Config, RepositoryEnv
5 5 import decouple
6 6 import json
7   -
  7 +import core.persistency
8 8  
9 9 # TODO:
10 10 # Load automatically if exists the persistency file
11 11 # Create a class that is a sigleton with configuration variables
  12 +# Search how to play `python -m volia.experiment deactivate` with the ``
  13 +# automatically.
12 14  
13   -VOLIA_ENV_VAR="VOLIA_EXPERIMENT"
14   -
  15 +def init(shell: str):
  16 + """
  17 + """
  18 + # TODO: add first scripts
  19 + if shell == "bash":
  20 + print("export VOLIA_EXPERIMENT=\"\"")
  21 + print("export VOLIA_BASH=\"bash\"")
  22 + else:
  23 + print("NO SHELL CHOOSE")
15 24  
16   -def init():
  25 +def initialize_files():
17 26 """
18   - Initialize volia
  27 + Initialize volia files : load environment file and persistency data.
19 28 """
20 29  
21 30 directory = VOLIA_HOME
22 31  
23 32 if not os.path.isdir(directory):
24 33 os.mkdir(directory)
25   -
26   - #config = decouple.AutoConfig(search_path=directory)
27   - #config = Config(RepositoryEnv(directory))
28 34  
29 35 persistency_file = os.path.join(directory, "persistency.json")
30   - env_file = os.path.join(directory, ".env")
31   -
32   - with open(env_file, "w") as f:
33   - f.write(f"VOLIA_PERSISTENCY=\"{persistency_file}\"")
34   - #exit(1)
  36 +
35 37 basic_persistency = {
36 38 "current_experiment": None
37 39 }
... ... @@ -41,87 +43,95 @@ def init():
41 43  
42 44  
43 45  
44   -def create():
  46 +def create(name: str):
  47 + print("Will allow the user to create an environment")
  48 + # TODO: check if the environment already exists
45 49 pass
46 50  
47 51 def show():
48   - print("Volia env var : " + VOLIA_ENV_VAR)
49   - print("Value : " + str(os.getenv(VOLIA_ENV_VAR)))
50   - config = decouple.AutoConfig(search_path="")
51   - pass
52   -
53   -def activate():
54   - pass
  52 + print("VOLIA_HOME=", VOLIA_HOME)
  53 + print("VOLIA_SHELL=", VOLIA_SHELL)
  54 + print("VOLIA EXPERIMENT=", VOLIA_EXPERIMENT)
  55 + print("Next feature: list the available environments")
  56 +
  57 +def activate(name: str):
  58 + # TODO: have a class hierarchy to manage all the possible
  59 + # shells. An abstract mother class will define all the needed methods.
  60 + # Or maybe an interface instead of abstract class.
  61 + if VOLIA_SHELL is None:
  62 + raise Exception("You need to define VOLIA_SHELL environment variable")
  63 +
  64 + if VOLIA_SHELL == "bash":
  65 + print(f"export VOLIA_EXPERIMENT={name}")
  66 +
  67 + else:
  68 + raise Exception(f"Do not know {VOLIA_SHELL} shell.")
  69 +
55 70  
56 71 def deactivate():
57   - #pycrosskit.envariables.SysEnv.set_var(VOLIA_ENV_VAR, "UNE VARIABLE")
58   - print("VARIABLE: " + pycrosskit.envariables.SysEnv.get_var(VOLIA_ENV_VAR, delete=True))
59   - print("Desactivation")
60   - print("New var : " + os.getenv(VOLIA_ENV_VAR))
61   - pass
  72 + if VOLIA_SHELL is None:
  73 + raise Exception("You need to define VOLIA_SHELL environment variable")
  74 +
  75 + if VOLIA_SHELL == "bash":
  76 + print("export VOLIA_EXPERIMENT=")
  77 +
  78 + else:
  79 + raise Exception(f"Do not know {VOLIA_SHELL} shell.")
  80 +
62 81  
63   -def remove():
  82 +def remove(name: str):
  83 + print("Under construction: Will soon allow you to remove an experimental environment.")
  84 + # TODO: check if the name exists
  85 + # Remove the experimental environment if the name exists.
64 86 pass
65 87  
66 88  
67 89 if __name__ == '__main__':
68 90  
69 91 # Load VOLIA_HOME env var
  92 + # TODO: encapsulate it into an environment class.
70 93 VOLIA_HOME = os.getenv("VOLIA_HOME")
  94 + VOLIA_SHELL = os.getenv("VOLIA_SHELL")
  95 + VOLIA_EXPERIMENT = os.getenv("VOLIA_EXPERIMENT")
71 96  
72 97 if VOLIA_HOME is None:
73 98 raise Exception("VOLIA_HOME is not defined. Please, define where you want to initialize VOLIA_HOME.\nWe suggest to set this variable to following value: /home/[your name]/.volia.")
74 99  
75   - # Load environment file if exists
76   - # TODO: Pas besoin de INIT finalement.
77   - CONFIG = None
78   - PERSISTENCY_DATA = None
79   - env_file = os.path.join(VOLIA_HOME, ".env")
80   - if os.path.isfile(env_file):
81   - # Load le fichier
82   - CONFIG = decouple.AutoConfig(search_path=VOLIA_HOME)
83   - # Then loading the persistency file
84   - persistency_file = os.path.join(VOLIA_HOME, "persistency.json")
85   -
86   - with open(persistency_file, "r") as f:
87   - PERSISTENCY_DATA = json.load(f)
88   -
89   - else:
90   - # Volia not initialized.
  100 + # Check if persistency file exists, or init
  101 + persistency_file = os.path.join(VOLIA_HOME, "persistency.json")
  102 + if not os.path.isfile(persistency_file):
91 103 print("VOLIA is not initialized. Initializing.")
92   - init()
93   - pass
  104 + initialize_files()
94 105  
95   - print("Config: ", CONFIG)
96   - print("PERSISTENCY_DATA: ", PERSISTENCY_DATA)
97   -
  106 + # Load persistency data
  107 + PERSISTENCY_DATA = core.persistency.PersistencyData(persistency_file)
98 108  
  109 + # PARSER
99 110 parser = argparse.ArgumentParser(description="...")
100 111 subparsers = parser.add_subparsers(title="actions")
101 112  
102   - # init : initialize the volia environment tool
  113 + # init : initialize the volia environment tool with shell script
103 114 parser_init = subparsers.add_parser("init", help="init volia environment")
  115 + parser_init.add_argument("--shell", type=str, default="bash", help="Help")
104 116 parser_init.set_defaults(which="init")
105 117  
106 118 # create a new env
107 119 parser_create = subparsers.add_parser("create", help="create an experimental environment")
  120 + parser_create.add_argument("name", type=str, help="name of the created environment")
108 121 parser_create.set_defaults(which="create")
109 122  
110 123 # activate env
111 124 parser_activate = subparsers.add_parser("activate", help="activate an experimental environment")
  125 + parser_activate.add_argument("name", type=str, help="environment name")
112 126 parser_activate.set_defaults(which="activate")
113   - '''
114   - Pour activater, il suffit d'ajouter la variable d'environnement.
115   - '''
  127 +
116 128 # deactivate env
117 129 parser_deactivate = subparsers.add_parser("deactivate", help="deactivate an experimental environment")
118 130 parser_deactivate.set_defaults(which="deactivate")
119   - '''
120   - Pour désactiver il suffit d'enlever la variable d'environnement.
121   - '''
122 131  
123 132 # remove env
124 133 parser_remove = subparsers.add_parser("remove", help="remove an experimental environment")
  134 + parser_remove.add_argument("name", type=str, help="name of the removed environment")
125 135 parser_remove.set_defaults(which="remove")
126 136  
127 137 # show env
... ... @@ -0,0 +1,58 @@
  1 +import argparse
  2 +import core.data
  3 +from utils import SubCommandRunner
  4 +
  5 +
  6 +def utt2char(features: str, outfile: str):
  7 + """Allow the user to generate utt2char file from masseffect features file.
  8 +
  9 + TODO: Don't forget to manage two cases: one with old ids, and an other with
  10 + new ones.
  11 +
  12 + Args:
  13 + features (str): [description]
  14 + outfile (str): [description]
  15 + """
  16 + data = core.data.read_features(features)
  17 + keys = list(data.keys())
  18 +
  19 + with open(outfile, "w") as f:
  20 + for key in keys:
  21 + splited = key.replace("\n", "").split(",")
  22 + character = splited[1]
  23 + f.write(",".join(splited) + " " + character + "\n")
  24 +
  25 +
  26 +def char2utt(features: str, outfile: str):
  27 + raise Exception("Not implemented yet")
  28 + pass
  29 +
  30 +
  31 +if __name__ == '__main__':
  32 + # Main parser
  33 + parser = argparse.ArgumentParser(description="...")
  34 + subparsers = parser.add_subparsers(title="action")
  35 +
  36 + # utt2char
  37 + parser_utt2char = subparsers.add_parser("utt2char")
  38 + parser_utt2char.add_argument("--features", type=str, help="features file")
  39 + parser_utt2char.add_argument("--outfile", type=str, help="output file")
  40 + parser_utt2char.set_defaults(which="utt2char")
  41 +
  42 + # char2utt
  43 + parser_char2utt = subparsers.add_parser("char2utt")
  44 + parser_char2utt.add_argument("--features", type=str, help="features file")
  45 + parser_char2utt.add_argument("--outfile", type=str, help="output file")
  46 + parser_char2utt.set_defaults(which="char2utt")
  47 +
  48 +
  49 + # Parse
  50 + args = parser.parse_args()
  51 +
  52 + # Run commands
  53 + runner = SubCommandRunner({
  54 + "utt2char" : utt2char,
  55 + "char2utt": char2utt,
  56 + })
  57 +
  58 + runner.run(args.which, args.__dict__, remove="which")
0 59 \ No newline at end of file
... ... @@ -53,7 +53,7 @@ if __name__ == '__main__':
53 53 parser = argparse.ArgumentParser(description="")
54 54 subparsers = parser.add_subparsers(title="action")
55 55  
56   - # with label
  56 + # scatter with labels
57 57 parser_scatter = subparsers.add_parser("scatter")
58 58 parser_scatter.add_argument("features", type=str, help="define the main features file")
59 59 parser_scatter.add_argument("--labels", default=None, type=str, help="specify the labels of each utterance/element")
... ... @@ -0,0 +1,114 @@
  1 +
  2 +import argparse
  3 +
  4 +import os
  5 +import core.data
  6 +import math
  7 +import numpy as np
  8 +import scipy.stats
  9 +import pickle
  10 +import matplotlib.pyplot as plt
  11 +import matplotlib.colors as mcolors
  12 +
  13 +
  14 +
  15 +from cycler import cycler
  16 +
  17 +def stats():
  18 + print("Decisions")
  19 +
  20 +
  21 +print(list(mcolors.TABLEAU_COLORS))
  22 +
  23 +
  24 +if __name__ == "__main__":
  25 +
  26 + # Parser
  27 + parser = argparse.ArgumentParser(description="")
  28 +
  29 + # Arguments
  30 + parser.add_argument("--predictions", type=str, help="prediction file", required=True)
  31 + parser.add_argument("--labels", type=str, help="label file", required=True)
  32 + parser.add_argument("--labelencoder", type=str, help="label encode pickle file", required=True)
  33 + parser.add_argument("--outdir", type=str, help="output file", required=True)
  34 +
  35 + args = parser.parse_args()
  36 +
  37 + predictions = core.data.read_id_values(args.predictions, float)
  38 + labels = core.data.read_labels(args.labels)
  39 +
  40 + le = None
  41 + with open(args.labelencoder, "rb") as f:
  42 + le = pickle.load(f)
  43 + stats = {}
  44 +
  45 + print("PREDICTIONS ---------------------------")
  46 + for id_, predictions_ in predictions.items():
  47 + label = labels[id_][0]
  48 + if label not in stats:
  49 + stats[label] = {
  50 + "nb_utt": 1,
  51 + "predictions": np.expand_dims(predictions_, axis=0)
  52 + }
  53 + else:
  54 + stats[label]["nb_utt"] = stats[label]["nb_utt"] + 1
  55 + stats[label]["predictions"] = np.append(stats[label]["predictions"], np.expand_dims(predictions_, axis=0), axis=0)
  56 +
  57 +
  58 + print("CALCULATING ---------------------------")
  59 +
  60 +
  61 + colors = [
  62 + "darkorange",
  63 + "red",
  64 + "blue"
  65 + ]
  66 + custom_cycler = (cycler(color=list(mcolors.TABLEAU_COLORS)) *
  67 + cycler(linestyle=['-', '--', '-.']))
  68 +
  69 +
  70 + for label, stats_ in stats.items():
  71 +
  72 + plt.gca().set_prop_cycle(custom_cycler)
  73 + stats_mean = np.mean(stats_["predictions"], axis=0)
  74 + stats_std = np.std(stats_["predictions"], axis=0)
  75 +
  76 + #print(label)
  77 + #print(stats_mean)
  78 + #print(stats_std)
  79 + kwargs = dict(alpha=0.5)
  80 +
  81 + for i in range(stats_["predictions"].shape[1]):
  82 + label_str = le.inverse_transform([i])[0]
  83 + #plt.hist(stats_["predictions"][:, i], bins=10, label=label_str, **kwargs)
  84 + mu = stats_mean[i]
  85 + variance = stats_std[i] * stats_std[i]
  86 + sigma = stats_std[i]
  87 + # math.sqrt(variance)
  88 + print(f"{i}: mu {mu}, var {variance}, sigma {sigma}")
  89 +
  90 + #x_values = np.arange(-1, 5, 0.1)
  91 +
  92 + #y_values = scipy.stats.norm(mu, variance)
  93 + #y = scipy.stats.norm.pdf(x,mean,std)
  94 +
  95 + #plt.plot(x_values, y_values.pdf(x_values,))
  96 +
  97 + #x, step = np.linspace(mu - 3*sigma, mu + 3*sigma, 1000, retstep=True)
  98 + x = np.linspace(0, 1, 1000)
  99 + #x = np.linspace(mu - 3*sigma, mu + 3*sigma, 1000)
  100 + #x, step = np.linspace(0, 1, 1000, retstep=True)
  101 +
  102 + P = scipy.stats.norm.cdf(x, mu, sigma)
  103 + #print(step)
  104 + plt.plot(x, P, label=label_str, **kwargs)
  105 + #plt.savefig("simple_gaussian.pdf")
  106 +
  107 + plt.legend()
  108 + plt.savefig(os.path.join(args.outdir, f"{label}_prediction_cdf.pdf"))
  109 + plt.clf()
  110 +
  111 +
  112 + # TODO:
  113 + # One graph for each label. Distribution of their predictions output are displayed.
  114 +