Blame view
volia/clustering.py
6.63 KB
3b960e0f1
|
1 2 3 |
import argparse from os import path, mkdir from utils import SubCommandRunner |
9191399c3
|
4 |
from core.data import read_features, read_lst, read_labels |
3b960e0f1
|
5 6 7 |
import numpy as np from sklearn.cluster import KMeans import pickle |
9191399c3
|
8 9 10 |
from clustering_modules.kmeans import kmeans from sklearn.preprocessing import LabelEncoder |
fea9649a7
|
11 |
from sklearn.metrics import v_measure_score, homogeneity_score, completeness_score |
9191399c3
|
12 13 |
import core.measures |
3e2abe83e
|
14 |
import json |
9191399c3
|
15 16 17 18 19 20 21 22 |
CLUSTERING_METHODS = { "k-means": kmeans() } EVALUATION_METHODS = { "entropy": core.measures.entropy_score, |
fea9649a7
|
23 24 25 26 |
"purity": core.measures.purity_score, "v-measure": v_measure_score, "homogeneity": homogeneity_score, "completeness": completeness_score, |
9191399c3
|
27 28 29 30 31 32 33 34 |
} def disequilibrium_run(): pass def measure_run(measure: str, features: str, lst: str, truelabels: str, model: str, modeltype: str): |
3e2abe83e
|
35 36 37 38 39 40 41 42 43 44 |
""" @param measure: @param features: @param lst: @param truelabels: @param model: @param modeltype: @return: """ |
9191399c3
|
45 46 |
module = CLUSTERING_METHODS[modeltype] module.load(model) |
9191399c3
|
47 |
|
3e2abe83e
|
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
eval = {} for ms in measure: evaluation = EVALUATION_METHODS[ms] feats_dict = read_features(features) labels_dict = read_labels(truelabels) lst_dict = read_lst(lst) lst_keys = [key for key in lst_dict] feats = np.asarray([feats_dict[key] for key in lst_keys]) Y_pred = module.predict(feats) Y_truth = [labels_dict[key][0] for key in lst_keys] le = LabelEncoder() le.fit(Y_truth) Y_truth = le.transform(Y_truth) eval[ms] = evaluation(Y_truth, Y_pred) |
9191399c3
|
64 |
|
3e2abe83e
|
65 |
print(json.dumps(eval)) |
9191399c3
|
66 |
|
3b960e0f1
|
67 68 69 70 71 72 73 74 75 76 77 78 |
def kmeans_run(features: str, lst: str, k:int, kmax: int, klist, output: str): """ @param features: output features @param lst: list file @param k: k (kmin if kmax specified) @param kmax: maximum k to compute @param klist: list of k values to compute, ignore k value @param output: output file if kmax not specified, else, output directory """ |
9191399c3
|
79 |
# -- READ FILES -- |
3b960e0f1
|
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
features_dict = read_features(features) lst_dict = read_lst(lst) X = np.asarray([features_dict[x] for x in lst_dict]) # Exception cases if kmax is None and klist is None and path.isdir(output): raise Exception("The \"output\" is an existing directory while the system is waiting the path of a file.") if (kmax is not None or klist is not None) and path.isfile(output): raise Exception("The \"output\" is an existing file while the system is waiting the path of a directory.") # Mono value case if kmax is None and klist is None: print(f"Computing clustering with k={k}") kmeans = KMeans(n_clusters=k, n_init=10, random_state=0).fit(X) preds = kmeans.predict(X) pickle.dump(kmeans, open(output, "wb")) # Multi values case with kmax if kmax is not None: if not path.isdir(output): mkdir(output) Ks = range(k, kmax + 1) for i in Ks: print(f"Computing clustering with k={i}") kmeans = KMeans(n_clusters=i, n_init=10, random_state=0).fit(X) preds = kmeans.predict(X) pickle.dump(kmeans, open(path.join(output, "clustering_" + str(i) + ".pkl"), "wb")) # Second multi values case with klist if klist is not None: if not path.isdir(output): mkdir(output) for k in klist: k = int(k) print(f"Computing clustering with k={k}") kmeans = KMeans(n_clusters=k, n_init=10, random_state=0).fit(X) preds = kmeans.predict(X) pickle.dump(kmeans, open(path.join(output, "clustering_" + str(k) + ".pkl"), "wb")) if __name__ == "__main__": # Main parser parser = argparse.ArgumentParser(description="Clustering methods to apply") subparsers = parser.add_subparsers(title="action") # kmeans parser_kmeans = subparsers.add_parser( "kmeans", help="Compute clustering using k-means algorithm") parser_kmeans.add_argument("--features", required=True, type=str, help="Features file (works with list)") parser_kmeans.add_argument("--lst", required=True, type=str, help="List file (.lst)") parser_kmeans.add_argument("-k", default=2, type=int, help="number of clusters to compute. It is kmin if kmax is specified.") parser_kmeans.add_argument("--kmax", default=None, type=int, help="if specified, k is kmin.") parser_kmeans.add_argument("--klist", nargs="+", help="List of k values to test. As kmax, activate the multi values mod.") parser_kmeans.add_argument("--output", default=".kmeans", help="output file if only k. Output directory if multiple kmax specified.") parser_kmeans.set_defaults(which="kmeans") |
9191399c3
|
139 140 141 142 143 144 |
# measure parser_measure = subparsers.add_parser( "measure", help="compute the entropy") parser_measure.add_argument("--measure", required=True, |
3e2abe83e
|
145 |
nargs="+", |
9191399c3
|
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 |
choices=[key for key in EVALUATION_METHODS], help="...") parser_measure.add_argument("--features", required=True, type=str, help="...") parser_measure.add_argument("--lst", required=True, type=str, help="...") parser_measure.add_argument("--truelabels", required=True, type=str, help="...") parser_measure.add_argument("--model", required=True, type=str, help="...") parser_measure.add_argument("--modeltype", required=True, choices=[key for key in CLUSTERING_METHODS], help="type of model for learning") parser_measure.set_defaults(which="measure") # disequilibrium parser_disequilibrium = subparsers.add_parser( "disequilibrium", help="...") parser_disequilibrium.add_argument("--features", required=True, type=str, help="...") parser_disequilibrium.add_argument("--lstrain", required=True, type=str, help="...") parser_disequilibrium.add_argument("--lstest", required=True, type=str, help="...") parser_disequilibrium.add_argument("--model", required=True, type=str, help="...") parser_disequilibrium.add_argument("--model-type", required=True, choices=["kmeans", "2", "3"], help="...") |
e82889087
|
170 |
parser_disequilibrium.set_defaults(which="disequilibrium") |
9191399c3
|
171 |
|
3b960e0f1
|
172 173 174 175 176 |
# Parse args = parser.parse_args() # Run commands runner = SubCommandRunner({ |
9191399c3
|
177 178 179 |
"kmeans": kmeans_run, "measure": measure_run, "disequilibrium": disequilibrium_run |
3b960e0f1
|
180 |
}) |
9191399c3
|
181 |
runner.run(args.which, args.__dict__, remove="which") |