Jean-François Rey / otmedia

Download as
Browse Code »

Commit fa8235ed834209e698f0b8d999baa5bf86eb7ad2

Authored by Jean-François Rey
1 parent 2168563763
Exists in master

update

Showing 3 changed files with 5 additions and 3 deletions Inline Diff

cfg/ConfidenceMeasure.cfg
Diff comments   View file @ fa8235e
1 ################################################## 1 ##################################################
2 # File : ConfidenceMeasure.cfg 2 # File : ConfidenceMeasure.cfg
3 # Brief : ConfidenceMeasure.sh configuration file 3 # Brief : ConfidenceMeasure.sh configuration file
4 # 4 #
5 ################################################## 5 ##################################################
6 6
7 . ${OTMEDIA_HOME}/cfg/main_cfg.cfg
8
7 ROOT=$OTMEDIA_HOME/tools/PACKAGE_MESURES_V1.0 9 ROOT=$OTMEDIA_HOME/tools/PACKAGE_MESURES_V1.0
8 SRILM_BIN=$SRILM/bin/$MACHINE_TYPE 10 SRILM_BIN=$SRILM/bin/$MACHINE_TYPE
9 HOME_SPEERAL=$OTMEDIA_HOME/tools/Speeral 11 HOME_SPEERAL=$OTMEDIA_HOME/tools/Speeral
10 12
11 #FICHIER_RES="res_p2" # nom du repertoire à traiter dans le show (il y faut les .treil, .res, .res.trace) 13 #FICHIER_RES="res_p2" # nom du repertoire à traiter dans le show (il y faut les .treil, .res, .res.trace)
12 REP_PLP="PLP" # plp correspondants dans le show 14 REP_PLP="PLP" # plp correspondants dans le show
13 PENALITE=-12 15 PENALITE=-12
14 FUDGE=27 # e x le fudge de speeral (le arpa est en log10) 16 FUDGE=27 # e x le fudge de speeral (le arpa est en log10)
15 ORDER=4 #ordre du ML 17 ORDER=4 #ordre du ML
16 18
17 MaxProcess=8 19 MaxProcess=8
18 #MaxProcess = 8 20 #MaxProcess = 8
19 21
20 mod_ms="$HOME_SPEERAL/AM/Best_WB_M.hmm" # les différents modèeles accoustiques 22 mod_ms="$HOME_SPEERAL/AM/Best_WB_M.hmm" # les différents modèeles accoustiques
21 mod_fs="$HOME_SPEERAL/AM/Best_WB_F.hmm" 23 mod_fs="$HOME_SPEERAL/AM/Best_WB_F.hmm"
22 mod_mt="$HOME_SPEERAL/AM/Best_NB_M.hmm" 24 mod_mt="$HOME_SPEERAL/AM/Best_NB_M.hmm"
23 mod_ft="$HOME_SPEERAL/AM/Best_NB_F.hmm" 25 mod_ft="$HOME_SPEERAL/AM/Best_NB_F.hmm"
24 26
25 ML="$HOME_SPEERAL/ARPA/ML_OTMEDIA_LEXIQUE_V6.4g.arpa" 27 ML="$HOME_SPEERAL/ARPA/ML_OTMEDIA_LEXIQUE_V6.4g.arpa"
26 pathML="$HOME_SPEERAL/LM/" 28 pathML="$HOME_SPEERAL/LM/"
27 nameML="ML_4gOTMEDIA_LEXIQUE_V6" 29 nameML="ML_4gOTMEDIA_LEXIQUE_V6"
28 TYPE_ML="PAF" 30 TYPE_ML="PAF"
29 PHON="$HOME_SPEERAL/FMT/LEXIQUE_V6.fmt.phon" 31 PHON="$HOME_SPEERAL/FMT/LEXIQUE_V6.fmt.phon"
30 32
31 EXTEND=1 # extension des treillis speeral avec le ML 33 EXTEND=1 # extension des treillis speeral avec le ML
32 FASTNC=1 # generation des posteriors et reseaux de confusion 34 FASTNC=1 # generation des posteriors et reseaux de confusion
33 PPL=1 # analyse de la perplexite 35 PPL=1 # analyse de la perplexite
34 ACOUST=1 # analyse de l'accoustique 36 ACOUST=1 # analyse de l'accoustique
35 EXTRACT=1 # extraction et combinaison de tous les parametres precedents 37 EXTRACT=1 # extraction et combinaison de tous les parametres precedents
36 BOOST=1 # classification via boosting + génération des mesures 38 BOOST=1 # classification via boosting + génération des mesures
37 39
1 #!/bin/bash 1 #!/bin/bash
2 2
3 #-------------------# 3 #-------------------#
4 # OTMEDIA LIA # 4 # OTMEDIA LIA #
5 # Install script # 5 # Install script #
6 # version : 1.1.0 # 6 # version : 1.1.0 #
7 #-------------------# 7 #-------------------#
8 8
9 # nyx login name 9 # nyx login name
10 username=${LOGNAME} 10 username=${LOGNAME}
11 11
12 # Color variables 12 # Color variables
13 txtred=$(tput setaf 1) # red 13 txtred=$(tput setaf 1) # red
14 txtgrn=$(tput setaf 2) # Green 14 txtgrn=$(tput setaf 2) # Green
15 txtylw=$(tput setaf 3) # Yellow 15 txtylw=$(tput setaf 3) # Yellow
16 txtblu=$(tput setaf 4) # Blue 16 txtblu=$(tput setaf 4) # Blue
17 txtred=$(tput setaf 5) # Purple 17 txtred=$(tput setaf 5) # Purple
18 txtcyn=$(tput setaf 6) # Cyan 18 txtcyn=$(tput setaf 6) # Cyan
19 txtwht=$(tput setaf 7) # White 19 txtwht=$(tput setaf 7) # White
20 txtrst=$(tput sgr0) # Text reset. 20 txtrst=$(tput sgr0) # Text reset.
21 #/color 21 #/color
22 22
23 # 23 #
24 ### Global Variables 24 ### Global Variables
25 # 25 #
26 PWD=$(pwd) 26 PWD=$(pwd)
27 OTMEDIA_HOME=$PWD 27 OTMEDIA_HOME=$PWD
28 test=$(arch) 28 test=$(arch)
29 if [ "$test" == "x86_64" ]; then ARCH=".64"; else ARCH=""; fi 29 if [ "$test" == "x86_64" ]; then ARCH=".64"; else ARCH=""; fi
30 #/Global 30 #/Global
31 31
32 32
33 # 33 #
34 # Put to 0 to disable dependencies of a pass 34 # Put to 0 to disable dependencies of a pass
35 # and 1 to enable 35 # and 1 to enable
36 # 36 #
37 PASS1=1 # First Pass 37 PASS1=1 # First Pass
38 PASS2=1 # Second and Third Pass 38 PASS2=1 # Second and Third Pass
39 CONFPASS=1 # Confidence Pass 39 CONFPASS=1 # Confidence Pass
40 EXPLOITCONFPASS=1 # SOLR query and trigg 40 EXPLOITCONFPASS=1 # SOLR query and trigg
41 41
42 echo -e "\nWill do install for :" 42 echo -e "\nWill do install for :"
43 if [ $PASS1 -eq 1 ];then echo "- Pass 1";fi 43 if [ $PASS1 -eq 1 ];then echo "- Pass 1";fi
44 if [ $PASS2 -eq 1 ];then echo "- Pass 2";fi 44 if [ $PASS2 -eq 1 ];then echo "- Pass 2";fi
45 if [ $CONFPASS -eq 1 ];then echo "- Confidence Pass";fi 45 if [ $CONFPASS -eq 1 ];then echo "- Confidence Pass";fi
46 if [ $EXPLOITCONFPASS -eq 1 ];then echo "- Exploit Confidence Pass";fi 46 if [ $EXPLOITCONFPASS -eq 1 ];then echo "- Exploit Confidence Pass";fi
47 47
48 # 48 #
49 ### CHECK Dependencies ### 49 ### CHECK Dependencies ###
50 # 50 #
51 echo -e "\n\t${txtblu}Check Dependencies${txtrst}\n" 51 echo -e "\n\t${txtblu}Check Dependencies${txtrst}\n"
52 52
53 ## make 53 ## make
54 test=$(whereis make) 54 test=$(whereis make)
55 if [ "$test" == "make:" ] 55 if [ "$test" == "make:" ]
56 then 56 then
57 echo -e "${txtred}ERROR${txtrst} make not found\n You have to install make\n sudo apt-get install make" 57 echo -e "${txtred}ERROR${txtrst} make not found\n You have to install make\n sudo apt-get install make"
58 exit 1; 58 exit 1;
59 fi 59 fi
60 echo -e "make \t ${txtgrn}OK${txtrst}" 60 echo -e "make \t ${txtgrn}OK${txtrst}"
61 61
62 ## CC 62 ## CC
63 test=$(whereis cc) 63 test=$(whereis cc)
64 if [ "$test" == "cc:" ] 64 if [ "$test" == "cc:" ]
65 then 65 then
66 echo -e "${txtred}ERROR${txtrst} cc not found\n You have to install cc\n sudo apt-get install gcc" 66 echo -e "${txtred}ERROR${txtrst} cc not found\n You have to install cc\n sudo apt-get install gcc"
67 exit 1; 67 exit 1;
68 fi 68 fi
69 echo -e "cc \t ${txtgrn}OK${txtrst}" 69 echo -e "cc \t ${txtgrn}OK${txtrst}"
70 70
71 ## AVCONV 71 ## AVCONV
72 test=$(whereis avconv) 72 test=$(whereis avconv)
73 if [ "$test" == "avconv:" ] 73 if [ "$test" == "avconv:" ]
74 then 74 then
75 echo -e "${txtred}ERROR${txtrst} avconv not found\n You have to install avconv\n sudo apt-get install libav-tools" 75 echo -e "${txtred}ERROR${txtrst} avconv not found\n You have to install avconv\n sudo apt-get install libav-tools"
76 exit 1; 76 exit 1;
77 fi 77 fi
78 echo -e "libav-tools : avconv \t ${txtgrn}OK${txtrst}" 78 echo -e "libav-tools : avconv \t ${txtgrn}OK${txtrst}"
79 79
80 ## JAVA 80 ## JAVA
81 test=$(whereis java) 81 test=$(whereis java)
82 if [ "$test" == "java:" ] 82 if [ "$test" == "java:" ]
83 then 83 then
84 echo -e "${txtred}ERROR${txtrst} java not found\n You have to install java JRE\n sudo apt-get install openjdk-7-jre" 84 echo -e "${txtred}ERROR${txtrst} java not found\n You have to install java JRE\n sudo apt-get install openjdk-7-jre"
85 exit 1; 85 exit 1;
86 fi 86 fi
87 echo -e "Java : JRE \t ${txtgrn}OK${txtrst}" 87 echo -e "Java : JRE \t ${txtgrn}OK${txtrst}"
88 test=$(whereis javac) 88 test=$(whereis javac)
89 if [ "$test" == "javac:" ] 89 if [ "$test" == "javac:" ]
90 then 90 then
91 echo -e "${txtred}ERROR${txtrst} javac not found\n You have to install java JDK\n sudo apt-get install openjdk-7-jdk" 91 echo -e "${txtred}ERROR${txtrst} javac not found\n You have to install java JDK\n sudo apt-get install openjdk-7-jdk"
92 exit 1; 92 exit 1;
93 fi 93 fi
94 echo -e "Java : JDK \t ${txtgrn}OK${txtrst}" 94 echo -e "Java : JDK \t ${txtgrn}OK${txtrst}"
95 95
96 if [ $EXPLOITCONFPASS -eq 1 ] 96 if [ $EXPLOITCONFPASS -eq 1 ]
97 then 97 then
98 ## Python 98 ## Python
99 test=$(whereis python) 99 test=$(whereis python)
100 if [ "$test" == "python:" ] 100 if [ "$test" == "python:" ]
101 then 101 then
102 echo -e "${txtred}ERROR${txtrst} python not found\n You have to install python\n sudo apt-get install python" 102 echo -e "${txtred}ERROR${txtrst} python not found\n You have to install python\n sudo apt-get install python"
103 exit 1; 103 exit 1;
104 fi 104 fi
105 echo -e "python : \t ${txtgrn}OK${txtrst}" 105 echo -e "python : \t ${txtgrn}OK${txtrst}"
106 106
107 ## csh shell 107 ## csh shell
108 test=$(whereis csh) 108 test=$(whereis csh)
109 if [ "$test" == "csh:" ] 109 if [ "$test" == "csh:" ]
110 then 110 then
111 echo -e "${txtred}ERROR${txtrst} csh shell not found\n You have to install csh shell\n sudo apt-get install csh" 111 echo -e "${txtred}ERROR${txtrst} csh shell not found\n You have to install csh shell\n sudo apt-get install csh"
112 exit 1; 112 exit 1;
113 fi 113 fi
114 echo -e "csh shell : \t ${txtgrn}OK${txtrst}" 114 echo -e "csh shell : \t ${txtgrn}OK${txtrst}"
115 fi 115 fi
116 116
117 ## Perl 117 ## Perl
118 test=$(whereis perl) 118 test=$(whereis perl)
119 if [ "$test" == "perl:" ] 119 if [ "$test" == "perl:" ]
120 then 120 then
121 echo -e "${txtred}ERROR${txtrst} perl not found\n You have to install perl\n sudo apt-get install perl" 121 echo -e "${txtred}ERROR${txtrst} perl not found\n You have to install perl\n sudo apt-get install perl"
122 exit 1; 122 exit 1;
123 fi 123 fi
124 echo -e "perl : \t ${txtgrn}OK${txtrst}" 124 echo -e "perl : \t ${txtgrn}OK${txtrst}"
125 125
126 ## iconv 126 ## iconv
127 test=$(whereis iconv) 127 test=$(whereis iconv)
128 if [ "$test" == "iconv:" ] 128 if [ "$test" == "iconv:" ]
129 then 129 then
130 echo -e "${txtred}ERROR${txtrst} iconv not found\n You have to install iconv\n sudo apt-cache search iconv" 130 echo -e "${txtred}ERROR${txtrst} iconv not found\n You have to install iconv\n sudo apt-cache search iconv"
131 exit 1; 131 exit 1;
132 fi 132 fi
133 echo -e "iconv : \t ${txtgrn}OK${txtrst}" 133 echo -e "iconv : \t ${txtgrn}OK${txtrst}"
134 134
135 ## SRI LM 135 ## SRI LM
136 if [ -z "$SRILM" ] && [ -z "$MACHINE_TYPE" ] 136 if [ -z "$SRILM" ] && [ -z "$MACHINE_TYPE" ]
137 then 137 then
138 echo -e "${txtred}ERROR${txtrst} SRILM toolkit variables are not defined (SRILM and MACHINE_TYPE)\n You have to install SRILM Toolkit\n" 138 echo -e "${txtred}ERROR${txtrst} SRILM toolkit variables are not defined (SRILM and MACHINE_TYPE)\n You have to install SRILM Toolkit\n"
139 exit 1; 139 exit 1;
140 fi 140 fi
141 export SRILM_BIN=$SRILM/bin/$MACHINE_TYPE 141 export SRILM_BIN=$SRILM/bin/$MACHINE_TYPE
142 echo -e "SRILM toolkit : \t ${txtgrn}OK${txtrst}" 142 echo -e "SRILM toolkit : \t ${txtgrn}OK${txtrst}"
143 143
144 ### Speeral Configuration ### 144 ### Speeral Configuration ###
145 145
146 echo -e "\n\t${txtblu}Speeral configuration${txtrst}\n" 146 echo -e "\n\t${txtblu}Speeral configuration${txtrst}\n"
147 read -e -p "Download Speeral bin and data ? (y/n) " speeral 147 read -e -p "Download Speeral bin and data ? (y/n) " speeral
148 if [ "$speeral" == "y" ] 148 if [ "$speeral" == "y" ]
149 then 149 then
150 echo -e "Download Speeral bin and data :" 150 echo -e "Download Speeral bin and data :"
151 scp -r ${username}@nyx:/local/OTMEDIA/OTMEDIA_DATA/Speeral $OTMEDIA_HOME/tools/ 151 scp -r ${username}@nyx:/local/OTMEDIA/OTMEDIA_DATA/Speeral $OTMEDIA_HOME/tools/
152 fi 152 fi
153 echo -e "\n\t${txtblu}Generating Speeral configuration files :${txtrst}\n" 153 echo -e "\n\t${txtblu}Generating Speeral configuration files :${txtrst}\n"
154 cat $PWD/tools/Speeral/CFG/SpeeralFirstPass.xml.tmp | sed -e "s|<nom>[^<]*</nom>|<nom>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer</nom>|g" \ 154 cat $PWD/tools/Speeral/CFG/SpeeralFirstPass.xml.tmp | sed -e "s|<nom>[^<]*</nom>|<nom>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer</nom>|g" \
155 | sed -e "s|<ngramme>[^<]*</ngramme>|<ngramme>$PWD/tools/Speeral/LM/ML_4gOTMEDIA_LEXIQUE_V6</ngramme>|g" \ 155 | sed -e "s|<ngramme>[^<]*</ngramme>|<ngramme>$PWD/tools/Speeral/LM/ML_4gOTMEDIA_LEXIQUE_V6</ngramme>|g" \
156 | sed -e "s|<binode>[^<]*</binode>|<binode>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer.bin</binode>|g" \ 156 | sed -e "s|<binode>[^<]*</binode>|<binode>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer.bin</binode>|g" \
157 > $PWD/tools/Speeral/CFG/SpeeralFirstPass.xml 157 > $PWD/tools/Speeral/CFG/SpeeralFirstPass.xml
158 echo $PWD/tools/Speeral/CFG/SpeeralFirstPass.xml 158 echo $PWD/tools/Speeral/CFG/SpeeralFirstPass.xml
159 cat $PWD/tools/Speeral/CFG/SpeeralSecondPass.xml.tmp | sed -e "s|<nom>[^<]*</nom>|<nom>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer</nom>|g" \ 159 cat $PWD/tools/Speeral/CFG/SpeeralSecondPass.xml.tmp | sed -e "s|<nom>[^<]*</nom>|<nom>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer</nom>|g" \
160 | sed -e "s|<ngramme>[^<]*</ngramme>|<ngramme>$PWD/tools/Speeral/LM/ML_4gOTMEDIA_LEXIQUE_V6</ngramme>|g" \ 160 | sed -e "s|<ngramme>[^<]*</ngramme>|<ngramme>$PWD/tools/Speeral/LM/ML_4gOTMEDIA_LEXIQUE_V6</ngramme>|g" \
161 | sed -e "s|<binode>[^<]*</binode>|<binode>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer.bin</binode>|g" \ 161 | sed -e "s|<binode>[^<]*</binode>|<binode>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer.bin</binode>|g" \
162 > $PWD/tools/Speeral/CFG/SpeeralSecondPass.xml 162 > $PWD/tools/Speeral/CFG/SpeeralSecondPass.xml
163 echo $PWD/tools/Speeral/CFG/SpeeralSecondPass.xml 163 echo $PWD/tools/Speeral/CFG/SpeeralSecondPass.xml
164 cat $PWD/tools/Speeral/CFG/SpeeralThirdPass.xml.tmp | sed -e "s|<nom>[^<]*</nom>|<nom>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer</nom>|g" \ 164 cat $PWD/tools/Speeral/CFG/SpeeralThirdPass.xml.tmp | sed -e "s|<nom>[^<]*</nom>|<nom>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer</nom>|g" \
165 | sed -e "s|<ngramme>[^<]*</ngramme>|<ngramme>$PWD/tools/Speeral/LM/ML_4gOTMEDIA_LEXIQUE_V6</ngramme>|g" \ 165 | sed -e "s|<ngramme>[^<]*</ngramme>|<ngramme>$PWD/tools/Speeral/LM/ML_4gOTMEDIA_LEXIQUE_V6</ngramme>|g" \
166 | sed -e "s|<binode>[^<]*</binode>|<binode>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer.bin</binode>|g" \ 166 | sed -e "s|<binode>[^<]*</binode>|<binode>$PWD/tools/Speeral/LEX/LEXIQUE_V6.speer.bin</binode>|g" \
167 > $PWD/tools/Speeral/CFG/SpeeralThirdPass.xml 167 > $PWD/tools/Speeral/CFG/SpeeralThirdPass.xml
168 echo $PWD/tools/Speeral/CFG/SpeeralThirdPass.xml 168 echo $PWD/tools/Speeral/CFG/SpeeralThirdPass.xml
169 169
170 170
171 if [ $EXPLOITCONFPASS -eq 1 ] 171 if [ $EXPLOITCONFPASS -eq 1 ]
172 then 172 then
173 ### LIA ltbox ### 173 ### LIA ltbox ###
174 echo -e "\t${txtblu}Install lia_ltbox${txtrst}\n" 174 echo -e "\t${txtblu}Install lia_ltbox${txtrst}\n"
175 export LIA_TAGG_LANG="french" 175 export LIA_TAGG_LANG="french"
176 export LIA_TAGG="$OTMEDIA_HOME/tools/lia_ltbox/lia_tagg/" 176 export LIA_TAGG="$OTMEDIA_HOME/tools/lia_ltbox/lia_tagg/"
177 export LIA_PHON_REP="$OTMEDIA_HOME/tools/lia_ltbox/lia_phon/" 177 export LIA_PHON_REP="$OTMEDIA_HOME/tools/lia_ltbox/lia_phon/"
178 export LIA_BIGLEX="$OTMEDIA_HOME/tools/lia_ltbox/lia_biglex/" 178 export LIA_BIGLEX="$OTMEDIA_HOME/tools/lia_ltbox/lia_biglex/"
179 179
180 ### config lia_phon 180 ### config lia_phon
181 cd $LIA_PHON_REP 181 cd $LIA_PHON_REP
182 make all > /dev/null 182 make all > /dev/null
183 make ressource > /dev/null 183 make ressource > /dev/null
184 ### config lia_tagg 184 ### config lia_tagg
185 cd $LIA_TAGG 185 cd $LIA_TAGG
186 make all > /dev/null 186 make all > /dev/null
187 make ressource.french > /dev/null 187 make ressource.french > /dev/null
188 ### config lia_biglex 188 ### config lia_biglex
189 cd $LIA_BIGLEX 189 cd $LIA_BIGLEX
190 make -f makefile.biglex > /dev/null 190 make -f makefile.biglex > /dev/null
191 cd $OTMEDIA_HOME 191 cd $OTMEDIA_HOME
192 192
193 193
194 ### SOLR DB ### 194 ### SOLR DB ###
195 # Tomcat fisrtly 195 # Tomcat fisrtly
196 test=$(dpkg -l | grep "^ii" | grep tomcat) 196 test=$(dpkg -l | grep "^ii" | grep tomcat)
197 if [ "$test" == "" ] 197 if [ "$test" == "" ]
198 then 198 then
199 echo -e "${txtred}ERROR${txtrst} TOMCAT seems to not be installed)\n You have to install TOMCAT\n" 199 echo -e "${txtred}ERROR${txtrst} TOMCAT seems to not be installed)\n You have to install TOMCAT\n"
200 #exit 1; 200 #exit 1;
201 fi 201 fi
202 echo -e "\nTOMCAT : \t ${txtgrn}OK${txtrst}\n" 202 echo -e "\nTOMCAT : \t ${txtgrn}OK${txtrst}\n"
203 # SOLR secondly 203 # SOLR secondly
204 echo -e "\t${txtblu}Install SOLR DB${txtrst}\n" 204 echo -e "\t${txtblu}Install SOLR DB${txtrst}\n"
205 echo -e "You will need 300 Go of free space to install SOLR DB" 205 echo -e "You will need 300 Go of free space to install SOLR DB"
206 read -p "Continue ? (y/n) " solr 206 read -p "Continue ? (y/n) " solr
207 if [ "$solr" == "y" ] 207 if [ "$solr" == "y" ]
208 then 208 then
209 209
210 echo -e "Download SOLR DB\r" 210 echo -e "Download SOLR DB\r"
211 mkdir -p $OTMEDIA_HOME/tools/SOLR 2> /dev/null 211 mkdir -p $OTMEDIA_HOME/tools/SOLR 2> /dev/null
212 scp -r ${username}@nyx:/local/OTMEDIA/OTMEDIA_DATA/SOLR/otmedia-2013-04.tar.gz $OTMEDIA_HOME/tools/SOLR 212 scp -r ${username}@nyx:/local/OTMEDIA/OTMEDIA_DATA/SOLR/otmedia-2013-04.tar.gz $OTMEDIA_HOME/tools/SOLR
213 echo -e "Unzip SOLR DB\r" 213 echo -e "Unzip SOLR DB\r"
214 res=0 214 res=0
215 #res = $(tar -xvzf "$OTMEDIA_HOME/tools/SOLR/otmedia-2013-04.tar.gz" "$OTMEDIA_HOME/tools/SOLR/") 215 #res = $(tar -xvzf "$OTMEDIA_HOME/tools/SOLR/otmedia-2013-04.tar.gz" "$OTMEDIA_HOME/tools/SOLR/")
216 if [ $res -eq 2 ]; then echo " ${txtred}NOT OK${txtrst}"; 216 if [ $res -eq 2 ]; then echo " ${txtred}NOT OK${txtrst}";
217 else echo " ${txtgrn}OK${txtrst}"; fi 217 else echo " ${txtgrn}OK${txtrst}"; fi
218 else 218 else
219 echo "Skipping SOLR install" 219 echo "Skipping SOLR install"
220 fi 220 fi
221 read -e -p "Configure SOLR DB server ? (y/n) " solr 221 read -e -p "Configure SOLR DB server ? (y/n) " solr
222 if [ "$solr" == "y" ] 222 if [ "$solr" == "y" ]
223 then 223 then
224 read -p "Enter SOLR server IP :" ip 224 read -p "Enter SOLR server IP :" ip
225 if [ "${ip}" == "" ];then ip="localhost";fi 225 if [ "${ip}" == "" ];then ip="localhost";fi
226 echo "machine = \"${ip}\"" > $OTMEDIA_HOME/tools/scripts/solrinfo.py 226 echo "machine = \"${ip}\"" > $OTMEDIA_HOME/tools/scripts/solrinfo.py
227 read -p "Enter SOLR server port :" port 227 read -p "Enter SOLR server port :" port
228 if [ "${port}" == "" ]; then port="8080";fi 228 if [ "${port}" == "" ]; then port="8080";fi
229 echo -e "\n\tSOLR server IP ${ip}" 229 echo -e "\n\tSOLR server IP ${ip}"
230 echo -e "\tSOLR server port ${port}" 230 echo -e "\tSOLR server port ${port}"
231 echo "port = \"${port}\"" >> $OTMEDIA_HOME/tools/scripts/solrinfo.py 231 echo "port = \"${port}\"" >> $OTMEDIA_HOME/tools/scripts/solrinfo.py
232 else 232 else
233 echo "Skipping SOLR DB Configuration" 233 echo "Skipping SOLR DB Configuration"
234 fi 234 fi
235 echo -e "\nSee SOLR.INSTALL file for more information\n" 235 echo -e "\nSee SOLR.INSTALL file for more information\n"
236 fi 236 fi
237 237
238 ### Set Variables in bashrc ### 238 ### Set Variables in bashrc ###
239 cat ~/.bashrc | grep -v "OTMEDIA_HOME" | grep -v "SRILM_BIN" | grep -v "main_tools" > ~/.bashrc.org 239 cat ~/.bashrc | grep -v "OTMEDIA_HOME" | grep -v "SRILM_BIN" | grep -v "main_tools" > ~/.bashrc.org
240 #cat ~/.bashrc | grep -v "OTMEDIA_HOME" | grep -v "SRILM_BIN" | grep -v "LIA_TAGG" | grep -v "LIA_PHON" | grep -v "LIA_BIGLEX" > ~/.bashrc.org 240 #cat ~/.bashrc | grep -v "OTMEDIA_HOME" | grep -v "SRILM_BIN" | grep -v "LIA_TAGG" | grep -v "LIA_PHON" | grep -v "LIA_BIGLEX" > ~/.bashrc.org
241 cp ~/.bashrc.org ~/.bashrc 241 cp ~/.bashrc.org ~/.bashrc
242 export OTMEDIA_HOME=$PWD 242 export OTMEDIA_HOME=$PWD
243 echo "export OTMEDIA_HOME=$PWD" >> ~/.bashrc 243 echo "export OTMEDIA_HOME=$PWD" >> ~/.bashrc
244 echo "export PATH=\$PATH:$PWD/main_tools" >> ~/.bashrc 244 echo "export PATH=\$PATH:$PWD/main_tools" >> ~/.bashrc
245 echo "export SRILM_BIN=$SRILM/bin/$MACHINE_TYPE" >> ~/.bashrc 245 echo "export SRILM_BIN=$SRILM/bin/$MACHINE_TYPE" >> ~/.bashrc
246 #echo "export LIA_TAGG_LANG=french" >> ~/.bashrc 246 #echo "export LIA_TAGG_LANG=french" >> ~/.bashrc
247 #echo "export LIA_TAGG=$OTMEDIA_HOME/tools/lia_ltbox/lia_tagg/" >> ~/.bashrc 247 #echo "export LIA_TAGG=$OTMEDIA_HOME/tools/lia_ltbox/lia_tagg/" >> ~/.bashrc
248 #echo "export LIA_PHON_REP=$OTMEDIA_HOME/tools/lia_ltbox/lia_phon/" >> ~/.bashrc 248 #echo "export LIA_PHON_REP=$OTMEDIA_HOME/tools/lia_ltbox/lia_phon/" >> ~/.bashrc
249 #echo "export LIA_BIGLEX=$OTMEDIA_HOME/tools/lia_ltbox/lia_biglex/" >> ~/.bashrc 249 #echo "export LIA_BIGLEX=$OTMEDIA_HOME/tools/lia_ltbox/lia_biglex/" >> ~/.bashrc
250 250
251 # set global configuration file 251 # set global configuration file
252 echo "OTMEDIA_HOME=$PWD" > $OTMEDIA_HOME/cfg/main_cfg.cfg 252 echo "OTMEDIA_HOME=\"$PWD\"" > $OTMEDIA_HOME/cfg/main_cfg.cfg
253 echo "ARCH=$ARCH" >> $OTMEDIA_HOME/cfg/main_cfg.cfg 253 echo "ARCH=\"$ARCH\"" >> $OTMEDIA_HOME/cfg/main_cfg.cfg
254 echo "VERBOSE=0" >> $OTMEDIA_HOME/cfg/main_cfg.cfg 254 echo "VERBOSE=0" >> $OTMEDIA_HOME/cfg/main_cfg.cfg
255 echo "DEBUG=0" >> $OTMEDIA_HOME/cfg/main_cfg.cfg 255 echo "DEBUG=0" >> $OTMEDIA_HOME/cfg/main_cfg.cfg
256 echo "CHECK=0" >> $OTMEDIA_HOME/cfg/main_cfg.cfg 256 echo "CHECK=0" >> $OTMEDIA_HOME/cfg/main_cfg.cfg
257 echo "RERUN=0" >> $OTMEDIA_HOME/cfg/main_cfg.cfg 257 echo "RERUN=0" >> $OTMEDIA_HOME/cfg/main_cfg.cfg
258 258
259 echo -e "\n\t${txtgrn}### Install completed ###${txtrst}\n" 259 echo -e "\n\t${txtgrn}### Install completed ###${txtrst}\n"
260 echo -e "do : source ~/.bashrc" 260 echo -e "do : source ~/.bashrc"
261 echo -e "or set variable :\n" 261 echo -e "or set variable :\n"
262 echo "export OTMEDIA_HOME=$PWD" 262 echo "export OTMEDIA_HOME=$PWD"
263 echo "export PATH=\$PATH:$OTMEDIA_HOME/main_tools" 263 echo "export PATH=\$PATH:$OTMEDIA_HOME/main_tools"
264 echo "export SRILM_BIN=$SRILM/bin/$MACHINE_TYPE" 264 echo "export SRILM_BIN=$SRILM/bin/$MACHINE_TYPE"
265 265
266 266
267 echo "${txtwht} \\\\ " 267 echo "${txtwht} \\\\ "
268 echo " ,-~~~-\\\\_" 268 echo " ,-~~~-\\\\_"
269 echo " ( .\ " 269 echo " ( .\ "
270 echo " @\___(__--'${txtrst}" 270 echo " @\___(__--'${txtrst}"
271 271
272 echo "${txtgrn}Yes${txtylw}I${txtred}Rastafari${txtrst}" 272 echo "${txtgrn}Yes${txtylw}I${txtred}Rastafari${txtrst}"
273 273
main_tools/ConfidenceMeasure.sh
Diff comments   View file @ fa8235e
1 #!/bin/bash 1 #!/bin/bash
2 #----------------------------------------------------------------------------------------- 2 #-----------------------------------------------------------------------------------------
3 # Author : Benjamin Lecouteux & Emmanuel FERREIRA (contact emmanuel.ferreira0194@gmail.com) 3 # Author : Benjamin Lecouteux & Emmanuel FERREIRA (contact emmanuel.ferreira0194@gmail.com)
4 # Brief: Determine les mesures de confiance d'une transcription (res de speeral) 4 # Brief: Determine les mesures de confiance d'une transcription (res de speeral)
5 #----------------------------------------------------------------------------------------- 5 #-----------------------------------------------------------------------------------------
6 6
7 # where is ConfidenceMeasure.sh 7 # where is ConfidenceMeasure.sh
8 if [ -z $MAIN_SCRIPT_PATH ]; then MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)); fi 8 if [ -z $MAIN_SCRIPT_PATH ]; then MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)); fi
9 9
10 # where is ConfidenceMeasure.cfg 10 # where is ConfidenceMeasure.cfg
11 CONFIDENCEMEASURE_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ConfidenceMeasure.cfg" 11 CONFIDENCEMEASURE_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ConfidenceMeasure.cfg"
12 if [ -e $CONFIDENCEMEASURE_CONFIG_FILE ] 12 if [ -e $CONFIDENCEMEASURE_CONFIG_FILE ]
13 then 13 then
14 . $CONFIDENCEMEASURE_CONFIG_FILE 14 . $CONFIDENCEMEASURE_CONFIG_FILE
15 else 15 else
16 echo "ERROR : Can't find configuration file $CONFIDENCEMEASURE_CONFIG_FILE" >&2 16 echo "ERROR : Can't find configuration file $CONFIDENCEMEASURE_CONFIG_FILE" >&2
17 exit 1 17 exit 1
18 fi 18 fi
19 19
20 PACKAGE_CONF_MEASURE=$CONFIDENCEMEASURE_CONFIG_FILE 20 PACKAGE_CONF_MEASURE=$CONFIDENCEMEASURE_CONFIG_FILE
21 #------------------ 21 #------------------
22 # Parser les options 22 # Parser les options
23 #------------------- 23 #-------------------
24 while getopts ":c:s:h" OPTION 24 while getopts ":c:s:h" OPTION
25 do 25 do
26 case $OPTION in 26 case $OPTION in
27 h) #Display help 27 h) #Display help
28 echo -e "$0 :" 28 echo -e "$0 :"
29 echo -e "\tAuthor : Benjamin Lecouteux & Emmanuel FERREIRA (contact: emmanuel.ferreira0194@gmail.com)" 29 echo -e "\tAuthor : Benjamin Lecouteux & Emmanuel FERREIRA (contact: emmanuel.ferreira0194@gmail.com)"
30 echo -e "\tVersion : 2.0" 30 echo -e "\tVersion : 2.0"
31 echo -e "\tBrief : Determine confidence measure of a transcription" 31 echo -e "\tBrief : Determine confidence measure of a transcription"
32 echo -e "\tUsage : $0 [OPTIONS] <(i) REP_IN> <REP_NAME>" 32 echo -e "\tUsage : $0 [OPTIONS] <(i) REP_IN> <REP_NAME>"
33 echo -e "\tOptions:" 33 echo -e "\tOptions:"
34 echo -e "\t\tc) specify the path of the configuration file (default $PACKAGE_CONF_MEASURE)" 34 echo -e "\t\tc) specify the path of the configuration file (default $PACKAGE_CONF_MEASURE)"
35 echo -e "\t\ts) specify PORT@HOST of a SRILM server" 35 echo -e "\t\ts) specify PORT@HOST of a SRILM server"
36 exit 1 36 exit 1
37 ;; 37 ;;
38 c) #Change the configuration file 38 c) #Change the configuration file
39 PACKAGE_CONF_MEASURE=$OPTARG 39 PACKAGE_CONF_MEASURE=$OPTARG
40 ;; 40 ;;
41 s) #use an SRILM server (avoid loading arpa model in memory) 41 s) #use an SRILM server (avoid loading arpa model in memory)
42 SERVER=$OPTARG 42 SERVER=$OPTARG
43 ;; 43 ;;
44 :) 44 :)
45 echo "BAD USAGE : OPTION $OPTARG need a value" 45 echo "BAD USAGE : OPTION $OPTARG need a value"
46 exit 1 46 exit 1
47 ;; 47 ;;
48 \?) 48 \?)
49 echo "BAD USAGE : unknow option '$OPTARG'" 49 echo "BAD USAGE : unknow option '$OPTARG'"
50 exit 1 50 exit 1
51 ;; 51 ;;
52 esac 52 esac
53 done 53 done
54 54
55 #------------------------------------------- 55 #-------------------------------------------
56 # Shift options pour recuperation arguments 56 # Shift options pour recuperation arguments
57 #------------------------------------------- 57 #-------------------------------------------
58 shift $((OPTIND-1)) 58 shift $((OPTIND-1))
59 59
60 if [ -z "$1" ] 60 if [ -z "$1" ]
61 then 61 then
62 echo "BAD USAGE: $0 [OPTIONS] <(i) repertoire (ex:20041006_0800_0900_CULTURE)> <REP_NAME (ex:res_p2)>" 62 echo "BAD USAGE: $0 [OPTIONS] <(i) repertoire (ex:20041006_0800_0900_CULTURE)> <REP_NAME (ex:res_p2)>"
63 exit 1 63 exit 1
64 fi 64 fi
65 65
66 if [ -z "$2" ] 66 if [ -z "$2" ]
67 then 67 then
68 echo "BAD USAGE: $0 [OPTIONS] <(i) repertoire (ex:20041006_0800_0900_CULTURE)> <REP_NAME (ex:res_p2)>" 68 echo "BAD USAGE: $0 [OPTIONS] <(i) repertoire (ex:20041006_0800_0900_CULTURE)> <REP_NAME (ex:res_p2)>"
69 exit 1 69 exit 1
70 fi 70 fi
71 71
72 . $PACKAGE_CONF_MEASURE 72 . $PACKAGE_CONF_MEASURE
73 73
74 #------------------------------------ 74 #------------------------------------
75 # INIT - Creation du workspace 75 # INIT - Creation du workspace
76 #------------------------------------ 76 #------------------------------------
77 NAME=`basename $1` 77 NAME=`basename $1`
78 CONF_DIR=$1/conf/$2 78 CONF_DIR=$1/conf/$2
79 FICHIER_RES=$2 79 FICHIER_RES=$2
80 REF=$CONF_DIR/ref 80 REF=$CONF_DIR/ref
81 POS=$CONF_DIR/pos 81 POS=$CONF_DIR/pos
82 MLCLASS=$CONF_DIR/mlclass 82 MLCLASS=$CONF_DIR/mlclass
83 GVALIGN=$CONF_DIR/gvalign 83 GVALIGN=$CONF_DIR/gvalign
84 HTK_POST=$CONF_DIR/htk_post 84 HTK_POST=$CONF_DIR/htk_post
85 HTK_LM=$CONF_DIR/htk_lm 85 HTK_LM=$CONF_DIR/htk_lm
86 WLAT=$CONF_DIR/wlat 86 WLAT=$CONF_DIR/wlat
87 LIKELIHOOD=$CONF_DIR/likelihood 87 LIKELIHOOD=$CONF_DIR/likelihood
88 GVCTM=$CONF_DIR/gvctm 88 GVCTM=$CONF_DIR/gvctm
89 SEGCTM=$CONF_DIR/segctm 89 SEGCTM=$CONF_DIR/segctm
90 SUPER_CTM=$CONF_DIR/super_ctm 90 SUPER_CTM=$CONF_DIR/super_ctm
91 SCORED_CTM=$CONF_DIR/scored_ctm 91 SCORED_CTM=$CONF_DIR/scored_ctm
92 mkdir -p $CONF_DIR > /dev/null 2>&1 92 mkdir -p $CONF_DIR > /dev/null 2>&1
93 mkdir -p $REF > /dev/null 2>&1 93 mkdir -p $REF > /dev/null 2>&1
94 mkdir -p $POS > /dev/null 2>&1 94 mkdir -p $POS > /dev/null 2>&1
95 mkdir -p $MLCLASS > /dev/null 2>&1 95 mkdir -p $MLCLASS > /dev/null 2>&1
96 mkdir -p $GVALIGN > /dev/null 2>&1 96 mkdir -p $GVALIGN > /dev/null 2>&1
97 mkdir -p $HTK_POST > /dev/null 2>&1 97 mkdir -p $HTK_POST > /dev/null 2>&1
98 #mkdir -p $HTK_LM ==> generer auto par SRILM si besoin 98 #mkdir -p $HTK_LM ==> generer auto par SRILM si besoin
99 mkdir -p $WLAT > /dev/null 2>&1 99 mkdir -p $WLAT > /dev/null 2>&1
100 mkdir -p $LIKELIHOOD > /dev/null 2>&1 100 mkdir -p $LIKELIHOOD > /dev/null 2>&1
101 mkdir -p $GVCTM > /dev/null 2>&1 101 mkdir -p $GVCTM > /dev/null 2>&1
102 mkdir -p $SEGCTM > /dev/null 2>&1 102 mkdir -p $SEGCTM > /dev/null 2>&1
103 mkdir -p $SUPER_CTM > /dev/null 2>&1 103 mkdir -p $SUPER_CTM > /dev/null 2>&1
104 mkdir -p $SCORED_CTM > /dev/null 2>&1 104 mkdir -p $SCORED_CTM > /dev/null 2>&1
105 if [ -z $BOOST_BIN ] && [ "$ARCH" == ".64" ] ; then 105 if [ -z $BOOST_BIN ] && [ "$ARCH" == ".64" ] ; then
106 BOOST_BIN=$ROOT/bin/icsiboost-64bit-static-r160 106 BOOST_BIN=$ROOT/bin/icsiboost-64bit-static-r160
107 fi 107 fi
108 if [ -z $BOOST_BIN ] ; then 108 if [ -z $BOOST_BIN ] ; then
109 BOOST_BIN=$ROOT/bin/icsiboost-32bit-static-r176 109 BOOST_BIN=$ROOT/bin/icsiboost-32bits-static-r176
110 fi 110 fi
111 #----------------------------------------------------------------- 111 #-----------------------------------------------------------------
112 # STEP 1 - Extension des treillis + ajout posteriors (format htk) 112 # STEP 1 - Extension des treillis + ajout posteriors (format htk)
113 #----------------------------------------------------------------- 113 #-----------------------------------------------------------------
114 if [ $EXTEND == 1 ] 114 if [ $EXTEND == 1 ]
115 then 115 then
116 echo "EXTEND step..." 116 echo "EXTEND step..."
117 rm -r $HTK_LM > /dev/null 2>&1 117 rm -r $HTK_LM > /dev/null 2>&1
118 rm $HTK_POST/* > /dev/null 2>&1 118 rm $HTK_POST/* > /dev/null 2>&1
119 # 119 #
120 # --> Ajout des scores linguistiques dans le HTK 120 # --> Ajout des scores linguistiques dans le HTK
121 # 121 #
122 ls $1/$FICHIER_RES/*.treil > $CONF_DIR/Liste_treil_${NAME}.lst 122 ls $1/$FICHIER_RES/*.treil > $CONF_DIR/Liste_treil_${NAME}.lst
123 123
124 LM_ACCESS="-lm $ML" 124 LM_ACCESS="-lm $ML"
125 if [ ! -z $SERVER ]; then 125 if [ ! -z $SERVER ]; then
126 LM_ACCESS="-use-server $SERVER -cache-served-ngrams" 126 LM_ACCESS="-use-server $SERVER -cache-served-ngrams"
127 fi 127 fi
128 echo "$SRILM_BIN/lattice-tool -read-htk -in-lattice-list $CONF_DIR/Liste_treil_${NAME}.lst $LM_ACCESS -order $ORDER -htk-logbase 10 -htk-lmscale $FUDGE -htk-wdpenalty $PENALITE -write-htk -out-lattice-dir $HTK_LM"; 128 echo "$SRILM_BIN/lattice-tool -read-htk -in-lattice-list $CONF_DIR/Liste_treil_${NAME}.lst $LM_ACCESS -order $ORDER -htk-logbase 10 -htk-lmscale $FUDGE -htk-wdpenalty $PENALITE -write-htk -out-lattice-dir $HTK_LM";
129 $SRILM_BIN/lattice-tool -read-htk -in-lattice-list $CONF_DIR/Liste_treil_${NAME}.lst $LM_ACCESS -order $ORDER -htk-logbase 10 -htk-lmscale $FUDGE -htk-wdpenalty $PENALITE -write-htk -out-lattice-dir $HTK_LM 129 $SRILM_BIN/lattice-tool -read-htk -in-lattice-list $CONF_DIR/Liste_treil_${NAME}.lst $LM_ACCESS -order $ORDER -htk-logbase 10 -htk-lmscale $FUDGE -htk-wdpenalty $PENALITE -write-htk -out-lattice-dir $HTK_LM
130 130
131 # 131 #
132 # --> Calcul des posteriors a partir des scores acoustiques et linguistiques present dans le HTK 132 # --> Calcul des posteriors a partir des scores acoustiques et linguistiques present dans le HTK
133 # 133 #
134 for file in `ls $HTK_LM/*.treil` 134 for file in `ls $HTK_LM/*.treil`
135 do 135 do
136 base=`basename $file .treil`; 136 base=`basename $file .treil`;
137 #echo "lattice-tool -read-htk -in-lattice $file -compute-posteriors -write-htk -out-lattice $HTK_POST/${base}.htk" 137 #echo "lattice-tool -read-htk -in-lattice $file -compute-posteriors -write-htk -out-lattice $HTK_POST/${base}.htk"
138 $SRILM_BIN/lattice-tool -read-htk -in-lattice $file -compute-posteriors -write-htk -out-lattice $HTK_POST/${base}.htk 138 $SRILM_BIN/lattice-tool -read-htk -in-lattice $file -compute-posteriors -write-htk -out-lattice $HTK_POST/${base}.htk
139 done 139 done
140 fi 140 fi
141 141
142 #--------------------------------------------------------------------------------------------------------------- 142 #---------------------------------------------------------------------------------------------------------------
143 # STEP 2 - alignement res et wlat pour creer res avec scores + infos (utilise un fastnc modifie) 143 # STEP 2 - alignement res et wlat pour creer res avec scores + infos (utilise un fastnc modifie)
144 # Exemple : 144 # Exemple :
145 # ok amendement 0.814885 ( time=36 nodes=3 min=0.0016862 max=0.814885 mean=0.333896 var=0.363849 svar=0.603199 ) 145 # ok amendement 0.814885 ( time=36 nodes=3 min=0.0016862 max=0.814885 mean=0.333896 var=0.363849 svar=0.603199 )
146 #---------------------------------------------------------------------------------------------------------------- 146 #----------------------------------------------------------------------------------------------------------------
147 if [ $FASTNC == 1 ] 147 if [ $FASTNC == 1 ]
148 then 148 then
149 echo "FASTNC step..." 149 echo "FASTNC step..."
150 rm -f $POS/* $WLAT/* > /dev/null 2>&1 150 rm -f $POS/* $WLAT/* > /dev/null 2>&1
151 for file in `ls $HTK_LM/*.treil` 151 for file in `ls $HTK_LM/*.treil`
152 do 152 do
153 base=`basename $file .treil`; 153 base=`basename $file .treil`;
154 #echo "$ROOT/bin/fastnc_v1.4 $HTK_POST/${base}.htk $WLAT/${base}.wlat $1/$FICHIER_RES/${base}.res rien -dtw2 > $POS/$base.pos2&" 154 #echo "$ROOT/bin/fastnc_v1.4 $HTK_POST/${base}.htk $WLAT/${base}.wlat $1/$FICHIER_RES/${base}.res rien -dtw2 > $POS/$base.pos2&"
155 $ROOT/bin/fastnc_v1.4 $HTK_POST/${base}.htk $WLAT/${base}.wlat $1/$FICHIER_RES/${base}.res rien -dtw2 > $POS/$base.pos2 155 $ROOT/bin/fastnc_v1.4 $HTK_POST/${base}.htk $WLAT/${base}.wlat $1/$FICHIER_RES/${base}.res rien -dtw2 > $POS/$base.pos2
156 done 156 done
157 fi 157 fi
158 158
159 #------------------------------------------------------------------------------------------------------------ 159 #------------------------------------------------------------------------------------------------------------
160 # STEP 3 - recuperation de la probabilite pour chaque mot + info relatives au modele de langue (backoff, ...) 160 # STEP 3 - recuperation de la probabilite pour chaque mot + info relatives au modele de langue (backoff, ...)
161 #------------------------------------------------------------------------------------------------------------ 161 #------------------------------------------------------------------------------------------------------------
162 if [ $PPL == 1 ] 162 if [ $PPL == 1 ]
163 then 163 then
164 echo "PPL step..." 164 echo "PPL step..."
165 rm -f $REF/* $CONF_DIR/${NAME}_ALLREF.* $MLCLASS/* > /dev/null 2>&1 165 rm -f $REF/* $CONF_DIR/${NAME}_ALLREF.* $MLCLASS/* > /dev/null 2>&1
166 # 166 #
167 # --> Creation des references a partir des .res (uniquement si .treil present) 167 # --> Creation des references a partir des .res (uniquement si .treil present)
168 # 168 #
169 for file in `ls $1/$FICHIER_RES/*.res` 169 for file in `ls $1/$FICHIER_RES/*.res`
170 do 170 do
171 base=`basename $file .res`; 171 base=`basename $file .res`;
172 if [ -f $1/$FICHIER_RES/$base.treil ];then 172 if [ -f $1/$FICHIER_RES/$base.treil ];then
173 cat $file | cut -f5 -d' ' | tr "\n" " " > $REF/${base}.ref 173 cat $file | cut -f5 -d' ' | tr "\n" " " > $REF/${base}.ref
174 fi 174 fi
175 done 175 done
176 176
177 # 177 #
178 # --> creation d'un fichier contenant l'ensemble des transcriptions du show 178 # --> creation d'un fichier contenant l'ensemble des transcriptions du show
179 # 179 #
180 compteur=0 180 compteur=0
181 for file in `du -sh $REF/*.ref | grep -v "^0" | cut -f2` 181 for file in `du -sh $REF/*.ref | grep -v "^0" | cut -f2`
182 do 182 do
183 base=`basename $file .ref`; 183 base=`basename $file .ref`;
184 cat $file >> $CONF_DIR/${NAME}_ALLREF.txt 184 cat $file >> $CONF_DIR/${NAME}_ALLREF.txt
185 echo "" >> $CONF_DIR/${NAME}_ALLREF.txt 185 echo "" >> $CONF_DIR/${NAME}_ALLREF.txt
186 ListeFichiers[$compteur]=$base.mlclass 186 ListeFichiers[$compteur]=$base.mlclass
187 compteur=$(( $compteur + 1 )) 187 compteur=$(( $compteur + 1 ))
188 done 188 done
189 189
190 # 190 #
191 # --> recuperation de la probabilite pour chaque mot provenant des resultats de l'ASR + informations linguistiques (backoff used, ngram,...) 191 # --> recuperation de la probabilite pour chaque mot provenant des resultats de l'ASR + informations linguistiques (backoff used, ngram,...)
192 # 192 #
193 $SRILM_BIN/ngram -lm $ML -order $ORDER -ppl $CONF_DIR/${NAME}_ALLREF.txt -debug 2 > $CONF_DIR/${NAME}_ALLREF.mlclass 193 $SRILM_BIN/ngram -lm $ML -order $ORDER -ppl $CONF_DIR/${NAME}_ALLREF.txt -debug 2 > $CONF_DIR/${NAME}_ALLREF.mlclass
194 194
195 # 195 #
196 # --> creation d'un fichier par fichier .ref 196 # --> creation d'un fichier par fichier .ref
197 # 197 #
198 compteur=0 198 compteur=0
199 cat $CONF_DIR/${NAME}_ALLREF.mlclass | while read line 199 cat $CONF_DIR/${NAME}_ALLREF.mlclass | while read line
200 do 200 do
201 echo $line | grep "^$" > /dev/null 201 echo $line | grep "^$" > /dev/null
202 if [ $? == 0 ];then 202 if [ $? == 0 ];then
203 compteur=$(( $compteur + 1 )) 203 compteur=$(( $compteur + 1 ))
204 else 204 else
205 echo "$line" | grep "p(" > /dev/null 205 echo "$line" | grep "p(" > /dev/null
206 if [ $? == 0 ];then 206 if [ $? == 0 ];then
207 echo "$line" >> $MLCLASS/${ListeFichiers[${compteur}]}; 207 echo "$line" >> $MLCLASS/${ListeFichiers[${compteur}]};
208 fi 208 fi
209 fi 209 fi
210 done 210 done
211 fi 211 fi
212 212
213 #---------------------------------------------------------- 213 #----------------------------------------------------------
214 # STEP 4 - recuperation du score acoustique de chaque mot 214 # STEP 4 - recuperation du score acoustique de chaque mot
215 #---------------------------------------------------------- 215 #----------------------------------------------------------
216 if [ $ACOUST == 1 ] 216 if [ $ACOUST == 1 ]
217 then 217 then
218 echo "ACOUST step..." 218 echo "ACOUST step..."
219 rm -f $GVALIGN/* > /dev/null 2>&1 219 rm -f $GVALIGN/* > /dev/null 2>&1
220 rm -f $GVCTM/* > /dev/null 2>&1 220 rm -f $GVCTM/* > /dev/null 2>&1
221 rm -f $SEGCTM/* > /dev/null 2>&1 221 rm -f $SEGCTM/* > /dev/null 2>&1
222 rm -f $LIKELIHOOD/* > /dev/null 2>&1 222 rm -f $LIKELIHOOD/* > /dev/null 2>&1
223 223
224 for file in `ls $1/$FICHIER_RES/*.res` 224 for file in `ls $1/$FICHIER_RES/*.res`
225 do 225 do
226 base=`basename $file .res` 226 base=`basename $file .res`
227 if [ -f $1/$FICHIER_RES/$base.treil ];then 227 if [ -f $1/$FICHIER_RES/$base.treil ];then
228 #echo "$ROOT/script/MakeListForGVAlign.pl $file $GVALIGN"; 228 #echo "$ROOT/script/MakeListForGVAlign.pl $file $GVALIGN";
229 $ROOT/script/MakeListForGVAlign.pl $file $GVALIGN 229 $ROOT/script/MakeListForGVAlign.pl $file $GVALIGN
230 fi 230 fi
231 done 231 done
232 232
233 for file in `ls $GVALIGN/*.gvalign` 233 for file in `ls $GVALIGN/*.gvalign`
234 do 234 do
235 base=`basename $file .gvalign`; 235 base=`basename $file .gvalign`;
236 236
237 type=`echo $base | cut -f2 -d: | cut -f2- -d\# | sed -e "s/[0-9]\+//"` 237 type=`echo $base | cut -f2 -d: | cut -f2- -d\# | sed -e "s/[0-9]\+//"`
238 238
239 case "$type" in 239 case "$type" in
240 "M#S") 240 "M#S")
241 HMM=$mod_ms 241 HMM=$mod_ms
242 ;; 242 ;;
243 "F#S") 243 "F#S")
244 HMM=$mod_fs 244 HMM=$mod_fs
245 ;; 245 ;;
246 "M#T") 246 "M#T")
247 HMM=$mod_mt 247 HMM=$mod_mt
248 ;; 248 ;;
249 "F#T") 249 "F#T")
250 HMM=$mod_ft 250 HMM=$mod_ft
251 ;; 251 ;;
252 esac 252 esac
253 253
254 #echo "$ROOT/bin/gvalign.old $HMM $PHON $file -e $1/${REP_PLP}/ -f .plp -r $GVALIGN -g .gv -C FAST -W $GVCTM -O CTM -s $SEGCTM > $LIKELIHOOD/${base}.likelihood | sed -e 's/Decoding/\\nDecoding/g' > $LIKELIHOOD/${base}.likelihood"; 254 #echo "$ROOT/bin/gvalign.old $HMM $PHON $file -e $1/${REP_PLP}/ -f .plp -r $GVALIGN -g .gv -C FAST -W $GVCTM -O CTM -s $SEGCTM > $LIKELIHOOD/${base}.likelihood | sed -e 's/Decoding/\\nDecoding/g' > $LIKELIHOOD/${base}.likelihood";
255 #$ROOT/bin/gvalign.old $HMM $PHON $file -e $1/${REP_PLP}/ -f .plp -r $GVALIGN -g .gv -C FAST -W $GVCTM -O CTM -s $SEGCTM | sed -e 's/Decoding/\nDecoding/g' > $LIKELIHOOD/${base}.likelihood 255 #$ROOT/bin/gvalign.old $HMM $PHON $file -e $1/${REP_PLP}/ -f .plp -r $GVALIGN -g .gv -C FAST -W $GVCTM -O CTM -s $SEGCTM | sed -e 's/Decoding/\nDecoding/g' > $LIKELIHOOD/${base}.likelihood
256 touch $LIKELIHOOD/${base}.likelihood 256 touch $LIKELIHOOD/${base}.likelihood
257 done 257 done
258 fi 258 fi
259 259
260 #-------------------------------------------------------------------------------------------------------------------------------- 260 #--------------------------------------------------------------------------------------------------------------------------------
261 # STEP 5 - Merge de tous les scores caclules => res (ctm) avec scores/params utiliser dans la classif 261 # STEP 5 - Merge de tous les scores caclules => res (ctm) avec scores/params utiliser dans la classif
262 # Format : 262 # Format :
263 # mot NbNode MinNode MaxNode MeanNode VarNode SVarNode Posterior AcousticLogLikelihood AcousticLogLikelihood/Frame ... 263 # mot NbNode MinNode MaxNode MeanNode VarNode SVarNode Posterior AcousticLogLikelihood AcousticLogLikelihood/Frame ...
264 # AcousticConfidenceLikelihood AcousticConstraintLikeLihood AcousticNoConstraint Likelihood ClasseRepliLinguistique ... 264 # AcousticConfidenceLikelihood AcousticConstraintLikeLihood AcousticNoConstraint Likelihood ClasseRepliLinguistique ...
265 # RepliLinguistique LogLinguistique LogUnigramme NbMotsFenetre NbNulNode NbTrame 265 # RepliLinguistique LogLinguistique LogUnigramme NbMotsFenetre NbNulNode NbTrame
266 #--------------------------------------------------------------------------------------------------------------------------------- 266 #---------------------------------------------------------------------------------------------------------------------------------
267 if [ $EXTRACT == 1 ] 267 if [ $EXTRACT == 1 ]
268 then 268 then
269 echo "EXTRACT step..." 269 echo "EXTRACT step..."
270 rm -f $SUPER_CTM/* > /dev/null 2>&1 270 rm -f $SUPER_CTM/* > /dev/null 2>&1
271 271
272 for file in `ls $1/$FICHIER_RES/*.res` 272 for file in `ls $1/$FICHIER_RES/*.res`
273 do 273 do
274 base=`basename $file .res`; 274 base=`basename $file .res`;
275 like=`echo "$base" | sed -e 's/\..*//'`; 275 like=`echo "$base" | sed -e 's/\..*//'`;
276 if [ -f $1/$FICHIER_RES/$base.treil ]; then 276 if [ -f $1/$FICHIER_RES/$base.treil ]; then
277 echo "$ROOT/scripts/ExtractData.pl $pathML $nameML $POS/${base}.pos2 $file $LIKELIHOOD/${like}.likelihood $MLCLASS/${base}.mlclass $TYPE_ML > $SUPER_CTM/${base}.ctm"; 277 echo "$ROOT/scripts/ExtractData.pl $pathML $nameML $POS/${base}.pos2 $file $LIKELIHOOD/${like}.likelihood $MLCLASS/${base}.mlclass $TYPE_ML > $SUPER_CTM/${base}.ctm";
278 $ROOT/script/ExtractData.pl $pathML $nameML $POS/${base}.pos2 $file $LIKELIHOOD/${like}.likelihood $MLCLASS/${base}.mlclass $TYPE_ML > $SUPER_CTM/${base}.ctm 278 $ROOT/script/ExtractData.pl $pathML $nameML $POS/${base}.pos2 $file $LIKELIHOOD/${like}.likelihood $MLCLASS/${base}.mlclass $TYPE_ML > $SUPER_CTM/${base}.ctm
279 # $ROOT/script/ExtractData.pl $pathML $nameML $POS/${base}.pos2 $file $LIKELIHOOD/${like}.likelihood $MLCLASS/${base}.mlclass $TYPE_ML > $SUPER_CTM/${base}.ctm 279 # $ROOT/script/ExtractData.pl $pathML $nameML $POS/${base}.pos2 $file $LIKELIHOOD/${like}.likelihood $MLCLASS/${base}.mlclass $TYPE_ML > $SUPER_CTM/${base}.ctm
280 fi 280 fi
281 done 281 done
282 fi 282 fi
283 283
284 #---------------------------------------------------------------- 284 #----------------------------------------------------------------
285 # STEP 6 - Calcul effectif du score de confiance pour chaque mot 285 # STEP 6 - Calcul effectif du score de confiance pour chaque mot
286 #---------------------------------------------------------------- 286 #----------------------------------------------------------------
287 if [ $BOOST == 1 ] 287 if [ $BOOST == 1 ]
288 then 288 then
289 echo "BOOST step..." 289 echo "BOOST step..."
290 rm -f $SCORED_CTM/* $CONF_DIR/${NAME}.sctm $CONF_DIR/${NAME}.boost* $CONF_DIR/${NAME}.resboost* $CONF_DIR/${NAME}.corres* > /dev/null 2>&1 290 rm -f $SCORED_CTM/* $CONF_DIR/${NAME}.sctm $CONF_DIR/${NAME}.boost* $CONF_DIR/${NAME}.resboost* $CONF_DIR/${NAME}.corres* > /dev/null 2>&1
291 # utilise pour le test sans etiquette 291 # utilise pour le test sans etiquette
292 $ROOT/script/DissociateErroneousFromDecoded.pl $SUPER_CTM 2 equilibre > $CONF_DIR/${NAME}.sctm 292 $ROOT/script/DissociateErroneousFromDecoded.pl $SUPER_CTM 2 equilibre > $CONF_DIR/${NAME}.sctm
293 293
294 $ROOT/script/ConvertSuperCTMtoDataSVM.pl $CONF_DIR/${NAME}.sctm boost 2 0 0 > $CONF_DIR/${NAME}.boost 294 $ROOT/script/ConvertSuperCTMtoDataSVM.pl $CONF_DIR/${NAME}.sctm boost 2 0 0 > $CONF_DIR/${NAME}.boost
295 $ROOT/script/ConvertSuperCTMtoDataSVM.pl $CONF_DIR/${NAME}.sctm boost 2 0 1 > $CONF_DIR/${NAME}.boost_refs 295 $ROOT/script/ConvertSuperCTMtoDataSVM.pl $CONF_DIR/${NAME}.sctm boost 2 0 1 > $CONF_DIR/${NAME}.boost_refs
296 296
297 $BOOST_BIN -S $ROOT/TRAIN -C --posteriors < $CONF_DIR/${NAME}.boost > $CONF_DIR/${NAME}.resboost 297 $BOOST_BIN -S $ROOT/TRAIN -C --posteriors < $CONF_DIR/${NAME}.boost > $CONF_DIR/${NAME}.resboost
298 298
299 cat $CONF_DIR/${NAME}.resboost | cut -f4 -d" " > $CONF_DIR/${NAME}.resboost2 299 cat $CONF_DIR/${NAME}.resboost | cut -f4 -d" " > $CONF_DIR/${NAME}.resboost2
300 300
301 cat $CONF_DIR/${NAME}.boost_refs | sed -e 's/.*ref=//' > $CONF_DIR/${NAME}.corres 301 cat $CONF_DIR/${NAME}.boost_refs | sed -e 's/.*ref=//' > $CONF_DIR/${NAME}.corres
302 302
303 paste $CONF_DIR/${NAME}.corres $CONF_DIR/${NAME}.resboost2 | sed -e 's/\.ctm/\.res/' > $CONF_DIR/${NAME}.corres2 303 paste $CONF_DIR/${NAME}.corres $CONF_DIR/${NAME}.resboost2 | sed -e 's/\.ctm/\.res/' > $CONF_DIR/${NAME}.corres2
304 304
305 $ROOT/script/AssociateScoreToCtm.pl $CONF_DIR/${NAME}.corres2 $1/$FICHIER_RES/ $SCORED_CTM/ 305 $ROOT/script/AssociateScoreToCtm.pl $CONF_DIR/${NAME}.corres2 $1/$FICHIER_RES/ $SCORED_CTM/
306 fi 306 fi
307 echo "END" 307 echo "END"
308 308