Jean-François Rey / otmedia

Download as
Browse Code »

Commit d7e9e4b9df6b3b9dab101921d6fd1cb357ef2cfa

Authored by Jean-François Rey
1 parent b427f103e1
Exists in master

update bugfix stderr

Showing 7 changed files with 34 additions and 31 deletions Inline Diff

main_tools/ConfPass.sh
Diff comments   View file @ d7e9e4b
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : ConfPass.sh # 4 # File : ConfPass.sh #
5 # Brief : Process the ASR Confidence pass # 5 # Brief : Process the ASR Confidence pass #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # (base on Emmanuel Ferreira # 7 # (base on Emmanuel Ferreira #
8 # and hugo Mauchrétien works) # 8 # and hugo Mauchrétien works) #
9 # Version : 1.0 # 9 # Version : 1.0 #
10 # Date : 17/06/13 # 10 # Date : 17/06/13 #
11 ##################################################### 11 #####################################################
12 12
13 echo "### ConfPass.sh ###" 13 echo "### ConfPass.sh ###"
14 14
15 #Check OTMEDIA_HOME env var 15 #Check OTMEDIA_HOME env var
16 if [ -z ${OTMEDIA_HOME} ] 16 if [ -z ${OTMEDIA_HOME} ]
17 then 17 then
18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
19 export OTMEDIA_HOME=$OTMEDIA_HOME 19 export OTMEDIA_HOME=$OTMEDIA_HOME
20 fi 20 fi
21 21
22 22
23 # where is ConfPath.sh 23 # where is ConfPath.sh
24 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 24 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
25 25
26 # Scripts Path 26 # Scripts Path
27 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 27 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
28 28
29 # Include scripts 29 # Include scripts
30 . $SCRIPT_PATH"/Tools.sh" 30 . $SCRIPT_PATH"/Tools.sh"
31 . $SCRIPT_PATH"/CheckConfPass.sh" 31 . $SCRIPT_PATH"/CheckConfPass.sh"
32 32
33 # where is FirstPass.cfg 33 # where is FirstPass.cfg
34 CONFPASS_CONFIG_FILE="$OTMEDIA_HOME/cfg/ConfPass.cfg" 34 CONFPASS_CONFIG_FILE="$OTMEDIA_HOME/cfg/ConfPass.cfg"
35 if [ -e $CONFPASS_CONFIG_FILE ] 35 if [ -e $CONFPASS_CONFIG_FILE ]
36 then 36 then
37 . $CONFPASS_CONFIG_FILE 37 . $CONFPASS_CONFIG_FILE
38 else 38 else
39 echo "ERROR : Can't find configuration file $CONFPASS_CONFIG_FILE" > /dev/stderr 39 echo "ERROR : Can't find configuration file $CONFPASS_CONFIG_FILE" >&2
40 exit 1 40 exit 1
41 fi 41 fi
42 42
43 #---------------# 43 #---------------#
44 # Parse Options # 44 # Parse Options #
45 #---------------# 45 #---------------#
46 while getopts ":hDv:cr" opt 46 while getopts ":hDv:cr" opt
47 do 47 do
48 case $opt in 48 case $opt in
49 h) 49 h)
50 echo -e "$0 [OPTIONS] <INPUT_DIRECTORY> <TREIL_DIRECTORY_NAME>\n" 50 echo -e "$0 [OPTIONS] <INPUT_DIRECTORY> <TREIL_DIRECTORY_NAME>\n"
51 echo -e "\t Options:" 51 echo -e "\t Options:"
52 echo -e "\t\t-h :\tprint this message" 52 echo -e "\t\t-h :\tprint this message"
53 echo -e "\t\t-D :\tDEBUG mode on" 53 echo -e "\t\t-D :\tDEBUG mode on"
54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
55 echo -e "\t\t-c :\t Check process, stop if error detected" 55 echo -e "\t\t-c :\t Check process, stop if error detected"
56 echo -e "\t\t-r :\tForce to rerun confpas without deleting existing files" 56 echo -e "\t\t-r :\tForce to rerun confpas without deleting existing files"
57 exit 1 57 exit 1
58 ;; 58 ;;
59 D) 59 D)
60 DEBUG=1 60 DEBUG=1
61 ;; 61 ;;
62 v) 62 v)
63 VERBOSE=$OPTARG 63 VERBOSE=$OPTARG
64 ;; 64 ;;
65 c) 65 c)
66 CHECK=1 66 CHECK=1
67 ;; 67 ;;
68 r) 68 r)
69 RERUN=1 69 RERUN=1
70 ;; 70 ;;
71 :) 71 :)
72 echo "Option -$OPTARG requires an argument." > /dev/stderr 72 echo "Option -$OPTARG requires an argument." >&2
73 exit 1 73 exit 1
74 ;; 74 ;;
75 \?) 75 \?)
76 echo "BAD USAGE : unknow opton -$OPTARG" 76 echo "BAD USAGE : unknow opton -$OPTARG"
77 #exit 1 77 #exit 1
78 ;; 78 ;;
79 esac 79 esac
80 done 80 done
81 81
82 # mode debug enable 82 # mode debug enable
83 if [ $DEBUG -eq 1 ] 83 if [ $DEBUG -eq 1 ]
84 then 84 then
85 set -x 85 set -x
86 echo -e "## Mode DEBUG ON ##" 86 echo -e "## Mode DEBUG ON ##"
87 fi 87 fi
88 88
89 # mode verbose enable 89 # mode verbose enable
90 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi 90 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
91 91
92 # Check USAGE by arguments number 92 # Check USAGE by arguments number
93 if [ $(($#-($OPTIND-1))) -ne 2 ] 93 if [ $(($#-($OPTIND-1))) -ne 2 ]
94 then 94 then
95 echo "BAD USAGE : ConfPass.sh [OPTIONS] <INPUT_DIR> <TREIL_DIRECTORY_NAME>" 95 echo "BAD USAGE : ConfPass.sh [OPTIONS] <INPUT_DIR> <TREIL_DIRECTORY_NAME>"
96 echo "$0 -h for more info" 96 echo "$0 -h for more info"
97 exit 1 97 exit 1
98 fi 98 fi
99 99
100 shift $((OPTIND-1)) 100 shift $((OPTIND-1))
101 # check input directory - first argument 101 # check input directory - first argument
102 if [ ! -e $1 ] 102 if [ ! -e $1 ]
103 then 103 then
104 print_error "can't open $1" 104 print_error "can't open $1"
105 exit 1 105 exit 1
106 fi 106 fi
107 # check treil input directory - second argument 107 # check treil input directory - second argument
108 if [ ! -e $1/$2 ] 108 if [ ! -e $1/$2 ]
109 then 109 then
110 print_error "can't open $1/$2" 110 print_error "can't open $1/$2"
111 exit 1 111 exit 1
112 fi 112 fi
113 113
114 print_info "[${BASENAME}] => Conf Pass start | $(date +'%d/%m/%y %H:%M:%S')" 1 114 print_info "[${BASENAME}] => Conf Pass start | $(date +'%d/%m/%y %H:%M:%S')" 1
115 115
116 #-------------# 116 #-------------#
117 # GLOBAL VARS # 117 # GLOBAL VARS #
118 #-------------# 118 #-------------#
119 INPUT_DIR=$(readlink -e $1) 119 INPUT_DIR=$(readlink -e $1)
120 OUTPUT_DIR=$INPUT_DIR 120 OUTPUT_DIR=$INPUT_DIR
121 BASENAME=$(basename $OUTPUT_DIR) 121 BASENAME=$(basename $OUTPUT_DIR)
122 RES_NAME=$2 122 RES_NAME=$2
123 RES_P="${INPUT_DIR}/${RES_NAME}" 123 RES_P="${INPUT_DIR}/${RES_NAME}"
124 USF_FILE=${INPUT_DIR}/${BASENAME}.${RES_NAME}.usf 124 USF_FILE=${INPUT_DIR}/${BASENAME}.${RES_NAME}.usf
125 CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME" 125 CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME"
126 RES_CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME/scored_ctm" 126 RES_CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME/scored_ctm"
127 LOGFILE="${OUTPUT_DIR_BASENAME}/info_conf.log" 127 LOGFILE="${OUTPUT_DIR_BASENAME}/info_conf.log"
128 ERRORFILE="${OUTPUT_DIR_BASENAME}/error_conf.log" 128 ERRORFILE="${OUTPUT_DIR_BASENAME}/error_conf.log"
129 129
130 #------------------# 130 #------------------#
131 # Create Workspace # 131 # Create Workspace #
132 #------------------# 132 #------------------#
133 # Lock directory 133 # Lock directory
134 if [ -e "$OUTPUT_DIR/CONFPASS.lock" ] && [ $RERUN -eq 0 ] 134 if [ -e "$OUTPUT_DIR/CONFPASS.lock" ] && [ $RERUN -eq 0 ]
135 then 135 then
136 print_warn "[${BASENAME}] Confpass is locked -> exit" 2 136 print_warn "[${BASENAME}] Confpass is locked -> exit" 2
137 exit 1 137 exit 1
138 fi 138 fi
139 rm "$OUTPUT_DIR/CONFPASS.unlock" > /dev/null 2>&1 139 rm "$OUTPUT_DIR/CONFPASS.unlock" > /dev/null 2>&1
140 touch "$OUTPUT_DIR/CONFPASS.lock" > /dev/null 2>&1 140 touch "$OUTPUT_DIR/CONFPASS.lock" > /dev/null 2>&1
141 if [ $RERUN -eq 0 ]; then rm -r $CONF_DIR > /dev/null 2>&1; fi 141 if [ $RERUN -eq 0 ]; then rm -r $CONF_DIR > /dev/null 2>&1; fi
142 if [ $RERUN -eq 1 ]; then rm $USF_FILE > /dev/null 2>&1; fi 142 if [ $RERUN -eq 1 ]; then rm $USF_FILE > /dev/null 2>&1; fi
143 mkdir -p $CONF_DIR > /dev/null 2>&1 143 mkdir -p $CONF_DIR > /dev/null 2>&1
144 mkdir -p $RES_CONF_DIR > /dev/null 2>&1 144 mkdir -p $RES_CONF_DIR > /dev/null 2>&1
145 rm $LOGFILE $ERRORFILE > /dev/null 2>&1 145 rm $LOGFILE $ERRORFILE > /dev/null 2>&1
146 146
147 #---------------# 147 #---------------#
148 # Check Pass # 148 # Check Pass #
149 #---------------# 149 #---------------#
150 print_info "[${BASENAME}] Check Conf Pass directory ${RES_NAME}" 1 150 print_info "[${BASENAME}] Check Conf Pass directory ${RES_NAME}" 1
151 # if usf contains more than 49% of 0.600 confidence -> usf error 151 # if usf contains more than 49% of 0.600 confidence -> usf error
152 if [ -s $USF_FILE ] 152 if [ -s $USF_FILE ]
153 then 153 then
154 conftozerosix=$(grep -c -E 'confidence="0.600"' "${USF_FILE}") 154 conftozerosix=$(grep -c -E 'confidence="0.600"' "${USF_FILE}")
155 confall=$(grep -c -E 'confidence=' "${USF_FILE}") 155 confall=$(grep -c -E 'confidence=' "${USF_FILE}")
156 if [ $confall -gt 0 ] 156 if [ $confall -gt 0 ]
157 then 157 then
158 pourcentageofzerosix=$((($conftozerosix*100)/$confall)) 158 pourcentageofzerosix=$((($conftozerosix*100)/$confall))
159 if [ $pourcentageofzerosix -gt 49 ] 159 if [ $pourcentageofzerosix -gt 49 ]
160 then 160 then
161 print_warn "[${BASENAME}] ${BASENAME}.${RES_NAME}.usf got $pourcentageofzerosix% of 0.600 confidence" 2 161 print_warn "[${BASENAME}] ${BASENAME}.${RES_NAME}.usf got $pourcentageofzerosix% of 0.600 confidence" 2
162 print_info "[${BASENAME}] bad usf ${RES_NAME}, will do it again" 1 162 print_info "[${BASENAME}] bad usf ${RES_NAME}, will do it again" 1
163 mv "${USF_FILE}" "${USF_FILE}.back" 163 mv "${USF_FILE}" "${USF_FILE}.back"
164 rm -r $CONF_DIR > /dev/null 2>&1 164 rm -r $CONF_DIR > /dev/null 2>&1
165 else 165 else
166 print_warn "[${BASENAME}] ${USF_FILE} already done, skipping it" 1 166 print_warn "[${BASENAME}] ${USF_FILE} already done, skipping it" 1
167 exit 0 167 exit 0
168 fi 168 fi
169 fi 169 fi
170 else 170 else
171 print_info "[${BASENAME}] No USF file already done, continue..." 1 171 print_info "[${BASENAME}] No USF file already done, continue..." 1
172 fi 172 fi
173 173
174 # Check if treil are here 174 # Check if treil are here
175 nbres_p1=$(cat ${INPUT_DIR}/plp.lst | wc -l) 175 nbres_p1=$(cat ${INPUT_DIR}/plp.lst | wc -l)
176 nbtreil_p=$(ls $RES_P/*.treil 2> /dev/null | wc -l) 176 nbtreil_p=$(ls $RES_P/*.treil 2> /dev/null | wc -l)
177 if [ $nbtreil_p -eq 0 ] 177 if [ $nbtreil_p -eq 0 ]
178 then 178 then
179 print_error "[${BASENAME}] No ${RES_NAME} Pass, No .treil -> exit ConfPass" 179 print_error "[${BASENAME}] No ${RES_NAME} Pass, No .treil -> exit ConfPass"
180 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ${RES_NAME} Pass, No .treil -> exit ConfPass" ;fi 180 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ${RES_NAME} Pass, No .treil -> exit ConfPass" ;fi
181 exit 1 181 exit 1
182 else 182 else
183 #Check if more then 89% of treil are done 183 #Check if more then 89% of treil are done
184 if [ $nbres_p1 -gt 0 ] 184 if [ $nbres_p1 -gt 0 ]
185 then 185 then
186 pourcentage=$((($nbtreil_p*100)/$nbres_p1)) 186 pourcentage=$((($nbtreil_p*100)/$nbres_p1))
187 if [ $pourcentage -gt 89 ] 187 if [ $pourcentage -gt 89 ]
188 then 188 then
189 print_info "[${BASENAME}] ${RES_NAME}/*.treil are here" 1 189 print_info "[${BASENAME}] ${RES_NAME}/*.treil are here" 1
190 else 190 else
191 print_warn "[${BASENAME}] not enough ${RES_NAME} treil" 2 191 print_warn "[${BASENAME}] not enough ${RES_NAME} treil" 2
192 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "Not enough ${RES_NAME} treil " 192 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "Not enough ${RES_NAME} treil "
193 fi 193 fi
194 fi 194 fi
195 fi 195 fi
196 196
197 #------# 197 #------#
198 # Save # 198 # Save #
199 #------# 199 #------#
200 cp $CONFPASS_CONFIG_FILE $OUTPUT_DIR/ConfPass.cfg 200 cp $CONFPASS_CONFIG_FILE $OUTPUT_DIR/ConfPass.cfg
201 echo "RES_CONF_DIR=$RES_CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg 201 echo "RES_CONF_DIR=$RES_CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg
202 echo "CONF_DIR=$CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg 202 echo "CONF_DIR=$CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg
203 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ConfPass.cfg" 1 203 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ConfPass.cfg" 1
204 204
205 #--------------------# 205 #--------------------#
206 # CONFIDENCE MEASURE # 206 # CONFIDENCE MEASURE #
207 #--------------------# 207 #--------------------#
208 208
209 # Check pourcentage of scored_ctm already done, if < 85% do confidence measure 209 # Check pourcentage of scored_ctm already done, if < 85% do confidence measure
210 nbres_p=$(ls ${RES_P}/*.treil | wc -l) 210 nbres_p=$(ls ${RES_P}/*.treil | wc -l)
211 nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l) 211 nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l)
212 if [ $nbres_p -gt 0 ] 212 if [ $nbres_p -gt 0 ]
213 then 213 then
214 pourcentageres=$((($nbconf*100)/$nbres_p)) 214 pourcentageres=$((($nbconf*100)/$nbres_p))
215 if [ $pourcentageres -lt 85 ] 215 if [ $pourcentageres -lt 85 ]
216 then 216 then
217 print_info "[${BASENAME}] Calcul Confidence $INPUT_DIR $RES_NAME" 1 217 print_info "[${BASENAME}] Calcul Confidence $INPUT_DIR $RES_NAME" 1
218 $MAIN_SCRIPT_PATH/ConfidenceMeasure.sh $INPUT_DIR $RES_NAME 218 $MAIN_SCRIPT_PATH/ConfidenceMeasure.sh $INPUT_DIR $RES_NAME
219 219
220 else 220 else
221 print_info "[${BASENAME}] Skipping Confidence Calcul $INPUT_DIR/$RES_NAME" 1 221 print_info "[${BASENAME}] Skipping Confidence Calcul $INPUT_DIR/$RES_NAME" 1
222 fi 222 fi
223 fi 223 fi
224 224
225 ### Check scored_ctm number res files ! 225 ### Check scored_ctm number res files !
226 if [ $CHECK -eq 1 ] 226 if [ $CHECK -eq 1 ]
227 then 227 then
228 nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l) 228 nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l)
229 if [ $nbres_p -ne $nbconf ] 229 if [ $nbres_p -ne $nbconf ]
230 then 230 then
231 print_warn "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" 2 231 print_warn "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" 2
232 print_log_file $LOGFILE "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" 232 print_log_file $LOGFILE "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ"
233 fi 233 fi
234 fi 234 fi
235 235
236 #---------------------------# 236 #---------------------------#
237 # FROM RES WITH CONF => USF # 237 # FROM RES WITH CONF => USF #
238 #---------------------------# 238 #---------------------------#
239 print_info "[${BASENAME}] Create USF file for $RES_P" 1 239 print_info "[${BASENAME}] Create USF file for $RES_P" 1
240 for f in `ls ${RES_CONF_DIR}`; do $SCRIPT_PATH/formatRES.pl $RES_CONF_DIR/$f; done 240 for f in `ls ${RES_CONF_DIR}`; do $SCRIPT_PATH/formatRES.pl $RES_CONF_DIR/$f; done
241 # create USF configuration file 241 # create USF configuration file
242 echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR/$BASENAME.seg" > $OUTPUT_DIR/$BASENAME.usf_cfg 242 echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR/$BASENAME.seg" > $OUTPUT_DIR/$BASENAME.usf_cfg
243 # create USF file 243 # create USF file
244 print_info "$SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg" 3 244 print_info "$SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg" 3
245 $SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg 245 $SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg
246 rm $OUTPUT_DIR/$BASENAME.usf_cfg > /dev/null 2>&1 246 rm $OUTPUT_DIR/$BASENAME.usf_cfg > /dev/null 2>&1
247 cat $USF_FILE.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f > $USF_FILE 247 cat $USF_FILE.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f > $USF_FILE
248 cp $USF_FILE ${OUTPUT_DIR}/${BASENAME}.usf 248 cp $USF_FILE ${OUTPUT_DIR}/${BASENAME}.usf
249 rm $USF_FILE.tmp > /dev/null 2>&1 249 rm $USF_FILE.tmp > /dev/null 2>&1
250 250
251 #----------------# 251 #----------------#
252 # Check USF file # 252 # Check USF file #
253 #----------------# 253 #----------------#
254 if [ $CHECK -eq 1 ] 254 if [ $CHECK -eq 1 ]
255 then 255 then
256 check_conf_pass_usf "$OUTPUT_DIR/$BASENAME.usf" 256 check_conf_pass_usf "$OUTPUT_DIR/$BASENAME.usf"
257 if [ $? -eq 1 ] 257 if [ $? -eq 1 ]
258 then 258 then
259 print_error "[${BASENAME}] Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf" 259 print_error "[${BASENAME}] Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf"
260 print_log_file $ERRORFILE "ERROR : Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf" 260 print_log_file $ERRORFILE "ERROR : Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf"
261 exit 1 261 exit 1
262 fi 262 fi
263 fi 263 fi
264 264
265 #-------# 265 #-------#
266 # CLOSE # 266 # CLOSE #
267 #-------# 267 #-------#
268 # Seem OK 268 # Seem OK
269 print_info "[${BASENAME}] <= ConfPass End | $(date +'%d/%m/%y %H:%M:%S')" 1 269 print_info "[${BASENAME}] <= ConfPass End | $(date +'%d/%m/%y %H:%M:%S')" 1
270 270
271 # unlock directory 271 # unlock directory
272 mv "$OUTPUT_DIR/CONFPASS.lock" "$OUTPUT_DIR/CONFPASS.unlock" 272 mv "$OUTPUT_DIR/CONFPASS.lock" "$OUTPUT_DIR/CONFPASS.unlock"
273 273
274 274
main_tools/ExploitConfidencePass.sh
Diff comments   View file @ d7e9e4b
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : ExploitConfidencePass.sh # 4 # File : ExploitConfidencePass.sh #
5 # Brief : Exploit the ASR confidence pass to : # 5 # Brief : Exploit the ASR confidence pass to : #
6 # -> boost the confident zone # 6 # -> boost the confident zone #
7 # -> find alternative in non confident zone 7 # -> find alternative in non confident zone
8 # -> dynamicly extend the lexicon # 8 # -> dynamicly extend the lexicon #
9 # Author : Jean-François Rey # 9 # Author : Jean-François Rey #
10 # (base on Emmanuel Ferreira # 10 # (base on Emmanuel Ferreira #
11 # and Hugo Mauchrétien works) # 11 # and Hugo Mauchrétien works) #
12 # Version : 1.0 # 12 # Version : 1.0 #
13 # Date : 25/06/13 # 13 # Date : 25/06/13 #
14 ##################################################### 14 #####################################################
15 15
16 echo "### ExploitConfidencePass.sh ###" 16 echo "### ExploitConfidencePass.sh ###"
17 17
18 # Check OTMEDIA_HOME env var 18 # Check OTMEDIA_HOME env var
19 if [ -z ${OTMEDIA_HOME} ] 19 if [ -z ${OTMEDIA_HOME} ]
20 then 20 then
21 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 21 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
22 export OTMEDIA_HOME=$OTMEDIA_HOME 22 export OTMEDIA_HOME=$OTMEDIA_HOME
23 fi 23 fi
24 24
25 # where is ExploitConfidencePass.sh 25 # where is ExploitConfidencePass.sh
26 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 26 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
27 27
28 if [ -z ${SCRIPT_PATH} ] 28 if [ -z ${SCRIPT_PATH} ]
29 then 29 then
30 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 30 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
31 fi 31 fi
32 32
33 # Include scripts 33 # Include scripts
34 . $SCRIPT_PATH"/Tools.sh" 34 . $SCRIPT_PATH"/Tools.sh"
35 . $SCRIPT_PATH"/CheckExploitConfPass.sh" 35 . $SCRIPT_PATH"/CheckExploitConfPass.sh"
36 36
37 # where is ExploitConfidencePass.cfg 37 # where is ExploitConfidencePass.cfg
38 EXPLOITCONFIDENCEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ExploitConfidencePass.cfg" 38 EXPLOITCONFIDENCEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ExploitConfidencePass.cfg"
39 if [ -e $EXPLOITCONFIDENCEPASS_CONFIG_FILE ] 39 if [ -e $EXPLOITCONFIDENCEPASS_CONFIG_FILE ]
40 then 40 then
41 . $EXPLOITCONFIDENCEPASS_CONFIG_FILE 41 . $EXPLOITCONFIDENCEPASS_CONFIG_FILE
42 else 42 else
43 echo "ERROR : Can't find configuration file $EXPLOITCONFIDENCEPASS_CONFIG_FILE" >&2 43 echo "ERROR : Can't find configuration file $EXPLOITCONFIDENCEPASS_CONFIG_FILE" >&2
44 exit 1 44 exit 1
45 fi 45 fi
46 46
47 #---------------# 47 #---------------#
48 # Parse Options # 48 # Parse Options #
49 #---------------# 49 #---------------#
50 while getopts ":hDv:cf:r" opt 50 while getopts ":hDv:cf:r" opt
51 do 51 do
52 case $opt in 52 case $opt in
53 h) 53 h)
54 echo -e "$0 [OPTIONS] <INPUT_DIRECTORY>\n" 54 echo -e "$0 [OPTIONS] <INPUT_DIRECTORY>\n"
55 echo -e "\t Options:" 55 echo -e "\t Options:"
56 echo -e "\t\t-h :\tprint this message" 56 echo -e "\t\t-h :\tprint this message"
57 echo -e "\t\t-D :\tDEBUG mode on" 57 echo -e "\t\t-D :\tDEBUG mode on"
58 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 58 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
59 echo -e "\t\t-c :\tCheck process, stop if error detected" 59 echo -e "\t\t-c :\tCheck process, stop if error detected"
60 echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)" 60 echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)"
61 echo -e "\t\t-r n :\tforce rerun without deleting files" 61 echo -e "\t\t-r n :\tforce rerun without deleting files"
62 exit 1 62 exit 1
63 ;; 63 ;;
64 D) 64 D)
65 DEBUG=1 65 DEBUG=1
66 ;; 66 ;;
67 v) 67 v)
68 VERBOSE=$OPTARG 68 VERBOSE=$OPTARG
69 ;; 69 ;;
70 c) 70 c)
71 CHECK=1 71 CHECK=1
72 ;; 72 ;;
73 f) 73 f)
74 FORKS="--forks $OPTARG" 74 FORKS="--forks $OPTARG"
75 ;; 75 ;;
76 r) 76 r)
77 RERUN=1 77 RERUN=1
78 ;; 78 ;;
79 :) 79 :)
80 echo "Option -$OPTARG requires an argument." > /dev/stderr 80 echo "Option -$OPTARG requires an argument." >&2
81 exit 1 81 exit 1
82 ;; 82 ;;
83 \?) 83 \?)
84 echo "BAD USAGE : unknow opton -$OPTARG" 84 echo "BAD USAGE : unknow opton -$OPTARG"
85 #exit 1 85 #exit 1
86 ;; 86 ;;
87 esac 87 esac
88 done 88 done
89 89
90 # mode debug enable 90 # mode debug enable
91 if [ $DEBUG -eq 1 ] 91 if [ $DEBUG -eq 1 ]
92 then 92 then
93 set -x 93 set -x
94 echo -e "## Mode DEBUG ON ##" 94 echo -e "## Mode DEBUG ON ##"
95 fi 95 fi
96 96
97 # mode verbose enable 97 # mode verbose enable
98 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi 98 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
99 99
100 # Check USAGE by arguments number 100 # Check USAGE by arguments number
101 if [ $(($#-($OPTIND-1))) -ne 1 ] 101 if [ $(($#-($OPTIND-1))) -ne 1 ]
102 then 102 then
103 echo "BAD USAGE : ExploitConfidencePass.sh [OPTIONS] <INPUT_DIRECTORY>" 103 echo "BAD USAGE : ExploitConfidencePass.sh [OPTIONS] <INPUT_DIRECTORY>"
104 echo "$0 -h for more info" 104 echo "$0 -h for more info"
105 exit 1 105 exit 1
106 fi 106 fi
107 107
108 shift $((OPTIND-1)) 108 shift $((OPTIND-1))
109 # check input directory - first argument 109 # check input directory - first argument
110 if [ ! -e $1 ] 110 if [ ! -e $1 ]
111 then 111 then
112 print_error "can't open $1" 112 print_error "can't open $1"
113 exit 1 113 exit 1
114 fi 114 fi
115 115
116 print_info "[${BASENAME}] => ExploitConfPass start | $(date +'%d/%m/%y %H:%M:%S')" 1 116 print_info "[${BASENAME}] => ExploitConfPass start | $(date +'%d/%m/%y %H:%M:%S')" 1
117 117
118 #-------------# 118 #-------------#
119 # GLOBAL VARS # 119 # GLOBAL VARS #
120 #-------------# 120 #-------------#
121 INPUT_DIR=$(readlink -e $1) 121 INPUT_DIR=$(readlink -e $1)
122 OUTPUT_DIR=$INPUT_DIR 122 OUTPUT_DIR=$INPUT_DIR
123 BASENAME=$(basename $OUTPUT_DIR) 123 BASENAME=$(basename $OUTPUT_DIR)
124 SHOW_DIR="$OUTPUT_DIR/shows/" 124 SHOW_DIR="$OUTPUT_DIR/shows/"
125 SOLR_RES="$OUTPUT_DIR/solr/" 125 SOLR_RES="$OUTPUT_DIR/solr/"
126 EXT_LEX="$OUTPUT_DIR/LEX/" 126 EXT_LEX="$OUTPUT_DIR/LEX/"
127 TRIGGER_CONFZONE="$OUTPUT_DIR/trigg/" 127 TRIGGER_CONFZONE="$OUTPUT_DIR/trigg/"
128 LOGFILE="$OUTPUT_DIR/info_exploitconf.log" 128 LOGFILE="$OUTPUT_DIR/info_exploitconf.log"
129 ERRORFILE="$OUTPUT_DIR/error_exploitconf.log" 129 ERRORFILE="$OUTPUT_DIR/error_exploitconf.log"
130 130
131 CONFPASS_CONFIG_FILE="$(readlink -e $1)/ConfPass.cfg" 131 CONFPASS_CONFIG_FILE="$(readlink -e $1)/ConfPass.cfg"
132 if [ -e $CONFPASS_CONFIG_FILE ] 132 if [ -e $CONFPASS_CONFIG_FILE ]
133 then 133 then
134 { 134 {
135 RES_CONF_DIR=$(cat $CONFPASS_CONFIG_FILE | grep "^RES_CONF_DIR=" | cut -f2 -d"=") 135 RES_CONF_DIR=$(cat $CONFPASS_CONFIG_FILE | grep "^RES_CONF_DIR=" | cut -f2 -d"=")
136 RES_CONF=$(cat $CONFPASS_CONFIG_FILE | grep "^CONF_DIR=" | cut -f2 -d"=") 136 RES_CONF=$(cat $CONFPASS_CONFIG_FILE | grep "^CONF_DIR=" | cut -f2 -d"=")
137 print_info "[${BASENAME}] Use confidence measure from : $RES_CONF" 2 137 print_info "[${BASENAME}] Use confidence measure from : $RES_CONF" 2
138 } 138 }
139 else 139 else
140 { 140 {
141 print_error "[${BASENAME}] Can't find $CONFPASS_CONFIG_FILE" 141 print_error "[${BASENAME}] Can't find $CONFPASS_CONFIG_FILE"
142 print_error "[${BASENAME}] -> use res_p2" 142 print_error "[${BASENAME}] -> use res_p2"
143 RES_CONF_DIR="$INPUT_DIR/conf/res_p2/scored_ctm" 143 RES_CONF_DIR="$INPUT_DIR/conf/res_p2/scored_ctm"
144 RES_CONF="$INPUT_DIR/conf/res_p2" 144 RES_CONF="$INPUT_DIR/conf/res_p2"
145 } 145 }
146 fi 146 fi
147 147
148 mkdir -p $SHOW_DIR > /dev/null 2>&1 148 mkdir -p $SHOW_DIR > /dev/null 2>&1
149 mkdir -p $SOLR_RES > /dev/null 2>&1 149 mkdir -p $SOLR_RES > /dev/null 2>&1
150 mkdir -p $EXT_LEX > /dev/null 2>&1 150 mkdir -p $EXT_LEX > /dev/null 2>&1
151 mkdir -p $TRIGGER_CONFZONE > /dev/null 2>&1 151 mkdir -p $TRIGGER_CONFZONE > /dev/null 2>&1
152 152
153 #------------------# 153 #------------------#
154 # Create Workspace # 154 # Create Workspace #
155 #------------------# 155 #------------------#
156 # Lock directory 156 # Lock directory
157 if [ -e "$OUTPUT_DIR_BASENAME/EXPLOITCONFPASS.lock" ] && [ $RERUN -eq 0 ] 157 if [ -e "$OUTPUT_DIR_BASENAME/EXPLOITCONFPASS.lock" ] && [ $RERUN -eq 0 ]
158 then 158 then
159 print_warn "[${BASENAME}] ExploitConfidencePass is locked -> exit" 2 159 print_warn "[${BASENAME}] ExploitConfidencePass is locked -> exit" 2
160 exit 1 160 exit 1
161 fi 161 fi
162 rm "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" > /dev/null 2>&1 162 rm "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" > /dev/null 2>&1
163 touch "$OUTPUT_DIR/EXPLOITCONFPASS.lock" > /dev/null 2>&1 163 touch "$OUTPUT_DIR/EXPLOITCONFPASS.lock" > /dev/null 2>&1
164 164
165 #------# 165 #------#
166 # Save # 166 # Save #
167 #------# 167 #------#
168 cp $EXPLOITCONFIDENCEPASS_CONFIG_FILE $OUTPUT_DIR/ExploitConfPass.cfg 168 cp $EXPLOITCONFIDENCEPASS_CONFIG_FILE $OUTPUT_DIR/ExploitConfPass.cfg
169 echo "TRIGGER_DIR=$TRIGGER_CONFZONE" >> $OUTPUT_DIR/ExploitConfPass.cfg 169 echo "TRIGGER_DIR=$TRIGGER_CONFZONE" >> $OUTPUT_DIR/ExploitConfPass.cfg
170 echo "TRIGGER_SPEERAL=$TRIGGER_CONFZONE/speeral/" >> $OUTPUT_DIR/ExploitConfPass.cfg 170 echo "TRIGGER_SPEERAL=$TRIGGER_CONFZONE/speeral/" >> $OUTPUT_DIR/ExploitConfPass.cfg
171 echo "LEX_SPEERAL=$EXT_LEX/speeral/${lexname}_ext" >> $OUTPUT_DIR/ExploitConfPass.cfg 171 echo "LEX_SPEERAL=$EXT_LEX/speeral/${lexname}_ext" >> $OUTPUT_DIR/ExploitConfPass.cfg
172 echo "LEX_BINODE_SPEERAL=$EXT_LEX/speeral/${lexname}_ext.bin" >> $OUTPUT_DIR/ExploitConfPass.cfg 172 echo "LEX_BINODE_SPEERAL=$EXT_LEX/speeral/${lexname}_ext.bin" >> $OUTPUT_DIR/ExploitConfPass.cfg
173 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ExploitConfPass.cfg" 1 173 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ExploitConfPass.cfg" 1
174 174
175 #---------------# 175 #---------------#
176 # Check Pass # 176 # Check Pass #
177 #---------------# 177 #---------------#
178 if [ $( ${RES_CONf_DIR}/*.res 2> /dev/null | wc -l) -eq 0 ] 178 if [ $( ${RES_CONf_DIR}/*.res 2> /dev/null | wc -l) -eq 0 ]
179 then 179 then
180 print_error "[${BASENAME}] No Conf Pass res -> exit ExploitConfPass" 180 print_error "[${BASENAME}] No Conf Pass res -> exit ExploitConfPass"
181 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ConfPass res in $[RES_CONf_DIR}" ;fi 181 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ConfPass res in $[RES_CONf_DIR}" ;fi
182 exit 1 182 exit 1
183 fi 183 fi
184 184
185 #-----------------------# 185 #-----------------------#
186 # Segmentation by show # 186 # Segmentation by show #
187 #-----------------------# 187 #-----------------------#
188 # create txt file from scored res 188 # create txt file from scored res
189 # tag pos and lemmatization of the txt file 189 # tag pos and lemmatization of the txt file
190 # merge the scored res and taglem file 190 # merge the scored res and taglem file
191 # segment using the last generated file 191 # segment using the last generated file
192 # and create a ctm file by show 192 # and create a ctm file by show
193 193
194 print_info "[${BASENAME}] Segmentation by show" 1 194 print_info "[${BASENAME}] Segmentation by show" 1
195 195
196 # -> to txt 196 # -> to txt
197 print_info "[${BASENAME}] Create txt from scored res" 3 197 print_info "[${BASENAME}] Create txt from scored res" 3
198 cat ${RES_CONF_DIR}/*.res > $INPUT_DIR/$BASENAME.sctm 198 cat ${RES_CONF_DIR}/*.res > $INPUT_DIR/$BASENAME.sctm
199 cat $INPUT_DIR/$BASENAME.seg | $SIGMUND_BIN/myConvert.pl $INPUT_DIR/$BASENAME.sctm $INPUT_DIR/$BASENAME.tmp 199 cat $INPUT_DIR/$BASENAME.seg | $SIGMUND_BIN/myConvert.pl $INPUT_DIR/$BASENAME.sctm $INPUT_DIR/$BASENAME.tmp
200 cat $INPUT_DIR/$BASENAME.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | sed -e "s/_/ /g" | sort -nt 'n' -k '2' > $INPUT_DIR/$BASENAME.txt 200 cat $INPUT_DIR/$BASENAME.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | sed -e "s/_/ /g" | sort -nt 'n' -k '2' > $INPUT_DIR/$BASENAME.txt
201 201
202 # -> to tagger + lemme 202 # -> to tagger + lemme
203 print_info "[${BASENAME}] Tag pos and lem in txt file" 3 203 print_info "[${BASENAME}] Tag pos and lem in txt file" 3
204 iconv -t ISO_8859-1 $INPUT_DIR/$BASENAME.txt > $INPUT_DIR/$BASENAME.tmp 204 iconv -t ISO_8859-1 $INPUT_DIR/$BASENAME.txt > $INPUT_DIR/$BASENAME.tmp
205 $SIGMUND_BIN/txt2lem.sh $INPUT_DIR/$BASENAME.tmp $INPUT_DIR/$BASENAME.taglem 205 $SIGMUND_BIN/txt2lem.sh $INPUT_DIR/$BASENAME.tmp $INPUT_DIR/$BASENAME.taglem
206 206
207 # merge sctm and taglem 207 # merge sctm and taglem
208 print_info "[${BASENAME}] Merge scored ctm with tag pos and lem file" 3 208 print_info "[${BASENAME}] Merge scored ctm with tag pos and lem file" 3
209 cat $INPUT_DIR/$BASENAME.sctm | $SCRIPT_PATH/BdlexUC.pl ${RULES}/basic -f | iconv -t ISO_8859-1 | $SCRIPT_PATH/scoredCtmAndTaggedLem2All.pl $INPUT_DIR/$BASENAME.taglem > $INPUT_DIR/$BASENAME.ctl 209 cat $INPUT_DIR/$BASENAME.sctm | $SCRIPT_PATH/BdlexUC.pl ${RULES}/basic -f | iconv -t ISO_8859-1 | $SCRIPT_PATH/scoredCtmAndTaggedLem2All.pl $INPUT_DIR/$BASENAME.taglem > $INPUT_DIR/$BASENAME.ctl
210 210
211 # -> new seg 211 # -> new seg
212 print_info "[${BASENAME}] Create xml file and run Topic Seg" 3 212 print_info "[${BASENAME}] Create xml file and run Topic Seg" 3
213 $SIGMUND_BIN/tagLem2xml.pl $INPUT_DIR/$BASENAME.taglem $INPUT_DIR/$BASENAME.doc.xml 213 $SIGMUND_BIN/tagLem2xml.pl $INPUT_DIR/$BASENAME.taglem $INPUT_DIR/$BASENAME.doc.xml
214 rm $INPUT_DIR/$BASENAME.tmp #$INPUT_DIR/$BASENAME.taglem 214 rm $INPUT_DIR/$BASENAME.tmp #$INPUT_DIR/$BASENAME.taglem
215 215
216 # Lia_topic_seg : bring together sentences into show 216 # Lia_topic_seg : bring together sentences into show
217 cp $INPUT_DIR/$BASENAME.doc.xml 0.xml 217 cp $INPUT_DIR/$BASENAME.doc.xml 0.xml
218 java -cp $LIATOPICSEG/bin Test > $INPUT_DIR/show.seg 218 java -cp $LIATOPICSEG/bin Test > $INPUT_DIR/show.seg
219 cat $INPUT_DIR/show.seg | $SIGMUND_BIN/toSegEmiss.pl $INPUT_DIR/$BASENAME.show.seg 219 cat $INPUT_DIR/show.seg | $SIGMUND_BIN/toSegEmiss.pl $INPUT_DIR/$BASENAME.show.seg
220 rm 0.xml $INPUT_DIR/show.seg 220 rm 0.xml $INPUT_DIR/show.seg
221 221
222 if [ $CHECK -eq 1 ] 222 if [ $CHECK -eq 1 ]
223 then 223 then
224 if [ ! -s $INPUT_DIR/$BASENAME.show.seg ] 224 if [ ! -s $INPUT_DIR/$BASENAME.show.seg ]
225 then 225 then
226 print_error "[${BASENAME}] No Topic segmentation ! " 226 print_error "[${BASENAME}] No Topic segmentation ! "
227 print_error "[${BASENAME}] Check $ERRORFILE " 227 print_error "[${BASENAME}] Check $ERRORFILE "
228 print_log_file "$ERRORFILE" "No Topic segmentation in ${BASENAME}.show.seg" 228 print_log_file "$ERRORFILE" "No Topic segmentation in ${BASENAME}.show.seg"
229 fi 229 fi
230 fi 230 fi
231 231
232 # Segment ctm into several show files and create a seg list by show 232 # Segment ctm into several show files and create a seg list by show
233 print_info "[${BASENAME}] Segment ctm into show files and a seg list by show" 1 233 print_info "[${BASENAME}] Segment ctm into show files and a seg list by show" 1
234 $SCRIPT_PATH/ctm2show.pl $INPUT_DIR/$BASENAME.ctl $INPUT_DIR/$BASENAME.show.seg $SHOW_DIR 234 $SCRIPT_PATH/ctm2show.pl $INPUT_DIR/$BASENAME.ctl $INPUT_DIR/$BASENAME.show.seg $SHOW_DIR
235 235
236 #-----------------------------------------------------------# 236 #-----------------------------------------------------------#
237 # SOLR QUERIES # 237 # SOLR QUERIES #
238 # -> Create Confidente Word # 238 # -> Create Confidente Word #
239 # Keep conf words and use Tags # 239 # Keep conf words and use Tags #
240 # -> Query SOLR (document & multimedia) # 240 # -> Query SOLR (document & multimedia) #
241 # concat word + add date 2 day before and after the show # 241 # concat word + add date 2 day before and after the show #
242 # query document & multimedia # 242 # query document & multimedia #
243 #-----------------------------------------------------------# 243 #-----------------------------------------------------------#
244 print_info "[${BASENAME}] Create SOLR queries and ask SOLR" 1 244 print_info "[${BASENAME}] Create SOLR queries and ask SOLR" 1
245 for show in $(ls $SHOW_DIR/*.ctm) 245 for show in $(ls $SHOW_DIR/*.ctm)
246 do 246 do
247 bn=$(basename $show .ctm) 247 bn=$(basename $show .ctm)
248 # Remove words with low confidence and keep useful tagger words 248 # Remove words with low confidence and keep useful tagger words
249 cat $show | $SCRIPT_PATH/KeepConfZone.pl | grep -e "MOTINC\|NMS\|NMP\|NFS\|NFP\|X[A-Z]{3,5}" | cut -f3 -d' ' > "$SHOW_DIR/$bn.confzone" 249 cat $show | $SCRIPT_PATH/KeepConfZone.pl | grep -e "MOTINC\|NMS\|NMP\|NFS\|NFP\|X[A-Z]{3,5}" | cut -f3 -d' ' > "$SHOW_DIR/$bn.confzone"
250 # Get date 2 day before and after the show 250 # Get date 2 day before and after the show
251 datePattern=`$SCRIPT_PATH/daybefore2after.sh $(echo $BASENAME | cut -c1-6)` 251 datePattern=`$SCRIPT_PATH/daybefore2after.sh $(echo $BASENAME | cut -c1-6)`
252 # Create SOLR queries 252 # Create SOLR queries
253 cat $SHOW_DIR/$bn".confzone" | $SCRIPT_PATH/GenerateSOLRQueries.pl | iconv -f ISO_8859-1 -t UTF-8 > "$SHOW_DIR/$bn.queries" 253 cat $SHOW_DIR/$bn".confzone" | $SCRIPT_PATH/GenerateSOLRQueries.pl | iconv -f ISO_8859-1 -t UTF-8 > "$SHOW_DIR/$bn.queries"
254 # Ask SOLR DB 254 # Ask SOLR DB
255 if [ $(wc -w "$SHOW_DIR/$bn.queries" | cut -f1 -d' ') -gt 0 ]; then 255 if [ $(wc -w "$SHOW_DIR/$bn.queries" | cut -f1 -d' ') -gt 0 ]; then
256 query=$(cat $SHOW_DIR/$bn.queries)"&fq=docDate:[$datePattern]" 256 query=$(cat $SHOW_DIR/$bn.queries)"&fq=docDate:[$datePattern]"
257 echo $query > $SHOW_DIR/$bn.queries 257 echo $query > $SHOW_DIR/$bn.queries
258 prnt_info "python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp" 3 258 prnt_info "python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp" 3
259 python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp 259 python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp
260 cat $SOLR_RES/$bn.keywords.tmp | sort -u > $SOLR_RES/$bn.keywords 260 cat $SOLR_RES/$bn.keywords.tmp | sort -u > $SOLR_RES/$bn.keywords
261 cat $SOLR_RES/$bn.txt.tmp | sort -u > $SOLR_RES/$bn.txt 261 cat $SOLR_RES/$bn.txt.tmp | sort -u > $SOLR_RES/$bn.txt
262 rm $SOLR_RES/*.tmp > /dev/null 2>&1 262 rm $SOLR_RES/*.tmp > /dev/null 2>&1
263 fi 263 fi
264 264
265 if [ $CHECK -eq 1 ] 265 if [ $CHECK -eq 1 ]
266 then 266 then
267 if [ ! -e $SOLR_RES/$bn.keywords ] || [ ! -e $SOLR_RES/$bn.txt ] 267 if [ ! -e $SOLR_RES/$bn.keywords ] || [ ! -e $SOLR_RES/$bn.txt ]
268 then 268 then
269 print_warn "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 2 269 print_warn "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 2
270 print_log_file "$LOGFILE" "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 270 print_log_file "$LOGFILE" "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !"
271 fi 271 fi
272 fi 272 fi
273 273
274 done 274 done
275 275
276 #----------------------------------------------------------------------------------------------- 276 #-----------------------------------------------------------------------------------------------
277 # Build trigger file 277 # Build trigger file
278 # 1) keywords are automatically boosted in the non confident zone of the current res 278 # 1) keywords are automatically boosted in the non confident zone of the current res
279 # confident zone are boosted 279 # confident zone are boosted
280 # previous words in sensible zone are penalized 280 # previous words in sensible zone are penalized
281 # 2) OOVs are extracted + phonetized 281 # 2) OOVs are extracted + phonetized
282 # 3) Try to find OOVs acousticly in the current segment 282 # 3) Try to find OOVs acousticly in the current segment
283 # 4) Generate the .trigg file 283 # 4) Generate the .trigg file
284 #------------------------------------------------------------------------------------------------ 284 #------------------------------------------------------------------------------------------------
285 print_info "[${BASENAME}] Build trigger files" 1 285 print_info "[${BASENAME}] Build trigger files" 1
286 for i in `ls $SOLR_RES/*.keywords` 286 for i in `ls $SOLR_RES/*.keywords`
287 do 287 do
288 basename=`basename $i .keywords` 288 basename=`basename $i .keywords`
289 289
290 # 290 #
291 # Tokenize & produce coverage report 291 # Tokenize & produce coverage report
292 # Use filter you need 292 # Use filter you need
293 # 293 #
294 print_info "[${BASENAME}] keywords filtering and produce coverage report" 3 294 print_info "[${BASENAME}] keywords filtering and produce coverage report" 3
295 # Default filter 295 # Default filter
296 cat $i | $SCRIPT_PATH/CleanFilter.sh | ${SCRIPT_PATH}/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t |\ 296 cat $i | $SCRIPT_PATH/CleanFilter.sh | ${SCRIPT_PATH}/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t |\
297 $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok 297 $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok
298 # do less filter 298 # do less filter
299 #cat $i | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex | $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok 299 #cat $i | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex | $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok
300 300
301 301
302 # 302 #
303 # Extract "real" OOV and phonetize them 303 # Extract "real" OOV and phonetize them
304 # -> petit filtrage persoo pour eviter d'avoir trop de bruits 304 # -> petit filtrage persoo pour eviter d'avoir trop de bruits
305 # 305 #
306 print_info "[${BASENAME}] Extract OOV and phonetize them" 3 306 print_info "[${BASENAME}] Extract OOV and phonetize them" 3
307 ${SCRIPT_PATH}/FindNormRules.pl $SOLR_RES/${basename}_tmp_report/report.oov $LEXICON.bdlex_tok | cut -f3 | grep -v "#" | grep -v "^[A-Z]\+$" | grep -v "^[0-9]" | grep --perl-regex -v "^([a-z']){1,3}$" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | iconv -t ISO_8859-1 -f UTF-8 | ${LIA_LTBOX}/lia_phon/script/lia_lex2phon_variante | grep -v "core dumped" | cut -d"[" -f1 | sort -u | ${SCRIPT_PATH}/PhonFormatter.pl | iconv -f ISO_8859-1 -t UTF-8 | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $SOLR_RES/${basename}.phon_oov 307 ${SCRIPT_PATH}/FindNormRules.pl $SOLR_RES/${basename}_tmp_report/report.oov $LEXICON.bdlex_tok | cut -f3 | grep -v "#" | grep -v "^[A-Z]\+$" | grep -v "^[0-9]" | grep --perl-regex -v "^([a-z']){1,3}$" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | iconv -t ISO_8859-1 -f UTF-8 | ${LIA_LTBOX}/lia_phon/script/lia_lex2phon_variante | grep -v "core dumped" | cut -d"[" -f1 | sort -u | ${SCRIPT_PATH}/PhonFormatter.pl | iconv -f ISO_8859-1 -t UTF-8 | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $SOLR_RES/${basename}.phon_oov
308 308
309 # 309 #
310 # Search INVOC & OOV in the current lattice 310 # Search INVOC & OOV in the current lattice
311 # 311 #
312 print_info "[${BASENAME}] Search INVOC and OOV in the current lattice" 3 312 print_info "[${BASENAME}] Search INVOC and OOV in the current lattice" 3
313 cat $SOLR_RES/${basename}_tmp_report/report.invoc | grep -v "\b0" | cut -f1 | grep -v --perl-regex -v "^[a-zA-Z']{1,3}$" | grep -v --perl-regex "^[a-zA-Z0-9]{1,3}$" | grep -v "<s>" | grep -v "</s>" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $TRIGGER_CONFZONE/$basename.tosearch 313 cat $SOLR_RES/${basename}_tmp_report/report.invoc | grep -v "\b0" | cut -f1 | grep -v --perl-regex -v "^[a-zA-Z']{1,3}$" | grep -v --perl-regex "^[a-zA-Z0-9]{1,3}$" | grep -v "<s>" | grep -v "</s>" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $TRIGGER_CONFZONE/$basename.tosearch
314 cat $SOLR_RES/${basename}.phon_oov | cut -f1 >> $TRIGGER_CONFZONE/$basename.tosearch 314 cat $SOLR_RES/${basename}.phon_oov | cut -f1 >> $TRIGGER_CONFZONE/$basename.tosearch
315 315
316 # For each treil 316 # For each treil
317 for baseseg in $(cat "$SHOW_DIR/$basename.lst") 317 for baseseg in $(cat "$SHOW_DIR/$basename.lst")
318 do 318 do
319 $OTMEDIA_HOME/tools/QUOTE_FINDER/bin/acousticFinder ${LEXICON}.speer_phon $RES_CONF/wlat/$baseseg.wlat $TRIGGER_CONFZONE/${basename}.tosearch $SOLR_RES/$basename.phon_oov > $TRIGGER_CONFZONE/$baseseg.acousticlyfound $OUTPUT_REDIRECTION 319 $OTMEDIA_HOME/tools/QUOTE_FINDER/bin/acousticFinder ${LEXICON}.speer_phon $RES_CONF/wlat/$baseseg.wlat $TRIGGER_CONFZONE/${basename}.tosearch $SOLR_RES/$basename.phon_oov > $TRIGGER_CONFZONE/$baseseg.acousticlyfound $OUTPUT_REDIRECTION
320 # 320 #
321 # Produce the boost file for the next decoding pass 321 # Produce the boost file for the next decoding pass
322 # 322 #
323 print_info "[${BASENAME}] Produce trigg file : $baseseg " 3 323 print_info "[${BASENAME}] Produce trigg file : $baseseg " 3
324 cat $RES_CONF_DIR/$baseseg.res | $SCRIPT_PATH/ScoreCtm2trigg.pl $TRIGGER_CONFZONE/$baseseg.acousticlyfound > $TRIGGER_CONFZONE/$baseseg.trigg 324 cat $RES_CONF_DIR/$baseseg.res | $SCRIPT_PATH/ScoreCtm2trigg.pl $TRIGGER_CONFZONE/$baseseg.acousticlyfound > $TRIGGER_CONFZONE/$baseseg.trigg
325 done 325 done
326 326
327 done 327 done
328 328
329 #----------------------------------------------------------------------------------------------- 329 #-----------------------------------------------------------------------------------------------
330 # Build the extended SPEERAL Lexicon 330 # Build the extended SPEERAL Lexicon
331 # 1) Merge OOVs + LEXICON 331 # 1) Merge OOVs + LEXICON
332 # 1) Related text are collected in order to find the invoc word with maximizing the ppl (LM proba) 332 # 1) Related text are collected in order to find the invoc word with maximizing the ppl (LM proba)
333 # 2) The current lexicon is extended with all the valid OOVs 333 # 2) The current lexicon is extended with all the valid OOVs
334 #----------------------------------------------------------------------------------------------- 334 #-----------------------------------------------------------------------------------------------
335 print_info "[${BASENAME}] Build extended Speeral Lexicon" 1 335 print_info "[${BASENAME}] Build extended Speeral Lexicon" 1
336 mkdir -p $EXT_LEX/final 336 mkdir -p $EXT_LEX/final
337 mkdir -p $EXT_LEX/tmp 337 mkdir -p $EXT_LEX/tmp
338 mkdir -p $EXT_LEX/tmp/txt 338 mkdir -p $EXT_LEX/tmp/txt
339 # 339 #
340 # Collect the acousticly found oov and their phonetisation 340 # Collect the acousticly found oov and their phonetisation
341 # 341 #
342 print_info "[${BASENAME}] Get all OOV and retrieve all phonetisation" 3 342 print_info "[${BASENAME}] Get all OOV and retrieve all phonetisation" 3
343 for i in `ls $SOLR_RES/*.phon_oov` 343 for i in `ls $SOLR_RES/*.phon_oov`
344 do 344 do
345 basename=`basename $i .phon_oov` 345 basename=`basename $i .phon_oov`
346 346
347 rm $EXT_LEX/$basename.acousticlyfound 2> /dev/null 347 rm $EXT_LEX/$basename.acousticlyfound 2> /dev/null
348 # list acousticly found for the show 348 # list acousticly found for the show
349 for baseseg in $(cat "$SHOW_DIR/$basename.lst") 349 for baseseg in $(cat "$SHOW_DIR/$basename.lst")
350 do 350 do
351 cat $TRIGGER_CONFZONE/$baseseg.acousticlyfound | cut -f1 | cut -f2 -d"=" >> $EXT_LEX/$basename.acousticlyfound 351 cat $TRIGGER_CONFZONE/$baseseg.acousticlyfound | cut -f1 | cut -f2 -d"=" >> $EXT_LEX/$basename.acousticlyfound
352 done 352 done
353 cat $EXT_LEX/$basename.acousticlyfound | sort -u > $EXT_LEX/.tmp 353 cat $EXT_LEX/$basename.acousticlyfound | sort -u > $EXT_LEX/.tmp
354 mv $EXT_LEX/.tmp $EXT_LEX/$basename.acousticlyfound 354 mv $EXT_LEX/.tmp $EXT_LEX/$basename.acousticlyfound
355 355
356 # 356 #
357 # Extract OOV really added 357 # Extract OOV really added
358 # 358 #
359 cat $SOLR_RES/$basename.phon_oov | cut -f1 | sort -u > $EXT_LEX/$basename.oov 359 cat $SOLR_RES/$basename.phon_oov | cut -f1 | sort -u > $EXT_LEX/$basename.oov
360 $SCRIPT_PATH/intersec.pl $EXT_LEX/$basename.oov $EXT_LEX/$basename.acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound 360 $SCRIPT_PATH/intersec.pl $EXT_LEX/$basename.oov $EXT_LEX/$basename.acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound
361 # 361 #
362 # Retrieve all phonetisation 362 # Retrieve all phonetisation
363 # 363 #
364 cat $SOLR_RES/${basename}.phon_oov | $SCRIPT_PATH/LexPhonFilter.pl $EXT_LEX/$basename.oov_acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound_phon 364 cat $SOLR_RES/${basename}.phon_oov | $SCRIPT_PATH/LexPhonFilter.pl $EXT_LEX/$basename.oov_acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound_phon
365 done 365 done
366 366
367 # 367 #
368 # Merge OOVs and their phonetisation 368 # Merge OOVs and their phonetisation
369 # 369 #
370 print_info "[${BASENAME}] Merge OOV and their phonetisation" 3 370 print_info "[${BASENAME}] Merge OOV and their phonetisation" 3
371 lexname=$(basename $LEXICON) 371 lexname=$(basename $LEXICON)
372 cat $EXT_LEX/*.oov_acousticlyfound_phon | sort -u > $EXT_LEX/final/all.oov_acousticlyfound_phon 372 cat $EXT_LEX/*.oov_acousticlyfound_phon | sort -u > $EXT_LEX/final/all.oov_acousticlyfound_phon
373 cat $EXT_LEX/*.oov_acousticlyfound | sort -u | grep --perl-regex -v "^([a-z']){3}$" > $EXT_LEX/final/all.oov_acousticlyfound 373 cat $EXT_LEX/*.oov_acousticlyfound | sort -u | grep --perl-regex -v "^([a-z']){3}$" > $EXT_LEX/final/all.oov_acousticlyfound
374 $SCRIPT_PATH/MergeLexicon.pl $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/final/${lexname}_ext.phon 374 $SCRIPT_PATH/MergeLexicon.pl $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/final/${lexname}_ext.phon
375 375
376 # 376 #
377 # Collect + clean retrieved txt 377 # Collect + clean retrieved txt
378 # 378 #
379 print_info "[${BASENAME}] Collect and clean SOLR txt answers" 2 379 print_info "[${BASENAME}] Collect and clean SOLR txt answers" 2
380 # choose filter 380 # choose filter
381 # default 381 # default
382 cat $SOLR_RES/*.txt | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $EXT_LEX/final/all.bdlex_txt 382 cat $SOLR_RES/*.txt | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $EXT_LEX/final/all.bdlex_txt
383 # low filter 383 # low filter
384 #cat $SOLR_RES/*.txt | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex > $EXT_LEX/final/all.bdlex_txt 384 #cat $SOLR_RES/*.txt | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex > $EXT_LEX/final/all.bdlex_txt
385 385
386 # 386 #
387 # Construct the map file 387 # Construct the map file
388 # 388 #
389 # Notes: 389 # Notes:
390 # - Expected format : 390 # - Expected format :
391 # <WORD1_STRING> <CANDIDATE1_STRING> <PHON_1> 391 # <WORD1_STRING> <CANDIDATE1_STRING> <PHON_1>
392 # 392 #
393 print_info "[${BASENAME}] Construct map file" 3 393 print_info "[${BASENAME}] Construct map file" 3
394 rm -f $EXT_LEX/final/${lexname}_ext.map 2>/dev/null 394 rm -f $EXT_LEX/final/${lexname}_ext.map 2>/dev/null
395 rm -f $EXT_LEX/final/${lexname}.unvalid_oov 2>/dev/null 395 rm -f $EXT_LEX/final/${lexname}.unvalid_oov 2>/dev/null
396 396
397 while read oov 397 while read oov
398 do 398 do
399 oov=`echo $oov | sed "s/\n//g"` 399 oov=`echo $oov | sed "s/\n//g"`
400 # 400 #
401 # Obtain the oov's tag 401 # Obtain the oov's tag
402 # 402 #
403 #oov_tag=`grep --perl-regex "^$oov\t" $DYNAMIC_TAGSTATS/all.tags | cut -f2` 403 #oov_tag=`grep --perl-regex "^$oov\t" $DYNAMIC_TAGSTATS/all.tags | cut -f2`
404 # 404 #
405 # Try to collect text containing the oov word 405 # Try to collect text containing the oov word
406 # 406 #
407 print_info "[${BASENAME}] Collect text containing the oov" 3 407 print_info "[${BASENAME}] Collect text containing the oov" 3
408 cat $EXT_LEX/final/all.bdlex_txt | grep --perl-regex " $oov " | $SCRIPT_PATH/NbMaxWordsFilter.pl 40 |uniq > $EXT_LEX/tmp/txt/$oov.bdlex_txt 408 cat $EXT_LEX/final/all.bdlex_txt | grep --perl-regex " $oov " | $SCRIPT_PATH/NbMaxWordsFilter.pl 40 |uniq > $EXT_LEX/tmp/txt/$oov.bdlex_txt
409 if [ -f $EXT_LEX/tmp/txt/$oov.bdlex_txt ]; then 409 if [ -f $EXT_LEX/tmp/txt/$oov.bdlex_txt ]; then
410 nbWords=`wc -l $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` 410 nbWords=`wc -l $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "`
411 if [ $nbWords -eq 0 ]; then 411 if [ $nbWords -eq 0 ]; then
412 print_warn "[${BASENAME}] UNVALID OOV: $oov => $nbWords occurrences" 2 412 print_warn "[${BASENAME}] UNVALID OOV: $oov => $nbWords occurrences" 2
413 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov 413 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov
414 else 414 else
415 # 415 #
416 # Find a candidate in a filtred invoc lexicon => a candidate which maximize the ppl in the overall txt collected 416 # Find a candidate in a filtred invoc lexicon => a candidate which maximize the ppl in the overall txt collected
417 # 417 #
418 #echo "$/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $LEXICON.bdlex_tok $EXT_LEX/tmp/txt/$oov.bdlex_txt" 418 #echo "$/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $LEXICON.bdlex_tok $EXT_LEX/tmp/txt/$oov.bdlex_txt"
419 print_info `$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` 3 419 print_info `$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` 3
420 candidate=`$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` 420 candidate=`$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "`
421 if [ ! "$candidate" == "" ]; then 421 if [ ! "$candidate" == "" ]; then
422 grep --perl-regex "^$oov\t" $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/tmp/$oov.phon 422 grep --perl-regex "^$oov\t" $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/tmp/$oov.phon
423 while read phonLine 423 while read phonLine
424 do 424 do
425 #<word> <phon> => <word> <candidate> <phon> 425 #<word> <phon> => <word> <candidate> <phon>
426 echo "$phonLine" | sed "s|\t|\t$candidate\t|" >> $EXT_LEX/final/${lexname}_ext.map 426 echo "$phonLine" | sed "s|\t|\t$candidate\t|" >> $EXT_LEX/final/${lexname}_ext.map
427 done < $EXT_LEX/tmp/$oov.phon 427 done < $EXT_LEX/tmp/$oov.phon
428 else 428 else
429 print_warn "[${BASENAME}] UNVALID OOV: $oov => no availaible Candidate word in LM" 2 429 print_warn "[${BASENAME}] UNVALID OOV: $oov => no availaible Candidate word in LM" 2
430 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov 430 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov
431 fi 431 fi
432 fi 432 fi
433 else 433 else
434 print_warn "[${BASENAME}] UNVALID OOV: $oov" 2 434 print_warn "[${BASENAME}] UNVALID OOV: $oov" 2
435 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov 435 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov
436 fi 436 fi
437 done < $EXT_LEX/final/all.oov_acousticlyfound 437 done < $EXT_LEX/final/all.oov_acousticlyfound
438 438
439 # 439 #
440 ### Speeral 440 ### Speeral
441 # 441 #
442 442
443 lexname=`basename $LEXICON` 443 lexname=`basename $LEXICON`
444 # 444 #
445 # Build the final trigger file 445 # Build the final trigger file
446 # 446 #
447 print_info "[${BASENAME}] Clean trigg files" 3 447 print_info "[${BASENAME}] Clean trigg files" 3
448 mkdir -p $TRIGGER_CONFZONE/speeral/ 2> /dev/null 448 mkdir -p $TRIGGER_CONFZONE/speeral/ 2> /dev/null
449 mkdir -p $EXT_LEX/speeral/ 2> /dev/null 449 mkdir -p $EXT_LEX/speeral/ 2> /dev/null
450 for i in `ls $TRIGGER_CONFZONE/*.trigg` 450 for i in `ls $TRIGGER_CONFZONE/*.trigg`
451 do 451 do
452 basename=`basename $i .trigg` 452 basename=`basename $i .trigg`
453 cat $i | $SCRIPT_PATH/RemoveLineContaining.pl $EXT_LEX/$lexname.unvalid_oov > $TRIGGER_CONFZONE/speeral/$basename.trigg 453 cat $i | $SCRIPT_PATH/RemoveLineContaining.pl $EXT_LEX/$lexname.unvalid_oov > $TRIGGER_CONFZONE/speeral/$basename.trigg
454 done 454 done
455 # 455 #
456 # Compile the speeral extended lexicon 456 # Compile the speeral extended lexicon
457 # 457 #
458 print_info "[${BASENAME}] Compile Speeral extended lexicon" 3 458 print_info "[${BASENAME}] Compile Speeral extended lexicon" 3
459 print_info "$SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext" 3 459 print_info "$SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext" 3
460 $SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext 460 $SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext
461 461
462 if [ $CHECK -eq 1 ] 462 if [ $CHECK -eq 1 ]
463 then 463 then
464 check_exploitconfpass_lex_check "${EXT_LEX}/speeral/${lexname}_ext" 464 check_exploitconfpass_lex_check "${EXT_LEX}/speeral/${lexname}_ext"
465 if [ $? -eq 1 ] 465 if [ $? -eq 1 ]
466 then 466 then
467 print_error "[${BASENAME}] Building Speeral Lexicon $INPUT_DIR -> exit" 467 print_error "[${BASENAME}] Building Speeral Lexicon $INPUT_DIR -> exit"
468 print_error "[${BASENAME}] Check $ERRORFILE" 468 print_error "[${BASENAME}] Check $ERRORFILE"
469 print_log_file $ERRORFILE "ERROR : Building Speeral Lexicon $INPUT_DIR" 469 print_log_file $ERRORFILE "ERROR : Building Speeral Lexicon $INPUT_DIR"
470 print_log_file $ERRORFILE "ERROR : ${EXT_LEX}/speeral/${lexname}_ext Empty after buildmappedbinode ?" 470 print_log_file $ERRORFILE "ERROR : ${EXT_LEX}/speeral/${lexname}_ext Empty after buildmappedbinode ?"
471 exit 1; 471 exit 1;
472 fi 472 fi
473 fi 473 fi
474 474
475 475
476 #-------# 476 #-------#
477 # CLOSE # 477 # CLOSE #
478 #-------# 478 #-------#
479 # Seem OK 479 # Seem OK
480 print_info "[${BASENAME}] <= ExploitConfidencePass End | $(date +'%d/%m/%y %H:%M:%S')" 1 480 print_info "[${BASENAME}] <= ExploitConfidencePass End | $(date +'%d/%m/%y %H:%M:%S')" 1
481 481
482 # unlok directory 482 # unlok directory
483 mv "$OUTPUT_DIR/EXPLOITCONFPASS.lock" "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" 483 mv "$OUTPUT_DIR/EXPLOITCONFPASS.lock" "$OUTPUT_DIR/EXPLOITCONFPASS.unlock"
484 484
485 485
486 486
main_tools/FirstPass.sh
Diff comments   View file @ d7e9e4b
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : FirstPass.sh # 4 # File : FirstPass.sh #
5 # Brief : ASR first pass and speaker diarization # 5 # Brief : ASR first pass and speaker diarization #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # (base on Emmanuel Ferreira # 7 # (base on Emmanuel Ferreira #
8 # and Hugo Mauchrétien works) # 8 # and Hugo Mauchrétien works) #
9 # Version : 1.1 # 9 # Version : 1.1 #
10 # Date : 18/06/13 # 10 # Date : 18/06/13 #
11 ##################################################### 11 #####################################################
12 12
13 echo "### FirstPass.sh ###" 13 echo "### FirstPass.sh ###"
14 14
15 # Check OTMEDIA_HOME env var 15 # Check OTMEDIA_HOME env var
16 if [ -z ${OTMEDIA_HOME} ] 16 if [ -z ${OTMEDIA_HOME} ]
17 then 17 then
18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
19 export OTMEDIA_HOME=$OTMEDIA_HOME 19 export OTMEDIA_HOME=$OTMEDIA_HOME
20 fi 20 fi
21 21
22 # where is FirstPass.sh 22 # where is FirstPass.sh
23 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 23 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
24 24
25 # scripts path 25 # scripts path
26 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 26 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
27 27
28 # Include scripts 28 # Include scripts
29 . $SCRIPT_PATH"/Tools.sh" 29 . $SCRIPT_PATH"/Tools.sh"
30 . $SCRIPT_PATH"/CheckFirstPass.sh" 30 . $SCRIPT_PATH"/CheckFirstPass.sh"
31 31
32 # where is FirstPass.cfg 32 # where is FirstPass.cfg
33 FIRSTPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/FirstPass.cfg" 33 FIRSTPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/FirstPass.cfg"
34 if [ -e $FIRSTPASS_CONFIG_FILE ] 34 if [ -e $FIRSTPASS_CONFIG_FILE ]
35 then 35 then
36 . $FIRSTPASS_CONFIG_FILE 36 . $FIRSTPASS_CONFIG_FILE
37 else 37 else
38 echo "ERROR : Can't find configuration file $FIRSTPASS_CONFIG_FILE" > /dev/stderr 38 echo "ERROR : Can't find configuration file $FIRSTPASS_CONFIG_FILE" >&2
39 echo "exit" > /dev/stderr 39 echo "exit" >&2
40 exit 1 40 exit 1
41 fi 41 fi
42 42
43 #---------------# 43 #---------------#
44 # Parse Options # 44 # Parse Options #
45 #---------------# 45 #---------------#
46 while getopts ":hDv:cf:r" opt 46 while getopts ":hDv:cf:r" opt
47 do 47 do
48 case $opt in 48 case $opt in
49 h) 49 h)
50 echo -e "$0 [OPTIONS] <WAV_FILE> <OUTPUT_DIRECTORY>\n" 50 echo -e "$0 [OPTIONS] <WAV_FILE> <OUTPUT_DIRECTORY>\n"
51 echo -e "\t Options:" 51 echo -e "\t Options:"
52 echo -e "\t\t-h :\tprint this message" 52 echo -e "\t\t-h :\tprint this message"
53 echo -e "\t\t-D :\tDEBUG mode on" 53 echo -e "\t\t-D :\tDEBUG mode on"
54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
55 echo -e "\t\t-c :\tCheck process, and log it into files, can stop if error detected" 55 echo -e "\t\t-c :\tCheck process, and log it into files, can stop if error detected"
56 echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)" 56 echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)"
57 echo -e "\t\t-r :\tforce rerun the wav file" 57 echo -e "\t\t-r :\tforce rerun the wav file"
58 exit 1 58 exit 1
59 ;; 59 ;;
60 D) 60 D)
61 DEBUG=1 61 DEBUG=1
62 ;; 62 ;;
63 v) 63 v)
64 VERBOSE=$OPTARG 64 VERBOSE=$OPTARG
65 ;; 65 ;;
66 c) 66 c)
67 CHECK=1 67 CHECK=1
68 ;; 68 ;;
69 f) 69 f)
70 FORKS="--forks $OPTARG" 70 FORKS="--forks $OPTARG"
71 ;; 71 ;;
72 r) 72 r)
73 RERUN=1 73 RERUN=1
74 ;; 74 ;;
75 :) 75 :)
76 echo "Option -$OPTARG requires an argument." > /dev/stderr 76 echo "Option -$OPTARG requires an argument." >&2
77 exit 1 77 exit 1
78 ;; 78 ;;
79 \?) 79 \?)
80 echo "BAD USAGE : unknow opton -$OPTARG" 80 echo "BAD USAGE : unknow opton -$OPTARG"
81 exit 1 81 exit 1
82 ;; 82 ;;
83 esac 83 esac
84 done 84 done
85 85
86 # mode debug enable 86 # mode debug enable
87 if [ $DEBUG -eq 1 ] 87 if [ $DEBUG -eq 1 ]
88 then 88 then
89 set -x 89 set -x
90 echo -e "## Mode DEBUG ON ##" 90 echo -e "## Mode DEBUG ON ##"
91 fi 91 fi
92 92
93 # mode verbose enable 93 # mode verbose enable
94 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi 94 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
95 95
96 # Check USAGE by arguments number 96 # Check USAGE by arguments number
97 if [ $(($#-($OPTIND-1))) -ne 2 ] 97 if [ $(($#-($OPTIND-1))) -ne 2 ]
98 then 98 then
99 echo "BAD USAGE : FirstPass.sh [OPTIONS] <WAV_FILE> <OUTPUT_DIR>" 99 echo "BAD USAGE : FirstPass.sh [OPTIONS] <WAV_FILE> <OUTPUT_DIR>"
100 echo "$0 -h for more info" 100 echo "$0 -h for more info"
101 exit 1 101 exit 1
102 fi 102 fi
103 103
104 shift $((OPTIND-1)) 104 shift $((OPTIND-1))
105 # check audio file - First argument 105 # check audio file - First argument
106 if [ -e $1 ] && [ -s $1 ] 106 if [ -e $1 ] && [ -s $1 ]
107 then 107 then
108 # absolute path to wav file 108 # absolute path to wav file
109 WAV_FILE=$(readlink -e $1) 109 WAV_FILE=$(readlink -e $1)
110 # wav filename 110 # wav filename
111 FILENAME=$(basename $WAV_FILE) 111 FILENAME=$(basename $WAV_FILE)
112 # wav filename without extension 112 # wav filename without extension
113 BASENAME=${FILENAME%.*} 113 BASENAME=${FILENAME%.*}
114 114
115 print_info "[${BASENAME}] => P1 start | $(date +'%d/%m/%y %H:%M:%S')" 1 115 print_info "[${BASENAME}] => P1 start | $(date +'%d/%m/%y %H:%M:%S')" 1
116 print_info "[${BASENAME}] $WAV_FILE OK" 2 116 print_info "[${BASENAME}] $WAV_FILE OK" 2
117 else 117 else
118 print_error " Can't find $1 OR file is empty" 118 print_error " Can't find $1 OR file is empty"
119 exit 1 119 exit 1
120 fi 120 fi
121 121
122 # check output directory - Second argument 122 # check output directory - Second argument
123 if [ ! -e $2 ] 123 if [ ! -e $2 ]
124 then 124 then
125 mkdir -p $2 125 mkdir -p $2
126 print_info "[${BASENAME}] Make directory $2" 2 126 print_info "[${BASENAME}] Make directory $2" 2
127 fi 127 fi
128 128
129 129
130 #-------------# 130 #-------------#
131 # GLOBAL VARS # 131 # GLOBAL VARS #
132 #-------------# 132 #-------------#
133 OUTPUT_DIR=$(readlink -e $2) # Output directory absolute path 133 OUTPUT_DIR=$(readlink -e $2) # Output directory absolute path
134 OUTPUT_DIR_BASENAME="$OUTPUT_DIR/$BASENAME/" # New OUTPUT_DIR with BASENAME 134 OUTPUT_DIR_BASENAME="$OUTPUT_DIR/$BASENAME/" # New OUTPUT_DIR with BASENAME
135 PLP_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.plp" # Global PLP file 135 PLP_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.plp" # Global PLP file
136 PLP_DIR="$OUTPUT_DIR_BASENAME/PLP/" # Segmented PLP files directory 136 PLP_DIR="$OUTPUT_DIR_BASENAME/PLP/" # Segmented PLP files directory
137 SEG_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.seg" # Global Seg file 137 SEG_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.seg" # Global Seg file
138 LBL_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.lbl" # Global LBL file 138 LBL_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.lbl" # Global LBL file
139 RES_DIR=$OUTPUT_DIR_BASENAME"/res_p1" 139 RES_DIR=$OUTPUT_DIR_BASENAME"/res_p1"
140 LOGFILE="$OUTPUT_DIR_BASENAME/info_p1.log" 140 LOGFILE="$OUTPUT_DIR_BASENAME/info_p1.log"
141 ERRORFILE="$OUTPUT_DIR_BASENAME/error_p1.log" 141 ERRORFILE="$OUTPUT_DIR_BASENAME/error_p1.log"
142 142
143 #------------------# 143 #------------------#
144 # Create WORKSPACE # 144 # Create WORKSPACE #
145 #------------------# 145 #------------------#
146 if [ ! -e $OUTPUT_DIR_BASENAME ] 146 if [ ! -e $OUTPUT_DIR_BASENAME ]
147 then 147 then
148 mkdir -p $OUTPUT_DIR_BASENAME 148 mkdir -p $OUTPUT_DIR_BASENAME
149 print_info "[${BASENAME}] Make directory $OUTPUT_DIR_BASENAME" 2 149 print_info "[${BASENAME}] Make directory $OUTPUT_DIR_BASENAME" 2
150 fi 150 fi
151 151
152 # Lock directory 152 # Lock directory
153 if [ -e $OUTPUT_DIR_BASENAME/FIRSTPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1; fi 153 if [ -e $OUTPUT_DIR_BASENAME/FIRSTPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1; fi
154 rm "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" > /dev/null 2>&1 154 rm "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" > /dev/null 2>&1
155 touch "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" > /dev/null 2>&1 155 touch "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" > /dev/null 2>&1
156 156
157 rm -r $PLP_DIR > /dev/null 2>&1; 157 rm -r $PLP_DIR > /dev/null 2>&1;
158 mkdir -p $PLP_DIR 158 mkdir -p $PLP_DIR
159 print_info "[${BASENAME}] Make directory $PLP_DIR" 2 159 print_info "[${BASENAME}] Make directory $PLP_DIR" 2
160 if [ $RERUN -eq 0 ]; 160 if [ $RERUN -eq 0 ];
161 then 161 then
162 rm -r $RES_DIR > /dev/null 2>&1; 162 rm -r $RES_DIR > /dev/null 2>&1;
163 else 163 else
164 rm $RES_DIR/*.lock > /dev/null 2>&1 164 rm $RES_DIR/*.lock > /dev/null 2>&1
165 fi 165 fi
166 mkdir -p $RES_DIR > /dev/null 2>&1 166 mkdir -p $RES_DIR > /dev/null 2>&1
167 print_info "[${BASENAME}] Make directory $RES_DIR" 2 167 print_info "[${BASENAME}] Make directory $RES_DIR" 2
168 rm $LOGFILE $ERRORFILE > /dev/null 2>&1 168 rm $LOGFILE $ERRORFILE > /dev/null 2>&1
169 169
170 #--------------------# 170 #--------------------#
171 # Save configuration # 171 # Save configuration #
172 #--------------------# 172 #--------------------#
173 cp $FIRSTPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/FirstPass.cfg 173 cp $FIRSTPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/FirstPass.cfg
174 echo "FIRSTPASS_SCRIPT_PATH=$MAIN_SCRIPT_PATH" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 174 echo "FIRSTPASS_SCRIPT_PATH=$MAIN_SCRIPT_PATH" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
175 echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 175 echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
176 echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 176 echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
177 echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 177 echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
178 echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 178 echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
179 echo "PLP_FILE=$PLP_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 179 echo "PLP_FILE=$PLP_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
180 echo "PLP_DIR=$PLP_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 180 echo "PLP_DIR=$PLP_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
181 echo "SEG_FILE=$SEG_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 181 echo "SEG_FILE=$SEG_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
182 echo "LBL_FILE=$LBL_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 182 echo "LBL_FILE=$LBL_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
183 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 183 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
184 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/FirstPass.cfg" 1 184 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/FirstPass.cfg" 1
185 185
186 #-------------------------# 186 #-------------------------#
187 # Check Audio File Format # 187 # Check Audio File Format #
188 #-------------------------# 188 #-------------------------#
189 error=0 189 error=0
190 temp=$(avconv -i $WAV_FILE 2>&1 | grep "16000 Hz") 190 temp=$(avconv -i $WAV_FILE 2>&1 | grep "16000 Hz")
191 if [ -z "$temp" ]; then error=1; fi 191 if [ -z "$temp" ]; then error=1; fi
192 temp=$(avconv -i $WAV_FILE 2>&1 | grep "1 channels") 192 temp=$(avconv -i $WAV_FILE 2>&1 | grep "1 channels")
193 if [ -z "$temp" ]; then error=1; fi 193 if [ -z "$temp" ]; then error=1; fi
194 temp=$(avconv -i $WAV_FILE 2>&1 | grep "s16") 194 temp=$(avconv -i $WAV_FILE 2>&1 | grep "s16")
195 if [ -z "$temp" ]; then error=1; fi 195 if [ -z "$temp" ]; then error=1; fi
196 196
197 if [ $error -eq 1 ] 197 if [ $error -eq 1 ]
198 then 198 then
199 print_message $WARNING 2 "[${BASENAME}] $WAV_FILE is not a wav file at 16000 Hz, 1 channel, 16bits\nhave to convert" 199 print_message $WARNING 2 "[${BASENAME}] $WAV_FILE is not a wav file at 16000 Hz, 1 channel, 16bits\nhave to convert"
200 print_message $INFO 3 "[${BASENAME}] avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav" 200 print_message $INFO 3 "[${BASENAME}] avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav"
201 avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav 201 avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav
202 WAV_FILE=$OUTPUT_DIR_BASENAME/$BASENAME.wav 202 WAV_FILE=$OUTPUT_DIR_BASENAME/$BASENAME.wav
203 FILENAME=$BASENAME.wav 203 FILENAME=$BASENAME.wav
204 print_message $INFO 1 "[${BASENAME}] new wav file : $WAV_FILE" 204 print_message $INFO 1 "[${BASENAME}] new wav file : $WAV_FILE"
205 fi 205 fi
206 206
207 #---------------# 207 #---------------#
208 # Get SRT file # 208 # Get SRT file #
209 #---------------# 209 #---------------#
210 if [ -s $(dirname $WAV_FILE)/$BASENAME.SRT ] 210 if [ -s $(dirname $WAV_FILE)/$BASENAME.SRT ]
211 then 211 then
212 cp $(dirname $WAV_FILE)/$BASENAME.SRT $OUTPUT_DIR_BASENAME/$BASENAME.SRT 212 cp $(dirname $WAV_FILE)/$BASENAME.SRT $OUTPUT_DIR_BASENAME/$BASENAME.SRT
213 print_info "[${BASENAME}] copy $BASENAME.SRT file into $OUTPUT_DIR_BASENAME" 3 213 print_info "[${BASENAME}] copy $BASENAME.SRT file into $OUTPUT_DIR_BASENAME" 3
214 fi 214 fi
215 215
216 #------------# 216 #------------#
217 # WAV -> PLP # 217 # WAV -> PLP #
218 #------------# 218 #------------#
219 print_info "[${BASENAME}] convert WAV -> PLP" 1 219 print_info "[${BASENAME}] convert WAV -> PLP" 1
220 echo $FILENAME > $OUTPUT_DIR_BASENAME/list.tmp 220 echo $FILENAME > $OUTPUT_DIR_BASENAME/list.tmp
221 print_info "[${BASENAME}] $BIN_PATH/lia_plp_mt.32 --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms" 3 221 print_info "[${BASENAME}] $BIN_PATH/lia_plp_mt.32 --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms" 3
222 222
223 $BIN_PATH/lia_plp_mt$ARCH --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms 223 $BIN_PATH/lia_plp_mt$ARCH --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms
224 224
225 if [ $CHECK -eq 1 ] 225 if [ $CHECK -eq 1 ]
226 then 226 then
227 check_first_pass_plp "$PLP_FILE" 227 check_first_pass_plp "$PLP_FILE"
228 if [ $? -eq 1 ] 228 if [ $? -eq 1 ]
229 then 229 then
230 print_log_file "$ERROFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating PLP file : $PLP_FILE" 230 print_log_file "$ERROFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating PLP file : $PLP_FILE"
231 echo "[${BASENAME}] ERROR : ${BASENAME} check $ERRORFILE file" > /dev/stderr 231 print_error "[${BASENAME}] -> exit, Check $ERRORFILE file"
232 exit 1 232 exit 1
233 fi 233 fi
234 fi 234 fi
235 235
236 rm $OUTPUT_DIR_BASENAME/list.tmp 236 rm $OUTPUT_DIR_BASENAME/list.tmp 2> /dev/null
237 237
238 #------------------------------# 238 #------------------------------#
239 # S/NS + SPEAKERS SEGMENTATION # 239 # S/NS + SPEAKERS SEGMENTATION #
240 #------------------------------# 240 #------------------------------#
241 print_info "[${BASENAME}] Launch speakers diarization" 1 241 print_info "[${BASENAME}] Launch speakers diarization" 1
242 # Calcul seg file 242 # Calcul seg file
243 print_info "[${BASENAME}] java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME" 3 243 print_info "[${BASENAME}] java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME" 3
244 #java -Xmx8000m -Xms2048 -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME 244 #java -Xmx8000m -Xms2048 -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME
245 java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME #–doCEClustering 245 java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME #–doCEClustering
246 246
247 if [ $CHECK -eq 1 ] && ( [ ! -e $SEG_FILE ] || [ -z $SEG_FILE ] ) 247 if [ $CHECK -eq 1 ] && ( [ ! -e $SEG_FILE ] || [ -z $SEG_FILE ] )
248 then 248 then
249 print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating SEG file : $SEG_FILE" 249 print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating SEG file : $SEG_FILE"
250 print_error "[${BASENAME}] Check $ERRORFILE file" 250 print_error "[${BASENAME}] Check $ERRORFILE file"
251 exit 1 251 exit 1
252 fi 252 fi
253 253
254 254
255 # Create LBL file 255 # Create LBL file
256 print_info "Extract LBL file from SEG file" 1 256 print_info "Extract LBL file from SEG file" 1
257 257
258 cat $SEG_FILE | grep -v ";;" | cut -f3,4,5,8 -d" " | tr " " "#" | sort -k1 -n | tr "#" " " > $LBL_FILE 258 cat $SEG_FILE | grep -v ";;" | cut -f3,4,5,8 -d" " | tr " " "#" | sort -k1 -n | tr "#" " " > $LBL_FILE
259 259
260 if [ $CHECK -eq 1 ] && ( [ ! -e $LBL_FILE ] || [ -z $LBL_FILE ] ) 260 if [ $CHECK -eq 1 ] && ( [ ! -e $LBL_FILE ] || [ -z $LBL_FILE ] )
261 then 261 then
262 print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating LBL file : $LBL_FILE" 262 print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating LBL file : $LBL_FILE"
263 print_error "[${BASENAME}] Check $ERRORFILE file" 263 print_error "[${BASENAME}] Check $ERRORFILE file"
264 exit 1 264 exit 1
265 fi 265 fi
266 266
267 267
268 #----------------------------------------------------# 268 #----------------------------------------------------#
269 # Cut global PLP file depending to LBL segmentations # 269 # Cut global PLP file depending to LBL segmentations #
270 #----------------------------------------------------# 270 #----------------------------------------------------#
271 print_info "[${BASENAME}] Cut PLP file depending to LBL segmentations" 1 271 print_info "[${BASENAME}] Cut PLP file depending to LBL segmentations" 1
272 print_info "[${BASENAME}] $BIN_PATH/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG" 3 272 print_info "[${BASENAME}] $BIN_PATH/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG" 3
273 273
274 $SPEERAL_TOOLS/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG 274 $SPEERAL_TOOLS/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG
275 275
276 if [ $CHECK -eq 1 ] 276 if [ $CHECK -eq 1 ]
277 then 277 then
278 check_first_pass_plps_lbl $PLP_DIR $LBL_FILE 278 check_first_pass_plps_lbl $PLP_DIR $LBL_FILE
279 if [ $? -eq 1 ] 279 if [ $? -eq 1 ]
280 then 280 then
281 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $PLP wrong number of .plp files" 281 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $PLP wrong number of .plp files"
282 print_error "[${BASENAME}] Check $ERRORFILE file" 282 print_error "[${BASENAME}] Check $ERRORFILE file"
283 exit 1 283 exit 1
284 fi 284 fi
285 fi 285 fi
286 286
287 # change plp files names 287 # change plp files names
288 cd $PLP_DIR; 288 cd $PLP_DIR;
289 rename -f s/_/#/g *plp 289 rename -f s/_/#/g *plp
290 rename -f s/#/_/ *plp 290 rename -f s/#/_/ *plp
291 cd $OLDPWD 291 cd $OLDPWD
292 292
293 #---------------------------------------------# 293 #---------------------------------------------#
294 # PLP files list depending to acoustic models # 294 # PLP files list depending to acoustic models #
295 #---------------------------------------------# 295 #---------------------------------------------#
296 print_info "[${BASENAME}] Create PLP list depending of the model" 1 296 print_info "[${BASENAME}] Create PLP list depending of the model" 1
297 # Create a list of plp files 297 # Create a list of plp files
298 find $PLP_DIR -type f -exec basename "{}" .plp \; | sort > $OUTPUT_DIR_BASENAME/plp.lst 298 find $PLP_DIR -type f -exec basename "{}" .plp \; | sort > $OUTPUT_DIR_BASENAME/plp.lst
299 299
300 rm $OUTPUT_DIR_BASENAME/plp_*.lst > /dev/null 2>&1 300 rm $OUTPUT_DIR_BASENAME/plp_*.lst > /dev/null 2>&1
301 for (( i=0; $i<${#MTAG[@]} ; i++ )) 301 for (( i=0; $i<${#MTAG[@]} ; i++ ))
302 do 302 do
303 a=`grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst` 303 a=`grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst`
304 if [ -n "$a" ]; then 304 if [ -n "$a" ]; then
305 print_info "[${BASENAME}] Creating $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst" 3 305 print_info "[${BASENAME}] Creating $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst" 3
306 grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst | sort > $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst 306 grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst | sort > $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst
307 fi 307 fi
308 done 308 done
309 309
310 #-----------------------# 310 #-----------------------#
311 # First Pass (DECODING) # 311 # First Pass (DECODING) #
312 #-----------------------# 312 #-----------------------#
313 # 313 #
314 # For all AM do decoding 314 # For all AM do decoding
315 # if Check error -> iter on undone decoding (max 1 times) 315 # if Check error -> iter on undone decoding (max 1 times)
316 # 316 #
317 print_info "[${BASENAME}] Launch decoding" 1 317 print_info "[${BASENAME}] Launch decoding" 1
318 for (( i=0; $i<${#MTAG[@]} ; i++ )) 318 for (( i=0; $i<${#MTAG[@]} ; i++ ))
319 do 319 do
320 redo=1; # nb of try if not all segs is decoded 320 redo=1; # nb of try if not all segs is decoded
321 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ] 321 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]
322 then 322 then
323 todo=$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst 323 todo=$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst
324 while [ $redo -gt 0 ]; do 324 while [ $redo -gt 0 ]; do
325 rm $RES_DIR/*.lock > /dev/null 2>&1 325 rm $RES_DIR/*.lock > /dev/null 2>&1
326 print_info "[${BASENAME}] $SPEERAL_BIN $todo $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3 326 print_info "[${BASENAME}] $SPEERAL_BIN $todo $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3
327 # Run speeral 327 # Run speeral
328 $SPEERAL_BIN ${todo} $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock 328 $SPEERAL_BIN ${todo} $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock
329 329
330 # Check if error 330 # Check if error
331 if [ $CHECK -eq 1 ] 331 if [ $CHECK -eq 1 ]
332 then 332 then
333 check_first_pass_output_speeral "${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" "$RES_DIR" 333 check_first_pass_output_speeral "${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" "$RES_DIR"
334 # if error 334 # if error
335 if [ $? -eq 1 ] 335 if [ $? -eq 1 ]
336 then 336 then
337 # rerun 337 # rerun
338 redo=$(($redo - 1)); 338 redo=$(($redo - 1));
339 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 339 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2
340 print_log_file $LOGFILE "WARN : Speeral number of output ERROR ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" 340 print_log_file $LOGFILE "WARN : Speeral number of output ERROR ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst"
341 # new plp list 341 # new plp list
342 # list .seg done and compare to list of seg to do 342 # list .seg done and compare to list of seg to do
343 ls $RES_DIR/*.seg | grep -e "${MTAG[$i]}" | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 343 ls $RES_DIR/*.seg | grep -e "${MTAG[$i]}" | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
344 diff ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" > ${OUTPUT_DIR_BASENAME}/todo.lst 344 diff ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" > ${OUTPUT_DIR_BASENAME}/todo.lst
345 rm ${OUTPUT_DIR_BASENAME}/.tmp 345 rm ${OUTPUT_DIR_BASENAME}/.tmp
346 # log seg to do 346 # log seg to do
347 print_log_file $LOGFILE "Segs not done :" 347 print_log_file $LOGFILE "Segs not done :"
348 cat ${OUTPUT_DIR_BASENAME}/todo.lst >> $LOGFILE 348 cat ${OUTPUT_DIR_BASENAME}/todo.lst >> $LOGFILE
349 todo=${OUTPUT_DIR_BASENAME}/todo.lst 349 todo=${OUTPUT_DIR_BASENAME}/todo.lst
350 print_warn "[${BASENAME}] Try $redo" 3 350 print_warn "[${BASENAME}] Try $redo" 3
351 fi 351 fi
352 fi 352 fi
353 done 353 done
354 rm ${OUTPUT_DIR_BASENAME}/todo.lst > /dev/null 2>&1 354 rm ${OUTPUT_DIR_BASENAME}/todo.lst > /dev/null 2>&1
355 #rm $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst 355 #rm $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst
356 rm $RES_DIR/*.lock > /dev/null 2>&1 356 rm $RES_DIR/*.lock > /dev/null 2>&1
357 fi 357 fi
358 done 358 done
359 359
360 ## Check missing seg and log it 360 ## Check missing seg and log it
361 if [ $CHECK -eq 1 ] 361 if [ "$CHECK" -eq 1 ]
362 then 362 then
363 ls $RES_DIR/*.seg | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 363 ls $RES_DIR/*.seg | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
364 todo=$(cat ${OUTPUT_DIR_BASENAME}/plp.lst | wc -l) 364 todo=$(cat ${OUTPUT_DIR_BASENAME}/plp.lst 2> /dev/null | wc -l)
365 if [ $todo -eq 0 ]; then todo=1;fi 365 if [ "$todo" -eq 0 ]; then todo=1;fi
366 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) 366 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l)))
367 pourcentage=$((($notdone*100)/$todo)) 367 pourcentage=$((($notdone*100)/$todo))
368 368
369 if [ $notdone -ne 0 ] 369 if [ "$notdone" -ne 0 ]
370 then 370 then
371 print_error "[${BASENAME}] Check $ERRORFILE" 371 print_error "[${BASENAME}] Check ${ERRORFILE}"
372 print_log_file "$ERRORFILE" "ERROR : Segs not done [" 372 print_log_file "${ERRORFILE}" "ERROR : Segs not done ["
373 diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE 373 diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> ${ERRORFILE}
374 print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" 374 print_log_file "${ERRORFILE}" "] $pourcentage% $BASENAME"
375 else 375 else
376 print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" 376 print_log_file "$LOGFILE" "P1 OK ${BASENAME} | $(date +'%d/%m/%y %H:%M:%S')"
377 fi 377 fi
378 rm ${OUTPUT_DIR_BASENAME}/.tmp 378 rm ${OUTPUT_DIR_BASENAME}/.tmp
379 fi 379 fi
380 380
381 #---------------# 381 #---------------#
382 # Convert res # 382 # Convert res #
383 #---------------# 383 #---------------#
384 print_info "[${BASENAME}] Convert .res into .ctm" 1 384 print_info "[${BASENAME}] Convert .res into .ctm" 1
385 # .res => .ctm 385 # .res => .ctm
386 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.ctm 386 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.ctm
387 print_info "[${BASENAME}] Convert .res into .trs" 1 387 print_info "[${BASENAME}] Convert .res into .trs" 1
388 # .res => .trs 388 # .res => .trs
389 echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR_BASENAME/$BASENAME.seg" > $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 389 echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR_BASENAME/$BASENAME.seg" > $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg
390 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.trs --trs_config $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 390 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.trs --trs_config $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg
391 rm $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 2> /dev/null 391 rm $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 2> /dev/null
392 print_info "[${BASENAME}] Convert .res into .txt" 1 392 print_info "[${BASENAME}] Convert .res into .txt" 1
393 # .res => .txt 393 # .res => .txt
394 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.txt 394 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.txt
395 395
396 print_info "[${BASENAME}] <= P1 End | $(date +'%d/%m/%y %H:%M:%S')" 1 396 print_info "[${BASENAME}] <= P1 End | $(date +'%d/%m/%y %H:%M:%S')" 1
397 # unlock directory 397 # unlock directory
398 mv "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" 398 mv "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock"
399 399
main_tools/RecomposePass.sh
Diff comments   View file @ d7e9e4b
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : RecomposePass.sh # 4 # File : RecomposePass.sh #
5 # Brief : Get missing res pass from previous pass # 5 # Brief : Get missing res pass from previous pass #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # Version : 1.0 # 7 # Version : 1.0 #
8 # Date : 25/07/13 # 8 # Date : 25/07/13 #
9 ##################################################### 9 #####################################################
10 10
11 echo "### RecomposePass.sh ###" 11 echo "### RecomposePass.sh ###"
12 12
13 13
14 # Check OTMEDIA_HOME env var 14 # Check OTMEDIA_HOME env var
15 if [ -z ${OTMEDIA_HOME} ] 15 if [ -z ${OTMEDIA_HOME} ]
16 then 16 then
17 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 17 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
18 export OTMEDIA_HOME=$OTMEDIA_HOME 18 export OTMEDIA_HOME=$OTMEDIA_HOME
19 fi 19 fi
20 20
21 # where is RecomposePass.sh 21 # where is RecomposePass.sh
22 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 22 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
23 23
24 # Scripts Path 24 # Scripts Path
25 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 25 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
26 26
27 # Include scripts 27 # Include scripts
28 . $SCRIPT_PATH"/Tools.sh" 28 . $SCRIPT_PATH"/Tools.sh"
29 . $SCRIPT_PATH"/CheckSecondPass.sh" 29 . $SCRIPT_PATH"/CheckSecondPass.sh"
30 30
31 # where is RescomposePass.cfg 31 # where is RescomposePass.cfg
32 RECOMPOSEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/RecomposePass.cfg" 32 RECOMPOSEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/RecomposePass.cfg"
33 if [ -e $RECOMPOSEPASS_CONFIG_FILE ] 33 if [ -e $RECOMPOSEPASS_CONFIG_FILE ]
34 then 34 then
35 . $RECOMPOSEPASS_CONFIG_FILE 35 . $RECOMPOSEPASS_CONFIG_FILE
36 else 36 else
37 echo "ERROR : Can't find configuration file $RECOMPOSEPASS_CONFIG_FILE" >&2 37 echo "ERROR : Can't find configuration file $RECOMPOSEPASS_CONFIG_FILE" >&2
38 exit 1 38 exit 1
39 fi 39 fi
40 40
41 #---------------# 41 #---------------#
42 # Parse Options # 42 # Parse Options #
43 #---------------# 43 #---------------#
44 while getopts ":hDv:" opt 44 while getopts ":hDv:" opt
45 do 45 do
46 case $opt in 46 case $opt in
47 h) 47 h)
48 echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" 48 echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n"
49 echo -e "\t Options:" 49 echo -e "\t Options:"
50 echo -e "\t\t-h :\tprint this message" 50 echo -e "\t\t-h :\tprint this message"
51 echo -e "\t\t-D :\tDEBUG mode on" 51 echo -e "\t\t-D :\tDEBUG mode on"
52 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 52 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
53 exit 1 53 exit 1
54 ;; 54 ;;
55 D) 55 D)
56 DEBUG=1 56 DEBUG=1
57 ;; 57 ;;
58 v) 58 v)
59 VERBOSE=$OPTARG 59 VERBOSE=$OPTARG
60 ;; 60 ;;
61 :) 61 :)
62 echo "Option -$OPTARG requires an argument." > /dev/null 62 echo "Option -$OPTARG requires an argument." >&2
63 exit 1 63 exit 1
64 ;; 64 ;;
65 \?) 65 \?)
66 echo "BAD USAGE : unknow opton -$OPTARG" 66 echo "BAD USAGE : unknow opton -$OPTARG"
67 #exit 1 67 #exit 1
68 ;; 68 ;;
69 esac 69 esac
70 done 70 done
71 71
72 # mode debug enable 72 # mode debug enable
73 if [ $DEBUG -eq 1 ] 73 if [ $DEBUG -eq 1 ]
74 then 74 then
75 set -x 75 set -x
76 echo -e "## Mode DEBUG ON ##" 76 echo -e "## Mode DEBUG ON ##"
77 fi 77 fi
78 78
79 # mode verbose enable 79 # mode verbose enable
80 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi 80 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi
81 81
82 # Check USAGE by arguments number 82 # Check USAGE by arguments number
83 if [ $(($#-($OPTIND-1))) -ne 1 ] 83 if [ $(($#-($OPTIND-1))) -ne 1 ]
84 then 84 then
85 echo "BAD USAGE : RecomposePass.sh [OPTIONS] <PASS_DIR>" 85 echo "BAD USAGE : RecomposePass.sh [OPTIONS] <PASS_DIR>"
86 echo "$0 -h for more info" 86 echo "$0 -h for more info"
87 exit 1 87 exit 1
88 fi 88 fi
89 89
90 shift $((OPTIND-1)) 90 shift $((OPTIND-1))
91 # check Pass directory - First argument 91 # check Pass directory - First argument
92 if [ -e $1 ] && [ -d $1 ] 92 if [ -e $1 ] && [ -d $1 ]
93 then 93 then
94 PASS_DIR=$(readlink -e $1) 94 PASS_DIR=$(readlink -e $1)
95 else 95 else
96 print_error "can't find $1 directory" 96 print_error "can't find $1 directory"
97 exit 1 97 exit 1
98 fi 98 fi
99 99
100 #-------------# 100 #-------------#
101 # GLOBAL VARS # 101 # GLOBAL VARS #
102 #-------------# 102 #-------------#
103 BASENAME=$(basename $PASS_DIR) 103 BASENAME=$(basename $PASS_DIR)
104 OUTPUT_DIR_BASENAME=$PASS_DIR 104 OUTPUT_DIR_BASENAME=$PASS_DIR
105 RES_DIR="$PASS_DIR/res_all" 105 RES_DIR="$PASS_DIR/res_all"
106 LOGFILE=$PASS_DIR"/info_recpass.log" 106 LOGFILE=$PASS_DIR"/info_recpass.log"
107 ERRORFILE=$PASS_DIR"/error_recpass.log" 107 ERRORFILE=$PASS_DIR"/error_recpass.log"
108 108
109 THIRDPASS_CONFIG_FILE="$PASS_DIR/ThirdPass.cfg" 109 THIRDPASS_CONFIG_FILE="$PASS_DIR/ThirdPass.cfg"
110 if [ -e $THIRDPASS_CONFIG_FILE ] 110 if [ -e $THIRDPASS_CONFIG_FILE ]
111 then 111 then
112 RES_P3=$(cat $THIRDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") 112 RES_P3=$(cat $THIRDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=")
113 else 113 else
114 print_warn "[${BASENAME}] can't find $THIRDPASS_CONFIG_FILE file" 2 114 print_warn "[${BASENAME}] can't find $THIRDPASS_CONFIG_FILE file" 2
115 #exit 1 115 #exit 1
116 fi 116 fi
117 SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg" 117 SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg"
118 if [ -e $SECONDPASS_CONFIG_FILE ] 118 if [ -e $SECONDPASS_CONFIG_FILE ]
119 then 119 then
120 RES_P2=$(cat $SECONDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") 120 RES_P2=$(cat $SECONDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=")
121 else 121 else
122 print_warn "[${BASENAME}] can't find $SECONDPASS_CONFIG_FILE file" 2 122 print_warn "[${BASENAME}] can't find $SECONDPASS_CONFIG_FILE file" 2
123 #exit 1 123 #exit 1
124 fi 124 fi
125 FIRSTPASS_CONFIG_FILE="$PASS_DIR/FirstPass.cfg" 125 FIRSTPASS_CONFIG_FILE="$PASS_DIR/FirstPass.cfg"
126 if [ -e $FIRSTPASS_CONFIG_FILE ] 126 if [ -e $FIRSTPASS_CONFIG_FILE ]
127 then 127 then
128 RES_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") 128 RES_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=")
129 else 129 else
130 print_warn "[${BASENAME}] can't find $FIRSTPASS_CONFIG_FILE file" 2 130 print_warn "[${BASENAME}] can't find $FIRSTPASS_CONFIG_FILE file" 2
131 #exit 1 131 #exit 1
132 fi 132 fi
133 133
134 if ( [ -z "$RES_P1" ] && [ -z "$RES_P2" ] ) || [ -z $RES_P3 ] 134 if ( [ -z "$RES_P1" ] && [ -z "$RES_P2" ] ) || [ -z $RES_P3 ]
135 then 135 then
136 print_error "[${BASENAME}] No configuration pass files in $PASS_DIR -> exit" 136 print_error "[${BASENAME}] No configuration pass files in $PASS_DIR -> exit"
137 if [ $CHECK -eq 1 ]; then print_log_file $ERROFILE "No configuration pass files in $PASS_DIR -> exit";fi 137 if [ $CHECK -eq 1 ]; then print_log_file $ERROFILE "No configuration pass files in $PASS_DIR -> exit";fi
138 exit 1 138 exit 1
139 fi 139 fi
140 140
141 print_info "[${BASENAME}] => RecomposePass start | $(date +'%d/%m/%y %H:%M:%S')" 1 141 print_info "[${BASENAME}] => RecomposePass start | $(date +'%d/%m/%y %H:%M:%S')" 1
142 142
143 #------------------# 143 #------------------#
144 # Create WORKSPACE # 144 # Create WORKSPACE #
145 #------------------# 145 #------------------#
146 146
147 # Lock directory 147 # Lock directory
148 if [ -e $OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock ] && [ $RERUN -eq 0 ] 148 if [ -e $OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock ] && [ $RERUN -eq 0 ]
149 then 149 then
150 print_warn "[${BASENAME}] RECOMPOSEPASS is locked -> exit" 2 150 print_warn "[${BASENAME}] RECOMPOSEPASS is locked -> exit" 2
151 exit 1 151 exit 1
152 fi 152 fi
153 rm "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" > /dev/null 2>&1 153 rm "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" > /dev/null 2>&1
154 touch "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" > /dev/null 2>&1 154 touch "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" > /dev/null 2>&1
155 155
156 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi 156 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi
157 mkdir -p $RES_DIR 2> /dev/null 157 mkdir -p $RES_DIR 2> /dev/null
158 print_info "Make directory $RES_DIR" 3 158 print_info "Make directory $RES_DIR" 3
159 159
160 #--------------------# 160 #--------------------#
161 # Save configuration # 161 # Save configuration #
162 #--------------------# 162 #--------------------#
163 cp $RECOMPOSEPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/RecomposePass.cfg 163 cp $RECOMPOSEPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/RecomposePass.cfg
164 echo "RES_ALL=$RES_DIR" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg 164 echo "RES_ALL=$RES_DIR" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
165 echo "RES_P1=$RES_P1" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg 165 echo "RES_P1=$RES_P1" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
166 echo "RES_P2=$RES_P2" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg 166 echo "RES_P2=$RES_P2" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
167 echo "RES_P3=$RES_P3" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg 167 echo "RES_P3=$RES_P3" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
168 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/RecomposePass.cfg" 1 168 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/RecomposePass.cfg" 1
169 169
170 #--------------------------------------------------# 170 #--------------------------------------------------#
171 # Recompose Pass using previous pass # 171 # Recompose Pass using previous pass #
172 #--------------------------------------------------# 172 #--------------------------------------------------#
173 print_info "[${BASENAME}] Launch Recompose Pass" 1 173 print_info "[${BASENAME}] Launch Recompose Pass" 1
174 174
175 ## Check seg in p3 175 ## Check seg in p3
176 cp $RES_P3/*.res $RES_DIR 176 cp $RES_P3/*.res $RES_DIR
177 ls $RES_P3/*.res | sed -e "s|$RES_P3\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p3.tmp 177 ls $RES_P3/*.res | sed -e "s|$RES_P3\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p3.tmp
178 ls $RES_P2/*.res | sed -e "s|$RES_P2\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p2.tmp 178 ls $RES_P2/*.res | sed -e "s|$RES_P2\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p2.tmp
179 ls $RES_P1/*.res | sed -e "s|$RES_P1\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p1.tmp 179 ls $RES_P1/*.res | sed -e "s|$RES_P1\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p1.tmp
180 diff_3_2=$(diff ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp | grep -e "^> " | sed -e "s/> //") 180 diff_3_2=$(diff ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp | grep -e "^> " | sed -e "s/> //")
181 for res in $diff_3_2 181 for res in $diff_3_2
182 do 182 do
183 cp $RES_P2/$res.res $RES_DIR 183 cp $RES_P2/$res.res $RES_DIR
184 done 184 done
185 ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_all.tmp 185 ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_all.tmp
186 diff_3_1=$(diff ${OUTPUT_DIR_BASENAME}/res_all.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp | grep -e "^> " | sed -e "s/> //") 186 diff_3_1=$(diff ${OUTPUT_DIR_BASENAME}/res_all.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp | grep -e "^> " | sed -e "s/> //")
187 for res in $diff_3_1 187 for res in $diff_3_1
188 do 188 do
189 cp $RES_P1/$res.res $RES_DIR 189 cp $RES_P1/$res.res $RES_DIR
190 done 190 done
191 191
192 rm ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp ${OUTPUT_DIR_BASENAME}/res_all.tmp 2> /dev/null 192 rm ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp ${OUTPUT_DIR_BASENAME}/res_all.tmp 2> /dev/null
193 193
194 #---------------# 194 #---------------#
195 # Convert res # 195 # Convert res #
196 #---------------# 196 #---------------#
197 197
198 print_info "[${BASENAME}] Convert .res into .ctm" 1 198 print_info "[${BASENAME}] Convert .res into .ctm" 1
199 # .res => .ctm 199 # .res => .ctm
200 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.ctm 200 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.ctm
201 print_info "[${BASENAME}] Convert .res into .trs" 1 201 print_info "[${BASENAME}] Convert .res into .trs" 1
202 # .res => .trs 202 # .res => .trs
203 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 203 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
204 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 204 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
205 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 205 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
206 print_info "[${BASENAME}] Convert .res into .txt" 1 206 print_info "[${BASENAME}] Convert .res into .txt" 1
207 # .res => .txt 207 # .res => .txt
208 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.txt 208 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.txt
209 209
210 210
211 print_info "[${BASENAME}] <= RecomposePass End | $(date +'%d/%m/%y %H:%M:%S')" 1 211 print_info "[${BASENAME}] <= RecomposePass End | $(date +'%d/%m/%y %H:%M:%S')" 1
212 212
213 # unlock directory 213 # unlock directory
214 mv "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" 214 mv "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock"
215 215
216 216
main_tools/SecondPass.sh
Diff comments   View file @ d7e9e4b
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : SecondPass.sh # 4 # File : SecondPass.sh #
5 # Brief : Speaker adaptation + ASR second pass # 5 # Brief : Speaker adaptation + ASR second pass #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # (base on Emmanuel Ferreira # 7 # (base on Emmanuel Ferreira #
8 # and Hugo Mauchrétien works) # 8 # and Hugo Mauchrétien works) #
9 # Version : 1.1 # 9 # Version : 1.1 #
10 # Date : 18/06/13 # 10 # Date : 18/06/13 #
11 ##################################################### 11 #####################################################
12 12
13 echo "### SecondPass.sh ###" 13 echo "### SecondPass.sh ###"
14 14
15 # Check OTMEDIA_HOME env var 15 # Check OTMEDIA_HOME env var
16 if [ -z ${OTMEDIA_HOME} ] 16 if [ -z ${OTMEDIA_HOME} ]
17 then 17 then
18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
19 export OTMEDIA_HOME=$OTMEDIA_HOME 19 export OTMEDIA_HOME=$OTMEDIA_HOME
20 fi 20 fi
21 21
22 # where is SecondPass.sh 22 # where is SecondPass.sh
23 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 23 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
24 24
25 # Scripts Path 25 # Scripts Path
26 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 26 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
27 27
28 # Include scripts 28 # Include scripts
29 . $SCRIPT_PATH"/Tools.sh" 29 . $SCRIPT_PATH"/Tools.sh"
30 . $SCRIPT_PATH"/CheckSecondPass.sh" 30 . $SCRIPT_PATH"/CheckSecondPass.sh"
31 31
32 # where is SecondPass.cfg 32 # where is SecondPass.cfg
33 SECONDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/SecondPass.cfg" 33 SECONDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/SecondPass.cfg"
34 if [ -e $SECONDPASS_CONFIG_FILE ] 34 if [ -e $SECONDPASS_CONFIG_FILE ]
35 then 35 then
36 . $SECONDPASS_CONFIG_FILE 36 . $SECONDPASS_CONFIG_FILE
37 else 37 else
38 echo "ERROR : Can't find configuration file $SECONDPASS_CONFIG_FILE" > /dev/stderr 38 echo "ERROR : Can't find configuration file $SECONDPASS_CONFIG_FILE" >&2
39 echo "exit" > /dev/stderr 39 echo "exit" >&2
40 exit 1 40 exit 1
41 fi 41 fi
42 42
43 #---------------# 43 #---------------#
44 # Parse Options # 44 # Parse Options #
45 #---------------# 45 #---------------#
46 while getopts ":hDv:crf:" opt 46 while getopts ":hDv:crf:" opt
47 do 47 do
48 case $opt in 48 case $opt in
49 h) 49 h)
50 echo -e "$0 [OPTIONS] <FIRST_PASS_DIRECTORY>\n" 50 echo -e "$0 [OPTIONS] <FIRST_PASS_DIRECTORY>\n"
51 echo -e "\t Options:" 51 echo -e "\t Options:"
52 echo -e "\t\t-h :\tprint this message" 52 echo -e "\t\t-h :\tprint this message"
53 echo -e "\t\t-D :\tDEBUG mode on" 53 echo -e "\t\t-D :\tDEBUG mode on"
54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
55 echo -e "\t\t-c :\t Check process, stop if error detected" 55 echo -e "\t\t-c :\t Check process, stop if error detected"
56 echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)" 56 echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)"
57 echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done" 57 echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done"
58 exit 1 58 exit 1
59 ;; 59 ;;
60 D) 60 D)
61 DEBUG=1 61 DEBUG=1
62 ;; 62 ;;
63 v) 63 v)
64 VERBOSE=$OPTARG 64 VERBOSE=$OPTARG
65 ;; 65 ;;
66 c) 66 c)
67 CHECK=1 67 CHECK=1
68 ;; 68 ;;
69 f) 69 f)
70 FORKS="--forks $OPTARG" 70 FORKS="--forks $OPTARG"
71 ;; 71 ;;
72 r) 72 r)
73 RERUN=1 73 RERUN=1
74 ;; 74 ;;
75 :) 75 :)
76 echo "Option -$OPTARG requires an argument." > /dev/stderr 76 echo "Option -$OPTARG requires an argument." >&2
77 exit 1 77 exit 1
78 ;; 78 ;;
79 \?) 79 \?)
80 echo "BAD USAGE : unknow opton -$OPTARG" > /dev/stderr 80 echo "BAD USAGE : unknow opton -$OPTARG" >&2
81 exit 1 81 exit 1
82 ;; 82 ;;
83 esac 83 esac
84 done 84 done
85 85
86 # mode debug enable 86 # mode debug enable
87 if [ $DEBUG -eq 1 ] 87 if [ $DEBUG -eq 1 ]
88 then 88 then
89 set -x 89 set -x
90 echo -e "## Mode DEBUG ON ##" 90 echo -e "## Mode DEBUG ON ##"
91 fi 91 fi
92 92
93 # mode verbose enable 93 # mode verbose enable
94 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi 94 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
95 95
96 # Check USAGE by arguments number 96 # Check USAGE by arguments number
97 if [ $(($#-($OPTIND-1))) -ne 1 ] 97 if [ $(($#-($OPTIND-1))) -ne 1 ]
98 then 98 then
99 echo "BAD USAGE : SecondPass.sh [OPTIONS] <FIRST_PASS_DIR>" 99 echo "BAD USAGE : SecondPass.sh [OPTIONS] <FIRST_PASS_DIR>"
100 echo "$0 -h for more info" 100 echo "$0 -h for more info"
101 exit 1 101 exit 1
102 fi 102 fi
103 103
104 shift $((OPTIND-1)) 104 shift $((OPTIND-1))
105 # check FirstPass directory - First argument 105 # check FirstPass directory - First argument
106 if [ -e $1 ] && [ -d $1 ] 106 if [ -e $1 ] && [ -d $1 ]
107 then 107 then
108 FIRSTPASS_DIR=$(readlink -e $1) 108 FIRSTPASS_DIR=$(readlink -e $1)
109 else 109 else
110 print_error "Can't find $1 directory" 110 print_error "Can't find $1 directory"
111 exit 1 111 exit 1
112 fi 112 fi
113 113
114 #-------------# 114 #-------------#
115 # GLOBAL VARS # 115 # GLOBAL VARS #
116 #-------------# 116 #-------------#
117 FIRSTPASS_CONFIG_FILE="$FIRSTPASS_DIR/FirstPass.cfg" 117 FIRSTPASS_CONFIG_FILE="$FIRSTPASS_DIR/FirstPass.cfg"
118 if [ -e $FIRSTPASS_CONFIG_FILE ] 118 if [ -e $FIRSTPASS_CONFIG_FILE ]
119 then 119 then
120 WAV_FILE=$(cat $FIRSTPASS_CONFIG_FILE | grep "WAV_FILE=" | cut -f2 -d"=") 120 WAV_FILE=$(cat $FIRSTPASS_CONFIG_FILE | grep "WAV_FILE=" | cut -f2 -d"=")
121 BASENAME=$(cat $FIRSTPASS_CONFIG_FILE | grep "^BASENAME=" | cut -f2 -d"=") 121 BASENAME=$(cat $FIRSTPASS_CONFIG_FILE | grep "^BASENAME=" | cut -f2 -d"=")
122 OUTPUT_DIR=$(cat $FIRSTPASS_CONFIG_FILE | grep "OUTPUT_DIR=" | cut -f2 -d"=") 122 OUTPUT_DIR=$(cat $FIRSTPASS_CONFIG_FILE | grep "OUTPUT_DIR=" | cut -f2 -d"=")
123 OUTPUT_DIR_BASENAME=$FIRSTPASS_DIR 123 OUTPUT_DIR_BASENAME=$FIRSTPASS_DIR
124 PLP_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_FILE=" | cut -f2 -d"=") 124 PLP_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_FILE=" | cut -f2 -d"=")
125 PLP_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_DIR=" | cut -f2 -d"=") 125 PLP_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_DIR=" | cut -f2 -d"=")
126 SEG_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "SEG_FILE=" | cut -f2 -d"=") 126 SEG_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "SEG_FILE=" | cut -f2 -d"=")
127 LBL_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "LBL_FILE=" | cut -f2 -d"=") 127 LBL_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "LBL_FILE=" | cut -f2 -d"=")
128 RES_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "RES_DIR=" | cut -f2 -d"=") 128 RES_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "RES_DIR=" | cut -f2 -d"=")
129 else 129 else
130 print_error "Can't find $FIRSTPASS_CONFIG_FILE file" 130 print_error "Can't find $FIRSTPASS_CONFIG_FILE file"
131 print_error "exit" 131 print_error "exit"
132 exit 1 132 exit 1
133 fi 133 fi
134 LST=$OUTPUT_DIR_BASENAME"/lists" 134 LST=$OUTPUT_DIR_BASENAME"/lists"
135 HMM=$OUTPUT_DIR_BASENAME"/hmm/" 135 HMM=$OUTPUT_DIR_BASENAME"/hmm/"
136 RES_DIR=$OUTPUT_DIR_BASENAME"/res_p2" 136 RES_DIR=$OUTPUT_DIR_BASENAME"/res_p2"
137 LOGFILE="$OUTPUT_DIR_BASENAME/info_p2.log" 137 LOGFILE="$OUTPUT_DIR_BASENAME/info_p2.log"
138 ERRORFILE="$OUTPUT_DIR_BASENAME/error_p2.log" 138 ERRORFILE="$OUTPUT_DIR_BASENAME/error_p2.log"
139 139
140 print_info "[${BASENAME}] => P2 start | $(date +'%d/%m/%y %H:%M:%S')" 1 140 print_info "[${BASENAME}] => P2 start | $(date +'%d/%m/%y %H:%M:%S')" 1
141 141
142 #------------------# 142 #------------------#
143 # Create WORKSPACE # 143 # Create WORKSPACE #
144 #------------------# 144 #------------------#
145 145
146 # Lock directory 146 # Lock directory
147 if [ -e $OUTPUT_DIR_BASENAME/SECONDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] SECONDPASS is locked -> exit" 2; exit 1;fi 147 if [ -e $OUTPUT_DIR_BASENAME/SECONDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] SECONDPASS is locked -> exit" 2; exit 1;fi
148 rm "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" > /dev/null 2>&1 148 rm "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" > /dev/null 2>&1
149 touch "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" > /dev/null 2>&1 149 touch "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" > /dev/null 2>&1
150 150
151 rm -r $LST > /dev/null 2>&1 151 rm -r $LST > /dev/null 2>&1
152 mkdir -p $LST 152 mkdir -p $LST
153 print_info "[${BASENAME}] Make directory $LST" 2 153 print_info "[${BASENAME}] Make directory $LST" 2
154 if [ $RERUN -eq 0 ]; then rm -r $HMM > /dev/null 2>&1; fi 154 if [ $RERUN -eq 0 ]; then rm -r $HMM > /dev/null 2>&1; fi
155 mkdir -p $HMM 155 mkdir -p $HMM
156 print_info "[${BASENAME}] Make directory $HMM" 2 156 print_info "[${BASENAME}] Make directory $HMM" 2
157 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi 157 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi
158 mkdir -p $RES_DIR > /dev/null 2>&1 158 mkdir -p $RES_DIR > /dev/null 2>&1
159 print_info "[${BASENAME}] Make directory $RES_DIR" 2 159 print_info "[${BASENAME}] Make directory $RES_DIR" 2
160 rm $LOGFILE $ERRORFILE > /dev/null 2>&1 160 rm $LOGFILE $ERRORFILE > /dev/null 2>&1
161 161
162 #-------------------# 162 #-------------------#
163 # Check Pass # 163 # Check Pass #
164 #-------------------# 164 #-------------------#
165 print_info "[${BASENAME}] Check Pass 2 directory" 1 165 print_info "[${BASENAME}] Check Pass 2 directory" 1
166 for treil in $(ls $RES_DIR/ | grep treil) 166 for treil in $(ls $RES_DIR/ | grep treil)
167 do 167 do
168 if [ ! -s $RES_DIR/$treil ] 168 if [ ! -s $RES_DIR/$treil ]
169 then 169 then
170 bn = $(basename $treil ".treil") 170 bn = $(basename $treil ".treil")
171 rm $RES_DIR/$treil $RES_DIR/$bn.seg $RES_DIR/$bn.res $RES_DIR/$bn.pho 2> /dev/null 171 rm $RES_DIR/$treil $RES_DIR/$bn.seg $RES_DIR/$bn.res $RES_DIR/$bn.pho 2> /dev/null
172 print_info "[${BASENAME}] $RES_DIR/$bn.* files deleted.." 2 172 print_info "[${BASENAME}] $RES_DIR/$bn.* files deleted.." 2
173 fi 173 fi
174 done 174 done
175 175
176 # Check if more then 89% of treil are done 176 # Check if more then 89% of treil are done
177 nbres_p1=$(ls $RES_DIR_P1/*.res 2> /dev/null | wc -l) 177 nbres_p1=$(ls $RES_DIR_P1/*.res 2> /dev/null | wc -l)
178 nbtreil_p2=$(ls $RES_DIR/*.treil 2> /dev/null | wc -l) 178 nbtreil_p2=$(ls $RES_DIR/*.treil 2> /dev/null | wc -l)
179 if [ $nbres_p1 -gt 0 ] 179 if [ $nbres_p1 -gt 0 ]
180 then 180 then
181 pourcentage=$((($nbtreil_p2*100)/$nbres_p1)) 181 pourcentage=$((($nbtreil_p2*100)/$nbres_p1))
182 if [ $pourcentage -gt 89 ] 182 if [ $pourcentage -gt 89 ]
183 then 183 then
184 print_info "[${BASENAME}] Lattice already done, skipping $BASENAME" 1 184 print_info "[${BASENAME}] Lattice already done, skipping $BASENAME" 1
185 exit 0 185 exit 0
186 fi 186 fi
187 else 187 else
188 print_error "[${BASENAME}] No First Pass, No .res -> exit P2" 188 print_error "[${BASENAME}] No First Pass, No .res -> exit P2"
189 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No First Pass, No .res -> exit P2" ;fi 189 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No First Pass, No .res -> exit P2" ;fi
190 exit 1 190 exit 1
191 fi 191 fi
192 192
193 #--------------------# 193 #--------------------#
194 # Save configuration # 194 # Save configuration #
195 #--------------------# 195 #--------------------#
196 cp $SECONDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/SecondPass.cfg 196 cp $SECONDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/SecondPass.cfg
197 echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 197 echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
198 echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 198 echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
199 echo "FIRSTPASS_DIR=$FIRSTPASS_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 199 echo "FIRSTPASS_DIR=$FIRSTPASS_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
200 echo "PLP_DIR_P1=$PLP_DIR_P1" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 200 echo "PLP_DIR_P1=$PLP_DIR_P1" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
201 echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 201 echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
202 echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 202 echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
203 echo "LST=$LST" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 203 echo "LST=$LST" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
204 echo "HMM=$HMM" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 204 echo "HMM=$HMM" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
205 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 205 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
206 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1 206 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1
207 207
208 208
209 #--------------------------------------------------# 209 #--------------------------------------------------#
210 # Speaker Adaptation (AM) + Second pass (DECODING) # 210 # Speaker Adaptation (AM) + Second pass (DECODING) #
211 #--------------------------------------------------# 211 #--------------------------------------------------#
212 print_info "[${BASENAME}] Launch Second Pass" 1 212 print_info "[${BASENAME}] Launch Second Pass" 1
213 213
214 # for all speaker 214 # for all speaker
215 for speaker in $(cat $LBL_FILE_P1 | cut -f4 -d" " | sort | uniq) 215 for speaker in $(cat $LBL_FILE_P1 | cut -f4 -d" " | sort | uniq)
216 do 216 do
217 ## get seg file from P1 containing the speaker 217 ## get seg file from P1 containing the speaker
218 find $RES_DIR_P1 -name "*${speaker}.seg" -exec basename "{}" .seg \; | sort > $LST/$speaker.lst 218 find $RES_DIR_P1 -name "*${speaker}.seg" -exec basename "{}" .seg \; | sort > $LST/$speaker.lst
219 print_info "[${BASENAME}] file for $speaker in $LST/$speaker.lst" 3 219 print_info "[${BASENAME}] file for $speaker in $LST/$speaker.lst" 3
220 if [ ! -s $LST/$speaker.lst ]; then print_warn "no ${speaker} file in $RES_DIR_P1" 3; continue; fi 220 if [ ! -s $LST/$speaker.lst ]; then print_warn "no ${speaker} file in $RES_DIR_P1" 3; continue; fi
221 221
222 222
223 # for all AM 223 # for all AM
224 for (( i=0; $i<${#MTAG[@]} ; i++ )) 224 for (( i=0; $i<${#MTAG[@]} ; i++ ))
225 do 225 do
226 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then 226 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then
227 type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst") 227 type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst")
228 ## if is the good AM for the speaker 228 ## if is the good AM for the speaker
229 if [ -n "$type" ] 229 if [ -n "$type" ]
230 then 230 then
231 ## HMM adaptation 231 ## HMM adaptation
232 if [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] ) 232 if [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] )
233 then 233 then
234 print_info "[${BASENAME}] $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/" 3 234 print_info "[${BASENAME}] $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/" 3
235 $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/ 235 $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/ 2> /dev/null
236 fi 236 fi
237 237
238 if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] ) 238 if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] )
239 then 239 then
240 print_warn "[${BASENAME}] No hmm files created for $speaker" 2 240 print_warn "[${BASENAME}] No hmm files created for $speaker" 2
241 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No hmm files created for $speaker" 241 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No hmm files created for $speaker"
242 print_error "[${BASENAME}] Check $ERRORFILE" 242 print_error "[${BASENAME}] Check $ERRORFILE"
243 #exit 1 243 #exit 1
244 fi 244 fi
245 245
246 ## cp map files 246 ## cp map files
247 cp $SPEERAL_AM/${MODS[$i]}.map $HMM/$speaker.map 247 cp $SPEERAL_AM/${MODS[$i]}.map $HMM/$speaker.map
248 248
249 ## class clustering 249 ## class clustering
250 if [ -s $HMM/$speaker.hmm ] && ( [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] )) 250 if [ -s $HMM/$speaker.hmm ] && ( [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] ))
251 then 251 then
252 print_info "[${BASENAME}] $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls" 3 252 print_info "[${BASENAME}] $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls" 3
253 $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls 253 $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls 2> /dev/null
254 fi 254 fi
255 if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] ) 255 if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] )
256 then 256 then
257 print_warn "[${BASENAME}] No cls file created for $speaker" 2 257 print_warn "[${BASENAME}] No cls file created for $speaker" 2
258 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No cls file created for $speakers" 258 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No cls file created for $speakers"
259 print_error "[${BASENAME}] Check $ERRORFILE" 259 print_error "[${BASENAME}] Check $ERRORFILE"
260 #exit 1 260 #exit 1
261 fi 261 fi
262 262
263 ## Speeral decoding 263 ## Speeral decoding
264 if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ] 264 if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ]
265 then 265 then
266 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3 266 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3
267 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock 267 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock
268 else 268 else
269 print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2 269 print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2
270 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3 270 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3
271 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock 271 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock
272 fi 272 fi
273 273
274 if [ $CHECK -eq 1 ] 274 if [ $CHECK -eq 1 ]
275 then 275 then
276 check_second_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR" 276 check_second_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR"
277 if [ $? -eq 1 ] 277 if [ $? -eq 1 ]
278 then 278 then
279 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 279 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2
280 print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst" 280 print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst"
281 ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 281 ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
282 print_log_file $LOGFILE "Segs (and treil) not done :\n[" 282 print_log_file $LOGFILE "Segs (and treil) not done :\n["
283 diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $LOGFILE 283 diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $LOGFILE
284 print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]" 284 print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]"
285 rm ${OUTPUT_DIR_BASENAME}/.tmp 285 rm ${OUTPUT_DIR_BASENAME}/.tmp
286 #exit 1 286 #exit 1
287 fi 287 fi
288 fi 288 fi
289 break 289 break
290 fi 290 fi
291 fi 291 fi
292 done 292 done
293 #rm "$HMM/$speaker.*" > /dev/null 2>&1 293 #rm "$HMM/$speaker.*" > /dev/null 2>&1
294 #rm "$LST/$speaker.lst" > /dev/null 2>&1 294 #rm "$LST/$speaker.lst" > /dev/null 2>&1
295 done 295 done
296 296
297 ## Check missing seg and log it 297 ## Check missing seg and log it
298 if [ $CHECK -eq 1 ] 298 if [ $CHECK -eq 1 ]
299 then 299 then
300 ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 300 ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
301 todo=$(cat ${PLP_FILE_P1} | wc -l) 301 todo=$(cat ${PLP_FILE_P1} | wc -l)
302 if [ $todo -eq 0 ]; then todo=1;fi 302 if [ $todo -eq 0 ]; then todo=1;fi
303 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) 303 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l)))
304 pourcentage=$((($notdone*100)/$todo)) 304 pourcentage=$((($notdone*100)/$todo))
305 if [ $notdone -ne 0 ] 305 if [ $notdone -ne 0 ]
306 then 306 then
307 print_error "[${BASENAME}] Check $ERRORFILE" 307 print_error "[${BASENAME}] Check $ERRORFILE"
308 print_log_file "$ERRORFILE" "ERROR : Treil not done [" 308 print_log_file "$ERRORFILE" "ERROR : Treil not done ["
309 diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE 309 diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE
310 print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" 310 print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME"
311 else 311 else
312 print_log_file "$LOGFILE" "P2 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" 312 print_log_file "$LOGFILE" "P2 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')"
313 fi 313 fi
314 rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1 314 rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1
315 fi 315 fi
316 316
317 #---------------# 317 #---------------#
318 # Convert res # 318 # Convert res #
319 #---------------# 319 #---------------#
320 320
321 print_info "[${BASENAME}] Convert .res into .ctm" 1 321 print_info "[${BASENAME}] Convert .res into .ctm" 1
322 # .res => .ctm 322 # .res => .ctm
323 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.ctm 323 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.ctm
324 print_info "[${BASENAME}] Convert .res into .trs" 1 324 print_info "[${BASENAME}] Convert .res into .trs" 1
325 # .res => .trs 325 # .res => .trs
326 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 326 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
327 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 327 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
328 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1 328 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1
329 print_info "[${BASENAME}] Convert .res into .txt" 1 329 print_info "[${BASENAME}] Convert .res into .txt" 1
330 # .res => .txt 330 # .res => .txt
331 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.txt 331 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.txt
332 332
333 print_info "[${BASENAME}] <= P2 End | $(date +'%d/%m/%y %H:%M:%S')" 1 333 print_info "[${BASENAME}] <= P2 End | $(date +'%d/%m/%y %H:%M:%S')" 1
334 # unlock directory 334 # unlock directory
335 mv "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" 335 mv "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock"
336 336
337 337
main_tools/ThirdPass.sh
Diff comments   View file @ d7e9e4b
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : ThirdPass.sh # 4 # File : ThirdPass.sh #
5 # Brief : ASR third pass using trigg files # 5 # Brief : ASR third pass using trigg files #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # Version : 1.0 # 7 # Version : 1.0 #
8 # Date : 18/07/13 # 8 # Date : 18/07/13 #
9 ##################################################### 9 #####################################################
10 10
11 echo "### ThirdPass.sh ###" 11 echo "### ThirdPass.sh ###"
12 12
13 # Check OTMEDIA_HOME env var 13 # Check OTMEDIA_HOME env var
14 if [ -z ${OTMEDIA_HOME} ] 14 if [ -z ${OTMEDIA_HOME} ]
15 then 15 then
16 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 16 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
17 export OTMEDIA_HOME=$OTMEDIA_HOME 17 export OTMEDIA_HOME=$OTMEDIA_HOME
18 fi 18 fi
19 19
20 # where is SecondPass.sh 20 # where is SecondPass.sh
21 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 21 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
22 22
23 # Scripts Path 23 # Scripts Path
24 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 24 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
25 25
26 # Include scripts 26 # Include scripts
27 . $SCRIPT_PATH"/Tools.sh" 27 . $SCRIPT_PATH"/Tools.sh"
28 . $SCRIPT_PATH"/CheckThirdPass.sh" 28 . $SCRIPT_PATH"/CheckThirdPass.sh"
29 29
30 # where is ThirdPass.cfg 30 # where is ThirdPass.cfg
31 THIRDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ThirdPass.cfg" 31 THIRDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ThirdPass.cfg"
32 if [ -e $THIRDPASS_CONFIG_FILE ] 32 if [ -e $THIRDPASS_CONFIG_FILE ]
33 then 33 then
34 . $THIRDPASS_CONFIG_FILE 34 . $THIRDPASS_CONFIG_FILE
35 else 35 else
36 echo "ERROR : Can't find configuration file $THIRDPASS_CONFIG_FILE" >&2 36 echo "ERROR : Can't find configuration file $THIRDPASS_CONFIG_FILE" >&2
37 exit 1 37 exit 1
38 fi 38 fi
39 39
40 #---------------# 40 #---------------#
41 # Parse Options # 41 # Parse Options #
42 #---------------# 42 #---------------#
43 while getopts ":hDv:crf:" opt 43 while getopts ":hDv:crf:" opt
44 do 44 do
45 case $opt in 45 case $opt in
46 h) 46 h)
47 echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" 47 echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n"
48 echo -e "\t Options:" 48 echo -e "\t Options:"
49 echo -e "\t\t-h :\tprint this message" 49 echo -e "\t\t-h :\tprint this message"
50 echo -e "\t\t-D :\tDEBUG mode on" 50 echo -e "\t\t-D :\tDEBUG mode on"
51 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 51 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
52 echo -e "\t\t-c :\t Check process, stop if error detected" 52 echo -e "\t\t-c :\t Check process, stop if error detected"
53 echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)" 53 echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)"
54 echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done" 54 echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done"
55 exit 1 55 exit 1
56 ;; 56 ;;
57 D) 57 D)
58 DEBUG=1 58 DEBUG=1
59 ;; 59 ;;
60 v) 60 v)
61 VERBOSE=$OPTARG 61 VERBOSE=$OPTARG
62 ;; 62 ;;
63 c) 63 c)
64 CHECK=1 64 CHECK=1
65 ;; 65 ;;
66 f) 66 f)
67 FORKS="--forks $OPTARG" 67 FORKS="--forks $OPTARG"
68 ;; 68 ;;
69 r) 69 r)
70 RERUN=1 70 RERUN=1
71 ;; 71 ;;
72 :) 72 :)
73 echo "Option -$OPTARG requires an argument." > /dev/null 73 echo "Option -$OPTARG requires an argument." >&2
74 exit 1 74 exit 1
75 ;; 75 ;;
76 \?) 76 \?)
77 echo "BAD USAGE : unknow opton -$OPTARG" 77 echo "BAD USAGE : unknow opton -$OPTARG"
78 #exit 1 78 #exit 1
79 ;; 79 ;;
80 esac 80 esac
81 done 81 done
82 82
83 # mode debug enable 83 # mode debug enable
84 if [ $DEBUG -eq 1 ] 84 if [ $DEBUG -eq 1 ]
85 then 85 then
86 set -x 86 set -x
87 echo -e "## Mode DEBUG ON ##" 87 echo -e "## Mode DEBUG ON ##"
88 fi 88 fi
89 89
90 # mode verbose enable 90 # mode verbose enable
91 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi 91 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi
92 92
93 # Check USAGE by arguments number 93 # Check USAGE by arguments number
94 if [ $(($#-($OPTIND-1))) -ne 1 ] 94 if [ $(($#-($OPTIND-1))) -ne 1 ]
95 then 95 then
96 echo "BAD USAGE : ThirdPass.sh [OPTIONS] <PASS_DIR>" 96 echo "BAD USAGE : ThirdPass.sh [OPTIONS] <PASS_DIR>"
97 echo "$0 -h for more info" 97 echo "$0 -h for more info"
98 exit 1 98 exit 1
99 fi 99 fi
100 100
101 shift $((OPTIND-1)) 101 shift $((OPTIND-1))
102 # check Pass directory - First argument 102 # check Pass directory - First argument
103 if [ -e $1 ] && [ -d $1 ] 103 if [ -e $1 ] && [ -d $1 ]
104 then 104 then
105 PASS_DIR=$(readlink -e $1) 105 PASS_DIR=$(readlink -e $1)
106 BASENAME=$(basename $PASS_DIR) 106 BASENAME=$(basename $PASS_DIR)
107 else 107 else
108 print_error "can't find $1 directory" 108 print_error "can't find $1 directory"
109 exit 1 109 exit 1
110 fi 110 fi
111 111
112 print_info "[${BASENAME}] => P3 start | $(date +'%d/%m/%y %H:%M:%S')" 1 112 print_info "[${BASENAME}] => P3 start | $(date +'%d/%m/%y %H:%M:%S')" 1
113 113
114 #-------------# 114 #-------------#
115 # GLOBAL VARS # 115 # GLOBAL VARS #
116 #-------------# 116 #-------------#
117 EXPLOITCONFPASS_CONFIG_FILE="$PASS_DIR/ExploitConfPass.cfg" 117 EXPLOITCONFPASS_CONFIG_FILE="$PASS_DIR/ExploitConfPass.cfg"
118 if [ -e $EXPLOITCONFPASS_CONFIG_FILE ] 118 if [ -e $EXPLOITCONFPASS_CONFIG_FILE ]
119 then 119 then
120 TRIGGER_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "TRIGGER_SPEERAL=" | cut -f2 -d"=") 120 TRIGGER_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "TRIGGER_SPEERAL=" | cut -f2 -d"=")
121 LEX_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_SPEERAL=" | cut -f2 -d"=") 121 LEX_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_SPEERAL=" | cut -f2 -d"=")
122 LEX_BINODE_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_BINODE_SPEERAL=" | cut -f2 -d"=") 122 LEX_BINODE_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_BINODE_SPEERAL=" | cut -f2 -d"=")
123 LST="" 123 LST=""
124 PLP_DIR_P1="" 124 PLP_DIR_P1=""
125 HMM="" 125 HMM=""
126 else 126 else
127 print_error "[${BASENAME}] Can't find $EXPLOITCONFPASS_CONFIG_FILE file" 127 print_error "[${BASENAME}] Can't find $EXPLOITCONFPASS_CONFIG_FILE file"
128 print_warn "[${BASENAME}] Will use default directories and files" 2 128 print_warn "[${BASENAME}] Will use default directories and files" 2
129 #exit 1 129 #exit 1
130 TRIGGER_SPEERAL=$PASS_DIR/trigg/speeral/ 130 TRIGGER_SPEERAL=$PASS_DIR/trigg/speeral/
131 LEX_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext 131 LEX_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext
132 LEX_BINODE_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext.bin 132 LEX_BINODE_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext.bin
133 fi 133 fi
134 134
135 SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg" 135 SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg"
136 if [ -e $SECONDPASS_CONFIG_FILE ] 136 if [ -e $SECONDPASS_CONFIG_FILE ]
137 then 137 then
138 LST=$(cat $SECONDPASS_CONFIG_FILE | grep "^LST=" | cut -f2 -d"=") 138 LST=$(cat $SECONDPASS_CONFIG_FILE | grep "^LST=" | cut -f2 -d"=")
139 HMM=$(cat $SECONDPASS_CONFIG_FILE | grep "^HMM=" | cut -f2 -d"=") 139 HMM=$(cat $SECONDPASS_CONFIG_FILE | grep "^HMM=" | cut -f2 -d"=")
140 PLP_DIR_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_DIR_P1=" | cut -f2 -d"=") 140 PLP_DIR_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_DIR_P1=" | cut -f2 -d"=")
141 PLP_FILE_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_FILE_P1=" | cut -f2 -d"=") 141 PLP_FILE_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_FILE_P1=" | cut -f2 -d"=")
142 else 142 else
143 print_error "[${BASENAME}] Can't find $SECONDPASS_CONFIG_FILE file" 143 print_error "[${BASENAME}] Can't find $SECONDPASS_CONFIG_FILE file"
144 print_warn "[${BASENAME}] Will use default directories and files" 2 144 print_warn "[${BASENAME}] Will use default directories and files" 2
145 #exit 1 145 #exit 1
146 LST=$PASS_DIR/lists 146 LST=$PASS_DIR/lists
147 HMM=$PASS_DIR/hmm 147 HMM=$PASS_DIR/hmm
148 PLP_DIR_P1=$PASS_DIR/PLP 148 PLP_DIR_P1=$PASS_DIR/PLP
149 PLP_FILE_P1=$PASS_DIR/plp.lst 149 PLP_FILE_P1=$PASS_DIR/plp.lst
150 fi 150 fi
151 151
152 OUTPUT_DIR_BASENAME=$PASS_DIR 152 OUTPUT_DIR_BASENAME=$PASS_DIR
153 RES_DIR="$PASS_DIR/res_p3" 153 RES_DIR="$PASS_DIR/res_p3"
154 LOGFILE=$PASS_DIR"/info_p3.log" 154 LOGFILE=$PASS_DIR"/info_p3.log"
155 ERRORFILE=$PASS_DIR"/error_p3.log" 155 ERRORFILE=$PASS_DIR"/error_p3.log"
156 156
157 #------------------# 157 #------------------#
158 # Create WORKSPACE # 158 # Create WORKSPACE #
159 #------------------# 159 #------------------#
160 160
161 # Lock directory 161 # Lock directory
162 if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] THIRDPASS is locked -> exit" 2; exit 1; fi 162 if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] THIRDPASS is locked -> exit" 2; exit 1; fi
163 rm "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" > /dev/null 2>&1 163 rm "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" > /dev/null 2>&1
164 touch "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" > /dev/null 2>&1 164 touch "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" > /dev/null 2>&1
165 165
166 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi 166 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi
167 mkdir -p $RES_DIR 167 mkdir -p $RES_DIR
168 print_info "[${BASENAME}] Make directory $RES_DIR" 3 168 print_info "[${BASENAME}] Make directory $RES_DIR" 3
169 169
170 #--------------------# 170 #--------------------#
171 # Save configuration # 171 # Save configuration #
172 #--------------------# 172 #--------------------#
173 cp $THIRDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/ThirdPass.cfg 173 cp $THIRDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/ThirdPass.cfg
174 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/ThirdPass.cfg 174 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/ThirdPass.cfg
175 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ThirdPass.cfg" 1 175 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ThirdPass.cfg" 1
176 176
177 #--------------------------------------------------# 177 #--------------------------------------------------#
178 # Third Pass using trigger file (DECODING) # 178 # Third Pass using trigger file (DECODING) #
179 #--------------------------------------------------# 179 #--------------------------------------------------#
180 print_info "[${BASENAME}] Launch Third Pass" 1 180 print_info "[${BASENAME}] Launch Third Pass" 1
181 181
182 ## Generate speeral config file adding trigger rep 182 ## Generate speeral config file adding trigger rep
183 cat $SPEERAL_CFG_PATH/$SPEERAL_CFG_FILE | sed -e "s|<nom>[^<]*</nom>|<nom>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext</nom>|g" \ 183 cat $SPEERAL_CFG_PATH/$SPEERAL_CFG_FILE | sed -e "s|<nom>[^<]*</nom>|<nom>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext</nom>|g" \
184 | sed -e "s|<binode>[^<]*</binode>|<binode>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext.bin</binode>|g" \ 184 | sed -e "s|<binode>[^<]*</binode>|<binode>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext.bin</binode>|g" \
185 | sed -e "s|<trigger><dir>[^<]*</dir></trigger>|<trigger><dir>$TRIGGER_SPEERAL</dir></trigger>|g" > $OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml 185 | sed -e "s|<trigger><dir>[^<]*</dir></trigger>|<trigger><dir>$TRIGGER_SPEERAL</dir></trigger>|g" > $OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml
186 SPEERAL_THIRD_CFG=$OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml 186 SPEERAL_THIRD_CFG=$OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml
187 print_info "[${BASENAME}] Third Pass Speeral Config file ${SPEERAL_THIRD_CFG}" 3 187 print_info "[${BASENAME}] Third Pass Speeral Config file ${SPEERAL_THIRD_CFG}" 3
188 188
189 # for all speaker 189 # for all speaker
190 for lspeaker in $(ls $LST/*.lst) 190 for lspeaker in $(ls $LST/*.lst)
191 do 191 do
192 speaker=$(basename $lspeaker ".lst") 192 speaker=$(basename $lspeaker ".lst")
193 # for all AM 193 # for all AM
194 for (( i=0; $i<${#MTAG[@]} ; i++ )) 194 for (( i=0; $i<${#MTAG[@]} ; i++ ))
195 do 195 do
196 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then 196 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then
197 type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst") 197 type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst")
198 ## if is the good AM for the speaker 198 ## if is the good AM for the speaker
199 if [ -n "$type" ] 199 if [ -n "$type" ]
200 then 200 then
201 ## Speeral decoding 201 ## Speeral decoding
202 if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ] 202 if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ]
203 then 203 then
204 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3 204 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3
205 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock 205 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock
206 else 206 else
207 print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2 207 print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2
208 print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3 208 print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3
209 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock 209 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock
210 fi 210 fi
211 211
212 if [ $CHECK -eq 1 ] 212 if [ $CHECK -eq 1 ]
213 then 213 then
214 check_third_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR" 214 check_third_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR"
215 if [ $? -eq 1 ] 215 if [ $? -eq 1 ]
216 then 216 then
217 217
218 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 218 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2
219 print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst" 219 print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst"
220 ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 220 ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
221 print_log_file $LOGFILE "Segs (and treil) not done :\n[" 221 print_log_file $LOGFILE "Segs (and treil) not done :\n["
222 diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $LOGFILE 222 diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $LOGFILE
223 print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]" 223 print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]"
224 rm ${OUTPUT_DIR_BASENAME}/.tmp 224 rm ${OUTPUT_DIR_BASENAME}/.tmp
225 fi 225 fi
226 fi 226 fi
227 break 227 break
228 fi 228 fi
229 fi 229 fi
230 done 230 done
231 done 231 done
232 232
233 ## Check missing seg and log it 233 ## Check missing seg and log it
234 234
235 ## Check missing seg and log it 235 ## Check missing seg and log it
236 if [ $CHECK -eq 1 ] 236 if [ $CHECK -eq 1 ]
237 then 237 then
238 ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 238 ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
239 todo=$(cat ${PLP_FILE_P1} | wc -l) 239 todo=$(cat ${PLP_FILE_P1} | wc -l)
240 if [ $todo -eq 0 ]; then todo=1;fi 240 if [ $todo -eq 0 ]; then todo=1;fi
241 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) 241 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l)))
242 pourcentage=$((($notdone*100)/$todo)) 242 pourcentage=$((($notdone*100)/$todo))
243 if [ $notdone -ne 0 ] 243 if [ $notdone -ne 0 ]
244 then 244 then
245 print_error "[${BASENAME}] Check $ERRORFILE" 245 print_error "[${BASENAME}] Check $ERRORFILE"
246 print_log_file "$ERRORFILE" "ERROR : Treil not done [" 246 print_log_file "$ERRORFILE" "ERROR : Treil not done ["
247 diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE 247 diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE
248 print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" 248 print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME"
249 else 249 else
250 print_log_file "$LOGFILE" "P3 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" 250 print_log_file "$LOGFILE" "P3 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')"
251 fi 251 fi
252 rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1 252 rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1
253 fi 253 fi
254 254
255 #---------------# 255 #---------------#
256 # Convert res # 256 # Convert res #
257 #---------------# 257 #---------------#
258 258
259 print_info "[${BASENAME}] Convert .res into .ctm" 1 259 print_info "[${BASENAME}] Convert .res into .ctm" 1
260 # .res => .ctm 260 # .res => .ctm
261 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.ctm 261 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.ctm
262 print_info "[${BASENAME}] Convert .res into .trs" 1 262 print_info "[${BASENAME}] Convert .res into .trs" 1
263 # .res => .trs 263 # .res => .trs
264 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 264 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
265 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 265 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
266 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1 266 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1
267 print_info "[${BASENAME}] Convert .res into .txt" 1 267 print_info "[${BASENAME}] Convert .res into .txt" 1
268 # .res => .txt 268 # .res => .txt
269 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.txt 269 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.txt
270 270
271 271
272 print_info "[${BASENAME}] <= P3 End | $(date +'%d/%m/%y %H:%M:%S')" 1 272 print_info "[${BASENAME}] <= P3 End | $(date +'%d/%m/%y %H:%M:%S')" 1
273 273
274 # unlock directory 274 # unlock directory
275 mv "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" 275 mv "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock"
276 276
277 277
tools/scripts/Tools.sh
Diff comments   View file @ d7e9e4b
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ################################################# 3 #################################################
4 # File : Tools.sh # 4 # File : Tools.sh #
5 # Brief : Tool functions # 5 # Brief : Tool functions #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # Version : 1.0 # 7 # Version : 1.0 #
8 # Date : 13/06/13 # 8 # Date : 13/06/13 #
9 ################################################# 9 #################################################
10 10
11 # Type 11 # Type
12 ERROR=2 12 ERROR=2
13 WARN=1 13 WARN=1
14 INFO=0 14 INFO=0
15 15
16 # Global Verbose variable (have to be define in scripts) 16 # Global Verbose variable (have to be define in scripts)
17 # 0 : mute 17 # 0 : mute
18 # 1 : basic info (start,config,end) 18 # 1 : basic info (start,config,end)
19 # 2 : add process 19 # 2 : add process
20 # 3 : all 20 # 3 : all
21 VERBOSE=0 21 VERBOSE=0
22 22
23 # print_message( verbose_level, message) 23 # print_message( verbose_level, message)
24 # - message_type : ERROR, WARN, INFO 24 # - message_type : ERROR, WARN, INFO
25 # - verbose_level : level to print message 25 # - verbose_level : level to print message
26 # - message : the message to print 26 # - message : the message to print
27 function print_message() 27 function print_message()
28 { 28 {
29 if [ $# -eq 3 ] 29 if [ $# -eq 3 ]
30 then 30 then
31 # print error 31 # print error
32 if [ $1 -eq $ERROR ]; then echo -ne "###ERROR : $3\n" > /dev/stderr; return 0; fi 32 if [ $1 -eq $ERROR ]; then echo -ne " ###ERROR : $3\n" >&2 ; return 0; fi
33 # check verbose level 33 # check verbose level
34 if [ $2 -le $VERBOSE ] 34 if [ $2 -le $VERBOSE ]
35 then 35 then
36 #for (( dec=1 ; $dec<$2 ; dec++ )); do echo -ne "\t"; done 36 #for (( dec=1 ; $dec<$2 ; dec++ )); do echo -ne "\t"; done
37 # print warn 37 # print warn
38 if [ $1 = $WARN ]; then echo -ne "##WARNING : $3\n"; return 0; fi 38 if [ $1 = $WARN ]; then echo -ne " ##WARNING : $3\n"; return 0; fi
39 # default info 39 # default info
40 echo -ne "#INFO : $3\n" 40 echo -ne " #INFO : $3\n"
41 fi 41 fi
42 fi 42 fi
43 } 43 }
44 44
45 # print_error(message) 45 # print_error(message)
46 # - message : error message to print 46 # - message : error message to print
47 function print_error() 47 function print_error()
48 { 48 {
49 print_message $ERROR 3 "$1" 49 if [ $# -eq 1 ]
50 then
51 print_message $ERROR 3 "$1"
52 fi
50 } 53 }
51 54
52 # print_warn(message,[level]) 55 # print_warn(message,[level])
53 # - message : message to print 56 # - message : message to print
54 # - level : optional option - verbose level to print 57 # - level : optional option - verbose level to print
55 function print_warn() 58 function print_warn()
56 { 59 {
57 if [ $# -eq 2 ] 60 if [ $# -eq 2 ]
58 then 61 then
59 print_message $WARN $2 "$1" 62 print_message $WARN $2 "$1"
60 else 63 else
61 print_message $WARN $VERBOSE "$1" 64 print_message $WARN $VERBOSE "$1"
62 fi 65 fi
63 } 66 }
64 67
65 # print_info(message,[level]) 68 # print_info(message,[level])
66 # - message : message to print 69 # - message : message to print
67 # - level : optional option - verbose level to print 70 # - level : optional option - verbose level to print
68 function print_info() 71 function print_info()
69 { 72 {
70 if [ $# -eq 2 ] 73 if [ $# -eq 2 ]
71 then 74 then
72 print_message $INFO $2 "$1" 75 print_message $INFO $2 "$1"
73 else 76 else
74 print_message $INFO $VERBOSE "$1" 77 print_message $INFO $VERBOSE "$1"
75 fi 78 fi
76 } 79 }
77 80
78 # print_log_file(filename,message) 81 # print_log_file(filename,message)
79 # - filename : full path to log file 82 # - filename : full path to log file
80 # - message : message to log 83 # - message : message to log
81 print_log_file() 84 print_log_file()
82 { 85 {
83 if [ $# -eq 2 ] 86 if [ $# -eq 2 ]
84 then 87 then
85 echo -e "$2" >> "$1" 88 echo -e "$2" >> "$1"
86 fi 89 fi
87 } 90 }
88 91