Jean-François Rey / otmedia

Download as
Browse Code »

Commit b427f103e12f3e8f120ff02a443be418e6f70a49

Authored by Jean-François Rey
1 parent 561670accd
Exists in master

update log info printed

Showing 7 changed files with 156 additions and 87 deletions Inline Diff

main_tools/ExploitConfidencePass.sh
Diff comments   View file @ b427f10
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : ExploitConfidencePass.sh # 4 # File : ExploitConfidencePass.sh #
5 # Brief : Exploit the ASR confidence pass to : # 5 # Brief : Exploit the ASR confidence pass to : #
6 # -> boost the confident zone # 6 # -> boost the confident zone #
7 # -> find alternative in non confident zone 7 # -> find alternative in non confident zone
8 # -> dynamicly extend the lexicon # 8 # -> dynamicly extend the lexicon #
9 # Author : Jean-François Rey # 9 # Author : Jean-François Rey #
10 # (base on Emmanuel Ferreira # 10 # (base on Emmanuel Ferreira #
11 # and Hugo Mauchrétien works) # 11 # and Hugo Mauchrétien works) #
12 # Version : 1.0 # 12 # Version : 1.0 #
13 # Date : 25/06/13 # 13 # Date : 25/06/13 #
14 ##################################################### 14 #####################################################
15 15
16 echo "### ExploitConfidencePass.sh ###" 16 echo "### ExploitConfidencePass.sh ###"
17 17
18 # Check OTMEDIA_HOME env var 18 # Check OTMEDIA_HOME env var
19 if [ -z ${OTMEDIA_HOME} ] 19 if [ -z ${OTMEDIA_HOME} ]
20 then 20 then
21 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 21 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
22 export OTMEDIA_HOME=$OTMEDIA_HOME 22 export OTMEDIA_HOME=$OTMEDIA_HOME
23 fi 23 fi
24 24
25 # where is ExploitConfidencePass.sh 25 # where is ExploitConfidencePass.sh
26 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 26 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
27 27
28 if [ -z ${SCRIPT_PATH} ] 28 if [ -z ${SCRIPT_PATH} ]
29 then 29 then
30 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 30 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
31 fi 31 fi
32 32
33 # Include scripts 33 # Include scripts
34 . $SCRIPT_PATH"/Tools.sh" 34 . $SCRIPT_PATH"/Tools.sh"
35 . $SCRIPT_PATH"/CheckExploitConfPass.sh" 35 . $SCRIPT_PATH"/CheckExploitConfPass.sh"
36 36
37 # where is ExploitConfidencePass.cfg 37 # where is ExploitConfidencePass.cfg
38 EXPLOITCONFIDENCEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ExploitConfidencePass.cfg" 38 EXPLOITCONFIDENCEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ExploitConfidencePass.cfg"
39 if [ -e $EXPLOITCONFIDENCEPASS_CONFIG_FILE ] 39 if [ -e $EXPLOITCONFIDENCEPASS_CONFIG_FILE ]
40 then 40 then
41 . $EXPLOITCONFIDENCEPASS_CONFIG_FILE 41 . $EXPLOITCONFIDENCEPASS_CONFIG_FILE
42 else 42 else
43 echo "ERROR : Can't find configuration file $EXPLOITCONFIDENCEPASS_CONFIG_FILE" >&2 43 echo "ERROR : Can't find configuration file $EXPLOITCONFIDENCEPASS_CONFIG_FILE" >&2
44 exit 1 44 exit 1
45 fi 45 fi
46 46
47 #---------------# 47 #---------------#
48 # Parse Options # 48 # Parse Options #
49 #---------------# 49 #---------------#
50 while getopts ":hDv:cf:r" opt 50 while getopts ":hDv:cf:r" opt
51 do 51 do
52 case $opt in 52 case $opt in
53 h) 53 h)
54 echo -e "$0 [OPTIONS] <INPUT_DIRECTORY>\n" 54 echo -e "$0 [OPTIONS] <INPUT_DIRECTORY>\n"
55 echo -e "\t Options:" 55 echo -e "\t Options:"
56 echo -e "\t\t-h :\tprint this message" 56 echo -e "\t\t-h :\tprint this message"
57 echo -e "\t\t-D :\tDEBUG mode on" 57 echo -e "\t\t-D :\tDEBUG mode on"
58 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 58 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
59 echo -e "\t\t-c :\tCheck process, stop if error detected" 59 echo -e "\t\t-c :\tCheck process, stop if error detected"
60 echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)" 60 echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)"
61 echo -e "\t\t-r n :\tforce rerun without deleting files" 61 echo -e "\t\t-r n :\tforce rerun without deleting files"
62 exit 1 62 exit 1
63 ;; 63 ;;
64 D) 64 D)
65 DEBUG=1 65 DEBUG=1
66 ;; 66 ;;
67 v) 67 v)
68 VERBOSE=$OPTARG 68 VERBOSE=$OPTARG
69 ;; 69 ;;
70 c) 70 c)
71 CHECK=1 71 CHECK=1
72 ;; 72 ;;
73 f) 73 f)
74 FORKS="--forks $OPTARG" 74 FORKS="--forks $OPTARG"
75 ;; 75 ;;
76 r) 76 r)
77 RERUN=1 77 RERUN=1
78 ;; 78 ;;
79 :) 79 :)
80 echo "Option -$OPTARG requires an argument." > /dev/stderr 80 echo "Option -$OPTARG requires an argument." > /dev/stderr
81 exit 1 81 exit 1
82 ;; 82 ;;
83 \?) 83 \?)
84 echo "BAD USAGE : unknow opton -$OPTARG" 84 echo "BAD USAGE : unknow opton -$OPTARG"
85 #exit 1 85 #exit 1
86 ;; 86 ;;
87 esac 87 esac
88 done 88 done
89 89
90 # mode debug enable 90 # mode debug enable
91 if [ $DEBUG -eq 1 ] 91 if [ $DEBUG -eq 1 ]
92 then 92 then
93 set -x 93 set -x
94 echo -e "## Mode DEBUG ON ##" 94 echo -e "## Mode DEBUG ON ##"
95 fi 95 fi
96 96
97 # mode verbose enable 97 # mode verbose enable
98 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi 98 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
99 99
100 # Check USAGE by arguments number 100 # Check USAGE by arguments number
101 if [ $(($#-($OPTIND-1))) -ne 1 ] 101 if [ $(($#-($OPTIND-1))) -ne 1 ]
102 then 102 then
103 echo "BAD USAGE : ExploitConfidencePass.sh [OPTIONS] <INPUT_DIRECTORY>" 103 echo "BAD USAGE : ExploitConfidencePass.sh [OPTIONS] <INPUT_DIRECTORY>"
104 echo "$0 -h for more info" 104 echo "$0 -h for more info"
105 exit 1 105 exit 1
106 fi 106 fi
107 107
108 shift $((OPTIND-1)) 108 shift $((OPTIND-1))
109 # check input directory - first argument 109 # check input directory - first argument
110 if [ ! -e $1 ] 110 if [ ! -e $1 ]
111 then 111 then
112 print_error "can't open $1" 112 print_error "can't open $1"
113 exit 1 113 exit 1
114 fi 114 fi
115 115
116 print_info "[${BASENAME}] => ExploitConfPass start | $(date +'%d/%m/%y %H:%M:%S')" 1 116 print_info "[${BASENAME}] => ExploitConfPass start | $(date +'%d/%m/%y %H:%M:%S')" 1
117 117
118 #-------------# 118 #-------------#
119 # GLOBAL VARS # 119 # GLOBAL VARS #
120 #-------------# 120 #-------------#
121 INPUT_DIR=$(readlink -e $1) 121 INPUT_DIR=$(readlink -e $1)
122 OUTPUT_DIR=$INPUT_DIR 122 OUTPUT_DIR=$INPUT_DIR
123 BASENAME=$(basename $OUTPUT_DIR) 123 BASENAME=$(basename $OUTPUT_DIR)
124 SHOW_DIR="$OUTPUT_DIR/shows/" 124 SHOW_DIR="$OUTPUT_DIR/shows/"
125 SOLR_RES="$OUTPUT_DIR/solr/" 125 SOLR_RES="$OUTPUT_DIR/solr/"
126 EXT_LEX="$OUTPUT_DIR/LEX/" 126 EXT_LEX="$OUTPUT_DIR/LEX/"
127 TRIGGER_CONFZONE="$OUTPUT_DIR/trigg/" 127 TRIGGER_CONFZONE="$OUTPUT_DIR/trigg/"
128 LOGFILE="$(dirname $OUTPUT_DIR)/info_exploitconf.log" 128 LOGFILE="$OUTPUT_DIR/info_exploitconf.log"
129 ERRORFILE="$(dirname $OUTPUT_DIR)/error_exploitconf.log" 129 ERRORFILE="$OUTPUT_DIR/error_exploitconf.log"
130 130
131 CONFPASS_CONFIG_FILE="$(readlink -e $1)/ConfPass.cfg" 131 CONFPASS_CONFIG_FILE="$(readlink -e $1)/ConfPass.cfg"
132 if [ -e $CONFPASS_CONFIG_FILE ] 132 if [ -e $CONFPASS_CONFIG_FILE ]
133 then 133 then
134 { 134 {
135 RES_CONF_DIR=$(cat $CONFPASS_CONFIG_FILE | grep "^RES_CONF_DIR=" | cut -f2 -d"=") 135 RES_CONF_DIR=$(cat $CONFPASS_CONFIG_FILE | grep "^RES_CONF_DIR=" | cut -f2 -d"=")
136 RES_CONF=$(cat $CONFPASS_CONFIG_FILE | grep "^CONF_DIR=" | cut -f2 -d"=") 136 RES_CONF=$(cat $CONFPASS_CONFIG_FILE | grep "^CONF_DIR=" | cut -f2 -d"=")
137 print_info "[${BASENAME}] Use confidence measure from : $RES_CONF" 2 137 print_info "[${BASENAME}] Use confidence measure from : $RES_CONF" 2
138 } 138 }
139 else 139 else
140 { 140 {
141 print_error "[${BASENAME}] Can't find $CONFPASS_CONFIG_FILE" 141 print_error "[${BASENAME}] Can't find $CONFPASS_CONFIG_FILE"
142 print_error "[${BASENAME}] -> use res_p2" 142 print_error "[${BASENAME}] -> use res_p2"
143 RES_CONF_DIR="$INPUT_DIR/conf/res_p2/scored_ctm" 143 RES_CONF_DIR="$INPUT_DIR/conf/res_p2/scored_ctm"
144 RES_CONF="$INPUT_DIR/conf/res_p2" 144 RES_CONF="$INPUT_DIR/conf/res_p2"
145 } 145 }
146 fi 146 fi
147 147
148 mkdir -p $SHOW_DIR > /dev/null 2>&1 148 mkdir -p $SHOW_DIR > /dev/null 2>&1
149 mkdir -p $SOLR_RES > /dev/null 2>&1 149 mkdir -p $SOLR_RES > /dev/null 2>&1
150 mkdir -p $EXT_LEX > /dev/null 2>&1 150 mkdir -p $EXT_LEX > /dev/null 2>&1
151 mkdir -p $TRIGGER_CONFZONE > /dev/null 2>&1 151 mkdir -p $TRIGGER_CONFZONE > /dev/null 2>&1
152 152
153 #------------------# 153 #------------------#
154 # Create Workspace # 154 # Create Workspace #
155 #------------------# 155 #------------------#
156 # Lock directory 156 # Lock directory
157 if [ -e "$OUTPUT_DIR_BASENAME/EXPLOITCONFPASS.lock" ] && [ $RERUN -eq 0 ] 157 if [ -e "$OUTPUT_DIR_BASENAME/EXPLOITCONFPASS.lock" ] && [ $RERUN -eq 0 ]
158 then 158 then
159 print_warn "[${BASENAME}] ExploitConfidencePass is locked -> exit" 2 159 print_warn "[${BASENAME}] ExploitConfidencePass is locked -> exit" 2
160 exit 1 160 exit 1
161 fi 161 fi
162 rm "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" > /dev/null 2>&1 162 rm "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" > /dev/null 2>&1
163 touch "$OUTPUT_DIR/EXPLOITCONFPASS.lock" > /dev/null 2>&1 163 touch "$OUTPUT_DIR/EXPLOITCONFPASS.lock" > /dev/null 2>&1
164 164
165 #------# 165 #------#
166 # Save # 166 # Save #
167 #------# 167 #------#
168 cp $EXPLOITCONFIDENCEPASS_CONFIG_FILE $OUTPUT_DIR/ExploitConfPass.cfg 168 cp $EXPLOITCONFIDENCEPASS_CONFIG_FILE $OUTPUT_DIR/ExploitConfPass.cfg
169 echo "TRIGGER_DIR=$TRIGGER_CONFZONE" >> $OUTPUT_DIR/ExploitConfPass.cfg 169 echo "TRIGGER_DIR=$TRIGGER_CONFZONE" >> $OUTPUT_DIR/ExploitConfPass.cfg
170 echo "TRIGGER_SPEERAL=$TRIGGER_CONFZONE/speeral/" >> $OUTPUT_DIR/ExploitConfPass.cfg 170 echo "TRIGGER_SPEERAL=$TRIGGER_CONFZONE/speeral/" >> $OUTPUT_DIR/ExploitConfPass.cfg
171 echo "LEX_SPEERAL=$EXT_LEX/speeral/${lexname}_ext" >> $OUTPUT_DIR/ExploitConfPass.cfg 171 echo "LEX_SPEERAL=$EXT_LEX/speeral/${lexname}_ext" >> $OUTPUT_DIR/ExploitConfPass.cfg
172 echo "LEX_BINODE_SPEERAL=$EXT_LEX/speeral/${lexname}_ext.bin" >> $OUTPUT_DIR/ExploitConfPass.cfg 172 echo "LEX_BINODE_SPEERAL=$EXT_LEX/speeral/${lexname}_ext.bin" >> $OUTPUT_DIR/ExploitConfPass.cfg
173 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ExploitConfPass.cfg" 1 173 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ExploitConfPass.cfg" 1
174 174
175 #---------------# 175 #---------------#
176 # Check Pass # 176 # Check Pass #
177 #---------------# 177 #---------------#
178 if [ $( ${RES_CONf_DIR}/*.res 2> /dev/null | wc -l) -eq 0 ]
179 then
180 print_error "[${BASENAME}] No Conf Pass res -> exit ExploitConfPass"
181 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ConfPass res in $[RES_CONf_DIR}" ;fi
182 exit 1
183 fi
178 184
179 #-----------------------# 185 #-----------------------#
180 # Segmentation by show # 186 # Segmentation by show #
181 #-----------------------# 187 #-----------------------#
182 # create txt file from scored res 188 # create txt file from scored res
183 # tag pos and lemmatization of the txt file 189 # tag pos and lemmatization of the txt file
184 # merge the scored res and taglem file 190 # merge the scored res and taglem file
185 # segment using the last generated file 191 # segment using the last generated file
186 # and create a ctm file by show 192 # and create a ctm file by show
187 193
188 print_info "[${BASENAME}] Segmentation by show" 1 194 print_info "[${BASENAME}] Segmentation by show" 1
189 195
190 # -> to txt 196 # -> to txt
191 print_info "[${BASENAME}] Create txt from scored res" 2 197 print_info "[${BASENAME}] Create txt from scored res" 3
192 cat ${RES_CONF_DIR}/*.res > $INPUT_DIR/$BASENAME.sctm 198 cat ${RES_CONF_DIR}/*.res > $INPUT_DIR/$BASENAME.sctm
193 cat $INPUT_DIR/$BASENAME.seg | $SIGMUND_BIN/myConvert.pl $INPUT_DIR/$BASENAME.sctm $INPUT_DIR/$BASENAME.tmp 199 cat $INPUT_DIR/$BASENAME.seg | $SIGMUND_BIN/myConvert.pl $INPUT_DIR/$BASENAME.sctm $INPUT_DIR/$BASENAME.tmp
194 cat $INPUT_DIR/$BASENAME.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | sed -e "s/_/ /g" | sort -nt 'n' -k '2' > $INPUT_DIR/$BASENAME.txt 200 cat $INPUT_DIR/$BASENAME.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | sed -e "s/_/ /g" | sort -nt 'n' -k '2' > $INPUT_DIR/$BASENAME.txt
195 201
196 # -> to tagger + lemme 202 # -> to tagger + lemme
197 print_info "[${BASENAME}] Tag pos and lem in txt file" 2 203 print_info "[${BASENAME}] Tag pos and lem in txt file" 3
198 iconv -t ISO_8859-1 $INPUT_DIR/$BASENAME.txt > $INPUT_DIR/$BASENAME.tmp 204 iconv -t ISO_8859-1 $INPUT_DIR/$BASENAME.txt > $INPUT_DIR/$BASENAME.tmp
199 $SIGMUND_BIN/txt2lem.sh $INPUT_DIR/$BASENAME.tmp $INPUT_DIR/$BASENAME.taglem 205 $SIGMUND_BIN/txt2lem.sh $INPUT_DIR/$BASENAME.tmp $INPUT_DIR/$BASENAME.taglem
200 206
201 # merge sctm and taglem 207 # merge sctm and taglem
202 print_info "[${BASENAME}] Merge scored ctm with tag pos and lem file" 2 208 print_info "[${BASENAME}] Merge scored ctm with tag pos and lem file" 3
203 cat $INPUT_DIR/$BASENAME.sctm | $SCRIPT_PATH/BdlexUC.pl ${RULES}/basic -f | iconv -t ISO_8859-1 | $SCRIPT_PATH/scoredCtmAndTaggedLem2All.pl $INPUT_DIR/$BASENAME.taglem > $INPUT_DIR/$BASENAME.ctl 209 cat $INPUT_DIR/$BASENAME.sctm | $SCRIPT_PATH/BdlexUC.pl ${RULES}/basic -f | iconv -t ISO_8859-1 | $SCRIPT_PATH/scoredCtmAndTaggedLem2All.pl $INPUT_DIR/$BASENAME.taglem > $INPUT_DIR/$BASENAME.ctl
204 210
205 # -> new seg 211 # -> new seg
206 print_info "[${BASENAME}] Create xml file and run Topic Seg" 2 212 print_info "[${BASENAME}] Create xml file and run Topic Seg" 3
207 $SIGMUND_BIN/tagLem2xml.pl $INPUT_DIR/$BASENAME.taglem $INPUT_DIR/$BASENAME.doc.xml 213 $SIGMUND_BIN/tagLem2xml.pl $INPUT_DIR/$BASENAME.taglem $INPUT_DIR/$BASENAME.doc.xml
208 rm $INPUT_DIR/$BASENAME.tmp #$INPUT_DIR/$BASENAME.taglem 214 rm $INPUT_DIR/$BASENAME.tmp #$INPUT_DIR/$BASENAME.taglem
209 215
210 # Lia_topic_seg : bring together sentences into show 216 # Lia_topic_seg : bring together sentences into show
211 cp $INPUT_DIR/$BASENAME.doc.xml 0.xml 217 cp $INPUT_DIR/$BASENAME.doc.xml 0.xml
212 java -cp $LIATOPICSEG/bin Test > $INPUT_DIR/show.seg 218 java -cp $LIATOPICSEG/bin Test > $INPUT_DIR/show.seg
213 cat $INPUT_DIR/show.seg | $SIGMUND_BIN/toSegEmiss.pl $INPUT_DIR/$BASENAME.show.seg 219 cat $INPUT_DIR/show.seg | $SIGMUND_BIN/toSegEmiss.pl $INPUT_DIR/$BASENAME.show.seg
214 rm 0.xml $INPUT_DIR/show.seg 220 rm 0.xml $INPUT_DIR/show.seg
215 221
216 if [ $CHECK -eq 1 ] 222 if [ $CHECK -eq 1 ]
217 then 223 then
218 if [ ! -s $INPUT_DIR/$BASENAME.show.seg ];then echo -e "[${BASENAME}] ERROR : no Topic segmentation" >> $ERRORFILE; fi 224 if [ ! -s $INPUT_DIR/$BASENAME.show.seg ]
225 then
226 print_error "[${BASENAME}] No Topic segmentation ! "
227 print_error "[${BASENAME}] Check $ERRORFILE "
228 print_log_file "$ERRORFILE" "No Topic segmentation in ${BASENAME}.show.seg"
229 fi
219 fi 230 fi
220 231
221 # Segment ctm into several show files and create a seg list by show 232 # Segment ctm into several show files and create a seg list by show
222 print_info "[${BASENAME}] Segment ctm into show files and a seg list by show" 2 233 print_info "[${BASENAME}] Segment ctm into show files and a seg list by show" 1
223 $SCRIPT_PATH/ctm2show.pl $INPUT_DIR/$BASENAME.ctl $INPUT_DIR/$BASENAME.show.seg $SHOW_DIR 234 $SCRIPT_PATH/ctm2show.pl $INPUT_DIR/$BASENAME.ctl $INPUT_DIR/$BASENAME.show.seg $SHOW_DIR
224 235
225 #-----------------------------------------------------------# 236 #-----------------------------------------------------------#
226 # SOLR QUERIES # 237 # SOLR QUERIES #
227 # -> Create Confidente Word # 238 # -> Create Confidente Word #
228 # Keep conf words and use Tags # 239 # Keep conf words and use Tags #
229 # -> Query SOLR (document & multimedia) # 240 # -> Query SOLR (document & multimedia) #
230 # concat word + add date 2 day before and after the show # 241 # concat word + add date 2 day before and after the show #
231 # query document & multimedia # 242 # query document & multimedia #
232 #-----------------------------------------------------------# 243 #-----------------------------------------------------------#
233 print_info "[${BASENAME}] Create SOLR queries and ASK SOLR" 1 244 print_info "[${BASENAME}] Create SOLR queries and ask SOLR" 1
234 for show in $(ls $SHOW_DIR/*.ctm) 245 for show in $(ls $SHOW_DIR/*.ctm)
235 do 246 do
236 bn=$(basename $show .ctm) 247 bn=$(basename $show .ctm)
237 # Remove words with low confidence and keep useful tagger words 248 # Remove words with low confidence and keep useful tagger words
238 cat $show | $SCRIPT_PATH/KeepConfZone.pl | grep -e "MOTINC\|NMS\|NMP\|NFS\|NFP\|X[A-Z]{3,5}" | cut -f3 -d' ' > "$SHOW_DIR/$bn.confzone" 249 cat $show | $SCRIPT_PATH/KeepConfZone.pl | grep -e "MOTINC\|NMS\|NMP\|NFS\|NFP\|X[A-Z]{3,5}" | cut -f3 -d' ' > "$SHOW_DIR/$bn.confzone"
239 # Get date 2 day before and after the show 250 # Get date 2 day before and after the show
240 datePattern=`$SCRIPT_PATH/daybefore2after.sh $(echo $BASENAME | cut -c1-6)` 251 datePattern=`$SCRIPT_PATH/daybefore2after.sh $(echo $BASENAME | cut -c1-6)`
241 # Create SOLR queries 252 # Create SOLR queries
242 cat $SHOW_DIR/$bn".confzone" | $SCRIPT_PATH/GenerateSOLRQueries.pl | iconv -f ISO_8859-1 -t UTF-8 > "$SHOW_DIR/$bn.queries" 253 cat $SHOW_DIR/$bn".confzone" | $SCRIPT_PATH/GenerateSOLRQueries.pl | iconv -f ISO_8859-1 -t UTF-8 > "$SHOW_DIR/$bn.queries"
243 # Ask SOLR DB 254 # Ask SOLR DB
244 if [ $(wc -w "$SHOW_DIR/$bn.queries" | cut -f1 -d' ') -gt 0 ]; then 255 if [ $(wc -w "$SHOW_DIR/$bn.queries" | cut -f1 -d' ') -gt 0 ]; then
245 query=$(cat $SHOW_DIR/$bn.queries)"&fq=docDate:[$datePattern]" 256 query=$(cat $SHOW_DIR/$bn.queries)"&fq=docDate:[$datePattern]"
246 echo $query > $SHOW_DIR/$bn.queries 257 echo $query > $SHOW_DIR/$bn.queries
258 prnt_info "python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp" 3
247 python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp 259 python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp
248 cat $SOLR_RES/$bn.keywords.tmp | sort -u > $SOLR_RES/$bn.keywords 260 cat $SOLR_RES/$bn.keywords.tmp | sort -u > $SOLR_RES/$bn.keywords
249 cat $SOLR_RES/$bn.txt.tmp | sort -u > $SOLR_RES/$bn.txt 261 cat $SOLR_RES/$bn.txt.tmp | sort -u > $SOLR_RES/$bn.txt
250 rm $SOLR_RES/*.tmp > /dev/null 2>&1 262 rm $SOLR_RES/*.tmp > /dev/null 2>&1
251 fi 263 fi
252 264
253 if [ $CHECK -eq 1 ] 265 if [ $CHECK -eq 1 ]
254 then 266 then
255 if [ ! -e $SOLR_RES/$bn.keywords ] || [ ! -e $SOLR_RES/$bn.txt ] 267 if [ ! -e $SOLR_RES/$bn.keywords ] || [ ! -e $SOLR_RES/$bn.txt ]
256 then 268 then
257 print_warn "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 1 269 print_warn "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 2
270 print_log_file "$LOGFILE" "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !"
258 fi 271 fi
259 fi 272 fi
260 273
261 done 274 done
262 275
263 #----------------------------------------------------------------------------------------------- 276 #-----------------------------------------------------------------------------------------------
264 # Build trigger file 277 # Build trigger file
265 # 1) keywords are automatically boosted in the non confident zone of the current res 278 # 1) keywords are automatically boosted in the non confident zone of the current res
266 # confident zone are boosted 279 # confident zone are boosted
267 # previous words in sensible zone are penalized 280 # previous words in sensible zone are penalized
268 # 2) OOVs are extracted + phonetized 281 # 2) OOVs are extracted + phonetized
269 # 3) Try to find OOVs acousticly in the current segment 282 # 3) Try to find OOVs acousticly in the current segment
270 # 4) Generate the .trigg file 283 # 4) Generate the .trigg file
271 #------------------------------------------------------------------------------------------------ 284 #------------------------------------------------------------------------------------------------
272 print_info "[${BASENAME}] Build trigger files" 1 285 print_info "[${BASENAME}] Build trigger files" 1
273 for i in `ls $SOLR_RES/*.keywords` 286 for i in `ls $SOLR_RES/*.keywords`
274 do 287 do
275 basename=`basename $i .keywords` 288 basename=`basename $i .keywords`
276 289
277 # 290 #
278 # Tokenize & produce coverage report 291 # Tokenize & produce coverage report
279 # Use filter you need 292 # Use filter you need
280 # 293 #
281 print_info "[${BASENAME}] keywords filtering and produce coverage report" 2 294 print_info "[${BASENAME}] keywords filtering and produce coverage report" 3
282 # Default filter 295 # Default filter
283 cat $i | $SCRIPT_PATH/CleanFilter.sh | ${SCRIPT_PATH}/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t |\ 296 cat $i | $SCRIPT_PATH/CleanFilter.sh | ${SCRIPT_PATH}/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t |\
284 $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok 297 $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok
285 # do less filter 298 # do less filter
286 #cat $i | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex | $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok 299 #cat $i | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex | $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok
287 300
288 301
289 # 302 #
290 # Extract "real" OOV and phonetize them 303 # Extract "real" OOV and phonetize them
291 # -> petit filtrage persoo pour eviter d'avoir trop de bruits 304 # -> petit filtrage persoo pour eviter d'avoir trop de bruits
292 # 305 #
293 print_info "[${BASENAME}] Extract OOV and phonetize them" 2 306 print_info "[${BASENAME}] Extract OOV and phonetize them" 3
294 ${SCRIPT_PATH}/FindNormRules.pl $SOLR_RES/${basename}_tmp_report/report.oov $LEXICON.bdlex_tok | cut -f3 | grep -v "#" | grep -v "^[A-Z]\+$" | grep -v "^[0-9]" | grep --perl-regex -v "^([a-z']){1,3}$" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | iconv -t ISO_8859-1 -f UTF-8 | ${LIA_LTBOX}/lia_phon/script/lia_lex2phon_variante | grep -v "core dumped" | cut -d"[" -f1 | sort -u | ${SCRIPT_PATH}/PhonFormatter.pl | iconv -f ISO_8859-1 -t UTF-8 | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $SOLR_RES/${basename}.phon_oov 307 ${SCRIPT_PATH}/FindNormRules.pl $SOLR_RES/${basename}_tmp_report/report.oov $LEXICON.bdlex_tok | cut -f3 | grep -v "#" | grep -v "^[A-Z]\+$" | grep -v "^[0-9]" | grep --perl-regex -v "^([a-z']){1,3}$" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | iconv -t ISO_8859-1 -f UTF-8 | ${LIA_LTBOX}/lia_phon/script/lia_lex2phon_variante | grep -v "core dumped" | cut -d"[" -f1 | sort -u | ${SCRIPT_PATH}/PhonFormatter.pl | iconv -f ISO_8859-1 -t UTF-8 | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $SOLR_RES/${basename}.phon_oov
295 308
296 # 309 #
297 # Search INVOC & OOV in the current lattice 310 # Search INVOC & OOV in the current lattice
298 # 311 #
299 print_info "[${BASENAME}] Search INVOC and OOV in the current lattice" 2 312 print_info "[${BASENAME}] Search INVOC and OOV in the current lattice" 3
300 cat $SOLR_RES/${basename}_tmp_report/report.invoc | grep -v "\b0" | cut -f1 | grep -v --perl-regex -v "^[a-zA-Z']{1,3}$" | grep -v --perl-regex "^[a-zA-Z0-9]{1,3}$" | grep -v "<s>" | grep -v "</s>" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $TRIGGER_CONFZONE/$basename.tosearch 313 cat $SOLR_RES/${basename}_tmp_report/report.invoc | grep -v "\b0" | cut -f1 | grep -v --perl-regex -v "^[a-zA-Z']{1,3}$" | grep -v --perl-regex "^[a-zA-Z0-9]{1,3}$" | grep -v "<s>" | grep -v "</s>" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $TRIGGER_CONFZONE/$basename.tosearch
301 cat $SOLR_RES/${basename}.phon_oov | cut -f1 >> $TRIGGER_CONFZONE/$basename.tosearch 314 cat $SOLR_RES/${basename}.phon_oov | cut -f1 >> $TRIGGER_CONFZONE/$basename.tosearch
302 315
303 # For each treil 316 # For each treil
304 for baseseg in $(cat "$SHOW_DIR/$basename.lst") 317 for baseseg in $(cat "$SHOW_DIR/$basename.lst")
305 do 318 do
306 $OTMEDIA_HOME/tools/QUOTE_FINDER/bin/acousticFinder ${LEXICON}.speer_phon $RES_CONF/wlat/$baseseg.wlat $TRIGGER_CONFZONE/${basename}.tosearch $SOLR_RES/$basename.phon_oov > $TRIGGER_CONFZONE/$baseseg.acousticlyfound $OUTPUT_REDIRECTION 319 $OTMEDIA_HOME/tools/QUOTE_FINDER/bin/acousticFinder ${LEXICON}.speer_phon $RES_CONF/wlat/$baseseg.wlat $TRIGGER_CONFZONE/${basename}.tosearch $SOLR_RES/$basename.phon_oov > $TRIGGER_CONFZONE/$baseseg.acousticlyfound $OUTPUT_REDIRECTION
307 # 320 #
308 # Produce the boost file for the next decoding pass 321 # Produce the boost file for the next decoding pass
309 # 322 #
310 print_info "[${BASENAME}] Produce trigg file : $baseseg " 3 323 print_info "[${BASENAME}] Produce trigg file : $baseseg " 3
311 cat $RES_CONF_DIR/$baseseg.res | $SCRIPT_PATH/ScoreCtm2trigg.pl $TRIGGER_CONFZONE/$baseseg.acousticlyfound > $TRIGGER_CONFZONE/$baseseg.trigg 324 cat $RES_CONF_DIR/$baseseg.res | $SCRIPT_PATH/ScoreCtm2trigg.pl $TRIGGER_CONFZONE/$baseseg.acousticlyfound > $TRIGGER_CONFZONE/$baseseg.trigg
312 done 325 done
313 326
314 done 327 done
315 328
316 #----------------------------------------------------------------------------------------------- 329 #-----------------------------------------------------------------------------------------------
317 # Build the extended SPEERAL Lexicon 330 # Build the extended SPEERAL Lexicon
318 # 1) Merge OOVs + LEXICON 331 # 1) Merge OOVs + LEXICON
319 # 1) Related text are collected in order to find the invoc word with maximizing the ppl (LM proba) 332 # 1) Related text are collected in order to find the invoc word with maximizing the ppl (LM proba)
320 # 2) The current lexicon is extended with all the valid OOVs 333 # 2) The current lexicon is extended with all the valid OOVs
321 #----------------------------------------------------------------------------------------------- 334 #-----------------------------------------------------------------------------------------------
322 print_info "[${BASENAME}] Build extended Speeral Lexicon" 1 335 print_info "[${BASENAME}] Build extended Speeral Lexicon" 1
323 mkdir -p $EXT_LEX/final 336 mkdir -p $EXT_LEX/final
324 mkdir -p $EXT_LEX/tmp 337 mkdir -p $EXT_LEX/tmp
325 mkdir -p $EXT_LEX/tmp/txt 338 mkdir -p $EXT_LEX/tmp/txt
326 # 339 #
327 # Collect the acousticly found oov and their phonetisation 340 # Collect the acousticly found oov and their phonetisation
328 # 341 #
329 print_info "[${BASENAME}] Get all OOV and retrieve all phonetisation" 2 342 print_info "[${BASENAME}] Get all OOV and retrieve all phonetisation" 3
330 for i in `ls $SOLR_RES/*.phon_oov` 343 for i in `ls $SOLR_RES/*.phon_oov`
331 do 344 do
332 basename=`basename $i .phon_oov` 345 basename=`basename $i .phon_oov`
333 346
334 rm $EXT_LEX/$basename.acousticlyfound 2> /dev/null 347 rm $EXT_LEX/$basename.acousticlyfound 2> /dev/null
335 # list acousticly found for the show 348 # list acousticly found for the show
336 for baseseg in $(cat "$SHOW_DIR/$basename.lst") 349 for baseseg in $(cat "$SHOW_DIR/$basename.lst")
337 do 350 do
338 cat $TRIGGER_CONFZONE/$baseseg.acousticlyfound | cut -f1 | cut -f2 -d"=" >> $EXT_LEX/$basename.acousticlyfound 351 cat $TRIGGER_CONFZONE/$baseseg.acousticlyfound | cut -f1 | cut -f2 -d"=" >> $EXT_LEX/$basename.acousticlyfound
339 done 352 done
340 cat $EXT_LEX/$basename.acousticlyfound | sort -u > $EXT_LEX/.tmp 353 cat $EXT_LEX/$basename.acousticlyfound | sort -u > $EXT_LEX/.tmp
341 mv $EXT_LEX/.tmp $EXT_LEX/$basename.acousticlyfound 354 mv $EXT_LEX/.tmp $EXT_LEX/$basename.acousticlyfound
342 355
343 # 356 #
344 # Extract OOV really added 357 # Extract OOV really added
345 # 358 #
346 cat $SOLR_RES/$basename.phon_oov | cut -f1 | sort -u > $EXT_LEX/$basename.oov 359 cat $SOLR_RES/$basename.phon_oov | cut -f1 | sort -u > $EXT_LEX/$basename.oov
347 $SCRIPT_PATH/intersec.pl $EXT_LEX/$basename.oov $EXT_LEX/$basename.acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound 360 $SCRIPT_PATH/intersec.pl $EXT_LEX/$basename.oov $EXT_LEX/$basename.acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound
348 # 361 #
349 # Retrieve all phonetisation 362 # Retrieve all phonetisation
350 # 363 #
351 cat $SOLR_RES/${basename}.phon_oov | $SCRIPT_PATH/LexPhonFilter.pl $EXT_LEX/$basename.oov_acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound_phon 364 cat $SOLR_RES/${basename}.phon_oov | $SCRIPT_PATH/LexPhonFilter.pl $EXT_LEX/$basename.oov_acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound_phon
352 done 365 done
353 366
354 # 367 #
355 # Merge OOVs and their phonetisation 368 # Merge OOVs and their phonetisation
356 # 369 #
357 print_info "[${BASENAME}] Merge OOV and their phonetisation" 2 370 print_info "[${BASENAME}] Merge OOV and their phonetisation" 3
358 lexname=$(basename $LEXICON) 371 lexname=$(basename $LEXICON)
359 cat $EXT_LEX/*.oov_acousticlyfound_phon | sort -u > $EXT_LEX/final/all.oov_acousticlyfound_phon 372 cat $EXT_LEX/*.oov_acousticlyfound_phon | sort -u > $EXT_LEX/final/all.oov_acousticlyfound_phon
360 cat $EXT_LEX/*.oov_acousticlyfound | sort -u | grep --perl-regex -v "^([a-z']){3}$" > $EXT_LEX/final/all.oov_acousticlyfound 373 cat $EXT_LEX/*.oov_acousticlyfound | sort -u | grep --perl-regex -v "^([a-z']){3}$" > $EXT_LEX/final/all.oov_acousticlyfound
361 $SCRIPT_PATH/MergeLexicon.pl $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/final/${lexname}_ext.phon 374 $SCRIPT_PATH/MergeLexicon.pl $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/final/${lexname}_ext.phon
362 375
363 # 376 #
364 # Collect + clean retrieved txt 377 # Collect + clean retrieved txt
365 # 378 #
366 print_info "[${BASENAME}] Collect and clean SOLR txt answers" 2 379 print_info "[${BASENAME}] Collect and clean SOLR txt answers" 2
367 # choose filter 380 # choose filter
368 # default 381 # default
369 cat $SOLR_RES/*.txt | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $EXT_LEX/final/all.bdlex_txt 382 cat $SOLR_RES/*.txt | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $EXT_LEX/final/all.bdlex_txt
370 # low filter 383 # low filter
371 #cat $SOLR_RES/*.txt | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex > $EXT_LEX/final/all.bdlex_txt 384 #cat $SOLR_RES/*.txt | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex > $EXT_LEX/final/all.bdlex_txt
372 385
373 # 386 #
374 # Construct the map file 387 # Construct the map file
375 # 388 #
376 # Notes: 389 # Notes:
377 # - Expected format : 390 # - Expected format :
378 # <WORD1_STRING> <CANDIDATE1_STRING> <PHON_1> 391 # <WORD1_STRING> <CANDIDATE1_STRING> <PHON_1>
379 # 392 #
380 print_info "[${BASENAME}] Construct map file" 2 393 print_info "[${BASENAME}] Construct map file" 3
381 rm -f $EXT_LEX/final/${lexname}_ext.map 2>/dev/null 394 rm -f $EXT_LEX/final/${lexname}_ext.map 2>/dev/null
382 rm -f $EXT_LEX/final/${lexname}.unvalid_oov 2>/dev/null 395 rm -f $EXT_LEX/final/${lexname}.unvalid_oov 2>/dev/null
383 396
384 while read oov 397 while read oov
385 do 398 do
386 oov=`echo $oov | sed "s/\n//g"` 399 oov=`echo $oov | sed "s/\n//g"`
387 # 400 #
388 # Obtain the oov's tag 401 # Obtain the oov's tag
389 # 402 #
390 #oov_tag=`grep --perl-regex "^$oov\t" $DYNAMIC_TAGSTATS/all.tags | cut -f2` 403 #oov_tag=`grep --perl-regex "^$oov\t" $DYNAMIC_TAGSTATS/all.tags | cut -f2`
391 # 404 #
392 # Try to collect text containing the oov word 405 # Try to collect text containing the oov word
393 # 406 #
407 print_info "[${BASENAME}] Collect text containing the oov" 3
394 cat $EXT_LEX/final/all.bdlex_txt | grep --perl-regex " $oov " | $SCRIPT_PATH/NbMaxWordsFilter.pl 40 |uniq > $EXT_LEX/tmp/txt/$oov.bdlex_txt 408 cat $EXT_LEX/final/all.bdlex_txt | grep --perl-regex " $oov " | $SCRIPT_PATH/NbMaxWordsFilter.pl 40 |uniq > $EXT_LEX/tmp/txt/$oov.bdlex_txt
395 if [ -f $EXT_LEX/tmp/txt/$oov.bdlex_txt ]; then 409 if [ -f $EXT_LEX/tmp/txt/$oov.bdlex_txt ]; then
396 nbWords=`wc -l $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` 410 nbWords=`wc -l $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "`
397 if [ $nbWords -eq 0 ]; then 411 if [ $nbWords -eq 0 ]; then
398 echo "[${BASENAME}] UNVALID OOV: $oov => $nbWords occurrences" 412 print_warn "[${BASENAME}] UNVALID OOV: $oov => $nbWords occurrences" 2
399 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov 413 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov
400 else 414 else
401 # 415 #
402 # Find a candidate in a filtred invoc lexicon => a candidate which maximize the ppl in the overall txt collected 416 # Find a candidate in a filtred invoc lexicon => a candidate which maximize the ppl in the overall txt collected
403 # 417 #
404 #echo "$/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $LEXICON.bdlex_tok $EXT_LEX/tmp/txt/$oov.bdlex_txt" 418 #echo "$/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $LEXICON.bdlex_tok $EXT_LEX/tmp/txt/$oov.bdlex_txt"
419 print_info `$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` 3
405 candidate=`$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` 420 candidate=`$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "`
406 if [ ! x$candidate = "x" ]; then 421 if [ ! "$candidate" == "" ]; then
407 grep --perl-regex "^$oov\t" $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/tmp/$oov.phon 422 grep --perl-regex "^$oov\t" $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/tmp/$oov.phon
408 while read phonLine 423 while read phonLine
409 do 424 do
410 #<word> <phon> => <word> <candidate> <phon> 425 #<word> <phon> => <word> <candidate> <phon>
411 echo "$phonLine" | sed "s|\t|\t$candidate\t|" >> $EXT_LEX/final/${lexname}_ext.map 426 echo "$phonLine" | sed "s|\t|\t$candidate\t|" >> $EXT_LEX/final/${lexname}_ext.map
412 done < $EXT_LEX/tmp/$oov.phon 427 done < $EXT_LEX/tmp/$oov.phon
413 else 428 else
414 echo "[${BASENAME}] UNVALID OOV: $oov => no availaible Candidate word in LM" 429 print_warn "[${BASENAME}] UNVALID OOV: $oov => no availaible Candidate word in LM" 2
415 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov 430 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov
416 fi 431 fi
417 fi 432 fi
418 else 433 else
419 echo "[${BASENAME}] UNVALID OOV: $oov" 434 print_warn "[${BASENAME}] UNVALID OOV: $oov" 2
420 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov 435 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov
421 fi 436 fi
422 done < $EXT_LEX/final/all.oov_acousticlyfound 437 done < $EXT_LEX/final/all.oov_acousticlyfound
423 438
424 # 439 #
425 ### Speeral 440 ### Speeral
426 # 441 #
427 442
428 lexname=`basename $LEXICON` 443 lexname=`basename $LEXICON`
429 # 444 #
430 # Build the final trigger file 445 # Build the final trigger file
431 # 446 #
432 print_info "[${BASENAME}] Clean trigg files" 2 447 print_info "[${BASENAME}] Clean trigg files" 3
433 mkdir -p $TRIGGER_CONFZONE/speeral/ 2> /dev/null 448 mkdir -p $TRIGGER_CONFZONE/speeral/ 2> /dev/null
434 mkdir -p $EXT_LEX/speeral/ 2> /dev/null 449 mkdir -p $EXT_LEX/speeral/ 2> /dev/null
435 for i in `ls $TRIGGER_CONFZONE/*.trigg` 450 for i in `ls $TRIGGER_CONFZONE/*.trigg`
436 do 451 do
437 basename=`basename $i .trigg` 452 basename=`basename $i .trigg`
438 cat $i | $SCRIPT_PATH/RemoveLineContaining.pl $EXT_LEX/$lexname.unvalid_oov > $TRIGGER_CONFZONE/speeral/$basename.trigg 453 cat $i | $SCRIPT_PATH/RemoveLineContaining.pl $EXT_LEX/$lexname.unvalid_oov > $TRIGGER_CONFZONE/speeral/$basename.trigg
439 done 454 done
440 # 455 #
441 # Compile the speeral extended lexicon 456 # Compile the speeral extended lexicon
442 # 457 #
443 print_info "[${BASENAME}] Compile Speeral extended lexicon" 2 458 print_info "[${BASENAME}] Compile Speeral extended lexicon" 3
459 print_info "$SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext" 3
444 $SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext 460 $SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext
445 461
446 if [ $CHECK -eq 1 ] 462 if [ $CHECK -eq 1 ]
447 then 463 then
448 check_exploitconfpass_lex_check "${EXT_LEX}/speeral/${lexname}_ext" 464 check_exploitconfpass_lex_check "${EXT_LEX}/speeral/${lexname}_ext"
449 if [ $? -eq 1 ] 465 if [ $? -eq 1 ]
450 then 466 then
451 echo -e "[${BASENAME}] ERROR : Building Speeral Lexicon $INPUT_DIR " >> $ERRORFILE 467 print_error "[${BASENAME}] Building Speeral Lexicon $INPUT_DIR -> exit"
468 print_error "[${BASENAME}] Check $ERRORFILE"
469 print_log_file $ERRORFILE "ERROR : Building Speeral Lexicon $INPUT_DIR"
470 print_log_file $ERRORFILE "ERROR : ${EXT_LEX}/speeral/${lexname}_ext Empty after buildmappedbinode ?"
452 exit 1; 471 exit 1;
453 fi 472 fi
454 fi 473 fi
455 474
456 475
457 #-------# 476 #-------#
458 # CLOSE # 477 # CLOSE #
459 #-------# 478 #-------#
460 # Seem OK 479 # Seem OK
461 print_info "[${BASENAME}] <= End $BASENAME Solr | $(date +'%d/%m/%y %H:%M:%S')" 1 480 print_info "[${BASENAME}] <= ExploitConfidencePass End | $(date +'%d/%m/%y %H:%M:%S')" 1
462 echo -e "[${BASENAME}] #Solr $BASENAME " >> $LOGFILE
463 481
464 # unlock directory 482 # unlok directory
465 mv "$OUTPUT_DIR/EXPLOITCONFPASS.lock" "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" 483 mv "$OUTPUT_DIR/EXPLOITCONFPASS.lock" "$OUTPUT_DIR/EXPLOITCONFPASS.unlock"
466 484
467 485
main_tools/FirstPass.sh
Diff comments   View file @ b427f10
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : FirstPass.sh # 4 # File : FirstPass.sh #
5 # Brief : ASR first pass and speaker diarization # 5 # Brief : ASR first pass and speaker diarization #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # (base on Emmanuel Ferreira # 7 # (base on Emmanuel Ferreira #
8 # and Hugo Mauchrétien works) # 8 # and Hugo Mauchrétien works) #
9 # Version : 1.1 # 9 # Version : 1.1 #
10 # Date : 18/06/13 # 10 # Date : 18/06/13 #
11 ##################################################### 11 #####################################################
12 12
13 echo "### FirstPass.sh ###" 13 echo "### FirstPass.sh ###"
14 14
15 # Check OTMEDIA_HOME env var 15 # Check OTMEDIA_HOME env var
16 if [ -z ${OTMEDIA_HOME} ] 16 if [ -z ${OTMEDIA_HOME} ]
17 then 17 then
18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
19 export OTMEDIA_HOME=$OTMEDIA_HOME 19 export OTMEDIA_HOME=$OTMEDIA_HOME
20 fi 20 fi
21 21
22 # where is FirstPass.sh 22 # where is FirstPass.sh
23 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 23 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
24 24
25 # scripts path 25 # scripts path
26 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 26 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
27 27
28 # Include scripts 28 # Include scripts
29 . $SCRIPT_PATH"/Tools.sh" 29 . $SCRIPT_PATH"/Tools.sh"
30 . $SCRIPT_PATH"/CheckFirstPass.sh" 30 . $SCRIPT_PATH"/CheckFirstPass.sh"
31 31
32 # where is FirstPass.cfg 32 # where is FirstPass.cfg
33 FIRSTPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/FirstPass.cfg" 33 FIRSTPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/FirstPass.cfg"
34 if [ -e $FIRSTPASS_CONFIG_FILE ] 34 if [ -e $FIRSTPASS_CONFIG_FILE ]
35 then 35 then
36 . $FIRSTPASS_CONFIG_FILE 36 . $FIRSTPASS_CONFIG_FILE
37 else 37 else
38 echo "ERROR : Can't find configuration file $FIRSTPASS_CONFIG_FILE" > /dev/stderr 38 echo "ERROR : Can't find configuration file $FIRSTPASS_CONFIG_FILE" > /dev/stderr
39 echo "exit" > /dev/stderr 39 echo "exit" > /dev/stderr
40 exit 1 40 exit 1
41 fi 41 fi
42 42
43 #---------------# 43 #---------------#
44 # Parse Options # 44 # Parse Options #
45 #---------------# 45 #---------------#
46 while getopts ":hDv:cf:r" opt 46 while getopts ":hDv:cf:r" opt
47 do 47 do
48 case $opt in 48 case $opt in
49 h) 49 h)
50 echo -e "$0 [OPTIONS] <WAV_FILE> <OUTPUT_DIRECTORY>\n" 50 echo -e "$0 [OPTIONS] <WAV_FILE> <OUTPUT_DIRECTORY>\n"
51 echo -e "\t Options:" 51 echo -e "\t Options:"
52 echo -e "\t\t-h :\tprint this message" 52 echo -e "\t\t-h :\tprint this message"
53 echo -e "\t\t-D :\tDEBUG mode on" 53 echo -e "\t\t-D :\tDEBUG mode on"
54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
55 echo -e "\t\t-c :\tCheck process, and log it into files, can stop if error detected" 55 echo -e "\t\t-c :\tCheck process, and log it into files, can stop if error detected"
56 echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)" 56 echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)"
57 echo -e "\t\t-r :\tforce rerun the wav file" 57 echo -e "\t\t-r :\tforce rerun the wav file"
58 exit 1 58 exit 1
59 ;; 59 ;;
60 D) 60 D)
61 DEBUG=1 61 DEBUG=1
62 ;; 62 ;;
63 v) 63 v)
64 VERBOSE=$OPTARG 64 VERBOSE=$OPTARG
65 ;; 65 ;;
66 c) 66 c)
67 CHECK=1 67 CHECK=1
68 ;; 68 ;;
69 f) 69 f)
70 FORKS="--forks $OPTARG" 70 FORKS="--forks $OPTARG"
71 ;; 71 ;;
72 r) 72 r)
73 RERUN=1 73 RERUN=1
74 ;; 74 ;;
75 :) 75 :)
76 echo "Option -$OPTARG requires an argument." > /dev/stderr 76 echo "Option -$OPTARG requires an argument." > /dev/stderr
77 exit 1 77 exit 1
78 ;; 78 ;;
79 \?) 79 \?)
80 echo "BAD USAGE : unknow opton -$OPTARG" 80 echo "BAD USAGE : unknow opton -$OPTARG"
81 exit 1 81 exit 1
82 ;; 82 ;;
83 esac 83 esac
84 done 84 done
85 85
86 # mode debug enable 86 # mode debug enable
87 if [ $DEBUG -eq 1 ] 87 if [ $DEBUG -eq 1 ]
88 then 88 then
89 set -x 89 set -x
90 echo -e "## Mode DEBUG ON ##" 90 echo -e "## Mode DEBUG ON ##"
91 fi 91 fi
92 92
93 # mode verbose enable 93 # mode verbose enable
94 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi 94 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
95 95
96 # Check USAGE by arguments number 96 # Check USAGE by arguments number
97 if [ $(($#-($OPTIND-1))) -ne 2 ] 97 if [ $(($#-($OPTIND-1))) -ne 2 ]
98 then 98 then
99 echo "BAD USAGE : FirstPass.sh [OPTIONS] <WAV_FILE> <OUTPUT_DIR>" 99 echo "BAD USAGE : FirstPass.sh [OPTIONS] <WAV_FILE> <OUTPUT_DIR>"
100 echo "$0 -h for more info" 100 echo "$0 -h for more info"
101 exit 1 101 exit 1
102 fi 102 fi
103 103
104 shift $((OPTIND-1)) 104 shift $((OPTIND-1))
105 # check audio file - First argument 105 # check audio file - First argument
106 if [ -e $1 ] && [ -s $1 ] 106 if [ -e $1 ] && [ -s $1 ]
107 then 107 then
108 # absolute path to wav file 108 # absolute path to wav file
109 WAV_FILE=$(readlink -e $1) 109 WAV_FILE=$(readlink -e $1)
110 # wav filename 110 # wav filename
111 FILENAME=$(basename $WAV_FILE) 111 FILENAME=$(basename $WAV_FILE)
112 # wav filename without extension 112 # wav filename without extension
113 BASENAME=${FILENAME%.*} 113 BASENAME=${FILENAME%.*}
114 114
115 print_info "[${BASENAME}] => P1 start | $(date +'%d/%m/%y %H:%M:%S')" 1 115 print_info "[${BASENAME}] => P1 start | $(date +'%d/%m/%y %H:%M:%S')" 1
116 print_info "[${BASENAME}] $WAV_FILE OK" 2 116 print_info "[${BASENAME}] $WAV_FILE OK" 2
117 else 117 else
118 print_error "can't find $1 OR file is empty" 118 print_error " Can't find $1 OR file is empty"
119 exit 1 119 exit 1
120 fi 120 fi
121 121
122 # check output directory - Second argument 122 # check output directory - Second argument
123 if [ ! -e $2 ] 123 if [ ! -e $2 ]
124 then 124 then
125 mkdir -p $2 125 mkdir -p $2
126 print_info "[${BASENAME}] Make directory $2" 2 126 print_info "[${BASENAME}] Make directory $2" 2
127 fi 127 fi
128 128
129 129
130 #-------------# 130 #-------------#
131 # GLOBAL VARS # 131 # GLOBAL VARS #
132 #-------------# 132 #-------------#
133 OUTPUT_DIR=$(readlink -e $2) # Output directory absolute path 133 OUTPUT_DIR=$(readlink -e $2) # Output directory absolute path
134 OUTPUT_DIR_BASENAME="$OUTPUT_DIR/$BASENAME/" # New OUTPUT_DIR with BASENAME 134 OUTPUT_DIR_BASENAME="$OUTPUT_DIR/$BASENAME/" # New OUTPUT_DIR with BASENAME
135 PLP_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.plp" # Global PLP file 135 PLP_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.plp" # Global PLP file
136 PLP_DIR="$OUTPUT_DIR_BASENAME/PLP/" # Segmented PLP files directory 136 PLP_DIR="$OUTPUT_DIR_BASENAME/PLP/" # Segmented PLP files directory
137 SEG_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.seg" # Global Seg file 137 SEG_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.seg" # Global Seg file
138 LBL_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.lbl" # Global LBL file 138 LBL_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.lbl" # Global LBL file
139 RES_DIR=$OUTPUT_DIR_BASENAME"/res_p1" 139 RES_DIR=$OUTPUT_DIR_BASENAME"/res_p1"
140 LOGFILE="$OUTPUT_DIR_BASENAME/info_p1.log" 140 LOGFILE="$OUTPUT_DIR_BASENAME/info_p1.log"
141 ERRORFILE="$OUTPUT_DIR_BASENAME/error_p1.log" 141 ERRORFILE="$OUTPUT_DIR_BASENAME/error_p1.log"
142 142
143 #------------------# 143 #------------------#
144 # Create WORKSPACE # 144 # Create WORKSPACE #
145 #------------------# 145 #------------------#
146 if [ ! -e $OUTPUT_DIR_BASENAME ] 146 if [ ! -e $OUTPUT_DIR_BASENAME ]
147 then 147 then
148 mkdir -p $OUTPUT_DIR_BASENAME 148 mkdir -p $OUTPUT_DIR_BASENAME
149 print_info "[${BASENAME}] Make directory $OUTPUT_DIR_BASENAME" 2 149 print_info "[${BASENAME}] Make directory $OUTPUT_DIR_BASENAME" 2
150 fi 150 fi
151 151
152 # Lock directory 152 # Lock directory
153 if [ -e $OUTPUT_DIR_BASENAME/FIRSTPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1; fi 153 if [ -e $OUTPUT_DIR_BASENAME/FIRSTPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1; fi
154 rm "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" > /dev/null 2>&1 154 rm "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" > /dev/null 2>&1
155 touch "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" > /dev/null 2>&1 155 touch "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" > /dev/null 2>&1
156 156
157 rm -r $PLP_DIR > /dev/null 2>&1; 157 rm -r $PLP_DIR > /dev/null 2>&1;
158 mkdir -p $PLP_DIR 158 mkdir -p $PLP_DIR
159 print_info "[${BASENAME}] Make directory $PLP_DIR" 2 159 print_info "[${BASENAME}] Make directory $PLP_DIR" 2
160 if [ $RERUN -eq 0 ]; 160 if [ $RERUN -eq 0 ];
161 then 161 then
162 rm -r $RES_DIR > /dev/null 2>&1; 162 rm -r $RES_DIR > /dev/null 2>&1;
163 else 163 else
164 rm $RES_DIR/*.lock > /dev/null 2>&1 164 rm $RES_DIR/*.lock > /dev/null 2>&1
165 fi 165 fi
166 mkdir -p $RES_DIR > /dev/null 2>&1 166 mkdir -p $RES_DIR > /dev/null 2>&1
167 print_info "[${BASENAME}] Make directory $RES_DIR" 2 167 print_info "[${BASENAME}] Make directory $RES_DIR" 2
168 rm $LOGFILE $ERRORFILE > /dev/null 2>&1 168 rm $LOGFILE $ERRORFILE > /dev/null 2>&1
169 169
170 #--------------------# 170 #--------------------#
171 # Save configuration # 171 # Save configuration #
172 #--------------------# 172 #--------------------#
173 cp $FIRSTPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/FirstPass.cfg 173 cp $FIRSTPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/FirstPass.cfg
174 echo "FIRSTPASS_SCRIPT_PATH=$MAIN_SCRIPT_PATH" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 174 echo "FIRSTPASS_SCRIPT_PATH=$MAIN_SCRIPT_PATH" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
175 echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 175 echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
176 echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 176 echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
177 echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 177 echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
178 echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 178 echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
179 echo "PLP_FILE=$PLP_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 179 echo "PLP_FILE=$PLP_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
180 echo "PLP_DIR=$PLP_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 180 echo "PLP_DIR=$PLP_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
181 echo "SEG_FILE=$SEG_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 181 echo "SEG_FILE=$SEG_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
182 echo "LBL_FILE=$LBL_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 182 echo "LBL_FILE=$LBL_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
183 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg 183 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg
184 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/FirstPass.cfg" 1 184 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/FirstPass.cfg" 1
185 185
186 #-------------------------# 186 #-------------------------#
187 # Check Audio File Format # 187 # Check Audio File Format #
188 #-------------------------# 188 #-------------------------#
189 error=0 189 error=0
190 temp=$(avconv -i $WAV_FILE 2>&1 | grep "16000 Hz") 190 temp=$(avconv -i $WAV_FILE 2>&1 | grep "16000 Hz")
191 if [ -z "$temp" ]; then error=1; fi 191 if [ -z "$temp" ]; then error=1; fi
192 temp=$(avconv -i $WAV_FILE 2>&1 | grep "1 channels") 192 temp=$(avconv -i $WAV_FILE 2>&1 | grep "1 channels")
193 if [ -z "$temp" ]; then error=1; fi 193 if [ -z "$temp" ]; then error=1; fi
194 temp=$(avconv -i $WAV_FILE 2>&1 | grep "s16") 194 temp=$(avconv -i $WAV_FILE 2>&1 | grep "s16")
195 if [ -z "$temp" ]; then error=1; fi 195 if [ -z "$temp" ]; then error=1; fi
196 196
197 if [ $error -eq 1 ] 197 if [ $error -eq 1 ]
198 then 198 then
199 print_message $WARNING 2 "[${BASENAME}] $WAV_FILE is not a wav file at 16000 Hz, 1 channel, 16bits\nhave to convert" 199 print_message $WARNING 2 "[${BASENAME}] $WAV_FILE is not a wav file at 16000 Hz, 1 channel, 16bits\nhave to convert"
200 print_message $INFO 3 "[${BASENAME}] avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav" 200 print_message $INFO 3 "[${BASENAME}] avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav"
201 avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav 201 avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav
202 WAV_FILE=$OUTPUT_DIR_BASENAME/$BASENAME.wav 202 WAV_FILE=$OUTPUT_DIR_BASENAME/$BASENAME.wav
203 FILENAME=$BASENAME.wav 203 FILENAME=$BASENAME.wav
204 print_message $INFO 1 "[${BASENAME}] new wav file : $WAV_FILE" 204 print_message $INFO 1 "[${BASENAME}] new wav file : $WAV_FILE"
205 fi 205 fi
206 206
207 #---------------# 207 #---------------#
208 # Get SRT file # 208 # Get SRT file #
209 #---------------# 209 #---------------#
210 if [ -s $(dirname $WAV_FILE)/$BASENAME.SRT ] 210 if [ -s $(dirname $WAV_FILE)/$BASENAME.SRT ]
211 then 211 then
212 cp $(dirname $WAV_FILE)/$BASENAME.SRT $OUTPUT_DIR_BASENAME/$BASENAME.SRT 212 cp $(dirname $WAV_FILE)/$BASENAME.SRT $OUTPUT_DIR_BASENAME/$BASENAME.SRT
213 print_info "[${BASENAME}] copy $BASENAME.SRT file into $OUTPUT_DIR_BASENAME" 3 213 print_info "[${BASENAME}] copy $BASENAME.SRT file into $OUTPUT_DIR_BASENAME" 3
214 fi 214 fi
215 215
216 #------------# 216 #------------#
217 # WAV -> PLP # 217 # WAV -> PLP #
218 #------------# 218 #------------#
219 print_info "[${BASENAME}] convert WAV -> PLP" 1 219 print_info "[${BASENAME}] convert WAV -> PLP" 1
220 echo $FILENAME > $OUTPUT_DIR_BASENAME/list.tmp 220 echo $FILENAME > $OUTPUT_DIR_BASENAME/list.tmp
221 print_info "[${BASENAME}] $BIN_PATH/lia_plp_mt.32 --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms" 3 221 print_info "[${BASENAME}] $BIN_PATH/lia_plp_mt.32 --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms" 3
222 222
223 $BIN_PATH/lia_plp_mt$ARCH --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms 223 $BIN_PATH/lia_plp_mt$ARCH --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms
224 224
225 if [ $CHECK -eq 1 ] 225 if [ $CHECK -eq 1 ]
226 then 226 then
227 check_first_pass_plp "$PLP_FILE" 227 check_first_pass_plp "$PLP_FILE"
228 if [ $? -eq 1 ] 228 if [ $? -eq 1 ]
229 then 229 then
230 print_log_file "$ERROFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating PLP file : $PLP_FILE" 230 print_log_file "$ERROFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating PLP file : $PLP_FILE"
231 echo "[${BASENAME}] ERROR : ${BASENAME} check $ERRORFILE file" > /dev/stderr 231 echo "[${BASENAME}] ERROR : ${BASENAME} check $ERRORFILE file" > /dev/stderr
232 exit 1 232 exit 1
233 fi 233 fi
234 fi 234 fi
235 235
236 rm $OUTPUT_DIR_BASENAME/list.tmp 236 rm $OUTPUT_DIR_BASENAME/list.tmp
237 237
238 #------------------------------# 238 #------------------------------#
239 # S/NS + SPEAKERS SEGMENTATION # 239 # S/NS + SPEAKERS SEGMENTATION #
240 #------------------------------# 240 #------------------------------#
241 print_info "[${BASENAME}] Launch speakers diarization" 1 241 print_info "[${BASENAME}] Launch speakers diarization" 1
242 # Calcul seg file 242 # Calcul seg file
243 print_info "[${BASENAME}] java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME" 3 243 print_info "[${BASENAME}] java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME" 3
244 #java -Xmx8000m -Xms2048 -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME 244 #java -Xmx8000m -Xms2048 -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME
245 java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME #–doCEClustering 245 java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME #–doCEClustering
246 246
247 if [ $CHECK -eq 1 ] && ( [ ! -e $SEG_FILE ] || [ -z $SEG_FILE ] ) 247 if [ $CHECK -eq 1 ] && ( [ ! -e $SEG_FILE ] || [ -z $SEG_FILE ] )
248 then 248 then
249 print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating SEG file : $SEG_FILE" 249 print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating SEG file : $SEG_FILE"
250 print_error "[${BASENAME}] ERROR : check $ERRORFILE file" 250 print_error "[${BASENAME}] Check $ERRORFILE file"
251 exit 1 251 exit 1
252 fi 252 fi
253 253
254 254
255 # Create LBL file 255 # Create LBL file
256 print_info "Extract LBL file from SEG file" 1 256 print_info "Extract LBL file from SEG file" 1
257 257
258 cat $SEG_FILE | grep -v ";;" | cut -f3,4,5,8 -d" " | tr " " "#" | sort -k1 -n | tr "#" " " > $LBL_FILE 258 cat $SEG_FILE | grep -v ";;" | cut -f3,4,5,8 -d" " | tr " " "#" | sort -k1 -n | tr "#" " " > $LBL_FILE
259 259
260 if [ $CHECK -eq 1 ] && ( [ ! -e $LBL_FILE ] || [ -z $LBL_FILE ] ) 260 if [ $CHECK -eq 1 ] && ( [ ! -e $LBL_FILE ] || [ -z $LBL_FILE ] )
261 then 261 then
262 print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating LBL file : $LBL_FILE" 262 print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating LBL file : $LBL_FILE"
263 print_error "[${BASENAME}] ERROR : check $ERRORFILE file" 263 print_error "[${BASENAME}] Check $ERRORFILE file"
264 exit 1 264 exit 1
265 fi 265 fi
266 266
267 267
268 #----------------------------------------------------# 268 #----------------------------------------------------#
269 # Cut global PLP file depending to LBL segmentations # 269 # Cut global PLP file depending to LBL segmentations #
270 #----------------------------------------------------# 270 #----------------------------------------------------#
271 print_info "[${BASENAME}] Cut PLP file depending to LBL segmentations" 1 271 print_info "[${BASENAME}] Cut PLP file depending to LBL segmentations" 1
272 print_info "[${BASENAME}] $BIN_PATH/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG" 3 272 print_info "[${BASENAME}] $BIN_PATH/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG" 3
273 273
274 $SPEERAL_TOOLS/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG 274 $SPEERAL_TOOLS/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG
275 275
276 if [ $CHECK -eq 1 ] 276 if [ $CHECK -eq 1 ]
277 then 277 then
278 check_first_pass_plps_lbl $PLP_DIR $LBL_FILE 278 check_first_pass_plps_lbl $PLP_DIR $LBL_FILE
279 if [ $? -eq 1 ] 279 if [ $? -eq 1 ]
280 then 280 then
281 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $PLP wrong number of .plp files" 281 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $PLP wrong number of .plp files"
282 print_error "[${BASENAME}] ERROR : check $ERRORFILE file" 282 print_error "[${BASENAME}] Check $ERRORFILE file"
283 exit 1 283 exit 1
284 fi 284 fi
285 fi 285 fi
286 286
287 # change plp files names 287 # change plp files names
288 cd $PLP_DIR; 288 cd $PLP_DIR;
289 rename -f s/_/#/g *plp 289 rename -f s/_/#/g *plp
290 rename -f s/#/_/ *plp 290 rename -f s/#/_/ *plp
291 cd $OLDPWD 291 cd $OLDPWD
292 292
293 #---------------------------------------------# 293 #---------------------------------------------#
294 # PLP files list depending to acoustic models # 294 # PLP files list depending to acoustic models #
295 #---------------------------------------------# 295 #---------------------------------------------#
296 print_info "[${BASENAME}] Create PLP list depending of the model" 1 296 print_info "[${BASENAME}] Create PLP list depending of the model" 1
297 # Create a list of plp files 297 # Create a list of plp files
298 find $PLP_DIR -type f -exec basename "{}" .plp \; | sort > $OUTPUT_DIR_BASENAME/plp.lst 298 find $PLP_DIR -type f -exec basename "{}" .plp \; | sort > $OUTPUT_DIR_BASENAME/plp.lst
299 299
300 rm $OUTPUT_DIR_BASENAME/plp_*.lst > /dev/null 2>&1 300 rm $OUTPUT_DIR_BASENAME/plp_*.lst > /dev/null 2>&1
301 for (( i=0; $i<${#MTAG[@]} ; i++ )) 301 for (( i=0; $i<${#MTAG[@]} ; i++ ))
302 do 302 do
303 a=`grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst` 303 a=`grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst`
304 if [ -n "$a" ]; then 304 if [ -n "$a" ]; then
305 print_info "[${BASENAME}] Creating $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst" 3 305 print_info "[${BASENAME}] Creating $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst" 3
306 grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst | sort > $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst 306 grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst | sort > $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst
307 fi 307 fi
308 done 308 done
309 309
310 #-----------------------# 310 #-----------------------#
311 # First Pass (DECODING) # 311 # First Pass (DECODING) #
312 #-----------------------# 312 #-----------------------#
313 # 313 #
314 # For all AM do decoding 314 # For all AM do decoding
315 # if Check error -> iter on undone decoding (max 1 times) 315 # if Check error -> iter on undone decoding (max 1 times)
316 # 316 #
317 print_info "[${BASENAME}] Launch decoding" 1 317 print_info "[${BASENAME}] Launch decoding" 1
318 for (( i=0; $i<${#MTAG[@]} ; i++ )) 318 for (( i=0; $i<${#MTAG[@]} ; i++ ))
319 do 319 do
320 redo=1; # nb of try if not all segs is decoded 320 redo=1; # nb of try if not all segs is decoded
321 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ] 321 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]
322 then 322 then
323 todo=$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst 323 todo=$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst
324 while [ $redo -gt 0 ]; do 324 while [ $redo -gt 0 ]; do
325 rm $RES_DIR/*.lock > /dev/null 2>&1 325 rm $RES_DIR/*.lock > /dev/null 2>&1
326 print_info "[${BASENAME}] $SPEERAL_BIN $todo $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3 326 print_info "[${BASENAME}] $SPEERAL_BIN $todo $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3
327 # Run speeral 327 # Run speeral
328 $SPEERAL_BIN ${todo} $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock 328 $SPEERAL_BIN ${todo} $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock
329 329
330 # Check if error 330 # Check if error
331 if [ $CHECK -eq 1 ] 331 if [ $CHECK -eq 1 ]
332 then 332 then
333 check_first_pass_output_speeral "${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" "$RES_DIR" 333 check_first_pass_output_speeral "${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" "$RES_DIR"
334 # if error 334 # if error
335 if [ $? -eq 1 ] 335 if [ $? -eq 1 ]
336 then 336 then
337 # rerun 337 # rerun
338 redo=$(($redo - 1)); 338 redo=$(($redo - 1));
339 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 339 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2
340 print_log_file $LOGFILE "WARN : Speeral number of output ERROR ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" 340 print_log_file $LOGFILE "WARN : Speeral number of output ERROR ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst"
341 # new plp list 341 # new plp list
342 # list .seg done and compare to list of seg to do 342 # list .seg done and compare to list of seg to do
343 ls $RES_DIR/*.seg | grep -e "${MTAG[$i]}" | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 343 ls $RES_DIR/*.seg | grep -e "${MTAG[$i]}" | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
344 diff ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" > ${OUTPUT_DIR_BASENAME}/todo.lst 344 diff ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" > ${OUTPUT_DIR_BASENAME}/todo.lst
345 rm ${OUTPUT_DIR_BASENAME}/.tmp 345 rm ${OUTPUT_DIR_BASENAME}/.tmp
346 # log seg to do 346 # log seg to do
347 print_log_file $LOGFILE "Segs not done :" 347 print_log_file $LOGFILE "Segs not done :"
348 cat ${OUTPUT_DIR_BASENAME}/todo.lst >> $LOGFILE 348 cat ${OUTPUT_DIR_BASENAME}/todo.lst >> $LOGFILE
349 todo=${OUTPUT_DIR_BASENAME}/todo.lst 349 todo=${OUTPUT_DIR_BASENAME}/todo.lst
350 print_warn "[${BASENAME}] Try $redo" 3 350 print_warn "[${BASENAME}] Try $redo" 3
351 fi 351 fi
352 fi 352 fi
353 done 353 done
354 rm ${OUTPUT_DIR_BASENAME}/todo.lst > /dev/null 2>&1 354 rm ${OUTPUT_DIR_BASENAME}/todo.lst > /dev/null 2>&1
355 #rm $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst 355 #rm $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst
356 rm $RES_DIR/*.lock > /dev/null 2>&1 356 rm $RES_DIR/*.lock > /dev/null 2>&1
357 fi 357 fi
358 done 358 done
359 359
360 ## Check missing seg and log it 360 ## Check missing seg and log it
361 if [ $CHECK -eq 1 ] 361 if [ $CHECK -eq 1 ]
362 then 362 then
363 ls $RES_DIR/*.seg | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 363 ls $RES_DIR/*.seg | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
364 todo=$(cat ${OUTPUT_DIR_BASENAME}/plp.lst | wc -l) 364 todo=$(cat ${OUTPUT_DIR_BASENAME}/plp.lst | wc -l)
365 if [ $todo -eq 0 ]; then todo=1;fi 365 if [ $todo -eq 0 ]; then todo=1;fi
366 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) 366 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l)))
367 pourcentage=$((($notdone*100)/$todo)) 367 pourcentage=$((($notdone*100)/$todo))
368 368
369 if [ $notdone -ne 0 ] 369 if [ $notdone -ne 0 ]
370 then 370 then
371 print_error "[${BASENAME}] ERROR : check $ERRORFILE" 371 print_error "[${BASENAME}] Check $ERRORFILE"
372 print_log_file "$ERRORFILE" "ERROR : Segs not done [" 372 print_log_file "$ERRORFILE" "ERROR : Segs not done ["
373 diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE 373 diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE
374 print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" 374 print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME"
375 else 375 else
376 print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" 376 print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')"
377 fi 377 fi
378 rm ${OUTPUT_DIR_BASENAME}/.tmp 378 rm ${OUTPUT_DIR_BASENAME}/.tmp
379 fi 379 fi
380 380
381 #---------------# 381 #---------------#
382 # Convert res # 382 # Convert res #
383 #---------------# 383 #---------------#
384 print_info "[${BASENAME}] Convert .res into .ctm" 1 384 print_info "[${BASENAME}] Convert .res into .ctm" 1
385 # .res => .ctm 385 # .res => .ctm
386 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.ctm 386 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.ctm
387 print_info "[${BASENAME}] Convert .res into .trs" 1 387 print_info "[${BASENAME}] Convert .res into .trs" 1
388 # .res => .trs 388 # .res => .trs
389 echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR_BASENAME/$BASENAME.seg" > $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 389 echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR_BASENAME/$BASENAME.seg" > $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg
390 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.trs --trs_config $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 390 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.trs --trs_config $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg
391 rm $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 2> /dev/null 391 rm $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 2> /dev/null
392 print_info "[${BASENAME}] Convert .res into .txt" 1 392 print_info "[${BASENAME}] Convert .res into .txt" 1
393 # .res => .txt 393 # .res => .txt
394 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.txt 394 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.txt
395 395
396 print_info "[${BASENAME}] <= P1 End | $(date +'%d/%m/%y %H:%M:%S')" 1 396 print_info "[${BASENAME}] <= P1 End | $(date +'%d/%m/%y %H:%M:%S')" 1
397 # unlock directory 397 # unlock directory
398 mv "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" 398 mv "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock"
399 399
main_tools/RecomposePass.sh
Diff comments   View file @ b427f10
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : RecomposePass.sh # 4 # File : RecomposePass.sh #
5 # Brief : Get missing res pass from previous pass # 5 # Brief : Get missing res pass from previous pass #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # Version : 1.0 # 7 # Version : 1.0 #
8 # Date : 25/07/13 # 8 # Date : 25/07/13 #
9 ##################################################### 9 #####################################################
10 10
11 echo "### RecomposePass.sh ###" 11 echo "### RecomposePass.sh ###"
12 12
13 13
14 # Check OTMEDIA_HOME env var 14 # Check OTMEDIA_HOME env var
15 if [ -z ${OTMEDIA_HOME} ] 15 if [ -z ${OTMEDIA_HOME} ]
16 then 16 then
17 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 17 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
18 export OTMEDIA_HOME=$OTMEDIA_HOME 18 export OTMEDIA_HOME=$OTMEDIA_HOME
19 fi 19 fi
20 20
21 # where is RecomposePass.sh 21 # where is RecomposePass.sh
22 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 22 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
23 23
24 # Scripts Path 24 # Scripts Path
25 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 25 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
26 26
27 # Include scripts 27 # Include scripts
28 . $SCRIPT_PATH"/Tools.sh" 28 . $SCRIPT_PATH"/Tools.sh"
29 . $SCRIPT_PATH"/CheckSecondPass.sh" 29 . $SCRIPT_PATH"/CheckSecondPass.sh"
30 30
31 # where is RescomposePass.cfg 31 # where is RescomposePass.cfg
32 RECOMPOSEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/RecomposePass.cfg" 32 RECOMPOSEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/RecomposePass.cfg"
33 if [ -e $RECOMPOSEPASS_CONFIG_FILE ] 33 if [ -e $RECOMPOSEPASS_CONFIG_FILE ]
34 then 34 then
35 . $RECOMPOSEPASS_CONFIG_FILE 35 . $RECOMPOSEPASS_CONFIG_FILE
36 else 36 else
37 echo "ERROR : Can't find configuration file $RECOMPOSEPASS_CONFIG_FILE" >&2 37 echo "ERROR : Can't find configuration file $RECOMPOSEPASS_CONFIG_FILE" >&2
38 exit 1 38 exit 1
39 fi 39 fi
40 40
41 #---------------# 41 #---------------#
42 # Parse Options # 42 # Parse Options #
43 #---------------# 43 #---------------#
44 while getopts ":hDv:" opt 44 while getopts ":hDv:" opt
45 do 45 do
46 case $opt in 46 case $opt in
47 h) 47 h)
48 echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" 48 echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n"
49 echo -e "\t Options:" 49 echo -e "\t Options:"
50 echo -e "\t\t-h :\tprint this message" 50 echo -e "\t\t-h :\tprint this message"
51 echo -e "\t\t-D :\tDEBUG mode on" 51 echo -e "\t\t-D :\tDEBUG mode on"
52 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 52 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
53 exit 1 53 exit 1
54 ;; 54 ;;
55 D) 55 D)
56 DEBUG=1 56 DEBUG=1
57 ;; 57 ;;
58 v) 58 v)
59 VERBOSE=$OPTARG 59 VERBOSE=$OPTARG
60 ;; 60 ;;
61 :) 61 :)
62 echo "Option -$OPTARG requires an argument." >&2 62 echo "Option -$OPTARG requires an argument." > /dev/null
63 exit 1 63 exit 1
64 ;; 64 ;;
65 \?) 65 \?)
66 echo "BAD USAGE : unknow opton -$OPTARG" 66 echo "BAD USAGE : unknow opton -$OPTARG"
67 #exit 1 67 #exit 1
68 ;; 68 ;;
69 esac 69 esac
70 done 70 done
71 71
72 # mode debug enable 72 # mode debug enable
73 if [ $DEBUG -eq 1 ] 73 if [ $DEBUG -eq 1 ]
74 then 74 then
75 set -x 75 set -x
76 echo -e "## Mode DEBUG ON ##" 76 echo -e "## Mode DEBUG ON ##"
77 fi 77 fi
78 78
79 # mode verbose enable 79 # mode verbose enable
80 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi 80 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi
81 81
82 # Check USAGE by arguments number 82 # Check USAGE by arguments number
83 if [ $(($#-($OPTIND-1))) -ne 1 ] 83 if [ $(($#-($OPTIND-1))) -ne 1 ]
84 then 84 then
85 echo "BAD USAGE : RecomposePass.sh [OPTIONS] <PASS_DIR>" 85 echo "BAD USAGE : RecomposePass.sh [OPTIONS] <PASS_DIR>"
86 echo "$0 -h for more info" 86 echo "$0 -h for more info"
87 exit 1 87 exit 1
88 fi 88 fi
89 89
90 shift $((OPTIND-1)) 90 shift $((OPTIND-1))
91 # check Pass directory - First argument 91 # check Pass directory - First argument
92 if [ -e $1 ] && [ -d $1 ] 92 if [ -e $1 ] && [ -d $1 ]
93 then 93 then
94 PASS_DIR=$(readlink -e $1) 94 PASS_DIR=$(readlink -e $1)
95 else 95 else
96 print_error "can't find $1 directory" 96 print_error "can't find $1 directory"
97 exit 1 97 exit 1
98 fi 98 fi
99 99
100 #-------------# 100 #-------------#
101 # GLOBAL VARS # 101 # GLOBAL VARS #
102 #-------------# 102 #-------------#
103 BASENAME=$(basename $PASS_DIR)
104 OUTPUT_DIR_BASENAME=$PASS_DIR
105 RES_DIR="$PASS_DIR/res_all"
106 LOGFILE=$PASS_DIR"/info_recpass.log"
107 ERRORFILE=$PASS_DIR"/error_recpass.log"
108
103 THIRDPASS_CONFIG_FILE="$PASS_DIR/ThirdPass.cfg" 109 THIRDPASS_CONFIG_FILE="$PASS_DIR/ThirdPass.cfg"
104 if [ -e $THIRDPASS_CONFIG_FILE ] 110 if [ -e $THIRDPASS_CONFIG_FILE ]
105 then 111 then
106 RES_P3=$(cat $THIRDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") 112 RES_P3=$(cat $THIRDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=")
107 else 113 else
108 print_warn "can't find $THIRDPASS_CONFIG_FILE file" 1 114 print_warn "[${BASENAME}] can't find $THIRDPASS_CONFIG_FILE file" 2
109 #exit 1 115 #exit 1
110 fi 116 fi
111 SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg" 117 SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg"
112 if [ -e $SECONDPASS_CONFIG_FILE ] 118 if [ -e $SECONDPASS_CONFIG_FILE ]
113 then 119 then
114 RES_P2=$(cat $SECONDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") 120 RES_P2=$(cat $SECONDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=")
115 else 121 else
116 print_warn "can't find $SECONDPASS_CONFIG_FILE file" 1 122 print_warn "[${BASENAME}] can't find $SECONDPASS_CONFIG_FILE file" 2
117 #exit 1 123 #exit 1
118 fi 124 fi
119 FIRSTPASS_CONFIG_FILE="$PASS_DIR/FirstPass.cfg" 125 FIRSTPASS_CONFIG_FILE="$PASS_DIR/FirstPass.cfg"
120 if [ -e $FIRSTPASS_CONFIG_FILE ] 126 if [ -e $FIRSTPASS_CONFIG_FILE ]
121 then 127 then
122 RES_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") 128 RES_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=")
123 else 129 else
124 print_warn "can't find $FIRSTPASS_CONFIG_FILE file" 130 print_warn "[${BASENAME}] can't find $FIRSTPASS_CONFIG_FILE file" 2
125 #exit 1 131 #exit 1
126 fi 132 fi
127 133
128 if ( [ -z "$RES_P1" ] && [ -z "$RES_P2" ] ) || [ -z $RES_P3 ]; then echo "ERROR no configuration pass files in $PASS_DIR"; exit 1; fi 134 if ( [ -z "$RES_P1" ] && [ -z "$RES_P2" ] ) || [ -z $RES_P3 ]
135 then
136 print_error "[${BASENAME}] No configuration pass files in $PASS_DIR -> exit"
137 if [ $CHECK -eq 1 ]; then print_log_file $ERROFILE "No configuration pass files in $PASS_DIR -> exit";fi
138 exit 1
139 fi
129 140
130 BASENAME=$(basename $PASS_DIR) 141 print_info "[${BASENAME}] => RecomposePass start | $(date +'%d/%m/%y %H:%M:%S')" 1
131 OUTPUT_DIR_BASENAME=$PASS_DIR
132 RES_DIR="$PASS_DIR/res_all"
133 LOGFILE=$(dirname $PASS_DIR)"/info_recpass.log"
134 ERRORFILE=$(dirname $PASS_DIR)"/error_recpass.log"
135 142
136 #------------------# 143 #------------------#
137 # Create WORKSPACE # 144 # Create WORKSPACE #
138 #------------------# 145 #------------------#
139 146
140 # Lock directory 147 # Lock directory
141 if [ -e $OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi 148 if [ -e $OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock ] && [ $RERUN -eq 0 ]
149 then
150 print_warn "[${BASENAME}] RECOMPOSEPASS is locked -> exit" 2
151 exit 1
152 fi
142 rm "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" > /dev/null 2>&1 153 rm "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" > /dev/null 2>&1
143 touch "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" > /dev/null 2>&1 154 touch "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" > /dev/null 2>&1
144 155
145 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi 156 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi
146 mkdir -p $RES_DIR 157 mkdir -p $RES_DIR 2> /dev/null
147 print_info "Make directory $RES_DIR" 1 158 print_info "Make directory $RES_DIR" 3
148 159
149 rm $RES_DIR 2> /dev/null
150 mkdir -p $RES_DIR
151
152 #--------------------# 160 #--------------------#
153 # Save configuration # 161 # Save configuration #
154 #--------------------# 162 #--------------------#
155 cp $RECOMPOSEPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/RecomposePass.cfg 163 cp $RECOMPOSEPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/RecomposePass.cfg
156 echo "RES_ALL=$RES_DIR" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg 164 echo "RES_ALL=$RES_DIR" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
157 echo "RES_P1=$RES_P1" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg 165 echo "RES_P1=$RES_P1" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
158 echo "RES_P2=$RES_P2" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg 166 echo "RES_P2=$RES_P2" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
159 echo "RES_P3=$RES_P3" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg 167 echo "RES_P3=$RES_P3" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
160 print_info "save config in $OUTPUT_DIR_BASENAME/RecomposePass.cfg" 1 168 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/RecomposePass.cfg" 1
161 169
162 #--------------------------------------------------# 170 #--------------------------------------------------#
163 # Recompose Pass using previous pass # 171 # Recompose Pass using previous pass #
164 #--------------------------------------------------# 172 #--------------------------------------------------#
165 print_info "Launch Recompose Pass" 1 173 print_info "[${BASENAME}] Launch Recompose Pass" 1
166 174
167 ## Check seg in p3 175 ## Check seg in p3
168 cp $RES_P3/*.res $RES_DIR 176 cp $RES_P3/*.res $RES_DIR
169 ls $RES_P3/*.res | sed -e "s|$RES_P3\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p3.tmp 177 ls $RES_P3/*.res | sed -e "s|$RES_P3\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p3.tmp
170 ls $RES_P2/*.res | sed -e "s|$RES_P2\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p2.tmp 178 ls $RES_P2/*.res | sed -e "s|$RES_P2\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p2.tmp
171 ls $RES_P1/*.res | sed -e "s|$RES_P1\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p1.tmp 179 ls $RES_P1/*.res | sed -e "s|$RES_P1\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p1.tmp
172 diff_3_2=$(diff ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp | grep -e "^> " | sed -e "s/> //") 180 diff_3_2=$(diff ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp | grep -e "^> " | sed -e "s/> //")
173 for res in $diff_3_2 181 for res in $diff_3_2
174 do 182 do
175 cp $RES_P2/$res.res $RES_DIR 183 cp $RES_P2/$res.res $RES_DIR
176 done 184 done
177 ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_all.tmp 185 ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_all.tmp
178 diff_3_1=$(diff ${OUTPUT_DIR_BASENAME}/res_all.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp | grep -e "^> " | sed -e "s/> //") 186 diff_3_1=$(diff ${OUTPUT_DIR_BASENAME}/res_all.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp | grep -e "^> " | sed -e "s/> //")
179 for res in $diff_3_1 187 for res in $diff_3_1
180 do 188 do
181 cp $RES_P1/$res.res $RES_DIR 189 cp $RES_P1/$res.res $RES_DIR
182 done 190 done
183 191
184 rm ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp ${OUTPUT_DIR_BASENAME}/res_all.tmp 192 rm ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp ${OUTPUT_DIR_BASENAME}/res_all.tmp 2> /dev/null
185 193
186 #---------------# 194 #---------------#
187 # Convert res # 195 # Convert res #
188 #---------------# 196 #---------------#
189 197
198 print_info "[${BASENAME}] Convert .res into .ctm" 1
190 # .res => .ctm 199 # .res => .ctm
191 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.ctm 200 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.ctm
201 print_info "[${BASENAME}] Convert .res into .trs" 1
192 # .res => .trs 202 # .res => .trs
193 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 203 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
194 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 204 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
195 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 205 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
206 print_info "[${BASENAME}] Convert .res into .txt" 1
196 # .res => .txt 207 # .res => .txt
197 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.txt 208 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.txt
198 209
main_tools/ScoringRes.sh
Diff comments   View file @ b427f10
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : ScoringRes.sh # 4 # File : ScoringRes.sh #
5 # Brief : Score result using SRT file # 5 # Brief : Score result using SRT file #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # Version : 1.0 # 7 # Version : 1.0 #
8 # Date : 11/07/13 # 8 # Date : 11/07/13 #
9 ##################################################### 9 #####################################################
10 10
11 echo "### ScoringRes.sh ###" 11 echo "### ScoringRes.sh ###"
12 12
13 # Check OTMEDIA_HOME env var 13 # Check OTMEDIA_HOME env var
14 if [ -z ${OTMEDIA_HOME} ] 14 if [ -z ${OTMEDIA_HOME} ]
15 then 15 then
16 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 16 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
17 export OTMEDIA_HOME=$OTMEDIA_HOME 17 export OTMEDIA_HOME=$OTMEDIA_HOME
18 fi 18 fi
19 19
20 # where is ScoringRes.sh 20 # where is ScoringRes.sh
21 MAIN_SCRIPT_PATH=$OTMEDIA_HOME/main_tools 21 MAIN_SCRIPT_PATH=$OTMEDIA_HOME/main_tools
22 22
23 # Scripts Path 23 # Scripts Path
24 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 24 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
25 25
26 # Include scripts 26 # Include scripts
27 . $SCRIPT_PATH"/Tools.sh" 27 . $SCRIPT_PATH"/Tools.sh"
28 28
29 # where is ScoringRes.cfg 29 # where is ScoringRes.cfg
30 SCORINGRES_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ScoringRes.cfg" 30 SCORINGRES_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ScoringRes.cfg"
31 if [ -e $SCORINGRES_CONFIG_FILE ] 31 if [ -e $SCORINGRES_CONFIG_FILE ]
32 then 32 then
33 . $SCORINGRES_CONFIG_FILE 33 . $SCORINGRES_CONFIG_FILE
34 else 34 else
35 echo "ERROR : Can't find configuration file $SCORINGRES_CONFIG_FILE" >&2 35 echo "ERROR : Can't find configuration file $SCORINGRES_CONFIG_FILE" >&2
36 exit 1 36 exit 1
37 fi 37 fi
38 38
39 #---------------# 39 #---------------#
40 # Parse Options # 40 # Parse Options #
41 #---------------# 41 #---------------#
42 while getopts ":hDv:" opt 42 while getopts ":hDv:" opt
43 do 43 do
44 case $opt in 44 case $opt in
45 h) 45 h)
46 echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" 46 echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n"
47 echo -e "\t Options:" 47 echo -e "\t Options:"
48 echo -e "\t\t-h :\tprint this message" 48 echo -e "\t\t-h :\tprint this message"
49 echo -e "\t\t-D :\tDEBUG mode on" 49 echo -e "\t\t-D :\tDEBUG mode on"
50 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 50 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
51 exit 1 51 exit 1
52 ;; 52 ;;
53 D) 53 D)
54 DEBUG=1 54 DEBUG=1
55 ;; 55 ;;
56 v) 56 v)
57 VERBOSE=$OPTARG 57 VERBOSE=$OPTARG
58 ;; 58 ;;
59 :) 59 :)
60 echo "Option -$OPTARG requires an argument." >&2 60 echo "Option -$OPTARG requires an argument." >&2
61 exit 1 61 exit 1
62 ;; 62 ;;
63 \?) 63 \?)
64 echo "BAD USAGE : unknow opton -$OPTARG" 64 echo "BAD USAGE : unknow opton -$OPTARG"
65 #exit 1 65 #exit 1
66 ;; 66 ;;
67 esac 67 esac
68 68
69 done 69 done
70 70
71 # mode debug enable 71 # mode debug enable
72 if [ $DEBUG -eq 1 ] 72 if [ $DEBUG -eq 1 ]
73 then 73 then
74 set -x 74 set -x
75 echo -e "## Mode DEBUG ON ##" 75 echo -e "## Mode DEBUG ON ##"
76 fi 76 fi
77 77
78 # mode verbose enable 78 # mode verbose enable
79 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi 79 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi
80 80
81 # Check USAGE by arguments number 81 # Check USAGE by arguments number
82 if [ $(($#-($OPTIND-1))) -ne 1 ] 82 if [ $(($#-($OPTIND-1))) -ne 1 ]
83 then 83 then
84 echo "BAD USAGE : SecondPass.sh [OPTIONS] <PASS_DIR>" 84 echo "BAD USAGE : SecondPass.sh [OPTIONS] <PASS_DIR>"
85 echo "$0 -h for more info" 85 echo "$0 -h for more info"
86 exit 1 86 exit 1
87 fi 87 fi
88 88
89 shift $((OPTIND-1)) 89 shift $((OPTIND-1))
90 # check Pass directory - First argument 90 # check Pass directory - First argument
91 if [ -e $1 ] && [ -d $1 ] 91 if [ -e $1 ] && [ -d $1 ]
92 then 92 then
93 PASS_DIR=$(readlink -e $1) 93 PASS_DIR=$(readlink -e $1)
94 else 94 else
95 print_error "can't find $1 directory" 95 print_error "can't find $1 directory"
96 exit 1 96 exit 1
97 fi 97 fi
98 98
99 #-------------# 99 #-------------#
100 # GLOBAL VARS # 100 # GLOBAL VARS #
101 #-------------# 101 #-------------#
102 BASENAME=$(basename $PASS_DIR) 102 BASENAME=$(basename $PASS_DIR)
103 SCORING_DIR=$PASS_DIR/scoring 103 SCORING_DIR=$PASS_DIR/scoring
104 LOGFILE="$(dirname $PASS_DIR)/info_scoring.log" 104 LOGFILE="$PASS_DIR/info_scoring.log"
105 ERRORFILE="$(dirname $PASS_DIR)/error_scoring.log" 105 ERRORFILE="$PASS_DIR/error_scoring.log"
106 106
107 print_info "[${BASENAME}] => ScoringRes start | $(date +'%d/%m/%y %H:%M:%S')" 1
108
107 #------------------# 109 #------------------#
108 # Create WORKSPACE # 110 # Create WORKSPACE #
109 #------------------# 111 #------------------#
110 112
111 # Lock directory 113 # Lock directory
112 if [ -e $PASS_DIR/*.lock ]; then echo "$PASS_DIR is lock"; exit 1;fi 114 if [ -e $PASS_DIR/*.lock ]
115 then
116 print_warn "[${BASENAME}] Directory is locked -> exit" 2;
117 exit 1
118 fi
113 119
114 rm -r $SCORING_DIR > /dev/null 2>&1 120 rm -r $SCORING_DIR > /dev/null 2>&1
115 mkdir -p $SCORING_DIR 121 mkdir -p $SCORING_DIR
116 print_info "Make directory $SCORING_DIR" 1 122 print_info "Make directory $SCORING_DIR" 1
117 123
118 if [ ! -e $PASS_DIR/$BASENAME.SRT ] 124 if [ ! -e $PASS_DIR/$BASENAME.SRT ]
119 then 125 then
120 echo -e "no SRT file for $BASENAME\n" 126 print_warn "[${BASENAME}] No SRT file -> exit" 2
121 exit 0; 127 exit 0;
122 else 128 else
123 SRT_FILE=$PASS_DIR/$BASENAME.SRT 129 SRT_FILE=$PASS_DIR/$BASENAME.SRT
130 print_info "[${BASENAME}] SRT file : $BASENAME.SRT" 1
124 fi 131 fi
125 132
126 if [ ! -e $PASS_DIR/$BASENAME.1pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.2pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.3pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.allpass.ctm ] 133 if [ ! -e $PASS_DIR/$BASENAME.1pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.2pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.3pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.allpass.ctm ]
127 then 134 then
128 echo -e "no CTM files for $BASENAME\n"; 135 print_error "[${BASENAME}] No CTM files -> exit\n";
129 exit 1; 136 exit 1;
130 fi 137 fi
131 138
139 print_info "[${BASENAME}] Create stm file from SRT" 1
132 i=$((-3)) 140 i=$((-3))
133 while read line 141 while read line
134 do 142 do
135 i=$(($i + 1)); 143 i=$(($i + 1));
136 if [ $i -eq 4 ] 144 if [ $i -eq 4 ]
137 then 145 then
138 echo " $line" 146 echo " $line"
139 i=$((0)); 147 i=$((0));
140 fi 148 fi
141 done < $SRT_FILE > $SCORING_DIR/$BASENAME.tmp.txt 149 done < $SRT_FILE > $SCORING_DIR/$BASENAME.tmp.txt
142 150
143 cat $SCORING_DIR/$BASENAME.tmp.txt | sed -e "s|\n| |g" | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -e "s|<s>||g" | sed -e "s|</s>||g" | sed -e "s|\n+| |g" > $SCORING_DIR/$BASENAME.tmp2.txt 151 cat $SCORING_DIR/$BASENAME.tmp.txt | sed -e "s|\n| |g" | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -e "s|<s>||g" | sed -e "s|</s>||g" | sed -e "s|\n+| |g" > $SCORING_DIR/$BASENAME.tmp2.txt
144 #cat $SCORING_DIR/$BASENAME.tmp.txt | sed -e "s|\n| |g" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex | sed -e "s|<s>||g" | sed -e "s|</s>||g" | sed -e "s|\n+| |g" > $SCORING_DIR/$BASENAME.tmp2.txt 152 #cat $SCORING_DIR/$BASENAME.tmp.txt | sed -e "s|\n| |g" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex | sed -e "s|<s>||g" | sed -e "s|</s>||g" | sed -e "s|\n+| |g" > $SCORING_DIR/$BASENAME.tmp2.txt
145 153
146 $SCRIPT_PATH/srt2stm.pl $SCORING_DIR/$BASENAME.tmp2.txt > "$SCORING_DIR/$BASENAME.stm" 154 $SCRIPT_PATH/srt2stm.pl $SCORING_DIR/$BASENAME.tmp2.txt > "$SCORING_DIR/$BASENAME.stm"
147 rm $SCORING_DIR/$BASENAME.tmp.txt $SCORING_DIR/$BASENAME.tmp2.txt 155 rm $SCORING_DIR/$BASENAME.tmp.txt $SCORING_DIR/$BASENAME.tmp2.txt
148 156
157 print_info "Scoring using sclite" 1
149 for ctmfile in $(ls $PASS_DIR/*.ctm) 158 for ctmfile in $(ls $PASS_DIR/*.ctm)
150 do 159 do
151 cp $ctmfile $SCORING_DIR/ 160 cp $ctmfile $SCORING_DIR/
161 print_info "$BIN_PATH/sclite -D -F -r $SCORING_DIR/$BASENAME.stm stm -h $SCORING_DIR/$(basename $ctmfile) ctm -o all" 3
152 $BIN_PATH/sclite -D -F -r $SCORING_DIR/$BASENAME.stm stm -h $SCORING_DIR/$(basename $ctmfile) ctm -o all 162 $BIN_PATH/sclite -D -F -r $SCORING_DIR/$BASENAME.stm stm -h $SCORING_DIR/$(basename $ctmfile) ctm -o all
153 done 163 done
164
165 print_info "[${BASENAME}] <= ScoringRes End | $(date +'%d/%m/%y %H:%M:%S')" 1
154 166
155 167
156 168
main_tools/SecondPass.sh
Diff comments   View file @ b427f10
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : SecondPass.sh # 4 # File : SecondPass.sh #
5 # Brief : Speaker adaptation + ASR second pass # 5 # Brief : Speaker adaptation + ASR second pass #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # (base on Emmanuel Ferreira # 7 # (base on Emmanuel Ferreira #
8 # and Hugo Mauchrétien works) # 8 # and Hugo Mauchrétien works) #
9 # Version : 1.1 # 9 # Version : 1.1 #
10 # Date : 18/06/13 # 10 # Date : 18/06/13 #
11 ##################################################### 11 #####################################################
12 12
13 echo "### SecondPass.sh ###" 13 echo "### SecondPass.sh ###"
14 14
15 # Check OTMEDIA_HOME env var 15 # Check OTMEDIA_HOME env var
16 if [ -z ${OTMEDIA_HOME} ] 16 if [ -z ${OTMEDIA_HOME} ]
17 then 17 then
18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
19 export OTMEDIA_HOME=$OTMEDIA_HOME 19 export OTMEDIA_HOME=$OTMEDIA_HOME
20 fi 20 fi
21 21
22 # where is SecondPass.sh 22 # where is SecondPass.sh
23 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 23 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
24 24
25 # Scripts Path 25 # Scripts Path
26 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 26 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
27 27
28 # Include scripts 28 # Include scripts
29 . $SCRIPT_PATH"/Tools.sh" 29 . $SCRIPT_PATH"/Tools.sh"
30 . $SCRIPT_PATH"/CheckSecondPass.sh" 30 . $SCRIPT_PATH"/CheckSecondPass.sh"
31 31
32 # where is SecondPass.cfg 32 # where is SecondPass.cfg
33 SECONDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/SecondPass.cfg" 33 SECONDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/SecondPass.cfg"
34 if [ -e $SECONDPASS_CONFIG_FILE ] 34 if [ -e $SECONDPASS_CONFIG_FILE ]
35 then 35 then
36 . $SECONDPASS_CONFIG_FILE 36 . $SECONDPASS_CONFIG_FILE
37 else 37 else
38 echo "ERROR : Can't find configuration file $SECONDPASS_CONFIG_FILE" > /dev/stderr 38 echo "ERROR : Can't find configuration file $SECONDPASS_CONFIG_FILE" > /dev/stderr
39 echo "exit" > /dev/stderr 39 echo "exit" > /dev/stderr
40 exit 1 40 exit 1
41 fi 41 fi
42 42
43 #---------------# 43 #---------------#
44 # Parse Options # 44 # Parse Options #
45 #---------------# 45 #---------------#
46 while getopts ":hDv:crf:" opt 46 while getopts ":hDv:crf:" opt
47 do 47 do
48 case $opt in 48 case $opt in
49 h) 49 h)
50 echo -e "$0 [OPTIONS] <FIRST_PASS_DIRECTORY>\n" 50 echo -e "$0 [OPTIONS] <FIRST_PASS_DIRECTORY>\n"
51 echo -e "\t Options:" 51 echo -e "\t Options:"
52 echo -e "\t\t-h :\tprint this message" 52 echo -e "\t\t-h :\tprint this message"
53 echo -e "\t\t-D :\tDEBUG mode on" 53 echo -e "\t\t-D :\tDEBUG mode on"
54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
55 echo -e "\t\t-c :\t Check process, stop if error detected" 55 echo -e "\t\t-c :\t Check process, stop if error detected"
56 echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)" 56 echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)"
57 echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done" 57 echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done"
58 exit 1 58 exit 1
59 ;; 59 ;;
60 D) 60 D)
61 DEBUG=1 61 DEBUG=1
62 ;; 62 ;;
63 v) 63 v)
64 VERBOSE=$OPTARG 64 VERBOSE=$OPTARG
65 ;; 65 ;;
66 c) 66 c)
67 CHECK=1 67 CHECK=1
68 ;; 68 ;;
69 f) 69 f)
70 FORKS="--forks $OPTARG" 70 FORKS="--forks $OPTARG"
71 ;; 71 ;;
72 r) 72 r)
73 RERUN=1 73 RERUN=1
74 ;; 74 ;;
75 :) 75 :)
76 echo "Option -$OPTARG requires an argument." > /dev/stderr 76 echo "Option -$OPTARG requires an argument." > /dev/stderr
77 exit 1 77 exit 1
78 ;; 78 ;;
79 \?) 79 \?)
80 echo "BAD USAGE : unknow opton -$OPTARG" > /dev/stderr 80 echo "BAD USAGE : unknow opton -$OPTARG" > /dev/stderr
81 exit 1 81 exit 1
82 ;; 82 ;;
83 esac 83 esac
84 done 84 done
85 85
86 # mode debug enable 86 # mode debug enable
87 if [ $DEBUG -eq 1 ] 87 if [ $DEBUG -eq 1 ]
88 then 88 then
89 set -x 89 set -x
90 echo -e "## Mode DEBUG ON ##" 90 echo -e "## Mode DEBUG ON ##"
91 fi 91 fi
92 92
93 # mode verbose enable 93 # mode verbose enable
94 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi 94 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
95 95
96 # Check USAGE by arguments number 96 # Check USAGE by arguments number
97 if [ $(($#-($OPTIND-1))) -ne 1 ] 97 if [ $(($#-($OPTIND-1))) -ne 1 ]
98 then 98 then
99 echo "BAD USAGE : SecondPass.sh [OPTIONS] <FIRST_PASS_DIR>" 99 echo "BAD USAGE : SecondPass.sh [OPTIONS] <FIRST_PASS_DIR>"
100 echo "$0 -h for more info" 100 echo "$0 -h for more info"
101 exit 1 101 exit 1
102 fi 102 fi
103 103
104 shift $((OPTIND-1)) 104 shift $((OPTIND-1))
105 # check FirstPass directory - First argument 105 # check FirstPass directory - First argument
106 if [ -e $1 ] && [ -d $1 ] 106 if [ -e $1 ] && [ -d $1 ]
107 then 107 then
108 FIRSTPASS_DIR=$(readlink -e $1) 108 FIRSTPASS_DIR=$(readlink -e $1)
109 else 109 else
110 print_error "can't find $1 directory" 110 print_error "Can't find $1 directory"
111 exit 1 111 exit 1
112 fi 112 fi
113 113
114 print_info "[${BASENAME}] => P2 start | $(date +'%d/%m/%y %H:%M:%S')" 1
115
116 #-------------# 114 #-------------#
117 # GLOBAL VARS # 115 # GLOBAL VARS #
118 #-------------# 116 #-------------#
119 FIRSTPASS_CONFIG_FILE="$FIRSTPASS_DIR/FirstPass.cfg" 117 FIRSTPASS_CONFIG_FILE="$FIRSTPASS_DIR/FirstPass.cfg"
120 if [ -e $FIRSTPASS_CONFIG_FILE ] 118 if [ -e $FIRSTPASS_CONFIG_FILE ]
121 then 119 then
122 WAV_FILE=$(cat $FIRSTPASS_CONFIG_FILE | grep "WAV_FILE=" | cut -f2 -d"=") 120 WAV_FILE=$(cat $FIRSTPASS_CONFIG_FILE | grep "WAV_FILE=" | cut -f2 -d"=")
123 BASENAME=$(cat $FIRSTPASS_CONFIG_FILE | grep "^BASENAME=" | cut -f2 -d"=") 121 BASENAME=$(cat $FIRSTPASS_CONFIG_FILE | grep "^BASENAME=" | cut -f2 -d"=")
124 OUTPUT_DIR=$(cat $FIRSTPASS_CONFIG_FILE | grep "OUTPUT_DIR=" | cut -f2 -d"=") 122 OUTPUT_DIR=$(cat $FIRSTPASS_CONFIG_FILE | grep "OUTPUT_DIR=" | cut -f2 -d"=")
125 OUTPUT_DIR_BASENAME=$FIRSTPASS_DIR 123 OUTPUT_DIR_BASENAME=$FIRSTPASS_DIR
126 PLP_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_FILE=" | cut -f2 -d"=") 124 PLP_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_FILE=" | cut -f2 -d"=")
127 PLP_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_DIR=" | cut -f2 -d"=") 125 PLP_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_DIR=" | cut -f2 -d"=")
128 SEG_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "SEG_FILE=" | cut -f2 -d"=") 126 SEG_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "SEG_FILE=" | cut -f2 -d"=")
129 LBL_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "LBL_FILE=" | cut -f2 -d"=") 127 LBL_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "LBL_FILE=" | cut -f2 -d"=")
130 RES_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "RES_DIR=" | cut -f2 -d"=") 128 RES_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "RES_DIR=" | cut -f2 -d"=")
131 else 129 else
132 print_error "can't find $FIRSTPASS_CONFIG_FILE file" 130 print_error "Can't find $FIRSTPASS_CONFIG_FILE file"
133 print_error "exit" 131 print_error "exit"
134 exit 1 132 exit 1
135 fi 133 fi
136 LST=$OUTPUT_DIR_BASENAME"/lists" 134 LST=$OUTPUT_DIR_BASENAME"/lists"
137 HMM=$OUTPUT_DIR_BASENAME"/hmm/" 135 HMM=$OUTPUT_DIR_BASENAME"/hmm/"
138 RES_DIR=$OUTPUT_DIR_BASENAME"/res_p2" 136 RES_DIR=$OUTPUT_DIR_BASENAME"/res_p2"
139 LOGFILE="$OUTPUT_DIR_BASENAME/info_p2.log" 137 LOGFILE="$OUTPUT_DIR_BASENAME/info_p2.log"
140 ERRORFILE="$OUTPUT_DIR_BASENAME/error_p2.log" 138 ERRORFILE="$OUTPUT_DIR_BASENAME/error_p2.log"
141 139
140 print_info "[${BASENAME}] => P2 start | $(date +'%d/%m/%y %H:%M:%S')" 1
141
142 #------------------# 142 #------------------#
143 # Create WORKSPACE # 143 # Create WORKSPACE #
144 #------------------# 144 #------------------#
145 145
146 # Lock directory 146 # Lock directory
147 if [ -e $OUTPUT_DIR_BASENAME/SECONDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] SECONDPASS is locked -> exit" 2; exit 1;fi 147 if [ -e $OUTPUT_DIR_BASENAME/SECONDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] SECONDPASS is locked -> exit" 2; exit 1;fi
148 rm "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" > /dev/null 2>&1 148 rm "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" > /dev/null 2>&1
149 touch "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" > /dev/null 2>&1 149 touch "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" > /dev/null 2>&1
150 150
151 rm -r $LST > /dev/null 2>&1 151 rm -r $LST > /dev/null 2>&1
152 mkdir -p $LST 152 mkdir -p $LST
153 print_info "[${BASENAME}] Make directory $LST" 2 153 print_info "[${BASENAME}] Make directory $LST" 2
154 if [ $RERUN -eq 0 ]; then rm -r $HMM > /dev/null 2>&1; fi 154 if [ $RERUN -eq 0 ]; then rm -r $HMM > /dev/null 2>&1; fi
155 mkdir -p $HMM 155 mkdir -p $HMM
156 print_info "[${BASENAME}] Make directory $HMM" 2 156 print_info "[${BASENAME}] Make directory $HMM" 2
157 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi 157 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi
158 mkdir -p $RES_DIR > /dev/null 2>&1 158 mkdir -p $RES_DIR > /dev/null 2>&1
159 print_info "[${BASENAME}] Make directory $RES_DIR" 2 159 print_info "[${BASENAME}] Make directory $RES_DIR" 2
160 rm $LOGFILE $ERRORFILE > /dev/null 2>&1 160 rm $LOGFILE $ERRORFILE > /dev/null 2>&1
161 161
162 #-------------------# 162 #-------------------#
163 # Check Pass # 163 # Check Pass #
164 #-------------------# 164 #-------------------#
165 print_info "[${BASENAME}] Check Pass 2 directory" 1 165 print_info "[${BASENAME}] Check Pass 2 directory" 1
166 for treil in $(ls $RES_DIR/ | grep treil) 166 for treil in $(ls $RES_DIR/ | grep treil)
167 do 167 do
168 if [ ! -s $RES_DIR/$treil ] 168 if [ ! -s $RES_DIR/$treil ]
169 then 169 then
170 bn = $(basename $treil ".treil") 170 bn = $(basename $treil ".treil")
171 rm $RES_DIR/$treil $RES_DIR/$bn.seg $RES_DIR/$bn.res $RES_DIR/$bn.pho 2> /dev/null 171 rm $RES_DIR/$treil $RES_DIR/$bn.seg $RES_DIR/$bn.res $RES_DIR/$bn.pho 2> /dev/null
172 print_info "[${BASENAME}] $RES_DIR/$bn.* files deleted.." 2 172 print_info "[${BASENAME}] $RES_DIR/$bn.* files deleted.." 2
173 fi 173 fi
174 done 174 done
175 175
176 # Check if more then 89% of treil are done 176 # Check if more then 89% of treil are done
177 nbres_p1=$(ls $RES_DIR_P1/*.res 2> /dev/null | wc -l) 177 nbres_p1=$(ls $RES_DIR_P1/*.res 2> /dev/null | wc -l)
178 nbtreil_p2=$(ls $RES_DIR/*.treil 2> /dev/null | wc -l) 178 nbtreil_p2=$(ls $RES_DIR/*.treil 2> /dev/null | wc -l)
179 if [ $nbres_p1 -gt 0 ] 179 if [ $nbres_p1 -gt 0 ]
180 then 180 then
181 pourcentage=$((($nbtreil_p2*100)/$nbres_p1)) 181 pourcentage=$((($nbtreil_p2*100)/$nbres_p1))
182 if [ $pourcentage -gt 89 ] 182 if [ $pourcentage -gt 89 ]
183 then 183 then
184 print_info "[${BASENAME}] Lattice already done, skipping $BASENAME" 1 184 print_info "[${BASENAME}] Lattice already done, skipping $BASENAME" 1
185 exit 0 185 exit 0
186 fi 186 fi
187 else 187 else
188 print_error "[${BASENAME}] No First Pass, No .res -> exit P2" 188 print_error "[${BASENAME}] No First Pass, No .res -> exit P2"
189 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No First Pass, No .res -> exit P2" ;fi 189 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No First Pass, No .res -> exit P2" ;fi
190 exit 1 190 exit 1
191 fi 191 fi
192 192
193 #--------------------# 193 #--------------------#
194 # Save configuration # 194 # Save configuration #
195 #--------------------# 195 #--------------------#
196 cp $SECONDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/SecondPass.cfg 196 cp $SECONDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/SecondPass.cfg
197 echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 197 echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
198 echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 198 echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
199 echo "FIRSTPASS_DIR=$FIRSTPASS_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 199 echo "FIRSTPASS_DIR=$FIRSTPASS_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
200 echo "PLP_DIR_P1=$PLP_DIR_P1" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 200 echo "PLP_DIR_P1=$PLP_DIR_P1" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
201 echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 201 echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
202 echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 202 echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
203 echo "LST=$LST" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 203 echo "LST=$LST" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
204 echo "HMM=$HMM" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 204 echo "HMM=$HMM" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
205 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg 205 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
206 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1 206 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1
207 207
208 208
209 #--------------------------------------------------# 209 #--------------------------------------------------#
210 # Speaker Adaptation (AM) + Second pass (DECODING) # 210 # Speaker Adaptation (AM) + Second pass (DECODING) #
211 #--------------------------------------------------# 211 #--------------------------------------------------#
212 print_info "[${BASENAME}] Launch Second Pass" 1 212 print_info "[${BASENAME}] Launch Second Pass" 1
213 213
214 # for all speaker 214 # for all speaker
215 for speaker in $(cat $LBL_FILE_P1 | cut -f4 -d" " | sort | uniq) 215 for speaker in $(cat $LBL_FILE_P1 | cut -f4 -d" " | sort | uniq)
216 do 216 do
217 ## get seg file from P1 containing the speaker 217 ## get seg file from P1 containing the speaker
218 find $RES_DIR_P1 -name "*${speaker}.seg" -exec basename "{}" .seg \; | sort > $LST/$speaker.lst 218 find $RES_DIR_P1 -name "*${speaker}.seg" -exec basename "{}" .seg \; | sort > $LST/$speaker.lst
219 print_info "[${BASENAME}] file for $speaker in $LST/$speaker.lst" 3 219 print_info "[${BASENAME}] file for $speaker in $LST/$speaker.lst" 3
220 if [ ! -s $LST/$speaker.lst ]; then print_warn "no ${speaker} file in $RES_DIR_P1" 3; continue; fi 220 if [ ! -s $LST/$speaker.lst ]; then print_warn "no ${speaker} file in $RES_DIR_P1" 3; continue; fi
221 221
222 222
223 # for all AM 223 # for all AM
224 for (( i=0; $i<${#MTAG[@]} ; i++ )) 224 for (( i=0; $i<${#MTAG[@]} ; i++ ))
225 do 225 do
226 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then 226 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then
227 type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst") 227 type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst")
228 ## if is the good AM for the speaker 228 ## if is the good AM for the speaker
229 if [ -n "$type" ] 229 if [ -n "$type" ]
230 then 230 then
231 ## HMM adaptation 231 ## HMM adaptation
232 if [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] ) 232 if [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] )
233 then 233 then
234 print_info "[${BASENAME}] $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/" 3 234 print_info "[${BASENAME}] $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/" 3
235 $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/ 235 $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/
236 fi 236 fi
237 237
238 if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] ) 238 if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] )
239 then 239 then
240 print_warn "[${BASENAME}] No hmm files created for $speaker" 2 240 print_warn "[${BASENAME}] No hmm files created for $speaker" 2
241 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No hmm files created for $speaker" 241 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No hmm files created for $speaker"
242 print_error "[${BASENAME}] Check $ERRORFILE" 242 print_error "[${BASENAME}] Check $ERRORFILE"
243 #exit 1 243 #exit 1
244 fi 244 fi
245 245
246 ## cp map files 246 ## cp map files
247 cp $SPEERAL_AM/${MODS[$i]}.map $HMM/$speaker.map 247 cp $SPEERAL_AM/${MODS[$i]}.map $HMM/$speaker.map
248 248
249 ## class clustering 249 ## class clustering
250 if [ -s $HMM/$speaker.hmm ] && ( [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] )) 250 if [ -s $HMM/$speaker.hmm ] && ( [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] ))
251 then 251 then
252 print_info "[${BASENAME}] $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls" 3 252 print_info "[${BASENAME}] $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls" 3
253 $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls 253 $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls
254 fi 254 fi
255 if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] ) 255 if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] )
256 then 256 then
257 print_warn "[${BASENAME}] No cls file created for $speaker" 2 257 print_warn "[${BASENAME}] No cls file created for $speaker" 2
258 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No cls file created for $speakers" 258 print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No cls file created for $speakers"
259 print_error "[${BASENAME}] Check $ERRORFILE" 259 print_error "[${BASENAME}] Check $ERRORFILE"
260 #exit 1 260 #exit 1
261 fi 261 fi
262 262
263 ## Speeral decoding 263 ## Speeral decoding
264 if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ] 264 if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ]
265 then 265 then
266 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3 266 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3
267 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock 267 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock
268 else 268 else
269 print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2 269 print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2
270 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3 270 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3
271 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock 271 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock
272 fi 272 fi
273 273
274 if [ $CHECK -eq 1 ] 274 if [ $CHECK -eq 1 ]
275 then 275 then
276 check_second_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR" 276 check_second_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR"
277 if [ $? -eq 1 ] 277 if [ $? -eq 1 ]
278 then 278 then
279 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 279 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2
280 print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst" 280 print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst"
281 ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 281 ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
282 print_log_file $LOGFILE "Segs (and treil) not done :\n[" 282 print_log_file $LOGFILE "Segs (and treil) not done :\n["
283 diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $LOGFILE 283 diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $LOGFILE
284 print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]" 284 print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]"
285 rm ${OUTPUT_DIR_BASENAME}/.tmp 285 rm ${OUTPUT_DIR_BASENAME}/.tmp
286 #exit 1 286 #exit 1
287 fi 287 fi
288 fi 288 fi
289 break 289 break
290 fi 290 fi
291 fi 291 fi
292 done 292 done
293 #rm "$HMM/$speaker.*" > /dev/null 2>&1 293 #rm "$HMM/$speaker.*" > /dev/null 2>&1
294 #rm "$LST/$speaker.lst" > /dev/null 2>&1 294 #rm "$LST/$speaker.lst" > /dev/null 2>&1
295 done 295 done
296 296
297 ## Check missing seg and log it 297 ## Check missing seg and log it
298 if [ $CHECK -eq 1 ] 298 if [ $CHECK -eq 1 ]
299 then 299 then
300 ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 300 ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
301 todo=$(cat ${PLP_FILE_P1} | wc -l) 301 todo=$(cat ${PLP_FILE_P1} | wc -l)
302 if [ $todo -eq 0 ]; then todo=1;fi 302 if [ $todo -eq 0 ]; then todo=1;fi
303 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) 303 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l)))
304 pourcentage=$((($notdone*100)/$todo)) 304 pourcentage=$((($notdone*100)/$todo))
305 if [ $notdone -ne 0 ] 305 if [ $notdone -ne 0 ]
306 then 306 then
307 print_error "[${BASENAME}] ERROR : check $ERRORFILE" 307 print_error "[${BASENAME}] Check $ERRORFILE"
308 print_log_file "$ERRORFILE" "ERROR : Treil not done [" 308 print_log_file "$ERRORFILE" "ERROR : Treil not done ["
309 diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE 309 diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE
310 print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" 310 print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME"
311 else 311 else
312 print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" 312 print_log_file "$LOGFILE" "P2 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')"
313 fi 313 fi
314 rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1 314 rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1
315 fi 315 fi
316 316
317 #---------------# 317 #---------------#
318 # Convert res # 318 # Convert res #
319 #---------------# 319 #---------------#
320 320
321 print_info "[${BASENAME}] Convert .res into .ctm" 1 321 print_info "[${BASENAME}] Convert .res into .ctm" 1
322 # .res => .ctm 322 # .res => .ctm
323 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.ctm 323 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.ctm
324 print_info "[${BASENAME}] Convert .res into .trs" 1 324 print_info "[${BASENAME}] Convert .res into .trs" 1
325 # .res => .trs 325 # .res => .trs
326 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 326 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
327 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 327 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
328 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1 328 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1
329 print_info "[${BASENAME}] Convert .res into .txt" 1 329 print_info "[${BASENAME}] Convert .res into .txt" 1
330 # .res => .txt 330 # .res => .txt
331 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.txt 331 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.txt
332 332
333 print_info "[${BASENAME}] <= P2 End | $(date +'%d/%m/%y %H:%M:%S')" 1 333 print_info "[${BASENAME}] <= P2 End | $(date +'%d/%m/%y %H:%M:%S')" 1
334 # unlock directory 334 # unlock directory
335 mv "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" 335 mv "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock"
main_tools/ThirdPass.sh
Diff comments   View file @ b427f10
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : ThirdPass.sh # 4 # File : ThirdPass.sh #
5 # Brief : ASR third pass using trigg files # 5 # Brief : ASR third pass using trigg files #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # Version : 1.0 # 7 # Version : 1.0 #
8 # Date : 18/07/13 # 8 # Date : 18/07/13 #
9 ##################################################### 9 #####################################################
10 10
11 echo "### ThirdPass.sh ###" 11 echo "### ThirdPass.sh ###"
12 12
13 # Check OTMEDIA_HOME env var 13 # Check OTMEDIA_HOME env var
14 if [ -z ${OTMEDIA_HOME} ] 14 if [ -z ${OTMEDIA_HOME} ]
15 then 15 then
16 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 16 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
17 export OTMEDIA_HOME=$OTMEDIA_HOME 17 export OTMEDIA_HOME=$OTMEDIA_HOME
18 fi 18 fi
19 19
20 # where is SecondPass.sh 20 # where is SecondPass.sh
21 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 21 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
22 22
23 # Scripts Path 23 # Scripts Path
24 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 24 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
25 25
26 # Include scripts 26 # Include scripts
27 . $SCRIPT_PATH"/Tools.sh" 27 . $SCRIPT_PATH"/Tools.sh"
28 . $SCRIPT_PATH"/CheckThirdPass.sh" 28 . $SCRIPT_PATH"/CheckThirdPass.sh"
29 29
30 # where is ThirdPass.cfg 30 # where is ThirdPass.cfg
31 THIRDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ThirdPass.cfg" 31 THIRDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ThirdPass.cfg"
32 if [ -e $THIRDPASS_CONFIG_FILE ] 32 if [ -e $THIRDPASS_CONFIG_FILE ]
33 then 33 then
34 . $THIRDPASS_CONFIG_FILE 34 . $THIRDPASS_CONFIG_FILE
35 else 35 else
36 echo "ERROR : Can't find configuration file $THIRDPASS_CONFIG_FILE" >&2 36 echo "ERROR : Can't find configuration file $THIRDPASS_CONFIG_FILE" >&2
37 exit 1 37 exit 1
38 fi 38 fi
39 39
40 #---------------# 40 #---------------#
41 # Parse Options # 41 # Parse Options #
42 #---------------# 42 #---------------#
43 while getopts ":hDv:crf:" opt 43 while getopts ":hDv:crf:" opt
44 do 44 do
45 case $opt in 45 case $opt in
46 h) 46 h)
47 echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" 47 echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n"
48 echo -e "\t Options:" 48 echo -e "\t Options:"
49 echo -e "\t\t-h :\tprint this message" 49 echo -e "\t\t-h :\tprint this message"
50 echo -e "\t\t-D :\tDEBUG mode on" 50 echo -e "\t\t-D :\tDEBUG mode on"
51 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 51 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
52 echo -e "\t\t-c :\t Check process, stop if error detected" 52 echo -e "\t\t-c :\t Check process, stop if error detected"
53 echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)" 53 echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)"
54 echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done" 54 echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done"
55 exit 1 55 exit 1
56 ;; 56 ;;
57 D) 57 D)
58 DEBUG=1 58 DEBUG=1
59 ;; 59 ;;
60 v) 60 v)
61 VERBOSE=$OPTARG 61 VERBOSE=$OPTARG
62 ;; 62 ;;
63 c) 63 c)
64 CHECK=1 64 CHECK=1
65 ;; 65 ;;
66 f) 66 f)
67 FORKS="--forks $OPTARG" 67 FORKS="--forks $OPTARG"
68 ;; 68 ;;
69 r) 69 r)
70 RERUN=1 70 RERUN=1
71 ;; 71 ;;
72 :) 72 :)
73 echo "Option -$OPTARG requires an argument." >&2 73 echo "Option -$OPTARG requires an argument." > /dev/null
74 exit 1 74 exit 1
75 ;; 75 ;;
76 \?) 76 \?)
77 echo "BAD USAGE : unknow opton -$OPTARG" 77 echo "BAD USAGE : unknow opton -$OPTARG"
78 #exit 1 78 #exit 1
79 ;; 79 ;;
80 esac 80 esac
81 done 81 done
82 82
83 # mode debug enable 83 # mode debug enable
84 if [ $DEBUG -eq 1 ] 84 if [ $DEBUG -eq 1 ]
85 then 85 then
86 set -x 86 set -x
87 echo -e "## Mode DEBUG ON ##" 87 echo -e "## Mode DEBUG ON ##"
88 fi 88 fi
89 89
90 # mode verbose enable 90 # mode verbose enable
91 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi 91 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi
92 92
93 # Check USAGE by arguments number 93 # Check USAGE by arguments number
94 if [ $(($#-($OPTIND-1))) -ne 1 ] 94 if [ $(($#-($OPTIND-1))) -ne 1 ]
95 then 95 then
96 echo "BAD USAGE : ThirdPass.sh [OPTIONS] <PASS_DIR>" 96 echo "BAD USAGE : ThirdPass.sh [OPTIONS] <PASS_DIR>"
97 echo "$0 -h for more info" 97 echo "$0 -h for more info"
98 exit 1 98 exit 1
99 fi 99 fi
100 100
101 shift $((OPTIND-1)) 101 shift $((OPTIND-1))
102 # check Pass directory - First argument 102 # check Pass directory - First argument
103 if [ -e $1 ] && [ -d $1 ] 103 if [ -e $1 ] && [ -d $1 ]
104 then 104 then
105 PASS_DIR=$(readlink -e $1) 105 PASS_DIR=$(readlink -e $1)
106 BASENAME=$(basename $PASS_DIR)
106 else 107 else
107 print_error "can't find $1 directory" 108 print_error "can't find $1 directory"
108 exit 1 109 exit 1
109 fi 110 fi
110 111
112 print_info "[${BASENAME}] => P3 start | $(date +'%d/%m/%y %H:%M:%S')" 1
113
111 #-------------# 114 #-------------#
112 # GLOBAL VARS # 115 # GLOBAL VARS #
113 #-------------# 116 #-------------#
114 EXPLOITCONFPASS_CONFIG_FILE="$PASS_DIR/ExploitConfPass.cfg" 117 EXPLOITCONFPASS_CONFIG_FILE="$PASS_DIR/ExploitConfPass.cfg"
115 if [ -e $EXPLOITCONFPASS_CONFIG_FILE ] 118 if [ -e $EXPLOITCONFPASS_CONFIG_FILE ]
116 then 119 then
117 TRIGGER_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "TRIGGER_SPEERAL=" | cut -f2 -d"=") 120 TRIGGER_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "TRIGGER_SPEERAL=" | cut -f2 -d"=")
118 LEX_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_SPEERAL=" | cut -f2 -d"=") 121 LEX_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_SPEERAL=" | cut -f2 -d"=")
119 LEX_BINODE_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_BINODE_SPEERAL=" | cut -f2 -d"=") 122 LEX_BINODE_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_BINODE_SPEERAL=" | cut -f2 -d"=")
120 LST="" 123 LST=""
121 PLP_DIR_P1="" 124 PLP_DIR_P1=""
122 HMM="" 125 HMM=""
123 else 126 else
124 print_error "can't find $EXPLOITCONFPASS_CONFIG_FILE file" 127 print_error "[${BASENAME}] Can't find $EXPLOITCONFPASS_CONFIG_FILE file"
128 print_warn "[${BASENAME}] Will use default directories and files" 2
125 #exit 1 129 #exit 1
126 TRIGGER_SPEERAL=$PASS_DIR/trigg/speeral/ 130 TRIGGER_SPEERAL=$PASS_DIR/trigg/speeral/
127 LEX_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext 131 LEX_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext
128 LEX_BINODE_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext.bin 132 LEX_BINODE_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext.bin
129 fi 133 fi
134
130 SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg" 135 SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg"
131 if [ -e $SECONDPASS_CONFIG_FILE ] 136 if [ -e $SECONDPASS_CONFIG_FILE ]
132 then 137 then
133 LST=$(cat $SECONDPASS_CONFIG_FILE | grep "^LST=" | cut -f2 -d"=") 138 LST=$(cat $SECONDPASS_CONFIG_FILE | grep "^LST=" | cut -f2 -d"=")
134 HMM=$(cat $SECONDPASS_CONFIG_FILE | grep "^HMM=" | cut -f2 -d"=") 139 HMM=$(cat $SECONDPASS_CONFIG_FILE | grep "^HMM=" | cut -f2 -d"=")
135 PLP_DIR_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_DIR_P1=" | cut -f2 -d"=") 140 PLP_DIR_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_DIR_P1=" | cut -f2 -d"=")
141 PLP_FILE_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_FILE_P1=" | cut -f2 -d"=")
136 else 142 else
137 print_error "can't find $SECONDPASS_CONFIG_FILE file" 143 print_error "[${BASENAME}] Can't find $SECONDPASS_CONFIG_FILE file"
144 print_warn "[${BASENAME}] Will use default directories and files" 2
138 #exit 1 145 #exit 1
139 LST=$PASS_DIR/lists 146 LST=$PASS_DIR/lists
140 HMM=$PASS_DIR/hmm 147 HMM=$PASS_DIR/hmm
141 PLP_DIR_P1=$PASS_DIR/PLP 148 PLP_DIR_P1=$PASS_DIR/PLP
149 PLP_FILE_P1=$PASS_DIR/plp.lst
142 fi 150 fi
143 151
144 BASENAME=$(basename $PASS_DIR)
145 OUTPUT_DIR_BASENAME=$PASS_DIR 152 OUTPUT_DIR_BASENAME=$PASS_DIR
146 RES_DIR="$PASS_DIR/res_p3" 153 RES_DIR="$PASS_DIR/res_p3"
147 LOGFILE=$(dirname $PASS_DIR)"/info_p3.log" 154 LOGFILE=$PASS_DIR"/info_p3.log"
148 ERRORFILE=$(dirname $PASS_DIR)"/error_p3.log" 155 ERRORFILE=$PASS_DIR"/error_p3.log"
149 156
150 #------------------# 157 #------------------#
151 # Create WORKSPACE # 158 # Create WORKSPACE #
152 #------------------# 159 #------------------#
153 160
154 # Lock directory 161 # Lock directory
155 if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi 162 if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] THIRDPASS is locked -> exit" 2; exit 1; fi
156 rm "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" > /dev/null 2>&1 163 rm "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" > /dev/null 2>&1
157 touch "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" > /dev/null 2>&1 164 touch "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" > /dev/null 2>&1
158 165
159 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi 166 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi
160 mkdir -p $RES_DIR 167 mkdir -p $RES_DIR
161 print_info "Make directory $RES_DIR" 1 168 print_info "[${BASENAME}] Make directory $RES_DIR" 3
162 169
163 #--------------------# 170 #--------------------#
164 # Save configuration # 171 # Save configuration #
165 #--------------------# 172 #--------------------#
166 cp $THIRDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/ThirdPass.cfg 173 cp $THIRDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/ThirdPass.cfg
167 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/ThirdPass.cfg 174 echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/ThirdPass.cfg
168 print_info "save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1 175 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ThirdPass.cfg" 1
169 176
170 #--------------------------------------------------# 177 #--------------------------------------------------#
171 # Third Pass using trigger file (DECODING) # 178 # Third Pass using trigger file (DECODING) #
172 #--------------------------------------------------# 179 #--------------------------------------------------#
173 print_info "Launch Third Pass" 1 180 print_info "[${BASENAME}] Launch Third Pass" 1
174 181
175 ## Generate speeral config file adding trigger rep 182 ## Generate speeral config file adding trigger rep
176 cat $SPEERAL_CFG_PATH/$SPEERAL_CFG_FILE | sed -e "s|<nom>[^<]*</nom>|<nom>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext</nom>|g" \ 183 cat $SPEERAL_CFG_PATH/$SPEERAL_CFG_FILE | sed -e "s|<nom>[^<]*</nom>|<nom>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext</nom>|g" \
177 | sed -e "s|<binode>[^<]*</binode>|<binode>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext.bin</binode>|g" \ 184 | sed -e "s|<binode>[^<]*</binode>|<binode>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext.bin</binode>|g" \
178 | sed -e "s|<trigger><dir>[^<]*</dir></trigger>|<trigger><dir>$TRIGGER_SPEERAL</dir></trigger>|g" > $OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml 185 | sed -e "s|<trigger><dir>[^<]*</dir></trigger>|<trigger><dir>$TRIGGER_SPEERAL</dir></trigger>|g" > $OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml
179 SPEERAL_THIRD_CFG=$OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml 186 SPEERAL_THIRD_CFG=$OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml
187 print_info "[${BASENAME}] Third Pass Speeral Config file ${SPEERAL_THIRD_CFG}" 3
180 188
181 # for all speaker 189 # for all speaker
182 for lspeaker in $(ls $LST/*.lst) 190 for lspeaker in $(ls $LST/*.lst)
183 do 191 do
184 speaker=$(basename $lspeaker ".lst") 192 speaker=$(basename $lspeaker ".lst")
185 # for all AM 193 # for all AM
186 for (( i=0; $i<${#MTAG[@]} ; i++ )) 194 for (( i=0; $i<${#MTAG[@]} ; i++ ))
187 do 195 do
188 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then 196 if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then
189 type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst") 197 type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst")
190 ## if is the good AM for the speaker 198 ## if is the good AM for the speaker
191 if [ -n "$type" ] 199 if [ -n "$type" ]
192 then 200 then
193 ## Speeral decoding 201 ## Speeral decoding
194 if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ] 202 if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ]
195 then 203 then
196 print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3 204 print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3
197 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock 205 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock
198 else 206 else
199 print_warn "$HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2 207 print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2
208 print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3
200 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock 209 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock
201 fi 210 fi
202 211
203 if [ $CHECK -eq 1 ] 212 if [ $CHECK -eq 1 ]
204 then 213 then
205 check_third_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR" 214 check_third_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR"
206 if [ $? -eq 1 ] 215 if [ $? -eq 1 ]
207 then 216 then
208 echo -e "ERROR : Speeral $LST/$speaker.lst\n[" >> $ERRORFILE 217
218 print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2
219 print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst"
209 ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp 220 ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
210 diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $ERRORFILE 221 print_log_file $LOGFILE "Segs (and treil) not done :\n["
211 echo -e "] " >> $ERRORFILE 222 diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $LOGFILE
223 print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]"
212 rm ${OUTPUT_DIR_BASENAME}/.tmp 224 rm ${OUTPUT_DIR_BASENAME}/.tmp
213 #exit 1
214 fi 225 fi
215 fi 226 fi
216 break 227 break
217 fi 228 fi
218 fi 229 fi
219 done 230 done
220 done 231 done
221 232
222 ## Check missing seg and log it 233 ## Check missing seg and log it
223 ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
224 echo -e "$BASENAME P3 END\n[" >> $LOGFILE
225 diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $LOGFILE
226 echo -e "] $BASENAME" >> $LOGFILE
227 rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1
228 234
235 ## Check missing seg and log it
236 if [ $CHECK -eq 1 ]
237 then
238 ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
239 todo=$(cat ${PLP_FILE_P1} | wc -l)
240 if [ $todo -eq 0 ]; then todo=1;fi
241 notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l)))
242 pourcentage=$((($notdone*100)/$todo))
243 if [ $notdone -ne 0 ]
244 then
245 print_error "[${BASENAME}] Check $ERRORFILE"
246 print_log_file "$ERRORFILE" "ERROR : Treil not done ["
247 diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE
248 print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME"
249 else
250 print_log_file "$LOGFILE" "P3 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')"
251 fi
252 rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1
253 fi
254
229 #---------------# 255 #---------------#
230 # Convert res # 256 # Convert res #
231 #---------------# 257 #---------------#
232 258
259 print_info "[${BASENAME}] Convert .res into .ctm" 1
233 # .res => .ctm 260 # .res => .ctm
234 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.ctm 261 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.ctm
262 print_info "[${BASENAME}] Convert .res into .trs" 1
235 # .res => .trs 263 # .res => .trs
236 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 264 echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
237 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg 265 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
238 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1 266 rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1
267 print_info "[${BASENAME}] Convert .res into .txt" 1
239 # .res => .txt 268 # .res => .txt
240 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.txt 269 $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.txt
241 270
tools/scripts/Tools.sh
Diff comments   View file @ b427f10
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ################################################# 3 #################################################
4 # File : Tools.sh # 4 # File : Tools.sh #
5 # Brief : Tool functions # 5 # Brief : Tool functions #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # Version : 1.0 # 7 # Version : 1.0 #
8 # Date : 13/06/13 # 8 # Date : 13/06/13 #
9 ################################################# 9 #################################################
10 10
11 # Type 11 # Type
12 ERROR=2 12 ERROR=2
13 WARN=1 13 WARN=1
14 INFO=0 14 INFO=0
15 15
16 # Global Verbose variable (have to be define in scripts) 16 # Global Verbose variable (have to be define in scripts)
17 # 0 : mute 17 # 0 : mute
18 # 1 : basic info (start,config,end) 18 # 1 : basic info (start,config,end)
19 # 2 : add process 19 # 2 : add process
20 # 3 : all 20 # 3 : all
21 VERBOSE=0 21 VERBOSE=0
22 22
23 # print_message( verbose_level, message) 23 # print_message( verbose_level, message)
24 # - message_type : ERROR, WARN, INFO 24 # - message_type : ERROR, WARN, INFO
25 # - verbose_level : level to print message 25 # - verbose_level : level to print message
26 # - message : the message to print 26 # - message : the message to print
27 function print_message() 27 function print_message()
28 { 28 {
29 if [ $# -eq 3 ] 29 if [ $# -eq 3 ]
30 then 30 then
31 # print error 31 # print error
32 if [ $1 -eq $ERROR ]; then echo -ne "###ERROR : $3\n" > /dev/stderr; exit 0; fi 32 if [ $1 -eq $ERROR ]; then echo -ne "###ERROR : $3\n" > /dev/stderr; return 0; fi
33 # check verbose level 33 # check verbose level
34 if [ $2 -le $VERBOSE ] 34 if [ $2 -le $VERBOSE ]
35 then 35 then
36 #for (( dec=1 ; $dec<$2 ; dec++ )); do echo -ne "\t"; done 36 #for (( dec=1 ; $dec<$2 ; dec++ )); do echo -ne "\t"; done
37 # print warn 37 # print warn
38 if [ $1 = $WARN ]; then echo -ne "##WARNING : $3\n"; exit 0; fi 38 if [ $1 = $WARN ]; then echo -ne "##WARNING : $3\n"; return 0; fi
39 # default info 39 # default info
40 echo -ne "#INFO : $3\n" 40 echo -ne "#INFO : $3\n"
41 fi 41 fi
42 fi 42 fi
43 } 43 }
44 44
45 # print_error(message) 45 # print_error(message)
46 # - message : error message to print 46 # - message : error message to print
47 function print_error() 47 function print_error()
48 { 48 {
49 print_message $ERROR 3 "$1" 49 print_message $ERROR 3 "$1"
50 } 50 }
51 51
52 # print_warn(message,[level]) 52 # print_warn(message,[level])
53 # - message : message to print 53 # - message : message to print
54 # - level : optional option - verbose level to print 54 # - level : optional option - verbose level to print
55 function print_warn() 55 function print_warn()
56 { 56 {
57 if [ $# -eq 2 ] 57 if [ $# -eq 2 ]
58 then 58 then
59 print_message $WARN $2 "$1" 59 print_message $WARN $2 "$1"
60 else 60 else
61 print_message $WARN $VERBOSE "$1" 61 print_message $WARN $VERBOSE "$1"
62 fi 62 fi
63 } 63 }
64 64
65 # print_info(message,[level]) 65 # print_info(message,[level])
66 # - message : message to print 66 # - message : message to print
67 # - level : optional option - verbose level to print 67 # - level : optional option - verbose level to print
68 function print_info() 68 function print_info()
69 { 69 {
70 if [ $# -eq 2 ] 70 if [ $# -eq 2 ]
71 then 71 then
72 print_message $INFO $2 "$1" 72 print_message $INFO $2 "$1"
73 else 73 else
74 print_message $INFO $VERBOSE "$1" 74 print_message $INFO $VERBOSE "$1"
75 fi 75 fi
76 } 76 }
77 77
78 # print_log_file(filename,message) 78 # print_log_file(filename,message)
79 # - filename : full path to log file 79 # - filename : full path to log file
80 # - message : message to log 80 # - message : message to log
81 print_log_file() 81 print_log_file()
82 { 82 {
83 if [ $# -eq 2 ] 83 if [ $# -eq 2 ]
84 then 84 then
85 echo -e "$2" >> $1 85 echo -e "$2" >> "$1"
86 fi 86 fi
87 } 87 }
88 88