Commit f37e72eafc1e5626cb65707c3d7994e73766bd6e
1 parent
d67610f632
Exists in
master
up
Showing 8 changed files with 18 additions and 2 deletions Inline Diff
main_tools/ConfPass.sh
1 | #!/bin/bash | 1 | #!/bin/bash |
2 | 2 | ||
3 | ##################################################### | 3 | ##################################################### |
4 | # File : ConfPass.sh # | 4 | # File : ConfPass.sh # |
5 | # Brief : Process the ASR Confidence pass # | 5 | # Brief : Process the ASR Confidence pass # |
6 | # Author : Jean-François Rey # | 6 | # Author : Jean-François Rey # |
7 | # (base on Emmanuel Ferreira # | 7 | # (base on Emmanuel Ferreira # |
8 | # and hugo Mauchrétien works) # | 8 | # and hugo Mauchrétien works) # |
9 | # Version : 1.0 # | 9 | # Version : 1.0 # |
10 | # Date : 17/06/13 # | 10 | # Date : 17/06/13 # |
11 | ##################################################### | 11 | ##################################################### |
12 | 12 | ||
13 | echo "### ConfPass.sh ###" | ||
14 | |||
13 | #Check OTMEDIA_HOME env var | 15 | #Check OTMEDIA_HOME env var |
14 | if [ -z ${OTMEDIA_HOME} ] | 16 | if [ -z ${OTMEDIA_HOME} ] |
15 | then | 17 | then |
16 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) | 18 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) |
17 | export OTMEDIA_HOME=$OTMEDIA_HOME | 19 | export OTMEDIA_HOME=$OTMEDIA_HOME |
18 | fi | 20 | fi |
19 | 21 | ||
20 | 22 | ||
21 | # where is ConfPath.sh | 23 | # where is ConfPath.sh |
22 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) | 24 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) |
23 | 25 | ||
24 | # Scripts Path | 26 | # Scripts Path |
25 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts | 27 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts |
26 | 28 | ||
27 | # Include scripts | 29 | # Include scripts |
28 | . $SCRIPT_PATH"/Tools.sh" | 30 | . $SCRIPT_PATH"/Tools.sh" |
29 | . $SCRIPT_PATH"/CheckConfPass.sh" | 31 | . $SCRIPT_PATH"/CheckConfPass.sh" |
30 | 32 | ||
31 | # where is FirstPass.cfg | 33 | # where is FirstPass.cfg |
32 | CONFPASS_CONFIG_FILE="$OTMEDIA_HOME/cfg/ConfPass.cfg" | 34 | CONFPASS_CONFIG_FILE="$OTMEDIA_HOME/cfg/ConfPass.cfg" |
33 | if [ -e $CONFPASS_CONFIG_FILE ] | 35 | if [ -e $CONFPASS_CONFIG_FILE ] |
34 | then | 36 | then |
35 | . $CONFPASS_CONFIG_FILE | 37 | . $CONFPASS_CONFIG_FILE |
36 | else | 38 | else |
37 | echo "ERROR : Can't find configuration file $CONFPASS_CONFIG_FILE" > /dev/stderr | 39 | echo "ERROR : Can't find configuration file $CONFPASS_CONFIG_FILE" > /dev/stderr |
38 | exit 1 | 40 | exit 1 |
39 | fi | 41 | fi |
40 | 42 | ||
41 | #---------------# | 43 | #---------------# |
42 | # Parse Options # | 44 | # Parse Options # |
43 | #---------------# | 45 | #---------------# |
44 | while getopts ":hDv:cr" opt | 46 | while getopts ":hDv:cr" opt |
45 | do | 47 | do |
46 | case $opt in | 48 | case $opt in |
47 | h) | 49 | h) |
48 | echo -e "$0 [OPTIONS] <INPUT_DIRECTORY> <TREIL_DIRECTORY_NAME>\n" | 50 | echo -e "$0 [OPTIONS] <INPUT_DIRECTORY> <TREIL_DIRECTORY_NAME>\n" |
49 | echo -e "\t Options:" | 51 | echo -e "\t Options:" |
50 | echo -e "\t\t-h :\tprint this message" | 52 | echo -e "\t\t-h :\tprint this message" |
51 | echo -e "\t\t-D :\tDEBUG mode on" | 53 | echo -e "\t\t-D :\tDEBUG mode on" |
52 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" | 54 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" |
53 | echo -e "\t\t-c :\t Check process, stop if error detected" | 55 | echo -e "\t\t-c :\t Check process, stop if error detected" |
54 | echo -e "\t\t-r :\tForce to rerun confpas without deleting existing files" | 56 | echo -e "\t\t-r :\tForce to rerun confpas without deleting existing files" |
55 | exit 1 | 57 | exit 1 |
56 | ;; | 58 | ;; |
57 | D) | 59 | D) |
58 | DEBUG=1 | 60 | DEBUG=1 |
59 | ;; | 61 | ;; |
60 | v) | 62 | v) |
61 | VERBOSE=$OPTARG | 63 | VERBOSE=$OPTARG |
62 | ;; | 64 | ;; |
63 | c) | 65 | c) |
64 | CHECK=1 | 66 | CHECK=1 |
65 | ;; | 67 | ;; |
66 | r) | 68 | r) |
67 | RERUN=1 | 69 | RERUN=1 |
68 | ;; | 70 | ;; |
69 | :) | 71 | :) |
70 | echo "Option -$OPTARG requires an argument." > /dev/stderr | 72 | echo "Option -$OPTARG requires an argument." > /dev/stderr |
71 | exit 1 | 73 | exit 1 |
72 | ;; | 74 | ;; |
73 | \?) | 75 | \?) |
74 | echo "BAD USAGE : unknow opton -$OPTARG" | 76 | echo "BAD USAGE : unknow opton -$OPTARG" |
75 | #exit 1 | 77 | #exit 1 |
76 | ;; | 78 | ;; |
77 | esac | 79 | esac |
78 | done | 80 | done |
79 | 81 | ||
80 | # mode debug enable | 82 | # mode debug enable |
81 | if [ $DEBUG -eq 1 ] | 83 | if [ $DEBUG -eq 1 ] |
82 | then | 84 | then |
83 | set -x | 85 | set -x |
84 | echo -e "## Mode DEBUG ON ##" | 86 | echo -e "## Mode DEBUG ON ##" |
85 | REDIRECTION_OUTPUT="" | 87 | REDIRECTION_OUTPUT="" |
86 | else | 88 | else |
87 | REDIRECTION_OUTPUT=" 2> /dev/null" | 89 | REDIRECTION_OUTPUT=" 2> /dev/null" |
88 | fi | 90 | fi |
89 | 91 | ||
90 | # mode verbose enable | 92 | # mode verbose enable |
91 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi | 93 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi |
92 | 94 | ||
93 | # Check USAGE by arguments number | 95 | # Check USAGE by arguments number |
94 | if [ $(($#-($OPTIND-1))) -ne 2 ] | 96 | if [ $(($#-($OPTIND-1))) -ne 2 ] |
95 | then | 97 | then |
96 | echo "BAD USAGE : ConfPass.sh [OPTIONS] <INPUT_DIR> <TREIL_DIRECTORY_NAME>" | 98 | echo "BAD USAGE : ConfPass.sh [OPTIONS] <INPUT_DIR> <TREIL_DIRECTORY_NAME>" |
97 | echo "$0 -h for more info" | 99 | echo "$0 -h for more info" |
98 | exit 1 | 100 | exit 1 |
99 | fi | 101 | fi |
100 | 102 | ||
101 | shift $((OPTIND-1)) | 103 | shift $((OPTIND-1)) |
102 | # check input directory - first argument | 104 | # check input directory - first argument |
103 | if [ ! -e $1 ] | 105 | if [ ! -e $1 ] |
104 | then | 106 | then |
105 | print_error "can't open $1" | 107 | print_error "can't open $1" |
106 | exit 1 | 108 | exit 1 |
107 | fi | 109 | fi |
108 | # check treil input directory - second argument | 110 | # check treil input directory - second argument |
109 | if [ ! -e $1/$2 ] | 111 | if [ ! -e $1/$2 ] |
110 | then | 112 | then |
111 | print_error "can't open $1/$2" | 113 | print_error "can't open $1/$2" |
112 | exit 1 | 114 | exit 1 |
113 | fi | 115 | fi |
114 | 116 | ||
115 | #-------------# | 117 | #-------------# |
116 | # GLOBAL VARS # | 118 | # GLOBAL VARS # |
117 | #-------------# | 119 | #-------------# |
118 | INPUT_DIR=$(readlink -e $1) | 120 | INPUT_DIR=$(readlink -e $1) |
119 | OUTPUT_DIR=$INPUT_DIR | 121 | OUTPUT_DIR=$INPUT_DIR |
120 | BASENAME=$(basename $OUTPUT_DIR) | 122 | BASENAME=$(basename $OUTPUT_DIR) |
121 | RES_NAME=$2 | 123 | RES_NAME=$2 |
122 | RES_P="${INPUT_DIR}/${RES_NAME}" | 124 | RES_P="${INPUT_DIR}/${RES_NAME}" |
123 | USF_FILE=${INPUT_DIR}/${BASENAME}.${RES_NAME}.usf | 125 | USF_FILE=${INPUT_DIR}/${BASENAME}.${RES_NAME}.usf |
124 | CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME" | 126 | CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME" |
125 | RES_CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME/scored_ctm" | 127 | RES_CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME/scored_ctm" |
126 | LOGFILE="${OUTPUT_DIR_BASENAME}/info_conf.log" | 128 | LOGFILE="${OUTPUT_DIR_BASENAME}/info_conf.log" |
127 | ERRORFILE="${OUTPUT_DIR_BASENAME}/error_conf.log" | 129 | ERRORFILE="${OUTPUT_DIR_BASENAME}/error_conf.log" |
128 | 130 | ||
129 | #------------------# | 131 | #------------------# |
130 | # Create Workspace # | 132 | # Create Workspace # |
131 | #------------------# | 133 | #------------------# |
132 | # Lock directory | 134 | # Lock directory |
133 | if [ -e "$OUTPUT_DIR/CONFPASS.lock" ] && [ $RERUN -eq 0 ] | 135 | if [ -e "$OUTPUT_DIR/CONFPASS.lock" ] && [ $RERUN -eq 0 ] |
134 | then | 136 | then |
135 | print_warn "[${BASENAME}] Confpass is locked -> exit" 2 | 137 | print_warn "[${BASENAME}] Confpass is locked -> exit" 2 |
136 | exit 1 | 138 | exit 1 |
137 | fi | 139 | fi |
138 | rm "$OUTPUT_DIR/CONFPASS.unlock" > /dev/null 2>&1 | 140 | rm "$OUTPUT_DIR/CONFPASS.unlock" > /dev/null 2>&1 |
139 | touch "$OUTPUT_DIR/CONFPASS.lock" > /dev/null 2>&1 | 141 | touch "$OUTPUT_DIR/CONFPASS.lock" > /dev/null 2>&1 |
140 | if [ $RERUN -eq 0 ]; then rm -r $CONF_DIR > /dev/null 2>&1; fi | 142 | if [ $RERUN -eq 0 ]; then rm -r $CONF_DIR > /dev/null 2>&1; fi |
141 | if [ $RERUN -eq 1 ]; then rm $USF_FILE > /dev/null 2>&1; fi | 143 | if [ $RERUN -eq 1 ]; then rm $USF_FILE > /dev/null 2>&1; fi |
142 | mkdir -p $CONF_DIR > /dev/null 2>&1 | 144 | mkdir -p $CONF_DIR > /dev/null 2>&1 |
143 | mkdir -p $RES_CONF_DIR > /dev/null 2>&1 | 145 | mkdir -p $RES_CONF_DIR > /dev/null 2>&1 |
144 | rm $LOGFILE $ERRORFILE > /dev/null 2>&1 | 146 | rm $LOGFILE $ERRORFILE > /dev/null 2>&1 |
145 | 147 | ||
146 | #---------------# | 148 | #---------------# |
147 | # Check Pass # | 149 | # Check Pass # |
148 | #---------------# | 150 | #---------------# |
149 | print_info "[${BASENAME}] Check Conf Pass directory ${RES_NAME}" 1 | 151 | print_info "[${BASENAME}] Check Conf Pass directory ${RES_NAME}" 1 |
150 | # if usf contains more than 49% of 0.600 confidence -> usf error | 152 | # if usf contains more than 49% of 0.600 confidence -> usf error |
151 | if [ -s $USF_FILE ] | 153 | if [ -s $USF_FILE ] |
152 | then | 154 | then |
153 | conftozerosix=$(grep -c -E 'confidence="0.600"' "${USF_FILE}") | 155 | conftozerosix=$(grep -c -E 'confidence="0.600"' "${USF_FILE}") |
154 | confall=$(grep -c -E 'confidence=' "${USF_FILE}") | 156 | confall=$(grep -c -E 'confidence=' "${USF_FILE}") |
155 | if [ $confall -gt 0 ] | 157 | if [ $confall -gt 0 ] |
156 | then | 158 | then |
157 | pourcentageofzerosix=$((($conftozerosix*100)/$confall)) | 159 | pourcentageofzerosix=$((($conftozerosix*100)/$confall)) |
158 | if [ $pourcentageofzerosix -gt 49 ] | 160 | if [ $pourcentageofzerosix -gt 49 ] |
159 | then | 161 | then |
160 | print_warn "[${BASENAME}] ${BASENAME}.${RES_NAME}.usf got $pourcentageofzerosix% of 0.600 confidence" 2 | 162 | print_warn "[${BASENAME}] ${BASENAME}.${RES_NAME}.usf got $pourcentageofzerosix% of 0.600 confidence" 2 |
161 | print_info "[${BASENAME}] bad usf ${RES_NAME}, will do it again" 1 | 163 | print_info "[${BASENAME}] bad usf ${RES_NAME}, will do it again" 1 |
162 | mv "${USF_FILE}" "${USF_FILE}.back" | 164 | mv "${USF_FILE}" "${USF_FILE}.back" |
163 | rm -r $CONF_DIR > /dev/null 2>&1 | 165 | rm -r $CONF_DIR > /dev/null 2>&1 |
164 | else | 166 | else |
165 | print_warn "[${BASENAME}] ${USF_FILE} already done, skipping it" 1 | 167 | print_warn "[${BASENAME}] ${USF_FILE} already done, skipping it" 1 |
166 | exit 0 | 168 | exit 0 |
167 | fi | 169 | fi |
168 | fi | 170 | fi |
169 | else | 171 | else |
170 | print_info "[${BASENAME}] No USF file already done, continue..." 1 | 172 | print_info "[${BASENAME}] No USF file already done, continue..." 1 |
171 | fi | 173 | fi |
172 | 174 | ||
173 | # Check if treil are here | 175 | # Check if treil are here |
174 | nbres_p1=$(cat ${INPUT_DIR}/plp.lst | wc -l) | 176 | nbres_p1=$(cat ${INPUT_DIR}/plp.lst | wc -l) |
175 | nbtreil_p=$(ls $RES_P/*.treil 2> /dev/null | wc -l) | 177 | nbtreil_p=$(ls $RES_P/*.treil 2> /dev/null | wc -l) |
176 | if [ $nbtreil_p -eq 0 ] | 178 | if [ $nbtreil_p -eq 0 ] |
177 | then | 179 | then |
178 | print_error "[${BASENAME}] No ${RES_NAME} Pass, No .treil -> exit ConfPass" | 180 | print_error "[${BASENAME}] No ${RES_NAME} Pass, No .treil -> exit ConfPass" |
179 | if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ${RES_NAME} Pass, No .treil -> exit ConfPass" ;fi | 181 | if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ${RES_NAME} Pass, No .treil -> exit ConfPass" ;fi |
180 | exit 1 | 182 | exit 1 |
181 | else | 183 | else |
182 | #Check if more then 89% of treil are done | 184 | #Check if more then 89% of treil are done |
183 | if [ $nbres_p1 -gt 0 ] | 185 | if [ $nbres_p1 -gt 0 ] |
184 | then | 186 | then |
185 | pourcentage=$((($nbtreil_p*100)/$nbres_p1)) | 187 | pourcentage=$((($nbtreil_p*100)/$nbres_p1)) |
186 | if [ $pourcentage -gt 89 ] | 188 | if [ $pourcentage -gt 89 ] |
187 | then | 189 | then |
188 | print_info "[${BASENAME}] ${RES_NAME}/*.treil are here" 1 | 190 | print_info "[${BASENAME}] ${RES_NAME}/*.treil are here" 1 |
189 | else | 191 | else |
190 | print_warn "[${BASENAME}] not enough ${RES_NAME} treil" 2 | 192 | print_warn "[${BASENAME}] not enough ${RES_NAME} treil" 2 |
191 | if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "Not enough ${RES_NAME} treil " | 193 | if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "Not enough ${RES_NAME} treil " |
192 | fi | 194 | fi |
193 | fi | 195 | fi |
194 | fi | 196 | fi |
195 | 197 | ||
196 | #------# | 198 | #------# |
197 | # Save # | 199 | # Save # |
198 | #------# | 200 | #------# |
199 | cp $CONFPASS_CONFIG_FILE $OUTPUT_DIR/ConfPass.cfg | 201 | cp $CONFPASS_CONFIG_FILE $OUTPUT_DIR/ConfPass.cfg |
200 | echo "RES_CONF_DIR=$RES_CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg | 202 | echo "RES_CONF_DIR=$RES_CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg |
201 | echo "CONF_DIR=$CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg | 203 | echo "CONF_DIR=$CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg |
202 | print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ConfPass.cfg" 1 | 204 | print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ConfPass.cfg" 1 |
203 | 205 | ||
204 | #--------------------# | 206 | #--------------------# |
205 | # CONFIDENCE MEASURE # | 207 | # CONFIDENCE MEASURE # |
206 | #--------------------# | 208 | #--------------------# |
207 | 209 | ||
208 | # Check pourcentage of scored_ctm already done, if < 85% do confidence measure | 210 | # Check pourcentage of scored_ctm already done, if < 85% do confidence measure |
209 | nbres_p=$(ls ${RES_P}/*.treil | wc -l) | 211 | nbres_p=$(ls ${RES_P}/*.treil | wc -l) |
210 | nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l) | 212 | nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l) |
211 | if [ $nbres_p -gt 0 ] | 213 | if [ $nbres_p -gt 0 ] |
212 | then | 214 | then |
213 | pourcentageres=$((($nbconf*100)/$nbres_p)) | 215 | pourcentageres=$((($nbconf*100)/$nbres_p)) |
214 | if [ $pourcentageres -lt 85 ] | 216 | if [ $pourcentageres -lt 85 ] |
215 | then | 217 | then |
216 | print_info "[${BASENAME}] Calcul Confidence $INPUT_DIR $RES_NAME" 1 | 218 | print_info "[${BASENAME}] Calcul Confidence $INPUT_DIR $RES_NAME" 1 |
217 | $MAIN_SCRIPT_PATH/ConfidenceMeasure.sh $INPUT_DIR $RES_NAME $REDIRECTION_OUTPUT | 219 | $MAIN_SCRIPT_PATH/ConfidenceMeasure.sh $INPUT_DIR $RES_NAME $REDIRECTION_OUTPUT |
218 | 220 | ||
219 | else | 221 | else |
220 | print_info "[${BASENAME}] Skipping Confidence Calcul $INPUT_DIR/$RES_NAME" 1 | 222 | print_info "[${BASENAME}] Skipping Confidence Calcul $INPUT_DIR/$RES_NAME" 1 |
221 | fi | 223 | fi |
222 | fi | 224 | fi |
223 | 225 | ||
224 | ### Check scored_ctm number res files ! | 226 | ### Check scored_ctm number res files ! |
225 | if [ $CHECK -eq 1 ] | 227 | if [ $CHECK -eq 1 ] |
226 | then | 228 | then |
227 | nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l) | 229 | nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l) |
228 | if [ $nbres_p -ne $nbconf ] | 230 | if [ $nbres_p -ne $nbconf ] |
229 | then | 231 | then |
230 | print_warn "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" 2 | 232 | print_warn "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" 2 |
231 | print_log_file $LOGFILE "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" | 233 | print_log_file $LOGFILE "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" |
232 | fi | 234 | fi |
233 | fi | 235 | fi |
234 | 236 | ||
235 | #---------------------------# | 237 | #---------------------------# |
236 | # FROM RES WITH CONF => USF # | 238 | # FROM RES WITH CONF => USF # |
237 | #---------------------------# | 239 | #---------------------------# |
238 | print_info "[${BASENAME}] Create USF file for $RES_P" 1 | 240 | print_info "[${BASENAME}] Create USF file for $RES_P" 1 |
239 | for f in `ls ${RES_CONF_DIR}`; do $SCRIPT_PATH/formatRES.pl $RES_CONF_DIR/$f; done | 241 | for f in `ls ${RES_CONF_DIR}`; do $SCRIPT_PATH/formatRES.pl $RES_CONF_DIR/$f; done |
240 | # create USF configuration file | 242 | # create USF configuration file |
241 | echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR/$BASENAME.seg" > $OUTPUT_DIR/$BASENAME.usf_cfg | 243 | echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR/$BASENAME.seg" > $OUTPUT_DIR/$BASENAME.usf_cfg |
242 | # create USF file | 244 | # create USF file |
243 | print_info "$SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg $REDIRECTION_OUTPUT" 3 | 245 | print_info "$SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg $REDIRECTION_OUTPUT" 3 |
244 | $SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg $REDIRECTION_OUTPUT | 246 | $SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg $REDIRECTION_OUTPUT |
245 | rm $OUTPUT_DIR/$BASENAME.usf_cfg > /dev/null 2>&1 | 247 | rm $OUTPUT_DIR/$BASENAME.usf_cfg > /dev/null 2>&1 |
246 | cat $USF_FILE.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f > $USF_FILE | 248 | cat $USF_FILE.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f > $USF_FILE |
247 | cp $USF_FILE ${OUTPUT_DIR}/${BASENAME}.usf | 249 | cp $USF_FILE ${OUTPUT_DIR}/${BASENAME}.usf |
248 | rm $USF_FILE.tmp > /dev/null 2>&1 | 250 | rm $USF_FILE.tmp > /dev/null 2>&1 |
249 | 251 | ||
250 | #----------------# | 252 | #----------------# |
251 | # Check USF file # | 253 | # Check USF file # |
252 | #----------------# | 254 | #----------------# |
253 | if [ $CHECK -eq 1 ] | 255 | if [ $CHECK -eq 1 ] |
254 | then | 256 | then |
255 | check_conf_pass_usf "$OUTPUT_DIR/$BASENAME.usf" | 257 | check_conf_pass_usf "$OUTPUT_DIR/$BASENAME.usf" |
256 | if [ $? -eq 1 ] | 258 | if [ $? -eq 1 ] |
257 | then | 259 | then |
258 | print_error "[${BASENAME}] Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf" | 260 | print_error "[${BASENAME}] Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf" |
259 | print_log_file $ERRORFILE "ERROR : Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf" | 261 | print_log_file $ERRORFILE "ERROR : Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf" |
260 | exit 1 | 262 | exit 1 |
261 | fi | 263 | fi |
262 | fi | 264 | fi |
263 | 265 | ||
264 | #-------# | 266 | #-------# |
265 | # CLOSE # | 267 | # CLOSE # |
266 | #-------# | 268 | #-------# |
267 | # Seem OK | 269 | # Seem OK |
268 | print_info "[${BASENAME}] <= ConfPass End | $(date +'%d/%m/%y %H:%M:%S')" 1 | 270 | print_info "[${BASENAME}] <= ConfPass End | $(date +'%d/%m/%y %H:%M:%S')" 1 |
269 | 271 | ||
270 | # unlock directory | 272 | # unlock directory |
271 | mv "$OUTPUT_DIR/CONFPASS.lock" "$OUTPUT_DIR/CONFPASS.unlock" | 273 | mv "$OUTPUT_DIR/CONFPASS.lock" "$OUTPUT_DIR/CONFPASS.unlock" |
272 | 274 | ||
273 | 275 |
main_tools/ExploitConfidencePass.sh
1 | #!/bin/bash | 1 | #!/bin/bash |
2 | 2 | ||
3 | ##################################################### | 3 | ##################################################### |
4 | # File : ExploitConfidencePass.sh # | 4 | # File : ExploitConfidencePass.sh # |
5 | # Brief : Exploit the ASR confidence pass to : # | 5 | # Brief : Exploit the ASR confidence pass to : # |
6 | # -> boost the confident zone # | 6 | # -> boost the confident zone # |
7 | # -> find alternative in non confident zone | 7 | # -> find alternative in non confident zone |
8 | # -> dynamicly extend the lexicon # | 8 | # -> dynamicly extend the lexicon # |
9 | # Author : Jean-François Rey # | 9 | # Author : Jean-François Rey # |
10 | # (base on Emmanuel Ferreira # | 10 | # (base on Emmanuel Ferreira # |
11 | # and Hugo Mauchrétien works) # | 11 | # and Hugo Mauchrétien works) # |
12 | # Version : 1.0 # | 12 | # Version : 1.0 # |
13 | # Date : 25/06/13 # | 13 | # Date : 25/06/13 # |
14 | ##################################################### | 14 | ##################################################### |
15 | 15 | ||
16 | echo "### ExploitConfidencePass.sh ###" | ||
17 | |||
18 | |||
16 | # Check OTMEDIA_HOME env var | 19 | # Check OTMEDIA_HOME env var |
17 | if [ -z ${OTMEDIA_HOME} ] | 20 | if [ -z ${OTMEDIA_HOME} ] |
18 | then | 21 | then |
19 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) | 22 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) |
20 | export OTMEDIA_HOME=$OTMEDIA_HOME | 23 | export OTMEDIA_HOME=$OTMEDIA_HOME |
21 | fi | 24 | fi |
22 | 25 | ||
23 | # where is ExploitConfidencePass.sh | 26 | # where is ExploitConfidencePass.sh |
24 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) | 27 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) |
25 | 28 | ||
26 | if [ -z ${SCRIPT_PATH} ] | 29 | if [ -z ${SCRIPT_PATH} ] |
27 | then | 30 | then |
28 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts | 31 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts |
29 | fi | 32 | fi |
30 | 33 | ||
31 | # Include scripts | 34 | # Include scripts |
32 | . $SCRIPT_PATH"/Tools.sh" | 35 | . $SCRIPT_PATH"/Tools.sh" |
33 | . $SCRIPT_PATH"/CheckExploitConfPass.sh" | 36 | . $SCRIPT_PATH"/CheckExploitConfPass.sh" |
34 | 37 | ||
35 | # where is ExploitConfidencePass.cfg | 38 | # where is ExploitConfidencePass.cfg |
36 | EXPLOITCONFIDENCEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ExploitConfidencePass.cfg" | 39 | EXPLOITCONFIDENCEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ExploitConfidencePass.cfg" |
37 | if [ -e $EXPLOITCONFIDENCEPASS_CONFIG_FILE ] | 40 | if [ -e $EXPLOITCONFIDENCEPASS_CONFIG_FILE ] |
38 | then | 41 | then |
39 | . $EXPLOITCONFIDENCEPASS_CONFIG_FILE | 42 | . $EXPLOITCONFIDENCEPASS_CONFIG_FILE |
40 | else | 43 | else |
41 | echo "ERROR : Can't find configuration file $EXPLOITCONFIDENCEPASS_CONFIG_FILE" >&2 | 44 | echo "ERROR : Can't find configuration file $EXPLOITCONFIDENCEPASS_CONFIG_FILE" >&2 |
42 | exit 1 | 45 | exit 1 |
43 | fi | 46 | fi |
44 | 47 | ||
45 | #---------------# | 48 | #---------------# |
46 | # Parse Options # | 49 | # Parse Options # |
47 | #---------------# | 50 | #---------------# |
48 | while getopts ":hDv:cf:r" opt | 51 | while getopts ":hDv:cf:r" opt |
49 | do | 52 | do |
50 | case $opt in | 53 | case $opt in |
51 | h) | 54 | h) |
52 | echo -e "$0 [OPTIONS] <INPUT_DIRECTORY>\n" | 55 | echo -e "$0 [OPTIONS] <INPUT_DIRECTORY>\n" |
53 | echo -e "\t Options:" | 56 | echo -e "\t Options:" |
54 | echo -e "\t\t-h :\tprint this message" | 57 | echo -e "\t\t-h :\tprint this message" |
55 | echo -e "\t\t-D :\tDEBUG mode on" | 58 | echo -e "\t\t-D :\tDEBUG mode on" |
56 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" | 59 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" |
57 | echo -e "\t\t-c :\tCheck process, stop if error detected" | 60 | echo -e "\t\t-c :\tCheck process, stop if error detected" |
58 | echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)" | 61 | echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)" |
59 | echo -e "\t\t-r n :\tforce rerun without deleting files" | 62 | echo -e "\t\t-r n :\tforce rerun without deleting files" |
60 | exit 1 | 63 | exit 1 |
61 | ;; | 64 | ;; |
62 | D) | 65 | D) |
63 | DEBUG=1 | 66 | DEBUG=1 |
64 | ;; | 67 | ;; |
65 | v) | 68 | v) |
66 | VERBOSE=$OPTARG | 69 | VERBOSE=$OPTARG |
67 | ;; | 70 | ;; |
68 | c) | 71 | c) |
69 | CHECK=1 | 72 | CHECK=1 |
70 | ;; | 73 | ;; |
71 | f) | 74 | f) |
72 | FORKS="--forks $OPTARG" | 75 | FORKS="--forks $OPTARG" |
73 | ;; | 76 | ;; |
74 | r) | 77 | r) |
75 | RERUN=1 | 78 | RERUN=1 |
76 | ;; | 79 | ;; |
77 | :) | 80 | :) |
78 | echo "Option -$OPTARG requires an argument." >&2 | 81 | echo "Option -$OPTARG requires an argument." >&2 |
79 | exit 1 | 82 | exit 1 |
80 | ;; | 83 | ;; |
81 | \?) | 84 | \?) |
82 | echo "BAD USAGE : unknow opton -$OPTARG" | 85 | echo "BAD USAGE : unknow opton -$OPTARG" |
83 | #exit 1 | 86 | #exit 1 |
84 | ;; | 87 | ;; |
85 | esac | 88 | esac |
86 | done | 89 | done |
87 | 90 | ||
88 | # mode debug enable | 91 | # mode debug enable |
89 | if [ $DEBUG -eq 1 ] | 92 | if [ $DEBUG -eq 1 ] |
90 | then | 93 | then |
91 | set -x | 94 | set -x |
92 | echo -e "## Mode DEBUG ON ##" | 95 | echo -e "## Mode DEBUG ON ##" |
93 | REDIRECTION_OUTPUT="" | 96 | REDIRECTION_OUTPUT="" |
94 | else | 97 | else |
95 | REDIRECTION_OUTPUT=" 2> /dev/null" | 98 | REDIRECTION_OUTPUT=" 2> /dev/null" |
96 | fi | 99 | fi |
97 | 100 | ||
98 | # mode verbose enable | 101 | # mode verbose enable |
99 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi | 102 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi |
100 | 103 | ||
101 | # Check USAGE by arguments number | 104 | # Check USAGE by arguments number |
102 | if [ $(($#-($OPTIND-1))) -ne 1 ] | 105 | if [ $(($#-($OPTIND-1))) -ne 1 ] |
103 | then | 106 | then |
104 | echo "BAD USAGE : ExploitConfidencePass.sh [OPTIONS] <INPUT_DIRECTORY>" | 107 | echo "BAD USAGE : ExploitConfidencePass.sh [OPTIONS] <INPUT_DIRECTORY>" |
105 | echo "$0 -h for more info" | 108 | echo "$0 -h for more info" |
106 | exit 1 | 109 | exit 1 |
107 | fi | 110 | fi |
108 | 111 | ||
109 | shift $((OPTIND-1)) | 112 | shift $((OPTIND-1)) |
110 | # check input directory - first argument | 113 | # check input directory - first argument |
111 | if [ ! -e $1 ] | 114 | if [ ! -e $1 ] |
112 | then | 115 | then |
113 | print_error "can't open $1" | 116 | print_error "can't open $1" |
114 | exit 1 | 117 | exit 1 |
115 | fi | 118 | fi |
116 | 119 | ||
117 | #-------------# | 120 | #-------------# |
118 | # GLOBAL VARS # | 121 | # GLOBAL VARS # |
119 | #-------------# | 122 | #-------------# |
120 | INPUT_DIR=$(readlink -e $1) | 123 | INPUT_DIR=$(readlink -e $1) |
121 | OUTPUT_DIR=$INPUT_DIR | 124 | OUTPUT_DIR=$INPUT_DIR |
122 | BASENAME=$(basename $OUTPUT_DIR) | 125 | BASENAME=$(basename $OUTPUT_DIR) |
123 | SHOW_DIR="$OUTPUT_DIR/shows/" | 126 | SHOW_DIR="$OUTPUT_DIR/shows/" |
124 | SOLR_RES="$OUTPUT_DIR/solr/" | 127 | SOLR_RES="$OUTPUT_DIR/solr/" |
125 | EXT_LEX="$OUTPUT_DIR/LEX/" | 128 | EXT_LEX="$OUTPUT_DIR/LEX/" |
126 | TRIGGER_CONFZONE="$OUTPUT_DIR/trigg/" | 129 | TRIGGER_CONFZONE="$OUTPUT_DIR/trigg/" |
127 | LOGFILE="$(dirname $OUTPUT_DIR)/info_exploitconf.log" | 130 | LOGFILE="$(dirname $OUTPUT_DIR)/info_exploitconf.log" |
128 | ERRORFILE="$(dirname $OUTPUT_DIR)/error_exploitconf.log" | 131 | ERRORFILE="$(dirname $OUTPUT_DIR)/error_exploitconf.log" |
129 | 132 | ||
130 | CONFPASS_CONFIG_FILE="$(readlink -e $1)/ConfPass.cfg" | 133 | CONFPASS_CONFIG_FILE="$(readlink -e $1)/ConfPass.cfg" |
131 | if [ -e $CONFPASS_CONFIG_FILE ] | 134 | if [ -e $CONFPASS_CONFIG_FILE ] |
132 | then | 135 | then |
133 | { | 136 | { |
134 | RES_CONF_DIR=$(cat $CONFPASS_CONFIG_FILE | grep "^RES_CONF_DIR=" | cut -f2 -d"=") | 137 | RES_CONF_DIR=$(cat $CONFPASS_CONFIG_FILE | grep "^RES_CONF_DIR=" | cut -f2 -d"=") |
135 | RES_CONF=$(cat $CONFPASS_CONFIG_FILE | grep "^CONF_DIR=" | cut -f2 -d"=") | 138 | RES_CONF=$(cat $CONFPASS_CONFIG_FILE | grep "^CONF_DIR=" | cut -f2 -d"=") |
136 | print_warn "Use confidence measure from : $RES_CONF" 1 | 139 | print_warn "[${BASENAME}] Use confidence measure from : $RES_CONF" 1 |
137 | } | 140 | } |
138 | else | 141 | else |
139 | { | 142 | { |
140 | print_error "Can't find $CONFPASS_CONFIG_FILE" 1 | 143 | print_error "Can't find $CONFPASS_CONFIG_FILE" 1 |
141 | RES_CONF_DIR="$INPUT_DIR/conf/res_p2/scored_ctm" | 144 | RES_CONF_DIR="$INPUT_DIR/conf/res_p2/scored_ctm" |
142 | RES_CONF="$INPUT_DIR/conf/res_p2" | 145 | RES_CONF="$INPUT_DIR/conf/res_p2" |
143 | } | 146 | } |
144 | fi | 147 | fi |
145 | 148 | ||
146 | mkdir -p $SHOW_DIR > /dev/null 2>&1 | 149 | mkdir -p $SHOW_DIR > /dev/null 2>&1 |
147 | mkdir -p $SOLR_RES > /dev/null 2>&1 | 150 | mkdir -p $SOLR_RES > /dev/null 2>&1 |
148 | mkdir -p $EXT_LEX > /dev/null 2>&1 | 151 | mkdir -p $EXT_LEX > /dev/null 2>&1 |
149 | mkdir -p $TRIGGER_CONFZONE > /dev/null 2>&1 | 152 | mkdir -p $TRIGGER_CONFZONE > /dev/null 2>&1 |
150 | 153 | ||
151 | #------------------# | 154 | #------------------# |
152 | # Create Workspace # | 155 | # Create Workspace # |
153 | #------------------# | 156 | #------------------# |
154 | # Lock directory | 157 | # Lock directory |
155 | if [ -e "$OUTPUT_DIR_BASENAME/EXPLOITCONFPASS.lock" ] && [ $RERUN -eq 0 ]; then exit 1; fi | 158 | if [ -e "$OUTPUT_DIR_BASENAME/EXPLOITCONFPASS.lock" ] && [ $RERUN -eq 0 ]; then exit 1; fi |
156 | rm "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" > /dev/null 2>&1 | 159 | rm "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" > /dev/null 2>&1 |
157 | touch "$OUTPUT_DIR/EXPLOITCONFPASS.lock" > /dev/null 2>&1 | 160 | touch "$OUTPUT_DIR/EXPLOITCONFPASS.lock" > /dev/null 2>&1 |
158 | 161 | ||
159 | #------# | 162 | #------# |
160 | # Save # | 163 | # Save # |
161 | #------# | 164 | #------# |
162 | cp $EXPLOITCONFIDENCEPASS_CONFIG_FILE $OUTPUT_DIR/ExploitConfPass.cfg | 165 | cp $EXPLOITCONFIDENCEPASS_CONFIG_FILE $OUTPUT_DIR/ExploitConfPass.cfg |
163 | echo "TRIGGER_DIR=$TRIGGER_CONFZONE" >> $OUTPUT_DIR/ExploitConfPass.cfg | 166 | echo "TRIGGER_DIR=$TRIGGER_CONFZONE" >> $OUTPUT_DIR/ExploitConfPass.cfg |
164 | echo "TRIGGER_SPEERAL=$TRIGGER_CONFZONE/speeral/" >> $OUTPUT_DIR/ExploitConfPass.cfg | 167 | echo "TRIGGER_SPEERAL=$TRIGGER_CONFZONE/speeral/" >> $OUTPUT_DIR/ExploitConfPass.cfg |
165 | echo "LEX_SPEERAL=$EXT_LEX/speeral/${lexname}_ext" >> $OUTPUT_DIR/ExploitConfPass.cfg | 168 | echo "LEX_SPEERAL=$EXT_LEX/speeral/${lexname}_ext" >> $OUTPUT_DIR/ExploitConfPass.cfg |
166 | echo "LEX_BINODE_SPEERAL=$EXT_LEX/speeral/${lexname}_ext.bin" >> $OUTPUT_DIR/ExploitConfPass.cfg | 169 | echo "LEX_BINODE_SPEERAL=$EXT_LEX/speeral/${lexname}_ext.bin" >> $OUTPUT_DIR/ExploitConfPass.cfg |
167 | 170 | ||
168 | 171 | ||
169 | #-----------------------# | 172 | #-----------------------# |
170 | # Segmentation by show # | 173 | # Segmentation by show # |
171 | #-----------------------# | 174 | #-----------------------# |
172 | # create txt file from scored res | 175 | # create txt file from scored res |
173 | # tag pos and lemmatization of the txt file | 176 | # tag pos and lemmatization of the txt file |
174 | # merge the scored res and taglem file | 177 | # merge the scored res and taglem file |
175 | # segment using the last generated file | 178 | # segment using the last generated file |
176 | # and create a ctm file by show | 179 | # and create a ctm file by show |
177 | 180 | ||
178 | print_info "Segmentation by show" 1 | 181 | print_info "Segmentation by show" 1 |
179 | 182 | ||
180 | # -> to txt | 183 | # -> to txt |
181 | print_info "Create txt from scored res" 2 | 184 | print_info "Create txt from scored res" 2 |
182 | cat ${RES_CONF_DIR}/*.res > $INPUT_DIR/$BASENAME.sctm | 185 | cat ${RES_CONF_DIR}/*.res > $INPUT_DIR/$BASENAME.sctm |
183 | cat $INPUT_DIR/$BASENAME.seg | $SIGMUND_BIN/myConvert.pl $INPUT_DIR/$BASENAME.sctm $INPUT_DIR/$BASENAME.tmp | 186 | cat $INPUT_DIR/$BASENAME.seg | $SIGMUND_BIN/myConvert.pl $INPUT_DIR/$BASENAME.sctm $INPUT_DIR/$BASENAME.tmp |
184 | cat $INPUT_DIR/$BASENAME.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | sed -e "s/_/ /g" | sort -nt 'n' -k '2' > $INPUT_DIR/$BASENAME.txt | 187 | cat $INPUT_DIR/$BASENAME.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | sed -e "s/_/ /g" | sort -nt 'n' -k '2' > $INPUT_DIR/$BASENAME.txt |
185 | 188 | ||
186 | # -> to tagger + lemme | 189 | # -> to tagger + lemme |
187 | print_info "Tag pos and lem in txt file" 2 | 190 | print_info "Tag pos and lem in txt file" 2 |
188 | iconv -t ISO_8859-1 $INPUT_DIR/$BASENAME.txt > $INPUT_DIR/$BASENAME.tmp | 191 | iconv -t ISO_8859-1 $INPUT_DIR/$BASENAME.txt > $INPUT_DIR/$BASENAME.tmp |
189 | $SIGMUND_BIN/txt2lem.sh $INPUT_DIR/$BASENAME.tmp $INPUT_DIR/$BASENAME.taglem | 192 | $SIGMUND_BIN/txt2lem.sh $INPUT_DIR/$BASENAME.tmp $INPUT_DIR/$BASENAME.taglem |
190 | 193 | ||
191 | # merge sctm and taglem | 194 | # merge sctm and taglem |
192 | print_info "Merge scored ctm with tag pos and lem file" 2 | 195 | print_info "Merge scored ctm with tag pos and lem file" 2 |
193 | cat $INPUT_DIR/$BASENAME.sctm | $SCRIPT_PATH/BdlexUC.pl ${RULES}/basic -f | iconv -t ISO_8859-1 | $SCRIPT_PATH/scoredCtmAndTaggedLem2All.pl $INPUT_DIR/$BASENAME.taglem > $INPUT_DIR/$BASENAME.ctl | 196 | cat $INPUT_DIR/$BASENAME.sctm | $SCRIPT_PATH/BdlexUC.pl ${RULES}/basic -f | iconv -t ISO_8859-1 | $SCRIPT_PATH/scoredCtmAndTaggedLem2All.pl $INPUT_DIR/$BASENAME.taglem > $INPUT_DIR/$BASENAME.ctl |
194 | 197 | ||
195 | # -> new seg | 198 | # -> new seg |
196 | print_info "Create xml file and run Topic Seg" 2 | 199 | print_info "Create xml file and run Topic Seg" 2 |
197 | $SIGMUND_BIN/tagLem2xml.pl $INPUT_DIR/$BASENAME.taglem $INPUT_DIR/$BASENAME.doc.xml | 200 | $SIGMUND_BIN/tagLem2xml.pl $INPUT_DIR/$BASENAME.taglem $INPUT_DIR/$BASENAME.doc.xml |
198 | rm $INPUT_DIR/$BASENAME.tmp #$INPUT_DIR/$BASENAME.taglem | 201 | rm $INPUT_DIR/$BASENAME.tmp #$INPUT_DIR/$BASENAME.taglem |
199 | 202 | ||
200 | # Lia_topic_seg : bring together sentences into show | 203 | # Lia_topic_seg : bring together sentences into show |
201 | cp $INPUT_DIR/$BASENAME.doc.xml 0.xml | 204 | cp $INPUT_DIR/$BASENAME.doc.xml 0.xml |
202 | java -cp $LIATOPICSEG/bin Test > $INPUT_DIR/show.seg | 205 | java -cp $LIATOPICSEG/bin Test > $INPUT_DIR/show.seg |
203 | cat $INPUT_DIR/show.seg | $SIGMUND_BIN/toSegEmiss.pl $INPUT_DIR/$BASENAME.show.seg | 206 | cat $INPUT_DIR/show.seg | $SIGMUND_BIN/toSegEmiss.pl $INPUT_DIR/$BASENAME.show.seg |
204 | rm 0.xml $INPUT_DIR/show.seg | 207 | rm 0.xml $INPUT_DIR/show.seg |
205 | 208 | ||
206 | if [ $CHECK -eq 1 ] | 209 | if [ $CHECK -eq 1 ] |
207 | then | 210 | then |
208 | if [ ! -s $INPUT_DIR/$BASENAME.show.seg ];then echo -e "ERROR : no Topic segmentation" >> $ERRORFILE; fi | 211 | if [ ! -s $INPUT_DIR/$BASENAME.show.seg ];then echo -e "ERROR : no Topic segmentation" >> $ERRORFILE; fi |
209 | fi | 212 | fi |
210 | 213 | ||
211 | # Segment ctm into several show files and create a seg list by show | 214 | # Segment ctm into several show files and create a seg list by show |
212 | print_info "Segment ctm into show files and a seg list by show" 2 | 215 | print_info "Segment ctm into show files and a seg list by show" 2 |
213 | $SCRIPT_PATH/ctm2show.pl $INPUT_DIR/$BASENAME.ctl $INPUT_DIR/$BASENAME.show.seg $SHOW_DIR $REDIRECTION_OUTPUT | 216 | $SCRIPT_PATH/ctm2show.pl $INPUT_DIR/$BASENAME.ctl $INPUT_DIR/$BASENAME.show.seg $SHOW_DIR $REDIRECTION_OUTPUT |
214 | 217 | ||
215 | #-----------------------------------------------------------# | 218 | #-----------------------------------------------------------# |
216 | # SOLR QUERIES # | 219 | # SOLR QUERIES # |
217 | # -> Create Confidente Word # | 220 | # -> Create Confidente Word # |
218 | # Keep conf words and use Tags # | 221 | # Keep conf words and use Tags # |
219 | # -> Query SOLR (document & multimedia) # | 222 | # -> Query SOLR (document & multimedia) # |
220 | # concat word + add date 2 day before and after the show # | 223 | # concat word + add date 2 day before and after the show # |
221 | # query document & multimedia # | 224 | # query document & multimedia # |
222 | #-----------------------------------------------------------# | 225 | #-----------------------------------------------------------# |
223 | print_info "Create SOLR queries and ASK SOLR" 1 | 226 | print_info "Create SOLR queries and ASK SOLR" 1 |
224 | for show in $(ls $SHOW_DIR/*.ctm) | 227 | for show in $(ls $SHOW_DIR/*.ctm) |
225 | do | 228 | do |
226 | bn=$(basename $show .ctm) | 229 | bn=$(basename $show .ctm) |
227 | # Remove words with low confidence and keep useful tagger words | 230 | # Remove words with low confidence and keep useful tagger words |
228 | cat $show | $SCRIPT_PATH/KeepConfZone.pl | grep -e "MOTINC\|NMS\|NMP\|NFS\|NFP\|X[A-Z]{3,5}" | cut -f3 -d' ' > "$SHOW_DIR/$bn.confzone" | 231 | cat $show | $SCRIPT_PATH/KeepConfZone.pl | grep -e "MOTINC\|NMS\|NMP\|NFS\|NFP\|X[A-Z]{3,5}" | cut -f3 -d' ' > "$SHOW_DIR/$bn.confzone" |
229 | # Get date 2 day before and after the show | 232 | # Get date 2 day before and after the show |
230 | datePattern=`$SCRIPT_PATH/daybefore2after.sh $(echo $BASENAME | cut -c1-6)` | 233 | datePattern=`$SCRIPT_PATH/daybefore2after.sh $(echo $BASENAME | cut -c1-6)` |
231 | # Create SOLR queries | 234 | # Create SOLR queries |
232 | cat $SHOW_DIR/$bn".confzone" | $SCRIPT_PATH/GenerateSOLRQueries.pl | iconv -f ISO_8859-1 -t UTF-8 > "$SHOW_DIR/$bn.queries" | 235 | cat $SHOW_DIR/$bn".confzone" | $SCRIPT_PATH/GenerateSOLRQueries.pl | iconv -f ISO_8859-1 -t UTF-8 > "$SHOW_DIR/$bn.queries" |
233 | # Ask SOLR DB | 236 | # Ask SOLR DB |
234 | if [ $(wc -w "$SHOW_DIR/$bn.queries" | cut -f1 -d' ') -gt 0 ]; then | 237 | if [ $(wc -w "$SHOW_DIR/$bn.queries" | cut -f1 -d' ') -gt 0 ]; then |
235 | query=$(cat $SHOW_DIR/$bn.queries)"&fq=docDate:[$datePattern]" | 238 | query=$(cat $SHOW_DIR/$bn.queries)"&fq=docDate:[$datePattern]" |
236 | echo $query > $SHOW_DIR/$bn.queries | 239 | echo $query > $SHOW_DIR/$bn.queries |
237 | python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp $REDIRECTION_OUTPUT | 240 | python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp $REDIRECTION_OUTPUT |
238 | cat $SOLR_RES/$bn.keywords.tmp | sort -u > $SOLR_RES/$bn.keywords | 241 | cat $SOLR_RES/$bn.keywords.tmp | sort -u > $SOLR_RES/$bn.keywords |
239 | cat $SOLR_RES/$bn.txt.tmp | sort -u > $SOLR_RES/$bn.txt | 242 | cat $SOLR_RES/$bn.txt.tmp | sort -u > $SOLR_RES/$bn.txt |
240 | rm $SOLR_RES/*.tmp > /dev/null 2>&1 | 243 | rm $SOLR_RES/*.tmp > /dev/null 2>&1 |
241 | fi | 244 | fi |
242 | 245 | ||
243 | if [ $CHECK -eq 1 ] | 246 | if [ $CHECK -eq 1 ] |
244 | then | 247 | then |
245 | if [ ! -e $SOLR_RES/$bn.keywords ] || [ ! -e $SOLR_RES/$bn.txt ] | 248 | if [ ! -e $SOLR_RES/$bn.keywords ] || [ ! -e $SOLR_RES/$bn.txt ] |
246 | then | 249 | then |
247 | print_warn "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 1 | 250 | print_warn "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 1 |
248 | fi | 251 | fi |
249 | fi | 252 | fi |
250 | 253 | ||
251 | done | 254 | done |
252 | 255 | ||
253 | #----------------------------------------------------------------------------------------------- | 256 | #----------------------------------------------------------------------------------------------- |
254 | # Build trigger file | 257 | # Build trigger file |
255 | # 1) keywords are automatically boosted in the non confident zone of the current res | 258 | # 1) keywords are automatically boosted in the non confident zone of the current res |
256 | # confident zone are boosted | 259 | # confident zone are boosted |
257 | # previous words in sensible zone are penalized | 260 | # previous words in sensible zone are penalized |
258 | # 2) OOVs are extracted + phonetized | 261 | # 2) OOVs are extracted + phonetized |
259 | # 3) Try to find OOVs acousticly in the current segment | 262 | # 3) Try to find OOVs acousticly in the current segment |
260 | # 4) Generate the .trigg file | 263 | # 4) Generate the .trigg file |
261 | #------------------------------------------------------------------------------------------------ | 264 | #------------------------------------------------------------------------------------------------ |
262 | print_info "Build trigger files" 1 | 265 | print_info "Build trigger files" 1 |
263 | for i in `ls $SOLR_RES/*.keywords` | 266 | for i in `ls $SOLR_RES/*.keywords` |
264 | do | 267 | do |
265 | basename=`basename $i .keywords` | 268 | basename=`basename $i .keywords` |
266 | 269 | ||
267 | # | 270 | # |
268 | # Tokenize & produce coverage report | 271 | # Tokenize & produce coverage report |
269 | # Use filter you need | 272 | # Use filter you need |
270 | # | 273 | # |
271 | print_info "keywords filtering and produce coverage report" 2 | 274 | print_info "keywords filtering and produce coverage report" 2 |
272 | # Default filter | 275 | # Default filter |
273 | cat $i | $SCRIPT_PATH/CleanFilter.sh | ${SCRIPT_PATH}/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t |\ | 276 | cat $i | $SCRIPT_PATH/CleanFilter.sh | ${SCRIPT_PATH}/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t |\ |
274 | $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok | 277 | $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok |
275 | # do less filter | 278 | # do less filter |
276 | #cat $i | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex | $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok | 279 | #cat $i | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex | $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok |
277 | 280 | ||
278 | 281 | ||
279 | # | 282 | # |
280 | # Extract "real" OOV and phonetize them | 283 | # Extract "real" OOV and phonetize them |
281 | # -> petit filtrage persoo pour eviter d'avoir trop de bruits | 284 | # -> petit filtrage persoo pour eviter d'avoir trop de bruits |
282 | # | 285 | # |
283 | print_info "Extract OOV and phonetize them" 2 | 286 | print_info "Extract OOV and phonetize them" 2 |
284 | ${SCRIPT_PATH}/FindNormRules.pl $SOLR_RES/${basename}_tmp_report/report.oov $LEXICON.bdlex_tok | cut -f3 | grep -v "#" | grep -v "^[A-Z]\+$" | grep -v "^[0-9]" | grep --perl-regex -v "^([a-z']){1,3}$" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | iconv -t ISO_8859-1 -f UTF-8 | ${LIA_LTBOX}/lia_phon/script/lia_lex2phon_variante | grep -v "core dumped" | cut -d"[" -f1 | sort -u | ${SCRIPT_PATH}/PhonFormatter.pl | iconv -f ISO_8859-1 -t UTF-8 | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $SOLR_RES/${basename}.phon_oov | 287 | ${SCRIPT_PATH}/FindNormRules.pl $SOLR_RES/${basename}_tmp_report/report.oov $LEXICON.bdlex_tok | cut -f3 | grep -v "#" | grep -v "^[A-Z]\+$" | grep -v "^[0-9]" | grep --perl-regex -v "^([a-z']){1,3}$" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | iconv -t ISO_8859-1 -f UTF-8 | ${LIA_LTBOX}/lia_phon/script/lia_lex2phon_variante | grep -v "core dumped" | cut -d"[" -f1 | sort -u | ${SCRIPT_PATH}/PhonFormatter.pl | iconv -f ISO_8859-1 -t UTF-8 | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $SOLR_RES/${basename}.phon_oov |
285 | 288 | ||
286 | # | 289 | # |
287 | # Search INVOC & OOV in the current lattice | 290 | # Search INVOC & OOV in the current lattice |
288 | # | 291 | # |
289 | print_info "Search INVOC and OOV in the current lattice" 2 | 292 | print_info "Search INVOC and OOV in the current lattice" 2 |
290 | cat $SOLR_RES/${basename}_tmp_report/report.invoc | grep -v "\b0" | cut -f1 | grep -v --perl-regex -v "^[a-zA-Z']{1,3}$" | grep -v --perl-regex "^[a-zA-Z0-9]{1,3}$" | grep -v "<s>" | grep -v "</s>" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $TRIGGER_CONFZONE/$basename.tosearch | 293 | cat $SOLR_RES/${basename}_tmp_report/report.invoc | grep -v "\b0" | cut -f1 | grep -v --perl-regex -v "^[a-zA-Z']{1,3}$" | grep -v --perl-regex "^[a-zA-Z0-9]{1,3}$" | grep -v "<s>" | grep -v "</s>" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $TRIGGER_CONFZONE/$basename.tosearch |
291 | cat $SOLR_RES/${basename}.phon_oov | cut -f1 >> $TRIGGER_CONFZONE/$basename.tosearch | 294 | cat $SOLR_RES/${basename}.phon_oov | cut -f1 >> $TRIGGER_CONFZONE/$basename.tosearch |
292 | 295 | ||
293 | # For each treil | 296 | # For each treil |
294 | for baseseg in $(cat "$SHOW_DIR/$basename.lst") | 297 | for baseseg in $(cat "$SHOW_DIR/$basename.lst") |
295 | do | 298 | do |
296 | $OTMEDIA_HOME/tools/QUOTE_FINDER/bin/acousticFinder ${LEXICON}.speer_phon $RES_CONF/wlat/$baseseg.wlat $TRIGGER_CONFZONE/${basename}.tosearch $SOLR_RES/$basename.phon_oov > $TRIGGER_CONFZONE/$baseseg.acousticlyfound $OUTPUT_REDIRECTION | 299 | $OTMEDIA_HOME/tools/QUOTE_FINDER/bin/acousticFinder ${LEXICON}.speer_phon $RES_CONF/wlat/$baseseg.wlat $TRIGGER_CONFZONE/${basename}.tosearch $SOLR_RES/$basename.phon_oov > $TRIGGER_CONFZONE/$baseseg.acousticlyfound $OUTPUT_REDIRECTION |
297 | # | 300 | # |
298 | # Produce the boost file for the next decoding pass | 301 | # Produce the boost file for the next decoding pass |
299 | # | 302 | # |
300 | print_info "Produce trigg file : $baseseg " 3 | 303 | print_info "Produce trigg file : $baseseg " 3 |
301 | cat $RES_CONF_DIR/$baseseg.res | $SCRIPT_PATH/ScoreCtm2trigg.pl $TRIGGER_CONFZONE/$baseseg.acousticlyfound > $TRIGGER_CONFZONE/$baseseg.trigg | 304 | cat $RES_CONF_DIR/$baseseg.res | $SCRIPT_PATH/ScoreCtm2trigg.pl $TRIGGER_CONFZONE/$baseseg.acousticlyfound > $TRIGGER_CONFZONE/$baseseg.trigg |
302 | done | 305 | done |
303 | 306 | ||
304 | done | 307 | done |
305 | 308 | ||
306 | #----------------------------------------------------------------------------------------------- | 309 | #----------------------------------------------------------------------------------------------- |
307 | # Build the extended SPEERAL Lexicon | 310 | # Build the extended SPEERAL Lexicon |
308 | # 1) Merge OOVs + LEXICON | 311 | # 1) Merge OOVs + LEXICON |
309 | # 1) Related text are collected in order to find the invoc word with maximizing the ppl (LM proba) | 312 | # 1) Related text are collected in order to find the invoc word with maximizing the ppl (LM proba) |
310 | # 2) The current lexicon is extended with all the valid OOVs | 313 | # 2) The current lexicon is extended with all the valid OOVs |
311 | #----------------------------------------------------------------------------------------------- | 314 | #----------------------------------------------------------------------------------------------- |
312 | print_info "Build extended Speeral Lexicon" 1 | 315 | print_info "Build extended Speeral Lexicon" 1 |
313 | mkdir -p $EXT_LEX/final | 316 | mkdir -p $EXT_LEX/final |
314 | mkdir -p $EXT_LEX/tmp | 317 | mkdir -p $EXT_LEX/tmp |
315 | mkdir -p $EXT_LEX/tmp/txt | 318 | mkdir -p $EXT_LEX/tmp/txt |
316 | # | 319 | # |
317 | # Collect the acousticly found oov and their phonetisation | 320 | # Collect the acousticly found oov and their phonetisation |
318 | # | 321 | # |
319 | print_info "Get all OOV and retrieve all phonetisation" 2 | 322 | print_info "Get all OOV and retrieve all phonetisation" 2 |
320 | for i in `ls $SOLR_RES/*.phon_oov` | 323 | for i in `ls $SOLR_RES/*.phon_oov` |
321 | do | 324 | do |
322 | basename=`basename $i .phon_oov` | 325 | basename=`basename $i .phon_oov` |
323 | 326 | ||
324 | rm $EXT_LEX/$basename.acousticlyfound 2> /dev/null | 327 | rm $EXT_LEX/$basename.acousticlyfound 2> /dev/null |
325 | # list acousticly found for the show | 328 | # list acousticly found for the show |
326 | for baseseg in $(cat "$SHOW_DIR/$basename.lst") | 329 | for baseseg in $(cat "$SHOW_DIR/$basename.lst") |
327 | do | 330 | do |
328 | cat $TRIGGER_CONFZONE/$baseseg.acousticlyfound | cut -f1 | cut -f2 -d"=" >> $EXT_LEX/$basename.acousticlyfound | 331 | cat $TRIGGER_CONFZONE/$baseseg.acousticlyfound | cut -f1 | cut -f2 -d"=" >> $EXT_LEX/$basename.acousticlyfound |
329 | done | 332 | done |
330 | cat $EXT_LEX/$basename.acousticlyfound | sort -u > $EXT_LEX/.tmp | 333 | cat $EXT_LEX/$basename.acousticlyfound | sort -u > $EXT_LEX/.tmp |
331 | mv $EXT_LEX/.tmp $EXT_LEX/$basename.acousticlyfound | 334 | mv $EXT_LEX/.tmp $EXT_LEX/$basename.acousticlyfound |
332 | 335 | ||
333 | # | 336 | # |
334 | # Extract OOV really added | 337 | # Extract OOV really added |
335 | # | 338 | # |
336 | cat $SOLR_RES/$basename.phon_oov | cut -f1 | sort -u > $EXT_LEX/$basename.oov | 339 | cat $SOLR_RES/$basename.phon_oov | cut -f1 | sort -u > $EXT_LEX/$basename.oov |
337 | $SCRIPT_PATH/intersec.pl $EXT_LEX/$basename.oov $EXT_LEX/$basename.acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound $REDIRECTION_OUTPUT | 340 | $SCRIPT_PATH/intersec.pl $EXT_LEX/$basename.oov $EXT_LEX/$basename.acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound $REDIRECTION_OUTPUT |
338 | # | 341 | # |
339 | # Retrieve all phonetisation | 342 | # Retrieve all phonetisation |
340 | # | 343 | # |
341 | cat $SOLR_RES/${basename}.phon_oov | $SCRIPT_PATH/LexPhonFilter.pl $EXT_LEX/$basename.oov_acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound_phon | 344 | cat $SOLR_RES/${basename}.phon_oov | $SCRIPT_PATH/LexPhonFilter.pl $EXT_LEX/$basename.oov_acousticlyfound > $EXT_LEX/$basename.oov_acousticlyfound_phon |
342 | done | 345 | done |
343 | 346 | ||
344 | # | 347 | # |
345 | # Merge OOVs and their phonetisation | 348 | # Merge OOVs and their phonetisation |
346 | # | 349 | # |
347 | print_info "Merge OOV and their phonetisation" 2 | 350 | print_info "Merge OOV and their phonetisation" 2 |
348 | lexname=$(basename $LEXICON) | 351 | lexname=$(basename $LEXICON) |
349 | cat $EXT_LEX/*.oov_acousticlyfound_phon | sort -u > $EXT_LEX/final/all.oov_acousticlyfound_phon | 352 | cat $EXT_LEX/*.oov_acousticlyfound_phon | sort -u > $EXT_LEX/final/all.oov_acousticlyfound_phon |
350 | cat $EXT_LEX/*.oov_acousticlyfound | sort -u | grep --perl-regex -v "^([a-z']){3}$" > $EXT_LEX/final/all.oov_acousticlyfound | 353 | cat $EXT_LEX/*.oov_acousticlyfound | sort -u | grep --perl-regex -v "^([a-z']){3}$" > $EXT_LEX/final/all.oov_acousticlyfound |
351 | $SCRIPT_PATH/MergeLexicon.pl $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/final/${lexname}_ext.phon $REDIRECTION_OUTPUT | 354 | $SCRIPT_PATH/MergeLexicon.pl $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/final/${lexname}_ext.phon $REDIRECTION_OUTPUT |
352 | 355 | ||
353 | # | 356 | # |
354 | # Collect + clean retrieved txt | 357 | # Collect + clean retrieved txt |
355 | # | 358 | # |
356 | print_info "Collect and clean SOLR txt answers" 2 | 359 | print_info "Collect and clean SOLR txt answers" 2 |
357 | # choose filter | 360 | # choose filter |
358 | # default | 361 | # default |
359 | cat $SOLR_RES/*.txt | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $EXT_LEX/final/all.bdlex_txt | 362 | cat $SOLR_RES/*.txt | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $EXT_LEX/final/all.bdlex_txt |
360 | # low filter | 363 | # low filter |
361 | #cat $SOLR_RES/*.txt | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex > $EXT_LEX/final/all.bdlex_txt | 364 | #cat $SOLR_RES/*.txt | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex > $EXT_LEX/final/all.bdlex_txt |
362 | 365 | ||
363 | # | 366 | # |
364 | # Construct the map file | 367 | # Construct the map file |
365 | # | 368 | # |
366 | # Notes: | 369 | # Notes: |
367 | # - Expected format : | 370 | # - Expected format : |
368 | # <WORD1_STRING> <CANDIDATE1_STRING> <PHON_1> | 371 | # <WORD1_STRING> <CANDIDATE1_STRING> <PHON_1> |
369 | # | 372 | # |
370 | print_info "Construct map file" 2 | 373 | print_info "Construct map file" 2 |
371 | rm -f $EXT_LEX/final/${lexname}_ext.map 2>/dev/null | 374 | rm -f $EXT_LEX/final/${lexname}_ext.map 2>/dev/null |
372 | rm -f $EXT_LEX/final/${lexname}.unvalid_oov 2>/dev/null | 375 | rm -f $EXT_LEX/final/${lexname}.unvalid_oov 2>/dev/null |
373 | 376 | ||
374 | while read oov | 377 | while read oov |
375 | do | 378 | do |
376 | oov=`echo $oov | sed "s/\n//g"` | 379 | oov=`echo $oov | sed "s/\n//g"` |
377 | # | 380 | # |
378 | # Obtain the oov's tag | 381 | # Obtain the oov's tag |
379 | # | 382 | # |
380 | #oov_tag=`grep --perl-regex "^$oov\t" $DYNAMIC_TAGSTATS/all.tags | cut -f2` | 383 | #oov_tag=`grep --perl-regex "^$oov\t" $DYNAMIC_TAGSTATS/all.tags | cut -f2` |
381 | # | 384 | # |
382 | # Try to collect text containing the oov word | 385 | # Try to collect text containing the oov word |
383 | # | 386 | # |
384 | cat $EXT_LEX/final/all.bdlex_txt | grep --perl-regex " $oov " | $SCRIPT_PATH/NbMaxWordsFilter.pl 40 |uniq > $EXT_LEX/tmp/txt/$oov.bdlex_txt | 387 | cat $EXT_LEX/final/all.bdlex_txt | grep --perl-regex " $oov " | $SCRIPT_PATH/NbMaxWordsFilter.pl 40 |uniq > $EXT_LEX/tmp/txt/$oov.bdlex_txt |
385 | if [ -f $EXT_LEX/tmp/txt/$oov.bdlex_txt ]; then | 388 | if [ -f $EXT_LEX/tmp/txt/$oov.bdlex_txt ]; then |
386 | nbWords=`wc -l $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` | 389 | nbWords=`wc -l $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` |
387 | if [ $nbWords -eq 0 ]; then | 390 | if [ $nbWords -eq 0 ]; then |
388 | echo "UNVALID OOV: $oov => $nbWords occurrences" | 391 | echo "UNVALID OOV: $oov => $nbWords occurrences" |
389 | echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov | 392 | echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov |
390 | else | 393 | else |
391 | # | 394 | # |
392 | # Find a candidate in a filtred invoc lexicon => a candidate which maximize the ppl in the overall txt collected | 395 | # Find a candidate in a filtred invoc lexicon => a candidate which maximize the ppl in the overall txt collected |
393 | # | 396 | # |
394 | #echo "$/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $LEXICON.bdlex_tok $EXT_LEX/tmp/txt/$oov.bdlex_txt" | 397 | #echo "$/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $LEXICON.bdlex_tok $EXT_LEX/tmp/txt/$oov.bdlex_txt" |
395 | candidate=`$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` | 398 | candidate=`$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` |
396 | if [ ! x$candidate = "x" ]; then | 399 | if [ ! x$candidate = "x" ]; then |
397 | grep --perl-regex "^$oov\t" $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/tmp/$oov.phon | 400 | grep --perl-regex "^$oov\t" $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/tmp/$oov.phon |
398 | while read phonLine | 401 | while read phonLine |
399 | do | 402 | do |
400 | #<word> <phon> => <word> <candidate> <phon> | 403 | #<word> <phon> => <word> <candidate> <phon> |
401 | echo "$phonLine" | sed "s|\t|\t$candidate\t|" >> $EXT_LEX/final/${lexname}_ext.map | 404 | echo "$phonLine" | sed "s|\t|\t$candidate\t|" >> $EXT_LEX/final/${lexname}_ext.map |
402 | done < $EXT_LEX/tmp/$oov.phon | 405 | done < $EXT_LEX/tmp/$oov.phon |
403 | else | 406 | else |
404 | echo "UNVALID OOV: $oov => no availaible Candidate word in LM" | 407 | echo "UNVALID OOV: $oov => no availaible Candidate word in LM" |
405 | echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov | 408 | echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov |
406 | fi | 409 | fi |
407 | fi | 410 | fi |
408 | else | 411 | else |
409 | echo "UNVALID OOV: $oov" | 412 | echo "UNVALID OOV: $oov" |
410 | echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov | 413 | echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov |
411 | fi | 414 | fi |
412 | done < $EXT_LEX/final/all.oov_acousticlyfound | 415 | done < $EXT_LEX/final/all.oov_acousticlyfound |
413 | 416 | ||
414 | # | 417 | # |
415 | ### Speeral | 418 | ### Speeral |
416 | # | 419 | # |
417 | 420 | ||
418 | lexname=`basename $LEXICON` | 421 | lexname=`basename $LEXICON` |
419 | # | 422 | # |
420 | # Build the final trigger file | 423 | # Build the final trigger file |
421 | # | 424 | # |
422 | print_info "Clean trigg files" 2 | 425 | print_info "Clean trigg files" 2 |
423 | mkdir -p $TRIGGER_CONFZONE/speeral/ 2> /dev/null | 426 | mkdir -p $TRIGGER_CONFZONE/speeral/ 2> /dev/null |
424 | mkdir -p $EXT_LEX/speeral/ 2> /dev/null | 427 | mkdir -p $EXT_LEX/speeral/ 2> /dev/null |
425 | for i in `ls $TRIGGER_CONFZONE/*.trigg` | 428 | for i in `ls $TRIGGER_CONFZONE/*.trigg` |
426 | do | 429 | do |
427 | basename=`basename $i .trigg` | 430 | basename=`basename $i .trigg` |
428 | cat $i | $SCRIPT_PATH/RemoveLineContaining.pl $EXT_LEX/$lexname.unvalid_oov > $TRIGGER_CONFZONE/speeral/$basename.trigg | 431 | cat $i | $SCRIPT_PATH/RemoveLineContaining.pl $EXT_LEX/$lexname.unvalid_oov > $TRIGGER_CONFZONE/speeral/$basename.trigg |
429 | done | 432 | done |
430 | # | 433 | # |
431 | # Compile the speeral extended lexicon | 434 | # Compile the speeral extended lexicon |
432 | # | 435 | # |
433 | print_info "Compile Speeral extended lexicon" 2 | 436 | print_info "Compile Speeral extended lexicon" 2 |
434 | $SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext $REDIRECTION_OUTPUT | 437 | $SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext $REDIRECTION_OUTPUT |
435 | 438 | ||
436 | if [ $CHECK -eq 1 ] | 439 | if [ $CHECK -eq 1 ] |
437 | then | 440 | then |
438 | check_exploitconfpass_lex_check "${EXT_LEX}/speeral/${lexname}_ext" | 441 | check_exploitconfpass_lex_check "${EXT_LEX}/speeral/${lexname}_ext" |
439 | if [ $? -eq 1 ] | 442 | if [ $? -eq 1 ] |
440 | then | 443 | then |
441 | echo -e "ERROR : Building Speeral Lexicon $INPUT_DIR " >> $ERRORFILE | 444 | echo -e "ERROR : Building Speeral Lexicon $INPUT_DIR " >> $ERRORFILE |
442 | exit 1; | 445 | exit 1; |
443 | fi | 446 | fi |
444 | fi | 447 | fi |
445 | 448 | ||
446 | 449 | ||
447 | #-------# | 450 | #-------# |
448 | # CLOSE # | 451 | # CLOSE # |
449 | #-------# | 452 | #-------# |
450 | # Seem OK | 453 | # Seem OK |
451 | print_info "<= End $BASENAME Solr | $(date +'%d/%m/%y %H:%M:%S')" 1 | 454 | print_info "<= End $BASENAME Solr | $(date +'%d/%m/%y %H:%M:%S')" 1 |
452 | echo -e "#Solr $BASENAME " >> $LOGFILE | 455 | echo -e "#Solr $BASENAME " >> $LOGFILE |
453 | 456 | ||
454 | # unlock directory | 457 | # unlock directory |
455 | mv "$OUTPUT_DIR/EXPLOITCONFPASS.lock" "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" | 458 | mv "$OUTPUT_DIR/EXPLOITCONFPASS.lock" "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" |
456 | 459 | ||
457 | 460 | ||
458 | 461 |
main_tools/FirstPass.sh
1 | #!/bin/bash | 1 | #!/bin/bash |
2 | 2 | ||
3 | ##################################################### | 3 | ##################################################### |
4 | # File : FirstPass.sh # | 4 | # File : FirstPass.sh # |
5 | # Brief : ASR first pass and speaker diarization # | 5 | # Brief : ASR first pass and speaker diarization # |
6 | # Author : Jean-François Rey # | 6 | # Author : Jean-François Rey # |
7 | # (base on Emmanuel Ferreira # | 7 | # (base on Emmanuel Ferreira # |
8 | # and Hugo Mauchrétien works) # | 8 | # and Hugo Mauchrétien works) # |
9 | # Version : 1.1 # | 9 | # Version : 1.1 # |
10 | # Date : 18/06/13 # | 10 | # Date : 18/06/13 # |
11 | ##################################################### | 11 | ##################################################### |
12 | 12 | ||
13 | echo "### FirstPass.sh ###" | ||
14 | |||
13 | # Check OTMEDIA_HOME env var | 15 | # Check OTMEDIA_HOME env var |
14 | if [ -z ${OTMEDIA_HOME} ] | 16 | if [ -z ${OTMEDIA_HOME} ] |
15 | then | 17 | then |
16 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) | 18 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) |
17 | export OTMEDIA_HOME=$OTMEDIA_HOME | 19 | export OTMEDIA_HOME=$OTMEDIA_HOME |
18 | fi | 20 | fi |
19 | 21 | ||
20 | # where is FirstPass.sh | 22 | # where is FirstPass.sh |
21 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) | 23 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) |
22 | 24 | ||
23 | # scripts path | 25 | # scripts path |
24 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts | 26 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts |
25 | 27 | ||
26 | # Include scripts | 28 | # Include scripts |
27 | . $SCRIPT_PATH"/Tools.sh" | 29 | . $SCRIPT_PATH"/Tools.sh" |
28 | . $SCRIPT_PATH"/CheckFirstPass.sh" | 30 | . $SCRIPT_PATH"/CheckFirstPass.sh" |
29 | 31 | ||
30 | # where is FirstPass.cfg | 32 | # where is FirstPass.cfg |
31 | FIRSTPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/FirstPass.cfg" | 33 | FIRSTPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/FirstPass.cfg" |
32 | if [ -e $FIRSTPASS_CONFIG_FILE ] | 34 | if [ -e $FIRSTPASS_CONFIG_FILE ] |
33 | then | 35 | then |
34 | . $FIRSTPASS_CONFIG_FILE | 36 | . $FIRSTPASS_CONFIG_FILE |
35 | else | 37 | else |
36 | echo "ERROR : Can't find configuration file $FIRSTPASS_CONFIG_FILE" > /dev/stderr | 38 | echo "ERROR : Can't find configuration file $FIRSTPASS_CONFIG_FILE" > /dev/stderr |
37 | echo "exit" > /dev/stderr | 39 | echo "exit" > /dev/stderr |
38 | exit 1 | 40 | exit 1 |
39 | fi | 41 | fi |
40 | 42 | ||
41 | #---------------# | 43 | #---------------# |
42 | # Parse Options # | 44 | # Parse Options # |
43 | #---------------# | 45 | #---------------# |
44 | while getopts ":hDv:cf:r" opt | 46 | while getopts ":hDv:cf:r" opt |
45 | do | 47 | do |
46 | case $opt in | 48 | case $opt in |
47 | h) | 49 | h) |
48 | echo -e "$0 [OPTIONS] <WAV_FILE> <OUTPUT_DIRECTORY>\n" | 50 | echo -e "$0 [OPTIONS] <WAV_FILE> <OUTPUT_DIRECTORY>\n" |
49 | echo -e "\t Options:" | 51 | echo -e "\t Options:" |
50 | echo -e "\t\t-h :\tprint this message" | 52 | echo -e "\t\t-h :\tprint this message" |
51 | echo -e "\t\t-D :\tDEBUG mode on" | 53 | echo -e "\t\t-D :\tDEBUG mode on" |
52 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" | 54 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" |
53 | echo -e "\t\t-c :\tCheck process, and log it into files, can stop if error detected" | 55 | echo -e "\t\t-c :\tCheck process, and log it into files, can stop if error detected" |
54 | echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)" | 56 | echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)" |
55 | echo -e "\t\t-r :\tforce rerun the wav file" | 57 | echo -e "\t\t-r :\tforce rerun the wav file" |
56 | exit 1 | 58 | exit 1 |
57 | ;; | 59 | ;; |
58 | D) | 60 | D) |
59 | DEBUG=1 | 61 | DEBUG=1 |
60 | ;; | 62 | ;; |
61 | v) | 63 | v) |
62 | VERBOSE=$OPTARG | 64 | VERBOSE=$OPTARG |
63 | ;; | 65 | ;; |
64 | c) | 66 | c) |
65 | CHECK=1 | 67 | CHECK=1 |
66 | ;; | 68 | ;; |
67 | f) | 69 | f) |
68 | FORKS="--forks $OPTARG" | 70 | FORKS="--forks $OPTARG" |
69 | ;; | 71 | ;; |
70 | r) | 72 | r) |
71 | RERUN=1 | 73 | RERUN=1 |
72 | ;; | 74 | ;; |
73 | :) | 75 | :) |
74 | echo "Option -$OPTARG requires an argument." > /dev/stderr | 76 | echo "Option -$OPTARG requires an argument." > /dev/stderr |
75 | exit 1 | 77 | exit 1 |
76 | ;; | 78 | ;; |
77 | \?) | 79 | \?) |
78 | echo "BAD USAGE : unknow opton -$OPTARG" | 80 | echo "BAD USAGE : unknow opton -$OPTARG" |
79 | exit 1 | 81 | exit 1 |
80 | ;; | 82 | ;; |
81 | esac | 83 | esac |
82 | done | 84 | done |
83 | 85 | ||
84 | # mode debug enable | 86 | # mode debug enable |
85 | if [ $DEBUG -eq 1 ] | 87 | if [ $DEBUG -eq 1 ] |
86 | then | 88 | then |
87 | set -x | 89 | set -x |
88 | echo -e "## Mode DEBUG ON ##" | 90 | echo -e "## Mode DEBUG ON ##" |
89 | REDIRECTION_OUTPUT="" | 91 | REDIRECTION_OUTPUT="" |
90 | else | 92 | else |
91 | REDIRECTION_OUTPUT=" 2> /dev/null" | 93 | REDIRECTION_OUTPUT=" 2> /dev/null" |
92 | fi | 94 | fi |
93 | 95 | ||
94 | # mode verbose enable | 96 | # mode verbose enable |
95 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi | 97 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi |
96 | 98 | ||
97 | # Check USAGE by arguments number | 99 | # Check USAGE by arguments number |
98 | if [ $(($#-($OPTIND-1))) -ne 2 ] | 100 | if [ $(($#-($OPTIND-1))) -ne 2 ] |
99 | then | 101 | then |
100 | echo "BAD USAGE : FirstPass.sh [OPTIONS] <WAV_FILE> <OUTPUT_DIR>" | 102 | echo "BAD USAGE : FirstPass.sh [OPTIONS] <WAV_FILE> <OUTPUT_DIR>" |
101 | echo "$0 -h for more info" | 103 | echo "$0 -h for more info" |
102 | exit 1 | 104 | exit 1 |
103 | fi | 105 | fi |
104 | 106 | ||
105 | shift $((OPTIND-1)) | 107 | shift $((OPTIND-1)) |
106 | # check audio file - First argument | 108 | # check audio file - First argument |
107 | if [ -e $1 ] && [ -s $1 ] | 109 | if [ -e $1 ] && [ -s $1 ] |
108 | then | 110 | then |
109 | # absolute path to wav file | 111 | # absolute path to wav file |
110 | WAV_FILE=$(readlink -e $1) | 112 | WAV_FILE=$(readlink -e $1) |
111 | # wav filename | 113 | # wav filename |
112 | FILENAME=$(basename $WAV_FILE) | 114 | FILENAME=$(basename $WAV_FILE) |
113 | # wav filename without extension | 115 | # wav filename without extension |
114 | BASENAME=${FILENAME%.*} | 116 | BASENAME=${FILENAME%.*} |
115 | 117 | ||
116 | print_info "[${BASENAME}] => P1 start | $(date +'%d/%m/%y %H:%M:%S')" 1 | 118 | print_info "[${BASENAME}] => P1 start | $(date +'%d/%m/%y %H:%M:%S')" 1 |
117 | print_info "[${BASENAME}] $WAV_FILE OK" 2 | 119 | print_info "[${BASENAME}] $WAV_FILE OK" 2 |
118 | else | 120 | else |
119 | print_error "can't find $1 OR file is empty" | 121 | print_error "can't find $1 OR file is empty" |
120 | exit 1 | 122 | exit 1 |
121 | fi | 123 | fi |
122 | 124 | ||
123 | # check output directory - Second argument | 125 | # check output directory - Second argument |
124 | if [ ! -e $2 ] | 126 | if [ ! -e $2 ] |
125 | then | 127 | then |
126 | mkdir -p $2 | 128 | mkdir -p $2 |
127 | print_info "[${BASENAME}] Make directory $2" 2 | 129 | print_info "[${BASENAME}] Make directory $2" 2 |
128 | fi | 130 | fi |
129 | 131 | ||
130 | 132 | ||
131 | #-------------# | 133 | #-------------# |
132 | # GLOBAL VARS # | 134 | # GLOBAL VARS # |
133 | #-------------# | 135 | #-------------# |
134 | OUTPUT_DIR=$(readlink -e $2) # Output directory absolute path | 136 | OUTPUT_DIR=$(readlink -e $2) # Output directory absolute path |
135 | OUTPUT_DIR_BASENAME="$OUTPUT_DIR/$BASENAME/" # New OUTPUT_DIR with BASENAME | 137 | OUTPUT_DIR_BASENAME="$OUTPUT_DIR/$BASENAME/" # New OUTPUT_DIR with BASENAME |
136 | PLP_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.plp" # Global PLP file | 138 | PLP_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.plp" # Global PLP file |
137 | PLP_DIR="$OUTPUT_DIR_BASENAME/PLP/" # Segmented PLP files directory | 139 | PLP_DIR="$OUTPUT_DIR_BASENAME/PLP/" # Segmented PLP files directory |
138 | SEG_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.seg" # Global Seg file | 140 | SEG_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.seg" # Global Seg file |
139 | LBL_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.lbl" # Global LBL file | 141 | LBL_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.lbl" # Global LBL file |
140 | RES_DIR=$OUTPUT_DIR_BASENAME"/res_p1" | 142 | RES_DIR=$OUTPUT_DIR_BASENAME"/res_p1" |
141 | LOGFILE="$OUTPUT_DIR_BASENAME/info_p1.log" | 143 | LOGFILE="$OUTPUT_DIR_BASENAME/info_p1.log" |
142 | ERRORFILE="$OUTPUT_DIR_BASENAME/error_p1.log" | 144 | ERRORFILE="$OUTPUT_DIR_BASENAME/error_p1.log" |
143 | 145 | ||
144 | #------------------# | 146 | #------------------# |
145 | # Create WORKSPACE # | 147 | # Create WORKSPACE # |
146 | #------------------# | 148 | #------------------# |
147 | if [ ! -e $OUTPUT_DIR_BASENAME ] | 149 | if [ ! -e $OUTPUT_DIR_BASENAME ] |
148 | then | 150 | then |
149 | mkdir -p $OUTPUT_DIR_BASENAME | 151 | mkdir -p $OUTPUT_DIR_BASENAME |
150 | print_info "[${BASENAME}] Make directory $OUTPUT_DIR_BASENAME" 2 | 152 | print_info "[${BASENAME}] Make directory $OUTPUT_DIR_BASENAME" 2 |
151 | fi | 153 | fi |
152 | 154 | ||
153 | # Lock directory | 155 | # Lock directory |
154 | if [ -e $OUTPUT_DIR_BASENAME/FIRSTPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1; fi | 156 | if [ -e $OUTPUT_DIR_BASENAME/FIRSTPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1; fi |
155 | rm "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" > /dev/null 2>&1 | 157 | rm "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" > /dev/null 2>&1 |
156 | touch "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" > /dev/null 2>&1 | 158 | touch "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" > /dev/null 2>&1 |
157 | 159 | ||
158 | rm -r $PLP_DIR > /dev/null 2>&1; | 160 | rm -r $PLP_DIR > /dev/null 2>&1; |
159 | mkdir -p $PLP_DIR | 161 | mkdir -p $PLP_DIR |
160 | print_info "[${BASENAME}] Make directory $PLP_DIR" 2 | 162 | print_info "[${BASENAME}] Make directory $PLP_DIR" 2 |
161 | if [ $RERUN -eq 0 ]; | 163 | if [ $RERUN -eq 0 ]; |
162 | then | 164 | then |
163 | rm -r $RES_DIR > /dev/null 2>&1; | 165 | rm -r $RES_DIR > /dev/null 2>&1; |
164 | else | 166 | else |
165 | rm $RES_DIR/*.lock > /dev/null 2>&1 | 167 | rm $RES_DIR/*.lock > /dev/null 2>&1 |
166 | fi | 168 | fi |
167 | mkdir -p $RES_DIR > /dev/null 2>&1 | 169 | mkdir -p $RES_DIR > /dev/null 2>&1 |
168 | print_info "[${BASENAME}] Make directory $RES_DIR" 2 | 170 | print_info "[${BASENAME}] Make directory $RES_DIR" 2 |
169 | rm $LOGFILE $ERRORFILE > /dev/null 2>&1 | 171 | rm $LOGFILE $ERRORFILE > /dev/null 2>&1 |
170 | 172 | ||
171 | #--------------------# | 173 | #--------------------# |
172 | # Save configuration # | 174 | # Save configuration # |
173 | #--------------------# | 175 | #--------------------# |
174 | cp $FIRSTPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/FirstPass.cfg | 176 | cp $FIRSTPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/FirstPass.cfg |
175 | echo "FIRSTPASS_SCRIPT_PATH=$MAIN_SCRIPT_PATH" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg | 177 | echo "FIRSTPASS_SCRIPT_PATH=$MAIN_SCRIPT_PATH" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
176 | echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg | 178 | echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
177 | echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg | 179 | echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
178 | echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg | 180 | echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
179 | echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg | 181 | echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
180 | echo "PLP_FILE=$PLP_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg | 182 | echo "PLP_FILE=$PLP_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
181 | echo "PLP_DIR=$PLP_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg | 183 | echo "PLP_DIR=$PLP_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
182 | echo "SEG_FILE=$SEG_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg | 184 | echo "SEG_FILE=$SEG_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
183 | echo "LBL_FILE=$LBL_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg | 185 | echo "LBL_FILE=$LBL_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
184 | echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg | 186 | echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
185 | print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/FirstPass.cfg" 1 | 187 | print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/FirstPass.cfg" 1 |
186 | 188 | ||
187 | #-------------------------# | 189 | #-------------------------# |
188 | # Check Audio File Format # | 190 | # Check Audio File Format # |
189 | #-------------------------# | 191 | #-------------------------# |
190 | error=0 | 192 | error=0 |
191 | temp=$(avconv -i $WAV_FILE 2>&1 | grep "16000 Hz") | 193 | temp=$(avconv -i $WAV_FILE 2>&1 | grep "16000 Hz") |
192 | if [ -z "$temp" ]; then error=1; fi | 194 | if [ -z "$temp" ]; then error=1; fi |
193 | temp=$(avconv -i $WAV_FILE 2>&1 | grep "1 channels") | 195 | temp=$(avconv -i $WAV_FILE 2>&1 | grep "1 channels") |
194 | if [ -z "$temp" ]; then error=1; fi | 196 | if [ -z "$temp" ]; then error=1; fi |
195 | temp=$(avconv -i $WAV_FILE 2>&1 | grep "s16") | 197 | temp=$(avconv -i $WAV_FILE 2>&1 | grep "s16") |
196 | if [ -z "$temp" ]; then error=1; fi | 198 | if [ -z "$temp" ]; then error=1; fi |
197 | 199 | ||
198 | if [ $error -eq 1 ] | 200 | if [ $error -eq 1 ] |
199 | then | 201 | then |
200 | print_message $WARNING 2 "[${BASENAME}] $WAV_FILE is not a wav file at 16000 Hz, 1 channel, 16bits\nhave to convert" | 202 | print_message $WARNING 2 "[${BASENAME}] $WAV_FILE is not a wav file at 16000 Hz, 1 channel, 16bits\nhave to convert" |
201 | print_message $INFO 3 "[${BASENAME}] avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav" | 203 | print_message $INFO 3 "[${BASENAME}] avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav" |
202 | avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav $REDIRECTION_OUTPUT | 204 | avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav $REDIRECTION_OUTPUT |
203 | WAV_FILE=$OUTPUT_DIR_BASENAME/$BASENAME.wav | 205 | WAV_FILE=$OUTPUT_DIR_BASENAME/$BASENAME.wav |
204 | FILENAME=$BASENAME.wav | 206 | FILENAME=$BASENAME.wav |
205 | print_message $INFO 1 "[${BASENAME}] new wav file : $WAV_FILE" | 207 | print_message $INFO 1 "[${BASENAME}] new wav file : $WAV_FILE" |
206 | fi | 208 | fi |
207 | 209 | ||
208 | #---------------# | 210 | #---------------# |
209 | # Get SRT file # | 211 | # Get SRT file # |
210 | #---------------# | 212 | #---------------# |
211 | if [ -s $(dirname $WAV_FILE)/$BASENAME.SRT ] | 213 | if [ -s $(dirname $WAV_FILE)/$BASENAME.SRT ] |
212 | then | 214 | then |
213 | cp $(dirname $WAV_FILE)/$BASENAME.SRT $OUTPUT_DIR_BASENAME/$BASENAME.SRT | 215 | cp $(dirname $WAV_FILE)/$BASENAME.SRT $OUTPUT_DIR_BASENAME/$BASENAME.SRT |
214 | print_info "[${BASENAME}] copy $BASENAME.SRT file into $OUTPUT_DIR_BASENAME" 3 | 216 | print_info "[${BASENAME}] copy $BASENAME.SRT file into $OUTPUT_DIR_BASENAME" 3 |
215 | fi | 217 | fi |
216 | 218 | ||
217 | #------------# | 219 | #------------# |
218 | # WAV -> PLP # | 220 | # WAV -> PLP # |
219 | #------------# | 221 | #------------# |
220 | print_info "[${BASENAME}] convert WAV -> PLP" 1 | 222 | print_info "[${BASENAME}] convert WAV -> PLP" 1 |
221 | echo $FILENAME > $OUTPUT_DIR_BASENAME/list.tmp | 223 | echo $FILENAME > $OUTPUT_DIR_BASENAME/list.tmp |
222 | print_info "[${BASENAME}] $BIN_PATH/lia_plp_mt.32 --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms" 3 | 224 | print_info "[${BASENAME}] $BIN_PATH/lia_plp_mt.32 --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms" 3 |
223 | 225 | ||
224 | $BIN_PATH/lia_plp_mt$ARCH --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms $REDIRECTION_OUTPUT | 226 | $BIN_PATH/lia_plp_mt$ARCH --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms $REDIRECTION_OUTPUT |
225 | 227 | ||
226 | if [ $CHECK -eq 1 ] | 228 | if [ $CHECK -eq 1 ] |
227 | then | 229 | then |
228 | check_first_pass_plp "$PLP_FILE" | 230 | check_first_pass_plp "$PLP_FILE" |
229 | if [ $? -eq 1 ] | 231 | if [ $? -eq 1 ] |
230 | then | 232 | then |
231 | print_log_file "$ERROFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating PLP file : $PLP_FILE" | 233 | print_log_file "$ERROFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating PLP file : $PLP_FILE" |
232 | echo "[${BASENAME}] ERROR : ${BASENAME} check $ERRORFILE file" > /dev/stderr | 234 | echo "[${BASENAME}] ERROR : ${BASENAME} check $ERRORFILE file" > /dev/stderr |
233 | exit 1 | 235 | exit 1 |
234 | fi | 236 | fi |
235 | fi | 237 | fi |
236 | 238 | ||
237 | rm $OUTPUT_DIR_BASENAME/list.tmp | 239 | rm $OUTPUT_DIR_BASENAME/list.tmp |
238 | 240 | ||
239 | #------------------------------# | 241 | #------------------------------# |
240 | # S/NS + SPEAKERS SEGMENTATION # | 242 | # S/NS + SPEAKERS SEGMENTATION # |
241 | #------------------------------# | 243 | #------------------------------# |
242 | print_info "[${BASENAME}] Launch speakers diarization" 1 | 244 | print_info "[${BASENAME}] Launch speakers diarization" 1 |
243 | # Calcul seg file | 245 | # Calcul seg file |
244 | print_info "[${BASENAME}] java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME" 3 | 246 | print_info "[${BASENAME}] java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME" 3 |
245 | #java -Xmx8000m -Xms2048 -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME | 247 | #java -Xmx8000m -Xms2048 -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME |
246 | java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME $REDIRECTION_OUTPUT #–doCEClustering | 248 | java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME $REDIRECTION_OUTPUT #–doCEClustering |
247 | 249 | ||
248 | if [ $CHECK -eq 1 ] && ( [ ! -e $SEG_FILE ] || [ -z $SEG_FILE ] ) | 250 | if [ $CHECK -eq 1 ] && ( [ ! -e $SEG_FILE ] || [ -z $SEG_FILE ] ) |
249 | then | 251 | then |
250 | print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating SEG file : $SEG_FILE" | 252 | print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating SEG file : $SEG_FILE" |
251 | print_error "[${BASENAME}] ERROR : check $ERRORFILE file" | 253 | print_error "[${BASENAME}] ERROR : check $ERRORFILE file" |
252 | exit 1 | 254 | exit 1 |
253 | fi | 255 | fi |
254 | 256 | ||
255 | 257 | ||
256 | # Create LBL file | 258 | # Create LBL file |
257 | print_info "Extract LBL file from SEG file" 1 | 259 | print_info "Extract LBL file from SEG file" 1 |
258 | 260 | ||
259 | cat $SEG_FILE | grep -v ";;" | cut -f3,4,5,8 -d" " | tr " " "#" | sort -k1 -n | tr "#" " " > $LBL_FILE | 261 | cat $SEG_FILE | grep -v ";;" | cut -f3,4,5,8 -d" " | tr " " "#" | sort -k1 -n | tr "#" " " > $LBL_FILE |
260 | 262 | ||
261 | if [ $CHECK -eq 1 ] && ( [ ! -e $LBL_FILE ] || [ -z $LBL_FILE ] ) | 263 | if [ $CHECK -eq 1 ] && ( [ ! -e $LBL_FILE ] || [ -z $LBL_FILE ] ) |
262 | then | 264 | then |
263 | print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating LBL file : $LBL_FILE" | 265 | print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating LBL file : $LBL_FILE" |
264 | print_error "[${BASENAME}] ERROR : check $ERRORFILE file" | 266 | print_error "[${BASENAME}] ERROR : check $ERRORFILE file" |
265 | exit 1 | 267 | exit 1 |
266 | fi | 268 | fi |
267 | 269 | ||
268 | 270 | ||
269 | #----------------------------------------------------# | 271 | #----------------------------------------------------# |
270 | # Cut global PLP file depending to LBL segmentations # | 272 | # Cut global PLP file depending to LBL segmentations # |
271 | #----------------------------------------------------# | 273 | #----------------------------------------------------# |
272 | print_info "[${BASENAME}] Cut PLP file depending to LBL segmentations" 1 | 274 | print_info "[${BASENAME}] Cut PLP file depending to LBL segmentations" 1 |
273 | print_info "[${BASENAME}] $BIN_PATH/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG" 3 | 275 | print_info "[${BASENAME}] $BIN_PATH/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG" 3 |
274 | 276 | ||
275 | $SPEERAL_TOOLS/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG $REDIRECTION_OUTPUT | 277 | $SPEERAL_TOOLS/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG $REDIRECTION_OUTPUT |
276 | 278 | ||
277 | if [ $CHECK -eq 1 ] | 279 | if [ $CHECK -eq 1 ] |
278 | then | 280 | then |
279 | check_first_pass_plps_lbl $PLP_DIR $LBL_FILE | 281 | check_first_pass_plps_lbl $PLP_DIR $LBL_FILE |
280 | if [ $? -eq 1 ] | 282 | if [ $? -eq 1 ] |
281 | then | 283 | then |
282 | print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $PLP wrong number of .plp files" | 284 | print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $PLP wrong number of .plp files" |
283 | print_error "[${BASENAME}] ERROR : check $ERRORFILE file" | 285 | print_error "[${BASENAME}] ERROR : check $ERRORFILE file" |
284 | exit 1 | 286 | exit 1 |
285 | fi | 287 | fi |
286 | fi | 288 | fi |
287 | 289 | ||
288 | # change plp files names | 290 | # change plp files names |
289 | cd $PLP_DIR; | 291 | cd $PLP_DIR; |
290 | rename -f s/_/#/g *plp | 292 | rename -f s/_/#/g *plp |
291 | rename -f s/#/_/ *plp | 293 | rename -f s/#/_/ *plp |
292 | cd $OLDPWD | 294 | cd $OLDPWD |
293 | 295 | ||
294 | #---------------------------------------------# | 296 | #---------------------------------------------# |
295 | # PLP files list depending to acoustic models # | 297 | # PLP files list depending to acoustic models # |
296 | #---------------------------------------------# | 298 | #---------------------------------------------# |
297 | print_info "[${BASENAME}] Create PLP list depending of the model" 1 | 299 | print_info "[${BASENAME}] Create PLP list depending of the model" 1 |
298 | # Create a list of plp files | 300 | # Create a list of plp files |
299 | find $PLP_DIR -type f -exec basename "{}" .plp \; | sort > $OUTPUT_DIR_BASENAME/plp.lst | 301 | find $PLP_DIR -type f -exec basename "{}" .plp \; | sort > $OUTPUT_DIR_BASENAME/plp.lst |
300 | 302 | ||
301 | rm $OUTPUT_DIR_BASENAME/plp_*.lst > /dev/null 2>&1 | 303 | rm $OUTPUT_DIR_BASENAME/plp_*.lst > /dev/null 2>&1 |
302 | for (( i=0; $i<${#MTAG[@]} ; i++ )) | 304 | for (( i=0; $i<${#MTAG[@]} ; i++ )) |
303 | do | 305 | do |
304 | a=`grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst` | 306 | a=`grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst` |
305 | if [ -n "$a" ]; then | 307 | if [ -n "$a" ]; then |
306 | print_info "[${BASENAME}] Creating $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst" 3 | 308 | print_info "[${BASENAME}] Creating $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst" 3 |
307 | grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst | sort > $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst | 309 | grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst | sort > $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst |
308 | fi | 310 | fi |
309 | done | 311 | done |
310 | 312 | ||
311 | #-----------------------# | 313 | #-----------------------# |
312 | # First Pass (DECODING) # | 314 | # First Pass (DECODING) # |
313 | #-----------------------# | 315 | #-----------------------# |
314 | # | 316 | # |
315 | # For all AM do decoding | 317 | # For all AM do decoding |
316 | # if Check error -> iter on undone decoding (max 1 times) | 318 | # if Check error -> iter on undone decoding (max 1 times) |
317 | # | 319 | # |
318 | print_info "[${BASENAME}] Launch decoding" 1 | 320 | print_info "[${BASENAME}] Launch decoding" 1 |
319 | for (( i=0; $i<${#MTAG[@]} ; i++ )) | 321 | for (( i=0; $i<${#MTAG[@]} ; i++ )) |
320 | do | 322 | do |
321 | redo=1; # nb of try if not all segs is decoded | 323 | redo=1; # nb of try if not all segs is decoded |
322 | if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ] | 324 | if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ] |
323 | then | 325 | then |
324 | todo=$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst | 326 | todo=$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst |
325 | while [ $redo -gt 0 ]; do | 327 | while [ $redo -gt 0 ]; do |
326 | rm $RES_DIR/*.lock > /dev/null 2>&1 | 328 | rm $RES_DIR/*.lock > /dev/null 2>&1 |
327 | print_info "[${BASENAME}] $SPEERAL_BIN $todo $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT" 3 | 329 | print_info "[${BASENAME}] $SPEERAL_BIN $todo $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT" 3 |
328 | # Run speeral | 330 | # Run speeral |
329 | $SPEERAL_BIN ${todo} $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT | 331 | $SPEERAL_BIN ${todo} $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT |
330 | 332 | ||
331 | # Check if error | 333 | # Check if error |
332 | if [ $CHECK -eq 1 ] | 334 | if [ $CHECK -eq 1 ] |
333 | then | 335 | then |
334 | check_first_pass_output_speeral "${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" "$RES_DIR" | 336 | check_first_pass_output_speeral "${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" "$RES_DIR" |
335 | # if error | 337 | # if error |
336 | if [ $? -eq 1 ] | 338 | if [ $? -eq 1 ] |
337 | then | 339 | then |
338 | # rerun | 340 | # rerun |
339 | redo=$(($redo - 1)); | 341 | redo=$(($redo - 1)); |
340 | print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 | 342 | print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 |
341 | print_log_file $LOGFILE "WARN : Speeral number of output ERROR ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" | 343 | print_log_file $LOGFILE "WARN : Speeral number of output ERROR ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" |
342 | # new plp list | 344 | # new plp list |
343 | # list .seg done and compare to list of seg to do | 345 | # list .seg done and compare to list of seg to do |
344 | ls $RES_DIR/*.seg | grep -e "${MTAG[$i]}" | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp | 346 | ls $RES_DIR/*.seg | grep -e "${MTAG[$i]}" | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp |
345 | diff ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" > ${OUTPUT_DIR_BASENAME}/todo.lst | 347 | diff ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" > ${OUTPUT_DIR_BASENAME}/todo.lst |
346 | rm ${OUTPUT_DIR_BASENAME}/.tmp | 348 | rm ${OUTPUT_DIR_BASENAME}/.tmp |
347 | # log seg to do | 349 | # log seg to do |
348 | print_log_file $LOGFILE "Segs not done :" | 350 | print_log_file $LOGFILE "Segs not done :" |
349 | cat ${OUTPUT_DIR_BASENAME}/todo.lst >> $LOGFILE | 351 | cat ${OUTPUT_DIR_BASENAME}/todo.lst >> $LOGFILE |
350 | todo=${OUTPUT_DIR_BASENAME}/todo.lst | 352 | todo=${OUTPUT_DIR_BASENAME}/todo.lst |
351 | print_warn "[${BASENAME}] Try $redo" 3 | 353 | print_warn "[${BASENAME}] Try $redo" 3 |
352 | fi | 354 | fi |
353 | fi | 355 | fi |
354 | done | 356 | done |
355 | rm ${OUTPUT_DIR_BASENAME}/todo.lst > /dev/null 2>&1 | 357 | rm ${OUTPUT_DIR_BASENAME}/todo.lst > /dev/null 2>&1 |
356 | #rm $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst | 358 | #rm $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst |
357 | rm $RES_DIR/*.lock > /dev/null 2>&1 | 359 | rm $RES_DIR/*.lock > /dev/null 2>&1 |
358 | fi | 360 | fi |
359 | done | 361 | done |
360 | 362 | ||
361 | ## Check missing seg and log it | 363 | ## Check missing seg and log it |
362 | if [ $CHECK -eq 1 ] | 364 | if [ $CHECK -eq 1 ] |
363 | then | 365 | then |
364 | ls $RES_DIR/*.seg | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp | 366 | ls $RES_DIR/*.seg | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp |
365 | todo=$(cat ${OUTPUT_DIR_BASENAME}/plp.lst | wc -l) | 367 | todo=$(cat ${OUTPUT_DIR_BASENAME}/plp.lst | wc -l) |
366 | if [ $todo -eq 0 ]; then todo=1;fi | 368 | if [ $todo -eq 0 ]; then todo=1;fi |
367 | notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) | 369 | notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) |
368 | pourcentage=$((($notdone*100)/$todo)) | 370 | pourcentage=$((($notdone*100)/$todo)) |
369 | 371 | ||
370 | if [ $notdone -ne 0 ] | 372 | if [ $notdone -ne 0 ] |
371 | then | 373 | then |
372 | print_error "[${BASENAME}] ERROR : check $ERRORFILE" | 374 | print_error "[${BASENAME}] ERROR : check $ERRORFILE" |
373 | print_log_file "$ERRORFILE" "ERROR : Segs not done [" | 375 | print_log_file "$ERRORFILE" "ERROR : Segs not done [" |
374 | diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE | 376 | diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE |
375 | print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" | 377 | print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" |
376 | else | 378 | else |
377 | print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" | 379 | print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" |
378 | fi | 380 | fi |
379 | rm ${OUTPUT_DIR_BASENAME}/.tmp | 381 | rm ${OUTPUT_DIR_BASENAME}/.tmp |
380 | fi | 382 | fi |
381 | 383 | ||
382 | #---------------# | 384 | #---------------# |
383 | # Convert res # | 385 | # Convert res # |
384 | #---------------# | 386 | #---------------# |
385 | print_info "[${BASENAME}] Convert .res into .ctm" 1 | 387 | print_info "[${BASENAME}] Convert .res into .ctm" 1 |
386 | # .res => .ctm | 388 | # .res => .ctm |
387 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.ctm $REDIRECTION_OUTPUT | 389 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.ctm $REDIRECTION_OUTPUT |
388 | print_info "[${BASENAME}] Convert .res into .trs" 1 | 390 | print_info "[${BASENAME}] Convert .res into .trs" 1 |
389 | # .res => .trs | 391 | # .res => .trs |
390 | echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR_BASENAME/$BASENAME.seg" > $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg | 392 | echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR_BASENAME/$BASENAME.seg" > $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg |
391 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.trs --trs_config $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg $REDIRECTION_OUTPUT | 393 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.trs --trs_config $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg $REDIRECTION_OUTPUT |
392 | rm $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 2> /dev/null | 394 | rm $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 2> /dev/null |
393 | print_info "[${BASENAME}] Convert .res into .txt" 1 | 395 | print_info "[${BASENAME}] Convert .res into .txt" 1 |
394 | # .res => .txt | 396 | # .res => .txt |
395 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.txt $REDIRECTION_OUTPUT | 397 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.txt $REDIRECTION_OUTPUT |
396 | 398 | ||
397 | print_info "[${BASENAME}] <= P1 End | $(date +'%d/%m/%y %H:%M:%S')" 1 | 399 | print_info "[${BASENAME}] <= P1 End | $(date +'%d/%m/%y %H:%M:%S')" 1 |
398 | # unlock directory | 400 | # unlock directory |
399 | mv "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" | 401 | mv "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" |
400 | 402 |
main_tools/RecomposePass.sh
1 | #!/bin/bash | 1 | #!/bin/bash |
2 | 2 | ||
3 | ##################################################### | 3 | ##################################################### |
4 | # File : RecomposePass.sh # | 4 | # File : RecomposePass.sh # |
5 | # Brief : Get missing res pass from previous pass # | 5 | # Brief : Get missing res pass from previous pass # |
6 | # Author : Jean-François Rey # | 6 | # Author : Jean-François Rey # |
7 | # Version : 1.0 # | 7 | # Version : 1.0 # |
8 | # Date : 25/07/13 # | 8 | # Date : 25/07/13 # |
9 | ##################################################### | 9 | ##################################################### |
10 | 10 | ||
11 | echo "### RecomposePass.sh ###" | ||
12 | |||
13 | |||
11 | # Check OTMEDIA_HOME env var | 14 | # Check OTMEDIA_HOME env var |
12 | if [ -z ${OTMEDIA_HOME} ] | 15 | if [ -z ${OTMEDIA_HOME} ] |
13 | then | 16 | then |
14 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) | 17 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) |
15 | export OTMEDIA_HOME=$OTMEDIA_HOME | 18 | export OTMEDIA_HOME=$OTMEDIA_HOME |
16 | fi | 19 | fi |
17 | 20 | ||
18 | # where is RecomposePass.sh | 21 | # where is RecomposePass.sh |
19 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) | 22 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) |
20 | 23 | ||
21 | # Scripts Path | 24 | # Scripts Path |
22 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts | 25 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts |
23 | 26 | ||
24 | # Include scripts | 27 | # Include scripts |
25 | . $SCRIPT_PATH"/Tools.sh" | 28 | . $SCRIPT_PATH"/Tools.sh" |
26 | . $SCRIPT_PATH"/CheckSecondPass.sh" | 29 | . $SCRIPT_PATH"/CheckSecondPass.sh" |
27 | 30 | ||
28 | # where is RescomposePass.cfg | 31 | # where is RescomposePass.cfg |
29 | RECOMPOSEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/RecomposePass.cfg" | 32 | RECOMPOSEPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/RecomposePass.cfg" |
30 | if [ -e $RECOMPOSEPASS_CONFIG_FILE ] | 33 | if [ -e $RECOMPOSEPASS_CONFIG_FILE ] |
31 | then | 34 | then |
32 | . $RECOMPOSEPASS_CONFIG_FILE | 35 | . $RECOMPOSEPASS_CONFIG_FILE |
33 | else | 36 | else |
34 | echo "ERROR : Can't find configuration file $RECOMPOSEPASS_CONFIG_FILE" >&2 | 37 | echo "ERROR : Can't find configuration file $RECOMPOSEPASS_CONFIG_FILE" >&2 |
35 | exit 1 | 38 | exit 1 |
36 | fi | 39 | fi |
37 | 40 | ||
38 | #---------------# | 41 | #---------------# |
39 | # Parse Options # | 42 | # Parse Options # |
40 | #---------------# | 43 | #---------------# |
41 | while getopts ":hDv:" opt | 44 | while getopts ":hDv:" opt |
42 | do | 45 | do |
43 | case $opt in | 46 | case $opt in |
44 | h) | 47 | h) |
45 | echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" | 48 | echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" |
46 | echo -e "\t Options:" | 49 | echo -e "\t Options:" |
47 | echo -e "\t\t-h :\tprint this message" | 50 | echo -e "\t\t-h :\tprint this message" |
48 | echo -e "\t\t-D :\tDEBUG mode on" | 51 | echo -e "\t\t-D :\tDEBUG mode on" |
49 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" | 52 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" |
50 | exit 1 | 53 | exit 1 |
51 | ;; | 54 | ;; |
52 | D) | 55 | D) |
53 | DEBUG=1 | 56 | DEBUG=1 |
54 | ;; | 57 | ;; |
55 | v) | 58 | v) |
56 | VERBOSE=$OPTARG | 59 | VERBOSE=$OPTARG |
57 | ;; | 60 | ;; |
58 | :) | 61 | :) |
59 | echo "Option -$OPTARG requires an argument." >&2 | 62 | echo "Option -$OPTARG requires an argument." >&2 |
60 | exit 1 | 63 | exit 1 |
61 | ;; | 64 | ;; |
62 | \?) | 65 | \?) |
63 | echo "BAD USAGE : unknow opton -$OPTARG" | 66 | echo "BAD USAGE : unknow opton -$OPTARG" |
64 | #exit 1 | 67 | #exit 1 |
65 | ;; | 68 | ;; |
66 | esac | 69 | esac |
67 | done | 70 | done |
68 | 71 | ||
69 | # mode debug enable | 72 | # mode debug enable |
70 | if [ $DEBUG -eq 1 ] | 73 | if [ $DEBUG -eq 1 ] |
71 | then | 74 | then |
72 | set -x | 75 | set -x |
73 | echo -e "## Mode DEBUG ON ##" | 76 | echo -e "## Mode DEBUG ON ##" |
74 | fi | 77 | fi |
75 | 78 | ||
76 | # mode verbose enable | 79 | # mode verbose enable |
77 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi | 80 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi |
78 | 81 | ||
79 | # Check USAGE by arguments number | 82 | # Check USAGE by arguments number |
80 | if [ $(($#-($OPTIND-1))) -ne 1 ] | 83 | if [ $(($#-($OPTIND-1))) -ne 1 ] |
81 | then | 84 | then |
82 | echo "BAD USAGE : RecomposePass.sh [OPTIONS] <PASS_DIR>" | 85 | echo "BAD USAGE : RecomposePass.sh [OPTIONS] <PASS_DIR>" |
83 | echo "$0 -h for more info" | 86 | echo "$0 -h for more info" |
84 | exit 1 | 87 | exit 1 |
85 | fi | 88 | fi |
86 | 89 | ||
87 | shift $((OPTIND-1)) | 90 | shift $((OPTIND-1)) |
88 | # check Pass directory - First argument | 91 | # check Pass directory - First argument |
89 | if [ -e $1 ] && [ -d $1 ] | 92 | if [ -e $1 ] && [ -d $1 ] |
90 | then | 93 | then |
91 | PASS_DIR=$(readlink -e $1) | 94 | PASS_DIR=$(readlink -e $1) |
92 | else | 95 | else |
93 | print_error "can't find $1 directory" | 96 | print_error "can't find $1 directory" |
94 | exit 1 | 97 | exit 1 |
95 | fi | 98 | fi |
96 | 99 | ||
97 | #-------------# | 100 | #-------------# |
98 | # GLOBAL VARS # | 101 | # GLOBAL VARS # |
99 | #-------------# | 102 | #-------------# |
100 | THIRDPASS_CONFIG_FILE="$PASS_DIR/ThirdPass.cfg" | 103 | THIRDPASS_CONFIG_FILE="$PASS_DIR/ThirdPass.cfg" |
101 | if [ -e $THIRDPASS_CONFIG_FILE ] | 104 | if [ -e $THIRDPASS_CONFIG_FILE ] |
102 | then | 105 | then |
103 | RES_P3=$(cat $THIRDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") | 106 | RES_P3=$(cat $THIRDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") |
104 | else | 107 | else |
105 | print_warn "can't find $THIRDPASS_CONFIG_FILE file" 1 | 108 | print_warn "can't find $THIRDPASS_CONFIG_FILE file" 1 |
106 | #exit 1 | 109 | #exit 1 |
107 | fi | 110 | fi |
108 | SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg" | 111 | SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg" |
109 | if [ -e $SECONDPASS_CONFIG_FILE ] | 112 | if [ -e $SECONDPASS_CONFIG_FILE ] |
110 | then | 113 | then |
111 | RES_P2=$(cat $SECONDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") | 114 | RES_P2=$(cat $SECONDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") |
112 | else | 115 | else |
113 | print_warn "can't find $SECONDPASS_CONFIG_FILE file" 1 | 116 | print_warn "can't find $SECONDPASS_CONFIG_FILE file" 1 |
114 | #exit 1 | 117 | #exit 1 |
115 | fi | 118 | fi |
116 | FIRSTPASS_CONFIG_FILE="$PASS_DIR/FirstPass.cfg" | 119 | FIRSTPASS_CONFIG_FILE="$PASS_DIR/FirstPass.cfg" |
117 | if [ -e $FIRSTPASS_CONFIG_FILE ] | 120 | if [ -e $FIRSTPASS_CONFIG_FILE ] |
118 | then | 121 | then |
119 | RES_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") | 122 | RES_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") |
120 | else | 123 | else |
121 | print_warn "can't find $FIRSTPASS_CONFIG_FILE file" | 124 | print_warn "can't find $FIRSTPASS_CONFIG_FILE file" |
122 | #exit 1 | 125 | #exit 1 |
123 | fi | 126 | fi |
124 | 127 | ||
125 | if ( [ -z "$RES_P1" ] && [ -z "$RES_P2" ] ) || [ -z $RES_P3 ]; then echo "ERROR no configuration pass files in $PASS_DIR"; exit 1; fi | 128 | if ( [ -z "$RES_P1" ] && [ -z "$RES_P2" ] ) || [ -z $RES_P3 ]; then echo "ERROR no configuration pass files in $PASS_DIR"; exit 1; fi |
126 | 129 | ||
127 | BASENAME=$(basename $PASS_DIR) | 130 | BASENAME=$(basename $PASS_DIR) |
128 | OUTPUT_DIR_BASENAME=$PASS_DIR | 131 | OUTPUT_DIR_BASENAME=$PASS_DIR |
129 | RES_DIR="$PASS_DIR/res_all" | 132 | RES_DIR="$PASS_DIR/res_all" |
130 | LOGFILE=$(dirname $PASS_DIR)"/info_recpass.log" | 133 | LOGFILE=$(dirname $PASS_DIR)"/info_recpass.log" |
131 | ERRORFILE=$(dirname $PASS_DIR)"/error_recpass.log" | 134 | ERRORFILE=$(dirname $PASS_DIR)"/error_recpass.log" |
132 | 135 | ||
133 | #------------------# | 136 | #------------------# |
134 | # Create WORKSPACE # | 137 | # Create WORKSPACE # |
135 | #------------------# | 138 | #------------------# |
136 | 139 | ||
137 | # Lock directory | 140 | # Lock directory |
138 | if [ -e $OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi | 141 | if [ -e $OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi |
139 | rm "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" > /dev/null 2>&1 | 142 | rm "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" > /dev/null 2>&1 |
140 | touch "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" > /dev/null 2>&1 | 143 | touch "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" > /dev/null 2>&1 |
141 | 144 | ||
142 | if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi | 145 | if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi |
143 | mkdir -p $RES_DIR | 146 | mkdir -p $RES_DIR |
144 | print_info "Make directory $RES_DIR" 1 | 147 | print_info "Make directory $RES_DIR" 1 |
145 | 148 | ||
146 | rm $RES_DIR 2> /dev/null | 149 | rm $RES_DIR 2> /dev/null |
147 | mkdir -p $RES_DIR | 150 | mkdir -p $RES_DIR |
148 | 151 | ||
149 | #--------------------# | 152 | #--------------------# |
150 | # Save configuration # | 153 | # Save configuration # |
151 | #--------------------# | 154 | #--------------------# |
152 | cp $RECOMPOSEPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/RecomposePass.cfg | 155 | cp $RECOMPOSEPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/RecomposePass.cfg |
153 | echo "RES_ALL=$RES_DIR" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg | 156 | echo "RES_ALL=$RES_DIR" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg |
154 | echo "RES_P1=$RES_P1" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg | 157 | echo "RES_P1=$RES_P1" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg |
155 | echo "RES_P2=$RES_P2" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg | 158 | echo "RES_P2=$RES_P2" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg |
156 | echo "RES_P3=$RES_P3" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg | 159 | echo "RES_P3=$RES_P3" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg |
157 | print_info "save config in $OUTPUT_DIR_BASENAME/RecomposePass.cfg" 1 | 160 | print_info "save config in $OUTPUT_DIR_BASENAME/RecomposePass.cfg" 1 |
158 | 161 | ||
159 | #--------------------------------------------------# | 162 | #--------------------------------------------------# |
160 | # Recompose Pass using previous pass # | 163 | # Recompose Pass using previous pass # |
161 | #--------------------------------------------------# | 164 | #--------------------------------------------------# |
162 | print_info "Launch Recompose Pass" 1 | 165 | print_info "Launch Recompose Pass" 1 |
163 | 166 | ||
164 | ## Check seg in p3 | 167 | ## Check seg in p3 |
165 | cp $RES_P3/*.res $RES_DIR | 168 | cp $RES_P3/*.res $RES_DIR |
166 | ls $RES_P3/*.res | sed -e "s|$RES_P3\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p3.tmp | 169 | ls $RES_P3/*.res | sed -e "s|$RES_P3\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p3.tmp |
167 | ls $RES_P2/*.res | sed -e "s|$RES_P2\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p2.tmp | 170 | ls $RES_P2/*.res | sed -e "s|$RES_P2\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p2.tmp |
168 | ls $RES_P1/*.res | sed -e "s|$RES_P1\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p1.tmp | 171 | ls $RES_P1/*.res | sed -e "s|$RES_P1\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_p1.tmp |
169 | diff_3_2=$(diff ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp | grep -e "^> " | sed -e "s/> //") | 172 | diff_3_2=$(diff ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp | grep -e "^> " | sed -e "s/> //") |
170 | for res in $diff_3_2 | 173 | for res in $diff_3_2 |
171 | do | 174 | do |
172 | cp $RES_P2/$res.res $RES_DIR | 175 | cp $RES_P2/$res.res $RES_DIR |
173 | done | 176 | done |
174 | ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_all.tmp | 177 | ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/res_all.tmp |
175 | diff_3_1=$(diff ${OUTPUT_DIR_BASENAME}/res_all.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp | grep -e "^> " | sed -e "s/> //") | 178 | diff_3_1=$(diff ${OUTPUT_DIR_BASENAME}/res_all.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp | grep -e "^> " | sed -e "s/> //") |
176 | for res in $diff_3_1 | 179 | for res in $diff_3_1 |
177 | do | 180 | do |
178 | cp $RES_P1/$res.res $RES_DIR | 181 | cp $RES_P1/$res.res $RES_DIR |
179 | done | 182 | done |
180 | 183 | ||
181 | rm ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp ${OUTPUT_DIR_BASENAME}/res_all.tmp | 184 | rm ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp ${OUTPUT_DIR_BASENAME}/res_all.tmp |
182 | 185 | ||
183 | #---------------# | 186 | #---------------# |
184 | # Convert res # | 187 | # Convert res # |
185 | #---------------# | 188 | #---------------# |
186 | 189 | ||
187 | # .res => .ctm | 190 | # .res => .ctm |
188 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.ctm | 191 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.ctm |
189 | # .res => .trs | 192 | # .res => .trs |
190 | echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg | 193 | echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg |
191 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg | 194 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg |
192 | rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg | 195 | rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg |
193 | # .res => .txt | 196 | # .res => .txt |
194 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.txt | 197 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.txt |
195 | 198 | ||
196 | 199 | ||
197 | print_info "<= End $BASENAME RecomposePass | $(date +'%d/%m/%y %H:%M:%S')" 1 | 200 | print_info "<= End $BASENAME RecomposePass | $(date +'%d/%m/%y %H:%M:%S')" 1 |
198 | 201 | ||
199 | # unlock directory | 202 | # unlock directory |
200 | mv "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" | 203 | mv "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" |
201 | 204 | ||
202 | 205 |
main_tools/ScoringRes.sh
1 | #!/bin/bash | 1 | #!/bin/bash |
2 | 2 | ||
3 | ##################################################### | 3 | ##################################################### |
4 | # File : ScoringRes.sh # | 4 | # File : ScoringRes.sh # |
5 | # Brief : Score result using SRT file # | 5 | # Brief : Score result using SRT file # |
6 | # Author : Jean-François Rey # | 6 | # Author : Jean-François Rey # |
7 | # Version : 1.0 # | 7 | # Version : 1.0 # |
8 | # Date : 11/07/13 # | 8 | # Date : 11/07/13 # |
9 | ##################################################### | 9 | ##################################################### |
10 | 10 | ||
11 | echo "### ScoringRes.sh ###" | ||
12 | |||
11 | # Check OTMEDIA_HOME env var | 13 | # Check OTMEDIA_HOME env var |
12 | if [ -z ${OTMEDIA_HOME} ] | 14 | if [ -z ${OTMEDIA_HOME} ] |
13 | then | 15 | then |
14 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) | 16 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) |
15 | export OTMEDIA_HOME=$OTMEDIA_HOME | 17 | export OTMEDIA_HOME=$OTMEDIA_HOME |
16 | fi | 18 | fi |
17 | 19 | ||
18 | # where is ScoringRes.sh | 20 | # where is ScoringRes.sh |
19 | MAIN_SCRIPT_PATH=$OTMEDIA_HOME/main_tools | 21 | MAIN_SCRIPT_PATH=$OTMEDIA_HOME/main_tools |
20 | 22 | ||
21 | # Scripts Path | 23 | # Scripts Path |
22 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts | 24 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts |
23 | 25 | ||
24 | # Include scripts | 26 | # Include scripts |
25 | . $SCRIPT_PATH"/Tools.sh" | 27 | . $SCRIPT_PATH"/Tools.sh" |
26 | 28 | ||
27 | # where is ScoringRes.cfg | 29 | # where is ScoringRes.cfg |
28 | SCORINGRES_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ScoringRes.cfg" | 30 | SCORINGRES_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ScoringRes.cfg" |
29 | if [ -e $SCORINGRES_CONFIG_FILE ] | 31 | if [ -e $SCORINGRES_CONFIG_FILE ] |
30 | then | 32 | then |
31 | . $SCORINGRES_CONFIG_FILE | 33 | . $SCORINGRES_CONFIG_FILE |
32 | else | 34 | else |
33 | echo "ERROR : Can't find configuration file $SCORINGRES_CONFIG_FILE" >&2 | 35 | echo "ERROR : Can't find configuration file $SCORINGRES_CONFIG_FILE" >&2 |
34 | exit 1 | 36 | exit 1 |
35 | fi | 37 | fi |
36 | 38 | ||
37 | #---------------# | 39 | #---------------# |
38 | # Parse Options # | 40 | # Parse Options # |
39 | #---------------# | 41 | #---------------# |
40 | while getopts ":hDv:" opt | 42 | while getopts ":hDv:" opt |
41 | do | 43 | do |
42 | case $opt in | 44 | case $opt in |
43 | h) | 45 | h) |
44 | echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" | 46 | echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" |
45 | echo -e "\t Options:" | 47 | echo -e "\t Options:" |
46 | echo -e "\t\t-h :\tprint this message" | 48 | echo -e "\t\t-h :\tprint this message" |
47 | echo -e "\t\t-D :\tDEBUG mode on" | 49 | echo -e "\t\t-D :\tDEBUG mode on" |
48 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" | 50 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" |
49 | exit 1 | 51 | exit 1 |
50 | ;; | 52 | ;; |
51 | D) | 53 | D) |
52 | DEBUG=1 | 54 | DEBUG=1 |
53 | ;; | 55 | ;; |
54 | v) | 56 | v) |
55 | VERBOSE=$OPTARG | 57 | VERBOSE=$OPTARG |
56 | ;; | 58 | ;; |
57 | :) | 59 | :) |
58 | echo "Option -$OPTARG requires an argument." >&2 | 60 | echo "Option -$OPTARG requires an argument." >&2 |
59 | exit 1 | 61 | exit 1 |
60 | ;; | 62 | ;; |
61 | \?) | 63 | \?) |
62 | echo "BAD USAGE : unknow opton -$OPTARG" | 64 | echo "BAD USAGE : unknow opton -$OPTARG" |
63 | #exit 1 | 65 | #exit 1 |
64 | ;; | 66 | ;; |
65 | esac | 67 | esac |
66 | 68 | ||
67 | done | 69 | done |
68 | 70 | ||
69 | # mode debug enable | 71 | # mode debug enable |
70 | if [ $DEBUG -eq 1 ] | 72 | if [ $DEBUG -eq 1 ] |
71 | then | 73 | then |
72 | set -x | 74 | set -x |
73 | echo -e "## Mode DEBUG ON ##" | 75 | echo -e "## Mode DEBUG ON ##" |
74 | fi | 76 | fi |
75 | 77 | ||
76 | # mode verbose enable | 78 | # mode verbose enable |
77 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi | 79 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi |
78 | 80 | ||
79 | # Check USAGE by arguments number | 81 | # Check USAGE by arguments number |
80 | if [ $(($#-($OPTIND-1))) -ne 1 ] | 82 | if [ $(($#-($OPTIND-1))) -ne 1 ] |
81 | then | 83 | then |
82 | echo "BAD USAGE : SecondPass.sh [OPTIONS] <PASS_DIR>" | 84 | echo "BAD USAGE : SecondPass.sh [OPTIONS] <PASS_DIR>" |
83 | echo "$0 -h for more info" | 85 | echo "$0 -h for more info" |
84 | exit 1 | 86 | exit 1 |
85 | fi | 87 | fi |
86 | 88 | ||
87 | shift $((OPTIND-1)) | 89 | shift $((OPTIND-1)) |
88 | # check Pass directory - First argument | 90 | # check Pass directory - First argument |
89 | if [ -e $1 ] && [ -d $1 ] | 91 | if [ -e $1 ] && [ -d $1 ] |
90 | then | 92 | then |
91 | PASS_DIR=$(readlink -e $1) | 93 | PASS_DIR=$(readlink -e $1) |
92 | else | 94 | else |
93 | print_error "can't find $1 directory" | 95 | print_error "can't find $1 directory" |
94 | exit 1 | 96 | exit 1 |
95 | fi | 97 | fi |
96 | 98 | ||
97 | #-------------# | 99 | #-------------# |
98 | # GLOBAL VARS # | 100 | # GLOBAL VARS # |
99 | #-------------# | 101 | #-------------# |
100 | BASENAME=$(basename $PASS_DIR) | 102 | BASENAME=$(basename $PASS_DIR) |
101 | SCORING_DIR=$PASS_DIR/scoring | 103 | SCORING_DIR=$PASS_DIR/scoring |
102 | LOGFILE="$(dirname $PASS_DIR)/info_scoring.log" | 104 | LOGFILE="$(dirname $PASS_DIR)/info_scoring.log" |
103 | ERRORFILE="$(dirname $PASS_DIR)/error_scoring.log" | 105 | ERRORFILE="$(dirname $PASS_DIR)/error_scoring.log" |
104 | 106 | ||
105 | #------------------# | 107 | #------------------# |
106 | # Create WORKSPACE # | 108 | # Create WORKSPACE # |
107 | #------------------# | 109 | #------------------# |
108 | 110 | ||
109 | # Lock directory | 111 | # Lock directory |
110 | if [ -e $PASS_DIR/*.lock ]; then echo "$PASS_DIR is lock"; exit 1;fi | 112 | if [ -e $PASS_DIR/*.lock ]; then echo "$PASS_DIR is lock"; exit 1;fi |
111 | 113 | ||
112 | rm -r $SCORING_DIR > /dev/null 2>&1 | 114 | rm -r $SCORING_DIR > /dev/null 2>&1 |
113 | mkdir -p $SCORING_DIR | 115 | mkdir -p $SCORING_DIR |
114 | print_info "Make directory $SCORING_DIR" 1 | 116 | print_info "Make directory $SCORING_DIR" 1 |
115 | 117 | ||
116 | if [ ! -e $PASS_DIR/$BASENAME.SRT ] | 118 | if [ ! -e $PASS_DIR/$BASENAME.SRT ] |
117 | then | 119 | then |
118 | echo -e "no SRT file for $BASENAME\n" | 120 | echo -e "no SRT file for $BASENAME\n" |
119 | exit 0; | 121 | exit 0; |
120 | else | 122 | else |
121 | SRT_FILE=$PASS_DIR/$BASENAME.SRT | 123 | SRT_FILE=$PASS_DIR/$BASENAME.SRT |
122 | fi | 124 | fi |
123 | 125 | ||
124 | if [ ! -e $PASS_DIR/$BASENAME.1pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.2pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.3pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.allpass.ctm ] | 126 | if [ ! -e $PASS_DIR/$BASENAME.1pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.2pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.3pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.allpass.ctm ] |
125 | then | 127 | then |
126 | echo -e "no CTM files for $BASENAME\n"; | 128 | echo -e "no CTM files for $BASENAME\n"; |
127 | exit 1; | 129 | exit 1; |
128 | fi | 130 | fi |
129 | 131 | ||
130 | i=$((-3)) | 132 | i=$((-3)) |
131 | while read line | 133 | while read line |
132 | do | 134 | do |
133 | i=$(($i + 1)); | 135 | i=$(($i + 1)); |
134 | if [ $i -eq 4 ] | 136 | if [ $i -eq 4 ] |
135 | then | 137 | then |
136 | echo " $line" | 138 | echo " $line" |
137 | i=$((0)); | 139 | i=$((0)); |
138 | fi | 140 | fi |
139 | done < $SRT_FILE > $SCORING_DIR/$BASENAME.tmp.txt | 141 | done < $SRT_FILE > $SCORING_DIR/$BASENAME.tmp.txt |
140 | 142 | ||
141 | cat $SCORING_DIR/$BASENAME.tmp.txt | sed -e "s|\n| |g" | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -e "s|<s>||g" | sed -e "s|</s>||g" | sed -e "s|\n+| |g" > $SCORING_DIR/$BASENAME.tmp2.txt | 143 | cat $SCORING_DIR/$BASENAME.tmp.txt | sed -e "s|\n| |g" | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -e "s|<s>||g" | sed -e "s|</s>||g" | sed -e "s|\n+| |g" > $SCORING_DIR/$BASENAME.tmp2.txt |
142 | #cat $SCORING_DIR/$BASENAME.tmp.txt | sed -e "s|\n| |g" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex | sed -e "s|<s>||g" | sed -e "s|</s>||g" | sed -e "s|\n+| |g" > $SCORING_DIR/$BASENAME.tmp2.txt | 144 | #cat $SCORING_DIR/$BASENAME.tmp.txt | sed -e "s|\n| |g" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t | sed -f $RULES/preprocess.regex | sed -f $RULES/lastprocess.regex | sed -e "s|<s>||g" | sed -e "s|</s>||g" | sed -e "s|\n+| |g" > $SCORING_DIR/$BASENAME.tmp2.txt |
143 | 145 | ||
144 | $SCRIPT_PATH/srt2stm.pl $SCORING_DIR/$BASENAME.tmp2.txt > "$SCORING_DIR/$BASENAME.stm" | 146 | $SCRIPT_PATH/srt2stm.pl $SCORING_DIR/$BASENAME.tmp2.txt > "$SCORING_DIR/$BASENAME.stm" |
145 | rm $SCORING_DIR/$BASENAME.tmp.txt $SCORING_DIR/$BASENAME.tmp2.txt | 147 | rm $SCORING_DIR/$BASENAME.tmp.txt $SCORING_DIR/$BASENAME.tmp2.txt |
146 | 148 | ||
147 | for ctmfile in $(ls $PASS_DIR/*.ctm) | 149 | for ctmfile in $(ls $PASS_DIR/*.ctm) |
148 | do | 150 | do |
149 | cp $ctmfile $SCORING_DIR/ | 151 | cp $ctmfile $SCORING_DIR/ |
150 | $BIN_PATH/sclite -D -F -r $SCORING_DIR/$BASENAME.stm stm -h $SCORING_DIR/$(basename $ctmfile) ctm -o all | 152 | $BIN_PATH/sclite -D -F -r $SCORING_DIR/$BASENAME.stm stm -h $SCORING_DIR/$(basename $ctmfile) ctm -o all |
151 | done | 153 | done |
152 | 154 | ||
153 | 155 | ||
154 | 156 | ||
155 | 157 |
main_tools/SecondPass.sh
1 | #!/bin/bash | 1 | #!/bin/bash |
2 | 2 | ||
3 | ##################################################### | 3 | ##################################################### |
4 | # File : SecondPass.sh # | 4 | # File : SecondPass.sh # |
5 | # Brief : Speaker adaptation + ASR second pass # | 5 | # Brief : Speaker adaptation + ASR second pass # |
6 | # Author : Jean-François Rey # | 6 | # Author : Jean-François Rey # |
7 | # (base on Emmanuel Ferreira # | 7 | # (base on Emmanuel Ferreira # |
8 | # and Hugo Mauchrétien works) # | 8 | # and Hugo Mauchrétien works) # |
9 | # Version : 1.1 # | 9 | # Version : 1.1 # |
10 | # Date : 18/06/13 # | 10 | # Date : 18/06/13 # |
11 | ##################################################### | 11 | ##################################################### |
12 | 12 | ||
13 | echo "### SecondPass.sh ###" | ||
14 | |||
13 | # Check OTMEDIA_HOME env var | 15 | # Check OTMEDIA_HOME env var |
14 | if [ -z ${OTMEDIA_HOME} ] | 16 | if [ -z ${OTMEDIA_HOME} ] |
15 | then | 17 | then |
16 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) | 18 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) |
17 | export OTMEDIA_HOME=$OTMEDIA_HOME | 19 | export OTMEDIA_HOME=$OTMEDIA_HOME |
18 | fi | 20 | fi |
19 | 21 | ||
20 | # where is SecondPass.sh | 22 | # where is SecondPass.sh |
21 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) | 23 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) |
22 | 24 | ||
23 | # Scripts Path | 25 | # Scripts Path |
24 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts | 26 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts |
25 | 27 | ||
26 | # Include scripts | 28 | # Include scripts |
27 | . $SCRIPT_PATH"/Tools.sh" | 29 | . $SCRIPT_PATH"/Tools.sh" |
28 | . $SCRIPT_PATH"/CheckSecondPass.sh" | 30 | . $SCRIPT_PATH"/CheckSecondPass.sh" |
29 | 31 | ||
30 | # where is SecondPass.cfg | 32 | # where is SecondPass.cfg |
31 | SECONDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/SecondPass.cfg" | 33 | SECONDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/SecondPass.cfg" |
32 | if [ -e $SECONDPASS_CONFIG_FILE ] | 34 | if [ -e $SECONDPASS_CONFIG_FILE ] |
33 | then | 35 | then |
34 | . $SECONDPASS_CONFIG_FILE | 36 | . $SECONDPASS_CONFIG_FILE |
35 | else | 37 | else |
36 | echo "ERROR : Can't find configuration file $SECONDPASS_CONFIG_FILE" > /dev/stderr | 38 | echo "ERROR : Can't find configuration file $SECONDPASS_CONFIG_FILE" > /dev/stderr |
37 | echo "exit" > /dev/stderr | 39 | echo "exit" > /dev/stderr |
38 | exit 1 | 40 | exit 1 |
39 | fi | 41 | fi |
40 | 42 | ||
41 | #---------------# | 43 | #---------------# |
42 | # Parse Options # | 44 | # Parse Options # |
43 | #---------------# | 45 | #---------------# |
44 | while getopts ":hDv:crf:" opt | 46 | while getopts ":hDv:crf:" opt |
45 | do | 47 | do |
46 | case $opt in | 48 | case $opt in |
47 | h) | 49 | h) |
48 | echo -e "$0 [OPTIONS] <FIRST_PASS_DIRECTORY>\n" | 50 | echo -e "$0 [OPTIONS] <FIRST_PASS_DIRECTORY>\n" |
49 | echo -e "\t Options:" | 51 | echo -e "\t Options:" |
50 | echo -e "\t\t-h :\tprint this message" | 52 | echo -e "\t\t-h :\tprint this message" |
51 | echo -e "\t\t-D :\tDEBUG mode on" | 53 | echo -e "\t\t-D :\tDEBUG mode on" |
52 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" | 54 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" |
53 | echo -e "\t\t-c :\t Check process, stop if error detected" | 55 | echo -e "\t\t-c :\t Check process, stop if error detected" |
54 | echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)" | 56 | echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)" |
55 | echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done" | 57 | echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done" |
56 | exit 1 | 58 | exit 1 |
57 | ;; | 59 | ;; |
58 | D) | 60 | D) |
59 | DEBUG=1 | 61 | DEBUG=1 |
60 | ;; | 62 | ;; |
61 | v) | 63 | v) |
62 | VERBOSE=$OPTARG | 64 | VERBOSE=$OPTARG |
63 | ;; | 65 | ;; |
64 | c) | 66 | c) |
65 | CHECK=1 | 67 | CHECK=1 |
66 | ;; | 68 | ;; |
67 | f) | 69 | f) |
68 | FORKS="--forks $OPTARG" | 70 | FORKS="--forks $OPTARG" |
69 | ;; | 71 | ;; |
70 | r) | 72 | r) |
71 | RERUN=1 | 73 | RERUN=1 |
72 | ;; | 74 | ;; |
73 | :) | 75 | :) |
74 | echo "Option -$OPTARG requires an argument." > /dev/stderr | 76 | echo "Option -$OPTARG requires an argument." > /dev/stderr |
75 | exit 1 | 77 | exit 1 |
76 | ;; | 78 | ;; |
77 | \?) | 79 | \?) |
78 | echo "BAD USAGE : unknow opton -$OPTARG" > /dev/stderr | 80 | echo "BAD USAGE : unknow opton -$OPTARG" > /dev/stderr |
79 | exit 1 | 81 | exit 1 |
80 | ;; | 82 | ;; |
81 | esac | 83 | esac |
82 | done | 84 | done |
83 | 85 | ||
84 | # mode debug enable | 86 | # mode debug enable |
85 | if [ $DEBUG -eq 1 ] | 87 | if [ $DEBUG -eq 1 ] |
86 | then | 88 | then |
87 | set -x | 89 | set -x |
88 | echo -e "## Mode DEBUG ON ##" | 90 | echo -e "## Mode DEBUG ON ##" |
89 | REDIRECTION_OUTPUT="" | 91 | REDIRECTION_OUTPUT="" |
90 | else | 92 | else |
91 | REDIRECTION_OUTPUT=" 2> /dev/null" | 93 | REDIRECTION_OUTPUT=" 2> /dev/null" |
92 | fi | 94 | fi |
93 | 95 | ||
94 | # mode verbose enable | 96 | # mode verbose enable |
95 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi | 97 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi |
96 | 98 | ||
97 | # Check USAGE by arguments number | 99 | # Check USAGE by arguments number |
98 | if [ $(($#-($OPTIND-1))) -ne 1 ] | 100 | if [ $(($#-($OPTIND-1))) -ne 1 ] |
99 | then | 101 | then |
100 | echo "BAD USAGE : SecondPass.sh [OPTIONS] <FIRST_PASS_DIR>" | 102 | echo "BAD USAGE : SecondPass.sh [OPTIONS] <FIRST_PASS_DIR>" |
101 | echo "$0 -h for more info" | 103 | echo "$0 -h for more info" |
102 | exit 1 | 104 | exit 1 |
103 | fi | 105 | fi |
104 | 106 | ||
105 | shift $((OPTIND-1)) | 107 | shift $((OPTIND-1)) |
106 | # check FirstPass directory - First argument | 108 | # check FirstPass directory - First argument |
107 | if [ -e $1 ] && [ -d $1 ] | 109 | if [ -e $1 ] && [ -d $1 ] |
108 | then | 110 | then |
109 | FIRSTPASS_DIR=$(readlink -e $1) | 111 | FIRSTPASS_DIR=$(readlink -e $1) |
110 | else | 112 | else |
111 | print_error "can't find $1 directory" | 113 | print_error "can't find $1 directory" |
112 | exit 1 | 114 | exit 1 |
113 | fi | 115 | fi |
114 | 116 | ||
115 | print_info "[${BASENAME}] => P2 start | $(date +'%d/%m/%y %H:%M:%S')" 1 | 117 | print_info "[${BASENAME}] => P2 start | $(date +'%d/%m/%y %H:%M:%S')" 1 |
116 | 118 | ||
117 | #-------------# | 119 | #-------------# |
118 | # GLOBAL VARS # | 120 | # GLOBAL VARS # |
119 | #-------------# | 121 | #-------------# |
120 | FIRSTPASS_CONFIG_FILE="$FIRSTPASS_DIR/FirstPass.cfg" | 122 | FIRSTPASS_CONFIG_FILE="$FIRSTPASS_DIR/FirstPass.cfg" |
121 | if [ -e $FIRSTPASS_CONFIG_FILE ] | 123 | if [ -e $FIRSTPASS_CONFIG_FILE ] |
122 | then | 124 | then |
123 | WAV_FILE=$(cat $FIRSTPASS_CONFIG_FILE | grep "WAV_FILE=" | cut -f2 -d"=") | 125 | WAV_FILE=$(cat $FIRSTPASS_CONFIG_FILE | grep "WAV_FILE=" | cut -f2 -d"=") |
124 | BASENAME=$(cat $FIRSTPASS_CONFIG_FILE | grep "^BASENAME=" | cut -f2 -d"=") | 126 | BASENAME=$(cat $FIRSTPASS_CONFIG_FILE | grep "^BASENAME=" | cut -f2 -d"=") |
125 | OUTPUT_DIR=$(cat $FIRSTPASS_CONFIG_FILE | grep "OUTPUT_DIR=" | cut -f2 -d"=") | 127 | OUTPUT_DIR=$(cat $FIRSTPASS_CONFIG_FILE | grep "OUTPUT_DIR=" | cut -f2 -d"=") |
126 | OUTPUT_DIR_BASENAME=$FIRSTPASS_DIR | 128 | OUTPUT_DIR_BASENAME=$FIRSTPASS_DIR |
127 | PLP_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_FILE=" | cut -f2 -d"=") | 129 | PLP_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_FILE=" | cut -f2 -d"=") |
128 | PLP_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_DIR=" | cut -f2 -d"=") | 130 | PLP_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_DIR=" | cut -f2 -d"=") |
129 | SEG_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "SEG_FILE=" | cut -f2 -d"=") | 131 | SEG_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "SEG_FILE=" | cut -f2 -d"=") |
130 | LBL_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "LBL_FILE=" | cut -f2 -d"=") | 132 | LBL_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "LBL_FILE=" | cut -f2 -d"=") |
131 | RES_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "RES_DIR=" | cut -f2 -d"=") | 133 | RES_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "RES_DIR=" | cut -f2 -d"=") |
132 | else | 134 | else |
133 | print_error "can't find $FIRSTPASS_CONFIG_FILE file" | 135 | print_error "can't find $FIRSTPASS_CONFIG_FILE file" |
134 | print_error "exit" | 136 | print_error "exit" |
135 | exit 1 | 137 | exit 1 |
136 | fi | 138 | fi |
137 | LST=$OUTPUT_DIR_BASENAME"/lists" | 139 | LST=$OUTPUT_DIR_BASENAME"/lists" |
138 | HMM=$OUTPUT_DIR_BASENAME"/hmm/" | 140 | HMM=$OUTPUT_DIR_BASENAME"/hmm/" |
139 | RES_DIR=$OUTPUT_DIR_BASENAME"/res_p2" | 141 | RES_DIR=$OUTPUT_DIR_BASENAME"/res_p2" |
140 | LOGFILE="$OUTPUT_DIR_BASENAME/info_p2.log" | 142 | LOGFILE="$OUTPUT_DIR_BASENAME/info_p2.log" |
141 | ERRORFILE="$OUTPUT_DIR_BASENAME/error_p2.log" | 143 | ERRORFILE="$OUTPUT_DIR_BASENAME/error_p2.log" |
142 | 144 | ||
143 | #------------------# | 145 | #------------------# |
144 | # Create WORKSPACE # | 146 | # Create WORKSPACE # |
145 | #------------------# | 147 | #------------------# |
146 | 148 | ||
147 | # Lock directory | 149 | # Lock directory |
148 | if [ -e $OUTPUT_DIR_BASENAME/SECONDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] SECONDPASS is locked -> exit" 2; exit 1;fi | 150 | if [ -e $OUTPUT_DIR_BASENAME/SECONDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] SECONDPASS is locked -> exit" 2; exit 1;fi |
149 | rm "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" > /dev/null 2>&1 | 151 | rm "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" > /dev/null 2>&1 |
150 | touch "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" > /dev/null 2>&1 | 152 | touch "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" > /dev/null 2>&1 |
151 | 153 | ||
152 | rm -r $LST > /dev/null 2>&1 | 154 | rm -r $LST > /dev/null 2>&1 |
153 | mkdir -p $LST | 155 | mkdir -p $LST |
154 | print_info "[${BASENAME}] Make directory $LST" 2 | 156 | print_info "[${BASENAME}] Make directory $LST" 2 |
155 | if [ $RERUN -eq 0 ]; then rm -r $HMM > /dev/null 2>&1; fi | 157 | if [ $RERUN -eq 0 ]; then rm -r $HMM > /dev/null 2>&1; fi |
156 | mkdir -p $HMM | 158 | mkdir -p $HMM |
157 | print_info "[${BASENAME}] Make directory $HMM" 2 | 159 | print_info "[${BASENAME}] Make directory $HMM" 2 |
158 | if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi | 160 | if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi |
159 | mkdir -p $RES_DIR > /dev/null 2>&1 | 161 | mkdir -p $RES_DIR > /dev/null 2>&1 |
160 | print_info "[${BASENAME}] Make directory $RES_DIR" 2 | 162 | print_info "[${BASENAME}] Make directory $RES_DIR" 2 |
161 | rm $LOGFILE $ERRORFILE > /dev/null 2>&1 | 163 | rm $LOGFILE $ERRORFILE > /dev/null 2>&1 |
162 | 164 | ||
163 | #-------------------# | 165 | #-------------------# |
164 | # Check Pass # | 166 | # Check Pass # |
165 | #-------------------# | 167 | #-------------------# |
166 | print_info "[${BASENAME}] Check Pass 2 directory" 1 | 168 | print_info "[${BASENAME}] Check Pass 2 directory" 1 |
167 | for treil in $(ls $RES_DIR/ | grep treil) | 169 | for treil in $(ls $RES_DIR/ | grep treil) |
168 | do | 170 | do |
169 | if [ ! -s $RES_DIR/$treil ] | 171 | if [ ! -s $RES_DIR/$treil ] |
170 | then | 172 | then |
171 | bn = $(basename $treil ".treil") | 173 | bn = $(basename $treil ".treil") |
172 | rm $RES_DIR/$treil $RES_DIR/$bn.seg $RES_DIR/$bn.res $RES_DIR/$bn.pho 2> /dev/null | 174 | rm $RES_DIR/$treil $RES_DIR/$bn.seg $RES_DIR/$bn.res $RES_DIR/$bn.pho 2> /dev/null |
173 | print_info "[${BASENAME}] $RES_DIR/$bn.* files deleted.." 2 | 175 | print_info "[${BASENAME}] $RES_DIR/$bn.* files deleted.." 2 |
174 | fi | 176 | fi |
175 | done | 177 | done |
176 | 178 | ||
177 | # Check if more then 89% of treil are done | 179 | # Check if more then 89% of treil are done |
178 | nbres_p1=$(ls $RES_DIR_P1/*.res 2> /dev/null | wc -l) | 180 | nbres_p1=$(ls $RES_DIR_P1/*.res 2> /dev/null | wc -l) |
179 | nbtreil_p2=$(ls $RES_DIR/*.treil 2> /dev/null | wc -l) | 181 | nbtreil_p2=$(ls $RES_DIR/*.treil 2> /dev/null | wc -l) |
180 | if [ $nbres_p1 -gt 0 ] | 182 | if [ $nbres_p1 -gt 0 ] |
181 | then | 183 | then |
182 | pourcentage=$((($nbtreil_p2*100)/$nbres_p1)) | 184 | pourcentage=$((($nbtreil_p2*100)/$nbres_p1)) |
183 | if [ $pourcentage -gt 89 ] | 185 | if [ $pourcentage -gt 89 ] |
184 | then | 186 | then |
185 | print_info "[${BASENAME}] Lattice already done, skipping $BASENAME" 1 | 187 | print_info "[${BASENAME}] Lattice already done, skipping $BASENAME" 1 |
186 | exit 0 | 188 | exit 0 |
187 | fi | 189 | fi |
188 | else | 190 | else |
189 | print_error "[${BASENAME}] No First Pass, No .res -> exit P2" | 191 | print_error "[${BASENAME}] No First Pass, No .res -> exit P2" |
190 | if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No First Pass, No .res -> exit P2" ;fi | 192 | if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No First Pass, No .res -> exit P2" ;fi |
191 | exit 1 | 193 | exit 1 |
192 | fi | 194 | fi |
193 | 195 | ||
194 | #--------------------# | 196 | #--------------------# |
195 | # Save configuration # | 197 | # Save configuration # |
196 | #--------------------# | 198 | #--------------------# |
197 | cp $SECONDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/SecondPass.cfg | 199 | cp $SECONDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/SecondPass.cfg |
198 | echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg | 200 | echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg |
199 | echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg | 201 | echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg |
200 | echo "FIRSTPASS_DIR=$FIRSTPASS_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg | 202 | echo "FIRSTPASS_DIR=$FIRSTPASS_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg |
201 | echo "PLP_DIR_P1=$PLP_DIR_P1" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg | 203 | echo "PLP_DIR_P1=$PLP_DIR_P1" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg |
202 | echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg | 204 | echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg |
203 | echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg | 205 | echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg |
204 | echo "LST=$LST" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg | 206 | echo "LST=$LST" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg |
205 | echo "HMM=$HMM" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg | 207 | echo "HMM=$HMM" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg |
206 | echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg | 208 | echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg |
207 | print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1 | 209 | print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1 |
208 | 210 | ||
209 | 211 | ||
210 | #--------------------------------------------------# | 212 | #--------------------------------------------------# |
211 | # Speaker Adaptation (AM) + Second pass (DECODING) # | 213 | # Speaker Adaptation (AM) + Second pass (DECODING) # |
212 | #--------------------------------------------------# | 214 | #--------------------------------------------------# |
213 | print_info "[${BASENAME}] Launch Second Pass" 1 | 215 | print_info "[${BASENAME}] Launch Second Pass" 1 |
214 | 216 | ||
215 | # for all speaker | 217 | # for all speaker |
216 | for speaker in $(cat $LBL_FILE_P1 | cut -f4 -d" " | sort | uniq) | 218 | for speaker in $(cat $LBL_FILE_P1 | cut -f4 -d" " | sort | uniq) |
217 | do | 219 | do |
218 | ## get seg file from P1 containing the speaker | 220 | ## get seg file from P1 containing the speaker |
219 | find $RES_DIR_P1 -name "*${speaker}.seg" -exec basename "{}" .seg \; | sort > $LST/$speaker.lst | 221 | find $RES_DIR_P1 -name "*${speaker}.seg" -exec basename "{}" .seg \; | sort > $LST/$speaker.lst |
220 | print_info "[${BASENAME}] file for $speaker in $LST/$speaker.lst" 3 | 222 | print_info "[${BASENAME}] file for $speaker in $LST/$speaker.lst" 3 |
221 | if [ ! -s $LST/$speaker.lst ]; then print_warn "no ${speaker} file in $RES_DIR_P1" 3; continue; fi | 223 | if [ ! -s $LST/$speaker.lst ]; then print_warn "no ${speaker} file in $RES_DIR_P1" 3; continue; fi |
222 | 224 | ||
223 | 225 | ||
224 | # for all AM | 226 | # for all AM |
225 | for (( i=0; $i<${#MTAG[@]} ; i++ )) | 227 | for (( i=0; $i<${#MTAG[@]} ; i++ )) |
226 | do | 228 | do |
227 | if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then | 229 | if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then |
228 | type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst") | 230 | type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst") |
229 | ## if is the good AM for the speaker | 231 | ## if is the good AM for the speaker |
230 | if [ -n "$type" ] | 232 | if [ -n "$type" ] |
231 | then | 233 | then |
232 | ## HMM adaptation | 234 | ## HMM adaptation |
233 | if [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] ) | 235 | if [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] ) |
234 | then | 236 | then |
235 | print_info "[${BASENAME}] $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/" 3 | 237 | print_info "[${BASENAME}] $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/" 3 |
236 | $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/ $REDIRECTION_OUTPUT | 238 | $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/ $REDIRECTION_OUTPUT |
237 | fi | 239 | fi |
238 | 240 | ||
239 | if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] ) | 241 | if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] ) |
240 | then | 242 | then |
241 | print_warn "[${BASENAME}] No hmm files created for $speaker" 2 | 243 | print_warn "[${BASENAME}] No hmm files created for $speaker" 2 |
242 | print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No hmm files created for $speaker" | 244 | print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No hmm files created for $speaker" |
243 | print_error "[${BASENAME}] Check $ERRORFILE" | 245 | print_error "[${BASENAME}] Check $ERRORFILE" |
244 | #exit 1 | 246 | #exit 1 |
245 | fi | 247 | fi |
246 | 248 | ||
247 | ## cp map files | 249 | ## cp map files |
248 | cp $SPEERAL_AM/${MODS[$i]}.map $HMM/$speaker.map | 250 | cp $SPEERAL_AM/${MODS[$i]}.map $HMM/$speaker.map |
249 | 251 | ||
250 | ## class clustering | 252 | ## class clustering |
251 | if [ -s $HMM/$speaker.hmm ] && ( [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] )) | 253 | if [ -s $HMM/$speaker.hmm ] && ( [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] )) |
252 | then | 254 | then |
253 | print_info "[${BASENAME}] $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls" 3 | 255 | print_info "[${BASENAME}] $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls" 3 |
254 | $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls $REDIRECTION_OUTPUT | 256 | $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls $REDIRECTION_OUTPUT |
255 | fi | 257 | fi |
256 | if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] ) | 258 | if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] ) |
257 | then | 259 | then |
258 | print_warn "[${BASENAME}] No cls file created for $speaker" 2 | 260 | print_warn "[${BASENAME}] No cls file created for $speaker" 2 |
259 | print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No cls file created for $speakers" | 261 | print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] No cls file created for $speakers" |
260 | print_error "[${BASENAME}] Check $ERRORFILE" | 262 | print_error "[${BASENAME}] Check $ERRORFILE" |
261 | #exit 1 | 263 | #exit 1 |
262 | fi | 264 | fi |
263 | 265 | ||
264 | ## Speeral decoding | 266 | ## Speeral decoding |
265 | if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ] | 267 | if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ] |
266 | then | 268 | then |
267 | print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock $REDIRECTION_OUTPUT" 3 | 269 | print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock $REDIRECTION_OUTPUT" 3 |
268 | $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock $REDIRECTION_OUTPUT | 270 | $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock $REDIRECTION_OUTPUT |
269 | else | 271 | else |
270 | print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2 | 272 | print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2 |
271 | print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT" 3 | 273 | print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT" 3 |
272 | $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT | 274 | $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT |
273 | fi | 275 | fi |
274 | 276 | ||
275 | if [ $CHECK -eq 1 ] | 277 | if [ $CHECK -eq 1 ] |
276 | then | 278 | then |
277 | check_second_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR" | 279 | check_second_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR" |
278 | if [ $? -eq 1 ] | 280 | if [ $? -eq 1 ] |
279 | then | 281 | then |
280 | print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 | 282 | print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 |
281 | print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst" | 283 | print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst" |
282 | ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp | 284 | ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp |
283 | print_log_file $LOGFILE "Segs (and treil) not done :\n[" | 285 | print_log_file $LOGFILE "Segs (and treil) not done :\n[" |
284 | diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $LOGFILE | 286 | diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $LOGFILE |
285 | print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]" | 287 | print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]" |
286 | rm ${OUTPUT_DIR_BASENAME}/.tmp | 288 | rm ${OUTPUT_DIR_BASENAME}/.tmp |
287 | #exit 1 | 289 | #exit 1 |
288 | fi | 290 | fi |
289 | fi | 291 | fi |
290 | break | 292 | break |
291 | fi | 293 | fi |
292 | fi | 294 | fi |
293 | done | 295 | done |
294 | #rm "$HMM/$speaker.*" > /dev/null 2>&1 | 296 | #rm "$HMM/$speaker.*" > /dev/null 2>&1 |
295 | #rm "$LST/$speaker.lst" > /dev/null 2>&1 | 297 | #rm "$LST/$speaker.lst" > /dev/null 2>&1 |
296 | done | 298 | done |
297 | 299 | ||
298 | ## Check missing seg and log it | 300 | ## Check missing seg and log it |
299 | if [ $CHECK -eq 1 ] | 301 | if [ $CHECK -eq 1 ] |
300 | then | 302 | then |
301 | ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp | 303 | ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp |
302 | todo=$(cat ${PLP_FILE_P1} | wc -l) | 304 | todo=$(cat ${PLP_FILE_P1} | wc -l) |
303 | if [ $todo -eq 0 ]; then todo=1;fi | 305 | if [ $todo -eq 0 ]; then todo=1;fi |
304 | notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) | 306 | notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) |
305 | pourcentage=$((($notdone*100)/$todo)) | 307 | pourcentage=$((($notdone*100)/$todo)) |
306 | if [ $notdone -ne 0 ] | 308 | if [ $notdone -ne 0 ] |
307 | then | 309 | then |
308 | print_error "[${BASENAME}] ERROR : check $ERRORFILE" | 310 | print_error "[${BASENAME}] ERROR : check $ERRORFILE" |
309 | print_log_file "$ERRORFILE" "ERROR : Treil not done [" | 311 | print_log_file "$ERRORFILE" "ERROR : Treil not done [" |
310 | diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE | 312 | diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE |
311 | print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" | 313 | print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" |
312 | else | 314 | else |
313 | print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" | 315 | print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" |
314 | fi | 316 | fi |
315 | rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1 | 317 | rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1 |
316 | fi | 318 | fi |
317 | 319 | ||
318 | #---------------# | 320 | #---------------# |
319 | # Convert res # | 321 | # Convert res # |
320 | #---------------# | 322 | #---------------# |
321 | 323 | ||
322 | print_info "[${BASENAME}] Convert .res into .ctm" 1 | 324 | print_info "[${BASENAME}] Convert .res into .ctm" 1 |
323 | # .res => .ctm | 325 | # .res => .ctm |
324 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.ctm $REDIRECTION_OUTPUT | 326 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.ctm $REDIRECTION_OUTPUT |
325 | print_info "[${BASENAME}] Convert .res into .trs" 1 | 327 | print_info "[${BASENAME}] Convert .res into .trs" 1 |
326 | # .res => .trs | 328 | # .res => .trs |
327 | echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg | 329 | echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg |
328 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg $REDIRECTION_OUTPUT | 330 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg $REDIRECTION_OUTPUT |
329 | rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1 | 331 | rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1 |
330 | print_info "[${BASENAME}] Convert .res into .txt" 1 | 332 | print_info "[${BASENAME}] Convert .res into .txt" 1 |
331 | # .res => .txt | 333 | # .res => .txt |
332 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.txt $REDIRECTION_OUTPUT | 334 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.txt $REDIRECTION_OUTPUT |
333 | 335 | ||
334 | print_info "[${BASENAME}] <= P2 End | $(date +'%d/%m/%y %H:%M:%S')" 1 | 336 | print_info "[${BASENAME}] <= P2 End | $(date +'%d/%m/%y %H:%M:%S')" 1 |
335 | # unlock directory | 337 | # unlock directory |
336 | mv "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" | 338 | mv "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" |
337 | 339 | ||
338 | 340 |
main_tools/ThirdPass.sh
1 | #!/bin/bash | 1 | #!/bin/bash |
2 | 2 | ||
3 | ##################################################### | 3 | ##################################################### |
4 | # File : ThirdPass.sh # | 4 | # File : ThirdPass.sh # |
5 | # Brief : ASR third pass using trigg files # | 5 | # Brief : ASR third pass using trigg files # |
6 | # Author : Jean-François Rey # | 6 | # Author : Jean-François Rey # |
7 | # Version : 1.0 # | 7 | # Version : 1.0 # |
8 | # Date : 18/07/13 # | 8 | # Date : 18/07/13 # |
9 | ##################################################### | 9 | ##################################################### |
10 | 10 | ||
11 | echo "### ThirdPass.sh ###" | ||
12 | |||
11 | # Check OTMEDIA_HOME env var | 13 | # Check OTMEDIA_HOME env var |
12 | if [ -z ${OTMEDIA_HOME} ] | 14 | if [ -z ${OTMEDIA_HOME} ] |
13 | then | 15 | then |
14 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) | 16 | OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) |
15 | export OTMEDIA_HOME=$OTMEDIA_HOME | 17 | export OTMEDIA_HOME=$OTMEDIA_HOME |
16 | fi | 18 | fi |
17 | 19 | ||
18 | # where is SecondPass.sh | 20 | # where is SecondPass.sh |
19 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) | 21 | MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) |
20 | 22 | ||
21 | # Scripts Path | 23 | # Scripts Path |
22 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts | 24 | SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts |
23 | 25 | ||
24 | # Include scripts | 26 | # Include scripts |
25 | . $SCRIPT_PATH"/Tools.sh" | 27 | . $SCRIPT_PATH"/Tools.sh" |
26 | . $SCRIPT_PATH"/CheckThirdPass.sh" | 28 | . $SCRIPT_PATH"/CheckThirdPass.sh" |
27 | 29 | ||
28 | # where is ThirdPass.cfg | 30 | # where is ThirdPass.cfg |
29 | THIRDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ThirdPass.cfg" | 31 | THIRDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ThirdPass.cfg" |
30 | if [ -e $THIRDPASS_CONFIG_FILE ] | 32 | if [ -e $THIRDPASS_CONFIG_FILE ] |
31 | then | 33 | then |
32 | . $THIRDPASS_CONFIG_FILE | 34 | . $THIRDPASS_CONFIG_FILE |
33 | else | 35 | else |
34 | echo "ERROR : Can't find configuration file $THIRDPASS_CONFIG_FILE" >&2 | 36 | echo "ERROR : Can't find configuration file $THIRDPASS_CONFIG_FILE" >&2 |
35 | exit 1 | 37 | exit 1 |
36 | fi | 38 | fi |
37 | 39 | ||
38 | #---------------# | 40 | #---------------# |
39 | # Parse Options # | 41 | # Parse Options # |
40 | #---------------# | 42 | #---------------# |
41 | while getopts ":hDv:crf:" opt | 43 | while getopts ":hDv:crf:" opt |
42 | do | 44 | do |
43 | case $opt in | 45 | case $opt in |
44 | h) | 46 | h) |
45 | echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" | 47 | echo -e "$0 [OPTIONS] <PASS_DIRECTORY>\n" |
46 | echo -e "\t Options:" | 48 | echo -e "\t Options:" |
47 | echo -e "\t\t-h :\tprint this message" | 49 | echo -e "\t\t-h :\tprint this message" |
48 | echo -e "\t\t-D :\tDEBUG mode on" | 50 | echo -e "\t\t-D :\tDEBUG mode on" |
49 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" | 51 | echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" |
50 | echo -e "\t\t-c :\t Check process, stop if error detected" | 52 | echo -e "\t\t-c :\t Check process, stop if error detected" |
51 | echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)" | 53 | echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)" |
52 | echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done" | 54 | echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done" |
53 | exit 1 | 55 | exit 1 |
54 | ;; | 56 | ;; |
55 | D) | 57 | D) |
56 | DEBUG=1 | 58 | DEBUG=1 |
57 | ;; | 59 | ;; |
58 | v) | 60 | v) |
59 | VERBOSE=$OPTARG | 61 | VERBOSE=$OPTARG |
60 | ;; | 62 | ;; |
61 | c) | 63 | c) |
62 | CHECK=1 | 64 | CHECK=1 |
63 | ;; | 65 | ;; |
64 | f) | 66 | f) |
65 | FORKS="--forks $OPTARG" | 67 | FORKS="--forks $OPTARG" |
66 | ;; | 68 | ;; |
67 | r) | 69 | r) |
68 | RERUN=1 | 70 | RERUN=1 |
69 | ;; | 71 | ;; |
70 | :) | 72 | :) |
71 | echo "Option -$OPTARG requires an argument." >&2 | 73 | echo "Option -$OPTARG requires an argument." >&2 |
72 | exit 1 | 74 | exit 1 |
73 | ;; | 75 | ;; |
74 | \?) | 76 | \?) |
75 | echo "BAD USAGE : unknow opton -$OPTARG" | 77 | echo "BAD USAGE : unknow opton -$OPTARG" |
76 | #exit 1 | 78 | #exit 1 |
77 | ;; | 79 | ;; |
78 | esac | 80 | esac |
79 | done | 81 | done |
80 | 82 | ||
81 | # mode debug enable | 83 | # mode debug enable |
82 | if [ $DEBUG -eq 1 ] | 84 | if [ $DEBUG -eq 1 ] |
83 | then | 85 | then |
84 | set -x | 86 | set -x |
85 | echo -e "## Mode DEBUG ON ##" | 87 | echo -e "## Mode DEBUG ON ##" |
86 | REDIRECTION_OUTPUT="" | 88 | REDIRECTION_OUTPUT="" |
87 | else | 89 | else |
88 | REDIRECTION_OUTPUT=" 2> /dev/null" | 90 | REDIRECTION_OUTPUT=" 2> /dev/null" |
89 | fi | 91 | fi |
90 | 92 | ||
91 | # mode verbose enable | 93 | # mode verbose enable |
92 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi | 94 | if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi |
93 | 95 | ||
94 | # Check USAGE by arguments number | 96 | # Check USAGE by arguments number |
95 | if [ $(($#-($OPTIND-1))) -ne 1 ] | 97 | if [ $(($#-($OPTIND-1))) -ne 1 ] |
96 | then | 98 | then |
97 | echo "BAD USAGE : ThirdPass.sh [OPTIONS] <PASS_DIR>" | 99 | echo "BAD USAGE : ThirdPass.sh [OPTIONS] <PASS_DIR>" |
98 | echo "$0 -h for more info" | 100 | echo "$0 -h for more info" |
99 | exit 1 | 101 | exit 1 |
100 | fi | 102 | fi |
101 | 103 | ||
102 | shift $((OPTIND-1)) | 104 | shift $((OPTIND-1)) |
103 | # check Pass directory - First argument | 105 | # check Pass directory - First argument |
104 | if [ -e $1 ] && [ -d $1 ] | 106 | if [ -e $1 ] && [ -d $1 ] |
105 | then | 107 | then |
106 | PASS_DIR=$(readlink -e $1) | 108 | PASS_DIR=$(readlink -e $1) |
107 | else | 109 | else |
108 | print_error "can't find $1 directory" | 110 | print_error "can't find $1 directory" |
109 | exit 1 | 111 | exit 1 |
110 | fi | 112 | fi |
111 | 113 | ||
112 | #-------------# | 114 | #-------------# |
113 | # GLOBAL VARS # | 115 | # GLOBAL VARS # |
114 | #-------------# | 116 | #-------------# |
115 | EXPLOITCONFPASS_CONFIG_FILE="$PASS_DIR/ExploitConfPass.cfg" | 117 | EXPLOITCONFPASS_CONFIG_FILE="$PASS_DIR/ExploitConfPass.cfg" |
116 | if [ -e $EXPLOITCONFPASS_CONFIG_FILE ] | 118 | if [ -e $EXPLOITCONFPASS_CONFIG_FILE ] |
117 | then | 119 | then |
118 | TRIGGER_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "TRIGGER_SPEERAL=" | cut -f2 -d"=") | 120 | TRIGGER_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "TRIGGER_SPEERAL=" | cut -f2 -d"=") |
119 | LEX_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_SPEERAL=" | cut -f2 -d"=") | 121 | LEX_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_SPEERAL=" | cut -f2 -d"=") |
120 | LEX_BINODE_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_BINODE_SPEERAL=" | cut -f2 -d"=") | 122 | LEX_BINODE_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_BINODE_SPEERAL=" | cut -f2 -d"=") |
121 | LST="" | 123 | LST="" |
122 | PLP_DIR_P1="" | 124 | PLP_DIR_P1="" |
123 | HMM="" | 125 | HMM="" |
124 | else | 126 | else |
125 | print_error "can't find $EXPLOITCONFPASS_CONFIG_FILE file" | 127 | print_error "can't find $EXPLOITCONFPASS_CONFIG_FILE file" |
126 | #exit 1 | 128 | #exit 1 |
127 | TRIGGER_SPEERAL=$PASS_DIR/trigg/speeral/ | 129 | TRIGGER_SPEERAL=$PASS_DIR/trigg/speeral/ |
128 | LEX_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext | 130 | LEX_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext |
129 | LEX_BINODE_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext.bin | 131 | LEX_BINODE_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext.bin |
130 | fi | 132 | fi |
131 | SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg" | 133 | SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg" |
132 | if [ -e $SECONDPASS_CONFIG_FILE ] | 134 | if [ -e $SECONDPASS_CONFIG_FILE ] |
133 | then | 135 | then |
134 | LST=$(cat $SECONDPASS_CONFIG_FILE | grep "^LST=" | cut -f2 -d"=") | 136 | LST=$(cat $SECONDPASS_CONFIG_FILE | grep "^LST=" | cut -f2 -d"=") |
135 | HMM=$(cat $SECONDPASS_CONFIG_FILE | grep "^HMM=" | cut -f2 -d"=") | 137 | HMM=$(cat $SECONDPASS_CONFIG_FILE | grep "^HMM=" | cut -f2 -d"=") |
136 | PLP_DIR_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_DIR_P1=" | cut -f2 -d"=") | 138 | PLP_DIR_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_DIR_P1=" | cut -f2 -d"=") |
137 | else | 139 | else |
138 | print_error "can't find $SECONDPASS_CONFIG_FILE file" | 140 | print_error "can't find $SECONDPASS_CONFIG_FILE file" |
139 | #exit 1 | 141 | #exit 1 |
140 | LST=$PASS_DIR/lists | 142 | LST=$PASS_DIR/lists |
141 | HMM=$PASS_DIR/hmm | 143 | HMM=$PASS_DIR/hmm |
142 | PLP_DIR_P1=$PASS_DIR/PLP | 144 | PLP_DIR_P1=$PASS_DIR/PLP |
143 | fi | 145 | fi |
144 | 146 | ||
145 | BASENAME=$(basename $PASS_DIR) | 147 | BASENAME=$(basename $PASS_DIR) |
146 | OUTPUT_DIR_BASENAME=$PASS_DIR | 148 | OUTPUT_DIR_BASENAME=$PASS_DIR |
147 | RES_DIR="$PASS_DIR/res_p3" | 149 | RES_DIR="$PASS_DIR/res_p3" |
148 | LOGFILE=$(dirname $PASS_DIR)"/info_p3.log" | 150 | LOGFILE=$(dirname $PASS_DIR)"/info_p3.log" |
149 | ERRORFILE=$(dirname $PASS_DIR)"/error_p3.log" | 151 | ERRORFILE=$(dirname $PASS_DIR)"/error_p3.log" |
150 | 152 | ||
151 | #------------------# | 153 | #------------------# |
152 | # Create WORKSPACE # | 154 | # Create WORKSPACE # |
153 | #------------------# | 155 | #------------------# |
154 | 156 | ||
155 | # Lock directory | 157 | # Lock directory |
156 | if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi | 158 | if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi |
157 | rm "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" > /dev/null 2>&1 | 159 | rm "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" > /dev/null 2>&1 |
158 | touch "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" > /dev/null 2>&1 | 160 | touch "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" > /dev/null 2>&1 |
159 | 161 | ||
160 | if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi | 162 | if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi |
161 | mkdir -p $RES_DIR | 163 | mkdir -p $RES_DIR |
162 | print_info "Make directory $RES_DIR" 1 | 164 | print_info "Make directory $RES_DIR" 1 |
163 | 165 | ||
164 | #--------------------# | 166 | #--------------------# |
165 | # Save configuration # | 167 | # Save configuration # |
166 | #--------------------# | 168 | #--------------------# |
167 | cp $THIRDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/ThirdPass.cfg | 169 | cp $THIRDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/ThirdPass.cfg |
168 | echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/ThirdPass.cfg | 170 | echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/ThirdPass.cfg |
169 | print_info "save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1 | 171 | print_info "save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1 |
170 | 172 | ||
171 | #--------------------------------------------------# | 173 | #--------------------------------------------------# |
172 | # Third Pass using trigger file (DECODING) # | 174 | # Third Pass using trigger file (DECODING) # |
173 | #--------------------------------------------------# | 175 | #--------------------------------------------------# |
174 | print_info "Launch Third Pass" 1 | 176 | print_info "Launch Third Pass" 1 |
175 | 177 | ||
176 | ## Generate speeral config file adding trigger rep | 178 | ## Generate speeral config file adding trigger rep |
177 | cat $SPEERAL_CFG_PATH/$SPEERAL_CFG_FILE | sed -e "s|<nom>[^<]*</nom>|<nom>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext</nom>|g" \ | 179 | cat $SPEERAL_CFG_PATH/$SPEERAL_CFG_FILE | sed -e "s|<nom>[^<]*</nom>|<nom>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext</nom>|g" \ |
178 | | sed -e "s|<binode>[^<]*</binode>|<binode>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext.bin</binode>|g" \ | 180 | | sed -e "s|<binode>[^<]*</binode>|<binode>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext.bin</binode>|g" \ |
179 | | sed -e "s|<trigger><dir>[^<]*</dir></trigger>|<trigger><dir>$TRIGGER_SPEERAL</dir></trigger>|g" > $OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml | 181 | | sed -e "s|<trigger><dir>[^<]*</dir></trigger>|<trigger><dir>$TRIGGER_SPEERAL</dir></trigger>|g" > $OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml |
180 | SPEERAL_THIRD_CFG=$OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml | 182 | SPEERAL_THIRD_CFG=$OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml |
181 | 183 | ||
182 | # for all speaker | 184 | # for all speaker |
183 | for lspeaker in $(ls $LST/*.lst) | 185 | for lspeaker in $(ls $LST/*.lst) |
184 | do | 186 | do |
185 | speaker=$(basename $lspeaker ".lst") | 187 | speaker=$(basename $lspeaker ".lst") |
186 | # for all AM | 188 | # for all AM |
187 | for (( i=0; $i<${#MTAG[@]} ; i++ )) | 189 | for (( i=0; $i<${#MTAG[@]} ; i++ )) |
188 | do | 190 | do |
189 | if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then | 191 | if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then |
190 | type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst") | 192 | type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst") |
191 | ## if is the good AM for the speaker | 193 | ## if is the good AM for the speaker |
192 | if [ -n "$type" ] | 194 | if [ -n "$type" ] |
193 | then | 195 | then |
194 | ## Speeral decoding | 196 | ## Speeral decoding |
195 | if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ] | 197 | if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ] |
196 | then | 198 | then |
197 | print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock $REDIRECTION_OUTPUT" 3 | 199 | print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock $REDIRECTION_OUTPUT" 3 |
198 | $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock $REDIRECTION_OUTPUT | 200 | $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock $REDIRECTION_OUTPUT |
199 | else | 201 | else |
200 | print_warn "$HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2 | 202 | print_warn "$HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2 |
201 | $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT | 203 | $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT |
202 | fi | 204 | fi |
203 | 205 | ||
204 | if [ $CHECK -eq 1 ] | 206 | if [ $CHECK -eq 1 ] |
205 | then | 207 | then |
206 | check_third_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR" | 208 | check_third_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR" |
207 | if [ $? -eq 1 ] | 209 | if [ $? -eq 1 ] |
208 | then | 210 | then |
209 | echo -e "ERROR : Speeral $LST/$speaker.lst\n[" >> $ERRORFILE | 211 | echo -e "ERROR : Speeral $LST/$speaker.lst\n[" >> $ERRORFILE |
210 | ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp | 212 | ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp |
211 | diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $ERRORFILE | 213 | diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $ERRORFILE |
212 | echo -e "] " >> $ERRORFILE | 214 | echo -e "] " >> $ERRORFILE |
213 | rm ${OUTPUT_DIR_BASENAME}/.tmp | 215 | rm ${OUTPUT_DIR_BASENAME}/.tmp |
214 | #exit 1 | 216 | #exit 1 |
215 | fi | 217 | fi |
216 | fi | 218 | fi |
217 | break | 219 | break |
218 | fi | 220 | fi |
219 | fi | 221 | fi |
220 | done | 222 | done |
221 | done | 223 | done |
222 | 224 | ||
223 | ## Check missing seg and log it | 225 | ## Check missing seg and log it |
224 | ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp | 226 | ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp |
225 | echo -e "$BASENAME P3 END\n[" >> $LOGFILE | 227 | echo -e "$BASENAME P3 END\n[" >> $LOGFILE |
226 | diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $LOGFILE | 228 | diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $LOGFILE |
227 | echo -e "] $BASENAME" >> $LOGFILE | 229 | echo -e "] $BASENAME" >> $LOGFILE |
228 | rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1 | 230 | rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1 |
229 | 231 | ||
230 | #---------------# | 232 | #---------------# |
231 | # Convert res # | 233 | # Convert res # |
232 | #---------------# | 234 | #---------------# |
233 | 235 | ||
234 | # .res => .ctm | 236 | # .res => .ctm |
235 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.ctm $REDIRECTION_OUTPUT | 237 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.ctm $REDIRECTION_OUTPUT |
236 | # .res => .trs | 238 | # .res => .trs |
237 | echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg | 239 | echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg |
238 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg $REDIRECTION_OUTPUT | 240 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg $REDIRECTION_OUTPUT |
239 | rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1 | 241 | rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1 |
240 | # .res => .txt | 242 | # .res => .txt |
241 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.txt $REDIRECTION_OUTPUT | 243 | $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.txt $REDIRECTION_OUTPUT |
242 | 244 | ||
243 | 245 | ||
244 | print_info "<= End $BASENAME P3 | $(date +'%d/%m/%y %H:%M:%S')" 1 | 246 | print_info "<= End $BASENAME P3 | $(date +'%d/%m/%y %H:%M:%S')" 1 |
245 | 247 | ||
246 | # unlock directory | 248 | # unlock directory |
247 | mv "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" | 249 | mv "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" |
248 | 250 | ||
249 | 251 |
tools/scripts/Tools.sh
1 | #!/bin/bash | 1 | #!/bin/bash |
2 | 2 | ||
3 | ################################################# | 3 | ################################################# |
4 | # File : Tools.sh # | 4 | # File : Tools.sh # |
5 | # Brief : Tool functions # | 5 | # Brief : Tool functions # |
6 | # Author : Jean-François Rey # | 6 | # Author : Jean-François Rey # |
7 | # Version : 1.0 # | 7 | # Version : 1.0 # |
8 | # Date : 13/06/13 # | 8 | # Date : 13/06/13 # |
9 | ################################################# | 9 | ################################################# |
10 | 10 | ||
11 | # Type | 11 | # Type |
12 | ERROR=2 | 12 | ERROR=2 |
13 | WARN=1 | 13 | WARN=1 |
14 | INFO=0 | 14 | INFO=0 |
15 | 15 | ||
16 | # Global Verbose variable (have to be define in scripts) | 16 | # Global Verbose variable (have to be define in scripts) |
17 | # 0 : mute | 17 | # 0 : mute |
18 | # 1 : basic info (start,config,end) | 18 | # 1 : basic info (start,config,end) |
19 | # 2 : add process | 19 | # 2 : add process |
20 | # 3 : all | 20 | # 3 : all |
21 | VERBOSE=0 | 21 | VERBOSE=0 |
22 | 22 | ||
23 | # print_message( verbose_level, message) | 23 | # print_message( verbose_level, message) |
24 | # - message_type : ERROR, WARN, INFO | 24 | # - message_type : ERROR, WARN, INFO |
25 | # - verbose_level : level to print message | 25 | # - verbose_level : level to print message |
26 | # - message : the message to print | 26 | # - message : the message to print |
27 | function print_message() | 27 | function print_message() |
28 | { | 28 | { |
29 | if [ $# -eq 3 ] | 29 | if [ $# -eq 3 ] |
30 | then | 30 | then |
31 | # print error | 31 | # print error |
32 | if [ $1 -eq $ERROR ]; then echo -ne "###ERROR : $3\n" > /dev/stderr; exit 0; fi | 32 | if [ $1 -eq $ERROR ]; then echo -ne "###ERROR : $3\n" > /dev/stderr; exit 0; fi |
33 | # check verbose level | 33 | # check verbose level |
34 | if [ $2 -le $VERBOSE ] | 34 | if [ $2 -le $VERBOSE ] |
35 | then | 35 | then |
36 | for (( dec=1 ; $dec<$2 ; dec++ )); do echo -ne "\t"; done | 36 | #for (( dec=1 ; $dec<$2 ; dec++ )); do echo -ne "\t"; done |
37 | # print warn | 37 | # print warn |
38 | if [ $1 = $WARN ]; then echo -ne "##WARNING : $3\n"; exit 0; fi | 38 | if [ $1 = $WARN ]; then echo -ne "##WARNING : $3\n"; exit 0; fi |
39 | # default info | 39 | # default info |
40 | echo -ne "#INFO : $3\n" | 40 | echo -ne "#INFO : $3\n" |
41 | fi | 41 | fi |
42 | fi | 42 | fi |
43 | } | 43 | } |
44 | 44 | ||
45 | # print_error(message) | 45 | # print_error(message) |
46 | # - message : error message to print | 46 | # - message : error message to print |
47 | function print_error() | 47 | function print_error() |
48 | { | 48 | { |
49 | print_message $ERROR 3 "$1" | 49 | print_message $ERROR 3 "$1" |
50 | } | 50 | } |
51 | 51 | ||
52 | # print_warn(message,[level]) | 52 | # print_warn(message,[level]) |
53 | # - message : message to print | 53 | # - message : message to print |
54 | # - level : optional option - verbose level to print | 54 | # - level : optional option - verbose level to print |
55 | function print_warn() | 55 | function print_warn() |
56 | { | 56 | { |
57 | if [ $# -eq 2 ] | 57 | if [ $# -eq 2 ] |
58 | then | 58 | then |
59 | print_message $WARN $2 "$1" | 59 | print_message $WARN $2 "$1" |
60 | else | 60 | else |
61 | print_message $WARN $VERBOSE "$1" | 61 | print_message $WARN $VERBOSE "$1" |
62 | fi | 62 | fi |
63 | } | 63 | } |
64 | 64 | ||
65 | # print_info(message,[level]) | 65 | # print_info(message,[level]) |
66 | # - message : message to print | 66 | # - message : message to print |
67 | # - level : optional option - verbose level to print | 67 | # - level : optional option - verbose level to print |
68 | function print_info() | 68 | function print_info() |
69 | { | 69 | { |
70 | if [ $# -eq 2 ] | 70 | if [ $# -eq 2 ] |
71 | then | 71 | then |
72 | print_message $INFO $2 "$1" | 72 | print_message $INFO $2 "$1" |
73 | else | 73 | else |
74 | print_message $INFO $VERBOSE "$1" | 74 | print_message $INFO $VERBOSE "$1" |
75 | fi | 75 | fi |
76 | } | 76 | } |
77 | 77 | ||
78 | # print_log_file(filename,message) | 78 | # print_log_file(filename,message) |
79 | # - filename : full path to log file | 79 | # - filename : full path to log file |
80 | # - message : message to log | 80 | # - message : message to log |
81 | print_log_file() | 81 | print_log_file() |
82 | { | 82 | { |
83 | if [ $# -eq 2 ] | 83 | if [ $# -eq 2 ] |
84 | then | 84 | then |
85 | echo -e "$message" >> $filename | 85 | echo -e "$message" >> $filename |
86 | fi | 86 | fi |
87 | } | 87 | } |
88 | 88 |