Jean-François Rey / otmedia

Download as
Browse Code »

Commit 48b8af9ef51fd19129552a47988c07ed4cf4f7a4

Authored by Jean-François Rey
1 parent 6db49dab66
Exists in master

bugfix

Showing 1 changed file with 4 additions and 4 deletions Inline Diff

main_tools/ConfPass.sh
Diff comments   View file @ 48b8af9
1 #!/bin/bash 1 #!/bin/bash
2 2
3 ##################################################### 3 #####################################################
4 # File : ConfPass.sh # 4 # File : ConfPass.sh #
5 # Brief : Process the ASR Confidence pass # 5 # Brief : Process the ASR Confidence pass #
6 # Author : Jean-François Rey # 6 # Author : Jean-François Rey #
7 # (base on Emmanuel Ferreira # 7 # (base on Emmanuel Ferreira #
8 # and hugo Mauchrétien works) # 8 # and hugo Mauchrétien works) #
9 # Version : 1.0 # 9 # Version : 1.0 #
10 # Date : 17/06/13 # 10 # Date : 17/06/13 #
11 ##################################################### 11 #####################################################
12 12
13 echo "### ConfPass.sh ###" 13 echo "### ConfPass.sh ###"
14 14
15 #Check OTMEDIA_HOME env var 15 #Check OTMEDIA_HOME env var
16 if [ -z ${OTMEDIA_HOME} ] 16 if [ -z ${OTMEDIA_HOME} ]
17 then 17 then
18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) 18 OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
19 export OTMEDIA_HOME=$OTMEDIA_HOME 19 export OTMEDIA_HOME=$OTMEDIA_HOME
20 fi 20 fi
21 21
22 22
23 # where is ConfPath.sh 23 # where is ConfPath.sh
24 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) 24 MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
25 25
26 # Scripts Path 26 # Scripts Path
27 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts 27 SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
28 28
29 # Include scripts 29 # Include scripts
30 . $SCRIPT_PATH"/Tools.sh" 30 . $SCRIPT_PATH"/Tools.sh"
31 . $SCRIPT_PATH"/CheckConfPass.sh" 31 . $SCRIPT_PATH"/CheckConfPass.sh"
32 32
33 # where is FirstPass.cfg 33 # where is FirstPass.cfg
34 CONFPASS_CONFIG_FILE="$OTMEDIA_HOME/cfg/ConfPass.cfg" 34 CONFPASS_CONFIG_FILE="$OTMEDIA_HOME/cfg/ConfPass.cfg"
35 if [ -e $CONFPASS_CONFIG_FILE ] 35 if [ -e $CONFPASS_CONFIG_FILE ]
36 then 36 then
37 . $CONFPASS_CONFIG_FILE 37 . $CONFPASS_CONFIG_FILE
38 else 38 else
39 echo "ERROR : Can't find configuration file $CONFPASS_CONFIG_FILE" >&2 39 echo "ERROR : Can't find configuration file $CONFPASS_CONFIG_FILE" >&2
40 exit 1 40 exit 1
41 fi 41 fi
42 42
43 #---------------# 43 #---------------#
44 # Parse Options # 44 # Parse Options #
45 #---------------# 45 #---------------#
46 while getopts ":hDv:cr" opt 46 while getopts ":hDv:cr" opt
47 do 47 do
48 case $opt in 48 case $opt in
49 h) 49 h)
50 echo -e "$0 [OPTIONS] <INPUT_DIRECTORY> <TREIL_DIRECTORY_NAME>\n" 50 echo -e "$0 [OPTIONS] <INPUT_DIRECTORY> <TREIL_DIRECTORY_NAME>\n"
51 echo -e "\t Options:" 51 echo -e "\t Options:"
52 echo -e "\t\t-h :\tprint this message" 52 echo -e "\t\t-h :\tprint this message"
53 echo -e "\t\t-D :\tDEBUG mode on" 53 echo -e "\t\t-D :\tDEBUG mode on"
54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" 54 echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
55 echo -e "\t\t-c :\t Check process, stop if error detected" 55 echo -e "\t\t-c :\t Check process, stop if error detected"
56 echo -e "\t\t-r :\tForce to rerun confpas without deleting existing files" 56 echo -e "\t\t-r :\tForce to rerun confpas without deleting existing files"
57 exit 1 57 exit 1
58 ;; 58 ;;
59 D) 59 D)
60 DEBUG=1 60 DEBUG=1
61 ;; 61 ;;
62 v) 62 v)
63 VERBOSE=$OPTARG 63 VERBOSE=$OPTARG
64 ;; 64 ;;
65 c) 65 c)
66 CHECK=1 66 CHECK=1
67 ;; 67 ;;
68 r) 68 r)
69 RERUN=1 69 RERUN=1
70 ;; 70 ;;
71 :) 71 :)
72 echo "Option -$OPTARG requires an argument." >&2 72 echo "Option -$OPTARG requires an argument." >&2
73 exit 1 73 exit 1
74 ;; 74 ;;
75 \?) 75 \?)
76 echo "BAD USAGE : unknow option -$OPTARG" 76 echo "BAD USAGE : unknow option -$OPTARG"
77 #exit 1 77 #exit 1
78 ;; 78 ;;
79 esac 79 esac
80 done 80 done
81 81
82 # mode debug enable 82 # mode debug enable
83 if [ $DEBUG -eq 1 ] 83 if [ $DEBUG -eq 1 ]
84 then 84 then
85 set -x 85 set -x
86 echo -e "## Mode DEBUG ON ##" 86 echo -e "## Mode DEBUG ON ##"
87 fi 87 fi
88 88
89 # mode verbose enable 89 # mode verbose enable
90 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi 90 if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
91 91
92 # Check USAGE by arguments number 92 # Check USAGE by arguments number
93 if [ $(($#-($OPTIND-1))) -ne 2 ] 93 if [ $(($#-($OPTIND-1))) -ne 2 ]
94 then 94 then
95 echo "BAD USAGE : ConfPass.sh [OPTIONS] <INPUT_DIR> <TREIL_DIRECTORY_NAME>" 95 echo "BAD USAGE : ConfPass.sh [OPTIONS] <INPUT_DIR> <TREIL_DIRECTORY_NAME>"
96 echo "$0 -h for more info" 96 echo "$0 -h for more info"
97 exit 1 97 exit 1
98 fi 98 fi
99 99
100 shift $((OPTIND-1)) 100 shift $((OPTIND-1))
101 # check input directory - first argument 101 # check input directory - first argument
102 if [ ! -e $1 ] 102 if [ ! -e $1 ]
103 then 103 then
104 print_error "can't open $1" 104 print_error "can't open $1"
105 exit 1 105 exit 1
106 fi 106 fi
107 # check treil input directory - second argument 107 # check treil input directory - second argument
108 if [ ! -e $1/$2 ] 108 if [ ! -e $1/$2 ]
109 then 109 then
110 print_error "can't open $1/$2" 110 print_error "can't open $1/$2"
111 exit 1 111 exit 1
112 fi 112 fi
113 113
114 print_info "[${BASENAME}] => Conf Pass start | $(date +'%d/%m/%y %H:%M:%S')" 1
115
116 #-------------# 114 #-------------#
117 # GLOBAL VARS # 115 # GLOBAL VARS #
118 #-------------# 116 #-------------#
119 INPUT_DIR=$(readlink -e $1) 117 INPUT_DIR=$(readlink -e $1)
120 OUTPUT_DIR=$INPUT_DIR 118 OUTPUT_DIR=$INPUT_DIR
121 BASENAME=$(basename $OUTPUT_DIR) 119 BASENAME=$(basename $OUTPUT_DIR)
122 RES_NAME=$2 120 RES_NAME=$2
123 RES_P="${INPUT_DIR}/${RES_NAME}" 121 RES_P="${INPUT_DIR}/${RES_NAME}"
124 USF_FILE=${INPUT_DIR}/${BASENAME}.${RES_NAME}.usf 122 USF_FILE=${INPUT_DIR}/${BASENAME}.${RES_NAME}.usf
125 CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME" 123 CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME"
126 RES_CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME/scored_ctm" 124 RES_CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME/scored_ctm"
127 LOGFILE="${OUTPUT_DIR_BASENAME}/info_conf.log" 125 LOGFILE="${OUTPUT_DIR_BASENAME}/info_conf.log"
128 ERRORFILE="${OUTPUT_DIR_BASENAME}/error_conf.log" 126 ERRORFILE="${OUTPUT_DIR_BASENAME}/error_conf.log"
129 127
128 print_info "[${BASENAME}] => Conf Pass start | $(date +'%d/%m/%y %H:%M:%S')" 1
129
130 #------------------# 130 #------------------#
131 # Create Workspace # 131 # Create Workspace #
132 #------------------# 132 #------------------#
133 # Lock directory 133 # Lock directory
134 if [ -e "$OUTPUT_DIR/CONFPASS.lock" ] && [ $RERUN -eq 0 ] 134 if [ -e "$OUTPUT_DIR/CONFPASS.lock" ] && [ $RERUN -eq 0 ]
135 then 135 then
136 print_warn "[${BASENAME}] Confpass is locked -> exit" 2 136 print_warn "[${BASENAME}] Confpass is locked -> exit" 2
137 exit 1 137 exit 1
138 fi 138 fi
139 rm "$OUTPUT_DIR/CONFPASS.unlock" > /dev/null 2>&1 139 rm "$OUTPUT_DIR/CONFPASS.unlock" > /dev/null 2>&1
140 touch "$OUTPUT_DIR/CONFPASS.lock" > /dev/null 2>&1 140 touch "$OUTPUT_DIR/CONFPASS.lock" > /dev/null 2>&1
141 if [ $RERUN -eq 0 ]; then rm -r $CONF_DIR > /dev/null 2>&1; fi 141 if [ $RERUN -eq 0 ]; then rm -r $CONF_DIR > /dev/null 2>&1; fi
142 if [ $RERUN -eq 1 ]; then rm $USF_FILE > /dev/null 2>&1; fi 142 if [ $RERUN -eq 1 ]; then rm $USF_FILE > /dev/null 2>&1; fi
143 mkdir -p $CONF_DIR > /dev/null 2>&1 143 mkdir -p $CONF_DIR > /dev/null 2>&1
144 mkdir -p $RES_CONF_DIR > /dev/null 2>&1 144 mkdir -p $RES_CONF_DIR > /dev/null 2>&1
145 rm $LOGFILE $ERRORFILE > /dev/null 2>&1 145 rm $LOGFILE $ERRORFILE > /dev/null 2>&1
146 146
147 #---------------# 147 #---------------#
148 # Check Pass # 148 # Check Pass #
149 #---------------# 149 #---------------#
150 print_info "[${BASENAME}] Check Conf Pass directory ${RES_NAME}" 1 150 print_info "[${BASENAME}] Check Conf Pass directory ${RES_NAME}" 1
151 # if usf contains more than 49% of 0.600 confidence -> usf error 151 # if usf contains more than 49% of 0.600 confidence -> usf error
152 if [ -s $USF_FILE ] 152 if [ -s $USF_FILE ]
153 then 153 then
154 conftozerosix=$(grep -c -E 'confidence="0.600"' "${USF_FILE}") 154 conftozerosix=$(grep -c -E 'confidence="0.600"' "${USF_FILE}")
155 confall=$(grep -c -E 'confidence=' "${USF_FILE}") 155 confall=$(grep -c -E 'confidence=' "${USF_FILE}")
156 if [ $confall -gt 0 ] 156 if [ $confall -gt 0 ]
157 then 157 then
158 pourcentageofzerosix=$((($conftozerosix*100)/$confall)) 158 pourcentageofzerosix=$((($conftozerosix*100)/$confall))
159 if [ $pourcentageofzerosix -gt 49 ] 159 if [ $pourcentageofzerosix -gt 49 ]
160 then 160 then
161 print_warn "[${BASENAME}] ${BASENAME}.${RES_NAME}.usf got $pourcentageofzerosix% of 0.600 confidence" 2 161 print_warn "[${BASENAME}] ${BASENAME}.${RES_NAME}.usf got $pourcentageofzerosix% of 0.600 confidence" 2
162 print_info "[${BASENAME}] bad usf ${RES_NAME}, will do it again" 1 162 print_info "[${BASENAME}] bad usf ${RES_NAME}, will do it again" 1
163 mv "${USF_FILE}" "${USF_FILE}.back" 163 mv "${USF_FILE}" "${USF_FILE}.back"
164 rm -r $CONF_DIR > /dev/null 2>&1 164 rm -r $CONF_DIR > /dev/null 2>&1
165 else 165 else
166 print_warn "[${BASENAME}] ${USF_FILE} already done, skipping it" 1 166 print_warn "[${BASENAME}] ${USF_FILE} already done, skipping it" 1
167 exit 0 167 exit 0
168 fi 168 fi
169 fi 169 fi
170 else 170 else
171 print_info "[${BASENAME}] No USF file already done, continue..." 1 171 print_info "[${BASENAME}] No USF file already done, continue..." 1
172 fi 172 fi
173 173
174 # Check if treil are here 174 # Check if treil are here
175 nbres_p1=$(cat ${INPUT_DIR}/plp.lst | wc -l) 175 nbres_p1=$(cat ${INPUT_DIR}/plp.lst | wc -l)
176 nbtreil_p=$(ls $RES_P/*.treil 2> /dev/null | wc -l) 176 nbtreil_p=$(ls ${RES_P}/*.treil 2> /dev/null | wc -l)
177 if [ $nbtreil_p -eq 0 ] 177 if [ $nbtreil_p -eq 0 ]
178 then 178 then
179 print_error "[${BASENAME}] No ${RES_NAME} Pass, No .treil -> exit ConfPass" 179 print_error "[${BASENAME}] No ${RES_NAME} Pass, No .treil -> exit ConfPass"
180 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ${RES_NAME} Pass, No .treil -> exit ConfPass" ;fi 180 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ${RES_NAME} Pass, No .treil -> exit ConfPass" ;fi
181 exit 1 181 exit 1
182 else 182 else
183 #Check if more then 89% of treil are done 183 #Check if more then 89% of treil are done
184 if [ $nbres_p1 -gt 0 ] 184 if [ $nbres_p1 -gt 0 ]
185 then 185 then
186 pourcentage=$((($nbtreil_p*100)/$nbres_p1)) 186 pourcentage=$((($nbtreil_p*100)/$nbres_p1))
187 if [ $pourcentage -gt 89 ] 187 if [ $pourcentage -gt 89 ]
188 then 188 then
189 print_info "[${BASENAME}] ${RES_NAME}/*.treil are here" 1 189 print_info "[${BASENAME}] ${RES_NAME}/*.treil are here" 1
190 else 190 else
191 print_warn "[${BASENAME}] not enough ${RES_NAME} treil" 2 191 print_warn "[${BASENAME}] not enough ${RES_NAME} treil" 2
192 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "Not enough ${RES_NAME} treil " 192 if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "Not enough ${RES_NAME} treil ";fi
193 fi 193 fi
194 fi 194 fi
195 fi 195 fi
196 196
197 #------# 197 #------#
198 # Save # 198 # Save #
199 #------# 199 #------#
200 cp $CONFPASS_CONFIG_FILE $OUTPUT_DIR/ConfPass.cfg 200 cp $CONFPASS_CONFIG_FILE $OUTPUT_DIR/ConfPass.cfg
201 echo "RES_CONF_DIR=$RES_CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg 201 echo "RES_CONF_DIR=$RES_CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg
202 echo "CONF_DIR=$CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg 202 echo "CONF_DIR=$CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg
203 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ConfPass.cfg" 1 203 print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ConfPass.cfg" 1
204 204
205 #--------------------# 205 #--------------------#
206 # CONFIDENCE MEASURE # 206 # CONFIDENCE MEASURE #
207 #--------------------# 207 #--------------------#
208 208
209 # Check pourcentage of scored_ctm already done, if < 85% do confidence measure 209 # Check pourcentage of scored_ctm already done, if < 85% do confidence measure
210 nbres_p=$(ls ${RES_P}/*.treil | wc -l) 210 nbres_p=$(ls ${RES_P}/*.treil | wc -l)
211 nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l) 211 nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l)
212 if [ $nbres_p -gt 0 ] 212 if [ $nbres_p -gt 0 ]
213 then 213 then
214 pourcentageres=$((($nbconf*100)/$nbres_p)) 214 pourcentageres=$((($nbconf*100)/$nbres_p))
215 if [ $pourcentageres -lt 85 ] 215 if [ $pourcentageres -lt 85 ]
216 then 216 then
217 print_info "[${BASENAME}] Calcul Confidence $INPUT_DIR $RES_NAME" 1 217 print_info "[${BASENAME}] Calcul Confidence $INPUT_DIR $RES_NAME" 1
218 $MAIN_SCRIPT_PATH/ConfidenceMeasure.sh $INPUT_DIR $RES_NAME 218 $MAIN_SCRIPT_PATH/ConfidenceMeasure.sh $INPUT_DIR $RES_NAME
219 219
220 else 220 else
221 print_info "[${BASENAME}] Skipping Confidence Calcul $INPUT_DIR/$RES_NAME" 1 221 print_info "[${BASENAME}] Skipping Confidence Calcul $INPUT_DIR/$RES_NAME" 1
222 fi 222 fi
223 fi 223 fi
224 224
225 ### Check scored_ctm number res files ! 225 ### Check scored_ctm number res files !
226 if [ $CHECK -eq 1 ] 226 if [ $CHECK -eq 1 ]
227 then 227 then
228 nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l) 228 nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l)
229 if [ $nbres_p -ne $nbconf ] 229 if [ $nbres_p -ne $nbconf ]
230 then 230 then
231 print_warn "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" 2 231 print_warn "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" 2
232 print_log_file $LOGFILE "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" 232 print_log_file $LOGFILE "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ"
233 fi 233 fi
234 fi 234 fi
235 235
236 #---------------------------# 236 #---------------------------#
237 # FROM RES WITH CONF => USF # 237 # FROM RES WITH CONF => USF #
238 #---------------------------# 238 #---------------------------#
239 print_info "[${BASENAME}] Create USF file for $RES_P" 1 239 print_info "[${BASENAME}] Create USF file for $RES_P" 1
240 for f in `ls ${RES_CONF_DIR}`; do $SCRIPT_PATH/formatRES.pl $RES_CONF_DIR/$f; done 240 for f in `ls ${RES_CONF_DIR}`; do $SCRIPT_PATH/formatRES.pl $RES_CONF_DIR/$f; done
241 # create USF configuration file 241 # create USF configuration file
242 echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR/$BASENAME.seg" > $OUTPUT_DIR/$BASENAME.usf_cfg 242 echo -e "name $AUTHOR\nfileName $BASENAME\nfileExt wav\nsegFile $OUTPUT_DIR/$BASENAME.seg" > $OUTPUT_DIR/$BASENAME.usf_cfg
243 # create USF file 243 # create USF file
244 print_info "$SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg" 3 244 print_info "$SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg" 3
245 $SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg 245 $SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg
246 rm $OUTPUT_DIR/$BASENAME.usf_cfg > /dev/null 2>&1 246 rm $OUTPUT_DIR/$BASENAME.usf_cfg > /dev/null 2>&1
247 cat $USF_FILE.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f > $USF_FILE 247 cat $USF_FILE.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f > $USF_FILE
248 cp $USF_FILE ${OUTPUT_DIR}/${BASENAME}.usf 248 cp $USF_FILE ${OUTPUT_DIR}/${BASENAME}.usf
249 rm $USF_FILE.tmp > /dev/null 2>&1 249 rm $USF_FILE.tmp > /dev/null 2>&1
250 250
251 #----------------# 251 #----------------#
252 # Check USF file # 252 # Check USF file #
253 #----------------# 253 #----------------#
254 if [ $CHECK -eq 1 ] 254 if [ $CHECK -eq 1 ]
255 then 255 then
256 check_conf_pass_usf "$OUTPUT_DIR/$BASENAME.usf" 256 check_conf_pass_usf "$OUTPUT_DIR/$BASENAME.usf"
257 if [ $? -eq 1 ] 257 if [ $? -eq 1 ]
258 then 258 then
259 print_error "[${BASENAME}] Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf" 259 print_error "[${BASENAME}] Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf"
260 print_log_file $ERRORFILE "ERROR : Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf" 260 print_log_file $ERRORFILE "ERROR : Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf"
261 exit 1 261 exit 1
262 fi 262 fi
263 fi 263 fi
264 264
265 #-------# 265 #-------#
266 # CLOSE # 266 # CLOSE #
267 #-------# 267 #-------#
268 # Seem OK 268 # Seem OK
269 print_info "[${BASENAME}] <= ConfPass End | $(date +'%d/%m/%y %H:%M:%S')" 1 269 print_info "[${BASENAME}] <= ConfPass End | $(date +'%d/%m/%y %H:%M:%S')" 1
270 270
271 # unlock directory 271 # unlock directory
272 mv "$OUTPUT_DIR/CONFPASS.lock" "$OUTPUT_DIR/CONFPASS.unlock" 272 mv "$OUTPUT_DIR/CONFPASS.lock" "$OUTPUT_DIR/CONFPASS.unlock"