Blame view

main_tools/ConfPass.sh 8.03 KB
e6be5137b   Jean-François Rey   reinitialized pro...
1
2
3
4
5
6
7
8
9
10
11
  #!/bin/bash
  
  #####################################################
  # File :    ConfPass.sh                             #
  # Brief :   Process the ASR Confidence pass         #
  # Author :  Jean-François Rey                       #
  #	        (base on Emmanuel Ferreira              #
  #	        and hugo Mauchrétien works)             #
  # Version : 1.0                                     #
  # Date :    17/06/13                                #
  #####################################################
f37e72eaf   Jean-François Rey   up
12
  echo "### ConfPass.sh ###"
e6be5137b   Jean-François Rey   reinitialized pro...
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
  #Check OTMEDIA_HOME env var
  if [ -z ${OTMEDIA_HOME} ]
  then
      OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
      export OTMEDIA_HOME=$OTMEDIA_HOME
  fi
  
  
  # where is ConfPath.sh
  MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
  
  # Scripts Path
  SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
  
  # Include scripts
  . $SCRIPT_PATH"/Tools.sh"
  . $SCRIPT_PATH"/CheckConfPass.sh"
  
  # where is FirstPass.cfg
  CONFPASS_CONFIG_FILE="$OTMEDIA_HOME/cfg/ConfPass.cfg"
  if [ -e $CONFPASS_CONFIG_FILE ]
  then
  	. $CONFPASS_CONFIG_FILE
  else
d7e9e4b9d   Jean-François Rey   update bugfix stderr
37
  	echo "ERROR : Can't find configuration file $CONFPASS_CONFIG_FILE" >&2
e6be5137b   Jean-François Rey   reinitialized pro...
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
  	exit 1
  fi
  
  #---------------#
  # Parse Options #
  #---------------#
  while getopts ":hDv:cr" opt
  do
  	case $opt in
  		h)
  			echo -e "$0 [OPTIONS] <INPUT_DIRECTORY> <TREIL_DIRECTORY_NAME>
  "
              echo -e "\t Options:"
              echo -e "\t\t-h :\tprint this message"
              echo -e "\t\t-D :\tDEBUG mode on"
              echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
              echo -e "\t\t-c :\t Check process, stop if error detected"
              echo -e "\t\t-r :\tForce to rerun confpas without deleting existing files"
  			exit 1
  			;;
  		D)
  			DEBUG=1
  			;;
          v)
              VERBOSE=$OPTARG
              ;;
          c)
              CHECK=1
              ;;
          r)
              RERUN=1
              ;;
  		:)
d7e9e4b9d   Jean-François Rey   update bugfix stderr
71
  			echo "Option -$OPTARG requires an argument." >&2
e6be5137b   Jean-François Rey   reinitialized pro...
72
73
74
  			exit 1
  			;;
  		\?)
6db49dab6   Jean-François Rey   bugfix arguments
75
  			echo "BAD USAGE : unknow option -$OPTARG"
e6be5137b   Jean-François Rey   reinitialized pro...
76
77
78
79
80
81
82
83
84
85
86
87
88
  			#exit 1
  			;;
  	esac
  done
  
  # mode debug enable
  if [ $DEBUG -eq 1 ]
  then
         set -x
         echo -e "## Mode DEBUG ON ##"
  fi
  
  # mode verbose enable
1fd315c89   Jean-François Rey   add Extract audio...
89
  if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
e6be5137b   Jean-François Rey   reinitialized pro...
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
   
  # Check USAGE by arguments number
  if [ $(($#-($OPTIND-1))) -ne 2 ]
  then
      echo "BAD USAGE : ConfPass.sh [OPTIONS] <INPUT_DIR> <TREIL_DIRECTORY_NAME>"
      echo "$0 -h for more info"
      exit 1
  fi
  
  shift $((OPTIND-1))
  # check input directory - first argument
  if [ ! -e $1 ]
  then
      print_error "can't open $1"
      exit 1
  fi
  # check treil input directory - second argument
  if [ ! -e $1/$2 ]
  then
      print_error "can't open $1/$2"
      exit 1
  fi
  
  #-------------#
  # GLOBAL VARS #
  #-------------#
  INPUT_DIR=$(readlink -e $1)
  OUTPUT_DIR=$INPUT_DIR 
  BASENAME=$(basename $OUTPUT_DIR)
  RES_NAME=$2
  RES_P="${INPUT_DIR}/${RES_NAME}"
  USF_FILE=${INPUT_DIR}/${BASENAME}.${RES_NAME}.usf
  CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME"
  RES_CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME/scored_ctm"
d7ca093e1   Jean-François Rey   update bugfix
124
125
  LOGFILE="${OUTPUT_DIR}/info_conf.log"
  ERRORFILE="${OUTPUT_DIR}/error_conf.log"
e6be5137b   Jean-François Rey   reinitialized pro...
126

48b8af9ef   Jean-François Rey   bugfix
127
  print_info "[${BASENAME}] => Conf Pass start | $(date +'%d/%m/%y %H:%M:%S')" 1
e6be5137b   Jean-François Rey   reinitialized pro...
128
129
130
131
132
133
  #------------------#
  # Create Workspace #
  #------------------#
  # Lock directory
  if [ -e "$OUTPUT_DIR/CONFPASS.lock" ] && [ $RERUN -eq 0 ]
  then
9456401f8   Jean-François Rey   modify info and l...
134
      print_warn "[${BASENAME}] Confpass is locked -> exit" 2
e6be5137b   Jean-François Rey   reinitialized pro...
135
136
137
138
139
140
      exit 1
  fi
  rm "$OUTPUT_DIR/CONFPASS.unlock" > /dev/null 2>&1
  touch "$OUTPUT_DIR/CONFPASS.lock" > /dev/null 2>&1
  if [ $RERUN -eq 0 ]; then rm -r $CONF_DIR > /dev/null 2>&1; fi
  if [ $RERUN -eq 1 ]; then rm $USF_FILE > /dev/null 2>&1; fi
1fd315c89   Jean-François Rey   add Extract audio...
141
142
  mkdir -p $CONF_DIR > /dev/null 2>&1
  mkdir -p $RES_CONF_DIR > /dev/null 2>&1
9456401f8   Jean-François Rey   modify info and l...
143
  rm $LOGFILE $ERRORFILE > /dev/null 2>&1
e6be5137b   Jean-François Rey   reinitialized pro...
144
145
146
147
  
  #---------------#
  # Check Pass    #
  #---------------#
9456401f8   Jean-François Rey   modify info and l...
148
149
  print_info "[${BASENAME}] Check Conf Pass directory ${RES_NAME}" 1
  # if usf contains more than 49% of 0.600 confidence -> usf error
e6be5137b   Jean-François Rey   reinitialized pro...
150
151
152
  if [ -s $USF_FILE ]
  then
      conftozerosix=$(grep -c -E 'confidence="0.600"' "${USF_FILE}")
9456401f8   Jean-François Rey   modify info and l...
153
154
      confall=$(grep -c -E 'confidence=' "${USF_FILE}")
      if [ $confall -gt 0 ]
e6be5137b   Jean-François Rey   reinitialized pro...
155
      then
9456401f8   Jean-François Rey   modify info and l...
156
157
          pourcentageofzerosix=$((($conftozerosix*100)/$confall))
          if [ $pourcentageofzerosix -gt 49 ]
e6be5137b   Jean-François Rey   reinitialized pro...
158
          then
9456401f8   Jean-François Rey   modify info and l...
159
160
              print_warn "[${BASENAME}] ${BASENAME}.${RES_NAME}.usf got $pourcentageofzerosix% of 0.600 confidence" 2
              print_info "[${BASENAME}] bad usf ${RES_NAME}, will do it again" 1
e6be5137b   Jean-François Rey   reinitialized pro...
161
162
163
              mv "${USF_FILE}" "${USF_FILE}.back"
              rm -r $CONF_DIR > /dev/null 2>&1
          else
9456401f8   Jean-François Rey   modify info and l...
164
              print_warn "[${BASENAME}] ${USF_FILE} already done, skipping it" 1
e6be5137b   Jean-François Rey   reinitialized pro...
165
166
167
              exit 0
          fi
      fi
9456401f8   Jean-François Rey   modify info and l...
168
169
170
171
172
173
  else
      print_info "[${BASENAME}] No USF file already done, continue..." 1
  fi
  
  # Check if treil are here
  nbres_p1=$(cat ${INPUT_DIR}/plp.lst | wc -l)
48b8af9ef   Jean-François Rey   bugfix
174
  nbtreil_p=$(ls ${RES_P}/*.treil 2> /dev/null | wc -l)
9456401f8   Jean-François Rey   modify info and l...
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
  if [ $nbtreil_p -eq 0 ]
  then
      print_error "[${BASENAME}] No ${RES_NAME} Pass, No .treil -> exit ConfPass"
      if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ${RES_NAME} Pass, No .treil -> exit ConfPass" ;fi
      exit 1
  else
      #Check if more then 89% of treil are done
      if [ $nbres_p1 -gt 0 ]
      then
          pourcentage=$((($nbtreil_p*100)/$nbres_p1))
          if [ $pourcentage -gt 89 ]
          then
              print_info "[${BASENAME}] ${RES_NAME}/*.treil are here" 1
          else
              print_warn "[${BASENAME}] not enough ${RES_NAME} treil" 2
48b8af9ef   Jean-François Rey   bugfix
190
              if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "Not enough ${RES_NAME} treil ";fi
9456401f8   Jean-François Rey   modify info and l...
191
192
          fi
      fi
e6be5137b   Jean-François Rey   reinitialized pro...
193
194
195
196
197
198
199
200
  fi
  
  #------#
  # Save #
  #------#
  cp $CONFPASS_CONFIG_FILE $OUTPUT_DIR/ConfPass.cfg
  echo "RES_CONF_DIR=$RES_CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg
  echo "CONF_DIR=$CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg
d7ca093e1   Jean-François Rey   update bugfix
201
  print_info "[${BASENAME}] Save config in $OUTPUT_DIR/ConfPass.cfg" 1
e6be5137b   Jean-François Rey   reinitialized pro...
202
203
204
205
  
  #--------------------#
  # CONFIDENCE MEASURE #
  #--------------------#
9456401f8   Jean-François Rey   modify info and l...
206
  # Check pourcentage of scored_ctm already done, if < 85% do confidence measure
f3043a8b9   Jean-François Rey   bugfix
207
208
  nbres_p=$(ls ${RES_P}/*.treil 2> /dev/null | wc -l)
  nbconf=$(ls ${RES_CONF_DIR}/*.res 2> /dev/null | wc -l)
e6be5137b   Jean-François Rey   reinitialized pro...
209
210
211
212
213
  if [ $nbres_p -gt 0 ]
  then
      pourcentageres=$((($nbconf*100)/$nbres_p))
      if [ $pourcentageres -lt 85 ]
      then
9456401f8   Jean-François Rey   modify info and l...
214
          print_info "[${BASENAME}] Calcul Confidence $INPUT_DIR $RES_NAME" 1
561670acc   Jean-François Rey   remove output red...
215
          $MAIN_SCRIPT_PATH/ConfidenceMeasure.sh $INPUT_DIR $RES_NAME
1fd315c89   Jean-François Rey   add Extract audio...
216

e6be5137b   Jean-François Rey   reinitialized pro...
217
      else
9456401f8   Jean-François Rey   modify info and l...
218
          print_info "[${BASENAME}] Skipping Confidence Calcul $INPUT_DIR/$RES_NAME" 1
e6be5137b   Jean-François Rey   reinitialized pro...
219
220
221
222
223
224
      fi
  fi
  
  ### Check scored_ctm number res files !
  if [ $CHECK -eq 1 ]
  then
f3043a8b9   Jean-François Rey   bugfix
225
      nbconf=$(ls ${RES_CONF_DIR}/*.res 2> /dev/null | wc -l)
9456401f8   Jean-François Rey   modify info and l...
226
227
228
229
230
      if [ $nbres_p -ne $nbconf ]
      then
          print_warn "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" 2
          print_log_file $LOGFILE "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ"
      fi
e6be5137b   Jean-François Rey   reinitialized pro...
231
232
233
234
235
  fi
  
  #---------------------------#
  # FROM RES WITH CONF => USF #
  #---------------------------#
9456401f8   Jean-François Rey   modify info and l...
236
  print_info "[${BASENAME}] Create USF file for $RES_P" 1
e6be5137b   Jean-François Rey   reinitialized pro...
237
238
239
240
241
242
243
  for f in `ls ${RES_CONF_DIR}`; do $SCRIPT_PATH/formatRES.pl $RES_CONF_DIR/$f; done
  # create USF configuration file
  echo -e "name $AUTHOR
  fileName $BASENAME
  fileExt wav
  segFile $OUTPUT_DIR/$BASENAME.seg" > $OUTPUT_DIR/$BASENAME.usf_cfg
  # create USF file
561670acc   Jean-François Rey   remove output red...
244
245
  print_info "$SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg" 3
  $SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg
1fd315c89   Jean-François Rey   add Extract audio...
246
  rm $OUTPUT_DIR/$BASENAME.usf_cfg > /dev/null 2>&1
0bf609bcc   Jean-François Rey   update and add sc...
247
  cat $USF_FILE.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f > $USF_FILE
e6be5137b   Jean-François Rey   reinitialized pro...
248
  cp $USF_FILE ${OUTPUT_DIR}/${BASENAME}.usf
1fd315c89   Jean-François Rey   add Extract audio...
249
  rm $USF_FILE.tmp > /dev/null 2>&1
e6be5137b   Jean-François Rey   reinitialized pro...
250
251
252
253
254
255
256
257
258
  
  #----------------#
  # Check USF file #
  #----------------#
  if [ $CHECK -eq 1 ]
  then
      check_conf_pass_usf "$OUTPUT_DIR/$BASENAME.usf"
      if [ $? -eq 1 ]
      then
9456401f8   Jean-François Rey   modify info and l...
259
260
          print_error "[${BASENAME}] Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf"
          print_log_file $ERRORFILE "ERROR : Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf"
e6be5137b   Jean-François Rey   reinitialized pro...
261
262
263
264
265
266
267
268
          exit 1
      fi
  fi
  
  #-------#
  # CLOSE #
  #-------#
  # Seem OK 
9456401f8   Jean-François Rey   modify info and l...
269
  print_info "[${BASENAME}] <= ConfPass End | $(date +'%d/%m/%y %H:%M:%S')" 1
e6be5137b   Jean-François Rey   reinitialized pro...
270
271
272
   
  # unlock directory
  mv "$OUTPUT_DIR/CONFPASS.lock" "$OUTPUT_DIR/CONFPASS.unlock"