Blame view

main_tools/ConfPass.sh 8.06 KB
e6be5137b   Jean-François Rey   reinitialized pro...
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
  #!/bin/bash
  
  #####################################################
  # File :    ConfPass.sh                             #
  # Brief :   Process the ASR Confidence pass         #
  # Author :  Jean-François Rey                       #
  #	        (base on Emmanuel Ferreira              #
  #	        and hugo Mauchrétien works)             #
  # Version : 1.0                                     #
  # Date :    17/06/13                                #
  #####################################################
  
  #Check OTMEDIA_HOME env var
  if [ -z ${OTMEDIA_HOME} ]
  then
      OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
      export OTMEDIA_HOME=$OTMEDIA_HOME
  fi
  
  
  # where is ConfPath.sh
  MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
  
  # Scripts Path
  SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
  
  # Include scripts
  . $SCRIPT_PATH"/Tools.sh"
  . $SCRIPT_PATH"/CheckConfPass.sh"
  
  # where is FirstPass.cfg
  CONFPASS_CONFIG_FILE="$OTMEDIA_HOME/cfg/ConfPass.cfg"
  if [ -e $CONFPASS_CONFIG_FILE ]
  then
  	. $CONFPASS_CONFIG_FILE
  else
9456401f8   Jean-François Rey   modify info and l...
37
  	echo "ERROR : Can't find configuration file $CONFPASS_CONFIG_FILE" > /dev/stderr
e6be5137b   Jean-François Rey   reinitialized pro...
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
  	exit 1
  fi
  
  #---------------#
  # Parse Options #
  #---------------#
  while getopts ":hDv:cr" opt
  do
  	case $opt in
  		h)
  			echo -e "$0 [OPTIONS] <INPUT_DIRECTORY> <TREIL_DIRECTORY_NAME>
  "
              echo -e "\t Options:"
              echo -e "\t\t-h :\tprint this message"
              echo -e "\t\t-D :\tDEBUG mode on"
              echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
              echo -e "\t\t-c :\t Check process, stop if error detected"
              echo -e "\t\t-r :\tForce to rerun confpas without deleting existing files"
  			exit 1
  			;;
  		D)
  			DEBUG=1
  			;;
          v)
              VERBOSE=$OPTARG
              ;;
          c)
              CHECK=1
              ;;
          r)
              RERUN=1
              ;;
  		:)
9456401f8   Jean-François Rey   modify info and l...
71
  			echo "Option -$OPTARG requires an argument." > /dev/stderr
e6be5137b   Jean-François Rey   reinitialized pro...
72
73
74
75
76
77
78
79
80
81
82
83
84
85
  			exit 1
  			;;
  		\?)
  			echo "BAD USAGE : unknow opton -$OPTARG"
  			#exit 1
  			;;
  	esac
  done
  
  # mode debug enable
  if [ $DEBUG -eq 1 ]
  then
         set -x
         echo -e "## Mode DEBUG ON ##"
1fd315c89   Jean-François Rey   add Extract audio...
86
87
88
         REDIRECTION_OUTPUT=""
     else
         REDIRECTION_OUTPUT=" 2> /dev/null"
e6be5137b   Jean-François Rey   reinitialized pro...
89
90
91
  fi
  
  # mode verbose enable
1fd315c89   Jean-François Rey   add Extract audio...
92
  if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
e6be5137b   Jean-François Rey   reinitialized pro...
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
   
  # Check USAGE by arguments number
  if [ $(($#-($OPTIND-1))) -ne 2 ]
  then
      echo "BAD USAGE : ConfPass.sh [OPTIONS] <INPUT_DIR> <TREIL_DIRECTORY_NAME>"
      echo "$0 -h for more info"
      exit 1
  fi
  
  shift $((OPTIND-1))
  # check input directory - first argument
  if [ ! -e $1 ]
  then
      print_error "can't open $1"
      exit 1
  fi
  # check treil input directory - second argument
  if [ ! -e $1/$2 ]
  then
      print_error "can't open $1/$2"
      exit 1
  fi
  
  #-------------#
  # GLOBAL VARS #
  #-------------#
  INPUT_DIR=$(readlink -e $1)
  OUTPUT_DIR=$INPUT_DIR 
  BASENAME=$(basename $OUTPUT_DIR)
  RES_NAME=$2
  RES_P="${INPUT_DIR}/${RES_NAME}"
  USF_FILE=${INPUT_DIR}/${BASENAME}.${RES_NAME}.usf
  CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME"
  RES_CONF_DIR="$OUTPUT_DIR/conf/$RES_NAME/scored_ctm"
9456401f8   Jean-François Rey   modify info and l...
127
128
  LOGFILE="${OUTPUT_DIR_BASENAME}/info_conf.log"
  ERRORFILE="${OUTPUT_DIR_BASENAME}/error_conf.log"
e6be5137b   Jean-François Rey   reinitialized pro...
129
130
131
132
133
134
135
  
  #------------------#
  # Create Workspace #
  #------------------#
  # Lock directory
  if [ -e "$OUTPUT_DIR/CONFPASS.lock" ] && [ $RERUN -eq 0 ]
  then
9456401f8   Jean-François Rey   modify info and l...
136
      print_warn "[${BASENAME}] Confpass is locked -> exit" 2
e6be5137b   Jean-François Rey   reinitialized pro...
137
138
139
140
141
142
      exit 1
  fi
  rm "$OUTPUT_DIR/CONFPASS.unlock" > /dev/null 2>&1
  touch "$OUTPUT_DIR/CONFPASS.lock" > /dev/null 2>&1
  if [ $RERUN -eq 0 ]; then rm -r $CONF_DIR > /dev/null 2>&1; fi
  if [ $RERUN -eq 1 ]; then rm $USF_FILE > /dev/null 2>&1; fi
1fd315c89   Jean-François Rey   add Extract audio...
143
144
  mkdir -p $CONF_DIR > /dev/null 2>&1
  mkdir -p $RES_CONF_DIR > /dev/null 2>&1
9456401f8   Jean-François Rey   modify info and l...
145
  rm $LOGFILE $ERRORFILE > /dev/null 2>&1
e6be5137b   Jean-François Rey   reinitialized pro...
146
147
148
149
  
  #---------------#
  # Check Pass    #
  #---------------#
9456401f8   Jean-François Rey   modify info and l...
150
151
  print_info "[${BASENAME}] Check Conf Pass directory ${RES_NAME}" 1
  # if usf contains more than 49% of 0.600 confidence -> usf error
e6be5137b   Jean-François Rey   reinitialized pro...
152
153
154
  if [ -s $USF_FILE ]
  then
      conftozerosix=$(grep -c -E 'confidence="0.600"' "${USF_FILE}")
9456401f8   Jean-François Rey   modify info and l...
155
156
      confall=$(grep -c -E 'confidence=' "${USF_FILE}")
      if [ $confall -gt 0 ]
e6be5137b   Jean-François Rey   reinitialized pro...
157
      then
9456401f8   Jean-François Rey   modify info and l...
158
159
          pourcentageofzerosix=$((($conftozerosix*100)/$confall))
          if [ $pourcentageofzerosix -gt 49 ]
e6be5137b   Jean-François Rey   reinitialized pro...
160
          then
9456401f8   Jean-François Rey   modify info and l...
161
162
              print_warn "[${BASENAME}] ${BASENAME}.${RES_NAME}.usf got $pourcentageofzerosix% of 0.600 confidence" 2
              print_info "[${BASENAME}] bad usf ${RES_NAME}, will do it again" 1
e6be5137b   Jean-François Rey   reinitialized pro...
163
164
165
              mv "${USF_FILE}" "${USF_FILE}.back"
              rm -r $CONF_DIR > /dev/null 2>&1
          else
9456401f8   Jean-François Rey   modify info and l...
166
              print_warn "[${BASENAME}] ${USF_FILE} already done, skipping it" 1
e6be5137b   Jean-François Rey   reinitialized pro...
167
168
169
              exit 0
          fi
      fi
9456401f8   Jean-François Rey   modify info and l...
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
  else
      print_info "[${BASENAME}] No USF file already done, continue..." 1
  fi
  
  # Check if treil are here
  nbres_p1=$(cat ${INPUT_DIR}/plp.lst | wc -l)
  nbtreil_p=$(ls $RES_P/*.treil 2> /dev/null | wc -l)
  if [ $nbtreil_p -eq 0 ]
  then
      print_error "[${BASENAME}] No ${RES_NAME} Pass, No .treil -> exit ConfPass"
      if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ${RES_NAME} Pass, No .treil -> exit ConfPass" ;fi
      exit 1
  else
      #Check if more then 89% of treil are done
      if [ $nbres_p1 -gt 0 ]
      then
          pourcentage=$((($nbtreil_p*100)/$nbres_p1))
          if [ $pourcentage -gt 89 ]
          then
              print_info "[${BASENAME}] ${RES_NAME}/*.treil are here" 1
          else
              print_warn "[${BASENAME}] not enough ${RES_NAME} treil" 2
              if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "Not enough ${RES_NAME} treil "
          fi
      fi
e6be5137b   Jean-François Rey   reinitialized pro...
195
196
197
198
199
200
201
202
  fi
  
  #------#
  # Save #
  #------#
  cp $CONFPASS_CONFIG_FILE $OUTPUT_DIR/ConfPass.cfg
  echo "RES_CONF_DIR=$RES_CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg
  echo "CONF_DIR=$CONF_DIR" >> $OUTPUT_DIR/ConfPass.cfg
9456401f8   Jean-François Rey   modify info and l...
203
  print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ConfPass.cfg" 1
e6be5137b   Jean-François Rey   reinitialized pro...
204
205
206
207
  
  #--------------------#
  # CONFIDENCE MEASURE #
  #--------------------#
9456401f8   Jean-François Rey   modify info and l...
208
  # Check pourcentage of scored_ctm already done, if < 85% do confidence measure
e6be5137b   Jean-François Rey   reinitialized pro...
209
210
211
212
213
214
215
  nbres_p=$(ls ${RES_P}/*.treil | wc -l)
  nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l)
  if [ $nbres_p -gt 0 ]
  then
      pourcentageres=$((($nbconf*100)/$nbres_p))
      if [ $pourcentageres -lt 85 ]
      then
9456401f8   Jean-François Rey   modify info and l...
216
          print_info "[${BASENAME}] Calcul Confidence $INPUT_DIR $RES_NAME" 1
1fd315c89   Jean-François Rey   add Extract audio...
217
          $MAIN_SCRIPT_PATH/ConfidenceMeasure.sh $INPUT_DIR $RES_NAME $REDIRECTION_OUTPUT
e6be5137b   Jean-François Rey   reinitialized pro...
218
      else
9456401f8   Jean-François Rey   modify info and l...
219
          print_info "[${BASENAME}] Skipping Confidence Calcul $INPUT_DIR/$RES_NAME" 1
e6be5137b   Jean-François Rey   reinitialized pro...
220
221
222
223
224
225
226
      fi
  fi
  
  ### Check scored_ctm number res files !
  if [ $CHECK -eq 1 ]
  then
      nbconf=$(ls ${RES_CONF_DIR}/*.res | wc -l)
9456401f8   Jean-François Rey   modify info and l...
227
228
229
230
231
      if [ $nbres_p -ne $nbconf ]
      then
          print_warn "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ" 2
          print_log_file $LOGFILE "WARN : ConfPass $INPUT_DIR/$RES_NAME number of res files differ"
      fi
e6be5137b   Jean-François Rey   reinitialized pro...
232
233
234
235
236
  fi
  
  #---------------------------#
  # FROM RES WITH CONF => USF #
  #---------------------------#
9456401f8   Jean-François Rey   modify info and l...
237
  print_info "[${BASENAME}] Create USF file for $RES_P" 1
e6be5137b   Jean-François Rey   reinitialized pro...
238
239
240
241
242
243
244
  for f in `ls ${RES_CONF_DIR}`; do $SCRIPT_PATH/formatRES.pl $RES_CONF_DIR/$f; done
  # create USF configuration file
  echo -e "name $AUTHOR
  fileName $BASENAME
  fileExt wav
  segFile $OUTPUT_DIR/$BASENAME.seg" > $OUTPUT_DIR/$BASENAME.usf_cfg
  # create USF file
9456401f8   Jean-François Rey   modify info and l...
245
  print_info "$SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg $REDIRECTION_OUTPUT" 3
1fd315c89   Jean-François Rey   add Extract audio...
246
247
  $SCRIPT_PATH/res2out.pl --dir $RES_CONF_DIR --format USF --ignore $RULES/asupp --out $USF_FILE.tmp --usf_config $OUTPUT_DIR/$BASENAME.usf_cfg $REDIRECTION_OUTPUT
  rm $OUTPUT_DIR/$BASENAME.usf_cfg > /dev/null 2>&1
0bf609bcc   Jean-François Rey   update and add sc...
248
  cat $USF_FILE.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f > $USF_FILE
e6be5137b   Jean-François Rey   reinitialized pro...
249
  cp $USF_FILE ${OUTPUT_DIR}/${BASENAME}.usf
1fd315c89   Jean-François Rey   add Extract audio...
250
  rm $USF_FILE.tmp > /dev/null 2>&1
e6be5137b   Jean-François Rey   reinitialized pro...
251
252
253
254
255
256
257
258
259
  
  #----------------#
  # Check USF file #
  #----------------#
  if [ $CHECK -eq 1 ]
  then
      check_conf_pass_usf "$OUTPUT_DIR/$BASENAME.usf"
      if [ $? -eq 1 ]
      then
9456401f8   Jean-François Rey   modify info and l...
260
261
          print_error "[${BASENAME}] Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf"
          print_log_file $ERRORFILE "ERROR : Wrong confidence measures in USF file : $OUTPUT_DIR/$BASENAME.usf"
e6be5137b   Jean-François Rey   reinitialized pro...
262
263
264
265
266
267
268
269
          exit 1
      fi
  fi
  
  #-------#
  # CLOSE #
  #-------#
  # Seem OK 
9456401f8   Jean-François Rey   modify info and l...
270
  print_info "[${BASENAME}] <= ConfPass End | $(date +'%d/%m/%y %H:%M:%S')" 1
e6be5137b   Jean-François Rey   reinitialized pro...
271
272
273
   
  # unlock directory
  mv "$OUTPUT_DIR/CONFPASS.lock" "$OUTPUT_DIR/CONFPASS.unlock"