Blame view

main_tools/ThirdPass.sh 9.89 KB
e6be5137b   Jean-François Rey   reinitialized pro...
1
2
3
4
5
6
7
8
9
  #!/bin/bash
  
  #####################################################
  # File :    ThirdPass.sh                            #
  # Brief :   ASR third pass using trigg files        #
  # Author :  Jean-François Rey                       #
  # Version : 1.0                                     #
  # Date :    18/07/13                                #
  #####################################################
f37e72eaf   Jean-François Rey   up
10
  echo "### ThirdPass.sh ###"
e6be5137b   Jean-François Rey   reinitialized pro...
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
  # Check OTMEDIA_HOME env var
  if [ -z ${OTMEDIA_HOME} ]
  then
      OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
      export OTMEDIA_HOME=$OTMEDIA_HOME
  fi
  
  # where is SecondPass.sh
  MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
  
  # Scripts Path
  SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
  
  # Include scripts
  . $SCRIPT_PATH"/Tools.sh"
0bf609bcc   Jean-François Rey   update and add sc...
26
  . $SCRIPT_PATH"/CheckThirdPass.sh"
e6be5137b   Jean-François Rey   reinitialized pro...
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
  
  # where is ThirdPass.cfg
  THIRDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/ThirdPass.cfg"
  if [ -e $THIRDPASS_CONFIG_FILE ]
  then
  	. $THIRDPASS_CONFIG_FILE
  else
  	echo "ERROR : Can't find configuration file $THIRDPASS_CONFIG_FILE" >&2
  	exit 1
  fi
  
  #---------------#
  # Parse Options #
  #---------------#
  while getopts ":hDv:crf:" opt
  do
  	case $opt in
  		h)
  			echo -e "$0 [OPTIONS] <PASS_DIRECTORY>
  "
              echo -e "\t Options:"
              echo -e "\t\t-h :\tprint this message"
              echo -e "\t\t-D :\tDEBUG mode on"
              echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
              echo -e "\t\t-c :\t Check process, stop if error detected"
              echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)"
              echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done"
  			exit 1
  			;;
  		D)
  			DEBUG=1
  			;;
          v)
              VERBOSE=$OPTARG
              ;;
          c)
              CHECK=1
              ;;
          f)
              FORKS="--forks $OPTARG"
              ;;
          r)
              RERUN=1
              ;;
  		:)
d7e9e4b9d   Jean-François Rey   update bugfix stderr
72
  			echo "Option -$OPTARG requires an argument." >&2
e6be5137b   Jean-François Rey   reinitialized pro...
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
  			exit 1
  			;;
  		\?)
  			echo "BAD USAGE : unknow opton -$OPTARG"
  			#exit 1
  			;;
  	esac
  done
  
  # mode debug enable
  if [ $DEBUG -eq 1 ]
  then
         set -x
         echo -e "## Mode DEBUG ON ##"
  fi
  
  # mode verbose enable
  if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ; fi
  
  # Check USAGE by arguments number
  if [ $(($#-($OPTIND-1))) -ne 1 ]
  then
  	echo "BAD USAGE : ThirdPass.sh [OPTIONS] <PASS_DIR>"
  	echo "$0 -h for more info"
      exit 1
  fi
  
  shift $((OPTIND-1))
  # check Pass directory - First argument
  if [ -e $1 ] && [ -d $1 ]
  then
      PASS_DIR=$(readlink -e $1)
b427f103e   Jean-François Rey   update log info p...
105
      BASENAME=$(basename $PASS_DIR)
e6be5137b   Jean-François Rey   reinitialized pro...
106
107
108
109
  else
  	print_error "can't find $1 directory"
  	exit 1
  fi
b427f103e   Jean-François Rey   update log info p...
110
  print_info "[${BASENAME}] => P3 start | $(date +'%d/%m/%y %H:%M:%S')" 1
e6be5137b   Jean-François Rey   reinitialized pro...
111
112
113
114
115
116
117
118
119
120
121
122
123
  #-------------#
  # GLOBAL VARS #
  #-------------#
  EXPLOITCONFPASS_CONFIG_FILE="$PASS_DIR/ExploitConfPass.cfg"
  if [ -e $EXPLOITCONFPASS_CONFIG_FILE ]
  then
      TRIGGER_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "TRIGGER_SPEERAL=" | cut -f2 -d"=")
      LEX_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_SPEERAL=" | cut -f2 -d"=")
      LEX_BINODE_SPEERAL=$(cat $EXPLOITCONFPASS_CONFIG_FILE | grep "LEX_BINODE_SPEERAL=" | cut -f2 -d"=")
      LST=""
      PLP_DIR_P1=""
      HMM=""
  else
b427f103e   Jean-François Rey   update log info p...
124
125
      print_error "[${BASENAME}] Can't find $EXPLOITCONFPASS_CONFIG_FILE file"
      print_warn "[${BASENAME}] Will use default directories and files" 2
e6be5137b   Jean-François Rey   reinitialized pro...
126
127
128
129
130
      #exit 1
      TRIGGER_SPEERAL=$PASS_DIR/trigg/speeral/
      LEX_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext
      LEX_BINODE_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext.bin
  fi
b427f103e   Jean-François Rey   update log info p...
131

e6be5137b   Jean-François Rey   reinitialized pro...
132
133
134
135
136
137
  SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg"
  if [ -e $SECONDPASS_CONFIG_FILE ]
  then
      LST=$(cat $SECONDPASS_CONFIG_FILE | grep "^LST=" | cut -f2 -d"=")
      HMM=$(cat $SECONDPASS_CONFIG_FILE | grep "^HMM=" | cut -f2 -d"=")
      PLP_DIR_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_DIR_P1=" | cut -f2 -d"=")
b427f103e   Jean-François Rey   update log info p...
138
      PLP_FILE_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_FILE_P1=" | cut -f2 -d"=")
e6be5137b   Jean-François Rey   reinitialized pro...
139
  else
b427f103e   Jean-François Rey   update log info p...
140
141
      print_error "[${BASENAME}] Can't find $SECONDPASS_CONFIG_FILE file"
      print_warn "[${BASENAME}] Will use default directories and files" 2
e6be5137b   Jean-François Rey   reinitialized pro...
142
143
144
145
      #exit 1
      LST=$PASS_DIR/lists
      HMM=$PASS_DIR/hmm
      PLP_DIR_P1=$PASS_DIR/PLP
b427f103e   Jean-François Rey   update log info p...
146
      PLP_FILE_P1=$PASS_DIR/plp.lst
e6be5137b   Jean-François Rey   reinitialized pro...
147
  fi
e6be5137b   Jean-François Rey   reinitialized pro...
148
149
  OUTPUT_DIR_BASENAME=$PASS_DIR
  RES_DIR="$PASS_DIR/res_p3"
b427f103e   Jean-François Rey   update log info p...
150
151
  LOGFILE=$PASS_DIR"/info_p3.log"
  ERRORFILE=$PASS_DIR"/error_p3.log"
e6be5137b   Jean-François Rey   reinitialized pro...
152
153
154
155
156
157
  
  #------------------#
  # Create WORKSPACE #
  #------------------#
  
  # Lock directory
b427f103e   Jean-François Rey   update log info p...
158
  if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] THIRDPASS is locked -> exit" 2; exit 1; fi
e6be5137b   Jean-François Rey   reinitialized pro...
159
160
161
162
163
  rm "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" > /dev/null 2>&1
  touch "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" > /dev/null 2>&1
  
  if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi
  mkdir -p $RES_DIR
b427f103e   Jean-François Rey   update log info p...
164
  print_info "[${BASENAME}] Make directory $RES_DIR" 3 
e6be5137b   Jean-François Rey   reinitialized pro...
165
166
167
168
169
170
  
  #--------------------#
  # Save configuration #
  #--------------------#
  cp $THIRDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/ThirdPass.cfg
  echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/ThirdPass.cfg
b427f103e   Jean-François Rey   update log info p...
171
  print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ThirdPass.cfg" 1
e6be5137b   Jean-François Rey   reinitialized pro...
172
173
174
175
  
  #--------------------------------------------------#
  # Third Pass using trigger file (DECODING)         #
  #--------------------------------------------------#
b427f103e   Jean-François Rey   update log info p...
176
  print_info "[${BASENAME}] Launch Third Pass" 1
e6be5137b   Jean-François Rey   reinitialized pro...
177
178
179
180
181
182
  
  ## Generate speeral config file adding trigger rep
  cat $SPEERAL_CFG_PATH/$SPEERAL_CFG_FILE | sed -e "s|<nom>[^<]*</nom>|<nom>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext</nom>|g" \
      | sed -e "s|<binode>[^<]*</binode>|<binode>$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext.bin</binode>|g" \
      | sed -e "s|<trigger><dir>[^<]*</dir></trigger>|<trigger><dir>$TRIGGER_SPEERAL</dir></trigger>|g" > $OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml
  SPEERAL_THIRD_CFG=$OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml
b427f103e   Jean-François Rey   update log info p...
183
  print_info "[${BASENAME}] Third Pass Speeral Config file ${SPEERAL_THIRD_CFG}" 3
e6be5137b   Jean-François Rey   reinitialized pro...
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
  
  # for all speaker
  for lspeaker in $(ls $LST/*.lst)
  do
      speaker=$(basename $lspeaker ".lst")
      # for all AM
      for (( i=0; $i<${#MTAG[@]} ; i++ ))
      do
          if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then
              type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst")
              ## if is the good AM for the speaker
              if [ -n "$type" ]
              then
                  ## Speeral decoding
                  if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ]
                  then
b427f103e   Jean-François Rey   update log info p...
200
                      print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3
561670acc   Jean-François Rey   remove output red...
201
                      $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock
e6be5137b   Jean-François Rey   reinitialized pro...
202
                  else
b427f103e   Jean-François Rey   update log info p...
203
204
                      print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2
                      print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3
561670acc   Jean-François Rey   remove output red...
205
                      $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock
e6be5137b   Jean-François Rey   reinitialized pro...
206
207
208
209
210
211
212
                  fi
  
                  if [ $CHECK -eq 1 ]
                  then
                      check_third_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR"
                      if [ $? -eq 1 ]
                      then 
b427f103e   Jean-François Rey   update log info p...
213
214
215
  
                          print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2
                          print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst"
e6be5137b   Jean-François Rey   reinitialized pro...
216
                          ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
b427f103e   Jean-François Rey   update log info p...
217
218
                          print_log_file $LOGFILE "Segs (and treil) not done :
  ["
d7e9e4b9d   Jean-François Rey   update bugfix stderr
219
                          diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //"  >> $LOGFILE
b427f103e   Jean-François Rey   update log info p...
220
                          print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]"
e6be5137b   Jean-François Rey   reinitialized pro...
221
                          rm ${OUTPUT_DIR_BASENAME}/.tmp
e6be5137b   Jean-François Rey   reinitialized pro...
222
223
224
225
226
227
228
229
230
                      fi
                  fi
                  break
              fi
          fi
      done
  done
  
  ## Check missing seg and log it
b427f103e   Jean-François Rey   update log info p...
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
  
  ## Check missing seg and log it
  if [ $CHECK -eq 1 ]
  then
      ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
      todo=$(cat ${PLP_FILE_P1} | wc -l)
      if [ $todo -eq 0 ]; then todo=1;fi
      notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l)))
      pourcentage=$((($notdone*100)/$todo))
      if [ $notdone -ne 0 ]
      then
          print_error "[${BASENAME}] Check $ERRORFILE"
          print_log_file "$ERRORFILE" "ERROR : Treil not done ["
          diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE
          print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME"
     else
          print_log_file "$LOGFILE" "P3 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')"
     fi
     rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1
  fi
e6be5137b   Jean-François Rey   reinitialized pro...
251
252
253
254
  
  #---------------#
  # Convert res   #
  #---------------#
b427f103e   Jean-François Rey   update log info p...
255
  print_info "[${BASENAME}] Convert .res into .ctm" 1
e6be5137b   Jean-François Rey   reinitialized pro...
256
  # .res => .ctm
561670acc   Jean-François Rey   remove output red...
257
  $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.ctm
b427f103e   Jean-François Rey   update log info p...
258
  print_info "[${BASENAME}] Convert .res into .trs" 1
e6be5137b   Jean-François Rey   reinitialized pro...
259
260
261
262
263
  # .res => .trs
  echo -e "name $AUTHOR
  fileName ${BASENAME}
  fileExt wav
  segFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
561670acc   Jean-François Rey   remove output red...
264
  $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
1fd315c89   Jean-François Rey   add Extract audio...
265
  rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1
b427f103e   Jean-François Rey   update log info p...
266
  print_info "[${BASENAME}] Convert .res into .txt" 1
e6be5137b   Jean-François Rey   reinitialized pro...
267
  # .res => .txt
561670acc   Jean-François Rey   remove output red...
268
  $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.txt
e6be5137b   Jean-François Rey   reinitialized pro...
269

b427f103e   Jean-François Rey   update log info p...
270
  print_info "[${BASENAME}] <= P3 End | $(date +'%d/%m/%y %H:%M:%S')" 1
e6be5137b   Jean-François Rey   reinitialized pro...
271
272
273
  
  # unlock directory
  mv "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock"