Jean-François Rey / otmedia

Blame view

main_tools/SecondPass.sh 11.3 KB
edit raw normal view history
e6be5137b   Jean-François Rey   reinitialized pro... 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
  #!/bin/bash
  
  #####################################################
  # File :    SecondPass.sh                           #
  # Brief :   Speaker adaptation + ASR second pass    #
  # Author :  Jean-François Rey                       #
  #	        (base on Emmanuel Ferreira              #
  #	        and Hugo Mauchrétien works)             #
  # Version : 1.1                                     #
  # Date :    18/06/13                                #
  #####################################################
  
  # Check OTMEDIA_HOME env var
  if [ -z ${OTMEDIA_HOME} ]
  then
      OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0)))
      export OTMEDIA_HOME=$OTMEDIA_HOME
  fi
  
  # where is SecondPass.sh
  MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0))
  
  # Scripts Path
  SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts
  
  # Include scripts
  . $SCRIPT_PATH"/Tools.sh"
  . $SCRIPT_PATH"/CheckSecondPass.sh"
  
  # where is SecondPass.cfg
  SECONDPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/SecondPass.cfg"
  if [ -e $SECONDPASS_CONFIG_FILE ]
  then
  	. $SECONDPASS_CONFIG_FILE
  else
  	echo "ERROR : Can't find configuration file $SECONDPASS_CONFIG_FILE" >&2
  	exit 1
  fi
  
  #---------------#
  # Parse Options #
  #---------------#
  while getopts ":hDv:crf:" opt
  do
  	case $opt in
  		h)
  			echo -e "$0 [OPTIONS] <FIRST_PASS_DIRECTORY>
  "
              echo -e "\t Options:"
              echo -e "\t\t-h :\tprint this message"
              echo -e "\t\t-D :\tDEBUG mode on"
              echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode"
              echo -e "\t\t-c :\t Check process, stop if error detected"
              echo -e "\t\t-f n :\tSpecify speeral forks number (default 1)"
              echo -e "\t\t-r n :\tforce rerun the show, without deleting works already done"
  			exit 1
  			;;
  		D)
  			DEBUG=1
  			;;
          v)
              VERBOSE=$OPTARG
              ;;
          c)
              CHECK=1
              ;;
          f)
              FORKS="--forks $OPTARG"
              ;;
          r)
              RERUN=1
              ;;
  		:)
  			echo "Option -$OPTARG requires an argument." >&2
  			exit 1
  			;;
  		\?)
  			echo "BAD USAGE : unknow opton -$OPTARG"
  			exit 1
  			;;
  	esac
  done
  
  # mode debug enable
  if [ $DEBUG -eq 1 ]
  then
         set -x
         echo -e "## Mode DEBUG ON ##"
1fd315c89   Jean-François Rey   add Extract audio... 
89
90
91
 
         REDIRECTION_OUTPUT=""
     else
         REDIRECTION_OUTPUT=" 2> /dev/null"
e6be5137b   Jean-François Rey   reinitialized pro... 
92
93
94
 
  fi
  
  # mode verbose enable
1fd315c89   Jean-François Rey   add Extract audio... 
95
 
  if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi
e6be5137b   Jean-François Rey   reinitialized pro... 
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
  
  # Check USAGE by arguments number
  if [ $(($#-($OPTIND-1))) -ne 1 ]
  then
  	echo "BAD USAGE : SecondPass.sh [OPTIONS] <FIRST_PASS_DIR>"
  	echo "$0 -h for more info"
      exit 1
  fi
  
  shift $((OPTIND-1))
  # check FirstPass directory - First argument
  if [ -e $1 ] && [ -d $1 ]
  then
      FIRSTPASS_DIR=$(readlink -e $1)
  else
  	print_error "can't find $1 directory"
  	exit 1
  fi
  
  #-------------#
  # GLOBAL VARS #
  #-------------#
  FIRSTPASS_CONFIG_FILE="$FIRSTPASS_DIR/FirstPass.cfg"
  if [ -e $FIRSTPASS_CONFIG_FILE ]
  then
      WAV_FILE=$(cat $FIRSTPASS_CONFIG_FILE | grep "WAV_FILE=" | cut -f2 -d"=")
      BASENAME=$(cat $FIRSTPASS_CONFIG_FILE | grep "^BASENAME=" | cut -f2 -d"=")
      OUTPUT_DIR=$(cat $FIRSTPASS_CONFIG_FILE | grep "OUTPUT_DIR=" | cut -f2 -d"=")
      OUTPUT_DIR_BASENAME=$FIRSTPASS_DIR
      PLP_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_FILE=" | cut -f2 -d"=")
      PLP_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "PLP_DIR=" | cut -f2 -d"=")
      SEG_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "SEG_FILE=" | cut -f2 -d"=")
      LBL_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "LBL_FILE=" | cut -f2 -d"=")
      RES_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "RES_DIR=" | cut -f2 -d"=")
  else
      print_error "can't find $FIRSTPASS_CONFIG_FILE file"
      exit 1
  fi
  LST=$OUTPUT_DIR_BASENAME"/lists"
  HMM=$OUTPUT_DIR_BASENAME"/hmm/"
  RES_DIR=$OUTPUT_DIR_BASENAME"/res_p2"
  LOGFILE="$OUTPUT_DIR/info_p2.log"
  ERRORFILE="$OUTPUT_DIR/error_p2.log"
  
  #------------------#
  # Create WORKSPACE #
  #------------------#
  
  # Lock directory
  if [ -e $OUTPUT_DIR_BASENAME/SECONDPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi
  rm "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock" > /dev/null 2>&1
  touch "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" > /dev/null 2>&1
  
  rm -r $LST > /dev/null 2>&1
  mkdir -p $LST
  print_info "Make directory $LST" 1
  if [ $RERUN -eq 0 ]; then rm -r $HMM > /dev/null 2>&1; fi
  mkdir -p $HMM
  print_info "Make directory $HMM" 1
  if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi
1fd315c89   Jean-François Rey   add Extract audio... 
156
 
  mkdir -p $RES_DIR > /dev/null 2>&1
e6be5137b   Jean-François Rey   reinitialized pro... 
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
  print_info "Make directory $RES_DIR" 1
  
  #-------------------#
  # Check Pass        #
  #-------------------#
  print_info "Check Pass 2 directory" 1
  for treil in $(ls $RES_DIR/ | grep treil)
  do
      if [ ! -s $RES_DIR/$treil ]
      then
          bn = $(basename $treil ".treil")
          rm $RES_DIR/$treil $RES_DIR/$bn.seg $RES_DIR/$bn.res $RES_DIR/$bn.pho 2> /dev/null
          print_info "$RES_DIR/$bn.* files deleted.." 2
      fi
  done
  
  # Check if more then 89% of treil are done
135404bcf   Jean-François Rey   Modify verbose an... 
174
175
 
  nbres_p1=$(ls $RES_DIR_P1/*.res 2> /dev/null | wc -l)
  nbtreil_p2=$(ls $RES_DIR/*.treil 2> /dev/null | wc -l)
e6be5137b   Jean-François Rey   reinitialized pro... 
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
 
  if [ $nbres_p1 -gt 0 ]
  then
      pourcentage=$((($nbtreil_p2*100)/$nbres_p1))
      if [ $pourcentage -gt 89 ]
      then
          echo "Lattice already done, skipping $BASENAME"
          exit 0
      fi
  fi
  
  #--------------------#
  # Save configuration #
  #--------------------#
  cp $SECONDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/SecondPass.cfg
  echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
  echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
  echo "FIRSTPASS_DIR=$FIRSTPASS_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
  echo "PLP_DIR_P1=$PLP_DIR_P1" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
  echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
  echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
  echo "LST=$LST" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
  echo "HMM=$HMM" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
  echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/SecondPass.cfg
  print_info "save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1
  
  
  #--------------------------------------------------#
  # Speaker Adaptation (AM) + Second pass (DECODING) #
  #--------------------------------------------------#
135404bcf   Jean-François Rey   Modify verbose an... 
205
 
  print_info "Launch Second Pass" 1
e6be5137b   Jean-François Rey   reinitialized pro... 
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
  
  # for all speaker
  for speaker in $(cat $LBL_FILE_P1 | cut -f4 -d" " | sort | uniq)
  do
      ## get seg file from P1 containing the speaker
      find $RES_DIR_P1 -name "*${speaker}.seg" -exec basename "{}" .seg \; | sort > $LST/$speaker.lst
      print_info "file for $speaker in $LST/$speaker.lst" 3
      if [ ! -s $LST/$speaker.lst ]; then print_warn "no ${speaker} file in $RES_DIR_P1" 3; continue; fi
  
  
      # for all AM
      for (( i=0; $i<${#MTAG[@]} ; i++ ))
      do
          if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ]; then
              type=$(grep -e "${speaker}$" "$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst")
              ## if is the good AM for the speaker
              if [ -n "$type" ]
              then
                  ## HMM adaptation
                  if [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] )
                  then
                      print_info "$SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/" 3
1fd315c89   Jean-François Rey   add Extract audio... 
228
 
                      $SPEERAL_TOOLS/mllrmap $SPEERAL_AM/${MODS[$i]}.hmm -LR -b$LST/$speaker.lst -d$PLP_DIR_P1/ -t.plp -i3 -c.seg -o$HMM/$speaker.hmm -a$RES_DIR_P1/ $REDIRECTION_OUTPUT
e6be5137b   Jean-François Rey   reinitialized pro... 
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
 
                  fi
  
                  if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.hmm ] || [ -z $HMM/$speaker.hmm ] )
                  then
                      echo "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $HMM/$speaker.hmm" >> $ERRORFILE
                      #exit 1
                  fi
  
                  ## cp map files
                  cp $SPEERAL_AM/${MODS[$i]}.map $HMM/$speaker.map
                  
                  ## class clustering
                  if [ -s $HMM/$speaker.hmm ] && ( [ $RERUN -eq 0 ] || ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] ))
                  then
                      print_info "$SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls" 3
1fd315c89   Jean-François Rey   add Extract audio... 
244
 
                      $SPEERAL_TOOLS/gclust_pdf $HMM/$speaker.hmm $HMM/$speaker.cls -i $SPEERAL_AM/${MODS[$i]}.cls $REDIRECTION_OUTPUT
e6be5137b   Jean-François Rey   reinitialized pro... 
245
246
247
248
249
250
251
252
253
254
 
                  fi
                  if [ $CHECK -eq 1 ] && ( [ ! -e $HMM/$speaker.cls ] || [ -z $HMM/$speaker.cls ] )
                  then
                      echo "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $HMM/$speaker.cls" >> $ERRORFILE
                      #exit 1
                  fi
  
                  ## Speeral decoding
                  if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ]
                  then
1fd315c89   Jean-François Rey   add Extract audio... 
255
256
 
                      print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock $REDIRECTION_OUTPUT" 3
                      $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock $REDIRECTION_OUTPUT
e6be5137b   Jean-François Rey   reinitialized pro... 
257
258
 
                  else
                      print_warn "$HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2
1fd315c89   Jean-François Rey   add Extract audio... 
259
 
                      $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT
e6be5137b   Jean-François Rey   reinitialized pro... 
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
 
                  fi
  
                  if [ $CHECK -eq 1 ]
                  then
                      check_second_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR"
                      if [ $? -eq 1 ]
                      then 
                          echo -e "ERROR : Speeral $LST/$speaker.lst
  [" >> $ERRORFILE
                          ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
                          diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp  >> $ERRORFILE
                          echo -e "] " >> $ERRORFILE
                          rm ${OUTPUT_DIR_BASENAME}/.tmp
                          #exit 1
                      fi
                  fi
                  break
              fi
          fi
      done
      #rm "$HMM/$speaker.*" > /dev/null 2>&1
      #rm "$LST/$speaker.lst" > /dev/null 2>&1
  done
  
  print_info "<= End $BASENAME P2 | $(date +'%d/%m/%y %H:%M:%S')" 1
  
  ## Check missing seg and log it
  ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
  echo -e "$BASENAME P2 END
  [" >> $LOGFILE
  diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $LOGFILE
  echo -e "] $BASENAME" >> $LOGFILE
1fd315c89   Jean-François Rey   add Extract audio... 
292
 
  rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1
e6be5137b   Jean-François Rey   reinitialized pro... 
293
294
295
296
297
298
 
  
  #---------------#
  # Convert res   #
  #---------------#
  
  # .res => .ctm
1fd315c89   Jean-François Rey   add Extract audio... 
299
 
  $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.ctm $REDIRECTION_OUTPUT
e6be5137b   Jean-François Rey   reinitialized pro... 
300
301
302
303
304
 
  # .res => .trs
  echo -e "name $AUTHOR
  fileName ${BASENAME}
  fileExt wav
  segFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
1fd315c89   Jean-François Rey   add Extract audio... 
305
306
 
  $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg $REDIRECTION_OUTPUT
  rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1
e6be5137b   Jean-François Rey   reinitialized pro... 
307
 
  # .res => .txt
1fd315c89   Jean-François Rey   add Extract audio... 
308
 
  $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.2pass.txt $REDIRECTION_OUTPUT
e6be5137b   Jean-François Rey   reinitialized pro... 
309
310
311
312
 
  
  
  # unlock directory
  mv "$OUTPUT_DIR_BASENAME/SECONDPASS.lock" "$OUTPUT_DIR_BASENAME/SECONDPASS.unlock"