Blame view
main_tools/FirstPass.sh
14.3 KB
e6be5137b
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 |
#!/bin/bash ##################################################### # File : FirstPass.sh # # Brief : ASR first pass and speaker diarization # # Author : Jean-François Rey # # (base on Emmanuel Ferreira # # and Hugo Mauchrétien works) # # Version : 1.1 # # Date : 18/06/13 # ##################################################### # Check OTMEDIA_HOME env var if [ -z ${OTMEDIA_HOME} ] then OTMEDIA_HOME=$(dirname $(dirname $(readlink -e $0))) export OTMEDIA_HOME=$OTMEDIA_HOME fi # where is FirstPass.sh MAIN_SCRIPT_PATH=$(dirname $(readlink -e $0)) # scripts path SCRIPT_PATH=$OTMEDIA_HOME/tools/scripts # Include scripts . $SCRIPT_PATH"/Tools.sh" . $SCRIPT_PATH"/CheckFirstPass.sh" # where is FirstPass.cfg FIRSTPASS_CONFIG_FILE=$OTMEDIA_HOME"/cfg/FirstPass.cfg" if [ -e $FIRSTPASS_CONFIG_FILE ] then . $FIRSTPASS_CONFIG_FILE else |
135404bcf
|
36 37 |
echo "ERROR : Can't find configuration file $FIRSTPASS_CONFIG_FILE" > /dev/stderr echo "exit" > /dev/stderr |
e6be5137b
|
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
exit 1 fi #---------------# # Parse Options # #---------------# while getopts ":hDv:cf:r" opt do case $opt in h) echo -e "$0 [OPTIONS] <WAV_FILE> <OUTPUT_DIRECTORY> " echo -e "\t Options:" echo -e "\t\t-h :\tprint this message" echo -e "\t\t-D :\tDEBUG mode on" echo -e "\t\t-v l :\tVerbose mode, l=(1|2|3) level mode" |
135404bcf
|
54 |
echo -e "\t\t-c :\tCheck process, and log it into files, can stop if error detected" |
e6be5137b
|
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
echo -e "\t\t-f n :\tspecify a speeral forks number (default 1)" echo -e "\t\t-r :\tforce rerun the wav file" exit 1 ;; D) DEBUG=1 ;; v) VERBOSE=$OPTARG ;; c) CHECK=1 ;; f) FORKS="--forks $OPTARG" ;; r) RERUN=1 ;; :) |
9456401f8
|
75 |
echo "Option -$OPTARG requires an argument." > /dev/stderr |
e6be5137b
|
76 77 78 |
exit 1 ;; \?) |
9456401f8
|
79 |
echo "BAD USAGE : unknow opton -$OPTARG" |
e6be5137b
|
80 81 82 83 84 85 86 87 88 89 |
exit 1 ;; esac done # mode debug enable if [ $DEBUG -eq 1 ] then set -x echo -e "## Mode DEBUG ON ##" |
0bf609bcc
|
90 91 |
REDIRECTION_OUTPUT="" else |
1fd315c89
|
92 |
REDIRECTION_OUTPUT=" 2> /dev/null" |
e6be5137b
|
93 94 95 |
fi # mode verbose enable |
1fd315c89
|
96 |
if [ $VERBOSE -gt 0 ]; then echo -e "## Verbose level : $VERBOSE ##" ;fi |
e6be5137b
|
97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 |
# Check USAGE by arguments number if [ $(($#-($OPTIND-1))) -ne 2 ] then echo "BAD USAGE : FirstPass.sh [OPTIONS] <WAV_FILE> <OUTPUT_DIR>" echo "$0 -h for more info" exit 1 fi shift $((OPTIND-1)) # check audio file - First argument if [ -e $1 ] && [ -s $1 ] then # absolute path to wav file WAV_FILE=$(readlink -e $1) # wav filename FILENAME=$(basename $WAV_FILE) # wav filename without extension BASENAME=${FILENAME%.*} |
135404bcf
|
116 117 |
print_info "[${BASENAME}] => P1 start | $(date +'%d/%m/%y %H:%M:%S')" 1 print_info "[${BASENAME}] $WAV_FILE OK" 2 |
e6be5137b
|
118 119 120 121 122 123 124 125 126 |
else print_error "can't find $1 OR file is empty" exit 1 fi # check output directory - Second argument if [ ! -e $2 ] then mkdir -p $2 |
135404bcf
|
127 |
print_info "[${BASENAME}] Make directory $2" 2 |
e6be5137b
|
128 129 130 131 132 133 134 135 136 137 138 139 140 |
fi #-------------# # GLOBAL VARS # #-------------# OUTPUT_DIR=$(readlink -e $2) # Output directory absolute path OUTPUT_DIR_BASENAME="$OUTPUT_DIR/$BASENAME/" # New OUTPUT_DIR with BASENAME PLP_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.plp" # Global PLP file PLP_DIR="$OUTPUT_DIR_BASENAME/PLP/" # Segmented PLP files directory SEG_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.seg" # Global Seg file LBL_FILE="$OUTPUT_DIR_BASENAME/$BASENAME.lbl" # Global LBL file RES_DIR=$OUTPUT_DIR_BASENAME"/res_p1" |
135404bcf
|
141 142 |
LOGFILE="$OUTPUT_DIR_BASENAME/info_p1.log" ERRORFILE="$OUTPUT_DIR_BASENAME/error_p1.log" |
e6be5137b
|
143 144 145 146 147 148 149 |
#------------------# # Create WORKSPACE # #------------------# if [ ! -e $OUTPUT_DIR_BASENAME ] then mkdir -p $OUTPUT_DIR_BASENAME |
135404bcf
|
150 |
print_info "[${BASENAME}] Make directory $OUTPUT_DIR_BASENAME" 2 |
e6be5137b
|
151 152 153 154 155 156 157 158 159 |
fi # Lock directory if [ -e $OUTPUT_DIR_BASENAME/FIRSTPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1; fi rm "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" > /dev/null 2>&1 touch "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" > /dev/null 2>&1 rm -r $PLP_DIR > /dev/null 2>&1; mkdir -p $PLP_DIR |
135404bcf
|
160 |
print_info "[${BASENAME}] Make directory $PLP_DIR" 2 |
e6be5137b
|
161 162 163 164 165 166 |
if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; else rm $RES_DIR/*.lock > /dev/null 2>&1 fi |
1fd315c89
|
167 |
mkdir -p $RES_DIR > /dev/null 2>&1 |
135404bcf
|
168 169 |
print_info "[${BASENAME}] Make directory $RES_DIR" 2 rm $LOGFILE $ERRORFILE > /dev/null 2>&1 |
e6be5137b
|
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 |
#--------------------# # Save configuration # #--------------------# cp $FIRSTPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/FirstPass.cfg echo "FIRSTPASS_SCRIPT_PATH=$MAIN_SCRIPT_PATH" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg echo "WAV_FILE=$WAV_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg echo "BASENAME=$BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg echo "OUTPUT_DIR=$OUTPUT_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg echo "OUTPUT_DIR_BASENAME=$OUTPUT_DIR_BASENAME" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg echo "PLP_FILE=$PLP_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg echo "PLP_DIR=$PLP_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg echo "SEG_FILE=$SEG_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg echo "LBL_FILE=$LBL_FILE" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/FirstPass.cfg |
9456401f8
|
185 |
print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/FirstPass.cfg" 1 |
e6be5137b
|
186 187 188 189 190 191 192 193 194 195 196 197 198 199 |
#-------------------------# # Check Audio File Format # #-------------------------# error=0 temp=$(avconv -i $WAV_FILE 2>&1 | grep "16000 Hz") if [ -z "$temp" ]; then error=1; fi temp=$(avconv -i $WAV_FILE 2>&1 | grep "1 channels") if [ -z "$temp" ]; then error=1; fi temp=$(avconv -i $WAV_FILE 2>&1 | grep "s16") if [ -z "$temp" ]; then error=1; fi if [ $error -eq 1 ] then |
135404bcf
|
200 201 202 |
print_message $WARNING 2 "[${BASENAME}] $WAV_FILE is not a wav file at 16000 Hz, 1 channel, 16bits have to convert" print_message $INFO 3 "[${BASENAME}] avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav" |
0bf609bcc
|
203 |
avconv -i $WAV_FILE -threads 4 -vn -f wav -ac 1 -ar 16000 -ab 256000 $OUTPUT_DIR_BASENAME/$BASENAME.wav $REDIRECTION_OUTPUT |
e6be5137b
|
204 205 |
WAV_FILE=$OUTPUT_DIR_BASENAME/$BASENAME.wav FILENAME=$BASENAME.wav |
135404bcf
|
206 |
print_message $INFO 1 "[${BASENAME}] new wav file : $WAV_FILE" |
e6be5137b
|
207 208 209 210 211 212 213 214 |
fi #---------------# # Get SRT file # #---------------# if [ -s $(dirname $WAV_FILE)/$BASENAME.SRT ] then cp $(dirname $WAV_FILE)/$BASENAME.SRT $OUTPUT_DIR_BASENAME/$BASENAME.SRT |
135404bcf
|
215 |
print_info "[${BASENAME}] copy $BASENAME.SRT file into $OUTPUT_DIR_BASENAME" 3 |
e6be5137b
|
216 217 218 219 220 |
fi #------------# # WAV -> PLP # #------------# |
135404bcf
|
221 |
print_info "[${BASENAME}] convert WAV -> PLP" 1 |
e6be5137b
|
222 |
echo $FILENAME > $OUTPUT_DIR_BASENAME/list.tmp |
135404bcf
|
223 |
print_info "[${BASENAME}] $BIN_PATH/lia_plp_mt.32 --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms" 3 |
e6be5137b
|
224 |
|
0bf609bcc
|
225 |
$BIN_PATH/lia_plp_mt$ARCH --lst $OUTPUT_DIR_BASENAME/list.tmp --input_dir $(dirname $WAV_FILE) --output_dir $OUTPUT_DIR_BASENAME --input_type WAV --output_type HTK --nb_coef 12 --cms $REDIRECTION_OUTPUT |
e6be5137b
|
226 227 228 229 230 231 |
if [ $CHECK -eq 1 ] then check_first_pass_plp "$PLP_FILE" if [ $? -eq 1 ] then |
135404bcf
|
232 233 |
print_log_file "$ERROFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating PLP file : $PLP_FILE" echo "[${BASENAME}] ERROR : ${BASENAME} check $ERRORFILE file" > /dev/stderr |
e6be5137b
|
234 235 236 237 238 239 240 241 242 |
exit 1 fi fi rm $OUTPUT_DIR_BASENAME/list.tmp #------------------------------# # S/NS + SPEAKERS SEGMENTATION # #------------------------------# |
135404bcf
|
243 |
print_info "[${BASENAME}] Launch speakers diarization" 1 |
e6be5137b
|
244 |
# Calcul seg file |
135404bcf
|
245 |
print_info "[${BASENAME}] java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME" 3 |
e6be5137b
|
246 |
#java -Xmx8000m -Xms2048 -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME |
0bf609bcc
|
247 |
java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FILE} --sOutputMask=${SEG_FILE} $BASENAME $REDIRECTION_OUTPUT #–doCEClustering |
e6be5137b
|
248 249 250 |
if [ $CHECK -eq 1 ] && ( [ ! -e $SEG_FILE ] || [ -z $SEG_FILE ] ) then |
135404bcf
|
251 |
print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating SEG file : $SEG_FILE" |
9456401f8
|
252 |
print_error "[${BASENAME}] ERROR : check $ERRORFILE file" |
e6be5137b
|
253 254 255 256 257 |
exit 1 fi # Create LBL file |
135404bcf
|
258 |
print_info "Extract LBL file from SEG file" 1 |
e6be5137b
|
259 260 261 262 263 |
cat $SEG_FILE | grep -v ";;" | cut -f3,4,5,8 -d" " | tr " " "#" | sort -k1 -n | tr "#" " " > $LBL_FILE if [ $CHECK -eq 1 ] && ( [ ! -e $LBL_FILE ] || [ -z $LBL_FILE ] ) then |
135404bcf
|
264 |
print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating LBL file : $LBL_FILE" |
9456401f8
|
265 |
print_error "[${BASENAME}] ERROR : check $ERRORFILE file" |
e6be5137b
|
266 267 268 269 270 271 272 |
exit 1 fi #----------------------------------------------------# # Cut global PLP file depending to LBL segmentations # #----------------------------------------------------# |
135404bcf
|
273 274 |
print_info "[${BASENAME}] Cut PLP file depending to LBL segmentations" 1 print_info "[${BASENAME}] $BIN_PATH/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG" 3 |
e6be5137b
|
275 |
|
0bf609bcc
|
276 |
$SPEERAL_TOOLS/gcep $PLP_FILE $LBL_FILE 500 $PLP_DIR -FSEG $REDIRECTION_OUTPUT |
e6be5137b
|
277 278 279 280 281 282 |
if [ $CHECK -eq 1 ] then check_first_pass_plps_lbl $PLP_DIR $LBL_FILE if [ $? -eq 1 ] then |
135404bcf
|
283 |
print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $PLP wrong number of .plp files" |
9456401f8
|
284 |
print_error "[${BASENAME}] ERROR : check $ERRORFILE file" |
e6be5137b
|
285 286 287 288 289 290 291 292 293 294 295 296 297 |
exit 1 fi fi # change plp files names cd $PLP_DIR; rename -f s/_/#/g *plp rename -f s/#/_/ *plp cd $OLDPWD #---------------------------------------------# # PLP files list depending to acoustic models # #---------------------------------------------# |
135404bcf
|
298 |
print_info "[${BASENAME}] Create PLP list depending of the model" 1 |
e6be5137b
|
299 300 301 302 303 304 305 306 |
# Create a list of plp files find $PLP_DIR -type f -exec basename "{}" .plp \; | sort > $OUTPUT_DIR_BASENAME/plp.lst rm $OUTPUT_DIR_BASENAME/plp_*.lst > /dev/null 2>&1 for (( i=0; $i<${#MTAG[@]} ; i++ )) do a=`grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst` if [ -n "$a" ]; then |
135404bcf
|
307 |
print_info "[${BASENAME}] Creating $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst" 3 |
e6be5137b
|
308 309 310 311 312 313 314 315 316 317 318 |
grep -e "${MTAG[$i]}" $OUTPUT_DIR_BASENAME/plp.lst | sort > $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst fi done #-----------------------# # First Pass (DECODING) # #-----------------------# # # For all AM do decoding # if Check error -> iter on undone decoding (max 1 times) # |
135404bcf
|
319 |
print_info "[${BASENAME}] Launch decoding" 1 |
e6be5137b
|
320 321 322 323 324 325 326 327 |
for (( i=0; $i<${#MTAG[@]} ; i++ )) do redo=1; # nb of try if not all segs is decoded if [ -e $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst ] then todo=$OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst while [ $redo -gt 0 ]; do rm $RES_DIR/*.lock > /dev/null 2>&1 |
135404bcf
|
328 |
print_info "[${BASENAME}] $SPEERAL_BIN $todo $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT" 3 |
e6be5137b
|
329 |
# Run speeral |
0bf609bcc
|
330 |
$SPEERAL_BIN ${todo} $RES_DIR ${SPEERAL_CFG[$i]} -r $PLP_DIR -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock $REDIRECTION_OUTPUT |
e6be5137b
|
331 332 333 334 335 336 337 338 339 340 |
# Check if error if [ $CHECK -eq 1 ] then check_first_pass_output_speeral "${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" "$RES_DIR" # if error if [ $? -eq 1 ] then # rerun redo=$(($redo - 1)); |
135404bcf
|
341 342 |
print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 print_log_file $LOGFILE "WARN : Speeral number of output ERROR ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst" |
e6be5137b
|
343 344 345 346 347 348 |
# new plp list # list .seg done and compare to list of seg to do ls $RES_DIR/*.seg | grep -e "${MTAG[$i]}" | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp diff ${OUTPUT_DIR_BASENAME}/plp_${MODS[$i]}.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" > ${OUTPUT_DIR_BASENAME}/todo.lst rm ${OUTPUT_DIR_BASENAME}/.tmp # log seg to do |
135404bcf
|
349 350 |
print_log_file $LOGFILE "Segs not done :" cat ${OUTPUT_DIR_BASENAME}/todo.lst >> $LOGFILE |
e6be5137b
|
351 |
todo=${OUTPUT_DIR_BASENAME}/todo.lst |
135404bcf
|
352 |
print_warn "[${BASENAME}] Try $redo" 3 |
e6be5137b
|
353 |
fi |
e6be5137b
|
354 355 |
fi done |
e6be5137b
|
356 357 358 359 360 |
rm ${OUTPUT_DIR_BASENAME}/todo.lst > /dev/null 2>&1 #rm $OUTPUT_DIR_BASENAME/plp_${MODS[$i]}.lst rm $RES_DIR/*.lock > /dev/null 2>&1 fi done |
e6be5137b
|
361 |
## Check missing seg and log it |
135404bcf
|
362 363 364 365 366 367 368 |
if [ $CHECK -eq 1 ] then ls $RES_DIR/*.seg | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp todo=$(cat ${OUTPUT_DIR_BASENAME}/plp.lst | wc -l) if [ $todo -eq 0 ]; then todo=1;fi notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) pourcentage=$((($notdone*100)/$todo)) |
e6be5137b
|
369 |
|
135404bcf
|
370 371 372 373 |
if [ $notdone -ne 0 ] then print_error "[${BASENAME}] ERROR : check $ERRORFILE" print_log_file "$ERRORFILE" "ERROR : Segs not done [" |
9456401f8
|
374 |
diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE |
135404bcf
|
375 376 377 378 379 380 |
print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" else print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" fi rm ${OUTPUT_DIR_BASENAME}/.tmp fi |
e6be5137b
|
381 382 383 384 |
#---------------# # Convert res # #---------------# |
135404bcf
|
385 |
print_info "[${BASENAME}] Convert .res into .ctm" 1 |
e6be5137b
|
386 |
# .res => .ctm |
0bf609bcc
|
387 |
$SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.ctm $REDIRECTION_OUTPUT |
135404bcf
|
388 |
print_info "[${BASENAME}] Convert .res into .trs" 1 |
e6be5137b
|
389 390 391 392 393 |
# .res => .trs echo -e "name $AUTHOR fileName $BASENAME fileExt wav segFile $OUTPUT_DIR_BASENAME/$BASENAME.seg" > $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg |
0bf609bcc
|
394 395 |
$SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.trs --trs_config $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg $REDIRECTION_OUTPUT rm $OUTPUT_DIR_BASENAME/$BASENAME.trs_cfg 2> /dev/null |
135404bcf
|
396 |
print_info "[${BASENAME}] Convert .res into .txt" 1 |
e6be5137b
|
397 |
# .res => .txt |
0bf609bcc
|
398 |
$SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out $OUTPUT_DIR_BASENAME/$BASENAME.1pass.txt $REDIRECTION_OUTPUT |
e6be5137b
|
399 |
|
135404bcf
|
400 |
print_info "[${BASENAME}] <= P1 End | $(date +'%d/%m/%y %H:%M:%S')" 1 |
e6be5137b
|
401 402 |
# unlock directory mv "$OUTPUT_DIR_BASENAME/FIRSTPASS.lock" "$OUTPUT_DIR_BASENAME/FIRSTPASS.unlock" |