From b427f103e12f3e8f120ff02a443be418e6f70a49 Mon Sep 17 00:00:00 2001 From: rey jean-Francois Date: Thu, 1 Aug 2013 15:36:32 +0200 Subject: [PATCH] update log info printed --- main_tools/ExploitConfidencePass.sh | 70 ++++++++++++++++++++++------------- main_tools/FirstPass.sh | 10 ++--- main_tools/RecomposePass.sh | 51 ++++++++++++++++---------- main_tools/ScoringRes.sh | 21 ++++++++--- main_tools/SecondPass.sh | 12 +++--- main_tools/ThirdPass.sh | 73 ++++++++++++++++++++++++++----------- tools/scripts/Tools.sh | 6 +-- 7 files changed, 156 insertions(+), 87 deletions(-) diff --git a/main_tools/ExploitConfidencePass.sh b/main_tools/ExploitConfidencePass.sh index 93183dd..c01c8e9 100755 --- a/main_tools/ExploitConfidencePass.sh +++ b/main_tools/ExploitConfidencePass.sh @@ -125,8 +125,8 @@ SHOW_DIR="$OUTPUT_DIR/shows/" SOLR_RES="$OUTPUT_DIR/solr/" EXT_LEX="$OUTPUT_DIR/LEX/" TRIGGER_CONFZONE="$OUTPUT_DIR/trigg/" -LOGFILE="$(dirname $OUTPUT_DIR)/info_exploitconf.log" -ERRORFILE="$(dirname $OUTPUT_DIR)/error_exploitconf.log" +LOGFILE="$OUTPUT_DIR/info_exploitconf.log" +ERRORFILE="$OUTPUT_DIR/error_exploitconf.log" CONFPASS_CONFIG_FILE="$(readlink -e $1)/ConfPass.cfg" if [ -e $CONFPASS_CONFIG_FILE ] @@ -175,6 +175,12 @@ print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ExploitConfPass.cf #---------------# # Check Pass # #---------------# +if [ $( ${RES_CONf_DIR}/*.res 2> /dev/null | wc -l) -eq 0 ] +then + print_error "[${BASENAME}] No Conf Pass res -> exit ExploitConfPass" + if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ConfPass res in $[RES_CONf_DIR}" ;fi + exit 1 +fi #-----------------------# # Segmentation by show # @@ -188,22 +194,22 @@ print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ExploitConfPass.cf print_info "[${BASENAME}] Segmentation by show" 1 # -> to txt -print_info "[${BASENAME}] Create txt from scored res" 2 +print_info "[${BASENAME}] Create txt from scored res" 3 cat ${RES_CONF_DIR}/*.res > $INPUT_DIR/$BASENAME.sctm cat $INPUT_DIR/$BASENAME.seg | $SIGMUND_BIN/myConvert.pl $INPUT_DIR/$BASENAME.sctm $INPUT_DIR/$BASENAME.tmp cat $INPUT_DIR/$BASENAME.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | sed -e "s/_/ /g" | sort -nt 'n' -k '2' > $INPUT_DIR/$BASENAME.txt # -> to tagger + lemme -print_info "[${BASENAME}] Tag pos and lem in txt file" 2 +print_info "[${BASENAME}] Tag pos and lem in txt file" 3 iconv -t ISO_8859-1 $INPUT_DIR/$BASENAME.txt > $INPUT_DIR/$BASENAME.tmp $SIGMUND_BIN/txt2lem.sh $INPUT_DIR/$BASENAME.tmp $INPUT_DIR/$BASENAME.taglem # merge sctm and taglem -print_info "[${BASENAME}] Merge scored ctm with tag pos and lem file" 2 +print_info "[${BASENAME}] Merge scored ctm with tag pos and lem file" 3 cat $INPUT_DIR/$BASENAME.sctm | $SCRIPT_PATH/BdlexUC.pl ${RULES}/basic -f | iconv -t ISO_8859-1 | $SCRIPT_PATH/scoredCtmAndTaggedLem2All.pl $INPUT_DIR/$BASENAME.taglem > $INPUT_DIR/$BASENAME.ctl # -> new seg -print_info "[${BASENAME}] Create xml file and run Topic Seg" 2 +print_info "[${BASENAME}] Create xml file and run Topic Seg" 3 $SIGMUND_BIN/tagLem2xml.pl $INPUT_DIR/$BASENAME.taglem $INPUT_DIR/$BASENAME.doc.xml rm $INPUT_DIR/$BASENAME.tmp #$INPUT_DIR/$BASENAME.taglem @@ -215,11 +221,16 @@ rm 0.xml $INPUT_DIR/show.seg if [ $CHECK -eq 1 ] then - if [ ! -s $INPUT_DIR/$BASENAME.show.seg ];then echo -e "[${BASENAME}] ERROR : no Topic segmentation" >> $ERRORFILE; fi + if [ ! -s $INPUT_DIR/$BASENAME.show.seg ] + then + print_error "[${BASENAME}] No Topic segmentation ! " + print_error "[${BASENAME}] Check $ERRORFILE " + print_log_file "$ERRORFILE" "No Topic segmentation in ${BASENAME}.show.seg" + fi fi # Segment ctm into several show files and create a seg list by show -print_info "[${BASENAME}] Segment ctm into show files and a seg list by show" 2 +print_info "[${BASENAME}] Segment ctm into show files and a seg list by show" 1 $SCRIPT_PATH/ctm2show.pl $INPUT_DIR/$BASENAME.ctl $INPUT_DIR/$BASENAME.show.seg $SHOW_DIR #-----------------------------------------------------------# @@ -230,7 +241,7 @@ $SCRIPT_PATH/ctm2show.pl $INPUT_DIR/$BASENAME.ctl $INPUT_DIR/$BASENAME.show.seg # concat word + add date 2 day before and after the show # # query document & multimedia # #-----------------------------------------------------------# -print_info "[${BASENAME}] Create SOLR queries and ASK SOLR" 1 +print_info "[${BASENAME}] Create SOLR queries and ask SOLR" 1 for show in $(ls $SHOW_DIR/*.ctm) do bn=$(basename $show .ctm) @@ -244,6 +255,7 @@ do if [ $(wc -w "$SHOW_DIR/$bn.queries" | cut -f1 -d' ') -gt 0 ]; then query=$(cat $SHOW_DIR/$bn.queries)"&fq=docDate:[$datePattern]" echo $query > $SHOW_DIR/$bn.queries + prnt_info "python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp" 3 python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp cat $SOLR_RES/$bn.keywords.tmp | sort -u > $SOLR_RES/$bn.keywords cat $SOLR_RES/$bn.txt.tmp | sort -u > $SOLR_RES/$bn.txt @@ -254,7 +266,8 @@ do then if [ ! -e $SOLR_RES/$bn.keywords ] || [ ! -e $SOLR_RES/$bn.txt ] then - print_warn "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 1 + print_warn "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 2 + print_log_file "$LOGFILE" "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" fi fi @@ -278,7 +291,7 @@ do # Tokenize & produce coverage report # Use filter you need # - print_info "[${BASENAME}] keywords filtering and produce coverage report" 2 + print_info "[${BASENAME}] keywords filtering and produce coverage report" 3 # Default filter cat $i | $SCRIPT_PATH/CleanFilter.sh | ${SCRIPT_PATH}/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t |\ $SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok @@ -290,13 +303,13 @@ do # Extract "real" OOV and phonetize them # -> petit filtrage persoo pour eviter d'avoir trop de bruits # - print_info "[${BASENAME}] Extract OOV and phonetize them" 2 + print_info "[${BASENAME}] Extract OOV and phonetize them" 3 ${SCRIPT_PATH}/FindNormRules.pl $SOLR_RES/${basename}_tmp_report/report.oov $LEXICON.bdlex_tok | cut -f3 | grep -v "#" | grep -v "^[A-Z]\+$" | grep -v "^[0-9]" | grep --perl-regex -v "^([a-z']){1,3}$" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | iconv -t ISO_8859-1 -f UTF-8 | ${LIA_LTBOX}/lia_phon/script/lia_lex2phon_variante | grep -v "core dumped" | cut -d"[" -f1 | sort -u | ${SCRIPT_PATH}/PhonFormatter.pl | iconv -f ISO_8859-1 -t UTF-8 | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $SOLR_RES/${basename}.phon_oov # # Search INVOC & OOV in the current lattice # - print_info "[${BASENAME}] Search INVOC and OOV in the current lattice" 2 + print_info "[${BASENAME}] Search INVOC and OOV in the current lattice" 3 cat $SOLR_RES/${basename}_tmp_report/report.invoc | grep -v "\b0" | cut -f1 | grep -v --perl-regex -v "^[a-zA-Z']{1,3}$" | grep -v --perl-regex "^[a-zA-Z0-9]{1,3}$" | grep -v "" | grep -v "" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $TRIGGER_CONFZONE/$basename.tosearch cat $SOLR_RES/${basename}.phon_oov | cut -f1 >> $TRIGGER_CONFZONE/$basename.tosearch @@ -326,7 +339,7 @@ mkdir -p $EXT_LEX/tmp/txt # # Collect the acousticly found oov and their phonetisation # -print_info "[${BASENAME}] Get all OOV and retrieve all phonetisation" 2 +print_info "[${BASENAME}] Get all OOV and retrieve all phonetisation" 3 for i in `ls $SOLR_RES/*.phon_oov` do basename=`basename $i .phon_oov` @@ -354,7 +367,7 @@ done # # Merge OOVs and their phonetisation # -print_info "[${BASENAME}] Merge OOV and their phonetisation" 2 +print_info "[${BASENAME}] Merge OOV and their phonetisation" 3 lexname=$(basename $LEXICON) cat $EXT_LEX/*.oov_acousticlyfound_phon | sort -u > $EXT_LEX/final/all.oov_acousticlyfound_phon cat $EXT_LEX/*.oov_acousticlyfound | sort -u | grep --perl-regex -v "^([a-z']){3}$" > $EXT_LEX/final/all.oov_acousticlyfound @@ -377,7 +390,7 @@ cat $SOLR_RES/*.txt | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrection # - Expected format : # # -print_info "[${BASENAME}] Construct map file" 2 +print_info "[${BASENAME}] Construct map file" 3 rm -f $EXT_LEX/final/${lexname}_ext.map 2>/dev/null rm -f $EXT_LEX/final/${lexname}.unvalid_oov 2>/dev/null @@ -391,19 +404,21 @@ do # # Try to collect text containing the oov word # + print_info "[${BASENAME}] Collect text containing the oov" 3 cat $EXT_LEX/final/all.bdlex_txt | grep --perl-regex " $oov " | $SCRIPT_PATH/NbMaxWordsFilter.pl 40 |uniq > $EXT_LEX/tmp/txt/$oov.bdlex_txt if [ -f $EXT_LEX/tmp/txt/$oov.bdlex_txt ]; then nbWords=`wc -l $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` if [ $nbWords -eq 0 ]; then - echo "[${BASENAME}] UNVALID OOV: $oov => $nbWords occurrences" + print_warn "[${BASENAME}] UNVALID OOV: $oov => $nbWords occurrences" 2 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov else # # Find a candidate in a filtred invoc lexicon => a candidate which maximize the ppl in the overall txt collected # #echo "$/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $LEXICON.bdlex_tok $EXT_LEX/tmp/txt/$oov.bdlex_txt" + print_info `$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` 3 candidate=`$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` - if [ ! x$candidate = "x" ]; then + if [ ! "$candidate" == "" ]; then grep --perl-regex "^$oov\t" $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/tmp/$oov.phon while read phonLine do @@ -411,12 +426,12 @@ do echo "$phonLine" | sed "s|\t|\t$candidate\t|" >> $EXT_LEX/final/${lexname}_ext.map done < $EXT_LEX/tmp/$oov.phon else - echo "[${BASENAME}] UNVALID OOV: $oov => no availaible Candidate word in LM" + print_warn "[${BASENAME}] UNVALID OOV: $oov => no availaible Candidate word in LM" 2 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov fi fi else - echo "[${BASENAME}] UNVALID OOV: $oov" + print_warn "[${BASENAME}] UNVALID OOV: $oov" 2 echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov fi done < $EXT_LEX/final/all.oov_acousticlyfound @@ -429,7 +444,7 @@ lexname=`basename $LEXICON` # # Build the final trigger file # -print_info "[${BASENAME}] Clean trigg files" 2 +print_info "[${BASENAME}] Clean trigg files" 3 mkdir -p $TRIGGER_CONFZONE/speeral/ 2> /dev/null mkdir -p $EXT_LEX/speeral/ 2> /dev/null for i in `ls $TRIGGER_CONFZONE/*.trigg` @@ -440,7 +455,8 @@ done # # Compile the speeral extended lexicon # -print_info "[${BASENAME}] Compile Speeral extended lexicon" 2 +print_info "[${BASENAME}] Compile Speeral extended lexicon" 3 +print_info "$SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext" 3 $SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext if [ $CHECK -eq 1 ] @@ -448,7 +464,10 @@ then check_exploitconfpass_lex_check "${EXT_LEX}/speeral/${lexname}_ext" if [ $? -eq 1 ] then - echo -e "[${BASENAME}] ERROR : Building Speeral Lexicon $INPUT_DIR " >> $ERRORFILE + print_error "[${BASENAME}] Building Speeral Lexicon $INPUT_DIR -> exit" + print_error "[${BASENAME}] Check $ERRORFILE" + print_log_file $ERRORFILE "ERROR : Building Speeral Lexicon $INPUT_DIR" + print_log_file $ERRORFILE "ERROR : ${EXT_LEX}/speeral/${lexname}_ext Empty after buildmappedbinode ?" exit 1; fi fi @@ -458,10 +477,9 @@ fi # CLOSE # #-------# # Seem OK -print_info "[${BASENAME}] <= End $BASENAME Solr | $(date +'%d/%m/%y %H:%M:%S')" 1 -echo -e "[${BASENAME}] #Solr $BASENAME " >> $LOGFILE +print_info "[${BASENAME}] <= ExploitConfidencePass End | $(date +'%d/%m/%y %H:%M:%S')" 1 -# unlock directory +# unlok directory mv "$OUTPUT_DIR/EXPLOITCONFPASS.lock" "$OUTPUT_DIR/EXPLOITCONFPASS.unlock" diff --git a/main_tools/FirstPass.sh b/main_tools/FirstPass.sh index 75ee49f..673c3b0 100755 --- a/main_tools/FirstPass.sh +++ b/main_tools/FirstPass.sh @@ -115,7 +115,7 @@ then print_info "[${BASENAME}] => P1 start | $(date +'%d/%m/%y %H:%M:%S')" 1 print_info "[${BASENAME}] $WAV_FILE OK" 2 else - print_error "can't find $1 OR file is empty" + print_error " Can't find $1 OR file is empty" exit 1 fi @@ -247,7 +247,7 @@ java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FIL if [ $CHECK -eq 1 ] && ( [ ! -e $SEG_FILE ] || [ -z $SEG_FILE ] ) then print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating SEG file : $SEG_FILE" - print_error "[${BASENAME}] ERROR : check $ERRORFILE file" + print_error "[${BASENAME}] Check $ERRORFILE file" exit 1 fi @@ -260,7 +260,7 @@ cat $SEG_FILE | grep -v ";;" | cut -f3,4,5,8 -d" " | tr " " "#" | sort -k1 -n | if [ $CHECK -eq 1 ] && ( [ ! -e $LBL_FILE ] || [ -z $LBL_FILE ] ) then print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating LBL file : $LBL_FILE" - print_error "[${BASENAME}] ERROR : check $ERRORFILE file" + print_error "[${BASENAME}] Check $ERRORFILE file" exit 1 fi @@ -279,7 +279,7 @@ then if [ $? -eq 1 ] then print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $PLP wrong number of .plp files" - print_error "[${BASENAME}] ERROR : check $ERRORFILE file" + print_error "[${BASENAME}] Check $ERRORFILE file" exit 1 fi fi @@ -368,7 +368,7 @@ then if [ $notdone -ne 0 ] then - print_error "[${BASENAME}] ERROR : check $ERRORFILE" + print_error "[${BASENAME}] Check $ERRORFILE" print_log_file "$ERRORFILE" "ERROR : Segs not done [" diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" diff --git a/main_tools/RecomposePass.sh b/main_tools/RecomposePass.sh index 72d8f3c..530e7e5 100755 --- a/main_tools/RecomposePass.sh +++ b/main_tools/RecomposePass.sh @@ -59,7 +59,7 @@ do VERBOSE=$OPTARG ;; :) - echo "Option -$OPTARG requires an argument." >&2 + echo "Option -$OPTARG requires an argument." > /dev/null exit 1 ;; \?) @@ -100,12 +100,18 @@ fi #-------------# # GLOBAL VARS # #-------------# +BASENAME=$(basename $PASS_DIR) +OUTPUT_DIR_BASENAME=$PASS_DIR +RES_DIR="$PASS_DIR/res_all" +LOGFILE=$PASS_DIR"/info_recpass.log" +ERRORFILE=$PASS_DIR"/error_recpass.log" + THIRDPASS_CONFIG_FILE="$PASS_DIR/ThirdPass.cfg" if [ -e $THIRDPASS_CONFIG_FILE ] then RES_P3=$(cat $THIRDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") else - print_warn "can't find $THIRDPASS_CONFIG_FILE file" 1 + print_warn "[${BASENAME}] can't find $THIRDPASS_CONFIG_FILE file" 2 #exit 1 fi SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg" @@ -113,7 +119,7 @@ if [ -e $SECONDPASS_CONFIG_FILE ] then RES_P2=$(cat $SECONDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") else - print_warn "can't find $SECONDPASS_CONFIG_FILE file" 1 + print_warn "[${BASENAME}] can't find $SECONDPASS_CONFIG_FILE file" 2 #exit 1 fi FIRSTPASS_CONFIG_FILE="$PASS_DIR/FirstPass.cfg" @@ -121,33 +127,35 @@ if [ -e $FIRSTPASS_CONFIG_FILE ] then RES_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=") else - print_warn "can't find $FIRSTPASS_CONFIG_FILE file" + print_warn "[${BASENAME}] can't find $FIRSTPASS_CONFIG_FILE file" 2 #exit 1 fi -if ( [ -z "$RES_P1" ] && [ -z "$RES_P2" ] ) || [ -z $RES_P3 ]; then echo "ERROR no configuration pass files in $PASS_DIR"; exit 1; fi +if ( [ -z "$RES_P1" ] && [ -z "$RES_P2" ] ) || [ -z $RES_P3 ] +then + print_error "[${BASENAME}] No configuration pass files in $PASS_DIR -> exit" + if [ $CHECK -eq 1 ]; then print_log_file $ERROFILE "No configuration pass files in $PASS_DIR -> exit";fi + exit 1 +fi -BASENAME=$(basename $PASS_DIR) -OUTPUT_DIR_BASENAME=$PASS_DIR -RES_DIR="$PASS_DIR/res_all" -LOGFILE=$(dirname $PASS_DIR)"/info_recpass.log" -ERRORFILE=$(dirname $PASS_DIR)"/error_recpass.log" +print_info "[${BASENAME}] => RecomposePass start | $(date +'%d/%m/%y %H:%M:%S')" 1 #------------------# # Create WORKSPACE # #------------------# # Lock directory -if [ -e $OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi +if [ -e $OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock ] && [ $RERUN -eq 0 ] +then + print_warn "[${BASENAME}] RECOMPOSEPASS is locked -> exit" 2 + exit 1 +fi rm "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" > /dev/null 2>&1 touch "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" > /dev/null 2>&1 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi -mkdir -p $RES_DIR -print_info "Make directory $RES_DIR" 1 - -rm $RES_DIR 2> /dev/null -mkdir -p $RES_DIR +mkdir -p $RES_DIR 2> /dev/null +print_info "Make directory $RES_DIR" 3 #--------------------# # Save configuration # @@ -157,12 +165,12 @@ echo "RES_ALL=$RES_DIR" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg echo "RES_P1=$RES_P1" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg echo "RES_P2=$RES_P2" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg echo "RES_P3=$RES_P3" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg -print_info "save config in $OUTPUT_DIR_BASENAME/RecomposePass.cfg" 1 +print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/RecomposePass.cfg" 1 #--------------------------------------------------# # Recompose Pass using previous pass # #--------------------------------------------------# -print_info "Launch Recompose Pass" 1 +print_info "[${BASENAME}] Launch Recompose Pass" 1 ## Check seg in p3 cp $RES_P3/*.res $RES_DIR @@ -181,23 +189,26 @@ do cp $RES_P1/$res.res $RES_DIR done -rm ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp ${OUTPUT_DIR_BASENAME}/res_all.tmp +rm ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp ${OUTPUT_DIR_BASENAME}/res_all.tmp 2> /dev/null #---------------# # Convert res # #---------------# +print_info "[${BASENAME}] Convert .res into .ctm" 1 # .res => .ctm $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.ctm +print_info "[${BASENAME}] Convert .res into .trs" 1 # .res => .trs echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg +print_info "[${BASENAME}] Convert .res into .txt" 1 # .res => .txt $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.txt -print_info "<= End $BASENAME RecomposePass | $(date +'%d/%m/%y %H:%M:%S')" 1 +print_info "[${BASENAME}] <= RecomposePass End | $(date +'%d/%m/%y %H:%M:%S')" 1 # unlock directory mv "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" diff --git a/main_tools/ScoringRes.sh b/main_tools/ScoringRes.sh index 4f56151..0719624 100755 --- a/main_tools/ScoringRes.sh +++ b/main_tools/ScoringRes.sh @@ -101,15 +101,21 @@ fi #-------------# BASENAME=$(basename $PASS_DIR) SCORING_DIR=$PASS_DIR/scoring -LOGFILE="$(dirname $PASS_DIR)/info_scoring.log" -ERRORFILE="$(dirname $PASS_DIR)/error_scoring.log" +LOGFILE="$PASS_DIR/info_scoring.log" +ERRORFILE="$PASS_DIR/error_scoring.log" + +print_info "[${BASENAME}] => ScoringRes start | $(date +'%d/%m/%y %H:%M:%S')" 1 #------------------# # Create WORKSPACE # #------------------# # Lock directory -if [ -e $PASS_DIR/*.lock ]; then echo "$PASS_DIR is lock"; exit 1;fi +if [ -e $PASS_DIR/*.lock ] +then + print_warn "[${BASENAME}] Directory is locked -> exit" 2; + exit 1 +fi rm -r $SCORING_DIR > /dev/null 2>&1 mkdir -p $SCORING_DIR @@ -117,18 +123,20 @@ print_info "Make directory $SCORING_DIR" 1 if [ ! -e $PASS_DIR/$BASENAME.SRT ] then - echo -e "no SRT file for $BASENAME\n" + print_warn "[${BASENAME}] No SRT file -> exit" 2 exit 0; else SRT_FILE=$PASS_DIR/$BASENAME.SRT + print_info "[${BASENAME}] SRT file : $BASENAME.SRT" 1 fi if [ ! -e $PASS_DIR/$BASENAME.1pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.2pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.3pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.allpass.ctm ] then - echo -e "no CTM files for $BASENAME\n"; + print_error "[${BASENAME}] No CTM files -> exit\n"; exit 1; fi +print_info "[${BASENAME}] Create stm file from SRT" 1 i=$((-3)) while read line do @@ -146,11 +154,14 @@ cat $SCORING_DIR/$BASENAME.tmp.txt | sed -e "s|\n| |g" | $SCRIPT_PATH/CleanFilte $SCRIPT_PATH/srt2stm.pl $SCORING_DIR/$BASENAME.tmp2.txt > "$SCORING_DIR/$BASENAME.stm" rm $SCORING_DIR/$BASENAME.tmp.txt $SCORING_DIR/$BASENAME.tmp2.txt +print_info "Scoring using sclite" 1 for ctmfile in $(ls $PASS_DIR/*.ctm) do cp $ctmfile $SCORING_DIR/ + print_info "$BIN_PATH/sclite -D -F -r $SCORING_DIR/$BASENAME.stm stm -h $SCORING_DIR/$(basename $ctmfile) ctm -o all" 3 $BIN_PATH/sclite -D -F -r $SCORING_DIR/$BASENAME.stm stm -h $SCORING_DIR/$(basename $ctmfile) ctm -o all done +print_info "[${BASENAME}] <= ScoringRes End | $(date +'%d/%m/%y %H:%M:%S')" 1 diff --git a/main_tools/SecondPass.sh b/main_tools/SecondPass.sh index 14fb2c6..869a39f 100755 --- a/main_tools/SecondPass.sh +++ b/main_tools/SecondPass.sh @@ -107,12 +107,10 @@ if [ -e $1 ] && [ -d $1 ] then FIRSTPASS_DIR=$(readlink -e $1) else - print_error "can't find $1 directory" + print_error "Can't find $1 directory" exit 1 fi -print_info "[${BASENAME}] => P2 start | $(date +'%d/%m/%y %H:%M:%S')" 1 - #-------------# # GLOBAL VARS # #-------------# @@ -129,7 +127,7 @@ then LBL_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "LBL_FILE=" | cut -f2 -d"=") RES_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "RES_DIR=" | cut -f2 -d"=") else - print_error "can't find $FIRSTPASS_CONFIG_FILE file" + print_error "Can't find $FIRSTPASS_CONFIG_FILE file" print_error "exit" exit 1 fi @@ -139,6 +137,8 @@ RES_DIR=$OUTPUT_DIR_BASENAME"/res_p2" LOGFILE="$OUTPUT_DIR_BASENAME/info_p2.log" ERRORFILE="$OUTPUT_DIR_BASENAME/error_p2.log" +print_info "[${BASENAME}] => P2 start | $(date +'%d/%m/%y %H:%M:%S')" 1 + #------------------# # Create WORKSPACE # #------------------# @@ -304,12 +304,12 @@ then pourcentage=$((($notdone*100)/$todo)) if [ $notdone -ne 0 ] then - print_error "[${BASENAME}] ERROR : check $ERRORFILE" + print_error "[${BASENAME}] Check $ERRORFILE" print_log_file "$ERRORFILE" "ERROR : Treil not done [" diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" else - print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" + print_log_file "$LOGFILE" "P2 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" fi rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1 fi diff --git a/main_tools/ThirdPass.sh b/main_tools/ThirdPass.sh index bf3bffd..77fa438 100755 --- a/main_tools/ThirdPass.sh +++ b/main_tools/ThirdPass.sh @@ -70,7 +70,7 @@ do RERUN=1 ;; :) - echo "Option -$OPTARG requires an argument." >&2 + echo "Option -$OPTARG requires an argument." > /dev/null exit 1 ;; \?) @@ -103,11 +103,14 @@ shift $((OPTIND-1)) if [ -e $1 ] && [ -d $1 ] then PASS_DIR=$(readlink -e $1) + BASENAME=$(basename $PASS_DIR) else print_error "can't find $1 directory" exit 1 fi +print_info "[${BASENAME}] => P3 start | $(date +'%d/%m/%y %H:%M:%S')" 1 + #-------------# # GLOBAL VARS # #-------------# @@ -121,62 +124,67 @@ then PLP_DIR_P1="" HMM="" else - print_error "can't find $EXPLOITCONFPASS_CONFIG_FILE file" + print_error "[${BASENAME}] Can't find $EXPLOITCONFPASS_CONFIG_FILE file" + print_warn "[${BASENAME}] Will use default directories and files" 2 #exit 1 TRIGGER_SPEERAL=$PASS_DIR/trigg/speeral/ LEX_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext LEX_BINODE_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext.bin fi + SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg" if [ -e $SECONDPASS_CONFIG_FILE ] then LST=$(cat $SECONDPASS_CONFIG_FILE | grep "^LST=" | cut -f2 -d"=") HMM=$(cat $SECONDPASS_CONFIG_FILE | grep "^HMM=" | cut -f2 -d"=") PLP_DIR_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_DIR_P1=" | cut -f2 -d"=") + PLP_FILE_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_FILE_P1=" | cut -f2 -d"=") else - print_error "can't find $SECONDPASS_CONFIG_FILE file" + print_error "[${BASENAME}] Can't find $SECONDPASS_CONFIG_FILE file" + print_warn "[${BASENAME}] Will use default directories and files" 2 #exit 1 LST=$PASS_DIR/lists HMM=$PASS_DIR/hmm PLP_DIR_P1=$PASS_DIR/PLP + PLP_FILE_P1=$PASS_DIR/plp.lst fi -BASENAME=$(basename $PASS_DIR) OUTPUT_DIR_BASENAME=$PASS_DIR RES_DIR="$PASS_DIR/res_p3" -LOGFILE=$(dirname $PASS_DIR)"/info_p3.log" -ERRORFILE=$(dirname $PASS_DIR)"/error_p3.log" +LOGFILE=$PASS_DIR"/info_p3.log" +ERRORFILE=$PASS_DIR"/error_p3.log" #------------------# # Create WORKSPACE # #------------------# # Lock directory -if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi +if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] THIRDPASS is locked -> exit" 2; exit 1; fi rm "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" > /dev/null 2>&1 touch "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" > /dev/null 2>&1 if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi mkdir -p $RES_DIR -print_info "Make directory $RES_DIR" 1 +print_info "[${BASENAME}] Make directory $RES_DIR" 3 #--------------------# # Save configuration # #--------------------# cp $THIRDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/ThirdPass.cfg echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/ThirdPass.cfg -print_info "save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1 +print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ThirdPass.cfg" 1 #--------------------------------------------------# # Third Pass using trigger file (DECODING) # #--------------------------------------------------# -print_info "Launch Third Pass" 1 +print_info "[${BASENAME}] Launch Third Pass" 1 ## Generate speeral config file adding trigger rep cat $SPEERAL_CFG_PATH/$SPEERAL_CFG_FILE | sed -e "s|[^<]*|$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext|g" \ | sed -e "s|[^<]*|$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext.bin|g" \ | sed -e "s|[^<]*|$TRIGGER_SPEERAL|g" > $OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml SPEERAL_THIRD_CFG=$OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml +print_info "[${BASENAME}] Third Pass Speeral Config file ${SPEERAL_THIRD_CFG}" 3 # for all speaker for lspeaker in $(ls $LST/*.lst) @@ -193,10 +201,11 @@ do ## Speeral decoding if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ] then - print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3 + print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock else - print_warn "$HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2 + print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2 + print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3 $SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock fi @@ -205,12 +214,14 @@ do check_third_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR" if [ $? -eq 1 ] then - echo -e "ERROR : Speeral $LST/$speaker.lst\n[" >> $ERRORFILE + + print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2 + print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst" ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp - diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $ERRORFILE - echo -e "] " >> $ERRORFILE + print_log_file $LOGFILE "Segs (and treil) not done :\n[" + diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $LOGFILE + print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]" rm ${OUTPUT_DIR_BASENAME}/.tmp - #exit 1 fi fi break @@ -220,27 +231,45 @@ do done ## Check missing seg and log it -ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp -echo -e "$BASENAME P3 END\n[" >> $LOGFILE -diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $LOGFILE -echo -e "] $BASENAME" >> $LOGFILE -rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1 + +## Check missing seg and log it +if [ $CHECK -eq 1 ] +then + ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp + todo=$(cat ${PLP_FILE_P1} | wc -l) + if [ $todo -eq 0 ]; then todo=1;fi + notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l))) + pourcentage=$((($notdone*100)/$todo)) + if [ $notdone -ne 0 ] + then + print_error "[${BASENAME}] Check $ERRORFILE" + print_log_file "$ERRORFILE" "ERROR : Treil not done [" + diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE + print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME" + else + print_log_file "$LOGFILE" "P3 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')" + fi + rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1 +fi #---------------# # Convert res # #---------------# +print_info "[${BASENAME}] Convert .res into .ctm" 1 # .res => .ctm $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.ctm +print_info "[${BASENAME}] Convert .res into .trs" 1 # .res => .trs echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1 +print_info "[${BASENAME}] Convert .res into .txt" 1 # .res => .txt $SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.txt -print_info "<= End $BASENAME P3 | $(date +'%d/%m/%y %H:%M:%S')" 1 +print_info "[${BASENAME}] <= P3 End | $(date +'%d/%m/%y %H:%M:%S')" 1 # unlock directory mv "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" diff --git a/tools/scripts/Tools.sh b/tools/scripts/Tools.sh index 4ff04de..daf034b 100755 --- a/tools/scripts/Tools.sh +++ b/tools/scripts/Tools.sh @@ -29,13 +29,13 @@ function print_message() if [ $# -eq 3 ] then # print error - if [ $1 -eq $ERROR ]; then echo -ne "###ERROR : $3\n" > /dev/stderr; exit 0; fi + if [ $1 -eq $ERROR ]; then echo -ne "###ERROR : $3\n" > /dev/stderr; return 0; fi # check verbose level if [ $2 -le $VERBOSE ] then #for (( dec=1 ; $dec<$2 ; dec++ )); do echo -ne "\t"; done # print warn - if [ $1 = $WARN ]; then echo -ne "##WARNING : $3\n"; exit 0; fi + if [ $1 = $WARN ]; then echo -ne "##WARNING : $3\n"; return 0; fi # default info echo -ne "#INFO : $3\n" fi @@ -82,6 +82,6 @@ print_log_file() { if [ $# -eq 2 ] then - echo -e "$2" >> $1 + echo -e "$2" >> "$1" fi } -- 1.8.2.3