diff --git a/main_tools/ExploitConfidencePass.sh b/main_tools/ExploitConfidencePass.sh
index 93183dd..c01c8e9 100755
--- a/main_tools/ExploitConfidencePass.sh
+++ b/main_tools/ExploitConfidencePass.sh
@@ -125,8 +125,8 @@ SHOW_DIR="$OUTPUT_DIR/shows/"
SOLR_RES="$OUTPUT_DIR/solr/"
EXT_LEX="$OUTPUT_DIR/LEX/"
TRIGGER_CONFZONE="$OUTPUT_DIR/trigg/"
-LOGFILE="$(dirname $OUTPUT_DIR)/info_exploitconf.log"
-ERRORFILE="$(dirname $OUTPUT_DIR)/error_exploitconf.log"
+LOGFILE="$OUTPUT_DIR/info_exploitconf.log"
+ERRORFILE="$OUTPUT_DIR/error_exploitconf.log"
CONFPASS_CONFIG_FILE="$(readlink -e $1)/ConfPass.cfg"
if [ -e $CONFPASS_CONFIG_FILE ]
@@ -175,6 +175,12 @@ print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ExploitConfPass.cf
#---------------#
# Check Pass #
#---------------#
+if [ $( ${RES_CONf_DIR}/*.res 2> /dev/null | wc -l) -eq 0 ]
+then
+ print_error "[${BASENAME}] No Conf Pass res -> exit ExploitConfPass"
+ if [ $CHECK -eq 1 ]; then print_log_file $ERRORFILE "No ConfPass res in $[RES_CONf_DIR}" ;fi
+ exit 1
+fi
#-----------------------#
# Segmentation by show #
@@ -188,22 +194,22 @@ print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ExploitConfPass.cf
print_info "[${BASENAME}] Segmentation by show" 1
# -> to txt
-print_info "[${BASENAME}] Create txt from scored res" 2
+print_info "[${BASENAME}] Create txt from scored res" 3
cat ${RES_CONF_DIR}/*.res > $INPUT_DIR/$BASENAME.sctm
cat $INPUT_DIR/$BASENAME.seg | $SIGMUND_BIN/myConvert.pl $INPUT_DIR/$BASENAME.sctm $INPUT_DIR/$BASENAME.tmp
cat $INPUT_DIR/$BASENAME.tmp | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | sed -e "s/_/ /g" | sort -nt 'n' -k '2' > $INPUT_DIR/$BASENAME.txt
# -> to tagger + lemme
-print_info "[${BASENAME}] Tag pos and lem in txt file" 2
+print_info "[${BASENAME}] Tag pos and lem in txt file" 3
iconv -t ISO_8859-1 $INPUT_DIR/$BASENAME.txt > $INPUT_DIR/$BASENAME.tmp
$SIGMUND_BIN/txt2lem.sh $INPUT_DIR/$BASENAME.tmp $INPUT_DIR/$BASENAME.taglem
# merge sctm and taglem
-print_info "[${BASENAME}] Merge scored ctm with tag pos and lem file" 2
+print_info "[${BASENAME}] Merge scored ctm with tag pos and lem file" 3
cat $INPUT_DIR/$BASENAME.sctm | $SCRIPT_PATH/BdlexUC.pl ${RULES}/basic -f | iconv -t ISO_8859-1 | $SCRIPT_PATH/scoredCtmAndTaggedLem2All.pl $INPUT_DIR/$BASENAME.taglem > $INPUT_DIR/$BASENAME.ctl
# -> new seg
-print_info "[${BASENAME}] Create xml file and run Topic Seg" 2
+print_info "[${BASENAME}] Create xml file and run Topic Seg" 3
$SIGMUND_BIN/tagLem2xml.pl $INPUT_DIR/$BASENAME.taglem $INPUT_DIR/$BASENAME.doc.xml
rm $INPUT_DIR/$BASENAME.tmp #$INPUT_DIR/$BASENAME.taglem
@@ -215,11 +221,16 @@ rm 0.xml $INPUT_DIR/show.seg
if [ $CHECK -eq 1 ]
then
- if [ ! -s $INPUT_DIR/$BASENAME.show.seg ];then echo -e "[${BASENAME}] ERROR : no Topic segmentation" >> $ERRORFILE; fi
+ if [ ! -s $INPUT_DIR/$BASENAME.show.seg ]
+ then
+ print_error "[${BASENAME}] No Topic segmentation ! "
+ print_error "[${BASENAME}] Check $ERRORFILE "
+ print_log_file "$ERRORFILE" "No Topic segmentation in ${BASENAME}.show.seg"
+ fi
fi
# Segment ctm into several show files and create a seg list by show
-print_info "[${BASENAME}] Segment ctm into show files and a seg list by show" 2
+print_info "[${BASENAME}] Segment ctm into show files and a seg list by show" 1
$SCRIPT_PATH/ctm2show.pl $INPUT_DIR/$BASENAME.ctl $INPUT_DIR/$BASENAME.show.seg $SHOW_DIR
#-----------------------------------------------------------#
@@ -230,7 +241,7 @@ $SCRIPT_PATH/ctm2show.pl $INPUT_DIR/$BASENAME.ctl $INPUT_DIR/$BASENAME.show.seg
# concat word + add date 2 day before and after the show #
# query document & multimedia #
#-----------------------------------------------------------#
-print_info "[${BASENAME}] Create SOLR queries and ASK SOLR" 1
+print_info "[${BASENAME}] Create SOLR queries and ask SOLR" 1
for show in $(ls $SHOW_DIR/*.ctm)
do
bn=$(basename $show .ctm)
@@ -244,6 +255,7 @@ do
if [ $(wc -w "$SHOW_DIR/$bn.queries" | cut -f1 -d' ') -gt 0 ]; then
query=$(cat $SHOW_DIR/$bn.queries)"&fq=docDate:[$datePattern]"
echo $query > $SHOW_DIR/$bn.queries
+ prnt_info "python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp" 3
python $SCRIPT_PATH/ProcessSOLRQueries.py $SHOW_DIR/$bn.queries $SOLR_RES/$bn.keywords.tmp $SOLR_RES/$bn.txt.tmp
cat $SOLR_RES/$bn.keywords.tmp | sort -u > $SOLR_RES/$bn.keywords
cat $SOLR_RES/$bn.txt.tmp | sort -u > $SOLR_RES/$bn.txt
@@ -254,7 +266,8 @@ do
then
if [ ! -e $SOLR_RES/$bn.keywords ] || [ ! -e $SOLR_RES/$bn.txt ]
then
- print_warn "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 1
+ print_warn "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !" 2
+ print_log_file "$LOGFILE" "$bn.keywords and $bn.txt are empty !\nMaybe SOLR server is down !"
fi
fi
@@ -278,7 +291,7 @@ do
# Tokenize & produce coverage report
# Use filter you need
#
- print_info "[${BASENAME}] keywords filtering and produce coverage report" 2
+ print_info "[${BASENAME}] keywords filtering and produce coverage report" 3
# Default filter
cat $i | $SCRIPT_PATH/CleanFilter.sh | ${SCRIPT_PATH}/ApplyCorrectionRules.pl ${LEXICON}.regex | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t |\
$SCRIPT_PATH/CoverageReportMaker.pl --out $SOLR_RES/${basename}_tmp_report $LEXICON.bdlex_tok
@@ -290,13 +303,13 @@ do
# Extract "real" OOV and phonetize them
# -> petit filtrage persoo pour eviter d'avoir trop de bruits
#
- print_info "[${BASENAME}] Extract OOV and phonetize them" 2
+ print_info "[${BASENAME}] Extract OOV and phonetize them" 3
${SCRIPT_PATH}/FindNormRules.pl $SOLR_RES/${basename}_tmp_report/report.oov $LEXICON.bdlex_tok | cut -f3 | grep -v "#" | grep -v "^[A-Z]\+$" | grep -v "^[0-9]" | grep --perl-regex -v "^([a-z']){1,3}$" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -f | iconv -t ISO_8859-1 -f UTF-8 | ${LIA_LTBOX}/lia_phon/script/lia_lex2phon_variante | grep -v "core dumped" | cut -d"[" -f1 | sort -u | ${SCRIPT_PATH}/PhonFormatter.pl | iconv -f ISO_8859-1 -t UTF-8 | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $SOLR_RES/${basename}.phon_oov
#
# Search INVOC & OOV in the current lattice
#
- print_info "[${BASENAME}] Search INVOC and OOV in the current lattice" 2
+ print_info "[${BASENAME}] Search INVOC and OOV in the current lattice" 3
cat $SOLR_RES/${basename}_tmp_report/report.invoc | grep -v "\b0" | cut -f1 | grep -v --perl-regex -v "^[a-zA-Z']{1,3}$" | grep -v --perl-regex "^[a-zA-Z0-9]{1,3}$" | grep -v "" | grep -v "" | $SCRIPT_PATH/BdlexUC.pl $RULES/basic -t > $TRIGGER_CONFZONE/$basename.tosearch
cat $SOLR_RES/${basename}.phon_oov | cut -f1 >> $TRIGGER_CONFZONE/$basename.tosearch
@@ -326,7 +339,7 @@ mkdir -p $EXT_LEX/tmp/txt
#
# Collect the acousticly found oov and their phonetisation
#
-print_info "[${BASENAME}] Get all OOV and retrieve all phonetisation" 2
+print_info "[${BASENAME}] Get all OOV and retrieve all phonetisation" 3
for i in `ls $SOLR_RES/*.phon_oov`
do
basename=`basename $i .phon_oov`
@@ -354,7 +367,7 @@ done
#
# Merge OOVs and their phonetisation
#
-print_info "[${BASENAME}] Merge OOV and their phonetisation" 2
+print_info "[${BASENAME}] Merge OOV and their phonetisation" 3
lexname=$(basename $LEXICON)
cat $EXT_LEX/*.oov_acousticlyfound_phon | sort -u > $EXT_LEX/final/all.oov_acousticlyfound_phon
cat $EXT_LEX/*.oov_acousticlyfound | sort -u | grep --perl-regex -v "^([a-z']){3}$" > $EXT_LEX/final/all.oov_acousticlyfound
@@ -377,7 +390,7 @@ cat $SOLR_RES/*.txt | $SCRIPT_PATH/CleanFilter.sh | $SCRIPT_PATH/ApplyCorrection
# - Expected format :
#
#
-print_info "[${BASENAME}] Construct map file" 2
+print_info "[${BASENAME}] Construct map file" 3
rm -f $EXT_LEX/final/${lexname}_ext.map 2>/dev/null
rm -f $EXT_LEX/final/${lexname}.unvalid_oov 2>/dev/null
@@ -391,19 +404,21 @@ do
#
# Try to collect text containing the oov word
#
+ print_info "[${BASENAME}] Collect text containing the oov" 3
cat $EXT_LEX/final/all.bdlex_txt | grep --perl-regex " $oov " | $SCRIPT_PATH/NbMaxWordsFilter.pl 40 |uniq > $EXT_LEX/tmp/txt/$oov.bdlex_txt
if [ -f $EXT_LEX/tmp/txt/$oov.bdlex_txt ]; then
nbWords=`wc -l $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "`
if [ $nbWords -eq 0 ]; then
- echo "[${BASENAME}] UNVALID OOV: $oov => $nbWords occurrences"
+ print_warn "[${BASENAME}] UNVALID OOV: $oov => $nbWords occurrences" 2
echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov
else
#
# Find a candidate in a filtred invoc lexicon => a candidate which maximize the ppl in the overall txt collected
#
#echo "$/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $LEXICON.bdlex_tok $EXT_LEX/tmp/txt/$oov.bdlex_txt"
+ print_info `$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "` 3
candidate=`$SPEERAL_PATH/bin/getCandidate $SPEER_LM_PATH $SPEER_LM_BASENAME $oov $CANDIDATE_LEXICON $EXT_LEX/tmp/txt/$oov.bdlex_txt | cut -f1 -d" "`
- if [ ! x$candidate = "x" ]; then
+ if [ ! "$candidate" == "" ]; then
grep --perl-regex "^$oov\t" $EXT_LEX/final/all.oov_acousticlyfound_phon > $EXT_LEX/tmp/$oov.phon
while read phonLine
do
@@ -411,12 +426,12 @@ do
echo "$phonLine" | sed "s|\t|\t$candidate\t|" >> $EXT_LEX/final/${lexname}_ext.map
done < $EXT_LEX/tmp/$oov.phon
else
- echo "[${BASENAME}] UNVALID OOV: $oov => no availaible Candidate word in LM"
+ print_warn "[${BASENAME}] UNVALID OOV: $oov => no availaible Candidate word in LM" 2
echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov
fi
fi
else
- echo "[${BASENAME}] UNVALID OOV: $oov"
+ print_warn "[${BASENAME}] UNVALID OOV: $oov" 2
echo "$oov" >> $EXT_LEX/final/${lexname}.unvalid_oov
fi
done < $EXT_LEX/final/all.oov_acousticlyfound
@@ -429,7 +444,7 @@ lexname=`basename $LEXICON`
#
# Build the final trigger file
#
-print_info "[${BASENAME}] Clean trigg files" 2
+print_info "[${BASENAME}] Clean trigg files" 3
mkdir -p $TRIGGER_CONFZONE/speeral/ 2> /dev/null
mkdir -p $EXT_LEX/speeral/ 2> /dev/null
for i in `ls $TRIGGER_CONFZONE/*.trigg`
@@ -440,7 +455,8 @@ done
#
# Compile the speeral extended lexicon
#
-print_info "[${BASENAME}] Compile Speeral extended lexicon" 2
+print_info "[${BASENAME}] Compile Speeral extended lexicon" 3
+print_info "$SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext" 3
$SPEERAL_PATH/bin/buildmappedbinode $LEXICON.bdlex_phon $EXT_LEX/final/${lexname}_ext.map $AM_SKL $EXT_LEX/speeral/${lexname}_ext
if [ $CHECK -eq 1 ]
@@ -448,7 +464,10 @@ then
check_exploitconfpass_lex_check "${EXT_LEX}/speeral/${lexname}_ext"
if [ $? -eq 1 ]
then
- echo -e "[${BASENAME}] ERROR : Building Speeral Lexicon $INPUT_DIR " >> $ERRORFILE
+ print_error "[${BASENAME}] Building Speeral Lexicon $INPUT_DIR -> exit"
+ print_error "[${BASENAME}] Check $ERRORFILE"
+ print_log_file $ERRORFILE "ERROR : Building Speeral Lexicon $INPUT_DIR"
+ print_log_file $ERRORFILE "ERROR : ${EXT_LEX}/speeral/${lexname}_ext Empty after buildmappedbinode ?"
exit 1;
fi
fi
@@ -458,10 +477,9 @@ fi
# CLOSE #
#-------#
# Seem OK
-print_info "[${BASENAME}] <= End $BASENAME Solr | $(date +'%d/%m/%y %H:%M:%S')" 1
-echo -e "[${BASENAME}] #Solr $BASENAME " >> $LOGFILE
+print_info "[${BASENAME}] <= ExploitConfidencePass End | $(date +'%d/%m/%y %H:%M:%S')" 1
-# unlock directory
+# unlok directory
mv "$OUTPUT_DIR/EXPLOITCONFPASS.lock" "$OUTPUT_DIR/EXPLOITCONFPASS.unlock"
diff --git a/main_tools/FirstPass.sh b/main_tools/FirstPass.sh
index 75ee49f..673c3b0 100755
--- a/main_tools/FirstPass.sh
+++ b/main_tools/FirstPass.sh
@@ -115,7 +115,7 @@ then
print_info "[${BASENAME}] => P1 start | $(date +'%d/%m/%y %H:%M:%S')" 1
print_info "[${BASENAME}] $WAV_FILE OK" 2
else
- print_error "can't find $1 OR file is empty"
+ print_error " Can't find $1 OR file is empty"
exit 1
fi
@@ -247,7 +247,7 @@ java -Xmx4096m -jar $BIN_PATH/LIUM_SpkDiarization-4.2.jar --fInputMask=${WAV_FIL
if [ $CHECK -eq 1 ] && ( [ ! -e $SEG_FILE ] || [ -z $SEG_FILE ] )
then
print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating SEG file : $SEG_FILE"
- print_error "[${BASENAME}] ERROR : check $ERRORFILE file"
+ print_error "[${BASENAME}] Check $ERRORFILE file"
exit 1
fi
@@ -260,7 +260,7 @@ cat $SEG_FILE | grep -v ";;" | cut -f3,4,5,8 -d" " | tr " " "#" | sort -k1 -n |
if [ $CHECK -eq 1 ] && ( [ ! -e $LBL_FILE ] || [ -z $LBL_FILE ] )
then
print_log_file "$ERRORFILE" "ERROR [$(date +'%d/%m/%y %H:%M:%S')] Creating LBL file : $LBL_FILE"
- print_error "[${BASENAME}] ERROR : check $ERRORFILE file"
+ print_error "[${BASENAME}] Check $ERRORFILE file"
exit 1
fi
@@ -279,7 +279,7 @@ then
if [ $? -eq 1 ]
then
print_log_file $ERRORFILE "ERROR [$(date +'%d/%m/%y %H:%M:%S')] $PLP wrong number of .plp files"
- print_error "[${BASENAME}] ERROR : check $ERRORFILE file"
+ print_error "[${BASENAME}] Check $ERRORFILE file"
exit 1
fi
fi
@@ -368,7 +368,7 @@ then
if [ $notdone -ne 0 ]
then
- print_error "[${BASENAME}] ERROR : check $ERRORFILE"
+ print_error "[${BASENAME}] Check $ERRORFILE"
print_log_file "$ERRORFILE" "ERROR : Segs not done ["
diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE
print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME"
diff --git a/main_tools/RecomposePass.sh b/main_tools/RecomposePass.sh
index 72d8f3c..530e7e5 100755
--- a/main_tools/RecomposePass.sh
+++ b/main_tools/RecomposePass.sh
@@ -59,7 +59,7 @@ do
VERBOSE=$OPTARG
;;
:)
- echo "Option -$OPTARG requires an argument." >&2
+ echo "Option -$OPTARG requires an argument." > /dev/null
exit 1
;;
\?)
@@ -100,12 +100,18 @@ fi
#-------------#
# GLOBAL VARS #
#-------------#
+BASENAME=$(basename $PASS_DIR)
+OUTPUT_DIR_BASENAME=$PASS_DIR
+RES_DIR="$PASS_DIR/res_all"
+LOGFILE=$PASS_DIR"/info_recpass.log"
+ERRORFILE=$PASS_DIR"/error_recpass.log"
+
THIRDPASS_CONFIG_FILE="$PASS_DIR/ThirdPass.cfg"
if [ -e $THIRDPASS_CONFIG_FILE ]
then
RES_P3=$(cat $THIRDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=")
else
- print_warn "can't find $THIRDPASS_CONFIG_FILE file" 1
+ print_warn "[${BASENAME}] can't find $THIRDPASS_CONFIG_FILE file" 2
#exit 1
fi
SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg"
@@ -113,7 +119,7 @@ if [ -e $SECONDPASS_CONFIG_FILE ]
then
RES_P2=$(cat $SECONDPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=")
else
- print_warn "can't find $SECONDPASS_CONFIG_FILE file" 1
+ print_warn "[${BASENAME}] can't find $SECONDPASS_CONFIG_FILE file" 2
#exit 1
fi
FIRSTPASS_CONFIG_FILE="$PASS_DIR/FirstPass.cfg"
@@ -121,33 +127,35 @@ if [ -e $FIRSTPASS_CONFIG_FILE ]
then
RES_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "^RES_DIR=" | cut -f2 -d"=")
else
- print_warn "can't find $FIRSTPASS_CONFIG_FILE file"
+ print_warn "[${BASENAME}] can't find $FIRSTPASS_CONFIG_FILE file" 2
#exit 1
fi
-if ( [ -z "$RES_P1" ] && [ -z "$RES_P2" ] ) || [ -z $RES_P3 ]; then echo "ERROR no configuration pass files in $PASS_DIR"; exit 1; fi
+if ( [ -z "$RES_P1" ] && [ -z "$RES_P2" ] ) || [ -z $RES_P3 ]
+then
+ print_error "[${BASENAME}] No configuration pass files in $PASS_DIR -> exit"
+ if [ $CHECK -eq 1 ]; then print_log_file $ERROFILE "No configuration pass files in $PASS_DIR -> exit";fi
+ exit 1
+fi
-BASENAME=$(basename $PASS_DIR)
-OUTPUT_DIR_BASENAME=$PASS_DIR
-RES_DIR="$PASS_DIR/res_all"
-LOGFILE=$(dirname $PASS_DIR)"/info_recpass.log"
-ERRORFILE=$(dirname $PASS_DIR)"/error_recpass.log"
+print_info "[${BASENAME}] => RecomposePass start | $(date +'%d/%m/%y %H:%M:%S')" 1
#------------------#
# Create WORKSPACE #
#------------------#
# Lock directory
-if [ -e $OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi
+if [ -e $OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock ] && [ $RERUN -eq 0 ]
+then
+ print_warn "[${BASENAME}] RECOMPOSEPASS is locked -> exit" 2
+ exit 1
+fi
rm "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock" > /dev/null 2>&1
touch "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" > /dev/null 2>&1
if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi
-mkdir -p $RES_DIR
-print_info "Make directory $RES_DIR" 1
-
-rm $RES_DIR 2> /dev/null
-mkdir -p $RES_DIR
+mkdir -p $RES_DIR 2> /dev/null
+print_info "Make directory $RES_DIR" 3
#--------------------#
# Save configuration #
@@ -157,12 +165,12 @@ echo "RES_ALL=$RES_DIR" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
echo "RES_P1=$RES_P1" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
echo "RES_P2=$RES_P2" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
echo "RES_P3=$RES_P3" >> $OUTPUT_DIR_BASENAME/RecomposePass.cfg
-print_info "save config in $OUTPUT_DIR_BASENAME/RecomposePass.cfg" 1
+print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/RecomposePass.cfg" 1
#--------------------------------------------------#
# Recompose Pass using previous pass #
#--------------------------------------------------#
-print_info "Launch Recompose Pass" 1
+print_info "[${BASENAME}] Launch Recompose Pass" 1
## Check seg in p3
cp $RES_P3/*.res $RES_DIR
@@ -181,23 +189,26 @@ do
cp $RES_P1/$res.res $RES_DIR
done
-rm ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp ${OUTPUT_DIR_BASENAME}/res_all.tmp
+rm ${OUTPUT_DIR_BASENAME}/res_p3.tmp ${OUTPUT_DIR_BASENAME}/res_p2.tmp ${OUTPUT_DIR_BASENAME}/res_p1.tmp ${OUTPUT_DIR_BASENAME}/res_all.tmp 2> /dev/null
#---------------#
# Convert res #
#---------------#
+print_info "[${BASENAME}] Convert .res into .ctm" 1
# .res => .ctm
$SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.ctm
+print_info "[${BASENAME}] Convert .res into .trs" 1
# .res => .trs
echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
$SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
+print_info "[${BASENAME}] Convert .res into .txt" 1
# .res => .txt
$SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.allpass.txt
-print_info "<= End $BASENAME RecomposePass | $(date +'%d/%m/%y %H:%M:%S')" 1
+print_info "[${BASENAME}] <= RecomposePass End | $(date +'%d/%m/%y %H:%M:%S')" 1
# unlock directory
mv "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.lock" "$OUTPUT_DIR_BASENAME/RECOMPOSEPASS.unlock"
diff --git a/main_tools/ScoringRes.sh b/main_tools/ScoringRes.sh
index 4f56151..0719624 100755
--- a/main_tools/ScoringRes.sh
+++ b/main_tools/ScoringRes.sh
@@ -101,15 +101,21 @@ fi
#-------------#
BASENAME=$(basename $PASS_DIR)
SCORING_DIR=$PASS_DIR/scoring
-LOGFILE="$(dirname $PASS_DIR)/info_scoring.log"
-ERRORFILE="$(dirname $PASS_DIR)/error_scoring.log"
+LOGFILE="$PASS_DIR/info_scoring.log"
+ERRORFILE="$PASS_DIR/error_scoring.log"
+
+print_info "[${BASENAME}] => ScoringRes start | $(date +'%d/%m/%y %H:%M:%S')" 1
#------------------#
# Create WORKSPACE #
#------------------#
# Lock directory
-if [ -e $PASS_DIR/*.lock ]; then echo "$PASS_DIR is lock"; exit 1;fi
+if [ -e $PASS_DIR/*.lock ]
+then
+ print_warn "[${BASENAME}] Directory is locked -> exit" 2;
+ exit 1
+fi
rm -r $SCORING_DIR > /dev/null 2>&1
mkdir -p $SCORING_DIR
@@ -117,18 +123,20 @@ print_info "Make directory $SCORING_DIR" 1
if [ ! -e $PASS_DIR/$BASENAME.SRT ]
then
- echo -e "no SRT file for $BASENAME\n"
+ print_warn "[${BASENAME}] No SRT file -> exit" 2
exit 0;
else
SRT_FILE=$PASS_DIR/$BASENAME.SRT
+ print_info "[${BASENAME}] SRT file : $BASENAME.SRT" 1
fi
if [ ! -e $PASS_DIR/$BASENAME.1pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.2pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.3pass.ctm ] && [ ! -e $PASS_DIR/$BASENAME.allpass.ctm ]
then
- echo -e "no CTM files for $BASENAME\n";
+ print_error "[${BASENAME}] No CTM files -> exit\n";
exit 1;
fi
+print_info "[${BASENAME}] Create stm file from SRT" 1
i=$((-3))
while read line
do
@@ -146,11 +154,14 @@ cat $SCORING_DIR/$BASENAME.tmp.txt | sed -e "s|\n| |g" | $SCRIPT_PATH/CleanFilte
$SCRIPT_PATH/srt2stm.pl $SCORING_DIR/$BASENAME.tmp2.txt > "$SCORING_DIR/$BASENAME.stm"
rm $SCORING_DIR/$BASENAME.tmp.txt $SCORING_DIR/$BASENAME.tmp2.txt
+print_info "Scoring using sclite" 1
for ctmfile in $(ls $PASS_DIR/*.ctm)
do
cp $ctmfile $SCORING_DIR/
+ print_info "$BIN_PATH/sclite -D -F -r $SCORING_DIR/$BASENAME.stm stm -h $SCORING_DIR/$(basename $ctmfile) ctm -o all" 3
$BIN_PATH/sclite -D -F -r $SCORING_DIR/$BASENAME.stm stm -h $SCORING_DIR/$(basename $ctmfile) ctm -o all
done
+print_info "[${BASENAME}] <= ScoringRes End | $(date +'%d/%m/%y %H:%M:%S')" 1
diff --git a/main_tools/SecondPass.sh b/main_tools/SecondPass.sh
index 14fb2c6..869a39f 100755
--- a/main_tools/SecondPass.sh
+++ b/main_tools/SecondPass.sh
@@ -107,12 +107,10 @@ if [ -e $1 ] && [ -d $1 ]
then
FIRSTPASS_DIR=$(readlink -e $1)
else
- print_error "can't find $1 directory"
+ print_error "Can't find $1 directory"
exit 1
fi
-print_info "[${BASENAME}] => P2 start | $(date +'%d/%m/%y %H:%M:%S')" 1
-
#-------------#
# GLOBAL VARS #
#-------------#
@@ -129,7 +127,7 @@ then
LBL_FILE_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "LBL_FILE=" | cut -f2 -d"=")
RES_DIR_P1=$(cat $FIRSTPASS_CONFIG_FILE | grep "RES_DIR=" | cut -f2 -d"=")
else
- print_error "can't find $FIRSTPASS_CONFIG_FILE file"
+ print_error "Can't find $FIRSTPASS_CONFIG_FILE file"
print_error "exit"
exit 1
fi
@@ -139,6 +137,8 @@ RES_DIR=$OUTPUT_DIR_BASENAME"/res_p2"
LOGFILE="$OUTPUT_DIR_BASENAME/info_p2.log"
ERRORFILE="$OUTPUT_DIR_BASENAME/error_p2.log"
+print_info "[${BASENAME}] => P2 start | $(date +'%d/%m/%y %H:%M:%S')" 1
+
#------------------#
# Create WORKSPACE #
#------------------#
@@ -304,12 +304,12 @@ then
pourcentage=$((($notdone*100)/$todo))
if [ $notdone -ne 0 ]
then
- print_error "[${BASENAME}] ERROR : check $ERRORFILE"
+ print_error "[${BASENAME}] Check $ERRORFILE"
print_log_file "$ERRORFILE" "ERROR : Treil not done ["
diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE
print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME"
else
- print_log_file "$LOGFILE" "P1 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')"
+ print_log_file "$LOGFILE" "P2 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')"
fi
rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1
fi
diff --git a/main_tools/ThirdPass.sh b/main_tools/ThirdPass.sh
index bf3bffd..77fa438 100755
--- a/main_tools/ThirdPass.sh
+++ b/main_tools/ThirdPass.sh
@@ -70,7 +70,7 @@ do
RERUN=1
;;
:)
- echo "Option -$OPTARG requires an argument." >&2
+ echo "Option -$OPTARG requires an argument." > /dev/null
exit 1
;;
\?)
@@ -103,11 +103,14 @@ shift $((OPTIND-1))
if [ -e $1 ] && [ -d $1 ]
then
PASS_DIR=$(readlink -e $1)
+ BASENAME=$(basename $PASS_DIR)
else
print_error "can't find $1 directory"
exit 1
fi
+print_info "[${BASENAME}] => P3 start | $(date +'%d/%m/%y %H:%M:%S')" 1
+
#-------------#
# GLOBAL VARS #
#-------------#
@@ -121,62 +124,67 @@ then
PLP_DIR_P1=""
HMM=""
else
- print_error "can't find $EXPLOITCONFPASS_CONFIG_FILE file"
+ print_error "[${BASENAME}] Can't find $EXPLOITCONFPASS_CONFIG_FILE file"
+ print_warn "[${BASENAME}] Will use default directories and files" 2
#exit 1
TRIGGER_SPEERAL=$PASS_DIR/trigg/speeral/
LEX_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext
LEX_BINODE_SPEERAL=$PASS_DIR/LEX/speeral/LEXIQUE_V6_ext.bin
fi
+
SECONDPASS_CONFIG_FILE="$PASS_DIR/SecondPass.cfg"
if [ -e $SECONDPASS_CONFIG_FILE ]
then
LST=$(cat $SECONDPASS_CONFIG_FILE | grep "^LST=" | cut -f2 -d"=")
HMM=$(cat $SECONDPASS_CONFIG_FILE | grep "^HMM=" | cut -f2 -d"=")
PLP_DIR_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_DIR_P1=" | cut -f2 -d"=")
+ PLP_FILE_P1=$(cat $SECONDPASS_CONFIG_FILE | grep "^PLP_FILE_P1=" | cut -f2 -d"=")
else
- print_error "can't find $SECONDPASS_CONFIG_FILE file"
+ print_error "[${BASENAME}] Can't find $SECONDPASS_CONFIG_FILE file"
+ print_warn "[${BASENAME}] Will use default directories and files" 2
#exit 1
LST=$PASS_DIR/lists
HMM=$PASS_DIR/hmm
PLP_DIR_P1=$PASS_DIR/PLP
+ PLP_FILE_P1=$PASS_DIR/plp.lst
fi
-BASENAME=$(basename $PASS_DIR)
OUTPUT_DIR_BASENAME=$PASS_DIR
RES_DIR="$PASS_DIR/res_p3"
-LOGFILE=$(dirname $PASS_DIR)"/info_p3.log"
-ERRORFILE=$(dirname $PASS_DIR)"/error_p3.log"
+LOGFILE=$PASS_DIR"/info_p3.log"
+ERRORFILE=$PASS_DIR"/error_p3.log"
#------------------#
# Create WORKSPACE #
#------------------#
# Lock directory
-if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then exit 1;fi
+if [ -e $OUTPUT_DIR_BASENAME/THIRDPASS.lock ] && [ $RERUN -eq 0 ]; then print_warn "[${BASENAME}] THIRDPASS is locked -> exit" 2; exit 1; fi
rm "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock" > /dev/null 2>&1
touch "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" > /dev/null 2>&1
if [ $RERUN -eq 0 ]; then rm -r $RES_DIR > /dev/null 2>&1; fi
mkdir -p $RES_DIR
-print_info "Make directory $RES_DIR" 1
+print_info "[${BASENAME}] Make directory $RES_DIR" 3
#--------------------#
# Save configuration #
#--------------------#
cp $THIRDPASS_CONFIG_FILE $OUTPUT_DIR_BASENAME/ThirdPass.cfg
echo "RES_DIR=$RES_DIR" >> $OUTPUT_DIR_BASENAME/ThirdPass.cfg
-print_info "save config in $OUTPUT_DIR_BASENAME/SecondPass.cfg" 1
+print_info "[${BASENAME}] Save config in $OUTPUT_DIR_BASENAME/ThirdPass.cfg" 1
#--------------------------------------------------#
# Third Pass using trigger file (DECODING) #
#--------------------------------------------------#
-print_info "Launch Third Pass" 1
+print_info "[${BASENAME}] Launch Third Pass" 1
## Generate speeral config file adding trigger rep
cat $SPEERAL_CFG_PATH/$SPEERAL_CFG_FILE | sed -e "s|[^<]*|$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext|g" \
| sed -e "s|[^<]*|$OUTPUT_DIR_BASENAME/LEX/speeral/LEXIQUE_V6_ext.bin|g" \
| sed -e "s|[^<]*|$TRIGGER_SPEERAL|g" > $OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml
SPEERAL_THIRD_CFG=$OUTPUT_DIR_BASENAME/SpeeralThirdPass.xml
+print_info "[${BASENAME}] Third Pass Speeral Config file ${SPEERAL_THIRD_CFG}" 3
# for all speaker
for lspeaker in $(ls $LST/*.lst)
@@ -193,10 +201,11 @@ do
## Speeral decoding
if [ -s $HMM/$speaker.hmm ] && [ -s $HMM/$speaker.cls ]
then
- print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3
+ print_info "[${BASENAME}] $SPEERAL_BIN $LST/$speaker.lst $RES_DIR $SPEERAL_THIRD_CFG -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock" 3
$SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $HMM/$speaker.hmm -c $HMM/$speaker.cls $FORKS --lock
else
- print_warn "$HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2
+ print_warn "[${BASENAME}] $HMM/$speaker.hmm and $speaker.cls empty, do default decoding..." 2
+ print_info "$SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock" 3
$SPEERAL_BIN $LST/$speaker.lst $RES_DIR ${SPEERAL_THIRD_CFG} -r $PLP_DIR_P1 -m $SPEERAL_AM/${MODS[$i]}.hmm -c $SPEERAL_AM/${MODS[$i]}.cls $FORKS --lock
fi
@@ -205,12 +214,14 @@ do
check_third_pass_output_speeral "$LST/$speaker.lst" "$RES_DIR"
if [ $? -eq 1 ]
then
- echo -e "ERROR : Speeral $LST/$speaker.lst\n[" >> $ERRORFILE
+
+ print_warn "[${BASENAME}] Speeral output error : check $LOGFILE" 2
+ print_log_file $LOGFILE "WARN : Speeral number of output ERROR $LST/$speaker.lst"
ls $RES_DIR/*.seg | grep -e "$speaker" | sed -e "s|$RES_DIR\/||" | sed -e 's/\.seg//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
- diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $ERRORFILE
- echo -e "] " >> $ERRORFILE
+ print_log_file $LOGFILE "Segs (and treil) not done :\n["
+ diff $LST/$speaker.lst ${OUTPUT_DIR_BASENAME}/.tmp >> $LOGFILE
+ print_log_file $LOGFILE "] [$(date +'%d/%m/%y %H:%M:%S')]"
rm ${OUTPUT_DIR_BASENAME}/.tmp
- #exit 1
fi
fi
break
@@ -220,27 +231,45 @@ do
done
## Check missing seg and log it
-ls $RES_DIR/*.res | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.res//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
-echo -e "$BASENAME P3 END\n[" >> $LOGFILE
-diff ${OUTPUT_DIR_BASENAME}/plp.lst ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $LOGFILE
-echo -e "] $BASENAME" >> $LOGFILE
-rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1
+
+## Check missing seg and log it
+if [ $CHECK -eq 1 ]
+then
+ ls $RES_DIR/*.treil | sed -e "s|$RES_DIR\/||g" | sed -e 's/\.treil//' | sort > ${OUTPUT_DIR_BASENAME}/.tmp
+ todo=$(cat ${PLP_FILE_P1} | wc -l)
+ if [ $todo -eq 0 ]; then todo=1;fi
+ notdone=$(($todo - $(cat ${OUTPUT_DIR_BASENAME}/.tmp | wc -l)))
+ pourcentage=$((($notdone*100)/$todo))
+ if [ $notdone -ne 0 ]
+ then
+ print_error "[${BASENAME}] Check $ERRORFILE"
+ print_log_file "$ERRORFILE" "ERROR : Treil not done ["
+ diff ${PLP_FILE_P1} ${OUTPUT_DIR_BASENAME}/.tmp | grep -e "^< " | sed -e "s/< //" >> $ERRORFILE
+ print_log_file "$ERRORFILE" "] $pourcentage% $BASENAME"
+ else
+ print_log_file "$LOGFILE" "P3 OK $BASENAME | $(date +'%d/%m/%y %H:%M:%S')"
+ fi
+ rm ${OUTPUT_DIR_BASENAME}/.tmp > /dev/null 2>&1
+fi
#---------------#
# Convert res #
#---------------#
+print_info "[${BASENAME}] Convert .res into .ctm" 1
# .res => .ctm
$SCRIPT_PATH/res2out.pl --dir $RES_DIR --format CTM --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.ctm
+print_info "[${BASENAME}] Convert .res into .trs" 1
# .res => .trs
echo -e "name $AUTHOR\nfileName ${BASENAME}\nfileExt wav\nsegFile ${OUTPUT_DIR_BASENAME}/${BASENAME}.seg" > ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
$SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TRS --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.trs --trs_config ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg
rm ${OUTPUT_DIR_BASENAME}/${BASENAME}.trs_cfg > /dev/null 2>&1
+print_info "[${BASENAME}] Convert .res into .txt" 1
# .res => .txt
$SCRIPT_PATH/res2out.pl --dir $RES_DIR --format TXT --ignore $RULES/asupp --out ${OUTPUT_DIR_BASENAME}/${BASENAME}.3pass.txt
-print_info "<= End $BASENAME P3 | $(date +'%d/%m/%y %H:%M:%S')" 1
+print_info "[${BASENAME}] <= P3 End | $(date +'%d/%m/%y %H:%M:%S')" 1
# unlock directory
mv "$OUTPUT_DIR_BASENAME/THIRDPASS.lock" "$OUTPUT_DIR_BASENAME/THIRDPASS.unlock"
diff --git a/tools/scripts/Tools.sh b/tools/scripts/Tools.sh
index 4ff04de..daf034b 100755
--- a/tools/scripts/Tools.sh
+++ b/tools/scripts/Tools.sh
@@ -29,13 +29,13 @@ function print_message()
if [ $# -eq 3 ]
then
# print error
- if [ $1 -eq $ERROR ]; then echo -ne "###ERROR : $3\n" > /dev/stderr; exit 0; fi
+ if [ $1 -eq $ERROR ]; then echo -ne "###ERROR : $3\n" > /dev/stderr; return 0; fi
# check verbose level
if [ $2 -le $VERBOSE ]
then
#for (( dec=1 ; $dec<$2 ; dec++ )); do echo -ne "\t"; done
# print warn
- if [ $1 = $WARN ]; then echo -ne "##WARNING : $3\n"; exit 0; fi
+ if [ $1 = $WARN ]; then echo -ne "##WARNING : $3\n"; return 0; fi
# default info
echo -ne "#INFO : $3\n"
fi
@@ -82,6 +82,6 @@ print_log_file()
{
if [ $# -eq 2 ]
then
- echo -e "$2" >> $1
+ echo -e "$2" >> "$1"
fi
}