Blame view
egs/gale_arabic/s5b/local/prepare_data.sh
3.45 KB
8dcb6dfcb
first commit
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
#!/bin/bash
# Copyright 2014 QCRI (author: Ahmed Ali)
# Apache 2.0
dir1=/export/corpora/LDC/LDC2013S02/
dir2=/export/corpora/LDC/LDC2013S07/
dir3=/export/corpora/LDC/LDC2014S07/
text1=/export/corpora/LDC/LDC2013T17/
text2=/export/corpora/LDC/LDC2013T04/
text3=/export/corpora/LDC/LDC2014T17/
gale_data=GALE
mkdir -p $gale_data
# check that sox is installed
which sox &>/dev/null
if [[ $? != 0 ]]; then
echo "$0: sox is not installed"; exit 1
fi
for dvd in $dir1 $dir2 $dir3; do
dvd_full_path=$(utils/make_absolute.sh $dvd)
if [[ ! -e $dvd_full_path ]]; then
echo "$0: missing $dvd_full_path"; exit 1;
fi
find $dvd_full_path \( -name "*.wav" -o -name "*.flac" \) | while read file; do
id=$(basename $file | awk '{gsub(".wav","");gsub(".flac","");print}')
echo "$id sox $file -r 16000 -t wav - |"
done
done | sort -u > $gale_data/wav.scp
echo "$0:data prep audio succeded"
gale_data=$(utils/make_absolute.sh "GALE" );
top_pwd=`pwd`
txtdir=$gale_data/txt
mkdir -p $txtdir; cd $txtdir
for cdx in $text1 $text2 $text3; do
echo "$0:Preparing $cdx"
if [[ $cdx == *.tgz ]] ; then
tar -xvf $cdx
elif [ -d "$cdx" ]; then
ln -s $cdx `basename $cdx`
else
echo "$0:I don't really know what I shall do with $cdx " >&2
fi
done
find -L . -type f -name "*.tdf" | while read file; do
sed '1,3d' $file # delete the first 3 lines
done > all.tmp$$
perl -e '
($inFile,$idFile,$txtFile)= split /\s+/, $ARGV[0];
open(IN, "$inFile");
open(ID, ">$idFile");
open(TXT, ">$txtFile");
while (<IN>) {
@arr= split /\t/,$_;
$start=sprintf ("%0.3f",$arr[2]);$rStart=$start;$start=~s/\.//; $start=~s/^0+$/0/; $start=~s/^0+([^0])/$1/; # remove zeros at the beginning
$end=sprintf ("%0.3f",$arr[3]);$rEnd=$end;$end=~s/^0+([^0])/$1/;$end=~s/\.//;
if ( ($arr[11] !~ m/report/) && ($arr[11] !~ m/conversational/) ){$arr[11]="UNK";}
$id="$arr[11] $arr[0] $arr[0]_${start}_${end} $rStart $rEnd
";
next if ($rStart == $rEnd);
$id =~ s/.sph//g;
print ID $id;
print TXT "$arr[7]
";
}' "all.tmp$$ allid.tmp$$ contentall.tmp$$"
perl ${top_pwd}/local/normalize_transcript_BW.pl contentall.tmp$$ contentall.buck.tmp$$
paste allid.tmp$$ contentall.buck.tmp$$ | sed 's: $::' | awk '{if (NF>5) {print $0}}' > all_1.tmp$$
awk '{$1="";print $0}' all_1.tmp$$ | sed 's:^ ::' > $gale_data/all
awk '{if ($1 == "report") {$1="";print $0}}' all_1.tmp$$ | sed 's:^ ::' > $gale_data/report
awk '{if ($1 == "conversational") {$1="";print $0}}' all_1.tmp$$ | sed 's:^ ::' > $gale_data/conversational
cd ..;
rm -fr $txtdir
cd $top_pwd
echo "$0:dat a prep text succeeded"
mkdir -p data
dir=$(utils/make_absolute.sh data/)
grep -f local/test_list $gale_data/all | grep -v -f local/bad_segments > $gale_data/all.test
grep -v -f local/test_list $gale_data/all | grep -v -f local/bad_segments > $gale_data/all.train
for x in test train; do
outdir=data/$x
file=$gale_data/all.$x
mkdir -p $outdir
awk '{print $2 " " $2}' $file | sort -u > $outdir/utt2spk
cp -pr $outdir/utt2spk $outdir/spk2utt
awk '{print $2 " " $1 " " $3 " " $4}' $file | sort -u > $outdir/segments
awk '{printf $2 " "; for (i=5; i<=NF; i++) {printf $i " "} printf "
"}' $file | sort -u > $outdir/text
done
grep -f local/test_list $gale_data/wav.scp > $dir/test/wav.scp
cat $gale_data/wav.scp | awk -v seg=$dir/train/segments 'BEGIN{while((getline<seg) >0) {seen[$2]=1;}}
{if (seen[$1]) { print $0}}' > $dir/train/wav.scp
echo "$0:data prep split succeeded"
exit 0
|