Blame view

egs/gale_arabic/s5c/local/prepare_data.sh 3.45 KB
8dcb6dfcb   Yannick Estève   first commit
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
  #!/bin/bash 
  
  # Copyright 2014 QCRI (author: Ahmed Ali)
  # Apache 2.0
  
  dir1=/export/corpora/LDC/LDC2013S02/
  dir2=/export/corpora/LDC/LDC2013S07/
  dir3=/export/corpora/LDC/LDC2014S07/
  text1=/export/corpora/LDC/LDC2013T17/
  text2=/export/corpora/LDC/LDC2013T04/
  text3=/export/corpora/LDC/LDC2014T17/
  gale_data=GALE
  
  mkdir -p $gale_data 
  # check that sox is installed 
  which sox  &>/dev/null
  if [[ $? != 0 ]]; then 
   echo "$0: sox is not installed"; exit 1
  fi
  
  for dvd in $dir1 $dir2 $dir3; do
    dvd_full_path=$(utils/make_absolute.sh $dvd)
    if [[ ! -e $dvd_full_path ]]; then 
      echo "$0: missing $dvd_full_path"; exit 1;
    fi
    find $dvd_full_path \( -name "*.wav" -o -name "*.flac" \)  | while read file; do
      id=$(basename $file | awk '{gsub(".wav","");gsub(".flac","");print}')
      echo "$id sox $file -r 16000 -t wav - |"
    done 
  done | sort -u > $gale_data/wav.scp
  echo "$0:data prep audio succeded"
  
  gale_data=$(utils/make_absolute.sh "GALE" );
  top_pwd=`pwd`
  txtdir=$gale_data/txt
  mkdir -p $txtdir; cd $txtdir
  
  for cdx in $text1 $text2 $text3; do
    echo "$0:Preparing $cdx"
    if [[ $cdx  == *.tgz ]] ; then
       tar -xvf $cdx
    elif [  -d "$cdx" ]; then
      ln -s $cdx `basename $cdx`
    else
      echo "$0:I don't really know what I shall do with $cdx " >&2
    fi
  done
  
  find -L . -type f -name "*.tdf" | while read file; do
  sed '1,3d' $file  # delete the first 3 lines
  done >  all.tmp$$
  
  perl -e '
      ($inFile,$idFile,$txtFile)= split /\s+/, $ARGV[0];
      open(IN, "$inFile");
      open(ID, ">$idFile");
      open(TXT, ">$txtFile");
      while (<IN>) {
        @arr= split /\t/,$_;
        $start=sprintf ("%0.3f",$arr[2]);$rStart=$start;$start=~s/\.//; $start=~s/^0+$/0/; $start=~s/^0+([^0])/$1/; # remove zeros at the beginning
        $end=sprintf ("%0.3f",$arr[3]);$rEnd=$end;$end=~s/^0+([^0])/$1/;$end=~s/\.//;
        if ( ($arr[11] !~ m/report/) && ($arr[11] !~ m/conversational/) ){$arr[11]="UNK";}
        $id="$arr[11] $arr[0] $arr[0]_${start}_${end} $rStart $rEnd
  ";
        next if ($rStart == $rEnd);
        $id =~ s/.sph//g;
        print ID $id;
        print TXT "$arr[7]
  ";
   }' "all.tmp$$ allid.tmp$$ contentall.tmp$$"
  
  perl ${top_pwd}/local/normalize_transcript_BW.pl contentall.tmp$$ contentall.buck.tmp$$
  paste allid.tmp$$ contentall.buck.tmp$$ | sed 's: $::' | awk '{if (NF>5) {print $0}}'  > all_1.tmp$$
  
  
  awk '{$1="";print $0}' all_1.tmp$$ | sed 's:^ ::' > $gale_data/all
  awk '{if ($1 == "report") {$1="";print $0}}' all_1.tmp$$ | sed 's:^ ::' >  $gale_data/report
  awk '{if ($1 == "conversational") {$1="";print $0}}' all_1.tmp$$ | sed 's:^ ::' > $gale_data/conversational
  
  cd ..;
  rm -fr $txtdir
  cd $top_pwd
  echo "$0:dat a prep text succeeded"
  
  mkdir -p data
  dir=$(utils/make_absolute.sh data/)
  grep -f local/test_list $gale_data/all | grep -v -f local/bad_segments > $gale_data/all.test
  grep -v -f local/test_list $gale_data/all | grep -v -f local/bad_segments > $gale_data/all.train 
  
  for x in test train; do
   outdir=data/$x
   file=$gale_data/all.$x 
   mkdir -p $outdir
   awk '{print $2 " " $2}' $file | sort -u > $outdir/utt2spk 
   cp -pr $outdir/utt2spk $outdir/spk2utt
   awk '{print $2 " " $1 " " $3 " " $4}' $file  | sort -u > $outdir/segments
   awk '{printf $2 " "; for (i=5; i<=NF; i++) {printf $i " "} printf "
  "}' $file | sort -u > $outdir/text
  done 
  
  grep -f local/test_list $gale_data/wav.scp > $dir/test/wav.scp
  
  cat $gale_data/wav.scp | awk -v seg=$dir/train/segments 'BEGIN{while((getline<seg) >0) {seen[$2]=1;}}
   {if (seen[$1]) { print $0}}' > $dir/train/wav.scp
   
  echo "$0:data prep split succeeded"
  exit 0