Blame view

Scripts/steps/tandem/.svn/text-base/train_deltas.sh.svn-base 6.17 KB
ec85f8892   bigot benjamin   first commit
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
  #!/bin/bash
  
  # Copyright 2012  Johns Hopkins University (Author: Daniel Povey)
  #                 Korbinian Riedhammer
  # Apache 2.0
  
  # Begin configuration.
  stage=-4 #  This allows restarting after partway, when something when wrong.
  config=
  cmd=run.pl
  scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1"
  realign_iters="10 20 30";
  num_iters=35    # Number of iterations of training
  max_iter_inc=25 # Last iter to increase #Gauss on.
  beam=10
  retry_beam=40
  boost_silence=1.0 # Factor by which to boost silence likelihoods in alignment
  power=0.2 # Exponent for number of gaussians according to occurrence counts
  cluster_thresh=-1  # for build-tree control final bottom-up clustering of leaves
  normft2=true  # typically, the tandem features will be normalized already b/c of pca
  # End configuration.
  
  echo "$0 $@"  # Print the command line for logging
  
  [ -f path.sh ] && . ./path.sh;
  . parse_options.sh || exit 1;
  
  if [ $# != 7 ]; then
     echo "Usage: steps/tandem/train_deltas.sh <num-leaves> <tot-gauss> <data1-dir> <data2-dir> <lang-dir> <alignment-dir> <exp-dir>"
     echo " e.g.: steps/tandem/train_deltas.sh 2000 10000 {mfcc,bottleneck}/data/train_si84_half data/lang exp/mono_ali exp/tri1"
     echo "main options (for others, see top of script file)"
     echo "  --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs."
     echo "  --config <config-file>                           # config containing options"
     echo "  --stage <stage>                                  # stage to do partial re-run from."
     echo "  --normft2 (true|false)                           # apply CMVN to second features?"
     exit 1;
  fi
  
  numleaves=$1
  totgauss=$2
  data1=$3
  data2=$4
  lang=$5
  alidir=$6
  dir=$7
  
  for f in $alidir/final.mdl $alidir/ali.1.gz $data1/feats.scp $data2/feats.scp $lang/phones.txt; do
    [ ! -f $f ] && echo "train_tandem.sh: no such file $f" && exit 1;
  done
  
  numgauss=$numleaves
  incgauss=$[($totgauss-$numgauss)/$max_iter_inc] # per-iter increment for #Gauss
  oov=`cat $lang/oov.int` || exit 1;
  ciphonelist=`cat $lang/phones/context_indep.csl` || exit 1;
  nj=`cat $alidir/num_jobs` || exit 1;
  mkdir -p $dir/log
  echo $nj > $dir/num_jobs
  
  # Set up features
  
  sdata1=$data1/split$nj;
  sdata2=$data2/split$nj;
  [[ -d $sdata1 && $data1/feats.scp -ot $sdata1 ]] || split_data.sh $data1 $nj || exit 1;
  [[ -d $sdata2 && $data2/feats.scp -ot $sdata2 ]] || split_data.sh $data2 $nj || exit 1;
  
  # Set up stream 1 (usually spectral features, so we use deltas)
  feats1="ark,s,cs:apply-cmvn --norm-vars=false --utt2spk=ark:$sdata1/JOB/utt2spk scp:$sdata1/JOB/cmvn.scp scp:$sdata1/JOB/feats.scp ark:- | add-deltas ark:- ark:- |"
  
  # Set up stream 2 (usually bottleneck/posteriors), normalize if desired
  feats2="scp:$sdata2/JOB/feats.scp"
  if [ "$normft2" == "true" ]; then
    feats2="ark,s,cs:apply-cmvn --norm-vars=false --utt2spk=ark:$sdata2/JOB/utt2spk scp:$sdata2/JOB/cmvn.scp $feats2 ark:- |"
  fi
  
  feats="ark,s,cs:paste-feats '$feats1' '$feats2' ark:- |"
  
  # save config
  echo $feats > $dir/tandem
  echo $normft2 > $dir/normft2
  
  rm $dir/.error 2>/dev/null
  
  if [ $stage -le -3 ]; then
    echo "$0: accumulating tree stats"
    $cmd JOB=1:$nj $dir/log/acc_tree.JOB.log \
      acc-tree-stats  --ci-phones=$ciphonelist $alidir/final.mdl "$feats" \
       "ark:gunzip -c $alidir/ali.JOB.gz|" $dir/JOB.treeacc || exit 1;
    sum-tree-stats $dir/treeacc $dir/*.treeacc 2>$dir/log/sum_tree_acc.log || exit 1;
    rm $dir/*.treeacc
  fi
  
  if [ $stage -le -2 ]; then
    echo "$0: getting questions for tree-building, via clustering"
    # preparing questions, roots file...
    cluster-phones $dir/treeacc $lang/phones/sets.int $dir/questions.int 2> $dir/log/questions.log || exit 1;
    cat $lang/phones/extra_questions.int >> $dir/questions.int
    compile-questions $lang/topo $dir/questions.int $dir/questions.qst 2>$dir/log/compile_questions.log || exit 1;
  
    echo "$0: building the tree"
    $cmd $dir/log/build_tree.log \
      build-tree --verbose=1 --max-leaves=$numleaves \
      --cluster-thresh=$cluster_thresh $dir/treeacc $lang/phones/roots.int \
      $dir/questions.qst $lang/topo $dir/tree || exit 1;
  
    gmm-init-model  --write-occs=$dir/1.occs  \
      $dir/tree $dir/treeacc $lang/topo $dir/1.mdl 2> $dir/log/init_model.log || exit 1;
    grep 'no stats' $dir/log/init_model.log && echo "This is a bad warning.";
  
    gmm-mixup --mix-up=$numgauss $dir/1.mdl $dir/1.occs $dir/1.mdl 2>$dir/log/mixup.log || exit 1;
    rm $dir/treeacc
  fi
  
  if [ $stage -le -1 ]; then
    # Convert the alignments.
    echo "$0: converting alignments from $alidir to use current tree"
    $cmd JOB=1:$nj $dir/log/convert.JOB.log \
      convert-ali $alidir/final.mdl $dir/1.mdl $dir/tree \
       "ark:gunzip -c $alidir/ali.JOB.gz|" "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1;
  fi
  
  if [ $stage -le 0 ]; then
    echo "$0: compiling graphs of transcripts"
    $cmd JOB=1:$nj $dir/log/compile_graphs.JOB.log \
      compile-train-graphs $dir/tree $dir/1.mdl  $lang/L.fst  \
       "ark:utils/sym2int.pl --map-oov $oov -f 2- $lang/words.txt < $data1/split$nj/JOB/text |" \
        "ark:|gzip -c >$dir/fsts.JOB.gz" || exit 1;
  fi
  
  x=1
  while [ $x -lt $num_iters ]; do
    echo "$0: training pass $x"
    if [ $stage -le $x ]; then
      if echo $realign_iters | grep -w $x >/dev/null; then
        echo "$0: aligning data"
        mdl="gmm-boost-silence --boost=$boost_silence `cat $lang/phones/optional_silence.csl` $dir/$x.mdl - |"
        $cmd JOB=1:$nj $dir/log/align.$x.JOB.log \
          gmm-align-compiled $scale_opts --beam=$beam --retry-beam=$retry_beam "$mdl" \
           "ark:gunzip -c $dir/fsts.JOB.gz|" "$feats" \
           "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1;
      fi
      $cmd JOB=1:$nj $dir/log/acc.$x.JOB.log \
        gmm-acc-stats-ali  $dir/$x.mdl "$feats" \
         "ark,s,cs:gunzip -c $dir/ali.JOB.gz|" $dir/$x.JOB.acc || exit 1;
      $cmd $dir/log/update.$x.log \
        gmm-est --mix-up=$numgauss --power=$power \
          --write-occs=$dir/$[$x+1].occs $dir/$x.mdl \
         "gmm-sum-accs - $dir/$x.*.acc |" $dir/$[$x+1].mdl || exit 1;
      rm $dir/$x.mdl $dir/$x.*.acc
      rm $dir/$x.occs
    fi
    [ $x -le $max_iter_inc ] && numgauss=$[$numgauss+$incgauss];
    x=$[$x+1];
  done
  
  rm $dir/final.mdl 2>/dev/null
  ln -s $x.mdl $dir/final.mdl
  ln -s $x.occs $dir/final.occs
  
  # Summarize warning messages...
  utils/summarize_warnings.pl  $dir/log
  
  echo "$0: Done training tandem system in $dir"