Blame view

Scripts/steps/tandem/.svn/text-base/train_sgmm2.sh.svn-base 13.2 KB
ec85f8892   bigot benjamin   first commit
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
  #!/bin/bash
  
  # Copyright 2012  Johns Hopkins University (Author: Daniel Povey).  Apache 2.0.
  #                 Korbinian Riedhammer
  
  # SGMM training, with speaker vectors.  This script would normally be called on
  # top of fMLLR features obtained from a conventional system, but it also works
  # on top of any type of speaker-independent features (based on
  # deltas+delta-deltas or LDA+MLLT).  For more info on SGMMs, see the paper "The
  # subspace Gaussian mixture model--A structured model for speech recognition".
  # (Computer Speech and Language, 2011).
  
  # Begin configuration section.
  nj=4
  cmd=run.pl
  stage=-6 # use this to resume partially finished training 
  context_opts= # e.g. set it to "--context-width=5 --central-position=2"  for a
  # quinphone system.
  scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1"
  num_iters=25   # Total number of iterations of training
  num_iters_alimdl=3 # Number of iterations for estimating alignment model.
  max_iter_inc=15 # Last iter to increase #substates on.
  realign_iters="5 10 15"; # Iters to realign on. 
  spkvec_iters="5 8 12 17" # Iters to estimate speaker vectors on.
  increase_iters="6 10 14"; # Iters on which to increase phn dim and/or spk dim;
      # rarely necessary, and if it is, only the 1st will normally be necessary.
  rand_prune=0.1 # Randomized-pruning parameter for posteriors, to speed up training.
                 # Bigger -> more pruning; zero = no pruning.
  phn_dim=  # You can use this to set the phonetic subspace dim. [default: feat-dim+1]
  spk_dim=  # You can use this to set the speaker subspace dim. [default: feat-dim]
  power=0.2 # Exponent for number of gaussians according to occurrence counts
  beam=8
  self_weight=0.9
  retry_beam=40
  leaves_per_group=5 # Relates to the SCTM (state-clustered tied-mixture) aspect:
                     # average number of pdfs in a "group" of pdfs.
  update_m_iter=4
  spk_dep_weights=true # [Symmetric SGMM] set this to false if you don't want "u" (i.e. to turn off
                        # symmetric SGMM.
  normft2=true
  # End configuration section.
  
  echo "$0 $@"  # Print the command line for logging
  
  if [ -f path.sh ]; then . ./path.sh; fi
  . parse_options.sh || exit 1;
  
  
  if [ $# != 8 ]; then
    echo "Usage: steps/tandem/train_sgmm2.sh <num-leaves> <num-substates> <data1> <data2> <lang> <ali-dir> <ubm> <exp-dir>"
    echo " e.g.: steps/tandem/train_sgmm2.sh 5000 8000 {mfcc,bottleneck}/data/train_si84 data/lang \\"
    echo "                      exp/tri3b_ali_si84 exp/ubm4a/final.ubm exp/sgmm4a"
    echo "main options (for others, see top of script file)"
    echo "  --config <config-file>                           # config containing options"
    echo "  --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs."
    echo "  --silence-weight <sil-weight>                    # weight for silence (e.g. 0.5 or 0.0)"
    echo "  --num-iters <#iters>                             # Number of iterations of E-M"
    echo "  --leaves-per-group <#leaves>                     # Average #leaves shared in one group"
    exit 1;
  fi
  
  num_pdfs=$1  # final #leaves, at 2nd level of tree.
  totsubstates=$2
  data1=$3
  data2=$4
  lang=$5
  alidir=$6
  ubm=$7
  dir=$8
  
  num_groups=$[$num_pdfs/$leaves_per_group]
  first_spkvec_iter=`echo $spkvec_iters | awk '{print $1}'` || exit 1;
  
  # Check some files.
  for f in $data1/feats.scp $data2/feats.scp $lang/L.fst $alidir/ali.1.gz $alidir/final.mdl $ubm; do
    [ ! -f $f ] && echo "$0: no such file $f" && exit 1;
  done
  
  
  # Set some variables.
  oov=`cat $lang/oov.int`
  silphonelist=`cat $lang/phones/silence.csl`
  if [ "$self_weight" == "1.0" ]; then
    numsubstates=$num_groups # Initial #-substates.
  else
    numsubstates=$num_pdfs # Initial #-substates.
  fi
  incsubstates=$[($totsubstates-$numsubstates)/$max_iter_inc] # per-iter increment for #substates
  feat_dim=`gmm-info $alidir/final.mdl 2>/dev/null | awk '/feature dimension/{print $NF}'` || exit 1;
  [ $feat_dim -eq $feat_dim ] || exit 1; # make sure it's numeric.
  [ -z $phn_dim ] && phn_dim=$[$feat_dim+1]
  [ -z $spk_dim ] && spk_dim=$feat_dim
  nj=`cat $alidir/num_jobs` || exit 1;
  
  mkdir -p $dir/log
  echo $nj > $dir/num_jobs
  
  sdata1=$data1/split$nj;
  sdata2=$data2/split$nj;
  [[ -d $sdata1 && $data1/feats.scp -ot $sdata1 ]] || split_data.sh $data1 $nj || exit 1;
  [[ -d $sdata2 && $data2/feats.scp -ot $sdata2 ]] || split_data.sh $data2 $nj || exit 1;
  
  spkvecs_opt=  # Empty option for now, until we estimate the speaker vectors.
  gselect_opt="--gselect=ark,s,cs:gunzip -c $dir/gselect.JOB.gz|"
  
  ## Set up features.
  
  
  # We will use the same settings as with the alidir
  splice_opts=`cat $alidir/splice_opts 2>/dev/null` # frame-splicing options.
  normft2=`cat $alidir/normft2 2>/dev/null`
  
  if [ -f $alidir/final.mat ]; then feat_type=lda; else feat_type=delta; fi
  
  case $feat_type in
    delta) 
    	echo "$0: feature type is $feat_type"
    	;;
    lda) 
    	echo "$0: feature type is $feat_type"
      cp $alidir/{lda,final}.mat $dir/ || exit 1;
      ;;
    *) echo "$0: invalid feature type $feat_type" && exit 1;
  esac
  
  # set up feature stream 1;  this are usually spectral features, so we will add
  # deltas or splice them
  feats1="ark,s,cs:apply-cmvn --norm-vars=false --utt2spk=ark:$sdata1/JOB/utt2spk scp:$sdata1/JOB/cmvn.scp scp:$sdata1/JOB/feats.scp ark:- |"
  
  if [ "$feat_type" == "delta" ]; then
    feats1="$feats1 add-deltas ark:- ark:- |"
  elif [ "$feat_type" == "lda" ]; then
    feats1="$feats1 splice-feats $splice_opts ark:- ark:- | transform-feats $dir/lda.mat ark:- ark:- |"
  fi
  
  # set up feature stream 2;  this are usually bottleneck or posterior features, 
  # which may be normalized if desired
  feats2="scp:$sdata2/JOB/feats.scp"
  
  if [ "$normft2" == "true" ]; then
    feats2="ark,s,cs:apply-cmvn --norm-vars=false --utt2spk=ark:$sdata2/JOB/utt2spk scp:$sdata2/JOB/cmvn.scp $feats2 ark:- |"
  fi
  
  # assemble tandem features
  feats="ark,s,cs:paste-feats '$feats1' '$feats2' ark:- |"
  
  # add transformation, if applicable
  if [ "$feat_type" == "lda" ]; then
    feats="$feats transform-feats $dir/final.mat ark:- ark:- |"
  fi
  
  # splicing/normalization options
  cp $alidir/{splice_opts,tandem,normft2} $dir 2>/dev/null
  
  if [ -f $alidir/trans.1 ]; then
    echo "$0: using transforms from $alidir"
    feats="$feats transform-feats --utt2spk=ark:$sdata1/JOB/utt2spk ark,s,cs:$alidir/trans.JOB ark:- ark:- |"
  fi
  ##
  
  
  if [ $stage -le -6 ]; then
    echo "$0: accumulating tree stats"
    $cmd JOB=1:$nj $dir/log/acc_tree.JOB.log \
      acc-tree-stats  --ci-phones=$ciphonelist $alidir/final.mdl "$feats" \
      "ark:gunzip -c $alidir/ali.JOB.gz|" $dir/JOB.treeacc || exit 1;
    [ "`ls $dir/*.treeacc | wc -w`" -ne "$nj" ] && echo "$0: Wrong #tree-stats" && exit 1;
    sum-tree-stats $dir/treeacc $dir/*.treeacc 2>$dir/log/sum_tree_acc.log || exit 1;
    rm $dir/*.treeacc
  fi
  
  if [ $stage -le -5 ]; then
    echo "$0: Getting questions for tree clustering."
    # preparing questions, roots file...
    cluster-phones $dir/treeacc $lang/phones/sets.int $dir/questions.int 2> $dir/log/questions.log || exit 1;
    cat $lang/phones/extra_questions.int >> $dir/questions.int
    compile-questions $lang/topo $dir/questions.int $dir/questions.qst 2>$dir/log/compile_questions.log || exit 1;
  
    echo "$0: Building the tree"
    $cmd $dir/log/build_tree.log \
      build-tree-two-level --binary=false --verbose=1 --max-leaves-first=$num_groups \
       --max-leaves-second=$num_pdfs $dir/treeacc $lang/phones/roots.int \
       $dir/questions.qst $lang/topo $dir/tree $dir/pdf2group.map || exit 1;
  fi
  
  if [ $stage -le -4 ]; then
    echo "$0: Initializing the model"  
    # Note: if phn_dim > feat_dim+1 or spk_dim > feat_dim, these dims
    # will be truncated on initialization.
    $cmd $dir/log/init_sgmm.log \
      sgmm2-init --spk-dep-weights=$spk_dep_weights --self-weight=$self_weight \
         --pdf-map=$dir/pdf2group.map --phn-space-dim=$phn_dim \
         --spk-space-dim=$spk_dim $lang/topo $dir/tree $ubm $dir/0.mdl || exit 1;
  fi
  
  if [ $stage -le -3 ]; then
    echo "$0: doing Gaussian selection"
    $cmd JOB=1:$nj $dir/log/gselect.JOB.log \
      sgmm2-gselect $dir/0.mdl "$feats" \
      "ark,t:|gzip -c >$dir/gselect.JOB.gz" || exit 1;
  fi
  
  if [ $stage -le -2 ]; then
    echo "$0: compiling training graphs"
    text="ark:sym2int.pl --map-oov $oov -f 2- $lang/words.txt < $sdata1/JOB/text|"
    $cmd JOB=1:$nj $dir/log/compile_graphs.JOB.log \
      compile-train-graphs $dir/tree $dir/0.mdl  $lang/L.fst  \
      "$text" "ark:|gzip -c >$dir/fsts.JOB.gz" || exit 1;
  fi
  
  if [ $stage -le -1 ]; then
    echo "$0: converting alignments" 
    $cmd JOB=1:$nj $dir/log/convert_ali.JOB.log \
      convert-ali $alidir/final.mdl $dir/0.mdl $dir/tree "ark:gunzip -c $alidir/ali.JOB.gz|" \
      "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1;
  fi
  
  
  x=0
  while [ $x -lt $num_iters ]; do
     echo "$0: training pass $x ... "
     if echo $realign_iters | grep -w $x >/dev/null && [ $stage -le $x ]; then
       echo "$0: re-aligning data"
       $cmd JOB=1:$nj $dir/log/align.$x.JOB.log  \
         sgmm2-align-compiled $spkvecs_opt $scale_opts "$gselect_opt" \
         --utt2spk=ark:$sdata1/JOB/utt2spk --beam=$beam --retry-beam=$retry_beam \
         $dir/$x.mdl "ark:gunzip -c $dir/fsts.JOB.gz|" "$feats" \
         "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1;
     fi
     if [ $spk_dim -gt 0 ] && echo $spkvec_iters | grep -w $x >/dev/null; then
       if [ $stage -le $x ]; then
         $cmd JOB=1:$nj $dir/log/spkvecs.$x.JOB.log \
           ali-to-post "ark:gunzip -c $dir/ali.JOB.gz|" ark:- \| \
           weight-silence-post 0.01 $silphonelist $dir/$x.mdl ark:- ark:- \| \
           sgmm2-est-spkvecs --rand-prune=$rand_prune --spk2utt=ark:$sdata1/JOB/spk2utt \
           $spkvecs_opt "$gselect_opt" $dir/$x.mdl "$feats" ark,s,cs:- \
           ark:$dir/tmp_vecs.JOB '&&' mv $dir/tmp_vecs.JOB $dir/vecs.JOB || exit 1;
       fi
       spkvecs_opt="--spk-vecs=ark:$dir/vecs.JOB"
     fi  
     if [ $x -eq 0 ]; then
       flags=vwcSt # on the first iteration, don't update projections M or N
     elif [ $spk_dim -gt 0 -a $[$x%2] -eq 1 -a $x -ge $first_spkvec_iter ]; then 
       # Update N if we have speaker-vector space and x is odd,
       # and we've already updated the speaker vectors...
       flags=vNwSct
     else
       if [ $x -ge $update_m_iter ]; then
         flags=vMwSct # udpate M.
       else
         flags=vwSct # no M on early iters, if --update-m-iter option given.
       fi
     fi
     $spk_dep_weights && [ $x -ge $first_spkvec_iter ] && flags=${flags}u; # update 
     # spk-weight projections "u".
     
     if [ $stage -le $x ]; then
       $cmd JOB=1:$nj $dir/log/acc.$x.JOB.log \
         sgmm2-acc-stats $spkvecs_opt --utt2spk=ark:$sdata1/JOB/utt2spk \
         --update-flags=$flags "$gselect_opt" --rand-prune=$rand_prune \
         $dir/$x.mdl "$feats" "ark,s,cs:gunzip -c $dir/ali.JOB.gz | ali-to-post ark:- ark:-|" \
         $dir/$x.JOB.acc || exit 1;
     fi
  
     # The next option is needed if the user specifies a phone or speaker sub-space
     # dimension that's higher than the "normal" one.
     increase_dim_opts=
     if echo $increase_dim_iters | grep -w $x >/dev/null; then
       increase_dim_opts="--increase-phn-dim=$phn_dim --increase-spk-dim=$spk_dim"
       # Note: the command below might have a null effect on some iterations.
       if [ $spk_dim -gt $feat_dim ]; then 
         cmd JOB=1:$nj $dir/log/copy_vecs.$x.JOB.log \
           copy-vector --print-args=false --change-dim=$spk_dim \
           ark:$dir/vecs.JOB ark:$dir/vecs_tmp.$JOB '&&' \
           mv $dir/vecs_tmp.JOB $dir/vecs.JOB || exit 1;
       fi
     fi
  
     if [ $stage -le $x ]; then
       $cmd $dir/log/update.$x.log \
         sgmm2-est --update-flags=$flags --split-substates=$numsubstates \
         $increase_dim_opts --power=$power --write-occs=$dir/$[$x+1].occs \
         $dir/$x.mdl "sgmm2-sum-accs - $dir/$x.*.acc|" $dir/$[$x+1].mdl || exit 1;
       rm $dir/$x.mdl $dir/$x.*.acc $dir/$x.occs 2>/dev/null
     fi
     if [ $x -lt $max_iter_inc ]; then
       numsubstates=$[$numsubstates+$incsubstates]
     fi
     x=$[$x+1];
  done
  
  rm $dir/final.mdl $dir/final.occs 2>/dev/null
  ln -s $x.mdl $dir/final.mdl
  ln -s $x.occs $dir/final.occs
  
  if [ $spk_dim -gt 0 ]; then
    # We need to create an "alignment model" that's been trained
    # without the speaker vectors, to do the first-pass decoding with.
    # in test time.
  
    # We do this for a few iters, in this recipe.
    final_mdl=$dir/$x.mdl
    cur_alimdl=$dir/$x.mdl
    while [ $x -lt $[$num_iters+$num_iters_alimdl] ]; do
      echo "$0: building alignment model (pass $x)"
      if [ $x -eq $num_iters ]; then # 1st pass of building alimdl.
        flags=MwcS # don't update v the first time.  Note-- we never update transitions.
        # they wouldn't change anyway as we use the same alignment as previously.
      else
        flags=vMwcS
      fi
      if [ $stage -le $x ]; then
        $cmd JOB=1:$nj $dir/log/acc_ali.$x.JOB.log \
          ali-to-post "ark:gunzip -c $dir/ali.JOB.gz|" ark:- \| \
          sgmm2-post-to-gpost $spkvecs_opt "$gselect_opt" \
           --utt2spk=ark:$sdata1/JOB/utt2spk $final_mdl "$feats" ark,s,cs:- ark:- \| \
          sgmm2-acc-stats-gpost --rand-prune=$rand_prune --update-flags=$flags \
            $cur_alimdl "$feats" ark,s,cs:- $dir/$x.JOB.aliacc || exit 1;
        $cmd $dir/log/update_ali.$x.log \
          sgmm2-est --update-flags=$flags --remove-speaker-space=true --power=$power \
          $cur_alimdl "sgmm2-sum-accs - $dir/$x.*.aliacc|" $dir/$[$x+1].alimdl || exit 1;
        rm $dir/$x.*.aliacc || exit 1;
        [ $x -gt $num_iters ]  && rm $dir/$x.alimdl
      fi
      cur_alimdl=$dir/$[$x+1].alimdl
      x=$[$x+1]
    done
    rm $dir/final.alimdl 2>/dev/null 
    ln -s $x.alimdl $dir/final.alimdl
  fi
  
  utils/summarize_warnings.pl $dir/log
  
  echo Done