Blame view

egs/bentham/v1/local/chain/tuning/run_cnn_e2eali_1a.sh 10.7 KB
8dcb6dfcb   Yannick Estève   first commit
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
  #!/bin/bash
  
  # local/chain/compare_wer.sh exp/chain/e2e_cnn_1a/ exp/chain/cnn_e2eali_1a
  # System                      e2e_cnn_1a cnn_e2eali_1a
  # WER                             13.72      8.14
  # WER (rescored)                  13.40      8.00
  # CER                              6.56      2.82
  # CER (rescored)                   6.33      2.73
  # WER val                         13.51      8.19
  # WER (rescored) val              13.38      7.97
  # CER val                          6.40      2.93
  # CER (rescored) val               6.29      2.90
  # Final train prob               0.1037   -0.0613
  # Final valid prob               0.0720   -0.0988
  # Final train prob (xent)                 -0.3706
  # Final valid prob (xent)                 -0.4669
  # Parameters                     11.54M     4.29M
  
  # steps/info/chain_dir_info.pl exp/chain/cnn_e2eali_1a
  # exp/chain/cnn_e2eali_1a: num-iters=20 nj=3..5 num-params=4.3M dim=40->336 combine=-0.066->-0.066 (over 1) xent:train/valid[12,19,final]=(-0.822,-0.437,-0.371/-0.859,-0.514,-0.467) logprob:train/valid[12,19,final]=(-0.188,-0.078,-0.061/-0.204,-0.114,-0.099)
  
  set -e -o pipefail
  
  stage=0
  
  nj=30
  train_set=train
  decode_val=true
  nnet3_affix=    # affix for exp dirs, e.g. it was _cleaned in tedlium.
  affix=_1a  #affix for TDNN+LSTM directory e.g. "1a" or "1b", in case we change the configuration.
  e2echain_model_dir=exp/chain/e2e_cnn_1a
  common_egs_dir=
  reporting_email=
  
  # chain options
  train_stage=-10
  xent_regularize=0.1
  frame_subsampling_factor=4
  # training chunk-options
  chunk_width=340,300,200,100
  num_leaves=500
  # we don't need extra left/right context for TDNN systems.
  chunk_left_context=0
  chunk_right_context=0
  tdnn_dim=550
  # training options
  srand=0
  remove_egs=true
  lang_decode=data/lang
  lang_rescore=data/lang_rescore_6g
  if $decode_val; then maybe_val=val; else maybe_val= ; fi
  dropout_schedule='0,0@0.20,0.2@0.50,0'
  # End configuration section.
  echo "$0 $@"  # Print the command line for logging
  
  
  . ./cmd.sh
  . ./path.sh
  . ./utils/parse_options.sh
  
  
  if ! cuda-compiled; then
    cat <<EOF && exit 1
  This script is intended to be used with GPUs but you have not compiled Kaldi with CUDA
  If you want to use GPUs (and have them), go to src/, and configure and make on a machine
  where "nvcc" is installed.
  EOF
  fi
  
  ali_dir=exp/chain/e2e_ali_train
  lat_dir=exp/chain${nnet3_affix}/e2e_${train_set}_lats
  dir=exp/chain${nnet3_affix}/cnn_e2eali${affix}
  train_data_dir=data/${train_set}
  tree_dir=exp/chain${nnet3_affix}/tree_e2e
  
  # the 'lang' directory is created by this script.
  # If you create such a directory with a non-standard topology
  # you should probably name it differently.
  lang=data/lang_chain
  for f in $train_data_dir/feats.scp $ali_dir/ali.1.gz $ali_dir/final.mdl; do
    [ ! -f $f ] && echo "$0: expected file $f to exist" && exit 1
  done
  
  
  if [ $stage -le 1 ]; then
    echo "$0: creating lang directory $lang with chain-type topology"
    # Create a version of the lang/ directory that has one state per phone in the
    # topo file. [note, it really has two states.. the first one is only repeated
    # once, the second one has zero or more repeats.]
    if [ -d $lang ]; then
      if [ $lang/L.fst -nt data/lang/L.fst ]; then
        echo "$0: $lang already exists, not overwriting it; continuing"
      else
        echo "$0: $lang already exists and seems to be older than data/lang..."
        echo " ... not sure what to do.  Exiting."
        exit 1;
      fi
    else
      cp -r data/lang $lang
      silphonelist=$(cat $lang/phones/silence.csl) || exit 1;
      nonsilphonelist=$(cat $lang/phones/nonsilence.csl) || exit 1;
      # Use our special topology... note that later on may have to tune this
      # topology.
      steps/nnet3/chain/gen_topo.py $nonsilphonelist $silphonelist >$lang/topo
    fi
  fi
  
  if [ $stage -le 2 ]; then
    # Get the alignments as lattices (gives the chain training more freedom).
    # use the same num-jobs as the alignments
    steps/nnet3/align_lats.sh --nj $nj --cmd "$cmd" \
                              --acoustic-scale 1.0 \
                              --scale-opts '--transition-scale=1.0 --self-loop-scale=1.0' \
                              ${train_data_dir} data/lang $e2echain_model_dir $lat_dir
    echo "" >$lat_dir/splice_opts
  fi
  
  if [ $stage -le 3 ]; then
    # Build a tree using our new topology.  We know we have alignments for the
    # speed-perturbed data (local/nnet3/run_ivector_common.sh made them), so use
    # those.  The num-leaves is always somewhat less than the num-leaves from
    # the GMM baseline.
    if [ -f $tree_dir/final.mdl ]; then
      echo "$0: $tree_dir/final.mdl already exists, refusing to overwrite it."
      exit 1;
    fi
  
    steps/nnet3/chain/build_tree.sh \
      --frame-subsampling-factor $frame_subsampling_factor \
      --alignment-subsampling-factor 1 \
      --context-opts "--context-width=2 --central-position=1" \
      --cmd "$cmd" $num_leaves ${train_data_dir} \
      $lang $ali_dir $tree_dir
  fi
  
  
  if [ $stage -le 4 ]; then
    mkdir -p $dir
    echo "$0: creating neural net configs using the xconfig parser";
  
    num_targets=$(tree-info $tree_dir/tree | grep num-pdfs | awk '{print $2}')
    learning_rate_factor=$(echo "print (0.5/$xent_regularize)" | python)
    cnn_opts="l2-regularize=0.03 dropout-proportion=0.0"
    tdnn_opts="l2-regularize=0.03"
    output_opts="l2-regularize=0.04"
    common1="$cnn_opts required-time-offsets= height-offsets=-2,-1,0,1,2 num-filters-out=36"
    common2="$cnn_opts required-time-offsets= height-offsets=-2,-1,0,1,2 num-filters-out=70"
    common3="$cnn_opts required-time-offsets= height-offsets=-1,0,1 num-filters-out=70"
    mkdir -p $dir/configs
    cat <<EOF > $dir/configs/network.xconfig
    input dim=40 name=input
  
    conv-relu-batchnorm-dropout-layer name=cnn1 height-in=40 height-out=40 time-offsets=-3,-2,-1,0,1,2,3 $common1
    conv-relu-batchnorm-dropout-layer name=cnn2 height-in=40 height-out=20 time-offsets=-2,-1,0,1,2 $common1 height-subsample-out=2
    conv-relu-batchnorm-dropout-layer name=cnn3 height-in=20 height-out=20 time-offsets=-4,-2,0,2,4 $common2
    conv-relu-batchnorm-dropout-layer name=cnn4 height-in=20 height-out=20 time-offsets=-4,-2,0,2,4 $common2
    conv-relu-batchnorm-dropout-layer name=cnn5 height-in=20 height-out=10 time-offsets=-4,-2,0,2,4 $common3 height-subsample-out=2
    conv-relu-batchnorm-dropout-layer name=cnn6 height-in=10 height-out=10 time-offsets=-4,0,4 $common3
    relu-batchnorm-dropout-layer name=tdnn1 input=Append(-4,0,4) dim=$tdnn_dim $tdnn_opts dropout-proportion=0.0
    relu-batchnorm-dropout-layer name=tdnn2 input=Append(-4,0,4) dim=$tdnn_dim $tdnn_opts dropout-proportion=0.0
    relu-batchnorm-dropout-layer name=tdnn3 input=Append(-4,0,4) dim=$tdnn_dim $tdnn_opts dropout-proportion=0.0
  
    ## adding the layers for chain branch
    relu-batchnorm-layer name=prefinal-chain dim=$tdnn_dim target-rms=0.5 $tdnn_opts
    output-layer name=output include-log-softmax=false dim=$num_targets max-change=1.5 $output_opts
  
    # adding the layers for xent branch
    # This block prints the configs for a separate output that will be
    # trained with a cross-entropy objective in the 'chain' mod?els... this
    # has the effect of regularizing the hidden parts of the model.  we use
    # 0.5 / args.xent_regularize as the learning rate factor- the factor of
    # 0.5 / args.xent_regularize is suitable as it means the xent
    # final-layer learns at a rate independent of the regularization
    # constant; and the 0.5 was tuned so as to make the relative progress
    # similar in the xent and regular final layers.
    relu-batchnorm-layer name=prefinal-xent input=tdnn3 dim=$tdnn_dim target-rms=0.5 $tdnn_opts
    output-layer name=output-xent dim=$num_targets learning-rate-factor=$learning_rate_factor max-change=1.5 $output_opts
  EOF
    steps/nnet3/xconfig_to_configs.py --xconfig-file $dir/configs/network.xconfig --config-dir $dir/configs/
  fi
  
  
  if [ $stage -le 5 ]; then
    if [[ $(hostname -f) == *.clsp.jhu.edu ]] && [ ! -d $dir/egs/storage ]; then
      utils/create_split_dir.pl \
       /export/b0{3,4,5,6}/$USER/kaldi-data/egs/iam-$(date +'%m_%d_%H_%M')/s5/$dir/egs/storage $dir/egs/storage
    fi
  
    steps/nnet3/chain/train.py --stage=$train_stage \
      --cmd="$cmd" \
      --feat.cmvn-opts="--norm-means=false --norm-vars=false" \
      --chain.xent-regularize $xent_regularize \
      --chain.leaky-hmm-coefficient=0.1 \
      --chain.l2-regularize=0.00005 \
      --chain.apply-deriv-weights=true \
      --chain.lm-opts="--ngram-order=2 --no-prune-ngram-order=1 --num-extra-lm-states=1000" \
      --chain.frame-subsampling-factor=$frame_subsampling_factor \
      --chain.alignment-subsampling-factor=1 \
      --chain.left-tolerance 3 \
      --chain.right-tolerance 3 \
      --trainer.srand=$srand \
      --trainer.max-param-change=2.0 \
      --trainer.num-epochs=5 \
      --trainer.frames-per-iter=1500000 \
      --trainer.optimization.num-jobs-initial=3 \
      --trainer.optimization.num-jobs-final=5 \
      --trainer.dropout-schedule $dropout_schedule \
      --trainer.optimization.initial-effective-lrate=0.001 \
      --trainer.optimization.final-effective-lrate=0.0001 \
      --trainer.optimization.shrink-value=1.0 \
      --trainer.num-chunk-per-minibatch=32,16 \
      --trainer.optimization.momentum=0.0 \
      --egs.chunk-width=$chunk_width \
      --egs.chunk-left-context=$chunk_left_context \
      --egs.chunk-right-context=$chunk_right_context \
      --egs.chunk-left-context-initial=0 \
      --egs.chunk-right-context-final=0 \
      --egs.dir="$common_egs_dir" \
      --egs.opts="--frames-overlap-per-eg 0 --constrained false" \
      --cleanup.remove-egs=$remove_egs \
      --use-gpu=true \
      --reporting.email="$reporting_email" \
      --feat-dir=$train_data_dir \
      --tree-dir=$tree_dir \
      --lat-dir=$lat_dir \
      --dir=$dir  || exit 1;
  fi
  
  if [ $stage -le 6 ]; then
    # The reason we are using data/lang here, instead of $lang, is just to
    # emphasize that it's not actually important to give mkgraph.sh the
    # lang directory with the matched topology (since it gets the
    # topology file from the model).  So you could give it a different
    # lang directory, one that contained a wordlist and LM of your choice,
    # as long as phones.txt was compatible.
  
    utils/mkgraph.sh \
      --self-loop-scale 1.0 $lang_decode \
      $dir $dir/graph || exit 1;
  fi
  
  if [ $stage -le 7 ]; then
    frames_per_chunk=$(echo $chunk_width | cut -d, -f1)
    for decode_set in test $maybe_val; do
      steps/nnet3/decode.sh --acwt 1.0 --post-decode-acwt 10.0 \
        --extra-left-context $chunk_left_context \
        --extra-right-context $chunk_right_context \
        --extra-left-context-initial 0 \
        --extra-right-context-final 0 \
        --frames-per-chunk $frames_per_chunk \
        --nj $nj --cmd "$cmd" \
        $dir/graph data/$decode_set $dir/decode_$decode_set || exit 1;
  
      steps/lmrescore_const_arpa.sh --cmd "$cmd" $lang_decode $lang_rescore \
                                  data/$decode_set $dir/decode_${decode_set}{,_rescored} || exit 1
    done
  fi
  
  
  echo "Done. Date: $(date). Results:"
  local/chain/compare_wer.sh $dir