Yannick Estève / ONTRAC-Kaldi

Blame view

egs/tedlium/s5/run.sh 6.48 KB
edit raw normal view history
8dcb6dfcb   Yannick Estève   first commit 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
  #!/bin/bash
  #
  # Based mostly on the Switchboard recipe. The training database is TED-LIUM,
  # it consists of TED talks with cleaned automatic transcripts:
  #
  # http://www-lium.univ-lemans.fr/en/content/ted-lium-corpus
  # http://www.openslr.org/resources (Mirror).
  #
  # Note: this only trains on the tedlium-1 data, there is now a second
  # release which we plan to incorporate in a separate directory, e.g
  # s5b or s5-release2.
  #
  # The data is distributed under 'Creative Commons BY-NC-ND 3.0' license,
  # which allow free non-commercial use, while only a citation is required.
  #
  # Copyright  2014  Nickolay V. Shmyrev
  #            2014  Brno University of Technology (Author: Karel Vesely)
  #            2016  Johs Hopkins University (Author: Daniel Povey)
  # Apache 2.0
  #
  
  . ./cmd.sh
  . ./path.sh
  
  nj=40
  decode_nj=8
  
  stage=0
  
  . utils/parse_options.sh  # accept options.. you can run this run.sh with the
                            # --stage option, for instance, if you don't want to
                            # change it in the script.
  
  # Data preparation
  if [ $stage -le 0 ]; then
    local/download_data.sh || exit 1
  
    local/prepare_data.sh || exit 1
  
    local/prepare_dict.sh || exit 1
  
    utils/prepare_lang.sh data/local/dict_nosp \
      "<unk>" data/local/lang_nosp data/lang_nosp || exit 1
  
    local/prepare_lm.sh || exit 1
  
  fi
  
  # Feature extraction
  if [ $stage -le 1 ]; then
    for set in test dev train; do
      dir=data/$set
      steps/make_mfcc.sh --nj 20 --cmd "$train_cmd" $dir $dir/log $dir/data || exit 1
      steps/compute_cmvn_stats.sh $dir $dir/log $dir/data || exit 1
    done
  fi
  
  # Now we have 118 hours of training data.
  # Let's create a subset with 10k short segments to make flat-start training easier:
  if [ $stage -le 2 ]; then
    utils/subset_data_dir.sh --shortest data/train 10000 data/train_10kshort || exit 1
    utils/data/remove_dup_utts.sh 10 data/train_10kshort data/train_10kshort_nodup || exit 1
  fi
  
  # Train
  if [ $stage -le 3 ]; then
    steps/train_mono.sh --nj 20 --cmd "$train_cmd" \
      data/train_10kshort_nodup data/lang_nosp exp/mono0a || exit 1
  
    steps/align_si.sh --nj $nj --cmd "$train_cmd" \
      data/train data/lang_nosp exp/mono0a exp/mono0a_ali || exit 1
  
    steps/train_deltas.sh --cmd "$train_cmd" \
      2500 30000 data/train data/lang_nosp exp/mono0a_ali exp/tri1 || exit 1
  
    utils/mkgraph.sh data/lang_nosp_test exp/tri1 exp/tri1/graph_nosp || exit 1
  
    steps/decode.sh --nj $decode_nj --cmd "$decode_cmd" \
      --num-threads 4 \
      exp/tri1/graph_nosp data/dev exp/tri1/decode_nosp_dev || exit 1
    steps/decode.sh --nj $decode_nj --cmd "$decode_cmd" \
      --num-threads 4 \
      exp/tri1/graph_nosp data/test exp/tri1/decode_nosp_test || exit 1
  fi
  
  if [ $stage -le 4 ]; then
    steps/align_si.sh --nj $nj --cmd "$train_cmd" \
      data/train data/lang_nosp exp/tri1 exp/tri1_ali || exit 1
  
    steps/train_lda_mllt.sh --cmd "$train_cmd" \
      4000 50000 data/train data/lang_nosp exp/tri1_ali exp/tri2 || exit 1
  
    utils/mkgraph.sh data/lang_nosp_test exp/tri2 exp/tri2/graph_nosp || exit 1
  
    steps/decode.sh --nj $decode_nj --cmd "$decode_cmd" \
      --num-threads 4 \
      exp/tri2/graph_nosp data/dev exp/tri2/decode_nosp_dev || exit 1
    steps/decode.sh --nj $decode_nj --cmd "$decode_cmd" \
      --num-threads 4 \
      exp/tri2/graph_nosp data/test exp/tri2/decode_nosp_test || exit 1
  fi
  
  if [ $stage -le 5 ]; then
    steps/get_prons.sh --cmd "$train_cmd" data/train data/lang_nosp exp/tri2
    utils/dict_dir_add_pronprobs.sh --max-normalize true \
      data/local/dict_nosp exp/tri2/pron_counts_nowb.txt \
      exp/tri2/sil_counts_nowb.txt \
      exp/tri2/pron_bigram_counts_nowb.txt data/local/dict
  
    utils/prepare_lang.sh data/local/dict "<unk>" data/local/lang data/lang
    cp -rT data/lang data/lang_test
    cp -rT data/lang data/lang_rescore
    cp data/lang_nosp_test/G.fst data/lang_test
    cp data/lang_nosp_rescore/G.carpa data/lang_rescore
  
    utils/mkgraph.sh data/lang_test exp/tri2 exp/tri2/graph || exit 1
  
    steps/decode.sh --nj $decode_nj --cmd "$decode_cmd" \
      --num-threads 4 \
      exp/tri2/graph data/dev exp/tri2/decode_dev || exit 1
    steps/decode.sh --nj $decode_nj --cmd "$decode_cmd" \
      --num-threads 4 \
      exp/tri2/graph data/test exp/tri2/decode_test || exit 1
  fi
  
  if [ $stage -le 6 ]; then
    steps/align_si.sh --nj $nj --cmd "$train_cmd" \
      data/train data/lang exp/tri2 exp/tri2_ali || exit 1
  
    steps/train_sat.sh --cmd "$train_cmd" \
      5000 100000 data/train data/lang exp/tri2_ali exp/tri3 || exit 1
  
    utils/mkgraph.sh data/lang_test exp/tri3 exp/tri3/graph || exit 1
  
    steps/decode_fmllr.sh --nj $decode_nj --cmd "$decode_cmd" \
      --num-threads 4 \
      exp/tri3/graph data/dev exp/tri3/decode_dev || exit 1
    steps/decode_fmllr.sh --nj $decode_nj --cmd "$decode_cmd" \
      --num-threads 4 \
      exp/tri3/graph data/test exp/tri3/decode_test || exit 1
  fi
  
  # steps/cleanup/debug_lexicon.sh --nj 100 --alidir exp/tri3 --cmd "$train_cmd" data/train data/lang exp/tri3 data/local/dict/lexicon.txt exp/tri3_debug_lexicon &
  
  if [ $stage -le 7 ]; then
    steps/align_fmllr.sh --nj $nj --cmd "$train_cmd" \
      data/train data/lang exp/tri3 exp/tri3_ali || exit 1
  
    steps/make_denlats.sh --transform-dir exp/tri3_ali --nj $nj --cmd "$decode_cmd" \
      data/train data/lang exp/tri3 exp/tri3_denlats || exit 1
  
    steps/train_mmi.sh --cmd "$train_cmd" --boost 0.1 \
      data/train data/lang exp/tri3_ali exp/tri3_denlats \
      exp/tri3_mmi_b0.1 || exit 1
  
    for iter in 4; do
    steps/decode.sh --transform-dir exp/tri3/decode_dev --nj $decode_nj --cmd "$decode_cmd" --iter $iter \
      --num-threads 4 \
      exp/tri3/graph data/dev exp/tri3_mmi_b0.1/decode_dev_it$iter || exit 1
    steps/decode.sh --transform-dir exp/tri3/decode_test --nj $decode_nj --cmd "$decode_cmd" --iter $iter \
      --num-threads 4 \
      exp/tri3/graph data/test exp/tri3_mmi_b0.1/decode_test_it$iter || exit 1
    done
  fi
  # Run the DNN recipe on fMLLR feats:
  local/nnet/run_dnn.sh || exit 1
  for decode_dir in "exp/dnn4_pretrain-dbn_dnn/decode_test" "exp/dnn4_pretrain-dbn_dnn_smbr_i1lats/decode_test_it4"; do
    steps/lmrescore_const_arpa.sh data/lang_test data/lang_rescore data/test $decode_dir $decode_dir.rescore
  done
  # DNN recipe with bottle-neck features
  #local/nnet/run_dnn_bn.sh
  # Rescore with 4-gram LM:
  #decode_dir=exp/dnn8f_BN_pretrain-dbn_dnn_smbr/decode_test_it4
  #steps/lmrescore_const_arpa.sh data/lang_test data/lang_rescore data/test $decode_dir $decode_dir.rescore || exit 1
  
  ## Run the nnet2 multisplice recipe
  # local/online/run_nnet2_ms.sh || exit 1;
  ## Run discriminative training on the top of multisplice recipe
  # local/online/run_nnet2_ms_disc.sh || exit 1;
  
  
  echo success...
  exit 0