#!/bin/bash
  
  
  # This script shows how you can do data-cleaning, and exclude data that has a
  # higher likelihood of being wrongly transcribed.  see the RESULTS file; this
  # made essentially no difference in our case-- indicating, perhaps, that Fisher
  # transcripts are already clean enough.
  
  
  . ./cmd.sh
  . ./path.sh
  set -e
  
  
  steps/cleanup/find_bad_utts.sh --nj 200 --cmd "$train_cmd" data/train data/lang \
    exp/tri5a exp/tri5a_cleanup
  
   # with threshold of 0.05 we keep 1.1 million out of 1.6 million utterances, and
   # around 8.7 million out of 18.1 million words
   # with threshold of 0.1 we keep 1.3 out of 1.6 million utterances, and around
   # 13.2 million out of 18.1 million words.
  thresh=0.1
  cat exp/tri5a_cleanup/all_info.txt | awk -v threshold=$thresh '{ errs=$2;ref=$3; if (errs <= threshold*ref) { print $1; } }' > uttlist
  utils/subset_data_dir.sh --utt-list uttlist data/train data/train.thresh$thresh
  
  steps/align_fmllr.sh --nj 30 --cmd "$train_cmd" \
    data/train.thresh$thresh data/lang exp/tri4a exp/tri4a_ali_$thresh
  
  steps/train_sat.sh  --cmd "$train_cmd" \
    10000 300000 data/train data/lang exp/tri4a_ali_$thresh  exp/tri5a_$thresh || exit 1;
  
  
  utils/mkgraph.sh data/lang_test exp/tri5a_$thresh exp/tri5a_$thresh/graph
  steps/decode_fmllr.sh --nj 25 --cmd "$decode_cmd" --config conf/decode.config \
    exp/tri5a_$thresh/graph data/dev exp/tri5a_$thresh/decode_dev