Yannick Estève / ONTRAC-Kaldi

Blame view

egs/tedlium/s5_r2/local/nnet3/compare_wer.sh 3.95 KB
edit raw normal view history
8dcb6dfcb   Yannick Estève   first commit 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
  #!/bin/bash
  
  # this script is used for comparing decoding results between systems.
  # e.g. local/nnet3/compare_wer.sh exp/nnet3_cleaned/tdnn_{c,d}_sp
  # For use with discriminatively trained systems you specify the epochs after a colon:
  # for instance,
  # local/nnet3/compare_wer.sh exp/nnet3_cleaned/tdnn_c_sp exp/nnet3_cleaned/tdnn_c_sp_smbr:{1,2,3}
  
  
  if [ $# == 0 ]; then
    echo "Usage: $0: [--looped] [--online] <dir1> [<dir2> ... ]"
    echo "e.g.: $0 exp/nnet3_cleaned/tdnn_{b,c}_sp"
    echo "or (with epoch numbers for discriminative training):"
    echo "$0 exp/nnet3_cleaned/tdnn_b_sp_disc:{1,2,3}"
    exit 1
  fi
  
  echo "# $0 $*"
  
  include_looped=false
  if [ "$1" == "--looped" ]; then
    include_looped=true
    shift
  fi
  include_online=false
  if [ "$1" == "--online" ]; then
    include_online=true
    shift
  fi
  
  
  
  used_epochs=false
  
  # this function set_names is used to separate the epoch-related parts of the name
  # [for discriminative training] and the regular parts of the name.
  # If called with a colon-free directory name, like:
  #  set_names exp/chain_cleaned/tdnn_lstm1e_sp_bi_smbr
  # it will set dir=exp/chain_cleaned/tdnn_lstm1e_sp_bi_smbr and epoch_infix=""
  # If called with something like:
  #  set_names exp/chain_cleaned/tdnn_d_sp_smbr:3
  # it will set dir=exp/chain_cleaned/tdnn_d_sp_smbr and epoch_infix="_epoch3"
  
  
  set_names() {
    if [ $# != 1 ]; then
      echo "compare_wer_general.sh: internal error"
      exit 1  # exit the program
    fi
    dirname=$(echo $1 | cut -d: -f1)
    epoch=$(echo $1 | cut -s -d: -f2)
    if [ -z $epoch ]; then
      epoch_infix=""
    else
      used_epochs=true
      epoch_infix=_epoch${epoch}
    fi
  }
  
  
  
  echo -n "# System               "
  for x in $*; do   printf "% 10s" " $(basename $x)";   done
  echo
  
  strings=("# WER on dev(orig)     " "# WER on dev(rescored) " "# WER on test(orig)    " "# WER on test(rescored)")
  
  for n in 0 1 2 3; do
     echo -n "${strings[$n]}"
     for x in $*; do
       set_names $x  # sets $dirname and $epoch_infix
       decode_names=(dev${epoch_infix} dev${epoch_infix}_rescore test${epoch_infix} test${epoch_infix}_rescore)
       wer=$(grep Sum $dirname/decode_${decode_names[$n]}/score*/*ys | utils/best_wer.sh | awk '{print $2}')
       printf "% 10s" $wer
     done
     echo
     if $include_looped; then
       echo -n "#         [looped:]    "
       for x in $*; do
         set_names $x  # sets $dirname and $epoch_infix
         decode_names=(dev${epoch_infix} dev${epoch_infix}_rescore test${epoch_infix} test${epoch_infix}_rescore)
         wer=$(grep Sum $dirname/decode_looped_${decode_names[$n]}/score*/*ys | utils/best_wer.sh | awk '{print $2}')
         printf "% 10s" $wer
       done
       echo
     fi
     if $include_online; then
       echo -n "#         [online:]    "
       for x in $*; do
         set_names $x  # sets $dirname and $epoch_infix
         decode_names=(dev${epoch_infix} dev${epoch_infix}_rescore test${epoch_infix} test${epoch_infix}_rescore)
         wer=$(grep Sum ${dirname}_online/decode_${decode_names[$n]}/score*/*ys | utils/best_wer.sh | awk '{print $2}')
         printf "% 10s" $wer
       done
       echo
     fi
  done
  
  
  if $used_epochs; then
    exit 0;  # the diagnostics aren't comparable between regular and discriminatively trained systems.
  fi
  
  
  echo -n "# Final train prob     "
  for x in $*; do
    prob=$(grep Overall $x/log/compute_prob_train.{final,combined}.log 2>/dev/null | grep log-like | awk '{printf("%.4f", $8)}')
    printf "% 10s" $prob
  done
  echo
  
  echo -n "# Final valid prob     "
  for x in $*; do
    prob=$(grep Overall $x/log/compute_prob_valid.{final,combined}.log 2>/dev/null | grep log-like | awk '{printf("%.4f", $8)}')
    printf "% 10s" $prob
  done
  echo
  
  echo -n "# Final train acc      "
  for x in $*; do
    prob=$(grep Overall $x/log/compute_prob_train.{final,combined}.log 2>/dev/null | grep accuracy | awk '{printf("%.4f", $8)}')
    printf "% 10s" $prob
  done
  echo
  
  echo -n "# Final valid acc      "
  for x in $*; do
    prob=$(grep Overall $x/log/compute_prob_valid.{final,combined}.log 2>/dev/null | grep accuracy | awk '{printf("%.4f", $8)}')
    printf "% 10s" $prob
  done
  echo
  
  echo