Blame view

egs/mini_librispeech/s5/local/nnet3/compare_wer.sh 3.55 KB
8dcb6dfcb   Yannick Estève   first commit
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
  #!/bin/bash
  
  # this script is used for comparing decoding results between systems.
  # e.g. local/chain/compare_wer.sh exp/chain/tdnn_{c,d}_sp
  # For use with discriminatively trained systems you specify the epochs after a colon:
  # for instance,
  # local/chain/compare_wer.sh exp/chain/tdnn_c_sp exp/chain/tdnn_c_sp_smbr:{1,2,3}
  
  
  if [ $# == 0 ]; then
    echo "Usage: $0: [--looped] [--online] <dir1> [<dir2> ... ]"
    echo "e.g.: $0 exp/chain/tdnn_{b,c}_sp"
    echo "or (with epoch numbers for discriminative training):"
    echo "$0 exp/chain/tdnn_b_sp_disc:{1,2,3}"
    exit 1
  fi
  
  echo "# $0 $*"
  
  include_looped=false
  if [ "$1" == "--looped" ]; then
    include_looped=true
    shift
  fi
  include_online=false
  if [ "$1" == "--online" ]; then
    include_online=true
    shift
  fi
  
  
  used_epochs=false
  
  # this function set_names is used to separate the epoch-related parts of the name
  # [for discriminative training] and the regular parts of the name.
  # If called with a colon-free directory name, like:
  #  set_names exp/chain/tdnn_lstm1e_sp_bi_smbr
  # it will set dir=exp/chain/tdnn_lstm1e_sp_bi_smbr and epoch_infix=""
  # If called with something like:
  #  set_names exp/chain/tdnn_d_sp_smbr:3
  # it will set dir=exp/chain/tdnn_d_sp_smbr and epoch_infix="_epoch3"
  
  
  set_names() {
    if [ $# != 1 ]; then
      echo "compare_wer_general.sh: internal error"
      exit 1  # exit the program
    fi
    dirname=$(echo $1 | cut -d: -f1)
    epoch=$(echo $1 | cut -s -d: -f2)
    if [ -z $epoch ]; then
      epoch_infix=""
    else
      used_epochs=true
      epoch_infix=_epoch${epoch}
    fi
  }
  
  
  
  echo -n "# System               "
  for x in $*; do   printf "% 10s" " $(basename $x)";   done
  echo
  
  strings=(
    "#WER dev_clean_2 (tgsmall) "
    "#WER dev_clean_2 (tglarge) ")
  
  for n in 0 1; do
     echo -n "${strings[$n]}"
     for x in $*; do
       set_names $x  # sets $dirname and $epoch_infix
      decode_names=(tgsmall_dev_clean_2 tglarge_dev_clean_2)
  
       wer=$(cat $dirname/decode_${decode_names[$n]}/wer_* | utils/best_wer.sh | awk '{print $2}')
       printf "% 10s" $wer
     done
     echo
     if $include_looped; then
       echo -n "#             [looped:]    "
       for x in $*; do
         set_names $x  # sets $dirname and $epoch_infix
         wer=$(cat $dirname/decode_looped_${decode_names[$n]}/wer_* | utils/best_wer.sh | awk '{print $2}')
         printf "% 10s" $wer
       done
       echo
     fi
     if $include_online; then
       echo -n "#             [online:]    "
       for x in $*; do
         set_names $x  # sets $dirname and $epoch_infix
         wer=$(cat ${dirname}_online/decode_${decode_names[$n]}/wer_* | utils/best_wer.sh | awk '{print $2}')
         printf "% 10s" $wer
       done
       echo
     fi
  done
  
  
  if $used_epochs; then
    exit 0;  # the diagnostics aren't comparable between regular and discriminatively trained systems.
  fi
  
  echo -n "# Final train prob     "
  for x in $*; do
    prob=$(grep Overall $x/log/compute_prob_train.{final,combined}.log 2>/dev/null | grep log-like | awk '{printf("%.4f", $8)}')
    printf "% 10s" $prob
  done
  echo
  
  echo -n "# Final valid prob     "
  for x in $*; do
    prob=$(grep Overall $x/log/compute_prob_valid.{final,combined}.log 2>/dev/null | grep log-like | awk '{printf("%.4f", $8)}')
    printf "% 10s" $prob
  done
  echo
  
  echo -n "# Final train acc      "
  for x in $*; do
    prob=$(grep Overall $x/log/compute_prob_train.{final,combined}.log 2>/dev/null | grep accuracy | awk '{printf("%.4f", $8)}')
    printf "% 10s" $prob
  done
  echo
  
  echo -n "# Final valid acc      "
  for x in $*; do
    prob=$(grep Overall $x/log/compute_prob_valid.{final,combined}.log 2>/dev/null | grep accuracy | awk '{printf("%.4f", $8)}')
    printf "% 10s" $prob
  done
  echo
  
  echo