Blame view
egs/chime4/s5_1ch/local/chain/compare_wer.sh
3.77 KB
8dcb6dfcb first commit |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 |
#!/bin/bash # this script is used for comparing decoding results between systems. # e.g. local/chain/compare_wer.sh exp/chain/tdnn_{c,d}_sp # For use with discriminatively trained systems you specify the epochs after a colon: # for instance, # local/chain/compare_wer.sh exp/chain/tdnn_c_sp exp/chain/tdnn_c_sp_smbr:{1,2,3} if [ $# == 0 ]; then echo "Usage: $0: [--looped] [--online] <dir1> [<dir2> ... ]" echo "e.g.: $0 exp/chain/tdnn_{b,c}_sp" echo "or (with epoch numbers for discriminative training):" echo "$0 exp/chain/tdnn_b_sp_disc:{1,2,3}" exit 1 fi echo "# $0 $*" include_looped=false if [ "$1" == "--looped" ]; then include_looped=true shift fi include_online=false if [ "$1" == "--online" ]; then include_online=true shift fi used_epochs=false # this function set_names is used to separate the epoch-related parts of the name # [for discriminative training] and the regular parts of the name. # If called with a colon-free directory name, like: # set_names exp/chain/tdnn_lstm1e_sp_bi_smbr # it will set dir=exp/chain/tdnn_lstm1e_sp_bi_smbr and epoch_infix="" # If called with something like: # set_names exp/chain/tdnn_d_sp_smbr:3 # it will set dir=exp/chain/tdnn_d_sp_smbr and epoch_infix="_epoch3" set_names() { if [ $# != 1 ]; then echo "compare_wer_general.sh: internal error" exit 1 # exit the program fi dirname=$(echo $1 | cut -d: -f1) epoch=$(echo $1 | cut -s -d: -f2) if [ -z $epoch ]; then epoch_infix="" else used_epochs=true epoch_infix=_epoch${epoch} fi } echo -n "# System " for x in $*; do printf "% 10s" " $(basename $x)"; done echo strings=( "#WER dev93 (tgpr) " "#WER dev93 (tg) " "#WER dev93 (big-dict,tgpr) " "#WER dev93 (big-dict,fg) " "#WER eval92 (tgpr) " "#WER eval92 (tg) " "#WER eval92 (big-dict,tgpr)" "#WER eval92 (big-dict,fg) ") for n in 0 1 2 3 4 5 6 7; do echo -n "${strings[$n]}" for x in $*; do set_names $x # sets $dirname and $epoch_infix decode_names=(tgpr_dev93 tg_dev93 bd_tgpr_dev93 bd_tgpr_dev93_fg tgpr_eval92 tg_eval92 bd_tgpr_eval92 bd_tgpr_eval92_fg) wer=$(cat $dirname/decode_${decode_names[$n]}/scoring_kaldi/best_wer | utils/best_wer.sh | awk '{print $2}') printf "% 10s" $wer done echo if $include_looped; then echo -n "# [looped:] " for x in $*; do set_names $x # sets $dirname and $epoch_infix wer=$(cat $dirname/decode_looped_${decode_names[$n]}/scoring_kaldi/best_wer | utils/best_wer.sh | awk '{print $2}') printf "% 10s" $wer done echo fi if $include_online; then echo -n "# [online:] " for x in $*; do set_names $x # sets $dirname and $epoch_infix wer=$(cat ${dirname}_online/decode_${decode_names[$n]}/scoring_kaldi/best_wer | utils/best_wer.sh | awk '{print $2}') printf "% 10s" $wer done echo fi done if $used_epochs; then exit 0; # the diagnostics aren't comparable between regular and discriminatively trained systems. fi echo -n "# Final train prob " for x in $*; do prob=$(grep Overall $x/log/compute_prob_train.final.log | grep -v xent | awk '{printf("%.4f", $8)}') printf "% 10s" $prob done echo echo -n "# Final valid prob " for x in $*; do prob=$(grep Overall $x/log/compute_prob_valid.final.log | grep -v xent | awk '{printf("%.4f", $8)}') printf "% 10s" $prob done echo echo -n "# Final train prob (xent)" for x in $*; do prob=$(grep Overall $x/log/compute_prob_train.final.log | grep -w xent | awk '{printf("%.4f", $8)}') printf "% 10s" $prob done echo echo -n "# Final valid prob (xent)" for x in $*; do prob=$(grep Overall $x/log/compute_prob_valid.final.log | grep -w xent | awk '{printf("%.4f", $8)}') printf "% 10s" $prob done echo |