swbd1_train_lms_fullfsh.sh
4.46 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
#!/bin/bash -v
# Copyright 2013 Arnab Ghoshal
# Johns Hopkins University (author: Daniel Povey)
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
# MERCHANTABLITY OR NON-INFRINGEMENT.
# See the Apache 2 License for the specific language governing permissions and
# limitations under the License.
# To be run from one directory above this script.
# Begin configuration section.
fisher=
weblm=
# end configuration sections
help_message="Usage: "`basename $0`" [options] <train-txt> <dict> <out-dir>
Train language models for Switchboard-1, and optionally for Fisher and web-data from University of Washington.\n
options:
--help # print this message and exit
--fisher DIR # directory for Fisher transcripts (default: '$srilm_opts')
--weblm DIR # directory for web-data from University of Washington
";
. utils/parse_options.sh
if [ $# -ne 3 ]; then
printf "$help_message\n";
exit 1;
fi
text=$1 # data/local/train/text
lexicon=$2 # data/local/dict/lexicon.txt
dir=$3 # data/local/lm
for f in "$text" "$lexicon"; do
[ ! -f $x ] && echo "$0: No such file $f" && exit 1;
done
loc=`which ngram-count`;
if [ -z $loc ]; then
if uname -a | grep 64 >/dev/null; then # some kind of 64 bit...
sdir=`pwd`/../../../tools/srilm/bin/i686-m64
else
sdir=`pwd`/../../../tools/srilm/bin/i686
fi
if [ -f $sdir/ngram-count ]; then
echo Using SRILM tools from $sdir
export PATH=$PATH:$sdir
else
echo You appear to not have SRILM tools installed, either on your path,
echo or installed in $sdir. See tools/install_srilm.sh for installation
echo instructions.
exit 1
fi
fi
set -o errexit
mkdir -p $dir
export LC_ALL=C
heldout_sent=0
cut -d' ' -f2- $text | gzip -c > $dir/train.all.gz
cut -d' ' -f2- $text | tail -n +$heldout_sent | gzip -c > $dir/train.gz
cut -d' ' -f2- $text | head -n $heldout_sent > $dir/heldout
cut -d' ' -f1 $lexicon > $dir/wordlist
cat data/eval2000/text | cut -d' ' -f2- | tr "[:upper:]" "[:lower:]" | local/fisher_map_words.pl > $dir/eval00text
ngram-count -text $dir/train.gz -order 3 -limit-vocab -vocab $dir/wordlist \
-unk -map-unk "<unk>" -kndiscount -interpolate -lm $dir/sw1.o3g.kn.gz
echo "PPL for SWBD1 LM:"
ngram -unk -lm $dir/sw1.o3g.kn.gz -ppl $dir/eval00text
ngram -unk -lm $dir/sw1.o3g.kn.gz -ppl $dir/eval00text -debug 2 >& $dir/ppl2
# file data/local/lm/heldout: 10000 sentences, 118254 words, 0 OOVs
# 0 zeroprobs, logprob= -250952 ppl= 90.5071 ppl1= 132.479
if [ ! -z "$fisher" ]; then
[ ! -d "$fisher/data/trans" ] \
&& echo "Cannot find transcripts in Fisher directory: '$fisher'" \
&& exit 1;
mkdir -p $dir/fisher
cat data/train_fisher/text | cut -d' ' -f2- \
| gzip -c > $dir/fisher/text0.gz
gunzip -c $dir/fisher/text0.gz | local/fisher_map_words.pl \
| gzip -c > $dir/fisher/text1.gz
ngram-count -text $dir/fisher/text1.gz -order 3 -limit-vocab \
-vocab $dir/wordlist -unk -map-unk "<unk>" -kndiscount -interpolate \
-lm $dir/fisher/fisher.o3g.kn.gz
echo "PPL for Fisher LM:"
ngram -unk -lm $dir/fisher/fisher.o3g.kn.gz -ppl $dir/eval00text
ngram -unk -lm $dir/fisher/fisher.o3g.kn.gz -ppl $dir/eval00text -debug 2 \
>& $dir/fisher/ppl2
compute-best-mix $dir/ppl2 $dir/fisher/ppl2 >& $dir/sw1_fsh_mix.log
grep 'best lambda' $dir/sw1_fsh_mix.log \
| perl -e '$_=<>; s/.*\(//; s/\).*//; @A = split; die "Expecting 2 numbers; found: $_" if(@A!=2); print "$A[0]\n$A[1]\n";' > $dir/sw1_fsh_mix.weights
swb1_weight=$(head -1 $dir/sw1_fsh_mix.weights)
fisher_weight=$(tail -n 1 $dir/sw1_fsh_mix.weights)
for w in {1..9}; do
new_dir=${dir}_$w
mkdir -p ${new_dir}
ngram -lm $dir/sw1.o3g.kn.gz -lambda 0.$w \
-mix-lm $dir/fisher/fisher.o3g.kn.gz \
-unk -write-lm ${new_dir}/sw1_fsh.o3g.kn.$w.gz
done
echo "PPL for SWBD1 + Fisher LM:"
ngram -unk -lm $dir/sw1_fsh.o3g.kn.gz -ppl $dir/eval00text
fi
if [ ! -z "$weblm" ]; then
echo "Interpolating web-LM not implemented yet"
fi
## The following takes about 11 minutes to download on Eddie:
# wget --no-check-certificate http://ssli.ee.washington.edu/data/191M_conversational_web-filt+periods.gz