#!/bin/bash
  
  . ./path.sh
  
  echo Preparing language models for test
  
  for lm_suffix in tg; do
    test=data/lang_test_${lm_suffix}
  
    rm -rf data/lang_test_${lm_suffix}
    cp -r data/lang data/lang_test_${lm_suffix}
  
    arpa2fst --disambig-symbol=#0 --read-symbol-table=$test/words.txt input/task.arpabo $test/G.fst
  
    fstisstochastic $test/G.fst
  
   # The output is like:
   # 9.14233e-05 -0.259833
   # we do expect the first of these 2 numbers to be close to zero (the second is
   # nonzero because the backoff weights make the states sum to >1).
   # Because of the <s> fiasco for these particular LMs, the first number is not
   # as close to zero as it could be.
  
    # Everything below is only for diagnostic.
    # Checking that G has no cycles with empty words on them (e.g. <s>, </s>);
    # this might cause determinization failure of CLG.
    # #0 is treated as an empty word.
    mkdir -p tmpdir.g
    awk '{if(NF==1){ printf("0 0 %s %s
  ", $1,$1); }} END{print "0 0 #0 #0"; print "0";}' \
      < data/local/dict/lexicon.txt  >tmpdir.g/select_empty.fst.txt
    fstcompile --isymbols=$test/words.txt --osymbols=$test/words.txt tmpdir.g/select_empty.fst.txt | \
     fstarcsort --sort_type=olabel | fstcompose - $test/G.fst > tmpdir.g/empty_words.fst
    fstinfo tmpdir.g/empty_words.fst | grep cyclic | grep -w 'y' &&
      echo "Language model has cycles with empty words" && exit 1
    rm -r tmpdir.g
  done
  
  echo "Succeeded in formatting data."