#!/bin/bash . path.sh echo Preparing language models for test for lm_suffix in tg; do test=data/lang_test_${lm_suffix} rm -rf data/lang_test_${lm_suffix} cp -r data/lang data/lang_test_${lm_suffix} cat input/task.arpabo | arpa2fst - | fstprint | utils/eps2disambig.pl | utils/s2eps.pl | fstcompile --isymbols=$test/words.txt --osymbols=$test/words.txt --keep_isymbols=false --keep_osymbols=false | fstrmepsilon > $test/G.fst #cat input/G.txt | utils/eps2disambig.pl | utils/s2eps.pl | fstcompile --isymbols=$test/words.txt --osymbols=$test/words.txt --keep_isymbols=false --keep_osymbols=false | fstrmepsilon > $test/G.fst fstisstochastic $test/G.fst # The output is like: # 9.14233e-05 -0.259833 # we do expect the first of these 2 numbers to be close to zero (the second is # nonzero because the backoff weights make the states sum to >1). # Because of the fiasco for these particular LMs, the first number is not # as close to zero as it could be. # Everything below is only for diagnostic. # Checking that G has no cycles with empty words on them (e.g. , ); # this might cause determinization failure of CLG. # #0 is treated as an empty word. mkdir -p tmpdir.g awk '{if(NF==1){ printf("0 0 %s %s\n", $1,$1); }} END{print "0 0 #0 #0"; print "0";}' \ < data/local/dict/lexicon.txt >tmpdir.g/select_empty.fst.txt fstcompile --isymbols=$test/words.txt --osymbols=$test/words.txt tmpdir.g/select_empty.fst.txt | \ fstarcsort --sort_type=olabel | fstcompose - $test/G.fst > tmpdir.g/empty_words.fst fstinfo tmpdir.g/empty_words.fst | grep cyclic | grep -w 'y' && echo "Language model has cycles with empty words" && exit 1 rm -r tmpdir.g done echo "Succeeded in formatting data."