# Note: these numbers are on eval2000 but don't compare them with the
# numbers Microsoft quotes in their DNN papers; those are just on the
# Switchboard portion of eval2000, excluding CallHome, which is
# substantially easier.

for x in exp/{mono,tri,sgmm,nnet}*/decode*; do [ -d $x ] && grep Sum $x/score_*/*.sys | utils/best_wer.sh; done 2>/dev/null
for x in exp/{mono,tri,sgmm,nnet}*/decode*; do [ -d $x ] && grep WER $x/wer_* | utils/best_wer.sh; done 2>/dev/null


%WER 37.2 | 1831 21395 | 65.9 24.1 10.0 3.1 37.2 67.8 | exp/tri1/decode_eval2000_sw1_fsh_tgpr/score_13/eval2000.ctm.swbd.filt.sys
%WER 37.4 | 1831 21395 | 65.9 24.1 10.0 3.4 37.4 67.9 | exp/tri1/decode_eval2000_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys
%WER 36.8 | 1831 21395 | 66.5 23.8 9.7 3.3 36.8 68.3 | exp/tri2/decode_eval2000_sw1_fsh_tgpr/score_13/eval2000.ctm.swbd.filt.sys
%WER 37.0 | 1831 21395 | 66.6 24.1 9.2 3.7 37.0 68.4 | exp/tri2/decode_eval2000_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys
%WER 29.1 | 1831 21395 | 74.0 18.3 7.7 3.1 29.1 65.4 | exp/tri3b/decode_eval2000_sw1_fsh_tgpr/score_14/eval2000.ctm.swbd.filt.sys
%WER 29.7 | 1831 21395 | 73.6 18.6 7.7 3.3 29.7 65.3 | exp/tri3b/decode_eval2000_sw1_tg/score_14/eval2000.ctm.swbd.filt.sys
%WER 24.3 | 1831 21395 | 77.9 15.0 7.1 2.3 24.3 61.5 | exp/tri4a/decode_eval2000_sw1_fsh_tgpr/score_17/eval2000.ctm.swbd.filt.sys
%WER 32.6 | 1831 21395 | 71.2 21.4 7.4 3.8 32.6 66.9 | exp/tri4a/decode_eval2000_sw1_fsh_tgpr.si/score_12/eval2000.ctm.swbd.filt.sys
%WER 25.0 | 1831 21395 | 77.7 15.6 6.6 2.8 25.0 62.4 | exp/tri4a/decode_eval2000_sw1_tg/score_15/eval2000.ctm.swbd.filt.sys
%WER 33.2 | 1831 21395 | 70.8 21.7 7.5 4.0 33.2 67.1 | exp/tri4a/decode_eval2000_sw1_tg.si/score_12/eval2000.ctm.swbd.filt.sys
%WER 23.5 | 1831 21395 | 79.0 14.7 6.3 2.5 23.5 61.3 | exp/tri4a_fmmi_b0.1/decode_eval2000_it4_sw1_fsh_tgpr/score_13/eval2000.ctm.swbd.filt.sys
%WER 23.7 | 1831 21395 | 78.7 14.8 6.5 2.4 23.7 62.0 | exp/tri4a_fmmi_b0.1/decode_eval2000_it4_sw1_tg/score_14/eval2000.ctm.swbd.filt.sys
%WER 22.1 | 1831 21395 | 80.0 13.3 6.7 2.1 22.1 60.1 | exp/tri4a_fmmi_b0.1/decode_eval2000_it5_sw1_fsh_tgpr/score_14/eval2000.ctm.swbd.filt.sys
%WER 22.6 | 1831 21395 | 79.5 13.9 6.6 2.2 22.6 60.6 | exp/tri4a_fmmi_b0.1/decode_eval2000_it5_sw1_tg/score_14/eval2000.ctm.swbd.filt.sys
%WER 21.9 | 1831 21395 | 80.6 13.8 5.6 2.5 21.9 59.3 | exp/tri4a_fmmi_b0.1/decode_eval2000_it6_sw1_fsh_tgpr/score_11/eval2000.ctm.swbd.filt.sys
%WER 22.2 | 1831 21395 | 80.2 13.9 5.9 2.4 22.2 60.2 | exp/tri4a_fmmi_b0.1/decode_eval2000_it6_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys
%WER 21.5 | 1831 21395 | 80.9 13.5 5.6 2.5 21.5 59.2 | exp/tri4a_fmmi_b0.1/decode_eval2000_it7_sw1_fsh_tgpr/score_11/eval2000.ctm.swbd.filt.sys
%WER 21.8 | 1831 21395 | 80.7 13.7 5.5 2.5 21.8 59.7 | exp/tri4a_fmmi_b0.1/decode_eval2000_it7_sw1_tg/score_11/eval2000.ctm.swbd.filt.sys
%WER 21.3 | 1831 21395 | 81.2 13.3 5.5 2.6 21.3 59.1 | exp/tri4a_fmmi_b0.1/decode_eval2000_it8_sw1_fsh_tgpr/score_11/eval2000.ctm.swbd.filt.sys
%WER 21.7 | 1831 21395 | 80.7 13.4 5.9 2.4 21.7 59.6 | exp/tri4a_fmmi_b0.1/decode_eval2000_it8_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys
%WER 23.0 | 1831 21395 | 79.2 14.2 6.5 2.2 23.0 60.5 | exp/tri4a_mmi_b0.1/decode_eval2000_1.mdl_sw1_fsh_tgpr/score_14/eval2000.ctm.swbd.filt.sys
%WER 23.5 | 1831 21395 | 79.0 14.8 6.2 2.5 23.5 60.8 | exp/tri4a_mmi_b0.1/decode_eval2000_1.mdl_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys
%WER 22.3 | 1831 21395 | 79.8 13.7 6.5 2.1 22.3 59.4 | exp/tri4a_mmi_b0.1/decode_eval2000_2.mdl_sw1_fsh_tgpr/score_14/eval2000.ctm.swbd.filt.sys
%WER 22.8 | 1831 21395 | 79.5 14.3 6.2 2.3 22.8 60.0 | exp/tri4a_mmi_b0.1/decode_eval2000_2.mdl_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys
%WER 22.0 | 1831 21395 | 80.4 13.8 5.8 2.3 22.0 59.3 | exp/tri4a_mmi_b0.1/decode_eval2000_3.mdl_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
%WER 22.4 | 1831 21395 | 79.9 13.9 6.2 2.3 22.4 59.4 | exp/tri4a_mmi_b0.1/decode_eval2000_3.mdl_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys
%WER 21.7 | 1831 21395 | 80.6 13.6 5.8 2.3 21.7 59.0 | exp/tri4a_mmi_b0.1/decode_eval2000_4.mdl_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
%WER 22.1 | 1831 21395 | 80.3 13.9 5.8 2.5 22.1 59.4 | exp/tri4a_mmi_b0.1/decode_eval2000_4.mdl_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys
%WER 21.8 | 1831 21395 | 80.5 13.7 5.8 2.3 21.8 59.3 | exp/tri4b/decode_eval2000_sw1_fsh_tgpr/score_15/eval2000.ctm.swbd.filt.sys
%WER 29.3 | 1831 21395 | 74.1 18.8 7.0 3.4 29.3 64.8 | exp/tri4b/decode_eval2000_sw1_fsh_tgpr.si/score_13/eval2000.ctm.swbd.filt.sys
%WER 22.4 | 1831 21395 | 80.0 13.9 6.1 2.4 22.4 60.0 | exp/tri4b/decode_eval2000_sw1_tg/score_16/eval2000.ctm.swbd.filt.sys
%WER 30.3 | 1831 21395 | 73.1 19.7 7.1 3.4 30.3 64.7 | exp/tri4b/decode_eval2000_sw1_tg.si/score_14/eval2000.ctm.swbd.filt.sys
%WER 20.7 | 1831 21395 | 81.3 12.8 6.0 2.0 20.7 59.3 | exp/tri4b_fmmi_b0.1/decode_eval2000_it4_sw1_fsh_tgpr/score_15/eval2000.ctm.swbd.filt.sys
%WER 21.4 | 1831 21395 | 81.0 13.2 5.8 2.4 21.4 59.3 | exp/tri4b_fmmi_b0.1/decode_eval2000_it4_sw1_tg/score_14/eval2000.ctm.swbd.filt.sys
%WER 19.6 | 1831 21395 | 82.2 12.0 5.8 1.9 19.6 57.2 | exp/tri4b_fmmi_b0.1/decode_eval2000_it5_sw1_fsh_tgpr/score_14/eval2000.ctm.swbd.filt.sys
%WER 20.2 | 1831 21395 | 81.9 12.5 5.6 2.1 20.2 56.9 | exp/tri4b_fmmi_b0.1/decode_eval2000_it5_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys
%WER 19.4 | 1831 21395 | 82.7 12.0 5.3 2.2 19.4 56.9 | exp/tri4b_fmmi_b0.1/decode_eval2000_it6_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
%WER 19.9 | 1831 21395 | 82.1 12.2 5.6 2.0 19.9 57.0 | exp/tri4b_fmmi_b0.1/decode_eval2000_it6_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys
%WER 19.3 | 1831 21395 | 82.9 12.0 5.2 2.1 19.3 56.4 | exp/tri4b_fmmi_b0.1/decode_eval2000_it7_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
%WER 19.8 | 1831 21395 | 82.4 12.3 5.3 2.2 19.8 56.6 | exp/tri4b_fmmi_b0.1/decode_eval2000_it7_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys
%WER 19.3 | 1831 21395 | 82.9 11.9 5.2 2.2 19.3 56.5 | exp/tri4b_fmmi_b0.1/decode_eval2000_it8_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
%WER 19.7 | 1831 21395 | 82.5 12.3 5.2 2.2 19.7 56.7 | exp/tri4b_fmmi_b0.1/decode_eval2000_it8_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys
%WER 20.2 | 1831 21395 | 81.6 12.4 5.9 1.9 20.2 57.6 | exp/tri4b_mmi_b0.1/decode_eval2000_1.mdl_sw1_fsh_tgpr/score_15/eval2000.ctm.swbd.filt.sys
%WER 20.7 | 1831 21395 | 81.4 12.8 5.7 2.1 20.7 57.9 | exp/tri4b_mmi_b0.1/decode_eval2000_1.mdl_sw1_tg/score_14/eval2000.ctm.swbd.filt.sys
%WER 19.7 | 1831 21395 | 82.2 12.1 5.7 1.9 19.7 57.3 | exp/tri4b_mmi_b0.1/decode_eval2000_2.mdl_sw1_fsh_tgpr/score_14/eval2000.ctm.swbd.filt.sys
%WER 20.3 | 1831 21395 | 81.9 12.6 5.5 2.2 20.3 57.8 | exp/tri4b_mmi_b0.1/decode_eval2000_2.mdl_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys
%WER 19.5 | 1831 21395 | 82.5 12.0 5.5 2.0 19.5 56.1 | exp/tri4b_mmi_b0.1/decode_eval2000_3.mdl_sw1_fsh_tgpr/score_13/eval2000.ctm.swbd.filt.sys
%WER 20.0 | 1831 21395 | 82.0 12.4 5.5 2.1 20.0 56.8 | exp/tri4b_mmi_b0.1/decode_eval2000_3.mdl_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys
%WER 19.5 | 1831 21395 | 82.7 12.5 4.8 2.3 19.5 56.4 | exp/tri4b_mmi_b0.1/decode_eval2000_4.mdl_sw1_fsh_tgpr/score_11/eval2000.ctm.swbd.filt.sys
%WER 19.9 | 1831 21395 | 82.3 12.5 5.2 2.2 19.9 56.7 | exp/tri4b_mmi_b0.1/decode_eval2000_4.mdl_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys
%WER 23.4 | 1831 21395 | 79.2 13.6 7.3 2.6 23.4 62.8 | exp/tri4c_reseg/decode_eval2000_sw1_fsh_tgpr/score_15/eval2000.ctm.swbd.filt.sys
%WER 30.4 | 1831 21395 | 73.0 18.7 8.3 3.4 30.4 68.1 | exp/tri4c_reseg/decode_eval2000_sw1_fsh_tgpr.si/score_13/eval2000.ctm.swbd.filt.sys
%WER 23.6 | 1831 21395 | 78.9 13.6 7.5 2.5 23.6 62.8 | exp/tri4c_reseg/decode_eval2000_sw1_tg/score_16/eval2000.ctm.swbd.filt.sys
%WER 31.0 | 1831 21395 | 72.7 19.0 8.3 3.7 31.0 68.5 | exp/tri4c_reseg/decode_eval2000_sw1_tg.si/score_13/eval2000.ctm.swbd.filt.sys
%WER 21.1 | 1831 21395 | 81.2 12.8 6.0 2.3 21.1 59.7 | exp/sgmm2_5a/decode_eval2000_sw1_fsh_tgpr/score_10/eval2000.ctm.swbd.filt.sys
%WER 21.3 | 1831 21395 | 80.9 13.0 6.2 2.2 21.3 59.5 | exp/sgmm2_5a/decode_eval2000_sw1_tg/score_11/eval2000.ctm.swbd.filt.sys
%WER 19.7 | 1831 21395 | 82.4 12.0 5.6 2.2 19.7 57.9 | exp/sgmm2_5a_mmi_b0.1/decode_eval2000_sw1_fsh_tgpr_it1/score_10/eval2000.ctm.swbd.filt.sys
%WER 19.1 | 1831 21395 | 82.8 11.5 5.7 1.9 19.1 56.7 | exp/sgmm2_5a_mmi_b0.1/decode_eval2000_sw1_fsh_tgpr_it2/score_11/eval2000.ctm.swbd.filt.sys
%WER 19.0 | 1831 21395 | 83.2 11.5 5.3 2.1 19.0 56.9 | exp/sgmm2_5a_mmi_b0.1/decode_eval2000_sw1_fsh_tgpr_it3/score_10/eval2000.ctm.swbd.filt.sys
%WER 18.9 | 1831 21395 | 83.3 11.6 5.1 2.2 18.9 56.9 | exp/sgmm2_5a_mmi_b0.1/decode_eval2000_sw1_fsh_tgpr_it4/score_10/eval2000.ctm.swbd.filt.sys
%WER 20.4 | 1831 21395 | 81.9 12.4 5.7 2.3 20.4 57.8 | exp/sgmm2_5a_mmi_b0.1/decode_eval2000_sw1_tg_it1/score_10/eval2000.ctm.swbd.filt.sys
%WER 19.8 | 1831 21395 | 82.5 12.1 5.4 2.3 19.8 57.3 | exp/sgmm2_5a_mmi_b0.1/decode_eval2000_sw1_tg_it2/score_10/eval2000.ctm.swbd.filt.sys
%WER 19.5 | 1831 21395 | 82.8 12.0 5.3 2.3 19.5 56.9 | exp/sgmm2_5a_mmi_b0.1/decode_eval2000_sw1_tg_it3/score_10/eval2000.ctm.swbd.filt.sys
%WER 19.5 | 1831 21395 | 82.9 12.0 5.1 2.4 19.5 56.5 | exp/sgmm2_5a_mmi_b0.1/decode_eval2000_sw1_tg_it4/score_10/eval2000.ctm.swbd.filt.sys





%WER 29.10 [ 14382 / 49427, 1963 ins, 3394 del, 9025 sub ] exp/tri4b/decode_train_dev_sw1_fsh_tgpr/wer_15
%WER 37.81 [ 18686 / 49427, 2078 ins, 4625 del, 11983 sub ] exp/tri4b/decode_train_dev_sw1_fsh_tgpr.si/wer_15
%WER 29.53 [ 14598 / 49427, 1885 ins, 3538 del, 9175 sub ] exp/tri4b/decode_train_dev_sw1_tg/wer_16
%WER 38.42 [ 18990 / 49427, 2154 ins, 4461 del, 12375 sub ] exp/tri4b/decode_train_dev_sw1_tg.si/wer_15



# some neural net GPU results.  You'll have to figure out from the scripts in local/nnet2/ what each one is.

%WER 18.8 | 1831 21395 | 82.9 11.2 5.9 1.7 18.8 56.1 | exp/nnet5a/decode_eval2000_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
%WER 19.4 | 1831 21395 | 82.4 11.6 5.9 1.8 19.4 56.2 | exp/nnet5a/decode_eval2000_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys
%WER 18.0 | 1831 21395 | 83.7 10.7 5.5 1.7 18.0 55.5 | exp/nnet5a_gpu/decode_eval2000_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
%WER 18.6 | 1831 21395 | 83.4 11.2 5.4 2.0 18.6 56.6 | exp/nnet5a_gpu/decode_eval2000_sw1_tg/score_11/eval2000.ctm.swbd.filt.sys
%WER 18.3 | 1831 21395 | 83.4 11.0 5.6 1.7 18.3 56.6 | exp/nnet5b.old/decode_eval2000_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
%WER 18.7 | 1831 21395 | 83.1 11.3 5.6 1.9 18.7 56.7 | exp/nnet5b.old/decode_eval2000_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys
%WER 18.0 | 1831 21395 | 83.7 10.7 5.5 1.7 18.0 55.5 | exp/nnet5b2.old/decode_eval2000_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
%WER 18.6 | 1831 21395 | 83.4 11.2 5.4 2.0 18.6 56.6 | exp/nnet5b2.old/decode_eval2000_sw1_tg/score_11/eval2000.ctm.swbd.filt.sys
%WER 15.3 | 1831 21395 | 86.5 9.0 4.5 1.8 15.3 53.0 | exp/nnet5c_gpu/decode_eval2000_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
%WER 15.7 | 1831 21395 | 86.1 9.3 4.5 1.9 15.7 53.3 | exp/nnet5c_gpu/decode_eval2000_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys
%WER 17.9 | 1831 21395 | 83.8 10.8 5.5 1.7 17.9 55.8 | exp/nnet6a_gpu/decode_eval2000_sw1_fsh_tgpr/score_13/eval2000.ctm.swbd.filt.sys
%WER 18.4 | 1831 21395 | 83.6 11.4 5.0 2.1 18.4 56.1 | exp/nnet6a_gpu/decode_eval2000_sw1_tg/score_11/eval2000.ctm.swbd.filt.sys

# Discriminative training on top of 5c (14.6 still not as Good as Karel's 14.1 number with this LM, but getting closer).
%WER 15.0 | 1831 21395 | 87.3 9.0 3.7 2.3 15.0 52.2 | exp/nnet6c_mpe_gpu/decode_eval2000_sw1_fsh_tgpr_epoch1/score_15/eval2000.ctm.swbd.filt.sys
%WER 14.8 | 1831 21395 | 87.3 8.9 3.8 2.2 14.8 51.9 | exp/nnet6c_mpe_gpu/decode_eval2000_sw1_fsh_tgpr_epoch2/score_16/eval2000.ctm.swbd.filt.sys
%WER 14.7 | 1831 21395 | 87.5 8.9 3.6 2.2 14.7 51.7 | exp/nnet6c_mpe_gpu/decode_eval2000_sw1_fsh_tgpr_epoch3/score_16/eval2000.ctm.swbd.filt.sys
%WER 14.6 | 1831 21395 | 87.6 8.9 3.6 2.2 14.6 51.5 | exp/nnet6c_mpe_gpu/decode_eval2000_sw1_fsh_tgpr_epoch4/score_16/eval2000.ctm.swbd.filt.sys
%WER 15.2 | 1831 21395 | 87.1 9.1 3.8 2.3 15.2 53.1 | exp/nnet6c_mpe_gpu/decode_eval2000_sw1_tg_epoch1/score_15/eval2000.ctm.swbd.filt.sys
%WER 15.1 | 1831 21395 | 87.4 9.1 3.5 2.5 15.1 52.6 | exp/nnet6c_mpe_gpu/decode_eval2000_sw1_tg_epoch2/score_13/eval2000.ctm.swbd.filt.sys
%WER 15.0 | 1831 21395 | 87.5 9.0 3.5 2.5 15.0 52.4 | exp/nnet6c_mpe_gpu/decode_eval2000_sw1_tg_epoch3/score_14/eval2000.ctm.swbd.filt.sys
%WER 14.9 | 1831 21395 | 87.4 9.1 3.5 2.4 14.9 52.3 | exp/nnet6c_mpe_gpu/decode_eval2000_sw1_tg_epoch4/score_15/eval2000.ctm.swbd.filt.sys




# Below are the results from the Karel's DNN recipe (Oct 11 2013) from 'local/run_dnn.sh'
# DNN with RBM pre-training, 6hidden layers, 2048 neurons each, ~9k outputs
%WER 15.4 | 1831 21395 | 86.2 9.0 4.8 1.6 15.4 52.2 | exp/tri4b_pretrain-dbn_dnn/decode_eval2000_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys
# sMBR sequence-discriminative training, 1+2 epochs (lattices re-generated after 1st epoch of sMBR training)
%WER 14.5 | 1831 21395 | 87.4 8.9 3.7 2.0 14.5 51.4 | exp/tri4b_pretrain-dbn_dnn_smbr/decode_eval2000_sw1_fsh_tgpr_it1/score_14/eval2000.ctm.swbd.filt.sys
%WER 14.2 | 1831 21395 | 87.7 8.7 3.6 1.9 14.2 50.4 | exp/tri4b_pretrain-dbn_dnn_smbr_iter1-lats/decode_eval2000_sw1_fsh_tgpr_it1/score_14/eval2000.ctm.swbd.filt.sys
%WER 14.1 | 1831 21395 | 87.8 8.7 3.5 1.9 14.1 50.4 | exp/tri4b_pretrain-dbn_dnn_smbr_iter1-lats/decode_eval2000_sw1_fsh_tgpr_it2/score_14/eval2000.ctm.swbd.filt.sys
# Final system rescored by sw1_fsh trigram (unpruned)
%WER 13.4 | 1831 21395 | 88.4 8.2 3.4 1.8 13.4 49.2 | exp/tri4b_pretrain-dbn_dnn_smbr_iter1-lats/decode_eval2000_sw1_fsh_tg.3_it2/score_14/eval2000.ctm.swbd.filt.sys