run_exp.sh

#!/bin/bash

# Theano-Kaldi RNNs
# Mirco Ravanelli - Fondazione Bruno Kessler
# May 2017

# Description: 
# this bash script runs a full speech recognition experiment. 
# The first part performs the main training loop (the theano part manages only a single epoch training) as well as learning rate annealing. 
# The last part performs decoding (with kaldi decoder) of the normalized posterior probabilties generated by the RNN.


# General Parameters
cfg_file=cfg/TIMIT_M_reluGRU.cfg
theano_opts='floatX=float32,allow_gc=False,nvcc.fastmath=true,lib.cnmem=0.80'
out_folder='TIMIT_M_reluGRU'
mkdir -p $out_folder

# Training parameters
N_epochs=22
learning_rate=0.001300
ImprovementTh=0.001
halving_factor=0.5
gpu_id=0
out_name='epoch'

# Decoding Parameters
graph_dir=/nfsmnt/ramsay0/home/mravanelli/RNN_official_Jan2017/kaldi_res/s5/exp/tri3/graph
data_dir=/nfsmnt/ramsay0/home/mravanelli/RNN_official_Jan2017/kaldi_res/s5/data/test
ali_dir=/nfsmnt/ramsay0/home/mravanelli/RNN_official_Jan2017/kaldi_res/s5/exp/tri3_ali


# Initialization
pt_file='none'
do_forward='no'

echo 'Training...'
# Main Training Loop (Training+Eval)
for epoch in $(seq -w 1 $N_epochs); do 

 out_file=$out_folder'/'$out_name$epoch'.pkl'
 res_file_tr=$out_folder'/'$out_name$epoch'-train.info'
 res_file_dev=$out_folder'/'$out_name$epoch'-dev.info'
 err_file=$out_folder'/'$out_name$epoch'.err'
  
 # First Epoch 
 if [ "$epoch" -gt "1" ]; then
 dev_err_prev=$dev_err
 fi
 
 # Last Epoch (Do forward)
 if [ "$epoch" -eq $N_epochs ]; then
 do_forward='yes'
 fi
  
 # Single Epoch Training-Eval
 THEANO_FLAGS=mode=FAST_RUN,device=gpu$gpu_id,$theano_opts python RNN.py  \
 --cfg $cfg_file \
 --pt_file=$pt_file \
 --out_file=$out_file \
 --do_training=yes \
 --do_eval=yes \
 --do_forward=$do_forward \
 --learning_rate=$learning_rate \
 --seed=$epoch\234 \
 &> $err_file
 
 # Print Results
 tr_cost=`cat $res_file_tr | grep 'tr_cost' | awk -F '=' '{print $2}'`
 tr_err=`cat $res_file_tr | grep 'tr_err' | awk -F '=' '{print $2}'`
 tr_time=`cat $res_file_tr | grep 'tr_time' | awk -F '=' '{print $2}'`
 
 dev_cost=`cat $res_file_dev | grep 'dev_cost' | awk -F '=' '{print $2}'`
 dev_err=`cat $res_file_dev | grep 'dev_err' | awk -F '=' '{print $2}'`
 
 printf "epoch %s tr_cost=%s tr_err=%s dev_cost=%s dev_err=%s learning_rate=%s time=%s sec. \n" $epoch $tr_cost $tr_err $dev_cost $dev_err $learning_rate $tr_time
 
 # Learning Rate Annealing
 if [ "$epoch" -gt "1" ]; then
  relative_imp=`echo "(($dev_err_prev-$dev_err)/$dev_err)<$ImprovementTh" | bc -l`
  if [ "$relative_imp" -eq "1" ]; then
   learning_rate=`echo "$learning_rate*$halving_factor" | bc -l`
  fi
 fi
 
 # Next Epoch
 pt_file=$out_file

done

# Do Decoding with Kaldi
cd kaldi_decoding_scripts
./decode_dnn_TIMIT.sh $graph_dir $data_dir $ali_dir ../$out_folder/decoding_test "cat ../$out_folder/*.ark"