#!/bin/bash
  
  # Copyright 2012  Johns Hopkins University (Author: Daniel Povey).  Apache 2.0.
  
  # This version of the train_sgmm2 script has several jobs on each machine, and adds the
  # accumulators up in memory.
  
  # SGMM training, with speaker vectors.  This script would normally be called on
  # top of fMLLR features obtained from a conventional system, but it also works
  # on top of any type of speaker-independent features (based on
  # deltas+delta-deltas or LDA+MLLT).  For more info on SGMMs, see the paper "The
  # subspace Gaussian mixture model--A structured model for speech recognition".
  # (Computer Speech and Language, 2011).
  
  # Begin configuration section.
  cmd=run.pl
  stage=-6 # use this to resume partially finished training 
  context_opts= # e.g. set it to "--context-width=5 --central-position=2"  for a
  # quinphone system.
  scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1"
  num_iters=25   # Total number of iterations of training
  num_iters_alimdl=3 # Number of iterations for estimating alignment model.
  max_iter_inc=15 # Last iter to increase #substates on.
  realign_iters="5 10 15"; # Iters to realign on. 
  spkvec_iters="5 8 12 17" # Iters to estimate speaker vectors on.
  increase_iters="6 10 14"; # Iters on which to increase phn dim and/or spk dim;
      # rarely necessary, and if it is, only the 1st will normally be necessary.
  rand_prune=0.1 # Randomized-pruning parameter for posteriors, to speed up training.
                 # Bigger -> more pruning; zero = no pruning.
  phn_dim=  # You can use this to set the phonetic subspace dim. [default: feat-dim+1]
  spk_dim=  # You can use this to set the speaker subspace dim. [default: feat-dim]
  power=0.25 # Exponent for number of gaussians according to occurrence counts
  beam=8
  self_weight=0.9
  retry_beam=40
  leaves_per_group=5 # Relates to the SCTM (state-clustered tied-mixture) aspect:
                     # average number of pdfs in a "group" of pdfs.
  update_m_iter=4
  spk_dep_weights=true # [Symmetric SGMM] set this to false if you don't want "u" (i.e. to turn off
                        # symmetric SGMM.
  group=3 # Number of jobs to group together on a single machine, and add the stats locally.
  parallel_opts="-pe smp 3" # You should make this match "group".
  
  echo "$0 $@"  # Print the command line for logging
  
  if [ -f path.sh ]; then . ./path.sh; fi
  . parse_options.sh || exit 1;
  
  
  if [ $# != 7 ]; then
    echo "Usage: steps/train_sgmm2.sh <num-leaves> <num-substates> <data> <lang> <ali-dir> <ubm> <exp-dir>"
    echo " e.g.: steps/train_sgmm2.sh 5000 8000 data/train_si84 data/lang \\"
    echo "                      exp/tri3b_ali_si84 exp/ubm4a/final.ubm exp/sgmm4a"
    echo "main options (for others, see top of script file)"
    echo "  --group <n>                                      # number of jobs on one machine, default 3."
    echo "  --parallel-opts <opts>                           # string to append to queue command, default '-pe smp 3'"
    echo "  --config <config-file>                           # config containing options"
    echo "  --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs."
    echo "  --silence-weight <sil-weight>                    # weight for silence (e.g. 0.5 or 0.0)"
    echo "  --num-iters <#iters>                             # Number of iterations of E-M"
    exit 1;
  fi
  
  num_pdfs=$1  # final #leaves, at 2nd level of tree.
  totsubstates=$2
  data=$3
  lang=$4
  alidir=$5
  ubm=$6
  dir=$7
  
  num_groups=$[$num_pdfs/$leaves_per_group]
  first_spkvec_iter=`echo $spkvec_iters | awk '{print $1}'` || exit 1;
  
  # Check some files.
  for f in $data/feats.scp $lang/L.fst $alidir/ali.1.gz $alidir/final.mdl $ubm; do
    [ ! -f $f ] && echo "$0: no such file $f" && exit 1;
  done
  
  # Set some variables.
  oov=`cat $lang/oov.int`
  silphonelist=`cat $lang/phones/silence.csl`
  if [ "$self_weight" == "1.0" ]; then
    numsubstates=$num_groups # Initial #-substates.
  else
    numsubstates=$num_pdfs # Initial #-substates.
  fi
  incsubstates=$[($totsubstates-$numsubstates)/$max_iter_inc] # per-iter increment for #substates
  feat_dim=`gmm-info $alidir/final.mdl 2>/dev/null | awk '/feature dimension/{print $NF}'` || exit 1;
  [ $feat_dim -eq $feat_dim ] || exit 1; # make sure it's numeric.
  [ -z $phn_dim ] && phn_dim=$[$feat_dim+1]
  [ -z $spk_dim ] && spk_dim=$feat_dim
  nj=`cat $alidir/num_jobs` || exit 1;
  splice_opts=`cat $alidir/splice_opts 2>/dev/null` # frame-splicing options.
  
  mkdir -p $dir/log
  cp $alidir/splice_opts $dir 2>/dev/null # frame-splicing options.
  echo $nj > $dir/num_jobs
  sdata=$data/split$nj;
  [[ -d $sdata && $data/feats.scp -ot $sdata ]] || split_data.sh $data $nj || exit 1;
  
  spkvecs_opt=  # Empty option for now, until we estimate the speaker vectors.
  gselect_opt="--gselect=ark,s,cs:gunzip -c $dir/gselect.JOB.gz|"
  
  ## Set up features.
  if [ -f $alidir/final.mat ]; then feat_type=lda; else feat_type=delta; fi
  echo "$0: feature type is $feat_type"
  
  case $feat_type in
    delta) feats="ark,s,cs:apply-cmvn --norm-vars=false --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | add-deltas ark:- ark:- |";;
    lda) feats="ark,s,cs:apply-cmvn --norm-vars=false --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | splice-feats $splice_opts ark:- ark:- | transform-feats $alidir/final.mat ark:- ark:- |"
      cp $alidir/final.mat $dir    
      ;;
    *) echo "$0: invalid feature type $feat_type" && exit 1;
  esac
  if [ -f $alidir/trans.1 ]; then
    echo "$0: using transforms from $alidir"
    feats="$feats transform-feats --utt2spk=ark:$sdata/JOB/utt2spk ark,s,cs:$alidir/trans.JOB ark:- ark:- |"
  fi
  ##
  
  
  if [ $stage -le -6 ]; then
    echo "$0: accumulating tree stats"
    $cmd JOB=1:$nj $dir/log/acc_tree.JOB.log \
      acc-tree-stats  --ci-phones=$ciphonelist $alidir/final.mdl "$feats" \
      "ark:gunzip -c $alidir/ali.JOB.gz|" $dir/JOB.treeacc || exit 1;
    [ "`ls $dir/*.treeacc | wc -w`" -ne "$nj" ] && echo "$0: Wrong #tree-stats" && exit 1;
    sum-tree-stats $dir/treeacc $dir/*.treeacc 2>$dir/log/sum_tree_acc.log || exit 1;
    rm $dir/*.treeacc
  fi
  
  if [ $stage -le -5 ]; then
    echo "$0: Getting questions for tree clustering."
    # preparing questions, roots file...
    cluster-phones $dir/treeacc $lang/phones/sets.int $dir/questions.int 2> $dir/log/questions.log || exit 1;
    cat $lang/phones/extra_questions.int >> $dir/questions.int
    compile-questions $lang/topo $dir/questions.int $dir/questions.qst 2>$dir/log/compile_questions.log || exit 1;
  
    echo "$0: Building the tree"
    $cmd $dir/log/build_tree.log \
      build-tree-two-level --binary=false --verbose=1 --max-leaves-first=$num_groups \
       --max-leaves-second=$num_pdfs $dir/treeacc $lang/phones/roots.int \
       $dir/questions.qst $lang/topo $dir/tree $dir/pdf2group.map || exit 1;
  fi
  
  if [ $stage -le -4 ]; then
    echo "$0: Initializing the model"  
    # Note: if phn_dim > feat_dim+1 or spk_dim > feat_dim, these dims
    # will be truncated on initialization.
    $cmd $dir/log/init_sgmm.log \
      sgmm2-init --spk-dep-weights=$spk_dep_weights --self-weight=$self_weight \
         --pdf-map=$dir/pdf2group.map --phn-space-dim=$phn_dim \
         --spk-space-dim=$spk_dim $lang/topo $dir/tree $ubm $dir/0.mdl || exit 1;
  fi
  
  if [ $stage -le -3 ]; then
    echo "$0: doing Gaussian selection"
    $cmd JOB=1:$nj $dir/log/gselect.JOB.log \
      sgmm2-gselect $dir/0.mdl "$feats" \
      "ark,t:|gzip -c >$dir/gselect.JOB.gz" || exit 1;
  fi
  
  if [ $stage -le -2 ]; then
    echo "$0: compiling training graphs"
    text="ark:sym2int.pl --map-oov $oov -f 2- $lang/words.txt < $sdata/JOB/text|"
    $cmd JOB=1:$nj $dir/log/compile_graphs.JOB.log \
      compile-train-graphs $dir/tree $dir/0.mdl  $lang/L.fst  \
      "$text" "ark:|gzip -c >$dir/fsts.JOB.gz" || exit 1;
  fi
  
  if [ $stage -le -1 ]; then
    echo "$0: converting alignments" 
    $cmd JOB=1:$nj $dir/log/convert_ali.JOB.log \
      convert-ali $alidir/final.mdl $dir/0.mdl $dir/tree "ark:gunzip -c $alidir/ali.JOB.gz|" \
      "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1;
  fi
  
  
  x=0
  while [ $x -lt $num_iters ]; do
     echo "$0: training pass $x ... "
     if echo $realign_iters | grep -w $x >/dev/null && [ $stage -le $x ]; then
       echo "$0: re-aligning data"
       $cmd JOB=1:$nj $dir/log/align.$x.JOB.log  \
         sgmm2-align-compiled $spkvecs_opt $scale_opts "$gselect_opt" \
         --utt2spk=ark:$sdata/JOB/utt2spk --beam=$beam --retry-beam=$retry_beam \
         $dir/$x.mdl "ark:gunzip -c $dir/fsts.JOB.gz|" "$feats" \
         "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1;
     fi
     if [ $spk_dim -gt 0 ] && echo $spkvec_iters | grep -w $x >/dev/null; then
       if [ $stage -le $x ]; then
         $cmd JOB=1:$nj $dir/log/spkvecs.$x.JOB.log \
           ali-to-post "ark:gunzip -c $dir/ali.JOB.gz|" ark:- \| \
           weight-silence-post 0.01 $silphonelist $dir/$x.mdl ark:- ark:- \| \
           sgmm2-est-spkvecs --rand-prune=$rand_prune --spk2utt=ark:$sdata/JOB/spk2utt \
           $spkvecs_opt "$gselect_opt" $dir/$x.mdl "$feats" ark,s,cs:- \
           ark:$dir/tmp_vecs.JOB '&&' mv $dir/tmp_vecs.JOB $dir/vecs.JOB || exit 1;
       fi
       spkvecs_opt="--spk-vecs=ark:$dir/vecs.JOB"
     fi  
     if [ $x -eq 0 ]; then
       flags=vwcSt # on the first iteration, don't update projections M or N
     elif [ $spk_dim -gt 0 -a $[$x%2] -eq 1 -a $x -ge $first_spkvec_iter ]; then 
       # Update N if we have speaker-vector space and x is odd,
       # and we've already updated the speaker vectors...
       flags=vNwSct
     else
       if [ $x -ge $update_m_iter ]; then
         flags=vMwSct # udpate M.
       else
         flags=vwSct # no M on early iters, if --update-m-iter option given.
       fi
     fi
     $spk_dep_weights && [ $x -ge $first_spkvec_iter ] && flags=${flags}u; # update 
     # spk-weight projections "u".
     
     # Submit separate jobs for small groups (of size $group) of accumulators.
     Args=() # bash array of training commands for 1:nj, that put accs to stdout.
     for n in `seq $nj`; do
       Args[$n]=`echo "sgmm2-acc-stats $spkvecs_opt --utt2spk=ark:$sdata/JOB/utt2spk \
               --update-flags=$flags '$gselect_opt' --rand-prune=$rand_prune \
               $dir/$x.mdl '$feats' 'ark,s,cs:gunzip -c $dir/ali.JOB.gz | ali-to-post ark:- ark:-|' - |" | sed s/JOB/$n/g`
     done
     
     g=0
     rm $dir/.error 2>/dev/null
     if [ $stage -le $x ]; then
       while [ $[$g*$group] -lt $nj ]; do
         if [ -s $dir/acc.$x.$g.gz ]; then
           echo "Skipping creation of acc $dir/acc.$x.$g.gz as it already exists."
         else 
           start=$[$g*$group + 1]; # start-position in array Args.
         # see http://www.thegeekstuff.com/2010/06/bash-array-tutorial/, this uses Bash arrays."
         # The syntax "${Args[@]:$start:$group}" is equivalent to, say,
         # "${Args[3]}" "${Args[4]}" if start=3 and group=2.  Except it's smart about the end
         # of the array, it won't give you empty quoted strings if the length "group" takes you off
         # the end of the array.
           $cmd $parallel_opts $dir/log/acc.$x.$g.log \
             sgmm2-sum-accs --parallel=true "|gzip -c >$dir/acc.$x.$g.gz" "${Args[@]:$start:$group}"  || touch $dir/.error &
         fi
         g=$[$g+1];
       done
       wait
       if [ -f $dir/.error ]; then
         echo "Something went wrong during accumulation on pass $x"
         exit 1;
       fi
     fi
     
     # The next option is needed if the user specifies a phone or speaker sub-space
     # dimension that's higher than the "normal" one.
     increase_dim_opts=
     if echo $increase_dim_iters | grep -w $x >/dev/null; then
       increase_dim_opts="--increase-phn-dim=$phn_dim --increase-spk-dim=$spk_dim"
       # Note: the command below might have a null effect on some iterations.
       if [ $spk_dim -gt $feat_dim ]; then 
         cmd JOB=1:$nj $dir/log/copy_vecs.$x.JOB.log \
           copy-vector --print-args=false --change-dim=$spk_dim \
           ark:$dir/vecs.JOB ark:$dir/vecs_tmp.$JOB '&&' \
           mv $dir/vecs_tmp.JOB $dir/vecs.JOB || exit 1;
       fi
     fi
  
     if [ $stage -le $x ]; then
       acc_sum="sgmm2-sum-accs - ";
       for j in `seq 0 $[$g-1]`; do acc_sum="$acc_sum 'gunzip -c $dir/acc.$x.$j.gz|'"; done
       $cmd $dir/log/update.$x.log \
         sgmm2-est --update-flags=$flags --split-substates=$numsubstates \
         $increase_dim_opts --power=$power --write-occs=$dir/$[$x+1].occs \
         $dir/$x.mdl "$acc_sum|" $dir/$[$x+1].mdl || exit 1;
       rm $dir/$x.mdl $dir/acc.$x.*.gz $dir/$x.occs 2>/dev/null
     fi
     if [ $x -lt $max_iter_inc ]; then
       numsubstates=$[$numsubstates+$incsubstates]
     fi
     x=$[$x+1];
  done
  
  rm $dir/final.mdl $dir/final.occs 2>/dev/null
  ln -s $x.mdl $dir/final.mdl
  ln -s $x.occs $dir/final.occs
  
  if [ $spk_dim -gt 0 ]; then
    # We need to create an "alignment model" that's been trained
    # without the speaker vectors, to do the first-pass decoding with.
    # in test time.
  
    # We do this for a few iters, in this recipe.
    final_mdl=$dir/$x.mdl
    cur_alimdl=$dir/$x.mdl
    while [ $x -lt $[$num_iters+$num_iters_alimdl] ]; do
      echo "$0: building alignment model (pass $x)"
      if [ $x -eq $num_iters ]; then # 1st pass of building alimdl.
        flags=MwcS # don't update v the first time.  Note-- we never update transitions.
        # they wouldn't change anyway as we use the same alignment as previously.
      else
        flags=vMwcS
      fi
      if [ $stage -le $x ]; then
        Args=() # bash array of training commands for 1:nj, that put accs to stdout.
        for n in `seq $nj`; do
          Args[$n]=`echo "ali-to-post 'ark:gunzip -c $dir/ali.JOB.gz|' ark:- | \
            sgmm2-post-to-gpost $spkvecs_opt '$gselect_opt' \
            --utt2spk=ark:$sdata/JOB/utt2spk $final_mdl '$feats' ark,s,cs:- ark:- | \
                    sgmm2-acc-stats-gpost --rand-prune=$rand_prune --update-flags=$flags \
            $cur_alimdl '$feats' ark,s,cs:- - |" | sed s/JOB/$n/g`
        done
        g=0
        rm $dir/.error 2>/dev/null
        while [ $[$g*$group] -lt $nj ]; do
          if [ -s $dir/acc.$x.$g.gz ]; then
            echo "Skipping creation of acc $dir/acc.$x.$g.gz as it already exists."
          else 
            start=$[$g*$group + 1]; # start-position in array Args.
            $cmd $parallel_opts $dir/log/acc.$x.$g.log \
              sgmm2-sum-accs --parallel=true "|gzip -c >$dir/acc.$x.$g.gz" "${Args[@]:$start:$group}"  || touch $dir/.error &
          fi
          g=$[$g+1];
        done
        wait
        if [ -f $dir/.error ]; then
          echo "Something went wrong during accumulation on pass $x"
          exit 1;
        fi
        acc_sum="sgmm2-sum-accs - ";
        for j in `seq 0 $[$g-1]`; do acc_sum="$acc_sum 'gunzip -c $dir/acc.$x.$j.gz|'"; done
        $cmd $dir/log/update_ali.$x.log \
          sgmm2-est --update-flags=$flags --remove-speaker-space=true --power=$power \
          $cur_alimdl "$acc_sum|" $dir/$[$x+1].alimdl || exit 1;
        rm $dir/acc.$x.*.gz || exit 1;
        [ $x -gt $num_iters ]  && rm $dir/$x.alimdl
      fi
      cur_alimdl=$dir/$[$x+1].alimdl
      x=$[$x+1]
    done
    rm $dir/final.alimdl 2>/dev/null 
    ln -s $x.alimdl $dir/final.alimdl
  fi
  
  utils/summarize_warnings.pl $dir/log
  
  echo Done