#!/usr/bin/env bash
  
  # This script validates the inputs for RNNLM training.
  
  
  if [ $# != 2 ]; then
    echo "Usage: $0 <text-dir> <rnnlm-config-dir>"
    echo "Validates <text-dir> and <rnnlm-config-dir>."
    echo "<text-dir> should be as validated by validate_data_dir.py."
    echo "<rnnlm-config-dir> contains various smallish user-provided"
    echo "files needed for RNNLM training:"
    echo "  words.txt  (vocabulary file with mapping to integers)"
    echo "  features.txt  [optional] File as generated by choose_features.py,"
    echo "              that determines the feature represenration."
    echo "              If not present, no feature representation will be "
    echo "              used, and each word's embedding is trained separately."
    echo "  data_weights.txt  File containing data multiplicities and"
    echo "     weighting factors for all data sources in <text-dir>,"
    echo "     except 'dev'.  e.g. with lines like"
    echo "       switchboard  1  0.5"
    echo "     Weights do not have to sum to one and can be greater"
    echo "     than one."
    echo "  oov.txt   Must either be a empty file, or contain the"
    echo "     written representation of the unknown word, usually"
    echo "     <unk> (only relevant if <text-dir> contains words not"
    echo "     present in words.txt)."
    echo "  xconfig   File containing xconfig representation of the"
    echo "     RNNLM to be created, as could be provided to"
    echo "     xconfig_to_configs.py"
    exit 1;
  fi
  
  
  text_dir=$1
  config_dir=$2
  
  set -e
  
  
  for f in words.txt data_weights.txt oov.txt xconfig; do
    if [ ! -f $config_dir/$f ]; then
      echo "$0: file $config_dir/$f is not present."
      exit 1
    fi
  done
  
  rnnlm/validate_text_dir.py --spot-check=true $text_dir
  
  if [ -f $config_dir/features.txt ]; then
    # features.txt is optional.
    rnnlm/validate_features.py $config_dir/features.txt
  fi
  
  # basic check of words.txt
  if ! echo 0 | utils/int2sym.pl $config_dir/words.txt >/dev/null; then
    echo "$0: detected a problem in $config_dir/words.txt"
    exit 1
  fi
  
  
  rnnlm/ensure_counts_present.sh $text_dir
  
  
  # rnnlm/get_unigram_probs.py validates the data-weights file, so we're
  # relying on that check rather than writing a special one.
  if ! rnnlm/get_unigram_probs.py --vocab-file=$config_dir/words.txt \
                             --unk-word=$(cat $config_dir/oov.txt) \
                             --data-weights-file=$config_dir/data_weights.txt \
                             $text_dir >/dev/null; then
    echo "$0: detected problem, most likely with data-weights file $config_dir/data_weights.txt"
    echo " ... see errors above."
  fi
  
  # for words.txt: check number of fields per line is 2 and that the
  # second is an integer; check that bos, eos and brk are in the
  # expected positions.
  
  if [ -s $config_dir/oov.txt ]; then
    # if oov.txt is nonempty...
    if ! awk '{if (NF!=1){ exit (1) }} END{if(NR != 1) exit(1)}' <$config_dir/oov.txt; then
      echo "$0: $config_dir/oov.txt does not look right."
      exit 1
    fi
    if ! utils/sym2int.pl $config_dir/words.txt <$config_dir/oov.txt >/dev/null; then
      echo "$0: the word in $config_dir/oov.txt does not exist in $config_dir/words.txt: '$(cat $config_dir/oov.txt)'"
      exit 1
    fi
  fi
  
  
  if grep '^\s*fixed-affine-layer' $config_dir/xconfig; then
    echo "$0: $config_dir/xconfig cannot contain a layer of type fixed-affine-layer."
    exit 1
  fi
  
  
  echo "$0: validated config dir $config_dir"
  exit 0;