#!/bin/bash
  
  # Copyright 2014 Vassil Panayotov
  # Apache 2.0
  
  # Performs text normalization for subsequent language model training
  
  echo $@
  
  . ./path.sh || exit 1
  
  if [[ $# -ne 2 ]]; then
    echo "Usage: $0 <input-book-dirs> <output-root>"
    exit 1
  fi
  
  in_list=$1
  out_root=$2
  
  [[ -f "$in_list" ]] || { echo "The input file '$in_list' does not exists!"; exit 1; }
  
  command -v nsw_expand 1>/dev/null 2>&1 || {
    echo ""
    echo "The Festival's NSW text normalization package is not found in PATH";
    echo "You can try to install it by running:";
    echo "  local/lm/install_festival.sh [--apply-gcc-patch false]";
    echo "Note however, that this script should only be considered as an example,";
    echo "so if you run into installation problems, it's up to you to resolve them";
    exit 1;
  }
  
  mkdir -p $out_root
  
  processed=0
  for b in $(cat $in_list); do
    id=$(basename $b)
    echo "Start processing $id at $(date '+%T %F')"
    in_file=$b/$id.txt
    [[ -f "$in_file" ]] || { echo "WARNING: $in_file does not exists"; continue; }
    out_file=$out_root/$id/$id.txt
    mkdir -p $out_root/$id
    $PYTHON local/lm/python/pre_filter.py $in_file /dev/stdout |\
      $PYTHON local/lm/python/text_pre_process.py /dev/stdin /dev/stdout |\
      nsw_expand -format opl /dev/stdin |\
      $PYTHON local/lm/python/text_post_process.py /dev/stdin $out_file /dev/null || exit 1
    processed=$((processed + 1))
    echo "Processing of $id has finished at $(date '+%T %F') [$processed texts ready so far]"
  done
  
  echo "$processed texts processed OK and stored under '$out_root'"
  
  exit 0