#!/usr/bin/env python3
  
  # Copyright  2018  Ashish Arora
  #                  Chun-Chieh Chang
  
  import argparse
  import os
  
  parser = argparse.ArgumentParser(description="""Creates the list of characters and words in lexicon""")
  parser.add_argument('dir', type=str, help='output path')
  parser.add_argument('--data-dir', type=str, default='data', help='Path to text file')
  args = parser.parse_args()
  
  ### main ###
  lex = {}
  text_path = os.path.join(args.data_dir, 'train', 'text')
  text_fh = open(text_path, 'r', encoding='utf-8')
  
  # Used specially for Chinese.
  # Uses the ChangJie keyboard input method to create subword units for Chinese.
  cj5_table = {}
  with open('download/cj5-cc.txt', 'r', encoding='utf-8') as f:
      for line in f:
          line_vect = line.strip().split()
          if not line_vect[0].startswith('yyy') and not line_vect[0].startswith('z'):
              cj5_table[line_vect[1]] = "cj5_" + " cj5_".join(list(line_vect[0]))
  
  with open(text_path, 'r', encoding='utf-8') as f:
      for line in f:
          line_vect = line.strip().split()
          for i in range(1, len(line_vect)):
              characters = list(line_vect[i])
  	    # Put SIL instead of "|". Because every "|" in the beginning of the words is for initial-space of that word
              characters = " ".join([ 'SIL' if char == '|' else cj5_table[char] if char in cj5_table else char for char in characters])
              characters = characters.replace('#','<HASH>')
              lex[line_vect[i]] = characters
  
  with open(os.path.join(args.dir, 'lexicon.txt'), 'w', encoding='utf-8') as fp:
      for key in sorted(lex):
          fp.write(key + " " + lex[key] + "
  ")