#!/bin/bash
  # Copyright 2019 QCRI (Author: Ahmed Ali)
  # Apache 2.0
  
  set -e -o pipefail
  
  
  ###
  # The script assumes you have downloaded to the MGB-5 corpus: https://arabicspeech.org/mgb5
  # DB/{dev.tar.gz,train.tar.gz}
  ###
  echo "Preparing train and dev data"
  
  if [[ ! -e "DB/train.tar.gz" || ! -e "DB/dev.tar.gz" ]]; then
    echo "You need to download the MGB-5 first and copy dev.tar.gz and train.tar.gz to DB folder"
    echo "check: https://arabicspeech.org/mgb5"
    exit 1
  fi
  
  # We will extract data again even if you did this before.
  (cd DB; rm -fr train dev;for x in *; do tar -xvf $x; done)
  
  mkdir -p data/local data/train data/dev
  
  for x in train dev; do
      sed -e 's:UNK: :g' -e 's:  : :g' DB/$x/$x.txt.bw > data/$x/text #removing words that annotators couldn't understand
      cp DB/$x/$x.segments.bw data/$x/segments
      awk '{print $1 " " $1}' DB/$x/$x.segments.bw > data/$x/spk2utt
      cp data/$x/spk2utt data/$x/utt2spk 
      find $PWD/DB/$x/ -name \*.wav | while read wav; do
          id=$(basename $wav | sed 's:.wav::')
          echo $id $wav
      done | sort -u > data/$x/wav.scp
      utils/fix_data_dir.sh data/$x
  done
  
  
  echo "Data preparation completed."