3.54 KB
# Copyright 2013  Hong Kong University of Science and Technology (Author: Ricky Chan Ho Yin);
#                 Brno University of Technology (Author: Karel Vesely);
#                 Johns Hopkins University (Author: Daniel Povey);
# Apache 2.0

# This script splits dataset to two parts : 
# training set from (100-P)% of speakers/utterances and 
# held-out set (or cross-validation) from P% of remaining speakers/remaining utterances,
# which will be later on used for neural network training
# There are two options for choosing held-out (or cross-validation) set, either by
# --cv-spk-percent P , which will give you CV set based on random chosen P% of speakers, or
# --cv-utt-percent P , which will give you CV set based on last P% utterances in the dataset
# If you don't apply the above two options, by default the script will use --cv-utt-percent option,
# and the default cross validation percentage portion is equal to 10% (i.e. P=10)
# The --cv-spk-percent option is useful if you would like to have subset chosen from random speakers order, 
# especially for the cases where dataset contains multiple different corpora,
# where type of speakers or recording channels may be quite different 

# Begin configuration.
cv_spk_percent= # % of speakers is parsed by option
cv_utt_percent=10 # default 10% of total utterances 
seed=777 # use seed for speaker shuffling
# End configuration.

echo "$0 $@"  # Print the command line for logging

uttbase=true; # by default, we choose last 10% utterances for CV

if [ "$1" == "--cv-spk-percent" ]; then

[ -f ] && . ./; 

. || exit 1;

if [ $# != 3 ]; then
  echo "Usage: $0 [--cv-spk-percent P|--cv-utt-percent P] <srcdir> <traindir> <crossvaldir>"
  echo "  --cv-spk-percent P  Cross Validation portion of the total speakers, recommend value is 10% (i.e. P=10)"
  echo "  --cv-utt-percent P  Cross Validation portion of the total utterances, default is 10% (i.e. P=10)"
  echo "  "
  exit 1;


## use simple last P% utterance for CV
if $uttbase; then
  if [ ! -f $srcdir/utt2spk ]; then
    echo "$0: no such file $srcdir/utt2spk"
    exit 1;

  #total number of lines
  N=$(cat $srcdir/utt2spk | wc -l)
  #get line number where (100-P)% of the data lies
  P_utt=$((N * cv_utt_percent / 100))
  N_head=$((N -P_utt))
  #move the boundary so it is located on speaker change
  N_head=$(cat $srcdir/utt2spk | uniq -f1 -c | awk '{ if(n+$1<='$N_head') { n += $1 } else { nextfile } } END{ print n }')
  #the rest of the data will be that big

  #now call the and fix the directories --first $srcdir $N_head $trndir --last $srcdir $N_tail $cvdir

  exit 0;

## use random chosen P% speakers for CV
if [ ! -f $srcdir/spk2utt ]; then
  echo "$0: no such file $srcdir/spk2utt" 
  exit 1;

#total, cv, train number of speakers
N=$(cat $srcdir/spk2utt | wc -l)
N_spk_cv=$((N * cv_spk_percent / 100))
N_spk_trn=$((N - N_spk_cv))

mkdir -p $cvdir $trndir

#shuffle the speaker list
awk '{print $1}' $srcdir/spk2utt | --srand $seed > $trndir/_tmpf_randspk

#split the train/cv
head -n $N_spk_cv $trndir/_tmpf_randspk > $cvdir/_tmpf_cvspk
tail -n $N_spk_trn $trndir/_tmpf_randspk > $trndir/_tmpf_trainspk

#now call the --spk-list $trndir/_tmpf_trainspk $srcdir $trndir --spk-list $cvdir/_tmpf_cvspk $srcdir $cvdir

rm -f $trndir/_tmpf_randspk $trndir/_tmpf_trainspk $cvdir/_tmpf_cvspk