// bin/compute-wer-bootci.cc
  
  // Copyright 2009-2011  Microsoft Corporation
  //                2014  Johns Hopkins University (authors: Jan Trmal, Daniel Povey)
  //                2015  Brno Universiry of technology (author: Karel Vesely)
  //                2016  Nicolas Serrano
  
  // See ../../COPYING for clarification regarding multiple authors
  //
  // Licensed under the Apache License, Version 2.0 (the "License");
  // you may not use this file except in compliance with the License.
  // You may obtain a copy of the License at
  //
  //  http://www.apache.org/licenses/LICENSE-2.0
  //
  // THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
  // WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
  // MERCHANTABLITY OR NON-INFRINGEMENT.
  // See the Apache 2 License for the specific language governing permissions and
  // limitations under the License.
  
  #include "base/kaldi-common.h"
  #include "util/common-utils.h"
  #include "util/parse-options.h"
  #include "tree/context-dep.h"
  #include "util/edit-distance.h"
  #include "base/kaldi-math.h"
  
  namespace kaldi {
  
  void GetEditsSingleHyp( const std::string &hyp_rspecifier,
        const std::string &ref_rspecifier,
        const std::string &mode,
        std::vector<std::pair<int32, int32> > & edit_word_per_hyp) {
  
      // Both text and integers are loaded as vector of strings,
      SequentialTokenVectorReader ref_reader(ref_rspecifier);
      RandomAccessTokenVectorReader hyp_reader(hyp_rspecifier);
      int32 num_words = 0, word_errs = 0, num_ins = 0, num_del = 0, num_sub = 0;
  
      // Main loop, store WER stats per hyp,
      for (; !ref_reader.Done(); ref_reader.Next()) {
        std::string key = ref_reader.Key();
        const std::vector<std::string> &ref_sent = ref_reader.Value();
        std::vector<std::string> hyp_sent;
        if (!hyp_reader.HasKey(key)) {
          if (mode == "strict")
            KALDI_ERR << "No hypothesis for key " << key << " and strict "
                "mode specifier.";
          if (mode == "present")  // do not score this one.
            continue;
        } else {
          hyp_sent = hyp_reader.Value(key);
        }
        num_words = ref_sent.size();
        word_errs = LevenshteinEditDistance(ref_sent, hyp_sent,
                                              &num_ins, &num_del, &num_sub);
        edit_word_per_hyp.push_back(std::pair<int32, int32>(word_errs, num_words));
      }
  }
  
  void GetEditsDualHyp(const std::string &hyp_rspecifier,
        const std::string &hyp_rspecifier2,
        const std::string &ref_rspecifier,
        const std::string &mode,
        std::vector<std::pair<int32, int32> > & edit_word_per_hyp,
        std::vector<std::pair<int32, int32> > & edit_word_per_hyp2) {
  
      // Both text and integers are loaded as vector of strings,
      SequentialTokenVectorReader ref_reader(ref_rspecifier);
      RandomAccessTokenVectorReader hyp_reader(hyp_rspecifier);
      RandomAccessTokenVectorReader hyp_reader2(hyp_rspecifier2);
      int32 num_words = 0, word_errs = 0,
              num_ins = 0, num_del = 0, num_sub = 0;
  
      // Main loop, store WER stats per hyp,
      for (; !ref_reader.Done(); ref_reader.Next()) {
        std::string key = ref_reader.Key();
        const std::vector<std::string> &ref_sent = ref_reader.Value();
        std::vector<std::string> hyp_sent, hyp_sent2;
        if (mode == "strict" &&
                (!hyp_reader.HasKey(key) || !hyp_reader2.HasKey(key))) {
            KALDI_ERR << "No hypothesis for key " << key << " in both transcripts "
                "comparison is not possible.";
        } else if (mode == "present" &&
                (!hyp_reader.HasKey(key) || !hyp_reader2.HasKey(key)))
            continue;
  
        num_words = ref_sent.size();
  
        //all mode, if a hypothesis is not present, consider as an error
        if(hyp_reader.HasKey(key)){
          hyp_sent = hyp_reader.Value(key);
          word_errs = LevenshteinEditDistance(ref_sent, hyp_sent,
                                              &num_ins, &num_del, &num_sub);
        }
        else
          word_errs = num_words;
        edit_word_per_hyp.push_back(std::pair<int32, int32>(word_errs, num_words));
  
        if(hyp_reader2.HasKey(key)){
          hyp_sent2 = hyp_reader2.Value(key);
          word_errs = LevenshteinEditDistance(ref_sent, hyp_sent2,
                                              &num_ins, &num_del, &num_sub);
        }
        else
          word_errs = num_words;
        edit_word_per_hyp2.push_back(std::pair<int32, int32>(word_errs, num_words));
      }
  }
  
  void GetBootstrapWERInterval(
        const std::vector<std::pair<int32, int32> > & edit_word_per_hyp,
        int32 replications,
        BaseFloat *mean, BaseFloat *interval) {
      BaseFloat wer_accum = 0.0, wer_mult_accum = 0.0;
  
      for (int32 i = 0; i < replications; ++i) {
        int32 num_words = 0, word_errs = 0;
        for (int32 j = 0; j < edit_word_per_hyp.size(); ++j) {
          int32 random_pos = kaldi::RandInt(0, edit_word_per_hyp.size() - 1);
          word_errs += edit_word_per_hyp[random_pos].first;
          num_words += edit_word_per_hyp[random_pos].second;
          }
  
        BaseFloat wer_rep = static_cast<BaseFloat>(word_errs) / num_words;
        wer_accum += wer_rep;
        wer_mult_accum += wer_rep*wer_rep;
      }
  
      // Compute mean WER and std WER
      *mean = wer_accum / replications;
      *interval = 1.96*sqrt(wer_mult_accum/replications-(*mean)*(*mean));
  }
  
  void GetBootstrapWERTwoSystemComparison(
        const std::vector<std::pair<int32, int32> > & edit_word_per_hyp,
        const std::vector<std::pair<int32, int32> > & edit_word_per_hyp2,
        int32 replications, BaseFloat *p_improv) {
      int32 improv_accum = 0.0;
  
      for (int32 i = 0; i < replications; ++i) {
        int32 word_errs = 0;
        for (int32 j = 0; j < edit_word_per_hyp.size(); ++j) {
          int32 random_pos = kaldi::RandInt(0, edit_word_per_hyp.size() - 1);
          word_errs += edit_word_per_hyp[random_pos].first -
                          edit_word_per_hyp2[random_pos].first;
          }
        if(word_errs > 0)
          ++improv_accum;
      }
      // Compute mean WER and std WER
      *p_improv = static_cast<BaseFloat>(improv_accum) / replications;
  }
  
  } //namespace kaldi
  
  int main(int argc, char *argv[]) {
    using namespace kaldi;
    typedef kaldi::int32 int32;
  
    try {
      const char *usage =
        "Compute a bootstrapping of WER to extract the 95% confidence interval.
  "
        "Take a reference and a transcription file, in integer or text format,
  "
        "and outputs overall WER statistics to standard output along with its
  "
        "confidence interval using the bootstrap method of Bisani and Ney.
  "
        "If a second transcription file corresponding to the same reference is
  "
        "provided, a bootstrap comparison of the two transcription is performed
  "
        "to estimate the probability of improvement.
  "
        "
  "
        "Usage: compute-wer-bootci [options] <ref-rspecifier> <hyp-rspecifier> [<hyp2-rspecifier>]
  "
        "E.g.: compute-wer-bootci --mode=present ark:data/train/text ark:hyp_text
  "
        "or compute-wer-bootci ark:data/train/text ark:hyp_text ark:hyp_text2
  "
        "See also: compute-wer
  ";
  
      ParseOptions po(usage);
  
      std::string mode = "strict";
      po.Register("mode", &mode,
                  "Scoring mode: \"present\"|\"all\"|\"strict\":
  "
                  "  \"present\" means score those we have transcriptions for
  "
                  "  \"all\" means treat absent transcriptions as empty
  "
                  "  \"strict\" means die if all in ref not also in hyp");
  
      int32 replications = 10000;
      po.Register("replications", &replications,
              "Number of replications to compute the intervals");
  
      po.Read(argc, argv);
  
      if (po.NumArgs() < 2 || po.NumArgs() > 3) {
        po.PrintUsage();
        exit(1);
      }
  
      std::string ref_rspecifier = po.GetArg(1);
      std::string hyp_rspecifier = po.GetArg(2);
      std::string hyp2_rspecifier = (po.NumArgs() == 3?po.GetArg(3):"");
  
      if (mode != "strict" && mode != "present" && mode != "all") {
        KALDI_ERR <<
            "--mode option invalid: expected \"present\"|\"all\"|\"strict\", got "
            << mode;
      }
  
      //Get editions per each utterance
      std::vector<std::pair<int32, int32> > edit_word_per_hyp, edit_word_per_hyp2;
      if(hyp2_rspecifier.empty())
        GetEditsSingleHyp(hyp_rspecifier, ref_rspecifier, mode, edit_word_per_hyp);
      else
        GetEditsDualHyp(hyp_rspecifier, hyp2_rspecifier, ref_rspecifier, mode,
                edit_word_per_hyp, edit_word_per_hyp2);
  
      //Extract WER for a number of replications of the same size
      //as the hypothesis extracted
      BaseFloat mean_wer = 0.0, interval = 0.0,
                mean_wer2 = 0.0, interval2 = 0.0,
                p_improv = 0.0;
  
      GetBootstrapWERInterval(edit_word_per_hyp, replications,
              &mean_wer, &interval);
  
      if(!hyp2_rspecifier.empty()) {
        GetBootstrapWERInterval(edit_word_per_hyp2, replications,
                &mean_wer2, &interval2);
  
        GetBootstrapWERTwoSystemComparison(edit_word_per_hyp, edit_word_per_hyp2,
               replications, &p_improv);
      }
  
      // Print the output,
      std::cout.precision(2);
      std::cerr.precision(2);
      std::cout << "Set1: %WER " << std::fixed << 100*mean_wer <<
                " 95% Conf Interval [ " << 100*mean_wer-100*interval <<
                ", " << 100*mean_wer+100*interval << " ]" << '
  ';
  
      if(!hyp2_rspecifier.empty()) {
          std::cout << "Set2: %WER " << std::fixed << 100*mean_wer2 <<
              " 95% Conf Interval [ " << 100*mean_wer2-100*interval2 <<
              ", " << 100*mean_wer2+100*interval2 << " ]" << '
  ';
  
          std::cout << "Probability of Set2 improving Set1: " << std::fixed <<
              100*p_improv << '
  ';
      }
  
      return 0;
    } catch(const std::exception &e) {
      std::cerr << e.what();
      return -1;
    }
  }