#!/usr/bin/perl
  # Copyright 2012  Johns Hopkins University (Author: Daniel Povey).
  # Apache 2.0.
  use File::Basename;
  use Cwd;
  
  # Modifie pour l'usage au LIA, avec mosrun
  
  # queue.pl has the same functionality as run.pl, except that
  # it runs the job in question on the queue (Sun GridEngine).
  # This version of queue.pl uses the task array functionality
  # of the grid engine.  Note: it's different from the queue.pl
  # in the s4 and earlier scripts.
  
  $qsub_opts = "";
  $sync = 0;
  
  for ($x = 1; $x <= 3; $x++) { # This for-loop is to 
    # allow the JOB=1:n option to be interleaved with the
    # options to qsub.
    while (@ARGV >= 2 && $ARGV[0] =~ m:^-:) {
      $switch = shift @ARGV;
      if ($switch eq "-V") {
        $qsub_opts .= "-V ";
      } else {
        $option = shift @ARGV;
        if ($switch eq "-sync" && $option =~ m/^[yY]/) {
          $sync = 1;
        }
        $qsub_opts .= "$switch $option ";
        if ($switch eq "-pe") { # e.g. -pe smp 5
          $option2 = shift @ARGV;
          $qsub_opts .= "$option2 ";
        }
      }
    }
    if ($ARGV[0] =~ m/^([\w_][\w\d_]*)+=(\d+):(\d+)$/) {
      $jobname = $1;
      $jobstart = $2;
      $jobend = $3;
      shift;
      if ($jobstart > $jobend) {
        die "queue.pl: invalid job range $ARGV[0]";
      }
    } elsif ($ARGV[0] =~ m/^([\w_][\w\d_]*)+=(\d+)$/) { # e.g. JOB=1.
      $jobname = $1;
      $jobstart = $2;
      $jobend = $2;
      shift;
    } elsif ($ARGV[0] =~ m/.+\=.*\:.*$/) {
      print STDERR "Warning: suspicious first argument to queue.pl: $ARGV[0]
  ";
    }
  }
  
  ### pas d'option dans mosrun dans cmd.sh => les options de mosix sont passés en durs dans ce script
  ### print "---=debug=--> $0 : $jobname : $jobstart : $jobend : $qsub_opts : $option : $option2
  ";
  
  
  
  if (@ARGV < 2) {
    print STDERR
     "Usage: queue.pl [options to qsub] [JOB=1:n] log-file command-line arguments...
  " .
     "e.g.: queue.pl foo.log echo baz
  " .
     " (which will echo \"baz\", with stdout and stderr directed to foo.log)
  " .
     "or: queue.pl -q all.q\@xyz foo.log echo bar \| sed s/bar/baz/ 
  " .
     " (which is an example of using a pipe; you can provide other escaped bash constructs)
  " .
     "or: queue.pl -q all.q\@qyz JOB=1:10 foo.JOB.log echo JOB 
  " .
     " (which illustrates the mechanism to submit parallel jobs; note, you can use 
  " .
     "  another string other than JOB)
  " .
     "Note: if you pass the \"-sync y\" option to qsub, this script will take note
  " .
     "and change its behavior.  Otherwise it uses qstat to work out when the job finished
  ";
    exit 1;
  }
  
  $cwd = getcwd();
  $logfile = shift @ARGV;
  
  
  ## recuperation du chemin vers les fichiers de log
  #print "------=debug=------> $cwd 
  ";
  #print "------=debug=------> $cwd $logfile
  ";
  
  #== verif syntaxe de la comande ==#
  if (defined $jobname && $logfile !~ m/$jobname/ && $jobend > $jobstart) {
    print STDERR "run.pl: you are trying to run a parallel job but "
      . "you are putting the output into just one log file ($logfile)
  ";
    exit(1);
  }
  
  # Work out the command; quote escaping is done here.
  # Note: the rules for escaping stuff are worked out pretty arbitrarily, based on what we want it to do.  Some things that
  # we pass as arguments to queue.pl, such as "|", we want to be  interpreted by bash, so we don't escape them.  Other things,
  # such as archive specifiers like 'ark:gunzip -c foo.gz|', we want  to be passed, in quotes, to the Kaldi program.  Our heuristic
  # is that stuff with spaces in should be quoted.  This doesn't  always work.
  
  $cmd = "";
  foreach $x (@ARGV) { 
    if ($x =~ m/^\S+$/) { $cmd .= $x . " "; } # If string contains no spaces, take
                                              # as-is.
    elsif ($x =~ m:\":) { $cmd .= "'\''$x'\'' "; } # else if no dbl-quotes, use single
    else { $cmd .= "\"$x\" "; }  # else use double.
  }
  
  # pattern de definition de la commande a repliquer
  #print"------=debug=---------> $cmd
  ";
  
  #
  # Work out the location of the script file, and open it for writing.
  #
  $dir = dirname($logfile);
  $base = basename($logfile);
  $qdir = "$dir/q";
  $qdir =~ s:/(log|LOG)/*q:/q:; # If qdir ends in .../log/q, make it just .../q.
  $queue_logfile = "$qdir/$base";
  
  #print "---=debug=---> $dir 
  ";
  #print "---=debug=---> $base 
  ";
  #print "---=debug=---> $qdir 
  ";
  #print "---=debug=---> $queue_logfile
  ";
  
  
  if (!-d $dir) { system "mkdir $dir 2>/dev/null"; } # another job may be doing this...
  if (!-d $dir) { die "Cannot make the directory $dir
  "; }
  
  
  
  
  
  
  # make a directory called "q",
  # where we will put the log created by qsub... normally this doesn't contain
  # anything interesting, evertyhing goes to $logfile.
  if (! -d "$qdir") { 
    system "mkdir $qdir 2>/dev/null";
    sleep(2); 
    #sleep(5); ## This is to fix an issue we encountered in denominator lattice creation,
    ## where if e.g. the exp/tri2b_denlats/log/15/q directory had just been
    ## created and the job immediately ran, it would die with an error because nfs
    ## had not yet synced.  I'm also decreasing the acdirmin and acdirmax in our
    ## NFS settings to something like 5 seconds.
  } 
  
  ##### if (defined $jobname) { # It's an array job.
  #####  $queue_array_opt = "-t $jobstart:$jobend"; 
  #####  $logfile =~ s/$jobname/\$SGE_TASK_ID/g; # This variable will get 
  #####  # replaced by qsub, in each job, with the job-id.
  #####  $cmd =~ s/$jobname/\$SGE_TASK_ID/g; # same for the command...
  #####  $queue_logfile =~ s/\.?$jobname//; # the log file in the q/ subdirectory
  #####  # is for the queue to put its log, and this doesn't need the task array subscript
  #####  # so we remove it.
  #####}
  # print "---=debug 1 =----> $jobname :  $queue_array_opt :  $logfile :  $cmd :  $queue_logfile
  ";
  
  # queue_scriptfile is as $queue_logfile [e.g. dir/q/foo.log] but
  # with the suffix .sh.
  
  $queue_scriptfile = $queue_logfile;
  ($queue_scriptfile =~ s/\.[a-zA-Z]{1,5}$/.sh/) || ($queue_scriptfile .= ".sh");
  if ($queue_scriptfile !~ m:^/:) {
    $queue_scriptfile = $cwd . "/" . $queue_scriptfile; # just in case.
  }
  
  
  ## BB harmonisation des chemins vers les fichiers
  ## BB : N fichiers make_mfcc.N.sh
  ## tentative numero1 : $nb fichies bash et un lanceur bash
  #print "---=debug 2=---> $queue_scriptfile
  ";
  #print "---=debug 3=---> $logfile
  ";
  
  
  ## BB : preparation d'un script appelé avec mosrun
  my $cmd_lanceur = "$cwd/$qdir/lanceurMosix.sh"  ;
  print "$cwd/$qdir/lanceurMosix.sh
  ";
  
  open(L, ">$cwd/$qdir/lanceurMosix.sh") or die "unable to open $cwd/$qdir/lanceurMosix_make_mfcc.sh
  ";
  print L "#!/bin/bash
  ";	
  
  
  for (my $j=$jobstart; $j <= $jobend; $j++){
  
  	$current_queue_scriptfile = $queue_scriptfile;
  	$current_logfile	  = $logfile;
  	$current_cmd		  = $cmd;
  	$current_queue_logfile	  = $queue_logfile;
  	$syncfile = "$qdir/done.$$.$j";
  
  	if (defined $jobname){
  		$current_queue_scriptfile =~ s/$jobname/$j/g; # This variable will get 
  		$current_logfile =~ s/$jobname/$j/g; # This variable will get 
  		$current_cmd =~ s/$jobname/$j/g; # same for the command...
  		$current_queue_logfile =~ s/\.?$jobname//; # the log file in the q/ subdirectory
  
  	}
  
  	#print "===---debug---===> $current_queue_scriptfile :: $current_logfile :: $current_queue_logfile ::  $syncfile 
  ";
  	system("rm $current_queue_logfile $syncfile 2>/dev/null");
  	
  	open(Q, ">$current_queue_scriptfile") || die "Failed to write to $current_queue_scriptfile";
  	
  
  	# ecriture de la comande dnas le lanceur mosix
  	print L "bash $current_queue_scriptfile  
  ";
  
  
  	print Q "#!/bin/bash
  ";
  	print Q "cd $cwd
  ";
  	print Q ". EXP_NEW/path.sh
  ";
  	print Q "( echo '#' Running on \`hostname\`
  ";
  	print Q "  echo '#' Started at \`date\`
  ";
  	print Q "  echo -n '# '; cat <<EOF
  ";
  	print Q "$current_cmd
  "; # this is a way of echoing the command into a comment in the log file,
  	print Q "EOF
  "; # without having to escape things like "|" and quote characters.
  	print Q ") >$current_logfile
  ";
  	print Q " ($current_cmd ) 2>>$current_logfile >>$current_logfile
  ";
  	print Q "ret=\$?
  ";
  	print Q "echo '#' Finished at \`date\` with status \$ret >>$current_logfile
  ";
  	if (!defined $jobname) { # not an array job
  		print Q "touch $syncfile
  "; # so we know it's done.
  	} else {
  		print Q "touch $syncfile
  "; # touch a bunch of sync-files.
  	}
  	print Q "exit \$[\$ret ? 1 : 0]
  "; # avoid status 100 which grid-engine
  	#print Q "## submitted with:
  ";       # treats specially.
  	#print Q "# $qsub_cmd
  ";
  	if (!close(Q)) { # close was not successful... || die "Could not close script file $shfile";
  		die "Failed to close the script file (full disk?)";
  	}
  
  	
  }
  
  if (!close(L)) { # close was not successful... || die "Could not close script file $shfile";
  	die "Failed to close the script file (full disk?)";
  }	
  
  
  #### exit;
  # We'll write to the standard input of "qsub" (the file-handle Q),
  # the job that we want it to execute.
  # Also keep our current PATH around, just in case there was something
  # in it that we need (although we also source ./path.sh)
  ##### $syncfile = "$qdir/done.$$";
  ##### print "---=debug 3=---> $syncfile
  ";
  ##### system("rm $queue_logfile $syncfile 2>/dev/null");
  #
  # Write to the script file, and then close it.
  #
  ##### open(Q, ">$queue_scriptfile") || die "Failed to write to $queue_scriptfile";
  ##### print Q "#!/bin/bash
  ";
  ##### print Q "cd $cwd
  ";
  ##### print Q ". ./path.sh
  ";
  ##### print Q "( echo '#' Running on \`hostname\`
  ";
  ##### print Q "  echo '#' Started at \`date\`
  ";
  ##### print Q "  echo -n '# '; cat <<EOF
  ";
  ##### print Q "$cmd
  "; # this is a way of echoing the command into a comment in the log file,
  ##### print Q "EOF
  "; # without having to escape things like "|" and quote characters.
  ##### print Q ") >$logfile
  ";
  ##### print Q " ( $cmd ) 2>>$logfile >>$logfile
  ";
  ##### print Q "ret=\$?
  ";
  ##### print Q "echo '#' Finished at \`date\` with status \$ret >>$logfile
  ";
  ##### if (!defined $jobname) { # not an array job
  #####   print Q "touch $syncfile
  "; # so we know it's done.
  ##### } else {
  #####   print Q "touch $syncfile.\$SGE_TASK_ID
  "; # touch a bunch of sync-files.
  ##### }
  ##### print Q "exit \$[\$ret ? 1 : 0]
  "; # avoid status 100 which grid-engine
  ##### print Q "## submitted with:
  ";       # treats specially.
  ##### print Q "# $qsub_cmd
  ";
  ##### if (!close(Q)) { # close was not successful... || die "Could not close script file $shfile";
  #####   die "Failed to close the script file (full disk?)";
  ##### }
  
  #print "system (qsub -S /bin/bash -v PATH -cwd -j y $qsub_opts $queue_array_opt $queue_scriptfile >> $queue_logfile );
  ";
  #print "system (mosrun -w  -b -m4000 -M $cwd  $cmd_lanceur );
  ";
  print "mosrun -q -S16 -e -b -m4000 $cmd_lanceur
  ";
  $ret = system ("mosrun -q -S16 -e  -b -m4000 $cmd_lanceur");
  exit ;
  $ret = system ("qsub -S /bin/bash -v PATH -cwd -j y -o $queue_logfile $qsub_opts $queue_array_opt $queue_scriptfile >>$queue_logfile 2>&1");
  
  if ($ret != 0) {
    if ($sync && $ret == 256) { # this is the exit status when a job failed (bad exit status)
      if (defined $jobname) { $logfile =~ s/\$SGE_TASK_ID/*/g; }
      print STDERR "queue.pl: job writing to $logfile failed
  ";
    } else {
      print STDERR "queue.pl: error submitting jobs to queue (return status was $ret)
  ";
      print STDERR `tail $queue_logfile`;
    }
    exit(1);
  }
  
  if (! $sync) { # We're not submitting with -sync y, so we
    # need to wait for the jobs to finish.  We wait for the
    # sync-files we "touched" in the script to exist.
    @syncfiles = ();
    if (!defined $jobname) { # not an array job.
      push @syncfiles, $syncfile;
    } else {
      for ($jobid = $jobstart; $jobid <= $jobend; $jobid++) {
        push @syncfiles, "$syncfile.$jobid";
      }
    }
    $wait = 0.1;
    foreach $f (@syncfiles) {
      # wait for them to finish one by one.
      while (! -f $f) {
        sleep($wait);
        $wait *= 1.2;
        if ($wait > 1.0) {
          $wait = 1.0; # never wait more than 1 second.
        }  
      }
    }
    $all_syncfiles = join(" ", @syncfiles);
    system("rm $all_syncfiles 2>/dev/null");
  }
  
  # OK, at this point we are synced; we know the job is done.
  # But we don't know about its exit status.  We'll look at $logfile for this.
  # First work out an array @logfiles of file-locations we need to
  # read (just one, unless it's an array job).
  @logfiles = ();
  if (!defined $jobname) { # not an array job.
    push @logfiles, $logfile;
  } else {
    for ($jobid = $jobstart; $jobid <= $jobend; $jobid++) {
      $l = $logfile; 
      $l =~ s/\$SGE_TASK_ID/$jobid/g;
      push @logfiles, $l;
    }
  }
  
  $num_failed = 0;
  foreach $l (@logfiles) {
    @wait_times = (0.1, 0.2, 0.2, 0.3, 0.5, 0.5, 1.0, 2.0, 5.0, 5.0, 5.0, 10.0, 25.0);
    for ($iter = 0; $iter <= @wait_times; $iter++) {
      $line = `tail -1 $l 2>/dev/null`;
      if ($line =~ m/with status (\d+)/) {
        $status = $1;
        last;
      } else {
        if ($iter < @wait_times) {
          sleep($wait_times[$iter]);
        } else {
          if (! -f $l) {
            print STDERR "Log-file $l does not exist.
  ";
          } else {
            print STDERR "The last line of log-file $l does not seem to indicate the "
              . "return status as expected
  ";
          }
          exit(1);                # Something went wrong with the queue, or the
          # machine it was running on, probably.
        }
      }
    }
    # OK, now we have $status, which is the return-status of
    # the command in the job.
    if ($status != 0) { $num_failed++; }
  }
  if ($num_failed == 0) { exit(0); }
  else { # we failed.
    if (@logfiles == 1) {
      if (defined $jobname) { $logfile =~ s/\$SGE_TASK_ID/$jobstart/g; }
      print STDERR "queue.pl: job failed with status $status, log is in $logfile
  ";
      if ($logfile =~ m/JOB/) {
        print STDERR "queue.pl: probably you forgot to put JOB=1:\$nj in your script.
  ";
      }
    } else {
      if (defined $jobname) { $logfile =~ s/\$SGE_TASK_ID/*/g; }
      $numjobs = 1 + $jobend - $jobstart;
      print STDERR "queue.pl: $num_failed / $numjobs failed, log is in $logfile
  ";
    }
    exit(1);
  }