#!/bin/sh
#=======================================================================
# Send run time diagnostic/plot jobs to the back end       --- pxrtd ---
# The run time diagnostic job will always be sent but the plot job will
# only be sent when pxrtd_with_plot=on
# $Id: pxrtd_jobdef 659 2012-03-21 21:56:58Z acrnrls $
#=======================================================================
#           keyword :: pxrtd
# short description :: send run time diagnostic/plot jobs to the back end
#              hide :: yes
 set -a
 . betapath2

#  * ........................... Parmsub Parameters ............................

 jobname=pxrtd; time="1800" ; memory="900mb" ;

 noprint=on
 nextjob=on

 username="acrnxxx"; user="XXX";
 runid=job000; uxxx='uxxx'; crawork="${runid}_job"

 # Temporary directory where this script will run
 CCRNTMP=$CCRNTMP

 # RUNPATH on execution machine
 RUNPATH=$RUNPATH

 # Alternate path to a directory where .queue/.crawork will be found
 JHOME=''

 if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then
   # Allow optional reset of DATAPATH/RUNPATH
   JHOME_DATA=''
   DATAPATH=${JHOME_DATA:=$DATAPATH}
   RUNPATH=${JHOME_DATA:=$RUNPATH}
   # Allow optional reset of CCRNTMP
   JHOME_RUN=''
   CCRNTMP=${JHOME_RUN:=$CCRNTMP}
 fi

 . comjcl.cdk
 
cat > Execute_Script <<'end_of_script'

  # bail is a simple error exit routine
  bail(){
    echo "pxrtd: $1"
    echo " " >> haltit
    exit 1
  }

#  * ........................... Parmsub Parameters ....................

  runid=job000; uxxx='uxxx'; pxrtd_uxxx=$uxxx; pxrtd_prefix="${pxrtd_uxxx}_${runid}";
  crawork="${runid}_job"; username="acrnxxx"; user="XXX";

  # Indicate how cccjob should be invoked
  # Setting CCCJOB_ROOT will allow a job specific version of cccjob to used
  CCCJOB_ROOT=''
  if [ -z "$CCCJOB_ROOT" ]; then
    CCCJOB_ENV=''
  else
    eval CCCJOB_ENV=\'env CCCJOB_ROOT\=$CCCJOB_ROOT\'
  fi

  # These variables are set when the job string is created
  run_start_year=NotSet
  run_start_month=NotSet
  run_stop_year=NotSet
  run_stop_month=NotSet
  current_year=NotSet
  current_month=NotSet
  previous_year=NotSet
  previous_month=NotSet
  next_year=NotSet
  next_month=NotSet

  [ $current_year = "NotSet" -o $current_month = "NotSet" ] && \
    bail "current_year or current_month is not set"

  if [ $previous_year = "NotSet" -o $previous_month = "NotSet" ]; then
    if [ $next_year = "NotSet" -o $next_month = "NotSet" ]; then
      bail "Neither previous_(year|month) nor next_(year|month) are set"
    else
      pxrtd_start_year=$current_year
      pxrtd_start_mon=$current_month
      pxrtd_end_year=$next_year
      pxrtd_end_mon=$next_month
    fi
  else
    pxrtd_start_year=$previous_year
    pxrtd_start_mon=$previous_month
    pxrtd_end_year=$current_year
    pxrtd_end_mon=$current_month
  fi

  [ -z "$pxrtd_start_year" ] && bail "pxrtd_start_year is null"
  [ -z "$pxrtd_end_year" ]   && bail "pxrtd_end_year is null"
  [ -z "$pxrtd_start_mon" ]  && bail "pxrtd_start_mon is null"
  [ -z "$pxrtd_end_mon" ]    && bail "pxrtd_end_mon is null"
  [ $pxrtd_start_mon -gt 12 -o $pxrtd_start_mon -lt 1 ] &&\
    bail "pxrtd_start_mon=$pxrtd_start_mon is out of range"
  [ $pxrtd_end_mon -gt 12 -o $pxrtd_end_mon -lt 1 ] &&\
    bail "pxrtd_end_mon=$pxrtd_end_mon is out of range"
  [ $pxrtd_start_year -gt $pxrtd_end_year ] &&\
    bail "pxrtd_start_year=$pxrtd_start_year is out of range"

  pxrtd_start_year=`echo $pxrtd_start_year|awk '{printf "%3.3d",$1}' -`
  pxrtd_start_mon=`echo $pxrtd_start_mon|awk '{printf "%2.2d",$1}' -`
  pxrtd_end_year=`echo $pxrtd_end_year|awk '{printf "%3.3d",$1}' -`
  pxrtd_end_mon=`echo $pxrtd_end_mon|awk '{printf "%2.2d",$1}' -`

  # Identify the name of the back end machine
  back_end_mach=''
  [ -z "$back_end_mach" ] && bail "back_end_mach is not set"

  # pxrtd_with_plot controls the addition of a plotting job to the job string
  # pdump_rtd_plot_freq is the frequency in years
  # at which these plots will be generated
  # Note: Setting pxrtd_with_plot on the cccjob command line when this module
  #       is used will override pdump_rtd_plot_freq
  pdump_rtd_plot_freq=2
  plot_rtd_year=`echo $pxrtd_end_year $pdump_rtd_plot_freq|awk '{n=$1%$2;printf "%d",n}' -`
  [ -z "$plot_rtd_year" ] && \
    bail "Invalid pxrtd_end_year = $pxrtd_end_year or pdump_rtd_plot_freq = $pdump_rtd_plot_freq"
  if [ \( $plot_rtd_year -eq 0 -a $pxrtd_end_year -ne $run_start_year \) \
       -o $pxrtd_end_year -eq $run_stop_year ]; then
    # Execute the run time diagnostics and run the plot job
    pxrtd_with_plot=on
  else
    # Execute the run time diagnostics without any plotting
    pxrtd_with_plot=off
  fi
  XXX=`echo $pxrtd_with_plot|sed 's/ //g'`
  eval pxrtd_with_plot\=$XXX
  [ "$pxrtd_with_plot" = 'on' ]  && eval pxrtd_with_plot\=1
  [ "$pxrtd_with_plot" = 'off' ] && eval pxrtd_with_plot\=0
  [ "$pxrtd_with_plot" = 'yes' ] && eval pxrtd_with_plot\=1
  [ "$pxrtd_with_plot" = 'no' ]  && eval pxrtd_with_plot\=0

  # Determine if the run time diagnostics will be submitted as a separate
  # job, in parallel with the current job string (rtd_in_series = off)
  # or inlined into the current string to run in series (rtd_in_series = on)
  rtd_in_series=off
  XXX=`echo $rtd_in_series|sed 's/ //g'`
  eval rtd_in_series\=$XXX
  [ -z "$rtd_in_series" ] && eval rtd_in_series\=0
  [ "$rtd_in_series" = 'on' ]  && eval rtd_in_series\=1
  [ "$rtd_in_series" = 'off' ] && eval rtd_in_series\=0
  [ "$rtd_in_series" = 'yes' ] && eval rtd_in_series\=1
  [ "$rtd_in_series" = 'no' ]  && eval rtd_in_series\=0

  # Determine which run time diagnostics/plotting to use
  pdump_rtd_diag=''
  pdump_rtd_plot=''
  rtd_diag=${pdump_rtd_diag:=rtdiag}
  rtd_plot=${pdump_rtd_plot:=rtdplot}

  rtd_diag_deck=''
  rtd_plot_deck=''
  rtd_path=''

  # Define a partial file name
  fid=${runid}_$current_year

  # Define a unique stamp for use in file names
  stamp=`date "+%j"$$`

  # Set the name of the associated lock file
  rtd_lock_file=""
  REMOVE_LOCK=''
  if [ -n "$rtd_lock_file" ]; then
    # Create the rtd lock file and append a job that will delete
    # this file after the run time diagnostics are complete
    touch $rtd_lock_file
    REMOVE_LOCK='remove_lock:s'
  fi

  # Define start/stop times for this job
  start_opt="--start_time=${pxrtd_start_year}:$pxrtd_start_mon"
  stop_opt="--stop_time=${pxrtd_end_year}:$pxrtd_end_mon"

  # Create a list of variable definition that may be required by the run
  # time diagnostica or plotting of the run time diagnostics
  year_rdiag_start=''
  year_rtdiag_start=''
  yearfirst=''
  PhysA=''
  PhysO=''
  CarbA=''
  CarbO=''
  CarbL=''
  resolution=''
  pldir=''
  varlist='uxxx
           runid
           CCCJOB_ROOT
           JHOME
           JHOME_DATA
           JHOME_RUN
           noprint
           year_rdiag_start
           year_rtdiag_start
           yearfirst
           PhysA
           PhysO
           CarbA
           CarbO
           CarbL
           rtd_lock_file
           resolution
           pldir
           rtd_diag_deck
           rtd_plot_deck
           rtd_path'

  # This eval will protect this crawork def from cccjob substitution
  eval craworkdef\=crawork\=rtd_${fid}_$stamp
  if [ -n "$rtd_lock_file" ]; then
    XXX=`basename $rtd_lock_file`
    XXX=`echo $XXX|sed -n 's/^lock_//p'`
    if [ -n "$XXX" ]; then
      eval craworkdef\=crawork\=$XXX
    fi
  fi
  if [ "$rtd_in_series" -eq 1 ]; then
    # The crawork definition for the job created here must be the same
    # as that in the current string
    craworkdef=crawork=$crawork
  fi
  cat > cccjob_defs <<EOF
$craworkdef
EOF
  for var in $varlist; do
    eval val=\$$var
    # If this variable is defined add it to the list
    [ -n "$val" ] && echo ${var}=\'$val\' >> cccjob_defs
  done

  echo "cccjob_defs:"
  [ -s "cccjob_defs" ] && cat cccjob_defs

  # Create a job to remove the lock file associated with this job

  # Using a here doc to create remove_lock seems to cause the submit
  # scripts grief. This is why the file is created as follows.
  # This eval is necessary to prevent the shebang line from being
  # blindly replaced with a different shell by submit3
  eval shebang=\#\!/\bin/\sh
  echo "$shebang"                                            > remove_lock
  echo '  set -a '                                          >> remove_lock
  echo '  . betapath2 '                                     >> remove_lock
  echo '  jobname=rmlock; crawork=xxyyzz '                  >> remove_lock
  echo '  username=acrnxxx; user=XXX '                      >> remove_lock
  echo ' '                                                  >> remove_lock
  echo '  noprint=on '                                      >> remove_lock
  echo '  nextjob=on '                                      >> remove_lock
  echo ' '                                                  >> remove_lock
  echo '  #  * ................... Parmsub Parameters'      >> remove_lock
  echo '  lopgm="lopgm"; stime="600"; memory1="24mb";'      >> remove_lock
  echo ' JHOME='                                            >> remove_lock
  echo ' '                                                  >> remove_lock
  echo ' . comjcl.cdk'                                      >> remove_lock
  echo ' '                                                  >> remove_lock
  echo "cat > Execute_Script << 'end_of_script' "           >> remove_lock
  echo ' '                                                  >> remove_lock
  echo '  # Define the name of the lock file, if any '      >> remove_lock
  echo '  rtd_lock_file=xyz '                               >> remove_lock
  echo '  [ -n "$rtd_lock_file" ] && rm -f $rtd_lock_file ' >> remove_lock
  echo ' '                                                  >> remove_lock
  echo 'end_of_script'                                      >> remove_lock
  echo ' '                                                  >> remove_lock
  echo ' . endjcl.cdk '                                     >> remove_lock
  echo ' '                                                  >> remove_lock
  echo '#end_of_job '                                       >> remove_lock

#  * ............................ Condef Parameters ............................

  noprint=on       # do not return stdout to ~/.queue
  nextjob=on       # more jobs in the job string will follow
  pdump_rtd_diag=''
  pdump_rtd_plot=''
  pdump_rtd_plot_freq=''

#  * ............................. Deck Definition .............................

  # Require cccjob
  which cccjob || bail "cccjob is not in your path"

  JOBDESC="${rtd_diag}:s $REMOVE_LOCK"
  if [ $pxrtd_with_plot -eq 1 ]; then
    JOBDESC="${rtd_diag}:s ${rtd_plot}:s $REMOVE_LOCK"
  fi

  # Create the job string
  $CCCJOB_ENV cccjob --out=pxrtd_job --job="$JOBDESC" $start_opt $stop_opt cccjob_defs

  if [ "$rtd_in_series" -eq 1 ]; then
    # Insert this job into the current string so that it will run in series

    # The variable "crawork" must be defined/exported in the current environment
    splice_job_string pxrtd_job || bail "Problem in splice_job_string"

  else
    # Submit this job to run on the back end in parallel with the current string

    # Create a temporary directory in ~/tmp to hold submission files
    tsubdir=${JHOME:-$HOME}/tmp/tmp_pxrtd_${fid}_$stamp
    mkdir -p $tsubdir || bail "Cannot create $tsubdir"

    # Submit this job to run on the back end
    cp pxrtd_job $tsubdir
    ssh $back_end_mach "cd ${tsubdir}; rsub mdest=$back_end_mach pxrtd_job" 2>&1 || \
      bail "***ERROR*** Remote submission failed: rsub mdest=$front_end_mach pxrtd_job\n"

    # Clean up the temporary directory
    rm -fr $tsubdir
  fi

  # sucessful completion
  exit 0

end_of_script

 . endjcl.cdk

#end_of_job