#!/bin/sh
#=======================================================================
# Dump history files to tape and optionally run           --- pxdump ---
# diagnostics on another machine or transport files to a remote location
# These tasks are done in parallel.
# $Id: pxdump_jobdef 671 2012-05-04 23:35:22Z acrnrls $
#=======================================================================
#
# Larry Solheim  Oct,2009
#
# Files transferred/dumped/diagnosed will be of the form
#    ${pxdump_prefix}_${year}_m${mon}_$suffix
# for all months from previous_year, previous_month to current_year,
# current_month or from current_year, current_month to next_year,
# next_month depending on which of previous_(year|month) or
# next_(year|month) are set.
#
# The variables current_year, current_month, previous_year and
# previous_month are set when the job string is created.
#
# A variable named pxdump_suffix_list may also be set to modify the list
# of files that will be transferred/dumped. See below for details.
#
# pxdump_prefix_ can be set to override the value of pxdump_prefix
# files transferred/dumped will then be of the form
#    ${pxdump_prefix_}${year}_m${mon}_$suffix
# note the missing underscore between pxdump_prefix_ and year.
#
# The user must set up public key authentication on the remote machine
# so that transport from alef to the remote machine does not require
# a password. This means, create a public/private key pair on alef
# using ssh-keygen (if nessecary) then copy the users public key from
# alef and append it to ~user/.ssh/authorized_keys on the remote machine
#
# A default remote user, remote machine and remote directory into which
# files are to be copied is set in the rtrans script (invoked below).
# The user can define rtrans_remusr, rtrans_remserver and rtrans_remdir
# on the cccjob command line which will override the default values.
#=======================================================================
#
#     keyword :: pxdump
# description :: parallel dump/diagnosing of history files
#        hide :: yes
#
 set -a
 . betapath2

 username=acrnxxx; user=MYNAME; crawork=pxdump_job

 nextjob=on
 noprint=on
 debug=off

 stime=1800; memory1=900mb
 runid=job000; jobname=pxdump; time=$stime ; memory=$memory1;
 nqsprfx="${runid}_"; nqsext=''

 # Temporary directory where this script will run
 CCRNTMP=$CCRNTMP

 # RUNPATH on execution machine
 RUNPATH=$RUNPATH

 # Alternate path to a directory where .queue/.crawork will be found
 JHOME=''

 if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then
   # Allow optional reset of DATAPATH/RUNPATH
   JHOME_DATA=''
   DATAPATH=${JHOME_DATA:=$DATAPATH}
   RUNPATH=${JHOME_DATA:=$RUNPATH}
   # Allow optional reset of CCRNTMP
   JHOME_RUN=''
   CCRNTMP=${JHOME_RUN:=$CCRNTMP}
 fi

 # RUNPATH on back end
 BERUNPATH=$BERUNPATH

 # pxdump_ssh_args may be used to supply additional
 # command line arguments to the ssh commands used below.
 # This is only useful for debugging.
 pxdump_ssh_args=''

 # pxdump_with_rtrans is used to control inclusion of the remote copy
 # If pxdump_with_rtrans is set to "off" then only the tdumper job will run.
 pxdump_with_rtrans=${pdump_with_rtrans:=off}
 XXX=`echo $pxdump_with_rtrans|sed 's/ //g'`
 eval pxdump_with_rtrans\=$XXX
 [ "$pxdump_with_rtrans" = 'on' ]  && eval pxdump_with_rtrans\=1
 [ "$pxdump_with_rtrans" = 'off' ] && eval pxdump_with_rtrans\=0
 [ "$pxdump_with_rtrans" = 'yes' ] && eval pxdump_with_rtrans\=1
 [ "$pxdump_with_rtrans" = 'no' ]  && eval pxdump_with_rtrans\=0

 # pxdump_with_mdump is used to control inclusion of the tape dump job
 # If pxdump_with_mdump is set to "off" then only the rtrans job will run.
 pxdump_with_mdump=${pdump_with_mdump:=on}
 XXX=`echo $pxdump_with_mdump|sed 's/ //g'`
 eval pxdump_with_mdump\=$XXX
 [ "$pxdump_with_mdump" = 'on' ]  && eval pxdump_with_mdump\=1
 [ "$pxdump_with_mdump" = 'off' ] && eval pxdump_with_mdump\=0
 [ "$pxdump_with_mdump" = 'yes' ] && eval pxdump_with_mdump\=1
 [ "$pxdump_with_mdump" = 'no' ]  && eval pxdump_with_mdump\=0

 # pdump_with_delete is used to control deleting history files after they are dumped
 pdump_with_delete=on
 XXX=`echo $pdump_with_delete|sed 's/ //g'`
 eval pdump_with_delete\=$XXX
 [ "$pdump_with_delete" = 'on' ]  && eval pdump_with_delete\=1
 [ "$pdump_with_delete" = 'off' ] && eval pdump_with_delete\=0
 [ "$pdump_with_delete" = 'yes' ] && eval pdump_with_delete\=1
 [ "$pdump_with_delete" = 'no' ]  && eval pdump_with_delete\=0

 # pxdump_with_diag is used to control inclusion of the diagnostic job
 # If pxdump_with_diag is set to "off" then no diagnostic job is launched.
 pxdump_with_diag=${pdump_with_diag:=on}
 XXX=`echo $pxdump_with_diag|sed 's/ //g'`
 eval pxdump_with_diag\=$XXX
 [ "$pxdump_with_diag" = 'on' ]  && eval pxdump_with_diag\=1
 [ "$pxdump_with_diag" = 'off' ] && eval pxdump_with_diag\=0
 [ "$pxdump_with_diag" = 'yes' ] && eval pxdump_with_diag\=1
 [ "$pxdump_with_diag" = 'no' ]  && eval pxdump_with_diag\=0

 # If pxdump_use_scp is set to "on" then scp will be used for transport.
 # The default if pxdump_use_scp = off is to use nrcp for transport.
 pxdump_use_scp=${pdump_use_scp:=on}
 XXX=`echo $pxdump_use_scp|sed 's/ //g'`
 eval pxdump_use_scp\=$XXX
 [ "$pxdump_use_scp" = 'on' ]  && eval pxdump_use_scp\=1
 [ "$pxdump_use_scp" = 'off' ] && eval pxdump_use_scp\=0
 [ "$pxdump_use_scp" = 'yes' ] && eval pxdump_use_scp\=1
 [ "$pxdump_use_scp" = 'no' ]  && eval pxdump_use_scp\=0

 . comjcl.cdk
 cat > Execute_Script <<'end_of_script'

  # bail is a simple error exit routine
  bail(){
    echo "pxdump: "$1
    echo " " >> haltit
    exit 1
  }

  username=acrnxxx; user=MYNAME; crawork=pxdump_job
  runid=job000; uxxx=uxxx; pxdump_uxxx=$uxxx;
  pxdump_prefix=${pxdump_uxxx}_${runid}; pxdump_prefix_=${pxdump_prefix}_

  noprint=on
  nextjob=on

  # ---Start_submit_ignore_code----

  # Indicate how cccjob should be invoked
  # Setting CCCJOB_ROOT will allow a job specific version of cccjob to used
  CCCJOB_ROOT=''
  if [ -z "$CCCJOB_ROOT" ]; then
    CCCJOB_ENV=''
  else
    eval CCCJOB_ENV=\'env CCCJOB_ROOT\=$CCCJOB_ROOT\'
  fi

  # pxdump_suffix_list is a white space separated list of suffixes
  # of file names to be transferred/dumped. Any suffix in this list may be
  # modified by appending a + followed by a comma separated list of
  # numbers (no white space is allowed within this modifier). Each
  # number within the modifier list will correspond to a month (1-12)
  # for which a file with this suffix is to be included. If the
  # modifier exists for a particular suffix then only those months
  # indicated in the modifier will be added to the file list.
  # e.g. the inclusion of rs+12 in this list will copy only DEC restarts
  mdump_suffix_list=''
  pxdump_suffix_list=${mdump_suffix_list:='gs ss td gz cm ab+12 ob+12 an+12 rs+12 cs+12 os+12'}

  # pxdump_suffix_ flags the addition of an underscore between the
  # suffix and the month (e.g. m01_gs) in file names generated here
  pxdump_suffix_=1
  XXX=`echo $pxdump_suffix_|sed 's/ //g'`
  eval pxdump_suffix_\=$XXX
  [ "$pxdump_suffix_" = 'on' ]  && eval pxdump_suffix_\=1
  [ "$pxdump_suffix_" = 'off' ] && eval pxdump_suffix_\=0
  [ "$pxdump_suffix_" = 'yes' ] && eval pxdump_suffix_\=1
  [ "$pxdump_suffix_" = 'no' ]  && eval pxdump_suffix_\=0

  # These variables are set when the job string is created
  run_start_year=NotSet
  run_start_month=NotSet
  run_stop_year=NotSet
  run_stop_month=NotSet
  current_year=NotSet
  current_month=NotSet
  previous_year=NotSet
  previous_month=NotSet
  next_year=NotSet
  next_month=NotSet
  [ $current_year = "NotSet" -o $current_month = "NotSet" ] && \
    bail "current_year or current_month is not set"

  # files transferred will be for months in the range from
  # pxdump_start_year,pxdump_start_mon to pxdump_end_year,pxdump_end_mon
  if [ $previous_year = "NotSet" -o $previous_month = "NotSet" ]; then
    if [ $next_year = "NotSet" -o $next_month = "NotSet" ]; then
      bail "Neither previous_(year|month) nor next_(year|month) are set"
    else
      pxdump_start_year=$current_year
      pxdump_start_mon=$current_month
      pxdump_end_year=$next_year
      pxdump_end_mon=$next_month
    fi
  else
    pxdump_start_year=$previous_year
    pxdump_start_mon=$previous_month
    pxdump_end_year=$current_year
    pxdump_end_mon=$current_month
  fi

  pxdump_start_year=`echo $pxdump_start_year|awk '{printf "%3.3d",$1}' -`
  pxdump_start_mon=`echo $pxdump_start_mon|awk '{printf "%2.2d",$1}' -`
  pxdump_end_year=`echo $pxdump_end_year|awk '{printf "%3.3d",$1}' -`
  pxdump_end_mon=`echo $pxdump_end_mon|awk '{printf "%2.2d",$1}' -`

  [ -z "$pxdump_start_year" ] && bail "pxdump_start_year is null"
  [ -z "$pxdump_end_year" ]   && bail "pxdump_end_year is null"
  [ -z "$pxdump_start_mon" ]  && bail "pxdump_start_mon is null"
  [ -z "$pxdump_end_mon" ]    && bail "pxdump_end_mon is null"
  [ $pxdump_start_mon -gt 12 -o $pxdump_start_mon -lt 1 ] &&\
    bail "pxdump_start_mon=$pxdump_start_mon is out of range"
  [ $pxdump_end_mon -gt 12 -o $pxdump_end_mon -lt 1 ] &&\
    bail "pxdump_end_mon=$pxdump_end_mon is out of range"
  [ $pxdump_start_year -gt $pxdump_end_year ] &&\
    bail "pxdump_start_year=$pxdump_start_year is out of range"

  # Generate a sequence of file names to dump
  pxdump_curr_year=`echo $pxdump_start_year|awk '{y=$1-1;printf "%3.3d", y}' -`
  nfile=0
  while [ $pxdump_curr_year -lt $pxdump_end_year ]; do
    pxdump_curr_year=`echo $pxdump_curr_year|awk '{y=1+$1;printf "%3.3d", y}' -`
    if [ $pxdump_curr_year -eq $pxdump_start_year ]; then
      mm=`echo $pxdump_start_mon|awk '{m=$1-1;printf "%2.2d", m}' -`
    else
      mm=0
    fi
    if [ $pxdump_curr_year -eq $pxdump_end_year ]; then
      mm_end=$pxdump_end_mon
    else
      mm_end=12
    fi
    while [ $mm -lt $mm_end ]; do
      mm=`echo $mm|awk '{m=1+$1;printf "%2.2d", m}' -`
      if [ "$pxdump_suffix_" = '1' ]; then
        bname=${pxdump_prefix_}${pxdump_curr_year}_m${mm}_
      else
        bname=${pxdump_prefix_}${pxdump_curr_year}_m${mm}
      fi
      for suffix in $pxdump_suffix_list; do
        mlist=`echo $suffix|awk -F'+' '{print $2}' -`
        mlist=`echo $mlist|sed 's/,/ /g'`
        if [ -n "$mlist" ]; then
          suffix=`echo $suffix|sed 's/+.*$//'`
          # assume that mlist is a white space separated list of numbers
          # indicating which months to dump
          for xx in $mlist; do
            if [ $xx -eq $mm ]; then
              nfile=`echo $nfile|awk '{printf "%d",$1+1}' -`
              nfile=`echo $nfile|sed -e 's/^ *//' -e 's/^0*//'`
              eval file${nfile}=$bname$suffix
            fi
          done
        else
          nfile=`echo $nfile|awk '{printf "%d",$1+1}' -`
          nfile=`echo $nfile|sed -e 's/^ *//' -e 's/^0*//'`
          eval file${nfile}=$bname$suffix
        fi
      done
    done
  done
  join=$nfile

  # Create a file containing the list of file names to process
  nsent=0
  rm -f file_list
  touch file_list
  while [ $nsent -lt $nfile ]; do
    nsent=`expr $nsent + 1`
    eval local_file=\$file$nsent
    [ -z "$local_file" ] && bail "Invalid file name in file$nsent"
    # Create this list in a format that may be inserted directly into a
    # a delete job
    echo "  file${nsent}=$local_file" >> file_list
  done

  # Append the number of files to file_list
  echo "  join=$nfile" >> file_list

  echo "pxdump: file_list"
  cat file_list

  # Insert a user supplied list of files that are to be deleted after
  # they have been dumped to tape. These file names must be assigned to
  # the shell variables fdel01,fdel02,... and the shell variable nfdel
  # must be set to the number in this list. If nfdel=0 then there will
  # be no delete job created below.
  nfdel=0
  # <<INSERT_PXDUMP>>

  if [ $nfdel -gt 0 ]; then
    # Create a file containing the list of files to be deleted from the
    # back end and define PXDELJOB to insert in the cccjob job description
    nn=0
    rm -f fdel_list
    touch fdel_list
    while [ $nn -lt $nfdel ]; do
      nn=`expr $nn + 1`
      eval local_fdel=\$fdel$nn
      [ -z "$local_fdel" ] && bail "Invalid file name in fdel$nn"
      # Create this list in a format that may be inserted directly into a
      # a delete job
      echo "  file${nn}=$local_fdel" >> fdel_list
    done
    # Append the number of files to fdel_list
    echo "  join=$nfdel" >> fdel_list

    echo "pxdump: fdel_list"
    cat fdel_list

    PXDELJOB="pxdel=fdel_list:s"
  else
    # Set PXDELJOB null so that no pxdel is inserted in the strings created below
    PXDELJOB=''
  fi

  if [ $pdump_with_delete -eq 0 ]; then
    PXDELJOB=''
  fi

  # Set a variable containing the name of the back end machine to which
  # the delete job will be sent
  bemach=''
  pdump_back_end=${bemach:=''}
  back_end_mach=${pdump_back_end:='hadar'}

  # gateway is the name of the machine to which files
  # are sent prior to being moved off site
  pdump_front_end=''
  pxdump_gateway=${pdump_front_end:=''}
  gateway=${pxdump_gateway:='pollux'}

  # Get the name of the machine running this script
  this_mach=`uname -n|awk -F\. '{print \$1}' -`
  case $this_mach in
     c1*) this_mach=spica  ;;
     c2*) this_mach=hadar  ;;
     c3*) this_mach=rigel  ;;
     c4*) this_mach=maia   ;;
     c5*) this_mach=naos   ;;
     c6*) this_mach=saiph  ;;
     c7*) this_mach=zeta   ;;
   alef*) this_mach=alef   ;;
    ib3*) this_mach=pollux ;;
  esac

  # running_on_gateway, if set, indicates that this script
  # is running on the gateway machine
  running_on_gateway=0
  if [ "$this_mach" = "$gateway" ]; then
    running_on_gateway=1
    echo "pxdump is running on $gateway"
  fi

  # pdump_lock is the name of the lock file associated with pdump.
  # This should only be set if this job string was created by the module pdump
  pdump_lock=''

  # Define a unique stamp for use in file names etc
  stamp=`date "+%j%H%M%S"$$`

  # Extra ssh args (use -v for debugging connections)
  ssh_args=${pxdump_ssh_args:-''}

  # Once all the files have been transferred, create a job to dump them
  # to tape, a job to transfer them to a remote machine and a job to run
  # diagnostics (assuming history files are being dumped).

  # ym_range is used as part of some file names defined below
  ym_range="${pxdump_start_year}m${pxdump_start_mon}_${pxdump_end_year}m${pxdump_end_mon}"

  # if set then mdump_months is used below to configure a serial mdump string
  # that will be comprised of a sequence of tdumper jobs, each of which will
  # dump mdump_months months of history files. It also signifies that no
  # history files should be deleted from the front end.
  mdump_months=''

  # Define partial file names for the dump, rtrans and diag jobs that will be
  # used in multiple places below
  fdump=mdump_${runid}_$ym_range
  ftran=rtrans_${runid}_$ym_range
  fdiag=diag_${runid}_$ym_range

  # Define names of lock files for each of tape/transfer/diag jobs

  if [ $running_on_gateway -eq 1 ]; then
    # If this script is running on the gateway machine
    # then use $HOME for rem_dir
    rem_home=${JHOME:-$HOME}
  else
    # If this script is not running on the gateway machine
    # then set rem_dir to the users home dir on the gateway machine
    rem_home=`ssh $gateway 'echo $HOME'` 2>&1 || \
      bail "***ERROR*** Unable to determine remote home directory.\n"
  fi
  mdump_lock="${rem_home}/.queue/.crawork/lock_${fdump}_$stamp"
  rtrans_lock="${rem_home}/.queue/.crawork/lock_${ftran}_$stamp"
  diag_lock="${rem_home}/.queue/.crawork/lock_${fdiag}_$stamp"
  mdump_string="${rem_home}/.queue/.crawork/${fdump}_${stamp}_string"
  rtrans_string="${rem_home}/.queue/.crawork/${ftran}_${stamp}_string"
  diag_string="${rem_home}/.queue/.crawork/${fdiag}_${stamp}_string"

  # If rtd_lock_file is defined when this job is created then it is
  # assumed that run time diagnostics are being used. When they are
  # we need to create the associated lock file here.
  rtd_lock_file=''
  [ -n "$rtd_lock_file" ] && touch $rtd_lock_file

  # Create the lock files unless this is a dry run
  if [ $running_on_gateway -eq 1 ]; then
    # If this script is running on the gateway machine
    # then create the lock files on this machine
    [ $pxdump_with_mdump -eq 1 ]  && cp -f file_list $mdump_lock
    [ $pxdump_with_rtrans -eq 1 ] && cp -f file_list $rtrans_lock
    [ $pxdump_with_diag -eq 1 ] && cp -f file_list $diag_lock
  else
    # If this script is not running on the gateway machine
    # then create the lock files on the gateway machine
    if [ $pxdump_with_mdump -eq 1 ]; then
      scp file_list ${gateway}:$mdump_lock 2>&1 || \
        bail "***ERROR*** Unable to create lock file $mdump_lock\n"
    fi
    if [ $pxdump_with_rtrans -eq 1 ]; then
      scp file_list ${gateway}:$rtrans_lock 2>&1 || \
        bail "***ERROR*** Unable to create lock file $rtrans_lock\n"
    fi
    if [ $pxdump_with_diag -eq 1 ]; then
      scp file_list ${gateway}:$diag_lock 2>&1 || \
        bail "***ERROR*** Unable to create lock file $diag_lock\n"
    fi
  fi
  
  # Each of the tape job, the rtrans job and the diag job string will contain
  # a final job in the string that will delete the files, but not until all
  # jobs have completed.

  # This eval is necessary to prevent the shebang line from being
  # blindly replaced with a different shell by submit3
  eval shebang=\#\!/\bin/\sh
  cat > del_part1 <<EOF
$shebang
  set -a
  . betapath2
  jobname=del_list; crawork=delete_job; username=acrnxxx; user=XXX;

  noprint=on
  nextjob=on

  #  * ........................... Parmsub Parameters ....................
  lopgm="lopgm"; stime="900"; memory1="24mb";

  # Define the name of the pdump_lock file, if any
  pdump_lock=$pdump_lock

  # Initialize join in case the file list is not included
  join=0

EOF

  cat > del_part2_mdump <<EOF
  # Remove the mdump lock file
  rm -f $mdump_lock

  # If the rtrans or diag lock file exists then do not delete any files but remove
  # this mdump job string from the crawork directory
  if [ -f "$rtrans_lock" -o -f "$diag_lock" ]; then
    rm -f "$mdump_string"
    exit 0
  fi

EOF

  cat > del_part2_rtrans <<EOF
  # Remove the rtrans lock file
  rm -f $rtrans_lock

  # If the mdump or diag lock file exists then do not delete any files but remove
  # this rtrans job string from the crawork directory
  if [ -f "$mdump_lock" -o -f "$diag_lock" ]; then
    rm -f "$rtrans_string"
    exit 0
  fi

EOF

  cat > del_part2_diag <<EOF
  # Remove the diag lock file
  rm -f $diag_lock

  # If the mdump or diag lock file exists then do not delete any files but remove
  # this diag job string from the crawork directory
  if [ -f "$mdump_lock" -o -f "$rtrans_lock" ]; then
    rm -f "$diag_string"
    exit 0
  fi

EOF

  cat > del_part4 <<'EOF'

  #  * ............................ Condef Parameters ............................
  noprint=on
  nextjob=on

  #  * ............................. Deck Definition .............................
  . comjcl.cdk
  cat > Execute_Script <<'end_of_exec_script'
    nn=0
    while [ $nn -lt $join ]; do
      nn=`expr $nn + 1`
      eval FILE=\$file$nn
      access $FILE $FILE nocp na
      delete $FILE na
    done

    # Remove the pdump lock file if it exists
    [ ! -z "$pdump_lock" ] && rm -f $pdump_lock

end_of_exec_script
EOF
  echo "  . endjcl.cdk" >> del_part4
  echo "#end_of_job"    >> del_part4

  if [ -n "$mdump_months" ]; then
    cat del_part1 del_part2_mdump           del_part4 > mdump_delete_job
  else
    cat del_part1 del_part2_mdump file_list del_part4 > mdump_delete_job
  fi
  cat del_part1 del_part2_rtrans file_list del_part4 > rtrans_delete_job
  cat del_part1 del_part2_diag   file_list del_part4 > diag_delete_job

  # Require cccjob
  which cccjob || bail "cccjob is not in your path"

  # Define start/stop year/month cccjob options to be used below
  curr_year=`echo $pxdump_start_year|awk '{printf "%d",$1}' -`
  curr_mon=`echo $pxdump_start_mon|awk '{printf "%d",$1}' -`
  next_year=`echo $pxdump_end_year|awk '{printf "%d",$1}' -`
  next_mon=`echo $pxdump_end_mon|awk '{printf "%d",$1}' -`
  start_opt="--start_time=${curr_year}:$curr_mon"
  stop_opt="--stop_time=${next_year}:$next_mon"

  ######### mdump job ###########

  # besc = on causes a tdumper job to run on the back end (spica/hadar)
  pxdump_besc=''
  besc=${pxdump_besc:=off}

  if [ $pxdump_with_mdump -eq 1 ]; then

    # These variable may be set when the pxdump job string is created
    # If a variable is set then it will be used in the mdump job

    mdump_uxxx=''
    mdump_prefix=''
    mdump_prefix_=''
    mdump_suffix_list="$pxdump_suffix_list"
    mdump_cfsuser=''
    mdump_CCRNTMP=''
    mdump_RUNPATH=''
    mdump_qsublog=''
    mdump_with_lock_file=''
    with_lsarc=''
    check_cfs_arcfile=''
    nontwrkchk=''
    cfsuser=''
    masterdir=off
    shortermdir=on
    nolist=''
    curr_mdump_lock_file=$mdump_lock

    # If mdump_months is defined then add this def to the cccjob command line
    # as the value of months. This will cause the resulting dump job to be
    # comprised of (possibly) more than 1 tdumper that will run in serial
    months_opt=''
    MDUMPJOB='mdump:s'
    if [ -n "$mdump_months" ]; then
      eval months_opt="months\=$mdump_months"
      MDUMPJOB='mdump:m'
      # Do not define an arclabel for this tape dump, let mdump do this
      mdump_arclabel=''
      # sv should always be off when history files are on the back end
      eval sv\=off
    else
      # Define an arclabel for this tape dump
      mdump_arclabel="${pxdump_uxxx}_${runid}_$ym_range"
      # sv should always be on when gateway is the front end
      # sv on means dump from front end disks
      eval sv\=on
    fi

    echo "MDUMPJOB = $MDUMPJOB"
    echo "mdump_months = $mdump_months"
    echo "months_opt = $months_opt"

    varlist='uxxx
             CCCJOB_ROOT
             runid
             noprint
             mdump_uxxx
             mdump_prefix
             mdump_prefix_
             mdump_suffix_list
             mdump_cfsuser
             mdump_CCRNTMP
             mdump_RUNPATH
             mdump_qsublog
             mdump_with_lock_file
             with_lsarc
             check_cfs_arcfile
             nontwrkchk
             cfsuser
             masterdir
             shortermdir
             nolist
             sv
             besc
             mdump_arclabel
             curr_mdump_lock_file
             rtd_lock_file
             back_end_mach
             JHOME
             JHOME_DATA
             JHOME_RUN
             BERUNPATH'

    # Create a file containing variable definitions used by the tape job

    # This eval will protect this crawork def from cccjob substitution
    eval craworkdef\=crawork\=${fdump}_$stamp
    cat > mdump_defs <<EOF
$craworkdef
EOF
    for var in $varlist; do
      eval val=\$$var
      # If this variable is defined add it to the list
      [ -n "$val" ] && echo ${var}=\'$val\' >> mdump_defs
    done

    # Create the tape job
    $CCCJOB_ENV cccjob --out=${fdump}_job --job="$MDUMPJOB mdump_delete_job:s ${PXDELJOB}" \
           $start_opt $stop_opt mdump_defs $months_opt

  fi

  ######### rtrans job ###########

  if [ $pxdump_with_rtrans -eq 1 ]; then

    # These variable may be set when the pxdump job string is created
    # If a variable is set then it will be used in the rtrans job
    rtrans_remusr=''
    rtrans_remserver=''
    rtrans_remdir=''
    rtrans_force=''
    rtrans_scp_args=''
    rtrans_ssh_args=''
    rtrans_getput=''
    rtrans_suffix_list='gs ss td gz'
    rtrans_uxxx=''
    rtrans_prefix=''
    rtrans_prefix_=''
    rtrans_suffix_=''
    varlist='uxxx
             CCCJOB_ROOT
             runid
             noprint
             rtrans_remusr
             rtrans_remserver
             rtrans_remdir
             rtrans_force
             rtrans_scp_args
             rtrans_ssh_args
             rtrans_getput
             rtrans_suffix_list
             rtrans_uxxx
             rtrans_prefix
             rtrans_prefix_
             rtrans_suffix_
             rtd_lock_file
             JHOME
             JHOME_DATA
             JHOME_RUN
             back_end_mach'

    # Create a file containing variable definitions used by the rtrans job

    # This eval will protect this crawork def from cccjob substitution
    eval craworkdef\=crawork\=${ftran}_$stamp
    cat > rtrans_defs <<EOF
$craworkdef
EOF
    for var in $varlist; do
      eval val=\$$var
      # If this variable is defined add it to the list
      [ -n "$val" ] && echo ${var}=\'$val\' >> rtrans_defs
    done

    # Create the rtrans job
    $CCCJOB_ENV cccjob --out=${ftran}_job --job="rtrans:s rtrans_delete_job:s" \
           $start_opt $stop_opt rtrans_defs

  fi

  ######### diag job ###########

  if [ $pxdump_with_diag -eq 1 ]; then

    # These variable may be set when the pxdump job string is created
    # If a variable is set then it will be used in the diag job

    dump_diag_start_year=`echo ${dump_diag_start_year:=0}|awk '{printf "%3.3d",$1}' -`
    dump_diag_start_mon=`echo ${dump_diag_start_mon:=0}|awk '{printf "%2.2d",$1}' -`
    dump_diag_end_year=`echo ${dump_diag_end_year:=0}|awk '{printf "%3.3d",$1}' -`
    dump_diag_end_mon=`echo ${dump_diag_end_mon:=0}|awk '{printf "%2.2d",$1}' -`
    diag_ym_range="${dump_diag_start_year}m${dump_diag_start_mon}_${dump_diag_end_year}m${dump_diag_end_mon}"

    dump_diag_uxxx=${dump_diag_uxxx:="dc"}
    dump_diag_prefix="${dump_diag_uxxx}_${runid}"
    dump_list_arclabel="${dump_diag_uxxx}_${runid}_$diag_ym_range"

    diag_uxxx=''
    model_uxxx=''
    hres=''
    modver=''
    xref01=''
    xref02=''
    xref03=''
    xref04=''
    xref05=''
    xref06=''
    xref07=''
    xref08=''
    xref09=''
    xref10=''
    xref11=''
    xref12=''
    xref13=''
    xref14=''
    xref15=''
    varlist='runid
             CCCJOB_ROOT
             JHOME
             JHOME_DATA
             JHOME_RUN
             noprint
             diag_uxxx
             model_uxxx
             hres
             modver
             xref01
             xref02
             xref03
             xref04
             xref05
             xref06
             xref07
             xref08
             xref09
             xref10
             xref11
             xref12
             xref13
             xref14
             xref15
             cfsuser
             masterdir
             shortermdir
             nolist
             sv
             dump_list_arclabel
             rtd_lock_file
             back_end_mach'

    # Create a file containing variable definitions used by the diag job

    # This eval will protect this crawork def from cccjob substitution
    eval craworkdef\=crawork\=${fdiag}_$stamp
    cat > diag_defs <<EOF
$craworkdef
EOF
    for var in $varlist; do
      eval val=\$$var
      # If this variable is defined add it to the list
      [ -n "$val" ] && echo ${var}=\'$val\' >> diag_defs
    done

    # Allow the user to supply their own diagnostic job
    # pdump_diag_job, if set, should be a full pathname or exist
    # as ~/cccjobs/${pdump_diag_job}_jobdef on the execution machine
    pxdump_diag_job=${pdump_diag_job:='stdiag'}

    JOBDESC="${pxdump_diag_job}:m diag_delete_job:s"

    dump_diag_files=${dump_diag_files:=0}
    if [ $dump_diag_files -eq 1 ]; then
      # If diagnostic files are to be dumped and deleted then add
      # this dump/delete to the job description
      [ $dump_diag_start_year -le 0 ] && bail "dump_diag_start_year not set"
      [ $dump_diag_start_mon -le 0 ]  && bail "dump_diag_start_mon not set"
      [ $dump_diag_end_year -le 0 ]   && bail "dump_diag_end_year not set"
      [ $dump_diag_end_mon -le 0 ]    && bail "dump_diag_end_mon not set"

      dump_diag_suffix_list=${dump_diag_suffix_list:="gp xp cp"}
      dump_diag_list="dump_diag_flist_$stamp"
      rm -f $dump_diag_list
      touch $dump_diag_list

      # Generate a list of diagnostic files to dump to cfs
      yyy=`echo $dump_diag_start_year|awk '{y=$1-1;printf "%3.3d", y}' -`
      join=0
      while [ $yyy -lt $dump_diag_end_year ]; do
        yyy=`echo $yyy|awk '{y=1+$1;printf "%3.3d", y}' -`
        if [ $yyy -eq $dump_diag_start_year ]; then
          mm=`echo $dump_diag_start_mon|awk '{printf "%2.2d",$1-1}' -`
        else
          mm=0
        fi
        if [ $yyy -eq $dump_diag_end_year ]; then
          mm_end=$dump_diag_end_mon
        else
          mm_end=12
        fi
        while [ $mm -lt $mm_end ]; do
          mm=`echo $mm|awk '{printf "%2.2d",$1+1}' -`
          bname=${dump_diag_prefix}_${yyy}_m${mm}_
          if [ -n "$dump_diag_suffix_list" ]; then
            # Iterate over the suffixes in dump_diag_suffix_list
            for suffix in $dump_diag_suffix_list; do
              mlist=`echo $suffix|awk -F'+' '{print $2}' -`
              mlist=`echo $mlist|sed 's/,/ /g'`
              if [ -n "$mlist" ]; then
                suffix=`echo $suffix|sed 's/+.*$//'`
                # assume that mlist is a white space separated list of numbers
                # indicating which months to dump
                for xx in $mlist; do
                  if [ $xx -eq $mm ]; then
                    echo "$bname$suffix" >> $dump_diag_list
                  fi
                done
              else
                echo "$bname$suffix" >> $dump_diag_list
              fi
            done
          else
            # No suffix
            echo "$bname" >> $dump_diag_list
          fi
        done
      done

      # Redefine the job description
      JOBDESC="${pxdump_diag_job}:m dump_list=${dump_diag_list}:s del_list=${dump_diag_list}:s diag_delete_job:s"

    fi

    # Create the diag job
    $CCCJOB_ENV cccjob --out=${fdiag}_job --job="$JOBDESC" $start_opt $stop_opt diag_defs

  fi

  ######### submit jobs ###########

  # Submit these jobs to run on the gateway machine

  # Create a temporary directory in ~/tmp to hold submission files
  host=`hostname` || bail "Problem in hostname"
  psubdir=${JHOME:-$HOME}/tmp/tmp_pxdump_${host}_$stamp
  mkdir -p $psubdir || bail "Cannot create $psubdir"

  if [ $pxdump_with_mdump -eq 1 ]; then
    if [ $running_on_gateway -eq 1 ]; then
      echo "pxdump: morigin=$morigin"
      rsub -x mdest=$gateway ${fdump}_job
    else
      this_remjob=${fdump}_job
      cp $this_remjob $psubdir
      ssh $ssh_args $gateway "cd ${psubdir}; rsub mdest=$gateway $this_remjob" 2>&1 || \
        bail "***ERROR*** Remote submission failed for $this_remjob\n"
    fi
  fi

  if [ $pxdump_with_rtrans -eq 1 ]; then
    if [ $running_on_gateway -eq 1 ]; then
      rsub mdest=$gateway ${ftran}_job
    else
      this_remjob=${ftran}_job
      cp $this_remjob $psubdir
      ssh $ssh_args $gateway "cd ${psubdir}; rsub mdest=$gateway $this_remjob" 2>&1 || \
        bail "***ERROR*** Remote submission failed for $this_remjob\n"
    fi
  fi

  if [ $pxdump_with_diag -eq 1 ]; then
    if [ $running_on_gateway -eq 1 ]; then
      rsub mdest=$gateway ${fdiag}_job
    else
      this_remjob=${fdiag}_job
      cp $this_remjob $psubdir
      ssh $ssh_args $gateway "cd ${psubdir}; rsub mdest=$gateway $this_remjob" 2>&1 || \
        bail "***ERROR*** Remote submission failed for $this_remjob\n"
    fi
  fi

  # Clean up the temporary directory
  rm -fr $psubdir

  # sucessful completion
  exit 0

  # ---Stop_submit_ignore_code----

end_of_script

 . endjcl.cdk

#end_of_job