#!/bin/sh

#   Mar 04/09 - F.Majaess (Major restructuring ...)
#   Jan 06/09 - F.Majaess (Revised for sa/saiph, ib/dorval-ib, af/alef)
#   Oct 16/06 - F.Majaess (Revised for ma/maia, ns/naos)
#   Mar 01/06 - F.Majaess (Revised for rg/rigel)
#   May 25/05 - F.Majaess (Implemented "nqsprfx" support)
#   Feb 15/05 - F.Majaess (Revised for AIX/Linux setup in Victoria)
#   Dec 07/04 - F.Majaess (Abort if "oldiag"/"TSTRDIR" is not accessible)
#   Dec 04/03 - F.Majaess (Added support for "nnodex" for MPI jobs)
#   Jun 20/03 - F.Majaess (Revised for az/azur)
#   Sep 24/02 - F.Majaess (Ensure "nextjob/crawork" and/or "auto/flabel" 
#                          proper settings, exclude comments lines from 
#                          extracted parameters check, add "run/runid" 
#                          parameters support)
#   Feb 25/02 - F.Majaess (Revised for ya/yata)
#   Jan 10/02 - F.Majaess (Revise to get rid of annoying NQE email messages)
#   Sep 19/01 - F.Majaess (Ensure at least "$X5minmem_mb" MB memory 
#                          is used for SX5 jobs)
#   May 22/01 - F.Majaess (Added support for "nprocx" on the SX-nodes)
#   Jan 05/00 - F.Majaess (Added kz/kaze)
#   Jun 11/99 - F.Majaess (Removed "oldmod" support)
#   Apr 21/99 - F.Majaess (Revise to make sure "TSTRDIR" is scanned)
#   Apr 08/99 - F.Majaess (Ensure error message is issued for missing deck)
#   Jan 21/99 - F.Majaess (Added yo/yonaka)
#   Jun 02/98 - F.Majaess (Revised for "orion" batch queue on pollux)
#   May 04/98 - F.Majaess (Removed o2000-2)
#   Mar 27/98 - F.Majaess (Added o2000-2)
#   Feb 24/98 - F.Majaess (Ensure proper handling of "[sv]vic=off")
#   Dec 05/97 - F.Majaess (Enforce extraction and evaluation of "[sv]vic")
#   Oct 24/97 - F.Majaess (Implemented "nqsext" functionality)
#   Jul 14/97 - F.Majaess (Added sx/hiru)
#   Jul 09/97 - D. Liu    (change #var_redef to #par_def)
#   Mar 26/97 - F.Majaess (Add RCMRUN for the list of evaluated parameters)
#   Feb 11/97 - F.Majaess (Ensure NQS jobname <= 14 characters long)
#   Jan 28/97 - D. Liu    (allow comments (beginning with a '#') after
#                          calling a deck by '. xxx.dk' in a submission job)
#   Nov 18/96 - F.Majaess (Implement preprocessing Victoria targeting job to
#                          ensure running under Bourne Shell ie. change
#                          'bin/sh' to '/bin/bsh')
#   Nov 04/96 - F.Majaess (Revised to honour and use "oldiag" environment 
#                          variable, when it's set by the user, in setting 
#                          the path to the standard diagnotic library to 
#                          use) 
#   Sep 17/96 - D.Liu     (fix for processing updates in multi-job string)
#   Sep 11/96 - F.Majaess (Adjust path massaging for Victoria, improve 
#                          running in the background option)
#   Sep 11/96 - D.Liu     (syntax check for certain variable assignments)
#   May 31/96 - D.Liu     (modified/added support for if-blocks
#                          in parameter section)
#   Apr 24/96 - D.Liu (Implement "#var_redef")
#   Mar 14/96 - F.Majaess (Added sx4)
#   Sep 27/95 - F.Majaess (Revise the path massaging for Victoria)
#   Jun 01/95 - F.Majaess (Replace cr01 by orion)
#   May 15/95 - F.Majaess (Allow 'cfsxfer' on tapejobs)
#   Apr 11/95 - F.Majaess (Modified to add Downsview as a Site_ID and
#                          replacing cidsv08 by cr01)
#   Mar 27/95 - E. Chan (add support for update decks in user $HOME/bin)
#   Feb 24/95 - E. Chan (add support for Victoria and save all string
#                        control files in .queue/.crawork on all machines) 
#   Nov 10/94 - E. Chan (add support for the SX-3R and new internal machine
#                        identifiers)
#   Jan 18/94 - E. Chan

#id submit3  - pre-process the submission file, if necessary, and
#id            submit the job to the desired machine  

#   AUTHOR  - E. Chan

#hd PURPOSE - "submit3" takes a diagnositic or model submission file written 
#hd           in the new CCRN standard (pure unix) and submits the job to the 
#hd           requested machine (defaults to the machine on which submit3 is 
#hd           invoked).  If the file contains update directives to
#hd           the deck, these will be processed.  If the job is already in the 
#hd           form of a stand alone Unix script, then it is simply 
#hd           submitted.
#hd
#hd           The name of the job submitted is taken from the "jobname" 
#hd           specified in the deck. If the "jobname" doesn't exist, 
#hd           then the name of the file is used instead. 
#hd   
#hd           The "script" option allows the generation of a pre-processed
#hd           job script ready to be executed on a Unix machine. The job is
#hd           not actually sent. This allows the user to peruse the script
#hd           for debugging purposes. The script is saved in the directory
#hd           from which "submit3" is invoked using the "jobname" as described
#hd           above followed by the extension ".script".
#hd 
#hd           The "log" option routes all printout back to the user's .queue
#hd           directory via NQS. This is useful if a job dies before the
#hd           printout is routed via the comjcl/endjcl common decks. Usually,
#hd           the printout does not return to the user in such situations. 
#hd           This option ensures that some printout comes back which may be
#hd           prove useful in debugging the problem. Also, this is a way to
#hd           get a listing of all parmsub and condef parameters as they are
#hd           defined at the beginning of the job.
#hd 
#hd           Note: "submit3" extracts and evaluates parmsub/condef variables
#hd                           from the submission job. 
#hd                           Section(s) bounded by:
#hd                            # ---Start_submit_ignore_code
#hd                           and
#hd                            # ---Stop_submit_ignore_code
#hd                           separator lines are skipped and ignored in this
#hd                           variables extraction and evaluation process.

#pr PARAMETERS:
#pr
#pr   POSITIONAL
#pr
#pr            fn = file containing submission job (filename may contain path
#pr                 information)
#pr
#pr   SECONDARY
#pr
#pr   hr or hare   = job to target hare.
#pr
#pr   br or brooks = job to target brooks.
#pr
#pr   dy or daley  = job to target daley.
#pr
#pr   ba or banting = job to target banting.
#pr
#pr   p1 or ppp1   = job to target ppp1.
#pr
#pr   p2 or ppp2   = job to target ppp2.
#pr
#pr   p3 or ppp3   = job to target ppp3.
#pr
#pr   p3 or ppp3   = job to target ppp3.
#pr
#pr   hadar or ha = job to target hadar (IBM).
#pr
#pr   pha         = job to target "preemptable" class on Hadar (IBM).
#pr
#pr   spica or sp = job to target spica (IBM).
#pr
#pr   psp         = job to target "preemptable" class on spica (IBM).
#pr
#pr        script = switch to generate a copy of the pre-processed job
#pr                 script without sending the job to NQS
#pr
#pr           log = switch to turn on routing of all printout via NQS 
#pr                 instead of relying on code in comjcl/endjcl (or
#pr                 comgcm/endgcm for the model).
#pr
#pr        resume = switch to resume a string 
#pr
#pr      continue = switch to continue a string (used only in endjcl.cdk)
#pr
#pr         JHOME = alternate path to use instead of $HOME for housing
#pr                 ".queue", ".crawork" and "tmp" subdirectories.
#pr                 Note: Honoured only if targeted subdirectory by "JHOME"
#pr                       already exists. And, presently is restricted to 
#pr                       Dorval site.
#pr
#pr   reset_jhome = switch controls targeting back $HOME instead of $JHOME
#pr                 and relocating "${crawork}_string" file into 
#pr                 $HOME/.queue/.crawork if "nextjob=on".
#pr                 NOTE: Presently, only valid in Dorval site.
#pr                 =on/yes; target "$HOME" and relocate "${crawork}_string" 
#pr                          file 
#pr                 =otherwise, no effect.
#pr
#pr     delaymin = optional switch valid only for job targeting one of AIX 
#pr                Power7 clusters at CMC in Dorval. If specified,it signals
#pr                to defer start execution of the job by "delaymin" time
#pr                in minutes.
#pr
#pr     premtn   = optional switch which allows targeting "preemptable" instead
#pr                of "development" class on the "Production" P7 cluster where,
#pr                the job will be subject to pre-emption. 
#pr                =on/yes; option enabled.

#ex EXAMPLE:
#ex   
#ex     submit3 jobfil br
#ex     
#ex     The above example pre-processes the submission file "jobfil"  
#ex     which is in the standard CCRD/N unix structure and submits it 
#ex     to Brooks.
#ex
#ex     submit3 myjob psp
#ex     submit3 myjob premtn sp
#ex
#ex     Either of the above 2 will result in submitting 'myjob' to Spica 
#ex     targeting "preemptable" class where, it'll be subject to "preemption"
#ex     provided Spica is running in "Production" cluster mode.
#ex
#ex     submit3 flabel_diag resume
#ex
#ex     The above example resumes a diagnostic string with flabel "flabel"
#ex     on the machine on which submit3 is invoked. The submission file is 
#ex     taken from the control file (stored in the user's .queue/.crawork 
#ex     subdirectory). Note that "_diag" must be specified by the user after 
#ex     "flabel".
#ex
#ex     submit3 crawork_string sp resume
#ex
#ex     The above example resumes a job string with crawork "crawork" on 
#ex     ZETA. The submission file is taken from the control file
#ex     stored in the user's $HOME/.queue/.crawork directory on Spica.
#ex     Note that "_string" must be specified by the user after "crawork".
#ex
#ex     submit3 jobfil script oldiag=old_diaglib
#ex  
#ex     The above example generates a pre-processed job script "jobfl" 
#ex     using codes from $CCRNSRC/source$OSbin/old_diaglib diagnostic library
#ex     in preprocessing the job.
#ex     The resulting job script is placed in the directory from which 
#ex     "submit3" was executed. The name of the script is composed 
#ex     of the "jobname" followed by a ".script" extension.
#ex
#ex     submit3 jobfil delaymin=30 ha
#ex
#ex     The above example results in submitting the job to Hadar with
#ex     deferred start execution by 30 minutes.

# set -a
# Make sure "queue" variable setting in the invoking shell is not used.
# set -x
Submit3_drctv="$@"
unset queue runid output_dir nqsprfx nqsext
delaymin=''

if [ -s "$CUSTOM_ENV_FILE" ] ; then 
 . $CUSTOM_ENV_FILE || : 
else
#  if [ -z "$SYSTEM_SETUP" -a -z "$CCRNSRC" -a -d "/users/tor/acrn/src/generic/."  ] ; then [ -s "/users/tor/acrn/src/generic/cccma_setup_profile" ] && . /users/tor/acrn/src/generic/cccma_setup_profile || : ; fi
 if [ -z "$SYSTEM_SETUP" -a -z "$CCRNSRC" -a -d /home/scrd101/generic/. ] ; then [ -s /home/scrd101/generic/sc_cccma_setup_profile ] && . /home/scrd101/generic/sc_cccma_setup_profile || : ; fi
fi

HOSTID=${HOSTID-`hostname | cut -d'.' -f1`}
HOSTID=`echo $HOSTID | sed -e 's/hpcr4-in/ppp4/' -e 's/hpcr3-in/ppp3/' `
#  * Set variable to the version of "awk" to be used. The original version
#  * of 'awk' does not have the capabilities required for this script
#  * and 'nawk' must be used instead. Fortunately, it seems that on most
#  * systems, 'awk' is actually 'nawk'. On systems where this is not the
#  * case, the variable AWK must be set to nawk.

# set -x
AWK=${AWK:-awk}

#  * Obtain the name of the submission file and any specified option.
unset file

# for arg in $@ 
for arg in $Submit3_drctv
do
  case $arg in 
          -*) set $arg                       ;;
         *=*) eval $arg                      ;;
         log) log=yes                        ;;
    hare|hr) mdest='hare'                  ;;
    brooks|br) mdest='brooks'                  ;;
    daley|dy) mdest='daley'                  ;;
    banting|ba) mdest='banting'                  ;;
    ppp1|p1) mdest='ppp1'                  ;;
    ppp2|p2) mdest='ppp2'                  ;;
    ppp3|p3) mdest='ppp3'                  ;;
    ppp4|p4) mdest='ppp4'                  ;;
    hadar|ha) mdest='hadar'                  ;;
    pha) mdest='hadar' ; premtn='yes'        ;;
    spica|sp) mdest='spica'                  ;;
    psp) mdest='spica' ; premtn='yes'        ;;
       lxsrv) mdest=lxsrv                    ;;
      lxsrv2) mdest=lxsrv2                   ;;
    lxsrvnew) mdest=lxsrvnew                 ;;
      lxwrk1) mdest=lxwrk1                   ;;
      lxwrk2) mdest=lxwrk2                   ;;
 lx[0-9][0-9]|ltop[0-9][0-9]) mdest="$arg"   ;;
              lxlp[0-9][0-9]) mdest="$arg"   ;;
      resume) resume=yes                     ;;
 reset_jhome) reset_jhome=yes                ;;
    continue) continue=yes                   ;; 
      script) script=yes                     ;;
       nonqs) nonqs=yes                      ;;
      topdog) topdog='yes'                   ;;
      premtn) premtn='yes'                   ;;
           *) file=${file:=$arg}      
  esac
done

#  * Obtain the name of the current directory, then move into a
#  * temporary directory to execute the script.
if [ "$SITE_ID" = 'DrvlSC' ] ; then
  CCCQSUB=${CCCQSUB:-'cccqsub'}
else
  CCCQSUB=${CCCQSUB:-'qsub'}
fi

CCwd=`pwd`
Stamp="${HOSTID}_"`date +%Y%j%H%M%S`
reset_jhome=${reset_jhome:='no'}
if [ -n "$JHOME" -a "${JHOME}" != "${HOME}" -a -d "${JHOME}/." -a \( "$SITE_ID" = 'Dorval' -o "$SITE_ID" = 'DrvlSC' \) ] ; then ( mkdir -m 755 ${JHOME}/.queue ${JHOME}/.queue/.crawork ${JHOME}/tmp 2>>/dev/null || : ) ; (chmod go+rx-w ${JHOME}/.queue ${JHOME}/.queue/.crawork ${JHOME}/tmp 2>>/dev/null || : ) ; fi
if [ -n "$JHOME" -a "${JHOME}" != "${HOME}" -a -d "${JHOME}/tmp/." -a \( "$SITE_ID" = 'Dorval' -o "$SITE_ID" = 'DrvlSC' \) ] ; then
 HOMETMP="$JHOME/tmp"
else
 HOMETMP="$HOME/tmp"
fi

host=${HOSTID-`hostname`}
mkdir $HOMETMP/tmp_submit3_$$_$Stamp
cd $HOMETMP/tmp_submit3_$$_$Stamp

HOSTHW=${HOSTHW-`uname -m`}

## if [ -n "$mdest" -a "$mdest" = 'orion' ] ; then
##  echo "" ; echo "Submit3: Error; $mdest destination is no longer supported!"
##  tty -s && InTrCtv='true' || InTrCtv='false'
##  if [ "$InTrCtv" = 'false' ] ; then
##   touch $CCwd/haltit
##   ( echo "" ; echo "Submit3: Error; $mdest destination is no longer supported!") >> $CCwd/haltit
##  fi
##  cd ..
##  \rm tmp_submit3_$$_$Stamp/*
##  \rm -r tmp_submit3_$$_$Stamp
##  exit 1
## fi

#  * Setup to use "USRXBIN" if set.

. usrxbin_split

 export USRXBIN USRXBIN1 USRXBIN2 USRTSTR TSTRDIR JHOME reset_jhome

#  * Set the defaults.

OSbin=${OSbin-`ostarget bin`}
# OSbin2="$OSbin"
OSbin2=''
# LJDIAG_DIR=${ljdiag:="$CCRNSRC/source${OSbin}/newdiag2"}
# ln -s $LJDIAG_DIR ljdiag_dir

ln -s $HOME/bin${OSbin2} homebin_dir
ln -s $HOME/bin          homebin_dr2

if [ "$USRTSTR" = 'true' -a -n "$TSTRDIR" ] ; then
 if [ -d "$TSTRDIR/." ] ; then
  (\rm -f usrtstr_dir || : )
  ln -s $TSTRDIR usrtstr_dir
 else
  echo "" ; echo "submit3: Unable to continue due to $TSTRDIR subdirectory is not accessible!" ; echo ""
  exit 2
 fi
else
 (\rm -f usrtstr_dir || : )
 ln -s $HOME/bin${OSbin2} usrtstr_dir
fi
(\rm -f usrxbin1_dir || : )
if [ -n "$USRXBIN1" -a -d "$USRXBIN1/." ] ; then
 ln -s -f $USRXBIN1 usrxbin1_dir
else
 ln -s -f usrtstr_dir usrxbin1_dir
fi
(\rm -f usrxbin2_dir || : )
if [ -n "$USRXBIN2" -a -d "$USRXBIN2/." ] ; then
 ln -s -f $USRXBIN2 usrxbin2_dir
else
 ln -s -f usrxbin1_dir usrxbin2_dir
fi

mdest=${mdest:=$host}

script=${script:='no'}
nonqs=${nonqs:=$NONQS}
nonqs=${nonqs:='no'}
qsublog=${qsublog:='off'}
export qsublog
export topdog
export premtn

#  * Extract name and path information from the job filename. The 
#  * default directory containing the job is set to the directory in
#  * which the script was invoked.

path_dir=`expr $file : '\(.*\)/'`
job=`expr //$file : '.*/\(.*\)'`

path_dir=${path_dir:=$CCwd}

home_dir=$HOME

if [ -n "$JHOME" -a "${JHOME}" != "${HOME}" -a -d "${JHOME}/." -a \( "$SITE_ID" = 'Dorval' -o "$SITE_ID" = 'DrvlSC' \) ] ; then ( mkdir -m 755 ${JHOME}/.queue ${JHOME}/.queue/.crawork ${JHOME}/tmp 2>>/dev/null || : ) ; (chmod go+rx-w ${JHOME}/.queue ${JHOME}/.queue/.crawork ${JHOME}/tmp 2>>/dev/null || : ) ; fi
# set -x
if [ -n "$JHOME" -a "${JHOME}" != "${HOME}" -a -d "${JHOME}/.queue/.crawork/." -a \( "$SITE_ID" = 'Dorval' -o "$SITE_ID" = 'DrvlSC' \) ] ; then
 string_dir=${JHOME}/.queue/.crawork
else
 string_dir=$home_dir/.queue/.crawork
fi

#  * Continuation of strings.
# set -x
if [ "$continue" = 'yes' ] ; then
  
  if [ \( "${reset_jhome}" = 'on' -o "${reset_jhome}" = 'yes' \) -a \( "$SITE_ID" = 'Dorval' -o "$SITE_ID" = 'DrvlSC' \) ] ; then
   if [ "$string_dir" != "${HOME}/.queue/.crawork" -a -s "$string_dir/$job" ] ; then
    (chmod g+r $string_dir/$job || : ) ; touch ${HOME}/.queue/.crawork/$job ; cp -p $string_dir/$job ${HOME}/.queue/.crawork/$job
    rm $string_dir/$job
    string_dir="${HOME}/.queue/.crawork"
   elif [ "$string_dir" != "${HOME}/.queue/.crawork" -a ! -s "$string_dir/$job" -a -s "${HOME}/.queue/.crawork/$job" ] ; then
    string_dir="${HOME}/.queue/.crawork" 
   fi
  else
   if [ "$string_dir" != "${HOME}/.queue/.crawork" -a -s ${HOME}/.queue/.crawork/$job -a ! -s "$string_dir/$job" ] ; then
    (chmod g+r ${HOME}/.queue/.crawork/$job || : ) ; touch $string_dir/$job ; cp -p ${HOME}/.queue/.crawork/$job $string_dir/$job
   #rm ${HOME}/.queue/.crawork/$job
   fi
  fi

  #  * Determine whether diagnostic or job string is to be continued.

  if [ ! -s "$string_dir/$job" ] ; then
   echo "" ; echo "Submit3: it appears file $string_dir/$job is not accessible:"
   (set -x ; \ls -al $string_dir || : )
   sleep 10
   echo "Submit3: Will re-attempt accessing it..."
  fi
  if [ ! -s "$string_dir/$job" ] ; then
    echo "" ; echo "Submit3: Error; file $string_dir/$job does not exist"
    tty -s && InTrCtv='true' || InTrCtv='false'
    if [ "$InTrCtv" = 'false' ] ; then
     touch $CCwd/haltit
     ( echo "" ; echo "Submit3: Error; file $string_dir/$job does not exist") >> $CCwd/haltit
    fi
    cd ..
    \rm tmp_submit3_$$_$Stamp/*
    \rm -r tmp_submit3_$$_$Stamp
    exit 1
  fi
  
  type=`expr $job : '.*_\([^_ ]*\)$'`

  #  * Diagnostic strings.

  if [ "$type" = 'diag' ] ; then

    #  * Continue with next job by editing diagnostic string control
    #  * file to strip out the deck that has just completed.
    #  * Filter out top blank/empty lines (if any)

    touch jobfile
    $AWK 'BEGIN {varflag=0}
          $1 ~ /#deck/ && num == 0 { next }
          $0 ~ /^jobname *=.*memory *=/ || $0 ~ /^ *\. *[^ ]*\.dk/ {
            num = num + 1
            if ( num == 2 ) { print > "next_deck" }
          }
          (num == 1 && varflag == 1) { print }
          ($0 ~ /^#par_def/ && num != 0) {varflag=1}
          num != 1 { print }
         ' $string_dir/$job | $AWK 'BEGIN {blank=0}
            $0 ~ /^ *$/ && blank == 0 { next }
            $0 !~ /^ *$/ {blank=1}
            blank != 0 { print }
         '>> jobfile

    #  * If there is a next deck, then save the modified control file.
    #  * Otherwise, the string has finished and the control file is
    #  * removed.

    if [ -s next_deck ] ; then
      # (\rm -f $string_dir/$job || : )
      # cp jobfile $string_dir/$job

      # cp -p jobfile jobfile_copy
      # mv jobfile $string_dir/$job
      # mv jobfile_copy jobfile

        (chmod g+r jobfile || : )
        [ -f $string_dir/.${job}_hold ] && \rm -f $string_dir/.${job}_hold && sleep 2 || : ; touch $string_dir/.${job}_hold ; sleep 1
        cp -p jobfile $string_dir/.${job}_hold && ExtSts=$? || ExtSts=$?
        if [ $ExtSts -eq 0 ] ; then
         [ -f $string_dir/$job ] && \rm -f $string_dir/$job && sleep 1 || : 
         mv $string_dir/.${job}_hold $string_dir/$job
        else
         echo "Submit3 - problem detected in --> cp -p jobfile $string_dir/.${job}_hold <--"
         echo "          process will terminate leaving jobfile in `pwd`"
         exit 1
        fi
    else
      [ -f $string_dir/$job ] && \rm -f $string_dir/$job && sleep 2 || : 
      # mv jobfile $CCwd/end_auto
      touch $CCwd/end_auto ; cp -p jobfile $CCwd/end_auto ; \rm jobfile
      cd ..
      \rm tmp_submit3_$$_$Stamp/*
      \rm -r tmp_submit3_$$_$Stamp
      exit
    fi

  fi

  #  * Job Strings.

  if [ "$type" = 'string' ] ; then

      #  * signal the target file is about to be updated
      [ -s "$string_dir/$job" ] && touch $string_dir/.${job}_updtng && sleep 1 || : 

      #  * Edit job string control file to strip out the job that has
      #  * just completed. 
      
      End_of_TapejobP=`head -100 $string_dir/$job | sed -n -e '/end_of_tapejob/p' | tail -1 | sed -e 's/^.*end_of_tapejob.*$/YES/'`

      if [ "$End_of_TapejobP" = 'YES' ] ; then

       #  * Filter out top blank/empty and/or "end_of_tapejob" bound header/trailer section lines (if any)

      #End_of_TapejobH=`sed -n -e "/cat.*'end_of_tapejob'/p" $string_dir/$job | tail -1 | sed -e 's/^.*end_of_tapejob.*$/YES/'`
       End_of_TapejobH=`head -100 $string_dir/$job | sed -n -e "/cat.*'end_of_tapejob'/p" | tail -1 | sed -e 's/^.*end_of_tapejob.*$/YES/'`
      #End_of_TapejobT=`sed -n -e "/^end_of_tapejob/p" $string_dir/$job | tail -1 | sed -e 's/^end_of_tapejob.*$/YES/'`
       End_of_TapejobT=`tail -100 $string_dir/$job | sed -n -e "/^end_of_tapejob/p" $string_dir/$job | tail -1 | sed -e 's/^end_of_tapejob.*$/YES/'`
       if [ "$End_of_TapejobH" = 'YES' ] ; then
          SedLineH='-e 1,/'"'end_of_tapejob'"'/d'
       fi
       if [ "$End_of_TapejobT" = 'YES' ] ; then
          SedLineT="-e "'/^end_of_tapejob/,$d'
       else
          SedLineT=" -n -e "'1,$p'
       fi
       touch job_string
       sed $SedLineH $SedLineT $string_dir/$job | $AWK 'num != 0 { print }
             num == 1 { 
               if ( $0 !~ /^ *$/ ) { flag = "on" }
               if ( flag == "on" ) { print > "jobfile" }
             }
             $1 ~ /# *end_of_job/ { num = num + 1 }
            ' | $AWK 'BEGIN {blank=0}
               $0 ~ /^ *$/ && blank == 0 { next }
               $0 !~ /^ *$/ {blank=1}
               blank != 0 { print }
            ' >> job_string

      else

       touch job_string
       $AWK 'num != 0 { print }
            num == 1 {
              if ( $0 !~ /^ *$/ ) { flag = "on" }
              if ( flag == "on" ) { print > "jobfile" }
            }
            $1 ~ /# *end_of_job/ { num = num + 1 }
           ' $string_dir/$job | $AWK 'BEGIN {blank=0}
              $0 ~ /^ *$/ && blank == 0 { next }
              $0 !~ /^ *$/ {blank=1}
              blank != 0 { print }
           ' >> job_string
      fi

      # Ensure a valid job exists in "jobfile"

      if [ -s jobfile ] ; then
       End_of_Job=`cat jobfile | sed -n -e '/^# *end_of_job/p' | tail -1 | sed -e 's/^# *end_of_job *$/Yes/'`
       End_of_Job=${End_of_Job='No'}
       if [ "$End_of_Job" != 'Yes' ] ; then
        echo "" ; echo "Submit3: Extracted jobfile from $string_dir/$job does not contain 'end_of_job' marker!" ; echo ""
       #\rm jobfile job_string
       #touch jobfile job_string
       fi
      fi

      #  * If there is a next job, then save the modified control file.
      #  * Otherwise, the string has finished and the the control file
      #  * is removed.

      if [ -s job_string -a -s jobfile ] ; then

       #[ -s $string_dir/$job ] && \rm -f $string_dir/$job || :
       #mv job_string $string_dir/$job

        (chmod g+r job_string || : )
        # mv job_string $string_dir/.${job}_hold
        [ -f $string_dir/.${job}_hold ] && \rm -f $string_dir/.${job}_hold && sleep 2 || :  ; touch $string_dir/.${job}_hold ; 
        cp -p job_string $string_dir/.${job}_hold && ExtSts=$? || ExtSts=$?
        if [ $ExtSts -eq 0 ] ; then
         \rm job_string
         [ -f $string_dir/$job ] && \rm -f $string_dir/$job && sleep 2 || : 
         mv $string_dir/.${job}_hold $string_dir/$job
         #  * Delete the updating signal file.
         (\rm -f $string_dir/.${job}_updtng || : )
        else
         echo "Submit3 - problem detected in --> cp -p job_string $string_dir/.${job}_hold <--"
         echo "          process will terminate leaving job_string in `pwd`"
         echo "          and skipping removal of $string_dir/.${job}_updtng"
         exit 1
        fi

      else
        \rm $string_dir/$job
        #  * Delete the updating signal file.
        (\rm -f $string_dir/.${job}_updtng || : )
        cd ..
        \rm tmp_submit3_$$_$Stamp/*
        \rm -r tmp_submit3_$$_$Stamp
        exit
      fi

  fi

elif [ "$resume" = 'yes' ] ; then

  #  * Determine whether diagnostic or job string is to be resumed.

  type=`expr $job : '.*_\([^_ ]*\)$'`

  if [ "$type" != 'diag' -a "$type" != 'string' ] ; then
    echo "" ; echo "Submit3: Error; filename $job must end with '_diag' for diagnostic"
    echo "                  strings or '_string' for job strings"
    tty -s && InTrCtv='true' || InTrCtv='false'
    if [ "$InTrCtv" = 'false' ] ; then
     touch $CCwd/haltit
     (echo "" ; echo "Submit3: Error; filename $job must end with '_diag' for diagnostic") >> $CCwd/haltit
     (echo "                strings or '_string' for job strings") >> $CCwd/haltit
    fi
    cd ..
    \rm tmp_submit3_$$_$Stamp/*
    \rm -r tmp_submit3_$$_$Stamp
    exit 2
  fi

  #  * Get submission file from the string directory.
  #  * Filter out top blank/empty lines (if any)

  if [ ! -s "$string_dir/$job" ] ; then
   echo "" ; echo "Submit3: it appears file $string_dir/$job is not accessible:"
   (set -x ; \ls -al $string_dir || : )
   sleep 10
   echo "Submit3: Will re-attempt accessing it..."
  fi
  if [ ! -s "$string_dir/$job" ] ; then
    echo "" ; echo "Submit3: Error; file $job does not exist in $string_dir"
    tty -s && InTrCtv='true' || InTrCtv='false'
    if [ "$InTrCtv" = 'false' ] ; then
     touch $CCwd/haltit
     ( echo "" ; echo "Submit3: Error; file $job does not exist in $string_dir") >> $CCwd/haltit
    fi
    cd ..
    \rm tmp_submit3_$$_$Stamp/*
    \rm -r tmp_submit3_$$_$Stamp
    exit 3
  fi

  if [ "$type" = 'diag' ] ; then
#   sed '1,/output_dir/d' $string_dir/$job > jobfile
#   cp $string_dir/$job jobfile
    touch jobfile
    $AWK 'BEGIN {blank=0}
              $0 ~ /^ *$/ && blank == 0 { next }
              $0 !~ /^ *$/ {blank=1}
              blank != 0 { print }
           ' $string_dir/$job >> jobfile
  else
    touch jobfile
    sed -n '1,/^ *#end_of_job/p' $string_dir/$job | $AWK 'BEGIN {blank=0}
              $0 ~ /^ *$/ && blank == 0 { next }
              $0 !~ /^ *$/ {blank=1}
              blank != 0 { print }
           ' >> jobfile
  fi

else

  #  * This is a new submission. Check to make sure the submission 
  #  * file exists and obtain a local copy.
  #  * Filter out top blank/empty lines (if any)

  if [ ! -s "$path_dir/$job" ] ; then
   echo "" ; echo "Submit3: it appears file $path_dir/$job is not accessible:"
   (set -x ; \ls -al $path_dir || : )
   sleep 10
   echo "Submit3: Will re-attempt accessing it..."
  fi
  if [ ! -s "$path_dir/$job" ] ; then
    echo "" ; echo "Submit3: Error; file $path_dir/$job does not exist"
    tty -s && InTrCtv='true' || InTrCtv='false'
    if [ "$InTrCtv" = 'false' ] ; then
     touch $CCwd/haltit
     (echo "" ; echo "Submit3: Error; file $path_dir/$job does not exist") >> $CCwd/haltit
    fi
    cd ..
    \rm tmp_submit3_$$_$Stamp/*
    \rm -r tmp_submit3_$$_$Stamp
    exit 4
  fi

  # cp $path_dir/$job jobfile
    touch jobfile
  $AWK 'BEGIN {blank=0}
              $0 ~ /^ *$/ && blank == 0 { next }
              $0 !~ /^ *$/ {blank=1}
              blank != 0 { print }
           ' $path_dir/$job >> jobfile

fi
besc='' ; export besc
TMPFS='' ; export TMPFS
bexfer='' ; export bexfer
fexfer='' ; export fexfer
#  * Check for variables setting.

    touch job.info.awk0
    cat >>job.info.awk0 <<'end_of_awk_script'
          BEGIN { copy=0 }
          $0 ~ /\.  *comjcl\.cdk/ { exit }
          $0 ~ /^# *comjcl.*---comjcl/ { exit }
          $0 ~ /^ *\. *[^ ]*\.dk/ { exit }
          $1 ~ /#deck/ { exit }
          $1 ~ /#end_of_job/ { exit }
          $0 ~ /Model.*Input/ { exit }
          $0 ~ /^ *# ---[Ss][Tt][Aa][Rr][Tt]_[Ss][Uu][Bb][Mm][Ii][Tt]_[Ii][Gg][Nn][Oo][Rr][Ee]_[Cc][Oo][Dd][Ee].*$/ { copy=1 }
          $0 ~ /^ *# ---[Ss][Tt][Oo][Pp]_[Ss][Uu][Bb][Mm][Ii][Tt]_[Ii][Gg][Nn][Oo][Rr][Ee]_[Cc][Oo][Dd][Ee].*$/ { copy=0 }
          { if ( copy == 0 ) {
          if ( $0 ~ /oldiag +=|oldiag= +/ && $0 !~ /^ *#/ ) {print "Submit3: Error in awk; improper assignment syntax: \"" $0 "\"\nexit 1";exit 3}
          {print}
          }
          { if ( $0 ~ /jobname=/ ) {
            print
            if ( $0 ~ /gcmjcl/ ) print "nextjob=off"
          }
          }
          }
end_of_awk_script
#         $0 ~ /^ *if +\[/ { print }
#         $0 ~ /^ *else *$/ { print ":\n" $0 }
#         $0 ~ /^ *fi *$/ { print ":\n" $0}

  # $AWK -f job.info.awk0 jobfile | sed -e 's/#.*$//' -e '/^ *set /d' -e '/^ *\. /d' -e '/^ *\* \.\.* /d' | sed -e '/^ *$/d' > job.info0
  # $AWK -f job.info.awk0 jobfile > job.info0

  # Filter out potential "sleep" in the extracted lines:

  touch job.info0
  $AWK -f job.info.awk0 jobfile | sed -e 's/ * sleep *[0-9]*[ ;]*//g' >> job.info0

 Status="$?"
 if [ "$Status" -ne 0 ] ; then
  if [ "$Status" -eq 3 ] ; then
   \fgrep 'Submit3: Error in awk; improper' job.info0
   tty -s && InTrCtv='true' || InTrCtv='false'
   if [ "$InTrCtv" = 'false' ] ; then
    touch $CCwd/haltit
    \fgrep 'Submit3: Error in awk; improper' job.info0 >> $CCwd/haltit
   fi
  else
   echo "" ; echo "Submit3: Error; problem with awk on $job"
   tty -s && InTrCtv='true' || InTrCtv='false'
   if [ "$InTrCtv" = 'false' ] ; then
    touch $CCwd/haltit
    (echo "" ; echo "Submit3: Error; problem with awk on $job") >> $CCwd/haltit
   fi
  fi
  cd ..
  \rm tmp_submit3_$$_$Stamp/*
  \rm -r tmp_submit3_$$_$Stamp
  exit 5
 fi

 touch job.info01
 DelBndLn=`cat job.info0 | sed -n '/^#!/=' | tail -1`
 if [ -n "$DelBndLn" ] ; then
  cat job.info0 | sed -e "1,${DelBndLn}d" | sed -e 's/#.*$//' -e '/^ *set /d' -e '/^ *\. /d' | sed -e '/^ *$/d' >> job.info01
 else
  cat job.info0 | sed -e 's/#.*$//' -e '/^ *set /d' -e '/^ *\. /d' | sed -e '/^ *$/d' >> job.info01
 fi
 mv job.info01 job.info0
 if [ -s job.info0 ] ; then

  # Scan for potential problem lines starting with " *" (possibly due to misdeleted '#' prefix
  # character, exclude '*)' cases/esac construct)

  Dangerous_Line=`cat job.info0 | sed -n -e '/^ *\*/p' | sed -e '/^ *\* *[)]/d' | tail -1 | sed -e 's/^ *\*.*$/YES/'`
  if [ "$Dangerous_Line" = 'YES' ] ; then
   echo "" ; echo ' Submit3: The following problem line(s) detected:' ; echo ""
   cat job.info0 | sed -n -e '/^ *\*/p' | sed -e '/^ *\* *[)]/d'
   echo "" ; echo "         Possibly due to deleted '#' character. Please correct ..." ; echo ""
   tty -s && InTrCtv='true' || InTrCtv='false'
   if [ "$InTrCtv" = 'false' ] ; then
    touch $CCwd/haltit
    (echo "" ; echo ' Submit3: The following problem line(s) detected:' ; echo "") >> $CCwd/haltit
    ( cat job.info0 | sed -n -e '/^ *\*/p' | sed -e '/^ *\* *[)]/d' ) >> $CCwd/haltit
    ( echo "" ; echo "         Possibly due to deleted '#' character. Please correct ..." ; echo "") >> $CCwd/haltit
   fi
   cd ..
   \rm tmp_submit3_$$_$Stamp/*
   \rm -r tmp_submit3_$$_$Stamp
   exit 6
  fi

  # Scan for other potential problem lines ('^qsub ' and/or first line with 'cd $HOME/tmp'/'end_of_tapejob')

  Dangerous_Line=`head -1 job.info0 | sed -n -e 's/^end_of_tapejob/YES/p'`
  Dangerous_Line=${Dangerous_Line=`head -1 job.info0 | sed -n -e 's/^cd \$HOME\/tmp *$/YES/p'`}
  Dangerous_Line=${Dangerous_Line=`cat job.info0 | sed -n -e 's/^qsub .*$/YES/p'`}
  if [ "$Dangerous_Line" = 'YES' ] ; then
   echo "" ; echo ' Submit3: Problem line(s) detected:' ; echo " -------------"
   cat job.info0 
   echo " -------------" ; echo " Submit3: Unable to proceed ..." ; echo ""
   tty -s && InTrCtv='true' || InTrCtv='false'
   if [ "$InTrCtv" = 'false' ] ; then
    touch $CCwd/haltit
    (echo "" ; echo ' Submit3: Problem line(s) detected:' ; echo " -------------") >> $CCwd/haltit
    ( cat job.info0 ) >> $CCwd/haltit
    ( echo " -------------" ; echo " Submit3: Unable to proceed ..." ; echo "") >> $CCwd/haltit
   fi
   cd ..
   \rm tmp_submit3_$$_$Stamp/*
   \rm -r tmp_submit3_$$_$Stamp
   exit 7
  fi

  #. ./job.info0
  #Toldiag=`((cat job.info0 ; echo 'echo "\\\$oldiag" ' ) | /bin/sh - | tail -1 | sed -e 's/^ *$//' )` ; [ -n "$Toldiag" ] && oldiag="$Toldiag"
   (\rm -f ./job.info0_sh ./job.info0_ext  && sleep 1 || : )
   touch job.info0_ext
   echo '[ -n "$oldiag" ] && echo "oldiag=$oldiag" >> ./job.info0_sh
         [ -n "$JHOME" ] && echo "JHOME=$JHOME" >> ./job.info0_sh
         [ -n "$reset_jhome" ] && echo "reset_jhome=$reset_jhome" >> ./job.info0_sh
         [ -n "$delaymin" ] && echo "delaymin=$delaymin" >> ./job.info0_sh
         [ -n "$premtn" ] && echo "premtn=$premtn" >> ./job.info0_sh
         [ -n "$USRXBIN" ] && echo "USRXBIN=$USRXBIN" >> ./job.info0_sh
         [ -n "$USRXBIN1" ] && echo "USRXBIN1=$USRXBIN1" >> ./job.info0_sh
         [ -n "$USRXBIN2" ] && echo "USRXBIN2=$USRXBIN2" >> ./job.info0_sh
         [ -n "$USRTSTR" ] && echo "USRTSTR=$USRTSTR" >> ./job.info0_sh
         [ -n "$TSTRDIR" ] && echo "TSTRDIR=$TSTRDIR" >> ./job.info0_sh' >> job.info0_ext
   # echo 'job.info0_ext:' ; cat job.info0_ext ; echo '---- end of job.info0_ext ---'
   touch ./job.info0_sh
   (cat job.info0 job.info0_ext ) | /bin/sh - 2>>/dev/null
   if [ -s "./job.info0_sh" ] ; then
    # echo 'job.info0_sh:' ; cat job.info0_sh ; echo '---- end of job.info0_sh ---'
    . ./job.info0_sh
    . usrxbin_split
   fi
   export oldiag USRXBIN USRXBIN1 USRXBIN2 USRTSTR TSTRDIR JHOME reset_jhome 
 fi
  
 if [ "$USRTSTR" = 'true' -a -n "$TSTRDIR" ] ; then
  if [ -d "$TSTRDIR/." ] ; then
   (\rm -f usrtstr_dir || : )
   ln -s -f $TSTRDIR usrtstr_dir
  else
   echo "" ; echo "submit3: Unable to continue due to $TSTRDIR subdirectory is not accessible!" ; echo ""
   # (echo "" ; echo "submit3: Unable to continue due to $TSTRDIR subdirectory is not accessible!" ; echo "") >> $CCwd/haltit
   cd ..
   \rm tmp_submit3_$$_$Stamp/*
   \rm -r tmp_submit3_$$_$Stamp
   exit 5
  fi
 else
  (\rm -f usrtstr_dir || : )
  ln -s -f $HOME/bin${OSbin2} usrtstr_dir
 fi
 (\rm -f usrxbin1_dir || : )
 if [ -n "$USRXBIN1" -a -d "$USRXBIN1/." ] ; then
  ln -s -f $USRXBIN1 usrxbin1_dir
 else
  ln -s -f usrtstr_dir usrxbin1_dir
 fi
 (\rm -f usrxbin2_dir || : )
 if [ -n "$USRXBIN2" -a -d "$USRXBIN2/." ] ; then
  ln -s -f $USRXBIN2 usrxbin2_dir
 else
  ln -s -f usrxbin1_dir usrxbin2_dir
 fi

#  * Set the defaults.

 if [ -n "$oldiag" ] ; then
  LJDIAG_DIR="$CCRNSRC/source${OSbin}/$oldiag"
  Optns="oldiag=$oldiag"
  setline_ext="oldiag=$oldiag; "
 else
  DEFDIAG=${DEFDIAG:='newdiag2'}
  LJDIAG_DIR=${ljdiag:="$CCRNSRC/source${OSbin}/$DEFDIAG"}
 fi
 if [ ! -d "$LJDIAG_DIR/." ] ; then
  echo "" ; echo "Submit3: Unable to continue due to $LJDIAG_DIR subdirectory is not accessible!" ; echo ""
  cd ..
  \rm tmp_submit3_$$_$Stamp/*
  \rm -r tmp_submit3_$$_$Stamp
  exit 6
 fi
 (\rm -f ljdiag_dir || : )
 ln -s $LJDIAG_DIR ljdiag_dir
 

#  * Obtain the name of the job from the "jobname" line of the deck.
#  * This is used in the names of the output files.  The time limit
#  * and the memory limits are obtained in the same way. If the line
#  * is missing, the name of the submission file is used, the time
#  * limit is set to 20 seconds, and the memory limit is set to 10 MB.

    touch job.info.awk
    cat >>job.info.awk <<'end_of_awk_script'
          BEGIN { copy=0 }
          $0 ~ /\.  *comjcl\.cdk/ { exit }
          $0 ~ /^# *comjcl.*---comjcl/ { exit }
          flag == "on" {
            if ( $0 ~ /^ *\. *[^ ]*\.dk/ || $1 ~ /#deck/ ||\
                 $1 ~ /#end_of_job/ || $0 ~ /Model.*Input/ ) exit
            if ( $0 ~ /^[%*]/ ) upflag = "on"
            if ( $0 ~ /^#par_def/ ) upflag = "off"
            sub(/^[%*]d /,"%D ")
            sub(/^[%*]i /,"%I ")
            sub(/^[%*]id /,"%ID ")
            if ( upflag == "on" ) print > "updates2"
            next
          }
          $0 ~ /^ *\. *[^ ]*\.dk/ {
            sub(/^ *\. */,"")
            sub(/ *#.*$/,"")
            sub(/ +/,"")
            deck = $0
            name = "homebin_dr2/" deck
            if ( getline < name < 1 ) {
             name = "homebin_dir/" deck
             if ( getline < name < 1 ) {
              name = "usrtstr_dir/" deck
              if ( getline < name < 1 ) {
               name = "usrxbin1_dir/" deck
               if ( getline < name < 1 ) {
                name = "usrxbin2_dir/" deck
                if ( getline < name < 1 ) {
                 name = "ljdiag_dir/" deck
                 if ( getline < name < 1 ) {print "Submit3: Error in awk; deck -> " $0 " <- does not exist!\nexit 1"; exit 6}
                }
               }
              }
             }
            }
            getline < name
            print
            print "%C " deck > "updates"
            flag = "on"
            next
          }
          $0 ~ /^ *# ---[Ss][Tt][Aa][Rr][Tt]_[Ss][Uu][Bb][Mm][Ii][Tt]_[Ii][Gg][Nn][Oo][Rr][Ee]_[Cc][Oo][Dd][Ee].*$/ { copy=1 }
          $0 ~ /^ *# ---[Ss][Tt][Oo][Pp]_[Ss][Uu][Bb][Mm][Ii][Tt]_[Ii][Gg][Nn][Oo][Rr][Ee]_[Cc][Oo][Dd][Ee].*$/ { copy=0 }
          { if ( copy == 0 ) {
          if ( $0 ~ /^ *if +\[/ ) { print }
          if ( $0 ~ /^ *else *$/ ) { print ":\n" $0 }
          if ( $0 ~ /^ *fi *$/ ) { print ":\n" $0}
          if ( $0 ~ /flabel +=|flabel= +|nqsext +=|nqsext= +|nqsprfx +=|nqsprfx= +|runid +=|runid= +|run +=|run= +|RCMRUN +=|RCMRUN= +|USRXBIN +=|USRXBIN= +|reset_jhome +=|reset_jhome= +|JHOME +=|JHOME= +|crawork +=|crawork= +|mdest +=|mdest= +|[sgp ]time +=|[sgp ]time= +|nnodex +=|nnodex= +|nprocx +=|nprocx= +|initmem +=|initmem= +|memory[0-9]* +=|memory[0-9]*= +|queue +=|queue= +|bexfer +=|bexfer= +|fexfer +=|fexfer= +|jobname +=|jobname= +|TMPFS +=|TMPFS= +/ && $0 !~ /^ *#/ ) {print "Submit3: Error in awk; improper assignment syntax: \"" $0 "\"\nexit 1";exit 7}
          if ( $0 ~ /auto +=|auto= +|besc +=|besc= +|delaymin +=|delaymin= +|premtn +=|premtn= +|phys_nodex +=|phys_nodex= +|geometryx +=|geometryx= +|nextjob +=|nextjob= +|[sv ]vic +=|[sv ]vic= +/ && $0 !~ /^ *#/ ) {print "Submit3: Error in awk; improper assignment syntax: \"" $0 "\"\nexit 1";exit 7}
          if ( $0 ~ /auto=/ ) {print}
          if ( $0 ~ /nonqs=/ ) {print}
          if ( $0 ~ /qsublog=/ ) {print}
          if ( $0 ~ /nextjob=/ ) {print}
          if ( $0 ~ /runid=/ ) { print }
          if ( $0 ~ /flabel=/ ) { print }
          if ( $0 ~ /RCMRUN=/ ) { print }
          if ( $0 ~ /USRXBIN=/ ) { print }
          if ( $0 ~ /JHOME=/ ) { print }
          if ( $0 ~ /reset_jhome=/ ) { print }
          if ( $0 ~ /delaymin=/ ) { print }
          if ( $0 ~ /premtn=/ ) { print }
          if ( $0 ~ /run=/ ) { print }
          if ( $0 ~ /nqsext=/ ) { print }
          if ( $0 ~ /nqsprfx=/ ) { print }
          if ( $0 ~ /crawork=/ ) { print }
          if ( $0 ~ /mdest=/  ) { print }
          if ( $0 ~ /[sgd ]p*time=/ ) { print }
          if ( $0 ~ /nnodex=/ ) { print }
          if ( $0 ~ /phys_nodex=/ ) { print }
          if ( $0 ~ /geometryx=/ ) { print }
          if ( $0 ~ /nprocx=/ ) { print }
          if ( $0 ~ /initmem=/ ) { print }
          if ( $0 ~ /[sv ]vic=/ ) { 
            if ( $0 !~ /^#.*/ ) {
             print 
             if ( $0 ~ /svvic=/ ) {
              if ( $0 !~ /svvic=on/ && $0 !~ /svvic='on'/ ) print "unset svvic"
             }
             else if ( $0 ~ /vic=/ ) {
              if ( $0 !~ /vic=on/ && $0 !~ /vic='on'/ )   print "unset vic"
             }
            }
           }
          if ( $0 ~ /memory[0-9]*=/ ) { print }
          if ( $0 ~ /queue=/ ) { print }
          if ( $0 ~ /besc=/ ) { 
            if ( $0 !~ /^#.*/ ) {
             print 
            }
           }
          if ( $0 ~ /TMPFS=/ ) { 
            if ( $0 !~ /^#.*/ ) {
             print 
            }
           }
          if ( $0 ~ /bexfer=/ ) { 
            if ( $0 !~ /^#.*/ ) {
             print 
            }
           }
          if ( $0 ~ /fexfer=/ ) { 
            if ( $0 !~ /^#.*/ ) {
             print 
            }
           }
          if ( $0 ~ /jobname=/ ) {
            print
            if ( $0 ~ /gcmjcl/ ) print "nextjob=off"
          }
          }
          }
end_of_awk_script

#            if ( $0 !~ /besc=on/ && $0 !~ /besc='on'/ ) print "unset besc" 

  touch job.info
  $AWK -f job.info.awk jobfile >> job.info

  Status="$?"
  if [ "$Status" -ne 0 ] ; then
    if [ "$Status" -eq 6 ] ; then 
#     echo "Error: deck $deck does not exist"
      \fgrep 'Submit3: Error in awk; deck' job.info
      tty -s && InTrCtv='true' || InTrCtv='false'
      if [ "$InTrCtv" = 'false' ] ; then
       touch $CCwd/haltit
       \fgrep 'Submit3; Error in awk; deck' job.info >> $CCwd/haltit
      fi
    elif [ "$Status" -eq 7 ] ; then
      \fgrep 'Submit3: Error in awk; improper' job.info
      tty -s && InTrCtv='true' || InTrCtv='false'
      if [ "$InTrCtv" = 'false' ] ; then
       touch $CCwd/haltit
       \fgrep 'Submit3: Error in awk; improper' job.info >> $CCwd/haltit
      fi
    else
      echo "" ; echo "Submit3: Error; problem with awk on $job"
      tty -s && InTrCtv='true' || InTrCtv='false'
      if [ "$InTrCtv" = 'false' ] ; then
       touch $CCwd/haltit
       (echo "" ; echo "Submit3: Error; problem with awk on $job") >> $CCwd/haltit
      fi
    fi
    cd ..
    \rm tmp_submit3_$$_$Stamp/*
    \rm -r tmp_submit3_$$_$Stamp
    exit 7
  fi
  touch job.info02
  cat job.info | sed -e 's/#.*$//' | sed -e '/^ *$/d' >> job.info02
  mv job.info02 job.info
  # (set -x ; egrep 'nodex=|nprocx=' job.info )
  auto=off
  nextjob=off
  time=600
  memory=250mb
  export jobname nextjob time memory queue vic svvic besc TMPFS bexfer fexfer
  nnodex=''; nprocx='';phys_nodex='';geometryx=''
  # X5minmem_mb=100
  if [ -s job.info ] ; then
   # echo 'job.info:' ; cat job.info ; echo '---- end of job.info ---'

   # Scan for potential problem lines starting with " *" (possibly due to misdeleted '#' prefix
   # character, exclude '*)' cases/esac construct)
 
   Dangerous_Line=`cat job.info | sed -n -e '/^ *\*/p' | sed -e '/^ *\* *[)]/d' | tail -1 | sed -e 's/^ *\*.*$/YES/'`
   if [ "$Dangerous_Line" = 'YES' ] ; then
    echo "" ; echo 'submit3: The following problem line(s) detected:' ; echo ""
    cat job.info | sed -n -e '/^ *\*/p' | sed -e '/^ *\* *[)]/d' 
    echo "" ; echo "        Possibly due to deleted '#' character. Please correct ..." ; echo ""
    tty -s && InTrCtv='true' || InTrCtv='false'
    if [ "$InTrCtv" = 'false' ] ; then
     touch $CCwd/haltit
     (echo "" ; echo 'submit3: The following problem line(s) detected:' ; echo "") >> $CCwd/haltit
     ( cat job.info | sed -n -e '/^ *\*/p' | sed -e '/^ *\* *[)]/d' ) >> $CCwd/haltit
     ( echo "" ; echo "        Possibly due to deleted '#' character. Please correct ..." ; echo "") >> $CCwd/haltit
    fi
    cd ..
    \rm tmp_submit3_$$_$Stamp/*
    \rm -r tmp_submit3_$$_$Stamp
    exit 6
   fi

   # Scan for other potential problem lines ('^qsub ' and/or first line with 'cd $HOME/tmp'/'end_of_tapejob')
 
   Dangerous_Line=`head -1 job.info | sed -n -e 's/^end_of_tapejob/YES/p'`
   Dangerous_Line=${Dangerous_Line=`head -1 job.info | sed -n -e 's/^cd \$HOME\/tmp *$/YES/p'`}
   Dangerous_Line=${Dangerous_Line=`cat job.info | sed -n -e 's/^qsub .*$/YES/p'`}
   if [ "$Dangerous_Line" = 'YES' ] ; then
    echo "" ; echo ' Submit3: Problem line(s) detected:' ; echo " -------------"
    cat job.info 
    echo " -------------" ; echo " Submit3: Unable to proceed ..." ; echo ""
    tty -s && InTrCtv='true' || InTrCtv='false'
    if [ "$InTrCtv" = 'false' ] ; then
     touch $CCwd/haltit
     (echo "" ; echo ' Submit3: Problem line(s) detected:' ; echo " -------------") >> $CCwd/haltit
     ( cat job.info ) >> $CCwd/haltit
     ( echo " -------------" ; echo " Submit3: Unable to proceed ..." ; echo "") >> $CCwd/haltit
    fi
    cd ..
    \rm tmp_submit3_$$_$Stamp/*
    \rm -r tmp_submit3_$$_$Stamp
    exit 7
   fi

   # . ./job.info
  #echo "---- mdest=$mdest" ; export mdest
  #Tjobname=`((cat job.info ; echo 'echo "\\\$jobname" ' ) | /bin/sh - 2>>/dev/null | tail -1 | sed -e 's/^ *$//' )` ; [ -n "$Tjobname" ] && jobname="$Tjobname"
   (\rm -f ./job.info_sh ./job.info_ext && sleep 1  || : )
   touch job.info_ext
   #     [ -n "$auto" ] && echo "auto=$auto" >> ./job.info_sh
   #     [ -n "$flabel" ] && echo "flabel=$flabel" >> ./job.info_sh
   echo '[ -n "$bexfer" ] && echo "bexfer=$bexfer" >> ./job.info_sh
         [ -n "$fexfer" ] && echo "fexfer=$fexfer" >> ./job.info_sh
         [ -n "$jobname" ] && echo "jobname=$jobname" >> ./job.info_sh
         [ -n "$nextjob" ] && echo "nextjob=$nextjob" >> ./job.info_sh
         [ -n "$crawork" ] && echo "crawork=$crawork" >> ./job.info_sh
         [ -n "$auto" ] && echo "auto=$auto" >> ./job.info_sh
         [ -n "$flabel" ] && echo "flabel=$flabel" >> ./job.info_sh
         [ -n "$time" ] && echo "time=$time" >> ./job.info_sh
         [ -n "$memory" ] && echo "memory=$memory" >> ./job.info_sh
         [ -n "$queue" ] && echo "queue=$queue" >> ./job.info_sh
         [ -n "$vic" ] && echo "vic=$vic" >> ./job.info_sh
         [ -n "$qsublog" ] && echo "qsublog=$qsublog" >> ./job.info_sh
         [ -n "$besc" ] && echo "besc=$besc" >> ./job.info_sh
         [ -n "$TMPFS" ] && echo "TMPFS=$TMPFS" >> ./job.info_sh
         [ -n "$JHOME" ] && echo "JHOME=$JHOME" >> ./job.info_sh
         [ -n "$reset_jhome" ] && echo "reset_jhome=$reset_jhome" >> ./job.info_sh
         [ -n "$delaymin" ] && echo "delaymin=$delaymin" >> ./job.info_sh
         [ -n "$premtn" ] && echo "premtn=$premtn" >> ./job.info_sh
         [ -n "$USRTSTR" ] && echo "USRTSTR=$USRTSTR" >> ./job.info_sh
         [ -n "$TSTRDIR" ] && echo "TSTRDIR=$TSTRDIR" >> ./job.info_sh
         [ -n "$USRXBIN" ] && echo "USRXBIN=$USRXBIN" >> ./job.info_sh
         [ -n "$USRXBIN1" ] && echo "USRXBIN1=$USRXBIN1" >> ./job.info_sh
         [ -n "$USRXBIN2" ] && echo "USRXBIN2=$USRXBIN2" >> ./job.info_sh
         [ -n "$nnodex" ] && echo "nnodex=$nnodex" >> ./job.info_sh
         [ -n "$nprocx" ] && echo "nprocx=$nprocx" >> ./job.info_sh
         [ -n "$phys_nodex" ] && echo "phys_nodex=$phys_nodex" >> ./job.info_sh
         [ -n "$svvic" ] && echo "svvic=$svvic" >> ./job.info_sh' >> job.info_ext
   # echo 'job.info_ext:' ; cat job.info_ext ; echo '---- end of job.info_ext ---'
   touch ./job.info_sh
   (cat job.info job.info_ext ) | /bin/sh - 2>>/dev/null
   # (echo 'set -x' ; cat job.info job.info_ext ) | /bin/sh - 
   # (set -a ; (echo 'set -x' ; cat job.info job.info_ext ) | /bin/sh - )
   if [ -s "./job.info_sh" ] ; then
    # echo 'job.info_sh:' ; cat job.info_sh ; echo '---- end of job.info_sh ---'
    . ./job.info_sh
   fi
   . usrxbin_split
  fi
  # nnodex=''; nprocx='';phys_nodex='';geometryx=''
 
#  * Attempt better extraction of key variable settings ...

  if [ -s job.info0 ] ; then

   # Extract time/memory and append to ./job.info0

   cat ./job.info | sed -n -e '/^ *jobname=/p' | sed -e 's/^ *jobname=.*; *time=/time=/' >> ./job.info0

   # echo 'job.info0:' ; cat job.info0 ; echo '---- end of job.info0 ---'
  #Tflabel=`((cat job.info0 ; echo 'echo "\\\$flabel" ' ) | /bin/sh - | tail -1 | sed -e 's/^ *$//' )` ; [ -n "$Tflabel" ] && flabel="$Tflabel"
   (\rm -f ./job.info0_sh ./job.info0_ext && sleep 1  || : )
   ## Be careful in using: [ -n "$nextjob" ] && echo "nextjob=$nextjob" >> ./job.info0_sh
   ## since it's intentionally turned "off" for "gcmjcl" job(s).
   # SKIP: [ -n "$morigin" ] && echo "morigin=$morigin" >> ./job.info0_sh
   touch job.info0_ext
   echo '[ -n "$bexfer" ] && echo "bexfer=$bexfer" >> ./job.info0_sh
         [ -n "$fexfer" ] && echo "fexfer=$fexfer" >> ./job.info0_sh
         [ -n "$jobname" ] && echo "jobname=$jobname" >> ./job.info0_sh
         [ -n "$flabel" ] && echo "flabel=$flabel" >> ./job.info0_sh
         [ -n "$nextjob" ] && echo "nextjob=$nextjob" >> ./job.info0_sh
         [ -n "$crawork" ] && echo "crawork=$crawork" >> ./job.info0_sh
         [ -n "$time" ] && echo "time=$time" >> ./job.info0_sh
         [ -n "$memory" ] && echo "memory=$memory" >> ./job.info0_sh
         [ -n "$queue" ] && echo "queue=$queue" >> ./job.info0_sh
         [ -n "$vic" ] && echo "vic=$vic" >> ./job.info0_sh
         [ -n "$svvic" ] && echo "svvic=$svvic" >> ./job.info0_sh
         [ -n "$JHOME" ] && echo "JHOME=$JHOME" >> ./job.info0_sh
         [ -n "$reset_jhome" ] && echo "reset_jhome=$reset_jhome" >> ./job.info0_sh
         [ -n "$delaymin" ] && echo "delaymin=$delaymin" >> ./job.info0_sh
         [ -n "$premtn" ] && echo "premtn=$premtn" >> ./job.info0_sh
         [ -n "$USRTSTR" ] && echo "USRTSTR=$USRTSTR" >> ./job.info0_sh
         [ -n "$TSTRDIR" ] && echo "TSTRDIR=$TSTRDIR" >> ./job.info0_sh
         [ -n "$USRXBIN" ] && echo "USRXBIN=$USRXBIN" >> ./job.info0_sh
         [ -n "$USRXBIN1" ] && echo "USRXBIN1=$USRXBIN1" >> ./job.info0_sh
         [ -n "$USRXBIN2" ] && echo "USRXBIN2=$USRXBIN2" >> ./job.info0_sh
         [ -n "$besc" ] && echo "besc=$besc" >> ./job.info0_sh
         [ -n "$TMPFS" ] && echo "TMPFS=$TMPFS" >> ./job.info0_sh
         [ -n "$nqsext" ] && echo "nqsext=$nqsext" >> ./job.info0_sh
         [ -n "$nqsprfx" ] && echo "nqsprfx=$nqsprfx" >> ./job.info0_sh
         [ -n "$runid" ] && echo "runid=$runid" >> ./job.info0_sh
         [ -n "$auto" ] && echo "auto=$auto" >> ./job.info0_sh
         [ -n "$mdest" ] && echo "mdest=$mdest" >> ./job.info0_sh
         [ -n "$nnodex" ] && echo "nnodex=$nnodex" >> ./job.info0_sh
         [ -n "$nprocx" ] && echo "nprocx=$nprocx" >> ./job.info0_sh
         [ -n "$phys_nodex" ] && echo "phys_nodex=$phys_nodex" >> ./job.info0_sh
         [ -n "$geometryx" ] && echo "geometryx='"'"'$geometryx'"'"'" >> ./job.info0_sh
         [ -n "$qsublog" ] && echo "qsublog=$qsublog" >> ./job.info0_sh
         [ -n "$nonqs" ] && echo "nonqs=$nonqs" >> ./job.info0_sh' >> job.info0_ext
   # echo 'job.info0_ext:' ; cat job.info0_ext ; echo '---- end of job.info0_ext ---'
   touch ./job.info0_sh
   (cat job.info0 job.info0_ext ) | /bin/sh - 2>>/dev/null
   if [ -s "./job.info0_sh" ] ; then
    # echo 'job.info0_sh:' ; cat job.info0_sh ; echo '---- end of job.info0_sh ---'
    . ./job.info0_sh
    . usrxbin_split
   fi
  fi
  echo " ------- nprocx=$nprocx , nnodex=$nnodex , phys_nodex=$phys_nodex"

  #if [ -n "${runid}" -a -n "${runid_env}" -a "${runid}" != "${runid_env}" ] ; then
  # echo "" ; echo "submit3: Mismatched runid=$runid and runid_env=$runid_env !" ; echo ""
  # exit 8
  #fi
  if [ -z "${runid_env}" ] ; then
   echo "" ; echo "submit3: Undefined runid_env environment variable !" ; echo ""
   exit 7
  fi
  if [ -n "${runid}" -a "${runid}" != "${runid_env}" ] ; then
   echo "" ; echo "submit3: Mismatched runid=$runid and runid_env=$runid_env !" ; echo ""
   exit 8
  fi

  if [ -z "${nqsext}" -a -z "${nqsprfx}" -a -n "${runid}" ] ; then nqsprfx="${runid}_" ; fi
 
#

  if [ "$SITE_ID" = 'DrvlSC' -a -n "$queue" ] ; then
   case $queue in
    xc1*|mom* ) queue='hare' ;;
    xc2* ) queue='brooks' ;;
    xc3* ) queue='banting' ;;
    xc4* ) queue='daley' ;;
    pp1|cs1* ) queue='ppp1' ;;
    pp2|cs2* ) queue='ppp2' ;;
    pp3|cs3*|hpcr3* ) queue='ppp3' ;;
    pp4|cs4*|hpcr4* ) queue='ppp4' ;;
      * ) :           ;;
   esac
  elif [ "$SITE_ID" = 'Dorval' -a -n "$queue" ] ; then
   case $queue in
    c1f*|c1h*|c1r*|c1s* ) queue=spica;;
    c2f*|c2h*|c2r*|c2s* ) queue=hadar;;
    ib3* ) queue='pollux' ;;
    ib4* ) queue='castor' ;;
    ib8* ) queue='mez' ;;
    dataq* ) queue='dataq' ;;
      * ) :           ;;
   esac
   #c3f* ) queue=rigel ;;
  fi

  # Ensure valid "nextjob/crawork" and/or "auto/flabel" settings.

  TsTatus='continue'
  if [ "$nextjob" = 'on' ] ; then
   unset TcRwrk
   TcRwrk=`echo $crawork | sed -e 's/[ _]//g'`
   if [ -n "$TcRwrk" ] ; then 
    unset TcRwrk
   else
    set -x
    TsTatus='abort'
    echo "" ; echo "Submit3: Error; detected invalid crawork<$crawork> setting for nextjob=$nextjob!"
    tty -s && InTrCtv='true' || InTrCtv='false'
    if [ "$InTrCtv" = 'false' ] ; then
     touch $CCwd/haltit 
     (echo "" ; echo "Submit3: Error; detected invalid crawork<$crawork> setting for nextjob=$nextjob!") >> $CCwd/haltit 
    fi
   fi
  fi
  if [ "$auto" = 'on' ] ; then
   unset TfLbL
   TfLbL=`echo $flabel | sed -e 's/[ _]//g'`
   if [ -n "$TfLbL" ] ; then 
    unset TfLbL
   else
    set -x
    TsTatus='abort'
    echo "" ; echo "Submit3: Error; detected invalid flabel<$flabel> setting for auto=$auto!"
    tty -s && InTrCtv='true' || InTrCtv='false'
    if [ "$InTrCtv" = 'false' ] ; then
     touch $CCwd/haltit
     (echo "" ; echo "Submit3: Error; detected invalid flabel<$flabel> setting for auto=$auto!") >> $CCwd/haltit 
    fi
   fi
  fi
  if [ "$TsTatus" != 'continue' ] ; then
   cd ..
   \rm tmp_submit3_$$_$Stamp/*
   \rm -r tmp_submit3_$$_$Stamp
   exit 1
  fi

  #  * If update is required, then process the update directives.
 
  if [ -s updates2 ] ; then
    EXTRA_LIB=${EXTRA_LIB:-"$HOME/bin${OSbin2}"}
    EXTRA_LIB2=${EXTRA_LIB2:-"$HOME/bin"}

    cat updates2 >> updates
#   $LJDIAG_DIR/update updates $LJDIAG_DIR upout usrlib extra_lib=$HOME/bin${OSbin2}
    update updates $LJDIAG_DIR upout usrlib extra_lib=$EXTRA_LIB extra_lb2=$EXTRA_LIB2
   
    if [ "$?" -ne 0 ] ; then
      echo "" ; echo "Submit3: Error; problem in update of deck $deck"
      tty -s && InTrCtv='true' || InTrCtv='false'
      if [ "$InTrCtv" = 'false' ] ; then
       touch $CCwd/haltit
       (echo "" ; echo "Submit3: Error; problem in update of deck $deck") >> $CCwd/haltit
      fi
      cd ..
      \rm tmp_submit3_$$_$Stamp/*
      \rm -r tmp_submit3_$$_$Stamp
      exit 9
    fi

   #  * Edit job file to replace update directives with updated script.

    touch up.awk
    cat <<'    endcat' >> up.awk
      $0 ~ /^ *\.  *[^ ]*\.dk/ || $0 ~ /^#deck/ ||\
           $0 ~ /^#end_of_job/ || $0 ~ /^#par_def/ || $0 ~ /Model.*Input/ { n = ++n }
      $0 ~ /^ *\.  *[^ ]*\.dk/ && n == 1 {
        while ( getline < "upout" > 0 ) {
          if ( $0 ~ /cat.*Execute_Script/ ) { 
            $0 = "cat > Execute_Script <<'end_of_script'"
          }
          print
        }
      }
      n != 1 { print } 
    endcat

    touch jobfile2
    $AWK -f up.awk jobfile >> jobfile2
    mv jobfile2 jobfile

  fi

#  * In order to send jobs and continue strings across more than one
#  * machine, it is useful for each job to know which is the 
#  * originating machine, and which is the machine on which the job
#  * is to actually run. 
#  *
#  * The host where the job or string should run is set in the 
#  * parameter "mdest". If an NQS queue to which the job should be
#  * be submitted is specified on the "jobname" line, then this is used
#  * instead for "mdest". The machine from which the job was sent is
#  * set in "morigin".
#  * 
#  * Also, set the output directory and name of the output files.
#  * A PID is added to the name of the output files to make them unique.

if [ -z "$morigin" -a "$SITE_ID" = 'DrvlSC' ] ; then 
 # HOSTIDf=`echo $HOSTID | cut -c 1-3`
 HOSTIDf=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | cut -c 1-3`
 if [ "$OS" = 'Linux' ] ; then
  case $HOSTIDf in
   xc1*|mom*) morigin='hare' ;;
   xc2*) morigin='brooks' ;;
   xc3*) morigin='banting' ;;
   xc4*) morigin='daley' ;;
   pp1|cs1*) morigin='ppp1' ;;
   pp2|cs2*) morigin='ppp2' ;;
   pp3|cs3*|hpcr3*) morigin='ppp3' ;;
   pp4|cs4*|hpcr4*) morigin='ppp4' ;;
     * ) morigin='azur'  ;;
  esac
 fi
fi
morigin=${morigin:=$host}
queue=${queue:=$mdest}
export morigin
 if [ -n "$JHOME" -a "${JHOME}" != "${HOME}" -a -d "${JHOME}/." -a \( "$SITE_ID" = 'Dorval' -o "$SITE_ID" = 'DrvlSC' \) -a "$reset_jhome" != 'on' -a "$reset_jhome" != 'yes' ] ; then ( mkdir -m 755 ${JHOME}/.queue ${JHOME}/.queue/.crawork ${JHOME}/tmp 2>>/dev/null || : ) ; (chmod go+rx-w ${JHOME}/.queue ${JHOME}/.queue/.crawork ${JHOME}/tmp 2>>/dev/null || : ) ; fi
 if [ -n "$JHOME" -a "${JHOME}" != "${HOME}" -a -d "${JHOME}/.queue/." -a \( "$SITE_ID" = 'Dorval' -o "$SITE_ID" = 'DrvlSC' \) -a "$reset_jhome" != 'on' -a "$reset_jhome" != 'yes' ] ; then
  output_dir=${output_dir:="${JHOME}/.queue"}
 else
  output_dir=${output_dir:="${HOME}/.queue"}
 fi
# fi
if [ -n "$JHOME" -a "${JHOME}" != "${HOME}" -a -d "${JHOME}/.queue/.crawork/." -a \( "$SITE_ID" = 'Dorval' -o "$SITE_ID" = 'DrvlSC' \) -a "$reset_jhome" != 'on' -a "$reset_jhome" != 'yes' ] ; then
 string_dir=${JHOME}/.queue/.crawork
else
 string_dir=$home_dir/.queue/.crawork
fi
jobname=${jobname:=$job} 
# nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext} | cut -c1-14`
nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}`

if [ "$log" = yes ] ; then
 unset logfile
 Optns="log $Optns"
 if [ "$nonqs" = 'yes' ] ; then
  logfile=${logfile:=$output_dir/${nqsprfx}${jobname}${nqsext}.out.$$}
 else
  logfile=${logfile:=$morigin:$output_dir/${nqsprfx}${jobname}${nqsext}.out.$$}
 fi
else
  if [ "$SITE_ID" = 'DrvlSC' -a "$nonqs" != 'yes' -a "$mdest" = "$queue" ] ; then
   ## case $mdest in
   ##        hare) logfile='hare:/dev/null'    ;;
   ##        brooks) logfile='brooks:/dev/null'    ;;
   ##        ppp1) logfile='ppp1:/dev/null'   ;;
   ##        ppp2) logfile='ppp2:/dev/null'   ;;
   ##             *) logfile=/dev/null            ;;
   ## esac
      logfile=/dev/null
  else
   logfile=/dev/null
  fi
fi

#  * Initialize variable to ensure Bourne shell is invoked.

  BSHELL=${CCCSHELL:='/bin/sh'}

#  * Modify the beginning of the script as necessary.

if [ -n "$premtn" ] ; then
 setline_ext="$setline_ext premtn=$premtn ; "
fi
if [ "$type" != 'diag' ] ; then

if [ "$log" = yes ] ; then
  if [ -n "$topdog" ] ; then
   setline="set -ax ; log=$log ; topdog=$topdog ; $setline_ext"
  else
   setline="set -ax ; log=$log ; $setline_ext"
  fi
else
  if [ -n "$topdog" ] ; then
   setline="set -a; topdog=$topdog ; $setline_ext"  
  else
   setline="set -a; $setline_ext"  
  fi
fi
# if [ -n "$USRXBIN" ] ; then
#  setline="$setline USRXBIN="'"'"$USRXBIN"'" ;'
# fi
# if [ $BSHELL = '/bin/bash' ] ; then
#  setline="$setline . $HOME/.bashrc_ext ; "
# fi
touch begin2
cat <<endcat >> begin2
#! $BSHELL
#@\$
$setline
if [ -s "\$CUSTOM_ENV_FILE" ] ; then 
 . \$CUSTOM_ENV_FILE || : 
else
#  if [ -z "\$SYSTEM_SETUP" -a -z "\$CCRNSRC" -a -d "/users/tor/acrn/src/generic/."  ] ; then [ -s "/users/tor/acrn/src/generic/cccma_setup_profile" ] && . /users/tor/acrn/src/generic/cccma_setup_profile || : ; fi
  if [ -z "\$SYSTEM_SETUP" -a -z "\$CCRNSRC" -a -d /home/scrd101/generic/. ] ; then [ -s /home/scrd101/generic/sc_cccma_setup_profile ] && . /home/scrd101/generic/sc_cccma_setup_profile || : ; fi
fi
# if [ "\$SHELL" = '/bin/bash' -a -z "\$BASH_ENV" -a -f "$HOME/.bashrc_batch" ] ; then
#  BASH_ENV="$HOME/.bashrc_batch" ; export BASH_ENV ; . $HOME/.bashrc_batch
# fi
morigin=$morigin ; mdest=$mdest
output_dir=$output_dir

endcat
# if [ -n "$USRXBIN" ] ; then
#  echo 'USRXBIN="'"$USRXBIN"'"' >> begin2
# fi

fi

#  * If this is a tape job but destined for the back-end, add the 
#  * necessary beginning and end sections to the script to ensure 
#  * that it is sent instead to the MIPS.

if [ "$queue" = 'dataq' -o \( "$morigin" != 'mez' -a "$morigin" != 'castor' -a "$mdest" != 'pollux' -a \( "$queue" = 'tape08' -o "$queue" = 'cfsxfer' -o "$queue" = 'ccrnftp' -o "$queue" = 'pollux' \) \) -o \( "$morigin" = 'mez' -a "$mdest" != 'mez' -a \( "$queue" = 'tape08' -o "$queue" = 'cfsxfer' -o "$queue" = 'ccrnftp' -o "$queue" = 'mez' \) \) -o \( "$morigin" = 'castor' -a "$mdest" != 'castor' -a \( "$queue" = 'tape08' -o "$queue" = 'cfsxfer' -o "$queue" = 'ccrnftp' -o "$queue" = 'castor' \) \) ] ; then

Tqueue=$queue
# case $morigin in
#   pollux*) queue='pollux' ;;
#     * ) CMCFEDEST=${CMCFEDEST:='pollux'};
#         queue=${CMCFEDEST} ;;
# esac
# if [ \( "$queue" = 'tape08' -o "$queue" = 'cfsxfer' \) -a "$morigin" = 'dorval-ib' ] ; then
if [ "$queue" != 'dataq' ] ; then
 if [ "$queue" = 'castor' -o "$morigin" = 'castor' ] ; then
  queue=castor
 elif [ "$queue" = 'mez' -o "$morigin" = 'mez' ] ; then
  queue='mez'
 else
  queue='pollux'
 fi
fi
sub=yes
touch begin1
cat <<endcat >> begin1
#! $BSHELL
# set -x
if [ -s "\$CUSTOM_ENV_FILE" ] ; then 
 . \$CUSTOM_ENV_FILE || : 
else
#  if [ -z "\$SYSTEM_SETUP" -a -z "\$CCRNSRC" -a -d "/users/tor/acrn/src/generic/."  ] ; then [ -s "/users/tor/acrn/src/generic/cccma_setup_profile" ] && . /users/tor/acrn/src/generic/cccma_setup_profile || : ; fi
  if [ -z "\$SYSTEM_SETUP" -a -z "\$CCRNSRC" -a -d /home/scrd101/generic/. ] ; then [ -s /home/scrd101/generic/sc_cccma_setup_profile ] && . /home/scrd101/generic/sc_cccma_setup_profile || : ; fi
fi
# if [ "\$SHELL" = '/bin/bash' -a -z "\$BASH_ENV" -a -f "$HOME/.bashrc_batch" ] ; then
#  BASH_ENV="$HOME/.bashrc_batch" ; export BASH_ENV ; . $HOME/.bashrc_batch
# fi
Stamp="\${HOSTID}__"\`date +%Y%j%H%M%S\`
mkdir \$HOME/tmp/tmp_submit3_\$\$_\$Stamp
cd \$HOME/tmp/tmp_submit3_\$\$_\$Stamp
[ -f $jobname ] && \rm -f $jobname && sleep 2 || : ; touch $jobname
cat <<'end_of_tapejob' >> $jobname
endcat

touch end1
cat <<endcat >> end1
end_of_tapejob
qsublog=$qsublog ; export qsublog
$CCCQSUB -q $Tqueue $jobname -r $nqs_jobname -eo -o $logfile || exit \$?
cd \$HOME/tmp
\rm -f tmp_submit3_\$\$_\$Stamp/\*
\rm -r -f tmp_submit3_\$\$_\$Stamp
endcat

fi

#  * If this is a new diagnostic string, then generate the necessary 
#  * beginning and end sections to the script to save and execute
#  * the control file.

if [ "$auto" = 'on' ] ; then
type=`expr $job : '.*_\([^_ ]*\)$'`
if [ \( -z "$type" -a "$nextjob" != 'on' \) -o "$type" = 'string' ] ; then
# if [ \( -z "$type" -a "$nextjob" != 'on' \) -o "$type" != 'diag' ] ; then

touch begin1
cat <<endcat >> begin1
#!/bin/sh
[ -f $string_dir/${flabel}_diag ] && \rm -f $string_dir/${flabel}_diag && sleep 2 || : ; touch $string_dir/${flabel}_diag ; sleep 1
cat <<'end_of_diag' >> $string_dir/${flabel}_diag
endcat

touch end1
cat <<endcat >> end1
end_of_diag
chmod g+r $string_dir/${flabel}_diag ; chmod u+x $string_dir/${flabel}_diag ; $string_dir/${flabel}_diag

endcat

fi
fi

#  * If this is a job string, then generate the necessary beginning
#  * and end sections to the script to save and execute the control
#  * file. Note that the control file may reside on another machine
#  * (e.g. a tape job running on the front-end but is actually part 
#  * of a job string on the back-end)

if [ "$nextjob" = 'on' -a "$continue" != 'yes' -a "$resume" != 'yes' ] ; then

  if [ "$auto" = 'on' ] ; then
   #line="submit3 $Optns JHOME=$JHOME reset_jhome=$reset_jhome ${crawork}_string resume mdest=$mdest"
    line="submit3 $Optns JHOME=$JHOME reset_jhome=$reset_jhome ${crawork}_string resume mdest=$mdest ; ( rmllfls || : ) "
   #line="submit3 $Optns JHOME=$JHOME reset_jhome=$reset_jhome $string_dir/${crawork}_string mdest=$mdest ; ( rmllfls || : ) "
    sub=yes
  else  
# later on
    #                        -o \( "$SITE_ID" = Dorval -a \( "$mdest" = 'mez' -o "$mdest" = 'castor' -o "$mdest" = 'pollux' \) -a "$OS" = 'Linux' \)  \
    if [ "$queue" = "$mdest"  \
                             -o \( "$SITE_ID" = DrvlSC -a \( "$mdest" = 'hare' -o "$mdest" = 'brooks' -o "$mdest" = 'banting' -o "$mdest" = 'daley' \
                                                          -o "$mdest" = 'ppp1' -o "$mdest" = 'ppp2' -o "$mdest" = 'ppp3' -o "$mdest" = 'ppp4' \) -a "$OS" = 'Linux' \) ] ; then
      line="chmod g+r $string_dir/${crawork}_string ; chmod u+x $string_dir/${crawork}_string ; $string_dir/${crawork}_string"
    else
      if [ "$sub" = 'yes' ] ; then
       if [ "$nonqs" = 'yes' ] ; then
        line="chmod u+x $string_dir/${crawork}_string; nohup $string_dir/${crawork}_string\
                   > $logfile 2>$logfile &"
       else
        # nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}_sub | cut -c1-14`
        nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}_sub`
        # if [ "$SITE_ID" = 'Dorval' -a "$HOSTID" != 'pollux' -a \( "$queue" = 'pollux' -o "$queue" = 'orion' \) ] ; then AnnoYance="$E_NOBODY" ; else unset AnnoYance || : ; fi
        unset AnnoYance || :
        line="$CCCQSUB -q $queue $string_dir/${crawork}_string\
                   ${AnnoYance} -eo -o $logfile -r $nqs_jobname && ( rmllfls || : ) || exit \$?"
        unset AnnoYance || :
       fi
      else
       if [ "$nonqs" = 'yes' ] ; then
        line="chmod u+x $string_dir/${crawork}_string ; nohup $string_dir/${crawork}_string \
                   > $logfile 2>$logfile &"
       else
        # nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext} | cut -c1-14`
        nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}`

        if [ "$SITE_ID" = 'DrvlSC' -o \( "$SITE_ID" = 'Dorval' \
                                      -a \( "$mdest" = 'hadar' -o "$mdest" = 'spica' \)  \) ] ; then
          NQSTlmt='-lR' 
          nprocx=${nprocx:='1'}
          NQSnproc="-c $nprocx -lu $nprocx"
          if [ -n "$nnodex" ] ; then
           Qnnode="$nnodex"
           export Qnnode
          fi
          if [ -n "$geometryx" ] ; then
           Qgeometry="$geometryx"
           export Qgeometry
          fi
          if [ -n "$phys_nodex" ] ; then
           Qphys_node="$phys_nodex"
           export Qphys_node
          fi
          if [ -n "$delaymin" ] ; then
           Qdelaymin="$delaymin"
           export Qdelaymin
          fi
        else
          NQSTlmt='-lT' 
          unset NQSnproc
        fi
        unset AnnoYance || :
        line="$CCCQSUB -q $queue $string_dir/${crawork}_string\
                   ${AnnoYance} $NQSTlmt $time -lM $memory -r $nqs_jobname $NQSnproc -eo -o $logfile && ( rmllfls || : ) || exit \$?"
        unset AnnoYance || :
       fi
       sub=yes
      fi
      ## if [ "$mdest" != 'sx' ] ; then
       queue=$mdest
      ## fi
    fi
  fi

#  Setup "create_string_now" to control creating the string file by "submit3" (=yes) 
#  or within the submitted job upon executing (=no).
if [ "$SITE_ID" = 'DrvlSC' -a "$string_dir" = "${HOME}/.queue/.crawork" -a "$script" != 'yes' -a "$nonqs" != 'yes' ] ; then
 create_string_now='yes'
else 
 create_string_now='no'
fi

touch begin0
# (\rm -f $string_dir/${crawork}_string || : ) ; touch $string_dir/${crawork}_string
if [ "$create_string_now" = 'yes' ] ; then

cat <<endcat >> begin0
#! $BSHELL
if [ -s "$CUSTOM_ENV_FILE" ] ; then . $CUSTOM_ENV_FILE || : ; fi
endcat

touch end2
cat <<endcat >> end2
$line 
endcat

else # create_string_now

cat <<endcat >> begin0
#! $BSHELL
if [ -s "$CUSTOM_ENV_FILE" ] ; then . $CUSTOM_ENV_FILE || : ; fi
[ -f $string_dir/${crawork}_string ] && \rm -f $string_dir/${crawork}_string && sleep 2 || : ; touch $string_dir/${crawork}_string ; sleep 1
cat <<'end_of_string' >> $string_dir/${crawork}_string
endcat

touch end2
cat <<endcat >> end2
end_of_string
$line 
endcat

fi # create_string_now

fi
#  * Ensure "jobfile" has "oldiag" definition before the call
#  * to betapath2.
 
if [ -n "$setline_ext" ] ; then
eval "( \ed -s jobfile << woof
/betapath2/i
$setline_ext
.
w
q
woof
) "
fi

if [ $BSHELL = '/bin/bash' ] ; then
 touch jobfile_select
#cat jobfile | sed -e 's/^ *select /ccc select /' >> jobfile_select
 cat jobfile | sed -e '/^ *select *case/b
 /^ *select .* in /b
 /^ *select /s/^ *select /ccc select /' >> jobfile_select
 mv jobfile_select jobfile
fi

#  * Put all the different sections together into the final job.
#  * Ensure Bourne shell is invoked if running in Victoria.

touch $jobname
if [ "$create_string_now" = 'yes' ] ; then
 [ -f $string_dir/${crawork}_string ] && \rm -f $string_dir/${crawork}_string && sleep 2 || : ; touch $string_dir/${crawork}_string ; sleep 1
 # ( cat begin2 jobfile | sed 's/\/bin\/sh/\/bin\/bash/g' | sed 's/\/bin\/bsh/\/bin\/sh/g' ) >> $string_dir/${crawork}_string 2>/dev/null
 ( cat jobfile | sed 's/\/bin\/sh/\/bin\/bash/g' | sed 's/\/bin\/bsh/\/bin\/sh/g' ) >> $string_dir/${crawork}_string 2>/dev/null
 ( cat begin* end* | sed 's/\/bin\/sh/\/bin\/bash/g' ) >> $jobname 2>/dev/null
 # echo "---- jobname file:" ; cat $jobname ; echo " ---- "
else
 if [ -n "$CCCSHELL" -a "$CCCSHELL" = '/bin/bsh' ] ; then
 
  ( cat begin* jobfile end* | sed 's/\/bin\/sh/\/bin\/bsh/g' ) >> $jobname 2>/dev/null
 
 elif [ -n "$CCCSHELL" -a "$CCCSHELL" = '/bin/ash' ] ; then
 
  ( cat begin* jobfile end* | sed 's/\/bin\/sh/\/bin\/ash/g' ) >> $jobname 2>/dev/null
 
 elif [ -n "$CCCSHELL" -a "$CCCSHELL" = '/bin/bash' ] ; then
 
  ( cat begin* jobfile end* | sed 's/\/bin\/sh/\/bin\/bash/g' ) >> $jobname 2>/dev/null
 
 elif [ -n "$CCCSHELL" -a "$CCCSHELL" = '/bin/dash' ] ; then
 
  ( cat begin* jobfile end* | sed 's/\/bin\/sh/\/bin\/dash/g' ) >> $jobname 2>/dev/null
 
 else
 
  cat begin* jobfile end* >> $jobname 2>/dev/null
 
 fi
fi

 if [ "$SITE_ID" = 'Dorval' -o "$SITE_ID" = 'DrvlSC' ] ; then

 jobname_bsh_sh="${jobname}_bsh_sh"
 touch $jobname_bsh_sh
 (cat $jobname | sed 's/\/bin\/bsh/\/bin\/sh/g' ) >> $jobname_bsh_sh 2>/dev/null
 \mv $jobname_bsh_sh $jobname

 else

  case $mdest in 
   lxsrv|lxsrv2|lxsrvnew|lxwrk1|lxwrk2|lx[0-9][0-9]|lxlp[0-9][0-9]|ltop[0-9][0-9]) jobname_sh_bsh="${jobname}_sh_bsh"
                                      touch $jobname_sh_bsh
                                      (cat $jobname | sed 's/\/bin\/sh/\/bin\/ash/g' ) >> $jobname_sh_bsh 2>/dev/null
                                      \mv $jobname_sh_bsh $jobname ;;
                                   *) : 
  esac
 
 fi

# fi

#!!!!
# echo " $jobname (start):"
# cat $jobname
# echo " $jobname (end):"
#!!!!

#  * Submit the script.
  
if [ "$script" = 'yes' ] ; then

  # mv $jobname $CCwd/$jobname.script
  touch $CCwd/$jobname.script ; cp -p $jobname $CCwd/$jobname.script ; \rm $jobname

else

  if [ "$SITE_ID" = Victoria ] ; then

    Cwd=`pwd` 
    # ln -s $Cwd/$jobname $HOMETMP/$jobname.$$_$Stamp 
    ln $Cwd/$jobname $HOMETMP/$jobname.$$_$Stamp 
    cd $HOMETMP

    if [ "$sub" = yes ] ; then
     if [ "$nonqs" = 'yes' ] ; then
      chmod u+x $jobname.$$_$Stamp
      nohup `pwd`/$jobname.$$_$Stamp > $logfile 2> $logfile &
##    cat $jobname.$$_$Stamp | nohup sh -s - 1>> $logfile 2>> $logfile &
#     LclCmd="rm -f `pwd`/$jobname.$$_$Stamp"
#     (echo $LclCmd | at now + 02 minutes 1>> /dev/null 2>> /dev/null ) > /dev/null 2>&1
#     sleep 5
#     unset LclCmd
      sleep 2 ; (\rm -f $jobname.$$_$Stamp $Cwd/$jobname || : )
     else
      # nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}_sub | cut -c1-14`
      nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}_sub`
      $CCCQSUB -q $queue $jobname.$$_$Stamp -eo -o $logfile -r $nqs_jobname || exit $?
      \rm $jobname.$$_$Stamp 
     fi
    else
     if [ "$nonqs" = 'yes' ] ; then
      chmod u+x $jobname.$$_$Stamp
      nohup `pwd`/$jobname.$$_$Stamp > $logfile 2> $logfile &
##    cat $jobname.$$_$Stamp | nohup sh -s - 1>> $logfile 2>> $logfile &
#     LclCmd="rm -f `pwd`/$jobname.$$_$Stamp"
#     (echo $LclCmd | at now + 02 minutes 1>> /dev/null 2>> /dev/null ) > /dev/null 2>&1
#     sleep 5
#     unset LclCmd
      sleep 2 ; (\rm -f $jobname.$$_$Stamp $Cwd/$jobname || : )
     else
      # nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}.$$ | cut -c1-14`
      nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}.$$`
      $CCCQSUB -q $queue $jobname.$$_$Stamp -r $nqs_jobname -lT $time -lM $memory -eo -o $logfile || exit $?
      \rm $jobname.$$_$Stamp 
     fi
    fi

#   \rm $jobname.$$_$Stamp 
#   ls -ld `pwd`/$jobname.$$_$Stamp
    cd $Cwd

  else

    Cwd=`pwd` 
    # ln -s $Cwd/$jobname $HOMETMP/$jobname.$$_$Stamp 
    ln $Cwd/$jobname $HOMETMP/$jobname.$$_$Stamp 
    cd $HOMETMP

    if [ "$sub" = yes ] ; then


     if [ "$nonqs" = 'yes' ] ; then
      chmod u+x $jobname.$$_$Stamp
      nohup `pwd`/$jobname.$$_$Stamp > $logfile 2> $logfile &
#     cat $jobname.$$_$Stamp | nohup sh -s - 1>> $logfile 2>> $logfile &
      sleep 5
     else
      # nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}_sub | cut -c1-14`
      nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}_sub`
      unset AnnoYance || :
      $CCCQSUB -q $queue $jobname.$$_$Stamp ${AnnoYance} -eo -o $logfile -r $nqs_jobname || exit $?
      ### Replace above by next line if need time/memory
      ## $CCCQSUB -q $queue $jobname.$$_$Stamp ${AnnoYance} -lT $time -lM $memory -eo -o $logfile -r $nqs_jobname || exit $?
      # unset AnnoYance || :
     fi
    else
     if [ "$nonqs" = 'yes' ] ; then
      chmod u+x $jobname.$$_$Stamp
      nohup `pwd`/$jobname.$$_$Stamp > $logfile 2> $logfile &
#     cat $jobname.$$_$Stamp | nohup sh -s - 1>> $logfile 2>> $logfile &
      sleep 5
     else
      # nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}.$$ | cut -c1-14`
      nqs_jobname=`echo ${nqsprfx}${jobname}${nqsext}.$$`
      if [ "$SITE_ID" = 'DrvlSC' \
                 -a \( "$mdest" = 'hare' -o "$mdest" = 'brooks' -o "$mdest" = 'banting' -o "$mdest" = 'daley' \
                    -o "$mdest" = 'ppp1' -o "$mdest" = 'ppp2' -o "$mdest" = 'ppp3' -o "$mdest" = 'ppp4' \) ] ; then
        #        -a \( "$mdest" = 'hare' -o "$mdest" = 'brooks' \) ] ; then
        NQSTlmt='-lR' 
        nprocx=${nprocx:='1'}
        NQSnproc="-c $nprocx -lu $nprocx"
        if [ -n "$nnodex" ] ; then
         Qnnode="$nnodex"
         export Qnnode
        fi
        # if [ -n "$geometryx" ] ; then
        #  Qgeometry="$geometryx"
        #  export Qgeometry
        # fi
        if [ -n "$phys_nodex" ] ; then
         Qphys_node="$phys_nodex"
         export Qphys_node
        fi
        if [ -n "$delaymin" ] ; then
         Qdelaymin="$delaymin"
         export Qdelaymin
        fi
      else
        NQSTlmt='-lT' 
        unset NQSnproc
      fi
      # if [ "$SITE_ID" = 'Dorval' -a "$HOSTID" != 'pollux' -a \( "$queue" = 'pollux' -o "$queue" = 'orion' \) ] ; then AnnoYance="$E_NOBODY" ; else unset AnnoYance || : ; fi
      unset AnnoYance || :
      $CCCQSUB -q $queue $jobname.$$_$Stamp ${AnnoYance} -r $nqs_jobname $NQSTlmt $time -lM $memory $NQSnproc -eo -o $logfile || exit $?
       unset AnnoYance || :
     fi

    fi

    \rm $jobname.$$_$Stamp
    cd $Cwd

  fi
# cp -p $jobname $OUTPUTQ/$jobname
# sleep 10

fi

#  * Move back into parent directory and clean up temporary directory
#  * before exiting from script.

cd ..
\rm tmp_submit3_$$_$Stamp/*
\rm -r tmp_submit3_$$_$Stamp

exit