#!/bin/sh # set -x # Dec 03/08 - F.Majaess (Revised for sa/saiph, ib/dorval-ib, af/alef) # Apr 04/06 - F.Majaess (Enforce time=600, mem=16mb for "xfer" jobs) # Mar 01/06 - F.Majaess (Revised for rg/rigel) # Feb 15/05 - F.Majaess (Ensure full node usage for MPI tasks > 1) # Apr 19/04 - F.Majaess (Switch from using "total_tasks"/"blocking" # for LL based jobs to "node"/"tasks_per_node") # Jun 26/03 - F.Majaess (Revise for IBM "azur" Loadleveler setup) # May 01/03 - F.Majaess (Revise for "user.[multi,openmp,mpi].sx6" changes) # Jan 20/03 - F.Majaess (Revise for NQS/SGE setup) # Jun 13/02 - F.Majaess (Revise for SX6 setup) # Mar 18/96 - F.Majaess (Based on Victoria's qsub script, # revise for SX4 to use nohup) #id cccqsub - script to massage NQS qsub parameters or convert #id into nohup style command. # AUTHOR - M.Berkley,F.Majaess #hd PURPOSE - This script cracks NQS qsub command options, #hd alter settings or convert to use nohup instead. #pr PARAMETERS: #pr #pr Should follow NQS qsub man page #pr # set -x echo "QSUB: $*" dirctv="$*" if [ -s "$CUSTOM_ENV_FILE" ] ; then . $CUSTOM_ENV_FILE || : Qstrng=". $CUSTOM_ENV_FILE " else # if [ -z "$SYSTEM_SETUP" -a -z "$CCRNSRC" -a -d "/users/tor/acrn/src/generic/." ] ; then [ -s "/users/tor/acrn/src/generic/cccma_setup_profile" ] && . /users/tor/acrn/src/generic/cccma_setup_profile || : ; fi if [ -z "$SYSTEM_SETUP" -a -z "$CCRNSRC" -a -d /home/scrd101/generic/. ] ; then [ -s /home/scrd101/generic/sc_cccma_setup_profile ] && . /home/scrd101/generic/sc_cccma_setup_profile || : ; fi Qstrng='. /home/scrd101/generic/sc_cccma_setup_profile ' : fi nonqs=${nonqs:=$NONQS} nonqs=${nonqs:='no'} if [ -n "$qsublog" ] ; then qsublog=`echo $qsublog | tr '[A-Z]' '[a-z]'` fi if [ -n "$topdog" ] ; then topdog=`echo $topdog | tr '[A-Z]' '[a-z]'` if [ "$topdog" = 'on' ] ; then topdog='yes' elif [ "$topdog" = 'off' ] ; then topdog='no' fi fi if [ -n "$premtn" ] ; then premtn=`echo $premtn | tr '[A-Z]' '[a-z]'` if [ "$premtn" = 'on' ] ; then premtn='yes' elif [ "$premtn" = 'off' ] ; then premtn='no' fi fi HOSTID=${HOSTID:=`hostname| cut -d'.' -f1 `} HOSTID=`echo $HOSTID | sed -e 's/hpcr4-in/ppp4/' -e 's/hpcr3-in/ppp3/' ` # HOSTIDf=`echo $HOSTID | cut -c 1-3` # HOSTIDf=`echo $HOSTID | sed -e 's/eccc.*-ppp/pp/g' | cut -c 1-3` # HOSTIDf=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' | cut -c 1-3` HOSTIDf=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | cut -c 1-3` HOSTIDf=${HOSTIDf:-`echo $HOSTID | sed -e 's/^ppp/cs/g' | cut -c 1-3`} # if [ "$HOSTIDf" = 'ale' -a "$OS" = 'Linux' ] ; then # touch $HOME/tmp/cccqsub_log # echo "`date`; $HOSTID: Error; Sorry, cccqsub use of (${dirctv}) is disabled on $HOSTID" >> $HOME/tmp/cccqsub_log # echo "$0 : Sorry, cccqsub use is disabled on $HOSTID:" # echo " ${dirctv}" # exit -1 # fi if [ "$nonqs" != 'yes' ] ; then dirctv=${dirctv:=$*} qName=`echo " $dirctv " | sed -n -e 's/^.* -q *\([^ ]*\) .*$/\1/gp'` if [ -z "$qName" ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to determine target queue in: ${dirctv} " >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to determine target queue in:" echo " ${dirctv}" exit -1 else qNamex="$qName" if [ "$bexfer" = 'on' -a "$qName" != 'bexfer' ] ; then qName='bexfer' fi if [ "$fexfer" = 'on' -a "$qName" != 'fexfer' ] ; then qName='fexfer' fi if [ "$qNamex" = 'ppp1' -o "$qNamex" = 'ppp2' ] ; then qNamex="eccc-${qNamex}" fi qNamexT="$qNamex" if [ "$qNamex" = 'brooks' ] ; then HOSTIDf6=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | cut -c 1-6` HOSTIDfm=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | sed -e 's/xc2mom.*-p2/xc2mom-p2/'` if [ "$HOSTIDf6" = 'xc2net' -o "$HOSTIDfm" = 'xc2mom-p2' ] ; then qNamexT='brooks-test' fi fi # echo "QSUB - `hostname` - qName=$qName , qNamex=$qNamex , qNamexT=$qNamexT" # set +x #if [ "$qName" = 'orion' -o "$qName" = 'alef' ] ; then # touch $HOME/tmp/cccqsub_log # echo "`date`; ERROR: $qName is no longer supported!" >> $HOME/tmp/cccqsub_log # echo "`date`; ERROR: $qName is no longer supported!" # exit -2 #fi HOSTID=${HOSTID:=`hostname| cut -d'.' -f1 `} HOSTID=`echo $HOSTID | sed -e 's/hpcr4-in/ppp4/' -e 's/hpcr3-in/ppp3/' ` HOSTHW=${HOSTHW:=`uname -m | tr '[a-z]' '[A-Z]'`} OS=${OS:=`uname -s`} # HOSTIDf=`echo $HOSTID | cut -c 1-3` # HOSTIDf=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' | cut -c 1-3` HOSTIDf=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | cut -c 1-3` HOSTIDf=${HOSTIDf:-`echo $HOSTID | sed -e 's/^ppp/cs/g' | cut -c 1-3`} unset Batch_mode if [ "$qName" = 'hare' -o "$qName" = 'brooks' -o "$qName" = 'banting' -o "$qName" = 'daley' -o "$qName" = 'ppp1' -o "$qName" = 'ppp2' -o "$qName" = 'ppp3' -o "$qName" = 'ppp4' -o "$qName" = 'xfer' -o "$qName" = 'fexfer' -o "$qName" = 'bexfer' ] ; then # if [ "$OS" = 'Linux' -a \( \( "$qName" = 'hare' -a \( "$HOSTIDf" = 'xc1' -o "$HOSTIDf" = 'mom' \) \) -o \( "$qName" = 'brooks' -a "$HOSTIDf" = 'xc2' \) \) ] ; then if [ "$OS" = 'Linux' -a \( "$qName" = 'hare' -o "$qName" = 'brooks' -o "$qName" = 'banting' -o "$qName" = 'daley' -o "$qName" = 'bexfer' -o \( "$qName" = 'xfer' -a \( "$HOSTIDf" = 'xc1' -o "$HOSTIDf" = 'xc2' -o "$HOSTIDf" = 'xc3' -o "$HOSTIDf" = 'xc4' \) \) \) ] ; then # Batch_mode='BEpbspro' Batch_mode='BEjobst' # elif [ "$OS" = 'Linux' -a \( \( "$qName" = 'ppp1' -a \( "$HOSTIDf" = 'cs1' -o "$HOSTIDf" = 'pp1' \) \) -o \( "$qName" = 'ppp2' -a \( "$HOSTIDf" = 'cs2' -o "$HOSTIDf" = 'pp2' \) \) \) ] ; then # elif [ "$OS" = 'Linux' -a \( \( "$qName" = 'ppp1' -o "$qName" = 'ppp2' \) -a \( "$HOSTIDf" = 'cs1' -o "$HOSTIDf" = 'pp1' -o "$HOSTIDf" = 'cs2' -o "$HOSTIDf" = 'pp2' \) \) ] ; then elif [ "$OS" = 'Linux' -a \( "$qName" = 'ppp1' -o "$qName" = 'ppp2' -o "$qName" = 'ppp3' -o "$qName" = 'ppp4' -o "$qName" = 'fexfer' -o \( "$qName" = 'xfer' -a \( "$HOSTIDf" = 'pp1' -o "$HOSTIDf" = 'pp2' -o "$HOSTIDf" = 'pp3' -o "$HOSTIDf" = 'pp4' \) \) \) ] ; then Batch_mode='PPPjobst' else Cwd=`pwd` ssh -n ${qNamexT} 'hostname' >> /dev/null && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; ${qName} not reachable ; will attempt in a short while cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; ${qName} not reachable ; will attempt in a short while cccqsub $*" >> $HOME/problem_log sleep 180 fi Qextra='' if [ -n "$Qdelaymin" ] ; then Qextra=" ; export Qdelaymin=$Qdelaymin " fi Qextra2='' if [ -n "$premtn" ] ; then Qextra=" $Qextra ; export premtn=$premtn " Qextra2=" ; export premtn=$premtn " fi if [ -n "$TMPFS" ] ; then Qextra=" $Qextra ; export TMPFS=$TMPFS " fi if [ -n "$qsublog" -a \( "$qsublog" = 'on' -o "$qsublog" = 'yes' \) ] ; then if [ -n "$topdog" ] ; then if [ -n "$Qnnode" -a -n "$Qgeometry" ] ; then # exec ssh ${qNamexT} "cd $Cwd $Qextra ; setenv qsublog $qsublog ; setenv topdog $topdog ; setenv Qnnode $Qnnode ; setenv Qgeometry ""'""$Qgeometry""'"" ; cccqsub $*" && Exit=0 || Exit="$?" Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export topdog=$topdog ; export Qnnode=$Qnnode ; export Qgeometry=""'""$Qgeometry""'"" ; cccqsub $*" #exec ssh -T ${qNamexT} " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" elif [ -n "$Qnnode" -a -n "$Qphys_node" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export topdog=$topdog ; export Qnnode=$Qnnode ; export Qphys_node=$Qphys_node ; cccqsub $*" elif [ -n "$Qnnode" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export topdog=$topdog ; export Qnnode=$Qnnode ; cccqsub $*" elif [ -n "$Qgeometry" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export topdog=$topdog ; export Qgeometry=""'""$Qgeometry""'"" ; cccqsub $*" elif [ -n "$Qphys_node" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export topdog=$topdog ; export Qphys_node=$Qphys_node ; cccqsub $*" else Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export topdog=$topdog ; cccqsub $*" fi exec ssh -T ${qNamexT} " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi else if [ -n "$Qnnode" -a -n "$Qgeometry" ] ; then # exec ssh ${qNamexT} "$Qstrng ; cd $Cwd $Qextra ; setenv qsublog $qsublog ; setenv Qnnode $Qnnode ; setenv Qgeometry ""'""$Qgeometry""'"" ; cccqsub $*" && Exit=0 || Exit="$?" Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export Qnnode=$Qnnode ; export Qgeometry=""'""$Qgeometry""'"" ; cccqsub $*" elif [ -n "$Qnnode" -a -n "$Qphys_node" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export Qnnode=$Qnnode ; export Qphys_node=$Qphys_node ; cccqsub $*" elif [ -n "$Qnnode" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export Qnnode=$Qnnode ; cccqsub $*" elif [ -n "$Qgeometry" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export Qgeometry=""'""$Qgeometry""'"" ; cccqsub $*" elif [ -n "$Qphys_node" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; export Qphys_node=$Qphys_node ; cccqsub $*" else Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export qsublog=$qsublog ; cccqsub $*" fi exec ssh -T ${qNamexT} " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi fi else if [ -n "$topdog" ] ; then if [ -n "$Qnnode" -a -n "$Qgeometry" ] ; then # exec ssh ${qNamexT} "$Qstrng ; cd $Cwd $Qextra ; setenv topdog $topdog ; setenv Qnnode $Qnnode ; setenv Qgeometry ""'""$Qgeometry""'"" ; cccqsub $*" && Exit=0 || Exit="$?" Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export topdog=$topdog ; export Qnnode=$Qnnode ; export Qgeometry=""'""$Qgeometry""'"" ; cccqsub $*" elif [ -n "$Qnnode" -a -n "$Qphys_node" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export topdog=$topdog ; export Qnnode=$Qnnode ; export Qphys_node=$Qphys_node ; cccqsub $*" elif [ -n "$Qnnode" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export topdog=$topdog ; export Qnnode=$Qnnode ; cccqsub $*" elif [ -n "$Qgeometry" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export topdog=$topdog ; export Qgeometry=""'""$Qgeometry""'"" ; cccqsub $*" elif [ -n "$Qphys_node" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export topdog=$topdog ; export Qphys_node=$Qphys_node ; cccqsub $*" else Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export topdog=$topdog ; cccqsub $*" fi exec ssh -T ${qNamexT} " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi else if [ -n "$Qnnode" -a -n "$Qgeometry" ] ; then # exec ssh ${qNamexT} "$Qstrng ; cd $Cwd $Qextra ; setenv Qnnode $Qnnode ; setenv Qgeometry ""'""$Qgeometry""'"" ; cccqsub $*" && Exit=0 || Exit="$?" Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export Qnnode=$Qnnode ; export Qgeometry=""'""$Qgeometry""'"" ; cccqsub $*" elif [ -n "$Qnnode" -a -n "$Qphys_node" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export Qnnode=$Qnnode ; export Qphys_node=$Qphys_node ; cccqsub $*" elif [ -n "$Qnnode" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export Qnnode=$Qnnode ; cccqsub $*" elif [ -n "$Qgeometry" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export Qgeometry=""'""$Qgeometry""'"" ; cccqsub $*" elif [ -n "$Qphys_node" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra ; export Qphys_node=$Qphys_node ; cccqsub $*" else Qstrng2="$Qstrng ; cd $Cwd $Qextra ; cccqsub $*" fi exec ssh -T ${qNamexT} " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi fi fi fi fi # if [ "$Batch_mode" != 'BEpbspro' -a "$Batch_mode" != 'PPPjobst' ] ; then if [ "$Batch_mode" != 'BEpbspro' -a "$Batch_mode" != 'BEjobst' -a "$Batch_mode" != 'PPPjobst' ] ; then # HOSTIDf=`echo $HOSTID | cut -c 1-3` # HOSTIDf=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' | cut -c 1-3` HOSTIDf=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | cut -c 1-3` HOSTIDf=${HOSTIDf:-`echo $HOSTID | sed -e 's/^ppp/cs/g' | cut -c 1-3`} if [ \( "$HOSTIDf" = 'cs1' -o "$HOSTIDf" = 'pp1' -o "$HOSTIDf" = 'xc2' \) -a "$OS" = 'Linux' ] ; then if [ "$qName" = 'ppp1' -o "$qName" = 'tape08' -o "$qName" = 'cfsxfer' -o "$qName" = 'ccrnftp' ] ; then PlxFEPrfx='cs1fe' # Enable the following to force submission through "ssh ppp1 ..." if not on "cs1fe" already. HOSTIDPrfx=`echo $HOSTID | sed -n -e 's/^.*cs1fe.*$/cs1fe/p'` # Works HOSTIDPrfx=${HOSTIDPrfx:-`echo $HOSTID | sed -n -e 's/^.*ppp1.*$/cs1fe/p'`} # Works HOSTIDPrfx=${HOSTIDPrfx:=$HOSTID} # Workaround #PlxFEPrfx="NA" #HOSTIDPrfx="NA" else PlxFEPrfx="NA" HOSTIDPrfx="NA" fi elif [ \( "$HOSTIDf" = 'cs2' -o "$HOSTIDf" = 'pp2' -o "$HOSTIDf" = 'xc1' \) -a "$OS" = 'Linux' ] ; then if [ "$qName" = 'ppp2' -o "$qName" = 'tape08' -o "$qName" = 'cfsxfer' -o "$qName" = 'ccrnftp' ] ; then CtrFEPrfx='cs2fe' # Enable the following to force submission through "ssh ppp2 ..." if not on "cs2fe" already. HOSTIDPrfx=`echo $HOSTID | sed -n -e 's/^.*cs2fe.*$/cs2fe/p'` # Works HOSTIDPrfx=${HOSTIDPrfx:-`echo $HOSTID | sed -n -e 's/^.*ppp2.*$/cs2fe/p'`} # Works HOSTIDPrfx=${HOSTIDPrfx:=$HOSTID} # Workaround #CtrFEPrfx="NA" #HOSTIDPrfx="NA" else CtrFEPrfx="NA" HOSTIDPrfx="NA" fi fi # PPP1 if [ \( \( "$qName" = 'ppp1' -o "$qName" = 'tape08' -o "$qName" = 'cfsxfer' -o "$qName" = 'ccrnftp' \) -a "$OS" = 'Linux' -a \( "$HOSTIDf" = 'cs1' -o "$HOSTIDf" = 'pp1' \) -a "$HOSTIDPrfx" != "$PlxFEPrfx" \) -o \( "$qName" = 'ppp1' -a "$OS" != 'Linux' \) -o \( "$qName" = 'ppp1' -a "$OS" = 'Linux' -a "$HOSTIDf" != 'cs1' -a "$HOSTIDf" != 'pp1' \) -o \( \( "$qName" = 'tape08' -o "$qName" = 'cfsxfer' -o "$qName" = 'ccrnftp' \) -a "$morigin" != 'ppp2' -a \( \( "$OS" != 'Linux' \) -o \( "$OS" = 'Linux' -a \( "$HOSTIDf" = 'gps' -o "$HOSTIDf" = 'usr' \) \) \) \) ] ; then Cwd=`pwd` ssh -n ppp1 'hostname' >> /dev/null && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; ppp1 not reachable ; will attempt in a short while cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; ppp1 not reachable ; will attempt in a short while cccqsub $*" >> $HOME/problem_log sleep 180 fi if [ -n "$qsublog" -a \( "$qsublog" = 'on' -o "$qsublog" = 'yes' \) ] ; then if [ -n "$topdog" ] ; then # exec ssh ppp1 "$Qstrng ; cd $Cwd $Qextra2 ; setenv qsublog $qsublog ; setenv topdog $topdog ; cccqsub $*" && Exit=0 || Exit="$?" Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; export qsublog=$qsublog ; export topdog=$topdog ; cccqsub $*" exec ssh -T eccc-ppp1 " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi else Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; export qsublog=$qsublog ; cccqsub $*" exec ssh -T eccc-ppp1 " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi fi else if [ -n "$topdog" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; export topdog=$topdog ; cccqsub $*" exec ssh -T eccc-ppp1 " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi else Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; cccqsub $*" exec ssh -T eccc-ppp1 " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi fi fi # PPP2 elif [ \( \( "$qName" = 'ppp2' -o "$qName" = 'tape08' -o "$qName" = 'cfsxfer' -o "$qName" = 'ccrnftp' \) -a "$OS" = 'Linux' -a \( "$HOSTIDf" = 'cs2' -o "$HOSTIDf" = 'pp2' \) -a "$HOSTIDPrfx" != "$CtrFEPrfx" \) -o \( "$qName" = 'ppp2' -a "$OS" != 'Linux' \) -o \( "$qName" = 'ppp2' -a "$OS" = 'Linux' -a "$HOSTIDf" != 'cs2' -a "$HOSTIDf" != 'pp2' \) -o \( \( "$qName" = 'tape08' -o "$qName" = 'cfsxfer' -o "$qName" = 'ccrnftp' \) -a "$morigin" != 'ppp1' -a "$OS" != 'Linux' -a \( \( "$OS" != 'Linux' \) -o \( "$OS" = 'Linux' -a \( "$HOSTIDf" = 'gps' -o "$HOSTIDf" = 'usr' \) \) \) \) ] ; then Cwd=`pwd` ssh -n ppp2 'hostname' >> /dev/null && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; ppp2 not reachable ; will attempt in a short while cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; ppp2 not reachable ; will attempt in a short while cccqsub $*" >> $HOME/problem_log sleep 180 fi if [ -n "$qsublog" -a \( "$qsublog" = 'on' -o "$qsublog" = 'yes' \) ] ; then if [ -n "$topdog" ] ; then # exec ssh ppp2 "$Qstrng ; cd $Cwd $Qextra2 ; setenv qsublog $qsublog ; setenv topdog $topdog ; cccqsub $*" && Exit=0 || Exit="$?" Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; export qsublog=$qsublog ; export topdog=$topdog ; cccqsub $*" exec ssh -T eccc-ppp2 " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi else Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; export qsublog=$qsublog ; cccqsub $*" exec ssh -T eccc-ppp2 " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi fi else if [ -n "$topdog" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; export topdog=$topdog ; cccqsub $*" exec ssh -T eccc-ppp2 " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi else Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; cccqsub $*" exec ssh -T eccc-ppp2 " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; failed cccqsub $*" >> $HOME/problem_log exit $Exit fi fi fi elif [ "$qName" = 'dataq' -a "$HOSTIDf" != 'dat' ] ; then Cwd=`pwd` if [ -n "$qsublog" -a \( "$qsublog" = 'on' -o "$qsublog" = 'yes' \) ] ; then ssh -n dataq 'hostname' >> /dev/null && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; dataq not reachable ; will attempt in a short while cccqsub $*" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; dataq not reachable ; will attempt in a short while cccqsub $*" >> $HOME/problem_log sleep 180 fi if [ -n "$topdog" ] ; then # exec ssh dataq "$Qstrng ; cd $Cwd $Qextra2 ; setenv qsublog $qsublog ; setenv topdog $topdog ; cccqsub $*" && Exit=0 || Exit="$?" Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; export qsublog=$qsublog ; export topdog=$topdog ; cccqsub $*" exec ssh -T dataq " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; failed cccqsub $*" >> $HOME/tmp/cccqsub_log exit $Exit fi else Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; export qsublog=$qsublog ; cccqsub $*" exec ssh -T dataq " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; failed cccqsub $*" >> $HOME/tmp/cccqsub_log exit $Exit fi fi else if [ -n "$topdog" ] ; then Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; export topdog=$topdog ; cccqsub $*" exec ssh -T dataq " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; failed cccqsub $*" >> $HOME/tmp/cccqsub_log exit $Exit fi else Qstrng2="$Qstrng ; cd $Cwd $Qextra2 ; cccqsub $*" exec ssh -T dataq " echo $Qstrng2 | /bin/bash -l -" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; failed cccqsub $*" >> $HOME/tmp/cccqsub_log exit $Exit fi fi fi # fi fi fi #!!!! if [ "$Batch_mode" = 'BEpbspro' ] ; then # set -x #!!!! # Back-end PBS pro case translate and submit using qsub... #???? LOGNAME=${LOGNAME:=`whoami`} # if [ -d '/home/ords/*/*/'"${LOGNAME}/." ] ; then if [ -d /home/ords/*/*/${LOGNAME}/. ] ; then BEPBSPRO_spool_dir='/home/ords/crd/ccrn/scrd101/BEPBSPRO_spool_dir' else eval BEPBSPRO_spool_dir='/home/scrd101/BEPBSPRO_spool_dir' fi stamp="${HOSTID}_"`date +%Y%j%H%M%S` stampid="${stamp}_$$" USR=${USR:=`whoami`} TMPJOBCMDFILE="${BEPBSPRO_spool_dir}/bepbstmpjbcmd_${USR}_${stamp}_$$.pbs" CKPTFILE="ckpt_file_${USR}_${stamp}_$$" RMFILES="$TMPJOBCMDFILE" # trap "(sleep 20 ; rm -f $TMPJOBCMDFILE) &" 0 # Set defaults... # Preemption=1 # target cluster where jobs will be subject to preemption # Preemption=0 # target cluster where jobs will not be subject to preemption # class: # JOBCLASS='workq' JOBCLASS='development' JOBMEMORY='' HJOBMEMORY='' if [ "$qName" = 'hare' ] ; then if [ "$topdog" = 'yes' -a \( "$LOGNAME" = 'acrnmr5' -o "$LOGNAME" = 'acrnmr6' -o "$LOGNAME" = 'acrnpd6' -o "$LOGNAME" = 'acrnpd7' \) ] ; then JOBCLASS='projects' elif [ "$premtn" = 'yes' ] ; then JOBCLASS='preemptable' else # JOBCLASS='workq' JOBCLASS='development' fi #Preemption=1 Preemption=0 elif [ "$qName" = 'brooks' ] ; then if [ "$topdog" = 'yes' -a \( "$LOGNAME" = 'acrnmr5' -o "$LOGNAME" = 'acrnmr6' -o "$LOGNAME" = 'acrnpd6' -o "$LOGNAME" = 'acrnpd7' \) ] ; then JOBCLASS='projects' elif [ "$premtn" = 'yes' ] ; then JOBCLASS='preemptable' else # JOBCLASS='workq' JOBCLASS='development' fi Preemption=1 #Preemption=0 fi # job_type: JOBTYPE='serial' # ConsumableCpus: JOBPES='1' set -- $dirctv while [ $# -gt 0 ] do case "$1" in -c) # Ignore shift ;; -mu) # Ignore shift ;; -lu) # setup based on maximum number of cpu per process JOBTYPE='parallel' shift if [ "$openmp" = 'on' ] ; then JOBTYPEP='openmp' else JOBTYPEP='multi' fi JOBPES="$1" ;; -pe) # Allow setting based on SGE parallel environment switch. JOBTYPE='parallel' shift JOBTYPEP=${JOBTYPEP:="$1"} shift JOBPES="$1" ;; -q) # queue name shift # JOBCLASS=$1 ;; -e) # error log location shift JOBERRLOG=`echo $1 | sed -n -e 's/^.*://' -e '1,$p'` ;; -eo) # both logs in same file JOBEO=EO ;; -o) # output log location shift JOBLOG=`echo $1 | sed -n -e 's/^.*://' -e '1,$p'` ;; -r) # job identification shift # JOBNAME=`echo $1 | cut -c1-20` JOBNAME=$1 ;; -lT) # time limit shift # JOBTIME=$1 JOBTIME=${JOBTIME:=$1} ;; -lR) # time limit shift # JOBTIME=$1 JOBTIME=${JOBTIME:=$1} ;; -lM) # memory limit shift # JOBMEMORY=$1 JOBMEMORY=`echo $1 |tr '[a-z]' '[A-Z]' | sed -n -e 's/B//' -e '1,$p'` JOBMEMORY=${JOBMEMORY:='1000M'} ;; *) # script to execute if [ -f $1 ] ; then JOBSCRIPT=$1 else JOBRest="$JOBRest $1" fi ;; esac shift done HMaxMemoryPerNode='' HJOBMEMORY=${JOBMEMORY:-'1500M'} tUnIt=`echo $HJOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*M/M/p'` tUnIt=${tUnIt:=`echo $HJOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*G/G/p'`} if [ -n "$tUnIt" -a "$tUnIt" = 'M' ] ; then tJOBMEMORY=`echo ${HJOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/M//' -e '1,$p'` if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 51000 ] ; then HMaxMemoryPerNode='102' else HMaxMemoryPerNode='51' fi elif [ -n "$tUnIt" -a "$tUnIt" = 'G' ] ; then tJOBMEMORY=`echo ${HJOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/G//' -e '1,$p'` if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 51 ] ; then HMaxMemoryPerNode='102' else HMaxMemoryPerNode='51' fi fi HMaxMemoryPerNode=${HMaxMemoryPerNode:='51'} HJOBMEMORY="$HMaxMemoryPerNode"'G' tJOBMEMORY='' tUnIt='' # Ensure "Qnnode" is set and "JOBTYPE" is defined accordingly. # Enforce for Qnnode = 1 & nproc = 1,...,8 --> 'serial' # (ie. OpenMP on single node --> 'serial') # for "parallel": # nproc=1,2,4 or 8 (only) for MPI tasks. Qnnode=${Qnnode:=1} if [ "$Qnnode" -gt 1 ] ; then # MPI job JOBTYPE='parallel' Ntasks=${Qnnode:='1'} if [ "$qName" = 'brooks' ] ; then MaxMemoryPerNode=`expr $HMaxMemoryPerNode \* 1000` if [ "$premtn" = 'yes' ] ; then MaxMemoryPerNode=`expr 51 \* 1000` ; fi elif [ "$qName" = 'hare' ] ; then MaxMemoryPerNode=`expr $HMaxMemoryPerNode \* 1000` if [ "$premtn" = 'yes' ] ; then MaxMemoryPerNode=`expr 51 \* 1000` ; fi else MaxMemoryPerNode='15000' fi if [ -z "$Qgeometry" ] ; then if [ -n "$Qphys_node" ] ; then if [ "$Qphys_node" -gt "$Qnnode" ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, Invalid setting of phys_node=${phys_node} exceeding $Qnnode MPI tasks!" >> $HOME/tmp/cccqsub_log echo "QSUB: Sorry, Invalid setting of phys_node=${phys_node} which" echo " is exceeding $Qnnode MPI tasks!" exit -1 fi Nodes="$Qphys_node" if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then Taskspernode=`expr $Qnnode / $Nodes ` MaxThrdspernode=`expr $Taskspernode \* $JOBPES` if [ "$MaxThrdspernode" -le 128 ] ; then CccMA_MCM='on' fi JOBPES=`expr 64 / $Taskspernode` if [ "$JOBPES" -lt 1 ] ; then JOBPES=1 fi JOBMEMORY=`expr $MaxMemoryPerNode / $Taskspernode`"M" else Taskspernode=`expr $Qnnode / $Nodes ` MaxThrdspernode=`expr $Taskspernode \* $JOBPES` JOBPES=`expr 8 / $Taskspernode` if [ "$JOBPES" -lt 1 ] ; then JOBPES=1 fi # if [ "$qName" = 'rigel' ] ; then # if [ "$MaxThrdspernode" -le 16 ] ; then # CccMA_MCM='on' # fi # fi JOBMEMORY=`expr $MaxMemoryPerNode / $Taskspernode`"M" fi else # -n "$Qphys_node" if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then if [ "$JOBPES" -eq 1 ] ; then BLOCKING=64 elif [ "$JOBPES" -eq 2 ] ; then BLOCKING=32 elif [ "$JOBPES" -eq 4 ] ; then BLOCKING=16 #BLOCKING=2 elif [ "$JOBPES" -le 8 ] ; then JOBPES=8 BLOCKING=8 elif [ "$JOBPES" -le 16 ] ; then JOBPES=16 BLOCKING=4 elif [ "$JOBPES" -le 32 ] ; then JOBPES=32 BLOCKING=2 else JOBPES=64 BLOCKING=1 fi if [ "$BLOCKING" -gt "$Qnnode" ] ; then BLOCKING="$Qnnode" fi Taskspernode="$BLOCKING" JOBMEMORY=`expr $MaxMemoryPerNode / $Taskspernode`"M" CPUpertask=${JOBPES} TtlCPU=`expr $Ntasks \* $CPUpertask` Tmphold=`expr 31 + $TtlCPU` Nodes=`expr $Tmphold / 32` TTLTasks=`expr $Nodes \* $Taskspernode` if [ "$TTLTasks" -ne "$Qnnode" ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, $Qnnode MPI tasks is not allowed; (no partial) node(s) use!" >> $HOME/tmp/cccqsub_log echo "QSUB: Sorry, $Qnnode MPI tasks is not allowed since the total number " echo " of tasks must maps into full (no partial) node(s) use!" exit -1 fi else if [ "$JOBPES" -eq 1 ] ; then BLOCKING=8 elif [ "$JOBPES" -eq 2 ] ; then BLOCKING=4 elif [ "$JOBPES" -le 4 ] ; then JOBPES=4 BLOCKING=2 else JOBPES=8 BLOCKING=1 fi if [ "$BLOCKING" -gt "$Qnnode" ] ; then BLOCKING="$Qnnode" fi Taskspernode="$BLOCKING" CPUpertask=${JOBPES} TtlCPU=`expr $Ntasks \* $CPUpertask` Tmphold=`expr 7 + $TtlCPU` Nodes=`expr $Tmphold / 8` TTLTasks=`expr $Nodes \* $Taskspernode` if [ "$TTLTasks" -ne "$Qnnode" ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, $Qnnode MPI tasks is not allowed; (no partial) node(s) use!" >> $HOME/tmp/cccqsub_log echo "QSUB: Sorry, $Qnnode MPI tasks is not allowed since the total number " echo " of tasks must maps into full (no partial) node(s) use!" exit -1 fi JOBMEMORY=`expr $MaxMemoryPerNode / $Taskspernode`"M" fi fi # -n "$Qphys_node" else # -z "$Qgeometry" # Figure out from "Qgeometry" setting, the number of physical nodes, # total number of tasks and maximum number of tasks per node. # Then set other parameters accordingly. QGlist=`echo "$Qgeometry" | sed -e 's/[{()}]\{1,\}/ /g' ` QGnode=0 QGmax_tasks_per_node=0 QGmax_tasks=0 # set -x for QGtasks in $QGlist do QGnode=` expr $QGnode + 1` QGtasks_per_node=`echo $QGtasks | $AWK -F, '{print NF}'` QGtasks_per_node=${QGtasks_per_node:-'0'} QGmax_tasks=` expr $QGmax_tasks + $QGtasks_per_node ` if [ "$QGtasks_per_node" -gt $QGmax_tasks_per_node ] ; then QGmax_tasks_per_node=$QGtasks_per_node fi done if [ "$QGmax_tasks" -ne "$Qnnode" ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, $Qnnode MPI tasks is not consistent with geometry $QGmax_tasks total MPI tasks!" >> $HOME/tmp/cccqsub_log echo "QSUB: Sorry, Invalid setting of specified $Qnnode MPI tasks which" echo " is not consistent with geometry $QGmax_tasks total MPI tasks!" exit -1 fi Nodes="$QGnode" Qnnode="$QGmax_tasks" Taskspernode="$QGmax_tasks_per_node" if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then MaxThrdspernode=`expr $Taskspernode \* $JOBPES` if [ "$MaxThrdspernode" -le 64 ] ; then CccMA_MCM='on' fi JOBPES=`expr 64 / $Taskspernode` if [ "$JOBPES" -lt 1 ] ; then JOBPES=1 fi JOBMEMORY=`expr $MaxMemoryPerNode / $Taskspernode`"M" else MaxThrdspernode=`expr $Taskspernode \* $JOBPES` JOBPES=`expr 8 / $Taskspernode` if [ "$JOBPES" -lt 1 ] ; then JOBPES=1 fi # if [ "$qName" = 'rigel' ] ; then # if [ "$MaxThrdspernode" -le 16 ] ; then # CccMA_MCM='on' # fi # fi JOBMEMORY=`expr $MaxMemoryPerNode / $Taskspernode`"M" fi fi # -z "$Qgeometry" else # if [ "$JOBPES" -eq 1 ] ; then JOBTYPE='serial' if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then if [ "$JOBPES" -gt 64 ] ; then JOBPES=64 fi else if [ "$JOBPES" -gt 8 ] ; then JOBPES=8 fi fi # else # # OpenMP case on single node # JOBPES=8 # fi fi # Set default time limit to 10 minutes... JOBTIME=${JOBTIME:='600'} # current pwd is job's pwd JOBPWD=$(pwd) # get user's home directory JOBHOMEDIR=$HOME # put output in .queue subdirectory if [ -n "$JHOME" -a "${JHOME}" != "${HOME}" -a -d "${JHOME}/.queue/." -a "$SITE_ID" = 'DrvlSC' ] ; then JOBQUEUEDIR="${JHOME}/.queue" else JOBQUEUEDIR=${JOBHOMEDIR}/.queue fi # use jobname from command line or make one up # JOBNAME=${JOBNAME-${JOBSCRIPT-stdin}} # JOBNAME=${JOBNAME-`basename ${JOBSCRIPT} | cut -c1-20`} JOBNAME=${JOBNAME-`basename ${JOBSCRIPT}`} JOBNAME=${JOBNAME-stdin} # If jobscript is not specified, then read from stdin. if [ -z "$JOBSCRIPT" ] ; then JOBSCRIPT="$BEPBSPRO_spool_dir/${JOBNAME}_pbsjbscrpt_${USR}_${stamp}_$$" RMFILES="$RMFILES $JOBSCRIPT" # trap "(sleep 20 ; rm -f $TMPJOBCMDFILE $JOBSCRIPT) &" 0 cat > $JOBSCRIPT if [ -s "$JOBSCRIPT" ] ; then chmod g+r $JOBSCRIPT : else [ -f $JOBSCRIPT ] && \rm -f $JOBSCRIPT || : touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed due to invalid/empty input script file!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed due to invalid/empty input script file!" exit -1 fi else if [ -s "$JOBSCRIPT" ] ; then JOBSCRIPTN="$BEPBSPRO_spool_dir/${JOBNAME}_pbsjbscrpt_${USR}_${stamp}_$$" \cp $JOBSCRIPT $JOBSCRIPTN JOBSCRIPT="$JOBSCRIPTN" RMFILES="$RMFILES $JOBSCRIPT" else touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed due to invalid/empty $JOBSCRIPT input script file!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed due to invalid/empty $JOBSCRIPT input script file!" exit -1 fi fi chmod u+x $JOBSCRIPT # setup output and error logs # if error and standard output to same file if [ "$JOBEO" = EO ] ; then # use standard output if it has been specified if [ -n "$JOBLOG" ] ; then JOBERRLOG=$JOBLOG # or use error output if it has been specified elif [ -n "$JOBERRLOG" ] ; then JOBLOG=$JOBERRLOG # or use defaults else JOBLOG=${JOBLOG-${JOBQUEUEDIR}/${JOBNAME}'.az.$(Cluster).$(Process).out'} JOBERRLOG=$JOBLOG fi else # otherwise make sure that defaults are set if [ -z "$JOBLOG" ] ; then JOBLOG=${JOBQUEUEDIR}/${JOBNAME}'.az.$(Cluster).$(Process).out' fi if [ -z "$JOBERRLOG" ] ; then JOBERRLOG=${JOBQUEUEDIR}/${JOBNAME}'.az.$(Cluster).$(Process).err' fi fi # if [ "$JOBCLASS" = 'projects' -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 ] ; then if [ \( "$JOBCLASS" = 'projects' -o "$JOBCLASS" = 'preemptable' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 ] ; then # JOBCLASS='workq' JOBCLASS='development' fi if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$bexfer" = 'on' ] ; then #if [ "$JOBTIME" -lt 1800 ] ; then JOBTIME=1800 ; fi #JOBTIME=1800 JOBTIME=2700 #JOBCLASS='dev_xfer' JOBCLASS='cfs_xfer' elif [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$JOBTIME" -lt 1800 ] ; then JOBTIME=1800 fi # create loadl command file if [ "$CCCSHELL" = '/bin/bash' ] ; then QSUBBSHELL="$CCCSHELL" else QSUBBSHELL='/bin/ksh' fi ### cat > $TMPJOBCMDFILE << EOF #!${QSUBBSHELL} # @ job_type = $JOBTYPE # @ job_name = $JOBNAME # @ executable = $JOBSCRIPT # @ initialdir = /tmp # @ input = /dev/null # @ output = $JOBLOG # @ error = $JOBERRLOG # @ class = $JOBCLASS # EOF ## TRY WITHOUT FOR NOW ##echo "#!${QSUBBSHELL} -l ## TRY WITHOUT FOR NOW ###PBS -S ${QSUBBSHELL} -V ## TRY WITHOUT FOR NOW ###PBS -N $JOBNAME ## TRY WITHOUT FOR NOW ####PBS -q $JOBCLASS " > $TMPJOBCMDFILE echo "#!${QSUBBSHELL} -l #PBS -S ${QSUBBSHELL} #PBS -N $JOBNAME #PBS -q $JOBCLASS " > $TMPJOBCMDFILE if [ -n "$JOBLOG" -a -n "$JOBERRLOG" -a "$JOBLOG" = "$JOBERRLOG" ] ; then echo "#PBS -j oe" >> $TMPJOBCMDFILE echo "#PBS -o $JOBLOG" >> $TMPJOBCMDFILE else [ -n "$JOBLOG" ] && echo "#PBS -o $JOBLOG" >> $TMPJOBCMDFILE || : [ -n "$JOBERRLOG" ] && echo "#PBS -e $JOBERRLOG" >> $TMPJOBCMDFILE || : fi chmod g+r $TMPJOBCMDFILE unset QSUBBSHELL if [ -n "$Qdelaymin" ] ; then TZ=UTC0; export TZ if [ "$Qdelaymin" -gt 0 ] ; then DatTimDlyx=`lldatimp ${Qdelaymin}` # echo "# @ startdate = $DatTimDlyx" >> $TMPJOBCMDFILE echo "#PBS -a $DatTimDlyx" >> $TMPJOBCMDFILE fi fi ### # ##### @ initialdir = $JOBHOMEDIR ### ### # echo "# @ notify_user = $USER@ec.gc.ca" >> $TMPJOBCMDFILE ### # LOGNAME=${LOGNAME:=`whoami | sed -e 's/^....//'`} ### LOGNAME=${LOGNAME:=`whoami`} ### echo "# @ notify_user = $LOGNAME@ec.gc.ca" >> $TMPJOBCMDFILE ### # echo "# @ notification = complete" >> $TMPJOBCMDFILE ### echo "# @ notification = error " >> $TMPJOBCMDFILE ### # echo "# @ notification = never" >> $TMPJOBCMDFILE ### # echo "# @ checkpoint = no" >> $TMPJOBCMDFILE ### # echo "# @ restart = no" >> $TMPJOBCMDFILE ### # if [ "$JOBTYPE" = 'serial' -a "$qName" = 'algol' ] ; then ### if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then ### #echo "# @ checkpoint = yes" >> $TMPJOBCMDFILE ### echo "# @ checkpoint = no" >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'algol' ] ; then ### # echo "# @ checkpoint = no" >> $TMPJOBCMDFILE ### else ### echo "# @ checkpoint = yes" >> $TMPJOBCMDFILE ### fi ### if [ "$qName" = 'hare' ] ; then ### echo '# @ ckpt_dir = /fs/dev/crb/had_src/LOADL_ckpt_dir' >> $TMPJOBCMDFILE ### elif [ "$qName" = 'brooks' ] ; then ### echo '# @ ckpt_dir = /fs/dev/crb/spd_src/LOADL_ckpt_dir' >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'saiph' ] ; then ### # #echo '# @ ckpt_dir = /fs/saiph/crb/sad_src/LOADL_ckpt_dir' >> $TMPJOBCMDFILE ### # echo '# @ ckpt_dir = /fs/dev/crb/mad_src/LOADL_ckpt_dir' >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'maia' -o "$qName" = 'zeta' ] ; then ### # echo '# @ ckpt_dir = /fs/dev/crb/mad_src/LOADL_ckpt_dir' >> $TMPJOBCMDFILE ### # # elif [ "$qName" = 'algol' -a "$JOBTYPE" != 'serial' ] ; then ### # elif [ "$qName" = 'algol' ] ; then ### # #echo '# @ ckpt_dir = /fs/c8/crb/ald_src/LOADL_ckpt_dir' >> $TMPJOBCMDFILE ### # : ### # # else ### # # echo '# @ ckpt_dir = /fs/crb/az7p_src/LOADL_ckpt_dir' >> $TMPJOBCMDFILE ### fi ### if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then ### echo "# @ ckpt_file = $CKPTFILE" >> $TMPJOBCMDFILE ### # # elif [ "$qName" = 'algol' -a "$JOBTYPE" != 'serial' ] ; then ### # elif [ "$qName" = 'algol' ] ; then ### # # echo "# @ ckpt_file = $CKPTFILE" >> $TMPJOBCMDFILE ### # : ### # else ### # echo "# @ ckpt_file = $CKPTFILE" >> $TMPJOBCMDFILE ### fi ### ### # add limits to command file if [ -n "$JOBTIME" ] ; then # echo "# @ job_cpu_limit = ${JOBTIME}" >> $TMPJOBCMDFILE # if [ "${JOBTIME}" -ge 400 -a \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" != 'serial' ] ; then # if [ "${JOBTIME}" -ge 400 -a \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" != 'serial' -a "$mpifdbg" != 'on' ] ; then # JOBTIMEsl=`expr $JOBTIME - 179 ` # echo "# @ wall_clock_limit = ${JOBTIME},${JOBTIMEsl}" >> $TMPJOBCMDFILE # JOBTIMEsl='' # else #echo "# @ wall_clock_limit = ${JOBTIME}" >> $TMPJOBCMDFILE JOBTIME=`echo ${JOBTIME} | $AWK '{ print strftime("%H:%M:%S", $1,1) ; }'` echo "#PBS -l walltime=${JOBTIME}" >> $TMPJOBCMDFILE # fi fi ### # if [ -n "$JOBMEMORY" ] ; then ### # echo "# @ data_limit = ${JOBMEMORY}" >> $TMPJOBCMDFILE ### # JOBMEMORYlwr=`echo $JOBMEMORY |tr '[A-Z]' '[a-z]'` ### # ### # fi ### ### # if [ "$qName" = 'maia' -o "$qName" = 'zeta' -o "$qName" = 'saiph' -o "$qName" = 'algol' -o "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then ### if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then ### JOBMEMORY=${JOBMEMORY:-'1200M'} ### if [ "$JOBTYPE" = 'serial' ] ; then ### tUnIt=`echo $JOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*M/M/p'` ### tUnIt=${tUnIt:=`echo $JOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*G/G/p'`} ### if [ -n "$tUnIt" -a "$tUnIt" = 'M' ] ; then ### tJOBMEMORY=`echo ${JOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/M//' -e '1,$p'` ### if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -lt 1200 ] ; then ### JOBMEMORY='1200M' ### fi ### elif [ -n "$tUnIt" -a "$tUnIt" = 'G' ] ; then ### tJOBMEMORY=`echo ${JOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/G//' -e '1,$p'` ### if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -lt 2 ] ; then ### JOBMEMORY='2G' ### fi ### fi ### tJOBMEMORY='' ### tUnIt='' ### fi ### JOBMEMORYlwr=${JOBMEMORY:-'1200M'} ### # JOBMEMORYlwr=${JOBMEMORYlwr:-'1200m'} ### JOBMEMORYlwr=`echo $JOBMEMORYlwr |tr '[A-Z]' '[a-z]'` ### fi ### ### # if [ -n "$JOBMEMORYlwr" ] ; then ### # echo "# @ resources = ConsumableCpus(${JOBPES}) ConsumableMemory(${JOBMEMORYlwr}b)" >> $TMPJOBCMDFILE ### # else ### # echo "# @ resources = ConsumableCpus(${JOBPES})" >> $TMPJOBCMDFILE ### # fi ### ### if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'parallel' ] ; then ### echo "# @ core_limit = unlimited" >> $TMPJOBCMDFILE ### fi ### if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'parallel' -a -n "$Qgeometry" ] ; then ### if [ "$JOBCLASS" = 'preemptable' ] ; then ### if [ "$qName" = 'brooks' ] ; then ### echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory(51gb)" >> $TMPJOBCMDFILE ### else ### echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory(51gb)" >> $TMPJOBCMDFILE ### fi ### else ### if [ -n "$HJOBMEMORY" ] ; then ### gb=`echo $HJOBMEMORY | tr '[A-Z]' '[a-z]' | sed -n -e '/g/p' | sed -e 's/g//'` ### if [ -n "$gb" -a "$gb" -gt 51 ] ; then ### if [ "$gb" -le 102 ] ; then ### # JOBMEMORY=`echo $HJOBMEMORY | tr '[A-Z]' '[a-z]'` ### JOBMEMORY="${gb}gb" ### else ### JOBMEMORY="102gb" ### fi ### else ### JOBMEMORY="51gb" ### fi ### else ### JOBMEMORY="51gb" ### fi ### if [ "$qName" = 'brooks' ] ; then ### # echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory(102gb)" >> $TMPJOBCMDFILE ### echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory($JOBMEMORY)" >> $TMPJOBCMDFILE ### else ### #echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory(116gb)" >> $TMPJOBCMDFILE ### #echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory(102gb)" >> $TMPJOBCMDFILE ### echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory($JOBMEMORY)" >> $TMPJOBCMDFILE ### fi ### fi ### else ### if [ -n "$JOBMEMORY" -a "$qName" != 'hare' -a "$qName" != 'brooks' ] ; then ### echo "# @ data_limit = ${JOBMEMORY}" >> $TMPJOBCMDFILE ### fi ### if [ -n "$JOBMEMORYlwr" ] ; then ### if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'parallel' ] ; then ### if [ "$JOBCLASS" = 'preemptable' ] ; then ### echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory(51gb)" >> $TMPJOBCMDFILE ### else ### HJOBMEMORY="$JOBMEMORYlwr" ### if [ -n "$HJOBMEMORY" ] ; then ### gb=`echo $HJOBMEMORY | tr '[A-Z]' '[a-z]' | sed -n -e '/g/p' | sed -e 's/g//'` ### if [ -n "$gb" -a "$gb" -gt 51 ] ; then ### if [ "$gb" -le 102 ] ; then ### # JOBMEMORY=`echo $HJOBMEMORY | tr '[A-Z]' '[a-z]'` ### JOBMEMORY="${gb}gb" ### else ### JOBMEMORY="102gb" ### fi ### else ### JOBMEMORY="51gb" ### fi ### else ### JOBMEMORY="51gb" ### fi ### #echo "# @ resources = ConsumableCpus(${JOBPES}) ConsumableMemory(${JOBMEMORYlwr}b)" >> $TMPJOBCMDFILE ### #echo "# @ node_resources = ConsumableCpus(${JOBPES}) ConsumableMemory(${JOBMEMORYlwr}b)" >> $TMPJOBCMDFILE ### #echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory(102gb)" >> $TMPJOBCMDFILE ### echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory($JOBMEMORY)" >> $TMPJOBCMDFILE ### fi ### else ### echo "# @ resources = ConsumableCpus(${JOBPES}) ConsumableMemory(${JOBMEMORYlwr}b)" >> $TMPJOBCMDFILE ### fi ### else ### if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'parallel' ] ; then ### if [ "$JOBCLASS" = 'preemptable' ] ; then ### echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory(51gb)" >> $TMPJOBCMDFILE ### else ### if [ -n "$HJOBMEMORY" ] ; then ### gb=`echo $HJOBMEMORY | tr '[A-Z]' '[a-z]' | sed -n -e '/g/p' | sed -e 's/g//'` ### if [ -n "$gb" -a "$gb" -gt 51 ] ; then ### if [ "$gb" -le 102 ] ; then ### # JOBMEMORY=`echo $HJOBMEMORY | tr '[A-Z]' '[a-z]'` ### JOBMEMORY="${gb}gb" ### else ### JOBMEMORY="102gb" ### fi ### else ### JOBMEMORY="51gb" ### fi ### else ### JOBMEMORY="51gb" ### fi ### #echo "# @ resources = ConsumableCpus(${JOBPES})" >> $TMPJOBCMDFILE ### #echo "# @ node_resources = ConsumableCpus(${JOBPES})" >> $TMPJOBCMDFILE ### #echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory(102gb)" >> $TMPJOBCMDFILE ### echo "# @ node_resources = ConsumableCpus(64) ConsumableMemory($JOBMEMORY)" >> $TMPJOBCMDFILE ### fi ### else ### echo "# @ resources = ConsumableCpus(${JOBPES})" >> $TMPJOBCMDFILE ### fi ### fi ### fi ### if [ "$JOBTYPE" = 'parallel' ] ; then ### ### ### if [ "$Ntasks" -gt 1 ] ; then ### if [ -n "$Qgeometry" ] ; then ### echo "# @ task_geometry = $Qgeometry" >> $TMPJOBCMDFILE ### echo "# @ node_usage = not_shared" >> $TMPJOBCMDFILE ### else ### ### # echo "# @ total_tasks = $Ntasks " >> $TMPJOBCMDFILE ### # echo "# @ blocking = ${BLOCKING} " >> $TMPJOBCMDFILE ### ### echo "# @ node = $Nodes " >> $TMPJOBCMDFILE ### echo "# @ tasks_per_node = ${Taskspernode}" >> $TMPJOBCMDFILE ### #if [ "$Nodes" -le 1 -a "$JOBPES" -lt 16 ] ; then ### #if [ "$Nodes" -le 1 -a "$JOBPES" -lt 32 ] ; then ### #echo "MaxThrdspernode=$MaxThrdspernode" ### if [ "$Nodes" -le 1 -a -n "$MaxThrdspernode" -a "$MaxThrdspernode" -lt 16 ] ; then ### : ### # echo "# @ node_usage = shared" >> $TMPJOBCMDFILE ### else ### echo "# @ node_usage = not_shared" >> $TMPJOBCMDFILE ### fi ### fi ### if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then ### T_MP_EUIDEVICE='sn_all' ### echo "# @ network.MPI = sn_all,,US,,instances=max,," >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'algol' ] ; then ### # T_MP_EUIDEVICE='sn_all' ### # echo "# @ network.MPI = sn_all,shared,US" >> $TMPJOBCMDFILE ### else ### T_MP_EUIDEVICE='csss' ### echo "# @ network.MPI = csss,shared,US" >> $TMPJOBCMDFILE ### # echo "# @ network.MPI = csss,not_shared,ip" >> $TMPJOBCMDFILE ### fi ### # echo "# @ environment = COPY_ALL; MP_SHARED_MEMORY=yes;" >> $TMPJOBCMDFILE ### # echo "# @ environment = MP_SHARED_MEMORY=yes;" >> $TMPJOBCMDFILE ### # echo "# @ environment = MP_SHARED_MEMORY=yes; MP_COREFILE_FORMAT=STDERR" >> $TMPJOBCMDFILE ### ### ## echo '# @ environment = MP_SHARED_MEMORY=yes; MP_COREFILE_FORMAT=STDERR ; \' >> $TMPJOBCMDFILE ### # echo '# @ environment = MP_SHARED_MEMORY=yes; MP_COREFILE_FORMAT=STDERR ; MP_BUFFER_MEM=256M ; LLAPIERRORMSGS=yes ; \' >> $TMPJOBCMDFILE ### # echo '# @ environment = MP_SHARED_MEMORY=yes; MP_COREFILE_FORMAT=STDERR ; MP_BUFFER_MEM=256M ; LLAPIERRORMSGS=yes ; MP_EUIDEVICE=${T_MP_EUIDEVICE} ; \' >> $TMPJOBCMDFILE ### if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'parallel' ] ; then ### #LPS echo '# @ environment = MP_SHARED_MEMORY=yes; MP_BUFFER_MEM=256M ; LLAPIERRORMSGS=yes ; MP_EUIDEVICE=${T_MP_EUIDEVICE} ; \' >> $TMPJOBCMDFILE ### echo "# @ environment = MP_SHARED_MEMORY=yes; MP_BUFFER_MEM=256M ; LLAPIERRORMSGS=yes ; MP_EUIDEVICE=${T_MP_EUIDEVICE} ; "'\' >> $TMPJOBCMDFILE ### else ### #LPS echo '# @ environment = MP_SHARED_MEMORY=yes; MP_COREFILE_FORMAT=STDERR ; MP_BUFFER_MEM=256M ; LLAPIERRORMSGS=yes ; MP_EUIDEVICE=${T_MP_EUIDEVICE} ; \' >> $TMPJOBCMDFILE ### echo "# @ environment = MP_SHARED_MEMORY=yes; MP_COREFILE_FORMAT=STDERR ; MP_BUFFER_MEM=256M ; LLAPIERRORMSGS=yes ; MP_EUIDEVICE=${T_MP_EUIDEVICE} ; "'\' >> $TMPJOBCMDFILE ### fi ### if [ "$CccMA_MCM" = 'on' ] ; then ### echo ' CCCMA_MCM=on ; \' >> $TMPJOBCMDFILE ### fi ### # if [ "$qName" = 'maia' -o "$qName" = 'zeta' -o "$qName" = 'saiph' -o "$qName" = 'algol' -o "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then ### if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then ### if [ "$qName" = 'hare' ] ; then ### echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/had_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### elif [ "$qName" = 'brooks' ] ; then ### # echo ' CCCMA_LL_RMFILES="'"$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl "'"' >> $TMPJOBCMDFILE ### echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'maia' -o "$qName" = 'zeta' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/mad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'saiph' ] ; then ### # #echo " CCCMA_LL_RMFILES=$RMFILES /fs/saiph/crb/sad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/mad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'algol' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/c8/crb/ald_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### fi ### echo '# @ bulkxfer = yes' >> $TMPJOBCMDFILE ### #X# echo '# @ rset = rset_mcm_affinity' >> $TMPJOBCMDFILE ### #X# echo '# @ mcm_affinity_options = mcm_distribute mcm_mem_pref mcm_sni_none' >> $TMPJOBCMDFILE ### else ### echo " CCCMA_LL_RMFILES=$RMFILES /fs/crb/az7p_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### fi ### #if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then ### # echo "# @ task_affinity = core ($JOBPES)" >> $TMPJOBCMDFILE ### ##echo '# @ cpus_per_core = 2' >> $TMPJOBCMDFILE ### # echo '# @ cpus_per_core = 1' >> $TMPJOBCMDFILE ### # echo "# @ parallel_threads = $JOBPES" >> $TMPJOBCMDFILE ### #fi ### ### else ### ### # echo "# @ total_tasks = 1 " >> $TMPJOBCMDFILE ### # echo "# @ blocking = 1 " >> $TMPJOBCMDFILE ### echo "# @ node = 1 " >> $TMPJOBCMDFILE ### echo "# @ tasks_per_node = 1 " >> $TMPJOBCMDFILE ### # echo "# @ environment = MP_COREFILE_FORMAT=STDERR" >> $TMPJOBCMDFILE ### # echo '# @ environment = MP_COREFILE_FORMAT=STDERR ; \' >> $TMPJOBCMDFILE ### #if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'parallel' ] ; then ### # : ### #else ### # echo '# @ environment = MP_COREFILE_FORMAT=STDERR ; \' >> $TMPJOBCMDFILE ### #fi ### if [ "$qName" = 'hare' ] ; then ### echo "# @ environment = CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/had_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### elif [ "$qName" = 'brooks' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # echo '# @ environment = CCCMA_LL_RMFILES="'"$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl "'"' >> $TMPJOBCMDFILE ### echo "# @ environment = CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'saiph' ] ; then ### # #echo " CCCMA_LL_RMFILES=$RMFILES /fs/saiph/crb/sad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/mad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'maia' -o "$qName" = 'zeta' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/mad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'algol' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/c8/crb/ald_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### else ### # echo " CCCMA_LL_RMFILES=$RMFILES " >> $TMPJOBCMDFILE ### echo "# @ environment = CCCMA_LL_RMFILES=$RMFILES /fs/crb/az7p_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### fi ### ### fi ### ### elif [ "$JOBPES" -gt 1 ] ; then ### ### # OpenMP ("serial") case... ### ### if [ "$JOBPES" -lt 32 ] ; then ### : ### # echo "# @ node_usage = not_shared" >> $TMPJOBCMDFILE ### else ### echo "# @ node_usage = not_shared" >> $TMPJOBCMDFILE ### fi ### #echo '# @ environment = MP_COREFILE_FORMAT=STDERR ; \' >> $TMPJOBCMDFILE ### if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'parallel' ] ; then ### : ### else ### echo '# @ environment = MP_COREFILE_FORMAT=STDERR ; \' >> $TMPJOBCMDFILE ### fi ### if [ "$qName" = 'hare' ] ; then ### echo "# @ environment = CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/had_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### elif [ "$qName" = 'brooks' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # echo '# @ environment = CCCMA_LL_RMFILES="'"$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl "'"' >> $TMPJOBCMDFILE ### echo "# @ environment = CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'saiph' ] ; then ### # #echo " CCCMA_LL_RMFILES=$RMFILES /fs/saiph/crb/sad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/mad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'maia' -o "$qName" = 'zeta' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/mad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'algol' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/c8/crb/ald_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### else ### echo " CCCMA_LL_RMFILES=$RMFILES /fs/crb/az7p_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### fi ### ### else ### ### # echo "# @ node_usage = not_shared" >> $TMPJOBCMDFILE ### ### #echo '# @ environment = MP_COREFILE_FORMAT=STDERR ; \' >> $TMPJOBCMDFILE ### # if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'parallel' ] ; then ### # : ### # else ### # echo '# @ environment = MP_COREFILE_FORMAT=STDERR ; \' >> $TMPJOBCMDFILE ### # fi ### if [ "$qName" = 'hare' ] ; then ### echo "# @ environment = CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/had_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### elif [ "$qName" = 'brooks' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # echo '# @ environment = CCCMA_LL_RMFILES="'"$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl "'"' >> $TMPJOBCMDFILE ### echo "# @ environment = CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/spd_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'saiph' ] ; then ### # #echo " CCCMA_LL_RMFILES=$RMFILES /fs/saiph/crb/sad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/mad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'maia' -o "$qName" = 'zeta' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/dev/crb/mad_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### # elif [ "$qName" = 'algol' ] ; then ### # echo " CCCMA_LL_RMFILES=$RMFILES /fs/c8/crb/ald_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### else ### echo "# @ environment = CCCMA_LL_RMFILES=$RMFILES /fs/crb/az7p_src/LOADL_ckpt_dir/${CKPTFILE}.cntl " >> $TMPJOBCMDFILE ### fi ### ### fi ### ### # echo "# @ requirements = (OpSys == "AIX") " >> $TMPJOBCMDFILE ### # echo "# @ requirements = (OpSys == "AIX51") " >> $TMPJOBCMDFILE ### ### # echo "# @ environment = QSUB_WORKDIR=$JOBPWD" >> $TMPJOBCMDFILE ### ### ### echo "# @ queue" >> $TMPJOBCMDFILE if [ "$JOBTYPE" = 'parallel' ] ; then #echo "#PBS -l select=$Nodes:vntype=cray_compute,place=scatter " >> $TMPJOBCMDFILE echo "#PBS -l select=$Nodes:ncpus=36:vntype=cray_compute,place=scatter " >> $TMPJOBCMDFILE else ## if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then ## JOBMEMORY=${JOBMEMORY:-'1200M'} ## if [ "$JOBTYPE" = 'serial' ] ; then ## tUnIt=`echo $JOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*M/M/p'` ## tUnIt=${tUnIt:=`echo $JOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*G/G/p'`} ## if [ -n "$tUnIt" -a "$tUnIt" = 'M' ] ; then ## tJOBMEMORY=`echo ${JOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/M//' -e '1,$p'` ## if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -lt 1200 ] ; then ## JOBMEMORY='1200M' ## fi ## elif [ -n "$tUnIt" -a "$tUnIt" = 'G' ] ; then ## tJOBMEMORY=`echo ${JOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/G//' -e '1,$p'` ## if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -lt 2 ] ; then ## JOBMEMORY='2G' ## fi ## fi ## tJOBMEMORY='' ## tUnIt='' ## fi ## JOBMEMORYlwr=${JOBMEMORY:-'1200M'} ## # JOBMEMORYlwr=${JOBMEMORYlwr:-'1200m'} ## JOBMEMORYlwr=`echo $JOBMEMORYlwr |tr '[A-Z]' '[a-z]'` ## fi #echo "#PBS -l select=1:vntype=cray_compute:ncpus=1:mem=3000M" >> $TMPJOBCMDFILE #echo "#PBS -l select=1:vntype=cray_compute:ncpus=1" >> $TMPJOBCMDFILE echo "#PBS -l select=1:vntype=cray_compute:ncpus=36" >> $TMPJOBCMDFILE #echo "#PBS -l select=1:ncpus=1:mem=3000M" >> $TMPJOBCMDFILE fi echo 'export CCCMA_LL_RMFILES="'"$RMFILES"'"' >> $TMPJOBCMDFILE echo "if [ -s $CUSTOM_ENV_FILE ] ; then . $CUSTOM_ENV_FILE || : ; "'else if [ -z "$SYSTEM_SETUP" -a -z "$CCRNSRC" -a -d /home/scrd101/generic/. ] ; then [ -s /home/scrd101/generic/sc_cccma_setup_profile ] && . /home/scrd101/generic/sc_cccma_setup_profile || : ; fi ; fi ' >> $TMPJOBCMDFILE if [ -s "$JOBSCRIPT" ] ; then if [ "$JOBTYPE" = 'parallel' ] ; then echo "$JOBSCRIPT" >> $TMPJOBCMDFILE else #echo "aprun -n1 -cc none $JOBSCRIPT" >> $TMPJOBCMDFILE echo "$JOBSCRIPT" >> $TMPJOBCMDFILE fi fi #!!!!!! if [ -s $TMPJOBCMDFILE ] ; then : # ls -ld $TMPJOBCMDFILE # cat $TMPJOBCMDFILE else echo "`date`; `whoami`;$HOSTID: Can not proceed due to invalid $TMPJOBCMDFILE !" touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Can not proceed due to invalid $TMPJOBCMDFILE !" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; `whoami`;$HOSTID: Can not proceed due to invalid $TMPJOBCMDFILE !" >> $HOME/problem_log exit 1 fi echo "QSUB - top of submitted file:" head -35 $TMPJOBCMDFILE echo '------------' echo "QSUB - associated files with the submitted job:" \ls -ld $RMFILES echo '------------' #!!!!!! # submit the job # if [ "$OS" = 'AIX' ] ; then if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then # CCC_PBS_JOBSUB=${CCC_PBS_JOBSUB:-'qsub'} if [ "$qName" = 'hare' ] ; then CCC_PBS_JOBSUB='jobsub -c hare' else # set -x HOSTIDf6=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | cut -c 1-6` HOSTIDfm=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | sed -e 's/xc2mom.*-p2/xc2mom-p2/'` if [ "$HOSTIDf6" = 'xc2net' -o "$HOSTIDfm" = 'xc2mom-p2' -o "$qNamexT" = 'brooks-test' ] ; then CCC_PBS_JOBSUB='jobsub -c brooks-test' else CCC_PBS_JOBSUB='jobsub -c brooks' fi fi [ -s "./.pbsqsub_jid_${stampid}" ] && (\rm -f ./.pbsqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) && sleep 1 || : touch ./.pbsqsub_jid_${stampid} ; sleep 1 echo "QSUB - $CCC_PBS_JOBSUB $TMPJOBCMDFILE" ; echo '------------' $CCC_PBS_JOBSUB $TMPJOBCMDFILE >> ./.pbsqsub_jid_${stampid} && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 1st failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 1st failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/problem_log sleep 115 (\rm -f ./.pbsqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) sleep 1 touch ./.pbsqsub_jid_${stampid} ; sleep 1 $CCC_PBS_JOBSUB $TMPJOBCMDFILE >> ./.pbsqsub_jid_${stampid} && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 2nd failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 2nd failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/problem_log echo "QSUB - top of submitted file:" head -35 $TMPJOBCMDFILE exit $Exit else if [ -s "./.pbsqsub_jid_${stampid}" ] ; then cat ./.pbsqsub_jid_${stampid} if [ "$qsublog" = 'yes' -o "$qsublog" = 'on' ] ; then JiD_cptred=`tail -1 ./.pbsqsub_jid_${stampid}` touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$PBS_JOBID on $HOSTID (${nqsprfx}${jobname}${nqsext}${id}.${stamp}) --> $CCC_PBS_JOBSUB $TMPJOBCMDFILE ==> $JiD_cptred " >> $HOME/tmp/cccqsub_log fi fi (\rm -f ./.pbsqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) fi else if [ -s "./.pbsqsub_jid_${stampid}" ] ; then cat ./.pbsqsub_jid_${stampid} if [ "$qsublog" = 'yes' -o "$qsublog" = 'on' ] ; then JiD_cptred=`tail -1 ./.pbsqsub_jid_${stampid}` touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$PBS_JOBID on $HOSTID (${nqsprfx}${jobname}${nqsext}${id}.${stamp}) --> $CCC_PBS_JOBSUB $TMPJOBCMDFILE ==> $JiD_cptred " >> $HOME/tmp/cccqsub_log fi fi (\rm -f ./.pbsqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) fi else # if [ "$qName" = 'saiph' -o "$qName" = 'maia' -o "$qName" = 'zeta' -o "$qName" = 'algol' -o "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then Cwd=`pwd` ssh -n ${qNamexT} 'hostname' >> /dev/null && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; ${qName} not reachable ; will attempt in a short while $CCC_PBS_JOBSUB $TMPJOBCMDFILE" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; ${qName} not reachable ; will attempt in a short while $CCC_PBS_JOBSUB $TMPJOBCMDFILE" >> $HOME/problem_log sleep 180 fi [ -s "./.pbsqsub_jid_${stampid}" ] && (\rm -f ./.pbsqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) && sleep 1 || : # ssh ${qNamexT} "$Qstrng ; cd $Cwd ; $CCC_PBS_JOBSUB $TMPJOBCMDFILE" > ./.pbsqsub_jid_${stampid} && Exit=0 || Exit="$?" Qstrng2="$Qstrng ; cd $Cwd ; $CCC_PBS_JOBSUB $TMPJOBCMDFILE" ssh -T ${qNamexT} "echo $Qstrng2 | /bin/bash -l -" > ./.pbsqsub_jid_${stampid} && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 1st failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; 1st failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/problem_log sleep 115 [ -s "./.pbsqsub_jid_${stampid}" ] && (\rm -f ./.pbsqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) && sleep 1 || : # ssh ${qNamexT} "$Qstrng ; cd $Cwd ; $CCC_PBS_JOBSUB $TMPJOBCMDFILE" > ./.pbsqsub_jid_${stampid} && Exit=0 || Exit="$?" ssh -T ${qNamexT} "echo $Qstrng2 | /bin/bash -l -" > ./.pbsqsub_jid_${stampid} && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 2nd failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; 2nd failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/problem_log echo "QSUB - top of submitted file:" head -35 $TMPJOBCMDFILE exit $Exit else if [ -s "./.pbsqsub_jid_${stampid}" ] ; then cat ./.pbsqsub_jid_${stampid} if [ "$qsublog" = 'yes' -o "$qsublog" = 'on' ] ; then JiD_cptred=`tail -1 ./.pbsqsub_jid_${stampid}` touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$PBS_JOBID on $HOSTID (${nqsprfx}${jobname}${nqsext}${id}.${stamp}) --> $CCC_PBS_JOBSUB $TMPJOBCMDFILE ==> $JiD_cptred " >> $HOME/tmp/cccqsub_log fi fi (\rm -f ./.pbsqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) fi else if [ -s "./.pbsqsub_jid_${stampid}" ] ; then cat ./.pbsqsub_jid_${stampid} if [ "$qsublog" = 'yes' -o "$qsublog" = 'on' ] ; then JiD_cptred=`tail -1 ./.pbsqsub_jid_${stampid}` touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$PBS_JOBID on $HOSTID (${nqsprfx}${jobname}${nqsext}${id}.${stamp}) --> $CCC_PBS_JOBSUB $TMPJOBCMDFILE ==> $JiD_cptred " >> $HOME/tmp/cccqsub_log fi fi (\rm -f ./.pbsqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) fi else (\rm -f ./.pbsqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; cccqsub not able to continue...!" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; cccqsub not able to continue...!" >> $HOME/problem_log Exit=99 echo "QSUB - top of submitted file:" head -35 $TMPJOBCMDFILE exit $Exit fi fi #!!!!!! # echo "QSUB - top of submitted file:" # head -35 $TMPJOBCMDFILE #!!!!!! # Clean-up # ((sleep 600 ; rm -f $RMFILES ) < /dev/null > /dev/null 2>&1 &) # ((sleep 300 ; rm -f $RMFILES ) < /dev/null > /dev/null 2>&1 &) # ((sleep 30 ; rm -f $RMFILES ) < /dev/null > /dev/null 2>&1 &) # ((sleep 10 ; rm -f $RMFILES ) < /dev/null > /dev/null 2>&1 &) # sleep 5 # Disable under the assumption that cleanup will be done via crontab... # sleep 30 # \rm -f $RMFILES #???? elif [ "$Batch_mode" = 'BEjobst' ] ; then # set -x #!!!! # Back-end PBS pro case translate and submit using jobsub... #???? ## if [ type jobst >> /dev/null 2>>/dev/null ] ; then ## : ## else ## if [ -z "$SSMUSE_PLATFORMS" ] ; then ## . /fs/ssm/main/env/ssmuse-boot.sh ## fi ## ### Load the jobctl package to submit and monitor jobs ## . ssmuse-sh -p main/base/master/jobctl_0.2_all ## fi LOGNAME=${LOGNAME:=`whoami`} # if [ -d '/home/ords/*/*/'"${LOGNAME}/." ] ; then if [ -d /home/ords/*/*/${LOGNAME}/. ] ; then BEJOBST_spool_dir='/home/ords/crd/ccrn/scrd101/BEJOBST_spool_dir' else eval BEJOBST_spool_dir='/home/scrd101/BEJOBST_spool_dir' fi stamp="${HOSTID}_"`date +%Y%j%H%M%S` stampid="${stamp}_$$" USR=${USR:=`whoami`} eval TMPJOBCMDFILE="${BEJOBST_spool_dir}/bejobsttmpjbcmd_${USR}_${stamp}_$$.pbs" CKPTFILE="ckpt_file_${USR}_${stamp}_$$" RMFILES="$TMPJOBCMDFILE" # trap "(sleep 20 ; rm -f $TMPJOBCMDFILE) &" 0 # Set defaults... # Preemption=1 # target cluster where jobs will be subject to preemption # Preemption=0 # target cluster where jobs will not be subject to preemption # class: # JOBCLASS='workq' JOBCLASS='development' JOBMEMORY='' HJOBMEMORY='' if [ "$qName" = 'hare' ] ; then #if [ "$topdog" = 'yes' -a \( "$LOGNAME" = 'acrnmr5' -o "$LOGNAME" = 'acrnmr6' -o "$LOGNAME" = 'acrnpd6' -o "$LOGNAME" = 'acrnpd7' \) ] ; then # JOBCLASS='projects' #elif [ "$premtn" = 'yes' ] ; then # JOBCLASS='preemptable' if [ "$TOPDOG" = 'on' -o "$TOPDOG" = 'yes' ] ; then JOBCLASS='project' else # JOBCLASS='workq' JOBCLASS='development' fi #Preemption=1 Preemption=0 elif [ "$qName" = 'brooks' ] ; then #if [ "$topdog" = 'yes' -a \( "$LOGNAME" = 'acrnmr5' -o "$LOGNAME" = 'acrnmr6' -o "$LOGNAME" = 'acrnpd6' -o "$LOGNAME" = 'acrnpd7' \) ] ; then # JOBCLASS='projects' #elif [ "$premtn" = 'yes' ] ; then # JOBCLASS='preemptable' if [ "$TOPDOG" = 'on' -o "$TOPDOG" = 'yes' ] ; then JOBCLASS='project' else # JOBCLASS='workq' JOBCLASS='development' fi Preemption=1 #Preemption=0 elif [ "$qName" = 'banting' ] ; then #if [ "$topdog" = 'yes' -a \( "$LOGNAME" = 'acrnmr5' -o "$LOGNAME" = 'acrnmr6' -o "$LOGNAME" = 'acrnpd6' -o "$LOGNAME" = 'acrnpd7' \) ] ; then # JOBCLASS='projects' #elif [ "$premtn" = 'yes' ] ; then # JOBCLASS='preemptable' if [ "$TOPDOG" = 'on' -o "$TOPDOG" = 'yes' ] ; then JOBCLASS='project' else # JOBCLASS='workq' JOBCLASS='development' fi Preemption=1 #Preemption=0 elif [ "$qName" = 'daley' ] ; then #if [ "$topdog" = 'yes' -a \( "$LOGNAME" = 'acrnmr5' -o "$LOGNAME" = 'acrnmr6' -o "$LOGNAME" = 'acrnpd6' -o "$LOGNAME" = 'acrnpd7' \) ] ; then # JOBCLASS='projects' #elif [ "$premtn" = 'yes' ] ; then # JOBCLASS='preemptable' if [ "$TOPDOG" = 'on' -o "$TOPDOG" = 'yes' ] ; then JOBCLASS='project' else # JOBCLASS='workq' JOBCLASS='development' fi Preemption=1 #Preemption=0 elif [ "$qName" = 'development' ] ; then JOBCLASS='development' Preemption=1 elif [ "$qName" = 'xfer' -o "$qName" = 'bexfer' ] ; then JOBCLASS='xfer' Preemption=0 fi # job_type: JOBTYPE='serial' # ConsumableCpus: JOBPES='1' set -- $dirctv while [ $# -gt 0 ] do case "$1" in -c) # Ignore shift ;; -mu) # Ignore shift ;; -lu) # setup based on maximum number of cpu per process JOBTYPE='parallel' shift if [ "$openmp" = 'on' ] ; then JOBTYPEP='openmp' else JOBTYPEP='multi' fi JOBPES="$1" ;; -pe) # Allow setting based on SGE parallel environment switch. JOBTYPE='parallel' shift JOBTYPEP=${JOBTYPEP:="$1"} shift JOBPES="$1" ;; -q) # queue name shift # JOBCLASS=$1 ;; -e) # error log location shift JOBERRLOG=`echo $1 | sed -n -e 's/^.*://' -e '1,$p'` ;; -eo) # both logs in same file JOBEO=EO ;; -o) # output log location shift JOBLOG=`echo $1 | sed -n -e 's/^.*://' -e '1,$p'` ;; -r) # job identification shift # JOBNAME=`echo $1 | cut -c1-20` JOBNAME=$1 ;; -lT) # time limit shift # JOBTIME=$1 JOBTIME=${JOBTIME:=$1} ;; -lR) # time limit shift # JOBTIME=$1 JOBTIME=${JOBTIME:=$1} ;; -lM) # memory limit shift # JOBMEMORY=$1 JOBMEMORY=`echo $1 |tr '[a-z]' '[A-Z]' | sed -n -e 's/B//' -e '1,$p'` JOBMEMORY=${JOBMEMORY:='1000M'} ;; *) # script to execute if [ -f $1 ] ; then JOBSCRIPT=$1 else JOBRest="$JOBRest $1" fi ;; esac shift done HMaxMemoryPerNode='' HJOBMEMORY=${JOBMEMORY:-'3000M'} tUnIt=`echo $HJOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*M/M/p'` tUnIt=${tUnIt:=`echo $HJOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*G/G/p'`} if [ -n "$tUnIt" -a "$tUnIt" = 'M' ] ; then tJOBMEMORY=`echo ${HJOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/M//' -e '1,$p'` if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -lt 3000 ] ; then JOBMEMORY='3000M' fi if [ "$qName" = 'banting' -o "$qName" = 'daley' ] ; then if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 96000 ] ; then HMaxMemoryPerNode='192' else HMaxMemoryPerNode='96' fi else if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 51000 ] ; then HMaxMemoryPerNode='102' else HMaxMemoryPerNode='51' fi fi elif [ -n "$tUnIt" -a "$tUnIt" = 'G' ] ; then tJOBMEMORY=`echo ${HJOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/G//' -e '1,$p'` if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 51 ] ; then HMaxMemoryPerNode='102' else HMaxMemoryPerNode='51' fi fi HMaxMemoryPerNode=${HMaxMemoryPerNode:='51'} HJOBMEMORY="$HMaxMemoryPerNode"'G' tJOBMEMORY='' tUnIt='' # Ensure "Qnnode" is set and "JOBTYPE" is defined accordingly. # Enforce for Qnnode = 1 & nproc = 1,...,8 --> 'serial' # (ie. OpenMP on single node --> 'serial') # for "parallel": # nproc=1,2,4 or 8 (only) for MPI tasks. Qnnode=${Qnnode:=1} Qphys_node=${Qphys_node:=1} # if [ "$Qnnode" -gt 1 ] ; then if [ "$Qnnode" -gt 1 -o "$Qphys_node" -gt 1 ] ; then # MPI job JOBTYPE='parallel' Ntasks=${Qnnode:='1'} if [ "$qName" = 'brooks' ] ; then MaxMemoryPerNode=`expr $HMaxMemoryPerNode \* 1000` if [ "$premtn" = 'yes' ] ; then MaxMemoryPerNode=`expr 51 \* 1000` ; fi elif [ "$qName" = 'hare' ] ; then MaxMemoryPerNode=`expr $HMaxMemoryPerNode \* 1000` if [ "$premtn" = 'yes' ] ; then MaxMemoryPerNode=`expr 51 \* 1000` ; fi elif [ "$qName" = 'banting' ] ; then MaxMemoryPerNode=`expr $HMaxMemoryPerNode \* 1000` if [ "$premtn" = 'yes' ] ; then MaxMemoryPerNode=`expr 96 \* 1000` ; fi elif [ "$qName" = 'daley' ] ; then MaxMemoryPerNode=`expr $HMaxMemoryPerNode \* 1000` if [ "$premtn" = 'yes' ] ; then MaxMemoryPerNode=`expr 96 \* 1000` ; fi else MaxMemoryPerNode='15000' fi if [ -n "$Qphys_node" ] ; then Nodes="$Qphys_node" fi # -n "$Qphys_node" else JOBTYPE='serial' if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then if [ "$JOBPES" -gt 72 ] ; then JOBPES=72 fi fi if [ "$qName" = 'banting' -o "$qName" = 'daley' ] ; then if [ "$JOBPES" -gt 80 ] ; then JOBPES=80 fi fi fi # Set default time limit to 10 minutes... JOBTIME=${JOBTIME:='600'} # current pwd is job's pwd JOBPWD=$(pwd) # get user's home directory JOBHOMEDIR=$HOME # put output in .queue subdirectory if [ -n "$JHOME" -a "${JHOME}" != "${HOME}" -a -d "${JHOME}/.queue/." -a "$SITE_ID" = 'DrvlSC' ] ; then JOBQUEUEDIR="${JHOME}/.queue" else JOBQUEUEDIR=${JOBHOMEDIR}/.queue fi # use jobname from command line or make one up # JOBNAME=${JOBNAME-${JOBSCRIPT-stdin}} # JOBNAME=${JOBNAME-`basename ${JOBSCRIPT} | cut -c1-20`} JOBNAME=${JOBNAME-`basename ${JOBSCRIPT}`} JOBNAME=${JOBNAME-stdin} # If jobscript is not specified, then read from stdin. if [ -z "$JOBSCRIPT" ] ; then eval JOBSCRIPT="$BEJOBST_spool_dir/${JOBNAME}_pbsjbscrpt_${USR}_${stamp}_$$" RMFILES="$RMFILES $JOBSCRIPT" # trap "(sleep 20 ; rm -f $TMPJOBCMDFILE $JOBSCRIPT) &" 0 cat > $JOBSCRIPT if [ -s "$JOBSCRIPT" ] ; then chmod g+r $JOBSCRIPT : else [ -f $JOBSCRIPT ] && \rm -f $JOBSCRIPT || : touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed due to invalid/empty input script file!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed due to invalid/empty input script file!" exit -1 fi else if [ -s "$JOBSCRIPT" ] ; then JOBSCRIPTN="$BEJOBST_spool_dir/${JOBNAME}_pbsjbscrpt_${USR}_${stamp}_$$" \cp $JOBSCRIPT $JOBSCRIPTN JOBSCRIPT="$JOBSCRIPTN" RMFILES="$RMFILES $JOBSCRIPT" else touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed due to invalid/empty $JOBSCRIPT input script file!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed due to invalid/empty $JOBSCRIPT input script file!" exit -1 fi fi chmod u+x $JOBSCRIPT # setup output and error logs # if error and standard output to same file if [ "$JOBEO" = EO ] ; then # use standard output if it has been specified if [ -n "$JOBLOG" ] ; then JOBERRLOG=$JOBLOG # or use error output if it has been specified elif [ -n "$JOBERRLOG" ] ; then JOBLOG=$JOBERRLOG # or use defaults else JOBLOG=${JOBLOG-${JOBQUEUEDIR}/${JOBNAME}'.az.$(Cluster).$(Process).out'} JOBERRLOG=$JOBLOG fi else # otherwise make sure that defaults are set if [ -z "$JOBLOG" ] ; then JOBLOG=${JOBQUEUEDIR}/${JOBNAME}'.az.$(Cluster).$(Process).out' fi if [ -z "$JOBERRLOG" ] ; then JOBERRLOG=${JOBQUEUEDIR}/${JOBNAME}'.az.$(Cluster).$(Process).err' fi fi # if [ "$JOBCLASS" = 'projects' -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 ] ; then if [ \( "$JOBCLASS" = 'projects' -o "$JOBCLASS" = 'preemptable' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 ] ; then # JOBCLASS='workq' JOBCLASS='development' fi # Toggle these 2 lines to switch the default for serial jobs from "development" (1st line active) # to "xfer_dev" (2nd line active) # if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$bexfer" = 'on' ] ; then # if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$bexfer" != 'off' ] ; then # if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' -o "$JOBCLASS" = 'project' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$bexfer" != 'off' ] ; then # if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' -o "$qName" = 'banting' -o "$qName" = 'daley' -o "$JOBCLASS" = 'project' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$bexfer" != 'on' ] ; then if [ \( "$qName" = 'hare' -o "$qName" = 'brooks' -o "$qName" = 'banting' -o "$qName" = 'daley' -o "$JOBCLASS" = 'project' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$bexfer" != 'off' ] ; then #JOBTIME=2700 JOBTIME=${JOBTIME:-1800} if [ "$JOBTIME" -lt 1800 ] ; then JOBTIME=1800 ; fi #JOBCLASS='dev_xfer' #JOBCLASS='cfs_xfer' JOBCLASS='xfer' # Enable later on #JOBCLASS='development' #123: Temporary workaround to use development until xfer is linked to xc4gpfs... instead of current xc4login... elif [ "$qName" = 'bexfer' -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 ] ; then #JOBTIME=2700 JOBTIME=${JOBTIME:-1800} if [ "$JOBTIME" -lt 1800 ] ; then JOBTIME=1800 ; fi JOBCLASS='xfer' elif [ \( "$qName" = 'hare' -o "$qName" = 'brooks' -o "$qName" = 'banting' -o "$qName" = 'daley' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$JOBTIME" -lt 1800 ] ; then JOBTIME=1800 fi # create loadl command file if [ "$CCCSHELL" = '/bin/bash' ] ; then QSUBBSHELL="$CCCSHELL" else QSUBBSHELL='/bin/ksh' fi if [ "$JOBCLASS" = 'project' -a "$JOBTYPE" = 'parallel' -a \( "$qName" = 'hare' -o "$qName" = 'brooks' -o "$qName" = 'banting' -o "$qName" = 'daley' \) ] ; then Nodes_spoken_for=`qstat -f | egrep 'queue = |Resource_List.nodect =' | $AWK ' BEGIN { nodes=0 ; } { if ( $3 == "prj_crd_cmip6" ) { getline ; nodes=nodes+$3 ; } ; } END { print nodes ;} '` Nodes_spoken_for=${Nodes_spoken_for:=0} if [ -n "$PBS_JOBID" ] ; then Nodes_current_job=`qstat -f $PBS_JOBID | egrep 'queue = |Resource_List.nodect =' | $AWK ' BEGIN { nodes=0 ; } { if ( $3 == "prj_crd_cmip6" ) { getline ; nodes=nodes+$3 ; } ; } END { print nodes ;} '` Nodes_current_job=${Nodes_current_job:=0} Nodes_spoken_for=`expr $Nodes_spoken_for - $Nodes_current_job` fi Nodes_remaining=`expr 104 - $Nodes_spoken_for` if [ $Nodes -gt $Nodes_remaining ] ; then JOBCLASS='development' echo "QSUB - JOBCLASS switched to -development- since -project- Nodes_spoken_for=${Nodes_spoken_for}, Nodes_remaining=${Nodes_remaining} < Nodes_needed=${Nodes}" fi fi ##123 # set -x touch $TMPJOBCMDFILE echo "#!${QSUBBSHELL} -l #PBS -S ${QSUBBSHELL} #PBS -N $JOBNAME #PBS -q $JOBCLASS " >> $TMPJOBCMDFILE if [ -n "$JOBLOG" -a -n "$JOBERRLOG" -a "$JOBLOG" = "$JOBERRLOG" ] ; then echo "#PBS -j oe" >> $TMPJOBCMDFILE echo "#PBS -o $JOBLOG" >> $TMPJOBCMDFILE else [ -n "$JOBLOG" ] && echo "#PBS -o $JOBLOG" >> $TMPJOBCMDFILE || : [ -n "$JOBERRLOG" ] && echo "#PBS -e $JOBERRLOG" >> $TMPJOBCMDFILE || : fi chmod g+r $TMPJOBCMDFILE unset QSUBBSHELL if [ -n "$Qdelaymin" ] ; then TZ=UTC0; export TZ if [ "$Qdelaymin" -gt 0 ] ; then DatTimDlyx=`lldatimp ${Qdelaymin}` # echo "# @ startdate = $DatTimDlyx" >> $TMPJOBCMDFILE echo "#PBS -a $DatTimDlyx" >> $TMPJOBCMDFILE fi fi ### # add limits to command file if [ -n "$JOBTIME" ] ; then JOBTIME=`echo ${JOBTIME} | $AWK '{ print strftime("%H:%M:%S", $1,1) ; }'` echo "#PBS -l walltime=${JOBTIME}" >> $TMPJOBCMDFILE fi if [ "$JOBTYPE" = 'parallel' ] ; then #echo "#PBS -l select=$Nodes:vntype=cray_compute,place=scatter " >> $TMPJOBCMDFILE if [ "$qName" = 'banting' -o "$qName" = 'daley' ] ; then echo "#PBS -l select=$Nodes:ncpus=40:vntype=cray_compute,place=scatter " >> $TMPJOBCMDFILE else echo "#PBS -l select=$Nodes:ncpus=36:vntype=cray_compute,place=scatter " >> $TMPJOBCMDFILE fi else if [ "$JOBCLASS" = 'xfer' ] ; then HMaxMemoryPerNode='' JOBMEMORY=${JOBMEMORY:-'3000M'} HJOBMEMORY=${JOBMEMORY:-'3000M'} tUnIt=`echo $HJOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*M/M/p'` tUnIt=${tUnIt:=`echo $HJOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*G/G/p'`} if [ -n "$tUnIt" -a "$tUnIt" = 'M' ] ; then tJOBMEMORY=`echo ${HJOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/M//' -e '1,$p'` if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -lt 3000 ] ; then HJOBMEMORY='3000M' fi if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 8000 ] ; then HMaxMemoryPerNode='8' fi elif [ -n "$tUnIt" -a "$tUnIt" = 'G' ] ; then tJOBMEMORY=`echo ${HJOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/G//' -e '1,$p'` if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -lt 3 ] ; then HJOBMEMORY='3G' fi if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 8 ] ; then HMaxMemoryPerNode='8' fi fi if [ -n "$HMaxMemoryPerNode" ] ; then HJOBMEMORY="$HMaxMemoryPerNode"'G' fi #echo "#PBS -l select=1:ncpus=1:mem=3000M:vntype=cray_transfer:arch=linux" >> $TMPJOBCMDFILE #echo "#PBS -l select=1:ncpus=1:mem=${JOBMEMORY}:vntype=cray_transfer:arch=linux" >> $TMPJOBCMDFILE # Hard-code to the 3000MB enforced maximum limit. HJOBMEMORY="3000M" echo "#PBS -l select=1:ncpus=1:mem=${HJOBMEMORY}:vntype=cray_transfer:arch=linux" >> $TMPJOBCMDFILE else #echo "#PBS -l select=1:vntype=cray_compute:ncpus=1:mem=3000M" >> $TMPJOBCMDFILE #echo "#PBS -l select=1:vntype=cray_compute:ncpus=1" >> $TMPJOBCMDFILE if [ "$qName" = 'banting' -o "$qName" = 'daley' ] ; then echo "#PBS -l select=1:vntype=cray_compute:ncpus=40" >> $TMPJOBCMDFILE else echo "#PBS -l select=1:vntype=cray_compute:ncpus=36" >> $TMPJOBCMDFILE fi #echo "#PBS -l select=1:ncpus=1:mem=3000M" >> $TMPJOBCMDFILE fi fi echo 'export CCCMA_LL_RMFILES="'"$RMFILES"'"' >> $TMPJOBCMDFILE echo "if [ -s $CUSTOM_ENV_FILE ] ; then . $CUSTOM_ENV_FILE || : ; "'else if [ -z "$SYSTEM_SETUP" -a -z "$CCRNSRC" -a -d /home/scrd101/generic/. ] ; then [ -s /home/scrd101/generic/sc_cccma_setup_profile ] && . /home/scrd101/generic/sc_cccma_setup_profile || : ; fi ; fi ' >> $TMPJOBCMDFILE if [ -s "$JOBSCRIPT" ] ; then if [ "$JOBTYPE" = 'parallel' ] ; then echo "$JOBSCRIPT" >> $TMPJOBCMDFILE else #echo "aprun -n1 -cc none $JOBSCRIPT" >> $TMPJOBCMDFILE echo "$JOBSCRIPT" >> $TMPJOBCMDFILE fi fi #!!!!!! if [ -s $TMPJOBCMDFILE ] ; then : # ls -ld $TMPJOBCMDFILE # cat $TMPJOBCMDFILE else echo "`date`; `whoami`;$HOSTID: Can not proceed due to invalid $TMPJOBCMDFILE !" touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Can not proceed due to invalid $TMPJOBCMDFILE !" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; `whoami`;$HOSTID: Can not proceed due to invalid $TMPJOBCMDFILE !" >> $HOME/problem_log exit 1 fi echo "QSUB - top of submitted file:" head -35 $TMPJOBCMDFILE echo '------------' echo "QSUB - associated files with the submitted job:" \ls -ld $RMFILES echo '------------' #!!!!!! # submit the job # if [ "$OS" = 'AIX' ] ; then # if [ "$qName" = 'hare' -o "$qName" = 'brooks' ] ; then HOSTIDf6=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | cut -c 1-6` HOSTIDfm=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | sed -e 's/xc2mom.*-p2/xc2mom-p2/'` # set -x #123 if [ "$qName" = 'hare' -o "$qName" = 'brooks' -o "$qName" = 'banting' -o "$qName" = 'daley' -o "$qName" = 'xfer' -o "$qName" = 'bexfer' ] ; then # CCC_PBS_JOBSUB=${CCC_PBS_JOBSUB:-'qsub'} # if [ "$qName" = 'hare' ] ; then if [ "$qNamex" = 'hare' -o "$qName" = 'hare' -o \( \( "$qName" = 'xfer' -o "$qName" = 'bexfer' \) -a \( "$HOSTIDf" = 'pp1' -o "$HOSTIDf" = 'xc1' -o "$HOSTIDf" = 'cs1' \) \) ] ; then CCC_PBS_JOBSUB='jobsub -c hare' elif [ "$qNamex" = 'banting' -o "$qName" = 'banting' -o \( \( "$qName" = 'xfer' -o "$qName" = 'bexfer' \) -a \( "$HOSTIDf" = 'pp3' -o "$HOSTIDf" = 'xc3' -o "$HOSTIDf" = 'cs3' \) \) ] ; then CCC_PBS_JOBSUB='jobsub -c banting' elif [ "$qNamex" = 'daley' -o "$qName" = 'daley' -o \( \( "$qName" = 'xfer' -o "$qName" = 'bexfer' \) -a \( "$HOSTIDf" = 'pp4' -o "$HOSTIDf" = 'xc4' -o "$HOSTIDf" = 'cs4' \) \) ] ; then # CCC_PBS_JOBSUB='jobsub -c daley' CCC_PBS_JOBSUB='jobsub -c daley.science.gc.ca' # CCC_PBS_JOBSUB='jobsub -c daley-broadwell.science.gc.ca' if [ -z "$ORDENV_SITE_PROFILE" ] ; then export SSMUSE_VERBOSE=1 export ORDENV_SITE_PROFILE=20191220 # export ORDENV_COMM_PROFILE=eccc/20190617-test . /fs/ssm/main/env/ordenv-boot-20200204.sh >> /dev/null 2>> /dev/null fi # if [ -z "$SSMUSE_PLATFORMS" ] ; then # . /fs/ssm/main/env/ssmuse-boot.sh # fi ### Load the jobctl package to submit and monitor jobs # . ssmuse-sh -p main/base/master/jobctl_0.2_all # elif [ "$qNamex" = 'brooks' -o "$qName" = 'brooks' -o \( \( "$qName" = 'xfer' -o "$qName" = 'bexfer' \) -a \( "$HOSTIDf6" = 'xc2net' \) \) ] ; then elif [ "$qNamexT" = 'brooks-test' -o \( \( "$qNamex" = 'brooks' -o "$qName" = 'brooks' -o "$qName" = 'xfer' -o "$qName" = 'bexfer' \) -a \( "$HOSTIDf6" = 'xc2net' -o "$HOSTIDfm" = 'xc2mom-p2' \) \) ] ; then CCC_PBS_JOBSUB='jobsub -c brooks-test' elif [ "$qNamex" = 'brooks' -o "$qName" = 'brooks' -o \( \( "$qName" = 'xfer' -o "$qName" = 'bexfer' \) -a \( "$HOSTIDf" = 'pp2' -o "$HOSTIDf" = 'xc2' -o "$HOSTIDf" = 'cs2' \) \) ] ; then CCC_PBS_JOBSUB='jobsub -c brooks' else CCC_PBS_JOBSUB='jobsub -c brooks' echo '------------' ; echo "QSUB - Due to parsing limitation, go with the default of CCC_PBS_JOBSUB=$CCC_PBS_JOBSUB" ; echo '------------' fi [ -s "./.jbstqsub_jid_${stampid}" ] && (\rm -f ./.jbstqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) && sleep 1 || : touch ./.jbstqsub_jid_${stampid} ; sleep 1 echo "QSUB - $CCC_PBS_JOBSUB $TMPJOBCMDFILE" ; echo '------------' Cwd=`pwd` $CCC_PBS_JOBSUB $TMPJOBCMDFILE >> ./.jbstqsub_jid_${stampid} && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 1st failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 1st failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/problem_log sleep 115 (\rm -f ./.jbstqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) sleep 1 touch ./.jbstqsub_jid_${stampid} ; sleep 1 $CCC_PBS_JOBSUB $TMPJOBCMDFILE >> ./.jbstqsub_jid_${stampid} && Exit=0 || Exit="$?" # ssh $HDNODE "cd $Cwd; $CCC_PBS_JOBSUB $TMPJOBCMDFILE >> ./.jbstqsub_jid_${stampid}" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 2nd failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 2nd failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/problem_log echo "QSUB - top of submitted file:" head -35 $TMPJOBCMDFILE (\rm -f ./.jbstqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) exit $Exit else sleep 1 if [ -s "./.jbstqsub_jid_${stampid}" ] ; then cat ./.jbstqsub_jid_${stampid} if [ "$qsublog" = 'yes' -o "$qsublog" = 'on' ] ; then JiD_cptred=`tail -1 ./.jbstqsub_jid_${stampid}` touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$PBS_JOBID on $HOSTID (${nqsprfx}${jobname}${nqsext}${id}.${stamp}) --> $CCC_PBS_JOBSUB $TMPJOBCMDFILE ==> $JiD_cptred " >> $HOME/tmp/cccqsub_log fi fi (\rm -f ./.jbstqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) fi else sleep 1 if [ -s "./.jbstqsub_jid_${stampid}" ] ; then cat ./.jbstqsub_jid_${stampid} if [ "$qsublog" = 'yes' -o "$qsublog" = 'on' ] ; then JiD_cptred=`tail -1 ./.jbstqsub_jid_${stampid}` touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$PBS_JOBID on $HOSTID (${nqsprfx}${jobname}${nqsext}${id}.${stamp}) --> $CCC_PBS_JOBSUB $TMPJOBCMDFILE ==> $JiD_cptred " >> $HOME/tmp/cccqsub_log fi fi (\rm -f ./.jbstqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) fi fi elif [ "$Batch_mode" = 'PPPjobst' ] ; then # set -x #!!!! if [ type jobst >> /dev/null 2>>/dev/null ] ; then : else ## if [ -z "$SSMUSE_PLATFORMS" ] ; then ## . /fs/ssm/main/env/ssmuse-boot.sh ## fi ### Load the jobctl package to submit and monitor jobs ## . ssmuse-sh -p main/base/master/jobctl_0.2_all ## # . ssmuse-sh -p main/opt/jobctl/jobctl-0.6.1 if [ -z "$ORDENV_SITE_PROFILE" ] ; then # export SSMUSE_VERBOSE=1 export ORDENV_SITE_PROFILE=20191220 # export ORDENV_COMM_PROFILE=eccc/20190617-test # export ORDENV_GROUP_PROFILE=eccc/cmc/1.7-beta.2 # . /fs/ssm/main/env/ordenv-boot-20190617-test.sh . /fs/ssm/main/env/ordenv-boot-20200204.sh >> /dev/null 2>> /dev/null fi fi # PPP jobst case translate and submit... #???? LOGNAME=${LOGNAME:=`whoami`} # if [ -d '/home/ords/*/*/'"${LOGNAME}/." ] ; then if [ -d /home/ords/*/*/${LOGNAME}/. ] ; then PPPjobst_spool_dir='/home/ords/crd/ccrn/scrd101/PPPjobst_spool_dir' else eval PPPjobst_spool_dir='/home/scrd101/PPPjobst_spool_dir' fi # stamp="${HOSTID}_"`date +%Y%j%H%M%S` # stamp="${qName}_"`date +%Y%j%H%M%S` stamp="${HOSTID}_trgt_${qName}_"`date +%Y%j%H%M%S` stampid="${stamp}_$$" USR=${USR:=`whoami`} TMPJOBCMDFILE="${PPPjobst_spool_dir}/pppjbsttmpjbcmd_${USR}_${stamp}_$$.jbst" CKPTFILE="ckpt_file_${USR}_${stamp}_$$" RMFILES="$TMPJOBCMDFILE" # trap "(sleep 20 ; rm -f $TMPJOBCMDFILE) &" 0 # Set defaults... # Preemption=1 # target cluster where jobs will be subject to preemption # Preemption=0 # target cluster where jobs will not be subject to preemption # class: # JOBCLASS='workq' JOBCLASS='development' JOBMEMORY='' HJOBMEMORY='' if [ "$qName" = 'ppp1' ] ; then if [ "$topdog" = 'yes' -a \( "$LOGNAME" = 'acrnmr5' -o "$LOGNAME" = 'acrnmr6' -o "$LOGNAME" = 'acrnpd6' -o "$LOGNAME" = 'acrnpd7' \) ] ; then JOBCLASS='projects' elif [ "$premtn" = 'yes' ] ; then JOBCLASS='preemptable' else # JOBCLASS='workq' JOBCLASS='development' fi #Preemption=1 Preemption=0 elif [ "$qName" = 'ppp3' ] ; then if [ "$topdog" = 'yes' -a \( "$LOGNAME" = 'acrnmr5' -o "$LOGNAME" = 'acrnmr6' -o "$LOGNAME" = 'acrnpd6' -o "$LOGNAME" = 'acrnpd7' \) ] ; then JOBCLASS='projects' elif [ "$premtn" = 'yes' ] ; then JOBCLASS='preemptable' else # JOBCLASS='workq' JOBCLASS='development' fi #Preemption=1 Preemption=0 elif [ "$qName" = 'ppp4' ] ; then if [ "$topdog" = 'yes' -a \( "$LOGNAME" = 'acrnmr5' -o "$LOGNAME" = 'acrnmr6' -o "$LOGNAME" = 'acrnpd6' -o "$LOGNAME" = 'acrnpd7' \) ] ; then JOBCLASS='projects' elif [ "$premtn" = 'yes' ] ; then JOBCLASS='preemptable' else # JOBCLASS='workq' JOBCLASS='development' fi #Preemption=1 Preemption=0 elif [ "$qName" = 'ppp2' ] ; then if [ "$topdog" = 'yes' -a \( "$LOGNAME" = 'acrnmr5' -o "$LOGNAME" = 'acrnmr6' -o "$LOGNAME" = 'acrnpd6' -o "$LOGNAME" = 'acrnpd7' \) ] ; then JOBCLASS='projects' elif [ "$premtn" = 'yes' ] ; then JOBCLASS='preemptable' else # JOBCLASS='workq' JOBCLASS='development' fi Preemption=1 #Preemption=0 fi # job_type: JOBTYPE='serial' # ConsumableCpus: JOBPES='1' set -- $dirctv while [ $# -gt 0 ] do case "$1" in -c) # Ignore shift ;; -mu) # Ignore shift ;; -lu) # setup based on maximum number of cpu per process JOBTYPE='parallel' shift if [ "$openmp" = 'on' ] ; then JOBTYPEP='openmp' else JOBTYPEP='multi' fi JOBPES="$1" ;; -pe) # Allow setting based on SGE parallel environment switch. JOBTYPE='parallel' shift JOBTYPEP=${JOBTYPEP:="$1"} shift JOBPES="$1" ;; -q) # queue name shift # JOBCLASS=$1 ;; -e) # error log location shift JOBERRLOG=`echo $1 | sed -n -e 's/^.*://' -e '1,$p'` ;; -eo) # both logs in same file JOBEO=EO ;; -o) # output log location shift JOBLOG=`echo $1 | sed -n -e 's/^.*://' -e '1,$p'` ;; -r) # job identification shift # JOBNAME=`echo $1 | cut -c1-20` JOBNAME=$1 ;; -lT) # time limit shift # JOBTIME=$1 JOBTIME=${JOBTIME:=$1} ;; -lR) # time limit shift # JOBTIME=$1 JOBTIME=${JOBTIME:=$1} ;; -lM) # memory limit shift # JOBMEMORY=$1 JOBMEMORY=`echo $1 |tr '[a-z]' '[A-Z]' | sed -n -e 's/B//' -e '1,$p'` JOBMEMORY=${JOBMEMORY:='1000M'} ;; *) # script to execute if [ -f $1 ] ; then JOBSCRIPT=$1 else JOBRest="$JOBRest $1" fi ;; esac shift done HMaxMemoryPerNode='' HJOBMEMORY=${JOBMEMORY:-'3000M'} tUnIt=`echo $HJOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*M/M/p'` tUnIt=${tUnIt:=`echo $HJOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*G/G/p'`} if [ -n "$tUnIt" -a "$tUnIt" = 'M' ] ; then tJOBMEMORY=`echo ${HJOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/M//' -e '1,$p'` if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -lt 3000 ] ; then JOBMEMORY='3000M' fi if [ "$qName" = 'ppp3' -o "$qName" = 'ppp4' ] ; then if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 96000 ] ; then HMaxMemoryPerNode='192' else HMaxMemoryPerNode='96' fi else if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 81000 ] ; then HMaxMemoryPerNode='202' else HMaxMemoryPerNode='81' fi fi elif [ -n "$tUnIt" -a "$tUnIt" = 'G' ] ; then tJOBMEMORY=`echo ${HJOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/G//' -e '1,$p'` if [ "$qName" = 'ppp3' -o "$qName" = 'ppp4' ] ; then if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 96 ] ; then HMaxMemoryPerNode='192' else HMaxMemoryPerNode='96' fi else if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -gt 81 ] ; then HMaxMemoryPerNode='202' else HMaxMemoryPerNode='81' fi fi fi HMaxMemoryPerNode=${HMaxMemoryPerNode:='81'} HJOBMEMORY="$HMaxMemoryPerNode"'G' tJOBMEMORY='' tUnIt='' # Ensure "Qnnode" is set and "JOBTYPE" is defined accordingly. # Enforce for Qnnode = 1 & nproc = 1,...,8 --> 'serial' # (ie. OpenMP on single node --> 'serial') # for "parallel": # nproc=1,2,4 or 8 (only) for MPI tasks. Qnnode=${Qnnode:=1} Qphys_node=${Qphys_node:=1} if [ "$Qnnode" -gt 1 -o "$Qphys_node" -gt 1 ] ; then # MPI job JOBTYPE='parallel' Ntasks=${Qnnode:='1'} if [ "$qName" = 'ppp4' ] ; then MaxMemoryPerNode=`expr $HMaxMemoryPerNode \* 1000` if [ "$premtn" = 'yes' ] ; then MaxMemoryPerNode=`expr 96 \* 1000` ; fi elif [ "$qName" = 'ppp3' ] ; then MaxMemoryPerNode=`expr $HMaxMemoryPerNode \* 1000` if [ "$premtn" = 'yes' ] ; then MaxMemoryPerNode=`expr 96 \* 1000` ; fi elif [ "$qName" = 'ppp2' ] ; then MaxMemoryPerNode=`expr $HMaxMemoryPerNode \* 1000` if [ "$premtn" = 'yes' ] ; then MaxMemoryPerNode=`expr 81 \* 1000` ; fi elif [ "$qName" = 'ppp1' ] ; then MaxMemoryPerNode=`expr $HMaxMemoryPerNode \* 1000` if [ "$premtn" = 'yes' ] ; then MaxMemoryPerNode=`expr 81 \* 1000` ; fi else MaxMemoryPerNode='15000' fi if [ -n "$Qphys_node" ] ; then Nodes="$Qphys_node" fi else # if [ "$JOBPES" -eq 1 ] ; then JOBTYPE='serial' if [ "$qName" = 'ppp1' -o "$qName" = 'ppp2' ] ; then if [ "$JOBPES" -gt 44 ] ; then JOBPES=44 fi elif [ "$qName" = 'ppp3' -o "$qName" = 'ppp4' ] ; then if [ "$JOBPES" -gt 40 ] ; then JOBPES=40 fi #else # if [ "$JOBPES" -gt 8 ] ; then # JOBPES=8 # fi fi # else # # OpenMP case on single node # JOBPES=8 # fi fi # Set default time limit to 10 minutes... JOBTIME=${JOBTIME:='600'} # current pwd is job's pwd JOBPWD=$(pwd) # get user's home directory JOBHOMEDIR=$HOME # put output in .queue subdirectory if [ -n "$JHOME" -a "${JHOME}" != "${HOME}" -a -d "${JHOME}/.queue/." -a "$SITE_ID" = 'DrvlSC' ] ; then JOBQUEUEDIR="${JHOME}/.queue" else JOBQUEUEDIR=${JOBHOMEDIR}/.queue fi # use jobname from command line or make one up # JOBNAME=${JOBNAME-${JOBSCRIPT-stdin}} # JOBNAME=${JOBNAME-`basename ${JOBSCRIPT} | cut -c1-20`} JOBNAME=${JOBNAME-`basename ${JOBSCRIPT}`} JOBNAME=${JOBNAME-stdin} # If jobscript is not specified, then read from stdin. if [ -z "$JOBSCRIPT" ] ; then JOBSCRIPT="$PPPjobst_spool_dir/${JOBNAME}_jbstjbscrpt_${USR}_${stamp}_$$" RMFILES="$RMFILES $JOBSCRIPT" # trap "(sleep 20 ; rm -f $TMPJOBCMDFILE $JOBSCRIPT) &" 0 cat > $JOBSCRIPT if [ -s "$JOBSCRIPT" ] ; then chmod g+r $JOBSCRIPT : else [ -f $JOBSCRIPT ] && \rm -f $JOBSCRIPT || : touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed due to invalid/empty input script file!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed due to invalid/empty input script file!" exit -1 fi else if [ -s "$JOBSCRIPT" ] ; then JOBSCRIPTN="$PPPjobst_spool_dir/${JOBNAME}_jbstjbscrpt_${USR}_${stamp}_$$" \cp $JOBSCRIPT $JOBSCRIPTN JOBSCRIPT="$JOBSCRIPTN" RMFILES="$RMFILES $JOBSCRIPT" else touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed due to invalid/empty $JOBSCRIPT input script file!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed due to invalid/empty $JOBSCRIPT input script file!" exit -1 fi fi chmod u+x $JOBSCRIPT # setup output and error logs # if error and standard output to same file if [ "$JOBEO" = EO ] ; then # use standard output if it has been specified if [ -n "$JOBLOG" ] ; then JOBERRLOG=$JOBLOG # or use error output if it has been specified elif [ -n "$JOBERRLOG" ] ; then JOBLOG=$JOBERRLOG # or use defaults else JOBLOG=${JOBLOG-${JOBQUEUEDIR}/${JOBNAME}'.az.$(Cluster).$(Process).out'} JOBERRLOG=$JOBLOG fi else # otherwise make sure that defaults are set if [ -z "$JOBLOG" ] ; then JOBLOG=${JOBQUEUEDIR}/${JOBNAME}'.az.$(Cluster).$(Process).out' fi if [ -z "$JOBERRLOG" ] ; then JOBERRLOG=${JOBQUEUEDIR}/${JOBNAME}'.az.$(Cluster).$(Process).err' fi fi # if [ "$JOBCLASS" = 'projects' -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 ] ; then if [ \( "$JOBCLASS" = 'projects' -o "$JOBCLASS" = 'preemptable' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 ] ; then # JOBCLASS='workq' JOBCLASS='development' fi # Toggle these 2 lines to switch the default for serial jobs from "development" (1st line active) # to "xxfer" (2nd line active) if [ \( "$qName" = 'ppp1' -o "$qName" = 'ppp2' -o "$qName" = 'ppp3' -o "$qName" = 'ppp4' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$fexfer" = 'on' ] ; then # if [ \( "$qName" = 'ppp1' -o "$qName" = 'ppp2' -o "$qName" = 'ppp3' -o "$qName" = 'ppp4' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$fexfer" != 'off' ] ; then #JOBTIME=2700 JOBTIME=${JOBTIME:-3600} if [ "$JOBTIME" -lt 3600 ] ; then JOBTIME=3600 ; fi #JOBCLASS='dev_xfer' # JOBCLASS='cfs_xfer' # JOBCLASS='xfer' # JOBCLASS='development' # JOBCLASS='xxfer' TDUMP=`echo $JOBNAME | tr '[A-Z]' '[a-z]' | sed -n -e 's/^.*tdumper.*$/yes/p'` TDUMP=${TDUMP:-'no'} if [ "$TDUMP" = 'yes' ] ; then JOBCLASS='archive' else # JOBCLASS='xxfer' JOBCLASS='dev_xxfer' fi # JOBMEMORY='3000M' JOBMEMORY='2048M' elif [ "$qName" = 'fexfer' -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 ] ; then #JOBCLASS='xfer' #JOBCLASS='development' #JOBCLASS='xxfer' TDUMP=`echo $JOBNAME | tr '[A-Z]' '[a-z]' | sed -n -e 's/^.*tdumper.*$/yes/p'` TDUMP=${TDUMP:-'no'} if [ "$TDUMP" = 'yes' ] ; then JOBCLASS='archive' else # JOBCLASS='xxfer' JOBCLASS='dev_xxfer' fi #JOBTIME=1800 JOBTIME=${JOBTIME:-3600} if [ "$JOBTIME" -lt 3600 ] ; then JOBTIME=3600 ; fi #JOBMEMORY='3000M' JOBMEMORY='2048M' JOBTYPE='serial' JOBPES=1 elif [ \( "$qName" = 'ppp1' -o "$qName" = 'ppp2' -o "$qName" = 'ppp3' -o "$qName" = 'ppp4' \) -a "$JOBTYPE" = 'serial' -a "$JOBPES" -eq 1 -a "$JOBTIME" -lt 1800 ] ; then JOBTIME=1800 fi # create loadl command file if [ "$CCCSHELL" = '/bin/bash' ] ; then QSUBBSHELL="$CCCSHELL" else QSUBBSHELL='/bin/ksh' fi ### cat > $TMPJOBCMDFILE << EOF #!${QSUBBSHELL} # @ job_type = $JOBTYPE # @ job_name = $JOBNAME # @ executable = $JOBSCRIPT # @ initialdir = /tmp # @ input = /dev/null # @ output = $JOBLOG # @ error = $JOBERRLOG # @ class = $JOBCLASS # EOF ## TRY WITHOUT FOR NOW ##echo "#!${QSUBBSHELL} -l ## TRY WITHOUT FOR NOW ###PBS -S ${QSUBBSHELL} -V ## TRY WITHOUT FOR NOW ###PBS -N $JOBNAME ## TRY WITHOUT FOR NOW ####PBS -q $JOBCLASS " > $TMPJOBCMDFILE echo "#!${QSUBBSHELL} -l #PBS -S ${QSUBBSHELL} #PBS -N $JOBNAME #PBS -q $JOBCLASS " > $TMPJOBCMDFILE if [ "$qName" = 'ppp1' ] ; then Rtmchn='cs1fe01' elif [ "$qName" = 'ppp2' ] ; then Rtmchn='cs2fe01' elif [ "$qName" = 'ppp3' ] ; then Rtmchn='eccc-ppp3' elif [ "$qName" = 'ppp4' ] ; then Rtmchn='eccc-ppp4' else Rtmchn='eccc-ppp1' fi if [ -n "$JOBLOG" -a -n "$JOBERRLOG" -a "$JOBLOG" = "$JOBERRLOG" ] ; then echo "#PBS -j oe" >> $TMPJOBCMDFILE echo "#PBS -o $JOBLOG" >> $TMPJOBCMDFILE # echo "#PBS -o $Rtmchn:$JOBLOG" >> $TMPJOBCMDFILE else [ -n "$JOBLOG" ] && echo "#PBS -o $JOBLOG" >> $TMPJOBCMDFILE || : [ -n "$JOBERRLOG" ] && echo "#PBS -e $JOBERRLOG" >> $TMPJOBCMDFILE || : # [ -n "$JOBLOG" ] && echo "#PBS -o $Rtmchn:$JOBLOG" >> $TMPJOBCMDFILE || : # [ -n "$JOBERRLOG" ] && echo "#PBS -e $Rtmchn:$JOBERRLOG" >> $TMPJOBCMDFILE || : fi ### Email on abort echo "#PBS -m a" >> $TMPJOBCMDFILE ### Email disabled # echo "#PBS -m n" >> $TMPJOBCMDFILE chmod g+r $TMPJOBCMDFILE unset QSUBBSHELL if [ -n "$Qdelaymin" ] ; then TZ=UTC0; export TZ if [ "$Qdelaymin" -gt 0 ] ; then DatTimDlyx=`lldatimp ${Qdelaymin}` # echo "# @ startdate = $DatTimDlyx" >> $TMPJOBCMDFILE echo "#PBS -a $DatTimDlyx" >> $TMPJOBCMDFILE fi fi if [ -n "$JOBTIME" ] ; then JOBTIME=`echo ${JOBTIME} | $AWK '{ print strftime("%H:%M:%S", $1,1) ; }'` echo "#PBS -l walltime=${JOBTIME}" >> $TMPJOBCMDFILE fi if [ "$JOBTYPE" = 'parallel' ] ; then # TMPFS=${TMPFS:-'50000'} TMPFS=${TMPFS:-'10000'} if [ "$qName" = 'ppp3' -o "$qName" = 'ppp4' ] ; then JOBPES=${JOBPES:-'40'} # echo "#PBS -l select=$Nodes:mem=112G:res_image=ppp_eccc_all_default_ubuntu-14.04-amd64_latest" >> $TMPJOBCMDFILE # echo "#PBS -l select=$Nodes:mem=5G:res_image=ppp_eccc_all_default_ubuntu-18.04-amd64_latest" >> $TMPJOBCMDFILE echo "#PBS -l select=$Nodes:mem=5G:res_image=eccc/eccc_all_ppp_ubuntu-18.04-amd64_latest" >> $TMPJOBCMDFILE echo "#PBS -l place=free" >> $TMPJOBCMDFILE ### The following 2 lines added for MPI based on migration documentation echo 'export OMPI_MCA_orte_tmpdir_base=/run/shm' >> $TMPJOBCMDFILE echo 'export OMPI_MCA_btl_openib_if_include=mlx5_0' >> $TMPJOBCMDFILE else JOBPES=${JOBPES:-'44'} # echo "#PBS -l select=$Nodes:ncpus=2:mem=3G:res_tmpfs=${TMPFS}:res_image=ppp_eccc_all_default_ubuntu-14.04-amd64_latest" >> $TMPJOBCMDFILE # echo "#PBS -l select=$Nodes:mem=112G:res_image=ppp_eccc_all_default_ubuntu-14.04-amd64_latest" >> $TMPJOBCMDFILE # echo "#PBS -l select=$Nodes:mem=112G:res_image=ppp_eccc_all_default_ubuntu-14.04-amd64_latest" >> $TMPJOBCMDFILE echo "#PBS -l select=$Nodes:mem=5G:res_image=ppp_eccc_all_default_ubuntu-14.04-amd64_latest" >> $TMPJOBCMDFILE echo "#PBS -l place=free" >> $TMPJOBCMDFILE ### The following 2 lines added for MPI based on migration documentation echo 'export OMPI_MCA_orte_tmpdir_base=/run/shm' >> $TMPJOBCMDFILE echo 'export OMPI_MCA_btl_openib_if_include=mlx5_0' >> $TMPJOBCMDFILE fi else if [ "$qName" = 'ppp1' -o "$qName" = 'ppp2' -o "$qName" = 'ppp3' -o "$qName" = 'ppp4' ] ; then JOBMEMORY=${JOBMEMORY:-'3000M'} if [ "$JOBTYPE" = 'serial' ] ; then tUnIt=`echo $JOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*M/M/p'` tUnIt=${tUnIt:=`echo $JOBMEMORY | tr '[a-z]' '[A-Z]' | sed -n -e 's/^.*G/G/p'`} if [ -n "$tUnIt" -a "$tUnIt" = 'M' ] ; then tJOBMEMORY=`echo ${JOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/M//' -e '1,$p'` # if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -lt 3000 ] ; then if [ -n "$tJOBMEMORY" ] ; then if [ "$tJOBMEMORY" -lt 3000 ] ; then JOBMEMORY='3000M' elif [ "$tJOBMEMORY" -gt 10000 ] ; then : #TMPFS=${TMPFS:-` expr $tJOBMEMORY - 5000 `} fi fi elif [ -n "$tUnIt" -a "$tUnIt" = 'G' ] ; then tJOBMEMORY=`echo ${JOBMEMORY} | tr '[a-z]' '[A-Z]' | sed -n -e 's/G//' -e '1,$p'` if [ -n "$tJOBMEMORY" -a "$tJOBMEMORY" -lt 2 ] ; then #JOBMEMORY='3G' JOBMEMORY='3072M' fi fi tJOBMEMORY='' tUnIt='' fi #JOBMEMORYlwr=${JOBMEMORY:-'1200M'} ## JOBMEMORYlwr=${JOBMEMORYlwr:-'1200m'} #JOBMEMORYlwr=`echo $JOBMEMORYlwr |tr '[A-Z]' '[a-z]'` fi # JOBMEMORY=${JOBMEMORY:-'3G'} JOBMEMORY=${JOBMEMORY:-'3072M'} # TMPFS=${TMPFS:-'20000'} # In MB # TMPFS=${TMPFS:-'10000'} # In MB # TMPFS=${TMPFS:-'7000'} # In MB JOBPES=${JOBPES:-'1'} # if [ "$qName" = 'ppp3' -o "$qName" = 'ppp4' ] ; then if [ "$qName" = 'ppp3' -o "$qName" = 'ppp4' -o "$JOBCLASS" = 'dev_xxfer' ] ; then # echo "#PBS -l select=1:ncpus=1:mem=${JOBMEMORY}:res_tmpfs=${TMPFS}:res_image=eccc/eccc_all_ppp_ubuntu-18.04-amd64_latest" >> $TMPJOBCMDFILE # echo "#PBS -l select=1:ncpus=${JOBPES}:mem=${JOBMEMORY}:res_image=eccc/eccc_all_ppp_ubuntu-18.04-amd64_latest" >> $TMPJOBCMDFILE if [ -n "$TMPFS" ] ; then echo "#PBS -l select=1:ncpus=${JOBPES}:mem=${JOBMEMORY}:res_tmpfs=${TMPFS}:res_image=eccc/eccc_all_ppp_ubuntu-18.04-amd64_latest" >> $TMPJOBCMDFILE else echo "#PBS -l select=1:ncpus=${JOBPES}:mem=${JOBMEMORY}:res_image=eccc/eccc_all_ppp_ubuntu-18.04-amd64_latest" >> $TMPJOBCMDFILE fi else # echo "#PBS -l select=1:ncpus=1:mem=${JOBMEMORY}:res_tmpfs=${TMPFS}:res_image=ppp_eccc_all_default_ubuntu-14.04-amd64_latest" >> $TMPJOBCMDFILE # echo "#PBS -l select=1:ncpus=${JOBPES}:mem=${JOBMEMORY}:res_image=ppp_eccc_all_default_ubuntu-14.04-amd64_latest" >> $TMPJOBCMDFILE if [ -n "$TMPFS" ] ; then echo "#PBS -l select=1:ncpus=${JOBPES}:mem=${JOBMEMORY}:res_tmpfs=${TMPFS}:res_image=ppp_eccc_all_default_ubuntu-14.04-amd64_latest" >> $TMPJOBCMDFILE else echo "#PBS -l select=1:ncpus=${JOBPES}:mem=${JOBMEMORY}:res_image=ppp_eccc_all_default_ubuntu-14.04-amd64_latest" >> $TMPJOBCMDFILE fi fi echo "#PBS -l place=free" >> $TMPJOBCMDFILE fi echo 'export CCCMA_LL_RMFILES="'"$RMFILES"'"' >> $TMPJOBCMDFILE echo "if [ -s $CUSTOM_ENV_FILE ] ; then . $CUSTOM_ENV_FILE || : ; "'else if [ -z "$SYSTEM_SETUP" -a -z "$CCRNSRC" -a -d /home/scrd101/generic/. ] ; then [ -s /home/scrd101/generic/sc_cccma_setup_profile ] && . /home/scrd101/generic/sc_cccma_setup_profile || : ; fi ; fi ' >> $TMPJOBCMDFILE if [ -s "$JOBSCRIPT" ] ; then if [ "$JOBTYPE" = 'parallel' ] ; then echo "$JOBSCRIPT" >> $TMPJOBCMDFILE else #echo "aprun -n1 -cc none $JOBSCRIPT" >> $TMPJOBCMDFILE echo "$JOBSCRIPT" >> $TMPJOBCMDFILE fi fi # sleep 5 #!!!!!! if [ -s $TMPJOBCMDFILE ] ; then : # ls -ld $TMPJOBCMDFILE # cat $TMPJOBCMDFILE else echo "`date`; `whoami`;$HOSTID: Can not proceed due to invalid $TMPJOBCMDFILE !" touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Can not proceed due to invalid $TMPJOBCMDFILE !" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; `whoami`;$HOSTID: Can not proceed due to invalid $TMPJOBCMDFILE !" >> $HOME/problem_log exit 1 fi echo "CCCQSUB - top of submitted file:" head -35 $TMPJOBCMDFILE echo '------------' echo "CCCQSUB - associated files with the submitted job:" \ls -ld $RMFILES echo '------------' #!!!!!! # submit the job # if [ "$OS" = 'AIX' ] ; then # if [ "$qName" = 'ppp1' -o "$qName" = 'ppp2' ] ; then if [ "$qName" = 'ppp1' -o "$qName" = 'ppp2' -o "$qName" = 'ppp3' -o "$qName" = 'ppp4' -o "$qName" = 'xfer' -o "$qName" = 'fexfer' ] ; then # CCC_PBS_JOBSUB=${CCC_PBS_JOBSUB:-'qsub'} if [ "$qNamex" = 'eccc-ppp3' -o "$qName" = 'ppp3' -o \( \( "$qName" = 'xfer' -o "$qName" = 'fexfer' \) -a \( "$HOSTIDf" = 'pp3' -o "$HOSTIDf" = 'xc3' -o "$HOSTIDf" = 'cs3' \) \) ] ; then CCC_PBS_JOBSUB='jobsub -c ppp3' elif [ "$qNamex" = 'eccc-ppp4' -o "$qName" = 'ppp4' -o \( \( "$qName" = 'xfer' -o "$qName" = 'fexfer' \) -a \( "$HOSTIDf" = 'pp4' -o "$HOSTIDf" = 'xc4' -o "$HOSTIDf" = 'cs4' \) \) ] ; then # CCC_PBS_JOBSUB='jobsub -c ppp4' CCC_PBS_JOBSUB='jobsub -c eccc-ppp4' elif [ "$qNamex" = 'eccc-ppp1' -o "$qName" = 'ppp1' -o \( \( "$qName" = 'xfer' -o "$qName" = 'fexfer' \) -a \( "$HOSTIDf" = 'pp1' -o "$HOSTIDf" = 'xc1' -o "$HOSTIDf" = 'cs1' \) \) ] ; then CCC_PBS_JOBSUB='jobsub -c ppp1' elif [ "$qNamex" = 'eccc-ppp2' -o "$qName" = 'ppp2' -o \( \( "$qName" = 'xfer' -o "$qName" = 'fexfer' \) -a \( "$HOSTIDf" = 'pp2' -o "$HOSTIDf" = 'xc2' -o "$HOSTIDf" = 'cs2' \) \) ] ; then CCC_PBS_JOBSUB='jobsub -c ppp2' else echo '------------' ; echo "QSUB - Due to parsing limitation, go with the default of CCC_PBS_JOBSUB=$CCC_PBS_JOBSUB" ; echo '------------' CCC_PBS_JOBSUB='jobsub -c ppp2' fi [ -s "./.jbstqsub_jid_${stampid}" ] && (\rm -f ./.jbstqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) && sleep 1 || : touch ./.jbstqsub_jid_${stampid} ; sleep 1 echo "QSUB - $CCC_PBS_JOBSUB $TMPJOBCMDFILE" ; echo '------------' $CCC_PBS_JOBSUB $TMPJOBCMDFILE >> ./.jbstqsub_jid_${stampid} && Exit=0 || Exit="$?" # Cwd=`pwd` # ssh $HDNODE "cd $Cwd; $CCC_PBS_JOBSUB $TMPJOBCMDFILE >> ./.jbstqsub_jid_${stampid}" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 1st failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 1st failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/problem_log sleep 115 (\rm -f ./.jbstqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) sleep 1 touch ./.jbstqsub_jid_${stampid} ; sleep 1 $CCC_PBS_JOBSUB $TMPJOBCMDFILE >> ./.jbstqsub_jid_${stampid} && Exit=0 || Exit="$?" # ssh $HDNODE "cd $Cwd; $CCC_PBS_JOBSUB $TMPJOBCMDFILE >> ./.jbstqsub_jid_${stampid}" && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 2nd failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; `whoami`;$HOSTID: Error; Exit=$Exit; 2nd failed attempt $CCC_PBS_JOBSUB $TMPJOBCMDFILE " >> $HOME/problem_log echo "QSUB - top of submitted file:" head -35 $TMPJOBCMDFILE (\rm -f ./.jbstqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) exit $Exit else sleep 1 if [ -s "./.jbstqsub_jid_${stampid}" ] ; then cat ./.jbstqsub_jid_${stampid} if [ "$qsublog" = 'yes' -o "$qsublog" = 'on' ] ; then JiD_cptred=`tail -1 ./.jbstqsub_jid_${stampid}` touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$PBS_JOBID on $HOSTID (${nqsprfx}${jobname}${nqsext}${id}.${stamp}) --> $CCC_PBS_JOBSUB $TMPJOBCMDFILE ==> $JiD_cptred " >> $HOME/tmp/cccqsub_log fi fi (\rm -f ./.jbstqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) fi else sleep 1 if [ -s "./.jbstqsub_jid_${stampid}" ] ; then cat ./.jbstqsub_jid_${stampid} if [ "$qsublog" = 'yes' -o "$qsublog" = 'on' ] ; then JiD_cptred=`tail -1 ./.jbstqsub_jid_${stampid}` touch $HOME/tmp/cccqsub_log echo "`date`; `whoami`;$PBS_JOBID on $HOSTID (${nqsprfx}${jobname}${nqsext}${id}.${stamp}) --> $CCC_PBS_JOBSUB $TMPJOBCMDFILE ==> $JiD_cptred " >> $HOME/tmp/cccqsub_log fi fi (\rm -f ./.jbstqsub_jid_${stampid} >> /dev/null 2>> /dev/null || : ) fi fi #!!!!!! # echo "QSUB - top of submitted file:" # head -35 $TMPJOBCMDFILE #!!!!!! # Clean-up # ((sleep 600 ; rm -f $RMFILES ) < /dev/null > /dev/null 2>&1 &) # ((sleep 300 ; rm -f $RMFILES ) < /dev/null > /dev/null 2>&1 &) # ((sleep 30 ; rm -f $RMFILES ) < /dev/null > /dev/null 2>&1 &) # ((sleep 10 ; rm -f $RMFILES ) < /dev/null > /dev/null 2>&1 &) # sleep 5 # Disable under the assumption that cleanup will be done via crontab... # sleep 30 # \rm -f $RMFILES #???? else # Batch_mode #!!!! # SGE case translate and submit... # set to mail on abort... mdirctv="-m a -M $LOGNAME@ec.gc.ca" unset Rest unset TR_time unset PE_target unset PE_set unset NPE_number unset Nproc_set unset SX6_target unset Xfer_que # mdirctv="" # parse the arguments list and convert... set -- $dirctv while [ $# -gt 0 ] do case "$1" in -c) # Ignore shift ;; -mu) # Ignore shift ;; -lu) # setup based on maximum number of cpu per process shift # mdirctv="$mdirctv -pe user.sx6 1" # mdirctv="$mdirctv -l cpus_per_slot=$1" if [ "$openmp" = 'on' ] ; then PE_target=${PE_target:="user.openmp.sx6"} else PE_target=${PE_target:="user.multi.sx6"} fi NPE_number="$1" # PE_set=${PE_set:="-pe ${PE_target} ${NPE_number}"} # mdirctv="$mdirctv -l cpus_per_slot=1" PE_set=${PE_set:="-l cpus_per_slot=1 -pe ${PE_target} ${NPE_number}"} ;; -pe) # setup for SGE parallel environment shift PE_target=${PE_target:="$1"} shift # if [ -z "$PE_set" ] ; then # PE_set=${PE_set:="-pe ${PE_target} $1"} # mdirctv="$mdirctv $PE_set" # fi NPE_number="$1" # PE_set="-pe ${PE_target} ${NPE_number}"} # PE_set="-pe ${PE_target} $1"} PE_set="-l cpus_per_slot=1 -pe ${PE_target} $1"} ;; -q) # which queue shift case "$1" in # new-pollux) mdirctv="$mdirctv -l qgroups.user.new-pollux=true" # ;; # pollux|orion) mdirctv="$mdirctv -S /bin/csh -l qgroups.user.pollux=true" # pollux|orion) mdirctv="$mdirctv -S /bin/bsh -l qgroups.user.pollux=true" # pollux|orion) mdirctv="$mdirctv -l qgroups.user.pollux=true" # pollux|orion) mdirctv="$mdirctv -l qgroups.user.pollux=true -pe user.pollux 1" # pollux|orion) mdirctv="$mdirctv -pe dev.s.pollux 1" pollux) if [ "$LOGNAME" = 'acrnsrc' ] ; then #mdirctv="$mdirctv "'-q dev -l res_cpus=4' #mdirctv="$mdirctv "'-pe dev 1 -l res_cpus=4' #mdirctv="$mdirctv "'-pe dev 1 -l res_cpus=2' mdirctv="$mdirctv "'-pe dev 1 -l res_cpus=1' else #mdirctv="$mdirctv "'-q dev -l res_cpus=4' #mdirctv="$mdirctv "'-pe dev 1 -l res_cpus=4' mdirctv="$mdirctv "'-pe dev 1 -l res_cpus=1' fi ;; mez) if [ "$LOGNAME" = 'acrnsrc' ] ; then #mdirctv="$mdirctv "'-pe dev 1 -l res_cpus=4' mdirctv="$mdirctv "'-pe dev 1 -l res_cpus=2' else mdirctv="$mdirctv "'-pe dev 1 -l res_cpus=4' fi ;; castor) if [ "$LOGNAME" = 'acrnsrc' ] ; then #mdirctv="$mdirctv "'-q dev -l res_cpus=4' mdirctv="$mdirctv "'-pe dev 1 -l res_cpus=4' else #mdirctv="$mdirctv "'-q dev -l res_cpus=4' mdirctv="$mdirctv "'-pe dev 1 -l res_cpus=4' fi ;; # dorval-ib) mdirctv="$mdirctv -pe dev.e.ib.x86-8-2330 1" # dorval-ib) mdirctv="$mdirctv -pe development.shared 1" # dorval-ib) mdirctv="$mdirctv -q dev.s.ib" # ;; ### # alef) mdirctv="$mdirctv -q dev.s.alef" ### # alef) mdirctv="$mdirctv -pe dev.e.alef"'.x86-8-2330 8' ### # alef) if [ `whoami` = 'acrnsrc' ] ; then ### alef) if [ "$LOGNAME" = 'acrnsrc' ] ; then ### #mdirctv="$mdirctv -q dev.s.alef" ### mdirctv="$mdirctv "'-q dev.s.alef -l tile_size=4' ### else ### #mdirctv="$mdirctv -pe dev.e.alef"'.x86-8-2330 8' ### mdirctv="$mdirctv "'-q dev.s.alef -l tile_size=4' ### fi ### ;; # ccrnftp) mdirctv="$mdirctv -l qgroups.xfer.pollux=true -l xfer.ccrnftp.pollux=1" # ccrnftp) mdirctv="$mdirctv -l ccrnftp.h.pollux=1" ccrnftp) if [ "$OS" = 'Linux' ] ; then if [ "$HOSTIDf" = 'ib3' ] ; then #pollux linux case #mdirctv="$mdirctv -q xfer -l res_cpus=1" mdirctv="$mdirctv -q xxfer -l res_cpus=1" #mdirctv="$mdirctv -q xxfer -l res_cpus=2" elif [ "$HOSTIDf" = 'ib8' ] ; then # mdirctv="$mdirctv -q xxfer -l res_cpus=1" mdirctv="$mdirctv -q xxfer -l res_cpu_shares=1" elif [ "$HOSTIDf" = 'ib4' ] ; then #castor linux case #mdirctv="$mdirctv -q xfer -l res_cpus=1" mdirctv="$mdirctv -q xxfer -l res_cpus=1" #mdirctv="$mdirctv -q xxfer -l res_cpus=2" else # Default Alef's case mdirctv="$mdirctv -l dev.ccrnftp=1" fi fi Xfer_que='yes' ;; # tape08) mdirctv="$mdirctv -l qgroups.xfer.pollux=true -l xfer.tape08.pollux=1" # tape08) mdirctv="$mdirctv -l tape08.h.pollux=1" tape08) if [ "$OS" = 'Linux' ] ; then if [ "$HOSTIDf" = 'ib3' ] ; then #pollux linux case mdirctv="$mdirctv -q xfer -l res_cpus=1" #mdirctv="$mdirctv -q xfer -l res_cpus=4" #mdirctv="$mdirctv -q xfer -l res_cpus=2" elif [ "$HOSTIDf" = 'ib8' ] ; then #mdirctv="$mdirctv -q xfer -l res_cpus=1" mdirctv="$mdirctv -q xfer -l res_cpu_shares=1" elif [ "$HOSTIDf" = 'ib4' ] ; then #castor linux case mdirctv="$mdirctv -q xfer -l res_cpus=1" #mdirctv="$mdirctv -q xfer -l res_cpus=4" #mdirctv="$mdirctv -q xfer -l res_cpus=2" else # Default Alef's case mdirctv="$mdirctv -l dev.tape08=1" fi fi Xfer_que='yes' ;; # cfsxfer) mdirctv="$mdirctv -l qgroups.xfer.pollux=true -l xfer.cfsxfer.pollux=1" # cfsxfer) mdirctv="$mdirctv -l cfsxfer.h.pollux=1" cfsxfer) if [ "$OS" = 'Linux' ] ; then if [ "$HOSTIDf" = 'ib3' ] ; then #pollux linux case # Don't work : mdirctv="$mdirctv -l xxfer=1" # Don't work : mdirctv="$mdirctv -pe xxfer 1 -l res_cpus=1" # Don't work: mdirctv="$mdirctv -q xfer" # Don't work: mdirctv="$mdirctv -pe xfer 1 -l res_cpus=1" # Either works: # mdirctv="$mdirctv -q xfer -l res_cpus=1" # mdirctv="$mdirctv -q xfer -l res_cpus=4" # mdirctv="$mdirctv -q cfsxfer -l res_cpus=1" # mdirctv="$mdirctv -q cfsxfer -l res_cpus=4" mdirctv="$mdirctv -q cfsxfer -l res_cpus=2" elif [ "$HOSTIDf" = 'ib8' ] ; then #mdirctv="$mdirctv -q cfsxfer -l res_cpus=2" mdirctv="$mdirctv -q cfsxfer -l res_cpu_shares=2" elif [ "$HOSTIDf" = 'ib4' ] ; then #castor linux case # Either works: # mdirctv="$mdirctv -q xfer -l res_cpus=1" # mdirctv="$mdirctv -q xfer -l res_cpus=4" # mdirctv="$mdirctv -q cfsxfer -l res_cpus=1" # mdirctv="$mdirctv -q cfsxfer -l res_cpus=4" mdirctv="$mdirctv -q cfsxfer -l res_cpus=2" else # Default Alef's case mdirctv="$mdirctv -l dev.cfsxfer=1" #mdirctv="$mdirctv -q dev.s.alef.xfer64" fi fi Xfer_que='yes' ;; dataq) if [ "$OS" = 'Linux' ] ; then if [ "$HOSTIDf" = 'dat' ] ; then #mdirctv="$mdirctv "'-q dev.s.dataq.datamover=1 -l h_cpu=900' # <-- NO WORK #mdirctv="$mdirctv "'-q datasvr=1 -l h_cpu=900' # <-- NO WORK #mdirctv="$mdirctv "'-l datasvr=1 -l h_cpu=900' # <-- WORKED; may be used to force running in one of dataq nodes #mdirctv="$mdirctv "'-l zeta=1 -l h_cpu=900' # <-- WORKED; Same effect as -l datasvr=1 running on dataq node #mdirctv="$mdirctv "'-l alef=1 -l h_cpu=900' # <-- WORKED; Same effect as -l datasvr=1 running on dataq node mdirctv="$mdirctv "'-q dev.cnfs.dataq -l h_cpu=900' # <-- WORKED ; Best to use this one UFN fi fi ;; # yata) mdirctv="$mdirctv -l qgroups.user.sx6=true -ckpt NEC" # yata) mdirctv="$mdirctv -l qgroups.user.sx6=true " # ;; *) # Unknown class - reject touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Unknown queue $1 !" >> $HOME/tmp/cccqsub_log echo 'Unknown queue: "'$1'"' exit -1 ;; esac ;; -o) # output destination shift JOBOUTFL=`echo $1 | sed -n -e 's/^.*://' -e '1,$p'` mdirctv="$mdirctv -o ${JOBOUTFL}" ;; -e) # error log location shift JOBERRFL=`echo $1 | sed -n -e 's/^.*://' -e '1,$p'` mdirctv="$mdirctv -e ${JOBERRFL}" ;; -eo) # both output and error logs in the same file mdirctv="$mdirctv -j y" ;; -r) # job name shift # JOBNAME=`echo $1 | cut -c1-14` # mdirctv="$mdirctv -N ${JOBNAME}" mdirctv="$mdirctv -N $1" ;; -lT)# time limit shift TR_time_val="$1" if [ "$Xfer_que" = 'yes' ] ; then # if [ "$TR_time_val" -lt 600 ] ; then ## TR_time_val=600 # fi # if [ "$OS" = 'Linux' ] ; then if [ "$HOSTIDf" = 'ib3' ] ; then #TR_time_val=3600 #TR_time_val=10800 TR_time_val=18000 elif [ "$HOSTIDf" = 'ib8' ] ; then #TR_time_val=3600 #TR_time_val=10800 TR_time_val=18000 elif [ "$HOSTIDf" = 'ib4' ] ; then #TR_time_val=3600 #TR_time_val=10800 TR_time_val=18000 else TR_time_val=3600 fi # else # TR_time_val=600 # fi fi if [ "$HOSTIDf" = 'dat' ] ; then TR_time=${TR_time:="-l h_cpu=900"} elif [ "$HOSTIDf" = 'ib3' ] ; then if [ "$TR_time_val" -lt 600 ] ; then TR_time_val=600 fi # Adjust to target devl queue if time exceeds 16200=(4h:30m) if [ "$TR_time_val" -gt 16200 ] ; then mdirctv=`echo $mdirctv | sed -e 's/-pe dev 1 -l res_cpus=./-pe devl 1 -l res_cpus=1/'` fi TR_time=${TR_time:="-l h_rt=$TR_time_val"} elif [ "$HOSTIDf" = 'ib8' ] ; then if [ "$TR_time_val" -lt 600 ] ; then TR_time_val=600 fi TR_time=${TR_time:="-l h_rt=$TR_time_val"} elif [ "$HOSTIDf" = 'ib4' ] ; then if [ "$TR_time_val" -lt 600 ] ; then TR_time_val=600 fi TR_time=${TR_time:="-l h_rt=$TR_time_val"} else TR_time=${TR_time:="-l h_cpu=$TR_time_val"} # TR_time=${TR_time:="-l h_cpu=$1"} # mdirctv="$mdirctv -l h_cpu=$1" fi ;; -lR)# Resident time limit shift # temporarily replace "h_cpurestm" use by "h_cpu": TR_time_val="$1" if [ "$Xfer_que" = 'yes' ] ; then # if [ "$TR_time_val" -lt 600 ] ; then ## TR_time_val=600 # fi # if [ "$OS" = 'Linux' ] ; then if [ "$HOSTIDf" = 'ib3' ] ; then #TR_time_val=3600 #TR_time_val=10800 TR_time_val=18000 elif [ "$HOSTIDf" = 'ib8' ] ; then #TR_time_val=3600 #TR_time_val=10800 TR_time_val=18000 elif [ "$HOSTIDf" = 'ib4' ] ; then #TR_time_val=3600 #TR_time_val=10800 TR_time_val=18000 else TR_time_val=3600 fi # else # TR_time_val=600 # fi fi if [ "$HOSTIDf" = 'dat' ] ; then TR_time=${TR_time:="-l h_rt=3600"} elif [ "$HOSTIDf" = 'ib3' ] ; then if [ "$TR_time_val" -lt 600 ] ; then TR_time_val=600 fi # Adjust to target devl queue if time exceeds 16200=(4h:30m) if [ "$TR_time_val" -gt 16200 ] ; then mdirctv=`echo $mdirctv | sed -e 's/-pe dev 1 -l res_cpus=./-pe devl 1 -l res_cpus=1/'` fi TR_time=${TR_time:="-l h_rt=$TR_time_val"} elif [ "$HOSTIDf" = 'ib8' ] ; then if [ "$TR_time_val" -lt 600 ] ; then TR_time_val=600 fi TR_time=${TR_time:="-l h_rt=$TR_time_val"} elif [ "$HOSTIDf" = 'ib4' ] ; then if [ "$TR_time_val" -lt 600 ] ; then TR_time_val=600 fi TR_time=${TR_time:="-l h_rt=$TR_time_val"} else TR_time=${TR_time:="-l h_cpurestm=$TR_time_val"} # TR_time=${TR_time:="-l h_cpurestm=$1"} # TR_time=${TR_time:="-l h_cpu=$1"} # mdirctv="$mdirctv -l h_rt=$1" # mdirctv="$mdirctv -l h_cpurestm=$1" fi ;; -lM)# memory limit shift if [ "$Xfer_que" = 'yes' ] ; then if [ "$OS" = 'Linux' ] ; then if [ "$HOSTIDf" = 'ib3' ] ; then #JOBMEMORY='320M' #JOBMEMORYval='320' JOBMEMORY='1000M' JOBMEMORYval='1000' elif [ "$HOSTIDf" = 'ib8' ] ; then #JOBMEMORY='320M' #JOBMEMORYval='320' JOBMEMORY='1000M' JOBMEMORYval='1000' elif [ "$HOSTIDf" = 'ib4' ] ; then #JOBMEMORY='320M' #JOBMEMORYval='320' JOBMEMORY='1000M' JOBMEMORYval='1000' else #JOBMEMORY='256M' #JOBMEMORYval='256' JOBMEMORY='640M' JOBMEMORYval='640' fi # else # Pollux irix seems to allow up to 32M # JOBMEMORY='16M' # JOBMEMORY='32M' # JOBMEMORYval='32' fi else JOBMEMORY=`echo $1 |tr '[a-z]' '[A-Z]' | sed -n -e 's/B//' -e 's/ //g' -e '1,$p'` # if [ "$OS" = 'Linux' -o "$OS" = 'IRIX64' ] ; then if [ "$OS" = 'Linux' ] ; then if [ -n "JOBMEMORY" ] ; then JOBMEMORYval=`echo $JOBMEMORY | sed -n -e 's/M *$//p'` if [ -n "$JOBMEMORYval" -a "$JOBMEMORYval" != "$JOBMEMORY" ] ; then #if [ "$JOBMEMORYval" -lt 3000 -a "$OS" = 'Linux' ] ; then if [ "$JOBMEMORYval" -lt 3000 ] ; then JOBMEMORY='3000M' JOBMEMORYval=3000 # elif [ "$JOBMEMORYval" -lt 250 -a "$OS" = 'IRIX64' ] ; then # JOBMEMORY='250M' fi fi fi fi # if [ "$OS" = 'Linux' ] ; then JOBMEMORY=${JOBMEMORY:='3000M'} JOBMEMORYval=${JOBMEMORYval:='3000'} # else # JOBMEMORY=${JOBMEMORY:='250M'} # fi fi if [ "$HOSTIDf" = 'dat' ] ; then JOBMEMORY='320M' JOBMEMORYval='320' #JOBMEMORY='1000M' #JOBMEMORYval='1000' mdirctv="$mdirctv -l h_vmem=${JOBMEMORY}" elif [ "$HOSTIDf" = 'ib3' ] ; then mdirctv="$mdirctv -l h_vmem=${JOBMEMORY},res_mem=${JOBMEMORYval}" elif [ "$HOSTIDf" = 'ib8' ] ; then mdirctv="$mdirctv -l h_vmem=${JOBMEMORY},res_mem=${JOBMEMORYval}" elif [ "$HOSTIDf" = 'ib4' ] ; then mdirctv="$mdirctv -l h_vmem=${JOBMEMORY},res_mem=${JOBMEMORYval}" else mdirctv="$mdirctv -l h_vmem=$JOBMEMORY" fi ;; *) # Rest of stuff, accumulate in "Rest" parameter if [ -f $1 ] ; then Script_file="$1" else Rest="$Rest $1" fi ;; esac shift done ##1 ##1 # Ensure the number of "cpus" is defined (default=1) ##1 # when the target is SX6. ##1 ##1 SX6_target=`echo $mdirctv | sed -n -e 's/^.* qgroups.user.sx6=true .*$/true/p'` ##1 if [ "$SX6_target" = 'true' ] ; then ##1 # Nproc_set=`echo $mdirctv | sed -n -e 's/^.* cpus_per_slot=.*$/yes/p'` ##1 # if [ "$Nproc_set" != 'yes' ] ; then ##1 # mdirctv=`echo $mdirctv | sed -n -e 's/ qgroups.user.sx6=true / qgroups.user.sx6=true -l cpus_per_slot=1 /p'` ##1 # fi ##1 PE_set=${PE_set:="-l cpus_per_slot=1 -pe user.multi.sx6 1"} ##1 fi ##1 #!!!!!!!!!!!!!!!!! if [ "$OS" = 'Linux' -a "$HOSTIDf" = 'ib3' -a -z "$TR_time" ] ; then #TR_time_val=3600 TR_time_val=10800 TR_time=${TR_time:="-l h_rt=$TR_time_val"} elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'ib8' -a -z "$TR_time" ] ; then #TR_time_val=3600 TR_time_val=10800 TR_time=${TR_time:="-l h_rt=$TR_time_val"} elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'ib4' -a -z "$TR_time" ] ; then #TR_time_val=3600 TR_time_val=10800 TR_time=${TR_time:="-l h_rt=$TR_time_val"} elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'dat' -a -z "$TR_time" ] ; then TR_time_val=3600 TR_time=${TR_time:="-l h_rt=$TR_time_val"} fi if [ "$OS" = 'Linux' -a "$HOSTIDf" = 'ib3' -a -z "$JOBMEMORY" ] ; then if [ "$Xfer_que" = 'yes' ] ; then #JOBMEMORY='320M' #JOBMEMORYval='320' JOBMEMORY='1000M' JOBMEMORYval='1000' else JOBMEMORY=${JOBMEMORY:='3000M'} JOBMEMORYval=${JOBMEMORYval:='3000'} fi mdirctv="$mdirctv -l h_vmem=${JOBMEMORY},res_mem=${JOBMEMORYval}" elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'ib8' -a -z "$JOBMEMORY" ] ; then if [ "$Xfer_que" = 'yes' ] ; then #JOBMEMORY='320M' #JOBMEMORYval='320' JOBMEMORY='1000M' JOBMEMORYval='1000' else JOBMEMORY=${JOBMEMORY:='3000M'} JOBMEMORYval=${JOBMEMORYval:='3000'} fi mdirctv="$mdirctv -l h_vmem=${JOBMEMORY},res_mem=${JOBMEMORYval}" elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'ib4' -a -z "$JOBMEMORY" ] ; then if [ "$Xfer_que" = 'yes' ] ; then #JOBMEMORY='320M' #JOBMEMORYval='320' JOBMEMORY='1000M' JOBMEMORYval='1000' else JOBMEMORY=${JOBMEMORY:='3000M'} JOBMEMORYval=${JOBMEMORYval:='3000'} fi mdirctv="$mdirctv -l h_vmem=${JOBMEMORY},res_mem=${JOBMEMORYval}" elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'dat' -a -z "$JOBMEMORY" ] ; then JOBMEMORY='320M' JOBMEMORYval='320' mdirctv="$mdirctv -l h_vmem=${JOBMEMORY}" fi #!!!!!!!!!!!!!!!!! # Rest=${Rest:=' - '} # mdirctv="$mdirctv $TR_time $Rest" mdirctv="$mdirctv $PE_set $TR_time $Rest" # swuse -p host/gridengine-5.3.C # if [ "$HOSTHW" = 'SX-6' ] ; then # eval `/sw/necsx-base-6/sw-2.06/bin/swset -style bsh -exitonerror -messages error -set P -first -p host/gridengine-5.3.C` if [ "$OS" = 'Linux' ] ; then #if [ "$Xfer_que" != 'yes' ] ; then if [ "$HOSTIDf" = 'ib3' ] ; then #mdirctv=`echo $mdirctv | sed -e 's/-l *h_vmem=/-l h_stack=32M -l res_ht=false -l h_vmem=/g'` #mdirctv=`echo $mdirctv | sed -e 's/-l *h_vmem=/-notify -l h_stack=32M -l res_ht=false -l h_vmem=/g'` mdirctv=`echo $mdirctv | sed -e 's/-l *h_vmem=/-l h_stack=32M -l h_vmem=/g'` #mdirctv=`echo $mdirctv | sed -e 's/-l *h_vmem=/-notify -l h_stack=32M -l h_vmem=/g'` elif [ "$HOSTIDf" = 'ib8' ] ; then mdirctv=`echo $mdirctv | sed -e 's/-l *h_vmem=/-l h_stack=32M -l h_vmem=/g'` #mdirctv=`echo $mdirctv | sed -e 's/-l *h_vmem=/-notify -l h_stack=32M -l h_vmem=/g'` elif [ "$HOSTIDf" = 'ib4' ] ; then mdirctv=`echo $mdirctv | sed -e 's/-l *h_vmem=/-l h_stack=32M -l h_vmem=/g'` #mdirctv=`echo $mdirctv | sed -e 's/-l *h_vmem=/-notify -l h_stack=32M -l h_vmem=/g'` elif [ "$HOSTIDf" = 'dat' ] ; then mdirctv=`echo $mdirctv | sed -e 's/-l *h_vmem=/-l h_stack=32M -l h_vmem=/g'` else mdirctv=`echo $mdirctv | sed -e 's/-l *h_vmem=/-l h_stack=32M -l h_vmem=/g'` fi #fi HOSTIDf=`echo $HOSTID | cut -c 1-3` ## if [ "$HOSTIDf" = 'ib-' ] ; then ## eval "`/opt/ssm/all/bin/ssmuse sh -d /opt/ssm -p gridengine_6.1_multi`" ## SGE_QMASTER_PORT=6444 ; export SGE_QMASTER_PORT ## elif [ "$HOSTIDf" = 'ale' ] ; then ## eval "`/opt/ssm/all/bin/ssmuse sh -d /opt/ssm -p gridengine_6.1u5_multi`" ## SGE_QMASTER_PORT=6444 ; export SGE_QMASTER_PORT ## elif [ "$HOSTIDf" = 'ib3' ] ; then if [ "$HOSTIDf" = 'ib3' ] ; then #eval "`/opt/ssm/all/bin/ssmuse sh -d /opt/ssm -p gridengine_6.1u3-cmc0-lucid_multi`" #eval "`/opt/ssm/all/bin/ssmuse sh -d /opt/ssm -p gridengine_6.1u3_multi`" #eval "`/opt/ssm/all/bin/ssmuse sh -d /opt/ssm `" #SGE_QMASTER_PORT=6444 ; export SGE_QMASTER_PORT . ssmuse-sh -d /ssm/net/gridengine/pollux.cmc.ec.gc.ca elif [ "$HOSTIDf" = 'ib8' ] ; then . ssmuse-sh -d /ssm/net/gridengine/mez.cmc.ec.gc.ca elif [ "$HOSTIDf" = 'ib4' ] ; then #eval "`/opt/ssm/all/bin/ssmuse sh -d /opt/ssm -p gridengine_6.1u3-cmc0-lucid_multi`" #eval "`/opt/ssm/all/bin/ssmuse sh -d /opt/ssm -p gridengine_6.1u3_multi`" #eval "`/opt/ssm/all/bin/ssmuse sh -d /opt/ssm `" #SGE_QMASTER_PORT=6444 ; export SGE_QMASTER_PORT . ssmuse-sh -d /ssm/net/gridengine/castor.cmc.ec.gc.ca elif [ "$HOSTIDf" = 'dat' ] ; then eval "`/opt/ssm/all/bin/ssmuse sh -d /opt/ssm -p gridengine_6.1u3_multi`" SGE_QMASTER_PORT=6444 ; export SGE_QMASTER_PORT else touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed with SGE setup on the local Linux platform!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed with SGE setup on the local Linux platform!" exit -1 fi else # eval `/software/base/bin/swset -style sh -exitonerror -messages error -set P -first -p host/gridengine-5.3.C` # SGE_QMASTER_PORT=6444 ; export SGE_QMASTER_PORT # eval "`ssmuse sh -d /opt/ssm -p gridengine_6.1u3_multi`" # eval "`/opt/ssm/all/bin/ssmuse sh -d /opt/ssm -p gridengine_6.1u3_multi`" # SGE_QMASTER_PORT=6444 ; export SGE_QMASTER_PORT touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed with SGE setup on the local $HOSTID platform!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed with SGE setup on the local $HOSTID platform!" exit -1 fi # LOGNAME=${LOGNAME:=`whoami`} PRFX=`echo $LOGNAME | cut -c1-4` if [ -d "/data/ords/${PRFX}/${LOGNAME}/." ] ; then SGE_spool_dir='/data/ords/acrn/acrnsrc/SGE_spool_dir' else SGE_spool_dir='/home/crb_ccrn/pollux/acrn/src/SGE_spool_dir' fi stamp="${HOSTID}_"`date +%Y%j%H%M%S` stampid="${stamp}_$$" USR=${USR:=`whoami | sed -e 's/^....//'`} if [ -z "$Script_file" ] ; then MScript_file="qsubjbcmd_${USR}_${stamp}_$$" cat > $SGE_spool_dir/$MScript_file if [ -s "$SGE_spool_dir/$MScript_file" ] ; then : else (\rm -f $SGE_spool_dir/$MScript_file || : ) touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed due to invalid/empty input script file!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed due to invalid/empty input script file!" exit -1 fi else if [ -s "$Script_file" ] ; then MScript_file=`basename $Script_file` MScript_file="${MScript_file}_${USR}_${stamp}_$$" \cp $Script_file $SGE_spool_dir/$MScript_file else touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed due to invalid/empty $Script_file input script file!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed due to invalid/empty $Script_file input script file!" exit -1 fi fi # set -x # Attempt piping the job via standard input # provided the size is less than SGE current # input file size limit of 102399 characters. # Numchar=`cat $Script_file | wc -c | sed -e 's/ *//g'` ## Sge_input_limit='102399' # Sge_input_limit='102300' # if [ "$Numchar" -le "$Sge_input_limit" ] ; then # echo "mod_qsub used: cat $Script_file | cccqsub $mdirctv" # cat $Script_file | exec cccqsub $mdirctv # sleep 5 # else echo "mod_qsub used: cccqsub $mdirctv $SGE_spool_dir/$MScript_file" ( touch $SGE_spool_dir/.$MScript_file && echo '#! /bin/sh' >> $SGE_spool_dir/.$MScript_file && echo 'set -x' >> $SGE_spool_dir/.$MScript_file && echo "sgeqsub $mdirctv $SGE_spool_dir/$MScript_file && sleep 2 && rm $SGE_spool_dir/$MScript_file $SGE_spool_dir/.$MScript_file || : " >> $SGE_spool_dir/.$MScript_file && chmod u+x $SGE_spool_dir/.$MScript_file || : ) ( chmod g+r $SGE_spool_dir/$MScript_file $SGE_spool_dir/.$MScript_file || : ) PlottrJob=` echo "$mdirctv $SGE_spool_dir/$MScript_file" | sed -e 's/^.* -N plottr .*$/YES/g'` PlottrJob=${PlottrJob:='NO'} # set -x ## exec cccqsub $mdirctv $SGE_spool_dir/$MScript_file qsub $mdirctv $SGE_spool_dir/$MScript_file && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; 1st failed attempt cccqsub $mdirctv $SGE_spool_dir/$MScript_file" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; 1st failed attempt cccqsub $mdirctv $SGE_spool_dir/$MScript_file" >> $HOME/problem_log sleep 180 qsub $mdirctv $SGE_spool_dir/$MScript_file && Exit=0 || Exit="$?" if [ $Exit -ne 0 ] ; then touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Exit=$Exit; 2nd failed attempt cccqsub $mdirctv $SGE_spool_dir/$MScript_file" >> $HOME/tmp/cccqsub_log touch $HOME/problem_log echo "`date`; $HOSTID: Error; Exit=$Exit; 2nd failed attempt cccqsub $mdirctv $SGE_spool_dir/$MScript_file" >> $HOME/problem_log exit $Exit fi fi if [ -n "$qsublog" ] ; then qsublog=`echo $qsublog | tr '[A-Z]' '[a-z]'` if [ "$qsublog" = 'on' -o "$qsublog" = 'yes' ] ; then touch $HOME/tmp/cccqsub_log if [ $Exit -ne 0 ] ; then echo "`date`; $HOSTID: FAILED mod_qsub used: cccqsub $mdirctv $SGE_spool_dir/$MScript_file" >> $HOME/tmp/cccqsub_log else echo "`date`; $HOSTID: mod_qsub used: cccqsub $mdirctv $SGE_spool_dir/$MScript_file" >> $HOME/tmp/cccqsub_log fi fi fi sleep 3 # if [ "$PlottrJob" = 'YES' ] ; then if [ $Exit -ne 0 -a "$PlottrJob" = 'YES' ] ; then : else ( \rm -f $SGE_spool_dir/$MScript_file $SGE_spool_dir/.$MScript_file || : ) fi # exit # fi fi # Batch_mode fi # [ -z "$qName" ] #### fi # [ nonqs ] ## Uncomment do disable " else -- nonqs" part. #BG -> Comment to enable "else -- nonqs" part. #### exit ## Uncomment do disable "else -- nonqs" part. #BG -> Comment to enable "else -- nonqs" part. else # [ nonqs ] unset saiphbg unset maiabg unset zetabg stamp="${HOSTID}_"`date +%Y%j%H%M%S` stampid="${stamp}_$$" USR=${USR:=`whoami | sed -e 's/^....//'`} unset RMFILES TMPJOBCMDFILE="/tmp/.nhpqsub_${USR}_${stamp}_$$" # RMFILES="$TMPJOBCMDFILE" # trap "(sleep 20 ; rm -f $TMPJOBCMDFILE) &" 0 STRNG=" nohup " # default class is vicmedium # JOBCLASS=vicmedium set -- $dirctv while [ $# -gt 0 ] do case "$1" in -c) # Ignore shift ;; -lu) # Ignore shift ;; -mu) # Ignore shift ;; -q) # queue name shift JOBCLASS=$1 ;; -e) # error log location shift JOBERRLOG=`echo $1 | sed -n -e 's/^.*://' -e '1,$p'` ;; -eo) # both logs in same file JOBEO=EO ;; -o) # output log location shift JOBLOG=`echo $1 | sed -n -e 's/^.*://' -e '1,$p'` ;; -r) # job identification shift # JOBNAME=`echo $1 | cut -c1-20` JOBNAME=$1 ;; -lT) # time limit shift # JOBTIME=$1 JOBTIME=${JOBTIME:=$1} ;; -lR) # time limit shift # JOBTIME=$1 JOBTIME=${JOBTIME:=$1} ;; -lM) # memory limit shift # JOBMEMORY=$1 JOBMEMORY=`echo $1 |tr '[a-z]' '[A-Z]' | sed -n -e 's/B//' -e '1,$p'` JOBMEMORY=${JOBMEMORY:='200M'} ;; *) # script to execute if [ -f $1 ] ; then JOBSCRIPT=$1 else JOBRest="$JOBRest $1" fi ;; esac shift done # if [ "$JOBCLASS" != "$HOSTID" ] ; then ## if [ "$JOBCLASS" != "$HOSTID" -a "$HOSTHW" != 'SX-6' ] ; then ## echo "$0 : Error, $JOBCLASS queue destination should be equal to $HOSTID!" ## exit 1 ## fi # current pwd is job's pwd JOBPWD=$(pwd) # get user's home directory JOBHOMEDIR=/home/$USER # put output in .queue subdirectory JOBQUEUEDIR=${JOBHOMEDIR}/.queue # use jobname from command line or make one up JOBNAME=${JOBNAME-${JOBSCRIPT-stdin}} # If jobscript is not specified, then read from stdin. if [ -z "$JOBSCRIPT" ] ; then # JOBSCRIPT="/tmp/.loadl_qsub_${USR}_${stamp}_$$" # RMFILES="$RMFILES $JOBSCRIPT" JOBSCRIPT="$TMPJOBCMDFILE" # trap "(sleep 20 ; rm -f $TMPJOBCMDFILE $JOBSCRIPT) &" 0 cat > $JOBSCRIPT if [ -s "$JOBSCRIPT" ] ; then : else (\rm -f $JOBSCRIPT || : ) touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed due to invalid/empty input script file!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed due to invalid/empty input script file!" exit -1 fi else if [ -s "$JOBSCRIPT" ] ; then cat $JOBSCRIPT > $TMPJOBCMDFILE chmod u+x $TMPJOBCMDFILE JOBSCRIPT="$TMPJOBCMDFILE" else touch $HOME/tmp/cccqsub_log echo "`date`; $HOSTID: Error; Sorry, unable to proceed due to invalid/empty $JOBSCRIPT input script file!" >> $HOME/tmp/cccqsub_log echo "$0 : Sorry, unable to proceed due to invalid/empty $JOBSCRIPT input script file!" exit -1 fi fi RMFILES="$TMPJOBCMDFILE" chmod u+x $JOBSCRIPT STRNG="${STRNG} $JOBSCRIPT" # setup output and error logs # if error and standard output to same file if [ "$JOBEO" = EO ] ; then # use standard output if it has been specified if [ -n "$JOBLOG" ] ; then JOBERRLOG=$JOBLOG # or use error output if it has been specified elif [ -n "$JOBERRLOG" ] ; then JOBLOG=$JOBERRLOG # or use defaults else JOBLOG=${JOBLOG-${JOBQUEUEDIR}/${JOBNAME}'.$(Cluster).$(Process).out'} JOBERRLOG=$JOBLOG fi else # otherwise make sure that defaults are set if [ -z "$JOBLOG" ] ; then JOBLOG=${JOBQUEUEDIR}/${JOBNAME}'.$(Cluster).$(Process).out' fi if [ -z "$JOBERRLOG" ] ; then JOBERRLOG=${JOBQUEUEDIR}/${JOBNAME}'.$(Cluster).$(Process).err' fi fi STRNG="${STRNG} > ${JOBLOG} 2> ${JOBERRLOG}" # submit the job set -x eval "$STRNG &" # sleep 10 sleep 5 if [ -n "$RMFILES" ] ; then # ((sleep 600 ; rm -f $RMFILES ) < /dev/null > /dev/null 2>&1 &) # ((sleep 240 ; rm -f $RMFILES ) < /dev/null > /dev/null 2>&1 &) case ${HOSTID} in c1f*|c1h*|c1r*|c1s*) Aixtrgt='brooks' ;; c2f*|c2h*|c2r*|c2s*) Aixtrgt='hare' ;; # c3f*) Aixtrgt='rigel' ;; c4f*) Aixtrgt='maia' ;; c7f*) Aixtrgt='zeta' ;; c8f*) Aixtrgt='algol' ;; c6f*) Aixtrgt='saiph' ;; ib*) Aixtrgt='dorval-ib' ;; al*) Aixtrgt='alef' ;; *) Aixtrgt='azur' ;; esac RmtCmd="ssh ${Aixtrgt} 'rm -f ""$RMFILES""'" echo $RmtCmd | ssh pollux "at now + 06 minutes" sleep 5 unset RmtCmd fi echo "Note: The job is submitted to run in the background on $HOSTID" fi # [ nonqs ] exit $Exit