#! /bin/sh
 
#    Dec 03/08 - F.Majaess
 
#id  felxq   - Used to check the batch queues on Linux clusters
 
#    AUTHOR  - F.Majaess
 
#hd  PURPOSE - "felxq" script is used to check the batch queues on 
#hd            "hare","brooks","brooks-test","daley" or "eccc-ppp[1-4]" 
#hd            Linux cluster at "ri" seconds repeat interval and up to 
#hd            "nt" times.
 
#pr  PARAMETERS:
#pr
#pr    PRIMARY
#pr
#pr      nt     = number of times to perform the check (=100)
#pr      ri     = interval time between the checks in seconds (=20)
#pr      dy     = interval delay time between NODES of a given 
#pr               check in seconds (=2)
#pr      target = target cluster/node to do the check on.
#pr               Valid options on Science: hare/hr,brooks/br,brooks-test/brt,
#pr                                         daley/dy,
#pr                                         eccc-ppp1/ppp1/p1,eccc-ppp2/ppp2/p2,
#pr                                         eccc-ppp3/ppp3/p3,eccc-ppp4/ppp4/p4.
#pr               ="local host")
#pr       user  = username - used only in conjunction with the "own"
#pr               parameter below (=$USER)
#pr
#pr   SECONDARY
#pr
#pr        own = switch to list only user's own files
#pr
#pr       wide = switch to enable wide column format
#pr              (=no/yes)
#pr
 
#ex  EXAMPLE: 
#ex
#ex    felxq nt=5 
#ex
#ex  The above will result in performing the check up to 5 times
#ex  at the default "ri" interval.
#ex  
#ex    felxq nt=5 p2
#ex
#ex  The above will result in performing the check up to 5 times
#ex  at the default "ri" interval on eccc-ppp2.
#ex  
#ex    felxq wide nt=5 brt
#ex
#ex  The above will result in performing the check up to 5 times
#ex  at the default "ri" interval on brooks-test with wide format 
#ex  enabled for "job name" column.
#ex  

FELXQT="$0" 
# FELXQT=`echo $FELXQT | sed -e 's/^\/home\//\~/'`
# echo "FELXQT=$FELXQT"
dirctv="$*"
[ -z "$CCRNSRC" ] && echo '' && echo '--- CCCma default setup invoked' &&  . /home/scrd101/generic/sc_cccma_setup_profile || : 
OS=${OS:-`uname -s`}
unset target
set -- $dirctv
# Check and possibly adjust for parameters specified on the 
# script call ...

if [ $# -gt 0 ] ; then
  for arg in $@
  do
    case $arg in
      -*) set $arg            ;;
     *=*) eval $arg           ;;
     own) own=yes             ;;
    wide) wide=yes            ;;
     *  ) if [ -z "$target" ] ; then
           target="$arg"
          else
           target="$target $arg" 
          fi ;;
    esac
  done
fi
HOSTID=${HOSTID:-`/bin/hostname| cut -d'.' -f1 `}
HOSTID=`echo $HOSTID | sed -e 's/hpcr4-in/ppp4/' -e 's/hpcr3-in/ppp3/' `

HOSTIDf6=`echo $HOSTID | sed -n -e 's/^ppp\(.\)-sh*.*$/bp\1/p' | cut -c 1-6`
HOSTIDf6=${HOSTIDf6:-`echo $HOSTID | sed -n -e 's/^ppp\(.\)-bionic*.*$/bp\1/p' | cut -c 1-6`}
HOSTIDf6=${HOSTIDf6:-`echo $HOSTID | sed -n -e 's/eccc.*-ppp/cs/gp' | cut -c 1-6`}
HOSTIDf6=${HOSTIDf6:-`echo $HOSTID | sed -e 's/^ppp/cs/g' | cut -c 1-6`}

# HOSTIDf=`echo $HOSTID | sed -n -e 's/^ppp\(.\)-sh*.*$/bp\1/p' | cut -c 1-3`
# HOSTIDf=${HOSTIDf:-`echo $HOSTID | sed -n -e 's/^ppp\(.\)-bionic*.*$/bp\1/p' | cut -c 1-3`}
# HOSTIDf=${HOSTIDf:-`echo $HOSTID | sed -n -e 's/eccc.*-ppp/cs/gp' | cut -c 1-3`}
# HOSTIDf=${HOSTIDf:-`echo $HOSTID | sed -e 's/^ppp/cs/g' | cut -c 1-3`}

HOSTIDf=`echo $HOSTIDf6 | cut -c 1-3`

if [ -z "$target" ] ; then
 if [ "$OS" = 'Linux' -a "$HOSTIDf" = 'xc4' ] ; then
  target='daley'
 elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'xc3' ] ; then
  target='banting'
 elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'xc1' ] ; then
  target='hare'
 elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'xc2' ] ; then
  if [ "$HOSTIDf6" = 'xc2net' ] ; then
    target='brooks-test'
  else
    target='brooks'
  fi
 elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'cs4' ] ; then
  target='eccc-ppp4'
 elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'cs3' ] ; then
  target='eccc-ppp3'
 elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'cs1' ] ; then
  target='eccc-ppp1'
 elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'cs2' ] ; then
  target='eccc-ppp2'
 elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'bp1' ] ; then
  target='ppp1-bionic'
 elif [ "$OS" = 'Linux' -a "$HOSTIDf" = 'bp2' ] ; then
  target='ppp2-bionic'
 else
  # target='pollux'
  echo "Sorry, $target cluster must be specified!"
  exit 1
 fi
fi
if [ "$target" = 'dy' -o "$target" = 'daley' ] ; then
 target='daley'
elif [ "$target" = 'ba' -o "$target" = 'banting' ] ; then
 target='banting'
elif [ "$target" = 'hr' -o "$target" = 'hare' ] ; then
 target='hare'
elif [ "$target" = 'brt' -o "$target" = 'brooks-test' ] ; then
 target='brooks-test'
elif [ "$target" = 'br' -o "$target" = 'brooks' ] ; then
 target='brooks'
elif [ "$target" = 'p4' -o "$target" = 'ppp4' -o "$target" = 'eccc-ppp4' ] ; then
 target='eccc-ppp4'
elif [ "$target" = 'p3' -o "$target" = 'ppp3' -o "$target" = 'eccc-ppp3' ] ; then
 target='eccc-ppp3'
elif [ "$target" = 'p1' -o "$target" = 'ppp1' -o "$target" = 'eccc-ppp1' ] ; then
 target='eccc-ppp1'
elif [ "$target" = 'p2' -o "$target" = 'ppp2' -o "$target" = 'eccc-ppp2' ] ; then
 target='eccc-ppp2'
elif [ "$target" = 'bp1' -o "$target" = 'ppp1-bionic' ] ; then
 target='ppp1-bionic'
elif [ "$target" = 'bp2' -o "$target" = 'ppp2-bionic' ] ; then
 target='ppp2-bionic'
else
 echo "Sorry, $target is invalid!" 
 exit 1
fi 

# if [ "$OS" != 'Linux' -o \( "$OS" = 'Linux' -a \( \( "$HOSTIDf" = 'usr' -o "$HOSTIDf" = 'sci' \) -o \( "$HOSTIDf" = 'xc1' -a "$target" != 'hare' \) -o \( "$HOSTIDf" = 'xc2' -a "$HOSTIDf6" != 'xc2net' -a "$target" != 'brooks' \) -o \( "$HOSTIDf6" = 'xc2net' -a "$target" != 'brooks-test' \) -o \( "$HOSTIDf" = 'cs1' -a "$target" != 'eccc-ppp1' \) -o \( "$HOSTIDf" = 'cs2' -a "$target" != 'eccc-ppp2' \) \) \)  ] ; then
# if [ "$OS" != 'Linux' -o \( "$OS" = 'Linux' -a \( \( "$HOSTIDf" = 'usr' -o "$HOSTIDf" = 'sci' \) -o \( "$HOSTIDf" = 'xc1' -a "$target" != 'hare' \) -o \( "$HOSTIDf" = 'xc2' -a "$HOSTIDf6" != 'xc2net' -a "$target" != 'brooks' \) -o \( "$HOSTIDf6" = 'xc2net' -a "$target" != 'brooks-test' \) -o \( "$HOSTIDf" = 'cs1' -a "$target" != 'eccc-ppp1' \) -o \( "$HOSTIDf" = 'cs2' -a "$target" != 'eccc-ppp2' \) -o \( "$HOSTIDf" = 'bp1' -a "$target" != 'ppp1-bionic' \) -o \( "$HOSTIDf" = 'bp2' -a "$target" != 'ppp2-bionic' \) \) \)  ] ; then
if [ "$OS" != 'Linux' -o \( "$OS" = 'Linux' -a \( \( "$HOSTIDf" = 'usr' -o "$HOSTIDf" = 'sci' \) -o \( "$HOSTIDf" = 'xc1' -a "$target" != 'hare' \) -o \( "$HOSTIDf" = 'xc2' -a "$HOSTIDf6" != 'xc2net' -a "$target" != 'brooks' \) -o \( "$HOSTIDf6" = 'xc2net' -a "$target" != 'brooks-test' \) -o \( "$HOSTIDf" = 'xc3' -a "$target" != 'banting' \) -o \( "$HOSTIDf" = 'xc4' -a "$target" != 'daley' \) -o \( "$HOSTIDf" = 'cs1' -a "$target" != 'eccc-ppp1' \) -o \( "$HOSTIDf" = 'cs2' -a "$target" != 'eccc-ppp2' \) -o \( "$HOSTIDf" = 'cs3' -a "$target" != 'eccc-ppp3' \) -o \( "$HOSTIDf" = 'cs4' -a "$target" != 'eccc-ppp4' \) -o \( "$HOSTIDf" = 'bp1' -a "$target" != 'ppp1-bionic' \) -o \( "$HOSTIDf" = 'bp2' -a "$target" != 'ppp2-bionic' \) \) \)  ] ; then
 if [ "$SITE_ID" = 'DrvlSC' ] ; then
  # exec echo "echo felxq $*" | ssh $target "bash -l"
  # exec ssh $target "felxq $*"
  # echo "FELXQT=$FELXQT"
  # set -x
  exec ssh $target "$FELXQT $*"
 else
  exec ssh $target "felxq $*"
 fi
 
fi

# Set the other defaults.
 
AWK=${AWK:='awk'} ; export AWK
nt=${nt:='500'}
ri=${ri:='20'}
dy=${dy:='2'}
own=${own:='no'}
wide=${wide:='no'}
# if [ "$wide" = 'yes' ] ; then
#  echo "Warning: wide option is currently disabled"
#  wide='no'
# fi
old=${old:='no'}
user=${user:=$USER}
if [ "$wide" = 'yes' -a "$SITE_ID" = 'DrvlSC' ] ; then
 line1="                                                                                                            Req'd     Req'd      Elap"
 line2='Job ID               Username Queue           Jobname                                            NDS  TSK   Memory    Time     S Time'
 line3='-------------------- -------- --------------- -------------------------------------------------- ---- ----- --------- -------- - --------'
#line3='12345678901234567890 12345678 123456789012345 12345678901234567890123456789012345678901234567890 1234 12345 123456789 12345678 1 12345678'
 # if [ "$HOSTIDf" = 'cs1' -o "$HOSTIDf" = 'cs2' ] ; then
 #  Optns=' -- -f 2>>/dev/null '
 # fi
 if [ "$own" = 'yes' ] ; then
  #Optns="$Optns -w -n -1 "
  FilTR="egrep ' ${user} '"
 else
  # Optns="$Optns -w -n -1 -f " # "-n -1" Can produce long list of nodes...
  # Optns="$Optns -w -f "
  # Optns="$Optns -w -a "
  FilTR=' cat '
 fi
# elif [ "$wide" = 'yes' -a "$SITE_ID" = 'DrvlSC' -a \( "$HOSTIDf" = 'cs1' -o "$HOSTIDf" = 'cs2' \) ] ; then
#  echo "Note - 'wide' option not supported yet on ppp and is ignored"
#  wide='no'
fi
if [ "$own" = 'yes' ] ; then
#if [ "$SITE_ID" = 'DrvlSC' -a \( "$HOSTIDf" = 'cs1' -o "$HOSTIDf" = 'cs2' \) ] ; then
# Optns="$Optns -u $user | egrep '"'^info:|^Job id|^-------|'" $user '"
#else
  Optns="$Optns -u $user"
#fi
fi

#set -x
# if [ "$HOSTIDf" = 'xc1' -o "$HOSTIDf" = 'xc2' -o "$HOSTIDf" = 'cs1' ] ; then
if [ "$HOSTIDf" = 'xc1' -o "$HOSTIDf" = 'xc2' -o "$HOSTIDf" = 'xc3' -o "$HOSTIDf" = 'xc4' ] ; then
 cmd=${cmd:="$Optns"}
 FEid_list="$target"
elif [ "$HOSTIDf" = 'cs1' -o "$HOSTIDf" = 'cs2' -o "$HOSTIDf" = 'cs3' -o "$HOSTIDf" = 'cs4' ] ; then
 cmd=${cmd:="$Optns"}
 FEid_list="$target"
elif [ "$HOSTIDf" = 'bp1' -o "$HOSTIDf" = 'bp2' ] ; then
 cmd=${cmd:="$Optns"}
 FEid_list="$target"
else
 echo "Sorry, 'felxq' is not valid on current platform!"
 exit 1
fi 
 
FEid_list=`echo $FEid_list | sed -e 's/eccc-//g'`

# set -x
#   ****   Task of the script...   ****
 
# * Perform the check "nt" times at "ri" sec. repeat interval.
# if [ "$wide" = 'yes' -a "$SITE_ID" != 'DrvlSC' ] ; then
#  export SGE_LONG_QNAMES=-1
# fi

# dashuse=`jobst -h | egrep -- '^--format' | sed -e 's/^--.*/skip/'`
# dashuse=${dashuse:-'use'}
dashuse=`jobst -h | egrep -- '^--format' | sed -e 's/^--.*/  /'`
dashuse=${dashuse:-'--'}
 
while [ $nt -gt 0 ] 
do
  echo " " ; echo "As of `date`:" ; echo " "
  for FEid in $FEid_list
    do
     echo "=== ${FEid} check:"
    #  set -x
     if [ "$wide" = 'yes' ] ; then
       echo "$line1" ; echo "$line2" ; echo "$line3"
#      if [ "$target" = 'daley' ] ; then
	#jobst -c $FEid -f 2>>/dev/null | egrep -v '^info:' | egrep 'job_id=|job_name=|job_owner=|resources_used.walltime=|job_state=|queue=|exec_host=|resource_list.mem=|resource_list.ncpus=|resource_list.nodect=|resource_list.walltime=' | sed -e 's/@.*$//g' -e 's/=/ = /' | $AWK ' BEGIN { cntr=0 ; } { if ( $1 == "resource_list.ncpus" ) { TSK=$3 ; getline ; RLWT=$3 ; getline ; JID=$3 ; getline ; JS=$3 ; getline ; NDS=$3 ; getline ;  OWNR=$3 ; getline ; JN=substr($3,1,50) ;  getline ; if ( $1 == "resources_used.walltime" ) { RUWT=$3 ; getline ; } else { RUWT="--:--:--" ; } ;  if ( $1 == "exec_host" ) { Ppos=index($3,"+") ; if ( Ppos > 1 ) { EXHST=substr($3,1,Ppos) ; } else { EXHST=$3 ;} ;  getline ; } else { EXHST=" " ; } ; if ( $1 == "resource_list.mem" ) { RLMEM=$3 ; getline ; } else { RLMEM="--" ; }  ; QU="---" ; if ( $1 == "queue" ) { QU=$3 ; } ; cntr=cntr+1 ; printf "%-20s%1s%-8s%1s%-15s%1s%-50s%1s%4s%1s%5s%1s%9s%1s%-s%1s%-s%1s%-s%1s%s\n",JID," ",OWNR," ",QU," ",JN," ",NDS," ",TSK," ",RLMEM," ",RLWT," ",JS," ",RUWT," ",EXHST ; } ; } '  | sort -b -k3,3 -k2,2 | eval $FilTR
#         qstat -f 2>>/dev/null | egrep -v '^info:' | egrep '^Job Id:|Job_Name =|Job_Owner =|resources_used.walltime =|job_state =|queue =|exec_host =|Resource_List.mem =|Resource_List.ncpus =|Resource_List.nodect =|Resource_List.walltime =' | sed -e 's/@.*$//g' | $AWK ' BEGIN { cntr=0 ; RLWT="--:--:--" ; RUWT="--:--:--" ; EXHST=" " ; RLMEM="--" ; QU="---" ; empty="yes" ; } { if ( $1 == "Job" && $2 == "Id:" ) { if ( empty == "no" ) { cntr=cntr+1 ; printf "%-20s%1s%-8s%1s%-15s%1s%-50s%1s%4s%1s%5s%1s%9s%1s%-s%1s%-s%1s%-s%1s%s\n",JID," ",OWNR," ",QU," ",JN," ",NDS," ",TSK," ",RLMEM," ",RLWT," ",JS," ",RUWT," ",EXHST ; RLWT="--:--:--" ; RUWT="--:--:--" ; EXHST=" " ; RLMEM="--" ; QU="---" ; empty="yes" ; } ; JID=$3 ; empty="no" ; } else { if (  $1 == "Resource_List.walltime" ) { RLWT=$3 ; } ; if (  $1 == "Resource_List.ncpus" ) { TSK=$3 ; } ; if (  $1 == "Job_Name" ) { JN=substr($3,1,50) ; } ; if (  $1 == "job_state" ) { JS=$3 ; } ; if (  $1 == "Resource_List.nodect" ) { NDS=$3 ; } ; if (  $1 == "Job_Owner" ) { OWNR=$3 ; } ; if (  $1 == "exec_host" ) { Ppos=index($3,"+") ; if ( Ppos > 1 ) { EXHST=substr($3,1,Ppos) ; } else { EXHST=$3 ;} ; } ; if (  $1 == "resources_used.walltime" ) { RUWT=$3 ; } ; if (  $1 == "Resource_List.mem" ) { RLMEM=$3 ; } ; if (  $1 == "queue" ) { QU=$3 ; } ; } ; } END { if ( empty == "no" ) { cntr=cntr+1 ; printf "%-20s%1s%-8s%1s%-15s%1s%-50s%1s%4s%1s%5s%1s%9s%1s%-s%1s%-s%1s%-s%1s%s\n",JID," ",OWNR," ",QU," ",JN," ",NDS," ",TSK," ",RLMEM," ",RLWT," ",JS," ",RUWT," ",EXHST ; } ; } ' | sort -b -k3,3 -k2,2 | eval $FilTR
#      else
        # if [ "$FEid" = 'eccc-ppp1' -o "$FEid" = 'eccc-ppp2' -o "$FEid" = 'ppp1' -o "$FEid" = 'ppp2' -o "$FEid" = 'brooks' -o "$FEid" = 'hare' -o "$FEid" = 'brooks-test' ] ; then
        if [ "$dashuse" = '--' ] ; then
         # jobst -c $FEid -- -f 2>>/dev/null | egrep -v '^info:' | egrep '^Job Id:|Job_Name =|Job_Owner =|resources_used.walltime =|job_state =|queue =|exec_host =|Resource_List.mem =|Resource_List.ncpus =|Resource_List.nodect =|Resource_List.walltime =' | sed -e 's/@.*$//g' | $AWK ' BEGIN { cntr=0 ; } { if ( $1 == "Job" && $2 == "Id:" ) { JID=$3 ; getline ; JN=substr($3,1,50) ; getline ; OWNR=$3 ; getline ; if ( $1 == "resources_used.walltime" ) { RUWT=$3 ; getline ; } else { RUWT="--:--:--" ; } ;  JS=$3 ; getline ; QU=$3 ; getline ; if ( $1 == "exec_host" ) { Ppos=index($3,"+") ; if ( Ppos > 1 ) { EXHST=substr($3,1,Ppos) ; } else { EXHST=$3 ;} ;  getline ; } else { EXHST=" " ; } ; if ( $1 == "Resource_List.mem" ) { RLMEM=$3 ; getline ; } else { RLMEM="--" ; }  ; TSK=$3 ; getline ; NDS=$3 ; getline ; RLWT=$3 ; cntr=cntr+1 ; printf "%-20s%1s%-8s%1s%-15s%1s%-50s%1s%4s%1s%5s%1s%9s%1s%-s%1s%-s%1s%-s%1s%s\n",JID," ",OWNR," ",QU," ",JN," ",NDS," ",TSK," ",RLMEM," ",RLWT," ",JS," ",RUWT," ",EXHST ; } ; } '  | sort -b -k3,3 -k2,2 | eval $FilTR
         jobst -c $FEid -- -f 2>>/dev/null | egrep -v '^info:' | egrep '^Job Id:|Job_Name =|Job_Owner =|resources_used.walltime =|job_state =|queue =|exec_host =|Resource_List.mem =|Resource_List.ncpus =|Resource_List.nodect =|Resource_List.walltime =' | sed -e 's/@.*$//g' | $AWK ' BEGIN { cntr=0 ; RLWT="--:--:--" ; RUWT="--:--:--" ; EXHST=" " ; RLMEM="--" ; QU="---" ; empty="yes" ; } { if ( $1 == "Job" && $2 == "Id:" ) { if ( empty == "no" ) { cntr=cntr+1 ; printf "%-20s%1s%-8s%1s%-15s%1s%-50s%1s%4s%1s%5s%1s%9s%1s%-s%1s%-s%1s%-s%1s%s\n",JID," ",OWNR," ",QU," ",JN," ",NDS," ",TSK," ",RLMEM," ",RLWT," ",JS," ",RUWT," ",EXHST ; RLWT="--:--:--" ; RUWT="--:--:--" ; EXHST=" " ; RLMEM="--" ; QU="---" ; empty="yes" ; } ; JID=$3 ; empty="no" ; } else { if (  $1 == "Resource_List.walltime" ) { RLWT=$3 ; } ; if (  $1 == "Resource_List.ncpus" ) { TSK=$3 ; } ; if (  $1 == "Job_Name" ) { JN=substr($3,1,50) ; } ; if (  $1 == "job_state" ) { JS=$3 ; } ; if (  $1 == "Resource_List.nodect" ) { NDS=$3 ; } ; if (  $1 == "Job_Owner" ) { OWNR=$3 ; } ; if (  $1 == "exec_host" ) { Ppos=index($3,"+") ; if ( Ppos > 1 ) { EXHST=substr($3,1,Ppos) ; } else { EXHST=$3 ;} ; } ; if (  $1 == "resources_used.walltime" ) { RUWT=$3 ; } ; if (  $1 == "Resource_List.mem" ) { RLMEM=$3 ; } ; if (  $1 == "queue" ) { QU=$3 ; } ; } ; } END { if ( empty == "no" ) { cntr=cntr+1 ; printf "%-20s%1s%-8s%1s%-15s%1s%-50s%1s%4s%1s%5s%1s%9s%1s%-s%1s%-s%1s%-s%1s%s\n",JID," ",OWNR," ",QU," ",JN," ",NDS," ",TSK," ",RLMEM," ",RLWT," ",JS," ",RUWT," ",EXHST ; } ; } ' | sort -b -k3,3 -k2,2 | eval $FilTR
        else
        #jobctl-qstat -f 2>>/dev/null | egrep -v '^info:' | egrep '^Job Id:|Job_Name =|Job_Owner =|resources_used.walltime =|job_state =|queue =|exec_host =|Resource_List.mem =|Resource_List.ncpus =|Resource_List.nodect =|Resource_List.walltime =' | sed -e 's/@.*$//g' | $AWK ' BEGIN { cntr=0 ; RLWT="--:--:--" ; RUWT="--:--:--" ; EXHST=" " ; RLMEM="--" ; QU="---" ; empty="yes" ; } { if ( $1 == "Job" && $2 == "Id:" ) { if ( empty == "no" ) { cntr=cntr+1 ; printf "%-20s%1s%-8s%1s%-15s%1s%-50s%1s%4s%1s%5s%1s%9s%1s%-s%1s%-s%1s%-s%1s%s\n",JID," ",OWNR," ",QU," ",JN," ",NDS," ",TSK," ",RLMEM," ",RLWT," ",JS," ",RUWT," ",EXHST ; RLWT="--:--:--" ; RUWT="--:--:--" ; EXHST=" " ; RLMEM="--" ; QU="---" ; empty="yes" ; } ; JID=$3 ; empty="no" ; } else { if (  $1 == "Resource_List.walltime" ) { RLWT=$3 ; } ; if (  $1 == "Resource_List.ncpus" ) { TSK=$3 ; } ; if (  $1 == "Job_Name" ) { JN=substr($3,1,50) ; } ; if (  $1 == "job_state" ) { JS=$3 ; } ; if (  $1 == "Resource_List.nodect" ) { NDS=$3 ; } ; if (  $1 == "Job_Owner" ) { OWNR=$3 ; } ; if (  $1 == "exec_host" ) { Ppos=index($3,"+") ; if ( Ppos > 1 ) { EXHST=substr($3,1,Ppos) ; } else { EXHST=$3 ;} ; } ; if (  $1 == "resources_used.walltime" ) { RUWT=$3 ; } ; if (  $1 == "Resource_List.mem" ) { RLMEM=$3 ; } ; if (  $1 == "queue" ) { QU=$3 ; } ; } ; } END { if ( empty == "no" ) { cntr=cntr+1 ; printf "%-20s%1s%-8s%1s%-15s%1s%-50s%1s%4s%1s%5s%1s%9s%1s%-s%1s%-s%1s%-s%1s%s\n",JID," ",OWNR," ",QU," ",JN," ",NDS," ",TSK," ",RLMEM," ",RLWT," ",JS," ",RUWT," ",EXHST ; } ; } ' | sort -b -k3,3 -k2,2 | eval $FilTR
	 jobst -c $FEid -f 2>>/dev/null | egrep -v '^info:' | egrep 'job_id=|job_name=|job_owner=|resources_used.walltime=|job_state=|queue=|exec_host=|resource_list.mem=|resource_list.ncpus=|resource_list.nodect=|resource_list.walltime=' | sed -e 's/@.*$//g' -e 's/=/ = /' | $AWK ' BEGIN { cntr=0 ; RLWT="--:--:--" ; RUWT="--:--:--" ; EXHST=" " ; RLMEM="--" ; QU="---" ; empty="yes" ; } { if ( $1 == "resource_list.ncpus" ) { if ( empty == "no" ) { cntr=cntr+1 ; printf "%-20s%1s%-8s%1s%-15s%1s%-50s%1s%4s%1s%5s%1s%9s%1s%-s%1s%-s%1s%-s%1s%s\n",JID," ",OWNR," ",QU," ",JN," ",NDS," ",TSK," ",RLMEM," ",RLWT," ",JS," ",RUWT," ",EXHST ; RLWT="--:--:--" ; RUWT="--:--:--" ; EXHST=" " ; RLMEM="--" ; QU="---" ; empty="yes" ; } ; TSK=$3 ; empty="no" ; } else { if (  $1 == "resource_list.walltime" ) { RLWT=$3 ; } ; if (  $1 == "job_id" ) { JID=$3 ; } ; if (  $1 == "job_name" ) { JN=substr($3,1,50) ; } ; if (  $1 == "job_state" ) { JS=$3 ; } ; if (  $1 == "resource_list.nodect" ) { NDS=$3 ; } ; if (  $1 == "job_owner" ) { OWNR=$3 ; } ; if (  $1 == "exec_host" ) { Ppos=index($3,"+") ; if ( Ppos > 1 ) { EXHST=substr($3,1,Ppos) ; } else { EXHST=$3 ;} ; } ; if (  $1 == "resources_used.walltime" ) { RUWT=$3 ; } ; if (  $1 == "resource_list.mem" ) { RLMEM=$3 ; } ; if (  $1 == "queue" ) { QU=$3 ; } ; } ; } END { if ( empty == "no" ) { cntr=cntr+1 ; printf "%-20s%1s%-8s%1s%-15s%1s%-50s%1s%4s%1s%5s%1s%9s%1s%-s%1s%-s%1s%-s%1s%s\n",JID," ",OWNR," ",QU," ",JN," ",NDS," ",TSK," ",RLMEM," ",RLWT," ",JS," ",RUWT," ",EXHST ; } ; } ' | sort -b -k3,3 -k2,2 | eval $FilTR
        fi
#      fi
     else
      # if [ "$FEid" = 'eccc-ppp1' -o "$FEid" = 'eccc-ppp2' -o "$FEid" = 'ppp1' -o "$FEid" = 'ppp2' ] ; then
      if [ "$FEid" = 'eccc-ppp1' -o "$FEid" = 'eccc-ppp2' -o "$FEid" = 'eccc-ppp3' -o "$FEid" = 'eccc-ppp4' -o "$FEid" = 'ppp1' -o "$FEid" = 'ppp2' -o "$FEid" = 'ppp3' -o "$FEid" = 'ppp4' -o "$FEid" = 'ppp1-bionic' -o "$FEid" = 'ppp2-bionic' ] ; then
       # if [ -n "$cmd" ] ; then
       #  eval "jobst -c $FEid -- $cmd"
       # else   
       #  eval "jobst -c $FEid $cmd"
       # fi 
       eval "jobst -c $FEid ${dashuse} $cmd"
      else
       eval "qstat $cmd"
      fi
     fi
#    echo "************************************************************" ; echo ""
#    echo "" ; echo ""
     echo ""
     if [ "$dy" -ge 1 ] ; then
      sleep ${dy}
     fi
    done
  
  if [ $nt -gt 1 ] ; then
    sleep ${ri}
  fi
  nt=`expr $nt - 1`
  echo " " ; echo " Remaining checks: ${nt} times at ${ri} sec. interval " ; echo " "
done