#! /bin/sh
 
#    Mar 20/09 - F.Majaess
 
#id  lnx2vic- Transfer data from CMC Linux cluster to Victoria.
 
#    AUTHOR  - F.Majaess
 
#hd  PURPOSE - "lnx2vic" script is used to transfer "target" from local 
#hd            platform "tarpath" directory to Victoria node "despath" 
#hd            directory.
#hd            NOTE : Edition number is added via "save" script if 
#hd                   "nooffsv=no" or if "target" is an official file.
#hd                   Removal of "target" is controlled by "rmv" parameter. 
#hd                   If "despath" is used to override RUNPATH use from 
#hd                   Victoria's side, ensure the path starts with 
#hd                   "/raid" prefix.
 
#pr  PARAMETERS:
#pr
#pr    POSITIONAL     
#pr
#pr      target = name of target file/subdirectory to transfer. 
#pr               (same name is used on destination machine) 
#pr
#pr    PRIMARY/SECONDARY
#pr
#pr      nooffsv= switch controlling saving on destination via "save"
#pr               script (=no/yes)
#pr      rmv    = switch controls deletion of "target" once the transfer
#pr               is complete (=no/yes). 
#pr      nosub  = switch controlling running the job as part of the 
#pr               current job instead of as a new separate job 
#pr               (=no/yes)
#pr      noprint= switch controls output generation; "yes" (or "on") no output
#pr               (=no/yes).
#pr      addhed = add highest edition found to "target" provided the latter
#pr               doesn't already have edition number suffix and "nooffsv=yes"
#pr               (=no/yes)
#pr
#pr    PRIMARY
#pr
#pr      tarpath= subdirectory in which "target" data is residing 
#pr               on local platform (=$RUNPATH).
#pr      despath= subdirectory in which the transfered data get stored
#pr               in Victoria (=$RUNPATH on Victoria's setup).
#pr               Note: Ensure path specified starts with "/raid" prefix.
#pr      time   = nqs CPU time in seconds (=300)
#pr      mfl    = nqs MFL                 (=50MB)
#pr      vmode  = access code on "target" (=640)
 
#ex  EXAMPLE: 
#ex
#ex      lnx2vic rmv=yes mydirectory 
#ex
#ex    The above command requests tansferring subdirectory 
#ex    "mydirectory" to Victoria and then removing it from 
#ex    local platform.
#ex
#ex      lnx2vic mxx99zz.001 
#ex
#ex    The above command requests transferring "mxx99zz.001" 
#ex    file from the user's RUNPATH subdirectory on the local 
#ex    platform to Victoria node and then officially save it.
#ex
#ex      lnx2vic tarpath=/data/ccrn/lnx0 sc_cad_1850_2000_ctem_rtd.001
#ex
#ex    The above command requests transferring 
#ex    "sc_cad_1850_2000_ctem_rtd.001" file from /data/ccrn/lnx0 
#ex    subdirectory path on the local platform to Victoria node 
#ex    and then officially save it.
#ex
#ex         lnx2vic myfiley despath=/raid/rd40/data addhed=yes
#ex
#ex    The above will result in transferring "myfiley" with
#ex    the highest edition number found into /rd40/data in
#ex    Victoria and perform official "save" on it.
# set -x
 
 
#   * code used to set switches and deal with parameter=value arguments 
#   * as well as setting parameters to their primary/secondary defaults.
#   * The list of other arguments (if any) is returned in "prmtrl" var...
 
#  *  check if switches/parameters specified on script call
 
if [ $# -gt 0 ] ; then
  
#  * put argument list in "prmtrl" ...
   
  prmtrl=$@
# prmtrl=`echo $prmtrl | tr '[A-Z]' '[a-z]'`
  
#  * for each argument specified do...
    
  for prmtr in $prmtrl
  do
    case $prmtr in
 
#   * check for standard shell switches and set them accordingly
 
     --|-|[-+][aefhkntuvx]) set $prmtr ;;
 
#   * case where a parameter is to be assigned a value
 
    *=*   ) eval $prmtr ;;
 
#   * check for keyword(s) then 
#   *     if the corresponding parameter is already set then
#   *        store "keyword" in "tprmtr" so that it is treated 
#   *        later on as a value to a positional parameter
#   *     otherwise set parameter whose name is "keyword" to 
#   *     its secondary default.
 
    rmv ) if [ -n "$rmv" ] ; then
            tprmtr="$tprmtr $prmtr"
          else
            rmv='yes' 
          fi ;;
 
 nooffsv ) if [ -n "$nooffsv" ] ; then
            tprmtr="$tprmtr $prmtr"
          else
            nooffsv='yes' 
          fi ;;

  nosub ) if [ -n "$nosub" ] ; then
            tprmtr="$tprmtr $prmtr"
          else
            nosub='yes' 
          fi ;;

  noprint) if [ -n "$noprint" ] ; then
            tprmtr="$tprmtr $prmtr"
          else
            noprint='yes'
          fi ;;

  addhed ) if [ -n "$addhed" ] ; then
            tprmtr="$tprmtr $prmtr"
          else
            addhed='yes'
          fi ;;
 
#   * anything else case accumulate in "tprmtr" temporary var...
 
    *  )  tprmtr="$tprmtr $prmtr" ;;
 
    esac
  done
  prmtrl=$tprmtr
  tprmtr=''
fi

 
#   * make sure arguments list variable "prmtrl" is initialized...
 
prmtrl=${prmtrl=}
  
#   * The following code deals with checking/setting positional 
#   * parameters ...
#   * Note: script will abort if too many arguments are specified.
 
if [ -n "$prmtrl" ] ; then
  i=1
  for prmtr in $prmtrl
  do
    case $i in
      1) target=$prmtr;;
      *) eval "echo '$0 : too many parameters !' "; exit 1;;
    esac
    i=`expr $i + 1`
  done
fi
 
#   * Prompt for a positional parameter value(s) if none was/were
#   * specified...
 
while [ -z "$target" ] ; do
  echo "please enter file\/subdirectory name ? > \\c"
  read target 
done
 
#   * code used to set hard coded primary defaults...
 
mtarget='ftp1'
COutputq=`ssh $CMCFEDEST ssh $VICSV01  'echo \\\$OUTPUTQ'`
time=${time='300'} 
mfl=${mfl='50MB'}
vmode=${vmode='640'}
dvmode=${dvmode='750'}
nooffsv=${nooffsv='no'}
noprint=${noprint='no'}
if [ "$noprint" = 'on' ] ; then
 noprint='yes'
elif [ "$noprint" = 'off' ] ; then
 noprint='no'
fi
rmv=${rmv='no'}
addhed=${addhed='no'}
nosub=${nosub='no'}
tarpath=${tarpath=$RUNPATH}
despath=${despath=`ssh $CMCFEDEST ssh $VICSV01  'echo \\\$RUNPATH'`}
nooffsv=`echo $nooffsv | tr '[A-Z]' '[a-z]'`
nosub=`echo $nosub | tr '[A-Z]' '[a-z]'`
rmv=`echo $rmv | tr '[A-Z]' '[a-z]'`
addhed=`echo $addhed | tr '[A-Z]' '[a-z]'`
mfl=`echo $mfl | tr '[a-z]' '[A-Z]'`

if [ "$addhed" = 'yes' ] ; then
 ed=`echo $target | sed -e 's/.*[.]\([0-9][0-9][0-9]\)/\1/'`
 if [ "$nooffsv" = 'yes' ] ; then
  echo "lnx2vic: Invalid use of nooffsv=$nooffsv with addhed=$addhed !"
  exit 1
 elif [ -n "$ed" -a "$ed" != "$target" ] ; then
  echo "lnx2vic: target=$target already has ed=$ed suffix!"
  exit 2
 else
  ttarget=`ls -1 $tarpath/${target}.[0-9][0-9][0-9] | sort | tail -1`
  if [ -n "$ttarget" ] ; then
   ed=`basename $ttarget | sed -e 's/.*[.]\([0-9][0-9][0-9]\)/\1/'`
   echo "lnx2vic: Adjusting $target to ${target}.${ed}"
   target="${target}.${ed}"
  else
   echo "lnx2vic: Sorry no matching ${target} filenmame with edition number suffix in $tarpath !"
   exit 3
  fi
 fi
fi
 
#   * Check if the file to be transfered, exist, readable and non-empty
#   * otherwise, issue an abort message and exit.
 
 
if [ ! -f "$tarpath/$target" -a ! -d  "$tarpath/$target/." ] ; then
 eval "echo $tarpath/$target is not valid !"
 exit 4
fi
 
cd $OUTPUTQ
Pid=$$
Lgname=${Lgname:-`logname`}
usr=`echo $Lgname | sed -e 's/^....//'`
Lhost=${HOSTID:=`hostname | cut -d'.' -f1`}
 
Outq=$OUTPUTQ
if [ "$nosub" != 'yes' ] ; then
 Mainfrm='ccrnftp' 
 sbmtcmd=" qsub -q $Mainfrm -r lnx2vic -eo -o "
 if [ "$noprint" = 'yes' ] ; then
  Outqfl="/dev/null"
 else
  Outqfl="$Outq/lnx2vic.${Lhost}.$target.outerr.$Pid" 
 fi
else
 Mainfrm='' 
 sbmtcmd=' sh -s  '
 if [ "$noprint" = 'yes' ] ; then
  Outqfl='1>/dev/null 2>&1 '
 else
  Outqfl='1>'"$Outq/lnx2vic.${Lhost}.$target.outerr.$Pid "'2>&1 ' 
 fi
 echo $sbmtcmd $Outqfl
fi

Tmpfl="/tmp/tmp.$usr.lnx2vic.ftptmp.$Pid"
Tmpflx="/tmp/tmp.$usr.lnx2vic.ftptmp.$Pid.X"
Ttarget=$target
if [ -d $tarpath/$target/. ] ; then
  targetdir='yes'
  cd $tarpath/$target 
  Tnlst=`echo *` 
  Nfils=`\ls -1 | wc -l`
  cd $OUTPUTQ
else
  targetdir='no'
  ed=`expr $target : '.*\.\([0-9][0-9][0-9]\)$'`
  Nfils='1'
  if [ -z "$ed" ] ; then
    Tnlst=$target
  else
    Tflnm=`echo $target | sed 's/....$//' | tr '[a-z]' '[A-Z]' `
    Ttarget=$Tflnm
    Tnlst=$Tflnm
    nooffsv='no'
    if [ "$rmv" = 'yes' -a "$tarpath/$target" != "$DATAPATH/$target" -a -h "$DATAPATH/$target" ] ; then
     OFFLink="$DATAPATH/$target"
    fi
  fi
fi
if [ "$nooffsv" != "yes" ] ; then
  besbmtcmd=" qsub -q ftppost -r lnx2vic -lT $time -lM $mfl -eo -o "
  if [ "$noprint" = 'yes' ] ; then
    beOutqfl="/dev/null"
  else
    beOutqfl="$COutputq/lnx2vic.Victoria.$target.outerr.$Pid"
  fi
fi
#   ****   Task of the script...   ****
 
#   * Assemble and submit the job ...
 
# set -x
# set -v
#cat << woofsub 
 cat << woofsub  | $sbmtcmd $Outqfl
#! /bin/sh
# set -xv
echo "0 lnx2vic: transferring $tarpath/$target : " &&
Tnlst="$Tnlst" &&
cd $tarpath  &&
\\ls -al $tarpath/$target &&
date | sed -n -e 's/^/0 lnx2vic : /p'  &&
## (ftp -i -v ftp1.cccma.ec.gc.ca << wooftp )  &&
## cd $despath
## quit 
## wooftp
ssh $CMCFEDEST "ssh ftp1.cccma.ec.gc.ca cd $despath" &&
if [ "$targetdir" = 'yes' ] ; then
 cd $tarpath/$target &&
 # ssh $CMCFEDEST 'ssh ftp1.cccma.ec.gc.ca "cd $despath ; mkdir $target"' &&
 ssh $CMCFEDEST "cd $tarpath; scp -r -p -o NoneSwitch=yes $target ftp1.cccma.ec.gc.ca:$despath/$Ttarget"  &&
 ssh $CMCFEDEST 'ssh ftp1.cccma.ec.gc.ca "cd $despath/$Ttarget ; \\ls -1"' | tail -$Nfils > $Tmpfl &&
 \\ls -1 > $Tmpflx
else
 cd $tarpath &&
 ssh $CMCFEDEST "cd $tarpath; scp -r -p -o NoneSwitch=yes $target ftp1.cccma.ec.gc.ca:$despath/$Ttarget"  &&
 ssh $CMCFEDEST 'ssh ftp1.cccma.ec.gc.ca "cd $despath ; \\ls -1 $Ttarget"' | tail -$Nfils > $Tmpfl &&
 echo "$Ttarget" > $Tmpflx
fi &&
date | sed -n -e 's/^/0 lnx2vic : /p'  &&
if [ -s "$Tmpfl" -a -s "$Tmpflx" ] ; then
 Continue='no' &&
 ( echo "$Tmpfl :" && cat $Tmpfl && echo "$Tmpflx :" && cat $Tmpflx || : ) &&
 cmp -s $Tmpfl $Tmpflx && Continue='yes'
fi &&
if [ "\$Continue" = 'yes' ] ; then 
 (\rm -f $Tmpfl $Tmpflx || : ) &&
 set -x &&
 if [ "$rmv" = "yes" ] ; then
  cd $tarpath && \rm -r -f $tarpath/$target $OFFLink 
 fi &&
 set +x 
else
 #chmod 770 $Tmpfl  &&
 (\rm -f $Tmpfl $Tmpflx || : ) &&
 exit 1
fi &&

if [ "$nooffsv" != "yes" ] ; then
# set -x &&
# ( cat << woofbesub | ssh ftp1.cccma.ec.gc.ca  "$besbmtcmd $beOutqfl " )
ssh $CMCFEDEST 'ssh ftp1.cccma.ec.gc.ca pwd' &&
# ssh $CMCFEDEST "ssh ftp1.cccma.ec.gc.ca $besbmtcmd $beOutqfl " << woofbesub
( cat << woofbesub ) | ( ssh $CMCFEDEST 'cat | ssh ftp1.cccma.ec.gc.ca "at now > ${beOutqfl} " ' ) &&
#! /bin/sh
 exec >> ${beOutqfl} 2>> ${beOutqfl}
# set -x
date | sed -n -e 's/^/0 lnx2vic : /p'  &&
echo "0 lnx2vic: transferring $tarpath/$target to Victoria " &&
cd $despath &&
RUNPATH=$despath &&
export RUNPATH &&
if [ "$targetdir" = 'yes' ] ; then
  chmod $dvmode $despath/$Ttarget &&
  cd $despath/$target
fi &&
Tnlst="$Tnlst" &&
chmod $vmode $Tnlst &&
echo " $despath/$Ttarget : " &&
\\ls -al $despath/$Ttarget &&
for prmtr in \\\$Tnlst
  do
   lprmtr=\\\`echo \\\$prmtr | tr '[A-Z]' '[a-z]' \\\` &&
   set -x &&
   save \\\$prmtr \\\$lprmtr &&
   release \\\$prmtr &&
   set +x &&
   :
  done &&
  cd $despath &&
  set -x &&
  if [ "$targetdir" = 'yes' ] ; then
   \rm -r -f $Ttarget
  else
   release $Ttarget 
  fi &&
  set +x &&
date | sed -n -e 's/^/0 lnx2vic : /p'  &&
exit ||
echo " abnormal termination" && exit 1 || exit 2
woofbesub
# set +x &&
: &&
echo "0 lnx2vic : Victoria output will be send to Victoria:$beOutqfl "
fi &&
exit ||
echo " abnormal termination" && exit 2 || exit 3
woofsub
date | sed -n -e 's/^/0 lnx2vic : /p'  &&
echo "0 lnx2vic: An nqs job is submitted, output will have $Pid suffix"  &&
exit || exit 5