#!/bin/bash
#=======================================================================
# --- xc2ppp ---
# Transfer files from the XC40 (lustre) to PPP (sitestore), save on PPP
#=======================================================================
#
#     keyword :: xc2ppp
# description :: Transfer files from the XC40 to PPP and save on PPP

set -a
. betapath2

  runid=xyz; year=$year;
  username=${username:-$invoking_user_name}; user=${user:-$invoking_user};
  uxxx=${uxxx:-${user:0:3}}; xc2ppp_uxxx=${uxxx:-mc}
  lopgm="LOPGM";
  xc2ppp_stime=''; stime=${xc2ppp_stime:-7200}; memory1="1gb";
  crawork=xc2ppp_$runid; nqsext="_${runid}_${year}";
  noprint=on

  # Use -e option if recognized by echo
  if [ "X`echo -e`" = "X-e" ]; then
    echo_e() { echo ${1+"$@"}; }
  else
    echo_e() { echo -e ${1+"$@"}; }
  fi

  stamp=`date "+%j%H%M%S"$$`

  # bail is a simple error exit routine
  # Note: we write the error directly to a file in ~/.queue so that this
  #       info is not lost if/when stdout is not returned
  error_out="${JHOME:-$HOME}/.queue/error_xc2ppp_${runid}_$stamp"
  [ ! -z "$error_out" ] && rm -f $error_out
  bail(){
    echo_e `date`" --- xc2ppp: $*"
    echo_e `date`" --- xc2ppp: $*" >>$error_out
    exit 1
  }
  warn(){
    echo_e `date`" --- xc2ppp: **WW** $*"
  }

#  * ........................... Parmsub Parameters ............................

  # These variables are set when the job string is created
  run_start_year=NotSet  # memory99=1
  run_start_month=NotSet # memory99=1
  run_stop_year=NotSet   # memory99=1
  run_stop_month=NotSet  # memory99=1
  previous_year=NotSet   # memory99=1
  previous_month=NotSet  # memory99=1
  current_year=NotSet    # memory99=1
  current_month=NotSet   # memory99=1
  next_year=NotSet       # memory99=1
  next_month=NotSet      # memory99=1
  # use eval to hide these assignments from cccjob substituion
  [ "$previous_year"  != "NotSet" ]  && eval with_previous_year\=1  || eval with_previous_year\=0
  [ "$previous_month" != "NotSet" ]  && eval with_previous_month\=1 || eval with_previous_month\=0
  [ "$current_year"  != "NotSet" ]   && eval with_current_year\=1   || eval with_current_year\=0
  [ "$current_month" != "NotSet" ]   && eval with_current_month\=1  || eval with_current_month\=0
  [ "$next_year"  != "NotSet" ]      && eval with_next_year\=1      || eval with_next_year\=0
  [ "$next_month" != "NotSet" ]      && eval with_next_month\=1     || eval with_next_month\=0
  prev_state="${with_previous_year}-${with_previous_month}"
  # Ensure that both previous_year and previous_month are defined or neither are defined
  case "$prev_state" in
    0-1) bail "previous_year=$previous_year but previous_month is not defined" ;;
    1-0) bail "previous_month=$previous_month but previous_year is not defined" ;;
      *) ;; # Anything else (e.g. 1-1 or 0-0) is valid
  esac
  curr_state="${with_current_year}-${with_current_month}"
  # Ensure that both current_year and current_month are defined or neither are defined
  case "$curr_state" in
    0-1) bail "current_year=$current_year but current_month is not defined" ;;
    1-0) bail "current_month=$current_month but current_year is not defined" ;;
      *) ;; # Anything else (e.g. 1-1 or 0-0) is valid
  esac
  next_state="${with_next_year}-${with_next_month}"
  # Ensure that both next_year and next_month are defined or neither are defined
  case "$next_state" in
    0-1) bail "next_year=$next_year but next_month is not defined" ;;
    1-0) bail "next_month=$next_month but next_year is not defined" ;;
      *) ;; # Anything else (e.g. 1-1 or 0-0) is valid
  esac
  # Define strings holding info about the default configuration for start/stop year/month
  default_state="${prev_state}-${curr_state}-${next_state}"

  # xc2ppp specific dates may be defined to override the defaults set above
  xc2ppp_start_year=''
  xc2ppp_start_month=''
  xc2ppp_stop_year=''
  xc2ppp_stop_month=''
  # If the user has supplied the start or stop year but not the month then assume
  # they want Jan for start year and Dec for stop year
  [ -n "$xc2ppp_start_year" -a -z "$xc2ppp_start_month" ] && eval xc2ppp_start_month\=1
  [ -n "$xc2ppp_stop_year"  -a -z "$xc2ppp_stop_month" ]  && eval xc2ppp_stop_month\=12
  # use eval to hide these assignments from cccjob substituion
  [ -n "$xc2ppp_start_year" ]  && eval with_start_year\=1  || eval with_start_year\=0
  [ -n "$xc2ppp_start_month" ] && eval with_start_month\=1 || eval with_start_month\=0
  [ -n "$xc2ppp_stop_year" ]   && eval with_stop_year\=1   || eval with_stop_year\=0
  [ -n "$xc2ppp_stop_month" ]  && eval with_stop_month\=1  || eval with_stop_month\=0
  # Ensure that both xc2ppp_start_year and xc2ppp_start_month are defined or neither are defined
  case "${with_start_year}-${with_start_month}" in
    0-1) bail "xc2ppp_start_year=$xc2ppp_start_year but xc2ppp_start_month is not defined" ;;
    1-0) bail "xc2ppp_start_month=$xc2ppp_start_month but xc2ppp_start_year is not defined" ;;
      *) ;; # Anything else (e.g. 1-1 or 0-0) is valid
  esac
  # Ensure that both xc2ppp_stop_year and xc2ppp_stop_month are defined or neither are defined
  case "${with_stop_year}-${with_stop_month}" in
    0-1) bail "xc2ppp_stop_year=$xc2ppp_stop_year but xc2ppp_stop_month is not defined" ;;
    1-0) bail "xc2ppp_stop_month=$xc2ppp_stop_month but xc2ppp_stop_year is not defined" ;;
      *) ;; # Anything else (e.g. 1-1 or 0-0) is valid
  esac
  user_state="${with_start_year}-${with_start_month}-${with_stop_year}-${with_stop_month}"
  case "$user_state" in
    1-1-1-1)
      # Start and stop dates are supplied by the user
      if [ "$prev_state" = "0-0" ]; then
        # Do not reassign previous year/month
        eval current_year\=$xc2ppp_start_year
        eval current_month\=$xc2ppp_start_month
        eval next_year\=$xc2ppp_stop_year
        eval next_month\=$xc2ppp_stop_month
      elif [ "$next_state" = "0-0" ]; then
        # Do not reassign next year/month
        eval previous_year\=$xc2ppp_start_year
        eval previous_month\=$xc2ppp_start_month
        eval current_year\=$xc2ppp_stop_year
        eval current_month\=$xc2ppp_stop_month
      else
        # Reassign all three
        eval previous_year\=NotSet
        eval previous_month\=NotSet
        eval current_year\=$xc2ppp_start_year
        eval current_month\=$xc2ppp_start_month
        eval next_year\=$xc2ppp_stop_year
        eval next_month\=$xc2ppp_stop_month
      fi
      ;;
    1-1-0-0) # Only start dates are supplied by the user
      if [ "$prev_state" = "0-0" ]; then
        # Do not reassign previous year/month
        eval current_year\=$xc2ppp_start_year
        eval current_month\=$xc2ppp_start_month
      elif [ "$next_state" = "0-0" ]; then
        # Do not reassign next year/month
        eval previous_year\=$xc2ppp_start_year
        eval previous_month\=$xc2ppp_start_month
      else
        # Reassign all three
        eval previous_year\=NotSet
        eval previous_month\=NotSet
        eval current_year\=$xc2ppp_start_year
        eval current_month\=$xc2ppp_start_month
        eval next_year\=$xc2ppp_stop_year
        eval next_month\=$xc2ppp_stop_month
      fi
      ;;
    0-0-1-1) # Only stop dates are supplied by the user
      if [ "$prev_state" = "0-0" ]; then
        # Do not reassign previous year/month
        eval next_year\=$xc2ppp_stop_year
        eval next_month\=$xc2ppp_stop_month
      elif [ "$next_state" = "0-0" ]; then
        # Do not reassign next year/month
        eval current_year\=$xc2ppp_stop_year
        eval current_month\=$xc2ppp_stop_month
      else
        # Reassign all three
        eval previous_year\=NotSet
        eval previous_month\=NotSet
        eval current_year\=$xc2ppp_start_year
        eval current_month\=$xc2ppp_start_month
        eval next_year\=$xc2ppp_stop_year
        eval next_month\=$xc2ppp_stop_month
      fi
      ;;
    *) bail "Invalid combination of xc2ppp_(start|stop)_(year|month) variables. state=$user_state"
      ;;
  esac

  # If defined, DESTSSPATH will determine the path on the front end (ppp1 or ppp2) into
  # which the files to be transferred will be copied
  # The default location is on site store in the same hall that is used by the back end machine
  # e.g. by default files copied from hare will go into
  #   /space/hall1/sitestore/eccc/crd/ccrn/users/$(whoami)
  # and files copied from brooks will go into
  #   /space/hall2/sitestore/eccc/crd/ccrn/users/$(whoami)
  DESTSSPATH=''

  # Determine the name of the invoking user
  nlines=0
  /usr/bin/perl -e '@info = (getpwuid($<)); $info[6]=~s/^.* ([A-Za-z]*)$/$1/; $info[6]=~tr/a-z/A-Z/; print "$info[0] $info[6]\n"' |
  while read invoking_user invoking_user_name; do
    # invoking_user is the user name according to the OS (as returned from getpwuid, e.g. xyz001)
    # invoking_user_name is the gcos entry from getpwuid (typically the users human name or equivalent)
    # invoking_user_name is emitted from the pipe as surname only (actually last word in field) and
    # is forced to upper case
    echo "xc2ppp: invoking_user=$invoking_user  invoking_user_name=$invoking_user_name"
    # Read only the first line
    nlines=`expr $nlines + 1`
    [ $nlines -gt 1 ] && bail "Found extraneous output while determining invoking user name."
    # This break is not normally required since there should be only 1 line ... but just in case
    break
  done
  # Alternatively (bash only) use the following here doc rather than the pipe used above if
  # the POSIX pipe/subshell problem raises its ugly head
  # done << end_info
  # $(/usr/bin/perl -e '...')
  # end_info

  # BERUNPATH must be set if files are to be copied from a machine that is
  # not the default back end machine
  bemach=''
  if [ -z "$bemach" ]; then
    # If bemach is not set explicitly then see if this job is executing on
    # a back end machine and use that machine name for bemach
    this_mach=`uname -n|awk -F\. '{print \$1}' -`
    on_back_end=0
    case $this_mach in
                    xc[0-9]*) on_back_end=1 ;;
                     c[0-9]*) on_back_end=1 ;;
    esac
    if [ $on_back_end -eq 1 ]; then
      # Set this_mach to its known alias
      case $this_mach in
        xc1*) this_mach=hare ;;
        xc2*) this_mach=brooks ;;
        xc3*) this_mach=banting  ;;
        xc4*) this_mach=daley  ;;
        *) warn "Unrecognized machine name $this_mach" ;;
      esac
      bemach=$this_mach
    fi
  fi
  if [ -n "$bemach" ]; then
    # Reset BERUNPATH if bemach is set
    # Ensure bemach contains a known alias
    case $bemach in
      xc1*) bemach=hare ;;
      xc2*) bemach=brooks ;;
      xc3*) bemach=banting ;;
      xc4*) bemach=daley ;;
    esac
    XXX=`ssh $bemach echo '$RUNPATH' 2>/dev/null` || bail "Unable to determine BERUNPATH"
    BERUNPATH=$XXX
  else
    BERUNPATH=${BERUNPATH:-''}
  fi

  # The variables xc2ppp_prefix_list and xc2ppp_suffix_list are strings containing
  # embedded colons and whitespace which are interpreted as list delimiters.
  # These strings may be thought of as 2 dimensional arrays, the rows of these
  # arrays are colon (:) separated strings and each row is divided into columns
  # by separating on white space.
  # These variables are used by make_file_name_list along with runid and year/mon
  # information from the *_year and *_month variables to generate file names.

  # In make_file_name_list the prefix_list and suffix_list strings are first
  # separated into colon delimited lists (rows). There must be an equal number of
  # rows in each of prefix_list and suffix_list because these rows will be
  # used in pairs.
  # Each pair of rows (one row from prefix_list and one row from suffix_list)
  # is separated into a white space separated list. Each element of these
  # white space separated lists is a single prefix or suffix (possibly modified
  # by appending a "+" followed by a comma separated list of integers in the
  # range 1-12). No white space is allowed within a single prefix or suffix.
  # These individual (pre|suf)fixes are then iterated over for each year and
  # month and for each pair of rows in prefix_list and suffix_list to form the
  # desired set of file names, each of which is of the form
  #
  # ${prefix}_${runid}_${year}_m${mon}_${suffix}

  # Any prefix or suffix in these  lists may be modified by appending a +
  # followed by a comma separated list of numbers (no white space is allowed
  # within this modifier). Each number within the modifier list will correspond
  # to a month (1-12) for which a file with this suffix is to be included.
  # If the modifier exists for a particular suffix then only those months
  # indicated in the modifier will be added to the file list.

  # If the above form of file name is inappropriate then the user may
  # provide a template or templates to produce arbitrary file names.
  # These templates are defined in the variable xc2ppp_prefix_list.
  # Any individual prefix will be treated as a file name template if it
  # begins with a "%" character. The template will consist of everthing
  # after the "%" character and up to the next colon or white space.
  # It can be composed of anything but must ultimately (after variable
  # substitution, etc) result in a valid file name. When a template
  # is encountered, it is used as the entire file name (ie the "normal" file
  # name form is disregarded as is the corresponding suffix(s)). However, it
  # is subject to the same interation procedure as a normal prefix and
  # does undergo variable substitution. Variables that are defined for
  # substitution include year, mon, runid, uxxx, start_year, start_mon,
  # stop_year, stop_mon, all of the *_year and *_month variables defined
  # above as well as any user supplied variable definitions passed to
  # make_file_name_list as a command line option of the form var=val.

  xc2ppp_suffix_list=''
  suffix_list="${xc2ppp_suffix_list:-gs ss}"

  xc2ppp_prefix_list=''
  prefix_list="${xc2ppp_prefix_list:-$xc2ppp_uxxx}"

  # Create a file containing a list of file names that may then be
  # "sourced" in the current environment to define the variables
  # file1, file2,..., file$join, join. These variables are used by
  # betdata to compile the list of files to be archived.
  join=0

  # make_file_name_list uses the variables current_year, current_month,
  # previous_year, previous_month, next_year and next_month to
  # determine start and stop dates for file name creation.
  # It also uses runid, prefix_list and suffix_list from the current
  # environment to build these file names.

  # Allow user supplied command line options for make_file_name_list
  # The following invocation of make_file_name_list will not allow multi-list
  # output so if any command line option is supplied that will turn on
  # multi-list output (e.g. --months_max=... --size_max=... --number_max=..)
  # then this script will abort.
  xc2ppp_file_list_opts=''
  fopts="${xc2ppp_file_list_opts:-}"

  xc2ppp_mon_offset=''
  if [ -n "$xc2ppp_mon_offset" ]; then
    # Set a user supplied month offset
    eval fopts=\"$fopts --mon_offset\=$xc2ppp_mon_offset\"
  fi

# ---Start_submit_ignore_code----

  # Create a temporary file containing the file list
  tmp_file_list="${JHOME:-$HOME}/.queue/xc2ppp_file_list_${runid}_${stamp}"
  fail=0
  make_file_name_list $fopts --nomulti_list $tmp_file_list >>$error_out 2>&1 || fail=1
  if [ $fail -eq 1 ]; then
    cat $error_out
    bail "Problem in make_file_name_list. See $error_out"
  fi
  rm -f $error_out

  [ ! -s "$tmp_file_list" ] && bail "Unable to create file list"

  # A file list was created ...source it
  # This will define start_year,start_mon,stop_year,stop_mon and file1,file2,...
  : ; . $tmp_file_list || bail "Unable to source $tmp_file_list"

  # At this point file1, file2,... are defined in the current environment
  # as well as certain other variables such as start_year, start_mon,
  # stop_year and stop_mon which correspond to the start and stop dates
  # for the file names that were created.
  ym_range="${start_year}m${start_mon}_${stop_year}m${stop_mon}"

  # Delete or keep the temporary file that contains the file list
  xc2ppp_keep_file_list=1
  XXX=`echo $xc2ppp_keep_file_list|sed 's/ //g'`
  eval xc2ppp_keep_file_list\=$XXX
  [ "$xc2ppp_keep_file_list" = 'on'  ] && eval xc2ppp_keep_file_list\=1
  [ "$xc2ppp_keep_file_list" = 'off' ] && eval xc2ppp_keep_file_list\=0
  [ "$xc2ppp_keep_file_list" = 'yes' ] && eval xc2ppp_keep_file_list\=1
  [ "$xc2ppp_keep_file_list" = 'no'  ] && eval xc2ppp_keep_file_list\=0
  if [ $xc2ppp_keep_file_list -eq 1 ]; then
    # Rename the file list to include the year/month range
    # and place it in the users ~/.queue dir
    saved_file_list="${JHOME:-$HOME}/.queue/xc2ppp_file_list_${runid}_${ym_range}_${stamp}"
    mv -f $tmp_file_list $saved_file_list
  else
    # Delete the file that contains the file list
    rm -f $tmp_file_list
  fi

  # ---Stop_submit_ignore_code----

#  * ............................ Condef Parameters ............................

  # delete_files = on means delete the files from XC40 upon successful transfer to PPP
  xc2ppp_delete_files=''
  delete_files=${xc2ppp_delete_files:-off}

  nextjob=on

#  * ............................. Deck Definition .............................

  . betdata.dk

#end_of_job