#!/bin/sh
#
#   Usage: qtrans [options] [RUNPATH=/non/default/run/path] filedesc [filedesc ...]
#
# Purpose: Copy files from a remote machine to the local machine or visa versa.
#          Optionally copy to a specified directory or "save" the files on RUNPATH.
#
# The command line will contain one or more file descriptors. Each file descriptor will
# be of the form
#
#    [[rem_user@]rem_mach:]filespec
#
# where rem_mach is an optional remote machine name or alias (e.g. erg, spica, lxwrk1, ...)
# and filespec is a file name specification that may contain shell wild cards
# (ie one or more of the characters "*", "?", "[" or "]").
#
# If filespec does contain wild cards then they must be quoted with single quotes to protect
# them against expansion by the shell.
#   e.g. If filspec is 'dc_*_1980_m0[0-6]_*' then files found will be eqivalent to the output
#        from the command "ls $data_dir/dc_*_1980_m0[0-6]_*"
#
# By default (when the -p option is not used) files will be transferred from the machine
# indicated by rem_mach in the file descriptor to the machine on which qtrans was invoked.
# Think of how the command "get" is used by ftp.
#
# If the -p option is used then files will be transferred from the machine on which qtrans
# was invoked to the remote machine specified by rem_mach in the file descriptor.
# Think of how the command "put" is used by ftp.
#
# Options:
# All options begin with a dash (-) and must appear before any other command line arg
# Single letter options may be combined, as in "qtrans -fpd dir ..."
#   -p     ...put files onto a remote machine
#             The default is to get files from a remote machine.
#             When -p is used, setting RUNPATH on the command line will change
#             RUNPATH on the remote machine.
#   -d dir ...destination directory
#             Copy files to the directory "dir" on the destination machine.
#             If -d is not used on the command line then files will be saved on RUNPATH
#   -x     ...Treat file specs that do not contain wild cards as a file name prefix.
#             e.g. If filspec is "dc_abc_1980_" then files found will be eqivalent
#                  to the output from the command "ls $data_dir/dc_abc_1980_*".
#             This will also ignore any numeric file name extension and use the file
#             with the highest edition number.
#   -B     ...batch mode, do not query the user to confirm the transfer
#   -l     ...do not copy but list files that would be copied
#   -h     ...show this usage info
#   -v     ...increase verbosity (multiple -v options are additive)
#
# Examples:
########################################################################
#
# Larry Solheim Jan,2011
#
# $Id: qtrans 670 2012-04-27 19:59:15Z acrnrls $

# *** DO NOT USE -f *** this option is currently incomplete
#   -f     ...force the creation of the destination directory if it does not already exist

FULLPATH=`type $0|awk '{print $3}'` # pathname of this script
Runame=`basename $FULLPATH`
usage() {
  err_exit=0
  while getopts e opt; do
    case $opt in
      e) err_exit=1 ;;
    esac
  done
  shift `expr $OPTIND - 1`

  [ -n "$1" ] && echo >&2 "${Runame}:" "$@"
  echo >&2 " "
  sed >&2 -n '/^###/q; s/^#$/# /; s/^ *$/# /; 3,$s/^# //p;' "$FULLPATH"
  if [ $err_exit -eq 0 ]; then
    exit
  else
    exit 1
  fi
}

bail(){
echo "${Runame}: *** ERROR *** $*"
exit 1
}

# echo without return (ie: emulate -n if not recognized by echo)
if [ "X`echo -n`" = "X-n" ]; then
  echo_n() { echo ${1+"$@"}'\c'; }
else
  echo_n() { echo -n ${1+"$@"}; }
fi

# A time stamp used with temporary file names
stamp=`date "+%j%H%M%S"$$`

# The name of the local machine
lmach=`uname -n|awk -F\. '{print \$1}' -`
case $this_mach in
    c1*) lmach=spica ;;
    c2*) lmach=hadar ;;
    c3*) lmach=rigel ;;
    c4*) lmach=maia  ;;
    c5*) lmach=naos  ;;
    c6*) lmach=saiph ;;
    c7*) lmach=zeta  ;;
  alef*) lmach=alef  ;;
   erg*) lmach=erg  ;;
   ib3*) lmach=pollux ;;
esac

# Set defaults

# fspec1 is a remote file name specification (e.g. erg:file_name)
# There may be more than one (fspec2, fspec3, ...) but there
# must be at least one file specification on the command line
fspec1=''

# list_only is a boolean flag that may be used to list the files that
# would be copied without actually copying them
list_only=0

# A boolean flag to determine if the remote file will be saved
# locally on DATAPATH/RUNPATH on the destination machine
dest_save=1
user_set_dest_save=0

# get_files and put_files are mutually exclusive boolean flags to indicate
# whether files are to be brought to the local machine from a remote machine
# (get_files) or sent from the local machine to a remote machine (put_files)
get_files=1
put_files=0

# The name of a directory into which files will be copied on the destination machine
# If dest_dir is NULL or undefined then files will be copied to RUNPATH prior to saving
dest_dir=`pwd`
user_set_dest_dir=0

# force=1 means create any non-existant destination directory, if nessecary
force=0

# For remote commands used below qtrans_abort flags either
#   1) continuation on error when qtrans_abort = 0
#   2) abort on error when qtrans_abort is not 0
qtrans_abort=1

# ssh_args may be used to set additional command line args to ssh commands
# used below. This is only useful for debugging.
ssh_args=''

# presh is used to prefix ssh commands run on certain machines in Dorval.
# Batch jobs on these machines (e.g. pollux, spica, hadar) will run on a compute node.
# This will cause problems with ssh commands. The presh will simply initiate the
# ssh command from a head node.
presh=''

# verbatim_filespec=1 means use file specs that do not contain wild cards, exactly
# as they are provided on the command line. Normally these file specs are treated
# as a file name prefix.
verbatim_filespec=1

# interactive is a boolean flag used to determine if the user will be queried
# about proceeding with the copy/save after viewing a list of files to be copied
interactive=1

# The value of verbose determines the amount of information written to stdout
verbose=1

# A boolean flag used to determine if the user has set RUNPATH on the command line
user_set_RUNPATH=0

# usr_list is the name of a file containing a list of files to be transferred
# nfile will be the number of files found in that list
usr_list=''
nfile=0

# process command line options
while getopts Bd:fhlpsvx opt
do
  case $opt in
    # The -s and -d options are mutually exclusive
    s) dest_dir=`pwd`;     dest_save=1; user_set_dest_save=1 ;;
    d) dest_dir="$OPTARG"; dest_save=0; user_set_dest_dir=1  ;;
    p) get_files=0; put_files=1 ;;
    B) interactive=0 ;;
    f) force=1 ;;
    x) verbatim_filespec=0 ;;
    l) list_only=1 ;;
    v) verbose=`expr $verbose + 1` ;;
    h) usage ;;
    -) shift; break ;; # end of options
    ?) usage -e $USAGE   ;;
  esac
done
shift `expr $OPTIND - 1`

if [ $user_set_dest_save -eq 1 -a $user_set_dest_dir -eq 1 ]; then
  bail "Only one of the options -s or -d is allowed on the command line."
fi

if [ $get_files -eq 0 -a $put_files -eq 1 ]; then
  # The -p option was specified on the command line
  # PUT files is operative
  if [ $user_set_dest_dir -eq 0 ]; then
    # The user did not specify the -d option
    # Set the default destination directory to ~/tmp on the remote machine
    # dest_dir='./tmp'
    # Save the file on the remote machine
    dest_save=1
  fi
elif [ $get_files -eq 1 -a $put_files -eq 0 ]; then
  # The deafult GET files is operative
  if [ $dest_save -eq 1 ]; then
    # RUNPATH will be required
    [ -z "$RUNPATH" ] && \
      bail "A save is requested but RUNPATH is not set in the environment"
  fi
else
  bail "One, and only one, of get_files or put_files may be set."
fi

[ $# -eq 0 ] && usage -e "A file name specification is required on the command line"

# nspec is the number of file name specifications found on the command line
nspec=0

# Process the remaining command line args
for arg in "$@"; do
  case $arg in
    *=*) var=`echo $arg|awk -F\= '{printf "%s",$1}' -`
         val=`echo "$arg"|awk '{i=index($0,"=")+1;printf "%s",substr($0,i)}' -`
         [ -n "$var" ] && eval ${var}=\"\$val\" # preserve quoted assignments
         case $var in
           CCRNTMP) export CCRNTMP ;;
           RUNPATH) export RUNPATH; user_set_RUNPATH=1 ;;
           list) usr_list=$val
                 [ -z "$usr_list" ] && bail "The user supplied list= without and argument."
                 [ ! -s $usr_list ] && bail "The user supplied list $usr_list is missing or empty."
                 # Remove comments and blank lines from usr_list
                 # Use only the first word found on each line

                 # Strip comments, leading whitespace and everthing but the first whitespace
                 # or semicolon separated word from each line of the input file list
                 USR_LIST=`sed -n '/^ *#/d; s/#.*$//g; s/^ *//g;
                                   s/ .*$//g; s/;.*$//g; p' $usr_list`
                 for FNAME in $USR_LIST; do
                   # Strip any fileN= from the beginning of the string
                   FNAME=`echo $FNAME|sed 's/^file[0-9][0-9]*=//g'`
                   # Strip single quotes
                   FNAME=`echo $FNAME|sed "s/'//g"`
                   # Strip double quotes
                   FNAME=`echo $FNAME|sed 's/"//g'`
                   if [ -n "$FNAME" ]; then
                     # Strip any numeric extension
                     FNAME=`echo $FNAME|sed 's/\.[0-9][0-9][0-9]$//'`
                   fi
                   [ -z "$FNAME" ] && continue
                   has_wild=`echo $FNAME|sed -n '/[][?*]/p'`
                   if [ -z "$has_wild" ]; then
                     # FNAME does not contain shell wild cards so it must be a valid
                     # file name containg only alphanumeric or underscore characters
                     FNAME=`echo $FNAME|sed -n '/^[A-Za-z0-9_][A-Za-z0-9_]*$/p'`
                   fi
                   [ -z "$FNAME" ] && continue
                   
                   # Add fileN to the current env
                   nfile=`expr $nfile + 1`
                   eval file${nfile}="$FNAME"
                   echo "$nfile  $FNAME"
                 done
                 bail "list=FLIST is not yet supported."
                 ;;
         esac
         ;;
      *) nspec=`expr $nspec + 1`
         eval fspec${nspec}=\"\$arg\" # preserve quoted command line arguments
         ;;
  esac
done

[ -z "$fspec1" ] &&
  usage -e "A file name specification is required on the command line"

if [ $get_files -eq 1 ]; then
  # Verify that RUNPATH is valid when geting remote files and saving them locally
  if [ $dest_save -eq 1 ]; then
    [ -z "$RUNPATH" ] && bail "A local save is requested but RUNPATH is not set."
    [ ! -d "$RUNPATH" ] && bail "RUNPATH=$RUNPATH is not a directory."
  fi
fi

ntrans=0
copied_list=''
while [ $ntrans -lt $nspec ]; do
  ntrans=`expr $ntrans + 1`
  eval filedesc=\$fspec$ntrans
  # Strip any white space from the file descriptor
  filedesc=`echo "$filedesc" | sed 's/ *//g'`

  # Reset these variables for each file descriptor
  REMOTE_RUNPATH=''
  REMOTE_TMPDIR=''
  create_remote_runpath=0
  create_remote_tmpdir=0
  presh=''

  # Ensure that filedesc contains zero or one colon characters
  ncolon=`echo "$filedesc" | awk '{print gsub(/:/,":")}' -`
  [ $ncolon -gt 1 ] && bail "Invalid use of : in remote file specification. $filedesc"

  # Detemine the name of the remote machine and the current file specification.
  # If filedesc contains a single colon character, then everything before that colon
  # will be the remote machine name or alias (ie whatever ssh needs to find it) and
  # everything after that colon will be a file specification (possibly containing
  # shell wild cards).
  # If there is no colon in filedesc then the entire string is the file specification
  # and files will be searched for on the local machine.

  if [ $ncolon -eq 1 ]; then
    rem_mach=`echo "$filedesc" | sed 's/^\(.*\):.*/\1/'`
    rem_user=`echo "$rem_mach" | sed -n 's/^\(.*@\).*/\1/p'`
    if [ -n "$rem_user" ]; then
      # Test for an empty remote user name (ie "@" is the first character in rem_mach)
      rem_user=`echo "$rem_user" | sed -n 's/^\(.*\)@.*/\1/p'`
      [ -z "$rem_user" ] && bail "Invalid remote user specified in --> $filedesc <--"
    fi
  else
    rem_mach=''
    rem_user=''
    # It does not make sense to PUT files onto a remote machine when
    # no remote machine name has been provided
    if [ $put_files -eq 1 ]; then
      echo_n "You have requested that files be copied to a remote machine "
      echo "but have not provided a remote machine name in a file descriptor."
      bail "Invalid filedesc --> $filedesc <--"
    fi
    # Disallow resaving a file to DATAPATH/RUNPATH
    if [ $get_files -eq 1 ]; then
      if [ -z "$dest_dir" ]; then
        echo "You have requested a save from ${lmach}:$DATAPATH to ${lmach}:$DATAPATH"
        echo_n "This is not allowed. Use the -d command line option to specify a directory"
        echo " into which the files will be saved."
        bail "Invalid filedesc --> $filedesc <--"
      fi
    fi
  fi

  if [ -n "$rem_mach" ]; then
    # Filter invalid remote machine names
    # Strip any "user@" from the beginning of rem_mach and assign to rem_mach_name
    rem_mach_name=`echo "$rem_mach" | sed 's/^.*@\(.*\)/\1/'`
    case $rem_mach_name in
# Setting presh is not currently implimented
#      alef|pollux|spica|hadar) presh="ssh $rem_mach_name" ;;
      erg|saiph|zeta|pollux|spica|hadar) ;;
      *.*) ;; # Assume a fully qualified domain name
      lx*) # Assume a CCCma machine alias and create a fully qualified domain name
           rem_mach="${rem_mach}.cccma.ec.gc.ca" ;;
      *) bail "Invalid remote machine name $rem_mach_name"
    esac

    # Determine the current file name specification
    curr_fspec=`echo "$filedesc" | sed 's/^.*:\(.*\)/\1/'`

    if [ $put_files -eq 1 ]; then
      if [ $dest_save -eq 1 ]; then
        # Determine the name of a remote temporary directory
        REMOTE_TMPDIR=`ssh $rem_mach 'echo $CCRNTMP'`
        [ $? -ne 0 ] && bail "Cannot determine CCRNTMP on $rem_mach"
        REMOTE_TMPDIR=$REMOTE_TMPDIR/tmp_qtrans_$stamp
        create_remote_tmpdir=1
      fi

      # Determine the name of the remote runpath
      if [ $user_set_RUNPATH -eq 1 ]; then
        REMOTE_RUNPATH=$RUNPATH
      else
        REMOTE_RUNPATH=`ssh $rem_mach 'echo $RUNPATH'`
      fi
      [ $force -eq 1 ] && create_remote_runpath=1
    fi

  else
    # Copy (link) files from the local DATAPATH to cwd

    # Determine the current file name specification
    curr_fspec="$filedesc"

  fi
  [ -z "$curr_fspec" ] && bail "Invalid file specification. --> $filedesc <--"

  # Remove any leading directory path and assign to get_list_path
  # This will be the originating directory containing files to be transferred
  get_list_path=`echo "$curr_fspec"|sed -n 's/^\(.*\/\).*/\1/p'`
  if [ -n "$get_list_path" ]; then
    # The user has provided a directory in which files are to be found
    # Strip this directory path from curr_fspec
    curr_fspec=`echo "$curr_fspec"|sed 's/^.*\/\(.*\)/\1/'`

    # Do not allow shell wildcards in the directory path
    nwild=`echo "$get_list_path" | awk '{print gsub(/[\*\?\]\[]/,"_")}' -`
    [ $nwild -gt 0 ] && \
      bail "Wild cards are not allowed in the directory path. --> $filedesc <--"
  fi
  [ -z "$curr_fspec" ] && bail "Invalid file specification. --> $filedesc <--"

  # Determine if there are any shell wildcards in the current filedesc
  nwild=`echo "$curr_fspec" | awk '{print gsub(/[\*\?\]\[]/,"_")}' -`

  # Get a list of file names that match the current file spec
  if [ -n "$rem_mach" ]; then
    if [ $get_files -eq 1 ]; then
      # GET a remote file and copy it to the local machine
      # Get the file list from a remote machine
      get_remote_file_list=1
    elif [ $put_files -eq 1 ]; then
      # PUT a local file onto a remote machine
      # Get the file list from the local machine
      get_remote_file_list=0
    else
      bail "Either get_files or put_files must be set."
    fi
  else
    # Get the file list from the local machine
    get_remote_file_list=0
  fi

  if [ $get_remote_file_list -eq 1 ]; then
    # Get the file list from a remote machine
    if [ -n "$get_list_path" ]; then
      # Get the file list from files in a user supplied dir
      if [ $nwild -gt 0 ]; then
        # The user has used shell wild cards in this file spec
        # Match the file spec exactly
        flist=`ssh $rem_mach ls -L $get_list_path/${curr_fspec}`
        lserr=$?
      else
        # There are no shell wild cards in this file spec
        if [ $verbatim_filespec -eq 1 ]; then
          # Do not add any wild cards but use this file spec verbatim
          flist=`ssh $rem_mach ls -L $get_list_path/${curr_fspec}`
          lserr=$?
        else
          # Append a '*' to whatever is there so that file spec becomes a common prefix
          flist=`ssh $rem_mach ls -L $get_list_path/${curr_fspec}'*'`
          lserr=$?
        fi
      fi
    else
      # Get the file list from files on DATAPATH
      if [ $nwild -gt 0 ]; then
        # The user has used shell wild cards in this file spec
        # Match the file spec exactly, except for the numeric extension
        flist=`ssh $rem_mach ls -L '$DATAPATH'/${curr_fspec}'.[0-9][0-9][0-9]'`
        lserr=$?
      else
        # There are no shell wild cards in this file spec
        if [ $verbatim_filespec -eq 1 ]; then
          # Do not add any wild cards except those to match the numeric extension
          flist=`ssh $rem_mach ls -L '$DATAPATH'/${curr_fspec}'.[0-9][0-9][0-9]'`
          lserr=$?
        else
          # Append a '*' to whatever is there so that file spec becomes a common prefix
          flist=`ssh $rem_mach ls -L '$DATAPATH'/${curr_fspec}'*'`
          lserr=$?
        fi
      fi
    fi
  else
    # Get the file list from the local machine
    if [ -n "$get_list_path" ]; then
      # Get the file list from files in a user supplied dir
      if [ $nwild -gt 0 ]; then
        # The user has used shell wild cards in this file spec
        # Match the file spec exactly
        flist=`ls -L $get_list_path/${curr_fspec}`
        lserr=$?
      else
        # There are no shell wild cards in this file spec
        if [ $verbatim_filespec -eq 1 ]; then
          # Do not add any wild cards but use this file spec verbatim
          flist=`ls -L $get_list_path/${curr_fspec}`
          lserr=$?
        else
          # Append a '*' to whatever is there so that file spec becomes a common prefix
          flist=`ls -L $get_list_path/${curr_fspec}*`
          lserr=$?
        fi
      fi
    else
      # Get the file list from files on DATAPATH
      [ -z "$DATAPATH" ] && bail "DATAPATH must be defined."
      if [ $nwild -gt 0 ]; then
        # The user has used shell wild cards in this file spec
        # Match the file spec exactly, except for the numeric extension
        flist=`ls -L $DATAPATH/${curr_fspec}.[0-9][0-9][0-9]`
        lserr=$?
      else
        # There are no shell wild cards in this file spec
        if [ $verbatim_filespec -eq 1 ]; then
          # Do not add any wild cards except those to match the numeric extension
          flist=`ls -L $DATAPATH/${curr_fspec}.[0-9][0-9][0-9]`
          lserr=$?
        else
          # Append a '*' to whatever is there so that file spec becomes a common prefix
          flist=`ls -L $DATAPATH/${curr_fspec}*`
          lserr=$?
        fi
      fi
    fi
  fi
  if [ $lserr -ne 0 ]; then
    echo "Unable to match any file names for filedesc --> $filedesc <--"
    continue
  fi

  if [ -n "$get_list_path" ]; then
    # The file list came from files in a user specified dir
    # In this case it is assumed that files do not have a 3 digit
    # numerical extension, so use the file list "as is"
    getlist="$flist"
  else
    # The file list came from files in DATAPATH
    # If there are multiple edition numbers for a single file name then
    # copy only the file with the highest edition number
    # The following lines will create a processed file list containing a single edition
    # number for each unique file name and put it in the variable getlist
    getlist=`echo $flist | perl -ne '
      @F = sort(split(/ /));
      ($name) = $F[0] =~ /^(.*)\.\d+/;
      foreach (@F) {
        ($this_name) = /^(.*)\.\d+/;
        ($this_ed) = /^.*\.(\d+)/;
        if ($this_name ne $name) {
          $edition = (sort @ed)[-1];
          push @flist, "${name}.$edition";
          $name = $this_name;
          undef @ed;
        }
        push @ed, $this_ed;
      }
      $edition = (sort @ed)[-1];
      push @flist, "${name}.$edition";
      print join("\n",@flist);'`
  fi

  if [ $list_only -eq 1 ]; then
    if [ -n "$getlist" ]; then
      echo "Files matching --> $filedesc <-- that would have been copied:"
      for curr_file in $getlist; do
        echo "    $curr_file"
      done
    else
      echo "No file names match filedesc --> $filedesc <--"
    fi
    continue
  fi

  if [ $verbose -gt 0 ]; then
    echo_n "Files matching --> $curr_fspec <-- to be copied and saved on "
    if [ $dest_save -eq 1 ]; then
      if [ $get_files -eq 1 ]; then
        echo_n "${lmach}:$RUNPATH"
      elif [ $put_files -eq 1 ]; then
        echo_n "${rem_mach}:$REMOTE_RUNPATH"
      fi
    elif [ -n "$dest_dir" ]; then
      if [ $get_files -eq 1 ]; then
        echo_n "${dest_dir}/"
      elif [ $put_files -eq 1 ]; then
        echo_n "${rem_mach}:${dest_dir}/"
      fi
    fi
    echo " "
    for curr_file in $getlist; do
      echo "    $curr_file"
    done
    echo " "
  fi
  if [ $interactive -eq 1 ]; then
    # Query user about proceeding with the copy/save
    echo_n "### Continue with the transfer (y/n) (default y) > "
    read ans
    ans=`echo $ans|sed -n '/^ *[nN]/p'`
    if [ -n "$ans" ]; then
      continue
    fi
  fi

  if [ $create_remote_tmpdir -eq 1 ]; then
    # Create a temorary directory on the remote machine to be used for transfer
    [ -z "$REMOTE_TMPDIR" ] && \
      bail "Attempting to create a tmpdir on $rem_mach but REMOTE_TMPDIR is NULL"
    dir_created=0
    ssh $rem_mach "ls -l $REMOTE_TMPDIR" >/dev/null 2>&1 || \
        dir_created=1; ssh $rem_mach "mkdir -m 755 $REMOTE_TMPDIR" || \
        bail "Cannot create remote directory ${rem_mach}:$REMOTE_TMPDIR"
    if [ $dir_created -eq 1 ]; then
      # Ensure that the directory is visible on the remote filesystem
      echo_n '.'
      sleep 10
      count=0
      success=0
      while [ $count -lt 5 ]; do
        count=`expr $count + 1`
        [ $count -gt 1 ] && sleep 10
        echo_n '.'
        ssh $rem_mach "ls -ld $REMOTE_TMPDIR" >/dev/null 2>&1 || continue
        success=1
        break
      done
      echo " "
      [ $success -eq 0 ] && bail "Unable to list ${rem_mach}:$REMOTE_TMPDIR"
    fi
  fi

  if [ $create_remote_runpath -eq 1 ]; then
    # Create the remote RUNPATH directory if it does not exist
    dir_created=0
    ssh $rem_mach "ls -l $REMOTE_RUNPATH" >/dev/null 2>&1 || \
      dir_created=1; ssh $rem_mach "mkdir -p -m 750 $REMOTE_RUNPATH" || \
        bail "Cannot create remote directory ${rem_mach}:$REMOTE_RUNPATH"
    if [ $dir_created -eq 1 ]; then
      # Ensure that the directory is visible on the remote filesystem
      count=0
      success=0
      while [ $count -lt 5 ]; do
        count=`expr $count + 1`
        [ $count -gt 1 ] && sleep 5
        echo_n '.'
        ssh $rem_mach "ls -ld $REMOTE_RUNPATH" >/dev/null 2>&1 || continue
        success=1
        break
      done
      echo " "
      [ $success -eq 0 ] && bail "Unable to list ${rem_mach}:$REMOTE_RUNPATH"
    fi
  fi

  # Loop over the processed file name list and copy each file in turn
  for curr_file in $getlist; do
    # The destination file name will be the current file name without
    # any numeric extension or any leading directory path
    dest_file=`echo $curr_file|sed 's/^\(.*\)\.[0-9][0-9][0-9]/\1/; s/^.*\/\(.*\)/\1/'`

    # Determine the full pathname for the destination file
    if [ $dest_save -eq 1 ]; then
      # Save to DATAPATH/RUNPATH
      # Copy destination file to RUNPATH to avoid a second copy during the save
      if [ $get_files -eq 1 ]; then
        curr_dest_dir=$RUNPATH
      elif [ $put_files -eq 1 ]; then
        curr_dest_dir=$REMOTE_TMPDIR
#        curr_dest_dir=$REMOTE_RUNPATH
      else
        bail "Either get_files or put_files must be set."
      fi
    else
      # Save to a user specified directory
      curr_dest_dir=$dest_dir
    fi
    dest_path=${curr_dest_dir}/$dest_file

    if [ -n "$rem_mach" ]; then
      # The user supplied file spec contained a remote machine name
      if [ $get_files -eq 1 ]; then
        # GET a remote file

        # Change permission on the local file, if it exists,
        # so that scp will overwrite it
        [ -s "$dest_path" ] && chmod u+w $dest_path

        # Copy a remote file to the local machine
        scp ${rem_mach}:$curr_file $dest_path
        [ $? -ne 0 ] && bail "Unable to:  scp ${rem_mach}:$curr_file $dest_path"

        if [ $dest_save -eq 1 ]; then
          # Save to RUNPATH on local machine
          CWD=`pwd`
          cd $RUNPATH
          save $dest_file $dest_file
          [ $? -ne 0 ] && bail "Unable to:  save $dest_file $dest_file"
          release $dest_file
          copied_list="$copied_list ${rem_mach}:${curr_file}%${lmach}:${dest_path}%SAVED"
        else
          copied_list="$copied_list ${rem_mach}:${curr_file}%${lmach}:$dest_path"
        fi
      elif [ $put_files -eq 1 ]; then
        # PUT a local file

        # Change permission on the remote file, if it exists,
        # so that scp will overwrite it
        ssh $rem_mach "chmod -f u+w $dest_path" 2>/dev/null || :

#        if [ $force -eq 1 ]; then
        if [ 2 -eq 1 ]; then

#TODO When the remote RUNPATH is created in this script the first (one or more) saves
#TODO done below will fail because "save" thinks that either the directory or the file
#TODO to be saved does not exist. Subsequent saves, after the first one (or two), will work.
#TODO Also if the dir was created outside this script then the saves will work.
#TODO Experimenting with the sleep time in the following loop seems to have some effect
#TODO but does not fix the problem.

          # Create the remote destination directory (remote RUNPATH) if it does not exist
          dir_created=0
          ssh $rem_mach "ls -l $curr_dest_dir" >/dev/null 2>&1 || \
            dir_created=1; ssh $rem_mach "mkdir -p -m 750 $curr_dest_dir" || \
              bail "Cannot create remote directory ${rem_mach}:$curr_dest_dir"
          if [ $dir_created -eq 1 ]; then
            # Ensure that the directory is visible on the remote filesystem
            count=0
            success=0
            while [ $count -lt 5 ]; do
              count=`expr $count + 1`
              [ $count -gt 1 ] && sleep 5
              ssh $rem_mach "ls -ld $curr_dest_dir" >/dev/null 2>&1 || continue
              success=1
             break
            done
            [ $success -eq 0 ] && bail "Unable to list ${rem_mach}:$curr_dest_dir"
          fi
        fi

        # Copy a local file to the remote machine
        success=0
        count=0
        while [ $count -lt 3 ]; do
          # retry up to 3 times
          count=`expr $count + 1`
          if [ $count -gt 1 ]; then
            sleep 5
            echo "*** RETRY ${count}: scp $curr_file ${rem_mach}:$dest_path"
          fi
          scp $curr_file ${rem_mach}:$dest_path 2>&1 || continue
          success=1
          break
        done
        [ $success -eq 0 ] && bail "Unable to:  scp $curr_file ${rem_mach}:$dest_path"

        if [ $dest_save -eq 1 ]; then
          # Ensure that the file just copied is visible on the remote filesystem
          # Even when the copy is successful, it is possible for the following
          # "save" command to fail because it cannot find the remote file.
          count=0
          success=0
          while [ $count -lt 5 ]; do
            count=`expr $count + 1`
            [ $count -gt 1 ] && sleep 5
            echo_n '.'
            ssh $rem_mach "ls -l $dest_path" >/dev/null 2>&1 || continue
            success=1
            break
          done
          echo " "
          [ $success -eq 0 ] && bail "Unable to list ${rem_mach}:$dest_path after copy"

          # The following remote command requires a bourne shell,
          # therefore explicitly use /bin/sh -c to ensure that this is so.
          # Note: submit3 blindly replaces /bin/sh with the name of a shell
          # that will not exist outside of the cccma environment. The following
          # eval will hide the name of the remote shell from this substitution.
          eval shebang=\'\#\!\/bin\/sh\'
          eval remshell=\'\/bin\/sh -c\'
          eval def_rmrun=RUNPATH\=$REMOTE_RUNPATH
          if [ $verbose -gt 1 ]; then
            SETX='set -x;'
          else
            SETX=''
          fi
          remcmd="${SETX}cd $curr_dest_dir || exit 1;env $def_rmrun save $dest_file $dest_file && release $dest_file"
          success=0
          if [ -z "$presh" ]; then
            success=0
            count=0
            while [ $count -lt 3 ]; do
              # retry up to 3 times
              count=`expr $count + 1`
              if [ $count -gt 1 ]; then
                sleep 5
                echo "*** RETRY ${count}: ssh $ssh_args $rem_mach $remshell $remcmd"
              fi
              ssh $ssh_args $rem_mach "$remshell '$remcmd'" 2>&1 || continue
              success=1
              break
            done
          else
#TODO This branch is currently not working
bail "Invalid branch to $presh ssh ..."
            this_dir=`pwd`
            cat >remsave <<EOF
$shebang
$remcmd
EOF
            chmod u+rx remsave
            $presh "cd $this_dir;scp $scp_args remsave ${rem_mach}:$curr_dest_dir" 2>&1 || success=1
            [ $success -eq 0 ] &&\
              $presh ssh $ssh_args $remloc "$remdir/remsave" 2>&1 || success=1
          fi
          if [ $success -eq 0 ]; then
            echo "***ERROR*** Attempting to save $REMOTE_RUNPATH/$dest_file\n"
            [ $qtrans_abort -ne 0 ] && bail "Unable to save remote file $dest_file\n"
          fi

          copied_list="$copied_list ${lmach}:${curr_file}%${rem_mach}:${dest_path}%SAVED"
        else
          copied_list="$copied_list ${lmach}:${curr_file}%${rem_mach}:${dest_path}"
        fi
      else
        bail "Either get_files or put_files must be set."
      fi
    else
      # Link to the file on a local RUNPATH
      # Remove any existing file
      rm -f $dest_path
      ln -s $curr_file $dest_path
      copied_list="$copied_list ${curr_file}%${dest_path}%LINKED"
    fi

  done

  if [ $create_remote_tmpdir -eq 1 ]; then
    # Delete the remote temporary dir created prior to this loop
    ssh $rem_mach "rm -fr $REMOTE_TMPDIR"
  fi

done

if [ $verbose -gt 0 ]; then
  if [ -n "$copied_list" ]; then
    echo "Files copied:"
    for f in $copied_list; do
      from_file=`echo $f|awk -F% '{print $1}' -`
      to_file=`echo $f|awk -F% '{print $2}' -`
      disposition=`echo $f|awk -F% '{print $3}' -`
      if [ "x$disposition" = "xSAVED" ]; then
        echo "  FROM $from_file\t TO $to_file\t AND SAVED"
      elif [ "x$disposition" = "xLINKED" ]; then
        echo "  FROM $from_file\t TO $to_file\t AS A LINK"
      else
        echo "  FROM $from_file\t TO $to_file"
      fi
    done
  else
    echo "No files were copied."
  fi
fi