#!/bin/sh #======================================================================= # Copy files to the front end --- pullfe --- # $Id: pullfe_jobdef 669 2012-04-24 20:44:05Z acrnrls $ #======================================================================= # # files copied will be of the form # ${pullfe_prefix}_${year}_m${mon}_$suffix # for all months from previous_year, previous_month to current_year, # current_month or from current_year, current_month to next_year, # next_month depending on which of previous_(year|month) or # next_(year|month) are set. # The default suffix list is "gs ss rs+12 ab+12 an+12" which means # that gs and ss files are copied every month while rs, ab and an files # are copied once every Dec that falls within this range. # # The variables current_year, current_month, previous_year and # previous_month are set when the job string is created. # # explicitly setting pullfe_start_year overrides previous_year # explicitly setting pullfe_start_mon overrides previous_month # explicitly setting pullfe_end_year overrides current_year # explicitly setting pullfe_end_mon overrides current_month # # A variable named pullfe_suffix_list may also be set to modify the list # of files that will be copied. See below for details. # # pullfe_prefix_ can be set to override the value of pullfe_prefix # files copied will then be of the form # ${pullfe_prefix_}${year}_m${mon}_$suffix # note the missing underscore between pullfe_prefix_ and year. #======================================================================= # # keyword :: pullfe # description :: copy files to the front end from another machine # set -a . betapath2 # * ........................... Parmsub Parameters ............................ jobname=pullfe; time="900" ; memory="900mb" ; username="acrnxxx"; user="XXX"; runid=job000; uxxx='uxxx'; crawork="${runid}_job" nqsprfx="${runid}_"; nqsext='' # Temporary directory where this script will run pullfe_CCRNTMP='' CCRNTMP=${pullfe_CCRNTMP:=$CCRNTMP} # RUNPATH on execution machine pullfe_RUNPATH='' if [ -n "$pullfe_RUNPATH" -a "$pullfe_RUNPATH" = "CCRNTMP" ]; then RUNPATH=$CCRNTMP else RUNPATH=${pullfe_RUNPATH:=$RUNPATH} fi # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi noprint=on nextjob=on . comjcl.cdk cat > Execute_Script <<'end_of_script' # bail is a simple error exit routine bail(){ echo "pullfe: "$1 echo " " >> haltit exit 1 } # Temporary directory where this script will run pullfe_CCRNTMP='' CCRNTMP=${pullfe_CCRNTMP:=$CCRNTMP} # RUNPATH on execution machine pullfe_RUNPATH='' if [ -n "$pullfe_RUNPATH" -a "$pullfe_RUNPATH" = "CCRNTMP" ]; then RUNPATH=$CCRNTMP else RUNPATH=${pullfe_RUNPATH:=$RUNPATH} fi # Define a unique stamp for use in file names etc stamp=`date "+%j"$$` # * ........................... Parmsub Parameters ............................ runid="job000"; uxxx='uxxx'; pullfe_uxxx=$uxxx; jobname=pullfe; pullfe_prefix="${pullfe_uxxx}_${runid}"; pullfe_prefix_=${pullfe_prefix}_ crawork="${runid}_job"; username="acrnxxx"; user="XXX"; year=yyy; mon=mm RUNID=`echo "${runid}"|tr '[a-z]' '[A-Z]'`; stime="900"; memory1="250mb"; lopgm="lopgm"; # These variables are set when the job string is created run_start_year=NotSet # memory99=1 run_start_month=NotSet # memory99=1 run_stop_year=NotSet # memory99=1 run_stop_month=NotSet # memory99=1 current_year=NotSet # memory99=1 current_month=NotSet # memory99=1 previous_year=NotSet # memory99=1 previous_month=NotSet # memory99=1 next_year=NotSet # memory99=1 next_month=NotSet # memory99=1 if [ $current_year = "NotSet" -o $current_month = "NotSet" ]; then bail "pullfe: current_year or current_month is not set" fi # files copied will be for months in the range from # pullfe_start_year,pullfe_start_mon to pullfe_end_year,pullfe_end_mon if [ $previous_year = "NotSet" -o $previous_month = "NotSet" ]; then if [ $next_year = "NotSet" -o $next_month = "NotSet" ]; then bail"pullfe: Neither previous_(year|month) nor next_(year|month) are set" else pullfe_start_year=$current_year pullfe_start_mon=$current_month pullfe_end_year=$next_year pullfe_end_mon=$next_month fi else pullfe_start_year=$previous_year pullfe_start_mon=$previous_month pullfe_end_year=$current_year pullfe_end_mon=$current_month fi pullfe_start_year=`echo $pullfe_start_year|awk '{printf "%3.3d",$1}' -` pullfe_start_mon=`echo $pullfe_start_mon|awk '{printf "%2.2d",$1}' -` pullfe_end_year=`echo $pullfe_end_year|awk '{printf "%3.3d",$1}' -` pullfe_end_mon=`echo $pullfe_end_mon|awk '{printf "%2.2d",$1}' -` [ -z "$pullfe_start_year" ] && bail "pullfe: pullfe_start_year is null" [ -z "$pullfe_end_year" ] && bail "pullfe: pullfe_end_year is null" [ -z "$pullfe_start_mon" ] && bail "pullfe: pullfe_start_mon is null" [ -z "$pullfe_end_mon" ] && bail "pullfe: pullfe_end_mon is null" [ $pullfe_start_mon -gt 12 -o $pullfe_start_mon -lt 1 ] &&\ bail "pullfe: pullfe_start_mon = $pullfe_start_mon is out of range" [ $pullfe_end_mon -gt 12 -o $pullfe_end_mon -lt 1 ] &&\ bail "pullfe: pullfe_end_mon = $pullfe_end_mon is out of range" [ $pullfe_start_year -gt $pullfe_end_year ] &&\ bail "pullfe: pullfe_start_year = $pullfe_start_year is out of range" # ym_range is used as part of some file names defined below ym_range="${pullfe_start_year}m${pullfe_start_mon}_${pullfe_end_year}m${pullfe_end_mon}" # Get the name of the machine running this script # and bail if this is not a front end machine is_fe=0 this_mach=`uname -n|awk -F\. '{print \$1}' -` case $this_mach in c1*) this_mach=spica ;; c2*) this_mach=hadar ;; c3*) this_mach=rigel ;; c4*) this_mach=maia ;; c5*) this_mach=naos ;; c6*) this_mach=saiph ;; c7*) this_mach=zeta ;; # ib*) this_mach=ib; is_fe=1 ;; alef*) this_mach=alef; is_fe=1 ;; ib3*) this_mach=pollux; is_fe=1 ;; esac [ $is_fe -eq 0 ] && bail "This machine $this_mach is not a front end machine" # pullfe_suffix_list is a white space separated list of suffixes # of file names to be generated here. Any suffix in this list may be # modified by appending a + followed by a comma separated list of # numbers (no white space is allowed within this modifier). Each # number within the modifier list will correspond to a month (1-12) # for which a file with this suffix is to be included. If the # modifier exists for a particular suffix then only those months # indicated in the modifier will be added to the file list. pullfe_suffix_list='gs ss rs+12 ab+12 an+12' # ---Start_submit_ignore_code---- # Create a lock file pullfe_lock=${JHOME:-$HOME}/.queue/.crawork/lock_pullfe_${runid}_${ym_range} if [ -n "$pullfe_lock" ]; then touch $pullfe_lock echo "Created lock file $pullfe_lock" fi # pullfe_suffix_ flags the addition of an underscore between the # suffix and the month (e.g. m01_gs) in file names generated here pullfe_suffix_=1 XXX=`echo $pullfe_suffix_|sed 's/ //g'` eval pullfe_suffix_\=$XXX [ "$pullfe_suffix_" = 'on' ] && eval pullfe_suffix_\=1 [ "$pullfe_suffix_" = 'off' ] && eval pullfe_suffix_\=0 [ "$pullfe_suffix_" = 'yes' ] && eval pullfe_suffix_\=1 [ "$pullfe_suffix_" = 'no' ] && eval pullfe_suffix_\=0 # If pullfe_name_once is defined then do not iterate over year/month # since the last invocation of pullfe to generate a list of file names # but rather use pullfe_name_once to form file names for each suffix name_once='' pullfe_name_once=${name_once:-''} # If pullfe_name_yearly is defined then iterate over year, but not month, # since the last invocation of pullfe to generate a list of file names # using pullfe_name_yearly as the file name (up to the suffix) name_yearly='' pullfe_name_yearly=${name_yearly:-''} if [ -n "$pullfe_name_once" ]; then join=0 # pullfe_name_once is a space separated list of file name templates # No file name template in this list may contain whitespace for bname in $pullfe_name_once; do if [ -n "$pullfe_suffix_list" ]; then # Iterate over the suffixes in pullfe_suffix_list for suffix in $pullfe_suffix_list; do # Strip off any "+" delimited modifier from suffix # These modifiers (used to limit processing to specific months) # are ignored when name_once is used to form file names suffix=`echo $suffix|sed 's/+.*$//'` # Use the variable fileN (N=1,2,...) to hold the next file name join=`echo $join|awk '{printf "%d",$1+1}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` eval file${join}=$bname$suffix done else # Use the name templates in pullfe_name_once "as is" join=`echo $join|awk '{printf "%d",$1+1}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` eval file${join}=$bname fi done elif [ -n "$pullfe_name_yearly" ]; then # Generate a list of history files for all years pullfe_curr_year=`echo $pullfe_start_year|awk '{y=$1-1;printf "%3.3d", y}' -` join=0 while [ $pullfe_curr_year -lt $pullfe_end_year ]; do pullfe_curr_year=`echo $pullfe_curr_year|awk '{y=1+$1;printf "%3.3d", y}' -` # Protect the year assignment from cccjob substitution eval year\=$pullfe_curr_year # These definitions must appear inside this loop name_yearly='' pullfe_name_yearly=${name_yearly:-''} # pullfe_name_yearly is a space separated list of file name templates # No file name template in this list may contain whitespace for bname in $pullfe_name_yearly; do if [ -n "$pullfe_suffix_list" ]; then # Iterate over the suffixes in pullfe_suffix_list for suffix in $pullfe_suffix_list; do # Strip off any "+" delimited modifier from suffix # These modifiers (used to limit processing to specific months) # are ignored when name_yearly is used to form file names suffix=`echo $suffix|sed 's/+.*$//'` # Use the variable fileN (N=1,2,...) to hold the next file name join=`echo $join|awk '{printf "%d",$1+1}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` eval file${join}=$bname$suffix done else # Use the name templates in pullfe_name_yearly "as is" join=`echo $join|awk '{printf "%d",$1+1}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` eval file${join}=$bname fi done done else # Generate a list of history files for all months pullfe_curr_year=`echo $pullfe_start_year|awk '{y=$1-1;printf "%3.3d", y}' -` join=0 while [ $pullfe_curr_year -lt $pullfe_end_year ]; do pullfe_curr_year=`echo $pullfe_curr_year|awk '{y=1+$1;printf "%3.3d", y}' -` # Protect the year assignment from cccjob substitution eval year\=$pullfe_curr_year if [ $pullfe_curr_year -eq $pullfe_start_year ]; then mm=`echo $pullfe_start_mon|awk '{m=$1-1;printf "%2.2d", m}' -` else mm=0 fi if [ $pullfe_curr_year -eq $pullfe_end_year ]; then mm_end=$pullfe_end_mon else mm_end=12 fi while [ $mm -lt $mm_end ]; do mm=`echo $mm|awk '{m=1+$1;printf "%2.2d", m}' -` # Protect the mon assignment from cccjob substitution eval mon\=$mm if [ "$pullfe_suffix_" = '1' ]; then bname=${pullfe_prefix_}${pullfe_curr_year}_m${mm}_ else bname=${pullfe_prefix_}${pullfe_curr_year}_m${mm} fi # pullfe_name_monthly, if defined, will be used to define # the file name (up to the suffix) name_monthly='' pullfe_name_monthly=${name_monthly:-$bname} # pullfe_name_monthly is a space separated list of file name templates # No file name template in this list may contain whitespace for bname in $pullfe_name_monthly; do if [ -n "$pullfe_suffix_list" ]; then # Iterate over the suffixes in pullfe_suffix_list for suffix in $pullfe_suffix_list; do mlist=`echo $suffix|awk -F'+' '{print $2}' -` mlist=`echo $mlist|sed 's/,/ /g'` if [ -n "$mlist" ]; then suffix=`echo $suffix|sed 's/+.*$//'` # assume that mlist is a white space separated list of numbers # indicating which months to dump for xx in $mlist; do if [ $xx -eq $mm ]; then join=`echo $join|awk '{j=1+$1;printf "%d",j}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` eval file${join}=$bname$suffix fi done else join=`echo $join|awk '{j=1+$1;printf "%d",j}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` eval file${join}=$bname$suffix fi done else # Use the name templates in pullfe_name_monthly "as is" join=`echo $join|awk '{printf "%d",$1+1}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` eval file${join}=$bname fi done done done fi # Insert a user supplied list of file names that will override # any defined above # <> # join must be included in the inserted list of files names and be # assigned an integer value which is the number of files in the list [ -z "$join" ] && bail "join is not set" [ $join -le 0 ] && bail "Invalid join=$join" # ---Stop_submit_ignore_code---- # * ............................ Condef Parameters ............................ noprint=on nextjob=on # * ............................. Deck Definition ............................. if [ 1 -eq 2 ]; then n=0 while [ $n -lt $join ]; do n=`expr $n + 1` eval echo "pullfe: file$n = \$file$n" done fi # Identify the name of the back end machine back_end_mach='' [ -z "$back_end_mach" ] && bail "back_end_mach is not set" # Use rem_mach as the name of the remote machine in what follows rem_mach=${back_end_mach:='hadar'} [ -z "$rem_mach" ] && bail "rem_mach is not set" # Determine DATAPATH on the remote machine and use it to create # a partial file spec for the remote file rem_dpath=`ssh $rem_mach 'echo $DATAPATH'` rem_dir="${rem_mach}:$rem_dpath" # Protect these echos from cccjob substitution eval echo \"pullfe: CCRNTMP\=$CCRNTMP\" eval echo \"pullfe: RUNPATH\=$RUNPATH\" total_save_time=0 total_transfer_time=0 total_size=0 total_time_in=`date '+%s.%N'` avgtrate=0 nn=0 while [ $nn -lt $join ]; do nn=`echo $nn|awk '{printf "%d",$1+1}' -` eval local_file=\$file$nn [ -z "$local_file" ] && bail "Invalid file name in file$nn" # List the remote file with the largest numeric extension rem_list=`ssh $rem_mach ls -Ll $rem_dpath/$local_file'.[0-9][0-9][0-9]' | tail -1` ||\ bail "Unable to list remote file $rem_dpath/$local_file" rem_file=`echo $rem_list|awk '{printf "%s",$9}' -` rem_size=`echo $rem_list|awk '{printf "%s",$5}' -` rem_file=`basename $rem_file` rem_file="${rem_dir}/$rem_file" time_in=`date '+%s.%N'` scp -o NoneSwitch=yes $rem_file $local_file 2>&1 ||\ bail "Error on scp of $local_file" time_out=`date '+%s.%N'` local_size=`ls -l $local_file|awk '{printf "%d",$5}' -` [ $rem_size -ne $local_size ] &&\ bail "Remote file size $rem_size does not match loacl file size $local_size" delta_time=`echo $time_in $time_out|awk '{t=$2-$1;printf "%g",t}' -` total_size=`echo $total_size $local_size|awk '{s=$1+$2;printf "%g",s}' -` # Accumulate transfer times total_transfer_time=`echo $total_transfer_time $delta_time|awk '{t=$1+$2;printf "%g",t}' -` # Determine transfer rate in MB/second trate=`echo $delta_time $local_size|\ awk '{s=(1.0*$2)/1048576.0;r=s/(1.0*$1);printf "%g",r}' -` avgtrate=`echo $avgtrate $trate|awk '{r=$1+$2;printf "%g",r}' -` echo "Time to transfer $local_size bytes is $delta_time seconds ($trate MB/s)" time_in=`date '+%s.%N'` save $local_file $local_file release $local_file time_out=`date '+%s.%N'` delta_time=`echo $time_in $time_out|awk '{t=$2-$1;printf "%g",t}' -` # Accumulate save times total_save_time=`echo $total_save_time $delta_time|awk '{t=$1+$2;printf "%g",t}' -` echo "Time to save $local_size bytes is $delta_time seconds" done total_time_out=`date '+%s.%N'` total_delta_time=`echo $total_time_in $total_time_out|awk '{t=$2-$1;printf "%g",t}' -` avgtrate=`echo $avgtrate $join|awk '{r=$1/(1.0*$2);printf "%g",r}' -` # total_size in GB total_size=`echo $total_size|awk '{s=$1/1.073741824e9;printf "%g",s}' -` # echo " " # echo "##############################" # echo "####### Total time: $total_delta_time seconds" # echo "####### Total transfer time: $total_transfer_time seconds" # echo "####### Total save time: $total_save_time seconds" # echo "####### Total size transferred: $total_size GB" # echo "####### Average transfer rate: $avgtrate MB/s" # echo "##############################" # echo " " rm -f local_log touch local_log curr_date=`date` echo "######### PULLFE: $curr_date" >> local_log echo " runid: $runid" >> local_log echo "start year/month: $pullfe_start_year / $pullfe_start_mon" >> local_log echo " stop year/month: $pullfe_end_year / $pullfe_end_mon" >> local_log echo " CCRNTMP: $CCRNTMP" >> local_log echo " RUNPATH: $RUNPATH" >> local_log echo " Total time: $total_delta_time seconds" >> local_log echo " Total transfer time: $total_transfer_time seconds" >> local_log echo " Total save time: $total_save_time seconds" >> local_log echo "Total size transferred: $total_size GB" >> local_log echo " Average transfer rate: $avgtrate MB/s" >> local_log echo " " >> local_log cat local_log # If pullfe_log is set to "on" then transfer_log is appended # with info related to the current transfer pullfe_log=on XXX=`echo $pullfe_log|sed 's/ //g'` eval pullfe_log\=$XXX [ "$pullfe_log" = 'on' ] && eval pullfe_log\=1 [ "$pullfe_log" = 'off' ] && eval pullfe_log\=0 [ "$pullfe_log" = 'yes' ] && eval pullfe_log\=1 [ "$pullfe_log" = 'no' ] && eval pullfe_log\=0 if [ $pullfe_log -eq 1 ]; then log_file=${JHOME:-$HOME}/tmp/transfer_log touch $log_file cat local_log >> $log_file # curr_date=`date` # echo "######### PULLFE: $curr_date" >> $log_file # echo " runid: $runid" >> $log_file # echo "start year/month: $pullfe_start_year / $pullfe_start_mon" >> $log_file # echo " stop year/month: $pullfe_end_year / $pullfe_end_mon" >> $log_file # echo " Total time: $total_delta_time seconds" >> $log_file # echo " Total transfer time: $total_transfer_time seconds" >> $log_file # echo " Total save time: $total_save_time seconds" >> $log_file # echo "Total size transferred: $total_size GB" >> $log_file # echo " Average transfer rate: $avgtrate MB/s" >> $log_file # echo " " >> $log_file fi # Clean up the lock file if [ -n "$pullfe_lock" ]; then rm -f $pullfe_lock echo "Removed lock file $pullfe_lock" fi end_of_script . endjcl.cdk #end_of_job