#!/bin/sh #========================================================================= # Save files written to the JHOME_DATA dir on DATAPATH --- resavejhome --- # $Id$ #========================================================================= # # Larry Solheim ...Nov 2013 #========================================================================= # # keyword :: resavejhome # description :: save files written to the JHOME_DATA dir on DATAPATH # # set -a . betapath2 # * ........................... Parmsub Parameters ...................... uxxx='uxxx'; runid="job000"; nqsprfx="${runid}_"; nqsext='' crawork="${runid}_job"; username="acrnxxx"; user="XXX"; jobname=resavejhome; stime=1800; time=$stime; memory1="1000mb"; memory=$memory1 noprint=on nextjob=on debug=off #=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=# #=#=# Begin execute script #=#=# #=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=# . comjcl.cdk cat > Execute_Script <<'end_of_script' # Auto export is required for make_file_name_list set -a # ---Start_submit_ignore_code---- # These variables are set when the job string is created previous_year=NotSet previous_month=NotSet current_year=NotSet current_month=NotSet next_year=NotSet next_month=NotSet run_start_year=NotSet run_start_month=NotSet run_stop_year=NotSet run_stop_month=NotSet # Indicate how cccjob should be invoked # Setting CCCJOB_ROOT will allow a job specific version of cccjob to used CCCJOB_ROOT='' if [ -z "$CCCJOB_ROOT" ]; then CCCJOB_ENV='' MAKE_FILE_NAME_LIST=make_file_name_list cccjob_root_def='' else eval CCCJOB_ENV=\'env CCCJOB_ROOT\=$CCCJOB_ROOT\' MAKE_FILE_NAME_LIST="$CCCJOB_ROOT/bin/make_file_name_list" eval cccjob_root_def=\'CCCJOB_ROOT\=$CCCJOB_ROOT\' fi # Define a date string that will be used in file names etc stamp=`date "+%j%H%M%S"$$` # this_host will simply be the output from uname this_host=`uname -n|awk -F\. '{print \$1}' -` # this_mach will be a known alias (or possibly the actual machine name) this_mach=$this_host case $this_mach in c1*) this_mach=spica ;; c2*) this_mach=hadar ;; ib3*) this_mach=pollux ;; joule*) this_mach=joule ;; esac # Emulate echo -n option if not recognized by echo (ie echo without return) if [ "X`echo -n`" = "X-n" ]; then echo_n() { echo ${1+"$@"}'\c'; } else echo_n() { echo -n ${1+"$@"}; } fi # Define a file name that may be used to emit error messages error_out="$HOME/.queue/error_resavejhome_${runid}_${this_mach}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out # bail is a simple error exit routine bail(){ # Send error message to stdout echo `date`" $this_host $runid --- resavejhome: $*" # Also copy to a file in case stdout goes missing in action echo `date`" $this_host $runid --- resavejhome: $*" >>$error_out exit 1 } ToF(){ # usage: ToF var_name # purpose: Possibly reset the value of var_name to "0" (false) or "1" (true) # If var_name is null or has a value of "off" or "no" then reset to "0" # If var_name has a value of "on" or "yes" then reset to "1" # Otherwise return with var_name unchanged [ -z "$1" ] && bail "ToF requires a variable name as an argument" eval ToF_var\=\$$1 XXX=`echo $ToF_var|sed 's/ //g'` eval ToF_var\=$XXX if [ -n "$ToF_var" ]; then if [ "$ToF_var" = 'on' ]; then eval ToF_var\=1 elif [ "$ToF_var" = 'off' ]; then eval ToF_var\=0 elif [ "$ToF_var" = 'yes' ]; then eval ToF_var\=1 elif [ "$ToF_var" = 'no' ]; then eval ToF_var\=0 else eval ToF_var\=\$$1 fi else eval ToF_var\=0 fi eval $1=$ToF_var } # If either reset_start_year or reset_stop_year are set then they must be # of the form old_year:new_year (ie a colon separated pair of integers) # where the first integer is the year that needs to be changed # and the second integer is the year that it will be changed to. # These may potentially change the value of start_year or stop_year that # are defined after the call to make_file_name_list below resavejhome_reset_start_year='' reset_start_year=${resavejhome_reset_start_year:=''} resavejhome_reset_stop_year='' reset_stop_year=${resavejhome_reset_stop_year:=''} # reset_end_year is defined here for backward compatibility # It is effectively equivalent to reset_stop_year. resavejhome_reset_end_year='' reset_end_year=${resavejhome_reset_end_year:=''} # This invocation of make_file_name_list will process the *_year and *_month # variables defined above and output a file containing definitions for # start_year, start_mon, stop_year, stop_mon fopts='' resavejhome_mon_offset='' if [ -n "$resavejhome_mon_offset" ]; then # Set a user supplied month offset eval fopts=\"--mon_offset\=$resavejhome_mon_offset\" fi tmp_file_list="tmp_file_list_${runid}_${stamp}" $MAKE_FILE_NAME_LIST $fopts --dates_only $tmp_file_list >>$error_out 2>&1 || \ bail "Problem in make_file_name_list" rm -f $error_out # Verify that the output list is not empty [ ! -s "$tmp_file_list" ] && bail "Unable to determine start/stop dates." # A file list was created ...source it to define # start_year, start_mon, stop_year, stop_mon in the current environment : ; . $tmp_file_list rm -f $tmp_file_list # Define a variable containing a string that will identify the # current year/month range, for possible use in file names etc. ym_range="${start_year}m${start_mon}_${stop_year}m${stop_mon}" echo "runid = $runid range = $ym_range" # # logall = on turns on creation of log files for all jobs in this chunk # # These log files will be copied to an external log directory at the end # # of each time series chunk # logall=off # ToF logall # Initialize a variable that will contain the name of the file created # by the user supplied code snippet below resavejhome_files='' # Insert a user supplied code snippet that defines a here document # containing a list of full pathnames, one path name per line # This here doc will create a local file named $resavejhome_files # <> [ -z "$resavejhome_files" ] && bail "Missing file list name." [ ! -s "$resavejhome_files" ] && bail "Empty file list." # The user supplied files will be saved in the current RUNPATH/DATAPATH cat $resavejhome_files | while read fname; do # Ignore blank lines [ -z "$fname" ] && continue # Ignore lines with "#" as the first non-whitespace character ch1=`echo $fname|awk '{print substr($1,1,1)}' -` [ x"$ch1" = x"#" ] && continue # If the file is a link then assume that it was created by a previous # invocation of this script and points to a regular file on DATAPATH if [ -L $fname ]; then # Determine the name of the file this link points to rname=`ls -l $fname|awk '{print $NF}' -` # If this is not a file or link then assume the link is broken [ -f "$rname" -o -L "$rname" ] || bail "Broken link --> $fname <--" continue fi # Only accept regular files [ -f $fname ] || bail "Invalid file name $fname" # Save this file bname=`basename $fname|sed 's/\.[0-9][0-9][0-9]$//'` [ -z "$bname" ] && bail "Problem extracting base name from $fname" save $fname $bname # Do not release fname so that a link remains in the JHOME data dir # This will allow resubmission of a job that failed after saving a partial # list of files since the link will exist and therefore the above test for # regular file or link will pass. Note that any files that are copied but # the job dies before the link is made in save will still fail but will # leave a log file in the .queue dir done # If the user has set curr_jhome then use it to add an empty file to indicate # that all files have been copied sucessfully from JHOME to DATAPATH # Also remove the autorestart file from the same dir, if present curr_jhome='' if [ -n "$curr_jhome" ]; then [ -d $curr_jhome ] || bail "$curr_jhome is not a directory." # touch the file that indicates the copy was successful touch $curr_jhome/jhome_files_on_datapath chmod a+r $curr_jhome/jhome_files_on_datapath # Remove any autorestart file that may exist rm -f $curr_jhome/autorestart || : fi # ---Stop_submit_ignore_code---- end_of_script . endjcl.cdk #end_of_job