#!/bin/sh #======================================================================= # Copy files from cfs --- mload --- # $Id: mload_jobdef 669 2012-04-24 20:44:05Z acrnrls $ #======================================================================= # # Files loaded will be of the form # # ${prefix}_${year}_m${mon}_${suffix} # # for all months from previous_year, previous_month to current_year, # current_month or from current_year, current_month to next_year, # next_month depending on which of previous_(year|month) or # next_(year|month) are set. # # Both prefix and suffix may have multiple values, in which case file # name generation iterates over these multiple values. Values for prefix # and suffix are defined by user supplied lists which are read from the # parmsub variables mload_prefix_list and mload_suffix_list. # See below for details of how these lists are composed. # # Larry Solheim Apr 2006 #======================================================================= # # keyword :: mload # description :: load model files from cfs # set -a . betapath2 # * ........................... Parmsub Parameters ............................ # These variables are set when the job string is created previous_year=NotSet previous_month=NotSet current_year=NotSet current_month=NotSet next_year=NotSet next_month=NotSet run_start_year=NotSet run_start_month=NotSet run_stop_year=NotSet run_stop_month=NotSet runid="job000"; uxxx='uxxx'; model_uxxx=$uxxx; mload_uxxx=$model_uxxx; mload_runid='' # memory99=1 eval runid\=${mload_runid:=$runid} # memory99=1 crawork="${runid}_job"; username="acrnxxx"; user="XXX"; year=yyy; mon=mm load_cfsuser=$username mload_cfsuser=$load_cfsuser cfsuser=$mload_cfsuser nqsprfx="${runid}_"; nqsext=''; jobname=mload stime="1800"; memory1="900mb"; lopgm="lopgm"; # If set, arcfile is the name of a cmcarc file in which # the files to be retrieved are assumed to exist # If not set, all cmcarc and non-cmcarc files will be searched arcfile='' # If arcfile is not set and arclabel is set then all cmcarc files that # contain the substring arclabel within their file name will be searched # for the files that are to be retrieved mload_arclabel='' arclabel=${mload_arclabel:=''} # Temporary directory where this script will run mload_CCRNTMP='' CCRNTMP=${mload_CCRNTMP:=$CCRNTMP} # RUNPATH on execution machine mload_RUNPATH='' if [ -n "$mload_RUNPATH" -a "$mload_RUNPATH" = "CCRNTMP" ]; then # Allow the special case of "mload_RUNPATH = CCRNTMP" to force RUNPATH # and CCRNTMP to be the same dir RUNPATH=$CCRNTMP else RUNPATH=${mload_RUNPATH:=$RUNPATH} fi # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi # RMTRUNPATH is used when vic = on is specified RMTRUNPATH=$RMTRUNPATH # Override md5sum check in tloader mload_nocheck=off nocheck=${mload_nocheck:=off} # ---Start_submit_ignore_code---- stamp=`date "+%j%H%M%S"$$` # Use -e option if recognized by echo if [ "X`echo -e`" = "X-e" ]; then echo_e() { echo ${1+"$@"}; } else echo_e() { echo -e ${1+"$@"}; } fi # bail is a simple error exit routine # Note: we write the error directly to a file in ~/.queue so that this # info is not lost if/when stdout is not returned error_out="${JHOME:-$HOME}/.queue/error_mload_${runid}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out bail(){ echo_e `date`" --- mload: $*" echo_e `date`" --- mload: $*" >>$error_out exit 1 } # BERUNPATH must be set if files are to be copied from a machine that is # not the default back end machine bemach='' if [ -z "$bemach" ]; then # If bemach is not set explicitly then see if this job is executing on # a back end machine and use that machine name for bemach this_mach=`uname -n|awk -F\. '{print \$1}' -` on_back_end=0 case $this_mach in c[0-9]*) on_back_end=1 ;; zeta|saiph|spica|hadar) on_back_end=1 ;; za|sa|sp|ha) on_back_end=1 ;; esac if [ $on_back_end -eq 1 ]; then # Set this_mach to its known alias case $this_mach in ha|ha*) this_mach=hadar ;; sp|sp*) this_mach=spica ;; c1*) this_mach=spica ;; c2*) this_mach=hadar ;; za|ze*) this_mach=zeta ;; sa|sa*) this_mach=saiph ;; c6*) this_mach=saiph ;; c7*) this_mach=zeta ;; esac bemach=$this_mach fi fi if [ -n "$bemach" ]; then # Reset BERUNPATH if bemach is set # Ensure bemach contains a known alias case $bemach in ha|ha*) bemach=hadar ;; sp|sp*) bemach=spica ;; za|ze*) bemach=zeta ;; sa|sa*) bemach=saiph ;; esac XXX=`ssh $bemach echo '$RUNPATH' 2>/dev/null` || bail "Unable to determine BERUNPATH" BERUNPATH=$XXX else BERUNPATH=$BERUNPATH fi # The variables mload_prefix_list and mload_suffix_list are strings containing # embedded colons and whitespace which are interpreted as list delimiters. # These strings may be thought of as 2 dimensional arrays, the rows of these # arrays are colon (:) separated strings and each row is divided into columns # by separating on white space. # These variables are used by make_file_name_list along with runid and year/mon # information from the *_year and *_month variables to generate file names. # In make_file_name_list the prefix_list and suffix_list strings are first # separated into colon delimited lists (rows). There must be an equal number of # rows in each of prefix_list and suffix_list because these rows will be # used in pairs. # Each pair of rows (one row from prefix_list and one row from suffix_list) # is separated into a white space separated list. Each element of these # white space separated lists is a single prefix or suffix (possibly modified # by appending a "+" followed by a comma separated list of integers in the # range 1-12). No white space is allowed within a single prefix or suffix. # These individual (pre|suf)fixes are then iterated over for each year and # month and for each pair of rows in prefix_list and suffix_list to form the # desired set of file names, each of which is of the form # # ${prefix}_${runid}_${year}_m${mon}_${suffix} # Any prefix or suffix in these lists may be modified by appending a + # followed by a comma separated list of numbers (no white space is allowed # within this modifier). Each number within the modifier list will correspond # to a month (1-12) for which a file with this suffix is to be included. # If the modifier exists for a particular suffix then only those months # indicated in the modifier will be added to the file list. # If the above form of file name is inappropriate then the user may # provide a template or templates to produce arbitrary file names. # These templates are defined in the variable mload_prefix_list. # Any individual prefix will be treated as a file name template if it # begins with a "%" character. The template will consist of everthing # after the "%" character and up to the next colon or white space. # It can be composed of anything but must ultimately (after variable # substitution, etc) result in a valid file name. When a template # is encountered, it is used as the entire file name (ie the "normal" file # name form is disregarded as is the corresponding suffix(s)). However, it # is subject to the same interation procedure as a normal prefix and # does undergo variable substitution. Variables that are defined for # substitution include year, mon, runid, uxxx, start_year, start_mon, # stop_year, stop_mon, all of the *_year and *_month variables defined # above as well as any user supplied variable definitions passed to # make_file_name_list as a command line option of the form var=val. mload_suffix_list='' suffix_list="${mload_suffix_list:-gs ss rs+12 ab+12 an+12}" mload_prefix_list='' prefix_list="${mload_prefix_list:-$mload_uxxx}" # Create a file containing a list of file names that may then be # "sourced" in the current environment to define the variables # file1, file2,..., file$join, join. These variables are used by # tdumper to compile the list of files to be archived. join=0 # make_file_name_list uses the variables current_year, current_month, # previous_year, previous_month, next_year and next_month to # determine start and stop dates for file name creation. # It also uses runid, prefix_list and suffix_list from the current # environment to build these file names. # Allow user supplied command line options for make_file_name_list # The following invocation of make_file_name_list will not allow multi-list # output so if any command line option is supplied that will turn on # multi-list output (e.g. --months_max=... --size_max=... --number_max=..) # then this script will abort. mload_file_list_opts='' fopts="${mload_file_list_opts:-}" mload_mon_offset='' if [ -n "$mload_mon_offset" ]; then # Set a user supplied month offset eval fopts=\"$fopts --mon_offset\=$mload_mon_offset\" fi # Create a temporary file containing the file list tmp_file_list="${JHOME:-$HOME}/.queue/mload_file_list_${runid}_${stamp}" make_file_name_list $fopts --nomulti_list $tmp_file_list >>$error_out 2>&1 ||\ bail "Problem in make_file_name_list" rm -f $error_out [ ! -s "$tmp_file_list" ] && bail "Unable to create file list" # A file list was created ...source it # This will define start_year,start_mon,stop_year,stop_mon and file1,file2,... : ; . $tmp_file_list # At this point file1, file2,... are defined in the current environment # as well as certain other variables such as start_year, start_mon, # stop_year and stop_mon which correspond to the start and stop dates # for the file names that were created. ym_range="${start_year}m${start_mon}_${stop_year}m${stop_mon}" # Delete or keep the temporary file that contains the file list mload_keep_file_list=0 XXX=`echo $mload_keep_file_list|sed 's/ //g'` eval mload_keep_file_list\=$XXX [ "$mload_keep_file_list" = 'on' ] && eval mload_keep_file_list\=1 [ "$mload_keep_file_list" = 'off' ] && eval mload_keep_file_list\=0 [ "$mload_keep_file_list" = 'yes' ] && eval mload_keep_file_list\=1 [ "$mload_keep_file_list" = 'no' ] && eval mload_keep_file_list\=0 if [ $mload_keep_file_list -eq 1 ]; then # Rename the file list to include the year/month range # and place it in the users ~/.queue dir saved_file_list="${JHOME:-$HOME}/.queue/mload_file_list_${runid}_${ym_range}_${stamp}" mv -f $tmp_file_list $saved_file_list else # Delete the file that contains the file list rm -f $tmp_file_list fi # ---Stop_submit_ignore_code---- # * ............................ Condef Parameters ............................ noprint=on nextjob=on load_masterdir=off mload_masterdir=${load_masterdir:=off} masterdir=${mload_masterdir:=off} load_shortermdir=on mload_shortermdir=${load_shortermdir:=on} shortermdir=${mload_shortermdir:=on} # The default if none of sv, both, vic or svvic are set to "on" is to # load files onto the back end machine # Note: Once again submit3 is causing a problem. It will not respect end of line # comments inside its internally created file named job.info. As a result a # constructs such as "vic =" are flagged as a syntax error even though they # are commented out. # sv : on means load files onto pollux load_sv=off mload_sv=${load_sv:=off} sv=${mload_sv:=off} both=off # both : on means load files onto pollux and onto back end machine vic=off # vic : on means send files to Victoria svvic=off # svvic : on means load files onto pollux and send to Victoria # cnfsallow = on will allow the user to use a cnfs dir for # RUNPATH and/or CCRNTMP when svsave = on # Normally this is not allowed because of the possibility that files # may get corrupted during transfer from/to the nfs file system mload_cnfsallow=''; cnfsallow=${mload_cnfsallow:=off} # besc = on causes a tdumper job to run on the back end (spica/hadar) # These definitions need to be on the same line to work around an error in the submit scripts mload_besc=''; besc=${mload_besc:=off} # * ............................. Deck Definition ............................. . tloader.dk #end_of_job