#!/bin/sh #======================================================================= # Copy files to the front end --- othr2fe --- # $Id: othr2fe_jobdef 651 2011-05-12 23:17:46Z acrnrls $ #======================================================================= # # files copied will be of the form # ${othr2fe_prefix}_${year}_m${mon}_$suffix # for all months from previous_year, previous_month to current_year, # current_month or from current_year, current_month to next_year, # next_month depending on which of previous_(year|month) or # next_(year|month) are set. # The default suffix list is "gs ss rs+12 ab+12 an+12" which means # that gs and ss files are copied every month while rs, ab and an files # are copied once every Dec that falls within this range. # # The variables current_year, current_month, previous_year and # previous_month are set when the job string is created. # # explicitly setting othr2fe_start_year overrides previous_year # explicitly setting othr2fe_start_mon overrides previous_month # explicitly setting othr2fe_end_year overrides current_year # explicitly setting othr2fe_end_mon overrides current_month # # A variable named othr2fe_suffix_list may also be set to modify the list # of files that will be copied. See below for details. # # othr2fe_prefix_ can be set to override the value of othr2fe_prefix # files copied will then be of the form # ${othr2fe_prefix_}${year}_m${mon}_$suffix # note the missing underscore between othr2fe_prefix_ and year. #======================================================================= # # keyword :: othr2fe # description :: copy files to the front end # set -a . betapath2 # * ........................... Parmsub Parameters ............................ # These variables are set when the job string is created previous_year=NotSet previous_month=NotSet current_year=NotSet current_month=NotSet next_year=NotSet next_month=NotSet run_start_year=NotSet run_start_month=NotSet run_stop_year=NotSet run_stop_month=NotSet runid="job000"; uxxx='uxxx'; model_uxxx=$uxxx; othr2fe_uxxx=$model_uxxx; othr2fe_runid='' # memory99=1 eval runid\=${othr2fe_runid:=$runid} # memory99=1 crawork="${runid}_job"; username="acrnxxx"; user="XXX"; year=yyy; mon=mm nqsprfx="${runid}_"; nqsext=''; jobname=othr2fe stime="1800"; memory1="900mb"; lopgm="lopgm"; # Temporary directory where this script will run othr2fe_CCRNTMP='' CCRNTMP=${othr2fe_CCRNTMP:=$CCRNTMP} # RUNPATH on execution machine othr2fe_RUNPATH='' RUNPATH=${othr2fe_RUNPATH:=$RUNPATH} # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi # The name of the front end machine CMCFEDEST=pollux # RMTRUNPATH is used when vic = on is specified RMTRUNPATH=$RMTRUNPATH # ---Start_submit_ignore_code---- stamp=`date "+%j%H%M%S"$$` # Use -e option if recognized by echo if [ "X`echo -e`" = "X-e" ]; then echo_e() { echo ${1+"$@"}; } else echo_e() { echo -e ${1+"$@"}; } fi # bail is a simple error exit routine # Note: we write the error directly to a file in ~/.queue so that this # info is not lost if/when stdout is not returned error_out="${JHOME:-$HOME}/.queue/error_othr2fe_${runid}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out bail(){ echo_e `date`" --- othr2fe: $*" echo_e `date`" --- othr2fe: $*" >>$error_out exit 1 } # The variables othr2fe_prefix_list and othr2fe_suffix_list are strings containing # embedded colons and whitespace which are interpreted as list delimiters. # These strings may be thought of as 2 dimensional arrays, the rows of these # arrays are colon (:) separated strings and each row is divided into columns # by separating on white space. # These variables are used by make_file_name_list along with runid and year/mon # information from the *_year and *_month variables to generate file names. # In make_file_name_list the prefix_list and suffix_list strings are first # separated into colon delimited lists (rows). There must be an equal number of # rows in each of prefix_list and suffix_list because these rows will be # used in pairs. # Each pair of rows (one row from prefix_list and one row from suffix_list) # is separated into a white space separated list. Each element of these # white space separated lists is a single prefix or suffix (possibly modified # by appending a "+" followed by a comma separated list of integers in the # range 1-12). No white space is allowed within a single prefix or suffix. # These individual (pre|suf)fixes are then iterated over for each year and # month and for each pair of rows in prefix_list and suffix_list to form the # desired set of file names, each of which is of the form # # ${prefix}_${runid}_${year}_m${mon}_${suffix} # Any prefix or suffix in these lists may be modified by appending a + # followed by a comma separated list of numbers (no white space is allowed # within this modifier). Each number within the modifier list will correspond # to a month (1-12) for which a file with this suffix is to be included. # If the modifier exists for a particular suffix then only those months # indicated in the modifier will be added to the file list. # If the above form of file name is inappropriate then the user may # provide a template or templates to produce arbitrary file names. # These templates are defined in the variable othr2fe_prefix_list. # Any individual prefix will be treated as a file name template if it # begins with a "%" character. The template will consist of everthing # after the "%" character and up to the next colon or white space. # It can be composed of anything but must ultimately (after variable # substitution, etc) result in a valid file name. When a template # is encountered, it is used as the entire file name (ie the "normal" file # name form is disregarded as is the corresponding suffix(s)). However, it # is subject to the same interation procedure as a normal prefix and # does undergo variable substitution. Variables that are defined for # substitution include year, mon, runid, uxxx, start_year, start_mon, # stop_year, stop_mon, all of the *_year and *_month variables defined # above as well as any user supplied variable definitions passed to # make_file_name_list as a command line option of the form var=val. othr2fe_suffix_list='' suffix_list="${othr2fe_suffix_list:-gs ss}" othr2fe_prefix_list='' prefix_list="${othr2fe_prefix_list:-$othr2fe_uxxx}" # Create a file containing a list of file names that may then be # "sourced" in the current environment to define the variables # file1, file2,..., file$join, join. These variables are used by # tdumper to compile the list of files to be archived. join=0 # make_file_name_list uses the variables current_year, current_month, # previous_year, previous_month, next_year and next_month to # determine start and stop dates for file name creation. # It also uses runid, prefix_list and suffix_list from the current # environment to build these file names. # Allow user supplied command line options for make_file_name_list # The following invocation of make_file_name_list will not allow multi-list # output so if any command line option is supplied that will turn on # multi-list output (e.g. --months_max=... --size_max=... --number_max=..) # then this script will abort. othr2fe_file_list_opts='' fopts="${othr2fe_file_list_opts:-}" othr2fe_mon_offset='' if [ -n "$othr2fe_mon_offset" ]; then # Set a user supplied month offset eval fopts=\"$fopts --mon_offset\=$othr2fe_mon_offset\" fi # Create a temporary file containing the file list tmp_file_list="${JHOME:-$HOME}/.queue/othr2fe_file_list_${runid}_${stamp}" make_file_name_list $fopts --nomulti_list $tmp_file_list >>$error_out 2>&1 ||\ bail "Problem in make_file_name_list" rm -f $error_out [ ! -s "$tmp_file_list" ] && bail "Unable to create file list" # A file list was created ...source it # This will define start_year,start_mon,stop_year,stop_mon and file1,file2,... : ; . $tmp_file_list # Delete the file that contains the file list rm -f $tmp_file_list # At this point file1, file2,... are defined in the current environment # as well as certain other variables such as start_year, start_mon, # stop_year and stop_mon which correspond to the start and stop dates # for the file names that were created. ym_range="${start_year}m${start_mon}_${stop_year}m${stop_mon}" # Insert a user supplied list of file names that will override # any defined above # <> # join must be included in the inserted list of files names and be # assigned an integer value which is the number of files in the list [ -z "$join" ] && bail "join is not set" [ $join -le 0 ] && bail "Invalid join=$join" # ---Stop_submit_ignore_code---- # * ............................ Condef Parameters ............................ noprint=on nextjob=on # The following must always be off eval sv\=off eval both\=off eval svvic\=off eval svcrd\=off # Files will be copied from the back end by default # If vic = on then files will be copied from Victoria vic=off # delet_files = on means delete files on the back end after the transfer. # Files will never be deleted from Vic disks even when delet_files = on. delet_files=off # * ............................. Deck Definition ............................. . othr2fe.dk #end_of_job