#!/bin/sh #======================================================================= # Copy pooled seasonal files to cfs --- psdump --- # $Id: psdump_jobdef 656 2011-12-28 19:51:43Z acrnrls $ #======================================================================= # # files dumped will be of the form # ${psdump_prefix}_${start_year}_${end_year}_${season}_$suffix # for all seasons from previous_year, previous_month to current_year, # current_month or from current_year, current_month to next_year, # next_month depending on which of previous_(year|month) or # next_(year|month) are set. # The default suffix list is "gp xp" which means that gp and xp files # are dumped each season # # The variables current_year, current_month, previous_year and # previous_month are set when the job string is created. # # explicitly setting psdump_start_year overrides previous_year # explicitly setting psdump_start_mon overrides previous_month # explicitly setting psdump_stop_year overrides current_year # explicitly setting psdump_stop_mon overrides current_month # # A variable named psdump_suffix_list may be set to modify the list # of files that will be dumped. See below for details. # # psdump_prefix_ can be set to override the value of psdump_prefix # files dumped will then be of the form # ${psdump_prefix_}${year}_${season}_$suffix # note the missing underscore between psdump_prefix_ and year. # # psdump_djf, psdump_mam, psdump_jja, psdump_son and psdump_ann may be set # to flag the inclusion of individual seasonal or annual average files. # # psdump_mon may be set to flag the inclusion of all monthly files. #======================================================================= # # keyword :: psdump # description :: dump pooled seasonal, monthly or annual averaged files to cfs # set -a . betapath2 # * ........................... Parmsub Parameters ............................ runid="job000"; uxxx='uxxx'; pool_uxxx=$uxxx; psdump_uxxx=$pool_uxxx; jobname=psdump; psdump_prefix="${psdump_uxxx}_${runid}"; psdump_prefix_=${psdump_prefix}_ crawork="${runid}_job"; username="acrnxxx"; user="XXX"; nqsprfx="${runid}_"; nqsext=''; dump_cfsuser=$username psdump_cfsuser=$dump_cfsuser cfsuser=$psdump_cfsuser RUNID=`echo "${runid}"|tr '[a-z]' '[A-Z]'`; stime="1800"; memory1="250mb"; lopgm="lopgm"; # Allow the user to reset CCRNTMP and/or RUNPATH pool_RUNPATH='' RUNPATH=${pool_RUNPATH:=$RUNPATH} pool_CCRNTMP='' CCRNTMP=${pool_CCRNTMP:=$CCRNTMP} # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi # BERUNPATH must be set if files are to be copied from a machine that is # not the default back end machine BERUNPATH=$BERUNPATH # RMTRUNPATH is used when vic = on is specified RMTRUNPATH=$RMTRUNPATH # ---Start_submit_ignore_code---- stamp=`date "+%j%H%M%S"$$` # Use -e option if recognized by echo if [ "X`echo -e`" = "X-e" ]; then echo_e() { echo ${1+"$@"}; } else echo_e() { echo -e ${1+"$@"}; } fi # bail is a simple error exit routine # Note: we write the error directly to a file in ~/.queue so that this # info is not lost if/when stdout is not returned error_out="${JHOME:-$HOME}/.queue/error_psdump_${runid}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out bail(){ echo_e `date`" --- psdump: $*" echo_e `date`" --- psdump: $*" >>$error_out exit 1 } # These variables are set when the job string is created previous_year=NotSet previous_month=NotSet current_year=NotSet current_month=NotSet next_year=NotSet next_month=NotSet run_start_year=NotSet run_start_month=NotSet run_stop_year=NotSet run_stop_month=NotSet # Pooling start and stop dates # These may differ from run start and stop dates, but are the same by default pool_start_year=$run_start_year pool_start_month=$run_start_month pool_stop_year=$run_stop_year pool_stop_month=$run_stop_month # If either reset_start_year or reset_stop_year are set then they must be # of the form old_year:new_year (ie a colon separated pair of integers) # where the first integer is the year that needs to be changed # and the second integer is the year that it will be changed to. # These may potentially change the value of start_year or stop_year that # are defined after the call to make_file_name_list below psdump_reset_start_year='' reset_start_year=${psdump_reset_start_year:=''} psdump_reset_end_year='' psdump_reset_stop_year=$psdump_reset_end_year reset_stop_year=${psdump_reset_stop_year:=''} # This invocation of make_file_name_list will process the *_year and *_months # variables defined above and output a file containing definitions for # start_year, start_mon, stop_year, stop_mon tmp_file_list="${JHOME:-$HOME}/tmp/psdump_date_list_${runid}_${stamp}" make_file_name_list --dates_only $tmp_file_list >>$error_out 2>&1 ||\ bail "Problem in make_file_name_list" rm -f $error_out # Verify that the output list is not empty [ ! -s "$tmp_file_list" ] && bail "Unable to create file list" # A file list was created ...source it # This will define start_year, start_mon, stop_year, stop_mon : ; . $tmp_file_list rm -f $tmp_file_list # Define start and stop dates psdump_start_year=$start_year psdump_start_mon=$start_mon psdump_stop_year=$stop_year psdump_stop_mon=$stop_mon # ym_range is used in the definition of file names to be dumped # define a range string in the form YYYYmMM_YYYYmMM to be used in file names ym_range="${psdump_start_year}m${psdump_start_mon}" ym_range="${ym_range}_${psdump_stop_year}m${psdump_stop_mon}" # In the special case that start and stop dates are identical use a YYYY format if [ $psdump_start_year -eq $psdump_stop_year ]; then if [ $psdump_start_mon -eq $psdump_stop_mon ]; then ym_range=$psdump_start_year fi fi # If arclabel is set then files to be dumped will be combined in a # cmcarc file with a name of the form {arclabel}_{Date-Time-Stamp}_arc # If arclabel is not set then files to be dumped will be combined in a # cmcarc file with a name of the form {common-prefix}_{Date-Time-Stamp}_arc psdump_arclabel="${psdump_uxxx}_${runid}_${ym_range}_pooled" arclabel=${psdump_arclabel:-''} # psdump_suffix_list is a white space separated list of suffixes # of file names to be generated here. Any suffix in this list may be # modified by appending a + followed by a comma separated list of # numbers (no white space is allowed within this modifier). Each # number within the modifier list will correspond to a month (1-12) # for which a file with this suffix is to be included. If the # modifier exists for a particular suffix then only those months # indicated in the modifier will be added to the file list. psdump_suffix_list='gp xp' # psdump_suffix_ flags the addition of an underscore between the # suffix and the month (e.g. djf_gs) in file names generated here psdump_suffix_=1 XXX=`echo $psdump_suffix_|sed 's/ //g'` eval psdump_suffix_\=$XXX [ "$psdump_suffix_" = 'on' ] && eval psdump_suffix_\=1 [ "$psdump_suffix_" = 'off' ] && eval psdump_suffix_\=0 [ "$psdump_suffix_" = 'yes' ] && eval psdump_suffix_\=1 [ "$psdump_suffix_" = 'no' ] && eval psdump_suffix_\=0 # psdump_djf, psdump_mam, psdump_jja and psdump_son flag the inclusion of # seasonal average files determined as file names of the form # ${psdump_prefix_}${year}_${season}_$suffix # for season = djf, mam, jja or son psdump_djf=1 XXX=`echo $psdump_djf|sed 's/ //g'` eval psdump_djf\=$XXX [ "$psdump_djf" = 'on' ] && eval psdump_djf\=1 [ "$psdump_djf" = 'off' ] && eval psdump_djf\=0 [ "$psdump_djf" = 'yes' ] && eval psdump_djf\=1 [ "$psdump_djf" = 'no' ] && eval psdump_djf\=0 psdump_mam=1 XXX=`echo $psdump_mam|sed 's/ //g'` eval psdump_mam\=$XXX [ "$psdump_mam" = 'on' ] && eval psdump_mam\=1 [ "$psdump_mam" = 'off' ] && eval psdump_mam\=0 [ "$psdump_mam" = 'yes' ] && eval psdump_mam\=1 [ "$psdump_mam" = 'no' ] && eval psdump_mam\=0 psdump_jja=1 XXX=`echo $psdump_jja|sed 's/ //g'` eval psdump_jja\=$XXX [ "$psdump_jja" = 'on' ] && eval psdump_jja\=1 [ "$psdump_jja" = 'off' ] && eval psdump_jja\=0 [ "$psdump_jja" = 'yes' ] && eval psdump_jja\=1 [ "$psdump_jja" = 'no' ] && eval psdump_jja\=0 psdump_son=1 XXX=`echo $psdump_son|sed 's/ //g'` eval psdump_son\=$XXX [ "$psdump_son" = 'on' ] && eval psdump_son\=1 [ "$psdump_son" = 'off' ] && eval psdump_son\=0 [ "$psdump_son" = 'yes' ] && eval psdump_son\=1 [ "$psdump_son" = 'no' ] && eval psdump_son\=0 # psdump_ann flags the inclusion of annual average files determined # as file names with ann replacing djf, mam, jja or son. psdump_ann=1 XXX=`echo $psdump_ann|sed 's/ //g'` eval psdump_ann\=$XXX [ "$psdump_ann" = 'on' ] && eval psdump_ann\=1 [ "$psdump_ann" = 'off' ] && eval psdump_ann\=0 [ "$psdump_ann" = 'yes' ] && eval psdump_ann\=1 [ "$psdump_ann" = 'no' ] && eval psdump_ann\=0 # psdump_mon flags the inclusion ofmonthly average files determined # as file names with m01,m02,... replacing djf, mam, jja or son. psdump_mon=1 XXX=`echo $psdump_mon|sed 's/ //g'` eval psdump_mon\=$XXX [ "$psdump_mon" = 'on' ] && eval psdump_mon\=1 [ "$psdump_mon" = 'off' ] && eval psdump_mon\=0 [ "$psdump_mon" = 'yes' ] && eval psdump_mon\=1 [ "$psdump_mon" = 'no' ] && eval psdump_mon\=0 # Create file names of pooled seasonal files to be dumped join=0 for season in djf mam jja son ann mon; do if [ "$season" = "djf" -a $psdump_djf -ne 1 ]; then continue fi if [ "$season" = "mam" -a $psdump_mam -ne 1 ]; then continue fi if [ "$season" = "jja" -a $psdump_jja -ne 1 ]; then continue fi if [ "$season" = "son" -a $psdump_son -ne 1 ]; then continue fi if [ "$season" = "ann" -a $psdump_ann -ne 1 ]; then continue fi if [ "$season" = "mon" -a $psdump_mon -ne 1 ]; then continue fi if [ "$season" = "mon" ]; then bname=${psdump_prefix_}${ym_range}_ elif [ "$season" = "ann" ]; then # Modify the file name for multi year annually average files when # the first and/or last year is incomplete, and therefore missing curr_start_year=$psdump_start_year curr_stop_year=$psdump_stop_year if [ $psdump_start_year -eq $pool_start_year ]; then if [ $pool_start_month -ne 1 ]; then # If this is annual pooling and pool_start_month is not JAN then increment # psdump_start_year because the first year will not have been pooled curr_start_year=`echo $psdump_start_year|awk '{printf "%3.3d",$1+1}' -` fi fi if [ $psdump_stop_year -eq $pool_stop_year ]; then if [ $pool_stop_month -ne 12 ]; then # If this is annual pooling and pool_stop_month is not DEC then decrement # psdump_stop_year because the last year will not have been pooled curr_stop_year=`echo $psdump_stop_year|awk '{printf "%3.3d",$1-1}' -` fi fi # Set bname using curr_start_year and curr_stop_year as defined above # For annual pooling files names, the first month is always 1 and the # last month is always 12. This will need to change if annual pooling # is done on an interval other than the calendar year if [ "$psdump_suffix_" = '1' ]; then bname=${psdump_prefix_}${curr_start_year}m01_${curr_stop_year}m12_${season}_ else bname=${psdump_prefix_}${curr_start_year}m01_${curr_stop_year}m12_$season fi else # Multi year seasonal pooled files (djf, mam, jja, son) if [ "$psdump_suffix_" = '1' ]; then bname=${psdump_prefix_}${ym_range}_${season}_ else bname=${psdump_prefix_}${ym_range}_$season fi fi for suffix in $psdump_suffix_list; do mlist=`echo $suffix|awk -F'+' '{print $2}' -` if [ -n "$mlist" ]; then # ignore any appended month list, it is not used suffix=`echo $suffix|sed 's/+.*$//'` fi if [ "$season" = "mon" ]; then mm=0 while [ $mm -lt 12 ]; do mm=`echo $mm|awk '{printf "%2.2d",$1+1}' -` join=`echo $join|awk '{j=1+$1;printf "%d",j}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` if [ "$psdump_suffix_" = '1' ]; then eval file${join}=${bname}m${mm}_$suffix else eval file${join}=${bname}m${mm}$suffix fi done else join=`echo $join|awk '{j=1+$1;printf "%d",j}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` eval file${join}=$bname$suffix fi done done [ $join -eq 0 ] && bail "No files selected for the year/mon range $psdump_start_year/$psdump_start_mon to $psdump_stop_year/$psdump_stop_mon" # ---Stop_submit_ignore_code---- # * ............................ Condef Parameters ............................ noprint=on nextjob=on dump_masterdir=off psdump_masterdir=${dump_masterdir:=off} masterdir=${psdump_masterdir:=off} dump_shortermdir=on psdump_shortermdir=${dump_shortermdir:=on} shortermdir=${psdump_shortermdir:=on} # The default if none of sv, both, vic or svvic are set to "on" is to # dump files to the back end machine # Note: Once again submit3 is causing a problem. It will not respect end of line # comments inside its internally created file named job.info. As a result a # constructs such as "vic =" are flagged as a syntax error even though they # are commented out. # sv : on means dump files from pollux dump_sv=off psdump_sv=${dump_sv:=off} sv=${psdump_sv:=off} both=off # both : on means dump files from pollux and the back end machine vic=off # vic : on means dump files from Victoria svvic=off # svvic : on means dump files from pollux and Victoria # nolist : off means save CFSDATA* files in ~/info # nolist : on means do not save CFSDATA* files in ~/info dump_nolist=off psdump_nolist=${dump_nolist:=off} nolist=${psdump_nolist:=off} # rmdskcpy = on means remove the copy of the file on the cfs disk buffer # as soon as it has been written to tape psdump_rmdskcpy='' rmdskcpy=${psdump_rmdskcpy:=off} # cnfsallow = on will allow the user to use a cnfs dir for # RUNPATH and/or CCRNTMP when svsave = on # Normally this is not allowed because of the possibility that files # may get corrupted during transfer from/to the nfs file system psdump_cnfsallow='' cnfsallow=${psdump_cnfsallow:=off} # dpalist = on means store DATAPATH information in a local file for use by access/save/delete # Since this local file is never deleted, dpalist = on should only be used when access, # save or delete are part of a job that runs in a temporary working directory which gets # removed after the job completes. Most decks do this, but not all. You have been warned! psdump_dpalist='' dpalist=${psdump_dpalist:=''} # besc = on causes a tdumper job to run on the back end (spica/hadar) # These definitions need to be on the same line to work around an error in the submit scripts psdump_besc=''; besc=${psdump_besc:=off} # * ............................. Deck Definition ............................. . tdumper.dk #end_of_job