#!/bin/sh #======================================================================= # Delete pooled seasonal files from DATAPATH/RUNPATH --- psdelete --- # $Id: psdelete_jobdef 655 2011-08-04 23:04:36Z acrnrls $ #======================================================================= # # files deleted will be of the form # ${psdelete_prefix}_${start_year}_${end_year}_${season}_$suffix # for all seasons from previous_year, previous_month to current_year, # current_month or from current_year, current_month to next_year, # next_month depending on which of previous_(year|month) or # next_(year|month) are set. # The default suffix list is "gp xp" which means that gp and xp files # are deleted each season # # The variables current_year, current_month, previous_year and # previous_month are set when the job string is created. # # explicitly setting psdelete_start_year overrides previous_year # explicitly setting psdelete_start_mon overrides previous_month # explicitly setting psdelete_stop_year overrides current_year # explicitly setting psdelete_stop_mon overrides current_month # # A variable named psdelete_suffix_list may be set to modify the list # of files that will be deleted. See below for details. # # psdelete_prefix_ can be set to override the value of psdelete_prefix # files deleted will then be of the form # ${psdelete_prefix_}${year}_${season}_$suffix # note the missing underscore between psdelete_prefix_ and year. # # psdelete_djf, psdelete_mam, psdelete_jja, psdelete_son and psdelete_ann # may be set to flag the inclusion of individual seasonal or annual # average files. # # psdelete_mon may be set to flag the inclusion of all monthly files. #======================================================================= # # keyword :: psdelete # description :: delete pooled seasonal, monthly or annual averaged files # set -a . betapath2 # * ........................... Parmsub Parameters ............................ runid="job000"; uxxx='uxxx'; pool_uxxx=$uxxx; psdelete_uxxx=$pool_uxxx; jobname=psdelete; psdelete_prefix="${psdelete_uxxx}_${runid}"; psdelete_prefix_=${psdelete_prefix}_ crawork="${runid}_job"; username="acrnxxx"; user="XXX"; RUNID=`echo "${runid}"|tr '[a-z]' '[A-Z]'`; nqsprfx="${runid}_"; nqsext=''; stime="1800"; memory1="250mb"; lopgm="lopgm"; # It is more efficient to set CCRNTMP = $RUNPATH if all files # to be deleted are on pollux:$RUNPATH CCRNTMP=$CCRNTMP # BERUNPATH must be set if files are to be copied from a machine that is # not the default back end machine BERUNPATH=$BERUNPATH # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi # ---Start_submit_ignore_code---- stamp=`date "+%j%H%M%S"$$` # Use -e option if recognized by echo if [ "X`echo -e`" = "X-e" ]; then echo_e() { echo ${1+"$@"}; } else echo_e() { echo -e ${1+"$@"}; } fi # bail is a simple error exit routine # Note: we write the error directly to a file in ~/.queue so that this # info is not lost if/when stdout is not returned error_out="${JHOME:-$HOME}/.queue/error_psdelete_${runid}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out bail(){ echo_e `date`" --- psdelete: $*" echo_e `date`" --- psdelete: $*" >>$error_out exit 1 } # These variables are set when the job string is created previous_year=NotSet previous_month=NotSet current_year=NotSet current_month=NotSet next_year=NotSet next_month=NotSet run_start_year=NotSet run_start_month=NotSet run_stop_year=NotSet run_stop_month=NotSet # Pooling start and stop dates # These may differ from run start and stop dates, but are the same by default pool_start_year=$run_start_year pool_start_month=$run_start_month pool_stop_year=$run_stop_year pool_stop_month=$run_stop_month pool_ann_start_year=$pool_start_year pool_ann_start_month=$pool_start_month pool_ann_stop_year=$pool_stop_year pool_ann_stop_month=$pool_stop_month # If either reset_start_year or reset_stop_year are set then they must be # of the form old_year:new_year (ie a colon separated pair of integers) # where the first integer is the year that needs to be changed # and the second integer is the year that it will be changed to. # These may potentially change the value of start_year or stop_year that # are defined after the call to make_file_name_list below psdelete_reset_start_year='' reset_start_year=${psdelete_reset_start_year:=''} psdelete_reset_end_year='' psdelete_reset_stop_year=$psdelete_reset_end_year reset_stop_year=${psdelete_reset_stop_year:=''} # This invocation of make_file_name_list will process the *_year and *_months # variables defined above and output a file containing definitions for # start_year, start_mon, stop_year, stop_mon tmp_file_list="${JHOME:-$HOME}/tmp/psdelete_date_list_${runid}_${stamp}" make_file_name_list --dates_only $tmp_file_list >>$error_out 2>&1 ||\ bail "Problem in make_file_name_list" rm -f $error_out # Verify that the output list is not empty [ ! -s "$tmp_file_list" ] && bail "Unable to create file list" # A file list was created ...source it # This will define start_year, start_mon, stop_year, stop_mon : ; . $tmp_file_list rm -f $tmp_file_list # Define start and stop dates psdelete_start_year=$start_year psdelete_start_mon=$start_mon psdelete_stop_year=$stop_year psdelete_stop_mon=$stop_mon # ym_range is used in the definition of file names to be deleted # define a range string in the form YYYYmMM_YYYYmMM to be used in file names ym_range="${psdelete_start_year}m${psdelete_start_mon}" ym_range="${ym_range}_${psdelete_stop_year}m${psdelete_stop_mon}" # In the special case that start and stop dates are identical use a YYYY format if [ $psdelete_start_year -eq $psdelete_stop_year ]; then if [ $psdelete_start_mon -eq $psdelete_stop_mon ]; then ym_range=$psdelete_start_year fi fi # psdelete_suffix_list is a white space separated list of suffixes # of file names to be generated here. Any suffix in this list may be # modified by appending a + followed by a comma separated list of # numbers (no white space is allowed within this modifier). Each # number within the modifier list will correspond to a month (1-12) # for which a file with this suffix is to be included. If the # modifier exists for a particular suffix then only those months # indicated in the modifier will be added to the file list. psdelete_suffix_list='gp xp' # psdelete_suffix_ flags the addition of an underscore between the # suffix and the month (e.g. djf_gs) in file names generated here psdelete_suffix_=1 XXX=`echo $psdelete_suffix_|sed 's/ //g'` eval psdelete_suffix_\=$XXX [ "$psdelete_suffix_" = 'on' ] && eval psdelete_suffix_\=1 [ "$psdelete_suffix_" = 'off' ] && eval psdelete_suffix_\=0 [ "$psdelete_suffix_" = 'yes' ] && eval psdelete_suffix_\=1 [ "$psdelete_suffix_" = 'no' ] && eval psdelete_suffix_\=0 # psdelete_djf, psdelete_mam, psdelete_jja and psdelete_son flag the inclusion of # seasonal average files determined as file names of the form # ${psdelete_prefix_}${year}_${season}_$suffix # for season = djf, mam, jja or son psdelete_djf=1 XXX=`echo $psdelete_djf|sed 's/ //g'` eval psdelete_djf\=$XXX [ "$psdelete_djf" = 'on' ] && eval psdelete_djf\=1 [ "$psdelete_djf" = 'off' ] && eval psdelete_djf\=0 [ "$psdelete_djf" = 'yes' ] && eval psdelete_djf\=1 [ "$psdelete_djf" = 'no' ] && eval psdelete_djf\=0 psdelete_mam=1 XXX=`echo $psdelete_mam|sed 's/ //g'` eval psdelete_mam\=$XXX [ "$psdelete_mam" = 'on' ] && eval psdelete_mam\=1 [ "$psdelete_mam" = 'off' ] && eval psdelete_mam\=0 [ "$psdelete_mam" = 'yes' ] && eval psdelete_mam\=1 [ "$psdelete_mam" = 'no' ] && eval psdelete_mam\=0 psdelete_jja=1 XXX=`echo $psdelete_jja|sed 's/ //g'` eval psdelete_jja\=$XXX [ "$psdelete_jja" = 'on' ] && eval psdelete_jja\=1 [ "$psdelete_jja" = 'off' ] && eval psdelete_jja\=0 [ "$psdelete_jja" = 'yes' ] && eval psdelete_jja\=1 [ "$psdelete_jja" = 'no' ] && eval psdelete_jja\=0 psdelete_son=1 XXX=`echo $psdelete_son|sed 's/ //g'` eval psdelete_son\=$XXX [ "$psdelete_son" = 'on' ] && eval psdelete_son\=1 [ "$psdelete_son" = 'off' ] && eval psdelete_son\=0 [ "$psdelete_son" = 'yes' ] && eval psdelete_son\=1 [ "$psdelete_son" = 'no' ] && eval psdelete_son\=0 # psdelete_ann flags the inclusion of annual average files # file names with ann replacing djf, mam, jja or son. psdelete_ann=1 XXX=`echo $psdelete_ann|sed 's/ //g'` eval psdelete_ann\=$XXX [ "$psdelete_ann" = 'on' ] && eval psdelete_ann\=1 [ "$psdelete_ann" = 'off' ] && eval psdelete_ann\=0 [ "$psdelete_ann" = 'yes' ] && eval psdelete_ann\=1 [ "$psdelete_ann" = 'no' ] && eval psdelete_ann\=0 # psdelete_mon flags the inclusion of monthly average files # file names with m01,m02,... replacing djf, mam, jja or son. psdelete_mon=1 XXX=`echo $psdelete_mon|sed 's/ //g'` eval psdelete_mon\=$XXX [ "$psdelete_mon" = 'on' ] && eval psdelete_mon\=1 [ "$psdelete_mon" = 'off' ] && eval psdelete_mon\=0 [ "$psdelete_mon" = 'yes' ] && eval psdelete_mon\=1 [ "$psdelete_mon" = 'no' ] && eval psdelete_mon\=0 # Create file names of pooled seasonal files to be deleted join=0 for season in djf mam jja son ann mon; do if [ "$season" = "djf" -a $psdelete_djf -ne 1 ]; then continue fi if [ "$season" = "mam" -a $psdelete_mam -ne 1 ]; then continue fi if [ "$season" = "jja" -a $psdelete_jja -ne 1 ]; then continue fi if [ "$season" = "son" -a $psdelete_son -ne 1 ]; then continue fi if [ "$season" = "ann" -a $psdelete_ann -ne 1 ]; then continue fi if [ "$season" = "mon" -a $psdelete_mon -ne 1 ]; then continue fi if [ "$season" = "mon" ]; then bname=${psdelete_prefix_}${ym_range}_ elif [ "$season" = "ann" ]; then # Modify the file name for multi year annually average files when # the first and/or last year is incomplete, and therefore missing curr_start_year=$psdelete_start_year curr_stop_year=$psdelete_stop_year if [ $psdelete_start_year -eq $pool_ann_start_year ]; then if [ $pool_start_month -ne 1 ]; then # If this is annual pooling and pool_start_month is not JAN then increment # psdelete_start_year because the first year will not have been pooled curr_start_year=`echo $psdelete_start_year|awk '{printf "%3.3d",$1+1}' -` fi fi if [ $psdelete_stop_year -eq $pool_ann_stop_year ]; then if [ $pool_stop_month -ne 12 ]; then # If this is annual pooling and pool_stop_month is not DEC then decrement # psdelete_stop_year because the last year will not have been pooled curr_stop_year=`echo $psdelete_stop_year|awk '{printf "%3.3d",$1-1}' -` fi fi # Set bname using curr_start_year and curr_stop_year as defined above # For annual pooling files names, the first month is always 1 and the # last month is always 12. This will need to change if annual pooling # is done on an interval other than the calendar year if [ "$psdelete_suffix_" = '1' ]; then bname=${psdelete_prefix_}${curr_start_year}m01_${curr_stop_year}m12_${season}_ else bname=${psdelete_prefix_}${curr_start_year}m01_${curr_stop_year}m12_$season fi else # Multi year seasonal pooled files (djf, mam, jja, son) pool_range=$ym_range if [ "$psdelete_suffix_" = '1' ]; then bname=${psdelete_prefix_}${pool_range}_${season}_ else bname=${psdelete_prefix_}${pool_range}_$season fi fi for suffix in $psdelete_suffix_list; do mlist=`echo $suffix|awk -F'+' '{print $2}' -` if [ -n "$mlist" ]; then # ignore any appended month list, it is not used suffix=`echo $suffix|sed 's/+.*$//'` fi if [ "$season" = "mon" ]; then mm=0 while [ $mm -lt 12 ]; do mm=`echo $mm|awk '{printf "%2.2d",$1+1}' -` join=`echo $join|awk '{j=1+$1;printf "%d",j}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` if [ "$psdelete_suffix_" = '1' ]; then eval file${join}=${bname}m${mm}_$suffix else eval file${join}=${bname}m${mm}$suffix fi done else join=`echo $join|awk '{j=1+$1;printf "%d",j}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` eval file${join}=$bname$suffix fi done done psdelete_leave_last_pool=1 XXX=`echo $psdelete_leave_last_pool|sed 's/ //g'` eval psdelete_leave_last_pool\=$XXX [ "$psdelete_leave_last_pool" = 'on' ] && eval psdelete_leave_last_pool\=1 [ "$psdelete_leave_last_pool" = 'off' ] && eval psdelete_leave_last_pool\=0 [ "$psdelete_leave_last_pool" = 'yes' ] && eval psdelete_leave_last_pool\=1 [ "$psdelete_leave_last_pool" = 'no' ] && eval psdelete_leave_last_pool\=0 if [ $psdelete_leave_last_pool -eq 1 ]; then year_bound=`echo $psdelete_start_year $psdelete_stop_year $pool_stop_year |\ awk '{b=$3-$2+$1;printf "%d",b}' -` if [ $psdelete_stop_year -ge $year_bound ]; then # Do not delete any files when psdelete_stop_year lies within # the pooling range of the end of the run # This is accomplished by supplying a single file name # that is unlikely to exist on DATAPATH/RUNPATH join=1 file1='p_____w____x___93475__y_z' fi fi [ $join -eq 0 ] && bail "No files selected for the year/mon range $psdelete_start_year/$psdelete_start_mon to $psdelete_stop_year/$psdelete_stop_mon" # ---Stop_submit_ignore_code---- # * ............................ Condef Parameters ............................ noprint=on nextjob=on # * ............................. Deck Definition ............................. . delet.dk #end_of_job