#!/bin/sh #======================================================================= # Delete model history files --- mdelete --- # $Id: mdelete_jobdef 656 2011-12-28 19:51:43Z acrnrls $ #======================================================================= # # Files deleted will be of the form # # ${prefix}_${runid}_${year}_m${mon}_${suffix} # # for all months from previous_year, previous_month to current_year, # current_month or from current_year, current_month to next_year, # next_month depending on which of previous_(year|month) or # next_(year|month) are set. # # Both prefix and suffix may have multiple values, in which case file # name generation iterates over these multiple values. Values for prefix # and suffix are defined by user supplied lists which are read from the # parmsub variables mdelete_prefix_list and mdelete_suffix_list. # See below for details of how these lists are composed. # # The last month will normally be kept on disk and the month prior to # the first month will be deleted (if it exists). This behaviour may be # modified by setting mdelete_leave_last_mon=0 on the cccjob # command line. If mdelete_leave_last_mon=0 then files will be deleted # for months from previous_year, previous_month to current_year, # current_month etc. # # Larry Solheim Apr 2006 #======================================================================= # # keyword :: mdelete # description :: delete files from DATAPATH # set -a . betapath2 # * ........................... Parmsub Parameters .................... # These variables are set when the job string is created previous_year=NotSet previous_month=NotSet current_year=NotSet current_month=NotSet next_year=NotSet next_month=NotSet run_start_year=NotSet run_start_month=NotSet run_stop_year=NotSet run_stop_month=NotSet runid="job000"; uxxx='uxxx'; model_uxxx=$uxxx; mdelete_uxxx=$model_uxxx; crawork="${runid}_job"; username="acrnxxx"; user="XXX"; year=yyy; mon=mm nqsprfx="${runid}_"; nqsext=''; # Allow the user to reset CCRNTMP and/or RUNPATH mdelete_RUNPATH='' RUNPATH=${mdelete_RUNPATH:=$RUNPATH} mdelete_CCRNTMP='' CCRNTMP=${mdelete_CCRNTMP:=$CCRNTMP} # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi jobname=mdelete lopgm="lopgm"; stime="1800"; memory1="1gb"; mdelete_bexfer='' bexfer="${mdelete_bexfer:off}" # ---Start_submit_ignore_code---- stamp=`date "+%j%H%M%S"$$` # Use -e option if recognized by echo if [ "X`echo -e`" = "X-e" ]; then echo_e() { echo ${1+"$@"}; } else echo_e() { echo -e ${1+"$@"}; } fi # bail is a simple error exit routine # Note: we write the error directly to a file in ~/.queue so that this # info is not lost if/when stdout is not returned error_out="${JHOME:-$HOME}/.queue/error_mdelete_${runid}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out bail(){ echo_e `date`" --- mdelete: $*" echo_e `date`" --- mdelete: $*" >>$error_out exit 1 } # The variables mdelete_prefix_list and mdelete_suffix_list are strings containing # embedded colons and whitespace which are interpreted as list delimiters. # These strings may be thought of as 2 dimensional arrays, the rows of these # arrays are colon (:) separated strings and each row is divided into columns # by separating on white space. # These variables are used by make_file_name_list along with runid and year/mon # information from the *_year and *_month variables to generate file names. # In make_file_name_list the prefix_list and suffix_list strings are first # separated into colon delimited lists (rows). There must be an equal number of # rows in each of prefix_list and suffix_list because these rows will be # used in pairs. # Each pair of rows (one row from prefix_list and one row from suffix_list) # is separated into a white space separated list. Each element of these # white space separated lists is a single prefix or suffix (possibly modified # by appending a "+" followed by a comma separated list of integers in the # range 1-12). No white space is allowed within a single prefix or suffix. # These individual (pre|suf)fixes are then iterated over for each year and # month and for each pair of rows in prefix_list and suffix_list to form the # desired set of file names, each of which is of the form # # ${prefix}_${runid}_${year}_m${mon}_${suffix} # Any prefix or suffix in these lists may be modified by appending a + # followed by a comma separated list of numbers (no white space is allowed # within this modifier). Each number within the modifier list will correspond # to a month (1-12) for which a file with this suffix is to be included. # If the modifier exists for a particular suffix then only those months # indicated in the modifier will be added to the file list. # If the above form of file name is inappropriate then the user may # provide a template or templates to produce arbitrary file names. # These templates are defined in the variable mdelete_prefix_list. # Any individual prefix will be treated as a file name template if it # begins with a "%" character. The template will consist of everthing # after the "%" character and up to the next colon or white space. # It can be composed of anything but must ultimately (after variable # substitution, etc) result in a valid file name. When a template # is encountered, it is used as the entire file name (ie the "normal" file # name form is disregarded as is the corresponding suffix(s)). However, it # is subject to the same interation procedure as a normal prefix and # does undergo variable substitution. Variables that are defined for # substitution include year, mon, runid, uxxx, start_year, start_mon, # stop_year, stop_mon, all of the *_year and *_month variables defined # above as well as any user supplied variable definitions passed to # make_file_name_list as a command line option of the form var=val. mdelete_suffix_list='' suffix_list="${mdelete_suffix_list:-gs ss rs+12 ab+12 an+12 _script}" mdelete_prefix_list='' prefix_list="${mdelete_prefix_list:-$mdelete_uxxx}" # Create a file containing a list of file names that may then be # "sourced" in the current environment to define the variables # file1, file2,..., file$join, join. These variables are used by # tdumper to compile the list of files to be archived. join=0 # make_file_name_list uses the variables current_year, current_month, # previous_year, previous_month, next_year and next_month to # determine start and stop dates for file name creation. # It also uses runid, prefix_list and suffix_list from the current # environment to build these file names. # Allow user supplied command line options for make_file_name_list # The following invocation of make_file_name_list will not allow multi-list # output so if any command line option is supplied that will turn on # multi-list output (e.g. --months_max=... --size_max=... --number_max=..) # then this script will abort. mdelete_file_list_opts='' fopts="${mdelete_file_list_opts:-}" # By default mdelete will leave the last month on disk # Setting mdelete_leave_last_mon = off will override this default # This is a typical requirement of a model run which needs the restart # from the previous month to continue after resubmission mdelete_leave_last_mon=on XXX=`echo $mdelete_leave_last_mon|sed 's/ //g'` eval mdelete_leave_last_mon\=$XXX [ "$mdelete_leave_last_mon" = 'on' ] && eval mdelete_leave_last_mon\=1 [ "$mdelete_leave_last_mon" = 'off' ] && eval mdelete_leave_last_mon\=0 [ "$mdelete_leave_last_mon" = 'yes' ] && eval mdelete_leave_last_mon\=1 [ "$mdelete_leave_last_mon" = 'no' ] && eval mdelete_leave_last_mon\=0 if [ $mdelete_leave_last_mon -eq 1 ]; then # Leave the last month on disk by shifting these dates back 1 month mdelete_mon_offset='-1' else # Otherwise delete files within the input start/stop dates mdelete_mon_offset='' fi if [ -n "$mdelete_mon_offset" ]; then # Set a user supplied month offset eval fopts=\"$fopts --mon_offset\=$mdelete_mon_offset\" fi # Create a temporary file containing the file list # tmp_file_list="${JHOME:-$HOME}/.queue/mdelete_file_list_${runid}_${stamp}" tmp_file_list="mdelete_file_list_${runid}_${stamp}" make_file_name_list $fopts --nomulti_list $tmp_file_list >>$error_out 2>&1 ||\ bail "Problem in make_file_name_list" rm -f $error_out [ ! -s "$tmp_file_list" ] && bail "Unable to create file list" # A file list was created ...source it # This will define start_year,start_mon,stop_year,stop_mon and file1,file2,... : ; . $tmp_file_list # At this point file1, file2,... are defined in the current environment # as well as certain other variables such as start_year, start_mon, # stop_year and stop_mon which correspond to the start and stop dates # for the file names that were created. ym_range="${start_year}m${start_mon}_${stop_year}m${stop_mon}" # Delete or keep the temporary file that contains the file list mdelete_keep_file_list=0 XXX=`echo $mdelete_keep_file_list|sed 's/ //g'` eval mdelete_keep_file_list\=$XXX [ "$mdelete_keep_file_list" = 'on' ] && eval mdelete_keep_file_list\=1 [ "$mdelete_keep_file_list" = 'off' ] && eval mdelete_keep_file_list\=0 [ "$mdelete_keep_file_list" = 'yes' ] && eval mdelete_keep_file_list\=1 [ "$mdelete_keep_file_list" = 'no' ] && eval mdelete_keep_file_list\=0 if [ $mdelete_keep_file_list -eq 1 ]; then # Rename the file list to include the year/month range # and place it in the users ~/.queue dir saved_file_list="${JHOME:-$HOME}/.queue/mdelete_file_list_${runid}_${ym_range}_${stamp}" mv -f $tmp_file_list $saved_file_list else # Delete the file that contains the file list rm -f $tmp_file_list fi # ---Stop_submit_ignore_code---- # * ............................ Condef Parameters ............................ noprint=on # do not return stdout to ~/.queue nextjob=on # more jobs in the job string will follow # * ............................. Deck Definition ............................. . delet.dk #end_of_job