#!/bin/sh #======================================================================= # multi-year pool of diagnostic and/or pooled files --- pool_mul --- # $Id: pool_mul_jobdef 656 2011-12-28 19:51:43Z acrnrls $ #======================================================================= # # pool up to 50 diagnostic and/or pooled files # When pooling pooled and diagnostic files together # the first input file must be a pooled file # # Files pooled will be of the form # ${pool_mul_model_prefix}_${year}_${pool_mul_id}_[gx]p # The pooled files that are created will be named # ${pool_mul_flabel_prefix}_${y1}_${y2}_${pool_mul_id}_[gx]p #======================================================================= # keyword :: pool_mul # description :: multi-year pooling of diagnostic and/or pooled files #****************************** #********* WARNING ************ #****************************** # # auto, nonqs, nextjob, flabel, RCMRUN, runid, run, nqsext, crawork, # mdest, [sgd]time, nnodex, nprocx, initmem, sv, vic, memory[0-9], # queue, jobname, gcmjcl # # These variables are special in that they are extracted from the # submission script prior to execution and used to parse file names, # among other things. Therefore any of these variables whose definition # depends on another variable that is not in this list will be improperly # defined in this preprocessing step. This could lead to file names being # incorrectly specified. The variables are extracted by copying the entire # line on which they are defined from the submission script. Therefore # variables that are used to define any of these variables must be defined # on the same line as that variable and prior to it or on a previous line # that contains any of these variables. # #****************************** set -a . betapath2 # * ........................... Parmsub Parameters ............................ runid="job000"; jobname=pool_mul crawork="${runid}_job" nqsprfx="${runid}_"; nqsext=''; username="acrnxxx"; user="XXX"; stime="1800"; memory1="800mb"; lopgm="lopgm"; # Allow the user to reset CCRNTMP and/or RUNPATH pool_RUNPATH='' RUNPATH=${pool_RUNPATH:=$RUNPATH} pool_CCRNTMP='' CCRNTMP=${pool_CCRNTMP:=$CCRNTMP} # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi # ---Start_submit_ignore_code---- stamp=`date "+%j%H%M%S"$$` # Use -e option if recognized by echo if [ "X`echo -e`" = "X-e" ]; then echo_e() { echo ${1+"$@"}; } else echo_e() { echo -e ${1+"$@"}; } fi # bail is a simple error exit routine # Note: we write the error directly to a file in ~/.queue so that this # info is not lost if/when stdout is not returned error_out="${JHOME:-$HOME}/.queue/error_pool_mul_${runid}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out bail(){ echo_e `date`" --- pool_mul: $*" echo_e `date`" --- pool_mul: $*" >>$error_out exit 1 } # These variables are set when the job string is created previous_year=NotSet previous_month=NotSet current_year=NotSet current_month=NotSet next_year=NotSet next_month=NotSet run_start_year=NotSet run_start_month=NotSet run_stop_year=NotSet run_stop_month=NotSet # Pooling start and stop dates # These may differ from run start and stop dates, but are the same by default pool_start_year=$run_start_year pool_start_month=$run_start_month pool_stop_year=$run_stop_year pool_stop_month=$run_stop_month # If either reset_start_year or reset_stop_year are set then they must be # of the form old_year:new_year (ie a colon separated pair of integers) # where the first integer is the year that needs to be changed # and the second integer is the year that it will be changed to. # These may potentially change the value of start_year or stop_year that # are defined after the call to make_file_name_list below pool_mul_reset_start_year='' reset_start_year=${pool_mul_reset_start_year:=''} pool_mul_reset_end_year='' pool_mul_reset_stop_year=$pool_mul_reset_end_year reset_stop_year=${pool_mul_reset_stop_year:=''} # This invocation of make_file_name_list will process the *_year and *_months # variables defined above and output a file containing definitions for # start_year, start_mon, stop_year, stop_mon tmp_file_list="${JHOME:-$HOME}/tmp/pool_mul_date_list_${runid}_${stamp}" make_file_name_list --dates_only $tmp_file_list >>$error_out 2>&1 ||\ bail "Problem in make_file_name_list" rm -f $error_out # Verify that the output list is not empty [ ! -s "$tmp_file_list" ] && bail "Unable to create file list" # A file list was created ...source it # This will define start_year, start_mon, stop_year, stop_mon : ; . $tmp_file_list rm -f $tmp_file_list # Define start and stop dates pool_mul_start_year=$start_year pool_mul_start_mon=$start_mon pool_mul_stop_year=$stop_year pool_mul_stop_mon=$stop_mon uxxx='uxxx'; pool_uxxx=$uxxx; pool_mul_uxxx=$pool_uxxx # memory99=1 pool_mul_prefix="${pool_mul_uxxx}_${runid}" # memory99=1 pool_mul_prefix_=${pool_mul_prefix}_ # memory99=1 pool_mul_flabel_uxxx=$pool_uxxx # memory99=1 diag_uxxx=$uxxx; pool_mul_model_uxxx=$diag_uxxx # memory99=1 # pool_mul_id is the month or season to be pooled # pool_mul_id is used to form part of the file name of monthly diag # or pooled diag files that are to be pooled here. # pool_mul_id should be one of ann,djf,mam,jja,son,m01..m12,jan..dec pool_mul_id=jja #xxx # Set flabel outside the "submit ignore" delimiters so that the submit #xxx # scripts can use it for the secondary crawork file name #xxx # flabel is reset below at run time to be used in the pooling deck #xxx flabel="pool_mul_${runid}_${pool_mul_id}_$stamp" # force_first_djf flags the inclusion of the first DJF of pool_start_year force_first_djf=off pool_mul_force_first_djf=${force_first_djf:-off} XXX=`echo $force_first_djf|sed 's/ //g'` eval force_first_djf\=$XXX [ "$force_first_djf" = 'on' ] && eval force_first_djf\=1 [ "$force_first_djf" = 'off' ] && eval force_first_djf\=0 [ "$force_first_djf" = 'yes' ] && eval force_first_djf\=1 [ "$force_first_djf" = 'no' ] && eval force_first_djf\=0 # force_first_sea flags pooling the first season in the string # even if pool_start_(year|month) falls after the first month of that season force_first_sea=off pool_mul_force_first_sea=${force_first_sea:-off} XXX=`echo $force_first_sea|sed 's/ //g'` eval force_first_sea\=$XXX [ "$force_first_sea" = 'on' ] && eval force_first_sea\=1 [ "$force_first_sea" = 'off' ] && eval force_first_sea\=0 [ "$force_first_sea" = 'yes' ] && eval force_first_sea\=1 [ "$force_first_sea" = 'no' ] && eval force_first_sea\=0 # Ensure that force_first_djf is true when force_first_sea is true [ $force_first_sea -eq 1 ] && eval force_first_djf\=1 # pool_mul_force_diag_uxxx = 1 means always use diag_uxxx as the model prefix pool_mul_force_diag_uxxx=0 [ "$pool_mul_force_diag_uxxx" = 'on' ] && eval pool_mul_force_diag_uxxx\=1 [ "$pool_mul_force_diag_uxxx" = 'off' ] && eval pool_mul_force_diag_uxxx\=0 [ "$pool_mul_force_diag_uxxx" = 'yes' ] && eval pool_mul_force_diag_uxxx\=1 [ "$pool_mul_force_diag_uxxx" = 'no' ] && eval pool_mul_force_diag_uxxx\=0 if [ "x$pool_mul_id" = "xann" -o "x$pool_mul_id" = "xdjf" -o "x$pool_mul_id" = "xmam" -o "x$pool_mul_id" = "xjja" -o "x$pool_mul_id" = "xson" ]; then # Seasons are already pooled so use pool_uxxx as the model prefix pool_mul_model_uxxx=$pool_uxxx # memory99=1 if [ $pool_mul_force_diag_uxxx -eq 1 ]; then # If requested, use diag_uxxx as the model prefix for seasonal # and annual pooling, even though they are pooled files pool_mul_model_uxxx=$diag_uxxx # memory99=1 fi fi # These variables are used to define flabel and model? below pool_mul_flabel_prefix=${pool_mul_flabel_uxxx}_${runid} # memory99=1 pool_mul_model_prefix=${pool_mul_model_uxxx}_${runid} # memory99=1 if [ "x$pool_mul_id" = "xann" ]; then if [ $pool_mul_start_year -eq $pool_start_year ]; then if [ $pool_start_month -ne 1 ]; then # If this is annual pooling and pool_start_month is not JAN then increment # pool_mul_start_year because the first year will not have been pooled pool_mul_start_year=`echo $pool_mul_start_year|awk '{printf "%3.3d",$1+1}' -` fi fi if [ $pool_mul_stop_year -eq $pool_stop_year ]; then if [ $pool_stop_month -ne 12 ]; then # If this is annual pooling and pool_stop_month is not DEC then decrement # pool_mul_stop_year because the last year will not have been pooled pool_mul_stop_year=`echo $pool_mul_stop_year|awk '{printf "%3.3d",$1-1}' -` fi fi # Unconditioanlly set start_mon=1 and stop_mon=12 # This may need to be revised if annual pooling is done on anything other # than a calendar year pool_mul_start_mon='01' pool_mul_stop_mon='12' fi # pool_suffix_list is used by poolseas4 pool_mul_suffix_list='gp xp' pool_suffix_list=$pool_mul_suffix_list # pool_mul_pool will be an integer indicating the month if pool_mul_id is # of the form m01,m02,m03,... or jan,feb,mar,... otherwise it will be null case $pool_mul_id in jan|m01) pool_mul_pool=1;; feb|m02) pool_mul_pool=2;; mar|m03) pool_mul_pool=3;; apr|m04) pool_mul_pool=4;; may|m05) pool_mul_pool=5;; jun|m06) pool_mul_pool=6;; jul|m07) pool_mul_pool=7;; aug|m08) pool_mul_pool=8;; sep|m09) pool_mul_pool=9;; oct|m10) pool_mul_pool=10;; nov|m11) pool_mul_pool=11;; dec|m12) pool_mul_pool=12;; *) pool_mul_pool='' ;; esac last_year_in_range=1 first_year_in_range=1 pool_mul_curr_year=`echo $pool_mul_start_year|awk '{y=$1-1;printf "%3.3d", y}' -` join=0 while [ $pool_mul_curr_year -lt $pool_mul_stop_year ]; do pool_mul_curr_year=`echo $pool_mul_curr_year|awk '{y=1+$1;printf "%3.3d", y}' -` pool_mul_curr_mon="12" if [ $pool_mul_curr_year -eq $pool_start_year ]; then if [ $force_first_djf -eq 1 -o $force_first_sea -eq 1 ]; then local_force_first_djf=1 else local_force_first_djf=0 fi if [ "x$pool_mul_id" = "xdjf" -a $local_force_first_djf -eq 0 ]; then # assume there is no djf available for the first year continue fi fi if [ $pool_mul_curr_year -eq $pool_mul_start_year ]; then pool_mul_curr_mon=$pool_mul_start_mon if [ -n "$pool_mul_pool" ]; then # this is a multi year pool of month $pool_mul_pool if [ $pool_mul_pool -lt $pool_mul_curr_mon ]; then # there will be no files available for months less than $pool_mul_curr_mon first_year_in_range=0 continue fi fi if [ "x$pool_mul_id" = "xdjf" ]; then if [ $pool_mul_curr_mon -gt 1 ]; then # Pooling was started on a month other than Jan # Assume no djf available first_year_in_range=0 continue fi fi if [ "x$pool_mul_id" = "xmam" ]; then if [ $pool_mul_curr_mon -gt 3 ]; then # Assume no mam available unless force_first_sea is set if [ $force_first_sea -eq 0 ]; then first_year_in_range=0 continue fi fi fi if [ "x$pool_mul_id" = "xjja" ]; then if [ $pool_mul_curr_mon -gt 6 ]; then # Assume no jja available unless force_first_sea is set if [ $force_first_sea -eq 0 ]; then first_year_in_range=0 continue fi fi fi if [ "x$pool_mul_id" = "xson" ]; then if [ $pool_mul_curr_mon -gt 9 ]; then # Assume no son available unless force_first_sea is set if [ $force_first_sea -eq 0 ]; then first_year_in_range=0 continue fi fi fi fi if [ $pool_mul_curr_year -eq $pool_mul_stop_year ]; then pool_mul_curr_mon=$pool_mul_stop_mon if [ -n "$pool_mul_pool" ]; then # this is a multi year pool of month $pool_mul_pool if [ $pool_mul_pool -gt $pool_mul_curr_mon ]; then # there are no files available for months greater than $pool_mul_curr_mon last_year_in_range=0 continue fi fi if [ "x$pool_mul_id" = "xdjf" ]; then if [ $pool_mul_curr_mon -lt 2 ]; then # no djf available last_year_in_range=0 continue fi fi if [ "x$pool_mul_id" = "xmam" ]; then if [ $pool_mul_curr_mon -lt 5 ]; then # no mam available last_year_in_range=0 continue fi fi if [ "x$pool_mul_id" = "xjja" ]; then if [ $pool_mul_curr_mon -lt 8 ]; then # no jja available last_year_in_range=0 continue fi fi if [ "x$pool_mul_id" = "xson" ]; then if [ $pool_mul_curr_mon -lt 11 ]; then # no son available last_year_in_range=0 continue fi fi fi join=`echo $join|awk '{j=1+$1;printf "%d",j}' -` join=`echo $join|sed -e 's/^ *//' -e 's/^0*//'` # Generate a list of diagnostic file prefixes in the model1,model2,... variables eval model${join}=${pool_mul_model_prefix}_${pool_mul_curr_year}_${pool_mul_id}_ # Parameters ny01, ny02, etc, specify the number of years pooled in input files. # If ny01, ... are not specified, or zero, the files are assumed to be # diagnostic files (ie not yet pooled). # Both ny01 and ny1 are accepted. eval ny${join}=0 done # define flabel if [ $last_year_in_range -eq 0 -a $first_year_in_range -eq 0 ]; then y1=`echo ${pool_mul_start_year}|awk '{printf "%3.3d",$1+1}' -` # memory99=1 y2=`echo ${pool_mul_stop_year}|awk '{printf "%3.3d",$1-1}' -` # memory99=1 elif [ $first_year_in_range -eq 0 ]; then y1=`echo ${pool_mul_start_year}|awk '{printf "%3.3d",$1+1}' -` # memory99=1 y2=`echo ${pool_mul_stop_year}|awk '{printf "%3.3d",$1}' -` # memory99=1 elif [ $last_year_in_range -eq 0 ]; then y1=`echo ${pool_mul_start_year}|awk '{printf "%3.3d",$1}' -` # memory99=1 y2=`echo ${pool_mul_stop_year}|awk '{printf "%3.3d",$1-1}' -` # memory99=1 else y1=`echo ${pool_mul_start_year}|awk '{printf "%3.3d",$1}' -` # memory99=1 y2=`echo ${pool_mul_stop_year}|awk '{printf "%3.3d",$1}' -` # memory99=1 fi range=${pool_mul_start_year}m${pool_mul_start_mon} # memory99=1 range=${range}_${pool_mul_stop_year}m${pool_mul_stop_mon} # memory99=1 flabel=${pool_mul_flabel_prefix}_${range}_${pool_mul_id}_ run=`echo ${pool_mul_flabel_prefix}|tr '[a-z]' '[A-Z]'`_${range} # ---Stop_submit_ignore_code---- # * ............................ Condef Parameters ............................ nextjob=on noprint=on # pool_var=on means pool variances as well as means pool_mul_pool_var=off pool_var=$pool_mul_pool_var # poolabort=off means do not abort when variables are missing poolabort=off # * ............................. Deck Definition ............................. . pool4.dk #end_of_job