#!/bin/sh #========================================================================= # Extract netcdf time series from a list of input files --- nccrip --- # $Id$ #========================================================================= # # Files processed will be of the form # # ${prefix}_${runid}_${year}_m${mon}_${suffix} # # for all months from previous_year, previous_month to current_year, # current_month or from current_year, current_month to next_year, # next_month depending on which of previous_(year|month) or # next_(year|month) are set. # # Both prefix and suffix may have multiple values, in which case file # name generation iterates over these multiple values. Values for prefix # and suffix may be defined by the user in the parmsub variables # nccrip_prefix_list and nccrip_suffix_list. # # Alternatively, the user may supply a list of file names when this job # string is created. These definitions may be defined in terms of certain # variables (e.g. start_year, start_mon,...) that are defined at run time. # # Larry Solheim ...Sept 2012 #========================================================================= # # keyword :: nccrip # description :: Extract netcdf time series from a list of input files # set -a . betapath2 # * ........................... Parmsub Parameters ...................... runid="job000"; nqsprfx="${runid}_"; nqsext='' crawork="${runid}_job"; username="acrnxxx"; user="XXX"; uxxx='uxxx'; nccrip_uxxx=$uxxx # memory99=1 jobname=nccrip; stime=7200; gptime=$stime; memory1="4000mb" noprint=on nextjob=on debug=off # Temporary directory where this script will run nccrip_CCRNTMP='' CCRNTMP=${nccrip_CCRNTMP:=$CCRNTMP} # RUNPATH on execution machine nccrip_RUNPATH='' RUNPATH=${nccrip_RUNPATH:=$RUNPATH} # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi #=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=# #=#=# Begin execute script #=#=# #=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=# . comjcl.cdk cat > Execute_Script <<'end_of_script' # Auto export is required for make_file_name_list set -a # ---Start_submit_ignore_code---- # These variables are set when the job string is created previous_year=NotSet previous_month=NotSet current_year=NotSet current_month=NotSet next_year=NotSet next_month=NotSet run_start_year=NotSet run_start_month=NotSet run_stop_year=NotSet run_stop_month=NotSet # Setting CCCJOB_ROOT will allow a job specific version # of make_file_name_list to be used CCCJOB_ROOT='' if [ -z "$CCCJOB_ROOT" ]; then MAKE_FILE_NAME_LIST=make_file_name_list else MAKE_FILE_NAME_LIST="$CCCJOB_ROOT/bin/make_file_name_list" fi # Define a date string that will be used in file names etc stamp=`date "+%j%H%M%S"$$` # this_host will simply be the output from uname this_host=`uname -n|awk -F\. '{print \$1}' -` # this_mach will be a known alias (or possibly the actual machine name) this_mach=$this_host case $this_mach in c1*) this_mach=spica ;; c2*) this_mach=hadar ;; alef*) this_mach=alef ;; ib3*) this_mach=pollux ;; joule*) this_mach=joule ;; esac # Emulate echo -n option if not recognized by echo (ie echo without return) if [ "X`echo -n`" = "X-n" ]; then echo_n() { echo ${1+"$@"}'\c'; } else echo_n() { echo -n ${1+"$@"}; } fi # Define a file name that may be used to emit error messages error_out="${JHOME:-$HOME}/.queue/error_nccrip_${runid}_${this_mach}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out # bail is a simple error exit routine bail(){ # Send error message to stdout echo `date`" $this_host $runid --- nccrip: $*" # Also copy to a file in case stdout goes missing in action echo `date`" $this_host $runid --- nccrip: $*" >>$error_out exit 1 } # If either reset_start_year or reset_stop_year are set then they must be # of the form old_year:new_year (ie a colon separated pair of integers) # where the first integer is the year that needs to be changed # and the second integer is the year that it will be changed to. # These may potentially change the value of start_year or stop_year that # are defined after the call to make_file_name_list below nccrip_reset_start_year='' reset_start_year=${nccrip_reset_start_year:=''} nccrip_reset_stop_year='' reset_stop_year=${nccrip_reset_stop_year:=''} # reset_end_year is defined here for backward compatibility # It is effectively equivalent to reset_stop_year. nccrip_reset_end_year='' reset_end_year=${nccrip_reset_end_year:=''} # This invocation of make_file_name_list will process the *_year and *_month # variables defined above and output a file containing definitions for # start_year, start_mon, stop_year, stop_mon fopts='' nccrip_mon_offset='' if [ -n "$nccrip_mon_offset" ]; then # Set a user supplied month offset eval fopts=\"--mon_offset\=$nccrip_mon_offset\" fi tmp_file_list="tmp_file_list_${runid}_${stamp}" $MAKE_FILE_NAME_LIST $fopts --dates_only $tmp_file_list >>$error_out 2>&1 || \ bail "Problem in make_file_name_list" rm -f $error_out # Verify that the output list is not empty [ ! -s "$tmp_file_list" ] && bail "Unable to determine start/stop dates." # A file list was created ...source it to define # start_year, start_mon, stop_year, stop_mon in the current environment : ; . $tmp_file_list rm -f $tmp_file_list # Define file names in the current environment in a sequence of variables # named file1, file2, ...,fileN. This can be done with a code snippet # supplied by the user at job string creation time or by make_file_name_list # at run time. The user supplied code snippet will always superceed the # make_file_name_list file name generation. # A user supplied list of file names may be inserted here when the job string # is created. If so then these file names will be used. # These file name definitions may use the variables start_year, start_mon, # stop_year, stop_mon, runid and nccrip_uxxx that were defined above. # <> if [ -z "$file1" ]; then # There is no user supplied list of file names # Use make_file_name_list to generate one at run time # prefix_list and suffix_list are used by make_file_name_list, together with # other variables from the current environment, to build a list of files names nccrip_prefix_list='' prefix_list="${nccrip_prefix_list:-$nccrip_uxxx}" nccrip_suffix_list='' suffix_list="${nccrip_suffix_list:-gp}" # This invocation of make_file_name_list uses runid, prefix_list and suffix_list # from the current environment, together with the *_year and *_month variables # defined above to build a list of file names and write that list to a file. # Note, fopts and tmp_file_list are defined above. $MAKE_FILE_NAME_LIST $fopts --nomulti_list $tmp_file_list >>$error_out 2>&1 || \ bail "Problem in make_file_name_list" rm -f $error_out # Verify that the output list is not empty [ ! -s "$tmp_file_list" ] && bail "Unable to create file list" # A file list was created ...source it # This will define fileN, join, start_year, start_mon, stop_year, stop_mon, # end_year and end_mon in the current environment : ; . $tmp_file_list rm -f $tmp_file_list fi # Ensure that at least 1 file name is defined [ -z "$file1" ] && bail "No file names have been defined." # Define a variable containing a string that will identify the # current year/month range, for possible use in file names etc. ym_range="${start_year}m${start_mon}_${stop_year}m${stop_mon}" # Define the name of a file to contain the list of file names input to nccrip input_flist="input_flist_$stamp" rm -f $input_flist touch $input_flist # Populate input_flist with file names, one per line nfile=1 eval fname=\$file$nfile while [ -n "$fname" ]; do echo "$fname" >> $input_flist nfile=`expr $nfile + 1` eval fname=\$file$nfile [ $nfile -gt 1000 ] && bail "Runaway file list. nfile=$nfile" done # If defined output_dir must be the name of a directory that is accessible from # the batch environment. Output files created below will be copied into output_dir # when it is defined. Otherwise output files will be saved to DATAPATH/RUNPATH. output_dir='' # nccrip_opts will contain user supplied command line options of the form "-opt" # None of these options require an argument # Valid options are: # # -coord_as_real ...Make coordinate variable type real (default is type double) # -add_time_bnds ...Add a time_bnds variable to indicate time intervals # -grll ...Input grid is a regular lat-lon grid # -grllc ...Input grid is a regular lat-lon grid with a cylic lon # -goll ...Input grid is an offset regular lat-lon grid # -gollc ...Input grid is an offset regular lat-lon grid with a cylic lon # -gocn ...Input grid is an ocean tracer grid # -gocnv ...Input grid is an ocean velocity grid # The input grid is assumed to be gaussian if none of # -grll, -grllc, -goll or -gollc are specified # -rem_cyclic_lon ...Remove the cyclic longitude from all output fields # -plev_in ...Input data is defined on pressure levels # -mlev_in ...Input data is defined on atmosphere model levels # -olev_in ...Input data is defined on ocean model levels # -rlev_out ...The output vertical coordinate will contain the verbatim values that # appear in the ibuf(4) field of labels read from the input file # -plev_out ...The output vertical coordinate will be on pressure levels # Normally input levels are output levels. plev_out is only # used when atm model levels are input and pressure levels # are desired in the output file. In this case the program # will calculate pressure levels from the atm model levels. # -reverse_levs ...Reverse the direction of the vertical coordinate # -disk ...By default all input CCCma records are kept in memory # until they get written to the output netcdf file # This option will force these records to be written to tmp files # then reread before they are finally written to the output file # These temporary files are removed when no longer needed # unless the "-keep" option is invoked # -keep ...Do not delete tmp files create by the "-disk" option # -ccout ...Create CCCma binary output files in addition to netcdf # -list_output ...Create text files containing lists of all output files # A file named "nccrip_file_list_netcdf" will contain a list of # netcdf output files. If -ccout is used then a second file named # "nccrip_file_list_cccbin" will contain a list CCCma output files # -nomatch_label ...do not compare LABL values when comparing input records to see if # the current record belongs to the same variable as a previous record # The default is to compare LABL vlaues for all input records # -nosp2gg ...do not convert SPEC records to GRID records # Normally SPEC records are always converted to GRID # -nopfx ...do not add a prefix to output file names # -noinv ...assume there are no time invariant records # The default is to assume that records with ibuf2=0 are invariant # but there are no time invariant records when time_offset is used # -no_default_meta ...Do not read default meta data # -ts ...The input file is a time series file containing a single variable # Using this option can speed up processing of time series files # Do not use this unless the input file contains a single variable # -h ...Display extended usage info # -v ...Increase verbosity (use -vv or -vvv for more output) nccrip_opts='' # Ensure the definition flist_out=... does not appear in the option list if [ -n "$nccrip_opts" ]; then XXX=`echo $nccrip_opts|sed 's/flist_out=[A-Za-z0-9_-][A-Za-z0-9_-]*//'` eval nccrip_opts\=\$XXX fi # nccrip will recognize and use certain variable definitions found on the command line # Valid variable definitions are: # # flist=Char ...The name of a file containing input file names # Each file name should appear on a separate line. # Comment lines (lines that begin with "#") and blank # lines are ignored. # flist_out=Char ...The name of a file containing output file names # This list will be created if either flist_out is defined # on the command line or the -list_output option is invoked. # If the file is to be written but flist_out is not defined then # a default name of nccrip_file_list_netcdf will be used. # desc=Char ...A file containing data description variables (e.g. PARM etc) # This file may contain an arbitrary set of variable but only a # certain subset of these variables will be used. # That subset will depend on the type of input file being processed. # var[s]=Char ...A white space separated list of variables to include # These may be variable names or variable definitions. # Note: variable definitions may not contain white space. # xvar[s]=Char ...A white space separated list of variables to exclude # These variable names will appear in ibuf(3) of input records # var[s] and xvar[s] are mutually exclusive # vlist=Char ...The name of a file containing variables to include # Each variable or variable definition should appear in this # file on a separate line. Comment lines (lines that begin # with "#") and blank lines are ignored. # This is an alternative to using var[s]=... but variable # definitions input here may contain white space # vdef[s]=Char ...A white space separated list of variable definitions # Each definition is of the form "ncname=def" where ncname # is the name of the variable as it will appear in the output # netcdf file and def is a linear combination of variable # names found in the input CCCma binary file. No whitespace # is allowed in ncname and def. Variables defined here will # be written to netcdf files in addition to all other variables # found in the input files. By contrast, variables defined via # var=... will result in only a subset of variables being written # to netcdf files (ie only those contained in the var=... list) # Examples of variable definitions would be # tas=ST+273.16 # balt=FSA+FSG+FLA+FLG # geo_h=PHIS/9.8 # vdef_list=Char ...The name of a file containing variable definitions # Each variable definition should appear in this file on a # separate line. Comment lines (lines that begin with "#") # and blank lines are ignored. This is an alternative to # using vdef[s]=... but variable definitions input here # may contain white space. Each variable definition must # contain one and only one equal sign. # lab[s]=Char ...A comma separated list of superlabels to include # xlab[s]=Char ...A comma separated list of superlabels to exclude # lab[s] and xlab[s] are mutually exclusive # lev[s]=Char ...A white space separated list of output levels (ibuf4 values) # This must be a subset of the levels found in the input files # plev[s]=Char ...A white space separated list of output pressure levels (hPa) # This is an alternative to levs when pressure levels are output. # Pressure values should be monotonically increasing with # array index and within the range 0 to 1200. # If plev is not defined a default set of levels is used # meta_data=Char ...The name of a file containing user supplied meta data # namelist=Char ...The name of a file containing namelist input # start=Int ...The smallest time (ibuf2 value) to appear in output files # stop=Int ...The largest time (ibuf2 value) to appear in output files # time_units=Char ...Indicate the units for the output time dimension # This string must be suitable as the "units" attribute # in a CF compiant netcdf file (default "days since 1850-1-1") # Valid values for the first word in this string are: # days, hours, minutes or seconds # ib2_fmt=Char ...Description of the format of the ibuf(2) value found # in the CCCma input file. This will be one of # MODEL OMODEL RAW YYYYMMDDHH YYYYMMDD YYYYMM YYYY # ib2_fmt is set internally if not defined on the command line # MODEL=Char ...A white space separated list of variables that require a MODEL ib2_fmt # OMODEL=Char ...A white space separated list of variables that require a OMODEL ib2_fmt # RAW=Char ...A white space separated list of variables that require a RAW ib2_fmt # YYYYMMDDHH=Char ...A white space separated list of variables that require a YYYYMMDDHH ib2_fmt # YYYYMMDD=Char ...A white space separated list of variables that require a YYYYMMDD ib2_fmt # YYYYMM=Char ...A white space separated list of variables that require a YYYYMM ib2_fmt # YYYY=Char ...A white space separated list of variables that require a YYYY ib2_fmt # ...Defining one or more of the above variable lists will force the # program to use the specified ib2_fmt for particular variables. # This ib2_fmt specification will superceed any other ib2_fmt # specification, for the variables in these lists. # time_shift=Int ...An integer flag used to determine if a forward or backward # time shift of half an interval is required. Valid values are: # -1 => shift backward, 0 => no shift, 1 => shift forward # time_interval=Char ...A string that will indicate the time interval between # successive records in the input data file. # This string must be of the form "R Unit" where R is numeric # and Unit is one of "days", "hours", "minutes" or "seconds". # If this is not set by the user then an internal default is # determined based on the format of the ibuf(2) value. # This time interval is used when a time shift is requested # or a time_bnds variable is required. # time_offset=Char ...A string that will indicate an offset to add to time values # read from the input file. A positive offset means move forward # in time and a negative offset means move backward in time. # This string is of the form "R Unit" where R is numeric # and Unit is one of "years", "days", "hours", "minutes" or "seconds". # pfx=Char ...A prefix to be used on all output file names # Use the command line option -nopfx to avoid any prefix # sfx=Char ...A suffix to be used on all output file names # psfile=Char ...The name of a file containing surface pressure in "hPa" # psfile_Pa=Char ...The name of a file containing surface pressure in "Pa" # psfile[_Pa] is used in conjunction with -mlev # For every record in the input CCCma binary time series file # there must be a corresponding record with the same time # (same ibuf2 value) in this psfile # nens=Int ...The number of ensemble members present in the input files # There must be nens separate files supplied on the command line. # All files will contain the same set of records and each file # will correspond with a single realization of the ensemble. # The order these files appear on the command line will determine # their index in the ensemble dimension of the output netcdf file. # fillvalue=Re ...Reset the fillvalue (default is 1e38) # verbose=Int ...Request more output (Int>3) than -vvv will provide # Initialize a variable that will contain unquoted variable definitions UQVARDEFS='' # Initialize a variable that will contain quoted variable definitions QVARDEFS='' # The following variables may be set by the user during job script creation to add # particular variable definitions to the nccrip command line # These variables contain a single word (no white space) so do not require quoting nccrip_start='' nccrip_stop='' nccrip_ib2_fmt='' nccrip_time_shift='' nccrip_pfx='' nccrip_sfx='' nccrip_psfile='' nccrip_psfile_Pa='' nccrip_fillvalue='' nccrip_verbose='' nccrip_desc='' nccrip_vdef_list='' nccrip_vlist='' nccrip_meta_data='' nccrip_namelist='' nccrip_time_shift='' nccrip_nens='' deflist='start stop ib2_fmt time_shift pfx sfx psfile psfile_Pa fillvalue verbose desc vdef_list vlist meta_data namelist time_shift nens' for var in $deflist; do eval optvar\=nccrip_\$var eval optval\=\$$optvar [ -z "$optval" ] && continue # This variable has a non null value # Ensure this value does not contain whitespace [ x`echo $optval|sed -n '/ /p'` != x ] && bail "Invalid nccrip variable definition --> ${optvar}=$optval <--" UQVARDEFS="$UQVARDEFS ${var}=$optval" done # These variables imply that external files are required ...ensure these files exist if [ -n "$nccrip_desc" ]; then echo "Reading data description from $nccrip_desc" if [ ! -s "$nccrip_desc" ]; then # There is no local version of this file ...attempt an access access $nccrip_desc $nccrip_desc || bail "Cannot find data description file $nccrip_desc" else # Ensure the local version is readable [ ! -r "$nccrip_desc" ] && bail "Cannot read data description file $nccrip_desc" fi fi if [ -n "$nccrip_vdef_list" ]; then echo "Reading variable list from $nccrip_vdef_list" if [ ! -s "$nccrip_vdef_list" ]; then # There is no local version of this file ...attempt an access access $nccrip_vdef_list $nccrip_vdef_list || bail "Cannot find variable list file $nccrip_vdef_list" else # Ensure the local version is readable [ ! -r "$nccrip_vdef_list" ] && bail "Cannot read variable list file $nccrip_vdef_list" fi fi if [ -n "$nccrip_vlist" ]; then echo "Reading variable list from $nccrip_vlist" if [ ! -s "$nccrip_vlist" ]; then # There is no local version of this file ...attempt an access access $nccrip_vlist $nccrip_vlist || bail "Cannot find variable list file $nccrip_vlist" else # Ensure the local version is readable [ ! -r "$nccrip_vlist" ] && bail "Cannot read variable list file $nccrip_vlist" fi fi if [ -n "$nccrip_meta_data" ]; then echo "Reading meta data from $nccrip_meta_data" if [ ! -s "$nccrip_meta_data" ]; then # There is no local version of this file ...attempt an access access $nccrip_meta_data $nccrip_meta_data || bail "Cannot find meta data file $nccrip_meta_data" else # Ensure the local version is readable [ ! -r "$nccrip_meta_data" ] && bail "Cannot read meta data file $nccrip_meta_data" fi fi if [ -n "$nccrip_namelist" ]; then echo "Reading the namelist from $nccrip_namelist" if [ ! -s "$nccrip_namelist" ]; then # There is no local version of this file ...attempt an access access $nccrip_namelist $nccrip_namelist || bail "Cannot find namelist file $nccrip_namelist" else # Ensure the local version is readable [ ! -r "$nccrip_namelist" ] && bail "Cannot read namelist file $nccrip_namelist" fi fi # These variables may contain a white space separated list of words and as such must # have their values quoted when they appear on the nccrip command line nccrip_vars='' nccrip_xvars='' nccrip_labs='' nccrip_vdefs='' nccrip_xlabs='' nccrip_levs='' nccrip_plevs='' nccrip_MODEL='' nccrip_OMODEL='' nccrip_RAW='' nccrip_YYYYMMDDHH='' nccrip_YYYYMMDD='' nccrip_YYYYMM='' nccrip_YYYY='' nccrip_time_units='' nccrip_time_interval='' nccrip_time_offset='' # Write the nccrip command line into a file for ease of dealing with # variable definitions that need to be quoted # This eval is necessary to prevent the shebang line from being # blindly replaced with a different shell by submit3 eval shebang=\#\!/\bin/\bash echo "$shebang" > xnccrip chmod u+x xnccrip # Add the nccrip command line with all args except those that require quotes echo_n "nccrip -list_output $nccrip_opts flist=$input_flist $UQVARDEFS" >> xnccrip # Append variable definitions that require quotes, one at a time deflist='vars xvars vdefs labs xlabs levs plevs MODEL OMODEL RAW YYYYMMDDHH YYYYMMDD YYYYMM YYYY time_units time_interval time_offset' for var in $deflist; do eval optvar\=nccrip_\$var eval optval\=\$$optvar [ -z "$optval" ] && continue # This variable has a non null value # Strip leading and trailing white space optval=`echo $optval|sed 's/^ *//; s/ *$//'` [ -z "$optval" ] && bail "Empty string (blank) found in variable $optvar" echo_n " ${var}=\"$optval\"" >> xnccrip done # Add a newline to xnccrip echo " " >> xnccrip # Run nccrip to generate the output time series files echo "Input file used during this invocation of nccrip:" cat $input_flist echo " " echo "nccrip command line to be executed:" cat xnccrip echo " " ./xnccrip || bail "Error executing nccrip" echo " " # Process output file if [ -s nccrip_file_list_netcdf ]; then echo "Netcdf files created by nccrip:" cat nccrip_file_list_netcdf | xargs -I{} ls -l {} if [ -n "$output_dir" ]; then # Move output files to a user supplied directory [ ! -d $output_dir ] && bail "Missing user supplied output_dir = $output_dir" for ofile in `cat nccrip_file_list_netcdf`; do echo "mv $ofile $output_dir/$ofile" mv $ofile $output_dir/$ofile || bail "**EE** mv $ofile $output_dir/$ofile" chmod a+r $output_dir/$ofile done else # Save output files to DATAPATH/RUNPATH for ofile in `cat nccrip_file_list_netcdf`; do save $ofile $ofile done fi else echo "No netcdf files were created." fi echo " " if [ -s nccrip_file_list_cccbin ]; then echo "CCCma binary files created by nccrip:" cat nccrip_file_list_cccbin | xargs -I{} ls -l {} if [ -n "$output_dir" ]; then # Move output files to a user supplied directory [ ! -d $output_dir ] && bail "Missing user supplied output_dir = $output_dir" for ofile in `cat nccrip_file_list_cccbin`; do echo "mv $ofile $output_dir/$ofile" mv $ofile $output_dir/$ofile || bail "**EE** mv $ofile $output_dir/$ofile" chmod a+r $output_dir/$ofile done else # Save output files to DATAPATH/RUNPATH for ofile in `cat nccrip_file_list_cccbin`; do save $ofile $ofile done fi else echo "No CCCma binary files were created." fi echo " " # ---Stop_submit_ignore_code---- end_of_script . endjcl.cdk #end_of_job