#!/bin/sh #======================================================================= # --- xcpd --- # Create a job that will dump/diagnose/delete history files in parallel # with the current job string. #======================================================================= # # Larry Solheim Nov,2017 # # Files dumped/diagnosed/deleted will be of the form # ${xcpd_prefix}_${year}_m${mon}_$suffix # for all months from previous_year, previous_month to current_year, # current_month or from current_year, current_month to next_year, # next_month depending on which of previous_(year|month) or # next_(year|month) are set. # # The variables current_year, current_month, previous_year and # previous_month are set when the job string is created. # # A variable named xcpd_suffix_list may also be set to modify the list # of files that will be transferred/dumped. See below for details. # #======================================================================= # # keyword :: xcpd # description :: parallel dump/diagnose of history files # # runid=job000 # uxxx=uxxx # # Run time diagnostics # xcpd_with_rtd=on # year_rdiag_start='' # year_rtdiag_start='' # yearfirst='' # PhysA='' # PhysO='' # CarbA='' # CarbO='' # CarbL='' # resolution='' # pldir='' # rtd_diag_deck='' # rtd_plot_deck='' # rtd_path='' # xcpd_rtd_plot_freq='' # # xcpd_with_time_series=on # xcpd_with_pool=on # # CCCJOB_ROOT # # # Chunk size (in months) for pooling, dumping diags and creating time series # xcpd_chunk_size='' # # # Chunk size (in months) of the first pooling interval # # This will default to chunk_size if not set here # xcpd_first_chunk_size='' # # # Include pooling in the parallel dump string # xcpd_with_pool='' # # # Do not include the first year of the run in multi year pooling # xcpd_pool_ignore_first_year='' # # # xcpd_with_partial_chunk flags whether or not pooling/time series etc will # # run on the last job in the job string if the interval since the last # # pooling/time series ran is less than xcpd_chunk_size # xcpd_with_partial_chunk='' # # # xcpd_with_time_series flags whether or not time series are created when # # files are dumped to cfs # xcpd_with_time_series='' # # # with_dd flags inclusion of "*_dd" files # xcpd_with_dd='' # # # with_dp flags inclusion of "*_dp" files # xcpd_with_dp='' # # # with_ds flags inclusion of "*_ds" files # xcpd_with_ds='' # # # with_dsd3D flags inclusion of additional CFMIP daily fields # xcpd_with_dsd3D='' # # # xcpd_dump_hist flags dumping (or not dumping) history files # xcpd_dump_hist='' # # # xcpd_dummy_diag flags the creation of dummy diagnostic file (used for debugging) # xcpd_dummy_diag='' # xcpd_mkdiag_clone='' # # # When history files are not dumped but simply copied to the front end # # (when xcpd_dump_or_copy_hist = off), del_hist_after_copy will flag deletion # # of the history files from the back end after they are copied # del_hist_after_copy='' # # # xcpd_transfer_time_series flags using rtrans to copy files to UVic # xcpd_transfer_time_series='' # # # If set, xcpd_RMTRUNPATH will be the directory on lxsrv (at UVic) into # # which these files will be copied # xcpd_RMTRUNPATH='' # # # Is this a coupled model run or an agcm only model run # coupled='' # # # Insert or comment out certain diagnostic decks # xcpd_gpintstat='' # xcpd_gsstats='' # xcpd_xstats='' # xcpd_cmstats='' # xcpd_xtrachem='' # xcpd_xtraconv='' # xcpd_cosp_sim='' # xcpd_cfmip_monthly_rad_force='' # xcpd_daily_cmip5_CanAM4='' # xcpd_daily_cmip5_cfmip_v2_dsd3D='' # xcpd_daily_cmip5_cosp_v2='' # xcpd_daily_cmip5='' # # # Is this is a special type of run (e.g. hansen) # xcpd_type='' # # xcpd_front_end is the name of the machine to which the diagnostic job will be submitted # xcpd_front_end='' # # xcpd_back_end is the name of the machine to which the job that deletes # history files will be sent, if that job is created # xcpd_back_end='' set -a . betapath2 username=acrnxxx; user=MYNAME; crawork=xcpd_job nextjob=on noprint=on debug=off xcpd_gptime='' xcpd_stime='' gptime=${xcpd_gptime:=3600}; stime=${xcpd_stime:=3600}; dtime=100; memory1=900mb jobname=xcpd; time=$stime ; memory=$memory1; # Temporary directory where this script will run CCRNTMP=$CCRNTMP # RUNPATH on execution machine RUNPATH=$RUNPATH # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi . comjcl.cdk cat > Execute_Script <<'end_of_script' # ---Start_submit_ignore_code---- set -a FULLPATH=`type $0|awk '{print $3}'` Runame=`basename $FULLPATH` # Define a unique stamp for use in file names etc stamp=`date "+%j%H%M%S"$$` runid=job000; uxxx=uxxx; xcpd_uxxx=$uxxx; xcpd_prefix=${xcpd_uxxx}_${runid} # Use -e option (enable interpretation of backslash escapes) if recognized by echo if [ "X`echo -e`" = "X-e" ]; then echo_e() { echo ${1+"$@"}; } else echo_e() { echo -e ${1+"$@"}; } fi # bail is a simple error exit routine # Note: we write the error directly to a file in ~/.queue so that this # info is not lost if/when stdout is not returned error_out="${JHOME:-$HOME}/.queue/error_xcpd_${runid}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out bail(){ echo_e `date`" --- xcpd: $*" echo_e `date`" --- xcpd: $*" >>$error_out exit 1 } # stand_alone_craworkval, if set, will indicate that the diagnostics are to be run # as a stand alone string of jobs run in series # If stand_alone_craworkval is not set or empty then it is assumed that the diagnostic # string is to be run in parallel with a model job stand_alone_craworkval='' # Use the second character from xcpd_uxxx to determine the type of # pooling and/or the prefix for diagnostic and time series files ch2=`echo $xcpd_uxxx|awk '{print substr($1,2,1)}' -` [ -z "$ch2" ] && bail "Invalid xcpd_uxxx = $xcpd_uxxx" case $ch2 in a|m|c|r|d|f) do_nothing=0 ;; # These are valid *) bail "Unable to determine pooling type from xcpd_uxxx = $xcpd_uxxx" ;; esac # xcpd_with_time_series is used to control inclusion of time series creation # If xcpd_with_time_series is set to "off" then no time series are created xcpd_with_time_series=off XXX=`echo $xcpd_with_time_series|sed 's/ //g'` eval xcpd_with_time_series\=$XXX [ "$xcpd_with_time_series" = 'on' ] && eval xcpd_with_time_series\=1 [ "$xcpd_with_time_series" = 'off' ] && eval xcpd_with_time_series\=0 [ "$xcpd_with_time_series" = 'yes' ] && eval xcpd_with_time_series\=1 [ "$xcpd_with_time_series" = 'no' ] && eval xcpd_with_time_series\=0 # xcpd_with_rtd is used to control inclusion of run time diagnostics # If xcpd_with_rtd is set to "off" then no run time diagnostics are used. xcpd_with_rtd=off XXX=`echo $xcpd_with_rtd|sed 's/ //g'` eval xcpd_with_rtd\=$XXX [ "$xcpd_with_rtd" = 'on' ] && eval xcpd_with_rtd\=1 [ "$xcpd_with_rtd" = 'off' ] && eval xcpd_with_rtd\=0 [ "$xcpd_with_rtd" = 'yes' ] && eval xcpd_with_rtd\=1 [ "$xcpd_with_rtd" = 'no' ] && eval xcpd_with_rtd\=0 # xcpd_rtd_with_plot can be used to force no plotting of run time diags # It is on by default. The variable xcpd_rtd_plot_freq may be used # to set the frequence (in years) of rtd plotting xcpd_rtd_with_plot=on XXX=`echo $xcpd_rtd_with_plot|sed 's/ //g'` eval xcpd_rtd_with_plot\=$XXX [ "$xcpd_rtd_with_plot" = 'on' ] && eval xcpd_rtd_with_plot\=1 [ "$xcpd_rtd_with_plot" = 'off' ] && eval xcpd_rtd_with_plot\=0 [ "$xcpd_rtd_with_plot" = 'yes' ] && eval xcpd_rtd_with_plot\=1 [ "$xcpd_rtd_with_plot" = 'no' ] && eval xcpd_rtd_with_plot\=0 # Indicate how cccjob should be invoked # Setting CCCJOB_ROOT will allow a job specific version of cccjob to used CCCJOB_ROOT='' if [ -z "$CCCJOB_ROOT" ]; then CCCJOB_ENV='' else eval CCCJOB_ENV=\'env CCCJOB_ROOT\=$CCCJOB_ROOT\' fi # Debugging options DBG_submit='' # When xcpd_dumprs_sublist = F use dump_list rather that dump_sublist # to dump restarts at the end of each time series chunk xcpd_dumprs_sublist='' # When xcpd_with_dumprs = T then restarts will be dumped even when # history files are not dumped (this is the default behaviour) xcpd_with_dumprs='' # These variables are set when the job string is created previous_year=NotSet previous_month=NotSet current_year=NotSet current_month=NotSet next_year=NotSet next_month=NotSet run_start_year=NotSet run_start_month=NotSet run_stop_year=NotSet run_stop_month=NotSet # Allow xcpd_first_year, if set, to override the value of run_start_year # The eval is required to hide this assignment from cccjob substitution xcpd_first_year='' [ -n "$xcpd_first_year" ] && eval run_start_year\=$xcpd_first_year # pdel_suffix_list is used to generate file names of history files that # are to be deleted after they have been processed. mdelete_suffix_list='' pdel_suffix_list=${mdelete_suffix_list:="gs ss gz cm rs os cs ab ob an _script"} # Pass diag2ts_suffix_list on to spltdiag if defined here diag2ts_suffix_list='' # Delete diag files from front/back end after theay have been used xcpd_delete_diag='' # These parameters are used to add/remove suffixes used by cmam20 xcpd_with_gp6='' xcpd_with_xp6='' xcpd_with_cc='' xcpd_with_td='' # Pass gssave to xcpd xcpd_gssave='' # This invocation of make_file_name_list will process the *_year and *_months # variables defined above and output a file containing definitions for # start_year, start_mon, stop_year, stop_mon tmp_file_list="xcpd_date_list_${runid}_${stamp}" make_file_name_list --dates_only $tmp_file_list >>$error_out 2>&1 ||\ bail "Problem in make_file_name_list" rm -f $error_out # Verify that the output list is not empty [ ! -s "$tmp_file_list" ] && bail "Unable to create file list" # A file list was created ...source it # This will define start_year, start_mon, stop_year, stop_mon : ; . $tmp_file_list rm -f $tmp_file_list # Define start and stop dates xcpd_start_year=$start_year xcpd_start_mon=$start_mon xcpd_stop_year=$stop_year xcpd_stop_mon=$stop_mon # model_version will be used to determine which diagnostic decks to use modver=${modver:=gcm15h} xcpd_model_version='' model_version=${xcpd_model_version:=$modver} # ym_range is used as part of some file names defined below ym_range="${xcpd_start_year}m${xcpd_start_mon}_${xcpd_stop_year}m${xcpd_stop_mon}" # Define a partial file name fid=${runid}_$ym_range # Define a directory used to hold persistent files (lock files etc) persistd="${JHOME:-$HOME}/.queue/.crawork" # Generate a lock file name (this should be a full pathname) xcpd_lock="${persistd}/lock_xcpd_${fid}_$stamp" # Check for the existence of previous lock files and abort if greater # than xcpd_lock_max lock files currently exist xcpd_lock_max=5 found_lock=`(ls -1 ${persistd}/lock_xcpd_${runid}_* || : ) 2>/dev/null` if [ ! -z "$found_lock" ]; then # If lock files exist then determine how many there are found_lock_n=`echo $found_lock|awk '{printf "%d",NF; exit}' -` echo "$found_lock_n xcpd lock files currently exist" if [ $found_lock_n -gt $xcpd_lock_max ]; then bail "$found_lock_n existing lock file(s): $found_lock" fi if [ -n "$stand_alone_craworkval" ]; then # The diagnostics are being run in a single job string # Check for xcpd lock files that are older than the current lock file ncheck=0 # Note, this loop will exit with an abort after 3 iterations while [ $ncheck -lt 10 ]; do ncheck=`expr $ncheck + 1` xcpd_lock_found=`(ls -1 ${persistd}/lock_xcpd_${runid}_* || : ) 2>/dev/null` if [ -z "$xcpd_lock_found" ]; then # There are no xcpd lock files in persistd # Assume that lock files are not being used lock_status=0 else # At least 1 xcpd lock file was found in persistd # These lock file names are of the form # lock_xcpd_${runid}_${start_date}_${stop_date}_$stamp # where stamp is a string that does not contain any underscore # and (start|stop)_date are both of the form YYYYmMM lock_status=1 nlock=0 for curr_dir in $xcpd_lock_found; do # "ls" will return a sorted list so if the current xcpd lock file # is not the first in the list then prior lock files exist nlock=`expr $nlock + 1` if [ x"$curr_dir" = x"$xcpd_lock" ]; then if [ $nlock -eq 1 ]; then # The current lock file is first in the list lock_status=0 fi # Ignore lock files that are newer than the current lock file break fi done fi if [ $lock_status -eq 0 ]; then # There are no xcpd lock files older than the current one break else # Prior lock files exist if [ $ncheck -ge 3 ]; then # Too many checks echo " " echo "xcpd lock files exist prior to $xcpd_lock" echo " " bail "xcpd lock files exist:\n$xcpd_lock_found" else # Try again in 2 minutes sleep 120 fi fi done fi fi ######### Create the parallel job ########### # These variables may be set when the xcpd job string is created # If a variable is set then it will be used in the diag job # xcpd_front_end is the name of the machine to which the diagnostic job # will be submitted. # xcpd_front_end superceeds xcpxd_gateway which has been depreciated. xcpd_front_end='' # gateway is the name of the machine to which files # are sent prior to being moved off site xcpxd_gateway=${xcpd_front_end:='pollux'} gateway_is_back_end=0 case $xcpxd_gateway in c[0-9]*) gateway_is_back_end=1 ;; zeta|saiph|spica|hadar) gateway_is_back_end=1 ;; za|sa|sp|ha) gateway_is_back_end=1 ;; esac if [ $gateway_is_back_end -eq 1 ]; then # When diags are run on the back end do not delete gz files until the time series are complete # This eval is required to hide this assignment from cccjob substitution xxx=`echo $pdel_suffix_list|sed 's/\g\z//'` eval pdel_suffix_list\=\"$xxx\" fi mdump_arclabel='' mdump_uxxx='' mdump_prefix='' mdump_CCRNTMP='' mdump_RUNPATH='' mdump_qsublog='' mdump_with_lock_file='' check_cfs_arcfile='' nontwrkchk='' # mdump_months is the frequency at which history files are dumped in xcpxd mdump_months='' # Ensure that mdump_months contains only integers or nothing at all XXX_mdump_months=`echo $mdump_months | sed 's/[0-9][0-9]*//'` [ -n "$XXX_mdump_months" ] && bail "Invalid mdump_months = $mdump_months" # xcpd_ddump_months is the frequency at which diagnostic files are dumped in xcpxd xcpd_ddump_months='' # Ensure that ddump_months contains only integers or nothing at all XXX_ddump_months=`echo $xcpd_ddump_months | sed 's/[0-9][0-9]*//'` [ -n "$XXX_ddump_months" ] && bail "Invalid xcpd_ddump_months = $xcpd_ddump_months" #xxx mdump_suffix_list=${xcpd_suffix_list:=''} mdump_cfsuser='' cfsuser='' masterdir='' shortermdir='' nolist='' # sv should always be on when gateway is a front end machine # sv on means dump from front end disks eval sv\=on # besc = on causes a tdumper job to run on the back end (spica/hadar) xcpd_besc='' besc=${xcpd_besc:=''} # BERUNPATH must be set if files are to be copied from a machine that is # not the default back end machine bemach='' if [ -z "$bemach" ]; then # If bemach is not set explicitly then see if this job is executing on # a back end machine and use that machine name for bemach this_mach=`uname -n|awk -F\. '{print \$1}' -` on_back_end=0 case $this_mach in c[0-9]*) on_back_end=1 ;; zeta|saiph|spica|hadar) on_back_end=1 ;; za|sa|sp|ha) on_back_end=1 ;; esac if [ $on_back_end -eq 1 ]; then # Set this_mach to its known alias case $this_mach in ha|ha*) this_mach=hadar ;; sp|sp*) this_mach=spica ;; c1*) this_mach=spica ;; c2*) this_mach=hadar ;; za|ze*) this_mach=zeta ;; sa|sa*) this_mach=saiph ;; c6*) this_mach=saiph ;; c7*) this_mach=zeta ;; esac bemach=$this_mach fi fi if [ -n "$bemach" ]; then # Reset BERUNPATH if bemach is set # Ensure bemach contains a known alias case $bemach in ha|ha*) bemach=hadar ;; sp|sp*) bemach=spica ;; za|ze*) bemach=zeta ;; sa|sa*) bemach=saiph ;; esac XXX=`ssh $bemach echo '$RUNPATH' 2>/dev/null` || bail "Unable to determine BERUNPATH" BERUNPATH=$XXX # Also assign back_end_mach eval back_end_mach\=$bemach else BERUNPATH=$BERUNPATH fi # xcpd_back_end is the name of the machine to which the job that deletes # history files will be sent, if that job is created and used xcpd_back_end=${back_end_mach:=''} # Define start/stop year/month cccjob options to be used below curr_year=`echo $xcpd_start_year|awk '{printf "%d",$1}' -` curr_mon=`echo $xcpd_start_mon|awk '{printf "%d",$1}' -` nxt_year=`echo $xcpd_stop_year|awk '{printf "%d",$1}' -` nxt_mon=`echo $xcpd_stop_mon|awk '{printf "%d",$1}' -` start_opt="--start_time=${curr_year}:$curr_mon" stop_opt="--stop_time=${nxt_year}:$nxt_mon" # Ensure that the value of months used in the cccjob command below is # consistent with the interval between start_opt and stop_opt xcpd_months=`echo $curr_year $curr_mon $nxt_year $nxt_mon|\ awk '{if ($1<=0 || $3<=0 || $3<$1) {print "0"; exit} if ($2<1 || $2>12 || $4<1 || $4>12) {print "0"; exit} if ($1 == $3) {m=$4-$2+1} else {m=($3-$1-1)*12+13-$2+$4}; printf "%d",m}' -` [ $xcpd_months -le 0 ] && \ bail "Invalid xcpd_months = $xcpd_months ...xcpd_start_year = $xcpd_start_year xcpd_start_mon = $xcpd_start_mon xcpd_stop_year = $xcpd_stop_year xcpd_stop_mon = $xcpd_stop_mon" eval months\=$xcpd_months rtrans_remusr='' rtrans_remserver='' rtrans_remdir='' rtrans_force='' rtrans_scp_args='' rtrans_ssh_args='' rtrans_getput='' rtrans_suffix_list='' rtrans_uxxx='' rtrans_prefix='' rtrans_prefix_='' rtrans_suffix_='' # If diag_job is set then it must be the name of diagnostic deck that # is found on the users path (or in a default location, see below) # If set then the diagnostics will run after history files are dumped # This is used in conjunction with mdump_months>0 (serial dumps) diag_job='' # load_hist will determine how history files get onto the front end # to be used by the diagnostics when diag_job has a non-null value # load_hist = off ==> Copy history files from the back end # load_hist = on ==> Load history files from the cfs load_hist='' # Chunk size (in months) for pooling, dumping diags and creating time series xcpd_chunk_size='' # Chunk size (in months) of the first pooling interval # This will default to chunk_size if not set here xcpd_first_chunk_size='' # Include pooling in the parallel dump string xcpd_with_pool='' # Do not include the first year of the run in multi year pooling xcpd_pool_ignore_first_year='' # xcpd_with_partial_chunk flags whether or not pooling/time series etc will # run on the last job in the job string if the interval since the last # pooling/time series is less than xcpd_chunk_size xcpd_with_partial_chunk='' # xcpd_with_time_series flags whether or not time series are created when # files are dumped to cfs xcpd_with_time_series='' # with_cplhist flags inclusion of "*_cplhist.nc" files xcpd_with_cplhist='' # with_cplrs flags inclusion of "*_cplrs.tar" files xcpd_with_cplrs='' # with_dd flags inclusion of "*_dd" files xcpd_with_dd='' # with_dp flags inclusion of "*_dp" files xcpd_with_dp='' # with_ds flags inclusion of "*_ds" files xcpd_with_ds='' # with_dsd flags inclusion of "*_dsd" files xcpd_with_dsd='' # with_dsd3D flags inclusion of additional CFMIP daily fields xcpd_with_dsd3D='' # with_dcosp flags inclusion of daily COSP files xcpd_with_dcosp='' # with_mcosp flags inclusion of monthly COSP files xcpd_with_mcosp='' # with_radforce flags inclusion of the cfmip_monthly_rad_force diagnostic deck xcpd_with_radforce='' # with_rad_flux_profs flags inclusion of radiative flux profiles xcpd_with_rad_flux_profs='' # xcpd_dump_hist flags dumping (or not dumping) history files (used for debugging) xcpd_dump_hist='' # This flag will turn on or turn off dumping of diagnostic files xcpd_dump_diag='' # A switch to cause either a dump to cfs of history files or a copy from the back # end to front end (prior to running diags) # This switch is depreciated xcpd_dump_or_copy_hist='' # xcpd_dummy_diag flags the creation of dummy diagnostic file (used for debugging) xcpd_dummy_diag='' xcpd_mkdiag_clone='' # Allow the user to supply the suffix list for all diag files that are to be dumped # xcpd_mdump_diag_suffix_list will override mdump_diag_suffix_list in xcpxd xcpd_mdump_diag_suffix_list='' # Allow the user to supply the suffix list for diag files created in xcpd # The user is responsible for supplying the correct suffix list as there are no checks done xcpd_diag_suffix_list='' # When history files are not dumped but simply copied to the front end # (when xcpd_dump_or_copy_hist = off), del_hist_after_copy will flag deletion # of the history files from the back end after they are copied del_hist_after_copy='' # xcpd_transfer_time_series flags using rtrans to copy files to UVic xcpd_transfer_time_series='' # xcpd_remusr is the login name of the user on the remote machine, if not invoking user xcpd_remusr='' # xcpd_remserver is the fully qualified domain name of the remote machine, if not at UVic xcpd_remserver='' # If set, xcpd_RMTRUNPATH will be the directory on lxsrv (at UVic) into # which these files will be copied xcpd_RMTRUNPATH='' # Is this a coupled model run or an agcm only model run coupled='' # year_offset is used by some diagnostic decks year_offset='' # with_cp can be set false to ingnore "*_cp" files even when coupled is true xcpd_with_cp='' # with_gz can be set false to ingnore "*_gz" files even when coupled is true xcpd_with_gz='' # Alow the user to override the internal defaults for prefix and suffix list xcpd_prefix_list='' xcpd_suffix_list='' # Insert or comment out certain diagnostic decks xcpd_gpintstat='' xcpd_gsstats='' xcpd_xstats='' xcpd_cmstats='' xcpd_xtrachem='' xcpd_xtraconv='' xcpd_cosp_sim='' xcpd_cfmip_monthly_rad_force='' xcpd_daily_cmip5_CanAM4='' xcpd_daily_cmip5_cfmip_v2_dsd3D='' xcpd_daily_cmip5_cosp_v2='' xcpd_daily_cmip5='' # Is this is a special type of diagnostics # valid values for xcpd_type :: spbc_dd_dp_rf cmam20 xcpd_type='' # Use cnfs for tdumper jobs xcpd_with_cnfs='' # Used with diag2ts to indicate the deck names used to create time series from gz files. diag2ts_with_gztsdiag='' diag2ts_gztsdiag_deck1='' diag2ts_gztsdiag_deck2='' # A flag that may be used to turn off deletion of time series files after they # have been created and put on cfs diag2ts_delete_time_series='' # A flag that may be used to turn off dumping time series files to cfs after they # have been created diag2ts_dump_time_series='' # A flag to toggle creation of a single spltdiag job for each suffix (e.g. gp, dd, ..) # that is associated with diag files used to create time series # This will also cause a separate tdumper job to be created for each suffix, # when time series files are dumped to cfs diag2ts_one_job_per_suffix='' # These variables are used to determine if and where spltdiag will copy # time series files to a remote machine # spltdiag_trans = on|off ...on means transfer time series to the remote machine spltdiag_trans='' # spltdiag_remserver is the name of the remote host used when spltdiag_trans = on spltdiag_remserver='' # spltdiag_remdir is the name of an existing directory on the remote host into which # the time series files will be copied when spltdiag_trans = on spltdiag_remdir='' # Allow user to define a list of specific variables (ie superlabels) that will # limit the number of files that spltdiag creates spltdiag_vars='' # If next_block_lock is defined it will be the name of a lock file that was created # by the "block" module. xcpxd should remove this lock file after it finishes # processing at the end of each time series chunk next_block_lock='' # If pxdel_lock_check = off then there will be no checks for lock files done # before history files are deleted from the back end in pxdel. # Do not use this if you care that history files could be removed before you use them. pxdel_lock_check='' # If pxdel_check_last_mon = on then only lock files with a date range ending with # the current year and month will be checked for existence. pxdel_check_last_mon='' # These variables are used by the [ca]pool alias # load_diag='' # dump_diag='' # delete_diag='' # dump_pooled='' # delete_pooled='' psdelete_leave_last_pool='' # transfer_pooled='' # This flag will turn on or turn off dumping of pooled files xcpd_dump_pool='' # This flag will turn on or turn off dumping of pooled files xcpd_delete_pool='' # xcpd_with_ondisk can be set to "off" if ondisk is not required xcpd_with_ondisk='' ondisk_prefix_list='' ondisk_suffix_list='' # PhysA = on/off use physical atmosphere variables # PhysO = on/off use physical ocean variables # CarbA = on/off use atmosphere carbon variables # CarbO = on/off use ocean carbon variables # CarbL = on/off use land carbon variables # These will all default to "on" if not set here PhysA='' PhysO='' CarbA='' CarbO='' CarbL='' # Misc other parameters with_lsarc='' mdump_with_lsarc='' mdump_diag_with_lsarc='' xcpd_diag_job='' diag_uxxx='' model_uxxx='' hres='' xref01='' xref02='' xref03='' xref04='' xref05='' xref06='' xref07='' xref08='' xref09='' xref10='' xref11='' xref12='' xref13='' xref14='' xref15='' varlist='uxxx runid months noprint xcpxd_gateway xcpd_lock mdump_uxxx mdump_prefix mdump_cfsuser mdump_CCRNTMP mdump_RUNPATH mdump_months mdump_qsublog mdump_with_lock_file psdelete_leave_last_pool check_cfs_arcfile nontwrkchk cfsuser masterdir shortermdir nolist sv besc mdump_arclabel back_end_mach with_lsarc mdump_with_lsarc mdump_diag_with_lsarc xcpd_diag_job diag_uxxx model_uxxx hres modver model_version xcpd_with_dumprs xcpd_dumprs_sublist year_offset xref01 xref02 xref03 xref04 xref05 xref06 xref07 xref08 xref09 xref10 xref11 xref12 xref13 xref14 xref15' varlist2='rtrans_remusr rtrans_remserver rtrans_remdir rtrans_force rtrans_scp_args rtrans_ssh_args rtrans_getput rtrans_suffix_list rtrans_uxxx rtrans_prefix rtrans_prefix_ rtrans_suffix_ run_start_year run_start_month run_stop_year run_stop_month diag_job load_hist bemach xcpd_ddump_months xcpd_with_pool xcpd_chunk_size xcpd_first_chunk_size xcpd_with_partial_chunk xcpd_dump_hist xcpd_dump_diag xcpd_dump_pool xcpd_delete_pool xcpd_dummy_diag xcpd_mkdiag_clone xcpd_with_time_series xcpd_transfer_time_series xcpd_RMTRUNPATH xcpd_with_dd xcpd_with_dp xcpd_with_ds xcpd_with_dsd xcpd_with_dsd3D xcpd_gssave xcpd_with_ondisk xcpd_mdump_diag_suffix_list xcpd_diag_suffix_list ondisk_prefix_list ondisk_suffix_list next_block_lock spltdiag_trans spltdiag_remserver spltdiag_remdir spltdiag_vars dtime gptime stime' varlist3='coupled xcpd_front_end xcpd_back_end xcpd_prefix_list xcpd_suffix_list xcpd_with_gz xcpd_with_cp xcpd_with_dcosp xcpd_with_mcosp xcpd_with_radforce xcpd_with_rad_flux_profs xcpd_gpintstat xcpd_gsstats xcpd_xstats xcpd_cmstats xcpd_xtrachem xcpd_xtraconv xcpd_cosp_sim xcpd_cfmip_monthly_rad_force xcpd_daily_cmip5_CanAM4 xcpd_daily_cmip5_cfmip_v2_dsd3D xcpd_daily_cmip5_cosp_v2 xcpd_daily_cmip5 xcpd_type xcpd_with_cnfs xcpd_with_gp6 xcpd_with_xp6 xcpd_with_cc xcpd_with_td xcpd_with_cplhist xcpd_with_cplrs diag2ts_suffix_list diag2ts_with_gztsdiag diag2ts_gztsdiag_deck1 diag2ts_gztsdiag_deck2 diag2ts_delete_time_series diag2ts_dump_time_series diag2ts_one_job_per_suffix xcpd_delete_diag pxdel_lock_check pxdel_check_last_mon del_hist_after_copy stand_alone_craworkval gateway_is_back_end DBG_submit CCCJOB_ROOT JHOME JHOME_DATA JHOME_RUN BERUNPATH' # Create a file containing variable definitions used by the diag job # This eval will protect this crawork def from cccjob substitution eval craworkdef\=crawork\=xcpd_${fid}_$stamp cat > cccjob_defs <> cccjob_defs done for var in $varlist2; do eval val=\$$var # If this variable is defined add it to the list [ -n "$val" ] && echo ${var}=\'$val\' >> cccjob_defs done for var in $varlist3; do eval val=\$$var # If this variable is defined add it to the list [ -n "$val" ] && echo ${var}=\'$val\' >> cccjob_defs done pass_phys_carb=off if [ x$pass_phys_carb = "xon" ]; then for var in PhysA PhysO CarbA CarbO CarbL pass_phys_carb; do eval val=\$$var # If this variable is defined add it to the list [ -n "$val" ] && echo ${var}=\'$val\' >> cccjob_defs done fi echo "cccjob_defs:" [ -s "cccjob_defs" ] && cat cccjob_defs # Create a file containing a list of history file names that are to be # deleted from the back end after the dump completes sucessfully. if [ x"$xcpd_dump_hist" = "xoff" ]; then # Modify pdel_suffix_list to remove any "rs cs os an ab ob" suffixes # so that these files do not get deleted before they can be saved to cfs pdel_suffix_list=`echo $pdel_suffix_list|perl -ne 's/\b(os|rs|cs|ab|an|ob)([+]?[,\d]+)?//g;print'` fi # Leave the last month on disk by shifting the xcpd dates back 1 month pxdel_vtmp=`echo $xcpd_start_year $xcpd_start_mon|\ awk '{m=$2-1;if(m==0){m=12;y=$1-1}else{y=$1}; printf "%2.2d %3.3d",m,y}' -` pxdel_start_mon=`echo $pxdel_vtmp|awk '{printf "%2.2d",$1}' -` pxdel_start_year=`echo $pxdel_vtmp|awk '{printf "%3.3d",$2}' -` pxdel_vtmp=`echo $xcpd_stop_year $xcpd_stop_mon|\ awk '{m=$2-1;if(m==0){m=12;y=$1-1}else{y=$1}; printf "%2.2d %3.3d",m,y}' -` pxdel_end_mon=`echo $pxdel_vtmp|awk '{printf "%2.2d",$1}' -` pxdel_end_year=`echo $pxdel_vtmp|awk '{printf "%3.3d",$2}' -` pxdel_curr_year=`echo $pxdel_start_year|awk '{y=$1-1;printf "%3.3d", y}' -` nfdel=0 while [ $pxdel_curr_year -lt $pxdel_end_year ]; do pxdel_curr_year=`echo $pxdel_curr_year|awk '{printf "%3.3d",$1+1}' -` if [ $pxdel_curr_year -eq $pxdel_start_year ]; then mm=`echo $pxdel_start_mon|awk '{printf "%2.2d",$1-1}' -` else mm=0 fi if [ $pxdel_curr_year -eq $pxdel_end_year ]; then mm_end=$pxdel_end_mon else mm_end=12 fi while [ $mm -lt $mm_end ]; do mm=`echo $mm|awk '{printf "%2.2d",$1+1}' -` bname=${xcpd_prefix}_${pxdel_curr_year}_m${mm}_ for suffix in $pdel_suffix_list; do mlist=`echo $suffix|awk -F'+' '{print $2}' -` mlist=`echo $mlist|sed 's/,/ /g'` if [ -n "$mlist" ]; then suffix=`echo $suffix|sed 's/+.*$//'` # assume that mlist is a white space separated list of numbers # indicating which months to dump for xx in $mlist; do if [ $xx -eq $mm ]; then nfdel=`echo $nfdel|awk '{printf "%d",$1+1}' -` nfdel=`echo $nfdel|sed -e 's/^ *//' -e 's/^0*//'` eval fdel${nfdel}=$bname$suffix fi done else nfdel=`echo $nfdel|awk '{printf "%d",$1+1}' -` nfdel=`echo $nfdel|sed -e 's/^ *//' -e 's/^0*//'` eval fdel${nfdel}=$bname$suffix fi done done done nn=0 rm -f fdel_list touch fdel_list while [ $nn -lt $nfdel ]; do nn=`expr $nn + 1` eval local_fdel=\$fdel$nn [ -z "$local_fdel" ] && bail "Invalid file name in fdel$nn" # Create this list in a format that may be inserted directly into a # a delete job echo " fdel${nn}=$local_fdel" >> fdel_list done # Append the number of files to fdel_list echo " nfdel=$nfdel" >> fdel_list echo "xcpd: fdel_list" cat fdel_list # Require cccjob which cccjob || bail "cccjob is not in your path" # Define the job description JOBDESC="xcpxd=fdel_list:s" # Create the job to be sent to the front end $CCCJOB_ENV cccjob --out=xcpd_${fid}_job --job="$JOBDESC" $start_opt $stop_opt cccjob_defs # Create the lock file for this job [ -z "$xcpd_lock" ] && bail "Lock file name is missing." cp xcpd_${fid}_job $xcpd_lock ######### submit the job ########### # Submit the remote job to the front end rem_mach=${xcpd_front_end:=pollux} # Create a temporary directory in ~/tmp to hold submission files host=`hostname` || bail "Problem in hostname" psubdir=${JHOME:-$HOME}/tmp/tmp_xcpd_${host}_$stamp mkdir -p $psubdir || bail "Cannot create $psubdir" # Submit this job to run on the front end this_remjob="xcpd_${fid}_job" cp $this_remjob $psubdir ssh $rem_mach "cd ${psubdir}; rsub mdest=$rem_mach $this_remjob" 2>&1 || \ bail "***ERROR*** Remote submission failed: rsub mdest=$rem_mach $this_remjob\n" # Clean up the temporary directory rm -fr $psubdir if [ $xcpd_with_rtd -eq 1 ]; then ######### Insert rtd job(s) into the model string ########### # run time diagnostics variables year_rdiag_start='' year_rtdiag_start='' yearfirst='' # These are now defined above # PhysA='' # PhysO='' # CarbA='' # CarbO='' # CarbL='' resolution='' pldir='' rtd_diag_deck='' rtd_plot_deck='' rtd_path='' xcpd_rtd_plot_freq='' if [ $stop_mon -eq 12 ]; then # Execute run time diagnostics at the end of every year # Create a string containing the rtd diag job and possibly the rtd plot job varlist='runid uxxx crawork noprint year_rdiag_start year_rtdiag_start yearfirst PhysA PhysO CarbA CarbO CarbL resolution pldir rtd_diag_deck rtd_plot_deck rtd_path rtd2vic JHOME JHOME_DATA JHOME_RUN CCCJOB_ROOT' # Determine the version number of the current rtd_diag_deck if [ -z "$rtd_diag_deck" ]; then # Use a default value of 65 if unable to determine the version number rtd_version=65 else # Determine the version from the name of the deck # The version number must be integer and at the end of the file name # just before the '.dk' suffix rtd_version=`expr $rtd_diag_deck : '.*[A-Za-z_-]\([0-9][0-9]*\)\.dk$'` [ -z "$rtd_version" ] && rtd_version=65 fi # For versions >= 67 do not plot run time diagnostics with the model # Instead the rtd files will be transferred back to Victoria and plotted there [ $rtd_version -ge 67 ] && eval xcpd_rtd_with_plot\=0 # For version >= 67 always copy rtd file back to lxsrv [ $rtd_version -ge 67 ] && eval rtd2vic\=on # Create a file containing variable definitions used by the rtd job rm -f cccjob_defs touch cccjob_defs for var in $varlist; do eval val=\$$var # If this variable is defined add it to the list [ -n "$val" ] && echo ${var}=\'$val\' >> cccjob_defs done # Create the rtd job string # rtd_with_plot controls the addition of a plotting job to the job string # rtd_plot_freq is the frequency in years at which these plots will be generated rtd_plot_freq=${xcpd_rtd_plot_freq:=2} plot_rtd_year=`echo $stop_year $rtd_plot_freq|awk '{n=$1%$2;printf "%d",n}' -` [ -z "$plot_rtd_year" ] && \ bail "Invalid stop_year = $stop_year or rtd_plot_freq = $rtd_plot_freq" if [ \( $plot_rtd_year -eq 0 -a $stop_year -ne $run_start_year \) \ -o $stop_year -eq $run_stop_year ]; then # Execute the run time diagnostics and run the plot job unless # the user has explicitly set xcpd_rtd_with_plot "off" rtd_with_plot=${xcpd_rtd_with_plot:=1} else # Execute the run time diagnostics without any plotting rtd_with_plot=0 fi if [ $rtd_with_plot -eq 1 ]; then JOBDESC="rtdiag:s rtdplot:s" else JOBDESC="rtdiag:s" fi job_string_to_insert="INSERT_JOB_STRING_${runid}_$stamp" rm -f $job_string_to_insert touch $job_string_to_insert $CCCJOB_ENV cccjob --out=$job_string_to_insert --job="$JOBDESC" \ --start="${start_year}" --stop="${stop_year}" cccjob_defs add_rtd67=off # Do not include a separate job to run rtd67 if the rtd version number is >= 67 [ $rtd_version -ge 67 ] && eval add_rtd67\=off XXX=`echo $add_rtd67|sed 's/ //g'` eval add_rtd67\=$XXX [ "$add_rtd67" = 'on' ] && eval add_rtd67\=1 [ "$add_rtd67" = 'off' ] && eval add_rtd67\=0 [ "$add_rtd67" = 'yes' ] && eval add_rtd67\=1 [ "$add_rtd67" = 'no' ] && eval add_rtd67\=0 if [ $add_rtd67 -eq 1 ]; then # Append an rtdiag67 job that will run after the current rtdiag job varlist='runid uxxx crawork noprint year_rdiag_start year_rtdiag_start yearfirst PhysA PhysO CarbA CarbO CarbL resolution pldir rtd_diag_deck rtd_plot_deck rtd_path rtd2vic JHOME JHOME_DATA JHOME_RUN CCCJOB_ROOT' # Hard code the name of the rtdiag deck to rtdiag67 eval rtd_diag_deck\=rtdiag67.dk # Always copy the rtdiag file back to UVic rtd2vic=on # Create a file containing variable definitions used by the rtd67 job rm -f cccjob_defs touch cccjob_defs for var in $varlist; do eval val=\$$var # If this variable is defined add it to the list [ -n "$val" ] && echo ${var}=\'$val\' >> cccjob_defs done # Create the rdtiag67 job string rtd67_job=tmp_$$_job $CCCJOB_ENV cccjob --out=$rtd67_job --job="rtdiag:s" \ --start="${start_year}" --stop="${stop_year}" cccjob_defs # append the rtdiag67 job string to the previous job that will be # inserted into the exisiting crawork string cat $rtd67_job >> $job_string_to_insert fi # Insert the job just created into the existing string from which this job came # The variable "crawork" must be defined/exported in the current environment splice_job_string $job_string_to_insert >>$error_out 2>&1 ||\ bail "Problem in splice_job_string" rm -f $error_out rm -f $job_string_to_insert else echo "Run time diagnostics is not invoked unless month=12" fi fi # sucessful completion exit 0 # ---Stop_submit_ignore_code---- end_of_script . endjcl.cdk #end_of_job