#!/bin/sh #======================================================================= # Copy a list of files to cfs --- dump_list --- # $Id: dump_list_jobdef 671 2012-05-04 23:35:22Z acrnrls $ #======================================================================= # # keyword :: dump_list # description :: dump a list of files to cfs set -a . betapath2 # * ........................... Parmsub Parameters ............................ # These variables are set when the job string is created previous_year=NotSet # memory99=1 previous_month=NotSet # memory99=1 current_year=NotSet # memory99=1 current_month=NotSet # memory99=1 next_year=NotSet # memory99=1 next_month=NotSet # memory99=1 run_start_year=NotSet # memory99=1 run_start_month=NotSet # memory99=1 run_stop_year=NotSet # memory99=1 run_stop_month=NotSet # memory99=1 runid="job000"; nqsprfx="${runid}_"; nqsext=''; uxxx='uxxx'; model_uxxx=$uxxx; dump_list_uxxx=$model_uxxx; crawork="${runid}_job"; username="acrnxxx"; user="XXX"; jobname=dump_list; stime="1800"; memory1="900mb"; lopgm="lopgm"; noprint=on nextjob=on # Temporary directory where this script will run default_CCRNTMP=$CCRNTMP dump_list_CCRNTMP='' CCRNTMP=${dump_list_CCRNTMP:=''} CCRNTMP=${CCRNTMP:=$default_CCRNTMP} # RUNPATH on execution machine default_RUNPATH=$RUNPATH dump_list_RUNPATH='' RUNPATH=${dump_list_RUNPATH:=''} RUNPATH=${RUNPATH:=$default_RUNPATH} # RMTRUNPATH is used when vic = on is specified default_RMTRUNPATH=$RMTRUNPATH dump_list_RMTRUNPATH='' RMTRUNPATH=${dump_list_RMTRUNPATH:=''} RMTRUNPATH=${RMTRUNPATH:=$default_RMTRUNPATH} # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi dump_cfsuser=$username dump_list_cfsuser=$dump_cfsuser cfsuser=$dump_list_cfsuser dump_masterdir=off dump_list_masterdir=${dump_masterdir:=off} masterdir=${dump_list_masterdir:=off} dump_shortermdir=on dump_list_shortermdir=${dump_shortermdir:=on} shortermdir=${dump_list_shortermdir:=on} # nontwrkchk = yes will turn off the md5sum check on the cfs # (but not on the front end) nontwrkchk='' # rtd parameters # PhysA = on/off physical atmosphere variables # PhysO = on/off physical ocean variables # CarbA = on/off atmosphere carbon variables # CarbO = on/off ocean carbon variables # CarbL = on/off land carbon variables PhysA=on PhysO=on CarbA=on CarbO=on CarbL=on # RHS for certain ocean variables rhs_avg=off # ---Start_submit_ignore_code---- stamp=`date "+%j%H%M%S"$$` # Use -e option if recognized by echo if [ "X`echo -e`" = "X-e" ]; then echo_e() { echo ${1+"$@"}; } else echo_e() { echo -e ${1+"$@"}; } fi # bail is a simple error exit routine # Note: we write the error directly to a file in ~/.queue so that this # info is not lost if/when stdout is not returned this_host=`hostname|cut -d'.' -f1` error_out="${JHOME:-$HOME}/.queue/error_dump_list_${runid}_${this_host}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out bail(){ echo_e `date`" $this_host $runid --- dump_list: $*" echo_e `date`" $this_host $runid --- dump_list: $*" >>$error_out exit 1 } # BERUNPATH must be set if files are to be copied from a machine that is # not the default back end machine bemach='' if [ -z "$bemach" ]; then # If bemach is not set explicitly then see if this job is executing on # a back end machine and use that machine name for bemach this_mach=`uname -n|awk -F\. '{print \$1}' -` on_back_end=0 case $this_mach in c[0-9]*) on_back_end=1 ;; zeta|saiph|spica|hadar) on_back_end=1 ;; za|sa|sp|ha) on_back_end=1 ;; esac if [ $on_back_end -eq 1 ]; then # Set this_mach to its known alias case $this_mach in ha|ha*) this_mach=hadar ;; sp|sp*) this_mach=spica ;; c1*) this_mach=spica ;; c2*) this_mach=hadar ;; za|ze*) this_mach=zeta ;; sa|sa*) this_mach=saiph ;; c6*) this_mach=saiph ;; c7*) this_mach=zeta ;; esac bemach=$this_mach fi fi if [ -n "$bemach" ]; then # Reset BERUNPATH if bemach is set # Ensure bemach contains a known alias case $bemach in ha|ha*) bemach=hadar ;; sp|sp*) bemach=spica ;; za|ze*) bemach=zeta ;; sa|sa*) bemach=saiph ;; esac XXX=`ssh $bemach echo '$RUNPATH' 2>/dev/null` || bail "Unable to determine BERUNPATH" BERUNPATH=$XXX else BERUNPATH=$BERUNPATH fi # If either reset_start_year or reset_stop_year are set then they must be # of the form old_year:new_year (ie a colon separated pair of integers) # where the first integer is the year that needs to be changed # and the second integer is the year that it will be changed to. # These may potentially change the value of start_year or stop_year that # are defined after the call to make_file_name_list below dump_list_reset_start_year='' reset_start_year=${dump_list_reset_start_year:=''} dump_list_reset_stop_year='' reset_stop_year=${dump_list_reset_stop_year:=''} # reset_end_year is defined here for backward compatibility # It is effectively equivalent to reset_stop_year. dump_list_reset_end_year='' reset_end_year=${dump_list_reset_end_year:=''} # This invocation of make_file_name_list will process the *_year and *_months # variables defined above and output a file containing definitions for # start_year, start_mon, stop_year, stop_mon, end_year, end_mon fopts='' dump_list_mon_offset='' if [ -n "$dump_list_mon_offset" ]; then # Set a user supplied month offset eval fopts=\"--mon_offset\=$dump_list_mon_offset\" fi tmp_file_list="tmp_file_list_${runid}_${stamp}" make_file_name_list $fopts --dates_only $tmp_file_list || \ bail "Problem in make_file_name_list" # Verify that the output list is not empty [ ! -s "$tmp_file_list" ] && bail "Unable to create file list" # A file list was created ...source it # This will define start_year, start_mon, stop_year, stop_mon, # end_year and end_mon in the current environment : ; . $tmp_file_list rm -f $tmp_file_list # Define a variable containing a string that will identify the # current year/month range, for possible use in file names etc. ym_range="${start_year}m${start_mon}_${stop_year}m${stop_mon}" # These variables are defined for backward compatability dump_list_start_year=$start_year dump_list_start_mon=$start_mon dump_list_end_year=$stop_year dump_list_end_mon=$stop_mon # arclabel may be used to determine the name of the cmcarc format file # that will be saved on the cfs tape archive. If set then the cmcarc filename # will be ${arclabel}_${timestamp}_arc. If not set then the cmcarc filename # will be ${commonprefix}_${timestamp}_arc where commonprefix is the common # prefix of all of the individual file names of the files that are dumped. dump_list_arclabel='' arclabel=${dump_list_arclabel:-''} # Insert a job that will query the CFS archive database for the names # of the files to be dumped by this tdumper job and abort the string # if any of the file names are not in the database. # This job will run after the current job exits successfully but before # the lock file delete job that was inserted above (if any) runs. dump_list_with_lsarc=''; with_lsarc=${dump_list_with_lsarc:=on} XXX=`echo $with_lsarc|sed 's/ //g'` eval with_lsarc\=$XXX [ "$with_lsarc" = 'on' ] && eval with_lsarc\=1 [ "$with_lsarc" = 'off' ] && eval with_lsarc\=0 [ "$with_lsarc" = 'yes' ] && eval with_lsarc\=1 [ "$with_lsarc" = 'no' ] && eval with_lsarc\=0 # A user supplied file name list, consisting of definitions for file1, file2,... # for $join files, will be inserted here. # <> if [ $with_lsarc -eq 1 ]; then # Create the lsarc job and provide it with the list of file names created above tmp_flst2="${JHOME:-$HOME}/.queue/dump_list_file_list2_${runid}_${stamp}" nn=0 while [ $nn -lt $join ]; do nn=`expr $nn + 1` eval fname=\$file$nn echo "$fname" >> $tmp_flst2 done # Indicate how cccjob should be invoked # Setting CCCJOB_ROOT will allow a job specific version of cccjob to used CCCJOB_ROOT='' if [ -z "$CCCJOB_ROOT" ]; then CCCJOB_ENV='' else eval CCCJOB_ENV=\'env CCCJOB_ROOT\=$CCCJOB_ROOT\' fi # Provide command line defs for JHOME and JHOME_DATA # if these variables are defined in the current env JHOME_opt='' [ -n "$JHOME" ] && JHOME_opt="JHOME=$JHOME " [ -n "$JHOME_DATA" ] && JHOME_opt="$JHOME_opt JHOME_DATA=$JHOME_DATA " [ -n "$JHOME_RUN" ] && JHOME_opt="$JHOME_opt JHOME_RUN=$JHOME_RUN " try_CFSDATA=0 job_string_to_insert="INSERT_JOB_STRING_${runid}_$stamp" rm -f $job_string_to_insert touch $job_string_to_insert $CCCJOB_ENV cccjob --out=$job_string_to_insert --job=lsarc=${tmp_flst2}:s \ --start="${start_year}:${start_mon}" --stop="${stop_year}:${stop_mon}" \ runid=$runid uxxx=$dump_list_uxxx masterdir=$masterdir cfsuser=$cfsuser \ crawork=$crawork noprint=$noprint try_CFSDATA=$try_CFSDATA \ $JHOME_opt >>$error_out 2>&1 rm -f $tmp_flst2 # Insert the job just created into the existing string from which this job came # The variable "crawork" must be defined/exported in the current environment splice_job_string $job_string_to_insert >>$error_out 2>&1 || bail "Problem in splice_job_string" rm -f $job_string_to_insert $error_out fi # ---Stop_submit_ignore_code---- # * ............................ Condef Parameters ............................ # The default if none of sv, both, vic or svvic are set to "on" is to # dump files from the back end machine # Note: Once again submit3 is causing a problem. It will not respect end of line # comments inside its internally created file named job.info. As a result a # construct such as "vic =" found within an end of line comment are flagged as # syntax errors even though they are commented out. # sv : on means dump files from pollux dump_sv=off dump_list_sv=${dump_sv:=off} sv=${dump_list_sv:=off} # svsave : on means save files on the front end when dumping # from the back end or from UVic dump_svsave=off dump_list_svsave=${dump_svsave:=off} svsave=${dump_list_svsave:=off} both=off # both : on means dump files from pollux and the back end machine vic=off # vic : on means dump files from Victoria svvic=off # svvic : on means dump files from pollux and Victoria # nolist : off means save CFSDATA* files in ~/info # nolist : on means do not save CFSDATA* files in ~/info dump_nolist=off dump_list_nolist=${dump_nolist:=off} nolist=${dump_list_nolist:=off} # rmdskcpy = on means remove the copy of the file on the cfs disk buffer # as soon as it has been written to tape dump_list_rmdskcpy='' rmdskcpy=${dump_list_rmdskcpy:=off} # cnfsallow = on will allow the user to use a cnfs dir for # RUNPATH and/or CCRNTMP when svsave = on # Normally this is not allowed because of the possibility that files # may get corrupted during transfer from/to the nfs file system dump_list_cnfsallow='' cnfsallow=${dump_list_cnfsallow:=off} # dpalist = on means store DATAPATH information in a local file for use by access/save/delete # Since this local file is never deleted, dpalist = on should only be used when access, # save or delete are part of a job that runs in a temporary working directory which gets # removed after the job completes. Most decks do this, but not all. You have been warned! dump_list_dpalist='' dpalist=${dump_list_dpalist:=''} # besc = on causes a tdumper job to run on the back end (spica/hadar) # These definitions need to be on the same line to work around an error in the submit scripts dump_list_besc=''; besc=${dump_list_besc:=off} # * ............................. Deck Definition ............................. if [ 2 -eq 1 ]; then n=0 while [ $n -lt $join ]; do n=`expr $n + 1` eval echo "dump: file$n = \$file$n" done fi # Dump the files defined above . tdumper.dk #end_of_job