#!/bin/sh #======================================================================= # Verify that a set of files is on disk --- ondisk --- # $Id: ondisk_jobdef 655 2011-08-04 23:04:36Z acrnrls $ #======================================================================= # # Larry Solheim May,2011 #======================================================================= # # keyword :: ondisk # description :: verify that a set of files is on disk # set -a . betapath2 # * ........................... Parmsub Parameters ............................ runid=job000; uxxx=uxxx; ondisk_uxxx=$uxxx; username=acrnxxx; user=MYNAME; crawork=ondisk_job nextjob=on noprint=on debug=off stime=${stime:=7200}; memory1=${memory1:=1000mb} jobname=ondisk; time=$stime ; memory=1000mb; # Alternate path to a directory where .queue/.crawork will be found JHOME='' if [ -n "$JHOME" -a x"$JHOME" != x"$HOME" ]; then # Allow optional reset of DATAPATH/RUNPATH JHOME_DATA='' DATAPATH=${JHOME_DATA:=$DATAPATH} RUNPATH=${JHOME_DATA:=$RUNPATH} # Allow optional reset of CCRNTMP JHOME_RUN='' CCRNTMP=${JHOME_RUN:=$CCRNTMP} fi . comjcl.cdk cat > Execute_Script <<'end_of_script' # Certain variables need to be exported so that # they are visible to make_file_name_list set -a # ---Start_submit_ignore_code---- # These variables are set when the job string is created previous_year=NotSet previous_month=NotSet current_year=NotSet current_month=NotSet next_year=NotSet next_month=NotSet run_start_year=NotSet run_start_month=NotSet run_stop_year=NotSet run_stop_month=NotSet runid=job000; uxxx=uxxx; ondisk_uxxx=$uxxx; stamp=`date "+%j%H%M%S"$$` # Use -e option if recognized by echo if [ "X`echo -e`" = "X-e" ]; then echo_e() { echo ${1+"$@"}; } else echo_e() { echo -e ${1+"$@"}; } fi # bail is a simple error exit routine # Note: we write the error directly to a file in ~/.queue so that this # info is not lost if/when stdout is not returned error_out="${JHOME:-$HOME}/.queue/error_ondisk_${runid}_$stamp" [ ! -z "$error_out" ] && rm -f $error_out bail(){ echo_e `date`" --- ondisk: $*" echo_e `date`" --- ondisk: $*" >>$error_out exit 1 } # If either reset_start_year or reset_stop_year are set then they must be # of the form old_year:new_year (ie a colon separated pair of integers) # where the first integer is the year that needs to be changed # and the second integer is the year that it will be changed to. # These may potentially change the value of start_year or stop_year that # are defined after the call to make_file_name_list below ondisk_reset_start_year='' reset_start_year=${ondisk_reset_start_year:=''} ondisk_reset_end_year='' ondisk_reset_stop_year=$ondisk_reset_end_year reset_stop_year=${ondisk_reset_stop_year:=''} # prefix_list and suffix_list are used by make_file_name_list, together with # other variables from th ecurrent environment, to build a list of files names ondisk_suffix_list='' suffix_list="${ondisk_suffix_list:-gs ss}" ondisk_prefix_list='' prefix_list="${ondisk_prefix_list:-$ondisk_uxxx}" # Create a file containing a list of file names that may then be # "sourced" in the current environment to define the variables # file1, file2,..., file$join, join. These variables are used by # tdumper to compile the list of files to be archived. join=0 # make_file_name_list uses the variables current_year, current_month, # previous_year, previous_month, next_year and next_month to # determine start and stop dates for file name creation. # It also uses runid, prefix_list and suffix_list from the current # environment to build these file names. # Allow user supplied command line options for make_file_name_list # The following invocation of make_file_name_list will not allow multi-list # output so if any command line option is supplied that will turn on # multi-list output (e.g. --months_max=... --size_max=... --number_max=..) # then this script will abort. ondisk_file_list_opts='' fopts="${ondisk_file_list_opts:-}" ondisk_mon_offset='' if [ -n "$ondisk_mon_offset" ]; then # Set a user supplied month offset eval fopts=\"$fopts --mon_offset\=$ondisk_mon_offset\" fi # Create a temporary file containing the file list tmp_file_list="./ondisk_file_list_${runid}_${stamp}" make_file_name_list $fopts --nomulti_list $tmp_file_list >>$error_out 2>&1 ||\ bail "Problem in make_file_name_list" rm -f $error_out [ ! -s "$tmp_file_list" ] && bail "Unable to create file list" # A file list was created ...source it : ; . $tmp_file_list rm -f $tmp_file_list # At this point file1, file2,... are defined in the current environment # as well as certain other variables such as start_year, start_mon, # stop_year and stop_mon which correspond to the start and stop dates # for the file names that were created. # These are defined for backward compatability ondisk_start_year=$start_year ondisk_start_mon=$start_mon ondisk_end_year=$stop_year ondisk_end_mon=$stop_mon # ondisk_comment, if set, will be written to stdout if the job aborts ondisk_comment='' # Pause for 2 minutes to allow the file system to catch up sleep 120 # Loop over all files and verify that they exist on disk missing=0 nfile=0 while [ $nfile -le $join ]; do nfile=`expr $nfile + 1` eval curr_file=\$file$nfile present=`ls -1 $DATAPATH/${curr_file}.[0-9][0-9][0-9] 2>/dev/null | tail -1` present=`echo $present|sed 's/\.[0-9][0-9][0-9]$//;s/^.*\///'` if [ x"$present" = x"$curr_file" ]; then # The current file is on disk eval miss${nfile}=0 else # The current file is not on disk missing=`expr $missing + 1` eval miss${nfile}=1 fi done if [ $missing -gt 0 ]; then # There is at least 1 missing file [ -n "$ondisk_comment" ] && echo "$ondisk_comment" if [ $missing -eq 1 ]; then echo "$missing file was missing from disk at run time: "`date` else echo "$missing files were missing from disk at run time: "`date` fi nfile=0 nmiss=0 while [ $nfile -le $join ]; do nfile=`expr $nfile + 1` eval amiss=\$miss$nfile if [ $amiss -eq 1 ]; then nmiss=`echo $nmiss|awk '{printf "%4d",$1+1}' -` eval curr_file=\$file$nfile echo " $nmiss MISSING: $curr_file" fi done exit 1 fi # ---Stop_submit_ignore_code---- end_of_script . endjcl.cdk #end_of_job