#! /bin/sh

#    May 31/2004 - F.Majaess

#id arc_mak - Creates "arcfile" from individual filenames list.

#   AUTHOR  - F.Majaess

#hd PURPOSE - "arc_mak" code is used to handle creating the "arcfile"
#hd           from the individual files.
#hd           Note: The script is not intended to be called directly by 
#hd                 the user, instead it's to be called by higher level
#hd                 scripts.

IFS=' '
export IFS
# set -x

#  * Set variable 'AWK' to the version of "awk" to be used.

AWK=${AWK:='awk'}

# echo "arc_mak start:"
if [ -z "$file_list" ] ; then
 echo "" ; echo "  arc_mak: no archive file list is found"
 exit 1
fi
# echo "file_list=$file_list"

# Set needed parameters...

Lgname=${Lgname:-`whoami`}
if [ "$SITE_ID" = 'DrvlSC' ] ; then
 LgnameEC=`accmap $Lgname`
 ArcSzlmt=17179869184 # <-- 16GB
 Arcmnsz=524288000  # <-- 500MB
else
 LgnameEC="$Lgname"
 # ArcSzlmt=6000000000
 # ArcSzlmt=8000000000
 # ArcSzlmt=9000000000
 # ArcSzlmt=9500000000
 #ArcSzlmt=10000000000
 #ArcSzlmt=10737418240 # <-- 10GB
 #ArcSzlmt=11274289150 # <-- 10.5GB
 #ArcSzlmt=11811160060 # <-- 11GB
 #ArcSzlmt=12348030976 # <-- 11.5GB
 #ArcSzlmt=12884901888 # <-- 12GB
  ArcSzlmt=17179869184 # <-- 16GB
 #ArcSzlmt=12000000000
 #ArcSzlmt=20000000000
   Arcmnsz=1000000000
fi
usr=`echo $LgnameEC | sed -e 's/^....//'`

# Disable "arc" mode if only a single individual file is specified.

Nfiles=0
for fname in $file_list
do
 Nfiles=`expr $Nfiles + 1`
done
if [ "$makearc" = 'yes' -a "$Nfiles" -le 1  ] ; then
 if [ -n "$arclabel" ] ; then
  echo "arc_mak: Only one file is specified, arclabel=$arclabel setting will be ignored"
 else
  echo "arc_mak: Only one file is specified!"
 fi
 echo "         $file_list will be archived directly on the archive server."
 unset makearc arclabel arcfile 
fi
# Proceed with creating the "arcfile" file from idividual files.

if [ "$makearc" = 'yes' ] ; then

 # Setup and switch to temporary subdirectory...

 Date=`date -u '+%Y%j'`
 Time=`date -u '+%H%M%S'`
 # Tdatetime=`date -u '+%Y%j%H%M%S'`
 Tdatetime="${Date}${Time}"
 Tmpcmcarcdir="tmp_${usr}_cmcarc_dir_${Tdatetime}_$$"
 Prevwd=`pwd`
 mkdir -m 755 $Tmpcmcarcdir
 if [ ! -d "$Tmpcmcarcdir/." ] ; then
  echo "arc_mak: Not able to create temporary subdirectory to make arcfile!"
  exit 1
 fi
 cd $Tmpcmcarcdir

 unset Tlist
 # set -x
 for fname in $file_list
 do
 
  # unset link 
  # link=`file $InfoH $fname 2>/dev/null | fgrep link` 
  # if [ -s "$fname" -o -n "$link" ] ; then
 
   #  * Extract name and path information from the source filename. The
   #  * default directory containing the source file is set to the
   #  * directory from which the script was invoked.
 
   # path=`expr $fname : '\(.*\)/'` 
   # name=`expr //$fname : '.*/\(.*\)'` 
 
   # path=${path:=$cwd} 
   
   name=`basename $fname`
 
   #  * Setup for filename on the archive server under which the 
   #  * transfered file get saved.
 
   rname=`expr "$name" : '\(.*\)\.[0-9][0-9][0-9]'` 
   rname=${rname:=$name} 
   rname=`echo $rname | tr '\*' '_'` 
   rname=`echo $rname | tr '[A-Z]' '[a-z]'` 

   FullPth=`dirname $fname | sed -e 's/^\/.*$/yes/'`
   if [ "$FullPth" = 'yes' ] ; then
    ln -s $fname $rname
   else
    ln -s $Prevwd/$fname $rname
   fi
   # Tlist=`clnstrng "$Tlist $rname" ' '`
   Tlist="$Tlist $rname"
  # fi
 done
 if [ -n "$Tlist" ] ; then Tlist=`clnstrng "$Tlist" ' '` ; fi

 # Map into "arcfile" variable the "arcfile" filename based on the
 # individual filenames whose list is in "Tlist" variable...

 # set -x
 . arc_fnm
 
 # Ensure "arcfile" name is not already in use...

 # Global "active" full search ...

 # Arc_conflict=`arc_lst full sngl $arcfile location`

 # Restricted search based on masterdir/shorterm switches setting ...

 Arc_conflict=`arc_lst own sngl $arcfile location`

 # Arc_conflict=`basename $Arc_conflict`
 # if [ "$Arc_conflict" = "$arcfile" ] ; then
 if [ -n "$Arc_conflict" ] ; then 
  Arc_conflict=`echo $Arc_conflict | sed -e 's/^.*\/active\///g'`
  echo "arc_mak: Sorry, not able to proceed due to:"
  echo "  $arcfile filename conflict with ...${Arc_conflict}"
  cd $Prevwd
  rm -r -f $Tmpcmcarcdir
  exit 1
 fi
 
 i=0
 Cache_sync_filek="./.arc_mak_cache_sync_""`date -u '+%Y%j%H%M%S%s'`""_$$"
 # HOSTIDf=${HOSTIDf:-`echo $HOSTID | cut -c 1-3`}
 HOSTIDf=`echo $HOSTID | sed -e 's/eccc.*-ppp/cs/g' -e 's/^ppp/cs/g' | cut -c 1-3`
 KCwd=`pwd` 
 KCnFs='no' 
 KCeTs='no' 
 if [ "$SITE_ID" != 'DrvlSC' ] ; then
  KCnFs=`echo $KCwd | sed -e 's/^\/cnfs\/.*$/yes/'` 
  KCeTs=`echo $KCwd | sed -e 's/^\/fs\/cetus\/.*$/yes/'` 
 ########### ENABLE LATER ON
  # if [ "$KCnFs" = 'yes' -a "$OS" != 'AIX' -a "$HOSTIDf" != 'cnf' ] ; then
  if [ "$OS" != 'AIX' -a \( \( "$KCnFs" = 'yes' -a "$HOSTIDf" != 'cnf' \) -o \( "$KCeTs" = 'yes' -a "$HOSTIDf" != 'cet' \) \) ] ; then
   nochksum='no'
  fi 
 fi
 # cmcarc_ver=`cmcarc -h | sed -n -e '/^cmcarc version /p' | $AWK '{print $3}'`
 cmcarc_ver=`cmcarc.64 -h | sed -n -e '/^cmcarc version /p' | $AWK '{print $3}'`
 if [ "$nochksum" != 'yes' ] ; then
  md5sum_ver=`md5sum --version | head -1 | $AWK '{print $NF}'`
  if [ -s "../.transfer_md5sum" ] ; then
   echo "" ; echo " md5sum of transferred files:"
             echo " ---------------------------"
   cat ../.transfer_md5sum  | sort -k2,2 | sed -e 's/^/ /'
   echo ""
  fi
 fi
 for rname in $Tlist
  do
   # Accumulate individual file in "arcfile".

   i=`expr $i + 1`
   Sz=`ls -ldL $rname | $AWK '{ print \$5 ; } '`
   if [ "$nochksum" != 'yes' ] ; then
    if [ -s "../.transfer_md5sum" ] ; then
     set -x
     FngrPrntfr=`cat ../.transfer_md5sum | sed -n -e "/ $rname"' *$/p' | $AWK '{ print \$1 ;}'`
     if [ -n "$FngrPrntfr" -a \( "$KCnFs" != 'yes' -o \( "$KCnFs" = 'yes' -a "$HOSTIDf" = 'cnf' \) \) \
                           -a \( "$KCeTs" != 'yes' -o \( "$KCeTs" = 'yes' -a "$HOSTIDf" = 'cet' \) \) ] ; then
      FngrPrnt="$FngrPrntfr"
     else
      FngrPrnt=`md5sum $rname | $AWK '{ print \$1 ; } '`
     fi
     if [ -n "$FngrPrntfr" ] ; then
      if [ "$FngrPrntfr" != "$FngrPrnt" ] ; then
       if [ "$KCnFs" = 'yes' -a "$HOSTIDf" != 'cnf' ] ; then
        KRmtsumckcnfs=`ssh datasvr md5sum $KCwd/$rname 2>/dev/null `
        KRmtsumcnfs=`echo $KRmtsumckcnfs | $AWK '{ print \$1 ; }' `
       elif [ "$KCeTs" = 'yes' -a "$HOSTIDf" != 'cet' ] ; then
        KRmtsumckcnfs=`ssh cetus md5sum $KCwd/$rname 2>/dev/null `
        KRmtsumcnfs=`echo $KRmtsumckcnfs | $AWK '{ print \$1 ; }' `
       else
        KRmtsumcnfs='N/A'
       fi 
       if [ -f "$CCRNSRC/info/.cfs_problems_log" ] ; then
         echo "ARC_MAK: `whoami`,`date`,`hostname`,......-$$; 1st check failed; mismatched md5sum for `pwd`/$rname ; FngrPrntfr=$FngrPrntfr ; FngrPrnt=$FngrPrnt; KRmtsumcnfs=$KRmtsumcnfs" >> $CCRNSRC/info/.cfs_problems_log
       fi
       (\rm -f ${Cache_sync_filek} || : ) && echo 'Sync' > ${Cache_sync_filek} && (\rm -f ${Cache_sync_filek} || : ) && sleep 5 || :
       cd $KCwd 
       FngrPrnt=`md5sum $rname | $AWK '{ print \$1 ; } '`
       if [ "$FngrPrntfr" != "$FngrPrnt" ] ; then
        if [ -f "$CCRNSRC/info/.cfs_problems_log" ] ; then
         echo "ARC_MAK: `whoami`,`date`,`hostname`,......-$$; 2nd check failed; mismatched md5sum for `pwd`/$rname ; FngrPrntfr=$FngrPrntfr ; FngrPrnt=$FngrPrnt; KRmtsumcnfs=$KRmtsumcnfs" >> $CCRNSRC/info/.cfs_problems_log
        fi
        (\rm -f ${Cache_sync_filek} || : ) && echo 'Sync' > ${Cache_sync_filek} && (\rm -f ${Cache_sync_filek} || : ) && sleep 5 || :
        cd $KCwd 
        FngrPrnt=`md5sum $rname | $AWK '{ print \$1 ; } '`
        if [ "$FngrPrntfr" != "$FngrPrnt" ] ; then
         echo "ARC_MAK: `whoami`,`date`,`hostname`,SEVERE-$$; mismatched md5sum for `pwd`/$rname ; FngrPrntfr=$FngrPrntfr ; FngrPrnt=$FngrPrnt ; KRmtsumcnfs=$KRmtsumcnfs" 
         if [ -f "$CCRNSRC/info/.cfs_problems_log" ] ; then
          echo "ARC_MAK: `whoami`,`date`,`hostname`,SEVERE-$$; mismatched md5sum for `pwd`/$rname ; FngrPrntfr=$FngrPrntfr ; FngrPrnt=$FngrPrnt; KRmtsumcnfs=$KRmtsumcnfs" >> $CCRNSRC/info/.cfs_problems_log
         fi
         exit 1
        fi
       fi
      fi
     fi
     set +x
    else
     FngrPrnt=`md5sum $rname | $AWK '{ print \$1 ; } '`
    fi
    # cmcarc -f $arcfile --md5 -p -a $rname
    cmcarc.64 -f $arcfile --md5 -p -a $rname
    # Line=`echo "$rname $Date $Time $arcfile $i $cmcarc_ver $Sz" | $AWK '{printf "%-40s%1s%8d%6.6d%1s%-40s%1s%4.3d%1s%4s%1s%12d\n",$1," ",$2,$3," ",$4," ",$5," ",$6," ",$7}'`
    # Line=`echo "$rname $Date $Time $arcfile $i $cmcarc_ver $Sz" | $AWK '{printf "%-40s%1s%8d%6.6d%1s%-40s%1s%4.3d%1s%4s%1s%12s\n",$1," ",$2,$3," ",$4," ",$5," ",$6," ",$7}'`
    Line=`echo "$rname $Date $Time $arcfile $i $cmcarc_ver $Sz $md5sum_ver $FngrPrnt" | $AWK '{printf "%-40s%1s%8d%6.6d%1s%-40s%1s%4.3d%1s%4s%1s%12s%1s%7s%1s%32s\n",$1," ",$2,$3," ",$4," ",$5," ",$6," ",$7," ",$8," ",$9}'`
   else
    # cmcarc -f $arcfile -p -a $rname
    # cmcarc.64 -f $arcfile -p -a $rname
    cmcarc.64 -f $arcfile --md5 -p -a $rname
    if [ -s "../.transfer_md5sum" ] ; then
     FngrPrnt=`cat ../.transfer_md5sum | sed -n -e "/ $rname"' *$/p' | $AWK '{ print \$1 ;}'`
     if [ -n "$FngrPrnt" ] ; then
      md5sum_ver=${md5sum_ver:-`md5sum --version | head -1 | $AWK '{print $NF}'`}
      Line=`echo "$rname $Date $Time $arcfile $i $cmcarc_ver $Sz $md5sum_ver $FngrPrnt" | $AWK '{printf "%-40s%1s%8d%6.6d%1s%-40s%1s%4.3d%1s%4s%1s%12s%1s%7s%1s%32s\n",$1," ",$2,$3," ",$4," ",$5," ",$6," ",$7," ",$8," ",$9}'`
     fi
    else 
     Line=`echo "$rname $Date $Time $arcfile $i $cmcarc_ver $Sz" | $AWK '{printf "%-40s%1s%8d%6.6d%1s%-40s%1s%4.3d%1s%4s%1s%12s\n",$1," ",$2,$3," ",$4," ",$5," ",$6," ",$7}'`
    fi
   fi
   touch $arcfile_list
   echo "$Line" >> $arcfile_list
   touch .cmcarc_xtrct_verify
   echo "$rname $Sz" | $AWK '{printf "%-40s%1s%12s\n",$1," ",$2}' >> .cmcarc_xtrct_verify
  done
 echo " " ; echo "List of --> $arcfile <-- table of content:" ; echo " "
 # cmcarc -f $arcfile -t -v
 # cmcarc.64 -f $arcfile -t -v
 cmcarc.64 -f $arcfile -t -v | tee .cmcarc_list_cptr | cat

 echo " "

 # Ensure all individual files made it into the "cmcarc" file.
 cat .cmcarc_list_cptr | sed -n -e '/^ *$/d' -e '3,$p' | $AWK '{printf "%-40s%1s%12s\n",$9," ",$4}' > .invdl_flist_verify
 # diff -q -w .cmcarc_xtrct_verify .invdl_flist_verify >> /dev/null 2>>/dev/null && unset arcmk_chkpss || arcmk_chkpss='no'
 arcmk_chkpss='yes'
 diff -w .cmcarc_xtrct_verify .invdl_flist_verify >> /dev/null 2>>/dev/null && unset arcmk_chkpss || arcmk_chkpss='no'
 
 if [ "$arcmk_chkpss" = 'no' ] ; then

   if [ -f "$CCRNSRC/info/.cfs_problems_log" ] ; then
     echo "ARC_MAK: `whoami`,`date`,`hostname`,SEVERE-$$; INCOMPLETE ARCHIVAL is detected, POSSIBLY DUE TO LACK OF DISK SPACE ; `pwd`/$arcfile " >> $CCRNSRC/info/.cfs_problems_log
   fi
   echo "" ; echo "" ; echo "  arc_mak: INCOMPLETE ARCHIVAL is detected, POSSIBLY DUE TO LACK OF DISK SPACE!"
   echo "                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ "
   echo "" ; echo "           Current partition usage:"
   echo "           ======================= " ; echo ""
   df -k ./ | sed -e 's/^/   /'
   echo "" ; echo "" ; echo "           What is expected based on the following list of filename/size:"
   echo "           ------------------------------------------------------------- " ; echo ""
   cat .cmcarc_xtrct_verify | sed -e 's/^/           /'
   echo "" ; echo "           What is extracted from <cmcarc file> listing:"
   echo "           -------------------------------------------- " ; echo ""
   cat .invdl_flist_verify | sed -e 's/^/           /'
   echo "" ; echo "           Differences consisted of:"
   echo "           ------------------------ " ; echo ""
   diff -w .cmcarc_xtrct_verify .invdl_flist_verify | sed -e 's/^/           /'
   (echo "" ; echo "=============================================================== " ; echo "") | sed -e 's/^/   /'
   cd $Prevwd
   rm -r -f $Tmpcmcarcdir
   exit 1

 fi

 if [ "$nochksum" != 'yes' ] ; then
  # Ensure all "md5sum" fingerprints are matching their counterparts in the "cmcarc" file.
  cmcarc.64 -f $arcfile -t -V | tee .cmcarc_list_cptr2 | cat
  cat .cmcarc_list_cptr2 | sed -n -e '/^ *$/d' -e '3,$p' | sed -n -e '/^----/,$d' -e '1,$p' |  $AWK '{printf "%-32s%1s%-40s\n",$11," ",$12}' > .cmcarc_xtrct_verify2
  cat $arcfile_list | $AWK '{printf "%-32s%1s%-40s\n",$8," ",$1}' > .invdl_flist_verify2
  arcmk_chkpss='yes'
  diff -w .invdl_flist_verify2 .cmcarc_xtrct_verify2 >> /dev/null 2>>/dev/null && unset arcmk_chkpss || arcmk_chkpss='no'
  
  if [ "$arcmk_chkpss" = 'no' ] ; then
 
    echo "" ; echo "ARC_MAK: `whoami`,`date`,`hostname`,SEVERE-$$; failed cmcarc md5sum check; `pwd`/$arcfile !"  ; echo " "
    if [ -f "$CCRNSRC/info/.cfs_problems_log" ] ; then
     echo "ARC_MAK: `whoami`,`date`,`hostname`,SEVERE-$$; failed cmcarc md5sum check; `pwd`/$arcfile !" >> $CCRNSRC/info/.cfs_problems_log
    fi
    echo "" ; echo "" ; echo "  arc_mak: INCOMPLETE ARCHIVAL is detected, POSSIBLY DUE TO LACK OF DISK SPACE or other PARTITION PROBLEM!"
                        echo "                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^          ^^^^^^^^^^^^^^^^^"
    echo "" ; echo "           Current partition usage:"
    echo "           ======================= " ; echo ""
    df -k ./ | sed -e 's/^/   /'
    echo "" ; echo "" ; echo "           What is generated for md5sum check for the filename(s):"
                        echo "           ------------------------------------------------------ " ; echo ""
    cat .invdl_flist_verify2 | sed -e 's/^/           /'
    echo "" ; echo "           What is extracted for md5sum check from <cmcarc file> listing:"
    echo "           ------------------------------------------------------------- " ; echo ""
    cat .cmcarc_xtrct_verify2 | sed -e 's/^/           /'
    echo "" ; echo "           Differences consisted of:"
    echo "           ------------------------ " ; echo ""
    diff -w .invdl_flist_verify2 .cmcarc_xtrct_verify2 | sed -e 's/^/           /'
    (echo "" ; echo "=============================================================== " ; echo "") | sed -e 's/^/   /'
    cd $Prevwd
    rm -r -f $Tmpcmcarcdir
    exit 2
 
  fi
 fi

 # Abort if the "arc" file size exceeds the allowable set limit.
 ArcflSz=`\ls -ld $arcfile | $AWK '{print $5}'`
 ArcSzchk=`echo ArcflSz ArcSzlmt | $AWK -v ArcflSz=$ArcflSz -v ArcSzlmt=$ArcSzlmt ' { if ( ArcflSz <= ArcSzlmt ) { print "ok" ; } else { print "notok" ; } ; }'`
 if [ "$ArcSzchk" != "ok" -a "$arc_large_size_permit" != 'on' ] ; then 
  echo "arc_mak: Sorry, not able to proceed due to $arcfile file size "
  echo "         of $ArcflSz exceeding the maximum allowable limit "
  echo "         of $ArcSzlmt bytes!"
  cd $Prevwd
  rm -r -f $Tmpcmcarcdir
  exit 1
 fi
 if [ "$ArcSzchk" = "ok" -a "$arc_large_size_permit" = 'on' ] ; then 
  echo "arc_mak: Sorry, not able to proceed since you are misusing arc_large_size_permit=$arc_large_size_permit"
  echo "         where, the $arcfile file size of $ArcflSz is still below the maximum allowable limit "
  echo "         of $ArcSzlmt bytes!"
  echo "         Reverse arc_large_size_permit setting and re-submit"
  cd $Prevwd
  rm -r -f $Tmpcmcarcdir
  exit 1
 fi

 # Check if "arc" file size is low...
 ArcSzchk=`echo ArcflSz Arcmnsz | $AWK -v ArcflSz=$ArcflSz -v Arcmnsz=$Arcmnsz ' { if ( ArcflSz >= Arcmnsz ) { print "ok" ; } else { print "notok" ; } ; }'`
 if [ "$ArcSzchk" != "ok" ] ; then 
  if [ "$SITE_ID" = 'DrvlSC' ] ; then
   if [ -f "$CCRNSRC/arc_dir/log/arc_low_file_size_log" ] ; then
    echo " SEVERE: $Lgname ${Date}${Time} Nfiles=$Nfiles Arcmnsz=$Arcmnsz ArcflSz=$ArcflSz arcfile=$arcfile" >> $CCRNSRC/arc_dir/log/arc_low_file_size_log
   fi
   echo "arc_mak: Sorry, not able to proceed due to $arcfile file size "
   echo "         of $ArcflSz below the set minimum of $Arcmnsz bytes!"
   cd $Prevwd
   rm -r -f $Tmpcmcarcdir
   exit 1
  else
   if [ "$Nfiles" -lt 12 ] ; then

    if [ -f "$CCRNSRC/arc_dir/log/arc_low_file_size_log" ] ; then
     echo " SEVERE: $Lgname ${Date}${Time} Nfiles=$Nfiles Arcmnsz=$Arcmnsz ArcflSz=$ArcflSz arcfile=$arcfile" >> $CCRNSRC/arc_dir/log/arc_low_file_size_log
    fi

    echo "arc_mak: Warning, $arcfile file size of $ArcflSz is low!"
    echo "         Can you please aim for more individual files,"
    echo "         and a higher _arc file size in future archival requests."

   else

    if [ -f "$CCRNSRC/arc_dir/log/arc_low_file_size_log" ] ; then
     echo " ALERT : $Lgname ${Date}${Time} Nfiles=$Nfiles Arcmnsz=$Arcmnsz ArcflSz=$ArcflSz arcfile=$arcfile" >> $CCRNSRC/arc_dir/log/arc_low_file_size_log
    fi

    echo "arc_mak: Warning, $arcfile file size of $ArcflSz is low!"
    echo "         Can you please aim for a higher _arc file size in future archival requests."

   fi
  fi
 fi

 # Relocate generated files into "mother" subdirectory and
 # then delete the temporary subdirectory.

 (rm -f $Prevwd/$arcfile || : )
 (rm -f $Prevwd/$arcfile_list || : )
 mv $arcfile $arcfile_list $Prevwd
 cd $Prevwd
 rm -r -f $Tmpcmcarcdir
 echo "" ; echo "old set of files to archive consisted of:" ; echo ""
 echo " $file_list"
 file_list="$arcfile"
 Tlist="$Tlist $arcfile"
 echo "" ; echo "New list of files to archive consists of the single file: $file_list " ; echo ""
 \ls -ld $arcfile
 echo ""
fi
# echo "arc_mak exit:"