#! /bin/sh
# set -x
# "compile_CanDIAG_banting" to be used to generate archive libraries
# and executable binaries under
# RTEXTBLS (="$RUNPATH_ROOT/${runid}/code/executables").
# "compile_CanDIAG_banting" must be invoked from within "tools"
# subdirectory on Brooks.
# NOTE: Added generation of "openmp" archive libraries due to "lpcrev2"
#       program issue.


echo "==== Start of generating archive libraries and executable binaries at - `date`"


## Parameters controlling

# check if a ref dir has been provided in env_setup_file
#  - if not, execute full compilation
#  - if yes, compare source code
if [ -z "$CANDIAG_REF_PATH" ]; then
  comp_full="yes"
else
  comp_full="no"
fi

# run ID:
# runid=${runid:-'gcm18_02'} ; export runid

if [ -n "$runid_env" -a -n "$runid" -a "$runid_env" != "$runid" ] ; then
  echo "compile_CanDIAG: Mismatched runid=$runid and runid_env=$runid_env !"
  exit 1
fi

# CCRNSRC=${CCRNSRC:-"${RUNID_ROOT}/code"}
CCRNSRC=${CCRNSRC:-"${RUNID_ROOT}"}
SRC_ROOT="/space/hall3/work/eccc/crd/btd_src"

# If "model_float1_mode=yes"; it leads to creating libraries and
# executables in support of running the model simulation in float1
# mode:
model_float1_mode=yes # float1 and float2 mode
model_float1_mode=${model_float1_mode:-'no'} # Default; just float2 mode
export model_float1_mode

# Plotting programs to be
plot_pgms=yes # generated
plot_pgms=${plot_pgms:-'no'} # skipped (default)
plot_pgms=no # Disable for now due to failures in building
export plot_pgms

# If "openmp_mode=yes"; it leads to creating "openmp" libraries.
# Presently, that only needed to produce "lpcrev2" program executable.
openmp_mode=yes # float1 and float2 mode
openmp_mode=${openmp_mode:-'no'} # Default; just float2 mode
export openmp_mode

# Setup to generate the archive libraries in interactive mode, serially
# and in the foreground:
febg_wait_option='febg=yes wait=yes'
export febg_wait_option

# Setup needed environment

if [ -s "${CCRNSRC}/generic/env_setup_file" ] ; then
  . ${CCRNSRC}/generic/env_setup_file
else
  echo "Sorry; missing ${CCRNSRC}/generic/env_setup_file file!"
  echo " compile_CanDIAG must be invoked from within tools subdirectory"
  exit 1
fi
OUTPUTQ=${ROUTPUTQ:-"$HOME/.queue"} ; export OUTPUTQ
#set -x
# Make sure "RTEXTBLS" is pointing to the valid subdirectory.
# Create it if necessary.


if [ ! -d "$RTEXTBLS/." ] ; then
  echo "Attempting creation of $RTEXTBLS subdirectory"
  mkdir -m 1755 $RTEXTBLS
fi
if [ ! -d "$RTEXTBLS/." ] ; then
  echo "Sorry; invalid $RTEXTBLS !"
  exit 1
fi

echo " " ; echo "=== Executables subdirectory: $RTEXTBLS" ; echo " "

# Define parameters used in compilation
# They may be skipped if relying on "generic/env_setup_file" settings

#-----------
EVF90=${EVF90:-'ftn'} ; export EVF90
F77_float2="${EVF90} -r8 -i8 -nbs -WB -g -O2 -traceback -m64 -static-intel -mavx -fixed -mp1 -fp-model strict -static -Bstatic  -fpe0 -ftrapuv" ; export F77_float2
LOCOMM_float2="$RTEXTBLS/libLOSUB_comm_float2.a" ; export LOCOMM_float2
LODIAG_float2="$RTEXTBLS/libLOSUB_diag_float2.a" ; export LODIAG_float2
LOMODEL_float2="$RTEXTBLS/libLOSUB_model_float2.a" ; export LOMODEL_float2
LOPLOT_float2="$RTEXTBLS/libLOSUB_plots_float2.a" ; export LOPLOT_float2
LINKNCAR_float2=" " ; export LINKNCAR_float2
LINKEXT_float2=" $LINKIMSL_float2 $LINKNAG_float2 " ; export LINKEXT_float2

if [ "$plot_pgms" = 'yes' ] ; then
  F77_float1="${EVF90}         -nbs -WB -static-intel -mavx " ; export F77_float1
  LOCOMM_float1="$RTEXTBLS/libLOSUB_comm_float1.a" ; export LOCOMM_float1
  LODIAG_float1="$RTEXTBLS/libLOSUB_diag_float1.a" ; export LODIAG_float1
  LOMODEL_float1="" ; export LOMODEL_float1
  LOPLOT_float1="$RTEXTBLS/libLOSUB_plots_float1.a" ; export LOPLOT_float1
  # NCARG_ROOT="${SRC_ROOT}/ncar_lnx64_32ix/ncarg_4.4.2" ; export NCARG_ROOT
  NCARG_ROOT="${SRC_ROOT}/ncar_lnx64_32_ifort_193/ncarg_4.4.2" ; export NCARG_ROOT
  NCAR_LIB_PATH="$NCARG_ROOT/lib" ; export NCAR_LIB_PATH
  LINKNCAR_float1="-L${NCAR_LIB_PATH} -lncarg -lncarg_gks -lncarg_c -L/usr/lib -lX11 -lXext -lxcb -ldl -lXau -lXdmcp -lm" ; export LINKNCAR_float1
  LINKEXT_float1='' ; export LINKEXT_float1
else
  F77_float1='' ; export F77_float1
fi

if [ "$model_float1_mode" = 'yes' ] ; then
  F77_float1_openmp="${EVF90}         -nbs -WB -g -O2 -traceback -m64 -static-intel -mavx -fixed -mp1 -fp-model strict -static -Bstatic  -fpe0 -ftrapuv -qopenmp" ; export F77_float1_openmp
  LOCOMM_float1_openmp="$RTEXTBLS/libLOSUB_comm_float1_openmp.a" ; export LOCOMM_float1_openmp
  LODIAG_float1_openmp="$RTEXTBLS/libLOSUB_diag_float1_openmp.a" ; export LODIAG_float1_openmp
  LOMODEL_float1_openmp="$RTEXTBLS/libLOSUB_model_float1_openmp.a" ; export LOMODEL_float1_openmp
  LOPLOT_float1_openmp="$RTEXTBLS/libLOSUB_plots_float1_openmp.a" ; export LOPLOT_float1_openmp
  LINKNCAR_float1_openmp='' ; export LINKNCAR_float1_openmp
  LINKEXT_float1_openmp='' ; export LINKEXT_float1_openmp
else
  F77_float1_openmp='' ; export F77_float1_openmp
fi

if [ "$openmp_mode" = 'yes' ] ; then
  F77_openmp="${EVF90} -r8 -i8 -nbs -WB -g -O0 -traceback -m64 -static-intel -mavx -fixed -mp1 -fp-model strict -static -Bstatic  -fpe0 -ftrapuv -qopenmp" ; export F77_openmp
  LOCOMM_openmp="$RTEXTBLS/libLOSUB_comm_openmp.a" ; export LOCOMM_openmp
  LODIAG_openmp="$RTEXTBLS/libLOSUB_diag_openmp.a" ; export LODIAG_openmp
  LOMODEL_openmp="$RTEXTBLS/libLOSUB_model_openmp.a" ; export LOMODEL_openmp
  LOPLOT_openmp="$RTEXTBLS/libLOSUB_plots_openmp.a" ; export LOPLOT_openmp
  LINKNCAR_openmp='' ; export LINKNCAR_openmp
  LINKEXT_openmp='' ; export LINKEXT_openmp
else
  F77_openmp='' ; export F77_openmp
fi

#-----------

############################
##### Archive libraries part:
#####

#---------------------------------------------
# Compare lssub directory to reference version
if [ "$comp_full" == "no" ]; then
  # this sequence of commands produces a list of files in the user's lssub directory
  # and then checks if these are all in the reference directory. If any of these files
  # are not contained in the reference dir, it tells the script that a full compilation is
  # required. If all files are present in the ref dir, it then compares the checksums
  # between the two and if any are different, activates full compilation, else an incremental
  # compilation is used.
  lssub_files=`find ${CCRNSRC}/CanESM/CanDIAG/lssub/ -type f -print0 | xargs -0`
  echo ${lssub_files} | sed "s#${CCRNSRC}#${CANDIAG_REF_PATH}#g" | xargs ls > /dev/null 2>&1
  if [ $? != 0 ]; then
    # new file present
    comp_full="yes"
  else
    # all files in ${CCRNSRC}/CanESM/CanDIAG/lssub/ are in the ref dir, compare checksums
    sha1sum $lssub_files | sed "s#${CCRNSRC}#${CANDIAG_REF_PATH}#g" | sha1sum -c - --quiet || comp_full="yes"
  fi
fi

if [ "$comp_full" == "no" ]; then
  echo " " ; echo "==== No change detected in source code for archive libraries.. link to reference directory"

  # Link lssub archive libraries instead of recompiling
  ref_ex=$(ls $CANDIAG_REF_PATH/executables/*.a)
  cd $RTEXTBLS
  for ex in $ref_ex; do
    ex_ln=$(basename $ex)
    ln -sf $ex $ex_ln
  done
  cd -
else
  echo " " ; echo "==== Full compilation required.. rebuilding"
  echo " " ; echo "==== Start of generating archive libraries at - `date`"
  # -----------------------------------------------------------
  # Locally on HADAR: Make sure to process "*.c" first for "plots/ncar".
  # -----------------------------------------------------------
  cd $CCRNSRC/source/lssub/plots/ncar
  if [ "$plot_pgms" = 'yes' ] ; then
    ( rm *.o 2>> /dev/null ; cc -c *.c ; ar -r $RTEXTBLS/libLOSUB_plots_float1.a *.o )
  fi
  ( chmod g+r $RTEXTBLS/libLOSUB_plots*.a 2>> /dev/null || : )
  ( ls -ld $RTEXTBLS/libLOSUB_plots*.a 2>> /dev/null || : )
  ( rm *.o 2>> /dev/null || : )
  # ----------------------------------------------------------
  # Process "readpltinfo.f & writepltinfo.f" called by
  #         "cofaps/ggaps/subarea" programs
  # ----------------------------------------------------------
  cd $CCRNSRC/source/lssub/plots/ccrn
  time archsub readpltinfo.f writepltinfo.f xdir="$RTEXTBLS" afname=LOSUB_plots_float2 float2 ${febg_wait_option} mdest=banting
  if [ "$model_float1_mode" = 'yes' ] ; then
    time archsub readpltinfo.f writepltinfo.f xdir="$RTEXTBLS" afname=LOSUB_plots_float1_openmp float1 openmp ${febg_wait_option} mdest=banting
  fi
  if [ "$openmp_mode" = 'yes' ] ; then
    time archsub readpltinfo.f writepltinfo.f xdir="$RTEXTBLS" afname=LOSUB_plots_openmp openmp ${febg_wait_option} mdest=banting
  fi

  # -----------------------------------------------------------
  # Setup some links in HADAR:$RTEXTBLS
  #
  cd $RTEXTBLS
  ln -sf  libLOSUB_model_gcm6u_float2.a libLOSUB_model_float2.a
  if [ "$model_float1_mode" = 'yes' ] ; then
    ln -sf  libLOSUB_model_gcm6u_float1_openmp.a libLOSUB_model_float1_openmp.a
  fi
  if [ "$openmp_mode" = 'yes' ] ; then
    ln -sf  libLOSUB_model_gcm6u_openmp.a libLOSUB_model_openmp.a
  fi
  if [ "$plot_pgms" = 'yes' ] ; then
    ln -sf  libLOSUB_model_gcm6u_float1.a libLOSUB_model_float1.a
  fi


  #**********
  # Process each of the LSSUB subdirectories
  #**********

  echo " " ; echo "==== Processing lssub/comm " ; echo " "
  cd $CCRNSRC/source/lssub/comm
  time archsub all xdir="$RTEXTBLS" afname=LOSUB_comm_float2 float2 ${febg_wait_option} mdest=banting
  if [ "$model_float1_mode" = 'yes' ] ; then
    time archsub all xdir="$RTEXTBLS" afname=LOSUB_comm_float1_openmp float1 openmp ${febg_wait_option} mdest=banting
  fi
  if [ "$openmp_mode" = 'yes' ] ; then
    time archsub all xdir="$RTEXTBLS" afname=LOSUB_comm_openmp openmp ${febg_wait_option} mdest=banting
  fi
  if [ "$plot_pgms" = 'yes' ] ; then
    time archsub all xdir="$RTEXTBLS" afname=LOSUB_comm_float1 float1 ${febg_wait_option} mdest=banting
  fi
  echo " " ; echo "==== Processing lssub/diag" ; echo " "
  cd $CCRNSRC/source/lssub/diag
  time archsub all xdir="$RTEXTBLS" afname=LOSUB_diag_float2 float2 ${febg_wait_option} mdest=banting
  if [ "$model_float1_mode" = 'yes' ] ; then
    time archsub all xdir="$RTEXTBLS" afname=LOSUB_diag_float1_openmp float1 openmp ${febg_wait_option} mdest=banting
  fi
  if [ "$openmp_mode" = 'yes' ] ; then
    time archsub all xdir="$RTEXTBLS" afname=LOSUB_diag_openmp openmp ${febg_wait_option} mdest=banting
  fi
  if [ "$plot_pgms" = 'yes' ] ; then
    time archsub all xdir="$RTEXTBLS" afname=LOSUB_diag_float1 float1  ${febg_wait_option} mdest=banting
  fi

  if [ "$plot_pgms" = 'yes' ] ; then
    echo " " ; echo "==== Processing lssub/plots" ; echo " "
    cd $CCRNSRC/source/lssub/plots
    time archsub all xdir="$RTEXTBLS" afname=LOSUB_plots_float1 float1 ${febg_wait_option} mdest=banting
  fi

  # Needed to satisfy references for some program executables:

  echo " " ; echo "==== Processing lssub/model/.../gcm6u" ; echo " "
  cd $CCRNSRC/source/lssub/model/agcm/gcm6u
  time archsub all xdir="$RTEXTBLS" afname=LOSUB_model_gcm6u_float2 float2 ${febg_wait_option} mdest=banting
  if [ "$model_float1_mode" = 'yes' ] ; then
    time archsub all xdir="$RTEXTBLS" afname=LOSUB_model_gcm6u_float1_openmp float1 openmp ${febg_wait_option} mdest=banting
  fi
  if [ "$openmp_mode" = 'yes' ] ; then
    time archsub all xdir="$RTEXTBLS" afname=LOSUB_model_gcm6u_openmp openmp ${febg_wait_option} mdest=banting
  fi
  if [ "$plot_pgms" = 'yes' ] ; then
    time archsub all xdir="$RTEXTBLS" afname=LOSUB_model_gcm6u_float1 float1 ${febg_wait_option} mdest=banting
  fi
fi
echo " " ; echo "=== End of generating archive libraries, and start of executable binaries part at - `date`" ; echo " "
############################
##### Programs executables part:
#####

# Setup subdirectory to house temporary files/subdirectories

HMTMP="$CCRNSRC/tmp" ; export HMTMP
[ ! -d "$HMTMP/." ] && mkdir -m 1755 $HMTMP

#### Proceed in setting up some links and generating
###  executables in $RTEXTBLS/[svpgm,lopgm_r8i8]

[ ! -d "$HMTMP/f_r8i8_linux/." ] && mkdir -m 1755 $HMTMP/f_r8i8_linux || :
chmod 1755 $HMTMP/f_r8i8_linux
( mkdir -m 1755 $RTEXTBLS/lopgm_r8i8 && cd $RTEXTBLS/lopgm_r8i8 && ln -sf gcmpar6 gcmpar6O && ln -sf gcmpar7 gcmpar7R || : )
chmod 1755 $RTEXTBLS/lopgm_r8i8


echo " " ; echo "==== Processing RTEXTBLS/svpgm " ; echo " "
## $RTEXTBLS/svpgm

(mkdir -m 1755 $RTEXTBLS/svpgm || : )
chmod 1755 $RTEXTBLS/svpgm
# [ ! -s $RTEXTBLS/svpgm/f90split ] && echo "Missing $RTEXTBLS/svpgm/f90split !" || :
# [ ! -s $RTEXTBLS/svpgm/dirplist ] && echo "Missing $RTEXTBLS/svpgm/dirplist !" || :

cd $CCRNSRC/source/lspgm/diag/init
## specific parmsub considerations
#   - note: at the time of adding the compare/link functionality (Dec 2017)
#     it seems that parmsub gets compiled in $RTEXTBLS/svpgm and /lopgm_r8i8.
#     it is not clear if this is required..but this behaviour has been preserved.

if [ "$comp_full" == "yes" ]; then
  # compile parmsub regardless
  echo "compiling all RTEXTBLS/svpgm programs"
  time genabs parmsub.f float2 nolist dir=$RTEXTBLS/svpgm
else
  # compare to reference directory and only compile if necessary
  echo "comparing RTEXTBLS/svpgm programs to reference directory"
  svcomp_parmsub="no"
  sha1sum $CCRNSRC/CanESM/CanDIAG/lspgm/diag/init/parmsub.f |
  sed "s#${CCRNSRC}#${CANDIAG_REF_PATH}#g" | sha1sum -c - --quiet || svcomp_parmsub="yes"

  if [ "$svcomp_parmsub" == "no" ]; then
    # link
    ln -sf $CANDIAG_REF_PATH/executables/svpgm/parmsub /$RTEXTBLS/svpgm/parmsub
  else
    # compile
    time genabs parmsub.f float2 nolist dir=$RTEXTBLS/svpgm
  fi
fi
cp -p *.f $HMTMP/f_r8i8_linux

# If desired to disable "ld" output, enable the next line.
# F77_float2=`echo $F77_float2 | sed -e 's/ -bnoquiet / /'` # On AIX

## Fortran files in lspgm/local/sv
cd $CCRNSRC/source/lspgm/local/sv
if [ "$comp_full" == "yes" ]; then
  # compile all files regardless
  time genabs *.f float2 nolist dir=$RTEXTBLS/svpgm
else
  # check for new files
  compare_list=""
  for f in `ls $CCRNSRC/CanESM/CanDIAG/lspgm/local/sv/*.f`; do
    echo $f | sed "s#${CCRNSRC}#${CANDIAG_REF_PATH}#g" | xargs ls > /dev/null 2>&1
    if [ $? != 0 ]; then
      # f isn't present in the ref dir, compile it
      echo $f >> compile_list
    else
      # f is present in ref dir, prep for comparison
      compare_list=${compare_list}" "${f}
    fi
  done

  # compare to refence directory
  [ ! -z "$compare_list" ] && sha1sum $compare_list | 	# calculate checksums for files to be compared
  sed "s#${CCRNSRC}#${CANDIAG_REF_PATH}#g" |            	# set up comparison to check if files have been altered
  sha1sum -c - | awk '{
  if ($2=="OK") print substr($1,1,length($1)-1) > "link_list";
  else if ($2=="FAILED") print substr($1,1,length($1)-1) >> "compile_list"
 }' -   # produce list of files to link and append to compile_list
  # note that the above files will have paths refering to the ref dir;
  #  swap them out
  [ -e compile_list ] && sed -i "s#${CANDIAG_REF_PATH}#${CCRNSRC}#g" compile_list
  [ -e link_list ] && sed -i "s#${CANDIAG_REF_PATH}#${CCRNSRC}#g" link_list

  # compile necessary files
  if [ -e compile_list ]; then
    # compile
    time genabs $(cat compile_list) float2 nolist dir=$RTEXTBLS/svpgm
    rm compile_list
  fi

  # link remaining
  if [ -e link_list ]; then
    for fl in $(cat link_list); do
      ln -sf $CANDIAG_REF_PATH/executables/svpgm/$(basename $fl .f) /$RTEXTBLS/svpgm/$(basename $fl .f)
    done
    rm link_list
  fi
fi
cp -p *.f $HMTMP/f_r8i8_linux
hash -r
chmod -R u+w $HMTMP/f_r8i8_linux
chmod -R go+r $HMTMP/f_r8i8_linux
#

echo " " ; echo "==== Processing RTEXTBLS/lopgm_r8i8" ; echo " "
## $RTEXTBLS/lopgm_r8i8

[ -d "$HMTMP/code_lspgm/." ] && \rm -rf $HMTMP/code_lspgm || :
(mkdir $HMTMP/code_lspgm || : )
(mkdir $HMTMP/code_lspgm/plots || : )

cd $HMTMP/code_lspgm

if [ "$comp_full" == "yes" ]; then
  # copy all files over for compilation, regardless
  /usr/bin/find -L $CCRNSRC/source/lspgm/diag -name '*.vd' -exec \cp {} . \;
  /usr/bin/find -L $CCRNSRC/source/lspgm/diag/init -name '*.f' -exec \cp {} . \;
  if [ -d "$CCRNSRC/source/lspgm/asis/." ] ; then
    /usr/bin/find -L $CCRNSRC/source/lspgm/asis -name '*.f' -exec \cp {} . \;
  fi

else
  # compare to reference directory
  # build list of files to be considered
  lspgm_diag_files=`/usr/bin/find $CCRNSRC/CanESM/CanDIAG/lspgm/diag -name '*.vd' -type f`
  lspgm_diag_files=${lspgm_diag_files}" "`/usr/bin/find $CCRNSRC/CanESM/CanDIAG/lspgm/diag/init/ -name '*.f'`
  if [ -d "$CCRNSRC/source/lspgm/asis/." ] ; then
    lspgm_diag_files=${lspgm_diag_files}" "`/usr/bin/find $CCRNSRC/CanESM/CanDIAG/lspgm/asis/ -name '*.f' -type f`
  fi

  # check for new files
  compare_list=""
  for f in $lspgm_diag_files; do
    echo $f | sed "s#${CCRNSRC}#${CANDIAG_REF_PATH}#g" | xargs ls > /dev/null 2>&1
    if [ $? != 0 ]; then
      # f isn't present in the ref dir, compile it
      echo $f >> compile_list
    else
      # f is present in ref dir, prep for comparison
      compare_list=${compare_list}" "${f}
    fi
  done

  # compare to reference directory
  [ ! -z "$compare_list" ] && sha1sum $compare_list | 	# calculate checksums for files to be compared
  sed "s#${CCRNSRC}#${CANDIAG_REF_PATH}#g" |            	# set up comparison to check if files have been altered
  sha1sum -c - | awk '{
  if ($2=="OK") print substr($1,1,length($1)-1) > "link_list";
  else if ($2=="FAILED") print substr($1,1,length($1)-1) >> "compile_list"
 }' -   # produce list of files to link and append to compile_list
  # note that the above files will have paths refering to the ref dir;
  #  these are swapped out below
  [ -e compile_list ] && sed -i "s#${CANDIAG_REF_PATH}#${CCRNSRC}#g" compile_list
  [ -e link_list ] && sed -i "s#${CANDIAG_REF_PATH}#${CCRNSRC}#g" link_list

  # copy necessary files into currect directory for compilation, else they will be linked
  if [ ! -e compile_list ]; then
    echo "lopgm_r8i8 files unchanged... will be linked to precompiled repo"
  else
    echo "lopgm_r8i8 files have been altered... necessary files will be copied and compiled"

    # copy files for compilation
    for fl in $(cat compile_list); do
      cp $fl .
    done
  fi
fi

if [ "$comp_full" == "yes" ] || [ -e compile_list ]; then
  [ -e compile_list ] && rm compile_list

  # compile necessary files
  chmod u+w *
  chmod go+r *.vd *.f

  time genabs *.f float2 nolist dir=$RTEXTBLS/lopgm_r8i8
  \cp -p *.f $HMTMP/f_r8i8_linux
  (\rm -f *.f || : )
  hash -r

  time genpgm *.vd nolist float2 fdir=`pwd` br << woof
 y
woof

  ## Due to encountered issue in buliding "lpcrev2" in
  ## "float2" mode, retrying with "openmp" mode seems to
  ## be better. Hence, the special handling for "lpcrev2"
  ## program:
  ## Only needed if the program needs to be recompiled, and thus
  ##  would exist in this directory.
  if [ -e lpcrev2.f ]; then
    (mkdir $HMTMP/special || : )
    mv lpcrev2.f $HMTMP/special
    cd $HMTMP/special
    time genabs lpcrev2.f openmp nolist dir=$RTEXTBLS/lopgm_r8i8
    \cp -p *.f $HMTMP/f_r8i8_linux
    (\rm -f *.f || : )

    cd $HMTMP/code_lspgm
    rmdir $HMTMP/special
  fi

  time genabs *.f float2 nolist dir=$RTEXTBLS/lopgm_r8i8
  \cp -p *.f $HMTMP/f_r8i8_linux
  (\rm -f *.f || : )
fi

if [ -e link_list ]; then
  # link to unchanged executables to ref repo
  for fl in $(cat link_list); do
    if [[ $fl == *.f ]]; then ln -sf $CANDIAG_REF_PATH/executables/lopgm_r8i8/$(basename $fl .f) /$RTEXTBLS/lopgm_r8i8/$(basename $fl .f)
    elif [[ $fl == *.vd ]]; then ln -sf $CANDIAG_REF_PATH/executables/lopgm_r8i8/$(basename $fl .vd) /$RTEXTBLS/lopgm_r8i8/$(basename $fl .vd)
    fi
  done
  rm link_list
fi

## Plotting programs:
if [ "$plot_pgms" = 'yes' ] ; then
  echo " " ; echo "==== Processing lspgm/plots " ; echo " "
  cd $HMTMP/code_lspgm/plots
  /usr/bin/find -L $CCRNSRC/source/lspgm/plots -name '*.vd' -exec \cp {} . \;
  chmod u+w *.vd
  chmod g+r *.vd
  time genpgm *.vd nolist float1 fdir=`pwd` br << woof
y
woof
  # If desired to disable "ld" output, enable the next line.
  # F77_float1=`echo $F77_float1 | sed -e 's/ -bnoquiet / /'` # On AIX
  time genabs *.f float1 nolist dir=$RTEXTBLS/lopgm_r8i8
  chmod a+r *.f
  \cp -p *.f $HMTMP/f_r8i8_linux
  (\rm -f *.f || : )
fi
# chmod -R u+w  $HMTMP/f_r8i8_linux
# chmod -R go+r $HMTMP/f_r8i8_linux

## $RTEXTBLS/svpgm_float1

if [ "$model_float1_mode" = 'yes' ] ; then
  echo " " ; echo "==== Processing RTEXTBLS/svpgm_float1 " ; echo " "
  (\rm -rf $HMTMP/code_lspgm/svpgm_float1 || : )
  (mkdir -m 1755 $HMTMP/code_lspgm/svpgm_float1 || : )
  cd $HMTMP/code_lspgm/svpgm_float1
  List='ggstat joinpio joinup murge pakgcm8 pakgcm9 pakrs pakrs2 paktnd separmc unpakrs gcmrcpy rcopy newnam select sub'
  List_asis='rstime perturb perturb_winds'

  if [ "$comp_full" == "yes" ]; then
    # copy compile all progress regardless
    for prog in $List; do
      /usr/bin/find -L $CCRNSRC/source/lspgm/diag -name "${prog}.vd" -exec \cp {} . \;
    done
    for prog in $List_asis; do
      /usr/bin/find -L $CCRNSRC/source/lspgm/asis -name "${prog}.f" -exec \cp {} . \;
    done
    List="$List $List_asis"
  else
    # compare to ref dir
    TMP_CANDIAG_SHA_FILE_LOPGM=/tmp/ref_dir_svpgm_fl1_sha1s_$$
    touch $TMP_CANDIAG_SHA_FILE_LOPGM
    for prog in $List; do
      /usr/bin/find -L $CCRNSRC/source/lspgm/diag -name "${prog}.vd" -type f -print0 | xargs -0 sha1sum |
      sed "s#${CCRNSRC}#${CANDIAG_REF_PATH}#g" >> $TMP_CANDIAG_SHA_FILE_LOPGM
    done
    for prog in $List_asis; do
      /usr/bin/find -L $CCRNSRC/source/lspgm/asis -name "${prog}.f"  -type f -print0 | xargs -0 sha1sum |
      sed "s#${CCRNSRC}#${CANDIAG_REF_PATH}#g" >> $TMP_CANDIAG_SHA_FILE_LOPGM
    done
    List="$List $List_asis"

    sha1sum -c $TMP_CANDIAG_SHA_FILE_LOPGM | awk '{
   if ($2=="OK")
     print substr($1,1,length($1)-1) > "link_list";
   else if ($2=="FAILED")
     print substr($1,1,length($1)-1) > "compile_list"
   }' - # create lists for linking and compiling
    # note that these paths will point to the reference directory;
    #  swap them out
    [ -e compile_list ] && sed -i "s#${CANDIAG_REF_PATH}#${CCRNSRC}#g" compile_list
    [ -e link_list    ] && sed -i "s#${CANDIAG_REF_PATH}#${CCRNSRC}#g" link_list

    rm $TMP_CANDIAG_SHA_FILE_LOPGM

    # link unchanged executables
    if [ -e link_list ]; then
      mkdir $RTEXTBLS/svpgm_float1
      for fl in $(cat link_list); do
	# test if the binary exists
	if [ -s $CANDIAG_REF_PATH/executables/svpgm_float1/$(basename $fl .vd) ] ; then
	  ln -sf $CANDIAG_REF_PATH/executables/svpgm_float1/$(basename $fl .vd) /$RTEXTBLS/svpgm_float1/$(basename $fl .vd)
	else
	  # the binary does not exists. Add it to the compile_list
	  echo $fl >> compile_list
	fi
      done
      rm link_list
    fi

    if [ -e compile_list ]; then
      echo "some svpgm_float1 source files have been modified..compiling necessary files and linking the rest"

      # copy over for compilation
      for fl in $(cat compile_list); do
	cp $fl .
      done
      rm compile_list
    fi
  fi

  # compile necessary files if any present
  vdcount=$(ls -l *.vd 2>/dev/null | wc -l)
  if [ $vdcount != 0 ]; then
    chmod u+w *.vd
    chmod a+r *.vd

    time genpgm *.vd nolist float1 openmp fdir=`pwd` br \
      bln=640 blt=481 lat=481 lmt=128 lon=640 lrt=128 ilev=100 plv=100 << woof
y
woof
  fi

  # If desired to disable "ld" output, enable the next line.
  # F77_float1_openmp=`echo $F77_float1_openmp | sed -e 's/ -bnoquiet / /'` # On AIX

  fcount=$(ls -l *.f 2>/dev/null | wc -l)
  if [ $fcount != 0 ]; then
    time genabs *.f nolist float1 openmp dir=$RTEXTBLS/svpgm_float1
    (\rm -f *.f || : )
  fi

  if [ -d "$RTEXTBLS/svpgm/." ] ; then
    cd $RTEXTBLS/svpgm/.
    for prog in $List
    do
      ln -sf ../svpgm_float1/${prog} ${prog}_float1
    done
  fi
fi
# cd
( \rm -rf $HMTMP/code_lspgm || : )
[ ! -s "$RTEXTBLS/svpgm/f90split" ] && echo "Missing $RTEXTBLS/svpgm/f90split binary!" && [ -s /home/ords/crd/ccrn/scrd101/ref_env/code/executables/svpgm/f90split ] && ln -sf /home/ords/crd/ccrn/scrd101/ref_env/code/executables/svpgm/f90split $RTEXTBLS/svpgm/f90split && echo "linked to /home/ords/crd/ccrn/scrd101/ref_env/code/executables/svpgm/f90split " || :
# [ ! -s "$RTEXTBLS/svpgm/f90split" ] && echo "Missing $RTEXTBLS/svpgm/f90split binary!" && [ -s ${SRC_ROOT}/Linux_executables64_3.2i_161/svpgm/f90split ] && cp -p ${SRC_ROOT}/Linux_executables64_3.2i_161/svpgm/f90split $RTEXTBLS/svpgm/f90split && echo "Copied from ${SRC_ROOT}/Linux_executables64_3.2i_161/svpgm/f90split " || :
# [ ! -s "$RTEXTBLS/svpgm/dirplist" ] && echo "Missing $RTEXTBLS/svpgm/dirplist binary!" && [ -s ${SRC_ROOT}/Linux_executables64_3.2i_161/svpgm/dirplist ] && cp -p ${SRC_ROOT}/Linux_executables64_3.2i_161/svpgm/dirplist $RTEXTBLS/svpgm/dirplist && echo "Copied from ${SRC_ROOT}/Linux_executables64_3.2i_161/svpgm/dirplist " || :

# A workaround the problem in creating plotting programs on Brooks
# is to copy PPP generated versions from standard executables location.
#
# UPDATE: December 19, 2017. Changed the cp to a softlink to save space on disk.
PLOTexecs_list='bar barg blt budplot budplt6 crvplot ggplot hdplot hovplot msdplot shadtst spplot txtplot xmplot xplot zlnplot zxplot'
for prgexec in $PLOTexecs_list ; do
  if [ ! -s "$RTEXTBLS/lopgm_r8i8/$prgexec" -a -s "/home/ords/crd/ccrn/scrd101/ref_env/code/executables/lopgm_r8i8/$prgexec" ] ; then
    #cp -p ${SRC_ROOT}/Linux_executables64_3.2i_161/lopgm_r8i8/$prgexec $RTEXTBLS/lopgm_r8i8/$prgexec
    ln -sf /home/ords/crd/ccrn/scrd101/ref_env/code/executables/lopgm_r8i8/$prgexec $RTEXTBLS/lopgm_r8i8/$prgexec
  fi
done
[ -d "$OUTPUTQ/." ] && (cd $OUTPUTQ ; \rm -f libLOSUB* || : ) || :
echo " " ; echo "==== End of generating archive libraries and executable binaries at - `date`"
# exit