#!/bin/bash
# Advanced canesm-setup
# This script is called by setup-canesm, and is used to do the work of
# setting up a run. setup-canesm just does the basic clone to /tmp, and then
# sources this file.
#
# Environment variables appearing here have been inherited from setup-canesm 
#
########################################################################
# 1. Environment and paths
#
#set -x

# Verify that necessary variables have been inherited from setup-canesm
[ -z "$CANESM_RUNID" ] && bail "adv-setup: CANESM_RUNID not inherited"
[ -z "$CANESM_VER" ] && bail "adv-setup: CANESM_VER not inherited" 
[ -z "$CANESM_REPO" ] && bail "adv-setup: CANESM_REPO not inherited"
[ -z "$CANESM_CONFIG" ] && bail "adv-setup: CANESM_CONFIG not inherited"
[ -z "$CANESM_CMIP" ] && bail "adv-setup: CANESM_CMIP not inherited"
[ -z "$WRK_DIR" ] && bail "adv-setup: WRK_DIR not inherited" 
if [ -z "$CANESM_SRCLNK" ]; then
    # only check if we are cloning, not linking
    [ -z "$TMP_ROOT" ] && bail "adv-setup: TMP_ROOT not inherited" 
fi

# Machine dependent path structure
case $(uname -n|awk -F'.' '{print $1}' -) in
  xc1*) # hare
        export TRUE_HOST=hare
        export FE_TRUE_HOST=eccc-ppp1
        export RUNPATH_ROOT=/space/hall1/work/eccc/crd/ccrn
        export TRUE_HOST_ALT=brooks
        export FE_TRUE_HOST_ALT=eccc-ppp2
        export RUNPATH_ROOT_ALT=/space/hall2/work/eccc/crd/ccrn
        ;;
  xc2*) # brooks
        export TRUE_HOST=brooks
        export FE_TRUE_HOST=eccc-ppp2
        export RUNPATH_ROOT=/space/hall2/work/eccc/crd/ccrn
        export TRUE_HOST_ALT=hare
        export FE_TRUE_HOST_ALT=eccc-ppp1
        export RUNPATH_ROOT_ALT=/space/hall1/work/eccc/crd/ccrn
        ;;
  xc3*) # banting
        export TRUE_HOST=banting
        export FE_TRUE_HOST=eccc-ppp3
        export RUNPATH_ROOT=/space/hall3/work/eccc/crd/ccrn
        export TRUE_HOST_ALT=daley
        export FE_TRUE_HOST_ALT=eccc-ppp4
        export RUNPATH_ROOT_ALT=/space/hall4/work/eccc/crd/ccrn
        ;;
  xc4*) # daley
        export TRUE_HOST=daley
        export FE_TRUE_HOST=eccc-ppp4
        export RUNPATH_ROOT=/space/hall4/work/eccc/crd/ccrn
        export TRUE_HOST_ALT=banting
        export FE_TRUE_HOST_ALT=eccc-ppp3
        export RUNPATH_ROOT_ALT=/space/hall3/work/eccc/crd/ccrn
        ;;
  *) echo "Invalid machine name $(uname -n|awk -F'.' '{print $1}' -). It must be banting or daley."
     exit 1 ;;
esac

# Root of where code goes on disk at /home/ords
export ORDS_ROOT=/home/ords/crd/ccrn
export HOME_ORDS=$ORDS_ROOT/$USER

# RUNID_ROOT is the "runtime directory" where the source will be placed.
RUNID_ROOT="$HOME_ORDS/${CANESM_RUNID}" ; export RUNID_ROOT
 
if [ -z "$CANESM_DFFSFX" ]; then
# Only if this is a run setup, not a diagnostic setup

    # Create RUNPATH dirs for this RUNID where data goes on disk on Backend and Frontend (FE)
    RUNPATH_RUNID=$RUNPATH_ROOT/users/$USER/${CANESM_RUNID}
    FE_RUNPATH_RUNID=`echo $RUNPATH_RUNID | sed 's/work/sitestore/'`
    export RUNPATH=${RUNPATH_RUNID}/data
    export FE_RUNPATH=${FE_RUNPATH_RUNID}/data
    [ ! -d "$RUNPATH/." ]  && mkdir -p -m 1755 $RUNPATH
    [ ! -d "$FE_RUNPATH/." ]  && mkdir -p -m 1755 $FE_RUNPATH

    # Create RUNPATH dirs on other halls (create a symlink on sitestore)
    RUNPATH_ALT_RUNID=$RUNPATH_ROOT_ALT/users/$USER/${CANESM_RUNID}
    FE_RUNPATH_ALT_RUNID=`echo $RUNPATH_ALT_RUNID | sed 's/work/sitestore/'`
    export RUNPATH_ALT=${RUNPATH_ALT_RUNID}/data
    export FE_RUNPATH_ALT=${FE_RUNPATH_ALT_RUNID}/data
    ssh $TRUE_HOST_ALT "[ ! -d $RUNPATH_ALT ]" && ssh $TRUE_HOST_ALT mkdir -p -m 1755 $RUNPATH_ALT
    ssh $FE_TRUE_HOST_ALT "[ ! -d $FE_RUNPATH_ALT ]" && ssh $FE_TRUE_HOST_ALT "ln -s $FE_RUNPATH_RUNID $FE_RUNPATH_ALT_RUNID"

    # Check if RUNID_ROOT already existing and give option to copy
    if [ -d $RUNID_ROOT/code ]; then
      echo "WARNING: directory $RUNID_ROOT/code already exists!"
      read -p "Do you want to rename it and continue? y/[n] " -n 1 -r
      echo
      if [[ ! $REPLY =~ ^[Yy]$ ]] ; then
	bail "ERROR: Cleanup or choose a new runid"
      else
	stamp=`date "+%j%H%M%S"$$`
	echo "Rename $RUNID_ROOT/code to $RUNID_ROOT/code.$stamp and continue."
	mv $RUNID_ROOT/code $RUNID_ROOT/code.$stamp
      fi
    fi

    # Create the runtime directory. This is where the source code is going to be placed later.
    [ ! -d "$RUNID_ROOT/." ]  && mkdir -p -m 1755 $RUNID_ROOT
    CCRNSRC=${RUNID_ROOT}/code ; export CCRNSRC

else
   # Choice for additional "diagnostics from the future"
   # In this case the RUNPATHs etc above have been created already, and here we add a new CCRNSRC directory
   # which contains newer code for the diagnostics (relative to the original run code). 
   # Note that while CCRNSRC must be different, runid, RUNPATH etc are all the same.
   

   # Define a new CCRNSRC for the future diagnostics code
   [ ! -d "$RUNID_ROOT/." ]  && bail "Cannot find $RUNID_ROOT, which should be there to do diagnostics on an existing run"
   CCRNSRC=${RUNID_ROOT}/code_$CANESM_DFFSFX ; export CCRNSRC

   # Create a new WRKDIR
   cd ${WRK_DIR}
   [ ! -d "${CANESM_RUNID}_${CANESM_DFFSFX}" ]  && mkdir -p ${CANESM_RUNID}_${CANESM_DFFSFX} || bail "diag-$CANESM_DFFSFX already exists!"
   export WRK_DIR=${WRK_DIR}/${CANESM_RUNID}_${CANESM_DFFSFX}
fi

mkdir -p -m 1755 ${CCRNSRC}

########################################################################
# 2. Copy repositories from tmp /clone location

#echo "about to clone"
cd $CCRNSRC

if  [ -z "$CANESM_SRCLNK" ]; then
    # CANESM_SRCLNK is not defind. Go ahead
    # and copy the source code from where it was cloned to.

    # TMP_ROOT is the /tmp location
    cp -r $TMP_ROOT/* . 
    rm -rf $TMP_ROOT

    ########################################################################
    # 4. Backwards compatibility / old CCRNSRC  
    #
    # Below we will construct the old structure of "CCRNSRC", using softlinks 
    # back to the REPOs. This is pretty ugly, but it supports all the existing 
    # scripting etc and so is required, at least for now.

    # Create the "old" structure.
    if [ -d "$CCRNSRC/." ] ; then
     cd $CCRNSRC
     HMTMP="$CCRNSRC/tmp" ; export HMTMP
     [ ! -d "$HMTMP/." ] && mkdir -m 1755 $HMTMP
     [ ! -d "$HMTMP/f_r8i8_linux/." ] && mkdir -m 1755 $HMTMP/f_r8i8_linux
     OUTPUTQ="$CCRNSRC/.queue" ; export OUTPUTQ
     [ ! -d "$OUTPUTQ/." ] && mkdir -m 1755 $OUTPUTQ
     [ ! -d "$OUTPUTQ/.data/." ] && mkdir -m 1755 $OUTPUTQ/.data
     [ ! -d "$OUTPUTQ/.crawork/." ] && mkdir -m 1755 $OUTPUTQ/.crawork
     RTEXTBLS="$CCRNSRC/executables" ; export RTEXTBLS
     [ ! -d "$RTEXTBLS/." ] && mkdir -m 1755 $RTEXTBLS
     SCRIPTS="$CCRNSRC/scripts" ; export SCRIPTS

     # CCRNOPT="$CCRNSRC" ; export CCRNOPT
     CCRNOPT="/home/scrd102" ; export CCRNOPT
     CCRNOPTBIN="$CCRNOPT/package/bin" ; export CCRNOPTBIN
     RUNID_BIN="$CCRNSRC/bin" ; export RUNID_BIN
     [ ! -d "$RUNID_BIN/." ] && mkdir -m 1755 $RUNID_BIN
     RUNID_LIB="$CCRNSRC/lib" ; export RUNID_LIB
     [ ! -d "$RUNID_LIB/." ] && mkdir -m 1755 $RUNID_LIB
     SITE_ID='DrvlSC' ; export SITE_ID

    else
     bail "Problem: Invalid $CCRNSRC subdirectory !"
    fi

    # Do the soft linking from the git repos into the old structure
    cd $CCRNSRC
    ln -s CanESM/CCCma_tools/bin_xc_linux64 bin_xc_linux64
    ln -s CanESM/CCCma_tools/bin_ppp_linux64 bin_ppp_linux64
    ln -s CanESM/CCCma_tools/cccjob_dir cccjob_dir
    ln -s CanESM/CCCma_tools/cccmake cccmake

    [ ! -d "$CCRNSRC/generic/." ] && mkdir -p $CCRNSRC/generic || bail "generic folder already exists in $WRK_DIR"

    cd $CCRNSRC/generic
    ln -s ../CanESM/CCCma_tools/generic/* .
    cd $CCRNSRC
    ln -s CanESM/CCCma_tools/scripts scripts
    ln -s scripts scripts_linux
    ln -s scripts scripts_linux64
    ln -s CanESM/CCCma_tools/tools tools

    mkdir -m 1755 $CCRNSRC/source
    cd $CCRNSRC/source
    ln -s ../CanESM/CanDIAG/diag4 diag4
    ln -s ../CanESM/CanDIAG/lspgm lspgm
    ln -s ../CanESM/CanDIAG/lssub lssub
    ln -s ../CanESM/CanAM/lsmod lsmod
    cd $CCRNSRC
    ln -s source source_linux
    ln -s source source_linux64
    ln -s source source_xc_linux64
    ln -s source source_ppp_linux64
    
    # clone down pycmor for nc conversion : 
    # POTENTIAL: this could be turned into a submodule
    ssh $FE_TRUE_HOST "cd $CCRNSRC && git clone --recursive git@gitlab.science.gc.ca:CanESM/ncconv.git"
    cd ncconv
    git checkout v1.3.1.9 && git submodule update
    NCCONV_SHA1=`git rev-parse HEAD`
    cd $CCRNSRC
else
    # In this case, we simply softlink CCRNSRC to another location.
    # Not really great for reproducibility, but sensible for ensembles.
    ln -s ${CANESM_SRCLNK}/* .
    rm -rf generic post_jobs
    cp -rp ${CANESM_SRCLNK}/generic .
    rm -f generic/env_setup_file
fi

cd $CCRNSRC/CanESM
# Repopulate index -> required for strict checking
git status > /dev/null || bail "Error in executing 'git status' in adv-setup !! Please try setup again."

# Get the checksums of each major subcomponent
CANESM_SHA1=`git rev-parse HEAD`
cd CanAM
CANAM_SHA1=`git rev-parse HEAD`
cd ../CanDIAG
CANDIAG_SHA1=`git rev-parse HEAD`
cd ../CanNEMO
CANNEMO_SHA1=`git rev-parse HEAD`
cd ../CanCPL
CANCPL_SHA1=`git rev-parse HEAD`
cd ../CCCma_tools
CANTOOLS_SHA1=`git rev-parse HEAD`

cd $CCRNSRC

# The section below creates the environment files to be sourced to get access to all the above.
# Basically, this sets up the "CCCma environment", but using the specific code just
# extracted from git, not something random on disk.

# File to use under "$RUNPATH/PRODUCTION/${runid}/generic"
# in setting up "env_setup_file" file under the same subdirectory
# where the latter will be executed to setup the run environment.
hst=${TRUE_HOST}

# define hpfx run-logging dir
HPFX_RUNLOG_DIR="/home/scrd104/public_html/canesm-run-logs"; export HPFX_RUNLOG_DIR

#[ -s "$CCRNSRC/generic/env_setup_file" ] && bail "$CCRNSRC/generic/env_setup_file already exists!"

#if [ -s "$CCRNSRC/generic/raw_env_setup_file" ] ; then
    echo "runid_env=${CANESM_RUNID} ; export runid_env" > $WRK_DIR/env_setup_file
    echo "CUSTOM_ENV_FILE=$WRK_DIR/env_setup_file ; export CUSTOM_ENV_FILE" >> $WRK_DIR/env_setup_file
    echo "CCRNSRC=$CCRNSRC ; export CCRNSRC" >> $WRK_DIR/env_setup_file
    echo "WRK_DIR=$WRK_DIR ; export WRK_DIR" >> $WRK_DIR/env_setup_file
    echo "HPFX_RUNLOG_DIR=${HPFX_RUNLOG_DIR} ; export HPFX_RUNLOG_DIR" >> $WRK_DIR/env_setup_file
    echo "DIAGJOB_SFX=`date "+%j%H%M%S"`$$ ; export DIAGJOB_SFX" >> $WRK_DIR/env_setup_file
    echo "CANESM_CMIP=$CANESM_CMIP ; export CANESM_CMIP" >> $WRK_DIR/env_setup_file
    echo "CANESM_DFFSFX=${CANESM_DFFSFX}; export CANESM_DFFSFX" >> $WRK_DIR/env_setup_file
    cat $CCRNSRC/generic/raw_env_setup_file >> $WRK_DIR/env_setup_file
#    mv $WRK_DIR/env_setup_file $WRK_DIR

    # prepend local database to the master database
    echo "export DATAPATH_DB=\"\$RUNID_ROOT/datapath_local.db:\$DATAPATH_DB\"" >> $WRK_DIR/env_setup_file

    # set TOPDOG=off by default
    echo "export TOPDOG=off" >> $WRK_DIR/env_setup_file

    ln -s $WRK_DIR/env_setup_file $CCRNSRC/generic/env_setup_file

#else
#    bail "Expected $CCRNSRC/generic/raw_env_setup_file file is missing!"
#fi

########################################################################
# 7. Extract helpful scripts into the PRODUCTION dir

# Below we extract some files for the user to setup their run. This includes a 
# "make_script/config" a basefile and a compilation file.

# Figure out which config to use / what files to extract
CONFIG_PATH="${CCRNSRC}/CanESM/CONFIG"
JOBDEF_PATH="${CCRNSRC}/CanESM/CCCma_tools/cccjob_dir/lib/jobdefs"
case $CANESM_CONFIG in
    ESM)       CANESM_CONFIG_PATH="${CONFIG_PATH}/ESM/" ; CANESM_basefile=canesm_cgcm_jobdef ;;
    AMIP)      CANESM_CONFIG_PATH="${CONFIG_PATH}/AMIP/"; CANESM_basefile=canesm_agcm_jobdef ;;
    OMIP)      CANESM_CONFIG_PATH="${CONFIG_PATH}/OMIP/"; CANESM_basefile=canesm_ogcm_jobdef ;;
                     *) bail "Invalid configuration --> $arg <-- ...config MUST be 'ESM' or 'AMIP'." ;;
esac

# Get the make_job, basefile and compile script in the PROD dir
[ -s ${CONFIG_PATH}/COMMON/make_job           ] && cp ${CONFIG_PATH}/COMMON/make_job           ${WRK_DIR}/make_job_${CANESM_RUNID}      || bail "Could not extract make_job"
[ -s ${CONFIG_PATH}/COMMON/make_diag_job      ] && cp ${CONFIG_PATH}/COMMON/make_diag_job      ${WRK_DIR}/make_diag_job_${CANESM_RUNID} || bail "Could not extract make_diag_job"
[ -s ${CONFIG_PATH}/COMMON/make_dpar_job      ] && cp ${CONFIG_PATH}/COMMON/make_dpar_job      ${WRK_DIR}/make_dpar_job_${CANESM_RUNID} || bail "Could not extract make_dpar_job"
[ -s ${CONFIG_PATH}/COMMON/make_pool_job      ] && cp ${CONFIG_PATH}/COMMON/make_pool_job      ${WRK_DIR}/make_pool_job_${CANESM_RUNID} || bail "Could not extract make_pool_job"
[ -s ${CONFIG_PATH}/COMMON/make_dump_job      ] && cp ${CONFIG_PATH}/COMMON/make_dump_job      ${WRK_DIR}/make_dump_job_${CANESM_RUNID} || bail "Could not extract make_dump_job"


[ -s ${CANESM_CONFIG_PATH}/compile-canesm.sh  ] && cp ${CANESM_CONFIG_PATH}/compile-canesm.sh  ${WRK_DIR}/compile_${CANESM_RUNID}       || bail "Could not extract compile script"
[ -s ${CANESM_CONFIG_PATH}/save_restart_files ] && cp ${CANESM_CONFIG_PATH}/save_restart_files ${WRK_DIR}/save_restart_files_${CANESM_RUNID} && rs_script=1 || rs_script=0
[ -s ${CANESM_CONFIG_PATH}/canesm.cfg         ] && cp ${CANESM_CONFIG_PATH}/canesm.cfg         ${WRK_DIR}/canesm.cfg                    || bail "Could not extract canesm.cfg"

[ -s ${CANESM_CONFIG_PATH}/cppdefs     ] && rm -f ${WRK_DIR}/cppdir_${CANESM_RUNID}   && ln -sf ${CANESM_CONFIG_PATH}/cppdefs     ${WRK_DIR}/cppdir_${CANESM_RUNID}  || bail "Could not extract ccpdefs"
[ -s ${JOBDEF_PATH}/${CANESM_basefile} ] && rm -f ${WRK_DIR}/basefile_${CANESM_RUNID} && ln -sf ${JOBDEF_PATH}/${CANESM_basefile} ${WRK_DIR}/basefile_${CANESM_RUNID} || bail "Could not extract basefile"

[ -s ${CONFIG_PATH}/COMMON/make_merged_diag_script ] && cp ${CONFIG_PATH}/COMMON/make_merged_diag_script ${WRK_DIR}/make_merged_diag_script    || bail "Could not extract make_merged_diag_script"
[ -s ${CONFIG_PATH}/COMMON/plotmix_job           ] && cp ${CONFIG_PATH}/COMMON/plotmix_job           ${WRK_DIR}/plotmix_job_${CANESM_RUNID}    || bail "Could not extract plotmix_job"
[ -s ${CONFIG_PATH}/COMMON/PHYS_PARM             ] && cp ${CONFIG_PATH}/COMMON/PHYS_PARM             ${WRK_DIR}/PHYS_PARM_${CANESM_RUNID}      || bail "Could not extract PHYS_PARM"
[ -s ${CONFIG_PATH}/COMMON/INLINE_DIAG_NL        ] && cp ${CONFIG_PATH}/COMMON/INLINE_DIAG_NL        ${WRK_DIR}/INLINE_DIAG_NL_${CANESM_RUNID} || bail "Could not extract INLINE_DIAG_NL"

[ -s ${CONFIG_PATH}/COMMON/tapeload_rs           ] && cp ${CONFIG_PATH}/COMMON/tapeload_rs           ${WRK_DIR}/tapeload_rs_${CANESM_RUNID} || echo "Could not extract tapeload_rs"

# Do some substitutions like inserting RUNID and the path to source
# in make_job
sed -i "s:REPLACEMENT_PATH_TO_SOURCE:${WRK_DIR}/env_setup_file:" ${WRK_DIR}/make_job_${CANESM_RUNID}
sed -i "s/JOBDESC=\".*_base:\(.*\)\"/JOBDESC=\"basefile_${CANESM_RUNID}:\1\"/g" ${WRK_DIR}/make_job_${CANESM_RUNID}

# in compile script
sed -i "s:REPLACEMENT_PATH_TO_SOURCE:${WRK_DIR}/env_setup_file:" ${WRK_DIR}/compile_${CANESM_RUNID}

# in canesm.cfg
sed -i "s/^runid=.*/runid=${CANESM_RUNID}/g" ${WRK_DIR}/canesm.cfg
CANESM_SHORT_SHA1=`echo ${CANESM_SHA1} | cut -c 1-8`
CANNEMO_SHORT_SHA1=`echo ${CANNEMO_SHA1} | cut -c 1-8`
CANCPL_SHORT_SHA1=`echo ${CANCPL_SHA1} | cut -c 1-8`
sed -i "s/nemo_exec=.*/nemo_exec=mc_${CANESM_RUNID}_nemo.exe/g" ${WRK_DIR}/canesm.cfg
sed -i "s:nemo_repo=.*:nemo_repo=${CCRNSRC}/CanESM/CanNEMO/:g" ${WRK_DIR}/canesm.cfg
sed -i "s:cancpl_repo=.*:cancpl_repo=${CCRNSRC}/CanESM/CanCPL/:g" ${WRK_DIR}/canesm.cfg
## Enable the next line to propagate $CANESM_CMIP into "production" setting in ${WRK_DIR}/canesm.cfg
sed -i "s/^production=.*/production=${CANESM_CMIP}/g" ${WRK_DIR}/canesm.cfg
# handle defaults for non-cmip runs
if (( CANESM_CMIP == 0 )) ; then
    sed -i "s/^override_activity_id=.*\#/override_activity_id=\"CCCma\" \#/" ${WRK_DIR}/canesm.cfg
    sed -i "s/^override_source_id=.*\#/override_source_id=\"CanESM5-${runid}\" \#/" ${WRK_DIR}/canesm.cfg
fi

# in plotmix_job
sed -i -e "s/runid=runid;/runid=${CANESM_RUNID};/" -e "s/username=username;/username=$NAME;/" -e "s/user=USER;/user=$USER;/" ${WRK_DIR}/plotmix_job_${CANESM_RUNID}

# Create post_jobs directory in $CCRNSRC and link it to $WRK_DIR
mkdir -p $CCRNSRC/post_jobs
rm -f $WRK_DIR/post_jobs
ln -s $CCRNSRC/post_jobs $WRK_DIR/post_jobs

# Link in environment files
#ln -sf $CCRNSRC/generic/env_setup_file ${WRK_DIR}/BE_env_setup_file
rm -f ${WRK_DIR}/CanESM_source_link ${WRK_DIR}/code_link
ln -s $CCRNSRC/CanESM ${WRK_DIR}/CanESM_source_link
ln -s $CCRNSRC        ${WRK_DIR}/code_link

########################################################################
# 8. Wrap-up 

### WRITE LOG FILE
cd ${WRK_DIR}
echo '~~~' >> .setup-canesm.log
echo `date` >> .setup-canesm.log
echo 'setup-canesm ' $@ >> .setup-canesm.log
echo 'Code corresponds to SHA1s:' >> .setup-canesm.log
echo 'CanESM:' $CANESM_SHA1 >> .setup-canesm.log
echo 'CanAM:' $CANAM_SHA1 >> .setup-canesm.log
echo 'CanDIAG:' $CANDIAG_SHA1 >> .setup-canesm.log
echo 'CanCPL:' $CANCPL_SHA1 >> .setup-canesm.log
echo 'CanNEMO:' $CANNEMO_SHA1 >> .setup-canesm.log
echo 'CCCma_tools:' $CANTOOLS_SHA1 >> .setup-canesm.log
echo 'NCCONV:' $NCCONV_SHA1 >> .setup-canesm.log
if [ ! -z "$CANESM_DFFSFX" ]; then
    echo "CanESM DFF suffix:" $CANESM_DFFSFX >> .setup-canesm.log
fi

### WRITE TO RUNS DB
datef=`date +"%Y-%m-%d %T"`
userf=`whoami`

sqlite3 /home/scrd104/dbs/setup-canesm.db  << EOF
                            INSERT INTO runs (runid, date, user, config, ver, repo, sha1, wrkdir)
                            values('${CANESM_RUNID}', '${datef}', '${userf}', '${CANESM_CONFIG}', 
                                   '${CANESM_VER}', '${CANESM_REPO}', '${CANESM_SHA1}', '${WRK_DIR}');
EOF

### Initiate .${runid}-log.db
#LOG_DB="$CCRNSRC/../${runid}-log.db"
if [ -z "$CANESM_DFFSFX" ]; then
    LOG_DB="$CCRNSRC/../${CANESM_RUNID}-log.db"
    CFG_BRANCH="${CANESM_RUNID}"
else
    LOG_DB="$CCRNSRC/../${CANESM_RUNID}_${CANESM_DFFSFX}-log.db"
    CFG_BRANCH="${CANESM_RUNID}_${CANESM_DFFSFX}"
fi


if [ ! -f $LOG_DB ]; then
    touch $LOG_DB
    sqlite3 $LOG_DB "CREATE TABLE events (Date VARCHAR, Event VARCHAR, Config_SHA1 VARCHAR, CanESM_SHA1 VARCHAR, Production VARCHAR)"
fi

### Define config files and initiate repo, hard copying in the files from the WRK_DIR
config_repo=".config-repo"
config_files="canesm.cfg make_*_${CANESM_RUNID} make_merged_diag_script compile_${CANESM_RUNID} save_restart_files_${CANESM_RUNID} tapeload_rs_${CANESM_RUNID}"

# clean old one if necessary and create new config repo
[ -d $config_repo ] && rm -rf $config_repo
git clone git@gitlab.science.gc.ca:CanESM/CanESM_run_config.git --branch master --single-branch $config_repo || 
    bail "Failed to create config repo!"

# create branch and override existing remote branch 
cd $config_repo
git checkout -b $CFG_BRANCH || bail "Failed to create config repo branch for this runid"
git push --force -u origin $CFG_BRANCH || bail "Failed to push config repo!"
cd $WRK_DIR

# copy config files into config repo
for cfg_f in .setup-canesm.log $config_files; do
    for f in `ls ${cfg_f}`; do
        rm -f ${config_repo}/${f} && cp $f ${config_repo}/${f}
    done
done
# set README for this branch and hard link it into the setup directory
printf "# ${CANESM_RUNID}\n\n" > README.md
printf "### Description\n\n" >> README.md
printf "### List of Changes/Interventions\n\n" >> README.md
rm -f ${config_repo}/README.md && cp README.md ${config_repo}/README.md

### Setup hpfx runlog directory and add entry into hpfx runlist
hpfx_rnlg_dir="${HPFX_RUNLOG_DIR}/${CFG_BRANCH}"
hpfx_rnlst_entry="<TR><TD>${CFG_BRANCH}</TD><TD>${datef}</TD><TD>${USER}</TD></TR>"
if [ ! -d $hpfx_rnlg_dir ] ; then
    # make runid directory
    mkdir -p $hpfx_rnlg_dir || bail "Failed to create the runlog hpfx directory : $hpfx_rnlg_dir"
    
    # copy in html templates and set permissions so they can be updated
    cp -r ${HPFX_RUNLOG_DIR}/templates/* ${hpfx_rnlg_dir}/ || bail "Failed to copy html templates into $hpfx_rnlg_dir"
    chmod g+w ${hpfx_rnlg_dir}/*.html

    # add run entry to hpfx run-list
    echo "$hpfx_rnlst_entry" >> ${HPFX_RUNLOG_DIR}/runlist
else
    # replace existing runlist entry with updated date
    entry_id="<TR><TD>${CFG_BRANCH}</TD>"
    grep -F "$entry_id" ${HPFX_RUNLOG_DIR}/runlist >> /dev/null || present="no"     # check if identifier is present in the runlist
    if [ "$present" = "no" ] ; then
        echo "adding new entry to hpfx runlist.."
        echo "$hpfx_rnlst_entry" >> ${HPFX_RUNLOG_DIR}/runlist                          
    else
        echo "runid entry exists in hpfx runlist.. updating" 
        sed -i "s#${entry_id}.*#${hpfx_rnlst_entry}#" ${HPFX_RUNLOG_DIR}/runlist
    fi
fi

### Log Setup Event
[ $CANESM_CMIP -eq 1 ] && flgs="-t" || flgs="-t -d"   # use 'development' mode if CANESM_CMIP flag = 0
$CCRNSRC/CanESM/CCCma_tools/scripts/comm/strict_check $flgs $CANESM_RUNID setup $config_files
[ $? -ne 0 ] && bail "Failed to log setup event!"

### Give them a message for that really fuzzy feeling that everything worked...
printf "\n\n \t *** SETUP-CANESM *** \n\n"
printf "\t Successfully cloned $CANESM_REPO into ${CCRNSRC} and checked out ${CANESM_VER}. \n\n"
printf "\t Setup the interactive CCCma environment (now) by executing the following line: \n"
printf "\n\t\t . env_setup_file \n\n\t *** \n\n"

### AUTOMATED compiling / saving / running
if [ "$ALL" == "1" ] ; then
   echo "ALL is on"
   COMPILE=1
   SAVE=1
   RUN=1
   . env_setup_file
fi

if [ "$COMPILE" == "1" ] ; then
   . env_setup_file
   echo "Trying to compile the default setup from $CANESM_SHORT_SHA1"
   ./make_job_${CANESM_RUNID}
   found_job=`ls -1 ${CANESM_RUNID}*_job | head -1`
   [ -z "$found_job" ] && { echo "Could not find a job to submit..." ; exit 1 ; }
   ./compile_${CANESM_RUNID} $found_job
fi
if [ "$SAVE" == "1" ] ; then
   . env_setup_file
   echo "Saving default restart files"
   ./save_restart_files_${CANESM_RUNID}
fi
if [ "$RUN" == "1" ] ; then
   . env_setup_file
   found_job=`ls -1 ${CANESM_RUNID}*_job | head -1`
   [ -z "$found_job" ] && { echo "Could not find a job to submit..." ; exit 1 ; }
   echo "Submitting default job $found_job to brooks"
   rsub $TRUE_HOST $found_job
fi