#!/bin/bash
set -e
# This script gets ran as part of the 'create_validate_plots' stage of the 
# nightly build ci pipeline, and as the stage suggests, produces validate plots.
#
# The environment variables are inherited from nb_config.sh

NCCONV_DIR=`pwd`
source activate val-env     # required to get jinja2 package

#=======================
# Setup validate scripts
#=======================
# clone/update the validate repo and 
if [ ! -d "$VALIDATE_DIR" ]; then
    git clone $VALIDATE_REPO --branch $VALIDATE_BRANCH $VALIDATE_DIR
else
    cd $VALIDATE_DIR
    git reset --hard                # make sure we are in a clean repo
    git checkout $VALIDATE_BRANCH   # checkout desired branch
    git fetch                       # update
    git pull
fi

# set the mass plot config settings
cp -f ${NCCONV_DIR}/${NB_SCRPT_DIR}/validate_cfg.py ${VALIDATE_DIR}/cmip6/batch_plotting/config.py

#=================================
# Create working directory and run
#=================================
if [ ! -d "$VALIDATE_WRKDIR" ]; then
    mkdir $VALIDATE_WRKDIR
    cd $VALIDATE_WRKDIR
else
    cd $VALIDATE_WRKDIR
    rm -rf *
fi

# launch jobs, storing jobids in a space delimited list
jobids=`python ${VALIDATE_DIR}/cmip6/batch_plotting/launch_verification_batches.py -f $VALIDATE_VARLST -vp $NB_STRG_DIR --silent`
jobids=`echo $jobids` # get rid of newline characters, else itll mess with the monitoring loops

#=============
# Monitor Jobs
#=============
echo "Monitoring plotting jobs.."
until [ "$jobsrunning" == "0" ] ; do
    sleep 30
    jobsrunning=0

    # check each jobid, if one is running, flip switch back to 1
    for jobid in $jobids; do 
        jobst -c $HDNODE -u $(whoami) 2> /dev/null | grep -F "$jobid" >> /dev/null 2>&1 && jobsrunning=1 || :
    done
done
echo "Plot jobs completed!"

#=============================
# Join plots and store on hpfx
#=============================
python ${VALIDATE_DIR}/cmip6/batch_plotting/sort_validate_plots.py --run_dir $VALIDATE_WRKDIR

# copy the plots into the plot viewer directory
cd $VALIDATE_PLTDIR
realms="atmos ocean land"
for realm in $realms; do
    # clean old plots
    [ -d "$realm" ] && rm -rf $realm
    cp -r ${VALIDATE_WRKDIR}/*_${realm} $realm
done