#!/bin/bash

# -- "big endian format conversion"
export F_UFMTENDIAN='big'
# -- likely for stack size per thread
export OMP_STACKSIZE=1G

export UCX_NET_DEVICES="mlx5_0:1"
export I_MPI_HYDRA_BOOTSTRAP="ssh"
export I_MPI_JOB_RESPECT_PROCESS_PLACEMENT="no"
export I_MPI_PIN_DOMAIN="auto:scatter"
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${CANESM_ESMF_INSTALL_DIR}/lib/"

#### === ADVANTAGEOUS
# -- enable "ifort" buffered I/O
export FORT_BUFFERED=TRUE # "-assume buffered_stdout"
export BUFFERED=yes #  "-assume buffered_io"
# -- beneficial with "-cc depth" use in "aprun" command
export KMP_AFFINITY=compact 

#### === DESIRABLE

# -- Potential workaround for hang in MPI_Init
export PMI_MMAP_SYNC_WAIT_TIME=180
export PMI_CONNECT_RETRIES=100

# export PMI_DEBUG=1

#### === USEFUL
# -- MPI Info 
export MPICH_VERSION_DISPLAY=1
export MPICH_ENV_DISPLAY=1
export MPICH_CPUMASK_DISPLAY=1
export MPICH_RANK_REORDER_DISPLAY=1

#### === MAY BE USEFUL; needs to be tested
# Controls the threshold for switching from eager to rendezvous protocols for inter-node messaging.
export MPICH_GNI_MAX_EAGER_MSG_SIZE=131072

# -- Increase stack size limit
ulimit -all
ulimit -s unlimited || :

# load compiler and netcdf libraries 
. ssmuse-sh -x main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl >> /dev/null 2>&1 || echo "IntelOneApi already loaded" # intel compiler
. ssmuse-sh -x /fs/ssm/main/opt/hdf5-netcdf4/serial/static/inteloneapi-2022.1.2/01  # netcdf libraries