#!/usr/bin/env bash
#PBS -N nwm_analysis_assim_coastal_hawaii_%CYC%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -j oe
#PBS -S /bin/bash
#PBS -q %QUEUE%
#PBS -l place=vscatter:exclhost,select=1:ncpus=128:mpiprocs=64
#PBS -l walltime=00:10:00
#PBS -l debug=true

export NODES=1
export NCORES=64
export NPROCS=$((NODES*NCORES))

export model=nwm

%include <head.h>
%include <envir-p1.h>

module load PrgEnv-intel/${PrgEnv_intel_ver}
module load craype/${craype_ver}
module load intel/${intel_ver}
module load cray-pals/${pals_ver}
module load cray-mpich/${cray_mpich_ver}
module load netcdf/${netcdf_ver}
module load hdf5/${hdf5_ver}
module load iobuf/${iobuf_ver}
module load cfp/${cfp_ver}
module load python/${python_ver}
module load wgrib2/${grib2_ver}
module list

export I_MPI_DEVICE=rdssm
export I_MPI_PIN_PROCESSOR_LIST=15
export SAVE_ALL_TASKS=yes
export OMP_NUM_THREADS=2
export IOBUF_PARAMS='*.LAKEOUT_DOMAIN1:size=64M:count=2:prefetch=1,*:size=32M:count=4:vbuffer_count=4096:prefetch=1'
export OMP_PLACES=cores
# ----------------: added (WCOSS2/Pete)
# set up MPI connections and buffers at start of run - helps efficiency of MPI later in run
export MPICH_OFI_STARTUP_CONNECT=1
# pace MPI_Bcast messaging when reading and distributing initial conditions - prevents the Bcast hangs
export MPICH_COLL_SYNC=MPI_Bcast
# turn off MPI_Reduce on node optimization - prevent MPI_Reduce hangs during time stepping
export MPICH_REDUCE_NO_SMP=1
export FI_OFI_RXM_SAR_LIMIT=3145728
export FI_MR_CACHE_MAX_COUNT=0

#Unlimit the stack size
ulimit -s unlimited

# EXPORT list here
export cyc=%CYC%
export CASETYPE=analysis_assim_coastal_hawaii
export COASTAL_DOMAIN='hawaii'
export mem=''

# CALL executable job script here
${HOMEnwm}/jobs/JNWM_COASTAL

%include <tail.h>
%manual
######################################################################
#
#  PURPOSE:
#
######################################################################

######################################################################
# Job specific troubleshooting instructions:
#  see generic troubleshoot manual page
#
######################################################################

# include manual page below
%end