#!/usr/bin/env bash
#PBS -N nwm_medium_range%MEM%_%CYC%_%STIME%
#PBS -j oe
#PBS -A "NWM-DEV" 
###PBS -l place=vscatter:exclhost,select=8:ncpus=64:mpiprocs=64
#PBS -l place=vscatter:exclhost,select=8:ncpus=128:mpiprocs=64
#PBS -q dev
#PBS -l walltime=02:30:00
#PBS -l debug=true

export NODES=8 
export NCORES=64
export NPROCS=$((NODES*NCORES))

export model=nwm
source $ECFTESTDIR/model_envir.h

module load PrgEnv-intel/${PrgEnv_intel_ver}
module load craype/${craype_ver}
module load intel/${intel_ver}
module load cray-pals/${pals_ver}
module load netcdf/${netcdf_ver}
module load hdf5/${hdf5_ver}
module load iobuf/${iobuf_ver}
module load cfp/${cfp_ver}
module load python/${python_ver}
module list

export I_MPI_DEVICE=rdssm
export I_MPI_PIN_PROCESSOR_LIST=15
export SAVE_ALL_TASKS=yes
export OMP_NUM_THREADS=1
export OMP_STACKSIZE=500M
export IOBUF_PARAMS='*.LAKEOUT_DOMAIN1:size=64M:count=2:prefetch=1,*:size=32M:count=4:vbuffer_count=4096:prefetch=1'
export OMP_PLACES=cores

#Unlimit the stack size
ulimit -s unlimited

# EXPORT list here
export cyc=%CYC%
export mem=%MEM%
export CASETYPE=medium_range_mem${mem}

# use --depth 4 for better runtime
export MPICOMMAND="mpiexec -n 512 -ppn 64 --depth 2 --cpu-bind depth "
export CFPCOMMAND="mpiexec -np ${NPROCS} --cpu-bind core cfp"

# CALL executable job script here
${HOMEnwm}/jobs/JNWM_MODEL


#%manual
######################################################################
#
#  PURPOSE:
#
######################################################################

######################################################################
# Job specific troubleshooting instructions:
#  see generic troubleshoot manual page
#
######################################################################

# include manual page below
#%end