#!/usr/bin/env bash
#PBS -N nwm_medium_range_blend_%CYC%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -j oe
#PBS -S /bin/bash
#PBS -q %QUEUE%
#PBS -l place=vscatter:exclhost,select=8:ncpus=128:mpiprocs=64
#PBS -l walltime=01:50:00
#PBS -l debug=true

export NODES=8
export NCORES=64
export NPROCS=$((NODES*NCORES))

export model=nwm

%include <head.h>
%include <envir-p1.h>

module load PrgEnv-intel/${PrgEnv_intel_ver}
module load craype/${craype_ver}
module load intel/${intel_ver}
module load cray-pals/${pals_ver}
module load netcdf/${netcdf_ver}
module load hdf5/${hdf5_ver}
module load iobuf/${iobuf_ver}
module load cfp/${cfp_ver}
module load python/${python_ver}
module load gsl/${gsl_ver}
module load udunits/${udunits_ver}
module load nco/${nco_ver}
module list

export I_MPI_DEVICE=rdssm
export I_MPI_PIN_PROCESSOR_LIST=15
export SAVE_ALL_TASKS=yes
export OMP_NUM_THREADS=1
export OMP_STACKSIZE=500M
export IOBUF_PARAMS='*.LAKEOUT_DOMAIN1:size=64M:count=2:prefetch=1,*:size=32M:count=4:vbuffer_count=4096:prefetch=1'
export OMP_PLACES=cores

#Unlimit the stack size
ulimit -s unlimited

# EXPORT list here
export cyc=%CYC%
export CASETYPE=medium_range_blend

# use --depth 2 for better runtime
export MPICOMMAND="mpiexec -n ${NPROCS} --depth 2 --cpu-bind depth "
export CFPCOMMAND="mpiexec -np ${NPROCS} --cpu-bind core cfp"

# CALL executable job script here
${HOMEnwm}/jobs/JNWM_MODEL

%include <tail.h>
%manual
######################################################################
#
#  PURPOSE:
#
######################################################################

######################################################################
# Job specific troubleshooting instructions:
#  see generic troubleshoot manual page
#
######################################################################

# include manual page below
%end