#PBS -N nam_catchup_forecast_tm%TMMARK%_%CYC%
#PBS -j oe
#PBS -q %QUEUE%
#PBS -S /bin/bash
#PBS -l place=vscatter:excl,select=16:ncpus=128:mpiprocs=64:ompthreads=2
#PBS -l walltime=00:30:00
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l debug=true

model=nam
%include <head.h>
%include <envir-p1.h>

# EXPORT list here

set -x

export ntasks=1024
export ppn=64
export threads=2

# OMP settings
ulimit -s unlimited
ulimit -a
export OMP_PROC_BIND=true
export OMP_NUM_THREADS=$threads
export OMP_STACKSIZE=1G

export MPICH_ABORT_ON_ERROR=1
export MALLOC_MMAP_MAX_=0
export MALLOC_TRIM_THRESHOLD_=134217728
export FORT_FMT_NO_WRAP_MARGIN=true
export MPICH_REDUCE_NO_SMP=1
export FOR_DISABLE_KMP_MALLOC=TRUE
export FI_OFI_RXM_RX_SIZE=40000
export FI_OFI_RXM_TX_SIZE=40000
export MPICH_OFI_STARTUP_CONNECT=1
export MPICH_OFI_VERBOSE=1
export MPICH_OFI_NIC_VERBOSE=1

#  export PMI_DEBUG=1
export FI_OFI_RXM_RX_SIZE=40000
export FI_OFI_RXM_TX_SIZE=40000

export cyc=%CYC%
export tmmark=tm%TMMARK%
export job=nam_catchup_forecast_tm%TMMARK%_%CYC%

export MPI_LABELIO=YES
export MP_STDOUTMODE="ORDERED"

# module purge
module load envvar/${envvar_ver}
module load PrgEnv-intel/${PrgEnv_intel_ver}
module load intel/${intel_ver}
module load craype/${craype_ver}
module load cray-mpich/${cray_mpich_ver}
module load cray-pals/${cray_pals_ver}
module load cfp/${cfp_ver}
module load hdf5/${hdf5_ver}
module load netcdf/${netcdf_ver}
module load nemsio/${nemsio_ver}
module load libpng/${libpng_ver}
module load libjpeg/${libjpeg_ver}
module load grib_util/${grib_util_ver}

#NAM forecast uses Spectrum MPI

# CALL executable job script here

${HOMEnam}/jobs/JNAM_FORECAST
%include <tail.h>
%manual
######################################################################
# Purpose: to prepare the initial input files for NAM
######################################################################


######################################################################
# Job specific troubleshooting instructions:
#  see generic troubleshoot manual page
#
######################################################################

# include manual page below
%end