#PBS -N nam_forecast_%CYC%
#PBS -j oe
#!#PBS -l place=vscatter:excl,select=40:ncpus=128:mpiprocs=64:ompthreads=2
#!#PBS -l place=vscatter:excl,select=40:ncpus=126
#PBS -l place=vscatter:excl,select=38:ncpus=128
#PBS -q %QUEUE%
#PBS -S /bin/bash
##PBS -l walltime=01:20:00
#PBS -l walltime=03:30:00
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l hyper=true
#PBS -l debug=true

model=nam
%include <head.h>
%include <envir-p1.h>

set -x


# EXPORT list here

# 01/12: export ntasks=2560
# export ppn=64
# 02/12 03Z: export ntasks=2520
# export ppn=63
export ntasks=2432
export ppn=64
export threads=2


module load envvar/${envvar_ver}
module load PrgEnv-intel/${PrgEnv_intel_ver}
module load intel/${intel_ver}
module load craype/${craype_ver}
module load cray-mpich/${cray_mpich_ver}
module load cray-pals/${cray_pals_ver}
module load cfp/${cfp_ver}
module load hdf5/${hdf5_ver}
module load netcdf/${netcdf_ver}
module load nemsio/${nemsio_ver}
module list

# OMP settings
ulimit -s unlimited
ulimit -a
export OMP_PROC_BIND=true
export OMP_NUM_THREADS=$threads
export OMP_STACKSIZE=1G

export MPICH_ABORT_ON_ERROR=1
export MALLOC_MMAP_MAX_=0
export MALLOC_TRIM_THRESHOLD_=134217728
export FORT_FMT_NO_WRAP_MARGIN=true
export MPICH_REDUCE_NO_SMP=1
export FOR_DISABLE_KMP_MALLOC=TRUE
export FI_OFI_RXM_RX_SIZE=40000
export FI_OFI_RXM_TX_SIZE=40000
export MPICH_OFI_STARTUP_CONNECT=1
export MPICH_OFI_VERBOSE=1
export MPICH_OFI_NIC_VERBOSE=1


export cyc=%CYC%
export tmmark=tm00
export job=nam_forecast_%CYC%

export MPI_LABELIO=YES
export MP_STDOUTMODE="ORDERED"


############################################################
#### Load modules
###############################################################
# module purge

#NAM forecast uses Spectrum MPI

# CALL executable job script here

${HOMEnam}/jobs/JNAM_FORECAST
%include <tail.h>
%manual
######################################################################
# Purpose: to run NAM forecast
######################################################################


######################################################################
# Job specific troubleshooting instructions:
#  see generic troubleshoot manual page
#
######################################################################

# include manual page below
%end