#PBS -N nosofs_nowcst_fcst_%OFS%_%CYC%%VER:%
#PBS -j oe
#PBS -S /bin/bash
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=10:30:00
#PBS -l place=vscatter:exclhost,select=2:ncpus=128:mpiprocs=64
#PBS -l debug=true

export model=nosofs
%include <head.h>
%include <envir-p1.h>
#
export cyc=%CYC%
export OFS=%OFS%

if [ -n "%DELT_MODEL:%" ]; then export DELT_MODEL=%DELT_MODEL:%; fi
if [ -n "%NTILE_I:%" ]; then export NTILE_I=%NTILE_I:%; fi
if [ -n "%NTILE_J:%" ]; then export NTILE_J=%NTILE_J:%; fi

# Loading Intel Compiler Suite
module load PrgEnv-intel/${PrgEnv_intel_ver}
module load craype/${craype_ver}
module load intel/${intel_ver}
#module load cray-mpich/${cray_mpich_ver}
module load cray-pals/${cray_pals_ver}
#Set other library variables
module load hdf5/${hdf5_ver}
module load ve/nosofs/${python_ve_ver}
module load libjpeg/${libjpeg_ver}
module load grib_util/${grib_util_ver}
module load cfp/${cfp_ver} 
module load gsl/${gsl_ver}
module load netcdf/${netcdf_ver}
module load udunits/${udunits_ver}
module load nco/${nco_ver}
module load cdo/${cdo_ver}
module load metis/${metis_ver}

# add the following lines to improve runtime performance based on GDIT recommendations
ulimit -a

# for Intel compiler
export MALLOC_MMAP_MAX_=0
export MALLOC_TRIM_THRESHOLD_=134217728
export FOR_DISABLE_KMP_MALLOC=TRUE

export OMP_NUM_THREADS=1

# Cray MPICH settings
export MPICH_ENV_DISPLAY=1
export MPICH_VERSION_DISPLAY=1
export MPICH_ABORT_ON_ERROR=1

# shasta NIC policy
export MPICH_OFI_VERBOSE=1
export MPICH_OFI_NIC_VERBOSE=1

# use this for cray-mpich/8.1.0 and newer
export FI_OFI_RXM_SAR_LIMIT=1572864
#PERFORMANCE settings
export MPICH_OFI_STARTUP_CONNECT=1
export MPICH_COLL_OPT_OFF=MPI_Bcast
export MPICH_COLL_SYNC=MPI_Bcast

#######################################################################
# Purpose: To execute the job that runs the NOS OFS model nowcast/forecast
#          on the IBMSP
#######################################################################
${HOMEnosofs}/jobs/JNOS_OFS_NOWCST_FCST

date
%include <tail.h>
%manual
%end