#!/bin/bash
#PBS  -N wcofs_da_nf_03
#PBS  -A NOSOFS-DEV
#PBS  -q devmax
#PBS  -o /lfs/h1/nos/ptmp/aijun.zhang/rpt/v3.7.2/wcofs_da_nf_03.out
#PBS  -e /lfs/h1/nos/ptmp/aijun.zhang/rpt/v3.7.2/wcofs_da_nf_03.err
#PBS  -l place=vscatter:excl,select=4:ncpus=128:mpiprocs=64
#PBS  -l walltime=12:00:00

. /lfs/h1/nos/nosofs/noscrub/$LOGNAME/packages/nosofs.v3.7.2/versions/run.ver

# cd to your working directory
RPTDIR=/lfs/h1/nos/ptmp/$LOGNAME/rpt/v3.7.2
WORKDIR=/lfs/h1/nos/ptmp/$LOGNAME/work/${nosofs_ver}/wcofs_da
if [ ! -r $WORKDIR ]; then
  mkdir -p -m 755 $WORKDIR
fi 
cd ${WORKDIR}

module purge
module load envvar/${envvars_ver}

# Loading Intel Compiler Suite
module load PrgEnv-intel/${PrgEnv_intel_ver}
module load craype/${craype_ver}
module load intel/${intel_ver}
module load cray-mpich/${cray_mpich_ver}
module load cray-pals/${cray_pals_ver}
#Set other library variables
module load netcdf/${netcdf_ver}
module load hdf5/${hdf5_ver}
module load libjpeg/${libjpeg_ver}
module load subversion/${subversion_ver}
#module load python/${python_ver}
#module load ve/nosofs/3.6
module load ve/nosofs/${python_ve_ver}
module load prod_envir/${prod_envir_ver}
module load prod_util/${prod_util_ver}
module load grib_util/${grib_util_ver}
module load cfp/${cfp_ver} 
module load udunits/${udunits_ver}
module load gsl/${gsl_ver}
module load nco/${nco_ver}

# EXPORT list here

set +x
export envir=dev
export OFS=wcofs_da
export cyc=03
export job=wcofs_da_nf_03_$envir
export platform=ptmp


# add the following lines to improve runtime performance based on GDIT recommendations
ulimit -a

# for Intel compiler
export MALLOC_MMAP_MAX_=0
export MALLOC_TRIM_THRESHOLD_=134217728
export FOR_DISABLE_KMP_MALLOC=TRUE

export OMP_NUM_THREADS=1

# Cray MPICH settings
export MPICH_ENV_DISPLAY=1
export MPICH_VERSION_DISPLAY=1
export MPICH_ABORT_ON_ERROR=1

# shasta NIC policy
export MPICH_OFI_VERBOSE=1
export MPICH_OFI_NIC_VERBOSE=1

# use this for cray-mpich/8.1.0 and newer
export FI_OFI_RXM_SAR_LIMIT=1572864
#PERFORMANCE settings
export MPICH_OFI_STARTUP_CONNECT=1
export MPICH_COLL_OPT_OFF=MPI_Bcast
export MPICH_COLL_SYNC=MPI_Bcast

####   For testing or development run
#export PDY=20210824
export KEEPDATA=YES
export SENDCOM=NO
export SENDDBN=NO
export SENDSMS=NO
export PACKAGEROOT=/lfs/h1/nos/nosofs/noscrub/$LOGNAME/packages
export COMPATH=/lfs/h1/ops/prod/com
export COMROOT=/lfs/h1/nos/ptmp/$LOGNAME/com
export DCOMROOT=/lfs/h1/ops/prod/dcom
export DATAROOT=/lfs/h1/nos/ptmp/$LOGNAME/work/${nosofs_ver}/${OFS}
export COMINnam=/lfs/h1/ops/prod/com/nam/${nam_ver}
export COMINhrrr=/lfs/h1/ops/prod/com/hrrr/${hrrr_ver}
export COMINrap=/lfs/h1/ops/prod/com/rap/${rap_ver}
export COMINgfs=/lfs/h1/ops/prod/com/gfs/${gfs_ver}
export COMINrtma=/lfs/h1/ops/prod/com/rtma/${rtma_ver}
export COMINetss=/lfs/h1/ops/prod/com/etss/${etss_ver}
export COMINrtofs_2d=/lfs/h1/ops/prod/com/rtofs/${rtofs_ver}
export COMINrtofs_3d=/lfs/h1/ops/prod/com/rtofs/${rtofs_ver}
export COMINnwm=/lfs/h1/ops/prod/com/nwm/${nwm_ver}
#####################################################

# CALL executable job script here

export pbsid=${PBS_JOBID%%.*}
export job=${job:-$PBS_JOBNAME}
export jobid=${jobid:-$job.$PBS_JOBID}

/lfs/h1/nos/nosofs/noscrub/$LOGNAME/packages/nosofs.${nosofs_ver}/jobs/JNOS_OFS_NOWCST_FCST

#s=${PBS_JOBID}
#mv ${RPTDIR}/wcofs_da_nf_03.out ${RPTDIR}/wcofs_da_nf_03.out."${s%%.*}"
#mv ${RPTDIR}/wcofs_da_nf_03.err ${RPTDIR}/wcofs_da_nf_03.err."${s%%.*}"
#######################################################################
# Purpose: To execute the job that runs the NOS OFS model nowcast/forecast
#          on the IBMSP
#######################################################################