#!/bin/sh
#PBS -N jgefs_chem_forecast_00
#PBS -j oe
#PBS -S /bin/bash
#PBS -q dev
#PBS -A GEFS-DEV
#PBS -l walltime=3:00:00
#PBS -l place=vscatter,select=2:ncpus=128:mpiprocs=128:mem=500GB
#PBS -l debug=true

##PBS -l place=vscatter,select=3:ncpus=110:mpiprocs=110     #3M
##PBS -l place=vscatter,select=4:prepost=true:ncpus=64:mpiprocs=64    #4R
##PBS -l place=vscatter,select=2:ncpus=128:mpiprocs=128:mem=500GB+1:ncpus=64:mpiprocs=64:mem=250GB #2.5M

set -x

cd $PBS_O_WORKDIR

#module purge

export model=gefs
# to get HOMEgefs
f=lsf_common.sh
if [[ -f "$f" ]]; then
    echo "$f exists."
    source $f
elif [[ -f "../$f" ]]; then
    echo "../$f exists."
    source ../$f
elif [[ -f "../../$f" ]]; then
    echo "../../$f exists."
    source ../../$f
elif [[ -f "../../../$f" ]]; then
    echo "../../../$f exists."
    source ../../../$f
fi
. ${HOMEgefs}/versions/run.ver

module purge
module load envvar/$envvar_ver
module load PrgEnv-intel/$PrgEnv_intel_ver
module load craype/$craype_ver
module load intel/$intel_ver

module load cray-mpich/$cray_mpich_ver
module load cray-pals/$cray_pals_ver

module load prod_util/$prod_util_ver
module load prod_envir/$prod_envir_ver

module load netcdf/$netcdf_ver
module load hdf5/$hdf5_ver

#module load lsf/$lsf_ver
module load cfp/$cfp_ver
export USE_CFP=YES

module list

# EXPORT list here
ulimit -s unlimited
ulimit -a

f=lsf_common.sh
if [[ -f "$f" ]]; then
    echo "$f exists."
    source $f
elif [[ -f "../$f" ]]; then
    echo "../$f exists."
    source ../$f
elif [[ -f "../../$f" ]]; then
    echo "../../$f exists."
    source ../../$f
elif [[ -f "../../../$f" ]]; then
    echo "../../../$f exists."
    source ../../../$f
fi
#---
export RUNMEM=geaer

# If needed, uncomment below statements to overwrite the setting in lsf_common.sh
#export layout_x_chem=6
#export layout_y_chem=6
#export WRITE_GROUP_chem=1
#export WRTTASK_PER_GROUP_chem=40 #24

#export parallel_threads_chem=1
#export restart_interval_aer=12

#export total_tasks=$((layout_x_chem * layout_y_chem * 6 + WRTTASK_PER_GROUP_chem * WRITE_GROUP_chem)) #330 #240
export total_tasks=256
#export taskspernode=128
export COREPERNODE=128
export FORECAST_SEGMENT=hr
export OMP_NUM_THREADS=1
#export RERUN="YES"

#For Debug of the forecast
#export ESMF_RUNTIME_PROFILE=ON
#export ESMF_RUNTIME_PROFILE_OUTPUT=SUMMARY
#export print_memory_usage=".true." #".false."

#export gefsmpexec="mpiexec -n $total_tasks" # -ppn 120"

export envir=prod
# CALL executable job script here
${HOMEgefs}/jobs/JGEFS_FORECAST