#! /bin/sh

# JHMON_FORECAST - this job runs the 126hr HMON forecast, either
# coupled or uncoupled.  The coupled forecast requires 5 more
# compute nodes (-n 640) than the uncoupled (-n 512). When the coupled
# forecast failed, it will fall back to run uncoupled forecast.
#
# Runs in parallel with:
#   JHMON_POST1
#   JHMON_POST2
#   JHMON_HYCOM_POST (coupled only)
#
# Dependencies:
#   JHMON_HYCOM_INIT1 (coupled only)
#   JHMON_HYCOM_INIT2 (coupled only)
#   JHMON_NPS_BC                                      
#   JHMON_RELOCATE 
#
# Next Job:  (above post jobs run in parallel)
#

#PBS -N cpnmmb_2021082406_hmon2
#PBS -A HMON-OPS
#PBS -q prod
#PBS -l place=excl,select=5:ncpus=128:mpiprocs=127
#PBS -l debug=true
#PBS -l walltime=02:30:00
#PBS -j oe

date
#export PS4='+ $SECONDS + '
set -xu

####################################
# Set default values if not yet set
# values from ecFlow or modules will override these
####################################
#export DATA=${DATA:-${DATAROOT:?}/hmon${storm_num:?}_${cyc:?}_${envir:?}}
#export DATA=/lfs/h2/emc/ptmp/Lin.L.Zhu/hmon.v3.2.0/2021082406/13E
export DATA=/lfs/f1/ops/prod/tmp/hmon2_06_prod
cyc=06
export cycle=${cycle:-t${cyc}z}
export jlogfile=${DATA}/jlogfile

####################################
# Initialize work area and cd there
####################################
mkdir -p $DATA
cd $DATA

####################################
# Variables for logging & DBnet
####################################
export NET=hmon RUN=hmon

####################################
# Set threading defaults
####################################
export OMP_NUM_THREADS=1
export OMP_STACKSIZE=128M
export MKL_NUM_THREADS=1

####################################
# Set HMON-specific vars
####################################
# Top of HMON installation tree:
#export HOMEhmon=${HOMEhmon:-${PACKAGEROOT:?}/hmon.${hmon_ver:?}}
export HOMEhmon=/lfs/h1/ops/prod/packages/hmon.v3.2.0
export USHhmon=$HOMEhmon/ush
export EXhmon=$HOMEhmon/scripts
export PARMhmon=$HOMEhmon/parm
export FIXhmon=$HOMEhmon/fix
export EXEChmon=$HOMEhmon/exec
export WORKhmon=$DATA

source ${HOMEhmon}/versions/run.ver1
module load PrgEnv-intel/${PrgEnv_intel_ver}
module load intel/${intel_ver}
module load craype/${craype_ver}
module load cray-mpich/${cray_mpich_ver}
module load cray-pals/${cray_pals_ver}

#module load iobuf/2.0.10
module load libjpeg/${libjpeg_ver}
module load grib_util/${grib_util_ver}
module load wgrib2/${wgrib2_ver}
module load python/${python_ver}
module load cfp/${cfp_ver}

module load atp/${atp_ver}
module load hdf5/${hdf5_ver}
module load netcdf/${netcdf_ver}
module load python/${python_ver}
module list

# Get PDY
#. ./PDY

# set up cycle
#export CYCLE=${PDY}${cyc}
export CYCLE=2021082406

storm_num=2
# Upstream and downstream areas:
#export stormlabel=${stormlabel:-storm$storm_num}
#export COMINmsg=${COMINmsg:-$(compath.py hwrf/${hwrf_ver}/inphmon)}

# HMON areas:
#priorymdh=$( $NDATE -6 "$PDY$cyc" )
priorymdh=2021082400
#export HISTDATA=${HISTDATA:-$(compath.py $NET/${hmon_ver})/$RUN.$priorymdh}
export HISTDATA=/lfs/h1/ops/canned/com/hmon/v3.2/hmon.2021082400
#export COMIN=${COMIN:-$(compath.py $NET/${hmon_ver})/$RUN.$PDY$cyc}
export COMIN=/lfs/h1/ops/prod/com/hmon/v3.2/hmon.2021082406
#export COMOUT=${COMOUT:-$(compath.py -o $NET/${hmon_ver})/$RUN.$PDY$cyc}
export COMOUT=/lfs/h1/ops/prod/com/hmon/v3.2/hmon.2021082406
#export COM=${COM:-${COMOUT}}
export COM=/lfs/h1/ops/prod/com/hmon/v3.2/hmon.2021082406

# Prepend $USHhmon to python package search path:
export PYTHONPATH=$USHhmon${PYTHONPATH:+:$PYTHONPATH}

#messagefile=$COMINmsg/message$storm_num
#message=$( head -1 "$messagefile" ) # full message contents
#basin1=${message:7:1} # L

# Make sure all mandatory variables are set: cyc, storm_num
set -u +x
echo "Checking for mandatory variables:"
echo "cyc=$cyc storm_num=$storm_num"
set +u -x
env
####################################
# Set/check job-specific vars if any
####################################

####################################
# Pass control to ex-script
####################################

#echo "$FORECAST_CONF_OVERRIDE" > "$COMOUT/storm$storm_num.forecast_override"
#postmsg "$jlogfile" "Starting exhmon_forecast.py $FORECAST_CONF_OVERRIDE"
#$EXhmon/exhmon_nmmb.py $FORECAST_CONF_OVERRIDE
# JY - if [[ ! ( "$storm_num" -le "$nstorms" ) ]] ; then
#if [[ "$basin1" == "L" ]]; then
#    export TOTAL_TASKS="412"
#    $EXhmon/exhmon_nmmb.py $TOTAL_TASKS
if [ `echo $job |grep noncouple` ]; then
    export TOTAL_TASKS=${TOTAL_TASKS:-"412"}
    $EXhmon/exhmon_nmmb.py $TOTAL_TASKS
else 
    export TOTAL_TASKS=${TOTAL_TASKS:-"624"}
    $EXhmon/exhmon_cpl.py $TOTAL_TASKS
fi

export err=$?; #err_chk
postmsg "$jlogfile" "exhmon_forecast.py ended normally."