#!/bin/bash
############################################
# Set up environment for ensemble products job
# initial program, Jun Du, 2001
# change log:
# 10/23/2003, Jun Du: use $REGPAIR (for Eta members)
#               and $RSMPAIR to replace $TOTALPAIR to
#               deal with different ensemble size in
#               the two components
# 06/16/2005, Jun Du: (1) extend forecast lead time from 63hr
#               to 87hr; (2) call multiple grids in stead of
#               212 one grid only
# 9/20/2009, Julia Zhu   Scripts are modified to be sharable
#                        between EMC and NCO
#         Please note that variable "RUN_ENVIR" is set and used
#         in the development enviroment only.
# 11/15/2011, Jun Du: modified to new 2012 version of SREF (v6.0)
#             and added hourly output capability
# 05/01/2012, Jun Du: added a new 16km grid (g132)
# 03/13/2019, Jun Du: Convert to Dell version
############################################
set -xa

export OMP_NUM_THREADS=1
export MP_EUIDEVICE=sn_all
export MP_EUILIB=us
export MP_MPILIB=mpich2
export MP_TASK_AFFINITY=core:1

#
# Specify whether the run is production or development
#
export RUN_ENVIR=${RUN_ENVIR:-nco}

#####################################################################################
# Run config file to get input parameters
# This config file should define the following variables
# DATA_IN: Location of working directory, default to ${DATAROOT}
# DEV_SMS: If the job is to be running using ecFflow, default to YES
# SENDDBN: Set to NO for developers, default to YES
# COM_IN:  Directory for input files, default to ${COMROOT}/$NET/${envir}
# COM_OUT: Directory for output file, default to ${COMROOT}/$NET/${envir}
# gespath: Directory for the guess or restart files, default to ${GESROOT}/${envir}
#####################################################################################
if [ "$RUN_ENVIR" != nco ]      ### For Developers, "group_name" is passed from the ecFflow script
then
  CONFIG_FILE=${CONFIG_FILE:-/gpfs/dell2/emc/modeling/noscrub//${LOGNAME}/sref.v7_dell/parm/sref_para_config_dell}
  . $CONFIG_FILE
  export userid=$LOGNAME
  export DATA_IN=${DATA_IN:-/gpfs/dell2/ptmp/$userid}
fi

export VERBOSE=${VERBOSE:-YES}

echo `date` $0 `date -u` begin
export PS4='$SECONDS + '

########################################################### 
# obtain unique process id (pid) and make temp directories
###########################################################
export DATA_IN=${DATA_IN:-${DATAROOT}}

export DATA=$DATA_IN/${jobid:?}
rm -rf $DATA
mkdir -p $DATA

cd $DATA

####################################
# File To Log Msgs
####################################
export jlogfile=${jlogfile:-$DATA/jlogfile.${job}.$$}

####################################
# Determine Job Output Name on System
####################################
export pgmout="OUTPUT.$$"

export cycle=t${cyc}z 

###############################
# Specify NET and RUN name
##############################
export NET=sref
export RUN=sref

##################################################
# SAVEGES  - Copy Files From TMPDIR to $GESdir
# SENDECF  - Flag Events on ecFflow
# SENDCOM  - Copy Files From TMPDIR to $COMOUT
# SENDDBN  - Issue DBNet Client Calls
# RERUN    - Rerun fcst from beginning (default no)
##################################################
export SENDCOM=${SENDCOM:-YES}
export SENDECF=${SENDECF:-YES}
export SENDDBN=${SENDDBN:-YES}

export HOMEsref=${HOMEsref:-${PACKAGEROOT}/${NET}.${sref_ver}}
export EXECsref=${EXECsref:-$HOMEsref/exec}
export FIXsref=${FIXsref:-$HOMEsref/fix}
export PARMsref=${PARMsref:-$HOMEsref/parm}
export USHsref=${USHsref:-$HOMEsref/ush}


export SPCfix=${SPCfix:-$HOMEsref/fix/spcsref}

#export HOMEspc=${HOMEspc:-${PACKAGEROOT}/spcsref.${spcsref_ver}}
#export SPCfix=${SPCfix:-$HOMEspc/fix}

# Since the grib_util cnvgrib tool at wcoss2 version doesn't work, 
# it needs this temporary one here:
export CNVGRIB=$HOMEsref/util/cnvgrib
# Run setpdy and initialize PDY variables
setpdy.sh
. ./PDY

#############################################
# COMIN and COMOUT
#############################################
export COM_IN=${COM_IN:-$(compath.py ${NET}/${sref_ver})}
export COM_OUT=${COM_OUT:-$(compath.py ${NET}/${sref_ver})}

export COMIN=${COMIN:-$COM_IN/${RUN}.${PDY}/${cyc}/pgrb}
export COMOUT=${COMOUT:-$COM_OUT/${RUN}.${PDY}/${cyc}/ensprod}


mkdir -p -m 775 $COMOUT 
 
env

#######################################################
# Pass information which is needed to run the Script
#######################################################
export ENDHOUR=87
export INCHOUR=3

########################################################
# Execute the script using POE
########################################################
rm -f $DATA/poescript

# 1. Special products for EMC verification and DTRA
mkdir -p $DATA/212_special
echo "$HOMEsref/scripts/exsref_enspost4special.sh  3hrly 212 $DATA/212_special 111 > $DATA/212_special.out" >>$DATA/poescript

mkdir -p $DATA/216_special
echo "$HOMEsref/scripts/exsref_enspost4special.sh  3hrly 216 $DATA/216_special 111 > $DATA/216_special.out" >>$DATA/poescript

mkdir -p $DATA/243_special
echo "$HOMEsref/scripts/exsref_enspost4special.sh  3hrly 243 $DATA/243_special 111 > $DATA/243_special.out" >>$DATA/poescript

# 2. SPC combined probabilities
mkdir -p $DATA/212_SPC_3hrly
echo "$HOMEsref/scripts/exsref_enspost4SPC.sh  3hrly 212 $DATA/212_SPC_3hrly 111 > $DATA/212_SPC_3hrly.out" >>$DATA/poescript
 
mkdir -p $DATA/243_SPC_3hrly
echo "$HOMEsref/scripts/exsref_enspost4SPC.sh  3hrly 243 $DATA/243_SPC_3hrly 111 > $DATA/243_SPC_3hrly.out" >>$DATA/poescript

mkdir -p $DATA/216_SPC_3hrly
echo "$HOMEsref/scripts/exsref_enspost4SPC.sh  3hrly 216 $DATA/216_SPC_3hrly 111 > $DATA/216_SPC_3hrly.out" >>$DATA/poescript

# Turned off because it runs too slow
#mkdir -p $DATA/132_SPC_3hrly
#echo "$HOMEsref/scripts/exsref_enspost4SPC.sh  3hrly 132 $DATA/132_SPC_3hrly 111 > $DATA/132_SPC_3hrly.out" >>$DATA/poescript

# 3. Regular ensemble products
mkdir -p $DATA/212_3hrly
echo "$HOMEsref/scripts/exsref_enspost.sh  3hrly 212 $DATA/212_3hrly 111 0 > $DATA/212_3hrly.out" >>$DATA/poescript

mkdir -p $DATA/216
echo "$HOMEsref/scripts/exsref_enspost.sh  3hrly 216 $DATA/216 111 0 > $DATA/216.out" >>$DATA/poescript

mkdir -p $DATA/243
echo "$HOMEsref/scripts/exsref_enspost.sh  3hrly 243 $DATA/243 111 0 > $DATA/243.out" >>$DATA/poescript

chmod 775 $DATA/poescript
export MP_PGMMODEL=mpmd
export MP_CMDFILE=$DATA/poescript

#
# Execute the script.
mpiexec --cpu-bind core --configfile $DATA/poescript

export err=$?; err_chk

wait
cd $DATA
cat $DATA/212_3hrly.out >>$pgmout
cat $DATA/216.out >>$pgmout
cat $DATA/243.out >>$pgmout

cat $DATA/212_special.out >>$pgmout
cat $DATA/216_special.out >>$pgmout
cat $DATA/243_special.out >>$pgmout

cat $DATA/212_SPC_3hrly.out >>$pgmout
cat $DATA/243_SPC_3hrly.out >>$pgmout
cat $DATA/216_SPC_3hrly.out >>$pgmout

########################################################
cat $pgmout

msg="JOB COMPLETED NORMALLY"
postmsg $jlogfile "$msg"

################################
# Remove the Working Directory
################################
cd $DATA_IN
if [ ${KEEPDATA:-NO} != YES ] ; then rm -rf $DATA ; fi

date