#! /bin/ksh 

# JHWRF_FORECAST -- runs an HWRF forecast, which may be either
#     coupled or uncoupled.  The job card used for this script
#     depends on whether ocean coupling is done, and that 
#     decision is made by the JHWRF_OCEAN_INIT job.  

# Last modified: Samuel Trahan, April 5, 2012.

# *** IMPORTANT JOB SUBMISSION NOTE **

# This job should be submitted with ocean coupling if MODEL=COUPLED
# in the $stormlabel.holdvars.txt,
#    *AND*
# the ${COMOUT}/ocean_status.${STORM}${STORMID}.${PDY}${cyc} contains
# RUN_COUPLED=YES.  

# OTHERWISE, it should be submitted without ocean coupling.  The only
# difference from the SMS level is to provide two fewer processors.
# Ocean coupling requires 248 processors, while uncoupled requires 246.
# See below for job cards.

# *** Operational Workflow ***

# Submitted By: JHWRF_MERGE
# Submits: None.
#
# Main Contacts: 
#   WRF Atmospheric Model -- Sam Trahan, Zhan Zhang, Young Kwon
#   POM Ocean Model -- URI
#   NCEP Coupler -- Dmitry Sheinin, Sam Trahan
#   scripts -- Sam Trahan, Zhan Zhang

# Primary scripts:
#
#   scripts/exhwrf_forecast.sh.sms -- determines whether a coupled or
#       uncoupled forecast is being done, and passes control to either
#       hwrf_nmm_fcst.sh or hwrf_coupled_fcst.sh
#   hwrf_nmm_fcst.sh -- performs an uncoupled (WRF-only) forecast
#   hwrf_coupled_fcst.sh -- sets up a working directory for a coupled
#       forecast, and then passes control to hwrf_c_execute.sh to
#       launch the MPI job
#   hwrf_c_execute.sh -- launches the WRF-POM coupled forecast

# Primary parameter files:
#
#   parm/hwrf_namelist.input -- WRF control file
#   parm/hwrf_gfdl_eastatl.parm.coup -- POM east atlantic ocean control file
#   parm/hwrf_gfdl_eastpac.parm.coup -- POM east pacific ocean control file
#   parm/hwrf_gfdl_united.parm.coup -- POM united (near-us) ocean control file

# Primary executables:
#
#   hwrf_wrf -- WRF atmospheric model
#   hwrf_wm3c -- NCEP coupler (the name comes from the fact that it is
#       capable of coupling to the wavewatch 3 model)
#   hwrf_ocean_eastatl -- POM East Atlantic ocean model
#   hwrf_ocean_eastatl_ext -- extended east atlantic, not currently used
#   hwrf_ocean_united -- POM United (near-US) ocean model
#   hwrf_ocean_eastpac -- POM East Pacific ocean model (NEW this year)
#   hwrf_swcorner_dynamic -- decides initial WRF domain 2 start location

# *** EMC-Suggested Job Cards ***

# OCEAN-COUPLED CONFIGURATION
#@ job_type = parallel 
#@ wall_clock_limit = 01:29:00
#@ total_tasks=248
#@ task_affinity=cpu(1)
#@ node_resources = ConsumableMemory(110 GB)
#@ node = 4
#@ network.MPI=csss,not_shared,us
#@ parallel_threads = 1
#@ bulkxfer = yes
#@ queue

# UNCOUPLED (ATMOSPHERE ONLY) CONFIGURATION

# The only difference is 246 processors instead of 248:

#@ job_type = parallel 
#@ wall_clock_limit = 01:29:00
#@ total_tasks=246
#@ task_affinity=cpu(1)
#@ node_resources = ConsumableMemory(110 GB)
#@ node = 4
#@ network.MPI=csss,not_shared,us
#@ parallel_threads = 1
#@ bulkxfer = yes
#@ queue


export PARAFLAG=${PARAFLAG:-NO}

if [ ${PARAFLAG} == NO ]; then
$SMSBIN/smsinit $LOADL_STEP_ID
fi
#----------------------------------------------
# JHWRF_COUPLED_FCST.sms  
#----------------------------------------------
set -xa
export MP_IOAGENT_CNT=all
export MP_IO_BUFFER_SIZE=8M

export PS4='$SECONDS + '
date

export PARAFLAG=${PARAFLAG:-NO}

if [ ${PARAFLAG} = 'YES' ];
then
  export yymmdd=` echo $PDY | cut -c3-8`
  export yymmddhh=${yymmdd}${cyc}
  export jlogfile=${DATA}/jlogfile
  export SENDSMS=YES
  export SENDCOM=YES
  export SENDDBN=NO

  if [ ! -d $outdir ]; then mkdir -p $outdir; fi
  if [ ! -d $COMOUT ]; then mkdir -p $COMOUT; fi
else
  export NWPROD=${NWPROD:-/nwprod}
fi

###############################################################
# This block can be modified for different Production test
# environment. This is used for operational testings
###############################################################
if [ ${PARAFLAG} == NO -a $envir != prod ]; then
  export SENDDBN=${SENDDBN:-NO}
  export jlogfile=${jlogfile:-/com/logs/${envir}/jlogfile}
  export DBN_ALERT_TYPE=TBD_PARA
  export DBNROOT=/nwprod/spa_util/fakedbn
fi

# #############################################
# SETUP HWRF FCST PROCESSING VARIABLES
# #############################################

####################################
# obtain unique process id (pid) and make temp directory
####################################
export pid=$$
export DATA=${DATA:-/tmpnwprd/hwrf${storm_num}_${cyc}_${envir}}
cd $DATA

export cycle=t${cyc}z

####################################
# Specify NET and RUN Name and model
####################################
export NET=hur 
export RUN=hwrf

####################################
# File To Log Msgs
####################################
export jlogfile=${jlogfile:-/com/logs/jlogfile}

####################################
# Determine Job Output Name on System
####################################
export outid="LL$job"
export jobid="${outid}.o${pid}"
export pgmout="OUTPUT.${pid}"
export pgmerr=errfile

####################################
# SENDSMS  - Flag Events on SMS
# SENDCOM  - Copy Files From TMPDIR to $COMOUT
# SENDDBN  - Issue DBNet Client Calls
# RERUN    - Rerun fcst from beginning (default no)
####################################
export SENDSMS=${SENDSMS:-YES}
export SENDCOM=${SENDCOM:-YES}
export SENDDBN=${SENDDBN:-YES}

####################################
# Specify HWRF version number
####################################
export HWRF_VERSION=${HWRF_VERSION:-${model_ver}}

####################################
# Specify Execution Areas
####################################
export HOMEhwrf=${HOMEhwrf:-/nw${envir}/hwrf.$HWRF_VERSION}
export EXEChwrf=${EXEChwrf:-$HOMEhwrf/exec}
export FIXhwrf=${FIXhwrf:-$HOMEhwrf/fix}
export PARMhwrf=${PARMhwrf:-$HOMEhwrf/parm}
export STOREhwrf=${STOREhwrf:-/com/hwrf/${envir}/store}
export EXhwrf=${EXhwrf:-$HOMEhwrf/scripts}
export USHhwrf=${USHhwrf:-$HOMEhwrf/ush}

##############################
# Set up the UTILITIES
##############################
export utilscript=$NWPROD/util/ush
export utilities=$NWPROD/util/ush
export utilexec=$NWPROD/util/exec

##############################
# Run setup to initialize working directory and utility scripts
##############################
sh $utilscript/setup.sh
#export PDY=20120101

##############################
# Run setpdy and initialize PDY variables
##############################
sh $utilscript/setpdy.sh
. PDY

##############################################
# Define COM directories
##############################################
export COMIN=${COMIN:-/com/${NET}/${envir}/${RUN}.${PDY}${cyc}}
export COMOUT=${COMOUT:-/com/${NET}/${envir}/${RUN}.${PDY}${cyc}}
mkdir -m 775 -p $COMOUT

##############################################
msg="HAS BEGUN on `hostname`"
postmsg "$jlogfile" "$msg"

#############################################################
# Execute the script

if [ ${PARAFLAG} = 'YES' ]
then
if [ ${OCEAN} == POM -o ${OCEAN} == NO ]; then
   ${EXhwrf}/exhwrf_coupled_fcst.sh.sms
elif  [ ${OCEAN} == HYCOM ]; then
   ${EXhwrf}/exhwrf_rtofs_coupled_fcst.sh.sms
fi
else
   ${EXhwrf}/exhwrf_coupled_fcst.sh.sms
#   /nw${envir}/scripts/exhwrf_coupled_fcst.sh.sms
fi


#############################################################

msg="ENDED NORMALLY."
postmsg "$jlogfile" "$msg"

date

if [ ${PARAFLAG} == NO ]; then
$SMSBIN/endt
fi