#!/bin/sh
#########################################################################
# hwrf_c_execute.sh 
######################### 
# This script is used in running the HWRF forecast model (AM) coupled with
# either GFDL (POM) or HYCOM ocean model (OM) and
# with Wave Watch wave model (WM).
# It is called from [hwrf_coupled_fcst.sh].
# April 2007 Written by Dmitry Sheinin
# April 2007 Modified for Operations by Vijay Tallapragada.
# Feb. 2009 Modified for 3-way coupled system by Dmitry Sheinin
# March 2009 Modified for Operations by Vijay Tallapragada.
# Script modifications for 2009 HWRF implementation - Vijay Tallapragada
# Added RESTART and AUTO_RSTART options June 2009
# Jan 2011: Added east pac domain for ocean coupling (Biju Thomas)
#########################################################################
# Vars, required:
#
# OM_EXE         # OM executable, may be undefined if there is no OM
# AM_EXE         # AM executable, may be undefined if there is no AM
# WMEXE          # WW3 executable, may be undefined if there is no WM
# C_EXE          # name of Coupler executable
# C_execdir      # Directory where the coupler executable resides 
# NPROCS_c       # number of CPUs for Coupler executable, must be 1 if
#                # Coupler is present and 0 to run OM (AM) standalone
# NPROCS_o       # number of CPUs for OM executable, must be 0 if OM is
#                # not present
# NPROCS_a       # number of CPUs for AM executable, must be 0 if AM is
#                # not present
# NPROCS_w       # number of CPUs for WM executable, must be 0 if WM is
#                # not present
#                #   The sum of the above 4 must = the number of CPUs requested
# WORK_DIR       # working directory, where execution is performed
# C_RESTART      # "y" if this is a continuation of a run with Coupler present,
#                # "n" otherwise
# C_PERIOD       # Coupling period in seconds
# C_NSECS        # Forcast length in seconds
#
# Vars, optional:    # so far, never mind
#
# C_NSBC                    (6)
# PRINT_DIR
# C_separate_stdout
####################################################################
set -x

PS4=' + hwrf_c_execute.sh line $LINENO: '

AM_EXE=wrf.exe
OMEXE=ocean_united.xc
OMEXE_EA=ocean_eastatl.xc
OMEXE_EP=ocean_eastpac.xc

##### echo variables exported into the script ##########
echo 'region     = ' ${region}
echo 'CASE       = ' ${CASE}
echo 'NPROCS_c   = ' ${NPROCS_c}
echo 'NPROCS_o   = ' ${NPROCS_o}
echo 'NPROCS_a   = ' ${NPROCS_a}
echo 'NPROCS_w   = ' ${NPROCS_w}
echo 'WMEXE      = ' ${WMEXE}
echo 'OMEXE      = ' ${OMEXE}
echo 'OMEXE_EA   = ' ${OMEXE_EA} 
echo 'OMEXE_EP   = ' ${OMEXE_EP}
echo 'CSTREAM    = ' ${CSTREAM}
echo 'WORK_DIR   = ' ${WORK_DIR}
echo 'C_RESTART  = ' ${C_RESTART}
echo 'C_PERIOD   = ' ${C_PERIOD}
echo 'C_NSECS    = ' ${C_NSECS}
echo 'C_NSBC     = ' ${C_NSBC}
echo 'C_EXE      = ' ${C_EXE} 
echo 'WAVEWATCH  = ' ${WAVEWATCH} 
########################################################

if [[ $region == united ]]; then
    OM_EXE=$OMEXE
elif [[ $region == eastatl ]]; then
    OM_EXE=$OMEXE_EA
elif [[ $region == eastpac ]]; then
   OM_EXE=$OMEXE_EP
else
   OM_EXE="UnDefined"
fi

echo "region = ${region}; OM_EXE = ${OM_EXE}"  
#-------------------------------------------
if [[ $CASE == FCST ]]; then
    PRINT_DIR=$CSTREAM/LOG
else 
    PRINT_DIR=$CSTREAM/LOG/3DVAR
fi
echo "CASE = ${CASE}; PRINT_DIR = ${PRINT_DIR}"  

#------------- BEGIN C.pre --------------------------

echo '*** BEGIN C.pre'`date`

if [ "$NPROCS_c" != 0 -a "$NPROCS_c" != 1 ]; then
    echo "NPROCS_c=$NPROCS_c not properly defined; aborting"
    sh ${utilscript}/setup.sh
    err=911
    err_exit "NPROCS_c=$NPROCS_c not properly defined; aborting"
fi
#----------------------------------------------------
if [ ! "$NPROCS_o" ]; then
    echo "NPROCS_o not defined; aborting"
    sh ${utilscript}/setup.sh
    err=911
    err_exit "NPROCS_o not defined; aborting"
fi
#----------------------------------------------------
if [ ! "$NPROCS_a" ]; then
    echo "NPROCS_a not defined; aborting"
    sh ${utilscript}/setup.sh
    err=911
    err_exit "NPROCS_a not defined; aborting"
fi
#----------------------------------------------------
if [ ! "$NPROCS_w" ]; then
    echo "NPROCS_w not defined; aborting"
    sh ${utilscript}/setup.sh
    err=911
    err_exit "NPROCS_w not defined; aborting"
fi
#----------------------------------------------------
if [ "$WORK_DIR" ]; then
    if [ -d $WORK_DIR ]; then
        workdir=$WORK_DIR
        cd $workdir
    else 
        echo "WORK_DIR=$WORK_DIR not a dir., aborting"
        sh ${utilscript}/setup.sh
        err=911
        err_exit "$WORK_DIR not a dir., aborting"
    fi
else
    echo "WORK_DIR not defined, aborting"
    sh ${utilscript}/setup.sh
    err=911
    err_exit "$WORK_DIR not defined, aborting"
fi
#----------------------------------------------------
if [ "$C_RESTART" = y -o "$C_RESTART" = Y -o "$C_RESTART" = yes -o "$C_RESTART" = YES ]; then
    C_RESTART=y
    C_restart=T
elif [ "$C_RESTART" = n -o "$C_RESTART" = N -o "$C_RESTART" = no -o "$C_RESTART" = NO ]; then
    C_RESTART=n
    C_restart=F
else
    echo 'C_RESTART not properly defined (must be y, yes, n, no in u. or l. case), aborting'
    sh ${utilscript}/setup.sh
    err=911
    err_exit 'C_RESTART not properly defined , aborting'
fi
#-----------------------------------------------------
if [ ! "$C_PERIOD" ]; then
    echo "C_PERIOD not defined; aborting"
    sh ${utilscript}/setup.sh
    err=911
    err_exit 'C_PERIOD not defined; aborting'
fi
#-----------------------------------------------------
if [ ! "$C_NSECS" ]; then
    echo "C_NSECS not defined; aborting"
    sh ${utilscript}/setup.sh
    err=911
    err_exit 'C_NSECS not defined; aborting'
fi
#-----------------------------------------------------
if [ ! "$C_NSBC" ]; then
    C_NSBC=10
    echo "C_NSBC not defined; C_NSBC=$C_NSBC set"
else
    echo "Externally defined C_NSBC=$C_NSBC"
    if [ $C_NSBC != 10 ]; then
        echo "WARNING: C_NSBC .ne. 10"
    fi
fi
#-------------------------------------------------------
if [ ! "$PRINT_DIR" ]; then
    PRINT_DIR=`pwd`
    echo "PRINT_DIR not defined, set to current $PRINT_DIR"
fi
#-------------------------------------------------------

if [ -x ${EXEChwrf}/$C_EXE ]; then :
else
    echo "No executable $C_execdir/$C_EXE : aborting"
    sh ${utilscript}/setup.sh
    err=911
    err_exit 'No executable $C_execdir/$C_EXE : aborting'
fi
echo "Coupler executable: $C_execdir/$C_EXE"

cp ${EXEChwrf}/$C_EXE .
#---------------------------------------------------------
ncpu=NPROCS_o
for fluxes in fluxes_for_[O,W]M; do
#
    if [ ! "$ncpu" ]; then continue; fi
    if [ $ncpu -eq 0 ]; then continue; fi
    if [ $C_RESTART = n ]; then
        echo "C cold start: removing $workdir/$fluxes if exists..."
        rm -f $fluxes
    else
        echo "C restart ($fluxes to be used): "
        if [ "$C_RESTART_DIR" ]; then
            if [ -d $C_RESTART_DIR ]; then
                echo "Copying $fluxes from $C_RESTART_DIR ..."
                if cp $C_RESTART_DIR/$fluxes .; then :
                else
                    echo "Copying $C_RESTART_DIR/$fluxes FAILED, aborting"
                    sh ${utilscript}/setup.sh
                    err=911
                    err_exit "Copying $C_RESTART_DIR/$fluxes FAILED, aborting"
                fi
            else
                echo "C_RESTART_DIR=$C_RESTART_DIR is not a dir., aborting"
                sh ${utilscript}/setup.sh
                err=911
                err_exit "C_RESTART_DIR=$C_RESTART_DIR is not a dir., aborting"
            fi
        else
            echo "RESTART_DIR not defined; $fluxes from WORK_DIR=$WORK_DIR to be used"
        fi
        if [ ! -f $fluxes ]; then
            echo "No file fluxes_for_OM in WORK_DIR=$WORK_DIR; aborting"
            sh ${utilscript}/setup.sh
            err=911
            err_exit "No file fluxes_for_OM in WORK_DIR=$WORK_DIR; aborting"
        fi
    fi
    ncpu=NPROCS_w
#
done
#----------------------------------------------------------
C_cstepmax=`expr $C_NSECS \/ $C_PERIOD`

echo "C_restart=$C_restart"
echo "C_cstepmax=$C_cstepmax"

nl=cpl_nml
echo "Creating C namelist file $nl ..."
echo '&CPL_SETTINGS'>$nl
echo "restart=$C_restart,">>$nl
echo "cstepmax=$C_cstepmax,">>$nl
if [ "$C_PERIOD" ]; then echo "dt_c=$C_PERIOD,">>$nl; fi
i=1
while [ $i -le $C_NSBC ]; do
    if [ "${C_SHT_SF[$i]}" ]; then
        echo "SHT_SF($i)=${C_SHT_SF[$i]}">>$nl
    fi
    i=`expr $i + 1`
done
if [ "$C_NO_AM_SST" ]; then echo "no_AM_SST=$C_NO_AM_SST,">>$nl; fi
if [ "$C_nprint" ]; then echo "nprint=$C_nprint,">>$nl; fi
if [ "$C_VL" ]; then echo "VerbLev=$C_VL,">>$nl; fi
if [ "$C_nprper" ]; then echo "nprper=$C_nprper,">>$nl; fi
if [ "$C_npr1st" ]; then echo "npr1st=$C_npr1st">>$nl; fi
echo '/'>>$nl

cat $nl

for file in C_printout.*; do
    if [ -f $file ]; then
        echo "removing $file"
        rm -f $file
    fi
done

echo 'END C.pre'

#----------------- END C.pre ----------------------------------#

#----------------- BEGIN modify task geometry -----------------#
if [[ -x "$USHhwrf/hwrf_task_geometry.sh" ]] ; then
    let 'ocean_procs=NPROC_o+NPROC_c+NPROC_w'
    "$USHhwrf/hwrf_task_geometry.sh"
fi
#----------------- END modify task geometry -------------------#

#----------------- BEGIN all.execute --------------------------#

echo 'BEGIN all.execute'

if [ ${WAVEWATCH} == YES ]; then

    if [ ${OCEAN} == POM -o ${OCEAN} == HYCOM ]; then
        NPROCS=`expr $NPROCS_c + $NPROCS_o + $NPROCS_a + $NPROCS_w`
        echo "NPROCS used for this model run is" $NPROCS 
    else
        NPROCS=`expr $NPROCS_c + $NPROCS_a + $NPROCS_w`
        echo "NPROCS used for this model run is" $NPROCS 
    fi

else

    if [ ${OCEAN} == POM -o ${OCEAN} == HYCOM ]; then

        NPROCS=`expr $NPROCS_c + $NPROCS_o + $NPROCS_a`
        echo "NPROCS used for this model run is" $NPROCS 

    else

        NPROCS=$NPROCS_a
        echo "NPROCS used for this model run is" $NPROCS 

        echo "Model can't run coupled, no ocean or wave coupling is requested"
        echo "Will run stand alone atmospheric model"

        ${USHhwrf}/hwrf_nmm_fcst.sh  ${STORM} ${YYYY} ${MM} ${DD} ${CYC} ${CASE}

        exit

    fi
fi

if [ $NPROCS -eq $NPROCS_w -o $NPROCS -eq $NPROCS_o -o $NPROCS -eq $NPROCS_a ]; then
    standalone=y
else
    npf_c=0
    npf_o=$NPROCS_c
    npf_a=`expr $npf_o + $NPROCS_o`
    npf_w=`expr $npf_a + $NPROCS_w`
    npl_c=`expr $npf_o - 1`
    npl_o=`expr $npf_a - 1`
    npl_a=`expr $npf_w - 1`
    npl_w=`expr $NPROCS - 1`
fi

rm -f out*

# run the executable in poe  
echo 'time control: before poe execution '`date`

export MP_SHARED_MEMORY=yes   # use shared memory for collective MPI routines
export MP_EAGER_LIMIT=131072  # size in bytes of send buffer

# Override number of processors in case task affinity is in use:
export MP_PROCS=$( echo $NPROCS ) # the "echo" gets rid of leading spaces

# For debugging purposes:
echo ===================== ENV BEFORE POE =====================
env
echo ===================== END ENV LISTING ====================

moreargs="-midtasks $(( NPROCS_a + NPROCS_w - IOSRV_PERGRP*IOSRV_GROUPS ))"
if [[ ! -z "$iomachines" ]] ; then
    moreargs="${moreargs:+$moreargs }-endtasks $iomachines"
fi
if [[ "$NPROCS_c" -gt 0 && "$NPROCS_o" -gt 0 ]] ; then
    moreargs="${moreargs:+$moreargs }-starttasks $(( NPROCS_c+NPROCS_o ))"
fi

"$USHhwrf/hwrf_run.sh" -mpmd $moreargs \
    -n "$NPROCS_c" "$C_EXE"  : \
    -n "$NPROCS_o" "$OM_EXE" : \
    -n "$NPROCS_w" "$WM_EXE" : \
    -n "$NPROCS_a" "$AM_EXE" \
    > "$PRINT_DIR/out"
err="$?"

if ( ! grep 'SUCCESS COMPLETE' rsl.out.0000 ) ; then
    echo "DID NOT FIND STRING \"SUCCESS COMPLETE\" IN rsl.out.0000.  WRF FAILED."
    echo "DID NOT FIND STRING \"SUCCESS COMPLETE\" IN rsl.out.0000.  WRF FAILED." 1>&2
    BOMB=1
fi

echo 'time control: after poe execution '`date`
cat $PRINT_DIR/out

if [[ "$PARAFLAG" == YES ]] ; then
    "$USHhwrf/hwrf_port_cachesync.sh"
fi

if [ "$BOMB" -eq 1 ]; then
    echo '#C# ** executable (poe invoked) ended ABNORMALLY (SUCCESS COMPLETE not found, exit status $err) **'
    echo "executable hwrf_wrf for poe, CASE = $CASE, return code err = $err"
else
    echo "#C# poe done"
fi

if [ "$BOMB" -eq 1 ]; then
    if [ $AUTO_RSTART == YES ]; then
        sh ${utilscript}/setup.sh
        err=911
        err_exit "COUPLED FORECAST FAILED BUT AUTO RESTART IS NOT IMPLEMENTED IN THIS VERSION"
    fi
fi

for s in OM AM WM; do
    if [ -f $PRINT_DIR/out.$s ]; then mv out.$s $PRINT_DIR; fi
    if [ -f out.$s ]; then mv out.$s $PRINT_DIR; fi
done

if [ ! "$standalone" ]; then
    separate_stdout=${C_separate_stdout:-/bin/true}
    if [ -x $separate_stdout ]; then
        echo $npf_c $npf_o $npf_a $npf_w $npl_c $npl_o $npl_a $npl_w | $separate_stdout $PRINT_DIR/out $PRINT_DIR/out_C $PRINT_DIR/out_OM $PRINT_DIR/out_AM $PRINT_DIR/out_WM
        rc=$?
        if [ $rc -ne 0 ]; then
            echo "$separate_stdout aborted"
        fi
    fi
fi

echo '*** END all.execute '`date`

#-------------- END all.execute -----------------------#

# end of script