#! /bin/ksh

if [[ "$#" -lt 1 || "$#" -gt 4 ]] ; then
    cat<<EOF 1>&2
This script checks an HWRF cycle for any errors that may have
occurred (unexpected cold start, no ocean coupling, etc.)  It prints
to stdout a ksh-readable list of variables describing the state of
the cycle.

Arguments:
  \$1 -- path to cycle's com directory
  \$2 -- Optional.  Path to tcvitals file (needed for previous cycle's info).
        If the cycle's \$USHhwrf directory contains a hwrf_syndat_tcvitals.sh,
        then that file will be used to find the tcvitals.
  \$3 -- Optional.  Path to cycle's data directory.  Default: use the
        data directory specified by the holdvars file.
  \$4 -- Optional: message number (1, 2, 3, 4, 5).  This is only relevant
        when run operationally (five storms can share the same com and
        data directory).  This defaults to 1.

This script returns status 0 if it was able to determine the cycle's
status, and non-zero otherwise.  (A failed cycle, for which the
status is known, will result in a zero return value.)

Author: Sam Trahan, October 2010

ERROR: ONE, TWO, THREE OR FOUR ARGUMENTS REQUIRED.  ABORTING.
EOF
    exit 1
fi

ASHhwrf=${ASHhwrf:-$USHhwrf}

# This script uses some variable name conventions to avoid clashing
# with holdvars variables:

# _arg_NAME -- an argument to this script
# F_NAME -- the path to file NAME
# i_name -- script loop variable
# c_name -- a constant
# v_name -- some other script variable

# Also, you must not send anything to stdout.  That means all echo
# commands must be followed by 1>&2 and any other commands that might
# send data to stdout (grep, sed, etc.) must either be followed by
# > /dev/null or must have their output captured some other way.

_arg_NOCYCLNG=''
if [[ "$1" == "-nocycling" ]] ; then
    _arg_NOCYCLING=YES
    shift 1
fi

_arg_COM="$1"
_arg_TCVITALS="$2"
_arg_DATA="$3" # will be filed in later if blank
_arg_ISTORM="${4:-1}"

########################################################################
## SUBROUTINES #########################################################
########################################################################

function die {  # sends arguments to stderr and exits script with status -2
    echo "$*" 1>&2
    exit -2
}

function information { # appends arguments to $v_information
    if [[ -z "$v_information" ]] ; then
        v_information="$*"
    else
        v_information="${v_information}  $*"
    fi
}

function warning { # appends arguments to $v_information and $v_warning
    information "$@"
    if [[ -z "$v_warning" ]] ; then
        v_warning="$*"
    else
        v_warning="${v_warning}  $*"
    fi
}

function failure { # appends args. to $v_information, $v_warning, $v_failure
    warning "$*"
    if [[ -z "$v_failure" ]] ; then
        v_failure="$*"
    else
        v_failure="${v_failure}  $*"
    fi
}

function report_and_exit {
    # This function generates the human-readable, script-readable output.
    # NO other code in this script should write to stdout.

   "$ASHhwrf/hwrf_report_heading.sh" "$F_COM" "$F_DATA" "$v_information" "$v_warning" "$v_failure"

    if [[ -z "$v_failure" ]] ; then
        echo "CYCLE_COMPLETE=\"YES\""
    else
        echo "CYCLE_COMPLETE=\"NO\""
    fi

cat<<EOF
STATUS="$v_status"
INIT_FAILURE="$v_init_failure"

# Is it safe to use this cycle as the prerequisite to a future cycle?
PREREQ_USE_OK="$v_prereq_use_ok"

# Cycling: was it used and should it have been used?
CYCLING="$v_cycling"
CYCLING_REMARK="$v_cycling_remark"
PREV_CYCLE_EXISTS="$v_prev_exists"
PREV_CYCLE_GUESS="$v_prev"
NEXT_CYCLE_EXISTS="$v_next_exists"
NEXT_CYCLE_GUESS="$v_next"
SAW_CYCLING_USED_MSG="$v_found_cycling"
SAW_COLD_START_MSG="$v_found_cold_start"

# Completion of ocean model initialization and presence of coupling:
OCEAN_MODEL="$v_ocean_model"
OCEAN_USED="$v_ocean_used"
OCEAN_REMARK="$v_ocean_remark"

# Completion of 12-hour forecast:
TRACK12="$v_track12"
MISSING_COM_WRFOUT_D01="$v_missing_d01"
MISSING_COM_WRFOUT_D02="$v_missing_d02"

# Completion of 126-hour forecast and post-processing:
LAST_GRIB_HOUR="$v_last_hour"
TRACK126="$v_track126"
EOF

    exit 0
}

# Uncomment the "set -x" line for detailed information about this script's
# logic while it runs:

set -x

########################################################################
## INITIALIZE VARIABLES AND SOURCE HOLDVARS FILE #######################
########################################################################

######## SOURCE THE HOLDVARS FILE
F_COM="$_arg_COM"
F_holdvars="$F_COM/storm${_arg_ISTORM}.holdvars.txt"
if [[ -s "$F_holdvars" ]] ; then
    . "$F_holdvars"
else
    die "$F_holdvars: holdvars file is empty or non-existant.  ABORTING"
fi

######## INITIALIZE SOME STATE VARIABLES AND CONSTANTS
c_relocates="hwrf_relocate.out hwrf_relocate.err hwrf_relocate_step1.out hwrf_relocate_step1.err hwrf_relocate_step2.out hwrf_relocate_step2.err"
c_cold_start="This is a cold start"
c_cycling_used="This is not a cold start"
c_ocean_vetoed="Model will run only atmosphere"
v_failure=""
v_init_failure=UNKNOWN
v_warning=""
v_information=""
v_lcstorm=$( echo "$STORM" | tr A-Z a-z )
v_outpre="$v_lcstorm$stormid.$YYYYMMDDHH" # ie: karl13l.2010091500
v_last_hour='UNKNOWN'
v_prereq_use_ok=NO

######## SET FILE LOCATIONS

F_DATA="${_arg_DATA:-$DATA}"
if [[ -x /bin/grep ]] ; then
    F_grep=/bin/grep
elif [[ -x /usr/bin/grep ]] ; then
    F_grep=/usr/bin/grep
else
    die "Unable to find grep.  Tried /bin/grep and /usr/bin/grep.  ABORTING."
fi
F_ndate="${utilexec}/ndate"
F_pre_atmos_out="$F_DATA/hwrf_pre_atmos.out"
F_ocean_status="$F_COM/ocean_status.$STORM$STORMID.$YYYYMMDDHH"
F_trak_12_parent="$F_COM/$stormid.trak.hwrf.atcfunix.$YYYYMMDDHH.parent"
F_trak_12_combine="$F_COM/$stormid.trak.hwrf.atcfunix.$YYYYMMDDHH.combine"
F_final_track="$F_COM/$v_outpre.trak.hwrf.atcfunix"
F_ocean_init_out="$F_DATA/hwrf_ocean_init.out"

######## GUESS TCVITALS LOCATION IF IT WAS NOT SPECIFIED

F_tcvitals="$_arg_TCVITALS"
if [[ -z "$F_tcvitals" ]] ; then
    if [[ -x "$USHhwrf/hwrf_syndat_tcvitals.sh" ]] ; then
        F_tcvitals=$( $USHhwrf/hwrf_syndat_tcvitals.sh $YYYYMMDDHH CACHED )
    elif [[ -x "$ASHhwrf/hwrf_syndat_tcvitals.sh" ]] ; then
        F_tcvitals=$( $ASHhwrf/hwrf_default_syndat_tcvitals.sh $YYYYMMDDHH CACHED )
    fi
fi
if [[ ! -s "$F_tcvitals" ]] ; then
    die "TCVITALS FILE IS EMPTY OR NONEXISTANT: \"$F_tcvitals\".  ABORTING."
fi

########################################################################
## FIGURE OUT WHAT THE NEXT CYCLE IS ###################################
########################################################################

v_next=$( $F_ndate 6 $YYYYMMDDHH )

# $v_tcpattern will contain a regular expression matching lines for the
# previous cycle in the tcvitals:
v_tcpattern=" $STORMID ......... "$( echo $v_next | cut -c 1-8 )" "$( echo $v_next | cut -c9-10 )

# Now check for a previous cycle in the tcvitals
if ( "$F_grep" -E "$v_tcpattern" "$F_tcvitals" < /dev/null > /dev/null ) ; then
    v_next_exists=YES
else
    v_next_exists=NO
fi

########################################################################
## CHECK FOR COMPLETION OF TWELVE-HOUR FORECAST ########################
########################################################################

######## DETERMINE IF THE CYCLE STARTED
if [[ -s "$F_pre_atmos_out" ]] ; then
    v_pre_atmos=PRESENT
else
    v_pre_atmos=MISSING
    v_status=NOT_STARTED
    failure "Cycle hwrf_pre_atmos.out file does not exist or has zero size, so the cycle probably has not yet started."
    report_and_exit
fi

######## DETERMINE IF THE CYCLE FINISHED ITS 12-HOUR TRACKS
if [[ -e "$F_trak_12_parent" && -e "$F_trak_12_combine" ]] ; then
    v_track12=PRESENT
else
    # The 12-hour forecast is not yet complete
    v_track12=MISSING
    v_status="BEFORE_12HR_FORECAST"
    failure "The cycle has not produced its 12-hour tracks yet."
fi

# We only get here if the 12-hour track file exists.  If that file
# exists, then the initialization has completed, and there should be
# some wrfout files in the com directory.  That means we can check
# both.

######## CHECK FOR THE WRFOUT FILES IN THE COM DIRECTORY

# At this point we know the cycle should have created its wrfout files.
# If it hasn't, something went wrong.
v_missing_d01=""
v_missing_d02=""
for i_hour in $( seq 0 6 6 ) ; do
    v_date=$( $F_ndate $i_hour $YYYYMMDDHH )
    v_wrfdate=$( echo $v_date | cut -c1-4 )"-"$( echo $v_date | cut -c5-6 )"-"$( echo $v_date | cut -c7-8 )"_"$( echo $v_date | cut -c9-10 )":00:00"
    v_wrfout1="$F_COM/$stormid.wrfout_d01_$v_wrfdate"
    v_wrfout2="$F_COM/$stormid.wrfout_d02_$v_wrfdate"
    if [[ ! -s "$v_wrfout1" ]] ; then
        v_missing_d01="${v_missing_d01} $i_hour"
    fi
    if [[ ! -s "$v_wrfout2" ]] ; then
        v_missing_d02="${v_missing_d02} $i_hour"
    fi
done
if [[ -z "$v_missing_d01" && -z "$v_missing_d02" ]] ; then
    v_com_wrfout=PRESENT
    v_missing_d01=NONE
    v_missing_d02=NONE
else
    v_com_wrfout=MISSING
    if [[ ! -z "$v_missing_d01" ]] ; then
        failure "Missing domain 1 wrfout files in com directory: $v_missing_d01."
    fi
    if [[ ! -z "$v_missing_d02" ]] ; then
        failure "Missing domain 2 wrfout files in com directory: $v_missing_d02."
    fi
fi

########################################################################
## CHECK CYCLE INITIALIZATION ##########################################
########################################################################

v_init_failure=NO

# If we get here, the cycle was able to run WRF and produce some GRIB
# files, so the initialization must have completed, though possibly
# not successfully.  Now we'll check to see if the initialization ran
# correctly.

######## CHECK FOR OCEAN INIT

v_ocean_model="${OCEAN}"
if [[ -z "$v_ocean_model" ]] ; then
    warning "No \$OCEAN variable was set in the holdvars file."
fi

# First, should we even run the ocean model?
if [[ $MODEL == COUPLED ]] ; then
    if ( "$F_grep" "$c_ocean_vetoed" "$F_ocean_init_out"  < /dev/null > /dev/null ) ; then
        v_ocean_used=WANTED_BUT_VETOED
        v_ocean_remark="Found $c_ocean_vetoed in $F_ocean_init_out so ocean init vetoed coupling."
    elif [[ -s "$F_ocean_status" ]] ; then
        v_run_coupled=$( . "$F_ocean_status" > /dev/null ; echo "$RUN_COUPLED" )
        if [[ "$v_run_coupled" == YES ]] ; then
            v_ocean_used=RUN_COUPLED_YES
            v_ocean_remark="Ocean coupling was used."
        elif [[ "$v_run_coupled" == NO ]] ; then
            v_ocean_used=RUN_COUPLED_NO
            v_ocean_remark="Ocean coupling was requested, but was unused (RUN_COUPLED=NO)."
            failure "$v_ocean_remark"
            v_init_failure=YES
        else
            v_ocean_used=RUN_COUPLED_UNKNOWN
            v_ocean_remark="Ocean coupling was requested but unused.  RUN_COUPLED had an unknown value of $v_run_coupled."
            failure "$v_ocean_remark"
            v_init_failure=YES
        fi
    else
        v_ocean_used=NO_OCEAN_STATUS_FILE
        v_ocean_remark="Ocean coupling was requested but no ocean status file was present."
        failure "$v_ocean_remark"
        v_init_failure=YES
    fi
else
    v_ocean_used=ATMOS_REQUESTED
    v_ocean_remark="No ocean coupling was requested (MODEL=ATMOS in holdvars)."
fi

######## CHECK FOR CYCLING

if [[ "$INIT" == NO ]] ; then
    v_cycling=INIT_DISABLED
    v_cycling_remark="In the holdvars file, initialization is disabled (\$INIT=NO)."
else
    v_prev=$( $F_ndate -6 $YYYYMMDDHH )

    # $v_tcpattern will contain a regular expression matching lines for the
    # previous cycle in the tcvitals:
    v_tcpattern=" $STORMID ......... "$( echo $v_prev | cut -c 1-8 )" "$( echo $v_prev | cut -c9-10 )

    # Now check for a previous cycle in the tcvitals
    if [[ _arg_NOCYCLING == "YES" ]] ; then
        v_prev_exists=NO
    else
        if ( "$F_grep" -E "$v_tcpattern" "$F_tcvitals" < /dev/null > /dev/null ) ; then
            v_prev_exists=YES
        else
            v_prev_exists=NO
        fi
    fi

    v_found_cold_start=NO
    v_found_cycling=NO
    for i_basename in $c_relocates ; do
        F_relocate="$F_DATA/$i_basename"
        if [[ ! -s "$F_relocate" ]] ; then
            # This relocate*.out file does not exist or has zero size,
            # so don't bother checking it.
            continue
        fi
        if ( grep "$c_cold_start" "$F_relocate" > /dev/null ) ; then
            v_found_cold_start=YES
        fi
        if ( grep "$c_cycling_used" "$F_relocate" > /dev/null ) ; then
            v_found_cycling=YES
        fi
    done

    if [[ "$v_found_cycling" == YES ]] ; then
        if [[ "$v_prev_exists" == YES ]] ; then
            v_cycling=CYCLING_USED
            v_cycling_remark="Cycling was used."
        else
            v_cycling=CYCLING_UNEXPECTEDLY_USED
            v_cycling_remark="Cycling was used, but no prior cycle was in the tcvitals (incorrect tcvitals to hwrf_check_cycle.sh?)"
            warning "$v_cycling_remark"
        fi
    elif [[ "$v_found_cold_start" == YES ]] ; then
        if [[ "$v_prev_exists" == YES ]] ; then
            v_cycling=UNEXPECTED_COLD_START
            v_cycling_remark="This cycle should have used $v_prev for cycling but it did not (found $c_cold_start but not $c_cycling_used in relocation out file(s))."
            v_init_failure=YES
            failure "$v_cycling_remark"
        else
            v_cycling=COLD_START
            v_cycling_remark="This is a cold start; no prior cycle existed."
            information "$v_cycling_remark"
        fi
    else
        v_cycling=UNKNOWN
        v_init_failure=YES
        failure "Could not determine if cycling succeeded or not, but it should be used (INIT=YES)"
    fi
fi

if [[ ! -z "$v_failure" ]] ; then
    # Either the 12-hour forecast has failed or initialization failed,
    # so don't check the 126-hour forecast
    v_status=STARTED
    report_and_exit
else
    # Initialization was successful, and a 12-hour forecast was created.
    v_status=HAVE_12HR_FORECAST
    v_prereq_use_ok=YES
fi


########################################################################
## CHECK THE 126-HOUR FORECAST #########################################
########################################################################

if [[ ! -e "$F_final_track" ]] ; then
    failure "Track file does not yet exist."
    v_track126=MISSING
elif [[ ! -s "$F_final_track" ]] ; then
    warning "Track file exists but is empty: $F_final_track"
    v_track126=EMPTY
else
    v_track126=PRESENT
fi

i_last_hour=-999
i_missing_total=0
i_expected_total=0
for i_pre in hwrfeprs_n hwrfprs_c hwrfprs_n hwrfprs_p ; do
    i_missing=0
    i_expected=0
    for i_hour in $( seq 0 6 126 ) ; do
        v_hour00=$( printf "%02d" $i_hour ) # zero-pad to two digits
        
        v_file="$F_COM/$v_outpre.$i_pre.grbf$v_hour00"
        i_expected=$(( i_expected + 1 ))
        if [[ ! -s "$v_file" ]] ; then
            i_missing=$(( i_missing + 1 ))
        elif (( i_hour > i_last_hour )) ; then
            i_last_hour=$i_hour
        fi
    done
    i_missing_total=$(( i_missing_total + i_missing ))
    i_expected_total=$(( i_expected_total + i_expected ))
    if (( i_missing > 0 )) ; then
        warning "Missing $i_missing of $i_expected six-hourly $v_outpre.$i_pre files"
    fi
done
if (( $i_last_hour < 0 )) ; then
    failure "No six-hourly grib files found yet."
elif (( i_missing_total > 0 )) ; then
    failure "Missing $i_missing_total of $i_expected_total six-hourly grib files.  Last hour found was $i_last_hour."
    v_last_hour=$i_last_hour
elif [[ "$v_track126" != MISSING ]] ; then
    v_status=HAVE_TRACK_AND_GRIBS
    v_last_hour=$i_last_hour
else
    v_status=HAVE_GRIBS_NO_TRACK
    v_last_hour=$i_last_hour
fi

report_and_exit