#!/bin/sh
###############################################################################
#                                                                             #
# This is the master script to generates GRIB2 file for the MWW3 forecast     #
#  model as the output is generated. The script is called together with the   #
#  script that starts the model run and is completed when the grib2 file      #
#  for the last step is generated or when the abort signal from the master    #
#  script is sent out. It runs a child script to generate grib2 files for     #
#  individual steps                                                           #
#                                                                             #
# Remarks :                                                                   #
# - This script runs side by side with the forecast script and relies on P6   #
#   IBM load leverler that allows multiple scripts to be launched from the    #
#   same job card                                                             # 
# - For each grid the script generates lock file in $DATA directory that      #
#   remains in place till all the grib2 files have been generated. Removing   #
#   this file will lead to a forced end of individual calls to grib2          #
#   generating shell scripts                                                  #
# - The script can also run Independantly. The flag for the ind. run is       #
#   INDRUN                                                                    # 
# - See section 0.b for variables that need to be set.                        # 
#                                                                             #
#                                                                Jan, 2009    #
# Last update : 03-30-2011                                                    #
#                                                                             #
###############################################################################

set -x

# Use LOUD variable to turn on/off trace.  Defaults to YES (on).
export LOUD=${LOUD:-YES}; [[ $LOUD = yes ]] && export LOUD=YES

[[ $LOUD != YES ]] && set +x

#
# --------------------------------------------------------------------------- #
# 0.  Preparations
# 0.a Basic modes of operation

# 0.a.1 Switch to indicate if the script is to run in parallel with the 
# exwave_${modID}_fcst.sh.sms script (INDRUN='no') or to be run independantly
# after the forecast script has finished (INDRUN='yes')

  export INDRUN=${INDRUN:-'no'}   #  default to no.  yes/no can be passed in from calling script.

  cd $DATA 

  postmsg "$jlogfile" "HAS BEGUN on `hostname`"

  msg="Starting MWW3 WAVE MODEL GRIB2 GENERATION SCRIPT for $modID"
  postmsg "$jlogfile" "$msg"

  set +x
  echo ' '
  echo '                      *******************************'
  echo '                      *** MWW3 MODEL GRIB2 SCRIPT ***'
  echo '                      *******************************'
  echo '                          Global multi-grid model with GFS forcing'
  echo '                          Model identifier : multi_1'
  echo ' '
  echo "Starting at : `date`"
  [[ $LOUD = YES ]] && set -x

  export MP_PGMMODEL=mpmd
  export MP_CMDFILE=cmdfile

  export MODNR=11
  err=0
  GribOK='yes'

# --------------------------------------------------------------------------- #
# 1.  Setup
# 1.a Cycle time info

  export date=$PDY
  export YMDH=${PDY}${cyc}

# The number of hours of forecast should match between this script and the master forecast script

  lsth=${lsth:-180}

  ymdh_end=`$utilexec/ndate $lsth $YMDH`
  export dtgrib=10800
 # export GRIBFLAGS='F F T F F  T F F F F  T T F F T  T F T F F  F F F F F  F F F F F  F'
  export GRIBFLAGS='F F T F F  F T F F F  F F T F F  T T T F T  F F F F F  F F F F F  F F F F F   F F F F F   F F F F F   F F F F F  F F F'

  set +x
  echo ' '
  echo "   Starting time     : $YMDH"
  echo "   Ending time       : $ymdh_end"
  echo "   Time step         : $dtgrib"
  echo "   GRIB field flags  : $GRIBFLAGS"
  echo ' '
  [[ $LOUD = YES ]] && set -x

# 1.b Grids used

  export grids='glo_30m ao_30m at_10m wc_10m ep_10m ak_10m at_4m wc_4m ak_4m'
  export int_grids='glo_30mext'

# --------------------------------------------------------------------------- #
# 2. Input files
# 2.a Mod def files. 
# If run in parallel mode with the forecast job these files should already
# exist as they will be generated by that job. Only if these files do not
# exist (when run in independant mode) will the new ones be generated 

  for grdID in $grids $int_grids
  do
    if [ ! -f mod_def.$grdID ]
    then
      echo "  mod_def.$grdID not available, generating a new one ..."
      if [ -f "$COMIN/multiwave_${modID}_${grdID}.moddef.${model_ver}" ]
      then
        echo " Mod def file for $grdID found in $COMIN. copying ...."
        cp $COMIN/multiwave_${modID}_${grdID}.moddef.${model_ver} mod_def.$grdID
      else
        if [ -f $FIXwave/multiwave_$grdID.inp ]
        then
          cp $FIXwave/multiwave_$grdID.inp $grdID.inp
        fi

        if [ -f $grdID.inp ]
        then
          echo "   $grdID.inp copied ($FIXwave/multiwave_$grdID.inp)."
        else
          msg="ABNORMAL EXIT: NO INP FILE FOR MODEL DEFINITION FILE"
          postmsg "$jlogfile" "$msg"
          set +x
          echo ' '
          echo '*********************************************************** '
          echo '*** FATAL ERROR : NO INP FILE FOR MODEL DEFINITION FILE *** '
          echo '*********************************************************** '
          echo "                                grdID = $grdID"
          echo ' '
          echo $msg
          [[ $LOUD = YES ]] && set -x
          echo "$modID grib2 $date $cycle : $grdID.inp missing." >> $wavelog
          err=1;export err;err_chk
        fi

        echo "   Generating mod_def file for $grdID ... "
        $USHwave/multiwavemod_def.sh $grdID > $grdID.out
      fi

      if [ ! -f mod_def.$grdID ]
      then
        msg="ABNORMAL EXIT: NO MODEL DEFINITION FILE"
        postmsg "$jlogfile" "$msg"
        set +x
        echo ' '
        echo ' ********************************************** '
        echo ' *** FATAL ERROR : NO MODEL DEFINITION FILE *** '
        echo ' ********************************************** '
        echo ' '
        echo $msg
        [[ $LOUD = YES ]] && set -x
        echo "$modID fcst grib2 $grdID $date $cycle : fixed file(s) missing." >> $wavelog
        if [ -f $grdID.out ]
        then
          sed "s/^/$grdID.out : /g"  $grdID.out
        fi 
        err=2;export err;err_chk
      fi

    fi
  done

# 2.b Template files

  if [ -f $FIXwave/multiwavegrib2_single.inp.tmpl ]
  then
    cp $FIXwave/multiwavegrib2_single.inp.tmpl .
  fi

  if [ ! -f multiwavegrib2_single.inp.tmpl ]
  then
    msg="ABNORMAL EXIT: NO GRIB2 TEMPLATE FILE"
    postmsg "$jlogfile" "$msg"
    set +x
    echo ' '
    echo ' ******************************************** '
    echo ' *** FATAL ERROR : NO GRIB2 TEMPLATE FILE *** '
    echo ' ******************************************** '
    echo ' '
    echo $msg
    [[ $LOUD = YES ]] && set -x
    echo "$modID fcst $date $cycle : grib2 template file missing." >> $wavelog
    err=3;export err;err_chk
  fi

  for grd in $int_grids
  do
    if [ -f $FIXwave/${grd}_interp.inp.tmpl ]
    then
      cp $FIXwave/${grd}_interp.inp.tmpl .
    fi

    if [ ! -f ${grd}_interp.inp.tmpl ]
    then
      msg="ABNORMAL EXIT: NO GRID INTERPOLATION TEMPLATE FILE"
      postmsg "$jlogfile" "$msg"
      set +x
      echo ' '
      echo ' ********************************************************* '
      echo ' *** FATAL ERROR : NO GRID INTERPOLATION TEMPLATE FILE *** '
      echo ' ********************************************************* '
      echo ' '
      echo $msg
      [[ $LOUD = YES ]] && set -x
      echo "$modID fcst $date $cycle : grid interpolation template file missing.  " >> $wavelog
      err=4;export err;err_chk
    fi
  done

# 2.c Awips Headers for grib files

  for grdID in $grids
  do

    if [ "$grdID" != 'ao_30m' ]
    then
      NR=`ls $PARMwave/grib2_awips_${modID}_fh*.$grdID | wc -l | awk '{print $1}'`
      if [ "$NR" -gt '0' ]
      then
        set +x
        echo " $grdID : Found $NR parm files for transmitting grib2 data via AWIPS "
        [[ $LOUD = YES ]] && set -x
        cp $PARMwave/grib2_awips_${modID}_fh*.$grdID .
      else
        msg="WARNING!! NO PARM FILES FOUND FOR GRID $grdID"
        postmsg "$jlogfile" "$msg"
        set +x
        echo ' '
        echo ' ************************************************ '
        echo ' *** WARNING : PARM FILES FOR AWIPS NOT FOUND *** '
        echo ' ************************************************ '
        echo ' '
        echo $msg
        [[ $LOUD = YES ]] && set -x
        echo "$modID fcst $date $cycle $grdID : Parm files missing. " >> $wavelog
      fi

      NR=`ls $PARMwave/grib2_awips_wstp_${modID}_fh*.$grdID | wc -l | awk '{print $1}'`
      if [ "$NR" -gt '0' ]
      then
        set +x
        echo " $grdID : Found $NR parm files for transmitting grib2 wave steepness data via AWIPS "
        [[ $LOUD = YES ]] && set -x
        cp $PARMwave/grib2_awips_wstp_${modID}_fh*.$grdID .
      else
        msg="WARNING!! NO WAVE STEEPNESS PARM FILES FOUND FOR GRID $grdID"
        postmsg "$jlogfile" "$msg"
        set +x
        echo ' '
        echo ' *************************************************************** '
        echo ' *** WARNING : WAVE STEEPNESS PARM FILES FOR AWIPS NOT FOUND *** '
        echo ' *************************************************************** '
        echo ' '
        echo $msg
        [[ $LOUD = YES ]] && set -x
        echo "$modID fcst $date $cycle $grdID : Wave Steepness Parm files missing. " >> $wavelog
      fi

    fi

  done

# 2.d Input files only if script running in independant settings  

  if [ "$INDRUN" = 'yes' ]
  then

# 2.d.i Raw output files and grid mod_def files 

    for grdID in $grids
    do
      if [ ! -f out_grd.$grdID ]
      then
        set +x
        echo "   Copying $modID.$grdID.$cycle.outgrd from $COMIN to out_grd.$grdID"
        [[ $LOUD = YES ]] && set -x
        cp $COMIN/$modID.$grdID.$cycle.outgrd out_grd.$grdID
      fi

      if [ ! -f out_grd.$grdID ]
      then
        set +x
        echo ' '
        echo '**************************************** '
        echo '*** ERROR : NO RAW FIELD OUTPUT FILE *** '
        echo '**************************************** '
        echo ' '
        [[ $LOUD = YES ]] && set -x
        echo "$modID post $grdID $date $cycle : field output missing." >> $wavelog
        postmsg "$jlogfile" "NON-FATAL ERROR : NO RAW FIELD OUTPUT FILE"
        err=5
        GribOK='no'
      fi

      if [ ! -f mod_def.$grdID ]
      then
        echo "   Copy mod_def.$grdID from $FIXwave/multiwave_$grdID.moddef.${model_ver}"
        if [ -f $FIXwave/multiwave_$grdID.moddef.${model_ver} ]
        then
          cp $FIXwave/multiwave_$grdID.moddef.${model_ver} mod_def.$grdID
        fi
      fi

      if [ ! -f mod_def.$grdID ]
      then
        msg="ABNORMAL EXIT: NO MODEL DEFINITION FILE"
        postmsg "$jlogfile" "$msg"
        set +x
        echo ' '
        echo ' ********************************************** '
        echo ' *** FATAL ERROR : NO MODEL DEFINITION FILE *** '
        echo ' ********************************************** '
        echo ' '
        echo $msg
        [[ $LOUD = YES ]] && set -x
        echo "$modID fcst grib2 $grdID $date $cycle : fixed file(s) missing." >> $wavelog
        err=6;export err;err_chk
      fi

    done

# 2.d.ii Processed grib2 files

    echo "    Checking completed grib2 files in $COMIN ..."

    rm -f gribdone.* 

    for grdID in $grids $int_grids
    do
      for file in `ls $COMIN/$modID.$grdID.$cycle.f???.grib2`
      do
       hr=`echo $file | cut -d. -f5 | cut -c3-5`
       ymdh=`$utilexec/ndate $hr $YMDH`
       touch gribdone.$ymdh.$grdID 
      done
    done

  fi
 
# 2.e Final output before execution of script

  if [ "$INDRUN" = 'yes' ]
  then
    set +x
    echo ' '
    echo '       ##########################################################'
    echo '       # NOTE : The script is executing in independent mode. If #'
    echo '       # this was not the intention then make sure that the tmp #'
    echo '       # $DATA is not deleted as this switches the script to    #'
    echo '       # independent mode. The script assumes that the forecast #'
    echo '       # of the model has been completed and all the files are  #'
    echo '       # available (either in $DATA or $COMIN) for running this #'
    echo '       # script                                                 #' 
    echo '       ##########################################################'
    echo ' '
    [[ $LOUD = YES ]] && set -x
  else
    set +x
    echo ' '
    echo '       ##########################################################'
    echo '       # NOTE : The script is executing in parallel mode. It is #'
    echo '       # expecting the tmp $DATA directory to be populated with #'
    echo '       # the necessary data from a parallel forecast script in  #'
    echo '       # execution (or to be executed soon).                    #'
    echo '       ##########################################################'
    echo ' '
    [[ $LOUD = YES ]] && set -x
  fi

# --------------------------------------------------------------------------- #
# 3. Start Grib2 generation
# 3.a Set up lock files

  for grdID in $grids $int_grids
  do
    rm -f grib2.$grdID.lock
    touch grib2.$grdID.lock
  done


# 3.b Setup poe command

  ifile=1
  nfile=`echo $LOADL_PROCESSOR_LIST | wc -w | awk '{ print $1}'`

  if [ "$nfile" -gt '1' ]
  then
    cmdtype='poe'
  else
    cmdtype='sh'
  fi

  rm -f cmdfile
  rm -f cmdfile.*

  while [ "$ifile" -le "$nfile" ]
  do
    touch cmdfile.$ifile
    chmod 700 cmdfile.$ifile
    echo "cmdfile.$ifile" >> cmdfile
    ifile=`expr $ifile + 1`
  done

  set +x
  echo ' '
  echo '  Setting up command for grib2 generation ... '
  echo "   Set up command file structure (type = $cmdtype)."
  echo "      Number of command files  : $nfile"
  echo ' '
  [[ $LOUD = YES ]] && set -x

  ifile=1

  for grdID in $grids
  do
    echo "$USHwave/multiwavegrib2_sbs.sh $grdID $ymdh_end > grib2_$grdID.out 2>&1" >> cmdfile.$ifile
    if [ "$nfile" -gt '1' ]
    then
      ifile=`expr $ifile + 1`
    fi

    if [ "$ifile" -gt "$nfile" ]
    then
      ifile=1
    fi
  done 

  for grdID in $int_grids
  do
    echo "$USHwave/multiwavegrib2int_sbs.sh $grdID $ymdh_end > grib2_$grdID.out 2>&1" >> cmdfile.$ifile
    if [ "$nfile" -gt '1' ]
    then
      ifile=`expr $ifile + 1`
    fi

    if [ "$ifile" -gt "$nfile" ]
    then
      ifile=1
    fi
  done 

# 3.c Submit poe command

  set +x
  echo "   Executing command file at : `date`"
  echo '   -------------------------- '
  echo ' '
  [[ $LOUD = YES ]] && set -x

  if [ "$nfile" -gt '1' ]
  then
    poe
    exit=$?
  else
    cmdfile.1
    exit=$?
  fi

# 3.d Timing info on steps

  set +x
  echo ' '
  echo "   Ending times for separate processors :"

  for file in `cat cmdfile`
  do
    if [ "`wc $file | awk '{ print $1 }'`" = '0' ]
    then
      echo "      $file : no commands in this file."
    else
      words="`tail -1 $file | wc | awk '{ print $2 }'`"
      wrdnr=`expr $words - 1`
      echo "{ print "'$'"$wrdnr }" > awkfile
      outfile="`tail -1 $file | awk -f awkfile`"
      rm -f awkfile
      echo "      $file : `tail -1 $outfile`"
    fi
  done
  [[ $LOUD = YES ]] && set -x

# --------------------------------------------------------------------------- #
# 4.  Check for errors

  for grdID in $grids $int_grids
  do
    if [ ! -f "grib2.$grdID.complete" ]
    then
      set +x
      echo "  Error in generating GRIB2 Files for $grdID. "
      [[ $LOUD = YES ]] && set -x
      postmsg "$jlogfile" "ERROR in Generating GRIB2 Files for $grdID."
    else
      set +x
      echo "  GRIB2 Files for $grdID completed succesfully "
      [[ $LOUD = YES ]] && set -x
      rm -f grib2_$grdID.out
    fi 
  done

  for grdID in $grids $int_grids
  do
    if [ -f "grib2_$grdID.out" ]
    then
      set +x
      echo ' '
      echo ' *********************************** '
      echo ' *** ERROR OUTPUT multiwavegrib2 *** ' 
      echo ' *********************************** '
      echo ' '
      [[ $LOUD = YES ]] && set -x
      echo "$modID grib2 $grdID $date $cycle : Error in GRIB2 Generation " >> $wavelog
      sed "s/^/grib2_$grdID.out : /g" grib2_$grdID.out
      rm -f grib2_$grdID.out
      err=7;export err;err_chk
    fi
  done

# --------------------------------------------------------------------------- #
# 5. End of script

  set +x
  echo ' '
  echo "Ending at : `date`"
  echo ' ----------- '
  echo ' '
  echo '                     *** End of MWW3 grib2 postprocessor *** '
  echo ' '
  [[ $LOUD = YES ]] && set -x

  msg=" Step $step of $job completed normally"
  postmsg "$jlogfile" "$msg"

  touch grib2.$cycle.complete

# End of exwave_multi_1_grib2.sh.sms ---------------------------------------- #