########################### EXENSSTAT ################################
echo "------------------------------------------------"
echo "Ensemble Postprocessing"
echo "create mean and spread members"
echo "------------------------------------------------"
echo "History: MAY 1997 - First implementation of this new script."
echo "AUTHOR: Yuejian Zhu (wd20yz)"
echo "Modified by: Mary Jacobs"
echo "Modified by: Yuejian Zhu ---- October 1997"
echo "Modified by: Maxine Brown for Yuejian Zhu ---- November 1997"
echo "Modified by: David Michaud to port to IBM SP ---- September 1999"
echo "Modified by: Larry Sager to add spaghetti ensembles -- Feb 2000"
echo "Modified by: Yuejian Zhu to add high resolution archive---- June 2000"
echo "Modified by: Richard Wobus to add ensstat statistics---- September 2001"
echo "Modified by: Yuejian Zhu to add more vars, 6-hr interval fcst output,"
echo "             apply to T00Z, T06Z, T12Z and T18Z cycles,               "
echo "             no more ensppf production                                "
echo "             move pqpf to ensstat process                ---- May 2003"
echo "Modified by: Richard Wobus to add more variables ---- MARCH 2004"
echo "Modified by: Richard Wobus to add 192hr zsfc  ---- August 2004"
echo "Modified by: Richard Wobus reorganize by hour  ---- March 2006"
echo "Modified by: Richard Wobus separate avg/spr from ensstat job  ---- June 2011"
echo "Modified by: Richard Wobus remove logic related to gfs ---- February 2012"

### need pass the values of CYC, YMD, DATA, COMIN and COMOUT

if [[ ! -d $DATA ]]; then
  mkdir -p $DATA
fi

set -x

cd $DATA

#####################################
# Define ensemble message switch
#   iens_msg=0 will add ensemble extension message
#   iens_msg=1 do not need to add ensemble extension message
#              only if evry member has
#              extension message
#####################################
export iens_msg=1

#####################################
# Define Script/Exec Variables
#####################################
export WGRIB=${WGRIB:-$EXECUTIL/wgrib}
export GRBIDX=${GRBIDX:-$EXECUTIL/grbindex}
export COPYGB=${COPYGB:-$EXECUTIL/copygb}
export CNVGRIB=${CNVGRIB:-$EXECUTIL/cnvgrib}
export GRB2IDX=${GRB2IDX:-$EXECUTIL/grb2index}
export WGRIB2=${WGRIB2:-$EXECUTIL/wgrib2}

export ENSPPF=${ENSPPF:-$USHgefs/global_ensppf.sh}
export ENSSTAT=${ENSSTAT:-$EXECgefs/gefs_ensstat}
export ENSPQPF=${ENSPQPF:-$USHgefs/global_enspqpf.sh}

#####################################
# START TO DUMP DATA FOR $cycle CYCLE
# START TO DUMP DATA FROM PGB FORECAST FILES
#####################################

memberlist="c00"
(( imem = 0 ))
while (( imem < npair * 2 ))
do
  (( imem = imem + 1 ))
  if (( imem < 10 )); then
    imem=0$imem
  fi
  memberlist="$memberlist p$imem"
done
echo memberlist=$memberlist


echo
echo "`date`  ==== START OF HOURLY PROCESSING OF PGB FORECAST FILES ====="
echo

SLEEP_LOOP_MAX=`expr $SLEEP_TIME / $SLEEP_INT`

export fh=$SHOUR
if test $fh -lt 10 -a $fh -gt 0
then
   export fhr="0$fh"
fi
############################################################
# Loop Through the Post Forecast Files
############################################################
while test $fh -le $FHOUR
do

  #  the order here should correspond to the order in
  #  which the pgrb files are produced by the post
  for res in hr lr
  do
    if [[ $res = lr ]]; then
      EXT=".2"
      lr=lr
      nenspost=0
      lfm=$lfmlr
    else
      EXT=
      lr=
      nenspost=0
      lfm=$lfmhr
    fi
    
    ###############################
    # Start Looping for the
    # existence of the pgrba files
    ###############################
    set +x
    ic=1
    while [ $ic -le $SLEEP_LOOP_MAX ]
    do
      nfiles=0
      nmem=0
      foundall=yes
      for mem in $memberlist
      do
	(( nmem = nmem + 1 ))
        testfile=$COMIN/$cyc/pgrba$lr/ge${mem}.${cycle}.pgrbaif$fh$EXT$cfsuffix
        if test -f $testfile
        then
	  echo testfile=$testfile found
	  (( nfiles = nfiles + 1 ))
	  echo mem=$mem nfiles=$nfiles foundall=$foundall ic=$ic fh=$fh found
        else
	  foundall=no
	  echo mem=$mem nfiles=$nfiles foundall=$foundall ic=$ic fh=$fh not found
	fi
      done

      if [[ $foundall = yes ]]; then
	echo Search process ends foundall=$foundall ic=$ic fh=$fh
	echo Process all $nfiles members
	break
      else
	ic=`expr $ic + 1`
	sleep $SLEEP_INT
      fi
      ###############################
      # If we reach this point assume
      # fcst job never reached restart
      # period and error exit
      ###############################
      echo $nfiles out of $nmem members were found
      if [ $ic -eq $SLEEP_LOOP_MAX ]
      then

        ###############################
        # MODIFY THIS STATEMENT TO
        # ALLOW A DIFFERENT NUMBER OF
        # MEMBERS TO BE MISSING
        #
        # CURRENTLY ALLOWS ONE MISSING
        ###############################
        (( nfilesmin = nmem - 0 ))

        if (( nfiles < nfilesmin )); then
	  echo Search process FAILS foundall=$foundall ic=$ic fh=$fh
          echo $nfiles IS FEWER THAN $nfilesmin MEMBERS
          export err=9
          err_chk
        else
	  echo Search process ends foundall=$foundall ic=$ic fh=$fh
	  echo Continue after timeout with $nfiles members
          break
	fi
      fi
    done
    set -x

    msg="Starting ensstat generation for fhr=$fh"
    postmsg "$jlogfile" "$msg"

    #
    #  Make namelist file
    #
    echo " &namdim" >namin
#   echo " nmemdim=100", >>namin
#   echo " nenspostdim=100", >>namin
    echo " lfdim=${lfm}", >>namin
    echo " /" >>namin
    echo " &namens" >>namin

    ifile=0
    for mem in $memberlist
    do
      (( ifile = ifile + 1 ))
      iskip=0
      for nskip in $statskiplist
      do
	if [[ $mem = $nskip ]]; then
	  iskip=1
	fi
      done
      echo " cfipg($ifile)"=\"$COMIN/$cyc/pgrba$lr/ge${mem}.${cycle}.pgrbaf$fh$EXT$cfsuffix\", >>namin
      echo " cfipi($ifile)"=\"$COMIN/$cyc/pgrba$lr/ge${mem}.${cycle}.pgrbaif$fh$EXT$cfsuffix\", >>namin
      echo " iskip($ifile)"=$iskip, >>namin
    done

    echo " nfiles=$ifile", >>namin
    echo " nenspost=$nenspost", >>namin

    echo " cfoag"=\"geavg.$cycle.pgrbaf$fh$EXT$cfsuffix\", >>namin
    echo " cfosg"=\"gespr.$cycle.pgrbaf$fh$EXT$cfsuffix\" >>namin

    echo ' /' >>namin

    echo
    cat namin
    echo

    echo "####################### `date` $fh $res ensstat begin" >$pgmout.$res$fh
    $ENSSTAT <namin | fold -w 2000 >$pgmout.$res$fh
    export err=$?; err_chk
    echo "####################### `date` $fh $res ensstat end">>$pgmout.$res$fh
    for fhout in $statoutfhlist
    do
      if (( fh == fhout )); then
	cat $pgmout.$res$fh >> $pgmout
      else
	lines=`cat $pgmout.$res$fh| wc -l`
	lobeg=5
	loend=40
	echo lines=$lines lobeg=$lobeg loend=$loend
	(( lskip = lines - lobeg - loend ))
	if (( lskip > 100 )); then
	  head -$lobeg $pgmout.$res$fh >>$pgmout
	    echo "####################### $lskip Lines Skipped">>$pgmout
	  tail -$loend $pgmout.$res$fh >>$pgmout
	else
	  cat $pgmout$res.$fh >> $pgmout
	fi
      fi
    done

    echo "`date` check for missing or zero-length output files"

    for run in geavg gespr
    do
      if [[ -s ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix ]]; then
        ls -al ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix
      else
        echo output file ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix IS MISSING
        export err=9
        err_chk
      fi
    done

    echo "`date` send pgrba output begin"

    if [ $SENDCOM = "YES" ]; then
      for run in geavg gespr
      do
        if [[ "$makepgrb2a" = "yes" ]]; then
           $CNVGRIB -g12 -p40 ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix ${run}.${cycle}.pgrb2af$fh$EXT$cfsuffix
        fi
	if [[ "$makegrb2i" = "yes" ]]; then
           $WGRIB2 -s ${run}.${cycle}.pgrb2af$fh$EXT$cfsuffix >${run}.${cycle}.pgrb2af$fh$EXT${cfsuffix}.idx
        fi
	if [[ "$makegrb1i" = "yes" ]]; then
	  $GRBIDX ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix ${run}.${cycle}.pgrbaif$fh$EXT$cfsuffix
	fi
	if [[ -s ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix ]]; then
	  mv ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix $COMOUT/$cyc/pgrba$lr
	  if [[ "$makegrb1i" = "yes" ]]; then
	    mv ${run}.${cycle}.pgrbaif$fh$EXT$cfsuffix $COMOUT/$cyc/pgrba$lr
	  fi
	fi
	if [[ -s ${run}.${cycle}.pgrb2af$fh$EXT$cfsuffix ]]; then
	  mv ${run}.${cycle}.pgrb2af$fh$EXT$cfsuffix $COMOUT/$cyc/pgrb2a$lr
	  mv ${run}.${cycle}.pgrb2af$fh$EXT${cfsuffix}.idx $COMOUT/$cyc/pgrb2a$lr
	fi
        if test "$SENDDBN" = 'YES'
        then
           if [[ ! -n "$cfsuffix" && ! -n "$lr" ]]
           then
              if [[ "$run" = "geavg" ]]
              then
                 #$DBNROOT/bin/dbn_alert MODEL ENS_PGBA_AVG $job ${COMOUT}/${cyc}/pgrba/geavg.${cycle}.pgrbaf${fh}${cfsuffix}
                 $DBNROOT/bin/dbn_alert MODEL ENS_PGB2A_AVG $job ${COMOUT}/${cyc}/pgrb2a/geavg.${cycle}.pgrb2af${fh}${cfsuffix}
                 $DBNROOT/bin/dbn_alert MODEL ENS_PGB2A_AVG_WIDX $job \
                         ${COMOUT}/${cyc}/pgrb2a/geavg.${cycle}.pgrb2af${fh}${cfsuffix}.idx
              fi
              if [[ "$run" = "gespr" ]]
              then
                 #$DBNROOT/bin/dbn_alert MODEL ENS_PGBA_SPR $job ${COMOUT}/${cyc}/pgrba/gespr.${cycle}.pgrbaf${fh}${cfsuffix}
                 $DBNROOT/bin/dbn_alert MODEL ENS_PGB2A_SPR $job ${COMOUT}/${cyc}/pgrb2a/gespr.${cycle}.pgrb2af${fh}${cfsuffix}
                 $DBNROOT/bin/dbn_alert MODEL ENS_PGB2A_SPR_WIDX $job \
                          ${COMOUT}/${cyc}/pgrb2a/gespr.${cycle}.pgrb2af${fh}${cfsuffix}.idx
              fi
           fi
           if [[ ! -n "$cfsuffix" &&  -n "$lr" ]]
           then
              if [[ "$run" = "geavg" ]]
              then
                 $DBNROOT/bin/dbn_alert MODEL ENS_PGB2A2_AVG $job ${COMOUT}/${cyc}/pgrb2alr/geavg.${cycle}.pgrb2af${fh}${cfsuffix}.2
              fi
              if [[ "$run" = "gespr" ]]
              then
                 $DBNROOT/bin/dbn_alert MODEL ENS_PGB2A2_SPR $job ${COMOUT}/${cyc}/pgrb2alr/gespr.${cycle}.pgrb2af${fh}${cfsuffix}.2
              fi
           fi
        fi
      done
    fi
  done

  echo "`date` send pgrba output end"

  export fh=`expr $fh + $FHINC`
  if test $fh -lt 10
  then
    export fh="0$fh"
  fi

done

echo ###############################`date` cat $pgmout begin
cat $pgmout
echo ###############################`date` cat $pgmout end

echo
echo "`date`  ==== END  OF  HOURLY PROCESSING OF PGB FORECAST FILES ====="
echo

############################################################################
###########  ADD DBN ALERTS FOR PPF AND PQPF FILES IF NEEDED  ##############
############################################################################

msg="HAS COMPLETED NORMALLY!"
postmsg "$jlogfile" "$msg"