########################### EXENSSTAT ################################
echo "------------------------------------------------"
echo "Ensemble Postprocessing"
echo "------------------------------------------------"
echo "History: MAY 1997 - First implementation of this new script."
echo "AUTHOR: Yuejian Zhu (wd20yz)"
echo "Modified by: Mary Jacobs"
echo "Modified by: Yuejian Zhu ---- October 1997"
echo "Modified by: Maxine Brown for Yuejian Zhu ---- November 1997"
echo "Modified by: David Michaud to port to IBM SP ---- September 1999"
echo "Modified by: Larry Sager to add spaghetti ensembles -- Feb 2000"
echo "Modified by: Yuejian Zhu to add high resolution archive---- June 2000"
echo "Modified by: Richard Wobus to add ensstat statistics---- September 2001"
echo "Modified by: Yuejian Zhu to add more vars, 6-hr interval fcst output,"
echo "             apply to T00Z, T06Z, T12Z and T18Z cycles,               "
echo "             no more ensppf production                                "
echo "             move pqpf to ensstat process                ---- May 2003"
echo "Modified by: Richard Wobus to add more variables ---- MARCH 2004"
echo "Modified by: Richard Wobus to add 192hr zsfc  ---- August 2004"
echo "Modified by: Richard Wobus reorganize by hour  ---- March 2006"

### need pass the values of CYC, YMD, DATA, COMIN and COMOUT

if [[ ! -d $DATA ]]; then
  mkdir -p $DATA
fi

set -x

cd $DATA

#####################################
# Define ensemble message switch
#   iens_msg=0 will add ensemble extension message
#   iens_msg=1 do not need to add ensemble extension message
#              only if evry members (include GFS) have
#              extension message
#####################################
export iens_msg=1

#####################################
# Define Script/Exec Variables
#####################################
export WGRIB=$EXECUTIL/wgrib
export GRBIDX=$EXECUTIL/grbindex
export COPYGB=$EXECUTIL/copygb
export CNVGRIB=$EXECUTIL/cnvgrib
export GRB2IDX=$EXECUTIL/grb2index
export WGRIB2=$EXECUTIL/wgrib2

export ENSPPF=$USHGLOBAL/global_ensppf.sh
export ENSSTAT=$EXECGLOBAL/gefs_ensstat
export ENSPQPF=$USHGLOBAL/global_enspqpf.sh

#####################################
# START TO DUMP DATA FOR $cycle CYCLE
# START TO DUMP DATA FROM PGB FORECAST FILES
#####################################

varlout=" \
  ensppf pqpf  pqsf  pqff  pqrf  pqif \
 "

if test "$SENDCOM" = 'YES'
then
  for file in enspost ensstat
  do
#   for FIELD in $varlboth $varlnostat $varlout
    for FIELD in $epnamhr $epnamlr $varlout
    do
      if [ -s $COMOUT/$cyc/ensstat/$file.${cycle}.${FIELD} ]
      then
         rm $COMOUT/$cyc/ensstat/$file.${cycle}.${FIELD}
         rm $COMOUT/$cyc/ensstat/$file.${cycle}.${FIELD}i
      fi
      if [ -s $COMOUT/$cyc/ensstat/$file.${cycle}.${FIELD}hr ]
      then
         rm $COMOUT/$cyc/ensstat/$file.${cycle}.${FIELD}hr
         rm $COMOUT/$cyc/ensstat/$file.${cycle}.${FIELD}hri
      fi
    done
  done
fi

memberlist="gfs c00"
(( imem = 0 ))
while (( imem < npair * 2 ))
do
  (( imem = imem + 1 ))
  if (( imem < 10 )); then
    imem=0$imem
  fi
  memberlist="$memberlist p$imem"
done
echo memberlist=$memberlist


echo
echo "`date`  ==== START OF HOURLY PROCESSING OF PGB FORECAST FILES ====="
echo

SLEEP_LOOP_MAX=`expr $SLEEP_TIME / $SLEEP_INT`

export fh=$SHOUR
if test $fh -lt 10 -a $fh -gt 0
then
   export fhr="0$fh"
fi
############################################################
# Loop Through the Post Forecast Files
############################################################
(( nfilesprev = 9999 ))
foundgfs=no
while test $fh -le $FHOUR
do

  #  the order here should correspond to the order in
  #  which the pgrb files are produced by the post
  for res in hr lr
  do
    if [[ $res = lr ]]; then
      EXT=".2"
      lr=lr
      nenspost=$nenspostlr
      lfm=$lfmlr
      set -A enspostnam x $epnamlr
      set -A enspostvar x $epvarlr
      set -A enspostlvt x $eplvtlr
      set -A enspostlev x $eplevlr
    else
      EXT=
      lr=
      nenspost=$nensposthr
      lfm=$lfmhr
      set -A enspostnam x $epnamhr
      set -A enspostvar x $epvarhr
      set -A enspostlvt x $eplvthr
      set -A enspostlev x $eplevhr
    fi
    
    ###############################
    # Start Looping for the
    # existence of the pgrba files
    ###############################
    set +x
    ic=1
    while [ $ic -le $SLEEP_LOOP_MAX ]
    do
      nfiles=0
      nmem=0
      foundall=yes
      previncr=no
      for mem in $memberlist
      do
	(( nmem = nmem + 1 ))
        testfile=$COMIN/$cyc/pgrba$lr/ge${mem}.${cycle}.pgrbaif$fh$EXT$cfsuffix
        if test -f $testfile
        then
	  echo testfile=$testfile found
	  (( nfiles = nfiles + 1 ))
	  if [[ $mem = gfs ]]; then
	    foundgfs=yes
	  fi
	  echo mem=$mem nfiles=$nfiles foundgfs=$foundgfs foundall=$foundall previncr=$previncr ic=$ic fh=$fh found
        else
	  if [[ $mem = gfs ]] && [[ $foundgfs = yes ]] && (( fh > 180 )) && (( fh % 12 > 0 )); then
	    previncr=yes
	  else
	    foundall=no
	  fi
	  echo mem=$mem nfiles=$nfiles foundgfs=$foundgfs foundall=$foundall previncr=$previncr ic=$ic fh=$fh not found
	fi
      done

      if [[ $foundall = yes ]]; then
	if [[ $previncr = yes ]]; then
	  (( nfilesprev = nfiles + 1 ))
	else
	  (( nfilesprev = nfiles ))
	fi
	echo Search process ends nfilesprev=$nfilesprev foundgfs=$foundgfs foundall=$foundall previncr=$previncr ic=$ic fh=$fh
	echo Process all $nfiles members
	break
      else
	if (( nfiles < nfilesprev )) then
	  ic=`expr $ic + 1`
	  sleep $SLEEP_INT
	else
	  if [[ $previncr = yes ]]; then
	    (( nfilesprev = nfiles + 1 ))
	  else
            (( nfilesprev = nfiles ))
          fi
	  echo Search process ends nfilesprev=$nfilesprev foundgfs=$foundgfs foundall=$foundall previncr=$previncr ic=$ic fh=$fh
	  echo Continue processing with $nfiles members
          break
        fi
      fi
      ###############################
      # If we reach this point assume
      # fcst job never reached restart
      # period and error exit
      ###############################
      echo $nfiles out of $nmem members were found
      if [ $ic -eq $SLEEP_LOOP_MAX ]
      then

        ###############################
        # MODIFY THIS STATEMENT TO
        # ALLOW A DIFFERENT NUMBER OF
        # MEMBERS TO BE MISSING
        #
        # CURRENTLY ALLOWS ONE MISSING
        ###############################
        (( nfilesmin = nmem - 1 ))

        if (( nfiles < nfilesmin )); then
	  echo Search process FAILS nfilesprev=$nfilesprev foundgfs=$foundgfs foundall=$foundall previncr=$previncr ic=$ic fh=$fh
          echo $nfiles IS FEWER THAN $nfilesmin MEMBERS
          export err=9
          err_chk
        else
          (( nfilesprev = nfiles ))
	  echo Search process ends nfilesprev=$nfilesprev foundgfs=$foundgfs foundall=$foundall previncr=$previncr ic=$ic fh=$fh
	  echo Continue after timeout with $nfiles members
          break
	fi
      fi
    done
    set -x

    msg="Starting ensstat generation for fhr=$fh"
    postmsg "$jlogfile" "$msg"

    #
    #  Make namelist file
    #
    echo " &namdim" >namin
#   echo " nmemdim=100", >>namin
#   echo " nenspostdim=100", >>namin
    echo " lfdim=${lfm}", >>namin
    echo " /" >>namin
    echo " &namens" >>namin

    ifile=0
    for mem in $memberlist
    do
      (( ifile = ifile + 1 ))
      iskip=0
      for nskip in $statskiplist
      do
	if [[ $mem = $nskip ]]; then
	  iskip=1
	fi
      done
      echo " cfipg($ifile)"=\"$COMIN/$cyc/pgrba$lr/ge${mem}.${cycle}.pgrbaf$fh$EXT$cfsuffix\", >>namin
      echo " cfipi($ifile)"=\"$COMIN/$cyc/pgrba$lr/ge${mem}.${cycle}.pgrbaif$fh$EXT$cfsuffix\", >>namin
      echo " iskip($ifile)"=$iskip, >>namin
    done

    echo " nfiles=$ifile", >>namin
    echo " nenspost=${nenspost}", >>namin

    (( ienspost = 0 ))
    (( iunitouts = 52 ))
    while (( ienspost < nenspost ))
    do
      (( ienspost = ienspost + 1 ))
      (( iunitouta = iunitouts + 1 ))
      (( iunitouts = iunitouts + 2 ))
      echo " ivar($ienspost)"=${enspostvar[$ienspost]}, >>namin
      echo " ilev($ienspost)"=${enspostlev[$ienspost]}, >>namin
      echo " ilvt($ienspost)"=${enspostlvt[$ienspost]}, >>namin
      echo " cfopg($ienspost)"=\"enspost.$cycle.${enspostnam[$ienspost]}\", >>namin
      echo " cfotg($ienspost)"=\"ensstat.$cycle.${enspostnam[$ienspost]}\", >>namin
    done

    echo " cfoag"=\"geavg.$cycle.pgrbaf$fh$EXT$cfsuffix\", >>namin
    echo " cfosg"=\"gespr.$cycle.pgrbaf$fh$EXT$cfsuffix\" >>namin

    echo ' /' >>namin

    echo
    cat namin
    echo

    echo "####################### `date` $fh $res ensstat begin" >$pgmout.$res$fh
    $ENSSTAT <namin | fold -w 2000 >$pgmout.$res$fh
    export err=$?; err_chk
    echo "####################### `date` $fh $res ensstat end">>$pgmout.$res$fh
    for fhout in $statoutfhlist
    do
      if (( fh == fhout )); then
	cat $pgmout.$res$fh >> $pgmout
      else
	lines=`cat $pgmout.$res$fh| wc -l`
	lobeg=5
	loend=40
	echo lines=$lines lobeg=$lobeg loend=$loend
	(( lskip = lines - lobeg - loend ))
	if (( lskip > 100 )); then
	  head -$lobeg $pgmout.$res$fh >>$pgmout
	    echo "####################### $lskip Lines Skipped">>$pgmout
	  tail -$loend $pgmout.$res$fh >>$pgmout
	else
	  cat $pgmout$res.$fh >> $pgmout
	fi
      fi
    done

    echo "`date` send pgrba output begin"

    if [ $SENDCOM = "YES" ]; then
      for run in geavg gespr
      do
        if [[ "$makepgrb2a" = "yes" ]]; then
           $CNVGRIB -g12 -p40 ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix ${run}.${cycle}.pgrb2af$fh$EXT$cfsuffix
        fi
	if [[ "$makegrb1i" = "yes" ]]; then
	  $GRBIDX ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix ${run}.${cycle}.pgrbaif$fh$EXT$cfsuffix
	fi
	if [[ -s ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix ]]; then
	  mv ${run}.${cycle}.pgrbaf$fh$EXT$cfsuffix $COMOUT/$cyc/pgrba$lr
	  if [[ "$makegrb1i" = "yes" ]]; then
	    mv ${run}.${cycle}.pgrbaif$fh$EXT$cfsuffix $COMOUT/$cyc/pgrba$lr
	  fi
	fi
	if [[ -s ${run}.${cycle}.pgrb2af$fh$EXT$cfsuffix ]]; then
	  mv ${run}.${cycle}.pgrb2af$fh$EXT$cfsuffix $COMOUT/$cyc/pgrb2a$lr
	  if [[ "$makegrb2i" = "yes" ]]; then
	    mv ${run}.${cycle}.pgrb2aif$fh$EXT$cfsuffix $COMOUT/$cyc/pgrb2a$lr
	  fi
	fi
        if test "$SENDDBN" = 'YES'
        then
           if [[ ! -n "$cfsuffix" && ! -n "$lr" ]]
           then
              if [[ "$run" = "geavg" ]]
              then
                 $DBNROOT/bin/dbn_alert MODEL ENS_PGBA_AVG $job ${COMOUT}/${cyc}/pgrba/geavg.${cycle}.pgrbaf${fh}${cfsuffix}
                 $DBNROOT/bin/dbn_alert MODEL ENS_PGB2A_AVG $job ${COMOUT}/${cyc}/pgrb2a/geavg.${cycle}.pgrb2af${fh}${cfsuffix}
              fi
              if [[ "$run" = "gespr" ]]
              then
                 $DBNROOT/bin/dbn_alert MODEL ENS_PGBA_SPR $job ${COMOUT}/${cyc}/pgrba/gespr.${cycle}.pgrbaf${fh}${cfsuffix}
                 $DBNROOT/bin/dbn_alert MODEL ENS_PGB2A_SPR $job ${COMOUT}/${cyc}/pgrb2a/gespr.${cycle}.pgrb2af${fh}${cfsuffix}
              fi
           fi
        fi
      done
    fi
  done

  echo "`date` send pgrba output end"

  export fh=`expr $fh + $FHINC`
  if test $fh -lt 10
  then
    export fh="0$fh"
  fi

done

for file in $epnamhr $epnamlr
do

  $GRBIDX enspost.$cycle.${file}   enspost.$cycle.${file}i
  $GRBIDX ensstat.$cycle.${file}   ensstat.$cycle.${file}i

  if [ "$SENDCOM" = "YES" ]
  then
     mv enspost.$cycle.${file}  $COMOUT/$cyc/ensstat/enspost.${cycle}.${file}
     mv enspost.$cycle.${file}i $COMOUT/$cyc/ensstat/enspost.${cycle}.${file}i
     mv ensstat.$cycle.${file}  $COMOUT/$cyc/ensstat/ensstat.${cycle}.${file}
     mv ensstat.$cycle.${file}i $COMOUT/$cyc/ensstat/ensstat.${cycle}.${file}i
  fi

done

echo ###############################`date` cat $pgmout begin
cat $pgmout
echo ###############################`date` cat $pgmout end

echo
echo "`date`  ==== END  OF  HOURLY PROCESSING OF PGB FORECAST FILES ====="
echo

#
# create ensemble statistics files
#

## part (1a): calculate PQPF (named ensppf ) for each 24 hours period,
## for 23 ens members 
## write out with ensemble extended messages
##
#
if test "$cycle" = "t00z"
then
   $ENSPPF $COMOUT/$cyc/ensstat/enspost.$cycle.prcp $COMOUT/$cyc/ensstat/enspost.$cycle.prcpi ensppf.$PDY$cyc
   $GRBIDX ensppf.$PDY$cyc ensppfi.$PDY$cyc

   ###########################
   # Convert to grib2 format:
   ###########################
   $CNVGRIB -g12 -p40 ensppf.$PDY$cyc ensppf.$PDY$cyc.grib2
   $WGRIB2 ensppf.$PDY$cyc.grib2 -s >ensppf.$PDY$cyc.grib2.idx 

   if [ $SENDCOM = "YES" ]
   then
     cp ensppf.$PDY$cyc $COMOUT/$cyc/ensstat/ensstat.$cycle.pqpf_24h
     cp ensppfi.$PDY$cyc $COMOUT/$cyc/ensstat/ensstat.$cycle.pqpfi_24h

     cp ensppf.$PDY$cyc.grib2 $COMOUT/$cyc/ensstat/ensstat.$cycle.pqpf_24h.grib2
     cp ensppf.$PDY$cyc.grib2.idx $COMOUT/$cyc/ensstat/ensstat.$cycle.pqpf_24h.grib2.idx
   fi
   if [ "$SENDDBN" = "YES" ]
   then
     $DBNROOT/bin/dbn_alert MODEL ENS_STAT $job $COMOUT/$cyc/ensstat/ensstat.${cycle}.pqpf_24h

     if [ $SENDDBN_GB2 = YES ]
     then

     $DBNROOT/bin/dbn_alert MODEL ENS_STAT_GB2 $job $COMOUT/$cyc/ensstat/ensstat.$cycle.pqpf_24h.grib2
     $DBNROOT/bin/dbn_alert MODEL ENS_STAT_GB2_WIDX $job $COMOUT/$cyc/ensstat/ensstat.$cycle.pqpf_24h.grib2.idx

     fi

   fi
fi


# part (1b): probabilistic forecasts ( PQPF, PQRF, PQFF, PQSF and PQIF )
export CDATE=$PDY$cyc; $ENSPQPF

for file in pqpf pqrf pqff pqsf pqif
do
   $GRBIDX $DATA/$file $DATA/${file}i
   if [ $SENDCOM = "YES" ]
   then
     mv $DATA/$file $COMOUT/$cyc/ensstat/ensstat.$cycle.$file
     mv $DATA/${file}i $COMOUT/$cyc/ensstat/ensstat.$cycle.${file}i
   fi
done

############################################################################
###########  ADD DBN ALERTS FOR PPF AND PQPF FILES IF NEEDED  ##############
############################################################################

msg="HAS COMPLETED NORMALLY!"
postmsg "$jlogfile" "$msg"