#!/bin/sh
######################################################################
#
#   Script:  excfs_postprocessing.sh.sms   
#   Author:  Shrinivas Moorthi
#   DATE:    February, 2004
#   Updated for CFSv2 in April 2010 by Shrinivas Moorthi
#
#   Based on SFM script by Hann-Ming Henry Juang
#
#
#   Main driver to do postprocessing for CFS (Climate Forecast SYSTEM)
#   Coupled Model Integrations
#   History: 2001.11.06 First Implementation  for SFM

# ###############################
#  START CFS POST PROCESSING
# ###############################

set +x
echo " ------------------------------------------------------------"
echo "  "
echo "               IBM-SP $ENVIR PROCESSING     "
echo "  "
echo "            SEASONAL FORECAST MODEL POSTPROCESSING     "
echo "  "
echo "                `date`     "
echo "  "
echo "                   JOB  $job  "
echo "  "
echo "  "
echo " ------------------------------------------------------------"
echo "          processing info for this execution"
echo " Home directory is ............................ $HOMEcfs"
echo " Processing directory for files.. ............. $DATA"
echo "  "
echo " Executable file directory is ................. $EXECcfs"
echo " Fixed field directory is ..................... $FIXcfs"
echo " Unix control language file is ................ $USHcfs"
echo "  "
echo " Network id is ................................ $NET"
echo " Run id for $com processing is ................ $RUN"
echo "  "
echo " standard output in file ...................... $pgmout"
echo " unique processing id for run ................. $pid"
echo " SENDCOM=YES means save com files ............. $SENDCOM"
echo " ------------------------------------------------------------"
set -x


###############################################
# check for post restart or recovery job
###############################################

export NDATE=${NDATE:-/nwprod/util/exec/ndate}
export CNVGRIB=${CNVGRIB:-/nwprod/util/exec/cnvgrib}
export WGRIB2=${WGRIB2:-/nwprod/util/exec/wgrib2}
export ENS_MEM=$((ENS_MEM+0))
export CNVGRIB=${CNVGRIB:-/nwprod/util/exec/cnvgrib}
export NCP=${NCP:-/bin/cp}
FH_45DAYS=${FH_45DAYS:-1080}
SAVE_TIME_45DAYS=${SAVE_TIME_45DAYS:-YES}
if [ $ENS_MEM -lt 10 ] ; then export ENS_MEM=0$ENS_MEM ; fi

################################################
# looping until no job to do or wait too long
################################################
#BREAK_ON_NENS=NO
BREAK_ON_RUN=NO
#
wait_count=0

while [ $wait_count -le 180 -a $BREAK_ON_RUN = NO ] ; do
  wait_count=` expr $wait_count + 1 `
  while [ -s $POSTDEFS/post_def$cyc.* ] ; do

      ###############################################
      # Run it if there is definition files for post and not time to resubmit
      ###############################################
      ls -rt $POSTDEFS/post_def$cyc.* | sed 's/.*\.//g' | read postid
    . $POSTDEFS/post_def$cyc.$postid

      export CDATE=$start_date
      if [ $((ENS_MEM+0)) -ne $((MEMBER+0)) ] ; then
        echo 'Member number confilct : ENS_MEM= '$ENS_MEM ', MEMBER= '$MEMBER
        echo ' exiting post job'
        exit
      fi
      export COMIN=$INPDIR
      export COM_YMDH=$OUTDIR
      export COM_YMDH_DG=$COM_YMDH/daily_grib
      export COM_YMDH_MG=$CLIMDIR/monthly_grib_${ENS_MEM}         
      export COM_DG=$CLIMDIR/time_grib_${ENS_MEM}
      export COM_HRLY=$CLIMDIR/${FHOUT}hrly_grib_${ENS_MEM}
      mkdir -p $COM_YMDH_DG
      if [ "$AVG_GRB" != NO ]; then
        mkdir -p $COM_YMDH_MG
      fi
      mkdir -p $COM_DG
      mkdir -p $COM_HRLY

      #############################
      # Create date related temp dir
      ###############################
      export YMDH=$start_date
      export SUFOUT=${SUFOUT:-.$ENS_MEM.$YMDH}
      cd $DATA

      ############################################
      # Copy necessary files from COM to temp dir
      ############################################
      if [[ $FHS -eq 0 ]] ; then
        FHINI=-$INTSIG
      else
        FHINI=$FHS
      fi
      export FHS=$FHS

      if [ $RUN_ENVIR = prod -o $RUN_ENVIR = devpara ]
      then
        if [ $FHE -gt $nhours ]
        then
          echo "Forecast hour is greater than end_hour for this chunk"
          echo "Should be processed in the next chunk"
          exit 0
        fi
      fi

      ###########################
      # Run postprocessing job
      ###########################

      cfs_postscript=${cfs_postscript:-$USHcfs/cfs_post.sh}
      $cfs_postscript $MEMBER
      rc=$?

      #############################################
      # Save the analysis files to 6hrly directory
      #############################################
      if [ $FHS -eq 0 ]; then
         $NCP $COM_YMDH/siganl.${ENS_MEM}.$YMDH     $COM_HRLY/.
         $NCP $COM_YMDH/sfcanl.${ENS_MEM}.$YMDH     $COM_HRLY/.
         $NCP $COM_YMDH/ocnanl.${ENS_MEM}.$YMDH.tar $COM_HRLY/.
         for fil in pgbanl ipvanl splanl
         do
           ifile=$COM_YMDH_DG/${fil}.${ENS_MEM}.$YMDH
           ofile=${fil}.${ENS_MEM}.${YMDH}.grb2
           $CNVGRIB -g12 -p40 $ifile $ofile
           $WGRIB2 $ofile -s >${ofile}.idx
           mv $ofile ${ofile}.idx $COM_HRLY/.
 
           if [ $SENDDBN = YES ]; then
              $DBNROOT/bin/dbn_alert MODEL CFS_FCST_ANL $job $COM_HRLY/$ofile
              $DBNROOT/bin/dbn_alert MODEL CFS_FCST_ANL_WIDX $job $COM_HRLY/${ofile}.idx
           fi
         done
      fi

      ##################################################################
      # Save the pgbf, ipvf, ocnf and flxf files to the 6hrly directory
      ##################################################################
      tot_err=0
      for file in $HRLY_FILES_TO_BE_KEPT ; do
        if [ $file = flxf -o $file = sigf -o $file = sfcf ] ; then
          fdir=$COM_YMDH
        else
          fdir=$COM_YMDH_DG
        fi

        start_hour=$FHS
        end_hour=$FHE

        if [ $ENS_MEM -eq 01 ]; then
           # Calculate the number of hours to keep 
            YYYYSS=$(echo $start_date | cut -c1-4)
            MMSS=$(echo $start_date | cut -c5-6)
            YYYY_KEEP=$YYYYSS
            MM_KEEP=$((MMSS+MON_TO_KEEP+1))
            while [[ $MM_KEEP -gt 12 ]] ; do
              MM_KEEP=$((MM_KEEP-12))
              YYYY_KEEP=$((YYYY_KEEP+1))
            done
           if [[ $MM_KEEP -lt 10 ]] ; then MM_KEEP=0$MM_KEEP; fi
           export end_date_KEEP=${YYYY_KEEP}${MM_KEEP}0100
           NHOUR=${NHOUR:-/nwprod/util/exec/nhour}
           export HOUR_TO_KEEP=$($NHOUR $end_date_KEEP $start_date) 
           if [ $HOUR_TO_KEEP -ge $ENDHOUR ]; then
              export HOUR_TO_KEEP=$ENDHOUR
           fi
         else
           export HOUR_TO_KEEP=$ENDHOUR
         fi   
    
         each_err=0
         if [ $file = sigf ]; then
            export HOUR_TO_KEEP=$HOUR_TO_KEEP_SIG
         elif [ $file = sfcf ]; then
            export HOUR_TO_KEEP=$HOUR_TO_KEEP_SFC
         fi

         if [ $start_hour -le $HOUR_TO_KEEP ]; then
           if [ $file = ocnf -a $FHS -eq 0 ]; then start_hour=6; fi
           if [ $start_hour -lt 10 ]; then start_hour=0$start_hour; fi
           $USHcfs/cfs_keepfile.sh $file $start_hour $end_hour $FHOUT $fdir $each_err
         fi

         tot_err=`expr $tot_err + $each_err`
       done

      if [[ $FHE = $nhours ]] ; then
#                             clean up temporary files and stop the post job
        if [ $rc =  0 ] ; then
          cd $DATA
          rm -rf $YMDH
        fi
        BREAK_ON_RUN=YES
        if [ ${DOALL_POSTDEFS:-NO} = YES ] ; then
          BREAK_ON_RUN=NO
        fi
      fi
      if [ $SAVE_TIME_45DAYS = YES -a $FHE -eq $FH_45DAYS ]; then

         ##############################################################
         # Run script to concatenate the time-series file and save them
         # in the "time_grib" directory for the first 45 days
         ##############################################################
         edate_45days=$($NDATE $FH_45DAYS $start_date)
         fh_inc=${INC_HOUR:-360}
         $USHcfs/cfs_savetimeser.sh $start_date $edate_45days $fh_inc
      fi

      ######################################################
      # At the very end of the POST job
      ######################################################

      if [ $FHE -ge $ENDHOUR ] ; then

#       if [ ${CPY_RES_TO_COM:-YES} = YES ] ; then
#         # Copy the restart file to COM for future use
#         $NCP $RECOVERY/*$YMDH* $COM_DG/.
#       fi

         ##############################################################################################
         # Run script to concatenate the time-series file and save them in the "time_grib" directory
         ##############################################################################################
         end_date=$($NDATE $ENDHOUR $start_date)
         fh_inc=${INC_HOUR:-360}
         $USHcfs/cfs_savetimeser.sh $start_date $end_date $fh_inc

###########################################################################
         ####################################################
         # Save the necessary files for restart in a tar file
         ####################################################
         RESDIR=$DATA/LAST_RESTART
         rm -rf $RESDIR
         mkdir -p $RESDIR
         $NCP $RECOVERY/stamp.cfs                      $RESDIR/.
         $NCP $RECOVERY/cfs_run.2restart               $RESDIR/.
         string1=${ENS_MEM}.${start_date}
         string2=${string1}.${end_date}
         for ifile in sigr1 sigr2 sfcr fluxes_for_OM
         do
           $NCP $RECOVERY/$ifile.$string2              $RESDIR/.
         done
         $NCP $RECOVERY/omrestart.$string2.tar         $RESDIR/.
         for ifile in sigf sfcf flxf logf ; do
           $NCP $COM_YMDH/${ifile}$ENDHOUR.$string1    $RESDIR/.
         done
         for ifile in pgbf ipvf ocnf ocnh ; do
           $NCP $COM_YMDH_DG/${ifile}$ENDHOUR.$string1 $RESDIR/.
         done

         # Tar up the files and save it in the 6-hrly directory
         cd $RESDIR
         #XXW tar -cvf $COM_HRLY/RESTART.$string1.tar * # should be restart.*
         tar -cvf $COM_HRLY/restart.$string1.tar *
         rc=$?

         # End saving the restart files
###########################################################################
         cd $DATA 
      fi

      ############################################################
      # rename the post define and delete them if a member run is completed
      ############################################################
      tot_rc=`expr $tot_err + $rc`
      if [ $tot_rc = 0 ] ; then
        mv $POSTDEFS/post_def$cyc.$postid $POSTDEFS/done$cyc.$postid
      else
        err_exit $tot_rc
      fi
#
      if [ $FHE -ge $ENDHOUR ] ; then
        
         #######################################################
         # Remove the hourly files in the COM directory
         #######################################################
         if [ $tot_rc -eq 0 ] ; then
           rm -rf $COM_YMDH
         fi

         ################################################
         # clean up temperary files and stop the post job
         ################################################
         if [ $tot_rc -eq 0 ] ; then
            cd $DATA
            rm -rf $YMDH
            if [ ${RM_POSTDEFS:-YES} = YES ] ; then
              rm -rf $POSTDEFS
#             rm $POSTDEFS/done*.*
#             rm $POSTDEFS/monthly.stamp
            fi
            if [ ${RM_RECOVERY:-YES} = YES ] ; then
              rm -rf $RECOVERY
            fi
         fi

         BREAK_ON_RUN=YES
      fi
#
    wait_count=0
    if [ $BREAK_ON_RUN = YES ] ; then break ; fi
  done
  ######################
  # wait for next post 
  ######################
  sleep 30

done
if [ $FHE -lt $ENDHOUR ] ; then
  if [ $RUN_ENVIR != prod -a $RUN_ENVIR != devpara ] ; then  # resubmit the post job for continuation run
    SUBSH=${SUBSH:-""}
    if [ ! -z $SUBSH ] ; then
      if [ -s $SUBSH ] ; then
        run_type=${run_type:-hindcast} ; class=${class:-dev} 
        $SUBSH post $run_type $CDATE $CONFIG_FILE $class
      fi
    fi
  else
    echo "Wait for forecast output for one and half hours, check if forecast job hung or silently failed!"
    err_exit 777
  fi
fi