####  UNIX Script Documentation Block
#
# Script name:   ingest_process_days
#
# JIF contact:  Keyser      org: NP22        date: 2008-01-24
#
# Abstract: Initiates the transfer of one requested, statically-named, file
#   from a remote unix maxchine.
#
# Script history log:
# 1996-10-03  Bert Katz   Original version for implementation.
# 1996-11-15  Bert Katz   Updated to handle snow field processing.
#     this involved separating the receiving of the file from the processing of
#     the file, with both events recorded separately in the "history" file.
# 1996-12-04  Bert Katz   Reorganized file structure to resemble the BUFR
#     tanking system.
# 1997-01-09  Bert Katz   Unified output in "stdout", added debug option
# 1997-01-23  Bert Katz   Corrected handling of first write to the "history"
#     file.
# 1997-01-31  Bert Katz   Inserted search of operational script directory ahead
#     of user's script directory to find the processing script to execute, if
#     any extra processing is done.
# 1997-02-12  Bert Katz   Improved "copying forward" so that it will update the
#     new day's directory with the newest version of the file when the file is
#     late and the previous day's directory receives a tardy update.
# 1998-05-12  Bert Katz   Changes for y2k compliance
# 2006-05-12  D. Keyser   Generalized name of remote machine (no longer
#     hardwired to CEMSCS) and changed to account for all remote machines now
#     being unix (since MVS CEMSCS machine was replaced with unix DDS machine).
#     Improved documentation and comments; more appropriate messages posted to
#     joblog.
# 2006-09-29  D. Keyser   If transfer fails on first attempt (for whatever
#     reason) now sleeps 30 sec and tries a second time, if this also fails
#     script gives up (allows transfer to bypass possible momentary ftp
#     glitches)
# 2008-01-24  D. Keyser   Checks value of new imported script variable
#     "MTYPSBT" - if "YES", now executes program bufr_tranmtypsbt to: 1) change
#     each BUFR message in output statically-named BUFR file to have BUFR type
#     and subtype as determined by input script variable "tankfile"
#     (bttt/xxsss, where ttt=BUFR type and sss=BUFR subtype); and 2) encode
#     the BUFR menmonic table bufrtab.ttt (where ttt is obtained from input
#     script variable "tankfile") into the top of the output statically-named
#     BUFR file (in place of any dictionary messages that may have previously
#     been there), output (stdout and stderr) from bufr_tranmtypsbt is written
#     into directory specified by imported script variable "OUTDIR"
#
# Usage: ingest_process_days
#
#   Script parameters: None
#
#   Modules and files referenced:
#     scripts    : $USHbufr/ingest_get
#                  $DATA/postmsg
#                  user-defined script in variable PROCSCRIPT
#     data cards : none
#     executables: $utilexec/cnvblk
#
# Remarks: Invoked by the script ingest_process_onetype_newdays.
#
# Condition codes:
#     0 - no problem encountered
#   > 0 - some problem encountered
#     Specifically:   1 - Tranfser of file failed in ingest_get or one of the
#                         following errors in program CNVBLK:
#                          - incorrect pumber of arguments passed in
#                          - option "-ib" not supported
#                          - option "-ob" not supported
#                          - invalid option
#                     2 - BAOPEN or BAOPENW error in program CNVBLK
#
# Attributes:
#   Language: aix unix script
#   Machine:  NCEP CCS
####

set -au

echo
echo "#######################################################################"
echo "                   START INGEST_PROCESS_DAYS                           "
echo "#######################################################################"
echo

if [ $DEBUGSCRIPTS = ON -o $DEBUGSCRIPTS = YES ] ; then
   set -x
fi

if [ -s $USERDIR/$dsname_hist.history ] ; then
   dsnfound=$(grep -c "$dsname (for $datecurr) RECEIVED" \
$USERDIR/$dsname_hist.history)
else
   dsnfound=0
fi

if [ $dsnfound -eq 0 ] ; then
   newday="$dsname"
   cd $DATA
###itries_max=1
   itries_max=2
   set +x
   echo
   echo "Time is now $(date)."
   echo
   [ $DEBUGSCRIPTS = ON -o $DEBUGSCRIPTS = YES ]  &&  set -x

   ftperror=99
   itries=1
   while [ $ftperror -gt 0 -a $itries -le $itries_max ]; do
      if [ $itries -gt 1 ]; then
         msg="TRANSFER OF $dsname FAILED!!!! - SLEEP 30 SEC AND TRY AGAIN."
         postmsg "$jlogfile" "$msg"
         sleep 30
      fi
      sh $USHbufr/ingest_get $MACHINE $dsname_local "$newday" 2>&1
      ftperror=$?
      itries=`expr $itries + 1`
   done
   itries=`expr $itries - 1`

   set +x
   echo
   echo "Time is now $(date)."
   echo
   [ $DEBUGSCRIPTS = ON -o $DEBUGSCRIPTS = YES ]  &&  set -x

   if [ $ftperror -ne 0 ] ; then
      msg="Exiting with rc = $ftperror - TRANSFER OF $dsname FAILED AFTER \
$itries TRIES!!!!"
      postmsg "$jlogfile" "$msg"
      set +x
      echo
      echo $msg
      echo "Time is now $(date)."
      echo
      exit $ftperror
   fi
   msg="TRANSFER OF $dsname successful on try no. ${itries}."
   postmsg "$jlogfile" "$msg"
   mv $dsname_local ${dsname_local}_temp
   if [ $CNVRT2F77 = YES ] ; then
      recfm=`echo $ATTRIBUTES | cut -f1 -d" "`
      lrecl=`echo $ATTRIBUTES | cut -f2 -d" "`
      if [ $recfm = F -o $recfm = FB ] ; then
         rform=$(echo $recfm | tr [A-Z] [a-z])
         $utilexec/cnvblk -ib $rform:$lrecl -ob f77 ${dsname_local}_temp \
          $dsname_local
         cnvblk_err=$?
      else
         cnvblk_err=1
      fi
      if [ $cnvblk_err -eq 0 ] ; then
         rm ${dsname_local}_temp
      else
         msg="Exiting with rc = $cnvblk_err - CNVBLK FAILED"
         postmsg "$jlogfile" "$msg"
         set +x
         echo
         echo $msg
         echo "Time is now $(date)."
         echo
         exit $cnvblk_err
      fi
   else
      cp ${dsname_local}_temp $dsname_local
      rm ${dsname_local}_temp
   fi 
   if [ $MTYPSBT = YES ] ; then
#==============================================================================
#                   Execute program $EXECbufr/bufr_tranmtypsbt
#==============================================================================
      export encode_bufrtable=YES # encodes bufrtab.ttt into top of output file
      outfile=$OUTDIR/bufr_tranmtypsbt.out
      tmpout=$DATA/bufr_tranmtypsbt.tempout$$
      errlog=$OUTDIR/bufr_tranmtypsbt.errlog
      tmperr=$DATA/bufr_tranmtypsbt.temperr$$
      outstring="bufr_tranmtypsbt will run on file $dsname_local"
      lenstring=${#outstring}
      dashes="---------------------------------------------------------------"
      if [ -s $outfile ] ; then
         lenoutfile=$(cat $outfile | wc -l)
      else
         lenoutfile=0
      fi
      if [ -s $errlog ] ; then
         lenerrlog=$(cat $errlog | wc -l)
      else
         lenerrlog=0
      fi
      datestring="bufr_tranmtypsbt will run on \
$(date -u '+%Y/%m/%d at %H:%M:%S') GMT"
      lendatestring=${#datestring}
      [ $lendatestring -gt $lenstring ]  &&  lenstring=$lendatestring
      dashstring=$(echo $dashes$dashes | cut -c1-$lenstring)
      echo $dashstring >> $tmpout
      echo $outstring  >> $tmpout
      echo $datestring >> $tmpout
      echo $dashstring >> $tmpout
      cp $tmpout $tmperr
      TANKFILE_save=$TANKFILE
      export TANKFILE=$tankfile
      mv $dsname_local ${dsname_local}_temp
      export XLFRTEOPTS="unit_vars=yes"
      export XLFUNIT_11="${dsname_local}_temp"
      pgm=bufr_tranmtypsbt
      msg="$pgm has BEGUN"
      postmsg "$jlogfile" "$msg"
      set +u
#  Note - must use "prep_step" here not ". prep_step" because the latter
#         will wipe out the XLFUNITs that have already been set!
#######   prep_step
      prep_step
      set -u
      export XLFUNIT_51="$dsname_local"
      $EXECbufr/bufr_tranmtypsbt >> $tmpout 2>&1
      ier=$?
      datestring="bufr_tranmtypsbt : rc = $ier on \
$(date -u '+%Y/%m/%d at %H:%M:%S') GMT"
      lendatestring=${#datestring}
      outstring="bufr_tranmtypsbt has run on file $dsname_local"
      len=${#outstring}
      [ $len -gt $lenstring ]  &&  lenstring=$len
      [ $lendatestring -gt $lenstring ]  &&  lenstring=$lendatestring
      dashstring=$(echo $dashes$dashes | cut -c1-$lenstring)
      [ $ier -ne 0 ]  &&  echo $dashstring >> $tmperr
      echo $dashstring >> $tmpout
      [ $ier -ne 0 ]  &&  echo $outstring >> $tmperr
      echo $outstring >> $tmpout
      if [ $ier -ne 0 ] ; then
         echo $datestring >> $tmperr
         echo $dashstring >> $tmperr
      fi
      echo $datestring >> $tmpout
      echo $dashstring >> $tmpout
      if [ $lenoutfile -gt 130000 ] ; then
         head -n 120000 $outfile > $outfile.head$$
         cat $tmpout $outfile.head$$ > $outfile
         rm $outfile.head$$
      else
         cat $tmpout $outfile > $outfile.temp$$
         mv $outfile.temp$$ $outfile
      fi
      cat $tmpout
      rm $tmpout
      TANKFILE=$TANKFILE_save
      if [ $ier -eq 0 ] ; then
         rm ${dsname_local}_temp
      else
         msg="Exiting with rc = $ier - BUFR_TRANMTYPSBT FAILED"
         postmsg "$jlogfile" "$msg"
         set +x
         echo
         echo $msg
         echo "Time is now $(date)."
         echo
         set +x
         if [ $lenerrlog -gt 4000 ] ; then
            head -n 3000 $errlog > $errlog.head$$
            cat $tmperr $errlog.head$$ > $errlog
            rm $errlog.head$$
         else
            cat $tmperr $errlog > $errlog.temp$$
            mv $errlog.temp$$ $errlog
         fi
         rm $tmperr
         exit $ier
      fi
#==============================================================================
   else
      cp ${dsname_local}_temp $dsname_local
      rm ${dsname_local}_temp
   fi 
   cd $TANKDIR/$datecurr/$TANKSUBDIR
   if [ ! -s $TANKFILE -a $COPYFORWARD != YES ] ; then
      idayarch=0
      found=0
      ymdhback=$(echo "$datecurr*100 \n quit" | bc)
      while [ $idayarch -lt $ndayarch -a $found -eq 0 ] ; do
         idayarch=$(($idayarch+1))
         ymdhback=$($utilexec/ndate -24 $ymdhback)
         dateback=$(echo "$ymdhback/100 \n quit" | bc)
         if [ -s $TANKDIR/$dateback/$TANKSUBDIR/$TANKFILE ] ; then
            cp $TANKDIR/$dateback/$TANKSUBDIR/$TANKFILE .
            found=1
         fi
      done
   fi
   if [ -s $TANKFILE ] ; then
      cmp -s $TANKFILE $DATA/$dsname_local
      diffrc=$?
      if [ $diffrc -eq 0 ] ; then
         if [ $COPYFORWARD != YES ] ; then
            rm $TANKFILE
         fi
         exit 0
      elif [ $CNVRT2F77 = YES ] ; then
         wc -c $TANKFILE | read curbytes curfilename
         wc -c $DATA/$dsname_local | read newbytes newfilename
         if [ $newbytes -lt $curbytes ] ; then
            rm $TANKFILE
            exit 0
         fi
      fi
   fi
   cp $DATA/$dsname_local $TANKFILE
   echo "$dsname (for $datecurr) RECEIVED AT \
`date -u +%Y/%m/%d' '%H:%M:%S' GMT'`" >> $USERDIR/$dsname_hist.history
   if [ $dsname_local = $TANKFILE ]; then
      msg="$dsname copied from remote unix machine to \
$TANKDIR/$datecurr/$TANKSUBDIR directory"
   else
      msg="$dsname copied from remote unix machine to \
$TANKDIR/$datecurr/$TANKSUBDIR/$TANKFILE"
   fi
   postmsg "$jlogfile" "$msg"
   if [ $COPYFORWARD = YES -a $timecurr -lt $timemade ] ; then
      cp $TANKFILE $TANKDIR/$datenext/$TANKSUBDIR
      if [ $PROCSCRIPT = nullexec ] ; then
         echo "$dsname (for $datenext) COPIED AGAIN (from $dateback) AT \
`date -u +%Y/%m/%d' '%H:%M:%S' GMT'`" >> $USERDIR/$dsname_hist.history
         msg="$dsname copied again to $TANKDIR/$datenext/$TANKSUBDIR/$TANKFILE"
         postmsg "$jlogfile" "$msg"
      fi
   fi
fi
if [ -s $USERDIR/$dsname_hist.history ] ; then
   procfound=$(grep -c "$dsname (for $datecurr) PROCESSED" \
$USERDIR/$dsname_hist.history)
else
   procfound=0
fi
if [ $procfound -eq 0 ] ; then
   if [ $PROCSCRIPT != nullexec ] ; then
      cd $TANKDIR/$datecurr/$TANKSUBDIR
      if [ -s $USHbufr/$PROCSCRIPT ] ; then
         sh $USHbufr/$PROCSCRIPT
         procrc=$?
      elif [ -s $USERDIR/$PROCSCRIPT ] ; then
         sh $USERDIR/$PROCSCRIPT $TANKFILE $datecurr 
         procrc=$?
      fi
      if [ $procrc -eq 0 ] ; then
         echo "$dsname (for $datecurr) PROCESSED AT \
`date -u +%Y/%m/%d' '%H:%M:%S' GMT'`" >> $USERDIR/$dsname_hist.history
      fi
      if [ $COPYFORWARD = YES -a $timecurr -lt $timemade ] ; then
         ymdhback=$ymdhcurr
         dateback=$datecurr
         ymdhcurr=$ymdhnext
         datecurr=$datenext
         cd $TANKDIR/$datecurr/$TANKSUBDIR
         if [ -s $USHbufr/$PROCSCRIPT ] ; then
            sh $USHbufr/$PROCSCRIPT 
            retcode=$?
         elif [ -s $USERDIR/$PROCSCRIPT ] ; then
            sh $USERDIR/$PROCSCRIPT $TANKFILE $dateback
            retcode=$?
         fi
         cd $TANKDIR/$dateback/$TANKSUBDIR
         if [ $retcode -eq 0 ] ; then
            echo "$dsname (for $datecurr) COPIED and PROCESSED AGAIN (from \
$dateback) AT `date -u +%Y/%m/%d' '%H:%M:%S' GMT'`" >> \
             $USERDIR/$dsname_hist.history
         fi
      fi
   fi
fi

histlen=`wc -l < $USERDIR/$dsname_hist.history`
if [ $histlen -gt $HISTLENMAX ] ; then 
   histcut=$(($histlen-$HISTLENMIN))
   awk ' BEGIN   { nlines = 0 }
                 { 
                   nlines = nlines + 1;
                   if(nlines>'$histcut') print $0
                 } 
       ' $USERDIR/$dsname_hist.history > $USERDIR/$dsname_hist.newhist
   mv $USERDIR/$dsname_hist.newhist $USERDIR/$dsname_hist.history
fi