####  UNIX Script Documentation Block
#
# Script name:   ingest_process_orbits
#
# JIF contact:  Keyser      org: NP22        date: 2011-01-05
#
# Abstract: Initiates the transfer and processing of time-stamped files from a
#   remote unix machine.  
#
# Script history log:
# 1996-10-03  Bert Katz   Original version for implementation.
# 1996-11-29  Bert Katz   Unified the output file.
# 1997-01-09  Bert Katz   Unified output goes to "stdout", difficulty in
#     reading exactly one file is now considered to be a benign problem, debug
#     option added.
# 1997-04-07  Bert Katz   Moved error handling up from script
#     "translate_orbits", and added an option to execute another script instead
#     of "translate_orbits", (such as "tranjb") if desired.  Provided a summary
#     of file processing activities, as well as a graceful exit if ftp transfer
#     of a file fails 5 consecutive times 
# 1997-10-14  Bert Katz   Modified to handle multiple input file families.
# 1997-12-15  Bert Katz   Inability to transfer one of a related N-tuple of
#     files from N file families will cause the rest of the N-tuple to be
#     skipped so that transfer of the next N-tuple  files will begin.
# 1998-12-02  Bert Katz   Modified to use Y2K compliant version of tranjb.
# 2001-05-15  L. Sager    Added option to unix uncompress data under control of
#     keyword UNCOMPRESS_UNIX in the event that the file transferred from
#     NESDIS is unix-compressed.
# 2006-02-08  D. Keyser   Replaced call to obsolete program GRABBUFR with call
#     to program CWORDSH (via ush script cwordsh) which has been augmented to
#     perform all functions which GRABBUFR had done (and more).
# 2006-05-12  D. Keyser    Combines/generalizes this script (previously for
#     CEMSCS machine only) and script ingest_unixproc_orbits (for unix machines
#     only).  Changed to account for all remote machines now being unix (since
#     MVS CEMSCS machine was replaced with unix DDS machine).  Improved
#     documentation and comments.  Now exits immediately with rc=230 and posts
#     message to joblog if no files submitted for processing (used to exit
#     immediately with rc=0 and did not post a message to joblog).
# 2007-05-14  D. Keyser    Now uses imported script variable ITRIES_MAX_GET to
#     determine maximum number of failed attempts to get a file from remote
#     machine via ftp before giving up (had been hardwired to "2"); minor
#     script changes regarding the posting of messages to the joblog file
# 2008-01-31  D. Keyser    "COPY_TO_TARGET" date processing now more
#     generalized {handles date qualifier Dyyddd in any qualifier in file name
#     vs. only in qualifier 5 before, handles files with date qualifier in the
#     form mmddyyyy (new autosnow files)}, streamlines yyddd to yyyymmdd
#     conversion
# 2011-01-05  D. Keyser   Modified to post a message to the joblog file after
#     the successful execution of cwordsh in the event that cwordsh is unable
#     to block 1 or more input BUFR messages.  Export DX_SKIP=YES prior to
#     calls to cwordsh so that dictionary messages will not be copied by that
#     program (see cwordsh for more information on DX_SKIP).
#
# Usage: ingest_process_orbits
#
#   Script parameters: none
#
#   Modules and files referenced:
#     scripts    : $USHbufr/ingest_get
#                  $utilscript/date2jday.sh
#                  $USHbufr/ingest_translate_orbits
#                  $USHbufr/tranjb
#                  $USHbufr/cwordsh
#                  $DATA/postmsg
#     data cards : none
#     executables: /nwprod/util/exec/cwordsh (invoked by USHbufr/cwordsh)
#                  $utilexec/cnvblk
#
# Remarks: Invoked by the script ingest_process_onetype_neworbits.
#
# Condition codes:
#     0 - no problem encountered
#   > 0 - some problem encountered
#     Specifically: 111 - All files to be processed in a particular group
#                         (family) were unprocessable
#                   199 - One or more files were untransferable in last 5 (or
#                         possibly some other number) runs of this job
#                   222 - All files to be processed were already processed
#                   230 - No files submitted for processing
#
# Attributes:
#   Language: aix unix
#   Machine:  NCEP CCS
####

set -au

echo
echo "#######################################################################"
echo "                   START INGEST_PROCESS_ORBITS                         "
echo "#######################################################################"
echo

if [ $DEBUGSCRIPTS = ON -o $DEBUGSCRIPTS = YES ] ; then
   set -x
fi

orbitcount=0
procorbcount=0
repeatcount=0
noproccount=0
noxfercount=0
unproccount=0
unxfercount=0

read line
rc=$?
if [ $rc -ne 0 ] ; then
   msg="Exiting with rc = 230 - no files submitted for processing --> non-fatal"
   cwd=`pwd`
   cd $DATA
   $DATA/postmsg "$jlogfile" "$msg"
   set +x
   echo
   echo $msg
   echo
   exit 230
fi

typeset -Z2 mm dd

while [ $rc -eq 0 ] ; do
   set -A FILEINFO $line
   iword=0
   dsnamelist=""
   toterr=0
   yy=-99
   ddd=999
   yyyy=9999
   mm=99
   dd=99
   while [ $iword -lt ${#FILEINFO[*]} ] ; do
      neworbit=${FILEINFO[$iword]}
      dsname_full=$neworbit
      dsname=$(basename $neworbit)
      iword=$(($iword+1))
      orbitnotp_max=5
      orbitcount=$(($orbitcount+1))
      if [ -s $ORBITLIST.history ] ; then
         orbitdone=$(grep -c "$neworbit PROCESSED" $ORBITLIST.history)
         orbitnotp=$(grep -c "COULD NOT PROCESS $neworbit" $ORBITLIST.history)
      else
         orbitdone=0
         orbitnotp=0
      fi
      if [ $orbitdone -eq 0 -a $orbitnotp -lt $orbitnotp_max ] ; then

         set +x
         echo
         echo "Time is now $(date)."
         echo
         [ $DEBUGSCRIPTS = ON -o $DEBUGSCRIPTS = YES ]  &&  set -x

         ftperror=99
         itries=1
         while [ $ftperror -gt 0 -a $itries -le $ITRIES_MAX_GET ]; do
            if [ $itries -gt 1 ]; then
               cwd=`pwd`
               cd $DATA
               msg="TRANSFER OF $dsname_full FAILED!!!! - SLEEP 30 SEC AND \
TRY AGAIN."
               $DATA/postmsg "$jlogfile" "$msg"
               cd $cwd
               sleep 30
            fi
            sh $USHbufr/ingest_get $MACHINE $DATA/$dsname "$neworbit" 2>&1
            ftperror=$?
            itries=`expr $itries + 1`
         done
         itries=`expr $itries - 1`

         set +x
         echo
         echo "Time is now $(date)."
         echo
         [ $DEBUGSCRIPTS = ON -o $DEBUGSCRIPTS = YES ]  &&  set -x

         if [ $ftperror -eq 0 ] ; then
         echo "$dsname_full RECEIVED AT $(date -u +%Y/%m/%d' '%H:%M:%S' GMT')" \
          >> $ORBITLIST.history
            cwd=`pwd`
            cd $DATA
            msg="TRANSFER OF $dsname_full successful on try no. ${itries}."
            $DATA/postmsg "$jlogfile" "$msg"
            fdcperr=0
            cd $cwd
            if [ $CNVRT2F77 = YES ] ; then
               recfm=`echo $ATTRIBUTES | cut -f1 -d" "`
               lrecl=`echo $ATTRIBUTES | cut -f2 -d" "`
               if [ $recfm = F -o $recfm = FB ] ; then
                  rform=$(echo $recfm | tr [A-Z] [a-z])
                  $utilexec/cnvblk -ib $rform:$lrecl -ob f77 $DATA/$dsname \
                   $DATA/tmporbit
                  fdcperr=$?
               else
                  fdcperr=1
               fi
               if [ $fdcperr -eq 0 ] ; then
                  mv $DATA/tmporbit $DATA/$dsname
               else
                  msg="CNVBLK FAILED with rc = $fdcperr"
                  cwd=`pwd`
                  cd $DATA
                  $DATA/postmsg "$jlogfile" "$msg"
                  cd $cwd
                  set +x
                  echo
                  echo $msg
                  echo "Time is now $(date)."
                  echo
               fi
            elif [ $FTYPE = bufr ] ; then
               if [ $UNCOMPRESS_UNIX = YES ] ; then
                  uncompress $DATA/$dsname
                  uncerror=$?
                  if [ $uncerror -eq 0 ] ; then
                     nchar=`echo ${#dsname}`
                     nchar=`expr $nchar - 2`
                     dsname=`echo $dsname | cut -c1-$nchar`
                     cwd=`pwd`
                     cd $DATA
                     msg="UNIX UNCOMPRESS OF $dsname_full successful."
                     $DATA/postmsg "$jlogfile" "$msg"
                     cd $cwd
                  else
                     cwd=`pwd`
                     cd $DATA
                     msg="UNIX UNCOMPRESS OF $dsname_full UNSUCCESSFUL \
--> non-fatal"
                     $DATA/postmsg "$jlogfile" "$msg"
                     cd $cwd
                  fi
               fi
               DX_SKIP=YES # tell cwordsh to not copy any dictionary messages
               $USHbufr/cwordsh block $DATA/$dsname $DATA/tmporbit \
                |tee $DATA/cwordsh.out.$$
               fdcperr=$?
               if [ $fdcperr -eq 0 ] ; then
                  mv $DATA/tmporbit $DATA/$dsname
                  cwd=`pwd`
                  cd $DATA
                  msg=`grep "***WARNING" $DATA/cwordsh.out.$$`
                  err_grep=$?
                  msg=`echo $msg, input file: $dsname_full.`
                  if [ $err_grep -eq 0 ]; then
                     [ -n "$jlogfile" ] && $DATA/postmsg "$jlogfile" "$msg"
                  fi
                  msg="CWORDSH successful for $dsname_full."
                  $DATA/postmsg "$jlogfile" "$msg"
                  cd $cwd
               else
                  cwd=`pwd`
                  cd $DATA
                  msg="CWORDSH UNSUCCESSFUL or INCOMPLETE for $dsname_full \
--> non-fatal"
                  $DATA/postmsg "$jlogfile" "$msg"
                  cd $cwd
               fi
               rm $DATA/cwordsh.out.$$
            fi 
            toterr=$(($toterr+$fdcperr))
         else
            cwd=`pwd`
            cd $DATA
            msg="TRANSFER OF $dsname_full FAILED AFTER $itries TRIES!!!!"
            $DATA/postmsg "$jlogfile" "$msg"
            cd $cwd
            noxfercount=$(($noxfercount+1))
            echo "COULD NOT TRANSFER $dsname_full AT \
$(date -u +%Y/%m/%d' '%H:%M:%S' GMT')" >> $ORBITLIST.history
            orbitnotr=$(grep "$dsname_full" $ORBITLIST.history | \
             tail -n$orbitnotp_max | grep -c "COULD NOT TRANSFER $dsname_full")
            if [ $orbitnotr -ge $orbitnotp_max ] ; then
               unxfercount=$(($unxfercount+1))
               unproccount=$(($unproccount+1))
               echo "$dsname_full DECLARED UNPROCESSABLE AT \
$(date -u +%Y/%m/%d' '%H:%M:%S' GMT')" >> $ORBITLIST.history
               print -u9 $dsname_full
            fi
         fi
         toterr=$(($toterr+$ftperror))
         dsnamelist="$dsnamelist $dsname"
      elif [ $orbitdone -gt 0 ] ; then
         repeatcount=$(($repeatcount+1))
         echo "$dsname_full REPEATED AT $(date -u +%Y/%m/%d' '%H:%M:%S' GMT')" \
          >> $ORBITLIST.repeats
         print -u9 $dsname_full
         toterr=$(($toterr+1))
      elif [ $orbitnotp -ge $orbitnotp_max ] ; then
         unproccount=$(($unproccount+1))
         echo "$dsname_full DECLARED UNPROCESSABLE AT \
$(date -u +%Y/%m/%d' '%H:%M:%S' GMT')" >> $ORBITLIST.history
         print -u9 $dsname_full
         toterr=$(($toterr+1))
      fi
      if [ $toterr -gt 0 ] ; then
         break
      fi
   done

   if [ $toterr -eq 0 ] ; then
      if [ $EXECUTE = nullexec ] ; then
         bufrerror=0
      elif [ $EXECUTE = copy_to_target ] ; then

#  COPY_TO_TARGET
#  --------------

         if [ $TARGETFILE != NO ] ; then

#  Need to obtain YYYYMMDD from file name, first look for a qualifier with the
#   date in the form Dyyddd
#  ---------------------------------------------------------------------------

            num_qual=`echo $dsname | awk -F"." '{print NF}'`
            i=1
            while [ $i -le $num_qual ]; do
               qual=$(echo $dsname | cut -d"." -f${i})
               if [ `echo $qual | cut -c1` = D ]; then
                  inum=${#qual}
                  if [ $inum -eq 6 ]; then
                     qual1=`echo $qual | cut -c2-6`
                     if [[ ! -z $( echo $qual1 | grep "^[0-9]*$" ) ]]; then
                        if [ `echo $qual | cut -c2-3` -ge 0 -a \
                             `echo $qual | cut -c2-3` -le 99 ]; then
                           if [ `echo $qual | cut -c4-6` -ge 0 -a \
                                `echo $qual | cut -c4-6` -le 366 ]; then
                              yy=$(echo $qual | cut -c2-3)
                              ddd=$(echo $qual | cut -c4-6)
                              break
                           fi
                        fi
                     fi
                  fi
               fi
               i=`expr $i + 1`
            done

            if [ $yy -eq -99 ]; then

#  No luck, next look for a qualifier with the date in the form mmddyyyy (can
#   come only after a "_" delimiter, but may have a "." in column 9)
#  --------------------------------------------------------------------------

               num_qual=`echo $dsname | awk -F"_" '{print NF}'`
               i=1
               while [ $i -le $num_qual ]; do
                  qual=$(echo $dsname | cut -d"_" -f${i})
                  inum=${#qual}
                  if [ $inum -eq 8 -o \
                  \( "$inum" -gt '8' -a "`echo $qual | cut -c9`" = '.' \) ];then
                     qual1_8=`echo $qual | cut -c1-8`
                     if [[ ! -z $( echo $qual1_8 | grep "^[0-9]*$" ) ]]; then
                        if [ `echo $qual1_8 | cut -c5-8` -ge 1800 -a \
                             `echo $qual 1_8 | cut -c5-8` -le 3000 ]; then
                           if [ `echo $qual1_8 | cut -c1-2` -ge 01 -a \
                                `echo $qual1_8 | cut -c1-2` -le 12 ]; then
                              if [ `echo $qual1_8 | cut -c3-4` -ge 01 -a \
                                   `echo $qual1_8 | cut -c3-4` -le 31 ]; then
                                 yyyy=$(echo $qual1_8 | cut -c5-8)
                                 mm=$(echo $qual1_8 | cut -c1-2)
                                 dd=$(echo $qual1_8 | cut -c3-4)
                              fi
                           fi
                        fi
                     fi
                  fi
                  i=`expr $i + 1`
               done
            fi

            if [ $yy -eq -99 -a $yyyy -eq 9999 ]; then

#  Still no luck, give up
#  ----------------------

               bufrerror=99
            else
               if [ $yyyy -eq 9999 ]; then

#  The date is of the form Dyyjjj - use windowing technique to convert 2-digit
#   yr to 4-digit yr, then convert year and day-of-year to year, month and day
#  ---------------------------------------------------------------------------

                  if [ $yy -gt 20 ]; then
                     yyyyddd=19${yy}${ddd}
                  else
                     yyyyddd=20${yy}${ddd}
                  fi

                  yyyymmdd=`$utilscript/date2jday.sh $yyyyddd`

               else
                  yyyymmdd=${yyyy}${mm}${dd}
               fi
            fi
            if [ ! -d $TANKDIR/$yyyymmdd/$TANKFILE ] ; then
               mkdir -p $TANKDIR/$yyyymmdd/$TANKFILE
            fi
            cp $dsname $TANKDIR/$yyyymmdd/$TANKFILE/$TARGETFILE
            bufrerror=$?
            if [ $bufrerror -eq 0 ] ; then
               echo "$dsname_full RECEIVED FROM REMOTE MACHINE, $TARGETFILE \
(for $yyyymmdd) COPIED AT `date -u +%Y/%m/%d' '%H:%M:%S' GMT'`" >> \
                $USERDIR/$TARGETFILE.history
               msg="$dsname_full received from remote unix machine, \
$TARGETFILE copied for $yyyymmdd"
               $DATA/postmsg "$jlogfile" "$msg"
            fi
         else
            bufrerror=99
         fi
      elif [ $EXECUTE = tranjb ] ; then
         give_rc="YES"
         if [ $FTYPE = ncepbufr ] ; then
            cword="yes"
            DX_SKIP=YES # tell cwordsh to not copy any dictionary messages
         else
            cword="NO"
         fi
         sh $USHbufr/tranjb $TANKDIR $DATA/$dsname 
         bufrerror=$?
      else
         sh $USHbufr/ingest_translate_orbits
         bufrerror=$?
      fi
      set +x
      echo
      echo "Time is now $(date)."
      echo
      [ $DEBUGSCRIPTS = ON -o $DEBUGSCRIPTS = YES ]  &&  set -x

      if [ $bufrerror -eq 0 ] ; then
         iword=0
         while [ $iword -lt ${#FILEINFO[*]} ] ; do
            neworbit=${FILEINFO[$iword]}
            dsname=$(basename $neworbit)
            print -u9 $neworbit
            procorbcount=$(($procorbcount+1))
           echo "$neworbit PROCESSED AT $(date -u +%Y/%m/%d' '%H:%M:%S' GMT')" \
            >> $ORBITLIST.history
            if [ $DELAFTPROC = YES ] ; then
               rm $DATA/$dsname
            fi
            iword=$(($iword+1))
         done
      else
         iword=0
         while [ $iword -lt ${#FILEINFO[*]} ] ; do
            neworbit=${FILEINFO[$iword]}
            dsname=$(basename $neworbit)
            noproccount=$(($noproccount+1))
            echo "COULD NOT PROCESS $neworbit AT \
$(date -u +%Y/%m/%d' '%H:%M:%S' GMT')" >> $ORBITLIST.history
            if [ $DELAFTPROC = YES ] ; then
               rm $DATA/$dsname
            fi
            iword=$(($iword+1))
         done
      fi 
   fi
   read line
   rc=$?
done

if [ -s $ORBITLIST.history ] ; then
   histlen=$(wc -l < $ORBITLIST.history)
   if [ $histlen -gt $HISTLENMAX ] ; then 
      histcut=$(($histlen-$HISTLENMIN))
      awk ' BEGIN   { nlines = 0 }
                    { 
                      nlines = nlines + 1;
                      if(nlines>'$histcut') print $0
                    } 
          ' $ORBITLIST.history > $ORBITLIST.newhist$$
      mv $ORBITLIST.newhist$$ $ORBITLIST.history
   fi
fi
if [ -s $ORBITLIST.repeats ] ; then
   reptlen=$(wc -l < $ORBITLIST.repeats)
   if [ $reptlen -gt $HISTLENMAX ] ; then 
      reptcut=$(($reptlen-$HISTLENMIN))
      awk ' BEGIN   { nlines = 0 }
                    { 
                      nlines = nlines + 1;
                      if(nlines>'$reptcut') print $0 
                    } 
          ' $ORBITLIST.repeats > $ORBITLIST.newrept$$
      mv $ORBITLIST.newrept$$ $ORBITLIST.repeats
   fi
fi

set +x
echo
echo "Exiting - summary of processing : "
echo
echo "$orbitcount files were submitted for processing."
echo
if [ $procorbcount -gt 0 ] ; then
   echo "$procorbcount files were sucessfully processed."
fi
if [ $noproccount -gt 0 ] ; then
   msg="$noproccount files were unsuccessfully processed --> non-fatal"
   echo $msg
   $DATA/postmsg "$jlogfile" "$msg"
fi
if [ $noxfercount -gt 0 ] ; then
   msg="$noxfercount files were unsuccessfully transferred --> non-fatal"
   echo $msg
   $DATA/postmsg "$jlogfile" "$msg"
fi
if [ $unproccount -gt 0 ] ; then
   msg="$unproccount files were declared unprocessable --> non-fatal"
   echo $msg
   $DATA/postmsg "$jlogfile" "$msg"
fi
if [ $repeatcount -gt 0 ] ; then
   msg="$repeatcount files had been previously processed --> non-fatal"
   echo $msg
   $DATA/postmsg "$jlogfile" "$msg"
fi
echo
[ $DEBUGSCRIPTS = ON -o $DEBUGSCRIPTS = YES ]  &&  set -x

if [ $unxfercount -gt 0 ] ; then
   msg="Exiting with rc = 199 - $unxfercount files were untransferable in last \
$orbitnotp_max runs of this job --> non-fatal"
   $DATA/postmsg "$jlogfile" "$msg"
   set +x
   echo
   echo $msg
   echo
   exit 199
fi
if [ $orbitcount -gt 1 ] ; then
   if [ $orbitcount -eq $repeatcount ] ; then
      msg="Exiting with rc = 222 - all files to be processed were already \
processed --> non-fatal"
      $DATA/postmsg "$jlogfile" "$msg"
      set +x
      echo
      echo $msg
      echo
      exit 222
   fi
   if [ $orbitcount -eq $noproccount ] ; then
      if [ -n "$igroup" ]; then
         msg="Exiting with rc = 111 - all group $igroup files to be processed \
were unprocessable --> non-fatal" 
      else
         msg="Exiting with rc = 111 - all files to be processed were \
unprocessable --> non-fatal" 
      fi
      $DATA/postmsg "$jlogfile" "$msg"
      set +x
      echo
      echo $msg
      echo
      exit 111
   fi
fi
exit 0