####  UNIX Script Documentation Block
#
# Script name:   ingest_translate_orbits
#
# JIF contact:  Keyser      org: NP22        date: 2009-09-30
#
# Abstract: Runs some type of translation processing on time-stamped data files
#   previously received from a remote unix machine.  This may consist of
#   generating BUFR messages to append to a BUFR tank file, or possibly
#   generating GRIB files from ASCII or binary format files.
#
# Script history log:
# 1996-10-03  Bert Katz   Original version for implementation.
# 1996-11-29  Bert Katz   Unified the output file.
# 1997-01-09  Bert Katz   Output file now goes to stdout, reduced size of
#     cumulative output file, reduced size of errlog file, added debug option,
#     reduced number of attempts to process an orbit before giving up from 10
#     to 5.
# 1997-02-27  Bert Katz   Added fix to limit to 5 the number of attempts to
#     process a zero-length file.
# 1997-03-20  Bert Katz   Moved error handling out of this script and into
#     scripts "process_orbits" and "unixproc_orbits".
# 1997-06-20  Bert Katz   Added option of using a BUFR table to unpack a
#     "foreign" input BUFR file.
# 1997-07-17  Bert Katz   Added code to limit the "errlog" file to 3000 lines,
#     and to add the newest lines to the head of the file.
# 1997-09-12  Bert Katz   Changed input file assign from -Fsystem to
#     -sunblocked (-Fcos and -Ff77 remain as they were).
# 1997-09-18  Bert Katz   Added logic to the "foreign" input BUFR file
#     processing option to point to a foreign BUFR table in /nwprod/bufr/parm
#     if no such BUFR table exists in TABLEDIR.
# 1997-10-20  Bert Katz   Modified to permit multiple input file families.
# 1998-12-23  Bert Katz   Program tranjb is now used to append BUFR messages to
#     tanks.  The "trans" executables now make a temporary file for input into
#     tranjb.
# 2001-03-15  R. Treadon  Add special logical to handle file renaming for TRMM
#     TMI hdf format files.
# 2006-05-12  D. Keyser   Incorporates special processing for SSM/I data which
#     had been in previous script ingest_transsmi.sh (which is now removed).
#     Improved documentation and comments.  Increased length of output files
#     from max/min 60000/50000 to max/min 130000/120000.
# 2007-05-14  D. Keyser   Incorporates special processing for new AVHRR data.
#     Prints message to joblog file for case where no eligible executable or
#     script is found (will be used later to stop executing job with a FATAL
#     error).
# 2009-09-30  D. Keyser   Added script variable UPDATE_ERRLOG which directs
#     script to write information to errlog file when there is an error coming
#     out of the executing program or script - this is always YES (write to
#     errlog) except when EXECUTE is ingest_script_omi.sh and error is 2,
#     meaning the program BUFR_TRANOMI could not open the input HDF5 file -
#     this is not a true error because it is not yet ready for processing and
#     will be opened and processed the next time the ingest processing for the
#     OMI ozone family runs (this error happens often, this change will prevent
#     the errlog file from being updated)
#
# Usage: ingest_translate_orbits
#
#   Script parameters: none
#
#   Modules and files referenced:
#     scripts    : $USHbufr/tranjb
#                  $DATA/postmsg
#                  $DATA/prep_step
#                  $EXECUTE contains name of script to perform translation into
#                  BUFR (unless $EXECUTE is a program, see below)
#     data cards : none
#     executables: $EXECUTE contains name of program to perform translation
#                  into BUFR (unless $EXECUTE is a script, see above)
#
# Remarks: Invoked by the script ingest_process_orbits.
#
# Condition codes:
#     0 - no problem encountered
#   > 0 - some problem encountered
#     Specifically: 191 - bad argument list       
#                   195 - directory $DATA does not exist
#                   196 - directory $TANKDIR does not exist
#                   198 - input file $DATA/$dsname does not exist
#                   244 - Neither $EXECbufr/$EXECUTE nor $USHbufr/$EXECUTE nor
#                         $USERDIR/$EXECUTE exists
#
# Attributes:
#   Language: aix unix script
#   Machine:  NCEP CCS
####

set -au

echo
echo "#######################################################################"
echo "                START INGEST_TRANSLATE_ORBITS                          "
echo "#######################################################################"
echo

if [ $DEBUGSCRIPTS = ON -o $DEBUGSCRIPTS = YES ] ; then
  set -x
fi

TANKDIR=${TANKDIR:-$DATA}
UPDATE_ERRLOG=YES  # UPDATE_ERRLOG is initialized to YES, write to errlog file
                   # if there is a problem
if [ ! -d $DATA ] ; then
   set +x
   echo
   echo "$0: Directory $DATA doesn't exist." 
   echo
   exit 195
fi
if [ ! -d $TANKDIR ] ; then
   set +x
   echo
   echo "$0: Directory $TANKDIR doesn't exist." 
   echo
   exit 196
fi
cd $TANKDIR

if [ -s $EXECbufr/$EXECUTE ] ; then
   texec=$EXECbufr/$EXECUTE
   outname=${EXECUTE}
   EXECTYPE=executable
elif [ -s $USHbufr/$EXECUTE ] ; then
   texec=$USHbufr/$EXECUTE
   outname=${EXECUTE%%.sh}
   EXECTYPE=script
elif [ -s $USERDIR/$EXECUTE ] ; then
   texec=$USERDIR/$EXECUTE
   outname=${EXECUTE%%.sh}
   EXECTYPE=script
   if [ $outname = $EXECUTE ] ; then
      outname=${EXECUTE}
      EXECTYPE=executable
   fi
else
   msg="No eligible executable or script named $EXECUTE."
   set +x
   echo
   echo $msg
   echo
   set -x
   cd $DATA
   $DATA/postmsg "$jlogfile" "$msg"
   cd $TANKDIR
   exit 244
fi

outfile=$OUTDIR/$outname.out; tmpout=$DATA/$outname.tempout$$
file=$DATA/$dsname
table=$TABLEDIR/$BUFRTABLE

forgntable=$TABLEDIR/$FORGNTABLE
[ ! -s $forgntable ]  &&  forgntable=$PARMbufr/$FORGNTABLE

errlog=$OUTDIR/$outname.errlog
tmperr=$DATA/$outname.temperr$$

set $dsnamelist
nargs=$#
iargs=0
lenstring=0
lenerror=0
outstring=""

if [ $EXECTYPE = executable ] ; then
   if [ ${#TANKFILE} -eq 0 ] ; then
      typsubdir="."
   else
      typsubdir=$(dirname $TANKFILE)
   fi
   if [ $typsubdir != "." ] ; then
      subtypfil=$(basename $TANKFILE)
   else
      set +x
      echo
      echo "    <tankfile> must be given as : subdirectory/tankfile>" 
      echo "    in the following form:                bYYY/xxZZZ ."
      echo "    Last 3 digits of subdirectory are BUFR type." 
      echo "    Last 3 digits of tankfile are BUFR sub-type." 
      echo "    BUFR type and sub-type may be coded as three" 
      echo "    question marks if translation code provides" 
      echo "    BUFR type and sub-type." 
      echo
      exit 191
   fi
fi

while [ $iargs -lt $nargs ] ; do
   iargs=$(($iargs+1))
   eval dsname=\$$iargs
   eval outstring$iargs='"$EXECUTE will run on file $dsname"'
   eval len=\${#outstring$iargs}
   [ $len -gt $lenstring ]  &&  lenstring=$len
   file=$DATA/$dsname
   if [ ! -s $file ] ; then
      outstring="$dsname does not exist or has zero length"
      len=${#outstring}
      [ $len -gt $lenerror ]  &&  lenerror=$len
      echo $outstring >> $tmperr
   else
      if [ $EXECTYPE = executable ] ; then
         iunit=$((10+$iargs))
         export XLFRTEOPTS="unit_vars=yes"
         export XLFUNIT_$iunit="$file"
      fi
   fi
done

if [ ! -s $table ] ; then
   if [ $BUFRTABLE != none -a $BUFRTABLE != nulltable -a \
        $BUFRTABLE != nobufrtable ] ; then
      outstring="$table does not exist or has zero length"
      len=${#outstring}
      [ $len -gt $lenerror ]  &&  lenerror=$len
      echo $outstring  >> $tmperr
   fi 
else
   [ $EXECTYPE = executable ]  &&  export XLFUNIT_20="$table"
fi

if [ ! -s $forgntable ] ; then
   if [ $FORGNTABLE != none -a $FORGNTABLE != nulltable -a \
        $FORGNTABLE != noforgntable ] ; then
      outstring="$forgntable does not exist or has zero length"
      len=${#outstring}
      [ $len -gt $lenerror ]  &&  lenerror=$len
      echo $outstring  >> $tmperr
   fi 
else
   [ $EXECTYPE = executable ]  &&  export XLFUNIT_19="$forgntable"
fi

dashes="---------------------------------------------------------------"

if [ -s $outfile ] ; then
   lenoutfile=$(cat $outfile | wc -l)
else
   lenoutfile=0
fi

if [ -s $errlog ] ; then
   lenerrlog=$(cat $errlog | wc -l)
else
   lenerrlog=0
fi

if [ $lenerror -ne 0 ] ; then
   if [ $lenoutfile -gt 130000 ] ; then
      head -n 120000 $outfile > $outfile.head$$ 
      cat $tmpout $tmperr $outfile.head$$ > $outfile
      rm $outfile.head$$
   else
      cat $tmpout $tmperr $outfile > $outfile.temp$$
      mv $outfile.temp$$ $outfile
   fi
   dashstring=$(echo $dashes$dashes | cut -c1-$lenstring)
   echo $dashstring >> $tmpout
   iargs=0
   while [ $iargs -lt $nargs ] ; do
      iargs=$(($iargs+1))
      eval echo \$outstring$iargs >> $tmpout
   done
   echo $dashstring >> $tmpout
   cat $tmpout $tmperr
   rm $tmpout 
   dashstring=$(echo $dashes$dashes | cut -c1-$lenerror)
   echo $dashstring >> $tmperr
   if [ $UPDATE_ERRLOG = YES ]; then
      if [ $lenerrlog -gt 4000 ] ; then
         head -n 3000 $errlog > $errlog.head$$
         echo $dashstring | cat - $tmperr $errlog.head$$ > $errlog
         rm $errlog.head$$
      else
         echo $dashstring | cat - $tmperr $errlog > $errlog.temp$$
         mv $errlog.temp$$ $errlog
      fi
   fi
   rm $tmperr $file 
   exit 198  
fi

datestring="$EXECUTE will run on $(date -u '+%Y/%m/%d at %H:%M:%S') GMT"

lendatestring=${#datestring}
[ $lendatestring -gt $lenstring ]  &&  lenstring=$lendatestring

dashstring=$(echo $dashes$dashes | cut -c1-$lenstring)
echo $dashstring >> $tmpout

iargs=0
while [ $iargs -lt $nargs ] ; do
   iargs=$(($iargs+1))
   eval echo \$outstring$iargs >> $tmpout
done
echo $datestring >> $tmpout
echo $dashstring >> $tmpout
cp $tmpout $tmperr

ymdh=$(date -u '+%Y%m%d%H')
if [ $TANKPROTECT = YES ] ; then
   apndstring=_A_P_N_$dsname
else
   apndstring="' '"
fi

if [ $EXECTYPE = executable ] ; then

#  Come here if a PROGRAM is to be executed
#  ----------------------------------------

   pgm=$EXECUTE
   cd $DATA
   set +u
#  Note - must use "prep_step" here not ". prep_step" because the latter
#         will wipe out the XLFUNITs that have already been set!
#######   . $DATA/prep_step
   $DATA/prep_step
   set -u
   cd $TANKDIR

   export XLFUNIT_51="$DATA/$typsubdir.$subtypfil.$dsname.tmpout$$"

   stdin="$typsubdir $subtypfil $apndstring $ymdh"

   second_tranjb_run=NO

   ssmi_sdr=1
   ssmi_sdr_err=0
   if [ $EXECUTE = bufr_transsmi ]; then
      echo $dsname | grep -q SDR
      ssmi_sdr=$?
      if [ ssmi_sdr -eq 0 ];then
 
         second_tranjb_run=YES

#  Special processing for SSMI SDR type
#  get sstgrb & sstgrb index & input into bufr_transsmi, along w/ land/sea tags
#  also processes NN3 products, output in unit 52
#  ----------------------------------------------------------------------------

         cendat=$(cat /com/date/t00z | cut -c7-16)
         $utilscript/getges.sh -t sstgrb -v $cendat $DATA/sstgrb
         err1=$?
         if [ $err1 -eq 0 ] ; then
            rm errfile
            /nwprod/util/exec/grbindex $DATA/sstgrb $DATA/sstgrb.index 2>errfile
            err2=$?
            [ $err2 -ne 0 ]  &&  cat errfile
            rm errfile
         else
            err2=99
         fi
         ssmi_sdr_err=$(($err1+$err2))
         if [ $ssmi_sdr_err -eq 0 ] ; then
            export XLFUNIT_31="$DATA/sstgrb"
            export XLFUNIT_32="$DATA/sstgrb.index"
            export XLFUNIT_33="$utilparm/nesdis.lstags"
            export XLFUNIT_52=\
"$DATA/$typsubdir.second_tranjb_file.$dsname.tmpout$$"
            stdin="$typsubdir $subtypfil $apndstring $ymdh \nNOTIMLIM \
\n$typsubdir xx103 $apndstring $ymdh"
         fi
      fi

   elif [ $EXECUTE = "bufr_trantmi" ] ; then

#  Special processing for TRMM/TMI type (unzip input file)
#  -------------------------------------------------------

      rm -f tmi.hdf
      cp $file tmi.hdf.gz
      /usr/bin/gzip -d tmi.hdf.gz
      export XLFUNIT_$iunit="tmi.hdf"

   elif [ $EXECUTE = "bufr_tranavhrr" ] ; then

#  Special processing for AVHRR type (uses land sea mask, runs tranjb twice)
#  -------------------------------------------------------------------------

      second_tranjb_run=YES
      if [ $subtypfil = xx051 ]; then
         subtypfil_p1=xx052
      else
         subtypfil_p1=xx054
      fi
      export XLFUNIT_37="$FIXbufr/lnd_sea_mask_dat"
      export XLFUNIT_38="$FIXbufr/sst2dvar_aoi.61.90.clim.halfdeg"
      export XLFUNIT_52="$DATA/$typsubdir.second_tranjb_file.$dsname.tmpout$$"
      stdin="$typsubdir $subtypfil $apndstring $ymdh \nNOTIMLIM \
\n$typsubdir $subtypfil_p1 $apndstring $ymdh"
   fi

   if [ $ssmi_sdr_err -eq 0 ]; then

      msg="$pgm has BEGUN"
      cd $DATA
      $DATA/postmsg "$jlogfile" "$msg"
      cd $TANKDIR

      echo $stdin|$texec >> $tmpout 2>&1
      ier=$?
      if [ $ier -eq 0 ] ; then
         [ $EXECUTE = "bufr_trantmi" ]  &&  rm -f tmi.hdf
         give_rc="YES"
         cword="NO"
         sh $USHbufr/tranjb $TANKDIR \
          $DATA/$typsubdir.$subtypfil.$dsname.tmpout$$ 
         ier=$?
         if [ $second_tranjb_run = YES ]; then

#  SSMI SDR type runs a second tranjb to ingest NN3 products
#   (first tranjb run ingests brightness temperature data)
#  AVHRR type runs a second tranjb to ingest land or cloudy data
#   (first tranjb run ingests sea and clear data)
#  -------------------------------------------------------------

            sh $USHbufr/tranjb $TANKDIR \
             $DATA/$typsubdir.second_tranjb_file.$dsname.tmpout$$
            rc2=$?
            retcode=$(($ier+$rc2))
            ier=$retcode
         fi
      fi
   else
      ier = $ssmi_sdr_err
   fi
else

#  Come here if a SCRIPT is to be executed
#  ---------------------------------------

   sh $texec $TABLEDIR/$BUFRTABLE $dsnamelist >> $tmpout 2>&1
   ier=$?
fi

datestring="$EXECUTE : rc = $ier on $(date -u '+%Y/%m/%d at %H:%M:%S') GMT"
lendatestring=${#datestring}

numbadate=$(grep "BAD DATE" $tmpout | wc -l)
if [ $numbadate -gt 100 ] ; then
   grep -v "BAD DATE" $tmpout > $tmpout.nobadate
   echo "$numbadate OCCURRENCES OF BAD DATE REMOVED on \
$(date -u '+%Y/%m/%d at %H:%M:%S') GMT" | cat - $tmpout.nobadate > $tmpout
   rm $tmpout.nobadate
fi

iargs=0
lenstring=0
while [ $iargs -lt $nargs ] ; do
   iargs=$(($iargs+1))
   eval dsname=\$$iargs
   eval outstring$iargs='"$EXECUTE has run on file $dsname"'
   eval len=\${#outstring$iargs}
   [ $len -gt $lenstring ]  &&  lenstring=$len
done

[ $lendatestring -gt $lenstring ]  &&  lenstring=$lendatestring

dashstring=$(echo $dashes$dashes | cut -c1-$lenstring)

[ $ier -ne 0 ]  &&  echo $dashstring >> $tmperr

echo $dashstring >> $tmpout

iargs=0
while [ $iargs -lt $nargs ] ; do
   iargs=$(($iargs+1))
   [ $ier -ne 0 ]  &&  eval echo \$outstring$iargs >> $tmperr
   eval echo \$outstring$iargs >> $tmpout
done

if [ $ier -ne 0 ] ; then
   echo $datestring >> $tmperr
   echo $dashstring >> $tmperr
fi
echo $datestring >> $tmpout
echo $dashstring >> $tmpout

if [ $lenoutfile -gt 130000 ] ; then
   head -n 120000 $outfile > $outfile.head$$ 
   cat $tmpout $outfile.head$$ > $outfile
   rm $outfile.head$$
else
   cat $tmpout $outfile > $outfile.temp$$
   mv $outfile.temp$$ $outfile
fi
cat $tmpout 
rm $tmpout
if [ $ier -ne 0 ] ; then

#  If script ingest_script_omi.sh returns with r.c. = 2, do not update the
#   errlog (program BUFR_TRANOMI could not open the input HDF5 file - this is
#   not a true error because it is not yet ready for processing and will be
#   opened and processed the next time the ingest processing for the OMI ozone
#   family runs)
#  ---------------------------------------------------------------------------

   [ $EXECUTE=ingest_script_omi.sh -a $ier -eq 2 ]  &&  UPDATE_ERRLOG=NO
   if [ $UPDATE_ERRLOG = YES ]; then
      if [ $lenerrlog -gt 4000 ] ; then
         head -n 3000 $errlog > $errlog.head$$ 
         cat $tmperr $errlog.head$$ > $errlog
         rm $errlog.head$$
      else
         cat $tmperr $errlog > $errlog.temp$$
         mv $errlog.temp$$ $errlog
      fi
   fi
   rm $tmperr 
   exit $ier
elif [ $TANKPROTECT = YES ] ; then 
   iertot=0
   for apndfile in $(ls */$typsubdir/$subtypfil$apndstring) ; do
      tank=${apndfile%%$apndstring}
      mv $apndfile $tank 
      iertot=$(($iertot+$?))
   done
   exit $iertot
else
   exit $ier
fi