####  UNIX Script Documentation Block
#
# Script name:   existore.sh.sms
#
# JIF contact:  Keyser      org: NP22        date: 2009-05-04
#
# Abstract: Entry point for a series of scripts that will handle the
#   processing/ingest of time-stamped files from a remote unix machine.
#
# Script history log:
# 1996-10-02  Bert Katz   Original version for implementation (existoru.sh.sms
#        and existorc.sh.sms).
# 1996-11-29  Bert Katz   Added check for interrupted processing and unified
#        the output file.
# 1997-01-10  Bert Katz   Sent unified output file to "stdout" and added a
#        debug option.
# 1997-06-06  Bert Katz   Added the ability to get different families of files
#        from different UNIX machines.
# 1997-06-20  Bert Katz   Added an option to use a BUFR table to unpack
#       "foreign" input BUFR files.
# 1997-10-21  Bert Katz   Added a new directory to locate unchanging input
#        files.  Also added an option to use a BUFR table to unpack "foreign"
#        input BUFR files.
# 1997-11-03  Bert Katz   Emergency fix to allow all file streams to complete
#        processing after a failure in one file stream.
# 1997-11-13  Bert Katz   Changed handling of lapses in processing so that such
#        errors are recorded in jlogfile, but are not displayed as fatal
#        errors.  In addition, modified the script so that different file
#        streams can be processed using different values of CNVRT2F77 and
#        ASSIGNFTYPE.
# 1998-01-06  Bert Katz   Changed handling of file transfer problems so that
#        such errors are recorded in jlogfile, but are not displayed as fatal
#        errors.
# 1998-05-12  Bert Katz   Changed use of check_lapsed_orbits to
#        check_lapsed_data, which is y2k compliant 
# 1998-09-15  Bert Katz   Added the ability to give different families of files
#        different file attributes.
# 1998-12-02  Bert Katz   Established "nobufrtable" as default for variable
#        BUFRTABLE.
# 1998-12-23  Bert Katz   Changed handling of repeated file processing so that
#        such errors are recorded in jlogfile, but are not displayed as fatal
#        errors.
# 2000-11-21  D. Keyser   Modified the script so that different file streams
#        can be processed using different values of HISTLENMIN and HISTLENMAX.
#        If not found for a particular file stream, the global value is used.
# 2001-03-21  D. Keyser   Modified the script to add new value COMPRESS
#        (different file streams can be processed using different values of
#        COMPRESS - if not found for a particular file stream, the global
#        value is used).
# 2001-03-21  L. Sager    Modified the script to add new value UNCOMPRESS_UNIX
#        (different file streams can be processed using different values of
#        UNCOMPRESS_UNIX - if not found for a particular file stream, the
#        global value is used).
# 2001-05-14  D. Keyser   Modified the script to add new value ATTRIBUTES
#        (different file streams can be processed using different values of
#        ATTRIBUTES, if not found for a particular file stream, the global
#        value is used).
# 2004-05-10  D. Keyser   Modified the script to add new values PROCESS_Ta and
#        PROCESS_Tb (different file streams can be processed using different
#        values of PROCESS_Ta and PROCESS_Tb - if not found for a particular
#        file stream, the global value is used).
# 2005-04-29  D. Keyser   Modified the script to add new values PROCESS_Ta
#        and PROCESS_Tb (different file streams can be processed using
#        different values of PROCESS_Ta and PROCESS_Tb, if not found for a
#        particular file stream, the global value is used).
# 2006-04-19 P. OReilly   Modified the script to accept unique values for the
#        value HOURS2ALARM from job scripts.
# 2006-06-14  P. OReilly  Modified the script to add variable "CRITICAL" which
#        is used to set flags for Big Brother monitoring in script
#        ingest_check_lapsed_data.
# 2006-08-12  D. Keyser   Combines/generalizes previous scripts existorc.sh.sms
#        (for CEMSCS machine only) and existoru.sh.sms (for unix machines
#        only) and changed to account for all remote machines now being unix
#        (since MVS CEMSCS machine was replaced with unix DDS machine).
#        Improved Docblock and comments.  Accepts imported value for path to
#        ingest_qmgr.sh, $utilscript, defaults to /nwprod/util/ush.  If
#        execution of ingest_qmgr.sh returns status code of 99, now posts a
#        message to the joblog file that another job with this same name is in
#        the system and this ingest job will continue but not ingest any
#        satellite data (before no message was posted).
# 2007-05-14  D. Keyser   Now stops with FATAL error for case where no eligible
#        executable or script is found in one or more groups (families) (return
#        code 244).  Modified the script to add new values ITRIES_MAX_QUERY and
#        ITRIES_MAX_GET to set the maximum number of failed attempts to ftp
#        query and ftp get before giving up, respectively (both default to "2")
#        and to add the new value IFILES_MAX_GET to set the maximum number of
#        new files for which ftp transfers for a file family will occur
#        (defaults to "999").  Different file streams can be processed using
#        different values of ITRIES_MAX_QUERY, ITRIES_MAX_GET and
#        IFILES_MAX_GET (if not found for a particular file stream, the global
#        value is used).  Minor docblock changes.
# 2008-01-02  D. Keyser   Modified the script to add new value COPYFILES which,
#        when "YES", attempts to copy individual files pulled over from the
#        remote unix machine to directory $TANKDIR/YYYYMMDD/wbufbul after
#        translating them into NCEP BUFR and then standardizing them into WMO
#        BUFR (defaults to "NO").  Different file streams can be processed
#        using different values of COPYFILES (if not found for a particular
#        file stream, the global value is used).
# 2009-05-04  D. Keyser   Modified the script to add new value mod_sec3_desc to
#        alert the  program bufr_tranmtypsbt (if it is run) to either overwrite
#        (=YES) or not overwrite (=NO) the top-level sequence descriptor in
#        Sec. 3 of every BUFR message with the read-in value from the external
#        BUFR mnemonic table (defaults to "YES").  Different file streams can
#        be processed using different values of mod_sec3_desc (if not found for
#        a particular file stream, the global value is used).
#
# Usage: existore.sh.sms
#
#   Script parameters: None
#
#   Modules and files referenced:
#     scripts    : $USHbufr/ingest_process_onetype_neworbits
#                  $USHbufr/ingest_check_lapsed_data
#                  $DATA/prep_step
#                  $DATA/postmsg
#     data cards : file(s) containing BUFR tables (see below)
#     executables: executable(s) to create BUFR messages and append to tank
#                  file (see below)
#
# Remarks:
#
#   Condition codes:
#       0 - no problem encountered
#     > 0 - some problem encountered
#       Specifically:  99 - Another job with this name is in the system, let
#                           that job run in lieu of this job
#                     333 - Job has set CNVRT2F77=YES but ATTRIBUTES was not set
#                     244 - No eligible executable or script found in one or
#                           more groups (families) -- this is a FATAL error
#       In addition, this script will complete with condition code ZERO but
#        will stamp out an ABNORMAL COMPLETION diagnostic if any one of the
#        groups (families) returns a status code from child script
#        ingest_process_onetype_neworbits that is > 0 - these include:
#                       1 - Query failed to produce any files to process
#                       2 - The number of new files for this family exceeds the
#                           limit (imported variable IFILES_MAX_GET)
#                     111 - All files to be processed in a particular group
#                           (family) were unprocessable
#                     199 - One or more files were untransferable in last 5 (or
#                           possibly some other number) runs of this job
#                     220 - No action specified for multiple file families
#                     222 - All files to be processed were already processed
#                     230 - No files submitted for processing
#                     254 - Failure in sort of list of files to be processed or
#                           in sort of files in history file
#
#
#   The following script variables are user controllable.  They must be
#    specified by the user in the script that invokes
#   $USHbufr/ingest_process_onetype_neworbits (except where there are default
#   values noted).
#
#     USHbufr : directory where the operational scripts are located.
#     USERDIR : directory where all housekeeping files (i.e., lists of files
#       and file-processing history) are to be generated.
#     DATA : directory where files from the remote unix machine are to be put.
#     TANKDIR : directory where BUFR holding tanks are being created.
#     OUTDIR : directory where output listing is to be put.
#     EXECbufr : directory where the BUFR processing executables are located.
#     TABLEDIR : directory where the BUFR tables are located.
#     FTYPE : file type used to determine if some preprocessing on the file
#       transferred from the remote unix machine is necessary.  One may also
#       specify a different file assignment for one or more file families as
#       FTYPE1, FTYPE2,. . . (see below).  Possible values are:
#              bufr     - execute $USHbufr/cwordsh to block file and strip off
#                         any extraneous characters from BUFR messages
#                         (records) prior to executing $USHbufr/tranjb
#              ncepbufr - execute $USHbufr/cwordsh to block file and strip off
#                         any extraneous characters from BUFR messages
#                         (records) within execution of $USHbufr/tranjb
#              none     - do nothing (default)
#     UNCOMPRESS_UNIX : if set to "YES", files from a remote unix machine are
#       assumed to be unix-compressed and must be uncompressed.  One may also
#       specify different values for one or more file families as
#       UNCOMPRESS_UNIX1, UNCOMPRESS_UNIX2,. . . (see below). Defaults to "NO".
#     HISTLENMIN, HISTLENMAX : minimum and maximum length of the file keeping
#       track of the file-processing history. Each line of the file corresponds
#       to one file successfully processed.  One may also specify a different
#       minimum and/or maximum length for one or more file families as
#       HISTLENMIN1, HISTLENMIN2,. . . or HISTLENMAX1, HISTLENMAX2,. . . (see
#       below). HISTLENMIN defaults to "900" and HISTLENMAX defaults to "1200".
#     TARGETFILE : the name of the file when copied from the current working
#       directory to sub-directory TANKFILE (or TANKFILEn).  One may also
#       specify a different filename for one or more file families,
#       TARGETFILE1, TARGETFILE2, . . .(see below).  Defaults to "NO".
#     timetype : if set to "LOCAL", all times for processing will be assumed to
#       be in local time.  If set to "GMT", all times for processing will be
#       assumed to be in GMT.  If set to "USER_SPEC", the calling program is
#       specifying a "hardwired" time for processing using the script variable
#       user_spec_timedatecurr (see next below).  One may also specify a
#       different value for timetype for one or more files as timetype1,
#       timetype2,. . .  (see below).  Defaults to "GMT".
#     user_spec_timedatecurr : hardwired time which will be used as the time
#       in all processing.  It is specified in the form "yyyymmdd doy dow HHMM",
#       where "yyyy" is year, "mm" is month, "dd" is day of month, "doy" is day
#       of year, "dow" is day of week (1=Monday, ..., 7=Sunday), "HH" is hour
#       and "MM" is minute.  This is only invoked if timetype is set to
#       "USER_SPEC" (see first above).  One may also specify a different value
#       for user_spec_timedatecurr for one or more files as
#       user_spec_timedatecurr1, user_spec_timedatecurr2,. . .  (see below).
#       Defaults to "`date -u '+%Y%m%d %j %w %H%M'" (current GMT time).
#     HOURS2ALARM : in case file transfers or processing come to a halt, this
#       is the number of hours that will pass before a return code will be
#       issued to trigger an alarm mechanism.  One may also specify a different
#       number of hours for one or more file families as HOURS2ALARM1,
#       HOURS2ALARM2,. . . (see below).  Defaults to "30".
#     CRITICAL : if data type is critical to NCEP production, variable is set
#       to "YES" and data will flag red in Big Brother.  If data type is not
#       critical, variable is set to default value of "NO". One may specify
#       different values for different file families such as CRITICAL1, 
#       CRITICAL2, . . . (see below).  As said, defaults to "NO".
#     ndayarch : number of days that files from the remote unix machine will be
#       kept in directory DATA.
#     DELAFTPROC : if set to "YES", files from the remote unix machine are not
#       kept after processing.
#     COPYFILES  : if set to "YES", individual files from the remote unix
#       machine are copied to $TANKDIR/YYYYMMDD/wbufbul directory after being
#       translated into NCEP BUFR and then standardized into WMO BUFR
#       Note: This currectly works for files with name:
#                           *.Dyyddd.S*.*
#                           *.Dyyddd.T*.*
#                           *.Dyyyymmddhh.T*.*
#                           *.Dyyyymmddhh.S*.*
#             (where yy is 2-digit year, ddd is day-of-year, yyyy is 4-digit
#              year, mm is month-of-year, dd is day-of-month, hh is hour-of-
#              day)
#       One may also specify a different value for one or more file families as
#       COPYFILES1, COPYFILES2,. . . (see below).  Defaults to "NO".
#     MACHINE : name of remote unix machine to be used in ftp requests.  One
#       may also specify a different machine for one or more file families as
#       MACHINE1, MACHINE2,. . . (see below).  Defaults to "none".
#     log : full path name of the file to which standard output from
#       BUFR_TRANJB should be directed.  One may also specify a different
#       BUFR_TRANJB standard output file name for one or more file families as
#       log1, log2,. . . (see below).  Defaults to "$OUTDIR/tranjb.out".
#     CNVRT2F77 : if set to "YES", files from the remote unix machine are
#       converted to f77-blocked format.  One may also specify a different 
#       conversion for one or more file families as CNVRT2F771, CNVRT2F772,
#       . . . (see below).  Defaults to "NO".
#       NOTE: If CNVRT2F77 is set to "YES", ATTRIBUTES (see next below) MUST
#             contain attribute information (i.e., it cannot pass through with
#             its default of "NO")
#     ATTRIBUTES: if not set to "NO", then contains attribute information
#       common to all files transferred from the remote machine in form of
#       record format and record length, separated by blank characters.  This
#       is used to convert files from the remote unix machine to f77-blocked
#       format in conjuction with CNVRT2F77 (see first above) being set to
#       "YES".  If set to "NO", then no file attribute information is available
#       and CNVRT2F77 MUST be set to "NO".  One may also specify a different
#       value for ATTRIBUTES for one or more file families as ATTRIBUTES1,
#       ATTRIBUTES2,. . . (see below).  Defaults to "NO".
#     TANKPROTECT : if set to "YES", the tank file being updated will be copied
#       to a temporary file before being updated, after which the temporary
#       tank file will be moved over the permanent tank file.  If set to
#       anything other than "YES", the tank file will be updated in situ.
#     COMPRESS : if set to "YES", the tank file being updated will be
#       compressed (in BUFR) if there is an option to compress the particular
#       data type.  If set to anything other than "YES", the tank file will
#       remain uncompressed BUFR.  One may also specify different values for
#       one or more file families as COMPRESS1, COMPRESS2,. . . (see below).
#       Defaults to "NO".
#     DEBUGSCRIPTS : if set to "ON" or "YES", all scripts will run with
#      "set -x" on. Intended for debugging.
#     PROCESS_Ta : if set to "YES",  then process antenna temperature (Ta)
#       reports into BUFR, otherwise do not.  One may also specify different
#       values for one or more file families as PROCESS_Ta1, PROCESS_Ta2,
#       . . . (see below).  (Note: Antenna temperature can only be processed
#       for ATOVS AMSU-A data.)  Defaults to "NO".
#     PROCESS_Tb : if set to "YES",  then process brightness temperature (Tb)
#       reports into BUFR, otherwise do not.  One may also specify different
#       values for one or more file families as PROCESS_Tb1, PROCESS_Tb2,
#       . . . (see below).  Defaults to "YES".
#     ITRIES_MAX_QUERY: the maximum number of failed ftp attempts to the remote
#       machine in order to query files before giving up.  One may also specify
#       a different number of ftp query attempts for one or more file families
#       as ITRIES_MAX_QUERY1, ITRIES_MAX_QUERY2,. . . (see below).  Defaults to
#       "2".
#     ITRIES_MAX_GET: the maximum number of failed ftp attempts to the remote
#       machine in get a file before giving up.  One may also specify a
#       different number of ftp get attempts for one or more file families
#       as ITRIES_MAX_GET1, ITRIES_MAX_GET2,. . . (see below).  Defaults to
#       "2".
#     IFILES_MAX_GET: the maximum number of new files on the remote machine for
#       which ftp transfers for a file family will occur.  If greater than
#       $IFILES_MAX_GET new files are found on the remote machine for a
#       particular file family, the job will end without transferring and
#       processing any new files in that family.  One may also specify a
#       different number of new file transfer limits for one or more file
#       families as IFILES_MAX_GET1, IFILES_MAX_GET2,. . . (see below).
#       Defaults to "999".
#     mod_sec3_desc: if set to "NO", program bufr_tranmtypsbt (if it is run)
#       will not overwrite the top-level sequence descriptor in Sec. 3 of every
#       BUFR message with the read-in value from the external BUFR mnemonic
#       table, otherwise it will overwrite this.  (See Docblock in program
#       bufr_tranmtypsbt for more information.)  One may also specify different
#       values for one or more file families as mod_sec3_desc1, mod_sec3_desc2,
#       . . . (see below).  Defaults to "YES".
#       
#
#
#       The following environmental variables allow multiple file families on
#       the remote unix machine to be processed with one invocation of
#       $USHbufr/ingest_process_onetype_neworbits :
#     
#     ngroup : The number of file families from the remote unix machine that
#       are being processed.  
#
#       In the definitions that follow, the suffixed "n" or "N" means that a
#       list of variables such as : ORBITLIST1, ORBITLIST2, . . ,
#       ORBITLIST$ngroup should be defined for use by the script
#       $USHbufr/ingest_process_onetype_neworbits (some values below do have
#       default values).
#
#     COPYFILESn : if set to "YES", individual files from the remote unix
#       machine are copied to $TANKDIR/YYYYMMDD/wbufbul directory after being
#       translated into NCEP BUFR and then standardized into WMO BUFR
#       Note: This currectly works for files with name:
#                           *.Dyyddd.S*.*
#                           *.Dyyddd.T*.*
#                           *.Dyyyymmddhh.T*.*
#                           *.Dyyyymmddhh.S*.*
#             (where yy is 2-digit year, ddd is day-of-year, yyyy is 4-digit
#              year, mm is month-of-year, dd is day-of-month, hh is hour-of-
#              day)
#       For a particular family "n", if $COPYFILESn does not exist, then the
#       global default value $COPYFILES is used (see above).
#     MACHINEn : name of remote unix machine to be used in ftp requests.  For a
#       particular family "n", if $MACHINEn does not exist, then the global
#       default value "$MACHINE" is used (see above).
#     ORBITLISTn : the file keeping track of a family of files to be processed
#       on the remote unix machine.  The file containing the file-processing
#       history will have the name ORBITLISTn.history.
#     REMOTEDSNGRPn : the leading portion of the name of a family of files from
#       the remote unix machine.
#     FTYPEn : file type used to determine if some preprocessing on the file
#       transferred from the remote unix machine is necessary.  For a
#       particular family "n", if $FTYPEn does not exist, then the default
#       global value "$FTYPE" is used (see above, also see above for possible
#       values).
#     logN : full path name of the file to which standard output from
#       BUFR_TRANJB should be directed.  For a particular family "N", if $logN
#       does not exist, then the global default value "$log" is used (see
#       above).
#     CNVRT2F77n : if set to "YES", files from the remote unix machine are
#       converted to f77-blocked format. For a particular family "n", if
#       $CNVRT2F77n does not exist, then the default global value "$CNVRT2F77"
#       is used (see above).
#       NOTE: If CNVRT2F77n is set to "YES", ATTRIBUTESn (see next below) for a
#             particular family "n" MUST contain attribute information (i.e.,
#             it cannot pass through with a value of "NO")
#     ATTRIBUTESn: if not set to "NO", then contains attribute information
#       common to all files transferred from the remote machine in form of
#       record format and record length, separated by blank characters.  This
#       is used to convert files from the remote unix machine to f77-blocked
#       format in conjuction with CNVRT2F77n (see first above) for a particular
#       family "n" being set to "YES".  If set to "NO", then no file attribute
#       information is available and CNVRT2F77n MUST be set to "NO".  For a
#       particular family "n", if $ATTRIBUTESn does not exist, then the default
#       global value "$ATTRIBUTES" is used (see above).
#     HISTLENMINn, HISTLENMAXn : minimum and maximum length of the file keeping
#       track of the file-processing history. Each line of the file corresponds
#       to one file successfully processed.  For a particular family "n", if
#       $HISTLENMINn or $HISTLENMAXn does not exist, then the default global
#       values "$HISTLENMIN" and "$HISTLENMAX" are used (see above).
#     TANKFILEn : the name of the tank file to be created/appended (this can
#       also be a sub-directory or a path including a sub-directory and a file).
#     EXECUTEn : the name of a program or script used to create/append BUFR
#       messages to the tank file.  The record of the transaction will be
#       written to EXECUTEn.out and EXECUTEn.errlog in the TANKDIR (see above)
#       directory.  An exception occurs when EXECUTEn is set to
#       "copy_to_target", in which case a file is just simply copied from the
#       current working directory to file TARGETFILE (or TARGETFILEn)  in
#       sub-directory TANKFILE (or TANKFILEn).  If EXECUTEn is not set or if it
#       is set to "copy_to_target", the script "ingest_translate_orbits" will
#       not be executed.
#     TARGETFILEn : the name of the file when copied from the current working
#       directory to sub-directory TANKFILE (or TANKFILEn).  For a particular
#       family "n", if $TARGETFILEn does not exist, then the default global
#       value "$TANKFILE" is used (see above).
#     timetypeN : if set to "LOCAL", all times for processing will be assumed
#       to be in local time.  If set to "GMT", all times for processing will be
#       assumed to be in GMT.  If set to "USER_SPEC", the calling program is
#       specifying a "hardwired" time for processing using the script variable
#       user_spec_timedatecurrN (for a particular file "N", see next below).
#       For a particular file "N", if $timetypeN does not exist, then the
#       global default value "$timetype" is used (see above).
#     user_spec_timedatecurrN : hardwired time which will be used as the time
#       in all processing.  It is specified in the form "yyyymmdd doy dow HHMM",
#       where "yyyy" is year, "mm" is month, "dd" is day of month, "doy" is day
#       of year, "dow" is day of week (1=Monday, ..., 7=Sunday), "HH" is hour
#       and "MM" is minute.  This is only invoked if timetypeN is set to
#       "USER_SPEC" (for a particular file "N", see first above).  For a
#       particular file "N", if $user_spec_timedatecurrN does not exist, then
#       the global default value "$user_spec_timedatecurr" is used (see above).
#     BUFRTABLEn : the file holding the BUFR table which controls the creation
#       of BUFR messages to be added to the tank file.  Defaults to
#       "nobufrtable".
#     FORGNTABLEn : the file holding the BUFR table which controls the reading
#       of foreign BUFR messages, if the input file is in BUFR.  Defaults to
#       "noforgntable".
#     HOURS2ALARMn : in case file transfers or processing come to a halt, this
#       is the number of hours that will pass before a return code will be
#       issued to trigger an alarm mechanism.  For a particular family "n", if
#       $HOURS2ALARMn does not exist, then the global default value
#       "$HOURS2ALARM" is used (see above).
#     CRITICALn : if data type is critical to NCEP production, variable is set
#       to "YES" and data will flag red in Big Brother.  If data type is not
#       critical, variable is set to default value of "NO". For a particular
#       family "n", if $CRITICALn does not exist, then the global default value
#       "$CRITICAL" is used (see above).
#     COMPRESSn : if set to "YES", the tank file being updated will be
#       compressed (in BUFR) if there is an option to compress the particular
#       data type.  If set to anything other than "YES", the tank file will
#       remain uncompressed BUFR.  For a particular family "n", if $COMPRESSn
#       does not exist, then the default global value "$COMPRESS" is used (see
#       above).
#     UNCOMPRESS_UNIXn : if set to "YES", files from a remote unix machine are
#       assumed to be unix-compressed and must be uncompressed.  For a
#       particular family "n", if $UNCOMPRESS_UNIXn does not exist, then the
#       global default value "$UNCOMPRESS_UNIX is used (see above).
#     PROCESS_TaN : if set to "YES",  then process antenna temperature (Ta)
#       reports into BUFR, otherwise do not.  For a particular family "N", if
#       $PROCESS_TaN does not exist, then the default "$PROCESS_Ta" is used
#       (see above).  (Note: Antenna temperature can only be processed for
#       ATOVS AMSU-A data.)
#     PROCESS_TbN : if set to "YES",  then process brightness temperature (Tb)
#       reports into BUFR, otherwise do not.  For a particular family "N", if
#       $PROCESS_TbN does not exist, then the default "$PROCESS_Tb" is used
#       (see above).
#     ITRIES_MAX_QUERYn: the maximum number of failed ftp attempts to the
#       remote machine in order to query files before giving up.  For a
#       particular family "n", if "$ITRIES_MAX_QUERYn" does not exist, then the
#       global default value "$ITRIES_MAX_QUERY" is used (see above).
#     ITRIES_MAX_GETn: the maximum number of failed ftp attempts to the remote
#       machine in order to get a file before giving up.  For a particular
#       family "n", if "$ITRIES_MAX_GETn" does not exist, then the global
#       default value "$ITRIES_MAX_GET" is used (see above).
#     IFILES_MAX_GETn: the maximum number of new files on the remote machine
#       for which ftp transfers for a file family will occur.  If greater than
#       $IFILES_MAX_GETn new files are found on the remote machine for a
#       particular file family, the job will end without transferring and
#       processing any new files in that family.  For a particular family "n",
#       if "$IFILES_MAX_GETn" does not exist, then the global default value
#       "$IFILES_MAX_GET" is used (see above).
#     mod_sec3_descN: if set to "NO", program bufr_tranmtypsbt (if it is run)
#       will not overwrite the top-level sequence descriptor in Sec. 3 of every
#       BUFR message with the read-in value from the external BUFR mnemonic
#       table, otherwise it will overwrite this.  (See Docblock in program
#       bufr_tranmtypsbt for more information.)  For a particular family "N",
#       if $mod_sec3_descN does not exist, then the default $mod_sec3_desc is
#       used (see above).
#
#     It should be noted that, for each family of files, the environmental
#       variables with a suffixed "n" or "N" (i.e., COPYFILES1, MACHINE1,
#       ORBITLIST1, REMOTEDSNGRP1, FTYPE1, log1, CNVRT2F771, ATTRIBUTES1,
#       HISTLENMIN1, HISTLENMAX1, TANKFILE1, EXECUTE1, TARGETFILE1, timetype1,
#       user_spec_timedatecurr1, BUFRTABLE1, FORGNTABLE1, HOURS2ALARM1,
#       CRITICAL1, COMPRESS1, UNCOMPRESS_UNIX1, PROCESS_Ta1, PROCESS_Tb1,
#       ITRIES_MAX_QUERY1, ITRIES_MAX_GET1, IFILES_MAX_GET1, mod_sec3_desc1)
#       are assigned to variables without the "n" or "N" (i.e., COPYFILES,
#       MACHINE, ORBITLIST, REMOTEDSNGRP, FTYPE, log, CNVRT2F77, ATTRIBUTES,
#       HISTLENMIN, HISTLENMAX, TANKFILE, EXECUTE, TARGETFILE, timetype,
#       user_spec_timedatecurr, BUFRTABLE, FORGNTABLE, HOURS2ALARM, CRITICAL,
#       COMPRESS, UNCOMPRESS_UNIX, PROCESS_Ta, PROCESS_Tb, ITRIES_MAX_QUERY,
#       ITRIES_MAX_GET, IFILES_MAX_GET, mod_sec3_desc) and exported for use by
#       the script $USHbufr/ingest_process_onetype_neworbits.
#
# Attributes:
#
#   Language: aix unix
#   Machine:  NCEP CCS
####

#####################################################################
#
# START FLOW OF CONTROL
#
#####################################################################

########################################
set -aux
msg="$DATATYPE PROCESSING FROM TIME-STAMPED FILES HAS BEGUN"
postmsg "$jlogfile" "$msg"

########################################

utilscript=${utilscript:-/nwprod/util/ush}
$utilscript/ingest_qmgr.sh
errsc=$?

cd $DATA

if [ $errsc -eq 99 ]; then
   msg="Another job with this name is in the system, this ingest job will \
continue but not ingest any satellite data"
   echo $msg
   postmsg "$jlogfile" "$msg"
   exit $errsc
fi


pwd
ls -ltr

cd $DATA

##########################################

set +x
echo " "
echo "############################################################"
echo "         INGEST DATA FROM REMOTE UNIX MACHINE               "
echo "                 (TIME-STAMPED FILES)                       "
echo "############################################################"
echo " "
set -x

pgm='ingest_process_onetype_neworbits'

set +u
. prep_step
set -u

COPYFILES=${COPYFILES:-NO}
MACHINE=${MACHINE:-none}
FTYPE=${FTYPE:-none}
log=${log:-$OUTDIR/tranjb.out}
HOURS2ALARM=${HOURS2ALARM:-30}
CRITICAL=${CRITICAL:-NO}
UNCOMPRESS_UNIX=${UNCOMPRESS_UNIX:-NO}
CNVRT2F77=${CNVRT2F77:-NO}
ATTRIBUTES=${ATTRIBUTES:-NO}
HISTLENMIN=${HISTLENMIN:-900}
HISTLENMAX=${HISTLENMAX:-1200}
COMPRESS=${COMPRESS:-NO}
PROCESS_Ta=${PROCESS_Ta:-NO}
PROCESS_Tb=${PROCESS_Tb:-YES}
ITRIES_MAX_QUERY=${ITRIES_MAX_QUERY:-2}
ITRIES_MAX_GET=${ITRIES_MAX_GET:-2}
IFILES_MAX_GET=${IFILES_MAX_GET:-999}
mod_sec3_desc=${mod_sec3_desc:-YES}
TARGETFILE=${TARGETFILE:-NO}
timetype=${timetype:-GMT}
user_spec_timedatecurr=\
${user_spec_timedatecurr:-`date -u '+%Y%m%d %j %w %H%M'`}

FTYPE_save=$FTYPE
log_save=$log
CNVRT2F77_save=$CNVRT2F77
ATTRIBUTES_save=$ATTRIBUTES
HISTLENMIN_save=$HISTLENMIN
HISTLENMAX_save=$HISTLENMAX
COMPRESS_save=$COMPRESS
COPYFILES_save=$COPYFILES
MACHINE_save=$MACHINE
HOURS2ALARM_save=$HOURS2ALARM
CRITICAL_save=$CRITICAL
UNCOMPRESS_UNIX_save=$UNCOMPRESS_UNIX
PROCESS_Ta_save=$PROCESS_Ta
PROCESS_Tb_save=$PROCESS_Tb
ITRIES_MAX_QUERY_save=$ITRIES_MAX_QUERY
ITRIES_MAX_GET_save=$ITRIES_MAX_GET
IFILES_MAX_GET_save=$IFILES_MAX_GET
mod_sec3_desc_save=$mod_sec3_desc
TARGETFILE_save=$TARGETFILE
timetype_save=$timetype
user_spec_timedatecurr_save=$user_spec_timedatecurr

igroup=0
errsave=0
lapseret=0

#  Process each group of data (family)
#  -----------------------------------

while [ $igroup -lt $ngroup ] ; do

  igroup=$(($igroup+1))

  msg="$pgm group $igroup has begun."
  postmsg "$jlogfile" "$msg"

  set +x
  echo
  echo "####################################################################"
  echo "####################################################################"
  echo "              PROCESSING FOR GROUP $igroup FAMILY HAS BEGUN         "
  echo "####################################################################"
  echo "####################################################################"
  echo
  set -x

  eval ORBITLIST=$USERDIR/\$ORBITLIST$igroup
  eval COPYFILES=\${COPYFILES$igroup:-\$COPYFILES_save}
  eval MACHINE=\${MACHINE$igroup:-\$MACHINE_save}
  eval REMOTEDSNGRP=\$REMOTEDSNGRP$igroup
  eval FTYPE=\${FTYPE$igroup:-\$FTYPE_save}
  eval log=\${log$igroup:-\$log_save}
  eval CNVRT2F77=\${CNVRT2F77$igroup:-\$CNVRT2F77_save}
  eval ATTRIBUTES=\${ATTRIBUTES$igroup:-\$ATTRIBUTES_save}
  eval HISTLENMIN=\${HISTLENMIN$igroup:-\$HISTLENMIN_save}
  eval HISTLENMAX=\${HISTLENMAX$igroup:-\$HISTLENMAX_save}
  eval TANKFILE=\$TANKFILE$igroup
  eval EXECUTE=\${EXECUTE$igroup:-nullexec}
  eval BUFRTABLE=\${BUFRTABLE$igroup:-nobufrtable}
  eval FORGNTABLE=\${FORGNTABLE$igroup:-noforgntable}
  eval COMPRESS=\${COMPRESS$igroup:-\$COMPRESS_save}
  eval HOURS2ALARM=\${HOURS2ALARM$igroup:-$HOURS2ALARM_save}
  eval CRITICAL=\${CRITICAL$igroup:-$CRITICAL_save}
  eval UNCOMPRESS_UNIX=\${UNCOMPRESS_UNIX$igroup:-$UNCOMPRESS_UNIX_save}
  eval PROCESS_Ta=\${PROCESS_Ta$igroup:-\$PROCESS_Ta_save}
  eval PROCESS_Tb=\${PROCESS_Tb$igroup:-\$PROCESS_Tb_save}
  eval ITRIES_MAX_QUERY=\${ITRIES_MAX_QUERY$igroup:-\$ITRIES_MAX_QUERY_save}
  eval ITRIES_MAX_GET=\${ITRIES_MAX_GET$igroup:-\$ITRIES_MAX_GET_save}
  eval IFILES_MAX_GET=\${IFILES_MAX_GET$igroup:-\$IFILES_MAX_GET_save}
  eval mod_sec3_desc=\${mod_sec3_desc$igroup:-\$mod_sec3_desc_save}
  eval TARGETFILE=\${TARGETFILE$igroup:-\$TARGETFILE_save}
  eval timetype=\${timetype$igroup:-\$timetype_save}
  eval user_spec_timedatecurr=\
\${user_spec_timedatecurr$igroup:-\$user_spec_timedatecurr_save}

  if [ $CNVRT2F77 = YES -a "$ATTRIBUTES" = NO ]; then
     msg="Job has set CNVRT2F77=YES but ATTRIBUTES is not set to record \
format, record length and blocksize - ABNORMAL EXIT"
     echo $msg
     postmsg "$jlogfile" "$msg"
     exit 333
  elif [ $CNVRT2F77 = NO -a "$ATTRIBUTES" != NO ]; then
     msg="WARNING: Job has set ATTRIBUTES to record format, record length and \
blocksize but CNVRT2F77 is set to NO  --> non-fatal"
     echo $msg
     postmsg "$jlogfile" "$msg"
  fi

  sh $USHbufr/ingest_process_onetype_neworbits
  err=$?

  echo "error from ingest_process_onetype_neworbits is " $err
  if [ $err -gt $errsave ] ; then
    errsave=$err
  fi

  sh $USHbufr/ingest_check_lapsed_data $ORBITLIST.history

done

err=$errsave

cp $jlogfile jlogfile_local

grep -q "$jobid-No eligible executable or script named" jlogfile_local
err_grep=$?
[ $err_grep -eq 0 ]  &&  err=244

if [ $err -ne 0 ] 
then

#####################################################################
# ABNORMAL RUN
set +x
echo " "
echo " ****** PROCESSING COMPLETED ABNORMALLY (R.C.=$err)"
echo " ****** PROCESSING COMPLETED ABNORMALLY (R.C.=$err)"
echo " ****** PROCESSING COMPLETED ABNORMALLY (R.C.=$err)"
echo " ****** PROCESSING COMPLETED ABNORMALLY (R.C.=$err)"
echo " "
set -x
#####################################################################

#################################


   if [ $err -ne 244 ]; then
      msg="$DATATYPE PROCESSING FROM TIME-STAMPED FILES HAS COMPLETED \
ABNORMALLY WITH R.C.=$err  --> non-fatal"
      set +x
      echo
      echo $msg
      echo $err
      echo
      set -x
      postmsg "$jlogfile" "$msg"
   else
      msg="**FATAL ERROR: NO ELIGIBLE EXECUTABLE OR SCRIPT IN ONE OR MORE \
GROUPS (FAMILIES) IN $DATATYPE PROCESSING FROM TIME-STAMPED FILES - R.C.=$err"
      set +x
      echo
      echo $msg
      echo $err
      echo
      set -x
      postmsg "$jlogfile" "$msg"
      errexit
   fi

else

#####################################################################
# GOOD RUN
set +x
echo " "
echo " ****** PROCESSING COMPLETED NORMALLY"
echo " ****** PROCESSING COMPLETED NORMALLY"
echo " ****** PROCESSING COMPLETED NORMALLY"
echo " ****** PROCESSING COMPLETED NORMALLY"
echo " "
set -x
#####################################################################

#################################


   msg="$DATATYPE PROCESSING FROM TIME-STAMPED FILES HAS COMPLETED NORMALLY."
   echo $msg
   postmsg "$jlogfile" "$msg"

fi

############## END OF SCRIPT #######################