####  UNIX Script Documentation Block
#
# Script name:   existday.sh.sms
#
# JIF contact:  Keyser      org: NP22        date: 2008-01-24
#
# Abstract: Entry point for a series of scripts that will handle the transfer
#   of statically-named files from a remote unix machine.  The contents of
#   these files are assumed to be updated at most once per day.
#
# Script history log:
# 1996-10-02  Bert Katz   Original version for implementation
#        (existdyc.sh.sms).
# 1996-12-04  Bert Katz   Reorganized file structure to resemble the BUFR
#        tanking system.
# 1997-01-10  Bert Katz   Sent unified output file to "stdout".  Added check
#        for lapsed processing.  Added a debug option.
# 1997-11-03  Bert Katz   Emergency fix to allow all file streams to complete
#        processing after a failure in one file stream.
# 1997-11-13  Bert Katz   Changed handling of lapses in processing so that such
#        errors are recorded in jlogfile, but are not displayed as fatal
#        errors.
# 1997-12-17  Bert Katz   Changed handling of file-transfer failures so that
#        such errors are recorded in jlogfile, but are not displayed as fatal
#        errors.
# 1998-05-12  Bert Katz   Changed use of check_lapsed_days to
#        check_lapsed_data, which is y2k compliant.
# 2005-11-02  Bert Katz   Modified to allow for different tasks to set
#        different times-of-creation for the NESDIS files being acquired
# 2006-06-14  P. OReilly   Modified the script to add variable "CRITICAL" which
#        is used to set flags for Big Brother monitoring in script
#        ingest_check_lapsed_data.
# 2006-08-12  D. Keyser   Renamed from existdyc.sh.sms.  Generalized name of
#        remote machine (no longer hardwired to CEMSCS) and changed to account
#        for all remote machines now being unix (since MVS CEMSCS machine was
#        replaced with unix DDS machine).  Improved Docblock and comments.
#        Accepts imported value for path to ingest_qmgr.sh, $utilscript,
#        defaults to /nwprod/util/ush.  If execution of ingest_qmgr.sh returns
#        status code of 99, now posts a message to the joblog file that another
#        job with this same name is in the system and this ingest job will
#        continue but not ingest any satellite data (before no message was
#        posted).
# 2008-01-24  D. Keyser   Added new script variable MTYPSBT.
#
# Usage: existday.sh.sms
#
#   Script parameters: None
#
#   Modules and files referenced:
#     scripts    : $USHbufr/ingest_process_onetype_newdays
#                  $USHbufr/ingest_check_lapsed_data
#                  $DATA/prep_step
#                  $DATA/postmsg
#     data cards : None
#     executables: None
#
# Remarks:
#
#   Condition codes:
#       0 - no problem encountered
#     > 0 - some problem encountered
#       Specifically:  99 - Another job with this name is in the system, let
#                           that job run in lieu of this job
#                     333 - Job has set CNVRT2F77=YES but ATTRIBUTES was not
#                           set
#       In addition, this script will complete with condition code ZERO but
#        will stamp out an ABNORMAL COMPLETION diagnostic if any one of the
#        files returns a status code from child script
#        ingest_process_onetype_newdays that is > 0 - these include:
#                             -- currently none !!
#
#
#   The following script variables are user controllable.  They should be
#   specified by the user in the script that invokes
#   $USHbufr/ingest_process_onetype_newdays (except where there are default
#   values noted).
#
#     USHbufr : directory where the operational scripts are located.
#     USERDIR : directory where all housekeeping files (i.e., file-
#       processing history) are to be generated.
#     DATA : directory where files from the remote unix machine are to be put.
#     TANKDIR : directory where files are put for operational use.
#     EXECbufr : directory where executables are located.
#     HISTLENMIN, HISTLENMAX : minimum and maximum length of the file keeping
#       track of the file-processing history. Each line of the file corresponds
#       to one file successfully processed.
#     daysavail : the schedule for the availablity of the file.
#       Default is "Sun Mon Tue Wed Thu Fri Sat".
#       The following options are available :
#       (1) Some subset of "Sun Mon Tue Wed Thu Fri Sat"
#       (2) "DAY OF MONTH = n"
#       (3) "DAY OF MONTH = n MODULO k"
#       (4) "DAY OF YEAR = n"
#       (5) "DAY OF YEAR = n MODULO k"
#       One may also specify a different schedule for availability for one or
#       more files as daysavail1, daysavail2,. . . (see below). Defaults to
#       "Sun Mon Tue Wed Thu Fri Sat".
#       Note: If set to "justcopy" or "justcopyforward" the file is NEVER
#             transferred from the remote unix machine, it is ALWAYS simply
#             copied from the previous day.
#     dayafter : If the file will always be available on the calendar day after
#       the date with which it should be date-stamped, then "YES" should be
#       specified.  One may also specify a different value for this for one or
#       more files as dayafter1, dayafter2,. . . (see below). Defaults to "NO".
#     HOURS2ALARM : in case file transfers or processing come to a halt, this
#       is the number of hours that will pass before a return code will be
#       issued to trigger an alarm mechanism.  One may also specify a different
#       number of hours for one or more files as HOURS2ALARM1, HOURS2ALARM2,
#       . . . (see below).  Defaults to "30".
#     CRITICAL : if data type is critical to NCEP production, variable is set
#       to "YES" and data will flag red in Big Brother.  If data type is not
#       critical, variable is set to default value of "NO". One may specify
#       different values for different file families such as CRITICAL1,
#       CRITICAL2, . . . (see below).  As said, defaults to "NO".
#     MACHINE : name of remote unix machine to be used in ftp requests.
#       Defaults to "none".
#     CNVRT2F77 : if set to "YES", files from the remote unix machine are
#       converted to f77-blocked format.  One may also specify a different
#       conversion for one or more files as CNVRT2F771, CNVRT2F772,. . . (see
#       below).  Defaults to "NO".
#       NOTE: If CNVRT2F77 is set to "YES", ATTRIBUTES (see next below) MUST
#             contain attribute information (i.e., it cannot pass through with
#             its default of "NO")
#     ATTRIBUTES : if not set to "NO", then contains attribute information
#       common to all files transferred from the remote machine in form of
#       record format and record length, separated by blank characters.  This
#       is used to convert files from the remote unix machine to f77-blocked
#       format in conjuction with CNVRT2F77 (see first above) being set to
#       "YES".  If set to "NO", then no file attribute information is available
#       and CNVRT2F77 MUST be set to "NO".  One may also specify a different
#       value for ATTRIBUTES for one or more files as ATTRIBUTES1,
#       ATTRIBUTES2,. . .  (see below).  Defaults to "NO".
#     DEBUGSCRIPTS : if set to "ON" or "YES", all scripts will run with
#       "set -x" on. Intended for debugging.  Defaults to "OFF".
#     ndayarch : number of days that files from the remote unix machine will be
#       kept in directory $DATA.  Defaults to "10".
#     COPYFORWARD : if set to "YES", the first request for a file on a given
#       day will result in the most recent available file being "copied
#       forward" from the older directory to the current day's directory.  If
#       the file being remotely provided changes later that day, the new copy
#       will be retrieved and overwrite #the file that was "copied forward".
#       Defaults to "YES".
#     timemade : the 24-hour clock time when the files to be requested from the
#       remote unix machine will be available.  One may also specify a
#       different value for timemade for one or more files as timemade1,
#       timemade2,. . . (see below).  Defaults to "0000".
#     timetype : if set to "LOCAL", all times for processing will be assumed to
#       be in local time.  If set to "GMT", all times for processing will be
#       assumed to be in GMT.  If set to "USER_SPEC", the calling program is
#       specifying a "hardwired" time for processing using the script variable
#       user_spec_timedatecurr (see next below).  One may also specify a
#       different value for timetype for one or more files as timetype1,
#       timetype2,. . .  (see below).  Defaults to "GMT".
#     user_spec_timedatecurr : hardwired time which will be used as the time
#       in all processing.  It is specified in the form "yyyymmdd doy dow HHMM",
#       where "yyyy" is year, "mm" is month, "dd" is day of month, "doy" is day
#       of year, "dow" is day of week (1=Monday, ..., 7=Sunday), "HH" is hour
#       and "MM" is minute.  This is only invoked if timetype is set to
#       "USER_SPEC" (see first above).  One may also specify a different value
#       for user_spec_timedatecurr for one or more files as
#       user_spec_timedatecurr1, user_spec_timedatecurr2,. . .  (see below).
#       Defaults to "`date -u '+%Y%m%d %j %w %H%M'" (current GMT time).
#     MTYPSBT : if set to "YES", the program $EXECbufr/bufr_tranmtypsbt will
#       be executed by the child script ingest_process_days in order to encode
#       the external BUFR table into the beginning of the statically-named
#       file and to change the BUFR type and subtype internally in each non-
#       dictionary BUFR message.  One may also specify a different value for
#       MTYPSBT for one or more files as MTYPSBT1, MTYPSBT2,. . .  (see below).
#       Defaults to "NO"
#
#       The following environmental variables allow multiple files on the
#       remote unix machine to be processed with one invocation of
#       $USHbufr/ingest_process_onetype_newdays :
#     
#     nfiles : The number of files from the remote unix machine that are being
#       processed.
#
#       In the definitions that follow, the suffixed "n" or "N" means that a
#       list of variables such as : daysavail1, daysavail2, . . . ,
#       daysavail$nfiles should be defined for use by the script
#       $USHbufr/ingest_process_onetype_newdays (some values below do have
#       default values).
#
#     daysavailN : the schedule for the availablity of the file.
#       Default is "Sun Mon Tue Wed Thu Fri Sat".
#       The following options are available :
#       (1) Some subset of "Sun Mon Tue Wed Thu Fri Sat"
#       (2) "DAY OF MONTH = n"
#       (3) "DAY OF MONTH = n MODULO k"
#       (4) "DAY OF YEAR = n"
#       (5) "DAY OF YEAR = n MODULO k"
#       For a particular file "N", if $daysavailN does not exist, then the
#       global default value "$daysavail" is used (see above).
#       Note: If set to "justcopy" or "justcopyforward" the file is NEVER
#             transferred from the remote unix machine, it is ALWAYS simply
#             copied from the previous day.
#     dayafterN : If the file will always be available on the calendar day
#       after the date with which it should be date-stamped, then "YES" should
#       be specified.  For a particular file "N", if $dayafterN does not exist,
#       then the global default value "$dayafter" is used (see above).
#     CNVRT2F77n : if set to "YES", files from the remote unix machine are
#       converted to f77-blocked format. For a particular file "n", if
#       $CNVRT2F77n does not exist, then the default global value "$CNVRT2F77"
#        is used (see above).
#       NOTE: If CNVRT2F77n is set to "YES", ATTRIBUTESn (see next below) for
#             a particular file "n" MUST contain attribute information (i.e.,
#             it cannot pass through with a value of "NO")
#     ATTRIBUTESn : if not set to "NO", then contains attribute information
#       common to all files transferred from the remote machine in form of
#       record format and record length, separated by blank characters.  This
#       is used to convert files from the remote unix machine to f77-blocked
#       format in conjuction with CNVRT2F77n (see first above) for a particular
#       file "n" being set to "YES".  If set to "NO", then no file attribute
#       information is available and CNVRT2F77n MUST be set to "NO".  For a
#       particular file "n", if $ATTRIBUTESn does not exist, then the default
#       global value "$ATTRIBUTES" is used (see above).
#     PROCSCRIPTn : the name of the script to process the file.  If PROCSCRIPTn
#       is not set, no further processing is required.
#     dsnameN : The name of the file to be transferred from the remote unix
#       machine.
#     dsname_histN : The leading qualifier(s) in the file name in $USERDIR
#       containing file-processing history (file path is thus
#       $USERDIR/$dsname_hist.history).  Default is $dsnameN (see above).
#     TANKSUBDIRn : sub-directory holding file TANKFILEn (for a particular file
#       "n").
#     TANKFILEn : the name of the file when copied to sub-directory TANKSUBDIRn
#       (for a particular file "n").  Default is $dsnameN (see above).
#     HOURS2ALARMn : in case file transfers or processing come to a halt, this
#       is the number of hours that will pass before a return code will be
#       issued to trigger an alarm mechanism.  For a particular file "n", if
#       $HOURS2ALARMn does not exist, then the global default value
#       "$HOURS2ALARM" is used (see above).
#     CRITICALn : if data type is critical to NCEP production, variable is set
#       to "YES" and data will flag red in Big Brother.  If data type is not
#       critical, variable is set to default value of "NO". For a particular
#       family "n", if $CRITICALn does not exist, then the global default value
#       "$CRITICAL" is used (see above).
#     timemadeN : the 24-hour clock time when the files to be requested from
#       the remote unix machine will be available.  For a particular file "N",
#       if $timemadeN does not exist, then the global default value "$timemade"
#       is used (see above).
#     timetypeN : if set to "LOCAL", all times for processing will be assumed
#       to be in local time.  If set to "GMT", all times for processing will be
#       assumed to be in GMT.  If set to "USER_SPEC", the calling program is
#       specifying a "hardwired" time for processing using the script variable
#       user_spec_timedatecurrN (for a particular file "N", see next below).
#       For a particular file "N", if $timetypeN does not exist, then the
#       global default value "$timetype" is used (see above).
#     user_spec_timedatecurrN : hardwired time which will be used as the time
#       in all processing.  It is specified in the form "yyyymmdd doy dow HHMM",
#       where "yyyy" is year, "mm" is month, "dd" is day of month, "doy" is day
#       of year, "dow" is day of week (1=Monday, ..., 7=Sunday), "HH" is hour
#       and "MM" is minute.  This is only invoked if timetypeN is set to
#       "USER_SPEC" (for a particular file "N", see first above).  For a
#       particular file "N", if $user_spec_timedatecurrN does not exist, then
#       the global default value "$user_spec_timedatecurr" is used (see above).
#     MTYPSBTn : if set to "YES", the program $EXECbufr/bufr_tranmtypsbt will
#       be executed by the child script ingest_process_days in order to encode
#       the external BUFR table into the beginning of the statically-named
#       file and to change the BUFR type and subtype internally in each non-
#       dictionary BUFR message (for a particular file "n").  For a particular
#       file "n", if $MTYPSBTn does not exist, then the global default value
#       "$MTYPSBT" is used (see above).
#
#     It should be noted that, for each file, the environmental variables with
#       a suffixed "n" or "N" (i.e., daysavailN, dayafterN, CNVRT2F77n,
#       ATTRIBUTESn, PROCSCRIPTn, dsnameN, dsname_histN, TANKSUBDIRn,
#       TANKFILEn, HOURS2ALARMn, CRITICALn, timemadeN, timetypeN,
#       user_spec_timedatecurrN, MTYPSBTn) are assigned to variables without
#       the "n" or "N" (i.e., daysavail, dayafter, CNVRT2F77, ATTRIBUTES,
#       PROCSCRIPT, dsname, dsname_hist, TANKSUBDIR, TANKFILE, HOURS2ALARM,
#       CRITICAL, timemade, timetype, user_spec_timedatecurr, MTYPSBT) and
#       exported for use by the script $USHbufr/ingest_process_onetype_newdays.
#
# Attributes:
#
#   Language: aix unix
#   Machine:  NCEP CCS
####

#####################################################################
#
# START FLOW OF CONTROL
#
#####################################################################

########################################
set -aux
msg="$DATATYPE PROCESSING FROM STATICALLY-NAMED FILES HAS BEGUN"
postmsg "$jlogfile" "$msg"

########################################

utilscript=${utilscript:-/nwprod/util/ush}
$utilscript/ingest_qmgr.sh
errsc=$?

cd $DATA

if [ $errsc -eq 99 ]; then
   msg="Another job with this name is in the system, this ingest job will \
continue but not ingest any satellite data"
   echo $msg
   postmsg "$jlogfile" "$msg"
   exit $errsc
fi


pwd
ls -ltr

cd $DATA
 
##########################################

set +x
echo " "
echo "############################################################"
echo "         INGEST DATA FROM REMOTE UNIX MACHINE               "
echo "               (STATICALLY-NAMED FILES)                     "
echo "############################################################"
echo " "
set -x

pgm='ingest_process_onetype_newdays'
set +u
. prep_step
set -u

MACHINE=${MACHINE:-none}
ndayarch=${ndayarch:-10}
DEBUGSCRIPTS=${DEBUGSCRIPTS:-OFF}
COPYFORWARD=${COPYFORWARD:-YES}

daysavail_save=${daysavail:-"Sun Mon Tue Wed Thu Fri Sat"}
dayafter_save=${dayafter:-NO}
CNVRT2F77_save=${CNVRT2F77:-NO}
ATTRIBUTES_save=${ATTRIBUTES:-NO}
HOURS2ALARM_save=${HOURS2ALARM:-30}
CRITICAL_save=${CRITICAL:-NO}
timemade_save=${timemade:-0000}
timetype_save=${timetype:-GMT}
user_spec_timedatecurr_save=\
${user_spec_timedatecurr:-`date -u '+%Y%m%d %j %w %H%M'`}
MTYPSBT_save=${MTYPSBT:-NO}

ifiles=0
errsave=0
lapseret=0

#  Process each data file
#  ----------------------

while [ $ifiles -lt $nfiles ] ; do

  ifiles=$(($ifiles+1)) 

  msg="$pgm files $ifiles has begun."
  postmsg "$jlogfile" "$msg"

  set +x
  echo
  echo "####################################################################"
  echo "####################################################################"
  echo "                PROCESSING FOR FILE $ifiles HAS BEGUN               "
  echo "####################################################################"
  echo "####################################################################"
  echo
  set -x

  eval daysavail=\${daysavail$ifiles:-$daysavail_save}
  eval dayafter=\${dayafter$ifiles:-$dayafter_save}
  eval CNVRT2F77=\${CNVRT2F77$ifiles:-$CNVRT2F77_save}
  eval ATTRIBUTES=\${ATTRIBUTES$ifiles:-$ATTRIBUTES_save}
  eval PROCSCRIPT=\${PROCSCRIPT$ifiles:-nullexec}
  eval dsname=\$dsname$ifiles
  eval dsname_hist=\${dsname_hist$ifiles:-\$dsname$ifiles}
  eval TANKSUBDIR=\$TANKSUBDIR$ifiles
  eval TANKFILE=\${TANKFILE$ifiles:-\$dsname$ifiles}
  eval HOURS2ALARM=\${HOURS2ALARM$ifiles:-$HOURS2ALARM_save}
  eval CRITICAL=\${CRITICAL$ifiles:-$CRITICAL_save}
  eval timemade=\${timemade$ifiles:-$timemade_save}
  eval timetype=\${timetype$ifiles:-$timetype_save}
  eval user_spec_timedatecurr=\
\${user_spec_timedatecurr$ifiles:-$user_spec_timedatecurr_save}
  eval MTYPSBT=\${MTYPSBT$ifiles:-$MTYPSBT_save}

  if [ $CNVRT2F77 = YES -a "$ATTRIBUTES" = NO ]; then
     msg="Job has set CNVRT2F77=YES but ATTRIBUTES is not set to record \
format, record length and blocksize - ABNORMAL EXIT"
     echo $msg
     postmsg "$jlogfile" "$msg"
     exit 333
  elif [ $CNVRT2F77 = NO -a "$ATTRIBUTES" != NO ]; then
     msg="WARNING: Job has set ATTRIBUTES to record format, record length and \
blocksize but CNVRT2F77 is set to NO  --> non-fatal"
     echo $msg
     postmsg "$jlogfile" "$msg"
  fi

  sh $USHbufr/ingest_process_onetype_newdays
  err=$?

  echo "error from ingest_process_onetype_newdays is " $err
  if [ $err -gt $errsave ] ; then
    errsave=$err
  fi

  sh $USHbufr/ingest_check_lapsed_data $USERDIR/$dsname_hist.history

done

err=$errsave

if [ $err -ne 0 ] # currently should NEVER happen!!!
then

#####################################################################
# ABNORMAL RUN (NON FATAL)
set +x
echo " "
echo " ****** PROCESSING COMPLETED ABNORMALLY (R.C.=$err)"
echo " ****** PROCESSING COMPLETED ABNORMALLY (R.C.=$err)"
echo " ****** PROCESSING COMPLETED ABNORMALLY (R.C.=$err)"
echo " ****** PROCESSING COMPLETED ABNORMALLY (R.C.=$err)"
echo " "
set -x
#####################################################################

#################################


   msg="$DATATYPE PROCESSING FROM STATICALLY-NAMED FILES HAS COMPLETED \
ABNORMALLY WITH R.C.=$err  --> non-fatal"
   echo $msg
   echo $err
###err_chk

else

#####################################################################
# GOOD RUN
set +x
echo " "
echo " ****** PROCESSING COMPLETED NORMALLY"
echo " ****** PROCESSING COMPLETED NORMALLY"
echo " ****** PROCESSING COMPLETED NORMALLY"
echo " ****** PROCESSING COMPLETED NORMALLY"
echo " "
set -x
#####################################################################

#################################


   msg="$DATATYPE PROCESSING FROM STATICALLY-NAMED FILES HAS COMPLETED \
NORMALLY."
   echo $msg

fi
postmsg "$jlogfile" "$msg"

############## END OF SCRIPT #######################