#### UNIX Script Documentation Block # # Script name: ingest_check_lapsed_data # # JIF contact: Keyser org: NP22 date: 2010-07-06 # # Abstract: Checks for lapses in the transfer and processing of data files from # a remote unix server. It uses the appropriate history files to determine # the last time a data file was successfully received and/or processed. # Depending upon the value of HOURS2ALARM, a return code is issued indicating # whether a significant break has occurred in file transfer or processing. # # Script history log: # 1998-05-12 Bert Katz Original version for implementation. # 1999-11-17 Bert Katz Added capability for HOURS2ALARM to be a list of # numbers. The list may be any length, but only that part of the the # list which corresponds (one-to-one) with the file families in # REMOTEDSNGRP will be used. If the list of numbers in HOURS2ALARM # contains only one number, that number will be used as the number of # hours before a break in transfer/processing is indicated for ALL file # families in REMOTEDSNGRP. If the list of numbers in HOURS2ALARM # contains fewer numbers than the number of file families in # REMOTEDSNGRP, then the last number in HOURS2ALARM will be used for all # remaining file families in REMOTEDSNGRP which lack a corresponding # number in HOURS2ALARM. If the list of numbers in HOURS2ALARM is in # one-to-one correspondence with the list of file families in # REMOTEDSNGRP, then each file family in REMOTEDSNGRP is paired with a # unique number of hours indicating a break in #transfer/processing. # 2005-10-11 D. Keyser Includes the NESDIS 4km (96'th mesh) IMS snow cover # data, file imssnow96.grb, as a type for which a lapsed date warning # will not be posted to the production joblog file. # 2006-03-30 P. O'Reilly Modified to create flag files for use with Big # Brother monitoring to alert the lapse in data. # 2006-04-25 P. O'Reilly Modified to add logic to distinguish between two # files in the snow job that are identically named, to help distinguish # them in Big Brother monitoring. # 2006-06-14 P. O'Reilly Modified to add logic to distinguish between files # that are of CRITICAL or non-CRITICAL nature for Big Brother monitoring. # 2006-08-12 D. Keyser Improved documentation and comments, more appropriate # messages posted to joblog. Changed to account for all remote machines # now being unix (since MVS CEMSCS machine was replaced with unix DDS # machine). # 2008-01-31 D. Keyser Now converts each embedded "." character in file # group name (normally delineates qualifiers in filename) to "\." # (escapes "." so it will be treated as a period and not as a special # character in the grep from the history files to check for lapses in # the transfer and processing of data files from the remote server). # Now converts each embedded "*" character in file group name (wildcard # matching any string of 1 or more characters) to ".*" so it will be # treated the same way in the grep. Now converts each "?" character in # file group name (wildcard matching exactly 1 character) to "." so it # will be treated the same way in the grep. In grep, includes a blank # space after converted file group name if it originally contained any # "*" or "?" characters - this will clearly define the end of the file # name. Otherwise, do not include the blank, as the end of the filename # is a wildcard string extending beyond the file group name here. # 2010-07-06 D. Keyser Will not expand filenames which include substitution # characters "?" and "*" in case files being sftp'd are on same CCS # machine as that in which the job is running. # # Usage: ingest_check_lapsed_data history_file # # Script parameters: (1) history_file (complete path to history file) # # Modules and files referenced: # scripts : $DATA/postmsg # data cards : none # executables: /nwprod/util/exec/nhour # # Remarks: Invoked by the model scripts existore.sh.sms and existday.sh.sms. # # Condition codes: # 0 - no problem encountered # > 0 - some problem encountered # # Attributes: # Language: aix unix script # Machine: NCEP CCS #### set -au echo echo "#######################################################################" echo " START CHECK_LAPSED_DATA " echo "#######################################################################" echo DEBUGSCRIPTS=${DEBUGSCRIPTS:-OFF} if [ $DEBUGSCRIPTS = ON -o $DEBUGSCRIPTS = YES ] ; then set -x fi if [ $# -lt 1 ] ; then echo "Arguments to ingest_check_lapsed_data are:" echo " (1) history_file" exit 0 fi dsname_hist=${dsname_hist:-null} if [ $dsname_hist.history = $(basename $1) ] ; then ftype=file filenames=$dsname else ftype=filegroup filenames="$REMOTEDSNGRP" fi mcount=0 if [ -s $1 ] ; then for RECPRO in RECEIVED PROCESSED ; do ncount=0 filenames_save="$filenames" echo "$filenames_save" | read firstword restofstring while [[ "$firstword" != "" ]] ; do filenames_save="$restofstring" eval fname=\$firstword # Undo typesets on $hourwait from previous loop (i.e., start fresh) # ----------------------------------------------------------------- typeset +LZ hourwait typeset +Z1 hourwait if [ "$fname" = concatenate_families -o "$fname" = correlate_familes -o \ "$fname" = concatenate_fams -o "$fname" = correlate_fams -o \ "$fname" = concatenate -o "$fname" = correlate -o \ "$fname" = concat_families -o "$fname" = correl_families -o \ "$fname" = concat_fams -o "$fname" = correl_fams -o \ "$fname" = concat -o "$fname" = correl ] ; then echo "$filenames_save" | read firstword restofstring continue fi # Prepare file group name for grep'ing in history file # ---------------------------------------------------- # Convert each "." character in file group name (normally delineates # qualifiers in filename) to "\." (escapes "." so it will be treated as a # period and not as a special character later in the grep) # ------------------------------------------------------------------------ fname_grep1=$(echo "$fname" | sed "s/\./\\\./g") # Convert each embedded "*" character in file group name (wildcard matching # any string of 1 or more characters) to ".*" (it will then also be treated # as a wildcard matching any string later in the grep) # --------------------------------------------------------------------------- fname_grep2=$(echo $fname_grep1 | sed "s/*/.*/g") # Convert each "?" character in file group name (wildcard matching exactly 1 # character) to "." (it will then also be treated as a wildcard matching # exactly one character later in the grep) # -------------------------------------------------------------------------- fname_grep3=$(echo $fname_grep2 | sed "s/?/./g") # In grep, include a blank space after converted file group name if it # originally contained any "*" or "?" characters - this will clearly define # the end of the file name. Otherwise, do not include the blank, as the # end of the filename is a wildcard string extending beyond the file group # name here. # -------------------------------------------------------------------------- echo "$fname" | grep -Fe "*" -Fe "?" iret=$? if [ $iret -eq 0 ]; then blank=" " else blank="" fi grep $RECPRO $1 | grep "$fname_grep3$blank" | tail -n1 | awk ' { n=split($0,ymdarr,"/"); m=split(ymdarr[n-2],yeararr," "); l=split(ymdarr[n],dayarr," "); k=split($0,hmsarr,":"); j=split(hmsarr[k-2],hourarr," "); print yeararr[m],ymdarr[n-1],dayarr[1],hourarr[j],hmsarr[k-1] } ' | read lyear lmonth lday lhour lminute retcode=$? if [ $retcode -ne 0 ] ; then echo "$filenames_save" | read firstword restofstring continue fi # Do not use "typeset -LZ2 lmonth lday lhour lminute cmonth cday chour cminute" # because: 1) the expressions in the first and third lines below DO NOT # interpret values with leading zeroes as octal # 2) if you do, then $lhour, $lminute, $chour and $cminute are # undefined when their value is 00 # ----------------------------------------------------------------------------- lymdh=$(echo "$lyear*1000000+$lmonth*10000+$lday*100+$lhour \n quit" | bc) date -u '+%Y %m %d %H %M' | read cyear cmonth cday chour cminute cymdh=$(echo "$cyear*1000000+$cmonth*10000+$cday*100+$chour \n quit" | bc) hourwait=$(/nwprod/util/exec/nhour $cymdh $lymdh) # The following will remove leading zeroes from $hourwait unless $hourwait is # numerically equal to 0, in which case it will force it to equal the string # '0' rather than '00' # --------------------------------------------------------------------------- if [ $hourwait -eq 0 ] ; then typeset -Z1 hourwait else typeset -LZ hourwait fi # The following line modified on later machines so that the shell does not # interpret the values for $cminute and $lminute as octal when there are # leading zeros # (Note: Cannot use "typeset -LZ2 cminute lminute" with original line below # for reasons noted in previous comment, reason 2) # ------------------------------------------------------------------------- ### minwait=$(($cminute-$lminute)) minwait=$(expr $cminute - $lminute) if [ $minwait -lt 0 ] ; then # Undo earlier typesets on $hourwait until you see what its new value is # ---------------------------------------------------------------------- typeset +LZ hourwait typeset +Z1 hourwait # The following line modified on later machines so that the shell does not # interpret the value for $hourwait as octal when there are leading zeros # (Note: Cannot use "typeset -LZ hourwait" with original line below because # $hourwait on the right hand side is then undefined when its value # is 0) # ------------------------------------------------------------------------- ### hourwait=$(($hourwait-1)) hourwait=$(expr $hourwait - 1) # The following will remove leading zeroes from $hourwait unless $hourwait is # numerically equal to 0, in which case it will force it to equal the string # '0' rather than '00' # --------------------------------------------------------------------------- if [ $hourwait -eq 0 ] ; then typeset -Z1 hourwait else typeset -LZ hourwait fi fi ncount=$(($ncount+1)) [ $RECPRO = RECEIVED ] && mcount=1 icount=0 for hourlimit in $HOURS2ALARM ; do icount=$(($icount+1)) if [ $icount -eq $ncount ] ; then break fi done if [ $hourwait -gt $hourlimit ] ; then if [ $RECPRO = PROCESSED ] ; then $DATA/postmsg "$jlogfile" "REMOTE FTP INGEST: No data from \ $fname processed in the last $hourwait hrs." echo "Processing has failed for $hourwait hours on $ftype $fname." echo "Processing has failed for $hourwait hours on $ftype $fname." echo "Processing has failed for $hourwait hours on $ftype $fname." elif [ $RECPRO = RECEIVED ] ; then $DATA/postmsg "$jlogfile" "REMOTE FTP INGEST: No data from \ $fname received in the last $hourwait hrs." echo "No new files received for $hourwait hours for $ftype $fname." echo "No new files received for $hourwait hours for $ftype $fname." echo "No new files received for $hourwait hours for $ftype $fname." fi if [ $SENDBB = 'YES' ]; then if [ $CRITICAL = 'YES' ] ; then echo 2 $hourwait > /com/logs/bb/ingest/`basename $fname` else if [ $fname = "/snow/6144asc/ims???????.asc" ] ; then echo 1 $hourwait > /com/logs/bb/ingest/ims???????96.asc else echo 1 $hourwait > /com/logs/bb/ingest/`basename $fname` fi fi fi else if [ $RECPRO = PROCESSED ] ; then $DATA/postmsg "$jlogfile" "REMOTE FTP INGEST: Data from $fname \ processed in the last $hourwait hrs." elif [ $RECPRO = RECEIVED ] ; then $DATA/postmsg "$jlogfile" "REMOTE FTP INGEST: Data from $fname \ received in the last $hourwait hrs." fi if [ $SENDBB = 'YES' ]; then if [ $fname = "/snow/6144asc/ims???????.asc" ] ; then echo 0 $hourwait > /com/logs/bb/ingest/ims???????96.asc else echo 0 $hourwait > /com/logs/bb/ingest/`basename $fname` fi fi fi echo "$filenames_save" | read firstword restofstring done done if [ $mcount -eq 0 ] ; then if [ $fname != imssnow.grb -a $fname != imssnow96.grb -a \ $fname != snowdepth.grb ] ; then oldest_date=`grep " AT " $1 | grep "$fname_grep3$blank" | head -n1 | \ awk -F" AT " '{print $2}' | cat` $DATA/postmsg "$jlogfile" "REMOTE INGEST: No data from $fname \ received since before ${oldest_date}." fi fi else exit 0 fi