#!/bin/ksh # Script to split out newly-arrived RFC stage3 QPE files (from the # /dcom/us007003/$day/wgrbbul/QPE.009.$rid # files (new arrives get appended to the QPE files) # # rid=105,150,152,..., 161,162 # # Split-out RFC stage3 files are placed in # $DATAST3/$yyyymmdd/$rid/$yyyymmddhh.${x}h [x=1 or 6] # # There is a directory, $ST3/$yyyymmdd/QPEcount # where we keep updated counts of the QPE.009.$rid files for each $rid. # These files are 'nfiles.$rid.$day', and they keep track of how many new # QPE files have been added to QPE.009.$rid since the last time we checked. # # This job is run once an hour. A log is kept for each day in the format of # (between the '---' lines:) # ---------------------------- # Apr 27 10:07 150.2002042700.1h # Apr 27 10:07 152.2002042700.1h # Apr 27 10:07 152.2002042701.1h # Apr 27 14:03 155.2002042701.1h # ---------------------------- # This log file is kept in # $ST4/$day0/getst3.$day0.log (i.e. the day - UTC - when this job is run) # # Check data for today and yesterday # # various 'time'/'date' (yyyymmddhh) and 'day' used in this script: # (all are in UTC): # pdyhh (yyyymmddhh) # pdyhh/day0 current time/re-run time (if retro) # timem24h/daym1 time0 - 24h # day daym1 or day0, used in "for day in $daym1 $day0" loop # dateg date stamp (reference time) of a grib record, # from wwgrib # daten/dayn ending date (dateg+'forecast length)') of the # aforementioned grib file set -x wgrib=$EXECutil/wgrib wwgrib=$USHnam_pcpn_anal/wwgrib.pl cd $DATAST3 sh $utilscript/setup.sh #Apr 29 18:49 #----:----|-- typeset -R12 timestamp # 'delt' is the accumulation period: typeset -R2 -Z delt timem24h=`$EXECutil/ndate -24 $pdyhh` day0=`echo $pdyhh | cut -c1-8` daym1=`echo $timem24h | cut -c1-8` msg="Stage4: getstage3, look for new ST3 in $day0 $daym1" postmsg "$jlogfile" "$msg" mkdir $day0 $daym1 # check today and yesterday's directory for new files. 'Today' is checked # first because the log file getst3.$day0.log will be kept in the current # day (i.e. 'today') so we want to make sure that the 'today' - i.e. # $ST4/$day0 is created first. for day in $day0 $daym1 do # First, fetch and unpack the tar file that contains the little # files (nfiles.$rid.$day) that each contains a number which tracks # how many grib records are in $DCOM/$day/wgrbbul/QPE.009.$rid, the last # time we checked (an hour ago): nfiletar=$COMIN/${RUN}.$day/nfiles.$day.tar if [ -e $nfiletar ]; then cp $nfiletar . tar xvf nfiles.$day.tar rm nfiles.$day.tar fi DATDIR=$DCOM/$day/wgrbbul # # For each QPE.009.### file, find out if new fields have been appended: # rid is the 3-digit RFC center ID (105, 152 etc.) # Get a list of RIDs from all the QPE.009 files for the day ($day): for file in `ls -1 $DATDIR/QPE.009.*` do rid=`echo $file | sed 's/\./ /g' | nawk '{print $3}'` if [ ! -d $DATAST3/$day/$rid ]; then mkdir -p $DATAST3/$day/$rid fi timestamp=`ls -l $file | nawk '{print $6,$7,$8}'` # how many fields in this file the last time we checked? if [ -e nfiles.$rid.$day ]; then num1=`cat nfiles.$rid.$day` else num1=0 fi # how many fields are there currently? num2=`$wgrib -s $file | wc -l` echo $num2 > nfiles.$rid.$day # if more fields have been added for this RFC center, split it out: let "i = $num1 + 1" while [ $i -le $num2 ]; do # split out one field: $wgrib -i -d $i $file -grib -o tmp.grb # Figure out what to name this file from the time markers in the grib # header: dateg=`$wwgrib tmp.grb | cut -c14-23` # 1 255 61 200204201200 18 24 4 #----:----|----:----|----:----|----:----|----:----|----:----| ib=`$wwgrib tmp.grb | cut -c28-29` ie=`$wwgrib tmp.grb | cut -c32-33` let "delt= $ie-$ib" daten=`/nwprod/util/exec/ndate $ie $dateg` dayn=`echo $daten | cut -c 1-8` if [ $dayn -ne $PDY ]; then if [ $dayn -ne $PDYm1 ]; then if [ $dayn -ne $PDYp1 ]; then echo " This is FORECAST FILE not ANALYSIS FILE! Skip to the next record." break fi fi fi onefile=ST3.$rid.$daten.${delt}h cp tmp.grb $onefile # Find out whether 'timestamp' shows today or yesterday: # e.g. # timestamp=Aug 22 10:19 # onefile=ST3.105.2002082201.01h # we want the entries to 'newst3s' to look like this: # Aug 22 10:19 105.2002082201.01h # so we use 'sed' to get rid of the string 'ST3'. echo $timestamp $onefile | sed 's/ST3.//' >> newst3s.$pdyhh let "i = $i + 1" done # proceed to process the next file in QPE.009.$rid done # proceed to the next QPE.009.$rid # save the little files that keep track of the latest count of records # in each QPE.009.$rid file for that day: tar cvf nfiles.$day.tar nfiles.*.$day cp nfiles.$day.tar $COMOUT/${RUN}.$day/ done # proceed to the next day # if any hour/6-hourly period has new Stage III data, then go back # to the /com directory to get the other Stage III files (tar'd up # for each hour) in preparation for the mosaicking. Re-make the # hour/6-hour's Stage III tar file (adding the new Stage IIIs to # the ones we get from earlier runs) for this hour/6-hour period # and put it back to $COMOUT. # 'todo4.2002080818' looks like this: # 2002080718.06h # 2002080800.06h # 2002080806.06h # 2002080812.01h # 2002080812.06h # 2002080813.01h # 2002080814.01h # 2002080815.01h # 2002080816.01h if [ -e newst3s.$pdyhh ]; then nawk '{print substr($4,5)}' newst3s.$pdyhh | sort -du > todo4.$pdyhh cp newst3s.$pdyhh $COMOUT/${RUN}.${PDY} cp todo4.$pdyhh $DATAST4 cp todo4.$pdyhh $COMOUT/${RUN}.${PDY} cd $DATAST4 for st3item in `cat todo4.$pdyhh` do dayitem=`echo $st3item | cut -c 1-8` if [ -e $COMIN/${RUN}.${dayitem}/ST3.$st3item.tar ]; then cp $COMIN/${RUN}.${dayitem}/ST3.$st3item.tar . tar xvf ST3.$st3item.tar rm -f ST3.$st3item.tar fi cp $DATAST3/ST3.*.$st3item . tar cvf ST3.$st3item.tar ST3.*.$st3item cp ST3.$st3item.tar $COMOUT/${RUN}.${dayitem} rm ST3.$st3item.tar done fi ##################################################################### # GOOD RUN set +x echo "**************JOB NAM_PCPN_GETSTAGE3 COMPLETED NORMALLY ON THE IBM SP" echo "**************JOB NAM_PCPN_GETSTAGE3 COMPLETED NORMALLY ON THE IBM SP" echo "**************JOB NAM_PCPN_GETSTAGE3 COMPLETED NORMALLY ON THE IBM SP" set -x ##################################################################### ############## END OF SCRIPT #######################