##################################################################### echo "----------------------------------------------------------" echo "exdump_post.sh.sms - If requested: " echo " 1) Generates combined dump STATUS file " echo " 2) Prepares data counts for the SDM " echo " 3) Removes or masks restricted data from " echo " dump files " echo " 4) Converts dump files to unblocked format " echo " 5) Lists the contents of dump files " echo " 6) Updates dump data count average tables " echo "----------------------------------------------------------" echo "History: Mar 29 2000 - Original script (Keyser) " echo " Jul 16 2003 - Added step to remove or mask " echo " restricted data (Keyser) " echo " MAR 01 2006 - Moved step to do data counts here " echo " (was in dump job) (Keyser) " echo " OCT 15 2011 - RAP replaces RUC (Keyser) " ##################################################################### # NOTE: NET is changed to gdas in the parent Job script for the gdas1 RUN # (was gfs - NET remains gfs for gfs RUN) # ----------------------------------------------------------------------- set -aux # Make sure we are in the $DATA directory cd $DATA msg="HAS BEGUN on `hostname`" $DATA/postmsg "$jlogfile" "$msg" cat break > $pgmout CHGRP_RSTPROD=${CHGRP_RSTPROD:-YES} UPDATE_AVERAGE_FILE=${UPDATE_AVERAGE_FILE:-YES} tmhr=`echo $tmmark|cut -c3-4` export dumptime=`/nwprod/util/exec/ndate -$tmhr $PDY$cyc` net=$NET if [ $NET = rucs ]; then if [[ $model = *ruc* ]]; then net=$model RUN=$model fi fi [[ $RUN = rap_p ]] && net=$RUN [[ $RUN = rap_e ]] && net=$RUN net_uc=$(echo $net | tr [a-z] [A-Z]) tmmark_uc=$(echo $tmmark | tr [a-z] [A-Z]) if [ "$NET" = 'rucs' ]; then msg="$CUTOFF CUTOFF SURFACE RUC ANALYSIS TIME IS $PDY$cyc" else msg="$net_uc ANALYSIS TIME IS $PDY$cyc" fi $DATA/postmsg "$jlogfile" "$msg" [[ $RUN = ndas ]] && net=$RUN # this doesn't seem to be used anywhere!! set +x echo if [ "$NET" = 'rucs' ]; then echo "CENTER DATA DUMP DATE-TIME FOR $CUTOFF CUTOFF SURFACE RUC IS $dumptime" else echo "CENTER DATA DUMP DATE-TIME FOR $tmmark_uc $net_uc IS $dumptime" fi echo set -x retcode=0 ######################################################################## # Generate a "combined" dump STATUS file if one is not yet present # # use STATUS files from TWO (SPLIT) dump jobs if they are present # ######################################################################## if [ ! -s $COMIN/$RUN.$cycle.status.$tmmark.bufr_d ]; then file1=$COMIN/$RUN.$cycle.status1.$tmmark.bufr_d file2=$COMIN/$RUN.$cycle.status2.$tmmark.bufr_d if [ -s $file1 -a -s $file2 ]; then mcountp1=`grep -n "##################################################\ #############################" $file1 | cut -f1 -d:` head -n `expr $mcountp1 - 3` $file1 > status1 mcount=`grep -n "PARENT SCRIPT IS" status1 | cut -f1 -d:` head -n $mcount status1 > top_part mcount=`expr \`cat bottom_part grep "PARENT SCRIPT IS" $file2 > insert cat top_part insert bottom_part > status1 mcount=`grep -n "THIS STATUS FILE CREATED" status1 | cut -f1 -d:` total=`cat top_part tail -n `expr $total - $mcount` status1 > bottom_part echo " THIS STATUS FILE CREATED `date -u`" > insert cat top_part insert bottom_part > status1 mcount=`grep -n "GEOGRAPHICAL DOMAIN IS DATA GROUP DEPENDENT" $file2 \ | cut -f1 -d:` total2=`cat <$file2 | wc -l` tail -n `expr $total2 - $mcount - 1` $file2 > bottom_part mcount=`grep -n "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\ %%%%%%%%%%%%%%%%%%%%%%%%%%%" bottom_part | cut -f1 -d:` head -n `expr $mcount - 1` bottom_part > insert cat status1 insert > status total1=`cat <$file1 | wc -l` tail -n `expr $total1 - $mcountp1 - 2` $file1 > bottom_part cat status bottom_part > status1 mcount=`grep -n " ++++ SPECIFIC INFORMATION ON EACH DATA GROUP DUMP \ ++++" $file2 | cut -f1 -d:` tail -n `expr $total2 - $mcount - 1` $file2 > bottom_part cat status1 bottom_part > $COMIN/$RUN.$cycle.status.$tmmark.bufr_d rm $file1 $file2 bottom_part insert status status1 top_part elif [ -s $file1 ]; then cp $file1 $COMIN/$RUN.$cycle.status.$tmmark.bufr_d rm $file1 msg="***WARNING: DUMP status file successfully generated but only from \ dump status1 file - dump status2 file not present" $DATA/postmsg "$jlogfile" "$msg" elif [ -s $file2 ]; then cp $file2 $COMIN/$RUN.$cycle.status.$tmmark.bufr_d rm $file2 msg="***WARNING: DUMP status file successfully generated but only from \ dump status2 file - dump status1 file not present" $DATA/postmsg "$jlogfile" "$msg" fi msg="DUMP status file successfully generated" $DATA/postmsg "$jlogfile" "$msg" fi ######################################################################## # Prepare Data Counts for the SDM # ######################################################################## [ "$PROCESS_DATACOUNTS" = 'YES' ] && $ushscript_datacount/bufr_datacount.sh if [ "$PROCESS_REMOREST" = 'YES' ]; then if [ "$NET" = 'rucs' ]; then msg="REMOVE OR MASK RESTRICTED DATA FROM $CUTOFF CUTOFF SURFACE RUC \ DATA DUMPS CENTERED ON $dumptime" else msg="REMOVE OR MASK RESTRICTED DATA FROM $tmmark_uc $net_uc DATA DUMPS \ CENTERED ON $dumptime" fi $DATA/postmsg "$jlogfile" "$msg" ######################################################################## # Remove or Mask Restricted Data from BUFR Data Dump Files # ######################################################################## cat <<\EOFparm > bufr_remorest.datadump.parm ========================================================================= Cards for DUMP Version of BUFR_REMOREST -- Version 18 November 2008 &SWITCHES MSG_RESTR = 'NC000000', ! These are the Table A Entries for 'NC004003', ! BUFR messages for which ALL reports 'NC004004', ! are RESTRICTED and will be REMOVED. 'NC004006', ! (up to 20) 'NC004007', 'NC004008', 'NC004009', 'NC004010', 'NC004012', 'NC004013', 'NC007001', 'NC007002', 'NC255xxx', MSG_MIXED = ' ', ! These are the Table A Entries for ' ', ! BUFR messages which contain a MIXTURE ' ', ! of restricted and unrestricted ' ', ! reports (based on mnemonic "RSRD"). All ' ', ! restricted reports will be REMOVED. ' ', ! (up to 20) ' ', MSG_MASKA = 'NC001001' ! These are the Table A Entries for ! BUFR messages for which ALL reports ! are RESTRICTED. They will not be removed, ! but all occurrences of their report ids will be ! unilaterally changed to either "MASKSTID" ! (where the id is stored by itself) or to all ! "X"'s where the number of "X"'s corresponds to ! the number of characters in the original report ! id (where the id is embedded in the raw report ! bulletin header string) ! (up to 20) / Note 1: A particular Table A entry should NEVER appear in more than one of MSG_RESTR, MSG_MIXED or MSG_MASKA. Note 2: Any Table A entry not in either MSG_RESTR, MSG_MIXED or MSG_MASKA is assumed to be a Table A entry for BUFR messages for which ALL reports are UNRESTRICTED (these messages are copied intact, no reports are unpacked). Note 3: Always fill in these arrays beginning with word 1. If there are less than 20 words filled in an array, either set the extra words to " " (8 blank characters) or do not specify them here (they default to " "). Note 4: For data dump Table A entries in the form "NCtttsss", where "ttt" is the BUFR message type and "sss" is the BUFR message subtype, if the last three characters (the subtype) is specified as 'xxx', then ALL BUFR messages of that type are either treated as having all restricted data all which is to be removed (if in MSG_RESTR), mixed data some of which is to be removed (if in MSG_MIXED) or all restricted data all of which is to have its report id masked (if in MSG_MASKA), regardless of the message subtype. (For example, if MSG_RESTR(1)='NC255xxx', then ALL mesonet BUFR messages are considered to have all restricted data and are all removed regardless of their subtype.) ========================================================================= EOFparm REMX=${REMX:-$EXECbufr/bufr_remorest} REMC=${REMC:-bufr_remorest.datadump.parm} for file in adpsfc aircar aircft msonet sfcshp lghtng do filestem=$RUN.$cycle.$file.$tmmark.bufr_d [ -f $COMIN/$filestem ] || continue cp $COMIN/$filestem $filestem $ushscript_remorest/bufr_remorest.sh $filestem rc=$? if [ $rc -gt 0 ] ; then [ $rc -gt $retcode ] && retcode=$rc msg="**WARNING: ERROR generating unrestricted $file BUFR file \ (rc = $rc)" $DATA/postmsg "$jlogfile" "$msg" else msg="Successful generation of unrestricted $file BUFR file" $DATA/postmsg "$jlogfile" "$msg" cp $filestem $COMOUT/$filestem.nr chmod 664 $COMOUT/$filestem.nr if [ $SENDDBN = "YES" ] ; then NETUP=`echo $RUN | tr {a-z} {A-Z}` $DBNROOT/bin/dbn_alert MODEL ${NETUP}_BUFR_${file}_nr $job \ $COMOUT/$filestem.nr fi fi done # endif loop $PROCESS_REMOREST fi if [ "$PROCESS_UNBLKBUFR" = 'YES' ]; then if [ "$NET" = 'rucs' ]; then msg="START THE UNBLOCKING OF $CUTOFF CUTOFF SURFACE RUC DATA DUMPS \ CENTERED ON $dumptime" else msg="START THE UNBLOCKING OF $tmmark_uc $net_uc DATA DUMPS CENTERED ON \ $dumptime" fi $DATA/postmsg "$jlogfile" "$msg" ######################################################################## # Unblock BUFR Data Dump Files # ######################################################################## for file in adpsfc adpupa aircar aircft satwnd sfcshp spssmi proflr \ vadwnd goesnd erscat sfcbog erswnd ssmip ssmipn ssmit \ atovs qkscat qkswnd trmm sptrmm geoimr 1bamua 1bamub \ 1bhrs3 1bmhs 1bhrs4 airs airswm amsre gpsipw msonet \ rassda nexrad gpsro airsev goesfv wndsat wdsatr osbuv8 \ ascatt ascatw mtiasi avcsam avcspm gome lghtng omi \ esamua esamub eshrs3 esmhs ssmisu sevcsr lgycld do for qual in "" ".nr" do qualdbn=$qual COMIN_here=$COMIN if [ "$qual" = .nr ]; then qualdbn=_nr COMIN_here=$COMOUT fi filestem=$RUN.$cycle.$file.$tmmark.bufr_d [ -f $COMIN_here/$filestem$qual ] || continue cp $COMIN_here/$filestem$qual $file.f77$qual $ushscript/cwordsh unblk $file.f77$qual $file.unblock$qual \ >> $pgmout 2>&1 rc=$? if [ $rc -gt 0 ] ; then [ $rc -gt $retcode ] && retcode=$rc msg="**WARNING: ERROR unblocking $file$qual BUFR file (rc = $rc)" $DATA/postmsg "$jlogfile" "$msg" else msg="$file$qual BUFR file SUCCESSFULLY unblocked" $DATA/postmsg "$jlogfile" "$msg" cp $file.unblock$qual $COMOUT/$filestem.unblok$qual chmod 664 $COMOUT/$filestem.unblok$qual if [ $SENDDBN = "YES" ] ; then NETUP=`echo $RUN | tr {a-z} {A-Z}` $DBNROOT/bin/dbn_alert MODEL \ ${NETUP}_BUFR_${file}_unblok${qualdbn} $job \ $COMOUT/$filestem.unblok$qual fi if [ "$CHGRP_RSTPROD" = 'YES' ]; then if [ "$qual" != .nr ]; then if [ -s $COMOUT/$filestem.nr ]; then chgrp rstprod $COMOUT/$filestem.unblok$qual errch=$? if [ $errch -eq 0 ]; then chmod 640 $COMOUT/$filestem.unblok$qual msg="NOTE: unblocked $file file contains RESTRICTED \ data, only users in rstprod group have read permission" $DATA/postmsg "$jlogfile" "$msg" echo " " echo "$msg" echo " " else cp /dev/null $COMOUT/$filestem.unblok$qual msg="**WARNING: unblocked $file file contains \ RESTRICTED data, since user $USER is not in rstprod group a null file is \ copied in its place" $DATA/postmsg "$jlogfile" "$msg" echo " " echo "$msg" echo " " fi fi fi fi fi done done # endif loop $PROCESS_UNBLKBUFR fi if [ "$PROCESS_LISTERS" = 'YES' ]; then if [ "$NET" = 'rucs' ]; then msg="START THE LISTING OF $CUTOFF CUTOFF SURFACE RUC DATA DUMPS \ CENTERED ON $dumptime" else msg="START THE LISTING OF $tmmark_uc $net_uc DATA DUMPS CENTERED ON \ $dumptime" fi $DATA/postmsg "$jlogfile" "$msg" retr=TRUE [ "$NET" = 'gfs' -o "$NET" = 'gdas' ] && retr=FALSE radn=TRUE [ "$NET" = 'rap' ] && radn=FALSE cat << EOFlistdumps > parms &PDATA IDAT10 = $dumptime, IWINDE = -3, IWINDL = +2, IPRINT = 0, SNDG = FALSE, RETR = $retr, RADN = $radn / EOFlistdumps cat <<\EOFthread > thread #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV # Make herefile "thread" which will run in background shells to list dumps #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV { echo set -x mkdir -p $DATA/thread${1} cd $DATA/thread${1} cp $COMIN/$filestem $file cat < parms1 &RDATA DSNAME_LC = "$file " / EOFc cat parms1 $DATA/parms > parms pgm=`basename $LSTX` export pgm . $DATA/prep_step echo " for $file BUFR file" >> $DATA/thread${1}/$pgmout export XLFUNIT_21=$file timex $LSTX < parms > $file.listing 2> errfile errlst=$? echo "$errlst" > $DATA/error${1} cp $file.listing $COMOUT/$filestem.listing chmod 664 $COMOUT/$filestem.listing cat errfile >> $DATA/thread${1}/$pgmout if [ "$errlst" -ne '0' ]; then set +x echo echo \ " ---------------------------------------------------------------------------" echo \ " ********** ERROR IN PROGRAM BUFR_LISTDUMPS for $file BUFR file **********" echo \ " ---------------------------------------------------------------------------" echo set -x msg="*WARNING: Program BUFR_LISTDUMPS DID NOT complete successfully \ for $file BUFR file (rc = $errlst)" $DATA/postmsg "$jlogfile" "$msg" else set +x echo echo \ " ---------------------------------------------------------------------------" echo \ " ********** COMPLETED PROGRAM BUFR_LISTDUMPS for $file BUFR file **********" echo \ " ---------------------------------------------------------------------------" echo set -x msg="Program BUFR_LISTDUMPS completed normally for $file BUFR file" $DATA/postmsg "$jlogfile" "$msg" if [ "$CHGRP_RSTPROD" = 'YES' ]; then if [ -s $COMOUT/$filestem.nr ]; then chgrp rstprod $COMOUT/$filestem.listing errch=$? if [ $errch -eq 0 ]; then chmod 640 $COMOUT/$filestem.listing msg="NOTE: $file listing file contains RESTRICTED data, \ only users in rstprod group have read permission" $DATA/postmsg "$jlogfile" "$msg" echo " " echo "$msg" echo " " else cp /dev/null $COMOUT/$filestem.listing msg="**WARNING: $file listing file contains RESTRICTED \ data, since user $USER is not in rstprod group a null file is copied in its \ place" $DATA/postmsg "$jlogfile" "$msg" echo " " echo "$msg" echo " " fi fi fi fi } > $DATA/thread${1}.out 2>&1 #AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA # End of herefile "thread" #AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA EOFthread chmod 775 thread index=0 ######################################################################## # List BUFR Data Dump Files # ######################################################################## LSTX=${LSTX:-$EXECbufr/bufr_listdumps} for file in adpsfc adpupa aircar aircft satwnd sfcshp spssmi proflr \ vadwnd goesnd goesfv erscat sfcbog qkswnd gpsipw msonet \ rassda wdsatr ascatw do filestem=$RUN.$cycle.$file.$tmmark.bufr_d [ -f $COMIN/$filestem ] || continue index=`expr $index + 1` export file filestem thread $index & done wait for n in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 do [ -d $DATA/thread${n} ] || break cat $DATA/thread${n}.out cat break $DATA/thread${n}/$pgmout break >> $pgmout err=`cat $DATA/error${n}` [ $err -gt $retcode ] && retcode=$err done # endif loop $PROCESS_LISTERS fi if [ "$PROCESS_AVGTABLES" = 'YES' ]; then ######################################################################## # If this is GDAS network (normally at 18Z only) ... # # Update Data Count Average Tables for gfs, gdas, nam Networks # # # # If this is RAP network (normally full cycle RAP at 23Z only) ... # # Update Data Count Average Tables for rap Network # # # # If this is RUCS network (normally LATE cutoff at 23Z only) ... # # Update Data Count Average Tables for rucs/early and late Networks # # # # If this is RTMA network (normally at 23Z only) ... # # Update Data Count Average Tables for rtma Network # ######################################################################## if [ "$NET" = 'gdas' ]; then networks="gfs gdas nam" elif [ "$NET" = 'rap' -a "$RUN" = 'rap' ]; then networks=rap elif [ "$NET" = 'rucs' ]; then networks="rucs1 rucs2" elif [ "$NET" = 'rtma' ]; then networks=rtma fi mkdir -p $AVGDarch_OUT for network in $networks do network_uc=$(echo $network | tr [a-z] [A-Z]) $ushscript_avgdata/bufr_avgdata.sh $network errsc=$? if [ $UPDATE_AVERAGE_FILE = YES ]; then if [ "$errsc" -eq '0' ]; then rm $AVGDarch_OUT/obcount_30davg.${network}.current cp obcount_30davg.${network}.current \ $AVGDarch_OUT/obcount_30davg.${network}.current chmod 775 $AVGDarch_OUT/obcount_30davg.${network}.current msg="Data count average table SUCCESSFULLY updated for $network_uc \ network" $DATA/postmsg "$jlogfile" "$msg" typeset -Z2 this_month last_month this_month=`echo $PDY | cut -c5-6` year=`echo $PDY | cut -c1-4` last_month=`expr $this_month - 1` if [ "$last_month" -eq '0' ]; then last_month=12 year=`expr $year - 1` fi if [ "$network" = 'nam' -o "$network" = 'gfs' -o "$network" = 'gdas' ] then if [ ! -s $AVGDarch_OUT/obcount_30davg.${network}.${year}${last_month} ] then # If no data count average table found for previous month, save this one rm $AVGDarch_OUT/obcount_30davg.${network}.*${last_month} cp obcount_30davg.${network}.current \ $AVGDarch_OUT/obcount_30davg.${network}.${year}${last_month} chmod 775 $AVGDarch_OUT/obcount_30davg.${network}.${year}${last_month} msg="DATA COUNT AVERAGE table for ${year}${last_month} saved \ for $network_uc network" $DATA/postmsg "$jlogfile" "$msg" fi fi else msg="Data count average table NOT updated for $network_uc network" $DATA/postmsg "$jlogfile" "$msg" fi else msg="Data count average table NOT updated for $network_uc network \ by choice" $DATA/postmsg "$jlogfile" "$msg" fi done # endif loop $PROCESS_AVGTABLES fi if [ "$NET" = 'rucs' ]; then if [ "$CUTOFF" = 'EARLY' ]; then rm ${COMOUT}/rucs1.${cycle}.early_post.flag echo "EARLY post job has run" > ${COMOUT}/rucs1.${cycle}.early_post.flag fi fi ####################### # GOOD RUN set +x echo " " echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" echo " " set -x # save standard output cat break $pgmout break > allout cat allout # rm allout if [ $retcode -eq 0 ] ; then msg="HAS COMPLETED NORMALLY. #################################" $DATA/postmsg "$jlogfile" "$msg" else msg="HAS COMPLETED WITH RETURN CODE $retcode. ################" $DATA/postmsg "$jlogfile" "$msg" fi ################## END OF SCRIPT #######################