##################################################################### echo "-------------------------------------------------------------" echo "exprep_post.sh.sms - Runs various post-analysis processing " echo " steps on the PREPBUFR files (removes or " echo " masks restricted data) " echo " - GDAS only: Identify TimeTwin duplicate " echo " upper-air (RAOB, PIBAL, DROP) wind " echo " report parts " echo " - 18Z GDAS only: Reformat GDAS received, " echo " selected, and assimilated data counts " echo " (both satellite and non-satellite) for " echo " all four cycles for today and save the " echo " result in the monthly archive directory " echo " (run monthly summary on the second day " echo " of the next month and post to web) " echo " - GDAS only: Generate a table of GDAS " echo " received, selected, and assimilated " echo " satellite data counts " echo "-------------------------------------------------------------" echo "History: Jul 16 2003 - Original script - D. Keyser " echo " OCT 15 2011 - RAP replaces RUC (Keyser) " ##################################################################### # NOTE: NET is gfs for the gdas1 RUN (as for the gfs RUN) # ------------------------------------------------------- set -aux export XLFRTEOPTS="unit_vars=yes" # Allow overriding default names. # Make sure we are in the $DATA directory cd $DATA msg="Post-analysis processing of PREPBUFR file has begun on `hostname`" $DATA/postmsg "$jlogfile" "$msg" cat break > $pgmout tmhr=`echo $tmmark|cut -c3-4` cdate10=`/nwprod/util/exec/ndate -$tmhr $PDY$cyc` cycp=`echo $cdate10|cut -c9-10` net=$NET [[ $RUN = rap_p ]] && net=rap_p [[ $RUN = rap_e ]] && net=rap_e [[ $RUN = gdas1 ]] && net=gdas net_uc=$(echo $net | tr [a-z] [A-Z]) tmmark_uc=$(echo $tmmark | tr [a-z] [A-Z]) msg="$net_uc ANALYSIS TIME IS $PDY$cyc" $DATA/postmsg "$jlogfile" "$msg" set +x echo echo "CENTER TIME FOR PREPBUFR PROCESSING FOR $tmmark_uc $net_uc IS $cdate10" echo set -x # The next line can be removed when all J*PREP*POST job scripts are updated # to export PROCESS_REMOREST into this script # ------------------------------------------------------------------------- PROCESS_REMOREST=${PROCESS_REMOREST:-YES} if [ "$PROCESS_REMOREST" = 'YES' ]; then msg="REMOVE OR MASK RESTRICTED DATA FROM $tmmark_uc $net_uc PREPBUFR files \ CENTERED ON $cdate10" $DATA/postmsg "$jlogfile" "$msg" set +x echo echo "$msg" echo set -x dot_tmmark=".$tmmark" [ $net = gdas -o $net = gfs -o $net = cdas ] && dot_tmmark="" ######################################################## # Remove or mask restricted data from PREPUFR files ######################################################## cat <<\EOFparm > bufr_remorest.prepbufr.parm ========================================================================= Cards for PREPBUFR Version of BUFR_REMOREST -- Version 12 September 2008 &SWITCHES MSG_RESTR = 'AIRCAR ', ! These are the Table A Entries for 'MSONET ', ! BUFR messages for which ALL reports ' ', ! are RESTRICTED and will be REMOVED. ' ', ! (up to 20) MSG_MIXED = 'ADPSFC ', ! These are the Table A Entries for 'AIRCFT ', ! BUFR messages which contain a MIXTURE ' ', ! of restricted and unrestricted ' ', ! reports (based on mnemonic "RSRD"). All ' ', ! restricted reports will be REMOVED. ' ', ! (up to 20) MSG_MASKA = 'SFCSHP ', ! These are the Table A Entries for ' ', ! BUFR messages for which ALL reports ' ', ! are RESTRICTED if their dump report type is ' ', ! one of up to 10 possible listed in switch ' ', ! IMASK_T29 below (each line in IMASK_T29 applies ' ', ! to the Table A entry in the same line number ' ', ! here). Restricted reports will not be removed, ' ', ! but their report ids will be unilaterally ' ', ! changed to "MASKSTID" ' ', ! (up to 20) ' ' IMASK_T29 = 522,523,8*99999, ! Dump report types restricted in MSG_MASKA(1) 10*99999, ! Dump report types restricted in MSG_MASKA(2) 10*99999 ! etc., {up to 20 for MSG_MASKA(20)} / Note 1: A particular Table A entry should NEVER appear in more than one of MSG_RESTR, MSG_MIXED or MSG_MASKA. Note 2: Any Table A entry not in either MSG_RESTR, MSG_MIXED or MSG_MASKA is assumed to be a Table A entry for BUFR messages for which ALL reports are UNRESTRICTED (these messages are copied intact, no reports are unpacked). Note 3: Always fill in the arrays MSG_RESTR, MSG_MIXED and MSG_MASKA beginning with word 1. If there are less than 20 words filled in an array, either set the extra words to " " (8 blank characters) or do not specify them here (they default to " "). Note 4: In array IMASK_T29, a value of "99999" means not applicable whereas a value of "000" means reports in all dump report types in the corresponding Table A entry in MSG_MASKA should be restricted (masked) {in this case IMASK_T29(1,x) would be set to 000 and IMASK_T29(2:10,x) would be set to 99999 for all reports in Table A entry MSG_MASKA(x) since they would all be ignored - this is the default for all Table A entries MSG_MASKA(1:20) if this is not set (i.e., for data dump files)} ========================================================================= EOFparm REMX=${REMX:-$EXECbufr/bufr_remorest} REMC=${REMC:-bufr_remorest.prepbufr.parm} if [ -f $COMIN/$RUN.$cycle.prepbufr${dot_tmmark} ]; then if [ ! -f $COMOUT/$RUN.$cycle.prepbufr${dot_tmmark}.nr ]; then cp $COMIN/$RUN.$cycle.prepbufr${dot_tmmark} \ $RUN.$cycle.prepbufr${dot_tmmark} $ushscript/bufr_remorest.sh $RUN.$cycle.prepbufr${dot_tmmark} errsc=$? [ "$errsc" -ne '0' ] && exit $errsc cp $RUN.$cycle.prepbufr${dot_tmmark} \ $COMOUT/$RUN.$cycle.prepbufr${dot_tmmark}.nr chmod 664 $COMOUT/$RUN.$cycle.prepbufr${dot_tmmark}.nr msg="$RUN.$cycle.prepbufr${dot_tmmark}.nr successfully created" $DATA/postmsg "$jlogfile" "$msg" if test "$SENDDBN" = "YES"; then if test "$net" = "gdas"; then $DBNROOT/bin/dbn_alert MODEL GDAS1_BUFR_PREPda_nr $job \ $COMOUT/$RUN.$cycle.prepbufr.nr elif test "$net" = "nam"; then $DBNROOT/bin/dbn_alert MODEL NAM_BUFR_PREPda_nr $job \ $COMOUT/$RUN.$cycle.prepbufr${dot_tmmark}.nr elif test "$net" = "gfs"; then $DBNROOT/bin/dbn_alert MODEL GFS_BUFR_PREPda_nr $job \ $COMOUT/$RUN.$cycle.prepbufr.nr fi fi else msg="$RUN.$cycle.prepbufr${dot_tmmark}.nr NOT created because it \ already exists" $DATA/postmsg "$jlogfile" "$msg" fi if [ -f $COMIN/$RUN.$cycle.prepbufr${dot_tmmark}.unblok -a \ ! -f $COMOUT/$RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr ]; then /nwprod/ush/cwordsh unblk $RUN.$cycle.prepbufr${dot_tmmark} \ $RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr err_cwordsh=$? if [ $err_cwordsh -eq 0 ]; then cp $RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr \ $COMOUT/$RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr chmod 664 $COMOUT/$RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr msg="$RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr successfully created" $DATA/postmsg "$jlogfile" "$msg" if test "$SENDDBN" = "YES"; then if test "$net" = "gdas"; then $DBNROOT/bin/dbn_alert MODEL GDAS1_BUFR_PREPda_unblok_nr $job \ $COMOUT/$RUN.$cycle.prepbufr.unblok.nr elif test "$net" = "gfs"; then $DBNROOT/bin/dbn_alert MODEL GFS_BUFR_PREPda_unblok_nr $job \ $COMOUT/$RUN.$cycle.prepbufr.unblok.nr fi fi else msg="$RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr NOT created \ because cwordsh had return code $err_cwordsh" $DATA/postmsg "$jlogfile" "$msg" fi else if [ ! -f $COMIN/$RUN.$cycle.prepbufr${dot_tmmark}.unblok ]; then msg="$RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr NOT created \ because unblocked prepbufr file does not exist" $DATA/postmsg "$jlogfile" "$msg" else msg="$RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr NOT created \ because it already exists" $DATA/postmsg "$jlogfile" "$msg" fi fi else msg="$RUN.$cycle.prepbufr${dot_tmmark}.nr NOT created because \ prepbufr file does not exist" $DATA/postmsg "$jlogfile" "$msg" if [ -f $COMIN/$RUN.$cycle.prepbufr${dot_tmmark}.unblok -a \ ! -f $COMOUT/$RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr ]; then msg="$RUN.$cycle.prepbufr${dot_tmmark}.unblok.nr NOT created \ because prepbufr file does not exist" fi fi if [ -f $COMIN/$RUN.$cycle.prepbufr_pre-qc${dot_tmmark} -a \ ! -f $COMOUT/$RUN.$cycle.prepbufr_pre-qc${dot_tmmark}.nr ]; then cp $COMIN/$RUN.$cycle.prepbufr_pre-qc${dot_tmmark} \ $RUN.$cycle.prepbufr_pre-qc${dot_tmmark} $ushscript/bufr_remorest.sh $RUN.$cycle.prepbufr_pre-qc${dot_tmmark} errsc=$? [ "$errsc" -ne '0' ] && exit $errsc cp $RUN.$cycle.prepbufr_pre-qc${dot_tmmark} \ $COMOUT/$RUN.$cycle.prepbufr_pre-qc${dot_tmmark}.nr chmod 664 $COMOUT/$RUN.$cycle.prepbufr_pre-qc${dot_tmmark}.nr msg="$RUN.$cycle.prepbufr_pre-qc${dot_tmmark}.nr successfully created" $DATA/postmsg "$jlogfile" "$msg" else if [ ! -f $COMIN/$RUN.$cycle.prepbufr_pre-qc${dot_tmmark} ]; then msg="$RUN.$cycle.prepbufr_pre-qc${dot_tmmark}.nr NOT created \ because prepbufr_pre-qc file does not exist" $DATA/postmsg "$jlogfile" "$msg" else msg="$RUN.$cycle.prepbufr_pre-qc${dot_tmmark}.nr NOT created because \ it already exists" $DATA/postmsg "$jlogfile" "$msg" fi fi fi # test for PROCESS_REMOREST=YES # The next 2 lines can be removed when all J*PREP*POST job scripts are updated # to export PROCESS_TIMETWINS (based on run) into this script # ---------------------------------------------------------------------------- PROCESS_TIMETWINS=${PROCESS_TIMETWINS:-YES} [ $net != gdas ] && PROCESS_TIMETWINS=NO if [ "$PROCESS_TIMETWINS" = 'YES' ]; then msg="FIND UPPER-AIR TIME-TWINS IN $tmmark_uc $net_uc PREPBUFR files \ CENTERED ON $cdate10" $DATA/postmsg "$jlogfile" "$msg" set +x echo echo "$msg" set -x ########################################################################### # Find upper-air TimeTwins (duplications in current RAOB, PIBAL, or DROP # wind report "parts" vs. those over the past 35 days) (executed only # for the GDAS run, for every cycle) ########################################################################## set +x echo "Upper-air TimeTwins are current reports that are duplicates of \ past reports" echo echo "Calculate date 35 days ago to limit old data accumulation." echo set -x F35DAYL=`/nwprod/util/exec/ndate -840 $PDY$cyc` cat << EOH > cardec ${F35DAYL} ${PDY} EOH if [ -f ${COMIN}/gdas1.t$cyc"z".prepbufr ]; then BUFFILE=${COMIN}/gdas1.t$cyc"z".prepbufr errbufr=0 elif [ -f ${COMINGFS}/gfs.t$cyc"z".prepbufr ]; then BUFFILE=${COMINGFS}/gfs.t$cyc"z".prepbufr errbufr=0 else errbufr=1 fi if [ $errbufr -eq 0 ]; then set +x echo echo "Wind reports taken from: " echo $BUFFILE echo set -x else set +x echo echo "Could not find the current" $cyc"z" "GFS or GDAS prepbufr file" echo set -x $DATA/err_chk fi pgm="timetwin" export pgm;. $DATA/prep_step THR=6 ICNT=1 error=FALSE while [ $error = FALSE ]; do prevgoodcyc=`/nwprod/util/exec/ndate -$THR $PDY$cyc` DATE_HIS=`echo $prevgoodcyc | cut -c1-8` CYC_HIS=`echo $prevgoodcyc | cut -c9-10` export XLFUNIT_10="$BUFFILE" export XLFUNIT_24="$COMIN1$DATE_HIS/upa_his_twinwind.ttaa.t$CYC_HIS"z"" export XLFUNIT_25="$COMIN1$DATE_HIS/upa_his_twinwind.ttbb.t$CYC_HIS"z"" export XLFUNIT_26="$COMIN1$DATE_HIS/upa_his_twinwind.ttcc.t$CYC_HIS"z"" export XLFUNIT_27="$COMIN1$DATE_HIS/upa_his_twinwind.ttdd.t$CYC_HIS"z"" export XLFUNIT_28="$COMIN1$DATE_HIS/upa_his_twinwind.ppbb.t$CYC_HIS"z"" export XLFUNIT_29="$COMIN1$DATE_HIS/upa_his_twinwind.ppdd.t$CYC_HIS"z"" export XLFUNIT_61="upa_cur_twinwind.ttaa.t$cyc"z"" export XLFUNIT_62="upa_cur_twinwind.ttbb.t$cyc"z"" export XLFUNIT_63="upa_cur_twinwind.ttcc.t$cyc"z"" export XLFUNIT_64="upa_cur_twinwind.ttdd.t$cyc"z"" export XLFUNIT_65="upa_cur_twinwind.ppbb.t$cyc"z"" export XLFUNIT_66="upa_cur_twinwind.ppdd.t$cyc"z"" export XLFUNIT_54="new.ttaa.t$cyc"z"" export XLFUNIT_55="new.ttbb.t$cyc"z"" export XLFUNIT_56="new.ttcc.t$cyc"z"" export XLFUNIT_57="new.ttdd.t$cyc"z"" export XLFUNIT_58="new.ppbb.t$cyc"z"" export XLFUNIT_59="new.ppdd.t$cyc"z"" export XLFUNIT_74="timetwin.diags.t$cyc"z"" export XLFUNIT_75="twin" #### export XLFUNIT_76="timetwin.debug1.diags.t$cyc"z"" #### export XLFUNIT_77="timetwin.debug2.diags.t$cyc"z"" startmsg $EXECutil/timetwin < cardec >>$pgmout 2> errfile export err=$? #### $DATA/err_chk if [ $err -eq 0 ]; then error=TRUE else let ICNT=$ICNT+1 if [ $ICNT -eq 28 ]; then set +x echo echo " Upperair wind reports are beyond 7 days...exit" echo set -x $DATA/err_chk fi let THR=$ICNT*6 echo "THR" $THR $ICNT fi done if [ $SENDCOM = "YES" ]; then cp new.ttaa.t$cyc"z" $COMOUT1/upa_his_twinwind.ttaa.t$cyc"z" cp new.ttbb.t$cyc"z" $COMOUT1/upa_his_twinwind.ttbb.t$cyc"z" cp new.ttcc.t$cyc"z" $COMOUT1/upa_his_twinwind.ttcc.t$cyc"z" cp new.ttdd.t$cyc"z" $COMOUT1/upa_his_twinwind.ttdd.t$cyc"z" cp new.ppbb.t$cyc"z" $COMOUT1/upa_his_twinwind.ppbb.t$cyc"z" cp new.ppdd.t$cyc"z" $COMOUT1/upa_his_twinwind.ppdd.t$cyc"z" cp upa_cur_twinwind.ttaa.t$cyc"z" $COMOUT1/upa_cur_twinwind.ttaa.t$cyc"z" cp upa_cur_twinwind.ttbb.t$cyc"z" $COMOUT1/upa_cur_twinwind.ttbb.t$cyc"z" cp upa_cur_twinwind.ttcc.t$cyc"z" $COMOUT1/upa_cur_twinwind.ttcc.t$cyc"z" cp upa_cur_twinwind.ttdd.t$cyc"z" $COMOUT1/upa_cur_twinwind.ttdd.t$cyc"z" cp upa_cur_twinwind.ppbb.t$cyc"z" $COMOUT1/upa_cur_twinwind.ppbb.t$cyc"z" cp upa_cur_twinwind.ppdd.t$cyc"z" $COMOUT1/upa_cur_twinwind.ppdd.t$cyc"z" cp timetwin.diags.t$cyc"z" $COMOUT1/timetwin.diags.t$cyc"z" if [ -f twin ]; then cp twin $COMOUT1/upa_time_twin.t$cyc"z" else echo "NO TWIN REPORTS" | cat >> upa_time_twin.t$cyc"z" cp upa_time_twin.t$cyc"z" $COMOUT1/upa_time_twin.t$cyc"z" fi fi fi # test for PROCESS_TIMETWINS=YES # The next 2 lines can be removed when all J*PREP*POST job scripts are updated # to export PROCESS_ALL_REPORT_COUNTS (based on run & cycle) into this script # ---------------------------------------------------------------------------- PROCESS_ALL_REPORT_COUNTS=${PROCESS_ALL_REPORT_COUNTS:-YES} [ $net != gdas -o $cyc != 18 ] && PROCESS_ALL_REPORT_COUNTS=NO if [ "$PROCESS_ALL_REPORT_COUNTS" = 'YES' ]; then msg="ARCHIVE ALL OBS COUNTS (SAT AND NON-SAT) FOR ALL 4 $net_uc CYCLES \ FOR $PDY" $DATA/postmsg "$jlogfile" "$msg" set +x echo echo "$msg" echo set -x ############################################################################# # Reformat GDAS received, selected, and assimilated data counts (both # satellite and non-satellite) for all four cycles for today and save the # result in the monthly archive directory (run monthly summary on the # second day of the next month and post to web) (executed only for the 18z # GDAS run) ############################################################################# $utilscript/gdas_counts.sh # Run monthly summary only on the second day of the month, post to web # -------------------------------------------------------------------- dat=`date +"%Y%m"02` if [ $dat -eq $PDY ]; then feb2_def=0202 feb2_cal=`echo $PDY | cut -c5-8` if [ $feb2_cal -eq $feb2_def ]; then msg="....it's Feb 2 - archive last year's monthly obs count summaries" $DATA/postmsg "$jlogfile" "$msg" set +x echo echo "$msg" echo set -x sh $utilscript/gdascounts_archive_pyear.sh fi msg="....it's day 2 of month - post last month's obs count summary to web" $DATA/postmsg "$jlogfile" "$msg" set +x echo echo "$msg" echo set -x $utilscript/gdas_summary.sh fi fi # test for PROCESS_ALL_REPORT_COUNTS=YES # The next 2 lines can be removed when all J*PREP*POST job scripts are updated # to export PROCESS_SATELLITE_COUNTS (based on run) into this script # ---------------------------------------------------------------------------- PROCESS_SATELLITE_COUNTS=${PROCESS_SATELLITE_COUNTS:-YES} [ $net != gdas ] && PROCESS_SATELLITE_COUNTS=NO if [ "$PROCESS_SATELLITE_COUNTS" = 'YES' ]; then msg="GENERATE A TABLE OF RECEIVED, SELECTED, & ASSIMILATED SAT OBS COUNTS \ FOR ${cyc}Z CYCLE ON $PDY FOR $net_uc" $DATA/postmsg "$jlogfile" "$msg" set +x echo echo "$msg" echo set -x ############################################################################## # Generate a table of GDAS received, selected, and assimilated satellite data # counts (executed only for the GDAS run, for every cycle) ############################################################################## export gesenvir=${envir} export JCAP=382 export JCAP_ges=382 export LEVS=64 export LEVS_ges=64 export SATCNT=$DATA/${RUN}.t${cyc}z.satcnt $utilscript/getges.sh -t satcnt -r ${JCAP_ges}${LEVS_ges} -v $PDY$cyc \ -e $gesenvir gsatcnt export GSATCNT=${GSATCNT:-$DATA/gsatcnt} export SATCNT=${SATCNT:-${COMIN}/${RUN}.t${cyc}z.satcnt} export SSISTAT=${SSISTAT:-${COMIN}/${RUN}.t${cyc}z.ssistat} export CDATE=${CDATE:-$PDY$cyc} if [[ -s $GSATCNT ]]; then ${NCP:-cp} $GSATCNT satcnt.ges fi # Extract satellite data usage counts # ----------------------------------- export ANLSTAT=$SSISTAT export DUMPNET=gdas export PREINP=${RUN}.${cycle}. export FILESTYLE='C' $ushscript/global_satcount.sh $CDATE $SATCNT $GSATCNT [ "$SENDCOM" = "YES" ] && cp $SATCNT $COMOUT/${RUN}.${cycle}.satcnt fi # on PROCESS_SATELLITE_COUNTS ####################### # GOOD RUN set +x echo " " echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" echo " " set -x # save standard output cat break $pgmout break > allout cat allout # rm allout sleep 10 msg='ENDED NORMALLY.' $DATA/postmsg "$jlogfile" "$msg" ################## END OF SCRIPT #######################