#!/bin/sh

set -x

############################################################################
echo "---------------------------------------------------------------------"
echo "cpc_format_metsyn.sh preprocessing"
echo "---------------------------------------------------------------------"
#
############################################################################
#
# Function:  Reformats Metar and Synoptic observations from the NCEP bufr
#            files for use in the Climate Prediction Center daily global 
#            precipitation and temperature summary file generation.
#
############################################################################
# author:    George G. Fulwood
#
# created:   10/28/2003
#
############################################################################
#
# Input Files: These files are generated by CPC/NCEP job JCPC_METSYN.sms.prod
#
#  cpc.t10ztot09z.metar.$PDY.bufr_d      - Metar obs from 10z yesterday
#                                          until 9z today 
#
#  cpc.t10ztot09z.metar.$PDYm1.bufr_d    - Metar obs from 10z 48 hours
#                                          ago until 9z yesterday 
#
#  cpc.t03ztot21z.synoptic.$PDYm2.bufr_d - 48 hour old 3z to 21z obs known 
#                                          in CPC as decode.dat
#  
#  cpc.t00ztot01z.synoptic.$PDYm1.bufr_d - Yesterday's 00z and 01z obs known
#                                          in CPC as dacode.dat 
#
#  cpc.t03ztot21z.synoptic.$PDYm1.bufr_d - Yesterday's 3z to 21z obs known in
#                                          CPC as dicode.dat 
#
#  cpc.t00ztot06z.synoptic.$PDY.bufr_d   - Today's 00z to 06z obs known in CPC
#                                          as docode.dat 
#
# Parm Files:
#
#  cpc.synoptic.hours.dat                - Synoptic processing times
#
#  cpc.regional.descriptor.table.dat     - Regional Descriptor File
#
#  cpc.temp.descriptor.table.dat         - Temperature Descriptor File
#
#  cpc.prcp.descriptor.table.dat         - Precipitation Descriptor File
#
##############################################################################
#
# Output Files:
#
#  cpc.t10z_$PDYm1.tot09z_$PDY.metar.hrlsr    - Formatted Metar obs from 10z
#                                               yesterday until 9z today known
#                                               by CPC as hrlsr.dat 
#
#  cpc.t10z_$PDYm2.tot09z_$PDYm1.metar.hslsr  - Formatted Metar obs from 10z
#                                               48 hours ago until 9z 
#                                               yesterday known by CPC as 
#                                               hslsr.dat 
#
#  cpc.t03z_$PDYm2.tot00z_$PDYm1.dzfile  - Formatted Synoptic obs from 3z
#                                          48 hours ago until 00z today,
#                                          known in CPC as dzfile.dat
#
#  cpc.t03z_$PDYm1.tot06z_$PDY.vzfile    - Formatted Synoptic obs from 3z
#                                          yesterday to 6z today, known in
#                                          CPC as vzfile.dat
#
#  cpc.t03z_$PDYm2.tot06z_$PDY.edfile    - 54 hours of formatted synoptic data
#                                          beginning 3z 48 hours ago and ending
#                                          6z today, known in CPC as the 
#                                          edfile.
#
###############################################################################
#
#  Copy CPC library and descriptor files to working directory
#
cp $CPCfix/cpc.synoptic.hours.dat               .
#
cp $CPCfix/cpc.regional.descriptor.table.dat    .
#
cp $CPCfix/cpc.temp.descriptor.table.dat        .
#
cp $CPCfix/cpc.prcp.descriptor.table.dat        .
#
cp $CPCfix/cpc.metar.library.dat                .
#
#  Initialize work files and copy required bufr file to working area
#
touch cpc.process.date_23z.dat  cpc.process.dates.card.dat
rm cpc*bufr_d  cpc*form_d
#
cp $COMIN/cpc.t10ztot09z.metar.$PDY.bufr_d  cpc.metar.bufr_d
touch cpc.hourly.dat
#
########################################################################
#
# Initiate prep step for using xlfunit
#
export pgm=format_metdat
  . prep_step 
#
#  CPC metar library and 24 hour old metar, bufr input file
#
export XLFUNIT_11="cpc.metar.library.dat" 
export XLFUNIT_12="cpc.metar.bufr_d"
#
#  24 Hour old formatted Metar output files
#
export XLFUNIT_51="cpc.hourly.dat"
export XLFUNIT_52="cpc.hlyprnt.dat"
#
########################################################################
#
#  Extract metar data from bufr file
#
startmsg
$CPCexec/format_metdat 
export err=$?;err_chk
#
#############################################################################
#
# sort file using field1 which is the station id.
#
sort -o sorted.cpc.hourly.dat -T $DATA -k1,1 cpc.hourly.dat

export pgm=dupelim_metar
  . prep_step 
#
#  Remove duplicates from sorted, formatted 24 hour old Metar input file
#
export XLFUNIT_11="sorted.cpc.hourly.dat"
#
#  Sorted, Formatted 24 hour old Metar Output File without duplicates
#
touch cpc.t10z_$PDYm1.tot09z_$PDY.metar.hrlsr   cpc.metar.hrlsr 
export XLFUNIT_51="cpc.metar.hrlsr"
#
startmsg
$CPCexec/dupelim_metar 
export err=$?;err_chk

cp cpc.metar.hrlsr  cpc.t10z_$PDYm1.tot09z_$PDY.metar.hrlsr 
#
date
#
############################################################################
#
# Initialize work files for 48 hour old metar processing
#
touch cpc.hourly.dat   cpc.hlyprnt.dat    sorted.cpc.hourly.dat
rm    cpc.hourly.dat   cpc.hlyprnt.dat    sorted.cpc.hourly.dat
touch cpc.hourly.dat   cpc.hlyprnt.dat    sorted.cpc.hourly.dat
#
############################################################################
#
#  CPC metar library and 48 hour old Metar bufr input file
# 
cp $COMIN/cpc.t10ztot09z.metar.$PDYm1.bufr_d      cpc.metar.bufr_d 
#
export pgm=format_metdat
  . prep_step 

export XLFUNIT_11="cpc.metar.library.dat"
export XLFUNIT_12="cpc.metar.bufr_d"
#
############################################################################
#
#  48 hour old Metar formatted output files
#
export XLFUNIT_51="cpc.hourly.dat"
export XLFUNIT_52="cpc.hlyprnt.dat"
#
#  Extract metar data from bufr file
#
startmsg
$CPCexec/format_metdat 
export err=$?;err_chk
#
#############################################################################
#
# sort file using field1 which is the station id.
#
sort -o sorted.cpc.hourly.dat -T $DATA -k1,1 cpc.hourly.dat
#
#############################################################################
#
#  Remove duplicate metar records from 48 hour old metar sorted, 
#  formatted input file
#
export pgm=dupelim_metar
  . prep_step

export XLFUNIT_11="sorted.cpc.hourly.dat"
#
##############################################################################
#
#  48 hour old metar,sorted, formatted output file without duplicates
#
touch cpc.t10z_$PDYm2.tot09z_$PDYm1.metar.hslsr      cpc.metar.hslsr 
export XLFUNIT_51="cpc.metar.hslsr"
#
startmsg
$CPCexec/dupelim_metar 
export err=$?;err_chk
cp cpc.metar.hslsr  cpc.t10z_$PDYm2.tot09z_$PDYm1.metar.hslsr 
rm cpc*bufr_d  cpc*form_d
#
##############################################################################
#
#   Copy formatted files to /com for use by CPC summarization code
#
cp cpc.t10z_$PDYm1.tot09z_$PDY.metar.hrlsr            $COMOUT/
cp cpc.t10z_$PDYm2.tot09z_$PDYm1.metar.hslsr          $COMOUT/
date
#
# Test filesize of last metar file to determine successful job completion 
#
if [ -s cpc.t10z_$PDYm2.tot09z_$PDYm1.metar.hslsr ]
   then
    msg='Metar formatting completed normally.'
   else
    msg='Job failure, check output files'
fi
echo $msg
postmsg "$jlogfile" "$msg"
# exit
#
echo 'format synoptic data'
#
cd $DATA
#
#   Initialize internal work variables
#
LABEL='CPCG03C'
BLANK=' '
#
########################################################################
#
#  Copy 48 hour old synoptic bufr file known to CPC as decode.dat, to 
#  workarea and initialize processing times.
#
cp $COMIN/cpc.t03ztot21z.synoptic.$PDYm2.bufr_d       cpc.synoptic.bufr_d
ls -l cpc.synoptic.bufr_d
#
LABEL='CPCG03C'
BLANK=' '
#
touch cpc.synoptic.dat
FD1=$BLANK$PDYm3
echo $FD1 $PDYm2 $LABEL >              cpc.process.dates.card.dat
#
FD2=$BLANK$PDYm2
echo $FD2 $PDYm1 $LABEL >              cpc.process.date_23z.dat
#
########################################################################
#
# Initiate prep step for using xlfunit
#
sh /nwprod/util/ush/setup.sh
export pgm=format_syndat
  . prep_step 
#
#
#  CPC process hours, bufr file, date file, and descriptor files 
#
export XLFUNIT_11="cpc.synoptic.hours.dat"
export XLFUNIT_12="cpc.synoptic.bufr_d"
export XLFUNIT_13="cpc.process.date_23z.dat"
export XLFUNIT_14="cpc.regional.descriptor.table.dat"
export XLFUNIT_15="cpc.temp.descriptor.table.dat"
export XLFUNIT_16="cpc.prcp.descriptor.table.dat"
export XLFUNIT_17="cpc.process.dates.card.dat"
#
########################################################################
#
#  48 Hour old formatted Synoptic output file.
#
export XLFUNIT_51="cpc.synoptic.dat"
#
########################################################################
#
echo 'formatting 48 hour old synoptic data '
#
#  Extract synoptic data from bufr file
#
startmsg
$CPCexec/format_syndat 
export err=$?;err_chk
#
echo '48 hour old synoptic format completed'
# exit
#
#############################################################################
#
# sort file using field4 which is the station id and field1 which is date.
#
echo 'sorting 48 hour old synoptic file'
#
sort -o cpc.sorted.synoptic.dat -T $DATA -k4,4 -k1,1 cpc.synoptic.dat
#rm cpc.synoptic.dat
#
echo '48 hour old synoptic sort completed'
#
# exit
#############################################################################
#
#   remove duplicate synoptic records
#
echo ' removing 48 hour old duplicate records'
#
#  Sorted, Formatted 48 hour old Synoptic input file
#
export pgm=dupelim_synoptic
  . prep_step
export XLFUNIT_11="cpc.sorted.synoptic.dat"
#
# Sorted, Formatted, duplicate-free 48 hour old synoptic file.
#
touch cpc.sorted.synoptic.p1.dat
export XLFUNIT_51="cpc.sorted.synoptic.p1.dat"
#
#############################################################################
#
startmsg
$CPCexec/dupelim_synoptic 
export err=$?;err_chk
#
# rm cpc.sorted.synoptic.dat
date
# exit
#
############################################################################
#
# Initialize work files for 24 hour old 00z synoptic processing
#
touch cpc.sorted.synoptic.p2.dat 
rm cpc.sorted.synoptic.p2.dat 
#
############################################################################
#
# 24 hour old 00z synoptic bufr file known to CPC as dacode.dat 
# 
rm cpc.synoptic.bufr_d
cp $COMIN/cpc.t00ztot01z.synoptic.$PDYm1.bufr_d        cpc.synoptic.bufr_d
ls -l cpc.synoptic.bufr_d
#
FD2=$BLANK$PDYm2
echo $FD2 $PDYm1 $LABEL >               cpc.process.dates.card.dat
#
########################################################################
#
#  CPC process hours, bufr file, date file, and descriptor files 
#

export pgm=format_syndat
  . prep_step

export XLFUNIT_11="cpc.synoptic.hours.dat"
export XLFUNIT_12="cpc.synoptic.bufr_d"
export XLFUNIT_13="cpc.process.date_23z.dat"
export XLFUNIT_14="cpc.regional.descriptor.table.dat"
export XLFUNIT_15="cpc.temp.descriptor.table.dat"
export XLFUNIT_16="cpc.prcp.descriptor.table.dat"
export XLFUNIT_17="cpc.process.dates.card.dat"
#
############################################################################
#
#  24 hour old 00z Synoptic formatted output files
#
touch cpc.synoptic.dat
export XLFUNIT_51="cpc.synoptic.dat"
#
#############################################################################
#
echo 'formatting 24 hour old 00z synoptic data '
#
#  Extract synoptic data from bufr file
#
startmsg
$CPCexec/format_syndat 
export err=$?;err_chk
#
echo '24 hour old 00z synoptic format completed'
#
#############################################################################
#
# sort file using field1 which is the station id.
#
echo 'sorting 24 hour old 00z synoptic file'
#
sort -o sorted.cpc.synoptic.dat -T $DATA -k4,4 -k1,1 cpc.synoptic.dat
# rm cpc.synoptic.dat
#
echo '24 hour old 00z synoptic sort completed'
# exit
#
#############################################################################
#
#   remove duplicate 24 hour old 00z synoptic records
#
echo ' removing 24 hour old 00z duplicate synoptic records'
#
##############################################################################
#
#  Setup 24 hour old synoptic I/O files
#

export pgm=dupelim_synoptic
  . prep_step

export XLFUNIT_11="sorted.cpc.synoptic.dat"
#
##############################################################################
#
#  24 hour old 00z synoptic,sorted, formatted output file without duplicates
#
touch  cpc.sorted.synoptic.p2.dat
export XLFUNIT_51="cpc.sorted.synoptic.p2.dat"
#
##############################################################################
#
startmsg
$CPCexec/dupelim_synoptic 
export err=$?;err_chk
#
##############################################################################
date
rm sorted.cpc.synoptic.dat
#
#  Create file beginning  3z 48 hours ago and ending 00z yesterday known
#  to CPC as dzfile.dat
#
cat cpc.sorted.synoptic.p1.dat cpc.sorted.synoptic.p2.dat > cpc.t03z_$PDYm2.tot00z_$PDYm1.dzfile
#
cp cpc.t03z_$PDYm2.tot00z_$PDYm1.dzfile       dzfile_ops.dat
mv cpc.sorted.synoptic.p1.dat                 dzfile_ops_p1.dat
mv cpc.sorted.synoptic.p2.dat                 dzfile_ops_p2.dat
# exit
#
###############################################################################
#
#  Copy 24 hour old synoptic 3z to 21z bufr file known to CPC as dicode.dat to
#  workarea and initalize processing times and date files.
#
rm cpc.synoptic.bufr_d
cp $COMIN/cpc.t03ztot21z.synoptic.$PDYm1.bufr_d     cpc.synoptic.bufr_d
#
touch cpc.synoptic.dat
rm cpc.synoptic.dat
touch cpc.synoptic.dat
#
FD2=$BLANK$PDYm2
FD3=$BLANK$PDYm1
#
echo $FD2 $PDYm1 $LABEL >             cpc.process.date.card.dat
echo $FD3 $PDY $LABEL   >             cpc.process.date_23z.dat
#
########################################################################
#
#  CPC process hours, bufr file, date file, and descriptor files 
#

export pgm=format_syndat
  . prep_step

export XLFUNIT_11="cpc.synoptic.hours.dat"
export XLFUNIT_12="cpc.synoptic.bufr_d"
export XLFUNIT_13="cpc.process.date_23z.dat"
export XLFUNIT_14="cpc.regional.descriptor.table.dat"
export XLFUNIT_15="cpc.temp.descriptor.table.dat"
export XLFUNIT_16="cpc.prcp.descriptor.table.dat"
export XLFUNIT_17="cpc.process.dates.card.dat"
#
#
########################################################################
#
#  24 Hour old formatted Synoptic output file.
#
export XLFUNIT_51="cpc.synoptic.dat"
#
########################################################################
#
echo 'formatting 24 hour old synoptic data '
#
#  Extract synoptic data from bufr file
#
startmsg
$CPCexec/format_syndat 
export err=$?;err_chk
#
echo '24 hour old synoptic format completed'
#
#############################################################################
#
# sort file using field4 which is the station id and field1 which is date.
#
echo 'sorting 24 hour old synoptic file'
#
sort -o cpc.sorted.synoptic.dat -T $DATA -k4,4 -k1,1 cpc.synoptic.dat
#
echo '24 hour old synoptic sort completed'
#
#############################################################################
#
#   remove duplicate synoptic records
#
echo ' removing 24 hour old duplicate records'
#
#  Sorted, Formatted 24 hour old Synoptic input file
#

export pgm=dupelim_synoptic
  . prep_step

export XLFUNIT_11="cpc.sorted.synoptic.dat"
#
# Sorted, Formatted, duplicate-free 24 hour old synoptic file.
#
touch cpc.sorted.synoptic.p1.dat
export XLFUNIT_51="cpc.sorted.synoptic.p1.dat"
#
#############################################################################
#
startmsg
$CPCexec/dupelim_synoptic 
export err=$?;err_chk
#
date
#
############################################################################
#
# Initialize work files for current 00z to 6z synoptic processing
#
touch cpc.sorted.synoptic.p2.dat 
rm cpc.sorted.synoptic.p2.dat 
touch cpc.sorted.synoptic.p2.dat 
#
############################################################################
#
# Current 00z to 06z synoptic bufr file known to CPC as docode.dat 
# Create date files using dates from yesterday($PDYm1) and today($PDY) 
#
rm cpc.synoptic.bufr_d
rm cpc.synoptic.dat
cp $COMIN/cpc.t00ztot06z.synoptic.$PDY.bufr_d      cpc.synoptic.bufr_d
#
FD3=$BLANK$PDYm1
echo $FD3 $PDY $LABEL >             cpc.process.dates.card.dat
echo $FD3 $PDY $LABEL >             cpc.process.date_23z.dat
#
########################################################################
#
#  CPC process hours, bufr file, date file, and descriptor files 
#

export pgm=format_syndat
  . prep_step

export XLFUNIT_11="cpc.synoptic.hours.dat"
export XLFUNIT_12="cpc.synoptic.bufr_d"
export XLFUNIT_13="cpc.process.date_23z.dat"
export XLFUNIT_14="cpc.regional.descriptor.table.dat"
export XLFUNIT_15="cpc.temp.descriptor.table.dat"
export XLFUNIT_16="cpc.prcp.descriptor.table.dat"
export XLFUNIT_17="cpc.process.dates.card.dat"
#
############################################################################
#
#  Current 00z to 6z Synoptic formatted output files
#
export XLFUNIT_51="cpc.synoptic.dat"
#
#############################################################################
#
echo 'formatting current 00z to 06z synoptic data '
#
#  Extract synoptic data from bufr file
#
startmsg
$CPCexec/format_syndat 
export err=$?;err_chk
#
echo 'Current 00z to 06z synoptic format completed'
#
#############################################################################
#
# sort file using field1 which is the station id.
#
echo 'sorting current 00z to 06z synoptic file'
#
sort -o sorted.cpc.synoptic.dat -T /ptmp/wx53gf -k4,4 -k1,1 cpc.synoptic.dat
#
echo 'Current 00z to 06z synoptic sort completed'
#
#############################################################################
#
#   remove duplicate current 00z to 06z synoptic records
#
echo ' removing current 00z to 06z duplicate synoptic records'
#
##############################################################################
#
#  24 hour old 00z synoptic sorted, formatted input file
#

export pgm=dupelim_synoptic
  . prep_step

export XLFUNIT_11="sorted.cpc.synoptic.dat"
#
##############################################################################
#
#  24 hour old 00z synoptic,sorted, formatted output file without duplicates
#
touch  cpc.sorted.synoptic.p2.dat
export XLFUNIT_51="cpc.sorted.synoptic.p2.dat"
#
##############################################################################
#
startmsg
$CPCexec/dupelim_synoptic 
export err=$?;err_chk
#
##############################################################################
date
#
#  Create file beginning  3z 24 hours ago and ending 06z today known to CPC as
#  vzfile.dat
#
cat cpc.sorted.synoptic.p1.dat cpc.sorted.synoptic.p2.dat > vzfile_ops.dat
#
#  Sort the vzfile
#
sort -o sorted.vzfile.dat -T $DATA -k4,4 -k1,1 vzfile_ops.dat
#
#
cp vzfile_ops.dat     cpc.t03z_$PDYm1.tot06z_$PDY.vzfile       
#
#  Create 54 hours of synoptic data
#
cat dzfile_ops.dat vzfile_ops.dat         > edfile_ops.dat
#
#############################################################################
#
# sort file using field4 which is the station id and field1 which is date.
#
echo 'sorting synoptic file with all available obs'
#
sort -o cpc.sorted.all.synoptic.dat -T $DATA -k4,4 -k1,1 edfile_ops.dat
#
echo 'All available synoptic obs sort completed'
#
#############################################################################
#
mv cpc.sorted.all.synoptic.dat                       edfile_ops.dat
cp edfile_ops.dat                cpc.t03z_$PDYm2.tot06z_$PDY.edfile
#
#  Remove work files
#
rm cpc*bufr_d cpc*form_d
#
#  Copy edited synoptic data files to /com for later CPC use
#
cp cpc.t03z_$PDYm2.tot00z_$PDYm1.dzfile  $COMOUT/
cp cpc.t03z_$PDYm1.tot06z_$PDY.vzfile    $COMOUT/       
cp cpc.t03z_$PDYm2.tot06z_$PDY.edfile    $COMOUT/
#
# Alert files to DBNet
#
if [ $SENDDBN = "YES" ]; then
   $DBNROOT/bin/dbn_alert MODEL CPCMETSYN  $job  $COMOUT/cpc.t10z_$PDYm1.tot09z_$PDY.metar.hrlsr
   $DBNROOT/bin/dbn_alert MODEL CPCMETSYN  $job  $COMOUT/cpc.t10z_$PDYm2.tot09z_$PDYm1.metar.hslsr
   $DBNROOT/bin/dbn_alert MODEL CPCMETSYN  $job  $COMOUT/cpc.synoptic.bufr_d
   $DBNROOT/bin/dbn_alert MODEL CPCMETSYN  $job  $COMOUT/cpc.t03z_$PDYm2.tot00z_$PDYm1.dzfile
   $DBNROOT/bin/dbn_alert MODEL CPCMETSYN  $job  $COMOUT/cpc.t03z_$PDYm1.tot06z_$PDY.vzfile
   $DBNROOT/bin/dbn_alert MODEL CPCMETSYN  $job  $COMOUT/cpc.t03z_$PDYm2.tot06z_$PDY.edfile
fi
#
# end of script
#
if [ -s cpc.t03z_$PDYm2.tot06z_$PDY.edfile ]
   then
    msg='Synoptic Data Formatting completed normally.'
   else
    msg='Abnormal termination, insufficient file size'
fi
echo $msg
postmsg "$jlogfile" "$msg"

############################### END OF SCRIPT #######################