echo
echo "------------------------------------------------"
echo "Datacount Averaging"             
echo "------------------------------------------------"
echo "History: AUG 2000 - First implementation of this new script."
echo "AUTHOR: Dennis Keyser "
echo "   2011-10-15 -- RAP replaces RUC (Keyser)"
echo

set -aux

# Imported positional parameters:
#   $1     - string indicating network ('nam', 'gfs', 'gdas', 'rap',
#            'rucs1', 'rucs2', 'rtma' or 'dump')
#           (Note: This is not defined the same as $NET in the j-job)
#
# Imported variables that must be passed in:
#   envir  - processing environment ('prod' or 'test')
#   DATA   - path to working directory
#   PDY    - today's YYYYMMDD
#   PDYm1  - yesterday's YYYYMMDD

# Imported variables that can be passed in:
#   COPY       - string indicating whether status,and possibly updated status,
#                file copying should be performed, along with cleaning of "old"
#                directories
#                (Default: YES)
#   UPDATE_AVERAGE_FILE
#              - string indicating whether the data count average file(s)
#                should be updated {one might one to set this to NO if only
#                interested in performing COPY (see above)}
#                (Default: YES)
#   pgmout     - string indicating path to for standard output file
#                (Default: none, if not exported into this scipt, ignored)
#   jlogfile   - string indicating path to job log file
#                (Default: none, if not exported into this scipt, ignored)
#   HOMEarch   - path to archive directory
#                (Default: /com/arch/${envir})
#   OBCNTarch  - path to archive (copy) of dump status (*.t*z.status*), and
#                possibly updated status (*.t*z.updated.status*), files over
#                many (30?) days (one directory back; actual directory is
#                hardwired to $OBCNTarch/${1}); status files will be copied to
#                $OBCNTarch/${1}/${1}.YYYYMMDD directory and updated status
#                files will be copied to
#                $OBCNTarch/${1}/${1}.YYYYMMDD/updated.status directory;
#                updated status files are copied only for nam, gfs or gdas
#                networks
#                (Default: $HOMEarch/obcount_30day);
#   comin      - path to today's dump status and updated status files
#                (Default: /com/${qual1}/$(envir}/${qual2}.${PDY})
#   comin_m1   - path to yesterday's dump status and updated status files
#                (Default: /com/${qual1}/$(envir}/${qual2}.${PDYm1})
#   FIXbufr    - path to bufr fixed field files
#                (Default: /nw${envir}/fix)
#   EXECbufr   - path to bufr executables
#                (Default: /nw${envir}/exec)
#   DUMPLIST   - path to the bufr_dumplist file
#                (Default: $FIXbufr/bufr_dumplist)
#   AVGX       - path to bufr_avgdata executable
#                (Default: $EXECbufr/bufr_avgdata)

err=0
net=$1
net_uc=$(echo $net | tr [a-z] [A-Z])
if [ $net = gdas ]; then
   qual1=gfs
   qual2=gdas
   qual3=gdas1
elif [ $net = rucs1 ]; then
   qual1=rucs
   qual2=rucs
   qual3=rucs1
elif [ $net = rucs2 ]; then
   qual1=rucs
   qual2=rucs
   qual3=rucs
else
   qual1=$net
   qual2=$net
   qual3=$net
fi


COPY=${COPY:-YES}
UPDATE_AVERAGE_FILE=${UPDATE_AVERAGE_FILE:-YES}

HOMEarch=${HOMEarch:-/com/arch/${envir}}
OBCNTarch=${OBCNTarch:-$HOMEarch/obcount_30day}
comin=${comin:-/com/${qual1}/${envir}/${qual2}.${PDY}}
comin_m1=${comin_m1:-/com/${qual1}/${envir}/${qual2}.${PDYm1}}
FIXbufr=${FIXbufr:-/nw${envir}/fix}
EXECbufr=${EXECbufr:-/nw${envir}/exec}
DUMPLIST=${DUMPLIST:-$FIXbufr/bufr_dumplist}
AVGX=${AVGX:-$EXECbufr/bufr_avgdata}

if [ $COPY = YES ]; then

#  Copy the status files, and possibly the updated status files, from today's
#  and yesterday's runs for this network (yesterday's is copied just in case
#  this didn't run yesterday) and add the EOF string to the end of each status
#  file (note: the updated status files are only copied if this is the nam, gfs
#  or gdas network - the EOF string is not added to the updated status files)
#  ----------------------------------------------------------------------------

mkdir -p $OBCNTarch/${net}/${net}.${PDY}
chmod 775 $OBCNTarch/${net}/${net}.${PDY}
mkdir -p $OBCNTarch/${net}/${net}.${PDYm1}
chmod 775 $OBCNTarch/${net}/${net}.${PDYm1}
[ $net = dump -o $net = rap -o $net = rucs1 -o $net = rucs2 -o \
 $net = rtma ] && set +x
cp -p ${comin}/${qual3}.*z.status*   $OBCNTarch/${net}/${net}.${PDY}
err_copy_status=$?
chmod 775 $OBCNTarch/${net}/${net}.${PDY}/*
if [ $err_copy_status -eq 0 ]; then
   msg="$net_uc status file(s) for today successfully copied to \
$OBCNTarch/${net}/${net}.${PDY}"
   set +x
   echo
   echo "$msg"
   echo
   set -x
   set +u
   [ -n "$jlogfile" ] && postmsg "$jlogfile" "$msg"
   set -u
fi
cp -p ${comin_m1}/${qual3}.*z.status* $OBCNTarch/${net}/${net}.${PDYm1}
err_copy_status_yesterday=$?
chmod 775 $OBCNTarch/${net}/${net}.${PDYm1}/*
if [ $err_copy_status_yesterday -eq 0 ]; then
   msg="$net_uc status file(s) for yesterday successfully copied to \
$OBCNTarch/${net}/${net}.${PDYm1}"
   set +x
   echo
   echo "$msg"
   echo
   set -x
   set +u
   [ -n "$jlogfile" ] && postmsg "$jlogfile" "$msg"
   set -u
fi
if [ $net = nam -o $net = gfs -o $net = gdas ]; then
   mkdir -p $OBCNTarch/${net}/${net}.${PDY}/updated.status
   chmod 775 $OBCNTarch/${net}/${net}.${PDY}/updated.status
   mkdir -p $OBCNTarch/${net}/${net}.${PDYm1}/updated.status
   chmod 775 $OBCNTarch/${net}/${net}.${PDYm1}/updated.status
   cp -p ${comin}/${qual3}.*z.updated.status* \
    $OBCNTarch/${net}/${net}.${PDY}/updated.status
   err_copy_updated_status=$?
   chmod 775 $OBCNTarch/${net}/${net}.${PDY}/updated.status/*
   if [ $err_copy_updated_status -eq 0 ]; then
      msg="$net_uc updated status file(s) for today successfully copied to \
$OBCNTarch/${net}/${net}.${PDY}/updated.status"
      set +x
      echo
      echo "$msg"
      echo
      set -x
      set +u
      [ -n "$jlogfile" ] && postmsg "$jlogfile" "$msg"
      set -u
   fi
   cp -p ${comin_m1}/${qual3}.*z.updated.status* \
    $OBCNTarch/${net}/${net}.${PDYm1}/updated.status
   err_copy_updated_status_yesterday=$?
   chmod 775 $OBCNTarch/${net}/${net}.${PDYm1}/updated.status/*
   if [ $err_copy_updated_status_yesterday -eq 0 ]; then
      msg="$net_uc updated status file(s) for yesterday successfully copied to \
$OBCNTarch/${net}/${net}.${PDYm1}/updated.status"
      set +x
      echo
      echo "$msg"
      echo
      set -x
      set +u
      [ -n "$jlogfile" ] && postmsg "$jlogfile" "$msg"
      set -u
   fi
fi
set -x

cd $OBCNTarch/${net}/${net}.${PDY}

ls *.bufr_d > $DATA/ls.files_today.bufr

for n in `cat $DATA/ls.files_today.bufr`; do
   [ -s ${comin}/${n} ] && echo "EOF @@@" >> $n
done

cd $OBCNTarch/${net}/${net}.${PDYm1}

ls *.bufr_d > $DATA/ls.files_yesterday.bufr

for n in `cat $DATA/ls.files_yesterday.bufr`; do
   [ -s ${comin_m1}/${n} ] && echo "EOF @@@" >> $n
done

cd $DATA


#  Clean out all status and updated status file directories older than 30-days
#  ---------------------------------------------------------------------------

pdy_earliest=`/nwprod/util/exec/ndate -720 ${PDY}00 | cut -c1-8`
set +x
echo
echo "pdy_earliest is " $pdy_earliest
echo
set -x

set +x
ls -d $OBCNTarch/${net}/* > ls.dir
set -x

if [ -s ls.dir ];then
   lines=`cat ls.dir | wc -l`
else
   lines=0
fi

n=1

until [ $n -gt $lines ]
do
   eval this_dir=`echo \`head -n ${n} ls.dir | tail -n 1\``
##echo "this_dir is " $this_dir
   eval this_dir_date=`echo \`head -n ${n} ls.dir | tail -n 1\` | \
    awk -F"${net}/${net}" '{print$2}' | cut -c2-9`
##echo "this_dir_date is " $this_dir_date
   if [ $this_dir_date -le $pdy_earliest ]; then
      rm -r -f $this_dir
   else
      break
   fi
   n=`expr $n + 1`
done
msg="All $net_uc status file and updated status file (where applicable) \
directories older than 30-days cleaned out"
set +x
echo
echo "$msg"
echo
set -x
set +u
[ -n "$jlogfile" ] && postmsg "$jlogfile" "$msg"
set -u

fi    # fi on COPY = YES test


if [ $UPDATE_AVERAGE_FILE = YES ]; then
 
msg="Attempt to update data count average table for $net_uc network"
set +x
echo
echo "$msg"
echo
set -x
set +u
[ -n "$jlogfile" ] && postmsg "$jlogfile" "$msg"
set -u

#  Grep out the individual counts for all types, based on cycle
#  ------------------------------------------------------------

if [ $net = dump -o $net = rap -o $net = rucs1 -o $net = rucs2 -o \
     $net = rtma ] ; then
   cycles="t00z t01z t02z t03z t04z t05z t06z t07z t08z t09z t10z t11z \
           t12z t13z t14z t15z t16z t17z t18z t19z t20z t21z t22z t23z"
else
   cycles="t00z t06z t12z t18z"
fi

for cycle_avg in $cycles; do
   set +x
   grep -h -e " REPORTS" -e "^EOF @@@" \
    $OBCNTarch/${net}/$net*/*${cycle_avg}.status*bufr_d | \
    grep -e " in data group " -e "^EOF @@@" | grep -e " HAS" -e "^EOF @@@" | \
    grep -v -e "Domain" > delete
   set -x
   awk -F" HAS" '{print$2}' delete | awk -F" REPORTS" '{print$1}' | \
    sed "s/[^0-9]/ /g" > delete.r
   awk -F" in data group " '{print $1}' delete | \
    sed "s/[^0-9][^0-9][^0-9]^\.[^0-9][^0-9][^0-9]/ /g" | \
    sed "s/[^0-9]/ /g" > delete.l
   paste -d" " delete.l delete.r | sed "s/^[^0-9]/EOF @@@       0/g" \
    > accum.obs.counts.${cycle_avg}
   rm delete delete.l delete.r
done

grep -e "^_" -e "^:" $DUMPLIST | grep -Fe "#>" > dumplist


#  Generate a new average file
#  ---------------------------

final_yr=`echo $PDY | cut -c1-4`
final_mn=`echo $PDY | cut -c5-6`
final_dy=`echo $PDY | cut -c7-8`
final_date=${final_mn}/${final_dy}/${final_yr}


cat <<EOFd > avgdata.cards
 &INPUT
   NETWORK='$net_uc',
   CURRDATE='$final_date'
 /
EOFd

pgm=`basename  $AVGX`
if [ -s prep_step ]; then
   set +u
   . prep_step
   set -u
else
   [ -f errfile ] && rm errfile
   export XLFUNITS=0
   unset `env | grep XLFUNIT | awk -F= '{print $1}'`

   set +u
   if [ -z "$XLFRTEOPTS" ]; then
     export XLFRTEOPTS="unit_vars=yes"
   else
     export XLFRTEOPTS="${XLFRTEOPTS}:unit_vars=yes"
   fi
   set -u

fi

export XLFUNIT_10=dumplist
# unit 21 opened internally by pgm and holds accumulated obs. for each cycle
# unit 25 opened internally by pgm and holds accumulated sat 1b for each cycle
export XLFUNIT_51=obcount_30davg.${net}.current
timex $AVGX< avgdata.cards > outout 2> errfile
err=$?
###cat errfile
cat errfile >> outout
cat outout >> avgdata.out
set +u
[ -n "$pgmout" ]  &&  cat outout >> $pgmout
set -u
rm outout
set +x
echo
echo 'The foreground exit status for BUFR_AVGDATA is ' $err
echo
set -x
if [ $err -eq 0 ]; then

   set +x
   echo " ------------------------------------------------------------------- "
   echo " ******** COMPLETED PROGRAM BUFR_AVGDATA for $net_uc NETWORK ********"
   echo " ------------------------------------------------------------------- "
   set -x
   msg="bufr_avgdata completed normally for $net_uc network"
   set +u
   [ -n "$jlogfile" ] && postmsg "$jlogfile" "$msg"
   set -u
else

msg="BUFR_AVGDATA TERMINATED ABNORMALLY WITH CONDITION CODE $err --> non-fatal"
   set +x
   echo
   echo "$msg"
   echo
   set -x
   set +u
   [ -n "$jlogfile" ] && postmsg "$jlogfile" "$msg"
   set -u

fi
fi    # fi on UPDATE_AVERAGE_FILE = YES test

if [ $COPY = YES ]; then

#  Remove any empty directories (these should never be empty unless
#   there is a problem - removing them here may help detect a 
#   "break" in the script sooner than if all 30-days are missing)
#  ----------------------------------------------------------------

   rmdir $OBCNTarch/${net}/${net}.${PDYm1}
   rmdir $OBCNTarch/${net}/${net}.${PDY}

fi

exit $err