echo "----------------------------------------------------------------------------"
echo "Executing exmodels_${RUN}.sh.sms to produce diagnostic asci data"
echo "regarding observational counts of a particular data type for"
echo "a given model, cycle (current), its past thirty days counts and its"
echo "monthly average count.  The monthly average counts are taken from"
echo "/com/arch/prod/avgdata and the daily counts are from"
echo "/com/$NET/prod/$NETM.YYYYMMDD .  This script produces data counts for"
echo "the following models NAM, GFS, GDAS.  RAP job is run separately"
echo "History: SEP 2004 - First implementation of this new script."
echo "Modified to add several new data types June 2006"
echo "Author: V. Krishna Kumar"
echo "NCO/Data Management & Quality Assessment Branch September 2004"
echo "NCO/System Integration Branch July 2008"
echo "Modified to automate the data type plotting from the bufr dump list Oct 2009"
echo "Modified to extend the time series to 30 days for all models May 2010"
echo "----------------------------------------------------------------------------"

cd $DATA

set -x
msg="$job HAS BEGUN!"

##############################################
# Define COM directories
##############################################
msg="$NETM ANALYSIS TIME IS $PDY$cyc"
postmsg "$jlogfile" "$msg"

if [ $NET = "gfs" -o $NET = "nam" ]; then
   export COM=/com/${NET}/prod/${NETM}.$PDY
   file_cur=filem00
   echo $file_cur >> file_${NETM}
   cp $COM/$BASERUN.t${cyc}z.updated.status.tm00.bufr_d filem00
   DATEminus=`sh /nwprod/util/ush/finddate.sh $PDY s-30`
   smod='01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30'
   for sm in $smod
   do
      file_cnt=filem${sm}
      echo ${file_cnt} >> file_${NETM}
      date_str=`echo $DATEminus | awk -v day=$sm '{ print $day }'`
      export COMmmod=/com/$ARCH/${envir}/$DLY/${NETM}/${NETM}.${date_str}/updated.status
      cp $COMmmod/$BASERUN.t${cyc}z.updated.status.tm00.bufr_d filem${sm}
   done
   mfiles=`cat file_${NETM}`
fi
### Truncate the files keeping only the data counts, 30-day average and data type info

### 1
dn=-1
for tfile in $mfiles
do
   dn=`expr $dn + 1`
   nr=`grep -n "receipt   average  " $tfile |  cut -d':' -f1`
   nc=`wc -l $tfile | tr -s " " | cut -d' ' -f2`
   nm=`expr $nc - 2`
   nt=`expr $nm - $nr - 1 `
   head -$nm $tfile | tail -$nt > ${tfile}_temp
   cut -c 2-19,22-79 ${tfile}_temp > ${tfile}_t
   cat ${tfile}_t > all_${tfile}_t

### Prepare the input file from all_${tfile}_t file for gempak graphics start !!!

### Create model_name as input for fortran

   echo ${NETMOD} > model

### Program will read the data files containing current data counts,
### monthly counts and mnemonics and converts to a format required for
### sffdsf (asci data) GEMPAK plotting program

   export pgm=realtime_models_convert_gem
   . prep_step
   startmsg

   export XLFRTEOPTS="unit_vars=yes"     # Allow overriding default names.
   export XLFUNIT_30="realtime_bufr_dumplist.txt"
   export XLFUNIT_31="all_${tfile}_t"
   export XLFUNIT_50="file_all_${tfile}_t"

   $EXECPREP/realtime_models_convert_gem <model 1>> $pgmout 2>errfile

   export err=$?;err_chk

### Sort for duplicate entries and change lower case letters to upper
### case letters - otherwise gempak screams and fails!!!

   sort -d -f -u file_all_${tfile}_t > file_all_${tfile}_t_sorted
   tr '[a-z]' '[A-Z]' < file_all_${tfile}_t_sorted > ${NETMOD}_alldatatype_${tfile}_t${cyc}z

   export dstring=`awk '{print $3}' ${tfile}_t`

   if [ $tfile = filem00 ]; then
      datecyc=$PDY$cyc
      echo $datecyc > cur_date
   else
      datecyc=`echo $DATEminus | awk -v dday=$dn '{ print $dday }'`
      echo $datecyc > curm${dn}_date
   fi

### 2
   for dtype in $dstring
   do
      echo $datecyc > date_$dtype

      if [ $dtype = synop ]; then
         grep 'synop  S' ${tfile}_t >file_${dtype}_${datecyc}
      elif [ $dtype = prflr ]; then
         grep 'NOAA/FSL' ${tfile}_t >file_${dtype}_${datecyc}
      elif [ $dtype = ssmip ]; then
         grep 'SSM/I - FNMOC' ${tfile}_t >file_${dtype}_${datecyc}
      elif [ $dtype = geost ]; then
         grep 'GOES/NESDIS (5x5 f-o-v)' ${tfile}_t >file_${dtype}_${datecyc}
      else
         grep $dtype ${tfile}_t >file_${dtype}_${datecyc}
      fi

      paste date_${dtype} file_${dtype}_${datecyc} >> ${NETMOD}_${dtype}_all_days
   done
### 2

done
### 1

### Convert the asci output files of only models nam, gfs and gdas
### to gempak surface file which is the input for creating time series.

$UTILPREP/exmodels_realtime_asci2gempak.sh

### The script exmodels_realtime_gempak.sh reads the gempak
### surface file and makes single panel time series
### data counts plots starting from previous 30 days for models
### gfs, gdas, and nam from cyc hour to the current day cyc hour
### and its corresponding monthly mean dump counts.

tr '[a-z]' '[A-Z]' < datatypes_plotnumbs > ${NETMOD}_dataplotlist_${cycle}
plot_file=${NETMOD}_dataplotlist_${cycle}

cat $plot_file | awk '{ print $1,$2 }'>datatypes_list
wc -l datatypes_list > nlines_temp_file
total_dtypes=`awk '{ print $1 }' nlines_temp_file`

listperproc=`expr $total_dtypes / 16`
echo "List per processor = $listperproc"

msg="Total number of data types is $total_dtypes - each parallel task will process\
 $listperproc data types"

###---------------------------------------------------------------------

cal_total_dtypes=`expr $listperproc \* 16`
ndtypes=0
while [ "$ndtypes" -lt "$total_dtypes" ]
do
   ndtypes=`expr $ndtypes + 1`
   if [ $ndtypes -lt $cal_total_dtypes ]; then
      procn=`expr \( $ndtypes - 1 \) / $listperproc + 1`
   else
      procn=`expr $ndtypes - $cal_total_dtypes + 1`
   fi
   read x
   echo $x >> list_$procn
   echo ndtypes= $ndtypes
done <datatypes_list

###---------------------------------------------------------------------

tlist=`ls list*`

nlists=0
for tl in $tlist
do
   nlists=`expr $nlists + 1`
   echo `cat $tl | awk '{ print $1}'` > file.dtype_$nlists
   echo `cat $tl | awk '{ print $2}'` > plot.dtype_$nlists
done

###---------------------------------------------------------------------

### delete if number_of_processors file is present already

rm -rf number_of_processors

### Create a dummy process if the number of processors actually required is
### less than the alloted number of processors

numlist=`ls list* | wc -l`

while [ "$numlist" -lt "16" ]
do
   numlist=`expr $numlist + 1`
   touch list_$numlist
done

mlists=0
for im in `ls list_*`
do
   mlists=`expr $mlists + 1`
   echo $mlists >> number_of_processors
done

export n_proc=`cat number_of_processors`
echo $n_proc

### MPI debugging variables.  Don't always need.

poe hostname
export MP_PGMMODEL=mpmd
export MP_PROCS=16
export MP_NODES=1

### Create individual poe scripts

export nproc=$n_proc

for proc in $nproc
do
   cat <<EOF > poescript$proc
   echo $DISPLAY
   hostname
   DATA_TMP=$DATA
   export DATA=\$DATA_TMP/TASK$proc
   mkdir \$DATA
   cd \$DATA
   cp $DATAO/file.dtype_${proc} .
   cp $DATAO/plot.dtype_${proc} .
   cp $DATAO/*.sfc .
   cp $DATAO/*.pack .
   cp $DATAO/*alldatatype*file*z .
   cp $FIXgempak/coltbl.xwp.wbg coltbl.xwp
   $UTILPREP/exmodels_realtime_gempak.sh $proc
EOF
done

### Create master poescript

for proc in $nproc
do
   chmod 775 ./poescript$proc
   if [ $proc -eq 1 ]; then
      echo "./poescript$proc" > poescript
   else
      echo "./poescript$proc" >> poescript
   fi
done

chmod 775 poescript
export MP_CMDFILE=poescript

### Execute the script.
msg="Begin poe"
$DATAO/postmsg "$jlogfile" "$msg"

pgm=poe

poe

echo "Ending Poe  : `date`"

export err=$?; err_chk

### copy the final time series sfc file and gif files
### to /com/.. directory

### copy the final time series sfc file and gif files to /com/.. directories

if test "$SENDCOM" = 'YES'
then
   cp ${NETMOD}_alldatatype_m*.sfc $COMOUTRT/.
   cp ${NETMOD}_* $COMOUTRT/.
   cp ${NETMOD}_* $COMOUTRT/.
   cp ${NETMOD}_sumtable $COMOUTRT/${NETMOD}_sumtable_${cycle}
   for proc in $nproc
   do
      cp $DATA/TASK$proc/*.gif $COMOUT/.
   done
fi

### Prepare the main web page (tables) by running exhrly_realtime_mainpage.sh
### script with three arguments

$UTILPREP/exhrly_realtime_mainpage.sh $COM1OUT $COM1OUT ${NETMOD}_sumtable

### Generates three output (i) realtime_mainpage.html (ii) ${NETMOD}_summary_table.html
### (iii) ${NETMOD}_table.html files in $COM1OUT. The index files (i) index.mainpage.shtml
### (ii) index.summary.shtml (iii) index.table.shtml files need to be copied to $COM1OUT
### (i) and $gifdir (ii) & (iii)

if test "$SENDCOM" = 'YES'
then
   cp $FIXWEB/index.mainpage.shtml $COM1OUT/.
   cp $FIXWEB/index.summary.shtml $COMOUT/.
   cp $COM1OUT/${NETMOD}_summary_table.html $COMOUT/summary.html
   cp $FIXWEB/index.table.shtml $COMOUT/index.shtml
   cp $COM1OUT/${NETMOD}_table.html $COMOUT/table.html
   chmod og+r $COM1OUT/*html
   chmod og+r $COMOUT/*html
fi

### Run the color code script on the mainpage.

bash /nw${envir}/scripts/exhrly_realtime_color.sh $COM1OUT $COM1OUT

if test "$SENDWEB" = 'YES'
then
###
### rsync the directory structure from $COMOUT to ncorzdm web server
###
   /usr/bin/ssh nwprod@ncorzdm.ncep.noaa.gov "pwd"
   if [ $? -eq 0 ] 
   then
      /usrx/local/bin/rsync -v --recursive --links --perms --rsh=/usr/bin/ssh --stats --delete --whole-file --update /com/web/${envir}/realtime/realtime nwprod@ncorzdm.ncep.noaa.gov:/home/www/nco/htdocs/pmb/nw${envir}/
      if [ $? -gt 0 ] 
      then
         echo "Error doing rsync"
         exit 30
      fi
   else
      echo "Cannot connect to nwprod\@ncorzdm.ncep.noaa.gov"
      exit 40
   fi
fi

### GOOD RUN
set +x
echo " "
echo " ****** $job PROCESSING COMPLETED NORMALLY"
echo " ****** $job PROCESSING COMPLETED NORMALLY"
echo " ****** $job PROCESSING COMPLETED NORMALLY"
echo " ****** $job PROCESSING COMPLETED NORMALLY"
echo " "
set -x

### Added to ensure the global variable is returned to proper working
### directory. 20081102 wx11rp
cd $DATAO

msg='HAS COMPLETED NORMALLY.'
echo $msg
postmsg "$jlogfile" "$msg"

################## END OF SCRIPT #######################