#!/bin/sh
################################################################################
####  UNIX Script Documentation Block
#                      .                                             .
# Script name: 		exmerge_files.sh
#
# Script description: Top level script for creation of MERGE files
#	MERGE: Merge of Eta,Ruc,Global-gfs by Emc
#
# Author:  Robert E. Kistler W/NP23 301-763-8000x7232 bkistler@ncep.noaa.gov
#
# Abstract:
#
# Script history log:
# 1999-11-20
# 1999-12-29 - Michaud - Modified for Test Prod Version for IBM SP
#
# Usage: exmerge_files.sh [DATA] [PDY] [cyc]
#
#   Input script positional parameters (my be exported by J-Job):
#		DATA=$1  working directory
#		PDY=$2   8 digit date
#		cyc=$3    2 digit initial conditon hour
#
#   Imported Shell Variables:
#	 -----------------------------------
#	 production directories
#	 -----------------------------------
#		COM:			production /com directory
#		PCOM:			production /pcom directory
#		DATA:			working directory           
#	 -----------------------------------
#	 production utilities / replacements
#	 -----------------------------------
#		NDATE:			date utility directory
#		NHOUR:			date utility	
#		GRBINDEX:		grib index utility
#		COPYGB:			grib interpolation utility
#		COPYGB_PARM:	grib interpolation utility parm file
#		WGRIB:			grib inventory/file editor utility
#	 -----------------
#	 MERGE directories
#	 -----------------
#		USHmerge		scripts
#		FIXmerge:		table input used in  merge_wmogrib.sh
#		EXECmerge:		executables
#        -----------------
#        MERGE executables
#        -----------------
#		WAFS_MERGE		$EXECmerge/merge_mergegb
#		MERGE_WMOGRIB		$EXECmerge/merge_wmogrib
#		WAFS_WEIGHTS		$EXECmerge/merge_weights
#		RUC_PRECIP		$EXECmerge/merge_ruc_precip
#		GFS_PRECIP		$EXECmerge/merge_gfs_precip
#		NAM_PRECIP		$EXECmerge/merge_nam_precip
#    	-----------------
#    	MERGE directory tree
#    	-----------------
#		./$DATA/merge.$wafs_grid/f$fhr
#		./$DATA
#			./merge.39  
#				./f00
#				./f03
#				./f06
#				./f09
#				./f12
#				./f18
#				./f24
#				./f30
#				./f36
#			./merge.40
#				./f00
#				./f03
#				./f06
#				./f09
#				./f12
#				./f18
#				./f24
#				./f30
#				./f36
#    	-----------------
#    	MERGE script tree
#    	-----------------
#		merge_files.sh
#			merge_gfs_precip.sh
#			merge_nam_precip.sh
#			merge_nam_precip.sh
#			merge_grids.sh
#				merge_grid2wafs.sh
#				merge_precip2wafs.sh
#				merge_weights.sh
#				merge_wmogrib.sh
#    	-----------------
#    	MERGE switches
#    	-----------------
#		PRECIP	:-YES precip processing
#		PURGE	:-YES prune directory tree  below ./$DATA/merge.$wafs_grid
#		MAPS    :-NO  create diagnostic grid
#    	-------------------------------------
#		 description of production environment
#    	-------------------------------------
#		RUNS			gfs ruc nam
#		
#	 run directory,file and index variable
#		RUC_DIR			NAM_DIR			GFS_DIR
#		RUC_FILE 		NAM_FILE		GFS_FILE	
#		RUC_INDEX		NAM_INDEX		GFS_INDEX		
#		
#	 frequncy of run accessed by merge
#		HH_INC_RUC     	HH_INC_NAM		HH_INC_GFS
#		
#  	 fcst extent accessed by merge
#		MAX_FHR_RUC		MAX_FHR_NAM		MAX_FHR_GFS		
#		
#	 expected inital conditon list for each cyc
#		RUC_EXPECTED 	NAM_EXPECTED	GFS_EXPECTED
#   	----------------------------------------
# 		variables applicable to MERGE environment
#   	-----------------------------------------
#		FHRLIST 	list of forecast hours
#		FILE_QUAL       file prefix:merge
#		MERGE_QUAL  	structure of output merge grib file
#					$FILE_QUAL.\$wafs_grid.\$t\$cyc\$z.f\$fhr
#		WAFS_GRIDS      grib grid id for North Americal wafs grids {39,40}
#   	-----------------------------------------
#	 	output files
#   	-----------------------------------------
#		command file:     $FILE_QUAL.t${cyc}z.command
# 		merge grib files: $FILE_QUAL.$wafs_grid.t${cyc}z.f$fhr
#		script output:    $FILE_QUAL.$wafs_grid.t${cyc}z.f$fhr.out
#		
#
# Remarks: must be run a with NPROCS=$((${#WAFS_FILES[*]}*${#fhrlist[*]))
#
#   Condition codes
#      0 - no problem encountered
#     >0 - some problem encountered
#
#  Control variable resolution priority
#    1 Command line argument.
#    2 Environment variable.
#    3 Inline default.
#
# Attributes:
#   Language: POSIX shell
#   Machine: IBM SP
#
#
################################################################################

if [ $# -ne 3 -a $# -ne 0 ]
then 
	echo "Usage: $0 [DATA] [PDY] [cyc] " 
	exit 8
fi

set -eaux

DATA=${DATA:-$1}
PDY=${PDY:-$2}
cyc=${cyc:-$3}

if [ ${#PDY} -ne 8 -o ${#cyc} -ne 2 ]
then 
   exit 1
fi

cd $DATA

msg="MERGE Processing HAS BEGUN"
postmsg "$jlogfile" "$msg"

cdate10=$PDY$cyc

host=`hostname`
t=t;z=z

# Build table $wafs_files_out with decision entries:
# for each run [ruc|nam|gfs] and each forecast hour, ($fhr in $fhrlist)
# We search for and record decision on candidate forecasts
# the decisons are based on latest availability and geographical area covered
# by each run. The script is executed immediately following complettion 
# the ruc when it makes a 12 hr forecst (HH={00,03,06,09,12,15,18,21})
# First the gfs forecasts are screened. the first complication enters - the gfs
# and nam run at 6hr frequency 

# array of output forecast hours 
#-------------------------------
set -A fhrlist ${FHRLIST:- "00  03  06  09  12  18  24  30  36"}
for fhr in ${fhrlist[*]}
do 
   best_fhr[fhr]=99999
done

set -A run     ${RUNS:-gfs nam ruc}

n=-1
while [ $((n+=1)) -lt ${#run[*]} ]
do
   RUN=`echo ${run[$n]}|tr "[a-z]" "[A-Z]"`
   eval hh_inc[$n]=\$HH_INC_$RUN  
   eval dir[$n]=\$${RUN}_DIR
   eval file[$n]=\$${RUN}_FILE
done

set -A max_fhr 0 0 0
n=2
inc=$(( 24 / $(echo $MAX_FHR_RUC|wc -w) ))
ihh=-$inc
for f in $MAX_FHR_RUC
do  
	ruc_max_fhr[$((ihh+=inc))]=$f
	if [ $f -gt ${max_fhr[$n]} ]
        then
           max_fhr[$n]=$f
        fi
done

n=1
inc=$(( 24 / $(echo $MAX_FHR_NAM|wc -w) ))
ihh=-$inc
for f in $MAX_FHR_NAM
do  
	nam_max_fhr[$((ihh+=inc))]=$f
	if [ $f -gt ${max_fhr[$n]} ]
        then
           max_fhr[$n]=$f
        fi
done

n=0
inc=$(( 24 / $(echo $MAX_FHR_GFS|wc -w) ))
ihh=-$inc
for f in $MAX_FHR_GFS
do  
	gfs_max_fhr[$((ihh+=inc))]=$f
	if [ $f -gt ${max_fhr[$n]} ]
        then
           max_fhr[$n]=$f
        fi
done

inc=$((24/$(echo $RUC_EXPECTED|wc -w) ))
ihh=-$inc
for f in $RUC_EXPECTED
do 
	ruc_expected[$((ihh+=inc))]=$f
done

inc=$((24/$(echo $NAM_EXPECTED|wc -w) ))
ihh=-$inc
for f in $NAM_EXPECTED
do 
	nam_expected[$((ihh+=inc))]=$f
done

inc=$((24/$(echo $GFS_EXPECTED|wc -w) ))
ihh=-$inc; for f in $GFS_EXPECTED
do
	gfs_expected[$((ihh+=inc))]=$f
done

# build decision table $DATA/wafs_files.t${cyc}z.out
# ----------------------------------------------------
#	 record "best available forecasts"
#	 gfs - whatever is available
#	 nam - only if as timely or newer than nam
#	 ruc - only if as timely or newer than ruc
>$DATA/$FILE_QUAL.t${cyc}z.out
wafs_files_out=$DATA/$FILE_QUAL.t${cyc}z.out

# loop over the 3 runs
# --------------------
n=-1
while [ $((n+=1)) -lt ${#run[*]} ]
do
	eval ${run[$n]}_found="--"
	cinc=${hh_inc[$n]}
	eval expected_hh=\${${run[n]}_expected[cyc]}
	if [ $expected_hh != "--" ]
        then 
		((hh_diff=cyc-expected_hh))
		if [ $hh_diff -lt 0 ]
                then
                   ((hh_diff=hh_diff+24))
                fi
	fi
	expected_pdy=`$NDATE -$hh_diff $cdate10|cut -c1-8`
	  
	# loop over the targeted output forecast hours
	# --------------------------------------------
	for fhr in ${fhrlist[*]}
        do
		vdate10=`$NDATE $fhr $cdate10`
		tmpd=$cdate10
		tmpc=$cyc
		tmpp=$PDY
		tmpf=$fhr

		# write run header as table entry
		# -------------------------------
		echo "${run[n]} $vdate10 $fhr $cdate10 expected=$expected_pdy$expected_hh" |tee -a $wafs_files_out

		# make copies of date/forecast variables 
		# re-use variable names in internal loops for readability
		# -------------------------------------------------------
		found=NO
		eval ${run[$n]}_file[`expr $fhr`]=/dev/null
		eval ${run[$n]}_precip[`expr $fhr`]=/dev/null
		if [ $expected_hh = "--" ]
                then
                   continue
                fi

		found=NO
		# loop over forecast extent of the run
		# ------------------------------------
		while [ $fhr -le ${max_fhr[$n]} ]
                do 
			while [ `expr $fhr : '.*'` -lt 2 ]
                        do
                           fhr=0$fhr
                        done
			# determine valid cyc times for the run
			# ----------------------------------------------------
			# 	write table entry (key: "new cyc=$cyc)
			# 	when cyc time is revised
echo $cyc $cinc
                        ((tmpmod=cyc%cinc))
echo $tmpmod $cyc $cinc
			if [ "$tmpmod" -ne 0 ]
                        then 
				inc=$tmpmod
				echo old cyc=$cyc fhr=$fhr inc=$inc tee -a $wafs_files_out
				cdate10=`$NDATE -$inc $cdate10`
				PDY=`echo $cdate10|cut -c1-8`
				cyc=`echo $cdate10 |cut -c9-10`
				((fhr=fhr+inc))
				while [ `expr $fhr : '.*'` -lt 2 ]
                                do
                                   fhr=0$fhr
                                done
				echo new cyc=$cyc fhr=$fhr |tee -a $wafs_files_out
			fi

			# check for hierarchy of cyc
			# i.e. use the latest forecasts starting with expected forecast
			#--------------------------------------------------------------

			# list variable is for entering in table 
			# --------------------------------------- 
			list="fhr=$tmpf $cdate10 $fhr " 

			# make four tests for the run
			# ---------------------------
			# 1) check ${run}_max_fhr to see if run is made (-ne -1)
			#    if not make a table entry keyed with "nohh"
			# 2) check to see if the last forecast hour file exists 
			#    if not make a table entry keyed with "last"
			# 3) check for forecast file existence
			#    if not make a table entry keyed with "miss"
			# 4) check to see if previous runs have better timeliness
			#    if not make a table entry keyed with "miss"
			#       
			#  when a forecast file passess these check  
			#  an entry keyed with "found" is made 
			#  
			#  we now determine the associated precip file 
			#  
			#  then we "break" the loop
			#  
            
			found="Not Found"
			eval max=\${${run[n]}_max_fhr[$cyc]}
			if [ $max -ne -1 ]
                        then 
				tmpl=$fhr
				eval fhr=\${${run[n]}_max_fhr[$cyc]}
				eval name=${dir[$n]}/${file[$n]}
				if [ ${run[$n]} = ruc -a $fhr -eq 0 ]
                                then
					eval name=${dir[$n]}/ruc2.t${cyc}z.pgrbanl
				fi
				eval last_test=$name
				fhr=$tmpl
				if [ -s $last_test ]
                                then 
					eval name=${dir[$n]}/${file[$n]}
					eval test=$name
					if [ ${run[$n]} = ruc -a $fhr -eq 0 ]
                                        then
						eval name=${dir[$n]}/ruc2.t${cyc}z.pgrbanl
						eval test=$name
					fi
					if [ $cdate10 -gt $expected_pdy$expected_hh ]
                                        then
						result="lte "
					else
					result=miss
					   if [ -s $test -a ${best_fhr[$tmpf]} -ge $fhr ]
                                           then 
						result=found
						if [ $tmpf -eq 0 ]
                                                then
                                                   eval ${run[n]}_found="$cyc  "
                                                fi
						best_fhr[$tmpf]=$fhr
						eval ${run[$n]}_file[`expr $tmpf`]=$test
						if [ $PRECIP = YES ]
                                                then
							eval run_dir=${dir[$n]}
							precip=$DATA/${run[$n]}_precip.t${cyc}z
							if [ ! -s $precip ]
                                                        then
								eval "$TIMEX $USHmerge/merge_${run[$n]}_precip.sh $run_dir ${cyc} $max"
							fi
							eval ${run[$n]}_precip[`expr $tmpf`]=$precip
							echo "precip $list $precip" >> $wafs_files_out
						fi
					   fi
					fi
					echo "$result $list $test" |tee -a $wafs_files_out
					if [ $result = found ]
                                        then
                                           break
                                        fi
				else
					echo "last   $list $last_test" |tee -a $wafs_files_out
				fi
			else
				echo "nohh cyc-$cyc $list" |tee -a $wafs_files_out
			fi

			# decrement cyc/increment fhr to preserve valid time 
			# ----------------------------------------------------
			set -e
			cdate10=`$NDATE -$cinc $cdate10`
			PDY=`echo $cdate10|cut -c1-8`
			cyc=`echo $cdate10 |cut -c9-10`
			((fhr=$fhr+${hh_inc[$n]}))
		done # end run forcast extent loop
		# ----------------------------------------
		cdate10=$tmpd
		cyc=$tmpc
		PDY=$tmpp
		fhr=$tmpf
	done # end wafs forecast hour loop
done # end run loop

echo expected $PDY $cyc gfs ${gfs_expected[cyc]} $gfs_found | tee -a $wafs_files_out
echo expected $PDY $cyc nam ${nam_expected[cyc]} $nam_found | tee -a $wafs_files_out
echo expected $PDY $cyc ruc ${ruc_expected[cyc]} $ruc_found | tee -a $wafs_files_out

# prints of the entire table
# print files found sorted by fhr
# print precip files found 
grep "^found" $wafs_files_out
echo 
grep found $wafs_files_out|sort -k5,5 
echo
if [ $PRECIP = YES ]
then
	grep precip $wafs_files_out
fi

gfslabel=`grep "^found fhr=00" $wafs_files_out | grep gfs | awk '{print $3}'`
namlabel=`grep "^found fhr=00" $wafs_files_out | grep nam | awk '{print $3}'`
ruclabel=`grep "^found fhr=00" $wafs_files_out | grep ruc | awk '{print $3}'`

if [ "$SENDSMS" = "YES" ]
then
   if [ "${gfs_expected[cyc]}" = "$gfs_found" -a \
        "${nam_expected[cyc]}" = "$nam_found" -a \
        "${ruc_expected[cyc]}" = "$ruc_found" ]
   then
      $SMSBIN/setev Got_expected_runs
   fi

   if [ "$gfslabel" -eq "" ]
   then
      $SMSBIN/smslabel gfs "None"
   else
      $SMSBIN/smslabel gfs $gfslabel
   fi
   if [ "$namlabel" -eq "" ]
   then
      $SMSBIN/smslabel nam "None"
   else
      $SMSBIN/smslabel nam $namlabel
   fi
   if [ "$ruclabel" -eq "" ]
   then
      $SMSBIN/smslabel ruc "None"
   else
      $SMSBIN/smslabel ruc $ruclabel
   fi
fi

if [ "$SENDCOM" = "YES" ]
then
   echo "gfs $gfslabel" >  $DATA/$FILE_QUAL.t${cyc}z.status
   echo "nam $namlabel" >> $DATA/$FILE_QUAL.t${cyc}z.status
   echo "ruc $ruclabel" >> $DATA/$FILE_QUAL.t${cyc}z.status
   mv $DATA/$FILE_QUAL.t${cyc}z.status $COMOUT/$FILE_QUAL.t${cyc}z.status
fi

# Build command file to process $wafs_grid={39,40} for all of ${fhrlist[*]}
# ----------------------------------------------------------------------
# scripts are designed to run in parallel
# this requires that each runs in a separate directory
# the directory structure is: $DATA/$PDY/wafs.$wafs_grid/f$fhr
# each command file entry produces a "stdout/stderr" file and a grib file

# command file:		$FILE_QUAL.t${cyc}z.command
# grib file: 		$FILE_QUAL.$wafs_grid.t${cyc}z.f$fhr
# script output:	$FILE_QUAL.$wafs_grid.t${cyc}z.f$fhr.out

# on the SP we do all at once as a POE execution

cmdfile=$DATA/$FILE_QUAL.t${cyc}z.cmdfile
>$cmdfile
for wafs_grid in $WAFS_GRIDS
do
	qual=$DATA/$FILE_QUAL.$wafs_grid.t${cyc}z
	# build command file
	# ------------------
	for fhr in ${fhrlist[*]}
        do 
		if [ `expr $fhr : '.*'` -lt 2 ]
                then
                   fhr=0$fhr
                fi
		n=-1
                while [ $((n+=1)) -lt ${#run[*]} ]
                do
			eval ${run[$n]}_file=\${${run[$n]}_file[`expr $fhr`]}
			eval ${run[$n]}_precip=\${${run[$n]}_precip[`expr $fhr`]}
		done
		echo "$TIMEX $USHmerge/merge_grids.sh $cdate10 $fhr $wafs_grid >$qual.f$fhr.out 2>&1" |tee -a $cmdfile
                # make the working subdirectory
		# critical to running scripts in parallel
		# ---------------------------------------
		if [ ! -d $DATA/merge.$wafs_grid/f$fhr ]
		then
  		 mkdir -p $DATA/merge.$wafs_grid/f$fhr
		fi
	done
done
set -x
chmod 755 $cmdfile


export MP_PGMMODEL=mpmd
export MP_CMDFILE=$cmdfile
export MP_LABELIO=YES
export MP_INFOLEVEL=3
export MP_STDOUTMODE=ordered
poe_out=$DATA/$FILE_QUAL.t${cyc}z.poe.out

# Effective with transition to frost/snow, DO NOT execute a time command
#  with poe command!!!
############$TIMEX /usr/bin/poe >$poe_out 2>&1
/usr/bin/poe >$poe_out 2>&1
tail -6 $poe_out

nproc=-1
for wafs_grid in $WAFS_GRIDS
do 
	# check for successful execution
	# ------------------------------
	rc=0
	for fhr in ${fhrlist[*]}
        do 
		while [ `expr $fhr : '.*'` -lt 2 ]
                do
                   fhr=0$fhr
                done
		qual=$DATA/$FILE_QUAL.$wafs_grid.t${cyc}z
		out=$qual.f$fhr.out
		set +e
                   grep "finished" $poe_out|grep " $((nproc+=1)):"
                   rc=$?
                set -e
		if [ $rc -ne 0 ]
                then 
			msg="######## f$fhr failed ###########"
                        postmsg "$jlogfile" "$msg"
			cat $out
                        export pgm=merge_grids.sh
                        export err=$rc;err_chk
		fi
	done
done
exit $rc