#!/bin/sh
 
# This job processes the Canadian radar data.
#
# Original Author:  Jeff Ator NCO/SIB 03-2011
# V. Krishna Kumar NCO/SIB 01-2013  Modified JOBS FOR WCOSS
# Jeff Ator NCO/SIB 01-2014  Modified for vertical structure
 
#############################################################
# Set shell processing variables
#############################################################

date
export PS4=' $SECONDS + '
set -xa

#############################################################
# Obtain unique process id (pid) and make temp directory
#############################################################
export pid=$$
export RUN_ENVIR=${RUN_ENVIR:-prod}
if [ $RUN_ENVIR = prod -a $envir != prod ]; then
    jlogfile=${jlogfile:-/com/logs/${envir}/jlogfile}
    TMPNWDIR=${TMPNWDIR:-/tmpnwprd2}
fi
export job=canrad
export jlogfile=${jlogfile:-/com/logs/jlogfiles/jlogfile.${job}.${pid}}
TMPNWDIR=${TMPNWDIR:-/tmpnwprd1}
export DATA=${DATA:-$TMPNWDIR/${job}.${pid}}
mkdir -p $DATA
cd $DATA

date
export time=`date -u "+%Y%m%d%H%M"`
echo $time
echo `date`

export cycle=t${cyc}z

#############################################################
# Specify NET and RUN Name and model
#############################################################

export NET=${NET:-hourly}
export model=radar

#############################################################
# Determine Job Output Name on System
#############################################################

export outid="LL$job"
export jobid="${outid}.o${pid}"
export pgmout="OUTPUT.${pid}"

#############################################################
# Specify execution areas
#############################################################

export utilscript=/nwprod/util/ush

export DCOMROOT=/dcom/us007003
export RAWDATA=${DCOMROOT}/ldmdata/obs/upperair/radar_Canada
export HIST=${DCOMROOT}/radar_Canada

export DECDROOT=/nw${envir}/decoders
export jobscript=${DECDROOT}/decod_dccrad/scripts
export FIXbufr=${DECDROOT}/decod_shared/fix

vfile=/nw${envir}/versions/obsproc_satingest.ver
if [ -s ${vfile} ]
then
    . $vfile
    export TRANJBush=/nw${envir}/obsproc_satingest.${obsproc_satingest_ver}/ush/bufr_tranjb.sh
    export EXECobsproc_satingest=/nw${envir}/obsproc_satingest.${obsproc_satingest_ver}/exec
    export USHobsproc_shared_bufr_cword=/nw${envir}/obsproc_shared/bufr_cword.${obsproc_shared_bufr_cword_ver}/ush
    export EXECobsproc_shared_bufr_cword=/nw${envir}/obsproc_shared/bufr_cword.${obsproc_shared_bufr_cword_ver}/exec
else
    echo "Version file ${vfile} not found"
    exit 1
fi

export GEMERR=/nwprod/gempak/nawips1/gempak/error

#############################################################
# Specify the maximum age (in minutes) of files to be processed.
# Raw files older than this limit will not be processed but will
# be listed in the history files as though they were processed.
# This prevents trying to process too many old data files
# following an extended outage.
#############################################################

export mxagemin=30

#############################################################
# Specify the maximum number of processors to use
#############################################################

export mxprocs=8

#############################################################
# SENDCOM  - Send output to $DCOMROOT
#############################################################

export SENDCOM=YES

#############################################################
# Initialize the working directory
#############################################################

$utilscript/setup.sh

$utilscript/setpdy.sh
. PDY

env

#############################################################
# Distribute the stations to be processed amongst the available processors
#############################################################
 
msg="CANADIAN RADAR PROCESSING HAS BEGUN on \
`hostname`"
postmsg "$jlogfile" "$msg"

set -x

ls $RAWDATA > stations_list
total_stns=`ls $RAWDATA | wc -l`

listperproc=`expr $total_stns / $mxprocs`
echo "List per processor = $listperproc"

msg="Total number of stations is $total_stns - each parallel task will process\
 $listperproc stations"
postmsg "$jlogfile" "$msg"

cal_total_stns=`expr $listperproc \* $mxprocs`
nstns=0
while [ "$nstns" -lt "$total_stns" ]
do
    nstns=`expr $nstns + 1`
    if [ $nstns -lt $cal_total_stns ]
    then
	procn=`expr \( $nstns - 1 \) / $listperproc + 1`
    else
	procn=`expr $nstns - $cal_total_stns + 1`
    fi
    read x
    echo $x >> list_$procn
    echo nstns= $nstns
done < stations_list

tlist=`ls list*`

nlists=0
for tl in $tlist
do
    nlists=`expr $nlists + 1`
    echo `cat $tl` > file.station_$nlists
done

## Create dummy processes if the number of processors actually required
## is less than the alloted number of processors

numlist=`ls list* | wc -l`

while [ "$numlist" -lt "$mxprocs" ] 
do
   numlist=`expr $numlist + 1`
   touch list_$numlist
done

##
mlists=0
for im in `ls list_*`
do
    mlists=`expr $mlists + 1`
    echo $mlists >> number_of_processors
done

export n_proc=`cat number_of_processors`
echo $n_proc

#############################################################
# Create individual poe scripts
#############################################################

poe hostname

export nproc=$n_proc
            
for proc in $nproc
do
    cat << EOF > poescript$proc

    hostname
    DATA_TMP=$DATA

    export DATA=\$DATA_TMP/TASK$proc
    mkdir \$DATA

    cd \$DATA

    export DBNBUFRT=120
    export DBNROOT=\$DATA
    mkdir \$DBNROOT/tmp

    cp \$DATA_TMP/file.station_${proc} .

    sh $jobscript/exradar_canrad.sh.ecf $proc
EOF
done

#############################################################
# Create a master poescript
#############################################################

for proc in $nproc
do
    chmod 775 ./poescript$proc
    if [ $proc -eq 1 ]
    then
       echo "./poescript$proc" > poescript
    else
       echo "./poescript$proc" >> poescript
    fi
done

chmod 775 poescript

#############################################################
# Execute the master poescript
#############################################################

export MP_PGMMODEL=mpmd
export MP_CMDFILE=poescript
export MP_LABELIO=NO

msg="Begin poe"
postmsg "$jlogfile" "$msg"

mpirun.lsf -cmdfile poescript


echo "Ending Poe  : `date`"

export err=$?; err_chk


#############################################################
# Concatenate the BUFR output files from each process into a single BUFR file
#############################################################

if test "$SENDCOM" = 'YES'
then
    for proc in $nproc
    do
        if [ -s $DATA/TASK$proc/bufrout.task$proc ]; then
            cat $DATA/TASK$proc/bufrout.task$proc >> bufrout_total.stations
        else
            msg="***WARNING: COMBINED STATION BUFR FILE MAY BE INCOMPLETE***"
            postmsg "$jlogfile" "$msg"
        fi
    done
fi

#############################################################
# Ingest the BUFR file into the database
#############################################################

if [ ! -s bufrout_total.stations ]
then

   msg1="***WARNING: COMBINED STATION BUFR FILE NOT FOUND, TRANJB DID NOT"
   msg2="RUN - NO CANADIAN RADAR DATA PROCESSED"
   postmsg "$jlogfile" "$msg1 msg2"

else

   set +x
   echo
   echo "--------------------------------------------------------------------"
   echo "START TRANJB TO PUT BUFR DATA INTO TANKS IN $DCOMROOT"
   echo "--------------------------------------------------------------------"
   echo
   set -x

   export SUBDATE_CHECK=NO

   export log=$DCOMROOT/tranjb_canrad.out

   $TRANJBush $DCOMROOT ./bufrout_total.stations
   err=$?
   if [ $err -eq 0 ]
   then
      msg="TRANJB completed normally"
   else
      msg="**ERROR: TRANJB FAILED with r.c. $err - NO CANADIAN RADAR DATA PROCESSED"
   fi
   postmsg "$jlogfile" "$msg"

fi

#############################################################
# Close up shop
#############################################################

date

msg="CANADIAN RADAR PROCESSING HAS COMPLETED NORMALLY!"
postmsg "$jlogfile" "$msg"

if [ `find $DATA -name 'core' -exec ls -1 {} \; | wc -l` -gt 0 ]
then
    echo "core file exists - preserving $DATA"
else
    cd $TMPNWDIR
    rm -rf $DATA
fi