#!/bin/sh
 
# This job processes the Canadian radar data.
#
# Original Author:  Jeff Ator NCO/SIB 03-2011
# V. Krishna Kumar NCO/SIB 01-2013  Modified JOBS FOR WCOSS
# Jeff Ator NCO/SIB 01-2014  Modified for vertical structure
# Jeff Ator EMC     03-2019  Modified for WCOSS phase3
 
#############################################################
# Set shell processing variables
#############################################################

date
export PS4=' $SECONDS + '
set -xa

# define ${GEMERR} for use with the decoder
pwd

if [ -d /opt/modules ]; then
    # WCOSS TO4 (Cray XC40)
    . /opt/modules/default/init/sh
elif [ -d /usrx/local/Modules ]; then
    # WCOSS Phase 1 & 2 (IBM iDataPlex)
    . /usrx/local/Modules/default/init/sh
else
    # WCOSS Phase 3 (Dell PowerEdge)
    . /usrx/local/prod/lmod/lmod/init/sh
fi


# DECDROOT=$(dirname $(dirname $(pwd)))
# JY - the following three lines will be moved to version files

DECDROOT=${DECDROOT:-${NWROOT}/decoders}
module load gempak/7.14.0

which setpdy.sh

#############################################################
# Obtain unique process id (pid) and make temp directory
#############################################################

export pid=${pid:-$$}
export DATA=${DATA:-${DATAROOT}/${jobid:?}}
mkdir -p $DATA
cd $DATA

jlogfile=${jlogfile:-${DATA}/jlogfile}

################################################################
# Copy utility scripts
# For the call in obsproc_satingest.v3.8.?/ush/bufr_tranjb.sh
################################################################
#cp $UTILROOT/ush/err_chk   .; chmod +x err_chk
#cp $UTILROOT/ush/err_exit  .; chmod +x err_exit
cp $UTILROOT/ush/prep_step .; chmod +x prep_step
cp $UTILROOT/ush/startmsg   .; chmod +x startmsg
cp $UTILROOT/ush/postmsg   .; chmod +x postmsg
##########################################

export RUN_ENVIR=${RUN_ENVIR:-prod}

# export job=${job:-canrad}

date
export time=`date -u "+%Y%m%d%H%M"`
echo $time
echo `date`

export cycle=t${cyc}z

#############################################################
# Specify NET and RUN Name and model
#############################################################

export NET=${NET:-hourly}
export model=radar

#############################################################
# Determine Job Output Name on System
#############################################################

export pgmout="OUTPUT.${pid}"

#############################################################
# Specify execution areas
#############################################################

# export utilscript=/nwprod/util/ush

export DCOM_ROOT=/lfs/h1/ops/${envir}/dcom
export DCOMIN_ROOT=/lfs/h1/ops/${envir}/dcom
export RAWDATA=${DCOMIN_ROOT}/ldmdata/obs/upperair/radar_Canada
export HIST=${DCOM_ROOT}/radar_Canada
# JY add following
mkdir -p $HIST $RAWDATA

export SCRIPTSdccrad=${DECDROOT}/decod_dccrad_${decod_dccrad_ver}/scripts
export FIXbufr=${DECDROOT}/decod_shared/fix

export HOMEobsproc_satingest=${HOMEobsproc_satingest:\
-${NWROOT:?}/obsproc_satingest.${obsproc_satingest_ver:?\
"###FATAL ERROR \"obsproc_satingest_ver\" is not set"}}

export HOMEobsproc_shared_bufr_cword=${HOMEobsproc_shared_bufr_cword:\
-$NWROOT/obsproc_shared/bufr_cword.${obsproc_shared_bufr_cword_ver:?\
"###FATAL ERROR \"obsproc_shared_bufr_cword_ver\" is not set"}}

export TRANJBush=${HOMEobsproc_satingest}/ush/bufr_tranjb.sh
export EXECobsproc_satingest=${HOMEobsproc_satingest}/exec
export USHobsproc_shared_bufr_cword=${HOMEobsproc_shared_bufr_cword}/ush
export EXECobsproc_shared_bufr_cword=${HOMEobsproc_shared_bufr_cword}/exec

#############################################################
# Specify the maximum age (in minutes) of files to be processed.
# Raw files older than this limit will not be processed but will
# be listed in the history files as though they were processed.
# This prevents trying to process too many old data files
# following an extended outage.
#############################################################

export mxagemin=30

#############################################################
# Specify the maximum number of processors to use
#############################################################

export mxprocs=8

#############################################################
# SENDCOM  - Send output to $DCOM_ROOT
#############################################################

export SENDCOM=YES

#############################################################
# Initialize the working directory
#############################################################

setpdy.sh
. PDY

env

#############################################################
# Distribute the stations to be processed amongst the available processors
#############################################################
 
msg="CANADIAN RADAR PROCESSING HAS BEGUN on \
`hostname`"
postmsg "$jlogfile" "$msg"

set -x

ls $RAWDATA > stations_list
total_stns=`ls $RAWDATA | wc -l`

listperproc=`expr $total_stns / $mxprocs`
echo "List per processor = $listperproc"

msg="Total number of stations is $total_stns - each parallel task will process\
 $listperproc stations"
postmsg "$jlogfile" "$msg"

cal_total_stns=`expr $listperproc \* $mxprocs`
nstns=0
while [ "$nstns" -lt "$total_stns" ]
do
    nstns=`expr $nstns + 1`
    if [ $nstns -lt $cal_total_stns ]
    then
	procn=`expr \( $nstns - 1 \) / $listperproc + 1`
    else
	procn=`expr $nstns - $cal_total_stns + 1`
    fi
    read x
    echo $x >> list_$procn
    echo nstns= $nstns
done < stations_list

tlist=`ls list*`

nlists=0
for tl in $tlist
do
    nlists=`expr $nlists + 1`
    echo `cat $tl` > file.station_$nlists
done

## Create dummy processes if the number of processors actually required
## is less than the alloted number of processors

numlist=`ls list* | wc -l`

while [ "$numlist" -lt "$mxprocs" ] 
do
   numlist=`expr $numlist + 1`
   touch list_$numlist
done

##
mlists=0
for im in `ls list_*`
do
    mlists=`expr $mlists + 1`
    echo $mlists >> number_of_processors
done

export n_proc=`cat number_of_processors`
echo $n_proc

#############################################################
# Create individual poe scripts
#############################################################

# poe hostname

export nproc=$n_proc
            
DATA_TMP=$DATA
for proc in $nproc
do
    cat << EOF > poescript$proc

    hostname
   # DATA_TMP=$DATA

    export DATA=\$DATA_TMP/TASK$proc
    mkdir -p \$DATA

    cd \$DATA

    export DBNBUFRT=120
    export DBNROOT=\$DATA
    mkdir -p \$DBNROOT/tmp

    cp \$DATA_TMP/file.station_${proc} .

    sh $SCRIPTSdccrad/exradar_canrad.sh.ecf $proc
EOF
done

#############################################################
# Create a master poescript
#############################################################

for proc in $nproc
do
    chmod 775 ./poescript$proc
    if [ $proc -eq 1 ]
    then
       echo "./poescript$proc" > poescript
    else
       echo "./poescript$proc" >> poescript
    fi
done

chmod 775 poescript

#############################################################
# Execute the master poescript
#############################################################

export MP_PGMMODEL=mpmd
export MP_CMDFILE=poescript
export MP_LABELIO=NO

msg="Begin poe"
postmsg "$jlogfile" "$msg"

# mpirun.lsf -cmdfile poescript
# mpirun -n 8 ./poescript
mpirun -n 8 cfp ./poescript


echo "Ending Poe  : `date`"

export err=$?; err_chk


#############################################################
# Concatenate the BUFR output files from each process into a single BUFR file
#############################################################

if test "$SENDCOM" = 'YES'
then
    for proc in $nproc
    do
        if [ -s $DATA/TASK$proc/bufrout.task$proc ]; then
            cat $DATA/TASK$proc/bufrout.task$proc >> bufrout_total.stations
        else
            msg="***WARNING: COMBINED STATION BUFR FILE MAY BE INCOMPLETE***"
            postmsg "$jlogfile" "$msg"
        fi
    done
fi

#############################################################
# Ingest the BUFR file into the database
#############################################################

# JY outDir=$DCOM_ROOT/canrad_logs
# mkdir -p $outDir

if [ ! -s bufrout_total.stations ]
then

   msg1="***WARNING: COMBINED STATION BUFR FILE NOT FOUND, TRANJB DID NOT"
   msg2="RUN - NO CANADIAN RADAR DATA PROCESSED"
   postmsg "$jlogfile" "$msg1 msg2"

else

   set +x
   echo
   echo "--------------------------------------------------------------------"
   echo "START TRANJB TO PUT BUFR DATA INTO TANKS IN $DCOM_ROOT"
   echo "--------------------------------------------------------------------"
   echo
   set -x

   export SUBDATE_CHECK=NO

   # export log=$DCOM_ROOT/tranjb_canrad.out
   export log=$HIST/tranjb_canrad.out

   $TRANJBush $DCOM_ROOT ./bufrout_total.stations
   err=$?
   if [ $err -eq 0 ]
   then
      msg="TRANJB completed normally"
   else
      msg="**ERROR: TRANJB FAILED with r.c. $err - NO CANADIAN RADAR DATA PROCESSED"
   fi
   postmsg "$jlogfile" "$msg"

fi

#############################################################
# Close up shop
#############################################################

date

msg="CANADIAN RADAR PROCESSING HAS COMPLETED NORMALLY!"
postmsg "$jlogfile" "$msg"

# JY - add following line for debug
cat $jlogfile

if [ `find $DATA -name 'core' -exec ls -1 {} \; | wc -l` -gt 0 ]
then
    echo "core file exists - preserving $DATA"
else
    cd $DATAROOT
    #cd $TMPNWDIR
    rm -rf $DATA
fi