#!/bin/ksh
# JOBSPROC_AIRNOW_DUMP - 24 Oct 2014
#  J-job script to run AIRNOW_DUMP processing (ozone dump & mini pb creation)
#
# Dump AIRNOW data and encode into its own mini-PREPBUFR file for use in
#  EPA air quality forecast model verification
#
# modification history:
# 10 Oct 2013 - cp'd prod JAIRNOW_DUMP ver of 17 Jun 2014 (15:52);
#               converted to run under vertical structure
#                bug fix (RUN=hourly model=aqm ; reverse of previous versions)
# 24 Oct 2014 - added printouts for obsproc_dump_ver & obsproc_prep_ver
#               added HOMEobsproc_prep assignment (w/ defaults)
#               set EXECPREP & EXECFIX to use HOMEobsproc_prep by default
# 17 Dec 2019 - modified to run on WCOSS ph3
# 19 Dec 2021 - modified to run on WCOSS2
#--
set -xa
date -u

echo ; echo "Starting job script JOBSPROC_AIRNOW_DUMP (`date -u`)"

###################################################################
# Set DEBUG LEVEL - modifies info in execution trace prompt string
###################################################################
export DEBUG_LEVEL=${DEBUG_LEVEL=1}
case $DEBUG_LEVEL in
  1) export PS4='+ ${SECONDS}s + ' ;;       # add seconds (this is typical case)
  0) export PS4='+ ';;                      # don't include seconds (useful if
                                            #  want to diff output from 2 runs)
  2) export PS4='+ ${SECONDS}s:L$LINENO + ';;         # add script line #
  3) export PS4='+ ${SECONDS}s:${0##*/}:L$LINENO + ';;# add script name & line #
  4) export PS4='+ ${0##*/}:L$LINENO + ';;  # script name & line (no seconds)
  *) echo "Unrecognized DEBUG_LEVEL.  Stay with current PS4 setting";;
esac

########################################
# Check that required variables are set
########################################
set +x
echo
echo cyc is ${cyc:?"###FATAL ERROR \"cyc\" is not set"}
echo envir is ${envir:?"###FATAL ERROR \"envir\" is not set"}
echo job is ${job:?"###FATAL ERROR \"job\" is not set"}
echo DATAROOT is ${DATAROOT:?"###FATAL ERROR \"DATAROOT\" is not set"}
echo
set -x

################################
# Print out version information
################################
set +x
echo
echo "####################################################################"
echo
echo "OBSPROC version is ${obsproc_ver:-not set}"
echo "BUFR_DUMP version used by this job, for dump and dumplist, is" \
     "${bufr_dump_ver:-not set}"
echo
echo "####################################################################"
echo
set -x

#####################################################################
# List modules loaded for this job
#####################################################################
set +x
echo
echo "####################################################################"
echo
      module list
echo
echo "####################################################################"
echo
set -x

##################################
# Specify NET, RUN and model Name
##################################
export obsNET=${obsNET:-obsproc}
export NET=${NET:-hourly}
# model was assigned to obsproc_airnow in ecflow
#export model=${model:-hourly}
export model=hourly
export RUN=${RUN:-aqm}  # Used in child scripts for file names

set +x
echo
echo obsNET is ${obsNET:?"###FATAL ERROR \"obsNET\" is not set"}
echo NET is ${NET:?"###FATAL ERROR \"NET\" is not set"}
echo model is ${model:?"###FATAL ERROR \"model\" is not set"}
echo RUN is ${RUN:?"###FATAL ERROR \"RUN\" is not set"}
echo
set -x

##############################
# Specify cycle and time-mark
##############################
export cycle=t${cyc}z
export tmmark=tm00

###############################
# Create new working directory
###############################
jobid=${jobid:-${job}.$(hostname -s).$$}
export DATA=${DATA:-$DATAROOT/${jobid}}
rm -rf $DATA
mkdir -p $DATA
cd $DATA
export pgmout=OUTPUT.$$
if [[ "$RUN_ENVIR" != nco ]]; then
  TMPDIR=$DATAROOT  # In the event TMPDIR is reset to a temporary LSF
                    #  directory when batch jobs are submitted, set it to
                    #  $DATAROOT so ensure it remains under the user's control
                    #  (normally needed only if child script dumpjb executed)
fi

######################################
# Set job log file for postmsg output
######################################
jlogfile=${jlogfile:-${DATA}/jlogfile}

#################################################
# SENDCOM      - Copy files to $COMOUT directory
#################################################
export SENDCOM=${SENDCOM:-YES}

##########################
# Specify Execution Areas
##########################

# -----------------------
# Paths for obsproc items
# -----------------------
export HOMEobsproc=${HOMEobsproc:\
-/${PACKAGEROOT:?}/${obsNET}/${obsproc_ver:?\
"###FATAL ERROR \"obsproc_ver\" is not set"}}

export SCRIPTSobsproc=${SCRIPTSobsproc:\
-$HOMEobsproc/scripts}

# ---------------------------------------------------------------------
# Paths for dump and bufr_dumplist items
# Default variables are pre-defined when the bufr_dump module is loaded
# ---------------------------------------------------------------------
export USHbufr_dump=${USHbufr_dump:-$HOMEbufr_dump/ush}
export FIXbufr_dump=${FIXbufr_dump:-$HOMEbufr_dump/fix}

# NOTE: path to dump executables defaults to $HOMEobsproc_dump/exec in dumpjb

# directory path to data dump script executed in model script
# -----------------------------------------------------------
export ushscript_dump=${ushscript_dump:-$USHbufr_dump}

# -----------------------------------------
# Paths for combined obsproc and dump items
# -----------------------------------------

# directory paths to Airnow dump re-processing executed in bufr_dump_obs
# ----------------------------------------------------------------------
export EXECPREP=${EXECPREP:-$HOMEobsproc/exec}
export FIXPREP=${FIXPREP:-$HOMEobsproc/fix}


# ---------------------------
# Paths for non-obsproc items
# ---------------------------

# TANK is root directory path to observational database used in dumpjb
# --------------------------------------------------------------------
export DCOMROOT=${DCOMROOT:-/lfs/h1/ops/prod/dcom}
export TANK=${TANK:-${DCOMROOT}}

#########################################################################
# Add some prod utilities to working directory
#########################################################################
echo "step ############# break ##############################" > ./break
cp $UTILROOT/ush/err_chk   .; chmod +x err_chk
cp $UTILROOT/ush/err_exit  .; chmod +x err_exit
cp $UTILROOT/ush/prep_step .; chmod +x prep_step
cp $UTILROOT/ush/postmsg   .; chmod +x postmsg
cp $UTILROOT/ush/setpdy.sh .; chmod +x setpdy.sh

##########################################
# Run setpdy and initialize PDY variables
##########################################
./setpdy.sh
. $DATA/PDY

DATA_save=$DATA

msg="Daily AIRNOW data dump and PREPBUFR processing has begun on `hostname`"
postmsg "$jlogfile" "$msg"

for pdy in $PDYm1 $PDYm5; do
  export pdy 
  export DATA=$DATA/$pdy
  mkdir -p $DATA
  cd $DATA
  cp $DATA_save/* $DATA

######################### 
# Define COM directories
#  NOTE: 
#   !! directory dates are defined by ** LowerCase pdy ** for these runs !!
#   !! model script exairnow_dump.sh.ecf keys on existence of LOWERCASE pdy !!
#########################
  export COMIN_ROOT=${COMIN_ROOT:-${COMROOT:-""}}

  if [ "$RUN_ENVIR" = nco ]; then
    export COMIN=$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${model}.${pdy})
    export COMOUT=$(compath.py -o ${obsNET}/${obsproc_ver}/${model}.${pdy})
    mkdir -m 775 -p $COMOUT
  else
    export COMIN=${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${model}.${pdy}

# COMOUT_ROOT for developers defaults to unique $DATA/com directory as root to
#  prevent job from writing into a non-unique directory (e.g.,
#  /gpfs/dell2/ptmp/$USER/com) that might already have a dump status file in it
#  from a previous checkout run (this could mess up some logic downstream)
# ----------------------------------------------------------------------------
    export COMOUT_ROOT=${COMOUT_ROOT:-${DATA}/com}
#    export COMOUT=${COMOUT:-${COMOUT_ROOT}/obsproc/${model}.${pdy}}
# ! Do not inherit value of COMOUT from previous run of pdy loop
    export COMOUT=${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${model}.${pdy}
    mkdir -m 755 -p $COMOUT
  fi # RUN_ENVIR = nco 

#########################################################
# Specify variables specific to this execution of script
#########################################################
# Set the execution environment for dumpjb script
#  - POE must be set to on if running poe (not currently supported)
# -- or -- BACK must be set to on if running background shells

###  export POE=on
  export BACK=on # "on" is default

  set +x ; echo "JAD: ---start of env list---" 
  env
  echo "JAD: ---end of env list---" ; echo ; set -x

#####################
# Execute the script
#####################
  $SCRIPTSobsproc/exairnow_dump.sh

  err_airnow_dump=$?
  [[ $err_airnow_dump -ne 0 ]] && $DATA/err_exit

  export DATA=$DATA_save
  cd $DATA

done # for pdy in PDYm1 PDYm5
unset pdy  # clear lowercase pdy so that model script uses UPPERCASE PDY value


#################
#################
## Run again in new format, putting d-1 and d-5 into today's /com directory
#################
#################
for tmmark in tm024 tm120; do

  export tmmark

  export DATA=$DATA/$tmmark
  mkdir -p $DATA
  cd $DATA

  cp $DATA_save/* $DATA

#########################
# Define COM directories
#  NOTE: 
#   !! directory dates are defined by ** UPPERCASE PDY ** for this run !!
#   !! model script exairnow_dump.sh.ecf keys on existence of LOWERCASE pdy !!
#########################
  export COMIN_ROOT=${COMIN_ROOT:-${COMROOT:-""}}

  if [ "$RUN_ENVIR" = nco ]; then
    export COMIN=$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${model}.${PDY})
    export COMOUT=$(compath.py -o ${obsNET}/${obsproc_ver}/${model}.${PDY})
    mkdir -m 775 -p $COMOUT
  else
    export COMIN=${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${model}.${PDY}

# COMOUT_ROOT for developers defaults to unique $DATA/com directory as root to
#  prevent job from writing into a non-unique directory (e.g.,
#  /gpfs/dell2/ptmp/$USER/com) that might already have a dump status file in it
#  from a previous checkout run (this could mess up some logic downstream)
# ----------------------------------------------------------------------------
    export COMOUT_ROOT=${COMOUT_ROOT:-${DATA}/com}
    export COMOUT=${COMOUT:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${model}.${PDY}}
#    export COMOUT=${COMOUT:-${COMOUT_ROOT}/${NET}/${envir}/${model}.${pdy}}
# ! Do not inherit value of COMOUT from previous run of pdy loop
    export COMOUT=${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${model}.${PDY}
    mkdir -m 755 -p $COMOUT
  fi # RUN_ENVIR = nco 

#########################################################
# Specify variables specific to this execution of script
#########################################################
# Set the execution environment for dumpjb script
#  - POE must be set to on if running poe (not currently supported)
# -- or -- BACK must be set to on if running background shells

###  export POE=on
  export BACK=on # "on" is default

  set +x ; echo "JAD: ---start of env list---" 
  env
  echo "JAD: ---end of env list---" ; echo ; set -x

#####################
# Execute the script
#####################
  $SCRIPTSobsproc/exairnow_dump.sh

  err_airnow_dump=$?
  [[ $err_airnow_dump -ne 0 ]] && $DATA/err_exit

  export DATA=$DATA_save
  cd $DATA

done # for tmmark in tm024 tm120


# GOOD RUN
set +x
echo " "
echo " ****** AIRNOW DUMP AND PREPBUFR PROCESSING COMPLETED NORMALLY"
echo " ****** AIRNOW DUMP AND PREPBUFR PROCESSING COMPLETED NORMALLY"
echo " ****** AIRNOW DUMP AND PREPBUFR PROCESSING COMPLETED NORMALLY"
echo " ****** AIRNOW DUMP AND PREPBUFR PROCESSING COMPLETED NORMALLY"
echo " "
set -x


# save standard output
cat allout
# rm allout

sleep 10

msg='ENDED NORMALLY.'
postmsg "$jlogfile" "$msg"

echo "$SITE `hostname`  --  `date -u`" > $COMOUT/where_${cycle}_${RUN}_dump_ran

> $COMOUT/obsproc_version_for_${cycle}_${RUN}_dump_run
[ -n "$obsproc_ver" ]  &&  \
 echo "OBSPROC version is $obsproc_ver" >> \
 $COMOUT/obsproc_version_for_${cycle}_${RUN}_dump_run
[ -n "$bufr_dump_ver" ]  &&  \
 echo "BUFR_DUMP version used by this job is $bufr_dump_ver" >> \
 $COMOUT/obsproc_version_for_${cycle}_${RUN}_dump_run

if [ "$KEEPDATA" != YES ]; then
   cd $DATAROOT
   rm -rf $DATA
fi

set -x ; echo ; echo "JAD: normal end of JOBSPROC_AIRNOW_DUMP ($(date))"
exit
#---end JOBSPROC_AIRNOW_DUMP