#!/bin/sh

########################################
# RUNTIME STAT GENERATION
########################################

set -xa
# ###################################
# SET SHELL PROCESSING VARIABLES
# ###################################
export PS4='$SECONDS + ' 
date

# 
# obtain unique process id (pid) and make temp directories
#

export DATA=${DATA:-${DATAROOT:?}/${jobid:?}}
mkdir -p $DATA
cd $DATA

export pgmout="OUTPUT.${pid}"

export cycle=t${cyc}z 

############################################
# SENDCOM=YES--Copy output file to /com
# SENDECF=YES--Allow to talk back to ECF
############################################
export SENDWEB=${SENDWEB:-YES}
export SENDCOM=${SENDCOM:-NO}
export SENDDBN=${SENDDBN:-NO}
export SENDECF=${SENDECF:-YES}
export SENDEMAIL=${SENDEMAIL:-NO}
export send_alert=${send_alert:-YES}

#################################
# Set up the NET and RUN
#################################
export NET=logs
export RUN=runtime
export MODEL=runtime                 # runtime model

# Specify Execution Areas
# JY export HOMEruntime=${HOMEruntime:-$NWROOT/runtime.$runtime_ver}
export SCRruntime=${SCRruntime:-${HOMEruntime}/scripts}
export FIXruntime=${FIXruntime:-${HOMEruntime}/fix}
export USHruntime=${USHruntime:-${HOMEruntime}/ush}

###################################
# Run the setup script
###################################
setpdy.sh
. ./PDY

########################################
# Set up the input/output directory
########################################

# comment out some of the following -08/11 - JY
#if [ "${envir:?}" = "prod" ]
#then
#   export COMIN=${COMIN:-$(compath.py  ${envir}/${NET}/runtime/)${ecf_envir}}
    export COMIN=${COMIN:-$COMROOT/logs/runtime/${ecf_envir}}
#   export COMOUT=${COMOUT:-$(compath.py -o ${NET}/runtime/)${ecf_envir}}
    export COMOUT=${COMOUT:-$COMROOT/logs/runtime/${ecf_envir}}
    export COMOUTruntime=${COMOUTruntime:-$COMOUT/runtime}
    export COMOUTdaily=${COMOUTdaily:-$COMOUT/daily}
    export COMOUTinter=$COMOUT/interProcess
    export COMOUTstat=${COMOUTstat:-$COMOUT/stat}
    export COMINrtinfodirm2=${COMINrtinfodirm2:-$COMIN/rtinfodir.${PDYm2}}
    export COMINrtinfodirm1=${COMINrtinfodirm1:-$COMIN/rtinfodir.${PDYm1}}
    export COMINrtinfodirp1=${COMINrtinfodirp1:-$COMIN/rtinfodir.${PDYp1}}
    export COMOUTrtinfodir=${COMOUTrtinfodir:-$COMOUT/rtinfodir.${PDY}}
    #export COMIN_OUTPUT=${PBS_O_WORKDIR:-/lfs/h1/ops/${envir}/output}
    export COMIN_OUTPUT=${COMIN_OUTPUT:-/lfs/h1/ops/${envir}/output}
    echo "DEBUG:PBS_O_WORKDIR=$PBS_O_WORKDIR"
    #export DIR=${DIR:-$COMIN_OUTPUT}
#else
#    export COMINdaily=${COMINdaily:-$COMROOT/logs/runtime/daily}
#    export COMOUT=${COMOUT:-$COMROOT/logs/${envir}/runtime}
#    export COMOUTstat=${COMOUTstat:-$COMROOT/logs/${envir}/runtime/stat}
#    export COMINrtinfodirm2=${COMINrtinfodirm1:-$COMROOT/logs/${envir}/runtime/rtinfodir.${PDYm2}}
#    export COMINrtinfodirm1=${COMINrtinfodirm1:-$COMROOT/logs/${envir}/runtime/rtinfodir.${PDYm1}}
#    export COMINrtinfodirp1=${COMINrtinfodirp1:-$COMROOT/logs/${envir}/runtime/rtinfodir.${PDYp1}}
#    export COMOUTrtinfodir=${COMOUTrtinfodir:-$COMROOT/logs/${envir}/runtime/rtinfodir.${PDY}}
#fi
#export COMOUTWalltimeComp=${COMOUTWalltimeComp:-$COMROOT/logs/${envir}/runtime/WalltimeComp}
export COMOUTWalltimeComp=${COMOUTWalltimeComp:-$COMOUT/WalltimeComp}
mkdir -p -m 775 $COMOUT $COMOUTrtinfodir $COMOUTstat $COMOUTWalltimeComp $COMOUTinter
export DES=${DES:-${COMOUTstat}}
export USH=${USH:-${USHruntime}}
export SRC=${SRC:-${COMINdaily}}

env
########################################################
# Execute the script.
#list="/prod/primary/corn /prod/primary/00 /prod/primary/06 /prod/primary/12 /prod/primary/18 /prod/primary/climate /para/primary/corn /para/primary/00 /para/primary/06 /para/primary/12 /para/primary/18 /para/primary/climate"

list="/${envir}/primary/corn /${envir}/primary/00 /${envir}/primary/06 /${envir}/primary/12 /${envir}/primary/18 /${envir}/primary/climate"
rm -rf poescript

for family in $list; do 
  name=`echo $family|tr "/" "."`
  echo "${SCRruntime}/exruntime_stat_poe.sh $PDY $family > $DATA/runtime_stat_poe$name 2>&1" >> poescript
done
echo "${SCRruntime}/exruntime_WalltimeComp.sh $PDY ${envir} $COMOUTWalltimeComp > $DATA/runtime_WalltimeComp.${envir}.log 2>&1 " >> poescript

#echo "${SCRruntime}/exruntime_stat_poe.sh $PDY /prod/primary/00 > $DATA/runtime_stat_poe.prod00.log 2>&1 " >> poescript
#echo "${SCRruntime}/exruntime_stat_poe.sh $PDY /prod/primary/00 > $DATA/runtime_stat_poe.prod06.log 2>&1 " >> poescript
#echo "${SCRruntime}/exruntime_stat_poe.sh $PDY /prod12 > $DATA/runtime_stat_poe.prod12.log 2>&1 " >> poescript
#echo "${SCRruntime}/exruntime_stat_poe.sh $PDY /prod18 > $DATA/runtime_stat_poe.prod18.log 2>&1 " >> poescript
#echo "${SCRruntime}/exruntime_stat_poe.sh $PDY /climprod > $DATA/runtime_stat_poe.climprod.log 2>&1 " >> poescript
#echo "${SCRruntime}/exruntime_stat_poe.sh $PDY /para00 > $DATA/runtime_stat_poe.para00.log 2>&1 " >> poescript
#echo "${SCRruntime}/exruntime_stat_poe.sh $PDY /para06 > $DATA/runtime_stat_poe.para06.log 2>&1 " >> poescript
#echo "${SCRruntime}/exruntime_stat_poe.sh $PDY /para12 > $DATA/runtime_stat_poe.para12.log 2>&1 " >> poescript
#echo "${SCRruntime}/exruntime_stat_poe.sh $PDY /para18 > $DATA/runtime_stat_poe.para18.log 2>&1 " >> poescript
#echo "${SCRruntime}/exruntime_WalltimeComp.sh $PDY prod $COMOUTWalltimeComp > $DATA/runtime_WalltimeComp.prod.log 2>&1 " >> poescript
#echo "${SCRruntime}/exruntime_WalltimeComp.sh $PDY para $COMOUTWalltimeComp > $DATA/runtime_WalltimeComp.para.log 2>&1 " >> poescript

NTASK=`cat $DATA/poescript |wc -l`
chmod 775 $DATA/poescript
export MP_PGMMODEL=mpmd
export MP_CMDFILE=$DATA/poescript
 
#mpirun -report-bindings -tag-output $DATA/poescript
#mpirun cfp  $DATA/poescript
mpiexec -np $NTASK --cpu-bind core cfp $DATA/poescript

export err=$?;err_chk
########################################################

# JY may need to comment the following line about PARATEST, set in in ex script, - 08/11/2021
if [ $PARATEST = 'YES' ]; then SENDWEB=NO; fi

if [ $SENDWEB = 'YES' ]; then
if [ $time_tag = '1216' ] || [  $time_tag = '1916' ] || [  $time_tag = '2346' ]; then
max=1;
MAX_TIMES=3;
while [ $max -le $MAX_TIMES ];   do
########################################################
# check and email for list of long run jobs
  $USHruntime/email_long_run_jobs.sh
########################################################
  export err=$?;
  if [[ $err != 0 ]]; then 
#   err_chk; 
    echo "err=$err retrying again ($max try)...."
    let max=max+1;
    sleep 10;
  else
    break;
  fi
done

max=1;
MAX_TIMES=3;
while [ $max -le $MAX_TIMES ];   do
########################################################
# check and email for list of long run jobs
  $USHruntime/email_WalltimeComp_jobs.sh
########################################################
  export err=$?;
  if [[ $err != 0 ]]; then
#   err_chk;
    echo "err=$err retrying again ($max try)...."
    let max=max+1;
    sleep 10;
  else
    break;
  fi
done
fi

else
  echo "$SENDWEB is NO, do not send data to intra ...."
fi

########################################################
# clean the COMINrtinfodir, only keep the last +/- one days...
########################################################
ls -d $COMOUT/rtinfodir*|grep -v $COMINrtinfodirm2|grep -v $COMINrtinfodirm1|grep -v $COMOUTrtinfodir|grep -v $COMINrtinfodirp1 > $DATA/rtinfodir.rm.list
if [ $KEEPDATA != 'YES' ]; then
  echo "will clean up/remove the following rtinfodir ..."
  cat $DATA/rtinfodir.rm.list
  for dir in `cat $DATA/rtinfodir.rm.list`; do
    echo "rm -rf $dir "
    rm -rf $dir
  done
  rm -rf $DATA;
fi

date