#
#   10/2023:   Public Domain Wesley Ebisuzaki
#
# CORe Postprocessing .. reads COMIN/COMINp6 and saves to $COMOUTpost
#
# 1. run ncep post on ensmean
# 2. make flx files for ensemble members and ensmean
# 3. create flx stats file
# 4. run spost (simple post) on ensemble members and ensmean
# 5. create spost stats file
# 6. copy files that are going to be archived to COMOUTpost
# 7. make *.idx files (for nomads)
#
# Runs in 6 hour chunks HR=00, 06, 12, 18 from COMIN and COMINp6
# saves data in COMOUTpost
# Note: some analyses are outside of the current month
#  ex (analysis) date == Jan 31, 18Z, uses observations from 15Z to 21Z of Jan 31.
#     the CORe analysis is based on the 1st guess, so the output pgb files
#        will be Jan 31 21Z and Feb 01 00Z
# Note: Want pgb files for individual ensemble members; however, ncep_post is
#       is a big program. So spost was adopted.  Spost reads the hybrid sigma-pressure
#       grib file, does vertical interpolation to constant pressure/sigma/Z surfaces
#       and writes out the results in grib2 format.  Spost does thes is a few seconds
#       only using a single core.
#
# Requirements: 8 CPUs
#   ncep post uses MPI (8 cores)
#   mpiexec is used to run up to 8 scripts at one time
#     used to get around of NCO's edict against background jobs
#
#    export OMP_NUM_THREADS=1
#    mpiexec -n 1 \
#     job1.sh : job2.sh : job3.sh : job4.sh \
#     job5.sh : job6.sh : job7.sh : job8.sh
#
#  Some programs using wgrib2 or libwgrib2 can run faster using OpenMP
#
# Usage:
#
#      excore_post.sh $date
#             date code is the data assimilation time, The date code must
#             be a multiple of 6.
#             Produeces analyses for $date+3hr and $date+6hr
#
# scripts called:
#       config.model.sh        define CENTER, SUBCENTER PROCESSID
#       core_flux.sh           make flux files from bfg nemsio files
#       core_ncep_post.sh      run ncep post
#       core_spost.sh          run simple post
#       repeat.sh              repeat a command line with one list of options
#       repeat2x.sh            repeat a command line with two lists of options
#
# executables used:
#       core_gfsnemsio2grb, core_gmerge, ncep_post, core_spost, core_uv2sv, $NDATE (ndate), wgrib2
#
# libraries used:
#       libwgrib2 (used by core_gfsnemsio2grb, core_spost, core_uv2sv)
#       ncep grib, nemsio, spectral, ipolates
#
#
#  if $SENDCOM = 'YES'
#    write output to $COMOUTpost
#  else
#    write output to $DATA/COMOUTpost
#  fi
#
#  $RESTART = 0  run complete job
#                run with auto restart (default)
#

# v1.0    10/2023 Wesley Ebisuzaki

set -aux
# for ifort buffered io
export FORT_BUFFERED=true
export OMP_NUM_THREADS=1

# make file descriptor 3 point to stderr (FD2)
exec 3>&2

. $USHcore/setup_fatal.sh
export pgm=$0
export errfile=${errfile:-$DATA/errfile}
export pgmout=${pgmout:-$errfile}

[ "$COMIN" = '' ] && fatal "$0: undefined \$COMIN"
[ "$COMINp6" = '' ] && fatal "$0: undefined \$COMINp6"
[ "$COMOUTpost" = '' ] && fatal "$0: undefined \$COMOUTpost"
[ "$date" = '' ] && fatal "$0: undefined \$date"
[ "$datep3" = '' ] && fatal "$0: undefined \$datep3"
[ "$datep6" = '' ] && fatal "$0: undefined \$datep6"

export WGRIB2=${WGRIB2:-$EXECcore/core_wgrib2}
[ ! -f "$WGRIB2" ] && fatal "$0: \$WGRIB2 missing $WGRIB2"

if [ "$SENDCOM" = 'NO' ] ; then
   export COMOUTpost=$DATA/COMOUTpost
fi

if [ "$SENDDBN" = "YES" ] ; then
   dbn="$DBNROOT/bin/dbn_alert"
else
   dbn="echo >>dbn_alert: "
fi

export job=${job:-JCORE_POST}

# read in CENTER, SUBCENTER and PROCESSID
. $PARMcore/config.model.sh

mkdir -p $COMOUTpost/nceppost $COMOUTpost/flx $COMOUTpost/flx_stats $COMOUTpost/spost $COMOUTpost/spost_stats
[ $? -ne 0 ] && fatal "$0: mkdir $COMOUTpost/nceppost $COMOUTpost/flx $COMOUTpost/flx_stats \
  $COMOUTpost/spost $COMOUTpost/spost_stats"
mkdir -p $COMOUTpost/diag $COMOUTpost/bfg_grb2 $COMOUTpost/bfg_nemsio $COMOUTpost/ensmean
[ $? -ne 0 ] && fatal "$0: mkdir -p $COMOUTpost/diag $COMOUTpost/bfg_grb2 \
  $COMOUTpost/bfg_nemsio $COMOUTpost/ensmean"

# RESTART = 0 .. start at step 0
# RESTART = 1 .. normal, read RESTART_file to find next stp

export RESTART=${RESTART:-1}
export RESTART_status=$COMOUTpost/post.job_step
echo "$0: RESTART=$RESTART RESTART-file=$RESTART_status"

[ $RESTART -le 0 ] && echo "0" > $RESTART_status
[ ! -f $RESTART_status ] && echo "0" > $RESTART_status

n=`cat $RESTART_status | cut -c1-4`
[ $? -ne 0 ] && fatal "$0: reading file $RESTART_status"
if [ "$n" = 'done' ] ; then
  echo "$0: job was previously completed, not rerunning"
  exit 0
fi

export RESTART_step=`expr $n + 0`
[ $? -ne 0 ] && fatal "$0: $RESTART_status is ($n) and not an integer"
echo "$0: RESTART_step=$RESTART_step"

HH=`echo $date | cut -c9-10`
HHp3=`echo $datep3 | cut -c9-10`
HHp6=`echo $datep6 | cut -c9-10`

[ ! -f $EXECcore/core_gfsnemsio2grb ] && fatal "$0: missing $EXECcore/core_gfsnemsio2grb"


#---------------- Start Processing --------------------------------

if [ $RESTART_step -lt 1 ] ; then
#   NCEP-POST

#   Run ncep post on two ensemble-mean analyses and ensemble-mean 1st guess 
#   core_ncep_post.sh is setup to use 8 cores

  $USHcore/core_ncep_post.sh $COMINp6/core.t${HHp6}z.sfg.f003.ensmean $COMINp6/core.t${HHp6}z.bfg.f003.ensmean \
    $COMOUTpost/nceppost/core.t${HHp3}z.pgb.ensmean.anl.grib2 
  [ $? -ne 0 ] && fatal "$0: core_ncep_post.sh failed"

  $USHcore/core_ncep_post.sh $COMINp6/core.t${HHp6}z.sfg.f006.ensmean $COMINp6/core.t${HHp6}z.bfg.f006.ensmean \
    $COMOUTpost/nceppost/core.t${HHp6}z.pgb.ensmean.anl.grib2
  [ $? -ne 0 ] && fatal "$0: core_ncep_post.sh failed"

  $USHcore/core_ncep_post.sh $COMINp6/core.t${HHp6}z.sfg.f009.ensmean $COMINp6/core.t${HHp6}z.bfg.f009.ensmean \
    $COMOUTpost/nceppost/core.t${HHp6}z.pgb.ensmean.f003.grib2
  [ $? -ne 0 ] && fatal "$0: core_ncep_post.sh failed"

  # make index files
  for f in $COMOUTpost/nceppost/*.anl.grib2 $COMOUTpost/nceppost/*.f003.grib2
  do
    echo "make idx of $f"
    $WGRIB2 $f >$f.idx
    [ $? -ne 0 ] && fatal "$0: bad wgrib2 of $f"
  done

  $USHcore/repeat.sh "$HHp3 $HHp6"  $dbn MODEL CORE_PGB $job $COMOUTpost/nceppost/core.t%%z.pgb.ensmean.anl.grib2
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 1"

  $USHcore/repeat.sh "$HHp3 $HHp6"  $dbn MODEL CORE_PGB $job $COMOUTpost/nceppost/core.t%%z.pgb.ensmean.anl.grib2.idx
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 2"

  $dbn MODEL CORE_PGB    $job $COMOUTpost/nceppost/core.t${HHp6}z.pgb.ensmean.f003.grib2
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 3"

  $dbn MODEL CORE_PGB    $job $COMOUTpost/nceppost/core.t${HHp6}z.pgb.ensmean.f003.grib2.idx
   [ $? -ne 0 ] && fatal "$0: bad dbn alert 4"

  echo 1 > $RESTART_status || fatal "$0: write to $RESTART_status"

fi

# names when running 8 processes

export names1="`seq -fmem%03.0f 1 10` ensmean"
export names2="`seq -fmem%03.0f 11 20`"
export names3="`seq -fmem%03.0f 21 30`"
export names4="`seq -fmem%03.0f 31 40`"
export names5="`seq -fmem%03.0f 41 50`"
export names6="`seq -fmem%03.0f 51 60`"
export names7="`seq -fmem%03.0f 61 70`"
export names8="`seq -fmem%03.0f 71 80`"


# names when running 4 processes

export names1_4="`seq -fmem%03.0f 1 20` ensmean"
export names2_4="`seq -fmem%03.0f 21 40`"
export names3_4="`seq -fmem%03.0f 41 60`"
export names4_4="`seq -fmem%03.0f 61 80`"

# names 
export names_all="`seq -fmem%03.0f 1 80` ensmean"

if [ $RESTART_step -lt 2 ] ; then

#   FLX

  echo "$0: computing flux pwd=`pwd`"

  # make custom template needed by core_flux.sh
  $WGRIB2 $FIXcore/template_512.grb -set center $CENTER -set subcenter $SUBCENTER \
    -set analysis_or_forecast_process_id $PROCESSID -grib ./template_512.grb

  export OMP_NUM_THREADS=1

# compute flx for each ensemble member and ensmean
  mpiexec 2>&3 -n 1 \
    ${USHcore}/save_stdout.sh ${DATA}/core_flux_1.out $USHcore/core_flux.sh '$names1' $datep6 $COMINp6 . : \
    ${USHcore}/save_stdout.sh ${DATA}/core_flux_2.out $USHcore/core_flux.sh '$names2' $datep6 $COMINp6 . : \
    ${USHcore}/save_stdout.sh ${DATA}/core_flux_3.out $USHcore/core_flux.sh '$names3' $datep6 $COMINp6 . : \
    ${USHcore}/save_stdout.sh ${DATA}/core_flux_4.out $USHcore/core_flux.sh '$names4' $datep6 $COMINp6 . : \
    ${USHcore}/save_stdout.sh ${DATA}/core_flux_5.out $USHcore/core_flux.sh '$names5' $datep6 $COMINp6 . : \
    ${USHcore}/save_stdout.sh ${DATA}/core_flux_6.out $USHcore/core_flux.sh '$names6' $datep6 $COMINp6 . : \
    ${USHcore}/save_stdout.sh ${DATA}/core_flux_7.out $USHcore/core_flux.sh '$names7' $datep6 $COMINp6 . : \
    ${USHcore}/save_stdout.sh ${DATA}/core_flux_8.out $USHcore/core_flux.sh '$names8' $datep6 $COMINp6 .

# save ensemble members and ens-mean
  mpiexec 2>&3 -n 1 \
    $USHcore/save_stdout.sh $DATA/core_flx_1.out $USHcore/repeat2x.sh "$datep3" '$names_all' cpreq flx_%1_%2.grb $COMOUTpost/flx/core.t${HHp3}z.flx.%2.grib2 : \
    $USHcore/save_stdout.sh $DATA/core_flx_2.out $USHcore/repeat2x.sh "$datep6" '$names_all' cpreq flx_%1_%2.grb $COMOUTpost/flx/core.t${HHp6}z.flx.%2.grib2 : \
    $USHcore/save_stdout.sh $DATA/core_flx_3.out $USHcore/repeat2x.sh '03' '$names_all' cpreq bfg_${datep6}_fhr%1_%2.grb $COMOUTpost/bfg_grb2/core.t${HHp6}z.bfg.f0%1.%2.grib2 : \
    $USHcore/save_stdout.sh $DATA/core_flx_4.out $USHcore/repeat2x.sh '06' '$names_all' cpreq bfg_${datep6}_fhr%1_%2.grb $COMOUTpost/bfg_grb2/core.t${HHp6}z.bfg.f0%1.%2.grib2  : \
    $USHcore/save_stdout.sh $DATA/core_flx_5.out $USHcore/repeat2x.sh '09' '$names_all' cpreq bfg_${datep6}_fhr%1_%2.grb $COMOUTpost/bfg_grb2/core.t${HHp6}z.bfg.f0%1.%2.grib2


# compute flx_stats

  export OMP_NUM_THREADS=7

  $EXECcore/core_gmerge - `seq -fflx_${datep3}_mem%03.0f.grb 1 80` | $WGRIB2 - -set_grib_type c3 \
    -ens_processing $COMOUTpost/flx_stats/core.t${HHp3}z.flx.ensstats.grib2 1 >/dev/null
  $EXECcore/core_gmerge - `seq -fflx_${datep6}_mem%03.0f.grb 1 80` | $WGRIB2 - -set_grib_type c3 \
    -ens_processing $COMOUTpost/flx_stats/core.t${HHp6}z.flx.ensstats.grib2 1 >/dev/null

# check to see whether all files have right number of records

  n=`$WGRIB2 $COMOUTpost/flx_stats/core.t${HHp3}z.flx.ensstats.grib2 | wc -l`
  [ "$n" -ne 1041 ] && fatal "$0: flx-stats failed"
  n=`$WGRIB2 $COMOUTpost/flx_stats/core.t${HHp6}z.flx.ensstats.grib2 | wc -l`
  [ "$n" -ne 1041 ] && fatal "$0: flx-stats failed"

#  make index files
  for f in $COMOUTpost/flx_stats/*ensstats.grib2 $COMOUTpost/flx/core.*flx.mem???.grib2 $COMOUTpost/flx/core.*flx.ensmean.grib2
  do
    echo "make index for $f"
    $WGRIB2 $f >$f.idx
    [ $? -ne 0 ] && fatal "$0: bad wgrib2 of $f"
  done

# dbn alerts
  $USHcore/repeat2x.sh "$HHp3 $HHp6" "$names_all"  $dbn MODEL CORE_FLX  $job $COMOUTpost/flx/core.t%1z.flx.%2.grib2
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 5"
  $USHcore/repeat2x.sh "$HHp3 $HHp6" "$names_all"  $dbn MODEL CORE_FLX  $job $COMOUTpost/flx/core.t%1z.flx.%2.grib2.idx
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 6"

  $USHcore/repeat.sh "$HHp3 $HHp6"  $dbn MODEL CORE_FLX  $job $COMOUTpost/flx_stats/core.t%%z.flx.ensstats.grib2
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 7"
  $USHcore/repeat.sh "$HHp3 $HHp6"  $dbn MODEL CORE_FLX  $job $COMOUTpost/flx_stats/core.t%%z.flx.ensstats.grib2.idx
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 8"

  echo 2 > $RESTART_status || fatal "$0: write to $RESTART_status"

fi

if [ $RESTART_step -lt 3 ] ; then
# SPOST

# add center, subcenter and process_id to template files

# templates needed by spost.sh
  if [ ! -s ./template_512.grb ] ; then
    $WGRIB2 $FIXcore/template_512.grb -set center $CENTER -set subcenter $SUBCENTER \
      -set analysis_or_forecast_process_id $PROCESSID -grib ./template_512.grb
  fi
  if [ ! -s ./template.grb ] ; then
    $WGRIB2 $FIXcore/template_512.grb -set center $CENTER -set subcenter $SUBCENTER \
     -set analysis_or_forecast_process_id $PROCESSID -grib ./template.grb
  fi


  export OMP_NUM_THREADS=1

# compute spost for all ensemble members and ensmean
  mpiexec 2>&3 -n 1 \
    $USHcore/save_stdout.sh $DATA/core_spost_1.out $USHcore/repeat.sh '$names1_4' $USHcore/core_spost.sh $COMINp6/core.t${HHp6}z.sfg.f003.%% pgb_${datep3}_%%.grb : \
    $USHcore/save_stdout.sh $DATA/core_spost_2.out $USHcore/repeat.sh '$names2_4' $USHcore/core_spost.sh $COMINp6/core.t${HHp6}z.sfg.f003.%% pgb_${datep3}_%%.grb : \
    $USHcore/save_stdout.sh $DATA/core_spost_3.out $USHcore/repeat.sh '$names3_4' $USHcore/core_spost.sh $COMINp6/core.t${HHp6}z.sfg.f003.%% pgb_${datep3}_%%.grb : \
    $USHcore/save_stdout.sh $DATA/core_spost_4.out $USHcore/repeat.sh '$names4_4' $USHcore/core_spost.sh $COMINp6/core.t${HHp6}z.sfg.f003.%% pgb_${datep3}_%%.grb : \
    $USHcore/save_stdout.sh $DATA/core_spost_5.out $USHcore/repeat.sh '$names1_4' $USHcore/core_spost.sh $COMINp6/core.t${HHp6}z.sfg.f006.%% pgb_${datep6}_%%.grb : \
    $USHcore/save_stdout.sh $DATA/core_spost_6.out $USHcore/repeat.sh '$names2_4' $USHcore/core_spost.sh $COMINp6/core.t${HHp6}z.sfg.f006.%% pgb_${datep6}_%%.grb : \
    $USHcore/save_stdout.sh $DATA/core_spost_7.out $USHcore/repeat.sh '$names3_4' $USHcore/core_spost.sh $COMINp6/core.t${HHp6}z.sfg.f006.%% pgb_${datep6}_%%.grb : \
    $USHcore/save_stdout.sh $DATA/core_spost_8.out $USHcore/repeat.sh '$names4_4' $USHcore/core_spost.sh $COMINp6/core.t${HHp6}z.sfg.f006.%% pgb_${datep6}_%%.grb 

# save spost to COMOUTpost
  $USHcore/repeat2x.sh "$datep3" '$names_all' cpreq -fp pgb_%1_%2.grb $COMOUTpost/spost/core.t${HHp3}z.spgb.%2.anl.grib2
  $USHcore/repeat2x.sh "$datep6" '$names_all' cpreq -fp pgb_%1_%2.grb $COMOUTpost/spost/core.t${HHp6}z.spgb.%2.anl.grib2
  
# spost-stats, wgrib2 -ens_processing can use all cores

  export OMP_NUM_THREADS=7

# compute spost ensemble stats

  $EXECcore/core_gmerge - `seq -fpgb_${datep3}_mem%03.0f.grb 1 80` | $WGRIB2 - -set_grib_type c3 \
    -ens_processing $COMOUTpost/spost_stats/core.t${HHp3}z.spgb.ensstats.grib2 1 >/dev/null
  $EXECcore/core_gmerge - `seq -fpgb_${datep6}_mem%03.0f.grb 1 80` | $WGRIB2 - -set_grib_type c3 \
    -ens_processing $COMOUTpost/spost_stats/core.t${HHp6}z.spgb.ensstats.grib2 1 >/dev/null

# check to see all files have right number of records

  n=`$WGRIB2 $COMOUTpost/spost_stats/core.t${HHp3}z.spgb.ensstats.grib2 | wc -l`
  [ "$n" -ne 729 ] && fatal "$0: creating spost-stats failed"
  n=`$WGRIB2 $COMOUTpost/spost_stats/core.t${HHp6}z.spgb.ensstats.grib2 | wc -l`
  [ "$n" -ne 729 ] && fatal "$0: creating spost-stats failed"

# mk idx files
for f in $COMOUTpost/spost_stats/*ensstats.grib2 $COMOUTpost/spost/*anl.grib2
do
  echo "mk idx for $f"
  $WGRIB2 $f >$f.idx
  [ $? -ne 0 ] && fatal "$0: wgrib2 $f"
done

# dbn alerts


  $USHcore/repeat2x.sh "$HHp3 $HHp6" "$names_all"  $dbn MODEL CORE_SPGB  $job $COMOUTpost/spost/core.t%1z.spgb.%2.anl.grib2
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 9"
  $USHcore/repeat2x.sh "$HHp3 $HHp6" "$names_all"  $dbn MODEL CORE_SPGB  $job $COMOUTpost/spost/core.t%1z.spgb.%2.anl.grib2.idx
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 10"

  $USHcore/repeat.sh "$HHp3 $HHp6" $dbn MODEL CORE_SPGB  $job $COMOUTpost/spost_stats/core.t%%z.spgb.ensstats.grib2
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 11"
  $USHcore/repeat.sh "$HHp3 $HHp6" $dbn MODEL CORE_SPGB  $job $COMOUTpost/spost_stats/core.t%%z.spgb.ensstats.grib2.idx
  [ $? -ne 0 ] && fatal "$0: bad dbn alert 12"

  echo 3 >$RESTART_status || fatal "$0: write to $RESTART_status"
fi

export OMP_NUM_THREADS=1

$USHcore/repeat.sh "anavinfo convinfo hybens.info ozinfo satinfo scaninfo gsistats.control gsistats.ensmean" \
  cpreq $COMIN/core.t${HH}z.%%  $COMOUTpost/diag/

cycle=`echo $date | cut -c9-10`
PREINP=core.t${cycle}z.

cpreq -fp $COMIN/${PREINP}abias $COMOUTpost/diag/core.t${HH}z.abias
cpreq -fp $COMIN/${PREINP}abias_pc $COMOUTpost/diag/core.t${HH}z.abias_pc
cpreq -fp $COMIN/${PREINP}abias_air $COMOUTpost/diag/core.t${HH}z.abias_air

[ -f $COMIN/core.t${HH}z.config.info ] && cpreq -fp $COMIN/core.t${HH}z.config.info $COMOUTpost/diag/


warning=no
for f in $COMIN/core.t${HH}z.diag_conv*nc4
do
  ff=`basename $f`
  gzip -fc $f > $COMOUTpost/diag/$ff.gz
  # make sure  diag files are group=rstprod
  if [ "$CHGRP_RSTPROD" = 'YES' ]; then
     file=$COMOUTpost/diag/${ff}
     if [ -s "$file" ] ; then
        chgrp rstprod "$file"
        errch=$?
        if [ $errch -eq 0 ]; then
           chmod 640 "$file"
        else
           cp /dev/null "$file"
           warning=yes 
        fi 
     fi    
  fi
done

# ENSMEAN (NEMSIO)

for f in bfg.f003 bfg.f006 bfg.f009 sfg.f003 sfg.f006 sfg.f009
do
   file=core.t${HHp6}z.${f}.ensmean
   gzip -cf $COMINp6/$file  >$COMOUTpost/ensmean/${file}.gz 
   [ $? -ne 0 ] && fatal "$0: gzip failed for $file"
done


# ENS-NEMSIO

hr=`echo $datep6 | cut -c9-10`
if [ "$hr" = '00' ] ; then
   mkdir -p $COMOUTpost/ens_nemsio/
   for name in mem012 mem024 mem036 mem048 mem060 mem072
   do
       bfile=core.t${HHp6}z.bfg.f006.${name}
       sfile=core.t${HHp6}z.sfg.f006.${name}
       gzip -cf $COMINp6/$bfile > $COMOUTpost/ens_nemsio/${bfile}.gz
       gzip -cf $COMINp6/$sfile > $COMOUTpost/ens_nemsio/${sfile}.gz
   done
fi

# BFG_NEMSIO
export fhr='03 06 09'

mpiexec 2>&3 -n 1 \
  $USHcore/save_stdout.sh $DATA/core_nemsio_1.out $USHcore/repeat2x.sh '$names1' '$fhr' cpreq $COMINp6/core.t${HHp6}z.bfg.f0%2.%1 $COMOUTpost/bfg_nemsio/ : \
  $USHcore/save_stdout.sh $DATA/core_nemsio_2.out $USHcore/repeat2x.sh '$names2' '$fhr' cpreq $COMINp6/core.t${HHp6}z.bfg.f0%2.%1 $COMOUTpost/bfg_nemsio/ : \
  $USHcore/save_stdout.sh $DATA/core_nemsio_3.out $USHcore/repeat2x.sh '$names3' '$fhr' cpreq $COMINp6/core.t${HHp6}z.bfg.f0%2.%1 $COMOUTpost/bfg_nemsio/ : \
  $USHcore/save_stdout.sh $DATA/core_nemsio_4.out $USHcore/repeat2x.sh '$names4' '$fhr' cpreq $COMINp6/core.t${HHp6}z.bfg.f0%2.%1 $COMOUTpost/bfg_nemsio/ : \
  $USHcore/save_stdout.sh $DATA/core_nemsio_5.out $USHcore/repeat2x.sh '$names5' '$fhr' cpreq $COMINp6/core.t${HHp6}z.bfg.f0%2.%1 $COMOUTpost/bfg_nemsio/ : \
  $USHcore/save_stdout.sh $DATA/core_nemsio_6.out $USHcore/repeat2x.sh '$names6' '$fhr' cpreq $COMINp6/core.t${HHp6}z.bfg.f0%2.%1 $COMOUTpost/bfg_nemsio/ : \
  $USHcore/save_stdout.sh $DATA/core_nemsio_7.out $USHcore/repeat2x.sh '$names7' '$fhr' cpreq $COMINp6/core.t${HHp6}z.bfg.f0%2.%1 $COMOUTpost/bfg_nemsio/ : \
  $USHcore/save_stdout.sh $DATA/core_nemsio_8.out $USHcore/repeat2x.sh '$names8' '$fhr' cpreq $COMINp6/core.t${HHp6}z.bfg.f0%2.%1 $COMOUTpost/bfg_nemsio/


mpiexec 2>&3 -n 1 \
  $USHcore/save_stdout.sh $DATA/core_gzip_1.out $USHcore/repeat2x.sh '$names1' '$fhr' gzip -f $COMOUTpost/bfg_nemsio/core.t${HHp6}z.bfg.f0%2.%1 : \
  $USHcore/save_stdout.sh $DATA/core_gzip_2.out $USHcore/repeat2x.sh '$names2' '$fhr' gzip -f $COMOUTpost/bfg_nemsio/core.t${HHp6}z.bfg.f0%2.%1 : \
  $USHcore/save_stdout.sh $DATA/core_gzip_3.out $USHcore/repeat2x.sh '$names3' '$fhr' gzip -f $COMOUTpost/bfg_nemsio/core.t${HHp6}z.bfg.f0%2.%1 : \
  $USHcore/save_stdout.sh $DATA/core_gzip_4.out $USHcore/repeat2x.sh '$names4' '$fhr' gzip -f $COMOUTpost/bfg_nemsio/core.t${HHp6}z.bfg.f0%2.%1 : \
  $USHcore/save_stdout.sh $DATA/core_gzip_5.out $USHcore/repeat2x.sh '$names5' '$fhr' gzip -f $COMOUTpost/bfg_nemsio/core.t${HHp6}z.bfg.f0%2.%1 : \
  $USHcore/save_stdout.sh $DATA/core_gzip_6.out $USHcore/repeat2x.sh '$names6' '$fhr' gzip -f $COMOUTpost/bfg_nemsio/core.t${HHp6}z.bfg.f0%2.%1 : \
  $USHcore/save_stdout.sh $DATA/core_gzip_7.out $USHcore/repeat2x.sh '$names7' '$fhr' gzip -f $COMOUTpost/bfg_nemsio/core.t${HHp6}z.bfg.f0%2.%1 : \
  $USHcore/save_stdout.sh $DATA/core_gzip_8.out $USHcore/repeat2x.sh '$names8' '$fhr' gzip -f $COMOUTpost/bfg_nemsio/core.t${HHp6}z.bfg.f0%2.%1

# save logs
#   SAVE_LOGS_ARKV is a regular expression
#   if matches DDHH, then save
#   SAVE_LOGS_ARKV = ....         matches all DDHH, save all logs
#                  = 0100         save 00 cycle for 1st of month
#                  = ..00         save 00 cycle
#                  = ..(00|12)    save 00 and 12 cycle
# note ensda.out is always saved

SAVE_LOGS_ARKV=${SAVE_LOGS_ARKV:-'NO'}

export DDHH=`echo ${date} | cut -c7-10`
if [ `echo $DDHH | grep -cE "$SAVE_LOGS_ARKV"` -eq 1 ] ; then
   mkdir -p $COMOUTpost/logs/
   cpreq -pf $COMIN/logs/core.t${HH}z.logs.gz $COMOUTpost/logs/
fi

# save ensda.out if assim jobs saved the logs
if [ -f $COMIN/logs/core.t${HH}z.logs.gz ] ; then
   mkdir -p $COMOUTpost/logs/
   tar -xzf $COMIN/logs/core.t${HH}z.logs.gz ensda.out
   gzip ensda.out
   cpreq ensda.out.gz $COMOUTpost/logs/
fi

if [ "$warning" = 'yes' ] ; then
   fatal "change to rstprod failed, file erased"
fi

echo "done `date`" > $RESTART_status
[ $? -ne 0 ] && fatal "$0: bad write to $RESTART_status"

echo "$0: finished"
exit 0