#!/bin/bash
# --------------------------------------------------------------------------- #
# matrix.go: Run matrix of regression tests on target machine.                #
#                                                                             #
# Remarks:                                                                    #
# - This version is set up for automatic w3_setenv script and for the         #
#   NOAA RDHPC systems. When using this for your own setup and                #
#   computer, please copy rather than modify.                                 #
#                                                                             #
#                                                      Hendrik L. Tolman      #
#                                                      August 2013            #
#                                                      December 2013          #
#                                                      April 2018             #
#                                                      March 2020             #
#                                                                             #
#    Copyright 2013 National Weather Service (NWS),                           #
#       National Oceanic and Atmospheric Administration.  All rights          #
#       reserved.  WAVEWATCH III is a trademark of the NWS.                   #
#       No unauthorized use without permission.                               #
#                                                                             #
# --------------------------------------------------------------------------- #
# 0. Environment file

  source $(dirname $0)/../../model/bin/w3_setenv
  main_dir=$WWATCH3_DIR
  temp_dir=$WWATCH3_TMP
  source=$WWATCH3_SOURCE
  list=$WWATCH3_LIST

  echo "Main directory    : $main_dir"
  echo "Scratch directory : $temp_dir"
  echo "Save source codes : $source"
  echo "Save listings     : $list"

# Compiler option. Choose appropriate compiler and set cmplOption to
# y if using for the first time or using a different compiler

  cmplr=intel
  export cmplOption='y'

# Set batchq queue, choose modules and other custom variables to fit system and 
# to define headers etc (default to original version if empty)
  iss4=`hostname | grep s4-submit`
  if [ $iss4 ]
  then
    cmplr='s4.intel'
    batchq='slurm'
    hpcstackpath='/data/prod/hpc-stack/modulefiles/stack'
    hpcstackversion='hpc/1.1.0'
    modcomp='hpc-intel/18.0.4'
    modmpi='hpc-impi/18.0.4'
    modnetcdf='netcdf/4.7.4'
    metispath='/data/users/dhuber/save/parmetis-4.0.3'
    modjasper='jasper/2.0.25'
    modzlib='zlib/1.2.11'
    modpng='png/1.6.35'
    modhdf5='hdf5/1.10.6'
    modbacio='bacio/2.4.1'
    modg2='g2/3.4.1'
    modw3emc='w3emc/2.9.2'
    modesmf='esmf/8_1_1'
  else
    echo This matrix setup script is only intended for S4!
    exit 123
  fi

# 1. Set up
   
  export  np='24'     #number of mpi tasks
  export  npl='140'   #number of mpi tasks for ufs applications and large setups
  export  npl1='100'  #number of mpi tasks for ufs/large setups (b4b check)
  export  nr='4'      #number of mpi tasks for hybrid
  export nth='6'      #number of threads
  export nth1='4'     #number of threads (b4b check)

# 1.a Computer/ user dependent set up

  echo '#!/bin/sh --login'                                   > matrix.head
  echo ' '                                                  >> matrix.head


    echo "#SBATCH -n ${np}"                                 >> matrix.head
    echo "##SBATCH --cpus-per-task=${nth}"                  >> matrix.head
    echo '#SBATCH -q s4'                                    >> matrix.head
    echo '#SBATCH -t 06:00:00'                              >> matrix.head
    echo '#SBATCH -A star'                                  >> matrix.head
    echo '#SBATCH -J ww3_regtest'                           >> matrix.head
    echo '#SBATCH -o matrix.out'                            >> matrix.head
    echo '#SBATCH -p s4'                                    >> matrix.head
  echo "  cd $(dirname $main_dir)/regtests"                 >> matrix.head
  echo ' '                                                  >> matrix.head

# Netcdf and Parmetis modules & variables
  echo "  module purge"                                     >> matrix.head
  echo "  module use $hpcstackpath"                         >> matrix.head
  echo "  module load license_intel/S4"                     >> matrix.head
  echo "  module load $hpcstackversion"                     >> matrix.head
  echo "  module load $modcomp"                             >> matrix.head
  echo "  module load $modmpi"                              >> matrix.head
  echo "  module load $modnetcdf"                           >> matrix.head
  echo "  module load $modjasper"                           >> matrix.head
  echo "  module load $modzlib"                             >> matrix.head
  echo "  module load $modpng"                              >> matrix.head
  echo "  module load $modhdf5"                             >> matrix.head
  echo "  module load $modbacio"                            >> matrix.head
  echo "  module load $modg2"                               >> matrix.head
  echo "  module load $modw3emc"                            >> matrix.head
  echo "  module load $modesmf"                             >> matrix.head
  echo "  export WWATCH3_NETCDF=NC4"                        >> matrix.head
  echo '  export NETCDF_CONFIG=$NETCDF_ROOT/bin/nc-config'  >> matrix.head
  echo "  export METIS_PATH=${metispath}"                   >> matrix.head
  echo '  export JASPER_LIB=$JASPER_ROOT/lib64/libjasper.a' >> matrix.head
  echo '  export PNG_LIB=$PNG_ROOT/lib64/libpng.a'          >> matrix.head
  echo '  export Z_LIB=$ZLIB_ROOT/lib/libz.a'               >> matrix.head
  echo '  export ESMFMKFILE=$ESMF_LIB/esmf.mk'              >> matrix.head
  echo "  export WW3_PARCOMPN=4"                            >> matrix.head
  echo ' ' 

  export mpi='srun'

# Compile option
  if [ "$cmplOption" = 'y' ]
  then
     opt="-o all -c $cmplr -S -T"
  else
     opt="-o all -S"
  fi
# Batch queue option
  opt="-b $batchq $opt"

# Base run_test command line
  export rtst="./bin/run_test $opt"

  export  ww3='../model'

# 1.b Flags to do course selection - - - - - - - - - - - - - - - - - - - - - -
#     Addition selection by commenting out lines as below

  export       shrd='y' # Do shared architecture tests
  export       dist='y' # Do distributed architecture (MPI) tests
  export        omp='y' # Threaded (OpenMP) tests
  export       hybd='y' # Hybrid options

  export     prop1D='y' # 1-D propagation tests (ww3_tp1.X)
  export     prop2D='y' # 2-D propagation tests (ww3_tp2.X)
  export       time='y' # time linmited growth
  export      fetch='y' # fetch linmited growth
  export     hur1mg='y' # Hurricane with one moving grid
  export      shwtr='y' # shallow water tests
  export      unstr='y' # unstructured grid tests
  export      pdlib='y' # unstr with pdlib for domain decomposition and implicit solver
  export      smcgr='y' # SMC grid test
  export        rtd='y' # Rotated pole test
  export     mudice='y' # Mud/Ice and wave interaction tests 
  export     infgrv='y' # Second harmonic generation tests
  export       uost='y' # ww3_ts4 Unresolved Obstacles Source Term (UOST)
  export      assim='y' # Restart spectra update
  export      oasis='y' # Atmosphere, ocean, and ice coupling using OASIS
  export   calendar='y' # Calendar type
  export   confignc='y' # Configurable netCDF meta data (ww3_ounf)

  export    multi01='y' # mww3_test_01 (wetting and drying)
  export    multi02='y' # mww3_test_02 (basic two-way nesting test))
  export    multi03='y' # mww3_test_03 (three high and three low res grids).
  export    multi04='y' # mww3_test_04 (swell on sea mount and/or current)
  export    multi05='y' # mww3_test_05 (three-grid moving hurricane)
  export    multi06='y' # mww3_test_06 (curvilinear grid tests)
  export    multi07='y' # mww3_test_07 (unstructured grid tests)
  export    multi08='y' # mww3_test_08 (wind and ice tests)
  export    multi09='y' # mww3_test_09 (SMC multi grid test)
  export        ufs='y' # The Unified Forecast System
  export  ufscoarse='n' # Option for small PCs
  export       grib='y' # grib file field output
  export  rstrt_b4b='y' # Restart Reproducibility
  export    npl_b4b='y' # MPI task Reproducibility
  export    nth_b4b='y' # Thread Reproducibility
  export       esmf='n' # ESMF coupling
# export   filter='PR3 ST2 UQ'
                      # The filter does a set of consecutive greps on the 
                      # command lines generated by filter.base with the above
                      # selected options.

# --------------------------------------------------------------------------- #
# 2.  Execute matrix.base ...                                                 #
# --------------------------------------------------------------------------- #
             
  $main_dir/../regtests/bin/matrix.base

  $main_dir/../regtests/bin/matrix_divider_p.sh 


  echo "#submit all of the diveded matrix files" > msuball.sh
  files=`ls matrix? matrix??`
  for file in $files 
  do
    echo "sbatch < $file"  >> msuball.sh
  done

 



# --------------------------------------------------------------------------- #
# End to the matrix                                                           #
# --------------------------------------------------------------------------- #