#!/bin/bash
# --------------------------------------------------------------------------- #
# matrix_ukmo_cray: Run matrix of regression tests on Cray XC architecture #
# as currently used at the UK Met Office. #
#
# This version runs the CMake compilation version
# #
# Remarks: #
# Currently, programs using the PDLIB switch (METIS library) crash when #
# compiled with the Cray CCE compiler. For those regtests that use PDLIB #
# it is required to use the GNU compiler. #
# #
# Chris Bunney #
# April 2022 #
# #
# Hendrik L. Tolman #
# August 2013 #
# #
# Copyright 2013 National Weather Service (NWS), #
# National Oceanic and Atmospheric Administration. All rights #
# reserved. WAVEWATCH III is a trademark of the NWS. #
# No unauthorized use without permission. #
# #
# --------------------------------------------------------------------------- #
# 0. Environment file
usage ()
{
cat 2>&1 << EOF
Usage: $myname model_dir
Required:
model_dir : path to model dir of WW3 source
EOF
}
# Get required arguments
if [ ! $# = 0 ]
then
main_dir="$1" ; shift
else
usage
exit 1
fi
# Convert main_dir to absolute path
main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`"
# Set Cray compiler variant: CCE (Cray Compiler Environment) or GNU.
# - ukmo_cray[_debug] : Cray Fortan (Cray Compiler Environment)
# - ukmo_cray_gnu[_dubug] : GNU Fortran using cray wrapper
# - ukmo_cray_intel[_dubug] : Ifort using cray wrapper
# Note: currently, Cray Fortran fails with some PDLIB regrests
cmplr="ukmo_cray_gnu"
# 1. Set up for compilation environemnt on Cray XC (broadwell processors)
echo '#!/bin/bash' > matrix.head
echo ' ' >> matrix.head
# Run everything on single node in shared queue (compilation on compute
# nodes via parallel queue is inefficient and wasteful of resources):
echo '#PBS -l ncpus=16' >> matrix.head
echo '#PBS -l mem=16GB' >> matrix.head
echo '#PBS -q shared' >> matrix.head
echo '#PBS -l walltime=04:00:00' >> matrix.head
echo '#PBS -N ww3_regtest' >> matrix.head
echo '#PBS -j oe' >> matrix.head
echo '#PBS -o matrix.out' >> matrix.head
echo ' ' >> matrix.head
echo " cd $(dirname $main_dir)/regtests" >> matrix.head
echo ' ' >> matrix.head
if [[ $cmplr == "ukmo_cray" ]] || [[ $cmplr == "ukmo_cray_debug" ]]; then
# Load targetted versions of Cray Development Tools (bug in Fortran StreamIO
# for older versions) and netCDF/HDF5 modules:
echo "module load cdt/18.12" >> matrix.head
echo "module load cray-netcdf/4.6.1.3" >> matrix.head
echo "module load cray-hdf5/1.10.2.0" >> matrix.head
echo "export METIS_PATH=/home/d02/frey/WW3/ParMETIS" >> matrix.head
elif [[ $cmplr == ukmo_cray_gnu* ]]; then
# ParMETIS library not currently working with Cray compiler.
# Use GNU compiler for programs that use PDLIB.
echo "module switch PrgEnv-cray PrgEnv-gnu/5.2.82" >> matrix.head
echo "module load cray-netcdf" >> matrix.head
echo "export METIS_PATH=/home/d02/frey/WW3/ParMETIS_GNU" >> matrix.head
elif [[ $cmplr == ukmo_cray_intel* ]]; then
echo "module switch PrgEnv-cray PrgEnv-intel" >> matrix.head
echo "module swap intel/15.0.0.090 intel/18.0.5.274" >> matrix.head
echo "module load cdt/18.12" >> matrix.head
echo "module load cray-netcdf/4.6.1.3" >> matrix.head
echo "module load cray-hdf5/1.10.2.0" >> matrix.head
else
echo "Unknown compiler for UKMO regression tests: $cmplr"
exit 1
fi
# Need newer cmake version on the XC
echo "module load cmake/3.21.3" >> matrix.head
# SNP Launcher 7.7.4 allows -np switch:
echo "module load cray-snplauncher/7.7.4" >> matrix.head
echo "export NETCDF_CONFIG=\$(which nc-config)" >> matrix.head
# On the Cray XC, we need to stop CMake from searching for
# the NetCDF libraries - they are provided by the ftn wrapper.
echo "export CMAKE_OPTIONS=-DEXCLUDE_FIND=\"netcdf\"" >> matrix.head
# Resources:
export mpi='mpiexec'
export np='16'
export nr='4'
export nth='4'
export rtst="./bin/run_cmake_test -o both -S"
export ww3='../model'
# 1.b Flags to do course selection - - - - - - - - - - - - - - - - - - - - - -
# Addition selection by commenting out lines as below
export shrd='y' # Do shared architecture tests
export dist='y' # Do distributed architecture (MPI) tests
export omp='y' # Threaded (OpenMP) tests
export hybd='y' # Hybrid options
export prop1D='y' # 1-D propagation tests (ww3_tp1.X)
export prop2D='y' # 2-D propagation tests (ww3_tp2.X)
export time='y' # time linmited growth
export fetch='y' # fetch linmited growth
export hur1mg='y' # Hurricane with one moving grid
export shwtr='y' # shallow water tests
export unstr='y' # unstructured grid tests
export pdlib='y' # unstr with pdlib for domain decomposition and implicit solver
export smcgr='y' # SMC grid test
export rtd='y' # Rotated pole test
export mudice='y' # Mud/Ice and wave interaction tests
export infgrv='y' # Second harmonic generation tests
export uost='y' # ww3_ts4 Unresolved Obstacles Source Term (UOST)
export assim='y' # Restart spectra update
export oasis='y' # Atmosphere, ocean, and ice coupling using oasis
export calendar='y' # Calendar type
export confignc='y' # Configurable netCDF meta data (ww3_ounf)
export multi01='y' # mww3_test_01 (wetting and drying)
export multi02='y' # mww3_test_02 (basic two-way nesting test))
export multi03='y' # mww3_test_03 (three high and three low res grids).
export multi04='y' # mww3_test_04 (swell on sea mount and/or current)
export multi05='y' # mww3_test_05 (three-grid moving hurricane)
export multi06='y' # mww3_test_06 (curvilinear grid tests)
export multi07='y' # mww3_test_07 (unstructured grid tests)
export multi08='y' # mww3_test_08 (wind and ice tests)
export multi09='y' # mww3_test_09 (SMC multi grid test)
export ufs='n' # The Unified Forecast System
export ufscoarse='n' # Option for small PCs
export grib='n' # grib file field output
export rstrt_b4b='y' # Restart Reproducibility
export npl_b4b='y' # MPI task Reproducibility
export nth_b4b='y' # Thread Reproducibility
export esmf='n' # ESMF coupling
# export filter='PR3 ST2 UQ'
# The filter does a set of consecutinve greps on the
# command lines generated by filter.base with the above
# selected options.
# --------------------------------------------------------------------------- #
# 2. Execute matrix.base ... #
# --------------------------------------------------------------------------- #
$main_dir/../regtests/bin/matrix.base
$main_dir/../regtests/bin/matrix_divider_cmake.sh
# --------------------------------------------------------------------------- #
# End to the matrix #
# --------------------------------------------------------------------------- #