#!/bin/bash # --------------------------------------------------------------------------- # # matrix_ukmo_cray: Run matrix of regression tests on Cray XC architecture # # as currently used at the UK Met Office. # # # This version runs the CMake compilation version # # # Remarks: # # Currently, programs using the PDLIB switch (METIS library) crash when # # compiled with the Cray CCE compiler. For those regtests that use PDLIB # # it is required to use the GNU compiler. # # # # Chris Bunney # # April 2022 # # # # Hendrik L. Tolman # # August 2013 # # # # Copyright 2013 National Weather Service (NWS), # # National Oceanic and Atmospheric Administration. All rights # # reserved. WAVEWATCH III is a trademark of the NWS. # # No unauthorized use without permission. # # # # --------------------------------------------------------------------------- # # 0. Environment file usage () { cat 2>&1 << EOF Usage: $myname model_dir Required: model_dir : path to model dir of WW3 source EOF } # Get required arguments if [ ! $# = 0 ] then main_dir="$1" ; shift else usage exit 1 fi # Convert main_dir to absolute path main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`" # Set Cray compiler variant: CCE (Cray Compiler Environment) or GNU. # - ukmo_cray[_debug] : Cray Fortan (Cray Compiler Environment) # - ukmo_cray_gnu[_dubug] : GNU Fortran using cray wrapper # - ukmo_cray_intel[_dubug] : Ifort using cray wrapper # Note: currently, Cray Fortran fails with some PDLIB regrests cmplr="ukmo_cray_gnu" # 1. Set up for compilation environemnt on Cray XC (broadwell processors) echo '#!/bin/bash' > matrix.head echo ' ' >> matrix.head # Run everything on single node in shared queue (compilation on compute # nodes via parallel queue is inefficient and wasteful of resources): echo '#PBS -l ncpus=16' >> matrix.head echo '#PBS -l mem=16GB' >> matrix.head echo '#PBS -q shared' >> matrix.head echo '#PBS -l walltime=04:00:00' >> matrix.head echo '#PBS -N ww3_regtest' >> matrix.head echo '#PBS -j oe' >> matrix.head echo '#PBS -o matrix.out' >> matrix.head echo ' ' >> matrix.head echo " cd $(dirname $main_dir)/regtests" >> matrix.head echo ' ' >> matrix.head if [[ $cmplr == "ukmo_cray" ]] || [[ $cmplr == "ukmo_cray_debug" ]]; then # Load targetted versions of Cray Development Tools (bug in Fortran StreamIO # for older versions) and netCDF/HDF5 modules: echo "module load cdt/18.12" >> matrix.head echo "module load cray-netcdf/4.6.1.3" >> matrix.head echo "module load cray-hdf5/1.10.2.0" >> matrix.head echo "export METIS_PATH=/home/d02/frey/WW3/ParMETIS" >> matrix.head elif [[ $cmplr == ukmo_cray_gnu* ]]; then # ParMETIS library not currently working with Cray compiler. # Use GNU compiler for programs that use PDLIB. echo "module switch PrgEnv-cray PrgEnv-gnu/5.2.82" >> matrix.head echo "module load cray-netcdf" >> matrix.head echo "export METIS_PATH=/home/d02/frey/WW3/ParMETIS_GNU" >> matrix.head elif [[ $cmplr == ukmo_cray_intel* ]]; then echo "module switch PrgEnv-cray PrgEnv-intel" >> matrix.head echo "module swap intel/15.0.0.090 intel/18.0.5.274" >> matrix.head echo "module load cdt/18.12" >> matrix.head echo "module load cray-netcdf/4.6.1.3" >> matrix.head echo "module load cray-hdf5/1.10.2.0" >> matrix.head else echo "Unknown compiler for UKMO regression tests: $cmplr" exit 1 fi # Need newer cmake version on the XC echo "module load cmake/3.21.3" >> matrix.head # SNP Launcher 7.7.4 allows -np switch: echo "module load cray-snplauncher/7.7.4" >> matrix.head echo "export NETCDF_CONFIG=\$(which nc-config)" >> matrix.head # On the Cray XC, we need to stop CMake from searching for # the NetCDF libraries - they are provided by the ftn wrapper. echo "export CMAKE_OPTIONS=-DEXCLUDE_FIND=\"netcdf\"" >> matrix.head # Resources: export mpi='mpiexec' export np='16' export nr='4' export nth='4' export rtst="./bin/run_cmake_test -o both -S" export ww3='../model' # 1.b Flags to do course selection - - - - - - - - - - - - - - - - - - - - - - # Addition selection by commenting out lines as below export shrd='y' # Do shared architecture tests export dist='y' # Do distributed architecture (MPI) tests export omp='y' # Threaded (OpenMP) tests export hybd='y' # Hybrid options export prop1D='y' # 1-D propagation tests (ww3_tp1.X) export prop2D='y' # 2-D propagation tests (ww3_tp2.X) export time='y' # time linmited growth export fetch='y' # fetch linmited growth export hur1mg='y' # Hurricane with one moving grid export shwtr='y' # shallow water tests export unstr='y' # unstructured grid tests export pdlib='y' # unstr with pdlib for domain decomposition and implicit solver export smcgr='y' # SMC grid test export rtd='y' # Rotated pole test export mudice='y' # Mud/Ice and wave interaction tests export infgrv='y' # Second harmonic generation tests export uost='y' # ww3_ts4 Unresolved Obstacles Source Term (UOST) export assim='y' # Restart spectra update export oasis='y' # Atmosphere, ocean, and ice coupling using oasis export calendar='y' # Calendar type export confignc='y' # Configurable netCDF meta data (ww3_ounf) export multi01='y' # mww3_test_01 (wetting and drying) export multi02='y' # mww3_test_02 (basic two-way nesting test)) export multi03='y' # mww3_test_03 (three high and three low res grids). export multi04='y' # mww3_test_04 (swell on sea mount and/or current) export multi05='y' # mww3_test_05 (three-grid moving hurricane) export multi06='y' # mww3_test_06 (curvilinear grid tests) export multi07='y' # mww3_test_07 (unstructured grid tests) export multi08='y' # mww3_test_08 (wind and ice tests) export multi09='y' # mww3_test_09 (SMC multi grid test) export ufs='n' # The Unified Forecast System export ufscoarse='n' # Option for small PCs export grib='n' # grib file field output export rstrt_b4b='y' # Restart Reproducibility export npl_b4b='y' # MPI task Reproducibility export nth_b4b='y' # Thread Reproducibility export esmf='n' # ESMF coupling # export filter='PR3 ST2 UQ' # The filter does a set of consecutinve greps on the # command lines generated by filter.base with the above # selected options. # --------------------------------------------------------------------------- # # 2. Execute matrix.base ... # # --------------------------------------------------------------------------- # $main_dir/../regtests/bin/matrix.base $main_dir/../regtests/bin/matrix_divider_cmake.sh # --------------------------------------------------------------------------- # # End to the matrix # # --------------------------------------------------------------------------- #