#!/bin/bash
#SBATCH --job-name=R17r        # Specify job name
#SBATCH --partition=compute    # Specify partition name
#SBATCH --nodes=40              # Specify number of nodes
#SBATCH --ntasks-per-node=60   # Specify number of (MPI) tasks on each node
#SBATCH --time=01:00:00        # Set a limit on the total run time
#SBATCH --mail-type=NONE      # Notify user by email in case of job failure
#SBATCH --account=gg0028          # Charge resources on this project account
#SBATCH --output=my_job.o%j    # File name for standard output

# Bind your OpenMP threads
export OMP_NUM_THREADS=2
export KMP_AFFINITY=verbose,granularity=fine,scatter

# limit stacksize ... adjust to your programs need
# and core file size
ulimit -s 132400
ulimit -c 0
export OMP_STACKSIZE=128M

#Load appropriate toolset fro gcc/intel etc
source /home/g/g260135/gcc_tool
#source /home/g/g260135/intel_tool

# Replace this block according to https://docs.dkrz.de/doc/levante/running-jobs/mpi-runtime-settings.rst
#echo "Replace this block according to https://docs.dkrz.de/doc/levante/running-jobs/mpi-runtime-settings.rst"
#exit 23
# End of block to replace
export OMPI_MCA_pml="ucx"
export OMPI_MCA_btl=self
export OMPI_MCA_osc="pt2pt"
export UCX_IB_ADDR_TYPE=ib_global
# for most runs one may or may not want to disable HCOLL
export OMPI_MCA_coll="^ml,hcoll"
export OMPI_MCA_coll_hcoll_enable="0"
export HCOLL_ENABLE_MCAST_ALL="0"
export HCOLL_MAIN_IB=mlx5_0:1
export UCX_NET_DEVICES=mlx5_0:1
export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
export UCX_UNIFIED_MODE=y
export HDF5_USE_FILE_LOCKING=FALSE
export OMPI_MCA_io="romio321"
export UCX_HANDLE_ERRORS=bt


# Use srun (not mpirun or mpiexec) command to launch
# programs compiled with any MPI library
srun -l --cpu_bind=verbose --hint=nomultithread \
  --distribution=block:cyclic:block ./pschism_LEVANTE_GCC_OMP_PREC_EVAP_TVD-VL 7