################################################################################
# Parallel SCHISM Makefile
#
# User makes environment settings for particular OS / PLATFORM / COMPILER / MPI
# below as well as setting flags having to do with included algorithms (e.g. sediment)
# and the compiler configuration (debug, timing).
#
# The environment settings are based on the following options.
#
# Compiler name:
#   FCS: Serial compiler (for utilities)
#   FCP: Parallel compiler
#   FLD: Linker (in general same as parallel compiler)
#
# Compilation flags
#   FCSFLAGS: Flags for serial compilation
#   FCPFLAGS: Flags for parallel compilation (including all pre-processing flags)
#   FLDFLAGS: Flags for linker (e.g., -O2)
#
# Preprocessor flags:
#   DEBUG: Enable debugging code
#   ORDERED_SUM: Enable globally ordered sums & dot-products for bit reproducibility
#     of state quantities independent of number of processors (note: this can
#     significantly degrade performance);
#   INCLUDE_TIMING: Enable wallclock timing of code (note: this can have slight
#     effect on performance);
#   MPI_VERSION = 1 or 2: Version of MPI (try 2 first, if compile fails due to mpi
#     related errors then switch to version 1;
#
# Libraries (needed for parallel code)
#   MTSLIBS: Flags for linking ParMeTiS/MeTiS libaries
################################################################################

################################################################################
## Environment for DKRZ (levante)
## Attention: need to add '-WF,' in front of all -D (including some before this section)!
######################################################################################
ENV = levante

################################################################################
# Environment
################################################################################

# The openmpi module provides the environement variable $MPIF90
FCP = $(MPIF90)
FLD = $(FCP)
FCS = $(shell which ifort)
PPFLAGS := $(PPFLAGS) -DMPIVERSION=2
FCPFLAGS = $(PPFLAGS) -O2 -mtune=core-avx2 -mcmodel=medium -assume byterecl #-qopenmp
FCSFLAGS = $(FCPFLAGS)
FLDFLAGS = -O2 -mtune=core-avx2 -mcmodel=medium -assume byterecl  #-qopenmp

#####Libraries
##MTSLIBS = -L./ParMetis-3.1-Sep2010/ -lparmetis -lmetis

$(info $(ENV) uses as serial compiler $(FCS) with flags $(FCSFLAGS))
$(info $(ENV) uses as parallel compiler $(FCP) with flags $(FCPFLAGS))

NFCONFIG=$(shell module unload python3; which nf-config; module load python3)
NCCONFIG=$(shell module unload python3; which nc-config; module load python3)
$(info $(ENV) uses as for netCDF C config $(NCCONFIG))
$(info $(ENV) uses as for netCDF Fortran config $(NFCONFIG))

CDFLIBS = $(shell $(NFCONFIG) --flibs) $(shell $(NCCONFIG) --libs)
$(info $(ENV) uses netCDF with CDFLIBS=$(CDFLIBS))
CDFMOD = -I$(shell $(NFCONFIG) --includedir)
$(info $(ENV) uses netCDF with CDFMOD=$(CDFMOD))

#CDFLIBS = -L/sw/rhel6-x64/netcdf/netcdf_fortran-4.4.2-intel14/lib -lnetcdff -Wl,-rpath,/sw/rhel6-x64/netcdf/netcdf_fortran-4.4.2-intel14/lib -L/sw/rhel6-x64/netcdf/netcdf_c-4.3.2-gcc48/lib -Wl,-rpath,/sw/rhel6-x64/netcdf/netcdf_c-4.3.2-gcc48/lib -L/sw/rhel6-x64/hdf5/hdf5-1.8.14-threadsafe-gcc48/lib -Wl,-rpath,/sw/rhel6-x64/hdf5/hdf5-1.8.14-threadsafe-gcc48/lib -L/sw/rhel6-x64/sys/libaec-0.3.2-gcc48/lib -Wl,-rpath,/sw/rhel6-x64/sys/libaec-0.3.2-gcc48/lib -lnetcdf -lhdf5_hl -lhdf5 -lsz -lcurl -lz -lnetcdf
#/sw/rhel6-x64/netcdf/netcdf_fortran-4.4.2-intel14/include

################################################################################
# Alternate executable name if you do not want the default.
################################################################################
EXEC   := pschism_$(ENV)

################################################################################
# Algorithm preference flags.
# Comment out unwanted modules and flags.
################################################################################

# -DSCHISM is always on and is defined elsewhere

include ../mk/include_modules

# Don't comment out the follow ifdef
ifdef USE_GOTM
   GTMMOD =  
   GTMLIBS = -L/home/yinglong/GOTM/gotm-3.2.5/TSUNAMI/lib/IFORT/ -lturbulence_prod  -lutil_prod
else
   GTMMOD =
   GTMLIBS =
endif

######### Specialty compiler flags and workarounds
# Add -DNO_TR_15581 like below for allocatable array problem in sflux_subs.F90
# PPFLAGS := $(PPFLAGS) -WF,-DNO_TR_15581

# Obsolete flags: use USE_WRAP flag to avoid problems in ParMetis lib (calling C from FORTRAN)
# PPFLAGS := $(PPFLAGS) -WF,-DUSE_WRAP