saeed-moghimi-noaa
commented
1 year ago Documenting compiling ATMESH + WW3 journey
compiling ATMESH + WW3
git clone --recursive https://github.com/noaa-ocs-modeling/CoastalApp
git checkout develop_build
git submodule sync
git submodule update --init --recursiveedit .submodules to: [submodule "WW3"] path = WW3 url = https://github.com/awest-noaa/WW3.git branch = scalability
rm -rf WW3
git submodule update --init --recursiveReplace switch and makefile within: WWW/model/esmf with: switch
F90
NOGRB
DIST
MPI
SCRIP
SCRIPNC
NC4
PR3
UQ
FLX0
SEED
FLD2
ST4
STAB0
NL1
BT1
DB1
MLIM
TR0
BS0
XX0
RWND
WNX1
WNT1
CRX1
CRT1
O0
O1
O2
O3
O4
O5
O6
O7
O14
O15
IC0
IS0
REF0Makefile
# GNU Makefile for test Wavewatch III coupled application
################################################################################
WW3_DIR := $(shell cd .. 1>/dev/null 2>&1 && pwd)
WW3_BASEDIR := $(shell cd ../.. 1>/dev/null 2>&1 && pwd)
WW3_BINDIR := $(WW3_DIR)/bin
WW3_TMPDIR := $(WW3_DIR)/tmp
WW3_EXEDIR := $(WW3_DIR)/exe
COMP_BINDIR ?= $(WW3_BASEDIR)/exe
WW3_COMP ?= gnu
WW3_CC := gcc
WW3_F90 := gfortran
SWITCHES := $(shell cat switch)
WWATCH3_ENV := $(WW3_BINDIR)/wwatch3.env
WWATCH3_NETCDF := NC4
NETCDF_CONFIG := $(shell which nc-config)
export WWATCH3_ENV WWATCH3_NETCDF NETCDF_CONFIG
EXE := $(WW3_EXEDIR)/ww3_multi_esmf
################################################################################
ifneq ($(origin ESMFMKFILE), environment)
$(error Environment variable ESMFMKFILE was not set.)
endif
include $(ESMFMKFILE)
################################################################################
# pgi
ifeq ($(WW3_COMP),Portland)
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -byteswapio
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","pgi" "datarmor_pgi" "datarmor_pgi_debug"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -byteswapio
# intel
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","theia" "Intel"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","wcoss_phase2" "wcoss_cray"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","intel" "datarmor_intel" "datarmor_intel_debug"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
# mpt
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","mpt" "datarmor_mpt" "datarmor_mpt_debug"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
# gnu
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","gnu" "datarmor_gnu" "datarmor_gnu_debug"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -fconvert=big-endian
# error
else
$(error Unsupported WW3_COMP: $(WW3_COMP))
endif
################################################################################
DEP_FRONTS := -DFRONT_ESM=ESM
DEP_FRONTS += -DFRONT_ATM=MDL
DEP_FRONTS += -DFRONT_OCN=MDL
DEP_FRONTS += -DFRONT_ICE=MDL
DEP_CMPL_OBJS := utl.o con.o mdl.o dum.o
DEP_LINK_OBJS := esm.o utl.o con.o mdl.o dum.o
ifeq ($(MAKECMDGOALS),ww3_multi_esmf)
include $(WW3_DIR)/nuopc.mk
DEP_FRONTS += -DFRONT_WAV=$(ESMF_DEP_FRONT)
DEP_INCS += $(addprefix -I, $(ESMF_DEP_INCPATH))
DEP_CMPL_OBJS += $(ESMF_DEP_CMPL_OBJS)
DEP_LINK_OBJS += $(ESMF_DEP_LINK_OBJS)
endif
################################################################################
################################################################################
.SUFFIXES: .F90
%.o : %.F90
$(ESMF_F90COMPILER) -c $(DEP_FRONTS) $(DEP_INCS) \
$(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) \
$(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) $<
# -----------------------------------------------------------------------------
# ------------------------------------------------------------------------------
default: env setup gout switch
$(WW3_BINDIR)/w3_make ww3_multi_esmf
$(MAKE) ww3_multi_esmf
ww3_nems: env setup gout switch
$(WW3_BINDIR)/w3_make ww3_multi_esmf
ww3_multi_esmf: esmApp.o $(DEP_LINK_OBJS)
$(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) -o $(EXE) $^ \
$(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) $(ESMF_F90ESMFLINKLIBS)
# module dependencies:
esmApp.o: macros.h utl.o esm.o
esm.o: macros.h $(DEP_CMPL_OBJS)
utl.o: macros.h
con.o: macros.h utl.o
mdl.o: macros.h utl.o
dum.o: macros.h
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
.PHONY: env setup switch dust clean
dust:
\rm -f PET*.ESMF_LogFile
clean:
\rm -f $(EXE) *.o *.mod
distclean: env clean
$(WW3_BINDIR)/w3_clean -c
\rm -fr $(WW3_TMPDIR)
\rm -f $(WWATCH3_ENV)
# ------------------------------------------------------------------------------
# Setup wwatch3.env file for WW3 build or clean
# ------------------------------------------------------------------------------
env:
@rm -f $(WWATCH3_ENV)
@touch $(WWATCH3_ENV)
@echo '#' >> $(WWATCH3_ENV)
@echo '# ---------------------------------------' >> $(WWATCH3_ENV)
@echo '# Environment variables for wavewatch III' >> $(WWATCH3_ENV)
@echo '# ---------------------------------------' >> $(WWATCH3_ENV)
@echo '#' >> $(WWATCH3_ENV)
@echo 'WWATCH3_LPR $(PRINTER)' >> $(WWATCH3_ENV)
@echo 'WWATCH3_F90 $(WW3_F90)' >> $(WWATCH3_ENV)
@echo 'WWATCH3_CC $(WW3_CC)' >> $(WWATCH3_ENV)
@echo 'WWATCH3_DIR $(WW3_DIR)' >> $(WWATCH3_ENV)
@echo 'WWATCH3_TMP $(WW3_TMPDIR)' >> $(WWATCH3_ENV)
@echo 'WWATCH3_SOURCE yes' >> $(WWATCH3_ENV)
@echo 'WWATCH3_LIST yes' >> $(WWATCH3_ENV)
@echo '' >> $(WWATCH3_ENV)
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# Setup comp & link files and auxiliary programs (this is done using w3_setup)
# ------------------------------------------------------------------------------
setup: env
@$(WW3_BINDIR)/w3_setup -q -c $(WW3_COMP) $(WW3_DIR)
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# Setup switch file
# ------------------------------------------------------------------------------
switch:
@echo "$(SWITCHES)" > $(WW3_BINDIR)/switch
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# Setup switch file and compile serial routines ww3_grid, ww3_outf and ww3_outp
# ------------------------------------------------------------------------------
gout:
@echo "$(SWITCHES)" > $(WW3_BINDIR)/tempswitch
@sed -e "s/DIST/SHRD/g"\
-e "s/OMPG/ /g"\
-e "s/OMPH/ /g"\
-e "s/MPIT/ /g"\
-e "s/MPI/ /g"\
-e "s/PDLIB/ /g"\
$(WW3_BINDIR)/tempswitch > $(WW3_BINDIR)/switch
\rm -f $(WW3_BINDIR)/tempswitch
$(WW3_BINDIR)/w3_make ww3_grid
$(WW3_BINDIR)/w3_make ww3_outf
$(WW3_BINDIR)/w3_make ww3_outp
$(WW3_BINDIR)/w3_make ww3_prep
$(WW3_BINDIR)/w3_make ww3_gint
$(WW3_BINDIR)/w3_make ww3_prnc
$(WW3_BINDIR)/w3_make ww3_ounf
$(WW3_BINDIR)/w3_make ww3_ounp
\mkdir -p $(COMP_BINDIR)
\cp -f $(WW3_EXEDIR)/ww3_* $(COMP_BINDIR)/
# ------------------------------------------------------------------------------``
Then execute:
./build.sh --component "ATMESH WW3 " --plat hera --compiler intel --clean -2 --thirdparty=parmetis
GM-Ling
aliabdolali
Here is the version that works for me:
commit 216b06032ee47a10ebb7762817752843fc945fec (HEAD -> feature/pahm, origin/feature/pahm)
Author: zacharyburnettNOAA 
pvelissariou1
@GM-Ling @pvelissariou1 @aliabdolali @FVCOM-GitHub
Guoming Ling 9:36 AM 1) follow up with UND (i.e. Guoming) and check if they can run coupled model with PDLIB activated in WW3? If the answer is yes, we should check their setup and version of WW3 in their test. Yes, I always run with PDLIB activated, but I don't have much knowledge about the PDLIB except the introductions in the user manual(https://github.com/NOAA-EMC/WW3/wiki/Manual). My setups and results for the Shinnecock test case were uploaded here: https://github.com/GM-Ling/ATMESH-ADCIRC-WW3-TestSuite.
works for Guoming
Guoming Ling 9:40 AM git log commit 216b06032ee47a10ebb7762817752843fc945fec (HEAD -> feature/pahm, origin/feature/pahm) Author: zacharyburnettNOAA zachary.burnett@noaa.gov Date: Mon Aug 9 12:36:59 2021 +0000
commit 0154723783823b7786f3930be3787dc9757f5d53 Author: pvelissariou1 panagiotis.velissariou@noaa.gov Date: Fri Aug 6 00:50:15 2021 -0500
commit 21faa6238ca8d274c1d1112e1a5f530fe6e1c5a0 Author: pvelissariou1 panagiotis.velissariou@noaa.gov