Closed saeed-moghimi-noaa closed 1 year ago
git clone --recursive https://github.com/noaa-ocs-modeling/CoastalApp
git checkout develop_build
git submodule sync
git submodule update --init --recursive
edit .submodules to: [submodule "WW3"] path = WW3 url = https://github.com/awest-noaa/WW3.git branch = scalability
rm -rf WW3
git submodule update --init --recursive
Replace switch and makefile within: WWW/model/esmf with: switch
F90
NOGRB
DIST
MPI
SCRIP
SCRIPNC
NC4
PR3
UQ
FLX0
SEED
FLD2
ST4
STAB0
NL1
BT1
DB1
MLIM
TR0
BS0
XX0
RWND
WNX1
WNT1
CRX1
CRT1
O0
O1
O2
O3
O4
O5
O6
O7
O14
O15
IC0
IS0
REF0
Makefile
# GNU Makefile for test Wavewatch III coupled application
################################################################################
WW3_DIR := $(shell cd .. 1>/dev/null 2>&1 && pwd)
WW3_BASEDIR := $(shell cd ../.. 1>/dev/null 2>&1 && pwd)
WW3_BINDIR := $(WW3_DIR)/bin
WW3_TMPDIR := $(WW3_DIR)/tmp
WW3_EXEDIR := $(WW3_DIR)/exe
COMP_BINDIR ?= $(WW3_BASEDIR)/exe
WW3_COMP ?= gnu
WW3_CC := gcc
WW3_F90 := gfortran
SWITCHES := $(shell cat switch)
WWATCH3_ENV := $(WW3_BINDIR)/wwatch3.env
WWATCH3_NETCDF := NC4
NETCDF_CONFIG := $(shell which nc-config)
export WWATCH3_ENV WWATCH3_NETCDF NETCDF_CONFIG
EXE := $(WW3_EXEDIR)/ww3_multi_esmf
################################################################################
ifneq ($(origin ESMFMKFILE), environment)
$(error Environment variable ESMFMKFILE was not set.)
endif
include $(ESMFMKFILE)
################################################################################
# pgi
ifeq ($(WW3_COMP),Portland)
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -byteswapio
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","pgi" "datarmor_pgi" "datarmor_pgi_debug"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -byteswapio
# intel
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","theia" "Intel"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","wcoss_phase2" "wcoss_cray"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","intel" "datarmor_intel" "datarmor_intel_debug"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
# mpt
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","mpt" "datarmor_mpt" "datarmor_mpt_debug"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
# gnu
else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","gnu" "datarmor_gnu" "datarmor_gnu_debug"))
ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -fconvert=big-endian
# error
else
$(error Unsupported WW3_COMP: $(WW3_COMP))
endif
################################################################################
DEP_FRONTS := -DFRONT_ESM=ESM
DEP_FRONTS += -DFRONT_ATM=MDL
DEP_FRONTS += -DFRONT_OCN=MDL
DEP_FRONTS += -DFRONT_ICE=MDL
DEP_CMPL_OBJS := utl.o con.o mdl.o dum.o
DEP_LINK_OBJS := esm.o utl.o con.o mdl.o dum.o
ifeq ($(MAKECMDGOALS),ww3_multi_esmf)
include $(WW3_DIR)/nuopc.mk
DEP_FRONTS += -DFRONT_WAV=$(ESMF_DEP_FRONT)
DEP_INCS += $(addprefix -I, $(ESMF_DEP_INCPATH))
DEP_CMPL_OBJS += $(ESMF_DEP_CMPL_OBJS)
DEP_LINK_OBJS += $(ESMF_DEP_LINK_OBJS)
endif
################################################################################
################################################################################
.SUFFIXES: .F90
%.o : %.F90
$(ESMF_F90COMPILER) -c $(DEP_FRONTS) $(DEP_INCS) \
$(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) \
$(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) $<
# -----------------------------------------------------------------------------
# ------------------------------------------------------------------------------
default: env setup gout switch
$(WW3_BINDIR)/w3_make ww3_multi_esmf
$(MAKE) ww3_multi_esmf
ww3_nems: env setup gout switch
$(WW3_BINDIR)/w3_make ww3_multi_esmf
ww3_multi_esmf: esmApp.o $(DEP_LINK_OBJS)
$(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) -o $(EXE) $^ \
$(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) $(ESMF_F90ESMFLINKLIBS)
# module dependencies:
esmApp.o: macros.h utl.o esm.o
esm.o: macros.h $(DEP_CMPL_OBJS)
utl.o: macros.h
con.o: macros.h utl.o
mdl.o: macros.h utl.o
dum.o: macros.h
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
.PHONY: env setup switch dust clean
dust:
\rm -f PET*.ESMF_LogFile
clean:
\rm -f $(EXE) *.o *.mod
distclean: env clean
$(WW3_BINDIR)/w3_clean -c
\rm -fr $(WW3_TMPDIR)
\rm -f $(WWATCH3_ENV)
# ------------------------------------------------------------------------------
# Setup wwatch3.env file for WW3 build or clean
# ------------------------------------------------------------------------------
env:
@rm -f $(WWATCH3_ENV)
@touch $(WWATCH3_ENV)
@echo '#' >> $(WWATCH3_ENV)
@echo '# ---------------------------------------' >> $(WWATCH3_ENV)
@echo '# Environment variables for wavewatch III' >> $(WWATCH3_ENV)
@echo '# ---------------------------------------' >> $(WWATCH3_ENV)
@echo '#' >> $(WWATCH3_ENV)
@echo 'WWATCH3_LPR $(PRINTER)' >> $(WWATCH3_ENV)
@echo 'WWATCH3_F90 $(WW3_F90)' >> $(WWATCH3_ENV)
@echo 'WWATCH3_CC $(WW3_CC)' >> $(WWATCH3_ENV)
@echo 'WWATCH3_DIR $(WW3_DIR)' >> $(WWATCH3_ENV)
@echo 'WWATCH3_TMP $(WW3_TMPDIR)' >> $(WWATCH3_ENV)
@echo 'WWATCH3_SOURCE yes' >> $(WWATCH3_ENV)
@echo 'WWATCH3_LIST yes' >> $(WWATCH3_ENV)
@echo '' >> $(WWATCH3_ENV)
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# Setup comp & link files and auxiliary programs (this is done using w3_setup)
# ------------------------------------------------------------------------------
setup: env
@$(WW3_BINDIR)/w3_setup -q -c $(WW3_COMP) $(WW3_DIR)
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# Setup switch file
# ------------------------------------------------------------------------------
switch:
@echo "$(SWITCHES)" > $(WW3_BINDIR)/switch
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# Setup switch file and compile serial routines ww3_grid, ww3_outf and ww3_outp
# ------------------------------------------------------------------------------
gout:
@echo "$(SWITCHES)" > $(WW3_BINDIR)/tempswitch
@sed -e "s/DIST/SHRD/g"\
-e "s/OMPG/ /g"\
-e "s/OMPH/ /g"\
-e "s/MPIT/ /g"\
-e "s/MPI/ /g"\
-e "s/PDLIB/ /g"\
$(WW3_BINDIR)/tempswitch > $(WW3_BINDIR)/switch
\rm -f $(WW3_BINDIR)/tempswitch
$(WW3_BINDIR)/w3_make ww3_grid
$(WW3_BINDIR)/w3_make ww3_outf
$(WW3_BINDIR)/w3_make ww3_outp
$(WW3_BINDIR)/w3_make ww3_prep
$(WW3_BINDIR)/w3_make ww3_gint
$(WW3_BINDIR)/w3_make ww3_prnc
$(WW3_BINDIR)/w3_make ww3_ounf
$(WW3_BINDIR)/w3_make ww3_ounp
\mkdir -p $(COMP_BINDIR)
\cp -f $(WW3_EXEDIR)/ww3_* $(COMP_BINDIR)/
# ------------------------------------------------------------------------------``
Then execute:
./build.sh --component "ATMESH WW3 " --plat hera --compiler intel --clean -2 --thirdparty=parmetis
@saeed-moghimi-noaa @pvelissariou1 @aliabdolali @FVCOM-GitHub Hi All,
By following Saeed's instruction to compile ATMESH and WW3, I was able to run my test case successfully using 240 cores (10 nodes) of Notre Dame cluster. There is no issue for me running multiple nodes of cluster. My test case is here as I pointed out before: https://github.com/GM-Ling/ATMESH-ADCIRC-WW3-TestSuite. Please follow the instruction in README.md, modify the 'ww3_grid.inp' and 'ww3_multi.inp' for running the scalability version of ww3. In my opinion, the owner or manager of the CoastalApp should provide official test cases that work, and test them before updating or merging to the official CoastalApp branch. At the end, the above link just my personal test without guarantee.
Best regards, Guoming
@saeed-moghimi-noaa @pvelissariou1 @aliabdolali @FVCOM-GitHub Hi Guoming
Would you also paste here the scripts (e.g. bash) that you use to run adcirc preps and ww3 preps and submit the mpirun?
Thanks, -Saeed
Hi Saeed(@saeed-moghimi-noaa ), I am using csh:
C shell script for running ww3_grid:
module load mvapich2 module load netcdf ww3_grid > ww3_grid.out
C shell script for running NEMS.x:
module load mvapich2 module load netcdf mpirun -np 240 ./NEMS.x
Hi Guoming
I meant the WW3 prep steps for grid inp and so on.
Thanks, -Saeed
Saeed Moghimi, PhD NOS Storm Surge Modeling https://nauticalcharts.noaa.gov/learn/storm-surge-modeling.html Team Lead Coastal Marine Modeling Branch, Coast Survey Development Laboratory, Office of Coast Survey at NOAA National Ocean Service. Address: 1315 East West Hwy, Room 6607, Silver Spring, Maryland 20910
Phone: (302) 827-3986
The contents of this message are mine personally and do not necessarily reflect any position of NOAA.
On Wed, Sep 14, 2022 at 9:46 AM Guoming @.***> wrote:
-
Bash script for running ww3_grid:
!/bin/csh
$ -M @.***
$ -m abe
$ -q *@@westerink_d12chas_1992
$ -N WW3_grid
module load mvapich2 module load netcdf /afs/ crc.nd.edu/user/g/gling/GitHub_repo/test/CoastalApp_develop_build/WW3/model/exe/ww3_grid
ww3_grid.out
Bash script for running NEMS.x:
!/bin/csh
$ -M @.***
$ -m abe
$ -pe mpi-24 240
$ -q *@@westerink_d12chas_1992
$ -hold_jid NEMS_prep
$ -N RUN_NEMS_ADC-WW3_shinnecock2
module load mvapich2 module load netcdf mpirun -np 240 ./NEMS.x
— Reply to this email directly, view it on GitHub https://github.com/noaa-ocs-modeling/CoastalApp/issues/134#issuecomment-1246790601, or unsubscribe https://github.com/notifications/unsubscribe-auth/APZULD3UTGWKUYSJEUYUYTDV6HJMDANCNFSM6AAAAAAQLO6VTI . You are receiving this because you were mentioned.Message ID: @.***>
Hi Saeed,
It is just very simple 4 steps as following:
Best regards!
I am also able to finish a run of ATMESH+ADCIRC+WW3 (two way coupling of ADCIRC and WW3), the same core setup in my test repo, core 0 for ATMESH, 1-120 for ADCIRC, 121-240 for WW3(there are 24 cores per node in the ND cluster). The log for WW3 is attached: log.inlet.txt
@aliabdolali @GM-Ling
Hi Ali and Guoming
Please see if the ww3 results in atm2wav and atm2wav2ocn make sense. Also let me know how you extract nc files from ww3 bin files.
Hera: /scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp/Saeed.Moghimi/data
Thanks, -Saeed
Hi @saeed-moghimi-noaa I looked at it quickly, yes the log.mww3 seems the run completed and all wave outputs are written in out_grd.inlet here is how you can process the binary files to netcdf
compile ww3_ounf cp out_grd.inlet out_grd.ww3 cp mod_def.inlet mod_def.ww3 cp ww3_ounf.inp to the run directory > execute ww3_ounf. It generates netcdf with all the fields specified in ww3_multi.inp
@GM-Ling @pvelissariou1 @aliabdolali @FVCOM-GitHub @platipodium @josephzhang8
@saeed-moghimi-noaa updates:
Start working on this item since Sep 15, 2022
Added a new branch 'feature/ww3-multi-nodes' at https://github.com/noaa-ocs-modeling/CoastalApp-testsuite.git
Utilized NSEM-workflow to streamline atmesh, ww3 and adcirc testing
On Hera these tests for Shinecock inlet with multi-cores are finalized:
[x] Semi automated bash + Job submission on Hera
[x] ATMESH + ADCIRC
[x] ATMESH + WW3
[x] ATMESH + ADCIRC + WW3
To reproduce on your cluster please follow steps here and report back:
export NSEMdir='/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite'
export ROOTDIR='/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp'
###
git clone -b feature/ww3-multi-nodes https://github.com/noaa-ocs-modeling/CoastalApp-testsuite.git $NSEMdir
### Check out codes
git clone --recursive https://github.com/noaa-ocs-modeling/CoastalApp $ROOTDIR
cd $ROOTDIR
git checkout develop_build
git submodule sync
git submodule update --init --recursive
cp -fv ${NSEMdir}/sorc/ww3_extra_files/* ${ROOTDIR}/WW3/model/esmf/.
###
# download parmetis
sh $ROOTDIR/scripts/download_parmetis.sh
# build on hera
./build.sh --component "ATMESH WW3 ADCIRC WW3DATA " --plat hera --compiler intel --clean -2 --thirdparty=parmetis
run_shinnecock.sh script
#!/bin/bash --login
# -----------------------------------------------------------
# UNIX Shell Script File
# Tested Operating System(s): RHEL 7
# Tested Run Level(s):
# Shell Used: BASH shell
# Original Author(s): Saeed Moghimi
# File Creation Date: 09/17/2022
# Date Last Modified:
#
# Version control: 1.00
#
# Support Team:
#
# Contributors:
#
# -----------------------------------------------------------
# ------------- Program Description and Details -------------
# -----------------------------------------------------------
#
# Execute script to perform CoastalApp test runs
#
# -----------------------------------------------------------
# Saeed next steps on 09/17/2022
#TODO add clone the code and compile CoastalApp
#TODO add tide_fac to ALLBIN folder
#TODO automated copy of the atmesh input to com folder
#TODO add ww3 to netcdf conversion
#TODO add py env yaml file
#TODO add plot basic mesh and timeseries
#
export PS4=' $SECONDS + '
set -x
export NSEMdir='/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite'
export ROOTDIR='/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp'
############
#echo 'Fetching externals...'
#mkdir -p ${NSEMdir}/fix/
#cp -rp /scratch2/COASTAL/coastal/save/shared/CoastalApp_test_fix/fix/* ${NSEMdir}/fix/*
############
# load python env
# for now using the one on hera
#TODO add py env env file to repo
# set COMROOT
export COMROOT=${COMROOT:-${NSEMdir}/../${USER}/com/}
# set exec folder
ln -sfv ${ROOTDIR}/ALLBIN_INSTALL ${NSEMdir}/exec
########
export STORM=shinnecock
#prep COMin
COMINatm=${COMROOT}/atm/para/${STORM}
mkdir -p ${COMINatm}
cp -fv ${NSEMdir}/fix/forcing/shinnecock/ATM/* ${COMINatm}/.
###
export RUN_TYPE=tide_spinup
spinup_jobid=$(sbatch ${NSEMdir}/ecf/jnsem_prep_spinup.ecf | awk '{print $NF}')
spinup_jobid=$(sbatch --dependency=afterok:$spinup_jobid ${NSEMdir}/ecf/jnsem_forecast_spinup.ecf | awk '{print $NF}')
echo $spinup_jobid
###
export RUN_TYPE=atm2ocn
jobid=$(sbatch --dependency=afterok:$spinup_jobid ${NSEMdir}/ecf/jnsem_prep.ecf | awk '{print $NF}')
jobid=$(sbatch --dependency=afterok:$jobid ${NSEMdir}/ecf/jnsem_forecast.ecf | awk '{print $NF}')
jobid=$(sbatch --dependency=afterok:$jobid ${NSEMdir}/ecf/jnsem_post.ecf | awk '{print $NF}')
###
export RUN_TYPE=atm2wav2ocn
jobid=$(sbatch --dependency=afterok:$spinup_jobid ${NSEMdir}/ecf/jnsem_prep.ecf | awk '{print $NF}')
jobid=$(sbatch --dependency=afterok:$jobid ${NSEMdir}/ecf/jnsem_forecast.ecf | awk '{print $NF}')
jobid=$(sbatch --dependency=afterok:$jobid ${NSEMdir}/ecf/jnsem_post.ecf | awk '{print $NF}')
###
export RUN_TYPE=atm2wav
jobid=$(sbatch ${NSEMdir}/ecf/jnsem_prep.ecf | awk '{print $NF}')
jobid=$(sbatch --dependency=afterok:$jobid ${NSEMdir}/ecf/jnsem_forecast.ecf | awk '{print $NF}')
jobid=$(sbatch --dependency=afterok:$jobid ${NSEMdir}/ecf/jnsem_post.ecf | awk '{print $NF}')
# display job queue with dependencies
squeue -u $USER -o "%.8i %3C %4D %16E %12R %j" --sort i
echo squeue -u $USER -o \"%.8i %3C %4D %16E %12R %j\" --sort i
@saeed-moghimi-noaa thanks Saeed for taking the lead on it. Great progress.
For Florence case on HSOFS (1.8M nodes) mesh I have got this tests working:
For the last one I can not pin point where things go wrong. @guoming would you please take a look. Here is the tar.gz of the run folder.
https://drive.google.com/drive/folders/1bJ9h4CTjrrmINceYvT3hsB5nKEjJ9izI?usp=sharing
Thanks, -Saeed
@saeed-moghimi-noaa Hi Saeed,
I see the error in 'out.log' was due to ww3, is it possible because you don't have 'ww3_multi.inp' file in the run folder? At least I don't see the file in the folder that you shared.
Best regards, Guoming
Good catch @GM-Ling. Working on it.
Hi Ali @aliabdolali Would you please take look here see what is the problem and see if you can get it working with the same vesion of stand alone ww3?
On Hera /scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/Saeed.Moghimi/data/florence.atm2wav.20220922/run
Thanks -Saeed
ALi in chat: Ali Abdolali - NOAA Affiliate, 2 min here is the problem 🙂 in ww3_multi.inp of inlet, and florence 'inlet' 'CPL:native' 'CPL:native' 'CPL:native' 'no' 'no' 'no' 'no' 'no' 'no' 1 1 0.00 1.00 F while in florece 'inlet' 'no' 'no' 'CPL:native' 'no' 'no' 'no' 'no' 1 1 0.00 1.00 F so add 2 'no' to it 'inlet' 'no' 'no' 'CPL:native' 'no' 'no' 'no' 'no' 'no' 'no' 1 1 0.00 1.00 F this is atmwav I assume
Saeed Moghimi - NOAA Federal, Now yes
Ali Abdolali - NOAA Affiliate, Now so you just need to add 2 'no' 'no' to that line in total, there should be 9 variables
Saeed Moghimi - NOAA Federal, Now Ok. I will try thanks
Ali Abdolali - NOAA Affiliate, Now sure
@saeed-moghimi-noaa from the communication you and I had, I add it here for reference. In WW3, we added 2 new fields to the forcings (almost 1 year ago), so instead of 7 variables, now we have 9 fields: 1) water levels 2) currents 3) winds 4) ice $ 5) momentum 6) air density and 7-9) assimilation data.
Some more tests suggested by Ali for HSOFS mesh:
could you test the following: comment points out at the begging change it 1 0 F 1 T T line 5: 1 0 T 1 T T > 1 0 F 1 T T line6: points > $points line 13: F F Also, if it did not run, could you make the wind also 'no' to see if it proceeds? line 9: 'inlet' 'no' 'no' 'no' 'no' 'no' 'no' 'no' 'no' 'no' 1 1 0.00 1.00 F but please first check with wind and if did not work, turn of the wind
Response: Still not go through
@aliabdolali
Hi See this folder at Hera from Andre runs: /scratch2/NCEPDEV/stmp1/Andre.VanderWesthuysen/data/florence.atm2wav.20210623/run Not sure which version he used.
@aliabdolali
Here is the crash log file. Please let em know if you can get the ww3 working for this set up (even the uncoupled ww3).
Thanks, -Saeed
++ ls /scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/parm/storms
+ export 'stormlist=florence
harvey
ida
ike
irma
isabel
lane
maria
michael
sandy
shinnecock'
+ stormlist='florence
harvey
ida
ike
irma
isabel
lane
maria
michael
sandy
shinnecock'
+ [[ florence
harvey
ida
ike
irma
isabel
lane
maria
michael
sandy
shinnecock != *florence* ]]
+ '[' atm2wav '!=' tide_spinup ']'
+ '[' atm2wav '!=' tide_baserun ']'
+ '[' atm2wav '!=' best_track2ocn ']'
+ '[' atm2wav '!=' 'wav&best_track2ocn' ']'
+ '[' atm2wav '!=' atm2ocn ']'
+ '[' atm2wav '!=' atm2wav ']'
+ export pgmout=OUTPUT.271336
+ pgmout=OUTPUT.271336
+ export pgm=NSEM
+ pgm=NSEM
++ date +%Y%m%d
+ PDY=20220922
+ export DATAROOT=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data
+ DATAROOT=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data
+ export DATA=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922
+ DATA=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922
+ export GESROOT=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/nwges
+ GESROOT=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/nwges
+ cd /scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922
+ export NET=nsem
+ NET=nsem
++ echo pnsem_forecast
++ awk -F_ '{print $2}'
+ export RUN=forecast
+ RUN=forecast
+ export HOMEnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite
+ HOMEnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite
+ export FIXnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/fix
+ FIXnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/fix
+ export EXECnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/exec
+ EXECnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/exec
+ export SORCnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/sorc
+ SORCnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/sorc
+ export PARMnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/parm
+ PARMnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/parm
+ export USHnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/ush
+ USHnsem=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/ush
+ export INPUTdir=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922/input
+ INPUTdir=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922/input
+ export RUNdir=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922/run
+ RUNdir=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922/run
+ export TMPdir=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922/tmp
+ TMPdir=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922/tmp
+ export LOGdir=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922/logs
+ LOGdir=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/florence.atm2wav.20220922/logs
+ export DEBUGGING=TRUE
+ DEBUGGING=TRUE
+ export DEBUG_LEVEL=1
+ DEBUG_LEVEL=1
+ export ISPRODUCTION=TRUE
+ ISPRODUCTION=TRUE
+ export SITETYPE=EMC
+ SITETYPE=EMC
+ echo pppppppppppppppp /scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/com/
+ export COMOUT=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/com//nsem/para/florence.atm2wav.20220922
+ COMOUT=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/com//nsem/para/florence.atm2wav.20220922
+ export GESOUT=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/nwges/para/nsem.20220922/florence
+ GESOUT=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/nwges/para/nsem.20220922/florence
+ export GESIN=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/nwges/para/nsem.20220922/florence
+ GESIN=/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/../Saeed.Moghimi/data/nwges/para/nsem.20220922/florence
+ /scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp3/CoastalApp-testsuite/scripts/exnsem_forecast.sh.ecf
forrtl: severe (174): SIGSEGV, segmentation fault occurred
Image PC Routine Line Source
NEMS.x 0000000000BFE92D Unknown Unknown Unknown
libpthread-2.17.s 00002B8C9740C630 Unknown Unknown Unknown
libmpi.so.12 00002B8C9BF03AF9 MPI_Irecv Unknown Unknown
libesmf.so 00002B8C98649362 _ZN5ESMCI9SparseM 209 ESMCI_SparseMsg.C
libesmf.so 00002B8C97E99735 _ZN5ESMCI7CommRel 619 ESMCI_CommRel.C
libesmf.so 00002B8C97E98272 _ZN5ESMCI7CommRel 108 ESMCI_CommRel.C
libesmf.so 00002B8C984393AA _ZN5ESMCI4Mesh18b 1711 ESMCI_Mesh.C
libesmf.so 00002B8C98533E8A _Z21ESMCI_meshadd 1345 ESMCI_Mesh_Glue.C
libesmf.so 00002B8C98D31D9B esmf_meshmod_mp_e 1430 ESMF_Mesh.F90
NEMS.x 00000000007F33A4 wmesmfmd_mp_creat 4485 wmesmfmd.F90
NEMS.x 00000000007FC690 wmesmfmd_mp_initi 1516 wmesmfmd.F90
libesmf.so 00002B8C980ECA3E _ZN5ESMCI6FTable1 2009 ESMCI_FTable.C
libesmf.so 00002B8C980F0686 ESMCI_FTableCallE 747 ESMCI_FTable.C
libesmf.so 00002B8C9868ADFA _ZN5ESMCI2VM5ente 1208 ESMCI_VM.C
libesmf.so 00002B8C980EE0D7 c_esmc_ftablecall 899 ESMCI_FTable.C
libesmf.so 00002B8C98918311 esmf_compmod_mp_e 1209 ESMF_Comp.F90
libesmf.so 00002B8C98BEE124 esmf_gridcompmod_ 1405 ESMF_GridComp.F90
libesmf.so 00002B8C9905D661 nuopc_driver_mp_l 2460 NUOPC_Driver.F90
libesmf.so 00002B8C99073AD5 nuopc_driver_mp_i 1751 NUOPC_Driver.F90
libesmf.so 00002B8C99086AC4 nuopc_driver_mp_i 439 NUOPC_Driver.F90
libesmf.so 00002B8C980ECA3E _ZN5ESMCI6FTable1 2009 ESMCI_FTable.C
libesmf.so 00002B8C980F0686 ESMCI_FTableCallE 747 ESMCI_FTable.C
libesmf.so 00002B8C9868ADFA _ZN5ESMCI2VM5ente 1208 ESMCI_VM.C
libesmf.so 00002B8C980EE0D7 c_esmc_ftablecall 899 ESMCI_FTable.C
libesmf.so 00002B8C98918311 esmf_compmod_mp_e 1209 ESMF_Comp.F90
libesmf.so 00002B8C98BEE124 esmf_gridcompmod_ 1405 ESMF_GridComp.F90
NEMS.x 00000000004FCE63 Unknown Unknown Unknown
libesmf.so 00002B8C980ECA3E _ZN5ESMCI6FTable1 2009 ESMCI_FTable.C
libesmf.so 00002B8C980F0686 ESMCI_FTableCallE 747 ESMCI_FTable.C
libesmf.so 00002B8C9868ADFA _ZN5ESMCI2VM5ente 1208 ESMCI_VM.C
libesmf.so 00002B8C980EE0D7 c_esmc_ftablecall 899 ESMCI_FTable.C
libesmf.so 00002B8C98918311 esmf_compmod_mp_e 1209 ESMF_Comp.F90
libesmf.so 00002B8C98BEE124 esmf_gridcompmod_ 1405 ESMF_GridComp.F90
NEMS.x 000000000041720F Unknown Unknown Unknown
NEMS.x 0000000000414E5E Unknown Unknown Unknown
libc-2.17.so 00002B8C9D1A5555 __libc_start_main Unknown Unknown
NEMS.x 0000000000414D69 Unknown Unknown Unknown
srun: error: h32m01: task 1: Exited with exit code 174
srun: launch/slurm: _step_signal: Terminating StepId=36018710.0
slurmstepd: error: *** STEP 36018710.0 ON h32m01 CANCELLED AT 2022-09-22T20:52:26 ***
forrtl: error (78): process killed (SIGTERM)
Image PC Routine Line Source
NEMS.x 0000000000BFE95E Unknown Unknown Unknown
libpthread-2.17.s 00002B7586FD1630 Unknown Unknown Unknown
libmpi.so.12 00002B758B8F0FB3 PMPIDI_CH3I_Progr Unknown Unknown
libmpi.so.12.0 00002B758BA561E0 Unknown Unknown Unknown
libmpi.so.12.0 00002B758B8D372C Unknown Unknown Unknown
libmpi.so.12.0 00002B758B8D9B67 Unknown Unknown Unknown
libmpi.so.12 00002B758B8D91B3 MPI_Bcast Unknown Unknown
libesmf.so 00002B7587D77038 _ZN5ESMCI13broadc 1581 ESMCI_Info.C
libesmf.so 00002B75882CE737 ESMC_InfoBaseSync 437 ESMC_InfoCDef.C
libesmf.so 00002B75882CE269 ESMC_InfoBaseSync 489 ESMC_InfoCDef.C
libesmf.so 00002B7588835AC3 esmf_infosyncmod_ 214 ESMF_InfoSync.F90
libesmf.so 00002B75884CF618 esmf_attributemod 47454 ESMF_Attribute.F90
libesmf.so 00002B7588C2039C nuopc_driver_mp_c 2514 NUOPC_Driver.F90
libesmf.so 00002B7588C22B56 nuopc_driver_mp_l 2478 NUOPC_Driver.F90
libesmf.so 00002B7588C38AD5 nuopc_driver_mp_i 1751 NUOPC_Driver.F90
libesmf.so 00002B7588C4BAC4 nuopc_driver_mp_i 439 NUOPC_Driver.F90
libesmf.so 00002B7587CB1A3E _ZN5ESMCI6FTable1 2009 ESMCI_FTable.C
libesmf.so 00002B7587CB5686 ESMCI_FTableCallE 747 ESMCI_FTable.C
libesmf.so 00002B758824FDFA _ZN5ESMCI2VM5ente 1208 ESMCI_VM.C
libesmf.so 00002B7587CB30D7 c_esmc_ftablecall 899 ESMCI_FTable.C
libesmf.so 00002B75884DD311 esmf_compmod_mp_e 1209 ESMF_Comp.F90
libesmf.so 00002B75887B3124 esmf_gridcompmod_ 1405 ESMF_GridComp.F90
NEMS.x 00000000004FCE63 Unknown Unknown Unknown
libesmf.so 00002B7587CB1A3E _ZN5ESMCI6FTable1 2009 ESMCI_FTable.C
libesmf.so 00002B7587CB5686 ESMCI_FTableCallE 747 ESMCI_FTable.C
libesmf.so 00002B758824FDFA _ZN5ESMCI2VM5ente 1208 ESMCI_VM.C
libesmf.so 00002B7587CB30D7 c_esmc_ftablecall 899 ESMCI_FTable.C
libesmf.so 00002B75884DD311 esmf_compmod_mp_e 1209 ESMF_Comp.F90
libesmf.so 00002B75887B3124 esmf_gridcompmod_ 1405 ESMF_GridComp.F90
NEMS.x 000000000041720F Unknown Unknown Unknown
NEMS.x 0000000000414E5E Unknown Unknown Unknown
libc-2.17.so 00002B758CD6A555 __libc_start_main Unknown Unknown
NEMS.x 0000000000414D69 Unknown Unknown Unknown
forrtl: error (78): process killed (SIGTERM)
Image PC Routine Line Source
NEMS.x 0000000000BFE95E Unknown Unknown Unknown
libpthread-2.17.s 00002B6D3A087630 Unknown Unknown Unknown
libmpi.so.12 00002B6D3E9A5136 PMPIDI_CH3I_Progr Unknown Unknown
libmpi.so.12 00002B6D3ECDD3D4 PMPI_Send Unknown Unknown
libesmf.so 00002B6D3B2C446C _ZN5ESMCI9SparseM 215 ESMCI_SparseMsg.C
libesmf.so 00002B6D3AB14735 _ZN5ESMCI7CommRel 619 ESMCI_CommRel.C
libesmf.so 00002B6D3AB13272 _ZN5ESMCI7CommRel 108 ESMCI_CommRel.C
libesmf.so 00002B6D3B0B43AA _ZN5ESMCI4Mesh18b 1711 ESMCI_Mesh.C
libesmf.so 00002B6D3B1AEE8A _Z21ESMCI_meshadd 1345 ESMCI_Mesh_Glue.C
libesmf.so 00002B6D3B9ACD9B esmf_meshmod_mp_e 1430 ESMF_Mesh.F90
NEMS.x 00000000007F33A4 wmesmfmd_mp_creat 4485 wmesmfmd.F90
NEMS.x 00000000007FC690 wmesmfmd_mp_initi 1516 wmesmfmd.F90
libesmf.so 00002B6D3AD67A3E _ZN5ESMCI6FTable1 2009 ESMCI_FTable.C
libesmf.so 00002B6D3AD6B686 ESMCI_FTableCallE 747 ESMCI_FTable.C
libesmf.so 00002B6D3B305DFA _ZN5ESMCI2VM5ente 1208 ESMCI_VM.C
libesmf.so 00002B6D3AD690D7 c_esmc_ftablecall 899 ESMCI_FTable.C
libesmf.so 00002B6D3B593311 esmf_compmod_mp_e 1209 ESMF_Comp.F90
libesmf.so 00002B6D3B869124 esmf_gridcompmod_ 1405 ESMF_GridComp.F90
libesmf.so 00002B6D3BCD8661 nuopc_driver_mp_l 2460 NUOPC_Driver.F90
libesmf.so 00002B6D3BCEEAD5 nuopc_driver_mp_i 1751 NUOPC_Driver.F90
libesmf.so 00002B6D3BD01AC4 nuopc_driver_mp_i 439 NUOPC_Driver.F90
libesmf.so 00002B6D3AD67A3E _ZN5ESMCI6FTable1 2009 ESMCI_FTable.C
libesmf.so 00002B6D3AD6B686 ESMCI_FTableCallE 747 ESMCI_FTable.C
libesmf.so 00002B6D3B305DFA _ZN5ESMCI2VM5ente 1208 ESMCI_VM.C
libesmf.so 00002B6D3AD690D7 c_esmc_ftablecall 899 ESMCI_FTable.C
libesmf.so 00002B6D3B593311 esmf_compmod_mp_e 1209 ESMF_Comp.F90
libesmf.so 00002B6D3B869124 esmf_gridcompmod_ 1405 ESMF_GridComp.F90
NEMS.x 00000000004FCE63 Unknown Unknown Unknown
libesmf.so 00002B6D3AD67A3E _ZN5ESMCI6FTable1 2009 ESMCI_FTable.C
libesmf.so 00002B6D3AD6B686 ESMCI_FTableCallE 747 ESMCI_FTable.C
libesmf.so 00002B6D3B305DFA _ZN5ESMCI2VM5ente 1208 ESMCI_VM.C
libesmf.so 00002B6D3AD690D7 c_esmc_ftablecall 899 ESMCI_FTable.C
libesmf.so 00002B6D3B593311 esmf_compmod_mp_e 1209 ESMF_Comp.F90
libesmf.so 00002B6D3B869124 esmf_gridcompmod_ 1405 ESMF_GridComp.F90
NEMS.x 000000000041720F Unknown Unknown Unknown
NEMS.x 0000000000414E5E Unknown Unknown Unknown
libc-2.17.so 00002B6D3FE20555 __libc_start_main Unknown Unknown
NEMS.x 0000000000414D69 Unknown Unknown Unknown
forrtl: error (78): process killed (SIGTERM)
Image PC Routine Line Source
NEMS.x 0000000000BFE95E Unknown Unknown Unknow
Hi @GM-Ling
Would you please see if you can run the atmesh-ww3 case for the HSOFS 1.8M node mesh that I shared with you?
Thanks, -Saeed
Hi @saeed-moghimi-noaa There is no wind file in the shared folder, so I cannot run the model. By the way, the scalability version ww3 is not well implemented yet, there is an issue reported here: https://github.com/NOAA-EMC/WW3/issues/649 It seems like no one can solve the issue yet. When coupling currents to WW3, the WW3 doesn't work for the Alaska model. By far, only the Shinnecock test case in my repo works for 2 way coupling.
I tried with my wind file, but atmesh-ww3 case didn't work, I got the error: *** WAVEWATCH III WARING IN W3WAVE : AT LEAST ONE PROCESSOR HAS 0 ACTIVE POINTS IN GRID 1
BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES PID 2829390 RUNNING AT d12chas082.crc.nd.edu EXIT CODE: 9 CLEANING UP REMAINING PROCESSES YOU CAN IGNORE THE BELOW CLEANUP MESSAGES
@saeed-moghimi-noaa Hi Saeed,
I just tried my old version of CoastalApp, I can run the atmesh-ww3 case: Here is the version that works for me: commit 216b06032ee47a10ebb7762817752843fc945fec (HEAD -> feature/pahm, origin/feature/pahm) Author: zacharyburnettNOAA zachary.burnett@noaa.gov Date: Mon Aug 9 12:36:59 2021 +0000
added aswip
So it looks like the new version of CoastalApp doesn't work. (The same input the latest CoastalApp didn't work for me)
Hi @GM-Ling How do you checkout the code? -Saeed
Hi Saeed, For the latest code: git clone --recurse-submodules https://github.com/noaa-ocs-modeling/CoastalApp.git -b develop_build Then followed your: "Documenting compiling ATMESH + WW3 journey". Best regards!
I meant the version that works :-)
Saeed Moghimi, PhD NOS Storm Surge Modeling https://nauticalcharts.noaa.gov/learn/storm-surge-modeling.html Team Lead Coastal Marine Modeling Branch, Coast Survey Development Laboratory, Office of Coast Survey at NOAA National Ocean Service. Address: 1315 East West Hwy, Room 6607, Silver Spring, Maryland 20910
Phone: (302) 827-3986
The contents of this message are mine personally and do not necessarily reflect any position of NOAA.
On Wed, Sep 28, 2022 at 1:59 PM Guoming @.***> wrote:
Hi Saeed, For the latest code: git clone --recurse-submodules https://github.com/noaa-ocs-modeling/CoastalApp.git -b develop_build Best regards!
— Reply to this email directly, view it on GitHub https://github.com/noaa-ocs-modeling/CoastalApp/issues/134#issuecomment-1261279942, or unsubscribe https://github.com/notifications/unsubscribe-auth/APZULDZ3NGNMVXZVDCZBV2LWASBQJANCNFSM6AAAAAAQLO6VTI . You are receiving this because you were mentioned.Message ID: @.***>
Then it will be something like this: git clone https://github.com/noaa-ocs-modeling/CoastalApp -b feature/pahm --recursive git submodule update --init --recursive --remote
It is the feature/pahm branch, and you can see the git log messages at the first block, so it should be the version after zacharyburnettNOAA consolidating on Aug 9, 2021. But then I just switched the WW3 to Andre's scalability version the same as your instruction, here is my git logs for the ww3:
When you switched the ww3, it is working or ???
It works. So it is working for:
Hi @GM-Ling
Do you have time for a quick tag up? I sent you an invite.
@aliabdolali and @pvelissariou1 Please join from my calendar if you have time.
-Saeed
Guoming, Saeed
Here are the links for Andre's video recordings for WW3.
Andre_WW3_08122022.mp4: https://drive.google.com/file/d/1BWIN9UizynN3sVzLMhhbwSixXiHZoSeP/view?usp=sharing Andre_WW3-1_08102022.mp4: https://drive.google.com/file/d/1EzGG4LX964xUP9Q2FSEf9MZtTh3Rb7QW/view?usp=sharing Andre_WW3-2_08102022.mp4: https://drive.google.com/file/d/1sDh5bUSFzD5DTO7igL9lUN4n0OU7Y6WV/view?usp=sharing
These are contained in the CMMB folder: Storm_Surge_CMMB_Technical_Videos: https://drive.google.com/drive/folders/1FZ-d1W3tF1FOgWA8fFI0hrZ4e8emgGbd?usp=sharing
Guoming, you do have access to this folder.
In the videos, Andre talks about setting WW3, its BCs and how to run it from the NSEM workflow.
Takis
Panagiotis Velissariou, Ph.D., P.E. UCAR Scientist National Ocean and Atmospheric Administration National Ocean Service Office of Coast Survey CSDL/CMMB Project Lead - Coastal Coupling cell: (205) 227-9141 email: @.***
On Wed, Sep 28, 2022 at 3:26 PM Saeed Moghimi @.***> wrote:
Thanks @GM-Ling https://github.com/GM-Ling
— Reply to this email directly, view it on GitHub https://github.com/noaa-ocs-modeling/CoastalApp/issues/134#issuecomment-1261427842, or unsubscribe https://github.com/notifications/unsubscribe-auth/APC7TP5UUVCHAV5MGBX4A4LWASSVDANCNFSM6AAAAAAQLO6VTI . You are receiving this because you were mentioned.Message ID: @.***>
Guoming, Thank you so much for following this.
Now, that version of CoastalApp was using the previous version of WaveWatch III (not the scalability version). The master branch of CoastalApp uses the previous version of WW3.
Summary: master branch: points to Guoming's forked branch ( GM-Ling/WW3) develop branch: WW3 points to Andre's scalability WW3 branch develop_build branch: WW3 points to WW3's development branch
So, if this is the case, then the issue might be with the new version of WW3. What I don't understand is that while Andre run successfully WW3+ATMESH, WW3+PAHM, WW3+ADCIRC+PAHM using the scalability version of WW3, we are facing this issue. Any ideas?
To me, the message: "AT LEAST ONE PROCESSOR HAS 0 ACTIVE POINTS IN GRID 1" tells that it might be an issue with the domain decomposition code.
Guoming, have you ever run WW3 outside CoastalApp using the same domain configuration?
Takis
Panagiotis Velissariou, Ph.D., P.E. UCAR Scientist National Ocean and Atmospheric Administration National Ocean Service Office of Coast Survey CSDL/CMMB Project Lead - Coastal Coupling cell: (205) 227-9141 email: @.***
On Wed, Sep 28, 2022 at 10:52 AM Guoming @.***> wrote:
@saeed-moghimi-noaa https://github.com/saeed-moghimi-noaa Hi Saeed,
I just tried my old version of CoastalApp, I can run the atmesh-ww3 case: [image: image] https://user-images.githubusercontent.com/48800504/192816150-ea047eea-0ce7-4dcc-93be-8debd35e47c6.png Here is the version that works for me: commit 216b060 https://github.com/noaa-ocs-modeling/CoastalApp/commit/216b06032ee47a10ebb7762817752843fc945fec (HEAD -> feature/pahm, origin/feature/pahm) Author: zacharyburnettNOAA @.*** Date: Mon Aug 9 12:36:59 2021 +0000
added aswip
So it looks like the new version of CoastalApp doesn't work.
— Reply to this email directly, view it on GitHub https://github.com/noaa-ocs-modeling/CoastalApp/issues/134#issuecomment-1261113958, or unsubscribe https://github.com/notifications/unsubscribe-auth/APC7TPYFOUO5OUA2MZLK3K3WARSUJANCNFSM6AAAAAAQLO6VTI . You are receiving this because you were mentioned.Message ID: @.***>
Hi Takis,
What I don't understand is that while Andre run successfully WW3+ATMESH, WW3+PAHM, WW3+ADCIRC+PAHM using the scalability version of WW3, we are facing this issue. Any ideas?
Why not just copy Andre's code and input data, then reproduce what he can do. He might have the correct configurations for WW3, when I asked for his configuration of link and comp, he never sent them to me.
Guoming, have you ever run WW3 outside CoastalApp using the same domain configuration?
Yes, I am able to run WW3 outside CoastalApp for the Alaska operational model. But I couldn't get it work inside CoastalApp for the same Alaska model when coupled with ADCIRC.
Best regards, Guoming
@saeed-moghimi-noaa CoastalApp (develop_build) has been updated to properly build ParMETIS (old version 4.0.3 and the newest github version). The download_parmetis.sh script downloads v4.0.3 (default) and if run as: "download_parmetis.sh github" it clones the latest version from github (not compatible with WW3 due to the split of the libraries). To compile CoastalApp with component WW3 without errors I copied Ali's switch and Makefile files from /scratch2/COASTAL/coastal/save/Ali.Abdolali/COASTAL_APP_Oct2022/HSOFS/WW3/model/esmf/ into CoastalApp/WW3/model/esmf/ All ww3 executables are installed in ALLBIN_INSTALL folder. Compilation using ParMETIS in CoastalApp and from the hpc_stack completed without any further WW3 source modifications (used PDLIB in switch file).
@aliabdolali @pvelissariou1 @GM-Ling
Here is how I set up test-suit and CoastalApp. At the current stage system runs:
Tests are going through for both setup for this testes:
Tests for this run types is still not working
Use this script to setup the test system
#!/bin/bash --login
# -----------------------------------------------------------
# UNIX Shell Script File
# Tested Operating System(s): RHEL 7
# Tested Run Level(s):
# Shell Used: BASH shell
# Original Author(s): Saeed Moghimi
# File Creation Date: 09/17/2022
# Date Last Modified:
#
# Version control: 1.00
#
# Support Team:
#
# Contributors:
#
# -----------------------------------------------------------
# ------------- Program Description and Details -------------
# -----------------------------------------------------------
#
# Execute script to perform CoastalApp test runs
#
# -----------------------------------------------------------
# Saeed next steps on 09/17/2022
#TODO add ww3 to netcdf conversion
#TODO add plot basic mesh and timeseries
# suggest to use interactive node to perform this:
# > srun -N 1 -A coastal --ntasks-per-node=8 -t 0-06:00 --pty bash
export PS4=' $SECONDS + '
set -x
### These need to be define and point to your location of interest:
## Test scripts
export NSEMdir='/scratch2/COASTAL/coastal/save/Saeed.Moghimi/models/NEMS/tests/CoastalApp_test/temp7/CoastalApp-testsuite'
##CoastalApp Code base
export ROOTDIR='/scratch2/COASTAL/coastal/save/Saeed.Moghimi/models/NEMS/tests/CoastalApp_test/temp7/CoastalApp'
############
### Clone CoastalApp-testsuite
##If not already done
git clone -b feature/ww3-multi-nodes https://github.com/noaa-ocs-modeling/CoastalApp-testsuite.git $NSEMdir
## echo 'Fetching externals... on Hera only'
mkdir -p ${NSEMdir}/fix/
cp -rpv /scratch2/COASTAL/coastal/save/shared/CoastalApp_test_fix/fix ${NSEMdir}
############
# Check out codes
git clone --recursive https://github.com/noaa-ocs-modeling/CoastalApp $ROOTDIR
cd $ROOTDIR
git checkout develop_build
git submodule sync
git submodule update --init --recursive
##just a hack to update WW3 submodule to point to Andre for the time being
cp -fv /scratch2/COASTAL/coastal/save/shared/CoastalApp_test_fix/ww3_extra_files/gitmodules $ROOTDIR/.gitmodules
rm -rf WW3
#git submodule update --init --recursive
git clone -b scalability https://github.com/awest-noaa/WW3.git $ROOTDIR/WW3
#copy extra files for ww3 compile
cp -fv /scratch2/COASTAL/coastal/save/shared/CoastalApp_test_fix/ww3_extra_files/* ${ROOTDIR}/WW3/model/esmf/.
cp -fv /scratch2/COASTAL/coastal/save/shared/CoastalApp_test_fix/ww3_extra_files/switch $ROOTDIR/WW3/model/bin/switch
# download parmatis
#TODO add a error message to perform below if not yet for ww3 compilation
#TODO or export the vars ...
sh $ROOTDIR/scripts/download_parmetis.sh
# build on hera
./build.sh --component "ATMESH WW3 ADCIRC WW3DATA PAHM " --plat hera --compiler intel --clean -2 --thirdparty=parmetis
## Tide_fac only for adecirc
### TODO we need to include this in build.sh
### This is a hack for now
source ${ROOTDIR}/modulefiles/envmodules_intel.hera
cd ${NSEMdir}/sorc/estofs_tide_fac
make
cp -fv ${NSEMdir}/sorc/estofs_tide_fac/tide_fac $ROOTDIR/ALLBIN_INSTALL
## Tide_fac only for adecirc
# point to executeable folder in CoastalApp from CoastalApp-testsuite
ln -sfv ${ROOTDIR}/ALLBIN_INSTALL ${NSEMdir}/exec
The external files are available at Hera (NOAA RDHPCS). Please let us know if you want to test with external files for HSOFS mesh as well.
@pvelissariou1 @aliabdolali
Here at Hera you can find actual tests mentioned here:
/scratch2/COASTAL/coastal/noscrub/shared/Saeed.Moghimi/coastalapp_test/temp7
#git clone -b scalability git@github.com:awest-noaa/WW3.git WW3
git clone -b scalability https://github.com/awest-noaa/WW3.git WW3
#Switch file should be in one line (i.e. no newline)
cp /scratch2/COASTAL/coastal/save/Andre.VanderWesthuysen/CoastalApp/WW3/model/esmf/switch WW3/model/bin
#load modules
module purge
module load cmake/3.20.1
module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
module load hpc/1.1.0
module load hpc-intel/18.0.5.274
module load hpc-impi/2018.0.4
module load netcdf/4.7.4
module load jasper/2.0.25
module load zlib/1.2.11
module load png/1.6.35
module load hdf5/1.10.6
module load bacio/2.4.1
module load g2/3.4.2
module load w3emc/2.9.2
export METIS_PATH=/scratch2/COASTAL/coastal/save/Ali.Abdolali/hpc-stack/parmetis-4.0.3
export NETCDF_CONFIG=/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/intel-18.0.5.274/impi-2018.0.4/netcdf/4.7.4/bin/nc-config
export WWATCH3_NETCDF=NC4
#
# Environment variables for wavewatch III
# ---------------------------------------
# From Ali's folder
#WWATCH3_LPR printer
#WWATCH3_F90 ifort
#WWATCH3_CC icc
#WWATCH3_DIR /scratch2/COASTAL/coastal/save/Ali.Abdolali/COASTAL_APP_Oct2022/Inlet/WW3/model
#WWATCH3_TMP /scratch2/COASTAL/coastal/save/Ali.Abdolali/COASTAL_APP_Oct2022/Inlet/WW3/model/tmp
#WWATCH3_SOURCE yes
#WWATCH3_LIST yes
cd WW3/model/
./bin/w3_setup ./ -c intel
cd bin
# remove PDLIB from switch and also put the switch in one row
./w3_make ww3_grid ww3_prnc ww3_bounc ww3_ounp ww3_ounf
#add PDLIB to the switch
./w3_make ww3_multi
#now all executables are under model/exe/
cd <work directory, here HSOFS>
./WW3/model/exe/ww3_grid > ww3_grid.out
#it generates mod_def.ww3 for implicit scheme
cp mod_def.ww3 mod_def.inlet
cp mod_def.ww3 mod_def.points
./WW3/model/exe/ww3_prnc > ww3_prnc.out
#it generates wind.ww3
mv wind.ww3 wind.inlet
./WW3/model/exe/ww3_bounc > ww3_bounc.out
#it generates nest.ww3
mv nest.ww3 nest.inlet
#if you remve it, it does not use open boundary condition with no problem
#!/bin/sh --login
#SBATCH -A coastal
#SBATCH -q debug
#SBATCH -n 720
#SBATCH --nodes=60 # Number of nodes
#SBATCH --ntasks-per-node=12 # How many tasks on each node
#SBATCH -t 0:30:00
#SBATCH -J ww3_regtest
#SBATCH -o jobslurm.out
module purge
module load cmake/3.20.1
module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
module load hpc/1.1.0
module load hpc-intel/18.0.5.274
module load hpc-impi/2018.0.4
module load netcdf/4.7.4
module list
ulimit -s unlimited
ulimit -c 0
export OMP_NUM_THREADS=1
export KMP_AFFINITY=disabled
export KMP_STACKSIZE=2G
export IOBUF_PARAMS='*:size=1M:count=4:vbuffer_count=4096:prefetch=1'
cd /scratch2/COASTAL/coastal/save/Ali.Abdolali/COASTAL_APP_Oct2022/HSOFS
srun /scratch2/COASTAL/coastal/save/Ali.Abdolali/COASTAL_APP_Oct2022/HSOFS/WW3/model/exe/ww3_multi > ww3_multi.out
For WW3 boundary condition, we need to do a preprocessing with ww3_bound (if boundary conditions are ascii) or ww3_bounc (if boundary files are netcdf). It generates nest.ww3 which you need to rename to nest.inlet in your case then, when you run the simulation, looking in log.inlet, you will see a column for boundary condition (b). at each time it reads a boundary data, you will see a X (Ali communicated)
In CoastalApp we don't need to have addition compilation to generate the ww3* executables. This is already taken care of in the esmf/Makefile as follows: 1) for the switch file, the line: SWITCHES := $(shell cat switch) in the Makefile converts the column format to row format and then the switch file in the bin directory is generated accordingly, see rule: switch: 2) The executables ww3* are created without the PDLIB switch, see rule "gout:" where PDLIB is filtered out via sed
Updated the develop_build branch to properly point to the WW3 scalability branch
the main executable ww3_multi_esmf should be compiled with pdlib in the switch, if that is the case, awesome, otherwise it should be corrected
Yes, I will double check the esmf/Makefile again (as we also switch between branches). Question1: Do we need to include bacio, g2 and w3emc libraries in the module list when compiling WW3 (we are not using grib functionality in CoastalApp). If yes, then I need to include these libraries in CoastalApp for users outside NOAA. Question2: In hpc_stack all libraries are static, so why do we need to load the netcdf module? I understand the reason for loading the modules for the Intel compilers.
I am trying to update/include the hpc_stack functionality for hera, orion, stampede as well. Any other HPC platforms using the hpc_stack?
Thank you Takis
Panagiotis Velissariou, Ph.D., P.E. UCAR Scientist National Ocean and Atmospheric Administration National Ocean Service Office of Coast Survey CSDL/CMMB Physical Scientist - Project Lead cell: (205) 227-9141 email: @.***
On Fri, Oct 21, 2022 at 12:11 PM Ali.Abdolali @.***> wrote:
the main executable ww3_multi_esmf should be compiled with pdlib in the switch, if that is the case, awesome, otherwise it should be corrected
— Reply to this email directly, view it on GitHub https://github.com/noaa-ocs-modeling/CoastalApp/issues/134#issuecomment-1287230884, or unsubscribe https://github.com/notifications/unsubscribe-auth/APC7TP6324AF5CLYZLC56ETWELFDNANCNFSM6AAAAAAQLO6VTI . You are receiving this because you were mentioned.Message ID: @.***>
Yes, I will double check the esmf/Makefile again (as we also switch between branches). Question1: Do we need to include bacio, g2 and w3emc libraries in the module list when compiling WW3 (we are not using grib functionality in CoastalApp). If yes, then I need to include these libraries in CoastalApp for users outside NOAA. Question2: In hpc_stack all libraries are static, so why do we need to load the netcdf module? I understand the reason for loading the modules for the Intel compilers.
I am trying to update/include the hpc_stack functionality for hera, orion, stampede as well. Any other HPC platforms using the hpc_stack?
Thank you Takis
Panagiotis Velissariou, Ph.D., P.E. UCAR Scientist National Ocean and Atmospheric Administration National Ocean Service Office of Coast Survey CSDL/CMMB Physical Scientist - Project Lead cell: (205) 227-9141 email: @.***
On Fri, Oct 21, 2022 at 12:11 PM Ali.Abdolali @.***> wrote:
the main executable ww3_multi_esmf should be compiled with pdlib in the switch, if that is the case, awesome, otherwise it should be corrected
— Reply to this email directly, view it on GitHub https://github.com/noaa-ocs-modeling/CoastalApp/issues/134#issuecomment-1287230884, or unsubscribe https://github.com/notifications/unsubscribe-auth/APC7TP6324AF5CLYZLC56ETWELFDNANCNFSM6AAAAAAQLO6VTI . You are receiving this because you were mentioned.Message ID: @.***>
No, we do not need them for now. Grib2 is the format in operation and we need it for post- processing.
Orion and Hera would be sufficient for now. Wcoss2 is also another platform but parmeris is not allowed on those by nco.
Thanks Ali for the clarification. I'll update CoastalApp accordingly after Saeed and I finish the WW3 testing.
Takis
Panagiotis Velissariou, Ph.D., P.E. UCAR Scientist National Ocean and Atmospheric Administration National Ocean Service Office of Coast Survey CSDL/CMMB Physical Scientist - Project Lead cell: (205) 227-9141 email: @.***
On Fri, Oct 21, 2022 at 12:37 PM Ali.Abdolali @.***> wrote:
Yes, I will double check the esmf/Makefile again (as we also switch between branches). Question1: Do we need to include bacio, g2 and w3emc libraries in the module list when compiling WW3 (we are not using grib functionality in CoastalApp). If yes, then I need to include these libraries in CoastalApp for users outside NOAA. Question2: In hpc_stack all libraries are static, so why do we need to load the netcdf module? I understand the reason for loading the modules for the Intel compilers.
I am trying to update/include the hpc_stack functionality for hera, orion, stampede as well. Any other HPC platforms using the hpc_stack?
Thank you Takis
Panagiotis Velissariou, Ph.D., P.E. UCAR Scientist National Ocean and Atmospheric Administration National Ocean Service Office of Coast Survey CSDL/CMMB Physical Scientist - Project Lead cell: (205) 227-9141 email: @.***
On Fri, Oct 21, 2022 at 12:11 PM Ali.Abdolali @.***> wrote:
the main executable ww3_multi_esmf should be compiled with pdlib in the switch, if that is the case, awesome, otherwise it should be corrected
— Reply to this email directly, view it on GitHub
134 (comment)
https://github.com/noaa-ocs-modeling/CoastalApp/issues/134#issuecomment-1287230884 , or unsubscribe
https://github.com/notifications/unsubscribe-auth/APC7TP6324AF5CLYZLC56ETWELFDNANCNFSM6AAAAAAQLO6VTI . You are receiving this because you were mentioned.Message ID: @.***>
No, we do not need them for now. Grib2 is the format in operation and we need it for post- processing.
Orion and Hera would be sufficient for now. Wcoss2 is also another platform but parmeris is not allowed on those by nco.
— Reply to this email directly, view it on GitHub https://github.com/noaa-ocs-modeling/CoastalApp/issues/134#issuecomment-1287255313, or unsubscribe https://github.com/notifications/unsubscribe-auth/APC7TP2GL2TCCZBRZ5KG223WELIELANCNFSM6AAAAAAQLO6VTI . You are receiving this because you were mentioned.Message ID: @.***>
@pvelissariou1 Lets concentrate only on Hera. Make sure things work fine then we need to decide on the next steps .i.e. whether or not hpc_stack support is needed at Stampede and so on
From @pvelissariou1 previous note.
I am trying to update/include the hpc_stack functionality for hera, orion,
stampede as well. Any other HPC platforms using the hpc_stack?
Hi @pvelissariou1
I think I found another bug. I do not see the ww3_bounc in the CoastalApp exe folder. Please add and let me know to fetch the code and test. Thanks
./WW3/model/exe/ww3_bounc > ww3_bounc.out
Hi @pvelissariou1
Seems like we still have this inconsistency. Please check and update here ASAP.
In ATMESH
subroutine ATMESH_FieldsSetup
integer :: rc
character(len=*),parameter :: subname='(ATMESH:ATMESH_FieldsSetup)'
!--------- import fields to ATMESH -------------
!--------- export fields from ATMESH -------------
call fld_list_add(num=fldsFrATM_num, fldlist=fldsFrATM, stdname="stdnamestdname" , shortname= "pmsl" )
call fld_list_add(num=fldsFrATM_num, fldlist=fldsFrATM, stdname="inst_zonal_wind_height10m" , shortname= "izwh10m" )
call fld_list_add(num=fldsFrATM_num, fldlist=fldsFrATM, stdname="inst_merid_wind_height10m" , shortname= "imwh10m" )
IN ww3 cap utl.F90
! ATM export fields
i = i+1; standardName(i) = 'air_pressure_at_sea_level'
canonicalUnits(i)='Pa'
i = i+1; standardName(i) = 'eastward_wind_at_10m_height'
canonicalUnits(i)='m s-1'
i = i+1; standardName(i) = 'northward_wind_at_10m_height'
canonicalUnits(i)='m s-1'
The last instance was in an email communication between you and Andre on
from: | Andre Van der Westhuysen - NOAA Affiliate <andre.vanderwesthuysen@noaa.gov>
-- | --
to: | Panagiotis Velissariou - NOAA Affiliate <panagiotis.velissariou@noaa.gov>
cc: | Saeed Moghimi - NOAA Affiliate <saeed.moghimi@noaa.gov>
date: | Jun 14, 2022, 2:13 PM
subject: | Re: Simulations PAHM+ADCIRC+WW3... on hera
Thanks @pvelissariou1
Here is from the email.
Hi Saeed,
The develop_branch has been updated as we discussed.
WW3 points to the scalability branch of Andre (forked) with my changes
Build scripts of CoastalApp have also been updated to accommodate the hpc-stack system.
The modulefiles folder contains the envmodules_intel.hera, envmodules_intel.hera.sys and
envmodules_intel.hera.hpc-stack modulefiles.
The "hpc-stack" loads the hpc-stack modules, and the "sys" loads the system's available
modules.
The envmodules_intel.hera file is a copy of the envmodules_intel.hera.sys file.
To switch to the "hpc-stack" just copy envmodules_intel.hera.hpc-stack to envmodules_intel.hera
WW3 compiles as expected and generates the ww3_multi as well (in ALLBIN_INSTALL)
I'll continue with the tests and I'll update the issue.
Takis
@GM-Ling @pvelissariou1 @aliabdolali @FVCOM-GitHub
Guoming Ling 9:36 AM 1) follow up with UND (i.e. Guoming) and check if they can run coupled model with PDLIB activated in WW3? If the answer is yes, we should check their setup and version of WW3 in their test. Yes, I always run with PDLIB activated, but I don't have much knowledge about the PDLIB except the introductions in the user manual(https://github.com/NOAA-EMC/WW3/wiki/Manual). My setups and results for the Shinnecock test case were uploaded here: https://github.com/GM-Ling/ATMESH-ADCIRC-WW3-TestSuite.
works for Guoming
Guoming Ling 9:40 AM git log commit 216b06032ee47a10ebb7762817752843fc945fec (HEAD -> feature/pahm, origin/feature/pahm) Author: zacharyburnettNOAA zachary.burnett@noaa.gov Date: Mon Aug 9 12:36:59 2021 +0000
commit 0154723783823b7786f3930be3787dc9757f5d53 Author: pvelissariou1 panagiotis.velissariou@noaa.gov Date: Fri Aug 6 00:50:15 2021 -0500
commit 21faa6238ca8d274c1d1112e1a5f530fe6e1c5a0 Author: pvelissariou1 panagiotis.velissariou@noaa.gov