GloSea6, GNU+nvptx 11.3.1 compile

HyunChaeJung commented 1 year ago

sudo 사용은 pass. gcc+nvptx를 계정내(~/offload/install/bin)에 컴파일 해놨기 때문...
sudo 사용하려면, 리눅스 loot 디렉토리(/usr/bin 등) 쪽에 gcc, g++, gfortran을 링크해야함.
gcc11 openmp 지시문 중 shared(??) 관련 버그 있음. 라이브러리/모델 openmp off 컴파일 진행함.

[openmpi 4.0.7]

vi ./configure
```
CC=gcc
CXX=g++
FC=gfortran
```
./configure --prefix=/data1/home/synim/Library/GNU1131/openmpi/4.0.7
make
make install

[GCOM]

vi kma_linux_gfortran_mpich3.cfg

$fflags           = -std=f2003 -Wmissing-include-dirs -fallow-argument-mismatch
$gcg_asum_flags   =     $fflags #-fopenmp
$ldflags          =       #-fopenmp

./kma_linux_ext_bld.scr

[zlib]

./configure --prefix=/data1/home/synim/Library/GNU1131/zlib/1.2.11
make
make install

[hdf5]

./configure --prefix=/data1/home/synim/Library/GNU1131/hdf5/1.10.3 --enable-fortran -enable-cxx --with-zlib=/data1/home/synim/Library/GNU1131/zlib/1.2.11 --enable-hl
make clean
make
make install

[netcdf-c]

vi configure

export CPPFLAGS=-I/data1/home/synim/Library/GNU1131/hdf5/1.10.3/include
export LDFLAGS=-L/data1/home/synim/Library/GNU1131/hdf5/1.10.3/lib

./configure --prefix=/data1/home/synim/Library/GNU1131/netcdf/4.7.3 --enable-netcdf-4 --disable-dab
make clean
make
make install

[netcdf-fortran]

vi configure

export CC=gcc
export FC=gfortran
export CPPFLAGS=-I/data1/home/synim/Library/GNU1131/netcdf/4.7.3/include
export LDFLAGS=-L/data1/home/synim/Library/GNU1131/netcdf/4.7.3/lib
export FCFLAGS=-fallow-argument-mismatch

./configure --prefix=/data1/home/synim/Library/GNU1131/netcdf/netcdf-fortran-4.4.4
make clean
make
make install

[cmake]

vi bootstrap

export CC=/data1/home/synim/offload/install/bin/gcc
export CXX=/data1/home/synim/offload/install/bin/g++

./bootstrap --prefix=/data1/home/synim/Library/GNU1131/cmake/3.20.2
make
make install

[eccodes]

vi CMakeLists.txt

set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch")


eccode 설치 시 주의 다음과 같이 옵션을 넣어줘야 11.3 버전의 컴파일러로 설치함.
지정하지 않을 경우 /usr/bin/cc 를 사용함
-D CMAKE_C_COMPILER=<컴파일러위치>
-D CMAKE_FORTRAN_COMPILER=<컴파일러위치>

cmake -D CMAKE_C_COMPILER=... -D CMAKE_C_COMPILER=... -D CMAKE_INSTALL_PREFIX=/usr/local/eccodes/2.9.2/GNU/11.3 ../

- cmake -DCMAKE_FORTRAN_COMPILER=/data1/home/synim/offload/install/bin/gfortran -DCMAKE_C_COMPILER=/data1/home/synim/offload/install/bin/gcc -DCMAKE_INSTALL_PREFIX=/data1/home/synim/Library/GNU1131/eccodes/2.9.2 ../
- make
- make install
- ctest

### [OASIS3-MCT]
- vi make.inc

include $(HOME)/oasis3-mct_4.0/util/make_dir/make.gfortran_openmpi_linux

- vi make.gfortran_openmpi_linux

ARCHDIR = $(COUPLE)/compile_oa3-mct_gnu_openmpi

MPIDIR = /data1/home/synim/Library/GNU1131/openmpi/4.0.7 NETCDF_INCLUDE1 = /data1/home/synim/Library/GNU1131/netcdf/4.7.3/include NETCDF_INCLUDE2 = /data1/home/synim/Library/GNU1131/netcdf/netcdf-fortran-4.4.4/include NETCDF_LIBRARY = -L/data1/home/synim/Library/GNU1131/netcdf/4.7.3/lib -L/data1/home/synim/Library/GNU1131/netcdf/netcdf-fortran-4.4.4/lib -lnetcdff -lnetcdf

F90 = $(MPIBIN)/mpif90 -I$(MPI_INCLUDE) -fallow-argument-mismatch F90FLAGS_1 = -ffree-line-length-none #-fopenmp LDFLAGS = #-fopenmp

- make cleanlibs -f TopMakefileOasis3
- make -f TopMakefileOasis3

### [XIOS]
- vi arch-GCC_LINUX.fcm

%BASE_FFLAGS -DNONE -Wno-maybe-uninitialized -Wno-unused-dummy-argument -Wno-unused-parameter -Wno-compare-reals -Wno-integer-division -ffree-line-length-none -fallow-argument-mismatch

- vi arch-GCC_LINUX.env

export MPI_INC_DIR=$MPICH_HOME/include export MPI_LIB_DIR=$MPICH_HOME/lib export HDF5_INC_DIR=$HDF5_HOME/include export HDF5_LIB_DIR=$HDF5_HOME/lib export NETCDF_INC_DIR=$NETCDF/include export NETCDF_LIB_DIR=$NETCDF/lib export NETCDFF_INC_DIR=$NETCDFF/include export NETCDFF_LIB_DIR=$NETCDFF/lib

- vi arch-GCC_LINUX.path

NETCDF_INCDIR="-I$NETCDF/include -I$NETCDFF/include" NETCDF_LIBDIR="-L$NETCDF/lib -L$NETCDFF/lib" NETCDF_LIB="-lnetcdff -lnetcdf"

MPI_INCDIR="-I$MPICH_HOME/include" MPI_LIBDIR="-L$MPICH_HOME/lib" MPI_LIB="-lmpi"

HDF5_INCDIR="-I$HDF5_HOME/include" HDF5_LIBDIR="-L$HDF5_HOME/lib" HDF5_LIB="-lhdf5_hl -lhdf5 -lhdf5 -lz"

BOOST_INCDIR="" BOOST_LIBDIR="" BOOST_LIB=""

OASIS_INCDIR="" OASIS_LIBDIR="" OASIS_LIB=""


- ./make_xios --prod --full --arch GCC_LINUX --jobs 8

HyunChaeJung commented 1 year ago

[CTLDATA1, fcm:um.xm/tags/vn11.3]

vi ./vn11.3/rose-stem/site/kma/family-linux.rc

PATH = /data1/home/synim/Library/GNU1131/openmpi/4.0.7/bin:$PATH

vi ./vn11.3/rose-stem/app/fcm_make_libs, fcm_make_utils_serial, fcm_make_utils_mpp, fcm_make_createbc/rose-app.conf
```
config_root_path=fcm:um_nims/branches/dev/synim/r20_um.xm_tr_vn11.3_LowGS6
```

vi ./vn11.3/rose-stem/rose-suite.conf

HOST_SOURCE_UM_BASE='fcm:um.xm_br/dev/synim/r20_um.xm_tr_vn11.3_LowGS6'

[CTLDATA2, fcm:um.xm_br/dev/synim/r20_um.xm_tr_vn11.3_LowGS6]

vi ~/r20_um.xm_tr_vn11.3_LowGS6/fcm-make/kma-x86-gnu/inc/parallel.cfg


$fcflags_prec{?} = -fdefault-integer-8 -fdefault-real-8 -fdefault-double-8 -fallow-argument-mismatch
$fc{?} = mpif90

$ccflags{?} = -g -O3 -std=c99 -Wall -Wextra -Wformat=2 -Winit-self \ \ -Wfloat-equal -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wcast-align \ \ -Wconversion -Wlogical-op -Wstrict-prototypes -Wmissing-declarations \ \ -Wredundant-decls -Wnested-externs -Woverlength-strings $ccflags_omp \ \ -fdiagnostics-show-option -Wno-format-truncation

- vi ~/r20_um.xm_tr_vn11.3_LowGS6/fcm-make/kma-x86-gnu/um-utils-mpp-high.cfg, um-createbc-high.cfg, um-libs-high.cfg, um-utils-serial-high.cfg

$fcflags_level = -O3 #-Werror


- vi ./r20_um.xm_tr_vn11.3_LowGS6/fcm-make/kma-x86-gnu/inc/external_paths.cfg

$gcom_root_path{?} = $lib_root_path/gcom/gcom6.8_gcc_openmpi $eccodes_root_path{?} = /data1/home/synim/Library/GNU1131/eccodes/2.9.2 $prism_path{?} = /data1/home/synim/oasis3-mct_4.0/compile_oa3-mct_gnu_openmpi $netcdf_path{?} = /data1/home/synim/Library/GNU1131/netcdf/4.7.3 $netcdff_path{?} = /data1/home/synim/Library/GNU1131/netcdf/netcdf-fortran-4.4.4

HyunChaeJung commented 1 year ago

[MOCI 설정 변경, fcm:moci.xm_br/dev/synim/r22_r2065_moci.xm_tr]

vi ./r22_r2065_moci.xm_tr/fcm-make/kma-x86-gnu/inc/parallel.cfg

$fcflags_prec{?} = -fdefault-integer-8 -fdefault-real-8 -mcmodel=medium -fallow-argument-mismatch
$cc{?} = mpicc 
$fc{?} = mpif90

vi ./r22_r2065_moci.xm_tr/fcm-make/kma-x86-gnu/inc/external_paths.cfg

$gcom_root_path{?} = /data1/home/synim/TI/gcom/gcom6.8_gcc_openmpi
$prism_path{?} = /data1/home/synim/oasis3-mct_4.0/compile_oa3-mct_gnu_openmpi
$netcdf_path{?} = /data1/home/synim/Library/GNU1131/netcdf/4.7.3
$netcdff_path{?} = /data1/home/synim/Library/GNU1131/netcdf/netcdf-fortran-4.4.4
$hdf5_path{?} = /data1/home/synim/Library/GNU1131/hdf5/1.10.3
$zlib_path{?} = /data1/home/synim/Library/GNU1131/zlib/1.2.11

HyunChaeJung commented 1 year ago

[Suite 설정 변경, fcm:kmt.xm/a/b/3/4/7/r3316_N96eORCA1_kmt-ab447_mirae]

vi ./r3316_N96eORCA1_kmt-ab447_mirae/suite.rc

UMDIR           = /data1/home/synim/TI/UM
owner = synim
REBUILD_NEMO = /data1/home/synim/TI/OCEAN/nemo/utils/src/REBUILD_NEMO/BLD/bin/rebuild_nemo.exe
ROSE_DATAC_FOAM  = /data1/home/synim/TI/OCEAN/hadgem3/initial/dumps/init_ORCA1/${YMDH_CYCLE_TIME}
ROSE_DATAC_GLOBAL = /data1/home/synim/TI/OCEAN/hadgem3/initial/dumps/init_ORCA1/${YMDH_CYCLE_TIME}
REBUILD_NEMO     = /data1/home/synim/TI/OCEAN/nemo/utils/src/REBUILD_NEMO/BLD/bin/rebuild_nemo.exe

vi ./r3316_N96eORCA1_kmt-ab447_mirae/app/nemo_cice/rose-app.conf

NEMO_INIT_DIR=/data1/home/synim/TI/OCEAN/hadgem3/initial/dumps/init_ORCA1

vi ./r3316_N96eORCA1_kmt-ab447_mirae/app/install_cold/rose-app.conf

AUX_ROOT=/data1/home/synim/TI
source=/data1/home/synim/oasis3-mct_4.0
source=/data1/home/synim/TI/OCEAN/nemo/utils/bin
source=/data1/home/synim/TI/OCEAN/nemo/nco/nco-4.4.7/bin
source=/data1/home/synim/TI/OCEAN/nemo/utils/src/REBUILD_NEMO/BLD/bin/rebuild_nemo.exe

vi ./r3316_N96eORCA1_kmt-ab447_mirae/app/fcm_make_um/file/fcm-make.cfg, fcm-make_main.cfg, fcm-make_slash.cfg

$flags_coupling = -I$prism_path/build/lib/psmile.MPI1 -I$prism_path/build/lib/mct -I$prism_path/build/lib/mctdir -I$prism_path/build/lib/scrip

vi ./r3316_N96eORCA1_kmt-ab447_mirae/app/fcm_make_ocean/rose-app.conf


!! -Wno-argument-mismatch -g -pedantic -fbacktrace 옵션 삭제
!! -Wmissing-include-dirs -fallow-argument-mismatch 옵션 추가
fcflags_cice_overrides=-fstack-arrays -ftree-vectorize -Wmissing-include-dirs -Wall -Wextra -fcheck=all -I/data1/home/synim/Library/GNU1131/netcdf/include -L/data1/home/synim/Library/GNU1131/netcdf/lib -I/data1/home/synim/Library/GNU1131/netcdf/netcdf-fortran-4.4.4/include -L/data1/home/synim/Library/GNU1131/netcdf/netcdf-fortran-4.4.4/lib -lnetcdf -lnetcdff -ffree-line-length-512 -Wno-compare-reals -finit-real=ZERO -finit-integer=0 -fdefault-real-8 -fdefault-double-8 -fallow-argument-mismatch

fcflags_nemo_overrides=-fstack-arrays -ftree-vectorize -Wmissing-include-dirs -Wall -Wextra -fcheck=all -I/data1/home/synim/Library/GNU1131/netcdf/4.7.3/include -L/data1/home/synim/Library/GNU1131/netcdf/4.7.3/lib -I/data1/home/synim/Library/GNU1131/netcdf/netcdf-fortran-4.4.4/include -L/data1/home/synim/Library/GNU1131/netcdf/netcdf-fortran-4.4.4/lib -lnetcdf -lnetcdff -ffree-line-length-512 -Wno-compare-reals -finit-real=ZERO -finit-integer=0 -fdefault-real-8 -fdefault-double-8 -fallow-argument-mismatch

fcflags_prec=-fdefault-real-8 -fdefault-double-8 -fallow-argument-mismatch

config_root_path=fcm:moci.xm_br/dev/synim/r22_r2065_moci.xm_tr prism_path = /data1/home/synim/oasis3-mct_4.0/compile_oa3-mct_gnu_openmpi xios_path=/data1/home/synim/xios-2.0_clena

- vi ./r3316_N96eORCA1_kmt-ab447_mirae/rose-suite.conf

gcom_path='/data1/home/synim/TI/gcom/gcom6.8_gcc_openmpi/kma_linux_gfortran_mpich3/build'

- vi ./r3316_N96eORCA1_kmt-ab447_mirae/app/fcm_make_um/rose-app.conf

config_root_path=fcm:um.xm_br/dev/synim/r20_um.xm_tr_vn11.3_LowGS6 prism_path=/data1/home/synim/oasis3-mct_4.0/compile_oa3-mct_gnu_openmpi

- vi ./r3316_N96eORCA1_kmt-ab447_mirae/app/fcm_make_ocean/file/fcm-make.cfg

$flags_coupling = -I\$prism_path/build/lib/psmile.MPI1 -I\$xios_path/inc -I\$prism_path/build/lib/mct -I\$prism_path/build/lib/mctdir -I\$prism_path/build/lib/scrip

HyunChaeJung commented 1 year ago

[UM src original code : rivision 38]

ctldata script, vn11.3

~/vn11.3/rose-stem/app/fcm_make_libs/rose-app.conf:um_sources=fcm:um_nims/branches/dev/synim/r68449_vn11.3_GloSeaFixes_gnu11
~/vn11.3/rose-stem/app/fcm_make_utils_mpp/rose-app.conf:um_sources=fcm:um_nims/branches/dev/synim/r68449_vn11.3_GloSeaFixes_gnu11
~/vn11.3/rose-stem/app/fcm_make_utils_serial/rose-app.conf:um_sources=fcm:um_nims/branches/dev/synim/r68449_vn11.3_GloSeaFixes_gnu11
~/vn11.3/rose-stem/app/fcm_make_createbc/rose-app.conf:um_sources=fcm:um_nims/branches/dev/synim/r68449_vn11.3_GloSeaFixes_gnu11

suite script, r3316_N96eORCA1_kmt-ab447_mirae


~/r3316_N96eORCA1_kmt-ab447_mirae/app/fcm_make_um/rose-app.conf:um_base=branches/dev/synim/r68449_vn11.3_GloSeaFixes_gnu11
                                                     um_sources=fcm:um_nims/branches/dev/synim/r68449_vn11.3_GloSeaFixes_gnu11

HyunChaeJung commented 1 year ago

[NEMO, CICE src original code : revision head]

r3316_N96eORCA1_kmt-ab447_mirae

~/r3316_N96eORCA1_kmt-ab447_mirae/app/fcm_make_ocean/rose-app.conf
: cice_base=branches/pkg/Config/vn5.1.2_GSI8.0_package_branch
: cice_sources=fcm:cice.xm/branches/pkg/Config/vn5.1.2_GSI8.0_package_branch@HEAD
: nemo_base=branches/UKMO/dev_r5518_GO6_package
: nemo_sources=fcm:NEMO.xm/branches/UKMO/dev_r5518_GO6_package

HyunChaeJung commented 1 year ago

[UM src 수정, fcm:um_nims/branches/dev/synim/r68449_vn11.3_GloSeaFixes_gnu11]

아래 에러 발생시,

Error: Variable ‘jro2’ at (1) not definable inside loop beginning at (2) as INTENT(INOUT) argument to subroutine ‘ereport’

fcm co fcm:um_nims/branches/dev/philipdavis/r68449_vn11.3_GloSeaFixes@38
mv r68449_vn11.3_GloSeaFixes r68449_vn11.3_GloSeaFixes_gnu11
vi ~/r68449_vn11.3_GloSeaFixes_gnu11/src/atmosphere/UKCA/ukca_chemistry_ctl_col_mod.F90, ukca_chemistry_ctl.F90

jro22 생성 및 적용 INTEGER :: jro2, jro22 ! loop variable - NTP RO2 species

      jro22=jro2
      WRITE(umMessage,'(A)') '** ERROR in ukca_chemistry_ctl'
      CALL umPrint(umMessage,src='ukca_chemistry_ctl')
      cmessage='ERROR: Indices for RO2 species do not match w/ nadvt'
      CALL ereport('UKCA_CHEMISTRY_CTL', jro22, cmessage)

fcm import r68449_vn11.3_GloSeaFixes_gnu11/ fcm:um_nims/branches/dev/synim/r68449_vn11.3_GloSeaFixes_gnu11
suite 스크립트 및 vn11.3 스크립트 의 um src 위치 수정

HCJung-jbnu / Autocoding