search

HCJung-jbnu / GRIMs-Chem-NEMO-coupling

GRIMS + NEMO coupling
0 stars 0 forks source link

GRIMs-Chem + NEMO coupling 개요 #17

Closed HyunChaeJung closed 2 weeks ago

HyunChaeJung commented 1 year ago

namcouple

 $NFIELDS
  7
 $END
##########################################################################
 $RUNTIME
  432000
 $END
##########################################################################
 $NLOGPRT
  10
 $END
##########################################################################
 $STRINGS
###############################OCN -> ATM
## NEMO -> GRIMs, SST
O_SSTSST sst 1 10800 1 restart.nc EXPORTED
182 149 192 94 ocnT atmT
P 2 P 0
SCRIPR
CONSERV LR SCALAR LATLON 50 FRACAREA SECOND
## NEMO -> GRIMs, Ice fraction
OIceFrc icefrac 1 10800 1 restart.ne EXPORTED
182 149 192 94 ocnT atmT
P 2 P 0
SCRIPR
CONSERV LR SCALAR LATLON 50 FRACAREA SECOND
###############################ATM -> OCN
## GRIMs -> NEMO, taux
taux O_OTaux1 1 10800 1 restart.nc EXPORTED
192 94 182 149 atmT ocnU
P 0 P 2
SCRIPR
BILINEAR LR SCALAR LATLON 1
## GRIMs -> NEMO, tauy
tauy O_OTauy1 1 10800 1 restart.nc EXPORTED
192 94 182 149 atmT ocnV
P 0 P 2
SCRIPR
BILINEAR LR SCALAR LATLON 1
## GRIMs -> NEMO, solar radiation
qsrmix O_QsrOce 1 10800 1 restart.nc EXPORTED
192 94 182 149 atmT ocnT
P 0 P 2
SCRIPR
CONSERV LR SCALAR LATLON 50 FRACAREA SECOND
## GRIMs -> NEMO, non solar radiation
#qnsmix O_QnsOce 1 28800 1 restart.nc EXPORTED
#192 94 182 149 atmT ocnT SEQ=+1
#P 0 P 2
#SCRIPR
#CONSERV LR SCALAR LATLON 50 FRACAREA SECOND
## GRIMs -> NEMO, non solar sensitivity
dqnsdt O_dQnsdT 1 10800 1 restart.nc EXPORTED
192 94 182 149 atmT ocnT
P 0 P 2
SCRIPR
CONSERV LR SCALAR LATLON 50 FRACAREA SECOND
##########################################################################
 $END
HyunChaeJung commented 1 year ago

GRIMs-Chem 소스코드 수정사항

~/GRIMs-Chem/srcs/src/cpl/sfc_cpl_oasis3Mct.F90 추가

~/GRIMs-Chem/srcs/src/mpi/mpbcastl.F90, commpi.F90, mpabort.F90, mpxk2f.F90, mpinit.F90, mpbcastc.F90, mpyk2f.F90, mpgetken.F90, mpgetspd.F90, mpsynall.F90, mpgf2p.F90, mpgf2xk.F90, mpgf2yk.F90, mpgfpk2fk.F90, mpgp2f.F90, mpsf2p.F90, mpsp2f.F90, mpbcasti.F90, mpbcastr.F90

oasis_init_localcomm에서 생성한 GRIMs 모델의 local_comm을 모델에 등록 및
mpi 통신에 활용하도록 수정
mpi_comm_world ==> local_comm

~/GRIMs-Chem/srcs/src/fcst/gmp_init_array.F90

   use commpi,    only   : mype, latlen, latstr, latdef, local_comm !!hcjung 20221207

!hcjung 20171222
   USE sfc_cpl_oasis3MCT, ONLY : cpl_init
!hcjung 20171222

   !!hcjung 20221211
   integer             :: localcomm
   !1hcjung 20221211

   call mpi_init(info)
!
   if( info.ne.0 ) then
     print *,'PE',mype,':error code from mpi_init = ',info
     call exit(1)
   endif
!

!hcjung 20221128
   CALL cpl_init(localcomm)
   local_comm=localcomm
!hcjung 20221128

~/GRIMs-Chem/srcs/src/fcst/gmp_start.F90

!!hcjung 20221207
   USE paramter, ONLY: idim, jdim
   USE sfc_cpl_oasis3Mct, ONLY : cpl_def_get_var, cpl_def_put_var
   USE sfc_cpl_oasis3Mct, ONLY : cpl_def_box_partition
   USE sfc_cpl_oasis3Mct, ONLY : cpl_def_orange_partition
   USE sfc_cpl_oasis3Mct, ONLY : cpl_enddef
!!hcjung 20221207

!!hcjung 20221207
print*,'hcjung.. def part'
   !! get(NEMO->sst->GRIMs) global box partition
   CALL cpl_def_box_partition(2, 1, idim, jdim)
   !! put(GRIMs->NEMO) local orange partition
   CALL cpl_def_orange_partition(2, 2, latlen(mype), latstr(mype), &
                                 latdef(:), idim, jdim)

   !! get var 2 = sst, icefrac
   CALL cpl_def_get_var(2, 1, 1, 'sst')
   CALL cpl_def_get_var(2, 2, 1, 'icefrac')
   !! put var 4 = taux, tauy, qsrmix, qnsmix, dqnsdt
   CALL cpl_def_put_var(4, 1, 2, 'taux')
   CALL cpl_def_put_var(4, 2, 2, 'tauy')
   CALL cpl_def_put_var(4, 3, 2, 'qsrmix')
!   CALL cpl_def_put_var(5, 4, 2, 'qnsmix')
   CALL cpl_def_put_var(4, 4, 2, 'dqnsdt')
   CALL cpl_enddef
!!hcjung 20221207

~/GRIMs-Chem/srcs/src/fcst/gmp_integrate.F90

   use commpi  !!, only                  : mype,npes, master !!hcjung 20221128

!!hcjung 20221208
   USE comgrad, ONLY : raddt
   USE paramter, ONLY : idim, jdim
   USE sfc_cpl_oasis3Mct, ONLY : cpl_send, cpl_recv, cpl_sec
   USE sfc_cpl_oasis3Mct, ONLY : sst_data, cpl_mask, save_cplvar
   USE module_trans
!!hcjung 20221208

!!hcjung 20221211
   REAL, ALLOCATABLE :: z_dqlw(:,:), qla_ice(:,:), dqla_ice(:,:), z_dqsb(:,:)
   REAL, ALLOCATABLE :: dummy_data(:,:)
   INTEGER :: ii, jj, kk, cnt
!!hcjung 20221211

!!hcjung save before u/v stress, qns
  IF ( .NOT. ALLOCATED(save_cplvar) ) ALLOCATE(save_cplvar(LONF2S, LATG2S, 4))
  save_cplvar(:,:,1) = dusfc(:,:)
  save_cplvar(:,:,2) = dvsfc(:,:)
  IF ( STEPONE .OR. MOD(REAL(cpl_sec), raddt) .EQ. 0 ) THEN
    save_cplvar(:,:,3) = fluxr(:,:,4)
  END IF
!!hcjung save before u/v stress, qns

........ after dyn -> phys

!! hcjung 20221207 send to NEMO (oasis3-mct)
!! use comfphys, only: dusfc, dvsfc, dlwsfc, dtsfc, dqsfc
!! use comfphys, only: t2m, q2m, u10m, v10m
!! use comsfc, only: tsea
!! use radiag, only: fluxr
IF ( kdt < 3 ) cpl_sec=INT(shour)
IF ( kdt >= 3 ) cpl_sec=cpl_sec+INT(deltim)

IF ( .NOT. ALLOCATED(z_dqlw) ) ALLOCATE(z_dqlw(LONF2S, LATG2S))
IF ( .NOT. ALLOCATED(qla_ice) ) ALLOCATE(qla_ice(LONF2S, LATG2S))
IF ( .NOT. ALLOCATED(dqla_ice) ) ALLOCATE(dqla_ice(LONF2S, LATG2S))
IF ( .NOT. ALLOCATED(z_dqsb) ) ALLOCATE(z_dqsb(LONF2S, LATG2S))
z_dqlw(:,:) = 4.0 * 0.95 * 5.67e-8 * tsea(:,:)**3
qla_ice(:,:) = 1 * MAX(0.e0, 1.22 * 2.839e6 * 1.4E-3 * sqrt(u10m(:,:)**2 + v10m(:,:)**2) &
             * ( 11637800.0 * EXP( -5897.8 / tsea(:,:) )  / 1.22 - q2m(:,:) ) )
DO jj = 1, LATG2S
DO ii = 1, LONF2S
  IF ( qla_ice(ii,jj) > 0.0 ) THEN
    dqla_ice(ii,jj) = 1 * (-2.839e6) * 1.4e-3 * 11637800.0 * (-5897.8) * sqrt(u10m(ii,jj)**2 + v10m(ii,jj)**2) &
                      / tsea(ii,jj)**2 * EXP( -5897.8 / tsea(ii,jj) )
  ELSE
    dqla_ice(ii,jj) = 0.0
  END IF
END DO
END DO
z_dqsb(:,:) = 1.22 * 1000.5 * 1.4e-3 * sqrt(u10m(ii,jj)**2 + v10m(ii,jj)**2)

IF ( mype .EQ. master ) THEN
        print*,'hcjung check u',u10m(1:10,1)
        print*,'hcjung check v',v10m(1:10,1)
        print*,'hcjung check tsea',tsea(1:10,5)
END IF

IF ( kdt .EQ. 1 ) THEN
  CALL cpl_send(1, 0, LONF2S, latlen(mype), (dusfc(:,:)-save_cplvar(:,:,1))/REAL(deltim))
  CALL cpl_send(2, 0, LONF2S, latlen(mype), (dvsfc(:,:)-save_cplvar(:,:,2))/REAL(deltim))
  CALL cpl_send(3, 0, LONF2S, latlen(mype), (fluxr(:,:,4)-save_cplvar(:,:,3))/REAL(raddt))
  !CALL cpl_send(4, 0, LONF2S, LATG2S, (dtsfc(:,:)+dqsfc(:,:))*rtime)
  CALL cpl_send(4, 0, LONF2S, latlen(mype), (z_dqlw(:,:)+dqla_ice(:,:)+z_dqsb(:,:)))
ELSE
  CALL cpl_send(1, cpl_sec, LONF2S, latlen(mype), (dusfc(:,:)-save_cplvar(:,:,1))/REAL(deltim))
  CALL cpl_send(2, cpl_sec, LONF2S, latlen(mype), (dvsfc(:,:)-save_cplvar(:,:,2))/REAL(deltim))
  CALL cpl_send(3, cpl_sec, LONF2S, latlen(mype), (fluxr(:,:,4)-save_cplvar(:,:,3))/REAL(raddt))
  !CALL cpl_send(4, cpl_sec, LONF2S, LATG2S, (dtsfc(:,:)+dqsfc(:,:))*rtime)
  CALL cpl_send(4, cpl_sec, LONF2S, latlen(mype), (z_dqlw(:,:)+dqla_ice(:,:)+z_dqsb(:,:)))
END IF
DEALLOCATE(z_dqlw, qla_ice, dqla_ice, z_dqsb)

!!==========hcjung, recieve from NEMO
IF ( kdt .EQ. 1 ) THEN !!dummy recieve
  ALLOCATE(dummy_data(LONF2S, LATG2S)); dummy_data=0.0
  CALL cpl_recv(1, 0, LONF2S, latlen(mype), dummy_data)
  dummy_data=0.0
  CALL cpl_recv(2, 0, LONF2S, latlen(mype), dummy_data) !!tmp icetemp
  dummy_data=0.0
  CALL cpl_recv(3, 0, LONF2S, latlen(mype), dummy_data) !!tmp icefrac
  DEALLOCATE(dummy_data)
ELSE
  ALLOCATE(dummy_data(LONF2S, LATG2S)); dummy_data=0.0
  CALL cpl_recv(1, cpl_sec, LONF2S, latlen(mype), dummy_data)

  IF ( cpl_sec .NE. 0 .AND. MOD(cpl_sec, 10800) .EQ. 0 ) THEN
    ALLOCATE(cpl_mask(LONF2S*LATG2S)); cpl_mask=0.0
    ALLOCATE(sst_data(LONF2S*LATG2S)); sst_data=0.0
    cnt = 0
    DO jj = 1, LATG2S
    DO ii = 1, LONF2S
      cnt = cnt + 1
      IF ( dummy_data(ii,jj) .NE. 0.0 ) THEN
        sst_data(cnt) = dummy_data(ii,jj)
      ELSE
        cpl_mask(cnt) = 1.0
      END IF
    END DO
    END DO
  END IF

  dummy_data=0.0
  CALL cpl_recv(2, cpl_sec, LONF2S, latlen(mype), dummy_data) !!tmp icetemp

  dummy_data=0.0
  CALL cpl_recv(3, cpl_sec, LONF2S, latlen(mype), dummy_data) !!tmp icefrac
  DEALLOCATE(dummy_data)
END IF
!!===========hcjung, recieve from NEMO
 ......

~/GRIMs-Chem/srcs/src/sfcl/sfc_grib_file.F90

   nsfc=1
   do nsfcv = 1,numsfcv
     ngrb=is2g(nsfc)
     if(ngrb.ne.9999) then
       if(fn(ngrb)(1:4).ne.'    ') then
         IF ( nsfcv .NE. 1 ) THEN !!exclude sst hcjung 20221226
           lsf(nsfc)=.true.
           call sfc_grib_solver(lugb,fn(ngrb),idim,jdim,slmask,                  &
                            ngrb,iy,im,id,ih,fh,                                 &
                            fnmskg,grbfld(1,nsfc))
         END IF !!hcjung 20221226
       else
         lsf(nsfc)=.false.
       endif
     endif
     nsfc=nsfc+ksfc(nsfc)
   enddo

~/GRIMs-Chem/srcs/src/sfcl/sfc_cycle_driver.F90


USE sfc_cpl_oasis3Mct, only: sst_data, cpl_mask !!hcjung 20221227

!!hcjung update sst & ocean mask(=0)
   grbfldp(:,1) = sst_data(:) !!hcjung 20221226
   slmaskp(:) = cpl_mask(:)   !!hcjung 20221226
   DEALLOCATE(sst_data, cpl_mask)
!!==================================
HyunChaeJung commented 1 year ago

NEMO 수정 사항

~/OPA_SRC/nemogcm.F90

         DO WHILE ( istp <= nitend .AND. nstop == 0 )
            IF (lk_oasis) CALL sbc_cpl_snd( istp ) !!Coupling to atmos !! hcjung 20221214
#if defined key_agrif
            CALL stp                         ! AGRIF: time stepping
#else
            CALL stp( istp )                 ! standard time stepping
#endif
            IF ( istp == nitend ) THEN !!hcjung 20221229
              IF (lk_oasis) CALL sbc_cpl_snd( istp+nit000 ) !!Coupling to atmos !! hcjung 20221214
              IF (lk_oasis) CALL sbc_cpl_rcv( istp+nit000, nn_fsbc, nn_ice) !!dummy rcv !!hcjung 20221214
            END IF  !!hcjung 20221229

            istp = istp + 1
!            IF (lk_oasis) CALL sbc_cpl_snd( istp ) !!Coupling to atmos !! hcjung 20221214
            IF( lk_mpp )   CALL mpp_max( nstop )
         END DO

~/OPA_SRC/step.F90

      !>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
      ! Coupled mode
      !<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
!      IF( lk_oasis         )   CALL sbc_cpl_snd( kstp )     ! coupled mode : field exchanges !!hcjung  <--- send 위치 수정
      !

~/OPA_SRC/SBC/sbccpl.F90

ln_cpl = .false. -> .true.

! send ##hcjung 20221117 sn_snd_temp = 'oce only' , 'no' , '' , '' , '' sn_snd_alb = 'none' , 'no' , '' , '' , '' sn_snd_thick = 'none' , 'no' , '' , '' , '' sn_snd_crt = 'none' , 'no' , 'spherical' , 'eastward-northward' , 'T' sn_snd_co2 = 'none' , 'no' , '' , '' , '' ! receive ##hcjung 20221117 sn_rcv_w10m = 'none' , 'no' , '' , '' , '' sn_rcv_taumod = 'none' , 'no' , '' , '' , '' sn_rcv_tau = 'oce only' , 'no' , 'spherical' , 'eastward-northward', 'U,V,F' sn_rcv_dqnsdt = 'coupled' , 'no' , '' , '' , '' sn_rcv_qsr = 'oce only' , 'no' , '' , '' , '' sn_rcv_qns = 'none' , 'no' , '' , '' , '' sn_rcv_emp = 'none' , 'no' , '' , '' , '' sn_rcv_rnf = 'none' , 'no' , '' , '' , '' sn_rcv_cal = 'none' , 'no' , '' , '' , '' sn_rcv_iceflx = 'none' , 'no' , '' , '' , '' sn_rcv_co2 = 'none' , 'no' , '' , '' , '' !! send

nn_cplmodel = 1 ! Maximum number of models to/from which NEMO is potentialy sending/receiving data

### GRIMs-NEMO 수행을 위한 ice namelist 설정 <- 연구필요
- error message: In coupled mode, use parsub = 0. or send dqla
- namelist_ice_ref

&namicethd ! ice thermodynamic parsub = 1.0 -> 0.0 ! switch for snow sublimation or not on 하기 위해서는 dqla 변수를 교환해야 할듯


**- nn_fsbc = 5->4 :: 혹시 coupling dt와 동일해야 하는가? 3=10800sec**
HyunChaeJung commented 1 year ago

GRIMs 스크립트 기반의 커플 수행 환경 setting

커플 수행 환경 setting

export NEMO_WORK_DIR=/home/grimsjung/hcjung/cpl_test/nemo_v3_6_stable export NEMO_INPUT_DIR=/home/grimsjung/hcjung/cpl_test/nemo_orca2_data export CPL_GRID_DIR=/home/grimsjung/hcjung/cpl_test/cpl_data/GRIMs-NEMO export NEMO_EXEC=${NEMO_WORK_DIR}/CONFIG/ORCA2_LIM_PISCES/BLD/bin/nemo.exe

cp -f ${CPL_GRID_DIR}/ ${EXECDIR} cp -f ${NEMO_WORK_DIR}/EXP00/ ${EXECDIR} cp -f ${NEMO_INPUT_DIR}/* ${EXECDIR} cp -f ${NEMO_EXEC} ${EXECDIR}

chmod 777 ${EXECDIR} ############################################

@FCSTENV@ ./$PROG.x 1>fcstout.ft$fh 2>&1 #GRIMs-Chem single mode

@FCSTENV@ -np @FCST_NPES@ ./$PROG.x : -np @NEMO_NPES@ nemo.exe 1>fcstout.ft$fh 2>&1

- ~/GRIMs-Chem/runs/configure-run  <-- MPI 수행을 위해 수정

NCPUS=1 # number of threads (when MARCH = thread or hybrid) GRIMS_NPES=1 NEMO_NPES=1 NPES=${GRIMS_NPES} TNPES=expr ${GRIMS_NPES} + ${NEMO_NPES} # number of MPI pes (when MARCH = mpi or hybrid) NCPU_PER_NODE=36 # number of total cores per node NCOL=1 # number of column (when MARCH=mpi or hybrid) #

if [ $march = 'mpi' ]; then

if [ $npes -le $NCPU_PER_NODE ]; then ##HCJUNG 20221229

    if [ ${TNPES} -le $NCPU_PER_NODE ]; then ##HCJUNG 20221229
            NNODES=1

NPES_PER_NODE=$npes ##hcjung 20221229

            NPES_PER_NODE=${TNPES}  ##hcjung 20221229

NCPU_PER_NODE=$npes # forced

npes=$npes

    else

NNODES=echo "($npes-1)/$NCPU_PER_NODE+1" | bc ##hcjung 20221229

            NNODES=`echo "(${TNPES}-1)/$NCPU_PER_NODE+1" | bc` ##hcjung 20221229
            NPES_PER_NODE=$NCPU_PER_NODE
            ncpus=1              # forced
    fi

fi

s%@NEMO_NPES@%${NEMO_NPES}%g

HyunChaeJung commented 1 year ago

GRIMs Makefile.in 및 configure 파일 수정

~/GRIMs-Chem/srcs/src/cpl/Makefile.in 생성

~/GRIMs-Chem/srcs/src/fcst/Makefile.in 수정

#hcjung 20171222
OASISDIR = @OASIS_DIR@
OASIS_LIB = $(OASISDIR)/lib/libpsmile.MPI1.a $(OASISDIR)/lib/libmct.a $(OASISDIR)/lib/libmpeu.a $(OASISDIR)/lib/libscrip.a
OASIS_INC = -I$(OASISDIR)/build/lib/mct -I$(OASISDIR)/build/lib/psmile.MPI1 -I$(OASISDIR)/build/lib/scrip -I$(OASISDIR)/include

STATIC_LIBS= @STATIC_LIBS@

NETCDFDIR = @NETCDF_DIR@
NETCDF_INC = -I$(NETCDFDIR)/include
NETCDF_LIB = $(NETCDFDIR)/lib/libnetcdff.a $(NETCDFDIR)/lib/libnetcdf.a
#hcjung 20171222

LIBS = @FCST_SFCL@ ${INSTALLDLIB}/share.a @FCST_COMMON@ ${LIB_DIR}/modelib.a ${LIB_DIR}/w3lib.a @FCST_MPI@ @FCST_CHEM@ @EXTRA_LIBS@ ${INSTALLDLIB}/cpl.a ${OASIS_LIB} ${NETCDF_LIB} ${STATIC_LIBS} ##hcjung 20221128

.F90.f90 :
        ${CPP} -P ${INCLUDE_DIR} ${OASIS_INC} $*.F90 >$*.i #hjcung 20171222
        sed '/^ *$$/d;s/\/\*/\!/' $*.i >$*.f90 ; rm $*.i

.F90.o :
        ${CPP} -P ${INCLUDE_DIR} ${OASIS_INC} $*.F90 >$*.i #hcjung 20171222
        sed '/^ *$$/d;s/\/\*/\!/' $*.i >$*.f90 ; rm $*.i
        ${F77} ${FORT_FLAGS} ${OASIS_INC} -c $*.f90 #hcjung 20171222

.f90.o :
        ${F77} ${FORT_FLAGS} ${OASIS_INC} -c $*.f90 #hcjung 20171222

~/GRIMs-Chem/srcs/src/sfcl_par/Makefile.in 수정

#hcjung 20221128
OASISDIR = @OASIS_DIR@
OASIS_LIB = $(OASISDIR)/lib/libpsmile.MPI1.a $(OASISDIR)/lib/libmct.a $(OASISDIR)/lib/libmpeu.a $(OASISDIR)/lib/libscrip.a
OASIS_INC = -I$(OASISDIR)/build/lib/mct -I$(OASISDIR)/build/lib/psmile.MPI1 -I$(OASISDIR)/build/lib/scrip -I$(OASISDIR)/include
#hcjung 20221127

INCLUDE_DIR = @INCLUDE_DIR@ ${OASIS_INC} ##hcjung 20221129

~/GRIMs-Chem/runs/configure-run

export CHEM_INPUT_DIR='/home/grimsjung/hcjung/cpl_test/GRIMs_data/data'

###hcjung add CHEM_INPUT_DIR 20221103
s%@CHEM_INPUT_DIR@%${CHEM_INPUT_DIR}%g

~/GRIMs-Chem/srcs/configure-src

##hcjung 20221101 add OASIS_DIR, NETCDF_DIR
s%@OASIS_DIR@%$OASIS_DIR%g
s%@NETCDF_DIR@%$NETCDF_DIR%g

        if [ $MARCH = mpi -o $MARCH = hybrid ]; then
                ##hcjung 20221128
                MARCH_SRC_DIRS="\
                share \
                common \
                common_par \
                mpi \
                cpl \
                "
        else
                ##hcjung 20221128
                MARCH_SRC_DIRS="\
                cpl \
                share \
                common \
                common_par \
                "
        fi

if [ $MARCH = mpi -o $MARCH = hybrid ]; then
                ##hcjung 20221128
                MARCH_SRC_DIRS="\
                share \
                common_par \
                rmpi \
                cpl \
                sfcl_par \
                chem \
                rgmp_par \
                rmpl_par \
                rpgb_par \
                rfcst_par \
                "
        else
                MARCH_SRC_DIRS="share"
        fi