From 6dcbdff08400e5de679304bd38a1125610a6037f Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 14 Sep 2021 11:55:18 -0600 Subject: [PATCH 01/10] use the mesh mask in time interpolation calculations --- streams/dshr_strdata_mod.F90 | 54 ++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/streams/dshr_strdata_mod.F90 b/streams/dshr_strdata_mod.F90 index 38fb8524..d172fb30 100644 --- a/streams/dshr_strdata_mod.F90 +++ b/streams/dshr_strdata_mod.F90 @@ -397,6 +397,7 @@ subroutine shr_strdata_init(sdat, model_clock, stream_name, rc) logical :: masterproc integer :: nvars integer :: i, stream_nlev, index + integer, allocatable :: mask(:) character(CL) :: stream_vectors character(len=*), parameter :: subname='(shr_sdat_init)' ! ---------------------------------------------- @@ -426,6 +427,10 @@ subroutine shr_strdata_init(sdat, model_clock, stream_name, rc) call shr_sys_abort(subName//"ERROR: file does not exist: "//trim(fileName)) end if endif + ! + ! We do not yet have mask information, but we are required to set it here and change it + ! later. + ! if(filename /= 'none') then stream_mesh = ESMF_MeshCreate(trim(filename), fileformat=ESMF_FILEFORMAT_ESMFMESH, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -811,6 +816,7 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) integer ,intent(out) :: rc ! local variables + integer :: ns ! stream index integer :: nf ! field index integer :: i,lev ! generic indices @@ -841,6 +847,8 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) integer :: year,month,day ! date year month day integer :: nstreams integer :: stream_index + integer :: lsize + integer ,allocatable :: mask(:) real(r8) ,parameter :: solZenMin = 0.001_r8 ! minimum solar zenith angle integer ,parameter :: tadj = 2 character(len=*) ,parameter :: timname = "_strd_adv" @@ -976,6 +984,15 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) ! --------------------------------------------------------- do ns = 1,nstreams + call ESMF_MeshGet(sdat%pstrm(ns)%stream_mesh, elementCount=lsize) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + call ESMF_MeshGet(sdat%pstrm(ns)%stream_mesh, elementCount=lsize) + if (chkerr(rc,__LINE__,u_FILE_u)) return + allocate(mask(lsize)) + call ESMF_MeshGet(sdat%pstrm(ns)%stream_mesh, elementMask=mask) + if (chkerr(rc,__LINE__,u_FILE_u)) return + if (trim(sdat%stream(ns)%tinterpalgo) == 'coszen') then ! Determine stream lower bound index @@ -1019,7 +1036,7 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return do i = 1,size(dataptr2d,dim=2) if (coszen(i) > solZenMin) then - dataptr2d(:,i) = dataptr2d_lb(:,i)*coszen(i)/sdat%tavCoszen(i) + dataptr2d(:,i) = mask(i)*dataptr2d_lb(:,i)*coszen(i)/sdat%tavCoszen(i) else dataptr2d(:,i) = 0._r8 endif @@ -1033,7 +1050,7 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return do i = 1,size(dataptr1d) if (coszen(i) > solZenMin) then - dataptr1d(i) = dataptr1d_lb(i)*coszen(i)/sdat%tavCoszen(i) + dataptr1d(i) = mask(i)*dataptr1d_lb(i)*coszen(i)/sdat%tavCoszen(i) else dataptr1d(i) = 0._r8 endif @@ -1071,7 +1088,7 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) sdat%pstrm(ns)%fldlist_model(nf), fldptr2=dataptr2d_ub, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return do lev = 1,sdat%pstrm(ns)%stream_nlev - dataptr2d(lev,:) = dataptr2d_lb(lev,:) * flb + dataptr2d_ub(lev,:) * fub + dataptr2d(lev,:) = mask(:)*(dataptr2d_lb(lev,:) * flb + dataptr2d_ub(lev,:) * fub) end do else call dshr_fldbun_getfldptr(sdat%pstrm(ns)%fldbun_model, & @@ -1083,7 +1100,7 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) call dshr_fldbun_getfldptr(sdat%pstrm(ns)%fldbun_data(sdat%pstrm(ns)%stream_ub), & sdat%pstrm(ns)%fldlist_model(nf), fldptr1=dataptr1d_ub, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - dataptr1d(:) = dataptr1d_lb(:) * flb + dataptr1d_ub(:) * fub + dataptr1d(:) = mask(:)*(dataptr1d_lb(:) * flb + dataptr1d_ub(:) * fub) end if end do call ESMF_TraceRegionExit(trim(lstr)//trim(timname)//'_tint') @@ -1111,7 +1128,7 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) call ESMF_TraceRegionExit(trim(lstr)//trim(timname)//'_zero') endif - + deallocate(mask) end do ! loop over ns (number of streams) deallocate(newData) @@ -1348,6 +1365,7 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & real(r8), allocatable :: data_dbl2d(:,:) ! stream input data integer(i2), allocatable :: data_short1d(:) ! stream input data integer(i2), allocatable :: data_short2d(:,:) ! stream input data + integer, allocatable :: mask(:) integer :: lsize, n integer :: spatialDim, numOwnedElements integer :: pio_iovartype @@ -1498,6 +1516,12 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & ! read the data call pio_setframe(pioid, varid, int(nt,kind=Pio_Offset_Kind)) + + call ESMF_MeshGet(per_stream%stream_mesh, elementCount=lsize) + if (chkerr(rc,__LINE__,u_FILE_u)) return + allocate(mask(lsize)) + call ESMF_MeshGet(per_stream%stream_mesh, elementMask=mask) + if (chkerr(rc,__LINE__,u_FILE_u)) return if (pio_iovartype == PIO_REAL) then ! ----------------------------- @@ -1649,14 +1673,14 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & if(data_short2d(n,lev) .eq. fillvalue_i2) then dataptr2d(lev,n) = r8fill else - dataptr2d(lev,n) = real(data_short2d(lev,n),r8) * scale_factor + add_offset + dataptr2d(lev,n) = mask(n)*(real(data_short2d(lev,n),r8) * scale_factor + add_offset) endif enddo end do else do lev = 1,stream_nlev do n = 1,lsize - dataptr2d(lev,n) = real(data_short2d(n,lev),r8) * scale_factor + add_offset + dataptr2d(lev,n) = mask(n)*(real(data_short2d(n,lev),r8) * scale_factor + add_offset) enddo end do end if @@ -1674,12 +1698,12 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & if(data_short1d(n).eq.fillvalue_i2) then dataptr1d(n) = r8fill else - dataptr1d(n) = real(data_short1d(n),r8) * scale_factor + add_offset + dataptr1d(n) = mask(n)*(real(data_short1d(n),r8) * scale_factor + add_offset) endif enddo else do n=1,lsize - dataptr1d(n) = real(data_short1d(n),r8) * scale_factor + add_offset + dataptr1d(n) = mask(n)*(real(data_short1d(n),r8) * scale_factor + add_offset) enddo endif end if @@ -1710,10 +1734,16 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & call ESMF_FieldFill(field_dst, dataFillScheme="const", const1=dataptr1d(1), rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return endif + deallocate(mask) enddo ! Both components of a vector stream must be in the same input stream file if (associated(dataptr2d_src) .and. associated(dataptr1d)) then + call ESMF_MeshGet(per_stream%stream_mesh, elementCount=lsize) + if (chkerr(rc,__LINE__,u_FILE_u)) return + allocate(mask(lsize)) + call ESMF_MeshGet(per_stream%stream_mesh, elementMask=mask) + if (chkerr(rc,__LINE__,u_FILE_u)) return ! get lon and lat of stream u and v fields lsize = size(dataptr1d) @@ -1736,8 +1766,8 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & lat = nu_coords(2*i) sinlon = sin(lon*deg2rad); coslon = cos(lon*deg2rad) sinlat = sin(lat*deg2rad); coslat = cos(lat*deg2rad) - dataptr2d_src(1,i) = coslon * dataptr(i) - sinlon * dataptr2d_src(2,i) - dataptr2d_src(2,i) = sinlon * dataptr(i) + coslon * dataptr2d_src(2,i) + dataptr2d_src(1,i) = mask(i)*(coslon * dataptr(i) - sinlon * dataptr2d_src(2,i)) + dataptr2d_src(2,i) = mask(i)*(sinlon * dataptr(i) + coslon * dataptr2d_src(2,i)) enddo vector_dst = ESMF_FieldCreate(sdat%model_mesh, ESMF_TYPEKIND_r8, name='vector_dst', & ungriddedLbound=(/1/), ungriddedUbound=(/2/), gridToFieldMap=(/2/), meshloc=ESMF_MESHLOC_ELEMENT, rc=rc) @@ -1762,7 +1792,7 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & data_u_dst(i) = coslon * dataptr2d_dst(1,i) + sinlon * dataptr2d_dst(2,i) data_v_dst(i) = -sinlon * dataptr2d_dst(1,i) + coslon * dataptr2d_dst(2,i) enddo - + deallocate(mask) deallocate(dataptr) endif From afce86ced060e8cabce8ad5386a052c28d9e95de Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 17 Sep 2021 15:30:20 -0600 Subject: [PATCH 02/10] changes needed for cime cmake build --- CMakeLists.txt | 24 ++++++++---------------- cime_config/buildlib | 1 + 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f465069..2e67b07c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,7 @@ cmake_minimum_required(VERSION 3.10) include(ExternalProject) include(FetchContent) +enable_language(Fortran) if (DEFINED CIMEROOT) message("Using CIME in ${CIMEROOT} with compiler ${COMPILER}") @@ -8,24 +9,15 @@ if (DEFINED CIMEROOT) if (${PIO_VERSION} LESS 2) message( FATAL_ERROR "Version 2 of the PIO library required") endif() - if (${MPILIB} STREQUAL "mpi-serial") - set(CMAKE_C_COMPILER ${SCC}) - set(CMAKE_Fortran_COMPILER ${SFC}) - set(CMAKE_CXX_COMPILER ${SCXX}) - else() - set(CMAKE_C_COMPILER ${MPICC}) - set(CMAKE_Fortran_COMPILER ${MPIFC}) - set(CMAKE_CXX_COMPILER ${MPICXX}) - endif() set(CMAKE_Fortran_FLAGS "${FFLAGS} ${CPPDEFS} -I${LIBROOT}/include -I${LIBROOT}/finclude -I${LIBROOT}/nuopc/esmf/${NINST_VALUE}/include") add_compile_definitions(CESMCOUPLED) + list(APPEND CMAKE_MODULE_PATH ${SRC_ROOT}/components/cdeps/cmake) else() set(BLD_STANDALONE TRUE) + project(NUOPC_DATA_MODELS LANGUAGES Fortran VERSION 0.1) endif() -project(NUOPC_DATA_MODELS LANGUAGES Fortran VERSION 0.1) - -list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) +list(APPEND CMAKE_MODULE_PATH ${SRC_ROOT}/components/cdeps/cmake) message("CMAKE_MODULE_PATH is ${CMAKE_MODULE_PATH}") @@ -61,15 +53,15 @@ endif() add_subdirectory(streams) add_subdirectory(dshr) -if(IS_DIRECTORY "${PROJECT_SOURCE_DIR}/fox") +if(IS_DIRECTORY "${SRC_ROOT}/components/cdeps/fox") message(STATUS "FoX library is already checked out!") - message(STATUS "FoX source dir: ${PROJECT_SOURCE_DIR}/fox") + message(STATUS "FoX source dir: ${SRC_ROOT}/components/cdeps/fox") else() FetchContent_Declare(fox GIT_REPOSITORY https://github.com/ESMCI/fox.git GIT_TAG 4.1.2.1 - SOURCE_DIR ${PROJECT_SOURCE_DIR}/fox - BINARY_DIR ${PROJECT_SOURCE_DIR} + SOURCE_DIR ${SRC_ROOT}/components/cdeps/fox + BINARY_DIR ${SRC_ROOT}/components/cdeps ) FetchContent_GetProperties(fox) if(NOT fox_POPULATED) diff --git a/cime_config/buildlib b/cime_config/buildlib index 552adecb..d982a719 100755 --- a/cime_config/buildlib +++ b/cime_config/buildlib @@ -95,6 +95,7 @@ def buildlib(bldroot, libroot, case, compname=None): bld_time = src_time - 1 # if any file in src is newer than CmakeFiles in the build directory, rerun cmake + if src_time > bld_time: logger.info("cmake_flags {}".format(cmake_flags)) s,o,e = run_cmd("cmake {} ".format(cmake_flags), from_dir=bldroot, verbose=True) From 0e973e8b2b43f92dd3452b929c0fb9b16756963a Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 28 Sep 2021 08:57:18 -0600 Subject: [PATCH 03/10] update cmake module path for standalone build --- .github/workflows/extbuild.yml | 1 + CMakeLists.txt | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/extbuild.yml b/.github/workflows/extbuild.yml index 12e57ea2..7f1260af 100644 --- a/.github/workflows/extbuild.yml +++ b/.github/workflows/extbuild.yml @@ -110,6 +110,7 @@ jobs: run: | export ESMFMKFILE=$HOME/ESMF/lib/libg/Linux.gfortran.64.openmpi.default/esmf.mk export PIO=$HOME/pio + export SRC_ROOT= git submodule init git submodule update mkdir build-cdeps diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e67b07c..aac06e87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,10 +15,9 @@ if (DEFINED CIMEROOT) else() set(BLD_STANDALONE TRUE) project(NUOPC_DATA_MODELS LANGUAGES Fortran VERSION 0.1) + list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) endif() -list(APPEND CMAKE_MODULE_PATH ${SRC_ROOT}/components/cdeps/cmake) - message("CMAKE_MODULE_PATH is ${CMAKE_MODULE_PATH}") if (TARGET esmf) From 088480b2b0d8c5a6a9605af59446f134f760969f Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 28 Sep 2021 09:10:35 -0600 Subject: [PATCH 04/10] more fixes for standalone build --- CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aac06e87..72e7189a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,10 +12,12 @@ if (DEFINED CIMEROOT) set(CMAKE_Fortran_FLAGS "${FFLAGS} ${CPPDEFS} -I${LIBROOT}/include -I${LIBROOT}/finclude -I${LIBROOT}/nuopc/esmf/${NINST_VALUE}/include") add_compile_definitions(CESMCOUPLED) list(APPEND CMAKE_MODULE_PATH ${SRC_ROOT}/components/cdeps/cmake) + set(FOX_ROOT ${SRC_ROOT}/components/cdeps/fox) else() set(BLD_STANDALONE TRUE) project(NUOPC_DATA_MODELS LANGUAGES Fortran VERSION 0.1) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) + set(FOX_ROOT ${CMAKE_SOURCE_DIR}/fox) endif() message("CMAKE_MODULE_PATH is ${CMAKE_MODULE_PATH}") @@ -52,15 +54,15 @@ endif() add_subdirectory(streams) add_subdirectory(dshr) -if(IS_DIRECTORY "${SRC_ROOT}/components/cdeps/fox") +if(IS_DIRECTORY "${FOX_ROOT}") message(STATUS "FoX library is already checked out!") - message(STATUS "FoX source dir: ${SRC_ROOT}/components/cdeps/fox") + message(STATUS "FoX source dir: ${FOX_ROOT}") else() FetchContent_Declare(fox GIT_REPOSITORY https://github.com/ESMCI/fox.git GIT_TAG 4.1.2.1 - SOURCE_DIR ${SRC_ROOT}/components/cdeps/fox - BINARY_DIR ${SRC_ROOT}/components/cdeps + SOURCE_DIR ${FOX_ROOT} + BINARY_DIR ${FOX_ROOT}/.. ) FetchContent_GetProperties(fox) if(NOT fox_POPULATED) From 73b47b31677c7d8ceeec1d5232b89764518585c6 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 30 Sep 2021 09:50:25 -0600 Subject: [PATCH 05/10] minor changes needed on izumi --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 72e7189a..98bcaef4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,6 @@ cmake_minimum_required(VERSION 3.10) include(ExternalProject) include(FetchContent) -enable_language(Fortran) if (DEFINED CIMEROOT) message("Using CIME in ${CIMEROOT} with compiler ${COMPILER}") @@ -9,6 +8,8 @@ if (DEFINED CIMEROOT) if (${PIO_VERSION} LESS 2) message( FATAL_ERROR "Version 2 of the PIO library required") endif() + set(CMAKE_Fortran_COMPILER ${MPIFC}) + set(CMAKE_C_COMPILER ${MPICC}) set(CMAKE_Fortran_FLAGS "${FFLAGS} ${CPPDEFS} -I${LIBROOT}/include -I${LIBROOT}/finclude -I${LIBROOT}/nuopc/esmf/${NINST_VALUE}/include") add_compile_definitions(CESMCOUPLED) list(APPEND CMAKE_MODULE_PATH ${SRC_ROOT}/components/cdeps/cmake) @@ -19,8 +20,8 @@ else() list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) set(FOX_ROOT ${CMAKE_SOURCE_DIR}/fox) endif() - -message("CMAKE_MODULE_PATH is ${CMAKE_MODULE_PATH}") +message("CMAKE_MODULE_PATH is ${CMAKE_MODULE_PATH}, CMAKE_Fortran_COMPILER is ${CMAKE_Fortran_COMPILER}") +enable_language(Fortran) if (TARGET esmf) message("Target esmf is already found. Skip find_package ...") From d5ca64d13119c42af46af9f8f308b9f1b13729b9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 1 Oct 2021 14:50:34 -0600 Subject: [PATCH 06/10] bilinear will work even when grids match --- datm/cime_config/stream_definition_datm.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index 22237102..af792834 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -654,7 +654,7 @@ null - redist + bilinear null $DATM_YR_ALIGN @@ -686,7 +686,7 @@ null - redist + bilinear null $DATM_YR_ALIGN @@ -722,7 +722,7 @@ null - redist + bilinear null $DATM_YR_ALIGN From a713943e1050c60e946651bc720b980635884106 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 7 Oct 2021 11:34:02 -0600 Subject: [PATCH 07/10] update to build mpi-serial correctly --- CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 98bcaef4..116f8aaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,8 +8,13 @@ if (DEFINED CIMEROOT) if (${PIO_VERSION} LESS 2) message( FATAL_ERROR "Version 2 of the PIO library required") endif() - set(CMAKE_Fortran_COMPILER ${MPIFC}) - set(CMAKE_C_COMPILER ${MPICC}) + if (MPILIB STREQUAL mpi-serial) + set(CMAKE_Fortran_COMPILER ${SFC}) + set(CMAKE_C_COMPILER ${SCC}) + else() + set(CMAKE_Fortran_COMPILER ${MPIFC}) + set(CMAKE_C_COMPILER ${MPICC}) + endif() set(CMAKE_Fortran_FLAGS "${FFLAGS} ${CPPDEFS} -I${LIBROOT}/include -I${LIBROOT}/finclude -I${LIBROOT}/nuopc/esmf/${NINST_VALUE}/include") add_compile_definitions(CESMCOUPLED) list(APPEND CMAKE_MODULE_PATH ${SRC_ROOT}/components/cdeps/cmake) From 6cce4032dec5bfc539a39e8572800bcb8e998fd9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 12 Oct 2021 17:11:44 -0600 Subject: [PATCH 08/10] should be model mesh here not stream mesh --- streams/dshr_strdata_mod.F90 | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/streams/dshr_strdata_mod.F90 b/streams/dshr_strdata_mod.F90 index d172fb30..16e80e7e 100644 --- a/streams/dshr_strdata_mod.F90 +++ b/streams/dshr_strdata_mod.F90 @@ -984,13 +984,10 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) ! --------------------------------------------------------- do ns = 1,nstreams - call ESMF_MeshGet(sdat%pstrm(ns)%stream_mesh, elementCount=lsize) - if (chkerr(rc,__LINE__,u_FILE_u)) return - - call ESMF_MeshGet(sdat%pstrm(ns)%stream_mesh, elementCount=lsize) + call ESMF_MeshGet(sdat%model_mesh, elementCount=lsize) if (chkerr(rc,__LINE__,u_FILE_u)) return allocate(mask(lsize)) - call ESMF_MeshGet(sdat%pstrm(ns)%stream_mesh, elementMask=mask) + call ESMF_MeshGet(sdat%model_mesh, elementMask=mask) if (chkerr(rc,__LINE__,u_FILE_u)) return if (trim(sdat%stream(ns)%tinterpalgo) == 'coszen') then From d39d940e0fef9c58bf727d9b7e610cd4aa3760a3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 22 Oct 2021 11:32:58 -0600 Subject: [PATCH 09/10] remove mask from time interp due to issues with unstructured grids --- datm/datm_datamode_clmncep_mod.F90 | 5 ++--- streams/dshr_strdata_mod.F90 | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/datm/datm_datamode_clmncep_mod.F90 b/datm/datm_datamode_clmncep_mod.F90 index 3db1165c..5a739b71 100644 --- a/datm/datm_datamode_clmncep_mod.F90 +++ b/datm/datm_datamode_clmncep_mod.F90 @@ -348,8 +348,8 @@ subroutine datm_datamode_clmncep_advance(masterproc, logunit, mpicom, rc) ! determine tbotmax (see below for use) rtmp = maxval(Sa_tbot(:)) call shr_mpi_max(rtmp, tbotmax, mpicom, 'datm_tbot', all=.true.) - write(logunit,*) trim(subname),' tbotmax = ',tbotmax - if(tbotmax <= 0) then + if (masterproc) write(logunit,*) trim(subname),' tbotmax = ',tbotmax + if(tbotmax <= 0 .or. tbotmax > 1000.0) then call shr_sys_abort(subname//'ERROR: bad value in tbotmax') endif @@ -418,7 +418,6 @@ subroutine datm_datamode_clmncep_advance(masterproc, logunit, mpicom, rc) else call shr_sys_abort(subname//'ERROR: cannot compute shum') endif - !--- density --- vp = (Sa_shum(n)*pbot) / (0.622_r8 + 0.378_r8 * Sa_shum(n)) Sa_dens(n) = (pbot - 0.378_r8 * vp) / (tbot*rdair) diff --git a/streams/dshr_strdata_mod.F90 b/streams/dshr_strdata_mod.F90 index 16e80e7e..302660a6 100644 --- a/streams/dshr_strdata_mod.F90 +++ b/streams/dshr_strdata_mod.F90 @@ -19,7 +19,7 @@ module dshr_strdata_mod use ESMF , only : ESMF_FieldReGridStore, ESMF_FieldRedistStore, ESMF_UNMAPPEDACTION_IGNORE use ESMF , only : ESMF_TERMORDER_SRCSEQ, ESMF_FieldRegrid, ESMF_FieldFill use ESMF , only : ESMF_REGION_TOTAL, ESMF_FieldGet, ESMF_TraceRegionExit, ESMF_TraceRegionEnter - use ESMF , only : ESMF_LOGMSG_INFO, ESMF_LogWrite + use ESMF , only : ESMF_LOGMSG_INFO, ESMF_LogWrite, ESMF_RC_ARG_VALUE use shr_kind_mod , only : r8=>shr_kind_r8, r4=>shr_kind_r4, i2=>shr_kind_I2 use shr_kind_mod , only : cs=>shr_kind_cs, cl=>shr_kind_cl, cxx=>shr_kind_cxx use shr_sys_mod , only : shr_sys_abort @@ -984,12 +984,16 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) ! --------------------------------------------------------- do ns = 1,nstreams - call ESMF_MeshGet(sdat%model_mesh, elementCount=lsize) + call ESMF_MeshGet(sdat%model_mesh, elementCount=lsize, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return allocate(mask(lsize)) - call ESMF_MeshGet(sdat%model_mesh, elementMask=mask) - if (chkerr(rc,__LINE__,u_FILE_u)) return - + ! call ESMF_MeshGet(sdat%model_mesh, elementMask=mask, rc=rc) + ! for unstructured grids mask is undetermined, setting it to 1 disables the feature. + ! if (rc == ESMF_RC_ARG_VALUE) then + mask = 1 + ! else if (chkerr(rc,__LINE__,u_FILE_u)) then + ! return + ! endif if (trim(sdat%stream(ns)%tinterpalgo) == 'coszen') then ! Determine stream lower bound index @@ -1514,12 +1518,12 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & ! read the data call pio_setframe(pioid, varid, int(nt,kind=Pio_Offset_Kind)) - call ESMF_MeshGet(per_stream%stream_mesh, elementCount=lsize) + call ESMF_MeshGet(per_stream%stream_mesh, elementCount=lsize, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return allocate(mask(lsize)) - call ESMF_MeshGet(per_stream%stream_mesh, elementMask=mask) + call ESMF_LogWrite(trim(subname)//' calling ESMF_MeshGet mask', ESMF_LOGMSG_INFO) + call ESMF_MeshGet(per_stream%stream_mesh, elementMask=mask, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - if (pio_iovartype == PIO_REAL) then ! ----------------------------- ! pio_iovartype is PIO_REAL From 6994d8ed29c6c68b174bbb028a4d2590b61eb121 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 22 Oct 2021 13:50:49 -0600 Subject: [PATCH 10/10] remove time interpolation mask --- datm/datm_datamode_clmncep_mod.F90 | 2 +- docn/ocn_comp_nuopc.F90 | 7 +++-- streams/dshr_strdata_mod.F90 | 47 +++++++----------------------- 3 files changed, 15 insertions(+), 41 deletions(-) diff --git a/datm/datm_datamode_clmncep_mod.F90 b/datm/datm_datamode_clmncep_mod.F90 index 5a739b71..86484151 100644 --- a/datm/datm_datamode_clmncep_mod.F90 +++ b/datm/datm_datamode_clmncep_mod.F90 @@ -349,7 +349,7 @@ subroutine datm_datamode_clmncep_advance(masterproc, logunit, mpicom, rc) rtmp = maxval(Sa_tbot(:)) call shr_mpi_max(rtmp, tbotmax, mpicom, 'datm_tbot', all=.true.) if (masterproc) write(logunit,*) trim(subname),' tbotmax = ',tbotmax - if(tbotmax <= 0 .or. tbotmax > 1000.0) then + if(tbotmax <= 0) then call shr_sys_abort(subname//'ERROR: bad value in tbotmax') endif diff --git a/docn/ocn_comp_nuopc.F90 b/docn/ocn_comp_nuopc.F90 index b132c3a8..942a0060 100644 --- a/docn/ocn_comp_nuopc.F90 +++ b/docn/ocn_comp_nuopc.F90 @@ -15,7 +15,7 @@ module cdeps_docn_comp use ESMF , only : ESMF_Alarm, ESMF_MethodRemove, ESMF_MethodAdd use ESMF , only : ESMF_GridCompSetEntryPoint, ESMF_ClockGetAlarm, ESMF_AlarmIsRinging use ESMF , only : ESMF_StateGet, operator(+), ESMF_AlarmRingerOff, ESMF_LogWrite - use ESMF , only : ESMF_Field, ESMF_FieldGet + use ESMF , only : ESMF_Field, ESMF_FieldGet, ESMF_VmLogMemInfo use NUOPC , only : NUOPC_CompDerive, NUOPC_CompSetEntryPoint, NUOPC_CompSpecialize use NUOPC , only : NUOPC_Advertise, NUOPC_CompAttributeGet use NUOPC_Model , only : model_routine_SS => SetServices @@ -306,7 +306,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) !------------------------------------------------------------------------------- rc = ESMF_SUCCESS - + call ESMF_VMLogMemInfo("Entering "//trim(subname)) ! Initialize model mesh, restart flag, logunit, model_mask and model_frac call ESMF_TraceRegionEnter('docn_strdata_init') call dshr_mesh_init(gcomp, sdat, nullstr, logunit, 'OCN', nx_global, ny_global, & @@ -355,6 +355,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) ! ******************* ! *** RETURN HERE *** ! ******************* + call ESMF_VMLogMemInfo("Leaving "//trim(subname)) RETURN end if @@ -374,7 +375,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return call dshr_state_SetScalar(dble(ny_global),flds_scalar_index_ny, exportState, flds_scalar_name, flds_scalar_num, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - + call ESMF_VMLogMemInfo("Leaving "//trim(subname)) end subroutine InitializeRealize !=============================================================================== diff --git a/streams/dshr_strdata_mod.F90 b/streams/dshr_strdata_mod.F90 index 302660a6..772ecb35 100644 --- a/streams/dshr_strdata_mod.F90 +++ b/streams/dshr_strdata_mod.F90 @@ -848,7 +848,6 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) integer :: nstreams integer :: stream_index integer :: lsize - integer ,allocatable :: mask(:) real(r8) ,parameter :: solZenMin = 0.001_r8 ! minimum solar zenith angle integer ,parameter :: tadj = 2 character(len=*) ,parameter :: timname = "_strd_adv" @@ -984,16 +983,6 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) ! --------------------------------------------------------- do ns = 1,nstreams - call ESMF_MeshGet(sdat%model_mesh, elementCount=lsize, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - allocate(mask(lsize)) - ! call ESMF_MeshGet(sdat%model_mesh, elementMask=mask, rc=rc) - ! for unstructured grids mask is undetermined, setting it to 1 disables the feature. - ! if (rc == ESMF_RC_ARG_VALUE) then - mask = 1 - ! else if (chkerr(rc,__LINE__,u_FILE_u)) then - ! return - ! endif if (trim(sdat%stream(ns)%tinterpalgo) == 'coszen') then ! Determine stream lower bound index @@ -1037,7 +1026,7 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return do i = 1,size(dataptr2d,dim=2) if (coszen(i) > solZenMin) then - dataptr2d(:,i) = mask(i)*dataptr2d_lb(:,i)*coszen(i)/sdat%tavCoszen(i) + dataptr2d(:,i) = dataptr2d_lb(:,i)*coszen(i)/sdat%tavCoszen(i) else dataptr2d(:,i) = 0._r8 endif @@ -1051,7 +1040,7 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return do i = 1,size(dataptr1d) if (coszen(i) > solZenMin) then - dataptr1d(i) = mask(i)*dataptr1d_lb(i)*coszen(i)/sdat%tavCoszen(i) + dataptr1d(i) = dataptr1d_lb(i)*coszen(i)/sdat%tavCoszen(i) else dataptr1d(i) = 0._r8 endif @@ -1089,7 +1078,7 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) sdat%pstrm(ns)%fldlist_model(nf), fldptr2=dataptr2d_ub, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return do lev = 1,sdat%pstrm(ns)%stream_nlev - dataptr2d(lev,:) = mask(:)*(dataptr2d_lb(lev,:) * flb + dataptr2d_ub(lev,:) * fub) + dataptr2d(lev,:) = dataptr2d_lb(lev,:) * flb + dataptr2d_ub(lev,:) * fub end do else call dshr_fldbun_getfldptr(sdat%pstrm(ns)%fldbun_model, & @@ -1101,7 +1090,7 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) call dshr_fldbun_getfldptr(sdat%pstrm(ns)%fldbun_data(sdat%pstrm(ns)%stream_ub), & sdat%pstrm(ns)%fldlist_model(nf), fldptr1=dataptr1d_ub, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - dataptr1d(:) = mask(:)*(dataptr1d_lb(:) * flb + dataptr1d_ub(:) * fub) + dataptr1d(:) = dataptr1d_lb(:) * flb + dataptr1d_ub(:) * fub end if end do call ESMF_TraceRegionExit(trim(lstr)//trim(timname)//'_tint') @@ -1129,7 +1118,6 @@ subroutine shr_strdata_advance(sdat, ymd, tod, logunit, istr, timers, rc) call ESMF_TraceRegionExit(trim(lstr)//trim(timname)//'_zero') endif - deallocate(mask) end do ! loop over ns (number of streams) deallocate(newData) @@ -1366,7 +1354,6 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & real(r8), allocatable :: data_dbl2d(:,:) ! stream input data integer(i2), allocatable :: data_short1d(:) ! stream input data integer(i2), allocatable :: data_short2d(:,:) ! stream input data - integer, allocatable :: mask(:) integer :: lsize, n integer :: spatialDim, numOwnedElements integer :: pio_iovartype @@ -1517,13 +1504,6 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & ! read the data call pio_setframe(pioid, varid, int(nt,kind=Pio_Offset_Kind)) - - call ESMF_MeshGet(per_stream%stream_mesh, elementCount=lsize, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - allocate(mask(lsize)) - call ESMF_LogWrite(trim(subname)//' calling ESMF_MeshGet mask', ESMF_LOGMSG_INFO) - call ESMF_MeshGet(per_stream%stream_mesh, elementMask=mask, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return if (pio_iovartype == PIO_REAL) then ! ----------------------------- ! pio_iovartype is PIO_REAL @@ -1674,14 +1654,14 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & if(data_short2d(n,lev) .eq. fillvalue_i2) then dataptr2d(lev,n) = r8fill else - dataptr2d(lev,n) = mask(n)*(real(data_short2d(lev,n),r8) * scale_factor + add_offset) + dataptr2d(lev,n) = real(data_short2d(lev,n),r8) * scale_factor + add_offset endif enddo end do else do lev = 1,stream_nlev do n = 1,lsize - dataptr2d(lev,n) = mask(n)*(real(data_short2d(n,lev),r8) * scale_factor + add_offset) + dataptr2d(lev,n) = real(data_short2d(n,lev),r8) * scale_factor + add_offset enddo end do end if @@ -1699,12 +1679,12 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & if(data_short1d(n).eq.fillvalue_i2) then dataptr1d(n) = r8fill else - dataptr1d(n) = mask(n)*(real(data_short1d(n),r8) * scale_factor + add_offset) + dataptr1d(n) = real(data_short1d(n),r8) * scale_factor + add_offset endif enddo else do n=1,lsize - dataptr1d(n) = mask(n)*(real(data_short1d(n),r8) * scale_factor + add_offset) + dataptr1d(n) = real(data_short1d(n),r8) * scale_factor + add_offset enddo endif end if @@ -1735,16 +1715,10 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & call ESMF_FieldFill(field_dst, dataFillScheme="const", const1=dataptr1d(1), rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return endif - deallocate(mask) enddo ! Both components of a vector stream must be in the same input stream file if (associated(dataptr2d_src) .and. associated(dataptr1d)) then - call ESMF_MeshGet(per_stream%stream_mesh, elementCount=lsize) - if (chkerr(rc,__LINE__,u_FILE_u)) return - allocate(mask(lsize)) - call ESMF_MeshGet(per_stream%stream_mesh, elementMask=mask) - if (chkerr(rc,__LINE__,u_FILE_u)) return ! get lon and lat of stream u and v fields lsize = size(dataptr1d) @@ -1767,8 +1741,8 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & lat = nu_coords(2*i) sinlon = sin(lon*deg2rad); coslon = cos(lon*deg2rad) sinlat = sin(lat*deg2rad); coslat = cos(lat*deg2rad) - dataptr2d_src(1,i) = mask(i)*(coslon * dataptr(i) - sinlon * dataptr2d_src(2,i)) - dataptr2d_src(2,i) = mask(i)*(sinlon * dataptr(i) + coslon * dataptr2d_src(2,i)) + dataptr2d_src(1,i) = (coslon * dataptr(i) - sinlon * dataptr2d_src(2,i)) + dataptr2d_src(2,i) = (sinlon * dataptr(i) + coslon * dataptr2d_src(2,i)) enddo vector_dst = ESMF_FieldCreate(sdat%model_mesh, ESMF_TYPEKIND_r8, name='vector_dst', & ungriddedLbound=(/1/), ungriddedUbound=(/2/), gridToFieldMap=(/2/), meshloc=ESMF_MESHLOC_ELEMENT, rc=rc) @@ -1793,7 +1767,6 @@ subroutine shr_strdata_readstrm(sdat, per_stream, stream, fldbun_data, & data_u_dst(i) = coslon * dataptr2d_dst(1,i) + sinlon * dataptr2d_dst(2,i) data_v_dst(i) = -sinlon * dataptr2d_dst(1,i) + coslon * dataptr2d_dst(2,i) enddo - deallocate(mask) deallocate(dataptr) endif