Merge upstream develop

.github/pull_request_template.md

@@ -1,13 +1,15 @@
 # PR Checklist
 
-- [ ] Ths PR is up-to-date with the top of all sub-component repositories except for those sub-components which are the subject of this PR. Please consult the ufs-weather-model [wiki](https://github.com/ufs-community/ufs-weather-model/wiki/Making-code-changes-in-the-UFS-weather-model-and-its-subcomponents) if you are unsure how to do this.
+- [ ] This PR is up-to-date with the top of all sub-component repositories except for those sub-components which are the subject of this PR. Please consult the ufs-weather-model [wiki](https://github.com/ufs-community/ufs-weather-model/wiki/Making-code-changes-in-the-UFS-weather-model-and-its-subcomponents) if you are unsure how to do this.
 
 - [ ] This PR has been tested using a branch which is up-to-date with the top of all sub-component repositories except for those sub-components which are the subject of this PR
 
 - [ ] An Issue describing the work contained in this PR has been created either in the subcomponent(s) or in the ufs-weather-model. The Issue should be created in the repository that is most relevant to the changes in contained in the PR. The Issue and the dependent sub-component PR 
 are specified below.
 
-- [ ] If new or updated input data is required by this PR, it is clearly stated in the text of the PR.
+- [ ] Results for one or more of the regression tests change and the reasons for the changes are understood and explained below.
+
+- [ ] New or updated input data is required by this PR. If checked, please work with the code managers to update input data sets on all platforms.
 
 ## Instructions: All subsequent sections of text should be filled in as appropriate.
 
@@ -37,6 +39,7 @@ How were these changes tested? What compilers / HPCs was it tested with? Are the
 - [ ] jet.intel
 - [ ] wcoss_cray
 - [ ] wcoss_dell_p3
+- [ ] opnReqTest for newly added/changed feature
 - [ ] CI
 
 ## Dependencies

.gitmodules

@@ -2,14 +2,10 @@
   path = FV3
   url = https://github.com/NOAA-EMC/fv3atm
   branch = develop
-[submodule "NEMS"]
-  path = NEMS
-  url = https://github.com/NOAA-EMC/NEMS
-  branch = develop
 [submodule "WW3"]
   path = WW3
   url = https://github.com/NOAA-EMC/WW3
-  branch = develop
+  branch = dev/ufs-weather-model
 [submodule "stochastic_physics"]
   path = stochastic_physics
   url = https://github.com/noaa-psd/stochastic_physics
@@ -34,10 +30,6 @@
   path = CICE-interface/CICE
   url = https://github.com/NOAA-EMC/CICE
   branch = emc/develop
-[submodule "DATM"]
-  path = DATM
-  url = https://github.com/NOAA-EMC/NEMSdatm
-  branch = develop
 [submodule "CDEPS"]
   path = CDEPS-interface/CDEPS
   url = https://github.com/NOAA-EMC/CDEPS

CDEPS-interface/CMakeLists.txt

@@ -7,7 +7,7 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
   set(CMAKE_Fortran_FLAGS "-g -traceback")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O -assume realloc_lhs")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fp-model precise")
-  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv -init=snan,arrays")
 else()
   message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")
 endif()

CDEPS-interface/cdeps_files.cmake

@@ -44,6 +44,7 @@ list(APPEND cdeps_datm_files
   CDEPS/datm/datm_datamode_cplhist_mod.F90
   CDEPS/datm/datm_datamode_era5_mod.F90
   CDEPS/datm/datm_datamode_gefs_mod.F90
+  CDEPS/datm/datm_datamode_gfs_mod.F90
   CDEPS/datm/datm_datamode_jra_mod.F90
 )
 

CICE-interface/CMakeLists.txt

@@ -9,7 +9,7 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
   set(CMAKE_Fortran_FLAGS "-g -traceback")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -FR -convert big_endian -assume byterecl -ftz -align array64byte -xHOST")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fp-model precise")
-  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -init=snan,arrays")
   set(CMAKE_Fortran_LINK_FLAGS "")
 else()
   message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")

CMEPS-interface/CMakeLists.txt

@@ -12,23 +12,18 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
   set(CMAKE_Fortran_FLAGS "-g -traceback")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O -assume realloc_lhs")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fp-model precise")
-  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv -init=snan,arrays")
   set(CMAKE_Fortran_LINK_FLAGS "")
 else()
   message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")
 endif()
 
 list(APPEND _nems_util_files
-  CMEPS/util/shr_abort_mod.F90
-  CMEPS/util/shr_log_mod.F90
-  CMEPS/util/shr_sys_mod.F90
-  CMEPS/util/shr_flux_mod.F90
-  CMEPS/util/shr_mpi_mod.F90
-  CMEPS/util/glc_elevclass_mod.F90
-  CMEPS/util/shr_mem_mod.F90
-  CMEPS/util/shr_kind_mod.F90
-  CMEPS/util/perf_mod.F90
-  CMEPS/util/shr_const_mod.F90)
+  CMEPS/ufs/flux_atmocn_mod.F90
+  CMEPS/ufs/glc_elevclass_mod.F90
+  CMEPS/ufs/ufs_kind_mod.F90
+  CMEPS/ufs/perf_mod.F90
+  CMEPS/ufs/ufs_const_mod.F90)
 
 list(APPEND _mediator_files
   CMEPS/mediator/med_phases_restart_mod.F90

CMakeLists.txt

@@ -16,7 +16,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMAERO ATMW S2S S2SW HAFS HAFSW HAFS-ALL NG-GODAS NG-GODAS-NEMSDATM)
+list(APPEND VALID_APPS ATM ATMAERO ATMW S2S S2SW HAFS HAFSW HAFS-ALL NG-GODAS)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
   message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
@@ -30,7 +30,6 @@ set(HYCOM           OFF CACHE BOOL "Enable HYCOM")
 set(CICE6           OFF CACHE BOOL "Enable CICE6")
 set(WW3             OFF CACHE BOOL "Enable WW3")
 set(STOCH_PHYS      OFF CACHE BOOL "Enable Stochastic Physics")
-set(NEMSdatm        OFF CACHE BOOL "Enable NEMSdatm")
 set(CMEPS           OFF CACHE BOOL "Enable CMEPS")
 set(CDEPS           OFF CACHE BOOL "Enable CDEPS")
 
@@ -47,7 +46,6 @@ message("HYCOM ............ ${HYCOM}")
 message("CICE6 ............ ${CICE6}")
 message("WW3 .............. ${WW3}")
 message("STOCH_PHYS ....... ${STOCH_PHYS}")
-message("NEMSdatm ......... ${NEMSdatm}")
 message("CDEPS ............ ${CDEPS}")
 message("CMEPS ............ ${CMEPS}")
 
@@ -81,7 +79,7 @@ endif()
 message("")
 message("32BIT ............ ${32BIT}")
 message("AVX2 ............. ${AVX2}")
-message("SIMDMULTIARCH ...  ${SIMDMULTIARCH}")
+message("SIMDMULTIARCH .... ${SIMDMULTIARCH}")
 message("DEBUG ............ ${DEBUG}")
 message("DEBUG_LINKMPI .... ${DEBUG_LINKMPI}")
 message("INLINE_POST ...... ${INLINE_POST}")
@@ -127,12 +125,9 @@ endif()
 find_package(bacio REQUIRED)
 find_package(sp REQUIRED)
 find_package(w3nco REQUIRED)
-if(INLINE_POST)
-  find_package(upp REQUIRED)
-endif()
 
 # Configure Python
-find_package(Python 3 REQUIRED COMPONENTS Interpreter)
+find_package(Python 3.6 REQUIRED COMPONENTS Interpreter)
 message("Found Python: ${Python_EXECUTABLE}")
 
 ###############################################################################
@@ -206,24 +201,16 @@ if(CMEPS)
 endif()
 
 ###############################################################################
-### Data Components [NEMSdatm, CDEPS]
+### Data Components [CDEPS]
 ###############################################################################
-if(NEMSdatm)
-  add_subdirectory(DATM/DATM)
-endif()
-
 if(CDEPS)
   add_subdirectory(CDEPS-interface)
 endif()
 
 ###############################################################################
-### UFS Library [NEMS]
+### UFS Library
 ###############################################################################
-list(APPEND _nems_srcs NEMS/src/module_EARTH_GRID_COMP.F90
-                       NEMS/src/module_NEMS_Rusage.F90
-                       NEMS/src/nems_c_rusage.c)
-
-add_library(ufs ${_nems_srcs})
+add_library(ufs driver/EARTH_GRID_COMP.F90)
 set_target_properties(ufs PROPERTIES Fortran_MODULE_DIRECTORY
                                      ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(ufs INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
@@ -252,12 +239,6 @@ if(STOCH_PHYS)
   list(APPEND _ufs_libs_public stochastic_physics)
 endif()
 
-if(NEMSdatm)
-  add_dependencies(ufs datatm)
-  list(APPEND _ufs_defs_private FRONT_NEMS_DATM=datm)
-  list(APPEND _ufs_libs_public datatm)
-endif()
-
 if(FV3)
   add_dependencies(ufs fv3atm)
   list(APPEND _ufs_defs_private FRONT_FV3=fv3gfs_cap_mod)
@@ -301,7 +282,7 @@ target_link_libraries(ufs PUBLIC "${_ufs_libs_public}")
 ###############################################################################
 ### UFS executable
 ###############################################################################
-add_executable(ufs_model NEMS/src/MAIN_NEMS.F90)
+add_executable(ufs_model driver/UFS.F90)
 add_dependencies(ufs_model ufs)
 target_link_libraries(ufs_model ufs esmf w3nco::w3nco_d)
 

HYCOM-interface/CMakeLists.txt

@@ -11,7 +11,7 @@ if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
 elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
   set(CMAKE_Fortran_FLAGS "-g -traceback -xSSE4.2 -mcmodel=small -r8")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -fp-model source -warn nogeneral")
-  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv -link_mpi=dbg")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv -link_mpi=dbg -init=snan,arrays")
   set(CMAKE_Fortran_LINK_FLAGS "-V ${CMAKE_Fortran_FLAGS} -static-intel")
 else()
   message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")

MOM6-interface/CMakeLists.txt

@@ -11,7 +11,7 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -i4 -r8")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -sox")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal -fp-model source")
-  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv -init=snan,arrays")
   set(CMAKE_Fortran_LINK_FLAGS "")
 else()
   message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")
@@ -46,6 +46,7 @@ target_include_directories(mom6_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURR
                                                   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
 target_link_libraries(mom6_nuopc_obj PRIVATE mom6_obj
                                              fms
+                                             stochastic_physics
                                              esmf)
 # OpenMP is disabled in MOM6
 #if(OpenMP_Fortran_FOUND)

MOM6-interface/mom6_files.cmake

@@ -181,6 +181,8 @@ list(APPEND mom6_src_files
   MOM6/src/parameterizations/CVmix/cvmix_tidal.F90
   MOM6/src/parameterizations/CVmix/cvmix_utils.F90
 
+  MOM6/src/parameterizations/stochastic/MOM_stochastics.F90
+
   MOM6/src/tracer/DOME_tracer.F90
 
   MOM6/src/tracer/ISOMIP_tracer.F90
@@ -189,6 +191,7 @@ list(APPEND mom6_src_files
   MOM6/src/tracer/MOM_generic_tracer.F90
   MOM6/src/tracer/MOM_lateral_boundary_diffusion.F90
   MOM6/src/tracer/MOM_neutral_diffusion.F90
+  MOM6/src/tracer/nw2_tracers.F90
   MOM6/src/tracer/MOM_offline_aux.F90
   MOM6/src/tracer/MOM_offline_main.F90
   MOM6/src/tracer/MOM_tracer_Z_init.F90
@@ -254,7 +257,6 @@ list(APPEND mom6_src_files
   MOM6/config_src/external/ODA_hooks/ocean_da_core.F90
   MOM6/config_src/external/ODA_hooks/ocean_da_types.F90
   MOM6/config_src/external/ODA_hooks/write_ocean_obs.F90
-  MOM6/config_src/external/OCEAN_stochastic_phyiscs/MOM_stochastics.F90
 
   MOM6/config_src/infra/FMS1/MOM_coms_infra.F90
   MOM6/config_src/infra/FMS1/MOM_constants.F90

build.sh

@@ -11,13 +11,9 @@ export CC=${CC:-mpicc}
 export CXX=${CXX:-mpicxx}
 export FC=${FC:-mpif90}
 
-export ESMFMKFILE=${ESMFMKFILE:?"Please set ESMFMKFILE environment variable"}
-
 BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build}
 mkdir -p ${BUILD_DIR}
 
-[[ -n "${MAPL_ROOT:-""}" ]] && CMAKE_FLAGS+=" -DCMAKE_MODULE_PATH=${MAPL_ROOT}/share/MAPL/cmake"
-
 cd ${BUILD_DIR}
 cmake ${UFS_MODEL_DIR} ${CMAKE_FLAGS}
 # Turn off OpenMP threading for parallel builds

cmake/GNU.cmake

@@ -27,7 +27,9 @@ else()
 endif()
 
 # For ccpp
-set(CMAKE_Fortran_FLAGS_DEFAULT_PREC "-fdefault-real-8 -fdefault-double-8")
 set(CMAKE_C_FLAGS_RELEASE       "-O2 -fPIC" CACHE STRING "" FORCE)
 set(CMAKE_CXX_FLAGS_RELEASE     "-O2 -fPIC" CACHE STRING "" FORCE)
 set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE)
+set(CMAKE_C_FLAGS_BITFORBIT       "-O2 -fPIC" CACHE STRING "" FORCE)
+set(CMAKE_CXX_FLAGS_BITFORBIT     "-O2 -fPIC" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE)

cmake/Intel.cmake

@@ -7,7 +7,7 @@ endif()
 
 if(DEBUG)
     add_definitions(-DDEBUG)
-    set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -ftrapuv")
+    set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -ftrapuv -init=snan,arrays")
     if(DEBUG_LINKMPI)
       if(OPENMP)
         set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -link_mpi=dbg_mt")
@@ -31,24 +31,25 @@ else()
     if(AVX2)
         set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2")
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2")
-        set(CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX2")
     elseif(SIMDMULTIARCH)
         set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,CORE-AVX2")
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,CORE-AVX2")
-        set(CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,CORE-AVX2")
      elseif(AVX)
         set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx-i")
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx-i")
-        set(CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX-I")
     endif()
 endif()
 
 # For ccpp
-set(CMAKE_Fortran_FLAGS_DEFAULT_PREC "-real-size 64")
 set(CMAKE_C_FLAGS_RELEASE       "-O2 -fPIC" CACHE STRING "" FORCE)
 set(CMAKE_CXX_FLAGS_RELEASE     "-O2 -fPIC" CACHE STRING "" FORCE)
 set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE)
-
 set(CMAKE_C_FLAGS_BITFORBIT       "-O2 -fPIC" CACHE STRING "" FORCE)
 set(CMAKE_CXX_FLAGS_BITFORBIT     "-O2 -fPIC" CACHE STRING "" FORCE)
 set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE)
+
+if(APPLE)
+  # The linker on macOS does not include `common symbols` by default
+  # Passing the -c flag includes them and fixes an error with undefined symbols
+  set(CMAKE_Fortran_ARCHIVE_FINISH "<CMAKE_RANLIB> -c <TARGET>")
+endif()

cmake/configure_apps.cmake

@@ -25,19 +25,14 @@ if(APP MATCHES "^(ATM|ATMW)$")
   endif()
 endif()
 
-if(APP MATCHES "^(NG-GODAS|NG-GODAS-NEMSDATM)$")
+if(APP MATCHES "^(NG-GODAS)$")
   set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
   set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
   set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
   set(MOM6       ON  CACHE BOOL "Enable MOM6"                FORCE)
   set(CICE6      ON  CACHE BOOL "Enable CICE6"               FORCE)
-  if(APP MATCHES "NG-GODAS-NEMSDATM")
-    set(NEMSdatm ON  CACHE BOOL "Enable NEMS DataAtm"        FORCE)
-    message("Configuring UFS app in (NEMS) Data Atmosphere mode")
-  elseif(APP MATCHES "NG-GODAS")
-    set(CDEPS    ON  CACHE BOOL "Enable CDEPS"               FORCE)
-    message("Configuring UFS app in (CDEPS) Data Atmosphere mode")
-  endif()
+  set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
+  message("Configuring UFS app in (CDEPS) Data Atmosphere mode")
 endif()
 
 if(APP MATCHES "^(S2S|S2SW)$")

cmake/configure_expanse.intel.cmake

@@ -0,0 +1,2 @@
+set(INLINE_POST     ON  CACHE BOOL "Enable inline post" FORCE)
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)

cmake/configure_macosx.gnu.cmake

@@ -1,4 +1,5 @@
-set(INLINE_POST     ON  CACHE BOOL "Enable inline post" FORCE)
+set(INLINE_POST     ON CACHE BOOL "Enable inline post" FORCE)
+set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)
 
 # OpenMP broken for clang compiler
 if(${CMAKE_C_COMPILER_ID} MATCHES "^(Clang|AppleClang)$")

cmake/configure_macosx.intel.cmake

@@ -0,0 +1,3 @@
+set(INLINE_POST     ON CACHE BOOL "Enable inline post" FORCE)
+set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)
+set(DEBUG_LINKMPI  OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)