Merge remote-tracking branch 'upstream/develop' into merra2

* upstream/develop:
  update MOM6 to GFDL 20210224 main branch commit (ufs-community#439)
  Add GNU and Cheyenne Support to Automated RT (ufs-community#444)
  Move Noah MP init to CCPP and update Noah MP regression tests, ice flux init bug fix in CCPP (ufs-community#425)
  Feature/rt automation (ufs-community#403)
  Update ccpp-physics. Make RRTMGP thread safe (ufs-community#418)
  Update regression tests from GFSv15+Thompson to GFSv16+Thompson, include "Add one regional regression test in DEBUG mode. (ufs-community#419)" (ufs-community#421)
  UGWP v0 v1 combined (ufs-community#396)
  add optional mesh in MOM6; add dz_min and min_seaice as configurable variables for coupled model (ufs-community#399)
  updates FMS to 2020.04.01 (ufs-community#392)
  Move LSM vegetation lookup tables into CCPP, clean up RUC snow cover on ice initialization (remove IPD step 2)  (ufs-community#407)
  Update CMEPS for HAFS integration; add datm and coupled-model tests on Gaea (ufs-community#401)
  Remove legacy gnumake build from fv3atm and NEMS, remove legacy Python 2.7 support, rename v16beta to v16 and RT updates (ufs-community#384)
  MOM6 bugfixes, GFDL update, update CDMBGWD settings; fix for restart reproducibility (without waves) when USE_LA_LI2016=True, sign error on fprec passed to ocean, GFDL update, resolution dependent cdmbgwd settings (ufs-community#379)
  dycore options to add zero-gradient BC to reconstruct interface u/v and change dz_min as input (ufs-community#369)
  Update develop from NOAA-GSL: RUC ice, MYNN sfclay, stochastic land perturbations (ufs-community#386)
  update cpl gfsv16 tests, rrtmgp fix and bug fixes in cmeps (ufs-community#378)
  point fv3 to EMC develop branch (ufs-community#377)
  Remove IPD steps 3 and 5 (ufs-community#357)
  Update CMEPS  (ufs-community#345)
  Implementation of CCPP timestep_init and timestep_final phases (ufs-community#337)
  Remove unnecessary SIMD instruction sets for Jet, first round of cleanup in rt.conf, initialize cld_amt to zero for regional runs (dycore) (ufs-community#353)
  add frac grid input, update and add additional cpld tests (ufs-community#354)
  Add checkpoint restarts for ufs-cpld (ufs-community#342)
  Update the format of rt.conf (ufs-community#349)
  Remove IPD (step 1) (ufs-community#331)
  Feature/ww3update (ufs-community#334)
  Replace old regional SDF with FV3_GFS_v15_thompson_mynn (ufs-community#333)
  Update modules with hpc-stack v1.1.0 (ufs-community#319)
  Regression test log for PR ufs-community#323 for jet.intel (ufs-community#336)
  RRTMGP and Thompson MP coupling (ufs-community#323)
  Add 2 new tests for DATM-MOM6-CICE6 application (ufs-community#332)
  Add optional bulk flux calculation in ufs-datm (ufs-community#266)
  Final-final GFS v16 updates / restart reproducibility bugfixes (ufs-community#325)
  Updates to build for JEDI linking/control, add wcoss2 (ufs-community#295)
  Update CICE, Move regression test input outside baseline directory (ufs-community#270)
  Feature/update mom6 and retain b4b results for 025x025 resolution (ufs-community#290)
  Update for Jet, bug fixes in running with frac_grid=T and GFDL MP, and in restarting with frac_grid=T  (ufs-community#304)
  Updates to stochastic_physics_wrapper (ufs-community#280)
  Update develop from gsd/develop 2020/11/20: Unified gravity wave drag, updates to other GSL physics (ufs-community#297)
  Fix to allow quilting with non-factors for layout (ufs-community#250)
  rt update (ufs-community#261)

.github/workflows/main.yml

@@ -68,7 +68,7 @@ jobs:
     - name: Free up disk space
       run: |
         sudo docker rmi $(sudo docker image ls | grep -E -m1 '<none>' | awk '{ print $3 }')
-        sudo docker rmi $(sudo docker image ls | awk '/ci-test-base/ { print $3 }')
+        sudo docker rmi $(sudo docker image ls | awk '/ubuntu-hpc/ { print $3 }')
 
     - name: Prepare artifacts
       run: |

.github/workflows/manage.yml

@@ -22,15 +22,15 @@ jobs:
           tr_id=$(cat ${GITHUB_EVENT_PATH} | ./json_helper.py get_trigger_id)
           tr_br=$(cat ${GITHUB_EVENT_PATH} | ./json_helper.py get_trigger_br)
           check=$(cat ${GITHUB_EVENT_PATH} | ./json_helper.py check_skip)
-          echo "::set-env name=TRIGGER_ID::${tr_id}"
-          echo "::set-env name=TRIGGER_BR::${tr_br}"
+          echo "TRIGGER_ID=${tr_id}" >> $GITHUB_ENV
+          echo "TRIGGER_BR=${tr_br}" >> $GITHUB_ENV
           echo "skip-ci: ${check}"
           if [[ $check == yes ]]; then
             echo "skip-ci is requested"
-            echo '::set-env name=CURR_JOB::cancelled'
+            echo "CURR_JOB=cancelled" >> $GITHUB_ENV
             curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github.v3+json" ${repo}/$tr_id/cancel
           else
-            echo '::set-env name=CURR_JOB::running'
+            echo "CURR_JOB=running" >> $GITHUB_ENV
           fi
 
       - name: Cancel redundant jobs

.gitmodules

@@ -11,7 +11,7 @@
 [submodule "FMS"]
   path = FMS
   url = https://github.com/NOAA-GFDL/FMS
-  branch = master
+  branch = release/2020.04
 [submodule "WW3"]
   path = WW3
   url = https://github.com/NOAA-EMC/WW3

CICE-interface/CMakeLists.txt

@@ -9,7 +9,7 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
   set(CMAKE_Fortran_FLAGS "-g -traceback")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -FR -convert big_endian -assume byterecl -ftz -align array64byte -xHOST")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fp-model precise")
-  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
   set(CMAKE_Fortran_LINK_FLAGS "")
 else()
   message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")

CICE-interface/cice_files.cmake

@@ -28,6 +28,7 @@ list(APPEND cice_shared_files
   CICE/cicecore/cicedynB/dynamics/ice_dyn_evp.F90
   CICE/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90
   CICE/cicecore/cicedynB/dynamics/ice_dyn_shared.F90
+  CICE/cicecore/cicedynB/dynamics/ice_dyn_vp.F90
   CICE/cicecore/cicedynB/dynamics/ice_transport_driver.F90
   CICE/cicecore/cicedynB/dynamics/ice_transport_remap.F90
 

CMEPS-interface/CMakeLists.txt

@@ -16,17 +16,17 @@ else()
 endif()
 
 list(APPEND _nems_util_files
-  CMEPS/nems/util/shr_abort_mod.F90
-  CMEPS/nems/util/shr_log_mod.F90
-  CMEPS/nems/util/shr_pio_mod.F90
-  CMEPS/nems/util/shr_sys_mod.F90
-  CMEPS/nems/util/shr_flux_mod.F90
-  CMEPS/nems/util/shr_mpi_mod.F90
-  CMEPS/nems/util/glc_elevclass_mod.F90
-  CMEPS/nems/util/shr_mem_mod.F90
-  CMEPS/nems/util/shr_kind_mod.F90
-  CMEPS/nems/util/perf_mod.F90
-  CMEPS/nems/util/shr_const_mod.F90)
+  CMEPS/util/shr_abort_mod.F90
+  CMEPS/util/shr_log_mod.F90
+  CMEPS/util/shr_pio_mod.F90
+  CMEPS/util/shr_sys_mod.F90
+  CMEPS/util/shr_flux_mod.F90
+  CMEPS/util/shr_mpi_mod.F90
+  CMEPS/util/glc_elevclass_mod.F90
+  CMEPS/util/shr_mem_mod.F90
+  CMEPS/util/shr_kind_mod.F90
+  CMEPS/util/perf_mod.F90
+  CMEPS/util/shr_const_mod.F90)
 
 list(APPEND _mediator_files
   CMEPS/mediator/med_phases_restart_mod.F90
@@ -56,7 +56,14 @@ list(APPEND _mediator_files
   CMEPS/mediator/med_fraction_mod.F90
   CMEPS/mediator/med_phases_aofluxes_mod.F90
   CMEPS/mediator/med_io_mod.F90
-  CMEPS/mediator/med_diag_mod.F90)
+  CMEPS/mediator/med_diag_mod.F90
+  CMEPS/mediator/med_phases_post_atm_mod.F90
+  CMEPS/mediator/med_phases_post_ice_mod.F90
+  CMEPS/mediator/med_phases_post_ocn_mod.F90
+  CMEPS/mediator/med_phases_post_lnd_mod.F90
+  CMEPS/mediator/med_phases_post_glc_mod.F90
+  CMEPS/mediator/med_phases_post_rof_mod.F90
+  CMEPS/mediator/med_phases_post_wav_mod.F90)
 
 list(APPEND _mediator_defs ESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR}
                            ESMF_VERSION_MINOR=${ESMF_VERSION_MINOR}

CMakeLists.txt

@@ -21,7 +21,6 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 set(32BIT           OFF CACHE BOOL "Enable 32BIT (single precision arithmetic in dycore)")
 set(AVX2            ON  CACHE BOOL "Enable AVX2 instruction set")
 set(SIMDMULTIARCH   OFF CACHE BOOL "Enable multi-target SIMD instruction sets")
-set(CCPP            ON  CACHE BOOL "Enable CCPP")
 set(DEBUG           OFF CACHE BOOL "Enable DEBUG mode")
 set(DEBUG_LINKMPI   ON  CACHE BOOL "Enable linkmpi option when DEBUG mode is on")
 set(INLINE_POST     OFF CACHE BOOL "Enable inline post")
@@ -32,13 +31,16 @@ set(QUAD_PRECISION  ON  CACHE BOOL "Enable QUAD_PRECISION (for certain grid metr
 set(REPRO           OFF CACHE BOOL "Enable REPRO mode")
 set(WW3             OFF CACHE BOOL "Enable WW3")
 set(S2S             OFF CACHE BOOL "Enable S2S")
+set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 set(DATM            OFF CACHE BOOL "Enable Data Atmosphere")
 
 ###############################################################################
 ### Set CMAKE_BUILD_TYPE for DEBUG mode
 ###############################################################################
 if(DEBUG)
   set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Set type of build to Debug." FORCE)
+else()
+  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Set type of build to Release." FORCE)
 endif()
 
 if(CMAKE_Platform)
@@ -49,13 +51,23 @@ if(CMAKE_Platform)
   else()
     message("Platform '${CMAKE_Platform}' configuration file does not exist")
   endif()
+
+  # DH* 20210208 temporary workaround for FMS issues on Cheyenne with GNU 9.1.0
+  if (CMAKE_Platform MATCHES "cheyenne.gnu")
+    message("Applying patch cheyenne_gnu_fms_mpp_util_mpi_inc.patch")
+    execute_process(COMMAND patch -N -p0 INPUT_FILE cheyenne_gnu_fms_mpp_util_mpi_inc.patch
+                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+                    OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/cheyenne_gnu_fms_mpp_util_mpi_inc.patch.out
+                    ERROR_FILE  ${CMAKE_CURRENT_BINARY_DIR}/cheyenne_gnu_fms_mpp_util_mpi_inc.patch.err
+                    RESULT_VARIABLE RC)
+    # *DH 20210208
+  endif()
 endif()
 
 message("")
 message("32BIT ............ ${32BIT}")
 message("AVX2 ............. ${AVX2}")
 message("SIMDMULTIARCH ...  ${SIMDMULTIARCH}")
-message("CCPP ............. ${CCPP}")
 message("DEBUG ............ ${DEBUG}")
 message("DEBUG_LINKMPI .... ${DEBUG_LINKMPI}")
 message("INLINE_POST ...... ${INLINE_POST}")
@@ -109,13 +121,9 @@ endif()
 # See https://github.com/NOAA-EMC/NCEPLIBS-nemsio/pull/22
 target_link_libraries(nemsio::nemsio INTERFACE w3emc::w3emc_d bacio::bacio_4)
 
-if(CCPP)
-  find_package(Python 3 QUIET COMPONENTS Interpreter)
-  if (NOT Python_Interpreter_FOUND)
-    find_package(Python 2.7 REQUIRED COMPONENTS Interpreter)
-  endif()
-  message("Found Python: ${Python_EXECUTABLE}")
-endif()
+# Configure Python
+find_package(Python 3 REQUIRED COMPONENTS Interpreter)
+message("Found Python: ${Python_EXECUTABLE}")
 
 ###############################################################################
 ### Checks
@@ -132,9 +140,7 @@ endif()
 ### FMS
 ###############################################################################
 include(fms_files.cmake)
-add_library(fms ${fms_src_files})
-# stupid cmake can not figure out dependency of fft.F90 on fft99.F90 because 'use fft99_mod' is inside ifdefs
-set_property(SOURCE FMS/fft/fft.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-DSGICRAY=0 -DNAGFFT=0")
+add_library(fms ${fms_src_files} ${fms_headers})
 
 list(APPEND _fms_defs_public use_libMPI
                              use_netCDF
@@ -143,26 +149,39 @@ list(APPEND _fms_defs_public use_libMPI
 if(QUAD_PRECISION)
   list(APPEND _fms_defs_public ENABLE_QUAD_PRECISION)
 endif()
+
+# check gettid
+include(CheckFunctionExists)
+check_function_exists(gettid HAVE_GETTID)
+if(HAVE_GETTID)
+  list(APPEND _fms_defs_public HAVE_GETTID)
+endif()
+
 target_compile_definitions(fms PUBLIC "${_fms_defs_public}")
 
 if(32BIT)
   list(APPEND _fms_defs_private OVERLOAD_R4
                                 OVERLOAD_R8)
 endif()
+
 target_compile_definitions(fms PRIVATE "${_fms_defs_private}")
 
 target_include_directories(fms PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/include>
                                       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/fms>
+                                      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/fms2_io/include>
+                                      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FMS/mod>
                                       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/mpp/include>)
+target_include_directories(fms INTERFACE
+                                      $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/include>
+                                      $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/mod>)
 target_link_libraries(fms PUBLIC MPI::MPI_Fortran
                                  NetCDF::NetCDF_Fortran)
 if(OpenMP_Fortran_FOUND)
   target_link_libraries(fms PRIVATE OpenMP::OpenMP_Fortran)
 endif()
 set_target_properties(fms PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS)
 set_target_properties(fms PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod)
-target_include_directories(fms INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FMS/mod>
-                                         $<INSTALL_INTERFACE:FMS/mod>)
+set_target_properties(fms PROPERTIES PUBLIC_HEADER "${fms_headers}" )
 
 ###############################################################################
 ### stochastic_physics
@@ -189,6 +208,10 @@ if(WW3)
       set(WW3_COMP "hera")
     elseif(${CMAKE_Platform} STREQUAL "orion.intel")
       set(WW3_COMP "orion")
+    elseif(${CMAKE_Platform} STREQUAL "cheyenne.intel")
+      set(WW3_COMP "cheyenne")
+    elseif(${CMAKE_Platform} STREQUAL "wcoss2")
+      set(WW3_COMP "wcoss_cray")
     elseif(${CMAKE_Platform} MATCHES "linux*" OR ${CMAKE_Platform} MATCHES "macosx*")
       if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
         set(WW3_COMP "gnu")
@@ -232,8 +255,7 @@ list(APPEND _nems_srcs NEMS/src/module_NEMS_UTILS.F90
                        NEMS/src/module_NEMS_INTERNAL_STATE.F90
                        NEMS/src/module_NEMS_GRID_COMP.F90
                        NEMS/src/module_NEMS_Rusage.F90
-                       NEMS/src/nems_c_rusage.c
-                       NEMS/src/ENS_Cpl/ENS_CplComp_ESMFMod_STUB.F90)
+                       NEMS/src/nems_c_rusage.c)
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/NEMS/src/ESMFVersionDefine_ESMF_NUOPC.h
                ${CMAKE_CURRENT_BINARY_DIR}/NEMS/src/ESMFVersionDefine.h
@@ -259,7 +281,9 @@ else()
   list(APPEND _ufs_defs_private FRONT_FV3=fv3gfs_cap_mod)
   add_dependencies(ufs fv3atm)
 endif()
-
+if(JEDI_DRIVER)
+  list(APPEND _ufs_defs_private JEDI_DRIVER=ON)
+endif()
 set_target_properties(ufs PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(ufs PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/NEMS/src)
 
@@ -314,7 +338,7 @@ endif()
 target_compile_definitions(ufs_model PRIVATE "${_ufs_model_defs_private}")
 
 if(DATM)
-  target_link_libraries(ufs_model PRIVATE ufs w3nco::w3nco_d)
+  target_link_libraries(ufs_model PUBLIC ufs w3nco::w3nco_d)
 endif()
 target_link_libraries(ufs_model PRIVATE ufs
                                         esmf
@@ -325,17 +349,28 @@ target_link_libraries(ufs_model PRIVATE ufs
 ### Install
 ###############################################################################
 install(
-  TARGETS ufs_model ufs fms
-  EXPORT  ufs-config
-  RUNTIME DESTINATION bin
+  TARGETS fms
+  EXPORT fms-config
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib
+  PUBLIC_HEADER DESTINATION include )
+
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
+
+install(EXPORT fms-config
+  DESTINATION lib/cmake
+)
+install(
+  TARGETS ufs
+  EXPORT ufs-config
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)
 
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
 
-install(EXPORT      ufs-config
-        DESTINATION lib/cmake)
-
+install(EXPORT ufs-config
+  DESTINATION lib/cmake
+)
 ###############################################################################
 ### Done
 ###############################################################################

MOM6-interface/CMakeLists.txt

@@ -11,11 +11,12 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -i4 -r8")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -sox")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal -fp-model source")
-  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv -link_mpi=dbg")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv")
   set(CMAKE_Fortran_LINK_FLAGS "")
 else()
   message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")
 endif()
+set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${MOM6_Extra_FORTRAN_FLAGS}")
 
 # User option to build MOM6 solo executable
 set(MOM6SOLO OFF CACHE BOOL "Build MOM6 solo")