CCPP cmake cleanup. Use ESMF 8.1.1. Update MOM6 (ufs-community#548)

Update compiler flags in cmake/Intel.cmake and cmake/GNU.cmke so that CCPP can use the flags set by UFS instead of maintaining it's own copy of all flags Use ESMF version 8.1.1 Update MOM6 submodule
MinsukJi-NOAA · May 9, 2021 · 0aa43a0 · 0aa43a0
1 parent 68cb84b
commit 0aa43a0
Show file tree

Hide file tree

Showing 41 changed files with 4,789 additions and 3,793 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,36 +1,27 @@
 cmake_minimum_required(VERSION 3.15)
 
-foreach(env_var IN ITEMS CMAKE_C_COMPILER CMAKE_CXX_COMPILER CMAKE_Fortran_COMPILER)
-  if(NOT DEFINED ENV{${env_var}})
-    message(FATAL_ERROR "${env_var} is not defined")
-  endif()
-endforeach()
-
-set(CMAKE_C_COMPILER $ENV{CMAKE_C_COMPILER})
-set(CMAKE_CXX_COMPILER $ENV{CMAKE_CXX_COMPILER})
-set(CMAKE_Fortran_COMPILER $ENV{CMAKE_Fortran_COMPILER})
-set(CMAKE_Platform $ENV{CMAKE_Platform})
-
 project(ufs
         VERSION 1.0
         LANGUAGES C CXX Fortran)
 
+if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 9.0.0)
+  message(FATAL_ERROR "GNU Compiler >= 9 is required")
+endif()
+
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 
-set(32BIT           OFF CACHE BOOL "Enable 32BIT (single precision arithmetic in dycore)")
-set(AVX2            ON  CACHE BOOL "Enable AVX2 instruction set")
-set(SIMDMULTIARCH   OFF CACHE BOOL "Enable multi-target SIMD instruction sets")
-set(DEBUG           OFF CACHE BOOL "Enable DEBUG mode")
-set(DEBUG_LINKMPI   ON  CACHE BOOL "Enable linkmpi option when DEBUG mode is on")
-set(INLINE_POST     OFF CACHE BOOL "Enable inline post")
-set(MULTI_GASES     OFF CACHE BOOL "Enable MULTI_GASES")
-set(OPENMP          ON  CACHE BOOL "Enable OpenMP threading")
-set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
-set(REPRO           OFF CACHE BOOL "Enable REPRO mode")
-set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
+###############################################################################
+### App and Component Options
+###############################################################################
+
+# Valid applications and choices
+list(APPEND VALID_APPS ATM ATMW S2S S2SW DATM DATM_NEMS)
+set(APP NONE CACHE BOOL "Application Name")
+if(NOT (APP IN_LIST VALID_APPS))
+  message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
+endif()
 
-# Component Options
 set(FMS             OFF CACHE BOOL "Enable FMS")
 set(FV3             OFF CACHE BOOL "Enable FV3")
 set(MOM6            OFF CACHE BOOL "Enable MOM6")
@@ -41,22 +32,37 @@ set(NEMSdatm        OFF CACHE BOOL "Enable NEMSdatm")
 set(CMEPS           OFF CACHE BOOL "Enable CMEPS")
 set(CDEPS           OFF CACHE BOOL "Enable CDEPS")
 
-# Valid applications and choices
-list(APPEND VALID_APPS ATM ATMW S2S S2SW DATM DATM_NEMS)
-set(APP NONE CACHE BOOL "Application Name")
-if(NOT (APP IN_LIST VALID_APPS))
-  message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
-endif()
+# Configure selected application specific components
+message("")
+include(cmake/configure_apps.cmake)
+
+message("")
+message("FMS .............. ${FMS}")
+message("FV3 .............. ${FV3}")
+message("MOM6 ............. ${MOM6}")
+message("CICE6 ............ ${CICE6}")
+message("WW3 .............. ${WW3}")
+message("STOCH_PHYS ....... ${STOCH_PHYS}")
+message("NEMSdatm ......... ${NEMSdatm}")
+message("CDEPS ............ ${CDEPS}")
+message("CMEPS ............ ${CMEPS}")
 
 ###############################################################################
-### Set CMAKE_BUILD_TYPE for DEBUG mode
+### Build Options
 ###############################################################################
-if(DEBUG)
-  set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Set type of build to Debug." FORCE)
-else()
-  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Set type of build to Release." FORCE)
-endif()
+set(32BIT           OFF CACHE BOOL "Enable 32BIT (single precision arithmetic in dycore)")
+set(AVX2            ON  CACHE BOOL "Enable AVX2 instruction set")
+set(SIMDMULTIARCH   OFF CACHE BOOL "Enable multi-target SIMD instruction sets")
+set(DEBUG           OFF CACHE BOOL "Enable DEBUG mode")
+set(DEBUG_LINKMPI   ON  CACHE BOOL "Enable linkmpi option when DEBUG mode is on")
+set(INLINE_POST     OFF CACHE BOOL "Enable inline post")
+set(MULTI_GASES     OFF CACHE BOOL "Enable MULTI_GASES")
+set(OPENMP          ON  CACHE BOOL "Enable OpenMP threading")
+set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
+set(REPRO           OFF CACHE BOOL "Enable REPRO mode")
+set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 
+set(CMAKE_Platform $ENV{CMAKE_Platform})
 if(CMAKE_Platform)
   message("")
   if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/cmake/configure_${CMAKE_Platform}.cmake)
@@ -79,6 +85,7 @@ if(CMAKE_Platform)
 endif()
 
 message("")
+message("32BIT ............ ${32BIT}")
 message("AVX2 ............. ${AVX2}")
 message("SIMDMULTIARCH ...  ${SIMDMULTIARCH}")
 message("DEBUG ............ ${DEBUG}")
@@ -88,41 +95,19 @@ message("MULTI_GASES ...... ${MULTI_GASES}")
 message("OPENMP ........... ${OPENMP}")
 message("PARALLEL_NETCDF .. ${PARALLEL_NETCDF}")
 message("REPRO ............ ${REPRO}")
+message("JEDI_DRIVER ...... ${JEDI_DRIVER}")
 message("")
 
-# Configure selected application specific components
-include(cmake/configure_apps.cmake)
-
-# Components
-message("")
-message("FMS .............. ${FMS}")
-message("FV3 .............. ${FV3}")
-message("MOM6 ............. ${MOM6}")
-message("CICE6 ............ ${CICE6}")
-message("WW3 .............. ${WW3}")
-message("STOCH_PHYS ....... ${STOCH_PHYS}")
-message("NEMSdatm ......... ${NEMSdatm}")
-message("CDEPS ............ ${CDEPS}")
-message("CMEPS ............ ${CMEPS}")
-
-# Options that applications reset
-message("32BIT ............ ${32BIT}")
-message("")
-
-get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
-get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
-get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
-message("C       compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
-message("CXX     compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
-message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
-message("")
-
-if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 9.0.0)
-  message(FATAL_ERROR "GNU Compiler >= 9 is required")
+###############################################################################
+### Set CMAKE_BUILD_TYPE for DEBUG mode
+###############################################################################
+if(DEBUG)
+  set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Set type of build to Debug." FORCE)
+else()
+  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Set type of build to Release." FORCE)
 endif()
 
 include(cmake/${CMAKE_Fortran_COMPILER_ID}.cmake)
-message("")
 
 ###############################################################################
 ### Find Dependencies

diff --git a/FV3 b/FV3
diff --git a/MOM6-interface/MOM6 b/MOM6-interface/MOM6
diff --git a/build.sh b/build.sh
@@ -7,18 +7,16 @@ else
   readonly UFS_MODEL_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P)
 fi
 
-export CMAKE_C_COMPILER=${CMAKE_C_COMPILER:-${CC:-mpicc}}
-export CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER:-${CXX:-mpicxx}}
-export CMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER:-${FC:-mpif90}}
+export CC=${CC:-mpicc}
+export CXX=${CXX:-mpicxx}
+export FC=${FC:-mpif90}
 
-export NETCDF=${NETCDF:-${NETCDF_ROOT:?"Please set NETCDF/NETCDF_ROOT environment variable"}}
 export ESMFMKFILE=${ESMFMKFILE:?"Please set ESMFMKFILE environment variable"}
 
 BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build}
 mkdir -p ${BUILD_DIR}
 
 [[ -n "${CCPP_SUITES:-""}" ]] && CMAKE_FLAGS+=" -DCCPP_SUITES=${CCPP_SUITES}"
-CMAKE_FLAGS+=" -DNETCDF_DIR=${NETCDF}"
 
 cd ${BUILD_DIR}
 cmake ${UFS_MODEL_DIR} ${CMAKE_FLAGS}

diff --git a/cmake/GNU.cmake b/cmake/GNU.cmake
@@ -1,4 +1,4 @@
-set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -fcray-pointer -ffree-line-length-none -fno-range-check")
+set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check")
 
 if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10)
     set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz")
@@ -25,3 +25,9 @@ else()
     set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2")
 endif()
+
+# For ccpp
+set(CMAKE_Fortran_FLAGS_DEFAULT_PREC "-fdefault-real-8 -fdefault-double-8")
+set(CMAKE_C_FLAGS_RELEASE       "-O2 -fPIC" CACHE STRING "" FORCE)
+set(CMAKE_CXX_FLAGS_RELEASE     "-O2 -fPIC" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE)
diff --git a/cmake/Intel.cmake b/cmake/Intel.cmake
@@ -27,8 +27,20 @@ else()
     if(AVX2)
         set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2")
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2")
+        set(CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX2")
     elseif(SIMDMULTIARCH)
         set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,CORE-AVX2")
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,CORE-AVX2")
+        set(CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,CORE-AVX2")
     endif()
 endif()
+
+# For ccpp
+set(CMAKE_Fortran_FLAGS_DEFAULT_PREC "-real-size 64")
+set(CMAKE_C_FLAGS_RELEASE       "-O2 -fPIC" CACHE STRING "" FORCE)
+set(CMAKE_CXX_FLAGS_RELEASE     "-O2 -fPIC" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE)
+
+set(CMAKE_C_FLAGS_BITFORBIT       "-O2 -fPIC" CACHE STRING "" FORCE)
+set(CMAKE_CXX_FLAGS_BITFORBIT     "-O2 -fPIC" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE)
diff --git a/modulefiles/ufs_cheyenne.gnu b/modulefiles/ufs_cheyenne.gnu
@@ -7,10 +7,6 @@ proc ModulesHelp {} {
 module-whatis "loads UFS Model prerequisites for Cheyenne/GNU"
 
 module load cmake/3.16.4
-setenv CMAKE_C_COMPILER mpicc
-setenv CMAKE_CXX_COMPILER mpicxx
-setenv CMAKE_Fortran_COMPILER mpif90
-setenv CMAKE_Platform cheyenne.gnu
 
 # load programming environment
 module load ncarenv/1.3
@@ -25,3 +21,8 @@ module load hpc-gnu/9.1.0
 module load hpc-mpt/2.22
 
 module load ufs_common
+
+setenv CC mpicc
+setenv CXX mpicxx
+setenv FC mpif90
+setenv CMAKE_Platform cheyenne.gnu
diff --git a/modulefiles/ufs_cheyenne.gnu_debug b/modulefiles/ufs_cheyenne.gnu_debug
@@ -7,10 +7,6 @@ proc ModulesHelp {} {
 module-whatis "loads UFS Model prerequisites for Cheyenne/GNU"
 
 module load cmake/3.16.4
-setenv CMAKE_C_COMPILER mpicc
-setenv CMAKE_CXX_COMPILER mpicxx
-setenv CMAKE_Fortran_COMPILER mpif90
-setenv CMAKE_Platform cheyenne.gnu
 
 # load programming environment
 module load ncarenv/1.3
@@ -25,3 +21,8 @@ module load hpc-gnu/9.1.0
 module load hpc-mpt/2.22
 
 module load ufs_common_debug
+
+setenv CC mpicc
+setenv CXX mpicxx
+setenv FC mpif90
+setenv CMAKE_Platform cheyenne.gnu
diff --git a/modulefiles/ufs_cheyenne.intel b/modulefiles/ufs_cheyenne.intel
@@ -7,10 +7,6 @@ proc ModulesHelp {} {
 module-whatis "loads UFS Model prerequisites for Cheyenne/Intel"
 
 module load cmake/3.16.4
-setenv CMAKE_C_COMPILER mpicc
-setenv CMAKE_CXX_COMPILER mpicxx
-setenv CMAKE_Fortran_COMPILER mpif90
-setenv CMAKE_Platform cheyenne.intel
 
 # load programming environment
 module load ncarenv/1.3
@@ -25,3 +21,8 @@ module load hpc-intel/19.1.1
 module load hpc-mpt/2.22
 
 module load ufs_common
+
+setenv CC mpicc
+setenv CXX mpicxx
+setenv FC mpif90
+setenv CMAKE_Platform cheyenne.intel
diff --git a/modulefiles/ufs_cheyenne.intel_debug b/modulefiles/ufs_cheyenne.intel_debug
@@ -7,10 +7,6 @@ proc ModulesHelp {} {
 module-whatis "loads UFS Model prerequisites for Cheyenne/Intel"
 
 module load cmake/3.16.4
-setenv CMAKE_C_COMPILER mpicc
-setenv CMAKE_CXX_COMPILER mpicxx
-setenv CMAKE_Fortran_COMPILER mpif90
-setenv CMAKE_Platform cheyenne.intel
 
 # load programming environment
 module load ncarenv/1.3
@@ -25,3 +21,8 @@ module load hpc-intel/19.1.1
 module load hpc-mpt/2.22
 
 module load ufs_common_debug
+
+setenv CC mpicc
+setenv CXX mpicxx
+setenv FC mpif90
+setenv CMAKE_Platform cheyenne.intel
diff --git a/modulefiles/ufs_common b/modulefiles/ufs_common
@@ -7,7 +7,7 @@ module load png/1.6.35
 module load hdf5/1.10.6
 module load netcdf/4.7.4
 module load pio/2.5.2
-module load esmf/8_1_0_beta_snapshot_47
+module load esmf/8_1_1
 
 module load bacio/2.4.1
 module load crtm/2.3.0

diff --git a/modulefiles/ufs_common_debug b/modulefiles/ufs_common_debug
@@ -7,7 +7,7 @@ module load png/1.6.35
 module load hdf5/1.10.6
 module load netcdf/4.7.4
 module load pio/2.5.2
-module load esmf/8_1_0_beta_snapshot_47-debug
+module load esmf/8_1_1-debug
 
 module load bacio/2.4.1
 module load crtm/2.3.0

diff --git a/modulefiles/ufs_gaea.intel b/modulefiles/ufs_gaea.intel
@@ -17,10 +17,6 @@ module load cray-python/3.7.3.2
 module load alps
 
 module load cmake/3.17.0
-setenv CMAKE_C_COMPILER cc
-setenv CMAKE_CXX_COMPILER CC
-setenv CMAKE_Fortran_COMPILER ftn
-setenv CMAKE_Platform gaea.intel
 
 #needed for WW3 build
 module load gcc/8.3.0
@@ -32,3 +28,8 @@ module load hpc-intel/18.0.6.288
 module load hpc-cray-mpich/7.7.11
 
 module load ufs_common
+
+setenv CC cc
+setenv CXX CC
+setenv FC ftn
+setenv CMAKE_Platform gaea.intel
diff --git a/modulefiles/ufs_gaea.intel_debug b/modulefiles/ufs_gaea.intel_debug
@@ -17,10 +17,6 @@ module load cray-python/3.7.3.2
 module load alps
 
 module load cmake/3.17.0
-setenv CMAKE_C_COMPILER cc
-setenv CMAKE_CXX_COMPILER CC
-setenv CMAKE_Fortran_COMPILER ftn
-setenv CMAKE_Platform gaea.intel
 
 module use /lustre/f2/pdata/esrl/gsd/ufs/hpc-stack-v1.1.0/modulefiles/stack
 
@@ -29,3 +25,8 @@ module load hpc-intel/18.0.6.288
 module load hpc-cray-mpich/7.7.11
 
 module load ufs_common_debug
+
+setenv CC cc
+setenv CXX CC
+setenv FC ftn
+setenv CMAKE_Platform gaea.intel
diff --git a/modulefiles/ufs_hera.gnu b/modulefiles/ufs_hera.gnu
@@ -10,14 +10,15 @@ module use /contrib/sutils/modulefiles
 module load sutils
 
 module load cmake/3.16.1
-setenv CMAKE_C_COMPILER mpicc
-setenv CMAKE_CXX_COMPILER mpicxx
-setenv CMAKE_Fortran_COMPILER mpif90
-setenv CMAKE_Platform hera.gnu
 
 module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
 module load hpc/1.1.0
 module load hpc-gnu/9.2.0
 module load hpc-mpich/3.3.2
 
 module load ufs_common
+
+setenv CC mpicc
+setenv CXX mpicxx
+setenv FC mpif90
+setenv CMAKE_Platform hera.gnu
+14 −51		CMakeLists.txt
+88 −193		ccpp/CMakeLists.txt
+141 −141		ccpp/config/ccpp_prebuild_config.py
+0 −43		ccpp/data/CMakeLists.txt
+0 −50		ccpp/driver/CMakeLists.txt
+148 −20		config_src/drivers/nuopc_cap/mom_cap.F90
+57 −35		config_src/drivers/nuopc_cap/mom_cap_methods.F90
+24 −0		src/framework/MOM_random.F90
+2 −1		src/parameterizations/vertical/MOM_set_diffusivity.F90
+3 −5		src/tracer/MOM_lateral_boundary_diffusion.F90
+4 −2		src/tracer/MOM_neutral_diffusion.F90