diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..21c5ba4f4 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# This is a comment. +# Each line is a file pattern followed by one or more owners. +# Order is important; the last matching pattern takes the most +# precedence. + +# The components are owned by their respective teams +/FV3/ @noaa-emc/fv3atm-team diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..faa664ab3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,56 @@ +--- +name: Bug report +about: Create a report to fix bugs +title: '' +labels: bug +assignees: '' + +--- + +## Description +Provide a clear and concise description of what the bug is. +Also give a description of how to fix the bug. + + +### To Reproduce: +What compilers/machines are you seeing this with? +Give explicit steps to reproduce the behavior. +1. do this +2. then that +3. then, oops, look at the bug + + +## Additional context +Add any other context about the problem here. +Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example: +- needs to be fixed also in noaa-emc/nems/issues/ +- needed for noaa-emc/fv3atm/pull/ + + +## Output + +**Screenshots** +If applicable, drag and drop screenshots to help explain your problem. + +**output logs** +If applicable, include relevant output logs. +Either drag and drop the entire log file here (if a long log) or + +``` +paste the code here (if a short section of log) +``` + +## Testing: + +1. Have you tested the code changes? On what platforms? + +2. Have you run regression test in ufs-weather-model or ufs-s2s-model with code changes? +- Will the baseline results change? +- If the baseline results change, is it expected? Please give brief explanation. + +## Dependent PRs: + +Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example: +- required to support noaa-emc/GFDL_atmos_cubed_sphere/issues/ +- ncar/ccpp-physics/pull/ +- associated ufs-weather-model/pull/ diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..12db08eda --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,32 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +## Description +Provide a clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +## Solution +Add a clear and concise description of what solution you provide. + +## Alternatives +If applicable, add a description of any alternative solutions or features you've considered. + +## Testing: + +1. Have you tested the code changes? On what platforms? +2. Have you run regression test in ufs-weather-model or ufs-s2s-model with code changes? +- Will the baseline results change? +- If yes, please give brief explanation of your code updates on the regression test. + +## Dependent PRs: + +Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example: +- required to support noaa-emc/GFDL_atmos_cubed_sphere/issues/ +- ncar/ccpp-physics/pull/ +- associated ufs-weather-model/pull/ + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..e4d67029f --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,37 @@ +## Description + +(Instructions: this, and all subsequent sections of text should be removed and filled in as appropriate.) +Provide a detailed description of what this PR does. +What bug does it fix, or what feature does it add? +Is a change of answers expected from this PR? + + + +### Issue(s) addressed + +Link the issues to be closed with this PR, whether in this repository, or in another repository. +(Remember, issues should always be created before starting work on a PR branch!) +- fixes # +- fixes noaa-emc/fv3atm/issues/ + + + +## Testing + +How were these changes tested? +What compilers / HPCs was it tested with? +Are the changes covered by regression tests? (If not, why? Do new tests need to be added?) +Have the ufs-weather-model regression test been run? On what platform? +- Will the code updates change regression test baseline? If yes, why? Please show the baseline directory below. +- Please commit the regression test log files in your ufs-weather-model branch + + +## Dependencies + +If testing this branch requires non-default branches in other repositories, list them. +Those branches should have matching names (ideally) + +Do PRs in upstream repositories need to be merged first? +If so add the "waiting for other repos" label and list the upstream PRs +- waiting on noaa-emc/nems/pull/ +- waiting on noaa-emc/fv3atm/pull/ diff --git a/CMakeLists.txt b/CMakeLists.txt index dadaf50f8..d5013e53e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,81 +1,134 @@ -if( NOT DEFINED PHYS) - set(PHYS gfs) -endif() +if(CCPP) -if (PHYS MATCHES "nam") - add_definitions(-DNAM_phys) -endif() + if(DEBUG) + set(_ccpp_debug_arg "--debug") + endif() + if(DEFINED CCPP_SUITES) + set(_ccpp_suites_arg "--suites=${CCPP_SUITES}") + message("Calling CCPP code generator (ccpp_prebuild.py) for suites ${_ccpp_suites_arg} ...") + else() + message("Calling CCPP code generator (ccpp_prebuild.py) for all available suites ...") + endif() + execute_process(COMMAND ${Python_EXECUTABLE} + "ccpp/framework/scripts/ccpp_prebuild.py" + "--config=ccpp/config/ccpp_prebuild_config.py" + "--builddir=${CMAKE_CURRENT_BINARY_DIR}" ${_ccpp_suites_arg} ${_ccpp_debug_arg} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/ccpp_prebuild.out + ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/ccpp_prebuild.err + RESULT_VARIABLE RC) + # Check return code from ccpp_prebuild.py + if(NOT RC EQUAL 0) + message(FATAL_ERROR "An error occured while running ccpp_prebuild.py, check ${CMAKE_CURRENT_BINARY_DIR}/ccpp_prebuild.{out,err}") + endif() + # this should not be necessary; including CCPP_*.cmake here and passing + # SCHEMES, CAPS and TYPEDEFS via environment variables to CCPP build. + # CCPP should be able to directly include those three .cmake files. + include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_SCHEMES.cmake) + include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_CAPS.cmake) + include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_TYPEDEFS.cmake) + set(ENV{CCPP_SCHEMES} "${SCHEMES}") + set(ENV{CCPP_CAPS} "${CAPS}") + set(ENV{CCPP_TYPEDEFS} "${TYPEDEFS}") -if(CCPP) - add_definitions(-DCCPP) endif() add_subdirectory(cpl) -add_subdirectory(${PHYS}physics) +add_subdirectory(gfsphysics) add_subdirectory(ipd) add_subdirectory(io) -# fv3core library +############################################################################### +### fv3dycore +############################################################################### +list(APPEND _fv3dycore_srcs + atmos_cubed_sphere/model/a2b_edge.F90 + atmos_cubed_sphere/model/multi_gases.F90 + atmos_cubed_sphere/model/boundary.F90 + atmos_cubed_sphere/model/dyn_core.F90 + atmos_cubed_sphere/model/fv_arrays.F90 + atmos_cubed_sphere/model/fv_control.F90 + atmos_cubed_sphere/model/fv_dynamics.F90 + atmos_cubed_sphere/model/fv_fill.F90 + atmos_cubed_sphere/model/fv_grid_utils.F90 + atmos_cubed_sphere/model/fv_mapz.F90 + atmos_cubed_sphere/model/fv_nesting.F90 + atmos_cubed_sphere/model/fv_regional_bc.F90 + atmos_cubed_sphere/model/fv_sg.F90 + atmos_cubed_sphere/model/fv_tracer2d.F90 + atmos_cubed_sphere/model/fv_update_phys.F90 + atmos_cubed_sphere/model/sw_core.F90 + atmos_cubed_sphere/model/tp_core.F90 + atmos_cubed_sphere/model/nh_core.F90 + atmos_cubed_sphere/model/nh_utils.F90 + atmos_cubed_sphere/tools/external_ic.F90 + atmos_cubed_sphere/tools/external_sst.F90 + atmos_cubed_sphere/tools/fv_diagnostics.F90 + atmos_cubed_sphere/tools/fv_eta.F90 + atmos_cubed_sphere/tools/fv_grid_tools.F90 + atmos_cubed_sphere/tools/fv_io.F90 + atmos_cubed_sphere/tools/fv_mp_mod.F90 + atmos_cubed_sphere/tools/fv_nudge.F90 + atmos_cubed_sphere/tools/fv_treat_da_inc.F90 + atmos_cubed_sphere/tools/fv_iau_mod.F90 + atmos_cubed_sphere/tools/fv_restart.F90 + atmos_cubed_sphere/tools/fv_surf_map.F90 + atmos_cubed_sphere/tools/fv_timing.F90 + atmos_cubed_sphere//tools/init_hydro.F90 + atmos_cubed_sphere/tools/sim_nc_mod.F90 + atmos_cubed_sphere/tools/sorted_index.F90 + atmos_cubed_sphere/tools/test_cases.F90 + atmos_cubed_sphere/driver/fvGFS/DYCORE_typedefs.F90 + atmos_cubed_sphere/driver/fvGFS/fv_nggps_diag.F90 + atmos_cubed_sphere/driver/fvGFS/atmosphere.F90) if(NOT CCPP) -set(FAST_PHYSICS_SRCS_F90 atmos_cubed_sphere/model/fv_cmp.F90) + list(APPEND _fv3dycore_srcs atmos_cubed_sphere/model/fv_cmp.F90) +endif() + +add_library(fv3dycore ${_fv3dycore_srcs}) + +list(APPEND _fv3dycore_defs_private SPMD + use_WRTCOMP + GFS_PHYS + USE_GFSL63 + MOIST_CAPPA + USE_COND) + +if(MULTI_GASES) + list(APPEND _fv3dycore_defs_private MULTI_GASES) +endif() + +if(32BIT) + list(APPEND _fv3dycore_defs_private OVERLOAD_R4 + OVERLOAD_R8) endif() -add_library( - fv3core - - atmos_cubed_sphere/model/a2b_edge.F90 - atmos_cubed_sphere/model/multi_gases.F90 - atmos_cubed_sphere/model/boundary.F90 - atmos_cubed_sphere/model/dyn_core.F90 - atmos_cubed_sphere/model/fv_arrays.F90 - ${FAST_PHYSICS_SRCS_F90} - atmos_cubed_sphere/model/fv_control.F90 - atmos_cubed_sphere/model/fv_dynamics.F90 - atmos_cubed_sphere/model/fv_fill.F90 - atmos_cubed_sphere/model/fv_grid_utils.F90 - atmos_cubed_sphere/model/fv_mapz.F90 - atmos_cubed_sphere/model/fv_nesting.F90 - atmos_cubed_sphere/model/fv_regional_bc.F90 - atmos_cubed_sphere/model/fv_sg.F90 - atmos_cubed_sphere/model/fv_tracer2d.F90 - atmos_cubed_sphere/model/fv_update_phys.F90 - atmos_cubed_sphere/model/sw_core.F90 - atmos_cubed_sphere/model/tp_core.F90 - atmos_cubed_sphere/model/nh_core.F90 - atmos_cubed_sphere/model/nh_utils.F90 - atmos_cubed_sphere/tools/external_ic.F90 - atmos_cubed_sphere/tools/external_sst.F90 - atmos_cubed_sphere/tools/fv_diagnostics.F90 - atmos_cubed_sphere/tools/fv_eta.F90 - atmos_cubed_sphere/tools/fv_grid_tools.F90 - atmos_cubed_sphere/tools/fv_io.F90 - atmos_cubed_sphere/tools/fv_mp_mod.F90 - atmos_cubed_sphere/tools/fv_nudge.F90 - atmos_cubed_sphere/tools/fv_treat_da_inc.F90 - atmos_cubed_sphere/tools/fv_iau_mod.F90 - atmos_cubed_sphere/tools/fv_restart.F90 - atmos_cubed_sphere/tools/fv_surf_map.F90 - atmos_cubed_sphere/tools/fv_timing.F90 - atmos_cubed_sphere//tools/init_hydro.F90 - atmos_cubed_sphere/tools/sim_nc_mod.F90 - atmos_cubed_sphere/tools/sorted_index.F90 - atmos_cubed_sphere/tools/test_cases.F90 - atmos_cubed_sphere/driver/fvGFS/DYCORE_typedefs.F90 - atmos_cubed_sphere/driver/fvGFS/fv_nggps_diag.F90 - atmos_cubed_sphere/driver/fvGFS/atmosphere.F90 -) +if(CCPP) + list(APPEND _fv3dycore_defs_private CCPP) +endif() + +if(OpenMP_Fortran_FOUND) + list(APPEND _fv3dycore_defs_private OPENMP) +endif() set_property(SOURCE atmos_cubed_sphere/model/nh_utils.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}") set_property(SOURCE atmos_cubed_sphere/model/fv_mapz.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}") -target_link_libraries(fv3core PRIVATE fms) -target_link_libraries(fv3core PRIVATE ${PHYS}physics) -target_link_libraries(fv3core PRIVATE ipd) +set_target_properties(fv3dycore PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) -target_include_directories(fv3core PRIVATE ${ESMF_MOD}) +target_compile_definitions(fv3dycore PRIVATE "${_fv3dycore_defs_private}") +target_include_directories(fv3dycore PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/atmos_cubed_sphere) +target_include_directories(fv3dycore INTERFACE $ + $) -# end of fv3core +target_link_libraries(fv3dycore PUBLIC fms + gfsphysics + ipd + esmf) +if(OpenMP_Fortran_FOUND) + target_link_libraries(fv3dycore PUBLIC OpenMP::OpenMP_Fortran) +endif() ############################################################################### ### ccpp @@ -83,46 +136,81 @@ target_include_directories(fv3core PRIVATE ${ESMF_MOD}) if(CCPP) add_subdirectory(ccpp) add_subdirectory(ccpp/driver) - add_dependencies(${PHYS}physics ccpp ccppphys) + add_dependencies(gfsphysics ccpp ccppphys) add_dependencies(ccppdriver ccpp ccppphys) add_dependencies(ccppphys ccpp) - - target_include_directories(fv3core PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) - target_include_directories(fv3core PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/driver) + target_include_directories(fv3dycore PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/ccpp/framework/src + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/driver) + target_link_libraries(ccppphys PRIVATE sp::sp_d + w3nco::w3nco_d) endif() -set_target_properties(fv3core PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) -target_include_directories(fv3core PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod) - +############################################################################### +### stochastic_physics +############################################################################### # When specifying an out-of-tree source a binary directory must be explicitly specified. #add_subdirectory(../stochastic_physics stochastic_physics) add_subdirectory(stochastic_physics) -add_library( - fv3cap +############################################################################### +### fv3atm +############################################################################### +add_library(fv3atm + atmos_model.F90 + fv3_cap.F90 + module_fv3_config.F90 + module_fcst_grid_comp.F90 + time_utils.F90) + +list(APPEND _fv3atm_defs_private GFS_PHYS + INTERNAL_FILE_NML + ESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR}) - atmos_model.F90 - fv3_cap.F90 - module_fv3_config.F90 - module_fcst_grid_comp.F90 - time_utils.F90 -) +set_target_properties(fv3atm PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) +target_include_directories(fv3atm INTERFACE $ + $) if(CCPP) -target_include_directories(fv3cap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/ccpp/driver/mod) -add_dependencies(fv3cap ccppdriver ccppphys) + list(APPEND _fv3atm_defs_private CCPP) + target_include_directories(fv3atm PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/ccpp/driver/mod) + set(CCPP_LIBRARIES ccppdriver ccppphys ccpp) + add_dependencies(fv3atm ccppdriver ccppphys ccpp) + target_link_libraries(fv3atm PUBLIC ccppdriver ccppphys ccpp) endif() -target_include_directories(fv3cap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/stochastic_physics) -target_compile_definitions(fv3cap PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR}) -target_include_directories(fv3cap PRIVATE ${ESMF_MOD}) - -target_link_libraries(fv3cap PRIVATE fms) -target_link_libraries(fv3cap PRIVATE fv3cpl) -target_link_libraries(fv3cap PRIVATE ${PHYS}physics) -target_link_libraries(fv3cap PRIVATE ipd) -target_link_libraries(fv3cap PRIVATE io) -target_link_libraries(fv3cap PRIVATE fv3core) -target_link_libraries(fv3cap PRIVATE stochastic_physics) - -set_target_properties(fv3cap PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) -target_include_directories(fv3cap PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod) +target_include_directories(fv3atm PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/stochastic_physics) +target_compile_definitions(fv3atm PRIVATE "${_fv3atm_defs_private}") + +target_link_libraries(fv3atm PUBLIC fv3dycore + cpl + io + stochastic_physics + stochastic_physics_wrapper) + +target_link_libraries(fv3atm PUBLIC nemsio::nemsio + w3emc::w3emc_d + w3nco::w3nco_d + sp::sp_d + bacio::bacio_4 + esmf) + +if(INLINE_POST) + target_link_libraries(fv3atm PUBLIC upp::upp) +endif() + +if(OpenMP_Fortran_FOUND) + target_link_libraries(fv3atm PUBLIC OpenMP::OpenMP_Fortran) +endif() + +############################################################################### +### Install +############################################################################### +install( + TARGETS fv3atm fv3dycore io ipd gfsphysics ${CCPP_LIBRARIES} cpl stochastic_physics stochastic_physics_wrapper + EXPORT fv3atm-config + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib) + +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX}) + +install(EXPORT fv3atm-config + DESTINATION lib/cmake) diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index f9d4483a4..61875852b 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit f9d4483a45fa9b538184b1bd607faf203511be92 +Subproject commit 61875852b52951f6c6215603a19c826b952fc534 diff --git a/atmos_model.F90 b/atmos_model.F90 index a002efdad..f61735eb8 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -78,7 +78,7 @@ module atmos_model_mod use atmosphere_mod, only: atmosphere_scalar_field_halo use atmosphere_mod, only: atmosphere_get_bottom_layer use atmosphere_mod, only: set_atmosphere_pelist -use atmosphere_mod, only: Atm, mytile +use atmosphere_mod, only: Atm, mygrid use block_control_mod, only: block_control_type, define_blocks_packed use DYCORE_typedefs, only: DYCORE_data_type, DYCORE_diag_type #ifdef CCPP @@ -99,15 +99,13 @@ module atmos_model_mod IPD_interstitial => GFS_interstitial use IPD_driver, only: IPD_initialize, IPD_initialize_rst use CCPP_driver, only: CCPP_step, non_uniform_blocks + +use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper,stochastic_physics_wrapper_end #else use IPD_driver, only: IPD_initialize, IPD_initialize_rst, IPD_step use physics_abstraction_layer, only: time_vary_step, radiation_step1, physics_step1, physics_step2 #endif -use stochastic_physics, only: init_stochastic_physics, & - run_stochastic_physics -use stochastic_physics_sfc, only: run_stochastic_physics_sfc - use FV3GFS_io_mod, only: FV3GFS_restart_read, FV3GFS_restart_write, & FV3GFS_IPD_checksum, & FV3GFS_diag_register, FV3GFS_diag_output, & @@ -133,7 +131,7 @@ module atmos_model_mod ! type atmos_data_type - integer :: axes(4) ! axis indices (returned by diag_manager) for the atmospheric grid + integer :: axes(4) ! axis indices (returned by diag_manager) for the atmospheric grid ! (they correspond to the x, y, pfull, phalf axes) integer, pointer :: pelist(:) =>null() ! pelist where atmosphere is running. integer :: layout(2) ! computer task laytout @@ -154,7 +152,7 @@ module atmos_model_mod type(time_type) :: Time ! current time type(time_type) :: Time_step ! atmospheric time step. type(time_type) :: Time_init ! reference time. - type(grid_box_type) :: grid ! hold grid information needed for 2nd order conservative flux exchange + type(grid_box_type) :: grid ! hold grid information needed for 2nd order conservative flux exchange type(IPD_diag_type), pointer, dimension(:) :: Diag end type atmos_data_type ! to calculate gradient on cubic sphere grid. @@ -222,9 +220,10 @@ module atmos_model_mod logical,parameter :: flip_vc = .true. #endif - real(kind=IPD_kind_phys), parameter :: zero = 0.0_IPD_kind_phys, & - one = 1.0_IPD_kind_phys, & - epsln = 1.0e-10_IPD_kind_phys + real(kind=IPD_kind_phys), parameter :: zero = 0.0_IPD_kind_phys, & + one = 1.0_IPD_kind_phys, & + epsln = 1.0e-10_IPD_kind_phys, & + zorlmin = 1.0e-7_IPD_kind_phys contains @@ -236,7 +235,7 @@ module atmos_model_mod ! atmospheric tendencies for dynamics, radiation, vertical diffusion of ! momentum, tracers, and heat/moisture. For heat/moisture only the ! downward sweep of the tridiagonal elimination is performed, hence -! the name "_down". +! the name "_down". ! !