Restructure alloc space calls into simpler smaller files (#2068)

TYPE: enhancement KEYWORDS: intel, compilation, llvm, memory SOURCE: internal DESCRIPTION OF CHANGES: Problem: The Intel oneAPI compilers (and others like nvhpc) struggle with some of the larger (15k+ lines of code) files within WRF. This causes intense memory usage that is not often available to the average user not in a resource-rich environment. This often limits compilation to single threaded if even possible or to a dedicated environment with enough memory if available. If neither of those is available to a user, they will be unable to use these configurations entirely. Solution: This PR focuses on the `module_alloc_space*` sections of code to reduce their individual file size to manageable levels. They are instead broken out into many smaller files as external subroutines, not requiring a wrapper module to house the subroutine call. The files are now fully generated source code from the registry, with the calls to the subroutines also being generated as well. This also makes it relatively easy to change the number of files generated from a source code perspective. Build rules would need to be modified accordingly as seen in these changes. TESTS CONDUCTED: Attached to this PR are plots of the respective effects of theses changes. Changes were tested with intel and gcc compilers, but only intel memory usage is shown as it exacerbates the memory usage issue.
wrf-model · Sep 26, 2024 · ed585bd · ed585bd
1 parent 958ce12
commit ed585bd
Show file tree

Hide file tree

Showing 24 changed files with 737 additions and 572 deletions.
diff --git a/arch/noopt_exceptions b/arch/noopt_exceptions
@@ -17,16 +17,6 @@ module_comm_nesting_dm.o : module_comm_nesting_dm.F
 module_configure.o : module_configure.F
 module_domain.o : module_domain.F
 module_domain_type.o : module_domain_type.F
-module_alloc_space_0.o : module_alloc_space_0.F
-module_alloc_space_1.o : module_alloc_space_1.F
-module_alloc_space_2.o : module_alloc_space_2.F
-module_alloc_space_3.o : module_alloc_space_3.F
-module_alloc_space_4.o : module_alloc_space_4.F
-module_alloc_space_5.o : module_alloc_space_5.F
-module_alloc_space_6.o : module_alloc_space_6.F
-module_alloc_space_7.o : module_alloc_space_7.F
-module_alloc_space_8.o : module_alloc_space_8.F
-module_alloc_space_9.o : module_alloc_space_9.F
 module_tiles.o : module_tiles.F
 module_initialize.o : module_initialize.F
 module_physics_init.o : module_physics_init.F
@@ -209,17 +199,6 @@ wrf_inputout.o \
 wrf_restartin.o \
 wrf_restartout.o \
 module_state_description.o \
-module_alloc_space.o \
-module_alloc_space_0.o \
-module_alloc_space_1.o \
-module_alloc_space_2.o \
-module_alloc_space_3.o \
-module_alloc_space_4.o \
-module_alloc_space_5.o \
-module_alloc_space_6.o \
-module_alloc_space_7.o \
-module_alloc_space_8.o \
-module_alloc_space_9.o \
 module_comm_dm.o \
 module_comm_dm_0.o \
 module_comm_dm_1.o \

diff --git a/arch/noopt_exceptions_f b/arch/noopt_exceptions_f
@@ -21,16 +21,6 @@ module_comm_nesting_dm.o : module_comm_nesting_dm.F
 module_configure.o : module_configure.F
 module_domain.o : module_domain.F
 module_domain_type.o : module_domain_type.F
-module_alloc_space_0.o : module_alloc_space_0.F
-module_alloc_space_1.o : module_alloc_space_1.F
-module_alloc_space_2.o : module_alloc_space_2.F
-module_alloc_space_3.o : module_alloc_space_3.F
-module_alloc_space_4.o : module_alloc_space_4.F
-module_alloc_space_5.o : module_alloc_space_5.F
-module_alloc_space_6.o : module_alloc_space_6.F
-module_alloc_space_7.o : module_alloc_space_7.F
-module_alloc_space_8.o : module_alloc_space_8.F
-module_alloc_space_9.o : module_alloc_space_9.F
 module_tiles.o : module_tiles.F
 module_initialize.o : module_initialize.F
 module_physics_init.o : module_physics_init.F 
@@ -124,17 +114,6 @@ wrf_inputout.o \
 wrf_restartin.o \
 wrf_restartout.o \
 module_state_description.o \
-module_alloc_space.o \
-module_alloc_space_0.o \
-module_alloc_space_1.o \
-module_alloc_space_2.o \
-module_alloc_space_3.o \
-module_alloc_space_4.o \
-module_alloc_space_5.o \
-module_alloc_space_6.o \
-module_alloc_space_7.o \
-module_alloc_space_8.o \
-module_alloc_space_9.o \
 module_comm_dm.o \
 module_comm_dm_0.o \
 module_comm_dm_1.o \

diff --git a/clean b/clean
@@ -21,7 +21,7 @@ endif
 
 ( cd tools/CodeBase ; make clean )
 
-( cd inc ; /bin/rm -f *.inc namelist.default commit_decl )
+( cd inc ; /bin/rm -f *.inc namelist.default commit_decl *.o *.F )
 
 find . -name \*.dSYM -exec rm -rf {} \; >& /dev/null
 

diff --git a/frame/CMakeLists.txt b/frame/CMakeLists.txt
@@ -109,16 +109,6 @@ target_sources(
                   module_cpl_oasis3.F
 
 
-                  module_alloc_space_0.F
-                  module_alloc_space_1.F
-                  module_alloc_space_2.F
-                  module_alloc_space_3.F
-                  module_alloc_space_4.F
-                  module_alloc_space_5.F
-                  module_alloc_space_6.F
-                  module_alloc_space_7.F
-                  module_alloc_space_8.F
-                  module_alloc_space_9.F
 
                   ${CMAKE_BINARY_DIR}/frame/module_state_description.F # GENERATED
                   ${nl_dyn_source}                                     # GENERATED

diff --git a/frame/Makefile b/frame/Makefile
@@ -35,16 +35,38 @@ MODULES =       module_driver_constants.o  \
 		module_cpl_oasis3.o        \
 		module_clear_halos.o
 ALOBJS =\
-                module_alloc_space_0.o       \
-                module_alloc_space_1.o       \
-                module_alloc_space_2.o       \
-                module_alloc_space_3.o       \
-                module_alloc_space_4.o       \
-                module_alloc_space_5.o       \
-                module_alloc_space_6.o       \
-                module_alloc_space_7.o       \
-                module_alloc_space_8.o       \
-                module_alloc_space_9.o
+                ../inc/allocs_0.o                  \
+                ../inc/allocs_1.o                  \
+                ../inc/allocs_2.o                  \
+                ../inc/allocs_3.o                  \
+                ../inc/allocs_4.o                  \
+                ../inc/allocs_5.o                  \
+                ../inc/allocs_6.o                  \
+                ../inc/allocs_7.o                  \
+                ../inc/allocs_8.o                  \
+                ../inc/allocs_9.o                  \
+                ../inc/allocs_10.o                  \
+                ../inc/allocs_11.o                  \
+                ../inc/allocs_12.o                  \
+                ../inc/allocs_13.o                  \
+                ../inc/allocs_14.o                  \
+                ../inc/allocs_15.o                  \
+                ../inc/allocs_16.o                  \
+                ../inc/allocs_17.o                  \
+                ../inc/allocs_18.o                  \
+                ../inc/allocs_19.o                  \
+                ../inc/allocs_20.o                  \
+                ../inc/allocs_21.o                  \
+                ../inc/allocs_22.o                  \
+                ../inc/allocs_23.o                  \
+                ../inc/allocs_24.o                  \
+                ../inc/allocs_25.o                  \
+                ../inc/allocs_26.o                  \
+                ../inc/allocs_27.o                  \
+                ../inc/allocs_28.o                  \
+                ../inc/allocs_29.o                  \
+                ../inc/allocs_30.o                  \
+                ../inc/allocs_31.o
 
 NLOBJS =\
                 nl_get_0_routines.o        \

diff --git a/frame/module_alloc_space.h b/frame/module_alloc_space.h
diff --git a/frame/module_alloc_space_0.F b/frame/module_alloc_space_0.F
diff --git a/frame/module_alloc_space_1.F b/frame/module_alloc_space_1.F
diff --git a/frame/module_alloc_space_2.F b/frame/module_alloc_space_2.F
diff --git a/frame/module_alloc_space_3.F b/frame/module_alloc_space_3.F
diff --git a/frame/module_alloc_space_4.F b/frame/module_alloc_space_4.F