Skip to content

Commit

Permalink
Breakout deallocation calls into simpler smaller files (#2070)
Browse files Browse the repository at this point in the history 
TYPE: enhancement

KEYWORDS: intel, compilation, llvm, memory

SOURCE: internal

DESCRIPTION OF CHANGES:
Problem:
The Intel oneAPI compilers (and others like nvhpc) struggle with some of
the larger (15k+ lines of code) files within WRF. This causes intense
memory usage that is not often available to the average user not in a
resource-rich environment. This often limits compilation to single
threaded if even possible or to a dedicated environment with enough
memory if available. If neither of those is available to a user, they
will be unable to use these configurations entirely.

Solution:
This PR focuses on the `deallocs.inc` sections of code used in
`module_domain` to reduce the include size to manageable levels. The
include is instead broken out into many smaller files as external
subroutines. The files are fully generated source code from the
registry, with the calls to the subroutines also being generated as
well. This also makes it relatively easy to change the number of files
generated from a source code perspective. Build rules would need to be
modified accordingly as seen in these changes.

TESTS CONDUCTED: 
Attached to this PR are plots of the respective effects of theses
changes. Changes were tested with intel and gcc compilers, but only
intel memory usage is shown as it exacerbates the memory usage issue.
  • Loading branch information
@islas
islas authored Oct 14, 2024
1 parent a9c2337 commit 5ffa840
Show file tree
Hide file tree
Showing 12 changed files with 243 additions and 17 deletions.
2 changes: 1 addition & 1 deletion clean
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ endif

( cd tools/CodeBase ; make clean )

( cd inc ; /bin/rm -f *.inc namelist.default commit_decl *.o *.F )
( cd inc ; /bin/rm -f *.inc namelist.default commit_decl *.o *.F *.f90 )

find . -name \*.dSYM -exec rm -rf {} \; >& /dev/null

Expand Down
4 changes: 1 addition & 3 deletions frame/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,6 @@ target_sources(
module_cpl_oasis3.F



${CMAKE_BINARY_DIR}/frame/module_state_description.F # GENERATED
${nl_dyn_source} # GENERATED

clog.c
Expand Down Expand Up @@ -166,4 +164,4 @@ set_source_files_properties(
install(
FILES ${WRF_INCLUDE_FILES}
DESTINATION include/${FOLDER_COMPILE_TARGET}
)
)
20 changes: 17 additions & 3 deletions frame/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,21 @@ NLOBJS =\
nl_set_4_routines.o \
nl_set_5_routines.o \
nl_set_6_routines.o \
nl_set_7_routines.o
nl_set_7_routines.o

DEALOBJS =\
../inc/deallocs_0.o \
../inc/deallocs_1.o \
../inc/deallocs_2.o \
../inc/deallocs_3.o \
../inc/deallocs_4.o \
../inc/deallocs_5.o \
../inc/deallocs_6.o \
../inc/deallocs_7.o \
../inc/deallocs_8.o \
../inc/deallocs_9.o \
../inc/deallocs_10.o \
../inc/deallocs_11.o

OBJS = \
wrf_num_bytes_between.o \
Expand Down Expand Up @@ -113,8 +127,8 @@ SPECIAL = module_internal_header_util.o pack_utils.o

LIBTARGET = framework
TARGETDIR = ./
$(LIBTARGET) : $(MODULES) $(OBJS) $(SPECIAL) $(NLOBJS) $(ALOBJS) $(RSL_OBJS)
$(AR) $(ARFLAGS) ../main/$(LIBWRFLIB) $(MODULES) $(OBJS) $(NLOBJS) $(ALOBJS) $(RSL_OBJS)
$(LIBTARGET) : $(MODULES) $(OBJS) $(SPECIAL) $(NLOBJS) $(ALOBJS) $(DEALOBJS) $(RSL_OBJS)
$(AR) $(ARFLAGS) ../main/$(LIBWRFLIB) $(MODULES) $(OBJS) $(NLOBJS) $(ALOBJS) $(DEALOBJS) $(RSL_OBJS)
$(RANLIB) ../main/$(LIBWRFLIB)

nl_set_0_routines.o : nl_access_routines.F module_configure.o
Expand Down
4 changes: 0 additions & 4 deletions frame/module_domain.F
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1646,10 +1646,6 @@ SUBROUTINE dealloc_space_field ( grid )

TYPE(domain) , POINTER :: grid

! Local data.

INTEGER :: ierr

# include "deallocs.inc"

END SUBROUTINE dealloc_space_field
Expand Down
2 changes: 2 additions & 0 deletions inc/.gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,5 @@
##############################################################################
*.inc
commit_decl
allocs_*.F
deallocs_*.F
88 changes: 88 additions & 0 deletions main/depend.common
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,82 @@ module_comm_nesting_dm.o: \
module_dm_stubs.F: \
module_domain.o

# For the following dealloc_* source files, as they are generated by the registry
# we must make that a dependency. There is no actual registry target and is
# instead facilitated via the module_state_description.F recipe. Placing a dependency on
# module_state_description.F forces the registry to be run if the following source
# files are required implicitly by their respective .o recipe
../inc/deallocs_0.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_0.F : module_state_description.F


../inc/deallocs_1.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_1.F : module_state_description.F


../inc/deallocs_2.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_2.F : module_state_description.F


../inc/deallocs_3.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_3.F : module_state_description.F


../inc/deallocs_4.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_4.F : module_state_description.F


../inc/deallocs_5.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_5.F : module_state_description.F


../inc/deallocs_6.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_6.F : module_state_description.F


../inc/deallocs_7.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_7.F : module_state_description.F


../inc/deallocs_8.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_8.F : module_state_description.F


../inc/deallocs_9.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_9.F : module_state_description.F


../inc/deallocs_10.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_10.F : module_state_description.F


../inc/deallocs_11.o : \
module_wrf_error.o \
module_domain_type.o
../inc/deallocs_11.F : module_state_description.F


module_domain.o: \
module_domain_type.o \
Expand Down Expand Up @@ -171,6 +247,18 @@ module_domain.o: \
../inc/allocs_29.o \
../inc/allocs_30.o \
../inc/allocs_31.o \
../inc/deallocs_0.o \
../inc/deallocs_1.o \
../inc/deallocs_2.o \
../inc/deallocs_3.o \
../inc/deallocs_4.o \
../inc/deallocs_5.o \
../inc/deallocs_6.o \
../inc/deallocs_7.o \
../inc/deallocs_8.o \
../inc/deallocs_9.o \
../inc/deallocs_10.o \
../inc/deallocs_11.o \
module_driver_constants.o \
module_configure.o \
module_machine.o \
Expand Down
14 changes: 13 additions & 1 deletion tools/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,14 +99,22 @@ elseif ( ${WRF_CORE} STREQUAL "DA" OR ${WRF_CORE} STREQUAL "DA_4D_VAR" )

endif()

set( dealloc_source )
foreach( n RANGE 0 11 )
list(
APPEND
dealloc_source
${CMAKE_BINARY_DIR}/inc/deallocs_${n}.F
)
endforeach()

set( allocs_source )
foreach( n RANGE 0 31 )
list(
APPEND
allocs_source
${CMAKE_BINARY_DIR}/inc/allocs_${n}.F
)

endforeach()

get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS )
Expand All @@ -122,6 +130,7 @@ add_custom_command(
OUTPUT
${CMAKE_BINARY_DIR}/inc/nl_config.inc
${CMAKE_BINARY_DIR}/frame/module_state_description.F
${dealloc_source}
${allocs_source}
COMMENT
"Generating registry code..."
Expand Down Expand Up @@ -151,12 +160,15 @@ add_custom_target(
DEPENDS
${CMAKE_BINARY_DIR}/inc/nl_config.inc
${CMAKE_BINARY_DIR}/frame/module_state_description.F
${dealloc_source}
${allocs_source}
)

target_sources(
${PROJECT_NAME}_Core
PRIVATE
${CMAKE_BINARY_DIR}/frame/module_state_description.F
${dealloc_source}
${allocs_source}
)

74 changes: 70 additions & 4 deletions tools/gen_allocs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,6 @@ gen_alloc1 ( char * dirname )
{
stop = start + ( primaryFields / numFiles );
}
printf( "Total [%d] : Setting [start,stop] to [%d, %d]", primaryFields, start, stop );
gen_alloc2( fpSub , "grid%", NULL, &Domain, start, stop, 1 ) ;
fprintf(
fpSub,
Expand Down Expand Up @@ -821,30 +820,97 @@ gen_dealloc1 ( char * dirname )
FILE * fp ;
char fname[NAMELEN] ;
char * fn = "deallocs.inc" ;
// Open array of deallocs_[n].inc
int numFiles = 12;
int idx = 0;
FILE * fpSub;
char * filename_prefix = "deallocs_" ;

if ( dirname == NULL ) return(1) ;
if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s",dirname,fn) ; }
else { sprintf(fname,"%s",fn) ; }
if ((fp = fopen( fname , "w" )) == NULL ) return(1) ;
print_warning(fp,fname) ;
gen_dealloc2( fp , "grid%", &Domain ) ;

fprintf(
fp,
"INTERFACE\n"
);
if ( dirname == NULL ) return(1) ;
for ( idx = 0; idx < numFiles; idx++ )
{
if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s%d.F",dirname,filename_prefix,idx) ; }
else { sprintf(fname,"%s%d.F",dirname,filename_prefix,idx ) ; }
if ((fpSub = fopen( fname , "w" )) == NULL ) return(1) ;

print_warning(fpSub,fname) ;

fprintf(
fp,
" SUBROUTINE %s%d( grid )\n"
" USE module_wrf_error\n"
" USE module_domain_type\n"
" IMPLICIT NONE\n"
" TYPE( domain ), POINTER :: grid\n END SUBROUTINE\n",
filename_prefix, idx
);

fprintf(
fpSub,
"SUBROUTINE %s%d( grid )\n"
" USE module_wrf_error\n"
" USE module_domain_type\n"
" IMPLICIT NONE\n"
" TYPE( domain ), POINTER :: grid\n INTEGER :: ierr\n",
filename_prefix, idx
);
gen_dealloc2( fpSub, "grid%", &Domain, idx, numFiles );
fprintf(
fpSub,
"END SUBROUTINE %s%d\n",
filename_prefix, idx
);
close_the_file( fpSub ) ;
}
fprintf(
fp,
"END INTERFACE\n"
);

// Call the functions in the inc
for ( idx = 0; idx < numFiles; idx++ )
{
fprintf(
fp,
"CALL %s%d( grid )\n", filename_prefix, idx
);
}
close_the_file( fp ) ;
return(0) ;
}

int
gen_dealloc2 ( FILE * fp , char * structname , node_t * node )
gen_dealloc2 ( FILE * fp , char * structname , node_t * node, int idx, int numFiles )
{
node_t * p ;
int tag ;
char post[NAMELEN] ;
char fname[NAMELEN] ;
char x[NAMELEN] ;
int currentIdx = -1;

if ( node == NULL ) return(1) ;

for ( p = node->fields ; p != NULL ; p = p->next )
{
// Modulo to divert each field based on index to a file
// Skip if this field is not part of that index and idx != -1, so -1 can be used to force output
currentIdx = ( currentIdx + 1 ) % numFiles;
if ( currentIdx != idx && idx != -1 )
{
continue;
}

if ( (p->ndims > 0 || p->boundary_array) && ( /* any array or a boundary array and... */
(p->node_kind & FIELD) || /* scalar arrays or */
(p->node_kind & FOURD) ) /* scalar arrays or */
Expand Down Expand Up @@ -914,7 +980,7 @@ structname, fname, structname, fname ) ;
else if ( p->type->type_type == DERIVED )
{
sprintf(x,"%s%s%%",structname,p->name ) ;
gen_dealloc2(fp,x, p->type) ;
gen_dealloc2(fp,x, p->type, idx, -1) ;
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion tools/protos.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ int gen_ddt_write ( char * );
int gen_ddt_write1 ( FILE *, char *, node_t *);
int gen_dealloc ( char * );
int gen_dealloc1 ( char * );
int gen_dealloc2 ( FILE *, char *, node_t *);
int gen_dealloc2 ( FILE *, char *, node_t *, int idx, int numFiles );
int gen_scalar_tables ( FILE *);
int AppendReg ( char *,int);
int irr_diag_scalar_indices ( char * );
Expand Down
12 changes: 12 additions & 0 deletions var/build/da.make
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,18 @@ WRFVAR_OBJS = \
inc/allocs_29.o \
inc/allocs_30.o \
inc/allocs_31.o \
inc/deallocs_0.o \
inc/deallocs_1.o \
inc/deallocs_2.o \
inc/deallocs_3.o \
inc/deallocs_4.o \
inc/deallocs_5.o \
inc/deallocs_6.o \
inc/deallocs_7.o \
inc/deallocs_8.o \
inc/deallocs_9.o \
inc/deallocs_10.o \
inc/deallocs_11.o \
nl_get_0_routines.o \
nl_get_1_routines.o \
nl_get_2_routines.o \
Expand Down
1 change: 1 addition & 0 deletions var/build/da_name_space.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
med_auxinput_in
med_restart_out
med_latbound_in
deallocs_
allocs_
module_configure
module_scalar_tables
Expand Down
Loading

0 comments on commit 5ffa840

Please sign in to comment.