Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HDF5 bug fix for vol-async and compression #3451

Merged
merged 5 commits into from
Jul 26, 2023
Merged

HDF5 bug fix for vol-async and compression #3451

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
765fa08
Fix a few issues with HDF5 when using vol-async and SZ compression
houjun Jul 25, 2023
0ecad05
Fix format
houjun Jul 25, 2023
3a0799b
HDF5 path
houjun Jul 25, 2023
3a43149
Merge branch 'AMReX-Codes:development' into development
houjun Jul 26, 2023
67b80a9
Add (comment out) Summit library paths
houjun Jul 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Src/Extern/HDF5/AMReX_ParticleHDF5.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -591,7 +591,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
hid_t dxpl_col, dxpl_ind, dcpl_int, dcpl_real;
dxpl_col = H5Pcreate(H5P_DATASET_XFER);
dxpl_ind = H5Pcreate(H5P_DATASET_XFER);
#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
H5Pset_dxpl_mpio(dxpl_col, H5FD_MPIO_COLLECTIVE);
#endif
dcpl_int = H5Pcreate(H5P_DATASET_CREATE);
Expand Down Expand Up @@ -721,7 +721,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
my_mfi_cnt++;
}

#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
// Collect the number of mf and total size of mf from each rank
MPI_Allgather(&my_mfi_cnt, 1, ParallelDescriptor::Mpi_typemap<int>::type(), &(all_mfi_cnt[0]), 1,
ParallelDescriptor::Mpi_typemap<int>::type(), ParallelDescriptor::Communicator());
Expand Down Expand Up @@ -767,7 +767,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
H5Sclose(int_dset_space);

// Create the real data
#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
MPI_Allgather(&my_mfi_real_total_size, 1, ParallelDescriptor::Mpi_typemap<ULong>::type(),
&(all_mfi_real_total_size[0]), 1, ParallelDescriptor::Mpi_typemap<ULong>::type(), ParallelDescriptor::Communicator());
#else
Expand Down Expand Up @@ -996,7 +996,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
int ret;

fapl = H5Pcreate (H5P_FILE_ACCESS);
#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
SetHDF5fapl(fapl, ParallelDescriptor::Communicator());
#else
SetHDF5fapl(fapl);
Expand Down
89 changes: 51 additions & 38 deletions Src/Extern/HDF5/AMReX_PlotFileUtilHDF5.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,13 @@ static int CreateWriteHDF5AttrString(hid_t loc, const char *name, const char* st
return 1;
}

#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
static void SetHDF5fapl(hid_t fapl, MPI_Comm comm)
#else
static void SetHDF5fapl(hid_t fapl)
#endif
{
#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
H5Pset_fapl_mpio(fapl, comm, MPI_INFO_NULL);

// Alignment and metadata block size
Expand All @@ -127,16 +127,18 @@ static void SetHDF5fapl(hid_t fapl)
H5Pset_all_coll_metadata_ops(fapl, true);

// Defer cache flush
H5AC_cache_config_t cache_config;
cache_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
H5Pget_mdc_config(fapl, &cache_config);
cache_config.set_initial_size = 1;
cache_config.initial_size = 16 * 1024 * 1024;
cache_config.evictions_enabled = 0;
cache_config.incr_mode = H5C_incr__off;
cache_config.flash_incr_mode = H5C_flash_incr__off;
cache_config.decr_mode = H5C_decr__off;
H5Pset_mdc_config (fapl, &cache_config);
// 7/19/23 Comment out the following as it is causing segfault
// with vol-async on Frontier
/* H5AC_cache_config_t cache_config; */
/* cache_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; */
/* H5Pget_mdc_config(fapl, &cache_config); */
/* cache_config.set_initial_size = 1; */
/* cache_config.initial_size = 16 * 1024 * 1024; */
/* cache_config.evictions_enabled = 0; */
/* cache_config.incr_mode = H5C_incr__off; */
/* cache_config.flash_incr_mode = H5C_flash_incr__off; */
/* cache_config.decr_mode = H5C_decr__off; */
/* H5Pset_mdc_config (fapl, &cache_config); */
#else
H5Pset_fapl_sec2(fapl);
#endif
Expand Down Expand Up @@ -372,7 +374,7 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
std::string filename(plotfilename + ".h5");

// Write out root level metadata
hid_t fapl, dxpl_col, dxpl_ind, dcpl_id, fid, grp;
hid_t fapl, dxpl_col, dxpl_ind, dcpl_id, lev_dcpl_id, fid, grp;

if(ParallelDescriptor::IOProcessor()) {
BL_PROFILE_VAR("H5writeMetadata", h5dwm);
Expand Down Expand Up @@ -432,7 +434,7 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
dxpl_col = H5Pcreate(H5P_DATASET_XFER);
dxpl_ind = H5Pcreate(H5P_DATASET_XFER);

#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
SetHDF5fapl(fapl, ParallelDescriptor::Communicator());
H5Pset_dxpl_mpio(dxpl_col, H5FD_MPIO_COLLECTIVE);
#else
Expand All @@ -441,6 +443,7 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,

dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_fill_time(dcpl_id, H5D_FILL_TIME_NEVER);
H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_INCR);

#if (defined AMREX_USE_HDF5_ZFP) || (defined AMREX_USE_HDF5_SZ)
const char *chunk_env = NULL;
Expand All @@ -453,7 +456,6 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
chunk_dim = atoi(chunk_env);

H5Pset_chunk(dcpl_id, 1, &chunk_dim);
H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_LATE);

std::string::size_type pos = compression.find('@');
if (pos != std::string::npos) {
Expand Down Expand Up @@ -668,13 +670,6 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
hid_t dataspace = H5Screate_simple(1, hs_allprocsize, NULL);
hid_t memdataspace = H5Screate_simple(1, hs_procsize, NULL);

/* fprintf(stderr, "Rank %d: level %d, offset %ld, size %ld, all size %ld\n", myProc, level, ch_offset[0], hs_procsize[0], hs_allprocsize[0]); */

if (hs_procsize[0] == 0)
H5Sselect_none(dataspace);
else
H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);

Vector<Real> a_buffer(procBufferSize[myProc], -1.0);
const MultiFab* data;
std::unique_ptr<MultiFab> mf_tmp;
Expand Down Expand Up @@ -705,24 +700,32 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,

BL_PROFILE_VAR("H5DwriteData", h5dwg);

lev_dcpl_id = H5Pcopy(dcpl_id);
#ifdef AMREX_USE_HDF5_SZ
if (mode_env == "SZ") {
size_t cd_nelmts;
unsigned int* cd_values = NULL;
unsigned filter_config;
SZ_metaDataToCdArray(&cd_nelmts, &cd_values, SZ_DOUBLE, 0, 0, 0, 0, hs_allprocsize[0]);
H5Pset_filter(dcpl_id, H5Z_FILTER_SZ, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values);
H5Pset_filter(lev_dcpl_id, H5Z_FILTER_SZ, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values);
}
#endif

#ifdef AMREX_USE_HDF5_ASYNC
hid_t dataset = H5Dcreate_async(grp, dataname.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT, es_id_g);
hid_t dataset = H5Dcreate_async(grp, dataname.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, lev_dcpl_id, H5P_DEFAULT, es_id_g);
#else
hid_t dataset = H5Dcreate(grp, dataname.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
hid_t dataset = H5Dcreate(grp, dataname.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, lev_dcpl_id, H5P_DEFAULT);
#endif
if(dataset < 0)
std::cout << ParallelDescriptor::MyProc() << "create data failed! ret = " << dataset << std::endl;

/* fprintf(stderr, "Rank %d: level %d, offset %ld, size %ld, all size %ld\n", myProc, level, ch_offset[0], hs_procsize[0], hs_allprocsize[0]); */

if (hs_procsize[0] == 0)
H5Sselect_none(dataspace);
else
H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);

#ifdef AMREX_USE_HDF5_ASYNC
ret = H5Dwrite_async(dataset, H5T_NATIVE_DOUBLE, memdataspace, dataspace, dxpl_col, a_buffer.dataPtr(), es_id_g);
#else
Expand All @@ -732,6 +735,7 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,

BL_PROFILE_VAR_STOP(h5dwg);

H5Pclose(lev_dcpl_id);
H5Sclose(memdataspace);
H5Sclose(dataspace);
H5Sclose(offsetdataspace);
Expand Down Expand Up @@ -812,7 +816,7 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
std::string filename(plotfilename + ".h5");

// Write out root level metadata
hid_t fapl, dxpl_col, dxpl_ind, fid, grp, dcpl_id;
hid_t fapl, dxpl_col, dxpl_ind, fid, grp, dcpl_id, lev_dcpl_id;

if(ParallelDescriptor::IOProcessor()) {
BL_PROFILE_VAR("H5writeMetadata", h5dwm);
Expand Down Expand Up @@ -872,7 +876,7 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
dxpl_col = H5Pcreate(H5P_DATASET_XFER);
dxpl_ind = H5Pcreate(H5P_DATASET_XFER);

#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
SetHDF5fapl(fapl, ParallelDescriptor::Communicator());
H5Pset_dxpl_mpio(dxpl_col, H5FD_MPIO_COLLECTIVE);
#else
Expand All @@ -881,6 +885,7 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,

dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_fill_time(dcpl_id, H5D_FILL_TIME_NEVER);
H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_INCR);

#if (defined AMREX_USE_HDF5_ZFP) || (defined AMREX_USE_HDF5_SZ)
const char *chunk_env = NULL;
Expand All @@ -893,7 +898,6 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
chunk_dim = atoi(chunk_env);

H5Pset_chunk(dcpl_id, 1, &chunk_dim);
H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_LATE);

std::string::size_type pos = compression.find('@');
if (pos != std::string::npos) {
Expand Down Expand Up @@ -1097,14 +1101,8 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
hs_procsize[0] = procBufferSize[myProc]; // ---- size of buffer on this proc
hs_allprocsize[0] = offsets[sortedGrids.size()]; // ---- size of buffer on all procs

hid_t dataspace = H5Screate_simple(1, hs_allprocsize, NULL);
hid_t memdataspace = H5Screate_simple(1, hs_procsize, NULL);

if (hs_procsize[0] == 0)
H5Sselect_none(dataspace);
else
H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);

Vector<Real> a_buffer(procBufferSize[myProc]*ncomp, -1.0);
Vector<Real> a_buffer_ind(procBufferSize[myProc], -1.0);
const MultiFab* data;
Expand All @@ -1123,13 +1121,14 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
hid_t dataset;
char dataname[64];

lev_dcpl_id = H5Pcopy(dcpl_id);
#ifdef AMREX_USE_HDF5_SZ
if (mode_env == "SZ") {
size_t cd_nelmts;
unsigned int* cd_values = NULL;
unsigned filter_config;
SZ_metaDataToCdArray(&cd_nelmts, &cd_values, SZ_DOUBLE, 0, 0, 0, 0, hs_allprocsize[0]);
H5Pset_filter(dcpl_id, H5Z_FILTER_SZ, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values);
H5Pset_filter(lev_dcpl_id, H5Z_FILTER_SZ, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values);
}
#endif

Expand Down Expand Up @@ -1158,26 +1157,40 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
writeDataSize += writeDataItems;
}

hid_t dataspace = H5Screate_simple(1, hs_allprocsize, NULL);
snprintf(dataname, sizeof dataname, "data:datatype=%d", jj);
#ifdef AMREX_USE_HDF5_ASYNC
dataset = H5Dcreate_async(grp, dataname, H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT, es_id_g);
dataset = H5Dcreate_async(grp, dataname, H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, lev_dcpl_id, H5P_DEFAULT, es_id_g);
if(dataset < 0) std::cout << ParallelDescriptor::MyProc() << "create data failed! ret = " << dataset << std::endl;

if (hs_procsize[0] == 0)
H5Sselect_none(dataspace);
else
H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);

ret = H5Dwrite_async(dataset, H5T_NATIVE_DOUBLE, memdataspace, dataspace, dxpl_col, a_buffer_ind.dataPtr(), es_id_g);
if(ret < 0) { std::cout << ParallelDescriptor::MyProc() << "Write data failed! ret = " << ret << std::endl; break; }
H5Dclose_async(dataset, es_id_g);
#else
dataset = H5Dcreate(grp, dataname, H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
dataset = H5Dcreate(grp, dataname, H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, lev_dcpl_id, H5P_DEFAULT);
if(dataset < 0) std::cout << ParallelDescriptor::MyProc() << "create data failed! ret = " << dataset << std::endl;

if (hs_procsize[0] == 0)
H5Sselect_none(dataspace);
else
H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);

ret = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, memdataspace, dataspace, dxpl_col, a_buffer_ind.dataPtr());
if(ret < 0) { std::cout << ParallelDescriptor::MyProc() << "Write data failed! ret = " << ret << std::endl; break; }
H5Dclose(dataset);
#endif
H5Sclose(dataspace);
}

BL_PROFILE_VAR_STOP(h5dwg);

H5Pclose(lev_dcpl_id);
H5Sclose(memdataspace);
H5Sclose(dataspace);
H5Sclose(offsetdataspace);
H5Sclose(centerdataspace);
H5Sclose(boxdataspace);
Expand Down
6 changes: 3 additions & 3 deletions Src/Extern/HDF5/AMReX_WriteBinaryParticleDataHDF5.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,13 @@ static int ReadHDF5Attr(hid_t loc, const char *name, void *data, hid_t dtype)
return 1;
}

#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
static void SetHDF5fapl(hid_t fapl, MPI_Comm comm)
#else
static void SetHDF5fapl(hid_t fapl)
#endif
{
#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
H5Pset_fapl_mpio(fapl, comm, MPI_INFO_NULL);

// Alignment and metadata block size
Expand Down Expand Up @@ -381,7 +381,7 @@ void WriteHDF5ParticleDataSync (PC const& pc,
pc.nOutFilesPrePost = nOutFiles;

fapl = H5Pcreate (H5P_FILE_ACCESS);
#ifdef BL_USE_MPI
#ifdef AMREX_USE_MPI
SetHDF5fapl(fapl, ParallelDescriptor::Communicator());
#else
SetHDF5fapl(fapl);
Expand Down
36 changes: 24 additions & 12 deletions Tests/HDF5Benchmark/GNUmakefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,26 @@ USE_PARTICLES = TRUE
EBASE = main

USE_HDF5 = TRUE
# HDF5_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/hdf5/develop_build/hdf5
HDF5_HOME = $(OLCF_HDF5_ROOT)
# HDF5_HOME = /lustre/orion/geo130/world-shared/gnu/hdf5.1.14.1.sync.cmake
# HDF5_HOME = $(HDF5_ROOT)
# HDF5_HOME = /lustre/orion/geo130/world-shared/gnu/hdf5.1.14.1.async.cmake

# To use HDF5 with ZFP compression, follow the instructions at https://h5z-zfp.readthedocs.io to compile ZFP and H5Z_ZFP plugin, or use the following gcc build on Summit
# To use HDF5 with ZFP compression, follow the instructions at https://h5z-zfp.readthedocs.io to compile ZFP and H5Z_ZFP plugin, or use the following gcc build on Frontier
USE_HDF5_ZFP = FALSE
H5Z_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/H5Z-ZFP/install
ZFP_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/zfp

# To use HDF5 with SZ compression, follow the instructions at https://github.com/szcompressor/SZ/tree/master/hdf5-filter/H5Z-SZ to compile SZ and H5Z_SZ plugin, or use the following gcc build on Summit
# Frontier
H5Z_HOME = /lustre/orion/geo130/world-shared/gnu/H5Z-ZFP
ZFP_HOME = /lustre/orion/geo130/world-shared/gnu/zfp
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it makes sense to leave the Summit one in, but commented out?

Also, are these available for everyone to use, or do you need access to geo130?

Copy link
Contributor Author

@houjun houjun Jul 26, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just added the Summit paths that are commented out.
Everyone should be able to access the "world-shared" locations on both Summit and Frontier.

# Summit
# H5Z_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/H5Z-ZFP/install
# ZFP_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/zfp

# To use HDF5 with SZ compression, follow the instructions at https://github.com/szcompressor/SZ/tree/master/hdf5-filter/H5Z-SZ to compile SZ and H5Z_SZ plugin, or use the following gcc build on Frontier
USE_HDF5_SZ = FALSE
SZ_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/SZ/install
# Frontier
SZ_HOME = /lustre/orion/geo130/world-shared/gnu/SZ
# Summit
# SZ_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/SZ/install

include $(AMREX_HOME)/Tools/GNUMake/Make.defs

Expand All @@ -43,13 +52,16 @@ include $(AMREX_HOME)/Src/Particle/Make.package

include $(AMREX_HOME)/Tools/GNUMake/Make.rules

# To use HDF5 asynchronous I/O VOL connector, follow the instructions at https://github.com/hpc-io/vol-async, add -DENABLE_WRITE_MEMCPY=1 to CFLAGS in src/Makefile, or use the following gcc build on Summit
# To use HDF5 asynchronous I/O VOL connector, follow the instructions at https://github.com/hpc-io/vol-async, add -DENABLE_WRITE_MEMCPY=1 to CFLAGS in src/Makefile, or use the following gcc build on Frontier
USE_HDF5_ASYNC = FALSE
ABT_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/argobots/install
ASYNC_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/vol-async-memcpy/src
# Frontier
ABT_HOME = /lustre/orion/geo130/world-shared/gnu/argobots
ASYNC_HOME = /lustre/orion/geo130/world-shared/gnu/vol-async-memcpy
# Summit
# ABT_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/argobots/install
# ASYNC_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/vol-async-memcpy/src

ifeq ($(USE_HDF5_ASYNC), TRUE)
DEFINES += -DAMREX_USE_HDF5_ASYNC
INCLUDE_LOCATIONS += $(ABT_HOME)/include $(ASYNC_HOME)
LIBRARIES += -L$(ABT_HOME)/lib -L$(ASYNC_HOME) -lh5async -lasynchdf5 -labt -Wl,-rpath=$(ABT_HOME)/lib -Wl,-rpath=$(ASYNC_HOME)
LIBRARIES += -L$(ABT_HOME)/lib -L$(ASYNC_HOME)/lib -lh5async -labt -Wl,-rpath=$(ABT_HOME)/lib -Wl,-rpath=$(ASYNC_HOME)/lib
endif
Loading