Skip to content

Commit

Permalink
Merge pull request #1036 from fortnern/parallel_selection_io
Browse files Browse the repository at this point in the history 
Implement parallel collective support for selection I/O.
  • Loading branch information
@fortnern
fortnern authored Sep 27, 2021
2 parents c1e73fd + 5115f8b commit 7c5c0bb
Show file tree
Hide file tree
Showing 14 changed files with 224 additions and 151 deletions.
13 changes: 6 additions & 7 deletions src/H5.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ H5__init_package(void)
herr_t
H5_init_library(void)
{
char *env_use_select_io = NULL;
herr_t ret_value = SUCCEED;
char * env_use_select_io = NULL;
herr_t ret_value = SUCCEED;

/* Set the 'library initialized' flag as early as possible, to avoid
* possible re-entrancy.
Expand Down Expand Up @@ -281,11 +281,10 @@ H5_init_library(void)

/* Check for HDF5_USE_SELECTION_IO env variable */
env_use_select_io = HDgetenv("HDF5_USE_SELECTION_IO");
if (NULL != env_use_select_io && HDstrcmp(env_use_select_io, "")
&& HDstrcmp(env_use_select_io, "0") && HDstrcmp(env_use_select_io, "no")
&& HDstrcmp(env_use_select_io, "No") && HDstrcmp(env_use_select_io, "NO")
&& HDstrcmp(env_use_select_io, "false") && HDstrcmp(env_use_select_io, "False")
&& HDstrcmp(env_use_select_io, "FALSE"))
if (NULL != env_use_select_io && HDstrcmp(env_use_select_io, "") && HDstrcmp(env_use_select_io, "0") &&
HDstrcmp(env_use_select_io, "no") && HDstrcmp(env_use_select_io, "No") &&
HDstrcmp(env_use_select_io, "NO") && HDstrcmp(env_use_select_io, "false") &&
HDstrcmp(env_use_select_io, "False") && HDstrcmp(env_use_select_io, "FALSE"))
H5_use_selection_io_g = TRUE;

/* Debugging? */
Expand Down
33 changes: 16 additions & 17 deletions src/H5Dchunk.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,9 +254,8 @@ typedef struct H5D_chunk_coll_info_t {
/* Chunked layout operation callbacks */
static herr_t H5D__chunk_construct(H5F_t *f, H5D_t *dset);
static herr_t H5D__chunk_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id);
static herr_t H5D__chunk_io_init(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
H5D_chunk_map_t *fm);
static herr_t H5D__chunk_io_init(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts,
const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *fm);
static herr_t H5D__chunk_io_init_selections(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
H5D_chunk_map_t *fm);
static herr_t H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts,
Expand Down Expand Up @@ -1064,10 +1063,10 @@ H5D__chunk_io_init(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsi
hssize_t old_offset[H5O_LAYOUT_NDIMS]; /* Old selection offset */
htri_t file_space_normalized = FALSE; /* File dataspace was normalized */
unsigned f_ndims; /* The number of dimensions of the file's dataspace */
int sm_ndims; /* The number of dimensions of the memory buffer's dataspace (signed) */
htri_t use_selection_io = FALSE; /* Whether to use selection I/O */
unsigned u; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
int sm_ndims; /* The number of dimensions of the memory buffer's dataspace (signed) */
htri_t use_selection_io = FALSE; /* Whether to use selection I/O */
unsigned u; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */

FUNC_ENTER_STATIC

Expand Down Expand Up @@ -2541,7 +2540,7 @@ H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_
H5S_t * chunk_file_spaces_static[8]; /* Static buffer for chunk_file_spaces */
haddr_t * chunk_addrs = NULL; /* Array of chunk addresses */
haddr_t chunk_addrs_static[8]; /* Static buffer for chunk_addrs */
herr_t ret_value = SUCCEED; /*return value */
herr_t ret_value = SUCCEED; /*return value */

FUNC_ENTER_STATIC

Expand Down Expand Up @@ -2650,10 +2649,10 @@ H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_
/* Issue selection I/O call (we can skip the page buffer because we've
* already verified it won't be used, and the metadata accumulator
* because this is raw data) */
if (num_chunks > 0 &&
H5F_shared_select_read(H5F_SHARED(io_info->dset->oloc.file), H5FD_MEM_DRAW, (uint32_t)num_chunks,
(const H5S_t * const *)chunk_mem_spaces, (const H5S_t * const *)chunk_file_spaces,
chunk_addrs, element_sizes, bufs) < 0)
if (H5F_shared_select_read(H5F_SHARED(io_info->dset->oloc.file), H5FD_MEM_DRAW, (uint32_t)num_chunks,
(const H5S_t *const *)chunk_mem_spaces,
(const H5S_t *const *)chunk_file_spaces, chunk_addrs, element_sizes,
bufs) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "chunk selection read failed")

/* Clean up memory */
Expand Down Expand Up @@ -2816,7 +2815,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize
H5S_t * chunk_file_spaces_static[8]; /* Static buffer for chunk_file_spaces */
haddr_t * chunk_addrs = NULL; /* Array of chunk addresses */
haddr_t chunk_addrs_static[8]; /* Static buffer for chunk_addrs */
herr_t ret_value = SUCCEED; /* Return value */
herr_t ret_value = SUCCEED; /* Return value */

FUNC_ENTER_STATIC

Expand Down Expand Up @@ -2985,10 +2984,10 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize
/* Issue selection I/O call (we can skip the page buffer because we've
* already verified it won't be used, and the metadata accumulator
* because this is raw data) */
if (num_chunks > 0 && H5F_shared_select_write(
H5F_SHARED(io_info->dset->oloc.file), H5FD_MEM_DRAW, (uint32_t)num_chunks,
(const H5S_t * const *)chunk_mem_spaces, (const H5S_t * const *)chunk_file_spaces,
chunk_addrs, element_sizes, bufs) < 0)
if (H5F_shared_select_write(H5F_SHARED(io_info->dset->oloc.file), H5FD_MEM_DRAW, (uint32_t)num_chunks,
(const H5S_t *const *)chunk_mem_spaces,
(const H5S_t *const *)chunk_file_spaces, chunk_addrs, element_sizes,
bufs) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "chunk selection read failed")

/* Clean up memory */
Expand Down
5 changes: 2 additions & 3 deletions src/H5Dcompact.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,8 @@
/* Layout operation callbacks */
static herr_t H5D__compact_construct(H5F_t *f, H5D_t *dset);
static hbool_t H5D__compact_is_space_alloc(const H5O_storage_t *storage);
static herr_t H5D__compact_io_init(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
H5D_chunk_map_t *cm);
static herr_t H5D__compact_io_init(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts,
const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *cm);
static ssize_t H5D__compact_readvv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t *dset_curr_seq,
size_t dset_size_arr[], hsize_t dset_offset_arr[], size_t mem_max_nseq,
size_t *mem_curr_seq, size_t mem_size_arr[], hsize_t mem_offset_arr[]);
Expand Down
38 changes: 18 additions & 20 deletions src/H5Dcontig.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,8 @@ typedef struct H5D_contig_writevv_ud_t {
/* Layout operation callbacks */
static herr_t H5D__contig_construct(H5F_t *f, H5D_t *dset);
static herr_t H5D__contig_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id);
static herr_t H5D__contig_io_init(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
H5D_chunk_map_t *cm);
static herr_t H5D__contig_io_init(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts,
const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *cm);
static ssize_t H5D__contig_readvv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t *dset_curr_seq,
size_t dset_len_arr[], hsize_t dset_offset_arr[], size_t mem_max_nseq,
size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[]);
Expand Down Expand Up @@ -555,8 +554,8 @@ H5D__contig_io_init(H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR_UNUSED
hsize_t H5_ATTR_UNUSED nelmts, const H5S_t H5_ATTR_UNUSED *file_space,
const H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *cm)
{
htri_t use_selection_io = FALSE; /* Whether to use selection I/O */
htri_t ret_value = SUCCEED; /* Return value */
htri_t use_selection_io = FALSE; /* Whether to use selection I/O */
htri_t ret_value = SUCCEED; /* Return value */

FUNC_ENTER_STATIC

Expand Down Expand Up @@ -601,10 +600,10 @@ H5D__contig_may_use_select_io(const H5D_io_info_t *io_info, H5D_io_op_type_t op_
/* Don't use selection I/O if it's globally disabled, if there is a type
* conversion, or if it's not a contiguous dataset, or if the sieve buffer
* exists (write) or is dirty (read) */
if (!H5_use_selection_io_g || io_info->io_ops.single_read != H5D__select_read
|| io_info->layout_ops.readvv != H5D__contig_readvv
|| (op_type == H5D_IO_OP_READ && io_info->dset->shared->cache.contig.sieve_dirty)
|| (op_type == H5D_IO_OP_WRITE && io_info->dset->shared->cache.contig.sieve_buf))
if (!H5_use_selection_io_g || io_info->io_ops.single_read != H5D__select_read ||
io_info->layout_ops.readvv != H5D__contig_readvv ||
(op_type == H5D_IO_OP_READ && io_info->dset->shared->cache.contig.sieve_dirty) ||
(op_type == H5D_IO_OP_WRITE && io_info->dset->shared->cache.contig.sieve_buf))
ret_value = FALSE;
else {
htri_t page_buf_enabled;
Expand Down Expand Up @@ -641,7 +640,7 @@ herr_t
H5D__contig_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts,
const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm)
{
herr_t ret_value = SUCCEED; /* Return value */
herr_t ret_value = SUCCEED; /* Return value */

FUNC_ENTER_PACKAGE

Expand All @@ -657,16 +656,16 @@ H5D__contig_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize

/* Issue selection I/O call (we can skip the page buffer because we've
* already verified it won't be used, and the metadata accumulator
* because this is raw data) Only call funciton if nelmts > 0. */
if (nelmts > 0 && H5F_shared_select_read(H5F_SHARED(io_info->dset->oloc.file), H5FD_MEM_DRAW, 1,
* because this is raw data) */
if (H5F_shared_select_read(H5F_SHARED(io_info->dset->oloc.file), H5FD_MEM_DRAW, nelmts > 0 ? 1 : 0,
&mem_space, &file_space, &(io_info->store->contig.dset_addr),
&dst_type_size, &(io_info->u.rbuf)) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "contiguous selection read failed")
} /* end if */
else
/* Read data through legacy (non-selection I/O) pathway */
if ((io_info->io_ops.single_read)(io_info, type_info, nelmts, file_space, mem_space) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "contiguous read failed")
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "contiguous read failed")

done:
FUNC_LEAVE_NOAPI(ret_value)
Expand All @@ -688,7 +687,7 @@ herr_t
H5D__contig_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts,
const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm)
{
herr_t ret_value = SUCCEED; /* Return value */
herr_t ret_value = SUCCEED; /* Return value */

FUNC_ENTER_PACKAGE

Expand All @@ -704,16 +703,16 @@ H5D__contig_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsiz

/* Issue selection I/O call (we can skip the page buffer because we've
* already verified it won't be used, and the metadata accumulator
* because this is raw data). Only call funciton if nelmts > 0. */
if (nelmts > 0 && H5F_shared_select_write(H5F_SHARED(io_info->dset->oloc.file), H5FD_MEM_DRAW, 1,
&mem_space, &file_space, &(io_info->store->contig.dset_addr),
&dst_type_size, &(io_info->u.wbuf)) < 0)
* because this is raw data) */
if (H5F_shared_select_write(H5F_SHARED(io_info->dset->oloc.file), H5FD_MEM_DRAW, nelmts > 0 ? 1 : 0,
&mem_space, &file_space, &(io_info->store->contig.dset_addr),
&dst_type_size, &(io_info->u.wbuf)) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "contiguous selection write failed")
} /* end if */
else
/* Write data through legacy (non-selection I/O) pathway */
if ((io_info->io_ops.single_write)(io_info, type_info, nelmts, file_space, mem_space) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "contiguous write failed")
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "contiguous write failed")

done:
FUNC_LEAVE_NOAPI(ret_value)
Expand Down Expand Up @@ -1649,4 +1648,3 @@ H5D__contig_copy(H5F_t *f_src, const H5O_storage_contig_t *storage_src, H5F_t *f

FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__contig_copy() */

6 changes: 3 additions & 3 deletions src/H5Defl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@ typedef struct H5D_efl_writevv_ud_t {
/********************/

/* Layout operation callbacks */
static herr_t H5D__efl_construct(H5F_t *f, H5D_t *dset);
static herr_t H5D__efl_io_init(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts,
const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *cm);
static herr_t H5D__efl_construct(H5F_t *f, H5D_t *dset);
static herr_t H5D__efl_io_init(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts,
const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *cm);
static ssize_t H5D__efl_readvv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t *dset_curr_seq,
size_t dset_len_arr[], hsize_t dset_offset_arr[], size_t mem_max_nseq,
size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[]);
Expand Down
24 changes: 13 additions & 11 deletions src/H5Dio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -845,11 +845,8 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, const H5S_t *file_
H5CX_set_mpio_actual_io_mode(H5D_MPIO_NO_COLLECTIVE);
} /* end if */

/* Make any parallel I/O adjustments. Do not use collective code path if
* we're using selection I/O - in this case the file driver will handle it.
*/
/* Check for selection/vector support in file driver? -NAF */
if (io_info->using_mpi_vfd /*&& !H5_use_selection_io_g*/) {
/* Make any parallel I/O adjustments */
if (io_info->using_mpi_vfd) {
H5FD_mpio_xfer_t xfer_mode; /* Parallel transfer for this request */
htri_t opt; /* Flag whether a selection is optimizable */

Expand All @@ -867,12 +864,17 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, const H5S_t *file_

/* Check if we can use the optimized parallel I/O routines */
if (opt == TRUE) {
/* Override the I/O op pointers to the MPI-specific routines */
io_info->io_ops.multi_read = dset->shared->layout.ops->par_read;
io_info->io_ops.multi_write = dset->shared->layout.ops->par_write;
io_info->io_ops.single_read = H5D__mpio_select_read;
io_info->io_ops.single_write = H5D__mpio_select_write;
} /* end if */
/* Override the I/O op pointers to the MPI-specific routines, unless
* selection I/O is to be used - in this case the file driver will
* handle collective I/O */
/* Check for selection/vector support in file driver? -NAF */
if (!io_info->use_select_io) {
io_info->io_ops.multi_read = dset->shared->layout.ops->par_read;
io_info->io_ops.multi_write = dset->shared->layout.ops->par_write;
io_info->io_ops.single_read = H5D__mpio_select_read;
io_info->io_ops.single_write = H5D__mpio_select_write;
} /* end if */
} /* end if */
else {
/* Check if there are any filters in the pipeline. If there are,
* we cannot break to independent I/O if this is a write operation;
Expand Down
5 changes: 2 additions & 3 deletions src/H5Dpkg.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,9 +121,8 @@ typedef herr_t (*H5D_layout_construct_func_t)(H5F_t *f, H5D_t *dset);
typedef herr_t (*H5D_layout_init_func_t)(H5F_t *f, const H5D_t *dset, hid_t dapl_id);
typedef hbool_t (*H5D_layout_is_space_alloc_func_t)(const H5O_storage_t *storage);
typedef hbool_t (*H5D_layout_is_data_cached_func_t)(const H5D_shared_t *shared_dset);
typedef herr_t (*H5D_layout_io_init_func_t)(struct H5D_io_info_t *io_info,
const H5D_type_info_t *type_info, hsize_t nelmts,
const H5S_t *file_space, const H5S_t *mem_space,
typedef herr_t (*H5D_layout_io_init_func_t)(struct H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
struct H5D_chunk_map_t *cm);
typedef herr_t (*H5D_layout_read_func_t)(struct H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
Expand Down
Loading

0 comments on commit 7c5c0bb

Please sign in to comment.