From 35fadc48a6f46ae60b717f3e0ea82218438273e4 Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Thu, 2 Nov 2023 14:40:38 -0500 Subject: [PATCH 01/11] Changes for ECP-344: Implement selection vector I/O with collective chunk filling. Also fix a bug in H5FD__mpio_write_vector() to account for fixed size optimization when computing max address. --- src/H5Dchunk.c | 268 ++++++----------------------------- src/H5FDmpio.c | 9 +- testpar/t_filters_parallel.c | 21 ++- 3 files changed, 66 insertions(+), 232 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 41d774d0d3e..5e733bdd612 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -314,7 +314,6 @@ static herr_t H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, bool new_unfilt #ifdef H5_HAVE_PARALLEL static herr_t H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_fill_info, const void *fill_buf, const void *partial_chunk_fill_buf); -static int H5D__chunk_cmp_coll_fill_info(const void *_entry1, const void *_entry2); #endif /* H5_HAVE_PARALLEL */ /* Debugging helper routine callback */ @@ -5536,9 +5535,7 @@ H5D__chunk_update_old_edge_chunks(H5D_t *dset, hsize_t old_dim[]) /*------------------------------------------------------------------------- * Function: H5D__chunk_collective_fill * - * Purpose: Use MPIO collective write to fill the chunks (if number of - * chunks to fill is greater than the number of MPI procs; - * otherwise use independent I/O). + * Purpose: Use MPIO selection vector I/O for writing fill chunks * * Return: Non-negative on success/Negative on failure * @@ -5548,211 +5545,61 @@ static herr_t H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_fill_info, const void *fill_buf, const void *partial_chunk_fill_buf) { - MPI_Comm mpi_comm = MPI_COMM_NULL; /* MPI communicator for file */ - int mpi_rank = (-1); /* This process's rank */ - int mpi_size = (-1); /* MPI Comm size */ - int mpi_code; /* MPI return code */ - size_t num_blocks; /* Number of blocks between processes. */ - size_t leftover_blocks; /* Number of leftover blocks to handle */ - int blocks, leftover; /* converted to int for MPI */ - MPI_Aint *chunk_disp_array = NULL; - MPI_Aint *block_disps = NULL; - int *block_lens = NULL; - MPI_Datatype mem_type = MPI_BYTE, file_type = MPI_BYTE; H5FD_mpio_xfer_t prev_xfer_mode; /* Previous data xfer mode */ bool have_xfer_mode = false; /* Whether the previous xffer mode has been retrieved */ - bool need_sort = false; size_t i; /* Local index variable */ + uint32_t io_count = 0; + haddr_t *io_addrs = NULL; + size_t *io_sizes = NULL; + const void **io_wbufs = NULL; + H5FD_mem_t io_types[2]; + bool all_same_block_len = true; + size_t io_2sizes[2]; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE - /* - * If a separate fill buffer is provided for partial chunks, ensure - * that the "don't filter partial edge chunks" flag is set. - */ - if (partial_chunk_fill_buf) - assert(dset->shared->layout.u.chunk.flags & H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS); - - /* Get the MPI communicator */ - if (MPI_COMM_NULL == (mpi_comm = H5F_mpi_get_comm(dset->oloc.file))) - HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI communicator"); - - /* Get the MPI rank */ - if ((mpi_rank = H5F_mpi_get_rank(dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank"); - - /* Get the MPI size */ - if ((mpi_size = H5F_mpi_get_size(dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI size"); - - /* Distribute evenly the number of blocks between processes. */ - if (mpi_size == 0) - HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Resulted in division by zero"); - num_blocks = - (size_t)(chunk_fill_info->num_chunks / (size_t)mpi_size); /* value should be the same on all procs */ - - /* After evenly distributing the blocks between processes, are there any - * leftover blocks for each individual process (round-robin)? - */ - leftover_blocks = (size_t)(chunk_fill_info->num_chunks % (size_t)mpi_size); - - /* Cast values to types needed by MPI */ - H5_CHECKED_ASSIGN(blocks, int, num_blocks, size_t); - H5_CHECKED_ASSIGN(leftover, int, leftover_blocks, size_t); - - /* Check if we have any chunks to write on this rank */ - if (num_blocks > 0 || (leftover && leftover > mpi_rank)) { - MPI_Aint partial_fill_buf_disp = 0; - bool all_same_block_len = true; - - /* Allocate buffers */ - if (NULL == (chunk_disp_array = (MPI_Aint *)H5MM_malloc((size_t)(blocks + 1) * sizeof(MPI_Aint)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk file displacement buffer"); - - if (partial_chunk_fill_buf) { - MPI_Aint fill_buf_addr; - MPI_Aint partial_fill_buf_addr; - - /* Calculate the displacement between the fill buffer and partial chunk fill buffer */ - if (MPI_SUCCESS != (mpi_code = MPI_Get_address(fill_buf, &fill_buf_addr))) - HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) - if (MPI_SUCCESS != (mpi_code = MPI_Get_address(partial_chunk_fill_buf, &partial_fill_buf_addr))) - HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) - -#if H5_CHECK_MPI_VERSION(3, 1) - partial_fill_buf_disp = MPI_Aint_diff(partial_fill_buf_addr, fill_buf_addr); -#else - partial_fill_buf_disp = partial_fill_buf_addr - fill_buf_addr; -#endif - - /* - * Allocate all-zero block displacements array. If a block's displacement - * is left as zero, that block will be written to from the regular fill - * buffer. If a block represents an unfiltered partial edge chunk, its - * displacement will be set so that the block is written to from the - * unfiltered fill buffer. - */ - if (NULL == (block_disps = (MPI_Aint *)H5MM_calloc((size_t)(blocks + 1) * sizeof(MPI_Aint)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate block displacements buffer"); - } - - /* - * Perform initial scan of chunk info list to: - * - make sure that chunk addresses are monotonically non-decreasing - * - check if all blocks have the same length - */ - for (i = 1; i < chunk_fill_info->num_chunks; i++) { - if (chunk_fill_info->chunk_info[i].addr < chunk_fill_info->chunk_info[i - 1].addr) - need_sort = true; - - if (chunk_fill_info->chunk_info[i].chunk_size != chunk_fill_info->chunk_info[i - 1].chunk_size) - all_same_block_len = false; - } - - if (need_sort) - qsort(chunk_fill_info->chunk_info, chunk_fill_info->num_chunks, - sizeof(struct chunk_coll_fill_info), H5D__chunk_cmp_coll_fill_info); - - /* Allocate buffer for block lengths if necessary */ - if (!all_same_block_len) - if (NULL == (block_lens = (int *)H5MM_malloc((size_t)(blocks + 1) * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk lengths buffer"); + assert(chunk_fill_info->num_chunks != 0); - for (i = 0; i < (size_t)blocks; i++) { - size_t idx = i + (size_t)(mpi_rank * blocks); + io_count = (uint32_t)chunk_fill_info->num_chunks; - /* store the chunk address as an MPI_Aint */ - chunk_disp_array[i] = (MPI_Aint)(chunk_fill_info->chunk_info[idx].addr); - - if (!all_same_block_len) - H5_CHECKED_ASSIGN(block_lens[i], int, chunk_fill_info->chunk_info[idx].chunk_size, size_t); - - if (chunk_fill_info->chunk_info[idx].unfiltered_partial_chunk) { - assert(partial_chunk_fill_buf); - block_disps[i] = partial_fill_buf_disp; - } - } /* end for */ - - /* Calculate if there are any leftover blocks after evenly - * distributing. If there are, then round-robin the distribution - * to processes 0 -> leftover. - */ - if (leftover && leftover > mpi_rank) { - chunk_disp_array[blocks] = - (MPI_Aint)chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].addr; - - if (!all_same_block_len) - H5_CHECKED_ASSIGN(block_lens[blocks], int, - chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].chunk_size, - size_t); - - if (chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].unfiltered_partial_chunk) { - assert(partial_chunk_fill_buf); - block_disps[blocks] = partial_fill_buf_disp; - } - - blocks++; + for (i = 1; i < io_count; i++) { + if (chunk_fill_info->chunk_info[i].chunk_size != chunk_fill_info->chunk_info[i - 1].chunk_size) { + all_same_block_len = false; + break; } + } - /* Create file and memory types for the write operation */ - if (all_same_block_len) { - int block_len; + if (all_same_block_len) { + io_2sizes[0] = chunk_fill_info->chunk_info[0].chunk_size; + io_2sizes[1] = 0; + } else { + if (NULL == (io_sizes = H5MM_malloc(io_count * sizeof(*io_sizes)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O sizes vector"); + } - H5_CHECKED_ASSIGN(block_len, int, chunk_fill_info->chunk_info[0].chunk_size, size_t); + io_types[0] = H5FD_MEM_DRAW; + io_types[1] = H5FD_MEM_NOLIST; - mpi_code = - MPI_Type_create_hindexed_block(blocks, block_len, chunk_disp_array, MPI_BYTE, &file_type); - if (mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed_block failed", mpi_code) + if (NULL == (io_addrs = H5MM_malloc(io_count * sizeof(*io_addrs)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate space for I/O addresses vector"); - if (partial_chunk_fill_buf) { - /* - * If filters are disabled for partial edge chunks, those chunks could - * potentially have the same block length as the other chunks, but still - * need to be written to using the unfiltered fill buffer. Use an hindexed - * block type rather than an hvector. - */ - mpi_code = - MPI_Type_create_hindexed_block(blocks, block_len, block_disps, MPI_BYTE, &mem_type); - if (mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed_block failed", mpi_code) - } - else { - mpi_code = MPI_Type_create_hvector(blocks, block_len, 0, MPI_BYTE, &mem_type); - if (mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) - } - } - else { - /* - * Currently, different block lengths implies that there are partial - * edge chunks and the "don't filter partial edge chunks" flag is set. - */ - assert(partial_chunk_fill_buf); - assert(block_lens); - assert(block_disps); + if (NULL == (io_wbufs = H5MM_malloc(io_count * sizeof(*io_wbufs)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate space for I/O buffers vector"); - mpi_code = MPI_Type_create_hindexed(blocks, block_lens, chunk_disp_array, MPI_BYTE, &file_type); - if (mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + for (i = 0; i < io_count; i++) { + io_addrs[i] = chunk_fill_info->chunk_info[i].addr; - mpi_code = MPI_Type_create_hindexed(blocks, block_lens, block_disps, MPI_BYTE, &mem_type); - if (mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) - } - - if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&mem_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - } /* end if */ - - /* Set MPI-IO VFD properties */ + if (!all_same_block_len) + io_sizes[i] = chunk_fill_info->chunk_info[i].chunk_size; - /* Set MPI datatypes for operation */ - if (H5CX_set_mpi_coll_datatypes(mem_type, file_type) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set MPI-I/O properties"); + if (chunk_fill_info->chunk_info[i].unfiltered_partial_chunk) + io_wbufs[i] = partial_chunk_fill_buf; + else + io_wbufs[i] = fill_buf; + } /* Get current transfer mode */ if (H5CX_get_io_xfer_mode(&prev_xfer_mode) < 0) @@ -5763,14 +5610,8 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ if (H5CX_set_io_xfer_mode(H5FD_MPIO_COLLECTIVE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set transfer mode"); - /* Low-level write (collective) */ - if (H5F_shared_block_write(H5F_SHARED(dset->oloc.file), H5FD_MEM_DRAW, (haddr_t)0, - (blocks) ? (size_t)1 : (size_t)0, fill_buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file"); - - /* Barrier so processes don't race ahead */ - if (MPI_SUCCESS != (mpi_code = MPI_Barrier(mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code) + if (H5F_shared_vector_write(H5F_SHARED(dset->oloc.file), io_count, io_types, io_addrs, all_same_block_len ? io_2sizes : io_sizes, io_wbufs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "vector write call failed"); done: if (have_xfer_mode) @@ -5778,33 +5619,14 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ if (H5CX_set_io_xfer_mode(prev_xfer_mode) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set transfer mode"); - /* free things */ - if (MPI_BYTE != file_type) - if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - if (MPI_BYTE != mem_type) - if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - H5MM_xfree(chunk_disp_array); - H5MM_xfree(block_disps); - H5MM_xfree(block_lens); + H5MM_xfree(io_addrs); + H5MM_xfree(io_wbufs); + if (!all_same_block_len) + H5MM_xfree(io_sizes); FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_collective_fill() */ -static int -H5D__chunk_cmp_coll_fill_info(const void *_entry1, const void *_entry2) -{ - const struct chunk_coll_fill_info *entry1; - const struct chunk_coll_fill_info *entry2; - - FUNC_ENTER_PACKAGE_NOERR - - entry1 = (const struct chunk_coll_fill_info *)_entry1; - entry2 = (const struct chunk_coll_fill_info *)_entry2; - - FUNC_LEAVE_NOAPI(H5_addr_cmp(entry1->addr, entry2->addr)) -} /* end H5D__chunk_cmp_coll_fill_info() */ #endif /* H5_HAVE_PARALLEL */ /*------------------------------------------------------------------------- diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 83a5ad45a9a..758728f1fbe 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -2473,8 +2473,13 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't build MPI datatypes for I/O"); /* Compute max address written to */ - if (count > 0) - max_addr = s_addrs[count - 1] + (haddr_t)(s_sizes[count - 1]); + /* Bug: need to account for fixed size optimization */ + if (count > 0) { + uint32_t idx = count - 1; + if (s_sizes[0] != 0 && s_sizes[1] == 0) + idx = 0; + max_addr = s_addrs[count - 1] + (haddr_t)(s_sizes[idx]); + } /* free sorted vectors if they exist */ if (!vector_was_sorted) { diff --git a/testpar/t_filters_parallel.c b/testpar/t_filters_parallel.c index 6c054085ed7..f6843a0fd71 100644 --- a/testpar/t_filters_parallel.c +++ b/testpar/t_filters_parallel.c @@ -556,11 +556,19 @@ verify_chunk_opt_status(size_t num_dsets, test_mode_t test_mode, bool any_io, bo * elements (because no elements were raw data), which is what happens when performing I/O on a * filtered dataset with no selection. Vector I/O does report an I/O call was made if passed a raw * data element of size 0, so this is consistent. */ + /* Changes: for allocation, the reported I/O is vector I/O */ if (!any_io) { - if (did_alloc || (num_dsets > 0 && test_mode == USE_MULTIPLE_DATASETS_MIXED_FILTERED)) + if (did_alloc && (num_dsets > 0 && test_mode == USE_MULTIPLE_DATASETS_MIXED_FILTERED)) { + VRFY((H5D_VECTOR_IO | H5D_SCALAR_IO) == actual_sel_io_mode_reduced, + "verified actual selection I/O mode was vector and scalar I/O"); + } else if (did_alloc) { + VRFY(H5D_VECTOR_IO == actual_sel_io_mode_reduced, + "verified actual selection I/O mode was vector I/O"); + } + else if (num_dsets > 0 && test_mode == USE_MULTIPLE_DATASETS_MIXED_FILTERED) { VRFY(H5D_SCALAR_IO == actual_sel_io_mode_reduced, "verified actual selection I/O mode was scalar I/O"); - else + } else VRFY(0 == actual_sel_io_mode_reduced, "verified actual selection I/O mode was 0 (no I/O)"); } @@ -592,15 +600,14 @@ verify_chunk_opt_status(size_t num_dsets, test_mode_t test_mode, bool any_io, bo * should be scalar I/O for allocation in addition to vector I/O for the actual data. * If we're reading from an unallocated dataset then there should be no actual I/O. * Otherwise there should only be vector I/O. */ - if (did_alloc) - VRFY((H5D_SCALAR_IO | H5D_VECTOR_IO) == actual_sel_io_mode_reduced, - "verified actual selection I/O mode was scalar and vector I/O"); - else if (unalloc_read) + /* Changes: for allocation, the reported I/O is vector I/O */ + if (unalloc_read) VRFY(0 == actual_sel_io_mode_reduced, "verified actual selection I/O mode was 0 (no I/O)"); - else + else { /* did_alloc || !unalloc_read */ VRFY(H5D_VECTOR_IO == actual_sel_io_mode_reduced, "verified actual selection I/O mode was vector I/O"); + } break; case USE_MULTIPLE_DATASETS_MIXED_FILTERED: From f018960c11dee8a1e2f47717e1c97cfc79ebba66 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Nov 2023 19:46:51 +0000 Subject: [PATCH 02/11] Committing clang-format changes --- src/H5Dchunk.c | 28 ++++++++++++++-------------- testpar/t_filters_parallel.c | 8 +++++--- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 5e733bdd612..8ea47093936 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -5547,14 +5547,14 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ { H5FD_mpio_xfer_t prev_xfer_mode; /* Previous data xfer mode */ bool have_xfer_mode = false; /* Whether the previous xffer mode has been retrieved */ - size_t i; /* Local index variable */ - uint32_t io_count = 0; - haddr_t *io_addrs = NULL; - size_t *io_sizes = NULL; - const void **io_wbufs = NULL; - H5FD_mem_t io_types[2]; - bool all_same_block_len = true; - size_t io_2sizes[2]; + size_t i; /* Local index variable */ + uint32_t io_count = 0; + haddr_t *io_addrs = NULL; + size_t *io_sizes = NULL; + const void **io_wbufs = NULL; + H5FD_mem_t io_types[2]; + bool all_same_block_len = true; + size_t io_2sizes[2]; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -5573,7 +5573,8 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ if (all_same_block_len) { io_2sizes[0] = chunk_fill_info->chunk_info[0].chunk_size; io_2sizes[1] = 0; - } else { + } + else { if (NULL == (io_sizes = H5MM_malloc(io_count * sizeof(*io_sizes)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O sizes vector"); } @@ -5582,12 +5583,10 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ io_types[1] = H5FD_MEM_NOLIST; if (NULL == (io_addrs = H5MM_malloc(io_count * sizeof(*io_addrs)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "couldn't allocate space for I/O addresses vector"); + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O addresses vector"); if (NULL == (io_wbufs = H5MM_malloc(io_count * sizeof(*io_wbufs)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "couldn't allocate space for I/O buffers vector"); + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O buffers vector"); for (i = 0; i < io_count; i++) { io_addrs[i] = chunk_fill_info->chunk_info[i].addr; @@ -5610,7 +5609,8 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ if (H5CX_set_io_xfer_mode(H5FD_MPIO_COLLECTIVE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set transfer mode"); - if (H5F_shared_vector_write(H5F_SHARED(dset->oloc.file), io_count, io_types, io_addrs, all_same_block_len ? io_2sizes : io_sizes, io_wbufs) < 0) + if (H5F_shared_vector_write(H5F_SHARED(dset->oloc.file), io_count, io_types, io_addrs, + all_same_block_len ? io_2sizes : io_sizes, io_wbufs) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "vector write call failed"); done: diff --git a/testpar/t_filters_parallel.c b/testpar/t_filters_parallel.c index f6843a0fd71..5473eaa3d5d 100644 --- a/testpar/t_filters_parallel.c +++ b/testpar/t_filters_parallel.c @@ -561,14 +561,16 @@ verify_chunk_opt_status(size_t num_dsets, test_mode_t test_mode, bool any_io, bo if (did_alloc && (num_dsets > 0 && test_mode == USE_MULTIPLE_DATASETS_MIXED_FILTERED)) { VRFY((H5D_VECTOR_IO | H5D_SCALAR_IO) == actual_sel_io_mode_reduced, "verified actual selection I/O mode was vector and scalar I/O"); - } else if (did_alloc) { + } + else if (did_alloc) { VRFY(H5D_VECTOR_IO == actual_sel_io_mode_reduced, "verified actual selection I/O mode was vector I/O"); - } + } else if (num_dsets > 0 && test_mode == USE_MULTIPLE_DATASETS_MIXED_FILTERED) { VRFY(H5D_SCALAR_IO == actual_sel_io_mode_reduced, "verified actual selection I/O mode was scalar I/O"); - } else + } + else VRFY(0 == actual_sel_io_mode_reduced, "verified actual selection I/O mode was 0 (no I/O)"); } From 5b501b91da5090ce2346e2b90d7dfc4965b27914 Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Fri, 3 Nov 2023 11:28:34 -0500 Subject: [PATCH 03/11] Fixes based on PR review comments: For H5Dchunk.c: fix H5MM_xfree() For H5FDmpio.c: 1) Revert the fix to H5FD__mpio_write_vector() 2) Apply the patch from Neil on the proper length of s_sizes reported by H5FD__mpio_vector_build_types() --- src/H5Dchunk.c | 3 +-- src/H5FDmpio.c | 30 ++++++++++++++++++------------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 8ea47093936..a83f5229c6c 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -5621,8 +5621,7 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ H5MM_xfree(io_addrs); H5MM_xfree(io_wbufs); - if (!all_same_block_len) - H5MM_xfree(io_sizes); + H5MM_xfree(io_sizes); FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_collective_fill() */ diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 758728f1fbe..35ea4c25ac4 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -106,7 +106,7 @@ static herr_t H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, co /* Other functions */ static herr_t H5FD__mpio_vector_build_types(uint32_t count, H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], H5_flexible_const_ptr_t bufs[], - haddr_t *s_addrs[], size_t *s_sizes[], + haddr_t *s_addrs[], size_t *s_sizes[], uint32_t *s_sizes_len, H5_flexible_const_ptr_t *s_bufs[], bool *vector_was_sorted, MPI_Offset *mpi_off, H5_flexible_const_ptr_t *mpi_bufs_base, int *size_i, MPI_Datatype *buf_type, bool *buf_type_created, @@ -1674,7 +1674,7 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, h */ static herr_t H5FD__mpio_vector_build_types(uint32_t count, H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], - H5_flexible_const_ptr_t bufs[], haddr_t *s_addrs[], size_t *s_sizes[], + H5_flexible_const_ptr_t bufs[], haddr_t *s_addrs[], size_t *s_sizes[], uint32_t *s_sizes_len, H5_flexible_const_ptr_t *s_bufs[], bool *vector_was_sorted, MPI_Offset *mpi_off, H5_flexible_const_ptr_t *mpi_bufs_base, int *size_i, MPI_Datatype *buf_type, bool *buf_type_created, MPI_Datatype *file_type, bool *file_type_created, @@ -1716,6 +1716,10 @@ H5FD__mpio_vector_build_types(uint32_t count, H5FD_mem_t types[], haddr_t addrs[ /* Get bio I/O transition point (may be lower than 2G for testing) */ bigio_count = H5_mpi_get_bigio_count(); + /* Start wiht s_sizes_len at count */ + if (s_sizes_len) + *s_sizes_len = count; + if (count == 1) { /* Single block. Just use a series of MPI_BYTEs for the file view. */ @@ -1808,8 +1812,13 @@ H5FD__mpio_vector_build_types(uint32_t count, H5FD_mem_t types[], haddr_t addrs[ if (!fixed_size) { if ((*s_sizes)[i] == 0) { assert(vector_was_sorted); + assert(i > 0); fixed_size = true; size = sizes[i - 1]; + + /* Return the used length of the s_sizes buffer */ + if (s_sizes_len) + *s_sizes_len = (uint32_t)i; } else { size = (*s_sizes)[i]; @@ -2098,7 +2107,7 @@ H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t cou if (xfer_mode == H5FD_MPIO_COLLECTIVE) { /* Build MPI types, etc. */ if (H5FD__mpio_vector_build_types(count, types, addrs, sizes, (H5_flexible_const_ptr_t *)bufs, - &s_addrs, &s_sizes, (H5_flexible_const_ptr_t **)&s_bufs, + &s_addrs, &s_sizes, NULL, (H5_flexible_const_ptr_t **)&s_bufs, &vector_was_sorted, &mpi_off, (H5_flexible_const_ptr_t *)&mpi_bufs_base, &size_i, &buf_type, &buf_type_created, &file_type, &file_type_created, &unused) < 0) @@ -2464,22 +2473,19 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode"); if (xfer_mode == H5FD_MPIO_COLLECTIVE) { + uint32_t s_sizes_len; + /* Build MPI types, etc. */ if (H5FD__mpio_vector_build_types(count, types, addrs, sizes, (H5_flexible_const_ptr_t *)bufs, - &s_addrs, &s_sizes, (H5_flexible_const_ptr_t **)&s_bufs, + &s_addrs, &s_sizes, &s_sizes_len, (H5_flexible_const_ptr_t **)&s_bufs, &vector_was_sorted, &mpi_off, (H5_flexible_const_ptr_t *)&mpi_bufs_base, &size_i, &buf_type, &buf_type_created, &file_type, &file_type_created, &unused) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't build MPI datatypes for I/O"); - /* Compute max address written to */ - /* Bug: need to account for fixed size optimization */ - if (count > 0) { - uint32_t idx = count - 1; - if (s_sizes[0] != 0 && s_sizes[1] == 0) - idx = 0; - max_addr = s_addrs[count - 1] + (haddr_t)(s_sizes[idx]); - } + /* Compute max address written to. Note s_sizes is indexed according to the length of that array as reported by H5FD__mpio_vector_build_types(), which may be shorter if using the compressed arrays feature. */ + if (count > 0) + max_addr = s_addrs[count - 1] + (haddr_t)(s_sizes[s_sizes_len - 1]); /* free sorted vectors if they exist */ if (!vector_was_sorted) { From be8f0e48630d1674c55028aaf3282de688905770 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 3 Nov 2023 16:41:07 +0000 Subject: [PATCH 04/11] Committing clang-format changes --- src/H5FDmpio.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 35ea4c25ac4..20ae4b70aab 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -1674,8 +1674,9 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, h */ static herr_t H5FD__mpio_vector_build_types(uint32_t count, H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], - H5_flexible_const_ptr_t bufs[], haddr_t *s_addrs[], size_t *s_sizes[], uint32_t *s_sizes_len, - H5_flexible_const_ptr_t *s_bufs[], bool *vector_was_sorted, MPI_Offset *mpi_off, + H5_flexible_const_ptr_t bufs[], haddr_t *s_addrs[], size_t *s_sizes[], + uint32_t *s_sizes_len, H5_flexible_const_ptr_t *s_bufs[], + bool *vector_was_sorted, MPI_Offset *mpi_off, H5_flexible_const_ptr_t *mpi_bufs_base, int *size_i, MPI_Datatype *buf_type, bool *buf_type_created, MPI_Datatype *file_type, bool *file_type_created, char *unused) @@ -2477,13 +2478,15 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co /* Build MPI types, etc. */ if (H5FD__mpio_vector_build_types(count, types, addrs, sizes, (H5_flexible_const_ptr_t *)bufs, - &s_addrs, &s_sizes, &s_sizes_len, (H5_flexible_const_ptr_t **)&s_bufs, - &vector_was_sorted, &mpi_off, + &s_addrs, &s_sizes, &s_sizes_len, + (H5_flexible_const_ptr_t **)&s_bufs, &vector_was_sorted, &mpi_off, (H5_flexible_const_ptr_t *)&mpi_bufs_base, &size_i, &buf_type, &buf_type_created, &file_type, &file_type_created, &unused) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't build MPI datatypes for I/O"); - /* Compute max address written to. Note s_sizes is indexed according to the length of that array as reported by H5FD__mpio_vector_build_types(), which may be shorter if using the compressed arrays feature. */ + /* Compute max address written to. Note s_sizes is indexed according to the length of that array as + * reported by H5FD__mpio_vector_build_types(), which may be shorter if using the compressed arrays + * feature. */ if (count > 0) max_addr = s_addrs[count - 1] + (haddr_t)(s_sizes[s_sizes_len - 1]); From b919f7378601f01c526263b9e6a8e546c1cc0121 Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Fri, 3 Nov 2023 11:50:04 -0500 Subject: [PATCH 05/11] Fix spelling error. --- src/H5FDmpio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 20ae4b70aab..ba3f4dc6374 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -1717,7 +1717,7 @@ H5FD__mpio_vector_build_types(uint32_t count, H5FD_mem_t types[], haddr_t addrs[ /* Get bio I/O transition point (may be lower than 2G for testing) */ bigio_count = H5_mpi_get_bigio_count(); - /* Start wiht s_sizes_len at count */ + /* Start with s_sizes_len at count */ if (s_sizes_len) *s_sizes_len = count; From 0f30b4f2542c5b00f9b1c69e8f7c7d826bad16f4 Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Mon, 6 Nov 2023 23:30:37 -0600 Subject: [PATCH 06/11] Put back the logoic of dividing up the work among all the mpi ranks similar to the original H5D__chunk_collective_fill() routine. --- src/H5Dchunk.c | 134 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 114 insertions(+), 20 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index a83f5229c6c..fd1a80aa4fc 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -314,6 +314,7 @@ static herr_t H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, bool new_unfilt #ifdef H5_HAVE_PARALLEL static herr_t H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_fill_info, const void *fill_buf, const void *partial_chunk_fill_buf); +static int H5D__chunk_cmp_coll_fill_info(const void *_entry1, const void *_entry2); #endif /* H5_HAVE_PARALLEL */ /* Debugging helper routine callback */ @@ -5545,15 +5546,23 @@ static herr_t H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_fill_info, const void *fill_buf, const void *partial_chunk_fill_buf) { - H5FD_mpio_xfer_t prev_xfer_mode; /* Previous data xfer mode */ - bool have_xfer_mode = false; /* Whether the previous xffer mode has been retrieved */ - size_t i; /* Local index variable */ - uint32_t io_count = 0; + MPI_Comm mpi_comm = MPI_COMM_NULL; /* MPI communicator for file */ + int mpi_rank = (-1); /* This process's rank */ + int mpi_size = (-1); /* MPI Comm size */ + int mpi_code; /* MPI return code */ + size_t num_blocks; /* Number of blocks between processes. */ + size_t leftover_blocks; /* Number of leftover blocks to handle */ + int blocks; /* converted to int for MPI */ + int leftover; /* converted to int for MPI */ + H5FD_mpio_xfer_t prev_xfer_mode; /* Previous data xfer mode */ + bool have_xfer_mode = false; /* Whether the previous xffer mode has been retrieved */ + size_t i; /* Local index variable */ haddr_t *io_addrs = NULL; size_t *io_sizes = NULL; const void **io_wbufs = NULL; H5FD_mem_t io_types[2]; bool all_same_block_len = true; + bool need_sort = false; size_t io_2sizes[2]; herr_t ret_value = SUCCEED; /* Return value */ @@ -5561,45 +5570,111 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ assert(chunk_fill_info->num_chunks != 0); - io_count = (uint32_t)chunk_fill_info->num_chunks; + /* + * If a separate fill buffer is provided for partial chunks, ensure + * that the "don't filter partial edge chunks" flag is set. + */ + if (partial_chunk_fill_buf) + assert(dset->shared->layout.u.chunk.flags & H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS); + + /* Get the MPI communicator */ + if (MPI_COMM_NULL == (mpi_comm = H5F_mpi_get_comm(dset->oloc.file))) + HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI communicator"); + + /* Get the MPI rank */ + if ((mpi_rank = H5F_mpi_get_rank(dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank"); + + /* Get the MPI size */ + if ((mpi_size = H5F_mpi_get_size(dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI size"); + + /* Distribute evenly the number of blocks between processes. */ + if (mpi_size == 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Resulted in division by zero"); + + num_blocks = + (size_t)(chunk_fill_info->num_chunks / (size_t)mpi_size); /* value should be the same on all procs */ + + /* After evenly distributing the blocks between processes, are there any + * leftover blocks for each individual process (round-robin)? + */ + leftover_blocks = (size_t)(chunk_fill_info->num_chunks % (size_t)mpi_size); + + /* Cast values to types needed by MPI */ + H5_CHECKED_ASSIGN(blocks, int, num_blocks, size_t); + H5_CHECKED_ASSIGN(leftover, int, leftover_blocks, size_t); - for (i = 1; i < io_count; i++) { - if (chunk_fill_info->chunk_info[i].chunk_size != chunk_fill_info->chunk_info[i - 1].chunk_size) { + /* Check if we have any chunks to write on this rank */ + if (num_blocks > 0 || (leftover && leftover > mpi_rank)) { + + if (NULL == (io_addrs = H5MM_malloc((size_t)(blocks + 1) * sizeof(*io_addrs)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O addresses vector"); + + if (NULL == (io_wbufs = H5MM_malloc((size_t)(blocks + 1) * sizeof(*io_wbufs)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O buffers vector"); + + } + + /* + * Perform initial scan of chunk info list to: + * - make sure that chunk addresses are monotonically non-decreasing + * - check if all blocks have the same length + */ + for (i = 1; i < chunk_fill_info->num_chunks; i++) { + if (chunk_fill_info->chunk_info[i].addr < chunk_fill_info->chunk_info[i - 1].addr) + need_sort = true; + + if (chunk_fill_info->chunk_info[i].chunk_size != chunk_fill_info->chunk_info[i - 1].chunk_size) all_same_block_len = false; - break; - } } + if (need_sort) + qsort(chunk_fill_info->chunk_info, chunk_fill_info->num_chunks, + sizeof(struct chunk_coll_fill_info), H5D__chunk_cmp_coll_fill_info); + if (all_same_block_len) { io_2sizes[0] = chunk_fill_info->chunk_info[0].chunk_size; io_2sizes[1] = 0; } else { - if (NULL == (io_sizes = H5MM_malloc(io_count * sizeof(*io_sizes)))) + if (NULL == (io_sizes = H5MM_malloc((size_t)(blocks + 1) * sizeof(*io_sizes)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O sizes vector"); } io_types[0] = H5FD_MEM_DRAW; io_types[1] = H5FD_MEM_NOLIST; - if (NULL == (io_addrs = H5MM_malloc(io_count * sizeof(*io_addrs)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O addresses vector"); - - if (NULL == (io_wbufs = H5MM_malloc(io_count * sizeof(*io_wbufs)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O buffers vector"); + for (i = 0; i < (size_t)blocks; i++) { + size_t idx = i + (size_t)(mpi_rank * blocks); - for (i = 0; i < io_count; i++) { - io_addrs[i] = chunk_fill_info->chunk_info[i].addr; + io_addrs[i] = chunk_fill_info->chunk_info[idx].addr; if (!all_same_block_len) - io_sizes[i] = chunk_fill_info->chunk_info[i].chunk_size; + io_sizes[i] = chunk_fill_info->chunk_info[idx].chunk_size; - if (chunk_fill_info->chunk_info[i].unfiltered_partial_chunk) + if (chunk_fill_info->chunk_info[idx].unfiltered_partial_chunk) io_wbufs[i] = partial_chunk_fill_buf; else io_wbufs[i] = fill_buf; } + if (leftover && leftover > mpi_rank) { + io_addrs[blocks] = + chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].addr; + + if (!all_same_block_len) + io_sizes[blocks] = chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].chunk_size; + + if (chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].unfiltered_partial_chunk) { + assert(partial_chunk_fill_buf); + io_wbufs[blocks] = partial_chunk_fill_buf; + } else + io_wbufs[blocks] = fill_buf; + + blocks++; + } + /* Get current transfer mode */ if (H5CX_get_io_xfer_mode(&prev_xfer_mode) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set transfer mode"); @@ -5609,10 +5684,15 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ if (H5CX_set_io_xfer_mode(H5FD_MPIO_COLLECTIVE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set transfer mode"); - if (H5F_shared_vector_write(H5F_SHARED(dset->oloc.file), io_count, io_types, io_addrs, + /* Barrier so processes don't race ahead */ + if (MPI_SUCCESS != (mpi_code = MPI_Barrier(mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code) + + if (H5F_shared_vector_write(H5F_SHARED(dset->oloc.file), (uint32_t)blocks, io_types, io_addrs, all_same_block_len ? io_2sizes : io_sizes, io_wbufs) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "vector write call failed"); + done: if (have_xfer_mode) /* Set transfer mode */ @@ -5626,6 +5706,20 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_collective_fill() */ +static int +H5D__chunk_cmp_coll_fill_info(const void *_entry1, const void *_entry2) +{ + const struct chunk_coll_fill_info *entry1; + const struct chunk_coll_fill_info *entry2; + + FUNC_ENTER_PACKAGE_NOERR + + entry1 = (const struct chunk_coll_fill_info *)_entry1; + entry2 = (const struct chunk_coll_fill_info *)_entry2; + + FUNC_LEAVE_NOAPI(H5_addr_cmp(entry1->addr, entry2->addr)) +} /* end H5D__chunk_cmp_coll_fill_info() */ + #endif /* H5_HAVE_PARALLEL */ /*------------------------------------------------------------------------- From 9059377477607e996a0442422368149611a19d90 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 05:38:32 +0000 Subject: [PATCH 07/11] Committing clang-format changes --- src/H5Dchunk.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index fd1a80aa4fc..ff5c339bab0 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -5554,15 +5554,15 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ size_t leftover_blocks; /* Number of leftover blocks to handle */ int blocks; /* converted to int for MPI */ int leftover; /* converted to int for MPI */ - H5FD_mpio_xfer_t prev_xfer_mode; /* Previous data xfer mode */ - bool have_xfer_mode = false; /* Whether the previous xffer mode has been retrieved */ + H5FD_mpio_xfer_t prev_xfer_mode; /* Previous data xfer mode */ + bool have_xfer_mode = false; /* Whether the previous xffer mode has been retrieved */ size_t i; /* Local index variable */ haddr_t *io_addrs = NULL; size_t *io_sizes = NULL; const void **io_wbufs = NULL; H5FD_mem_t io_types[2]; bool all_same_block_len = true; - bool need_sort = false; + bool need_sort = false; size_t io_2sizes[2]; herr_t ret_value = SUCCEED; /* Return value */ @@ -5605,15 +5605,15 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ H5_CHECKED_ASSIGN(blocks, int, num_blocks, size_t); H5_CHECKED_ASSIGN(leftover, int, leftover_blocks, size_t); - /* Check if we have any chunks to write on this rank */ - if (num_blocks > 0 || (leftover && leftover > mpi_rank)) { + /* Check if we have any chunks to write on this rank */ + if (num_blocks > 0 || (leftover && leftover > mpi_rank)) { if (NULL == (io_addrs = H5MM_malloc((size_t)(blocks + 1) * sizeof(*io_addrs)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O addresses vector"); + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate space for I/O addresses vector"); if (NULL == (io_wbufs = H5MM_malloc((size_t)(blocks + 1) * sizeof(*io_wbufs)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O buffers vector"); - } /* @@ -5630,8 +5630,8 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ } if (need_sort) - qsort(chunk_fill_info->chunk_info, chunk_fill_info->num_chunks, - sizeof(struct chunk_coll_fill_info), H5D__chunk_cmp_coll_fill_info); + qsort(chunk_fill_info->chunk_info, chunk_fill_info->num_chunks, sizeof(struct chunk_coll_fill_info), + H5D__chunk_cmp_coll_fill_info); if (all_same_block_len) { io_2sizes[0] = chunk_fill_info->chunk_info[0].chunk_size; @@ -5660,8 +5660,7 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ } if (leftover && leftover > mpi_rank) { - io_addrs[blocks] = - chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].addr; + io_addrs[blocks] = chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].addr; if (!all_same_block_len) io_sizes[blocks] = chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].chunk_size; @@ -5669,7 +5668,8 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ if (chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].unfiltered_partial_chunk) { assert(partial_chunk_fill_buf); io_wbufs[blocks] = partial_chunk_fill_buf; - } else + } + else io_wbufs[blocks] = fill_buf; blocks++; @@ -5692,7 +5692,6 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ all_same_block_len ? io_2sizes : io_sizes, io_wbufs) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "vector write call failed"); - done: if (have_xfer_mode) /* Set transfer mode */ From 69f61b5b87ea7a793b25aa212d9449ea59fa005e Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Wed, 8 Nov 2023 20:57:14 -0600 Subject: [PATCH 08/11] Modifications based on PR review comments. --- src/H5Dchunk.c | 32 ++++++++++++++++++++++++++++---- testpar/t_filters_parallel.c | 16 ++++++++-------- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index ff5c339bab0..0da01be4044 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -5538,7 +5538,7 @@ H5D__chunk_update_old_edge_chunks(H5D_t *dset, hsize_t old_dim[]) * * Purpose: Use MPIO selection vector I/O for writing fill chunks * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -5606,7 +5606,7 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ H5_CHECKED_ASSIGN(leftover, int, leftover_blocks, size_t); /* Check if we have any chunks to write on this rank */ - if (num_blocks > 0 || (leftover && leftover > mpi_rank)) { + if (num_blocks > 0 || leftover > mpi_rank) { if (NULL == (io_addrs = H5MM_malloc((size_t)(blocks + 1) * sizeof(*io_addrs)))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, @@ -5629,10 +5629,21 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ all_same_block_len = false; } + /* + * Note that we sort all of the chunks here, and not just a subset + * corresponding to this rank. We do this since we have found MPI I/O to work + * better when each rank writes blocks that are contiguous in the file, + * and by sorting the full list we maximize the chance of that happening. + */ if (need_sort) qsort(chunk_fill_info->chunk_info, chunk_fill_info->num_chunks, sizeof(struct chunk_coll_fill_info), H5D__chunk_cmp_coll_fill_info); + /* + * If all the chunks have the same length, use the compressed feature + * to store the size. + * Otherwise, allocate the array of sizes for storing chunk sizes. + */ if (all_same_block_len) { io_2sizes[0] = chunk_fill_info->chunk_info[0].chunk_size; io_2sizes[1] = 0; @@ -5642,9 +5653,17 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O sizes vector"); } + /* + * Since the type of all chunks is raw data, use the compressed feature + * to store the chunk type. + */ io_types[0] = H5FD_MEM_DRAW; io_types[1] = H5FD_MEM_NOLIST; + /* + * For the chunks corresponding to this rank, fill in the + * address, size and buf pointer for each chunk. + */ for (i = 0; i < (size_t)blocks; i++) { size_t idx = i + (size_t)(mpi_rank * blocks); @@ -5659,7 +5678,11 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ io_wbufs[i] = fill_buf; } - if (leftover && leftover > mpi_rank) { + /* + * For the leftover chunk corresponding to this rank, fill in the + * address, size and buf pointer for the chunk. + */ + if (leftover > mpi_rank) { io_addrs[blocks] = chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].addr; if (!all_same_block_len) @@ -5688,13 +5711,14 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ if (MPI_SUCCESS != (mpi_code = MPI_Barrier(mpi_comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code) + /* Perform the selection vector I/O for the chunks */ if (H5F_shared_vector_write(H5F_SHARED(dset->oloc.file), (uint32_t)blocks, io_types, io_addrs, all_same_block_len ? io_2sizes : io_sizes, io_wbufs) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "vector write call failed"); done: if (have_xfer_mode) - /* Set transfer mode */ + /* Restore transfer mode */ if (H5CX_set_io_xfer_mode(prev_xfer_mode) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set transfer mode"); diff --git a/testpar/t_filters_parallel.c b/testpar/t_filters_parallel.c index 5473eaa3d5d..148aa78db9b 100644 --- a/testpar/t_filters_parallel.c +++ b/testpar/t_filters_parallel.c @@ -547,8 +547,9 @@ verify_chunk_opt_status(size_t num_dsets, test_mode_t test_mode, bool any_io, bo /* Verify selection I/O mode on rank 0 */ if (mpi_rank == 0) { - /* No actual I/O performed, only reported I/O will be from allocation, even if "no" datasets were - * involved (num_dsets == 0 implies the call was expected to fail, but it fails after allocation). + /* No actual I/O performed, the only reported I/O will be from allocation which is vector I/O, + * even if "no" datasets were involved (num_dsets == 0 implies the call was expected to fail, + * but it fails after allocation). * Also if the test mode is mixed filtered and unfiltered and the call did not fail, then there * will always be an I/O callback made with raw data. This is because unfiltered datasets fall * back to scalar I/O when mixed with filtered, and scalar I/O reports an I/O call was made even @@ -556,7 +557,6 @@ verify_chunk_opt_status(size_t num_dsets, test_mode_t test_mode, bool any_io, bo * elements (because no elements were raw data), which is what happens when performing I/O on a * filtered dataset with no selection. Vector I/O does report an I/O call was made if passed a raw * data element of size 0, so this is consistent. */ - /* Changes: for allocation, the reported I/O is vector I/O */ if (!any_io) { if (did_alloc && (num_dsets > 0 && test_mode == USE_MULTIPLE_DATASETS_MIXED_FILTERED)) { VRFY((H5D_VECTOR_IO | H5D_SCALAR_IO) == actual_sel_io_mode_reduced, @@ -598,11 +598,11 @@ verify_chunk_opt_status(size_t num_dsets, test_mode_t test_mode, bool any_io, bo switch (test_mode) { case USE_SINGLE_DATASET: case USE_MULTIPLE_DATASETS: - /* Collective case with only filtered datasets. If we performed allocation then there - * should be scalar I/O for allocation in addition to vector I/O for the actual data. - * If we're reading from an unallocated dataset then there should be no actual I/O. - * Otherwise there should only be vector I/O. */ - /* Changes: for allocation, the reported I/O is vector I/O */ + /* Collective case with only filtered datasets. + * If we're reading from an unallocated dataset then there + * should be no actual I/O. + * Otherwise, only vector I/O is reported whether or not + * allocation happened. */ if (unalloc_read) VRFY(0 == actual_sel_io_mode_reduced, "verified actual selection I/O mode was 0 (no I/O)"); From d7786bf5ae9c8ea2b2841b00f2274d8f782bde33 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 03:10:17 +0000 Subject: [PATCH 09/11] Committing clang-format changes --- src/H5Dchunk.c | 20 ++++++++++---------- testpar/t_filters_parallel.c | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 0da01be4044..c8bad760e71 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -5629,17 +5629,17 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ all_same_block_len = false; } - /* - * Note that we sort all of the chunks here, and not just a subset + /* + * Note that we sort all of the chunks here, and not just a subset * corresponding to this rank. We do this since we have found MPI I/O to work - * better when each rank writes blocks that are contiguous in the file, + * better when each rank writes blocks that are contiguous in the file, * and by sorting the full list we maximize the chance of that happening. */ if (need_sort) qsort(chunk_fill_info->chunk_info, chunk_fill_info->num_chunks, sizeof(struct chunk_coll_fill_info), H5D__chunk_cmp_coll_fill_info); - /* + /* * If all the chunks have the same length, use the compressed feature * to store the size. * Otherwise, allocate the array of sizes for storing chunk sizes. @@ -5653,15 +5653,15 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate space for I/O sizes vector"); } - /* - * Since the type of all chunks is raw data, use the compressed feature + /* + * Since the type of all chunks is raw data, use the compressed feature * to store the chunk type. */ io_types[0] = H5FD_MEM_DRAW; io_types[1] = H5FD_MEM_NOLIST; - /* - * For the chunks corresponding to this rank, fill in the + /* + * For the chunks corresponding to this rank, fill in the * address, size and buf pointer for each chunk. */ for (i = 0; i < (size_t)blocks; i++) { @@ -5678,8 +5678,8 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_ io_wbufs[i] = fill_buf; } - /* - * For the leftover chunk corresponding to this rank, fill in the + /* + * For the leftover chunk corresponding to this rank, fill in the * address, size and buf pointer for the chunk. */ if (leftover > mpi_rank) { diff --git a/testpar/t_filters_parallel.c b/testpar/t_filters_parallel.c index 148aa78db9b..f60431b0dad 100644 --- a/testpar/t_filters_parallel.c +++ b/testpar/t_filters_parallel.c @@ -547,8 +547,8 @@ verify_chunk_opt_status(size_t num_dsets, test_mode_t test_mode, bool any_io, bo /* Verify selection I/O mode on rank 0 */ if (mpi_rank == 0) { - /* No actual I/O performed, the only reported I/O will be from allocation which is vector I/O, - * even if "no" datasets were involved (num_dsets == 0 implies the call was expected to fail, + /* No actual I/O performed, the only reported I/O will be from allocation which is vector I/O, + * even if "no" datasets were involved (num_dsets == 0 implies the call was expected to fail, * but it fails after allocation). * Also if the test mode is mixed filtered and unfiltered and the call did not fail, then there * will always be an I/O callback made with raw data. This is because unfiltered datasets fall @@ -598,8 +598,8 @@ verify_chunk_opt_status(size_t num_dsets, test_mode_t test_mode, bool any_io, bo switch (test_mode) { case USE_SINGLE_DATASET: case USE_MULTIPLE_DATASETS: - /* Collective case with only filtered datasets. - * If we're reading from an unallocated dataset then there + /* Collective case with only filtered datasets. + * If we're reading from an unallocated dataset then there * should be no actual I/O. * Otherwise, only vector I/O is reported whether or not * allocation happened. */ From 22a2bcb766ea4c4a9e299219c01e70bd790af0fe Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Mon, 13 Nov 2023 15:29:43 -0600 Subject: [PATCH 10/11] Add a test to verify the fix for the illegal reference problem in H5FD__mpio_write_vector(). --- testpar/t_vfd.c | 300 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 293 insertions(+), 7 deletions(-) diff --git a/testpar/t_vfd.c b/testpar/t_vfd.c index ac524ac8647..82cc9aee656 100644 --- a/testpar/t_vfd.c +++ b/testpar/t_vfd.c @@ -40,13 +40,14 @@ const char *FILENAMES[] = {"mpio_vfd_test_file_0", /*0*/ "mpio_vfd_test_file_4", /*4*/ "mpio_vfd_test_file_5", /*5*/ "mpio_vfd_test_file_6", /*6*/ - "subfiling_vfd_test_file_0", /*7*/ - "subfiling_vfd_test_file_1", /*8*/ - "subfiling_vfd_test_file_2", /*9*/ - "subfiling_vfd_test_file_3", /*10*/ - "subfiling_vfd_test_file_4", /*11*/ - "subfiling_vfd_test_file_5", /*12*/ - "subfiling_vfd_test_file_6", /*13*/ + "mpio_vfd_test_file_7", /*7*/ + "subfiling_vfd_test_file_0", /*8*/ + "subfiling_vfd_test_file_1", /*9*/ + "subfiling_vfd_test_file_2", /*10*/ + "subfiling_vfd_test_file_3", /*11*/ + "subfiling_vfd_test_file_4", /*12*/ + "subfiling_vfd_test_file_5", /*13*/ + "subfiling_vfd_test_file_6", /*14*/ NULL}; /* File Test Images @@ -100,6 +101,8 @@ static unsigned vector_write_test_6(int file_name_id, int mpi_rank, int mpi_size H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); static unsigned vector_write_test_7(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); +static unsigned vector_write_test_8(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name); /* * Tests for selection I/O: * They are derived from test_selection_io() in test/vfd.c and modified for parallel testing. @@ -4159,6 +4162,282 @@ vector_write_test_7(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer } /* vector_write_test_7() */ + +/*------------------------------------------------------------------------- + * Function: vector_write_test_8() + * + * Purpose: This test is to verify the fix for the following problem + * in H5FD__mpio_write_vector when calculating max_addr: + * --illegal reference occurs when referencing the s_sizes array + * with due to exceeding the length of the + * size array which uses the compressed feature. + * + * 1) Open the test file with the specified VFD, and set + * the eoa. + * + * 2) Set the test file in a known state by writing zeros + * to all bytes in the test file. Since we have already + * tested this, do this via a vector write of zero_fi_buf. + * + * 3) Barrier + * + * 4) For each rank, define base_index equal to: + * + * mpi_rank * INTS_PER_RANK + * + * and define base_addr equal to + * + * base_index * sizeof(int32_t). + * + * Setup a vector of length INTS_PER_RANK - 1. + * Set up the size array with the compressed feature: + * --The first element has size (2 * sizeof(int32_t)) + * --The second and third elements are of size sizeof(int32_t) + * --The fourth element is zero. + * Set up addrs and bufs accordingly. + * + * Write the vector. + * + * 5) Barrier + * + * 6) On each rank, read the entire file into the read_fi_buf, + * and compare against increasing_fi_buf. + * Report failure if any differences are detected. + * + * 7) Close the test file. On rank 0, delete the test file. + * + * Return: false on success, true if any errors are detected. + * + *------------------------------------------------------------------------- + */ +static unsigned +vector_write_test_8(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer_t xfer_mode, + H5FD_mpio_collective_opt_t coll_opt_mode, const char *vfd_name) +{ + const char *fcn_name = "vector_write_test_8()"; + char test_title[120]; + char filename[512]; + haddr_t eoa; + haddr_t base_addr; + bool show_progress = false; + hid_t fapl_id = H5I_INVALID_HID; /* file access property list ID */ + hid_t dxpl_id = H5I_INVALID_HID; /* data access property list ID */ + H5FD_t *lf = NULL; /* VFD struct ptr */ + int cp = 0; + int i; + int base_index; + uint32_t count = 0; + size_t sizes[4]; + H5FD_mem_t types[2]; + + haddr_t *tt_addrs = NULL; /* For storing addrs */ + const void **tt_bufs = NULL; /* For storing buf pointers */ + + pass = true; + + if (mpi_rank == 0) { + + if (xfer_mode == H5FD_MPIO_INDEPENDENT) { + + snprintf(test_title, sizeof(test_title), "parallel vector write test 8 -- %s / independent", + vfd_name); + } + else if (coll_opt_mode == H5FD_MPIO_INDIVIDUAL_IO) { + + snprintf(test_title, sizeof(test_title), "parallel vector write test 8 -- %s / col op / ind I/O", + vfd_name); + } + else { + + assert(coll_opt_mode == H5FD_MPIO_COLLECTIVE_IO); + + snprintf(test_title, sizeof(test_title), "parallel vector write test 8 -- %s / col op / col I/O", + vfd_name); + } + + TESTING(test_title); + } + + show_progress = ((show_progress) && (mpi_rank == 0)); + + if (show_progress) + fprintf(stdout, "\n%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Allocate local buffers for addrs and bufs, + open the test file with the specified VFD, set the eoa, and setup the dxpl */ + if (pass) { + + tt_addrs = (haddr_t *)malloc((INTS_PER_RANK) * sizeof(haddr_t *)); + tt_bufs = (const void **)malloc((INTS_PER_RANK) * sizeof(void *)); + + if (tt_addrs == NULL || tt_bufs == NULL) { + pass = false; + failure_mssg = "Can't allocate local addrs and bufs buffers."; + } + + if (pass) { + eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); + + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, + &fapl_id, &dxpl_id); + } + } + + if (show_progress) + fprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 2) Using rank zero, write the entire negative_fi_buf to + * the file. + */ + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (mpi_rank == 0) { + + if (H5FDwrite(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)zero_fi_buf) < 0) { + + pass = false; + failure_mssg = "H5FDwrite() on rank 0 failed.\n"; + } + } + } + + /* 3) Barrier */ + MPI_Barrier(comm); + + if (show_progress) + fprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 4) For each rank, define base_index equal to: + * + * mpi_rank * INTS_PER_RANK + * + * and define base_addr equal to + * + * base_index * sizeof(int32_t). + * + * Set up the array of sizes and types with the compressed feature + * as described in the routine header description. + */ + if (pass) { + + base_index = (mpi_rank * INTS_PER_RANK); + base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t); + + count = INTS_PER_RANK - 1; + + types[0] = H5FD_MEM_DRAW; + types[1] = H5FD_MEM_NOLIST; + + sizes[0] = 2 * sizeof(int32_t); + sizes[1] = sizeof(int32_t); + sizes[2] = sizeof(int32_t); + sizes[3] = 0; + + tt_addrs[0] = base_addr; + tt_bufs[0] = (const void *)(&(increasing_fi_buf[base_index])); + + tt_addrs[0] = base_addr; + base_index += 2; + base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t); + + for (i = 1; i < (INTS_PER_RANK - 1); i++) { + + tt_addrs[i] = base_addr + ((haddr_t)(i - 1) * (haddr_t)sizeof(int32_t)); + tt_bufs[i] = (const void *)(&(increasing_fi_buf[base_index + (i - 1)])); + } + + if (H5FDwrite_vector(lf, dxpl_id, count, types, tt_addrs, sizes, tt_bufs) < 0) { + + pass = false; + failure_mssg = "H5FDwrite_vector() failed (1).\n"; + } + } + + if (show_progress) + fprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 5) Barrier */ + MPI_Barrier(comm); + + if (show_progress) + fprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 6) On each rank, read the entire file into the read_fi_buf, + * and compare against increasing_fi_buf + * Report failure if any differences are detected. + */ + if (pass) { + + size_t image_size = (size_t)mpi_size * (size_t)INTS_PER_RANK * sizeof(int32_t); + + if (H5FDread(lf, H5FD_MEM_DRAW, H5P_DEFAULT, (haddr_t)0, image_size, (void *)read_fi_buf) < 0) { + + pass = false; + failure_mssg = "H5FDread() failed.\n"; + } + + for (i = 0; ((pass) && (i < mpi_size * INTS_PER_RANK)); i++) { + + if (read_fi_buf[i] != increasing_fi_buf[i]) { + + pass = false; + failure_mssg = "unexpected data read from file (1)"; + } + } + } /* end if */ + + if (show_progress) + fprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 7) Barrier */ + MPI_Barrier(comm); + + if (show_progress) + fprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 8) Close the test file and delete it (on rank 0 only). + * Close FAPL and DXPL. + */ + takedown_vfd_test_file(mpi_rank, filename, &lf, &fapl_id, &dxpl_id); + + /* Free the local buffers */ + if (tt_addrs) { + free(tt_addrs); + tt_addrs = NULL; + } + + if (tt_bufs) { + free(tt_bufs); + tt_bufs = NULL; + } + + if (show_progress) + fprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* report results */ + if (mpi_rank == 0) { + + if (pass) { + + PASSED(); + } + else { + + H5_FAILED(); + + if ((disp_failure_mssgs) || (show_progress)) { + fprintf(stdout, "%s: failure_mssg = \"%s\"\n", fcn_name, failure_mssg); + } + } + } + + return (!pass); + +} /* vector_write_test_8() */ + + static void test_vector_io(int mpi_rank, int mpi_size) { @@ -4249,6 +4528,13 @@ test_vector_io(int mpi_rank, int mpi_size) nerrs += vector_write_test_7(6, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + nerrs += + vector_write_test_8(7, mpi_rank, mpi_size, H5FD_MPIO_INDEPENDENT, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_8(7, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_INDIVIDUAL_IO, "mpio"); + nerrs += + vector_write_test_8(7, mpi_rank, mpi_size, H5FD_MPIO_COLLECTIVE, H5FD_MPIO_COLLECTIVE_IO, "mpio"); + MPI_Barrier(comm); #ifdef H5_HAVE_SUBFILING_VFD From 22fdd885679719d61899b1ee239991fd0ed84b2d Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 21:32:59 +0000 Subject: [PATCH 11/11] Committing clang-format changes --- testpar/t_vfd.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/testpar/t_vfd.c b/testpar/t_vfd.c index 82cc9aee656..79b7e0149d8 100644 --- a/testpar/t_vfd.c +++ b/testpar/t_vfd.c @@ -4162,14 +4162,13 @@ vector_write_test_7(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer } /* vector_write_test_7() */ - /*------------------------------------------------------------------------- * Function: vector_write_test_8() * * Purpose: This test is to verify the fix for the following problem * in H5FD__mpio_write_vector when calculating max_addr: - * --illegal reference occurs when referencing the s_sizes array - * with due to exceeding the length of the + * --illegal reference occurs when referencing the s_sizes array + * with due to exceeding the length of the * size array which uses the compressed feature. * * 1) Open the test file with the specified VFD, and set @@ -4230,8 +4229,8 @@ vector_write_test_8(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer size_t sizes[4]; H5FD_mem_t types[2]; - haddr_t *tt_addrs = NULL; /* For storing addrs */ - const void **tt_bufs = NULL; /* For storing buf pointers */ + haddr_t *tt_addrs = NULL; /* For storing addrs */ + const void **tt_bufs = NULL; /* For storing buf pointers */ pass = true; @@ -4268,8 +4267,8 @@ vector_write_test_8(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer if (pass) { tt_addrs = (haddr_t *)malloc((INTS_PER_RANK) * sizeof(haddr_t *)); - tt_bufs = (const void **)malloc((INTS_PER_RANK) * sizeof(void *)); - + tt_bufs = (const void **)malloc((INTS_PER_RANK) * sizeof(void *)); + if (tt_addrs == NULL || tt_bufs == NULL) { pass = false; failure_mssg = "Can't allocate local addrs and bufs buffers."; @@ -4278,8 +4277,8 @@ vector_write_test_8(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer if (pass) { eoa = (haddr_t)mpi_size * (haddr_t)INTS_PER_RANK * (haddr_t)(sizeof(int32_t)); - setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, &lf, - &fapl_id, &dxpl_id); + setup_vfd_test_file(file_name_id, filename, mpi_size, xfer_mode, coll_opt_mode, vfd_name, eoa, + &lf, &fapl_id, &dxpl_id); } } @@ -4325,7 +4324,7 @@ vector_write_test_8(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer base_index = (mpi_rank * INTS_PER_RANK); base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t); - count = INTS_PER_RANK - 1; + count = INTS_PER_RANK - 1; types[0] = H5FD_MEM_DRAW; types[1] = H5FD_MEM_NOLIST; @@ -4337,14 +4336,14 @@ vector_write_test_8(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer tt_addrs[0] = base_addr; tt_bufs[0] = (const void *)(&(increasing_fi_buf[base_index])); - + tt_addrs[0] = base_addr; base_index += 2; - base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t); + base_addr = (haddr_t)base_index * (haddr_t)sizeof(int32_t); for (i = 1; i < (INTS_PER_RANK - 1); i++) { - tt_addrs[i] = base_addr + ((haddr_t)(i - 1) * (haddr_t)sizeof(int32_t)); + tt_addrs[i] = base_addr + ((haddr_t)(i - 1) * (haddr_t)sizeof(int32_t)); tt_bufs[i] = (const void *)(&(increasing_fi_buf[base_index + (i - 1)])); } @@ -4437,7 +4436,6 @@ vector_write_test_8(int file_name_id, int mpi_rank, int mpi_size, H5FD_mpio_xfer } /* vector_write_test_8() */ - static void test_vector_io(int mpi_rank, int mpi_size) {