From 8119b9b439ae157d83137a175f48820792cf13a8 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Wed, 19 May 2021 11:46:08 -0500 Subject: [PATCH 01/12] Hdf5 1 12 doxygen version correction (#658) * OESS-98 fix tools test for plugins * sync fork * Merge of changes from dev * Move problem option to bottom of the list until fixed * HDFFV-11106 - fix parsing optional args * HDFFV-11106 add note * grammer fix * Whitespace after clang formatting * Undo format version 11 changes * Update check to working version * Merge workflow and minor changes from develop * Update supported platforms * PR#3 merge from develop * Merge gcc 10 diagnostics option from develop * Merge #318 OSX changes from develop * Merge serval small changes from dev * fix typo * Minor non-space formatting changes * GH #386 copyright corrections for java folder * revert because logic requires false return * Merges from develop * Remove case statement for H5I_EVENTSET * Correct call with versioning * Remove tabs * Double underscore change * Merges from develop Remove more underscores from header guards * Merge #380 from develop * Correct date entry * Split format source and commit changes on repo push * remove pre-split setting * Change windows TS to use older VS. * HDFFV-11212 JNI export util and Javadoc * Suggested review changes * Another change found * Committing clang-format changes * Some Javadoc warning fixes * Committing clang-format changes * Updated javadoc fixes * HDFFV-11228/9 merges from develop * remove obsolete debug comment * Fix conflict * HDFFV-11229 merge changes from develop * HDFFV-11229 merge second compare from develop * HDFFV-11229 fix reference file * HDFFV-11229 update autotools test script for two ref files * HDFFV-11229 merge dev changes for long double display in tools * Committing clang-format changes * Update with changes from develop * Add "option" command for clang options * Rework CMake add_custom to use the BYPRODUCTS argument Update pkgconfig scripts for parallel builds. Fix install COPYING file reference. Remove unused round defines. Change CMake default setting of BUILD_CPP to off. * Whitespace changes * Rework CMake add_custom to use the BYPRODUCTS argument * Revert CMake configure checks for round defines * With VS 2015 minimum strdup is supported * Doxygen comments merged from develop * doxygen build updates * Correct version string for map functions * TRILAB-227 and tools debug merge from develop * TRILAB-227 Enable test * Quote subset args * Use MATCHES in compiler id compare, merge from dev * Revert test enable * Add file to list * doxygen version errors * if blocks needed for build modes * Update list of test platforms Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- release_docs/RELEASE.txt | 17 ++++++++++++----- src/H5Spublic.h | 28 ++++++++++++++-------------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 32d0be175ff..9502375a69c 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -881,15 +881,22 @@ The following platforms are not supported but have been tested for this release. #1 SMP ppc64le GNU/Linux gcc/7.3 (ray) xl/2016,2019 - Fedora33 5.10.10-200.fc33.x86_64 - #1 SMP x86_64 GNU/Linux GNU gcc (GCC) 10.2.1 20201125 (Red Hat 10.2.1-9) - GNU Fortran (GCC) 10.2.1 20201125 (Red Hat 10.2.1-9) + Fedora33 5.11.18-200.fc33.x86_64 + #1 SMP x86_64 GNU/Linux GNU gcc (GCC) 10.3.1 20210422 (Red Hat 10.3.1-1) + GNU Fortran (GCC) 10.3.1 20210422 (Red Hat 10.3.1-1) clang version 11.0.0 (Fedora 11.0.0-2.fc33) (cmake and autotools) - Ubuntu20.10 5.8.0-41-generic-x86_64 - #46-Ubuntu SMP x86_64 GNU/Linux GNU gcc (GCC) 10.2.0-13ubuntu1 + Ubuntu20.04 5.8.0-53-generic-x86_64 + #60~20.04-Ubuntu SMP x86_64 GNU/Linux GNU gcc (GCC) 9.3.0-17ubuntu1 + GNU Fortran (GCC) 9.3.0-17ubuntu1 + clang version 10.0.0-4ubuntu1 + (cmake and autotools) + + Ubuntu20.10 5.8.0-53-generic-x86_64 + #60-Ubuntu SMP x86_64 GNU/Linux GNU gcc (GCC) 10.2.0-13ubuntu1 GNU Fortran (GCC) 10.2.0-13ubuntu1 + Ubuntu clang version 11.0.0-2 (cmake and autotools) SUSE15sp2 5.3.18-22-default diff --git a/src/H5Spublic.h b/src/H5Spublic.h index fd85dcc40cf..331e5ee8070 100644 --- a/src/H5Spublic.h +++ b/src/H5Spublic.h @@ -70,7 +70,7 @@ typedef enum H5S_class_t { /* Different ways of combining selections */ typedef enum H5S_seloper_t { H5S_SELECT_NOOP = -1, /* error */ - H5S_SELECT_SET = 0, /* Select "set" operation */ + H5S_SELECT_SET = 0, /* Select "set" operation */ H5S_SELECT_OR, /* Binary "or" operation for hyperslabs * (add new selection to existing selection) * Original region: AAAAAAAAAA @@ -108,12 +108,12 @@ typedef enum H5S_seloper_t { /* Enumerated type for the type of selection */ typedef enum { - H5S_SEL_ERROR = -1, /* Error */ - H5S_SEL_NONE = 0, /* Nothing selected */ - H5S_SEL_POINTS = 1, /* Points / elements selected */ + H5S_SEL_ERROR = -1, /* Error */ + H5S_SEL_NONE = 0, /* Nothing selected */ + H5S_SEL_POINTS = 1, /* Points / elements selected */ H5S_SEL_HYPERSLABS = 2, /* Hyperslab selected */ - H5S_SEL_ALL = 3, /* Entire extent selected */ - H5S_SEL_N /*THIS MUST BE LAST */ + H5S_SEL_ALL = 3, /* Entire extent selected */ + H5S_SEL_N /*THIS MUST BE LAST */ } H5S_sel_type; #ifdef __cplusplus @@ -164,7 +164,7 @@ H5_DLL herr_t H5Sclose(hid_t space_id); * composing the entire current extent). If either \p stride or * \p block is NULL, then it will be set to \p 1. * - * \since 1.12.0 + * \since 1.10.6 * */ H5_DLL hid_t H5Scombine_hyperslab(hid_t space_id, H5S_seloper_t op, const hsize_t start[], @@ -187,7 +187,7 @@ H5_DLL hid_t H5Scombine_hyperslab(hid_t space_id, H5S_seloper_t op, const hsize_ * from \p space1_id is copied for the dataspace extent of the * newly created dataspace. * - * \since 1.12.0 + * \since 1.10.6 * */ H5_DLL hid_t H5Scombine_select(hid_t space1_id, H5S_seloper_t op, hid_t space2_id); @@ -788,7 +788,7 @@ H5_DLL htri_t H5Sis_simple(hid_t space_id); * \p space2_id. The first selection is modified to contain the * result of \p space1_id operated on by \p space2_id. * - * \since 1.12.0 + * \since 1.10.6 * */ H5_DLL herr_t H5Smodify_select(hid_t space1_id, H5S_seloper_t op, hid_t space2_id); @@ -933,7 +933,7 @@ H5_DLL herr_t H5Ssel_iter_reset(hid_t sel_iter_id, hid_t space_id); * * \note This can be useful for VOL developers to implement chunked datasets. * - * \since 1.12.0 + * \since 1.10.6 */ H5_DLL herr_t H5Sselect_adjust(hid_t spaceid, const hssize_t *offset); /** @@ -970,7 +970,7 @@ H5_DLL herr_t H5Sselect_all(hid_t spaceid); * offset) from the source dataspace \p src_id to the destination * dataspace \p dst_id. * - * \since 1.12.0 + * \since 1.10.6 * */ H5_DLL herr_t H5Sselect_copy(hid_t dst_id, hid_t src_id); @@ -1198,7 +1198,7 @@ H5_DLL herr_t H5Sselect_hyperslab(hid_t space_id, H5S_seloper_t op, const hsize_ * \note Assumes that \p start & \p end block bounds are inclusive, so * \p start == \p end value is OK. * - * \since 1.12.0 + * \since 1.10.6 * */ H5_DLL htri_t H5Sselect_intersect_block(hid_t space_id, const hsize_t *start, const hsize_t *end); @@ -1237,7 +1237,7 @@ H5_DLL herr_t H5Sselect_none(hid_t spaceid); * into a third selection.This can be useful for VOL developers to * implement chunked or virtual datasets. * - * \since 1.12.0 + * \since 1.10.6 * */ H5_DLL hid_t H5Sselect_project_intersection(hid_t src_space_id, hid_t dst_space_id, @@ -1258,7 +1258,7 @@ H5_DLL hid_t H5Sselect_project_intersection(hid_t src_space_id, hid_t dst_space_ * This is primarily used for reading the entire selection in * one swoop. * - * \since 1.12.0 + * \since 1.10.6 * */ H5_DLL htri_t H5Sselect_shape_same(hid_t space1_id, hid_t space2_id); From 0a71283300491b7dd1f9ad8fedbdfee38908e946 Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Wed, 19 May 2021 15:09:55 -0500 Subject: [PATCH 02/12] Attempt format uniformity in RELEASE.txt. --- release_docs/RELEASE.txt | 521 +++++++++++++++++++-------------------- 1 file changed, 255 insertions(+), 266 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 9502375a69c..312f2b6e7e5 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -49,35 +49,35 @@ New Features ------------- - Added an option to make the global thread-safe lock a recursive R/W lock - Prior to this release, the HDF5 library supported multi-threaded - applications by placing a recursive global lock on the entire library, - thus allowing only one thread into the library at a time. + Prior to this release, the HDF5 library supported multi-threaded + applications by placing a recursive global lock on the entire library, + thus allowing only one thread into the library at a time. - While this is still the default, the library can now be built with the - recursive global lock replaced with a recursive read / write (R/W) lock - that allows recursive writer locks. + While this is still the default, the library can now be built with the + recursive global lock replaced with a recursive read / write (R/W) lock + that allows recursive writer locks. - Currently, this change results in no functional change in the HDF5 - library, as all threads will have to acquire a write lock on entry, and - thus obtain exclusive access to the HDF5 library as before. However, the - addition of the recursive R/W lock is a prerequisite for further work - directed at allowing some subset of the HDF5 API calls to enter the - library with read locks. + Currently, this change results in no functional change in the HDF5 + library, as all threads will have to acquire a write lock on entry, and + thus obtain exclusive access to the HDF5 library as before. However, the + addition of the recursive R/W lock is a prerequisite for further work + directed at allowing some subset of the HDF5 API calls to enter the + library with read locks. - CMake: HDF5_USE_RECURSIVE_RW_LOCKS (default: OFF, advanced) + CMake: HDF5_USE_RECURSIVE_RW_LOCKS (default: OFF, advanced) - Autotools: --enable-recursive-rw-locks [default=no] + Autotools: --enable-recursive-rw-locks [default=no] - This feature only works with Pthreads. Win32 threads are not supported. + This feature only works with Pthreads. Win32 threads are not supported. - (DER - 2021/05/10) + (2021/05/10) - CMake no longer builds the C++ library by default HDF5_BUILD_CPP_LIB now defaults to OFF, which is in line with the Autotools build defaults. - (DER - 2021/04/20) + (2021/04/20) - Removal of pre-VS2015 work-arounds @@ -91,7 +91,7 @@ New Features * va_copy * struct timespec - (DER - 2021/03/22) + (2021/03/22) - Add CMake variable HDF5_LIB_INFIX @@ -100,14 +100,14 @@ New Features This name is used in packages on debian based systems. (see https://packages.debian.org/jessie/amd64/libhdf5-openmpi-8/filelist) - (barcode - 2021/03/22) + (2021/03/22) - On macOS, Universal Binaries can now be built, allowing native execution on both Intel and Apple Silicon (ARM) based Macs. To do so, set CMAKE_OSX_ARCHITECTURES="x86_64;arm64" - (SAM - 2021/02/07, https://github.com/HDFGroup/hdf5/issues/311) + (2021/02/07, https://github.com/HDFGroup/hdf5/issues/311) - Added a configure-time option to control certain compiler warnings diagnostics @@ -126,23 +126,23 @@ New Features CMake: HDF5_ENABLE_BUILD_DIAGS - (ADB - 2021/02/05, HDFFV-11213) + (2021/02/05, HDFFV-11213) - CMake option to build the HDF filter plugins project as an external project - The HDF filter plugins project is a collection of registered compression - filters that can be dynamically loaded when needed to access data stored - in a hdf5 file. This CMake-only option allows the plugins to be built and - distributed with the hdf5 library and tools. Like the options for szip and - zlib, either a tgz file or a git repository can be specified for the source. + The HDF filter plugins project is a collection of registered compression + filters that can be dynamically loaded when needed to access data stored + in a hdf5 file. This CMake-only option allows the plugins to be built and + distributed with the hdf5 library and tools. Like the options for szip and + zlib, either a tgz file or a git repository can be specified for the source. - The option was refactored to use the CMake FetchContent process. This allows - more control over the filter targets, but required external project command - options to be moved to a CMake include file, HDF5PluginCache.cmake. Also - enabled the filter examples to be used as tests for operation of the - filter plugins. + The option was refactored to use the CMake FetchContent process. This allows + more control over the filter targets, but required external project command + options to be moved to a CMake include file, HDF5PluginCache.cmake. Also + enabled the filter examples to be used as tests for operation of the + filter plugins. - (ADB - 2020/12/10, OESS-98) + (2020/12/10, OESS-98) - FreeBSD Autotools configuration now defaults to 'cc' and 'c++' compilers @@ -156,7 +156,7 @@ New Features and g++ options will now be set if that compiler is being used (an omission from the former functionality). - (DER - 2020/11/28, HDFFV-11193) + (2020/11/28, HDFFV-11193) - Fixed POSIX problems when building w/ gcc on Solaris @@ -169,7 +169,7 @@ New Features differs slightly from the gcc norm, where we set the standard to c99 and manually set POSIX #define symbols. - (DER - 2020/11/25, HDFFV-11191) + (2020/11/25, HDFFV-11191) - Added a configure-time option to consider certain compiler warnings as errors @@ -191,120 +191,120 @@ New Features CMake: HDF5_ENABLE_WARNINGS_AS_ERRORS - (DER - 2020/11/23, HDFFV-11189) + (2020/11/23, HDFFV-11189) - Autotools and CMake target added to produce doxygen generated documentation - The default is OFF or disabled. - Autoconf option is '--enable-doxygen' + The default is OFF or disabled. + Autoconf option is '--enable-doxygen' autotools make target is 'doxygen' and will build all doxygen targets - CMake configure option is 'HDF5_BUILD_DOC'. + CMake configure option is 'HDF5_BUILD_DOC'. CMake target is 'doxygen' for all available doxygen targets CMake target is 'hdf5lib_doc' for the src subdirectory - (ADB - 2020/11/13) + (2020/11/13) - CMake option to use MSVC naming conventions with MinGW - HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions - when using a MinGW toolchain + HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions + when using a MinGW toolchain - (xan - 2020/10/30) + (2020/10/30) - CMake option to statically link gcc libs with MinGW - HDF5_MINGW_STATIC_GCC_LIBS allows to statically link libg/libstdc++ - with the MinGW toolchain + HDF5_MINGW_STATIC_GCC_LIBS allows to statically link libg/libstdc++ + with the MinGW toolchain - (xan - 2020/10/30) + (2020/10/30) - CMake option to build the HDF filter plugins project as an external project - The HDF filter plugins project is a collection of registered compression - filters that can be dynamically loaded when needed to access data stored - in a hdf5 file. This CMake-only option allows the plugins to be built and - distributed with the hdf5 library and tools. Like the options for szip and - zlib, either a tgz file or a git repository can be specified for the source. + The HDF filter plugins project is a collection of registered compression + filters that can be dynamically loaded when needed to access data stored + in a hdf5 file. This CMake-only option allows the plugins to be built and + distributed with the hdf5 library and tools. Like the options for szip and + zlib, either a tgz file or a git repository can be specified for the source. - The necessary options are (see the INSTALL_CMake.txt file): + The necessary options are (see the INSTALL_CMake.txt file): HDF5_ENABLE_PLUGIN_SUPPORT PLUGIN_TGZ_NAME or PLUGIN_GIT_URL - There are more options necessary for various filters and the plugin project - documents should be referenced. + There are more options necessary for various filters and the plugin project + documents should be referenced. - (ADB - 2020/10/16, OESS-98) + (2020/10/16, OESS-98) - Added CMake option to format source files - HDF5_ENABLE_FORMATTERS option will enable creation of targets using the - pattern - HDF5_*_SRC_FORMAT - where * corresponds to the source folder - or tool folder. All sources can be formatted by executing the format target; - make format + HDF5_ENABLE_FORMATTERS option will enable creation of targets using the + pattern - HDF5_*_SRC_FORMAT - where * corresponds to the source folder + or tool folder. All sources can be formatted by executing the format target; + make format - (ADB - 2020/09/24) + (2020/09/24) - CMake option to link the generated Fortran MOD files into the include directory. - The Fortran generation of MOD files by a Fortran compile can produce - different binary files between SHARED and STATIC compiles with different - compilers and/or different platforms. Note that it has been found that - different versions of Fortran compilers will produce incompatible MOD - files. Currently, CMake will locate these MOD files in subfolders of - the include directory and add that path to the Fortran library target - in the CMake config file, which can be used by the CMake find library - process. For other build systems using the binary from a CMake install, - a new CMake configuration can be used to copy the pre-chosen version - of the Fortran MOD files into the install include directory. - - The default will depend on the configuration of - BUILD_STATIC_LIBS and BUILD_SHARED_LIBS: + The Fortran generation of MOD files by a Fortran compile can produce + different binary files between SHARED and STATIC compiles with different + compilers and/or different platforms. Note that it has been found that + different versions of Fortran compilers will produce incompatible MOD + files. Currently, CMake will locate these MOD files in subfolders of + the include directory and add that path to the Fortran library target + in the CMake config file, which can be used by the CMake find library + process. For other build systems using the binary from a CMake install, + a new CMake configuration can be used to copy the pre-chosen version + of the Fortran MOD files into the install include directory. + + The default will depend on the configuration of + BUILD_STATIC_LIBS and BUILD_SHARED_LIBS: YES YES Default to SHARED YES NO Default to STATIC NO YES Default to SHARED NO NO Default to SHARED - The defaults can be overridden by setting the config option + The defaults can be overridden by setting the config option HDF5_INSTALL_MOD_FORTRAN to one of NO, SHARED, or STATIC - (ADB - 2020/07/09, HDFFV-11116) + (2020/07/09, HDFFV-11116) - CMake option to use AEC (open source SZip) library instead of SZip - The open source AEC library is a replacement library for SZip. In - order to use it for hdf5 the libaec CMake source was changed to add - "-fPIC" and exclude test files. Autotools does not build the - compression libraries within hdf5 builds. New option USE_LIBAEC is - required to compensate for the different files produced by AEC build. + The open source AEC library is a replacement library for SZip. In + order to use it for hdf5 the libaec CMake source was changed to add + "-fPIC" and exclude test files. Autotools does not build the + compression libraries within hdf5 builds. New option USE_LIBAEC is + required to compensate for the different files produced by AEC build. - (ADB - 2020/04/22, OESS-65) + (2020/04/22, OESS-65) - CMake ConfigureChecks.cmake file now uses CHECK_STRUCT_HAS_MEMBER - Some handcrafted tests in HDFTests.c has been removed and the CMake - CHECK_STRUCT_HAS_MEMBER module has been used. + Some handcrafted tests in HDFTests.c has been removed and the CMake + CHECK_STRUCT_HAS_MEMBER module has been used. - (ADB - 2020/03/24, TRILAB-24) + (2020/03/24, TRILAB-24) - Both build systems use same set of warnings flags - GNU C, C++ and gfortran warnings flags were moved to files in a config - sub-folder named gnu-warnings. Flags that only are available for a specific - version of the compiler are in files named with that version. - Clang C warnings flags were moved to files in a config sub-folder - named clang-warnings. - Intel C, Fortran warnings flags were moved to files in a config sub-folder - named intel-warnings. + GNU C, C++ and gfortran warnings flags were moved to files in a config + sub-folder named gnu-warnings. Flags that only are available for a specific + version of the compiler are in files named with that version. + Clang C warnings flags were moved to files in a config sub-folder + named clang-warnings. + Intel C, Fortran warnings flags were moved to files in a config sub-folder + named intel-warnings. - There are flags in named "error-xxx" files with warnings that may - be promoted to errors. Some source files may still need fixes. + There are flags in named "error-xxx" files with warnings that may + be promoted to errors. Some source files may still need fixes. - There are also pairs of files named "developer-xxx" and "no-developer-xxx" - that are chosen by the CMake option:HDF5_ENABLE_DEV_WARNINGS or the - configure option:--enable-developer-warnings. + There are also pairs of files named "developer-xxx" and "no-developer-xxx" + that are chosen by the CMake option:HDF5_ENABLE_DEV_WARNINGS or the + configure option:--enable-developer-warnings. - In addition, CMake no longer applies these warnings for examples. + In addition, CMake no longer applies these warnings for examples. - (ADB - 2020/03/24, TRILAB-192) + (2020/03/24, TRILAB-192) Library: @@ -324,7 +324,7 @@ New Features The Doxygen documentation has been updated and passing values larger than UINT32_MAX for size_hint will now produce a normal HDF5 error. - (DER - 2021/04/29, HDFFV-11241) + (2021/04/29, HDFFV-11241) - H5Pset_fapl_log() no longer crashes when passed an invalid fapl ID @@ -338,7 +338,7 @@ New Features The pointer is now correctly initialized and the API call now produces a normal HDF5 error when fed an invalid fapl ID. - (DER - 2021/04/28, HDFFV-11240) + (2021/04/28, HDFFV-11240) - Fixes a segfault when H5Pset_mdc_log_options() is called multiple times @@ -350,7 +350,7 @@ New Features The string is now handled properly and the segfault no longer occurs. - (DER - 2021/04/27, HDFFV-11239) + (2021/04/27, HDFFV-11239) - HSYS_GOTO_ERROR now emits the results of GetLastError() on Windows @@ -377,7 +377,7 @@ New Features for those inclined to parse it for error values. - (DER - 2021/03/21) + (2021/03/21) - File locking now works on Windows @@ -393,241 +393,245 @@ New Features same scheme as POSIX systems. We lock the entire file when we set up the locks (by passing DWORDMAX as both size parameters to LockFileEx()). - (DER - 2021/03/19, HDFFV-10191) + (2021/03/19, HDFFV-10191) - H5Epush_ret() now requires a trailing semicolon - H5Epush_ret() is a function-like macro that has been changed to - contain a `do {} while(0)` loop. Consequently, a trailing semicolon - is now required to end the `while` statement. Previously, a trailing - semi would work, but was not mandatory. This change was made to allow - clang-format to correctly format the source code. + H5Epush_ret() is a function-like macro that has been changed to + contain a `do {} while(0)` loop. Consequently, a trailing semicolon + is now required to end the `while` statement. Previously, a trailing + semi would work, but was not mandatory. This change was made to allow + clang-format to correctly format the source code. - (SAM - 2021/03/03) + (2021/03/03) - Improved performance of H5Sget_select_elem_pointlist - Modified library to cache the point after the last block of points - retrieved by H5Sget_select_elem_pointlist, so a subsequent call to the - same function to retrieve the next block of points from the list can - proceed immediately without needing to iterate over the point list. + Modified library to cache the point after the last block of points + retrieved by H5Sget_select_elem_pointlist, so a subsequent call to the + same function to retrieve the next block of points from the list can + proceed immediately without needing to iterate over the point list. - (NAF - 2021/01/19) + (2021/01/19) - Added H5VL_VERSION macro that indicates the version of the VOL framework - implemented by a version of the library. Currently, compatibility - checking enforces that the 'version' field in the H5VL_class_t for - a VOL connector must match the version of the VOL framework for the - library when it is registered or dynamically loaded. + implemented by a version of the library. Currently, compatibility + checking enforces that the 'version' field in the H5VL_class_t for + a VOL connector must match the version of the VOL framework for the + library when it is registered or dynamically loaded. - (QAK - 2020/12/10) + (2020/12/10) - Added two new API routines for tracking library memory use: - H5get_alloc_stats() and H5get_free_list_sizes(). + H5get_alloc_stats() and H5get_free_list_sizes(). + + (2020/03/25) - (QAK - 2020/03/25) Java Library: ------------- - Added new H5S functions. - H5Sselect_copy, H5Sselect_shape_same, H5Sselect_adjust, - H5Sselect_intersect_block, H5Sselect_project_intersection, - H5Scombine_hyperslab, H5Smodify_select, H5Scombine_select - wrapper functions added. + H5Sselect_copy, H5Sselect_shape_same, H5Sselect_adjust, + H5Sselect_intersect_block, H5Sselect_project_intersection, + H5Scombine_hyperslab, H5Smodify_select, H5Scombine_select + wrapper functions added. - (ADB - 2020/10/27, HDFFV-10868) + (2020/10/27, HDFFV-10868) Tools: ------ - h5repack added help text for user-defined filters. - Added help text line that states the valid values of the filter flag - for user-defined filters; - filter_flag: 1 is OPTIONAL or 0 is MANDATORY + Added help text line that states the valid values of the filter flag + for user-defined filters; + filter_flag: 1 is OPTIONAL or 0 is MANDATORY - (ADB - 2021/01/14, HDFFV-11099) + (2021/01/14, HDFFV-11099) - h5repack added options to control how external links are handled. - Currently h5repack preserves external links and cannot copy and merge - data from the external files. Two options, merge and prune, were added to - control how to merge data from an external link into the resulting file. + Currently h5repack preserves external links and cannot copy and merge + data from the external files. Two options, merge and prune, were added to + control how to merge data from an external link into the resulting file. --merge Follow external soft link recursively and merge data. --prune Do not follow external soft links and remove link. --merge --prune Follow external link, merge data and remove dangling link. - (ADB - 2020/08/05, HDFFV-9984) + (2020/08/05, HDFFV-9984) Support for new platforms, languages and compilers ================================================== - - + + - Added macOS 11.2 Big Sur + Bug Fixes since HDF5-1.12.0 release =================================== + Library ------- - Fixed CVE-2018-14460 - The tool h5repack produced a segfault when the rank in dataspace - message was corrupted, causing invalid read while decoding the - dimension sizes. + The tool h5repack produced a segfault when the rank in dataspace + message was corrupted, causing invalid read while decoding the + dimension sizes. - The problem was fixed by ensuring that decoding the dimension sizes - and max values will not go beyong the end of the buffer. + The problem was fixed by ensuring that decoding the dimension sizes + and max values will not go beyong the end of the buffer. - (BMR - 2021/05/12, HDFFV-11223) + (2021/05/12, HDFFV-11223) - Fixed CVE-2018-11206 - The tool h5dump produced a segfault when the size of a fill value - message was corrupted and caused a buffer overflow. + The tool h5dump produced a segfault when the size of a fill value + message was corrupted and caused a buffer overflow. - The problem was fixed by verifying the fill value's size - against the buffer size before attempting to access the buffer. + The problem was fixed by verifying the fill value's size + against the buffer size before attempting to access the buffer. - (BMR - 2021/03/15, HDFFV-10480) + (2021/03/15, HDFFV-10480) - Fixed CVE-2018-14033 (same issue as CVE-2020-10811) - The tool h5dump produced a segfault when the storage size message - was corrupted and caused a buffer overflow. + The tool h5dump produced a segfault when the storage size message + was corrupted and caused a buffer overflow. - The problem was fixed by verifying the storage size against the - buffer size before attempting to access the buffer. + The problem was fixed by verifying the storage size against the + buffer size before attempting to access the buffer. - (BMR - 2021/03/15, HDFFV-11159/HDFFV-11049) + (2021/03/15, HDFFV-11159/HDFFV-11049) - Remove underscores on header file guards - Header file guards used a variety of underscores at the beginning of the define. + Header file guards used a variety of underscores at the beginning of the define. - Removed all leading (some trailing) underscores from header file guards. + Removed all leading (some trailing) underscores from header file guards. - (ADB - 2021/03/03, #361) + (2021/03/03, #361) - Fixed issue with MPI communicator and info object not being copied into new FAPL retrieved from H5F_get_access_plist - Added logic to copy the MPI communicator and info object into - the output FAPL. MPI communicator is retrieved from the VFD, while - the MPI info object is retrieved from the file's original FAPL. + Added logic to copy the MPI communicator and info object into + the output FAPL. MPI communicator is retrieved from the VFD, while + the MPI info object is retrieved from the file's original FAPL. - (JTH - 2021/02/15, HDFFV-11109) + (2021/02/15, HDFFV-11109) - Fixed problems with vlens and refs inside compound using H5VLget_file_type() - Modified library to properly ref count H5VL_object_t structs and only - consider file vlen and reference types to be equal if their files are - the same. + Modified library to properly ref count H5VL_object_t structs and only + consider file vlen and reference types to be equal if their files are + the same. - (NAF - 2021/01/22) + (2021/01/22) - Fix bug and simplify collective metadata write operation when some ranks - have no entries to contribute. This fixes parallel regression test - failures with IBM SpectrumScale MPI on the Summit system at ORNL. + have no entries to contribute. This fixes parallel regression test + failures with IBM SpectrumScale MPI on the Summit system at ORNL. - (QAK - 2020/09/02) + (2020/09/02) - Avoid setting up complex MPI types with 0-length vectors, which some - MPI implementations don't handle well. (In particular, IBM - SpectrumScale MPI on the Summit system at ORNL) + MPI implementations don't handle well. (In particular, IBM + SpectrumScale MPI on the Summit system at ORNL) - (QAK - 2020/08/21) + (2020/08/21) - Explicitly declared dlopen to use RTLD_LOCAL - dlopen documentation states that if neither RTLD_GLOBAL nor - RTLD_LOCAL are specified, then the default behavior is unspecified. - The default on linux is usually RTLD_LOCAL while macos will default - to RTLD_GLOBAL. + dlopen documentation states that if neither RTLD_GLOBAL nor + RTLD_LOCAL are specified, then the default behavior is unspecified. + The default on linux is usually RTLD_LOCAL while macos will default + to RTLD_GLOBAL. - (ADB - 2020/08/12, HDFFV-11127) + (2020/08/12, HDFFV-11127) - H5Sset_extent_none() sets the dataspace class to H5S_NO_CLASS which causes asserts/errors when passed to other dataspace API calls. - H5S_NO_CLASS is an internal class value that should not have been - exposed via a public API call. + H5S_NO_CLASS is an internal class value that should not have been + exposed via a public API call. - In debug builds of the library, this can cause assert() function to - trip. In non-debug builds, it will produce normal library errors. + In debug builds of the library, this can cause assert() function to + trip. In non-debug builds, it will produce normal library errors. - The new library behavior is for H5Sset_extent_none() to convert - the dataspace into one of type H5S_NULL, which is better handled - by the library and easier for developers to reason about. + The new library behavior is for H5Sset_extent_none() to convert + the dataspace into one of type H5S_NULL, which is better handled + by the library and easier for developers to reason about. - (DER - 2020/07/27, HDFFV-11027) + (2020/07/27, HDFFV-11027) - Fixed the segmentation fault when reading attributes with multiple threads - It was reported that the reading of attributes with variable length string - datatype will crash with segmentation fault particularly when the number of - threads is high (>16 threads). The problem was due to the file pointer that - was set in the variable length string datatype for the attribute. That file - pointer was already closed when the attribute was accessed. + It was reported that the reading of attributes with variable length string + datatype will crash with segmentation fault particularly when the number of + threads is high (>16 threads). The problem was due to the file pointer that + was set in the variable length string datatype for the attribute. That file + pointer was already closed when the attribute was accessed. - The problem was fixed by setting the file pointer to the current opened file pointer - when the attribute was accessed. Similar patch up was done before when reading - dataset with variable length string datatype. + The problem was fixed by setting the file pointer to the current opened file pointer + when the attribute was accessed. Similar patch up was done before when reading + dataset with variable length string datatype. - (VC - 2020/07/13, HDFFV-11080) + (2020/07/13, HDFFV-11080) - Reduce overhead for H5open(), which is involved in public symbols like - H5T_NATIVE_INT, etc. + H5T_NATIVE_INT, etc. - (QAK - 2020/06/18) + (2020/06/18) - Cache last ID looked up for an ID type (dataset, datatype, file, etc), - improving performance when accessing the same ID repeatedly. + improving performance when accessing the same ID repeatedly. - (QAK - 2020/06/11) + (2020/06/11) - Streamline I/O to a single element, improving performance for record - appends to chunked datasets. + appends to chunked datasets. - (QAK - 2020/06/11) + (2020/06/11) - Remove redundant tagging of metadata cache entries for some chunked - dataset operations, slightly improving performance for chunked - datasets. + dataset operations, slightly improving performance for chunked + datasets. - (QAK - 2020/06/10) + (2020/06/10) - Better detect selections with the same shape, improving performance for - some uses of H5DOappend (and other situations). + some uses of H5DOappend (and other situations). - (QAK - 2020/06/07) + (2020/06/07) - Don't allocate an empty (0-dimensioned) chunked dataset's chunk - index, until the dataset's dimensions are increased. + index, until the dataset's dimensions are increased. - (QAK - 2020/05/07) + (2020/05/07) Java Library ------------ - JNI utility function does not handle new references. - The JNI utility function for converting reference data to string did - not use the new APIs. In addition to fixing that function, added new - java tests for using the new APIs. + The JNI utility function for converting reference data to string did + not use the new APIs. In addition to fixing that function, added new + java tests for using the new APIs. - (ADB - 2021/02/16, HDFFV-11212) + (2021/02/16, HDFFV-11212) - The H5FArray.java class, in which virtually the entire execution time - is spent using the HDFNativeData method that converts from an array - of bytes to an array of the destination Java type. + is spent using the HDFNativeData method that converts from an array + of bytes to an array of the destination Java type. 1. Convert the entire byte array into a 1-d array of the desired type, rather than performing 1 conversion per row; 2. Use the Java Arrays method copyOfRange to grab the section of the array from (1) that is desired to be inserted into the destination array. - (PGT,ADB - 2020/12/29, HDFFV-10865) + (2020/12/29, HDFFV-10865) Configuration @@ -643,7 +647,7 @@ Bug Fixes since HDF5-1.12.0 release Visual Studio warnings C4100, C4706, and C4127 have been moved to developer warnings, HDF5_ENABLE_DEV_WARNINGS, and are disabled for normal builds. - (ADB - 2021/03/22, HDFFV-11228) + (2021/03/22, HDFFV-11228) - Reclassify CMake messages, to allow new modes and --log-level option @@ -655,7 +659,7 @@ Bug Fixes since HDF5-1.12.0 release version 3.17 or above is used, the user can use the command line option of "--log-level" to further restrict which message commands are displayed. - (ADB - 2021/01/11, HDFFV-11144) + (2021/01/11, HDFFV-11144) - Fixes Autotools determination of the stat struct having an st_blocks field @@ -666,47 +670,47 @@ Bug Fixes since HDF5-1.12.0 release CMake. This #define is only used in the tests and does not affect the HDF5 C library. - (DER - 2021/01/07, HDFFV-11201) + (2021/01/07, HDFFV-11201) Tools ----- - Changed how h5dump and h5ls identify long double. - Long double support is not consistent across platforms. Tools will always - identify long double as 128-bit [little/big]-endian float nn-bit precision. - New test file created for datasets with attributes for float, double and - long double. In addition any unknown integer or float datatype will now - also show the number of bits for precision. - These files are also used in the java tests. + Long double support is not consistent across platforms. Tools will always + identify long double as 128-bit [little/big]-endian float nn-bit precision. + New test file created for datasets with attributes for float, double and + long double. In addition any unknown integer or float datatype will now + also show the number of bits for precision. + These files are also used in the java tests. - (ADB - 2021/03/24, HDFFV-11229) + (2021/03/24, HDFFV-11229) - Fixed tools argument parsing. - Tools parsing used the length of the option from the long array to match - the option from the command line. This incorrectly matched a shorter long - name option that happened to be a subset of another long option. - Changed to match whole names. + Tools parsing used the length of the option from the long array to match + the option from the command line. This incorrectly matched a shorter long + name option that happened to be a subset of another long option. + Changed to match whole names. - (ADB - 2021/01/19, HDFFV-11106) + (2021/01/19, HDFFV-11106) Fortran API ----------- - Fixed configure issue when building HDF5 with NAG Fortran 7.0. - HDF5 now accounts for the addition of half-precision floating-point - in NAG 7.0 with a KIND=16. + HDF5 now accounts for the addition of half-precision floating-point + in NAG 7.0 with a KIND=16. - (MSB - 2020/02/28, HDFFV-11033) + (2020/02/28, HDFFV-11033) High-Level Library ------------------ - Eliminated unnecessary code in H5DOappend(), improving its performance. - (QAK - 2020/06/05) + (2020/06/05) Fortran High-Level APIs @@ -718,7 +722,7 @@ Bug Fixes since HDF5-1.12.0 release ------------- - Updated doxygen comments with changes for release - (ADB - 2021/05/03) + (2021/05/03) F90 APIs @@ -735,30 +739,24 @@ Bug Fixes since HDF5-1.12.0 release ------- - Stopped java/test/junit.sh.in installing libs for testing under ${prefix} - Lib files needed are now copied to a subdirectory in the java/test - directory, and on Macs the loader path for libhdf5.xxxs.so is changed - in the temporary copy of libhdf5_java.dylib. + Lib files needed are now copied to a subdirectory in the java/test + directory, and on Macs the loader path for libhdf5.xxxs.so is changed + in the temporary copy of libhdf5_java.dylib. - (LRK, 2020/07/02, HDFFV-11063) + (2020/07/02, HDFFV-11063) Supported Platforms =================== - Linux 2.6.32-696.16.1.el6.ppc64 gcc (GCC) 4.4.7 20120313 (Red Hat 4.4.7-18) - #1 SMP ppc64 GNU/Linux g++ (GCC) 4.4.7 20120313 (Red Hat 4.4.7-18) - (ostrich) GNU Fortran (GCC) 4.4.7 20120313 (Red Hat 4.4.7-18) - IBM XL C/C++ V13.1 - IBM XL Fortran V15.1 - Linux 3.10.0-327.10.1.el7 GNU C (gcc), Fortran (gfortran), C++ (g++) #1 SMP x86_64 GNU/Linux compilers: - (kituo/moohan) Version 4.8.5 20150623 (Red Hat 4.8.5-4) + (jelly/kituo/moohan) Version 4.8.5 20150623 (Red Hat 4.8.5-4) Version 4.9.3, 5.2.0, 7.1.0 Intel(R) C (icc), C++ (icpc), Fortran (icc) compilers: Version 17.0.0.098 Build 20160721 - MPICH 3.1.4 + MPICH 3.2, 3.3 Windows 10 x64 Visual Studio 2015 w/ Intel Fortran 18 (cmake) Visual Studio 2017 w/ Intel Fortran 19 (cmake) @@ -769,10 +767,6 @@ Supported Platforms 64-bit gfortran GNU Fortran (GCC) 6.3.0 (bear) Intel icc/icpc/ifort version 19.0.4 - MacOS Mojave 10.14.6 Apple LLVM version 10.0.1 (clang-1001.0.46.4) - 64-bit gfortran GNU Fortran (GCC) 6.3.0 - (bobcat) Intel icc/icpc/ifort version 19.0.4 - MacOS Big Sur 11.2 Apple clang version 12.0.5 (clang-1205.0.22.9) gfortran GNU Fortran (Homebrew GCC 10.2.0_3) 10.2.0 Intel icc/icpc (ICC) 2021.1 Beta 20201112 @@ -792,36 +786,28 @@ Tested Configuration Features Summary Platform C F90/ F90 C++ zlib SZIP parallel F2003 parallel -Solaris2.11 32-bit n y/y n y y y -Solaris2.11 64-bit n y/n n y y y Windows 10 y y/y n y y y Windows 10 x64 y y/y n y y y MacOS High Sierra 10.13.6 64-bit n y/y n y y y -MacOS Mojave 10.14.6 64-bit n y/y n y y y MacOS Big Sur 11.2 64-bit n y/y n y y y CentOS 6.7 Linux 2.6.18 x86_64 GNU n y/y n y y y -CentOS 6.7 Linux 2.6.18 x86_64 Intel n y/y n y y y CentOS 6.7 Linux 2.6.32 x86_64 PGI n y/y n y y y CentOS 7.2 Linux 2.6.32 x86_64 GNU y y/y y y y y CentOS 7.2 Linux 2.6.32 x86_64 Intel n y/y n y y y -Linux 2.6.32-573.18.1.el6.ppc64 n y/n n y y y +Linux 3.10.0-1127.10.1.1.ch6.ppc64le n y/n n y y y Platform Shared Shared Shared Thread- C libs F90 libs C++ libs safe -Solaris2.11 32-bit y y y y -Solaris2.11 64-bit y y y y Windows 10 y y y y Windows 10 x64 y y y y MacOS High Sierra 10.13.6 64-bit y n y y -MacOS Mojave 10.14.6 64-bit y n y y MacOS Big Sur 11.2 64-bit y n y y CentOS 6.7 Linux 2.6.18 x86_64 GNU y y y y -CentOS 6.7 Linux 2.6.18 x86_64 Intel y y y n CentOS 6.7 Linux 2.6.32 x86_64 PGI y y y n CentOS 7.2 Linux 2.6.32 x86_64 GNU y y y n CentOS 7.2 Linux 2.6.32 x86_64 Intel y y y n -Linux 2.6.32-573.18.1.el6.ppc64 y y y n +Linux 3.10.0-1127.10.1.1.ch6.ppc64le y y y n Compiler versions for each platform are listed in the preceding "Supported Platforms" table. @@ -829,6 +815,7 @@ Compiler versions for each platform are listed in the preceding More Tested Platforms ===================== + The following platforms are not supported but have been tested for this release. Linux 2.6.32-573.22.1.el6 GNU C (gcc), Fortran (gfortran), C++ (g++) @@ -858,28 +845,28 @@ The following platforms are not supported but have been tested for this release. #1 SMP x86_64 GNU/Linux (moohan) - Linux-3.10.0-1127.0.0.1chaos openmpi-4.0.0 - #1 SMP x86_64 GNU/Linux clang/3.9.0, 8.0.1 + Linux-3.10.0-1127.0.0.1chaos openmpi/4.0.0 + #1 SMP x86_64 GNU/Linux clang/6.0.0, 11.0.1 (quartz) gcc/7.3.0, 8.1.0 - intel/16.0.4 + intel/16.0.4, 18.0.2, 19.0.4 Linux-4.14.0-115.10.1.1 spectrum-mpi/rolling-release - #1 SMP ppc64le GNU/Linux clang/coral-2018.08.08 - (lassen) gcc/7.3.1 - xl/2019.02.07 + #1 SMP ppc64le GNU/Linux clang/7.1.0, 10.0.1 + (lassen) gcc/4.9.3, 8.3.1 + xl/2019.02.07,2021.03.11 Linux-4.12.14-150.52-default cray-mpich/7.7.10 - #1 SMP x86_64 GNU/Linux gcc/7.3.0, 8.2.0 + #1 SMP x86_64 GNU/Linux gcc/8.3.0 (cori) intel/19.0.3 Linux-4.4.180-94.107-default cray-mpich/7.7.6 # 1SMP x86_64 GNU/Linux gcc/7.2.0, 8.2.0 (mutrino) intel/17.0.4, 18.0.2, 19.0.4 - Linux-3.10.0- spectrum-mpi/rolling-release with cmake>3.10 and - 862.14.4.1chaos.ch6.ppc64le clang/3.9,8.0 - #1 SMP ppc64le GNU/Linux gcc/7.3 - (ray) xl/2016,2019 + Linux-3.10.0- spectrum-mpi/rolling-release + 1127.10.1.1chaos.ch6.ppc64le clang/7.1,11.0.1 + #1 SMP ppc64le GNU/Linux gcc/4.9.3,8.3.1 + (ray) xl/2019.08.20,2021.03.11 Fedora33 5.11.18-200.fc33.x86_64 #1 SMP x86_64 GNU/Linux GNU gcc (GCC) 10.3.1 20210422 (Red Hat 10.3.1-1) @@ -912,6 +899,7 @@ The following platforms are not supported but have been tested for this release. Known Problems ============== + The t_bigio test fails on several HPC platforms, generally by timeout with OpenMPI 4.0.0 or with this error from spectrum-mpi: *** on communicator MPI_COMM_WORLD @@ -937,6 +925,7 @@ Known Problems CMake vs. Autotools installations ================================= + While both build systems produce similar results, there are differences. Each system produces the same set of folders on linux (only CMake works on standard Windows); bin, include, lib and share. Autotools places the From c1acf63b80d7cf3e92eaa06b79bfb186394e3597 Mon Sep 17 00:00:00 2001 From: Lee Newberg <35611400+Leengit@users.noreply.github.com> Date: Wed, 19 May 2021 16:48:20 -0400 Subject: [PATCH 03/12] BUG: DataSet assignment operator is missing (#503) * BUG: DataSet assignment operator is missing * Some compilers complain if the copy constructor is given explicitly but the assignment operator is implicitly set to default. * Explicitly defining the assignment operator allows us to properly handle reference counters for shared resources. * BUG: DataSet assignment operator is missing. * Mimicking code of H5DataType::operator() as suggested by @bmribler. * Added test Description: Added test for DataSet::operator= that Leengit added Platform tested: Linux/64 (jelly) * Removed Author field. * Commit clang format changes. * Entry for Leengit's github PR #503 * Removed lines left by mistake Co-authored-by: Binh-Minh Ribler Co-authored-by: Larry Knox --- c++/src/H5DataSet.cpp | 62 ++++++++++------------------ c++/src/H5DataSet.h | 3 ++ c++/test/dsets.cpp | 89 +++++++++++++++++++++++++--------------- release_docs/RELEASE.txt | 7 +++- 4 files changed, 88 insertions(+), 73 deletions(-) diff --git a/c++/src/H5DataSet.cpp b/c++/src/H5DataSet.cpp index 3a1497580f5..47031dbfaec 100644 --- a/c++/src/H5DataSet.cpp +++ b/c++/src/H5DataSet.cpp @@ -42,7 +42,6 @@ using std::endl; //-------------------------------------------------------------------------- // Function: DataSet default constructor ///\brief Default constructor: creates a stub DataSet. -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- DataSet::DataSet() : H5Object(), AbstractDs(), id(H5I_INVALID_HID) { @@ -52,7 +51,6 @@ DataSet::DataSet() : H5Object(), AbstractDs(), id(H5I_INVALID_HID) // Function: DataSet overloaded constructor ///\brief Creates an DataSet object using the id of an existing dataset. ///\param existing_id - IN: Id of an existing dataset -// Programmer Binh-Minh Ribler - 2000 // Description // incRefCount() is needed here to prevent the id from being closed // prematurely. That is, when application uses the id of an @@ -69,13 +67,26 @@ DataSet::DataSet(const hid_t existing_id) : H5Object(), AbstractDs(), id(existin // Function: DataSet copy constructor ///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: DataSet instance to copy -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- DataSet::DataSet(const DataSet &original) : H5Object(), AbstractDs(), id(original.id) { incRefCount(); // increment number of references to this id } +//-------------------------------------------------------------------------- +// Function: DataSet assignment operator +///\brief Assignment operator: same HDF5 object as \a original +///\param original - IN: DataSet instance to copy +//-------------------------------------------------------------------------- +DataSet & +DataSet::operator=(const DataSet &original) +{ + if (this != &original) { + setId(original.id); + } + return (*this); +} + //-------------------------------------------------------------------------- // Function: DataSet overload constructor - dereference ///\brief Given a reference, ref, to an hdf5 location, creates a @@ -89,7 +100,6 @@ DataSet::DataSet(const DataSet &original) : H5Object(), AbstractDs(), id(origina ///\par Description /// \c loc can be DataSet, Group, H5File, or named DataType, that /// is a datatype that has been named by DataType::commit. -// Programmer Binh-Minh Ribler - Oct, 2006 //-------------------------------------------------------------------------- DataSet::DataSet(const H5Location &loc, const void *ref, H5R_type_t ref_type, const PropList &plist) : H5Object(), AbstractDs(), id(H5I_INVALID_HID) @@ -106,7 +116,6 @@ DataSet::DataSet(const H5Location &loc, const void *ref, H5R_type_t ref_type, co ///\param ref_type - IN: Reference type - default to H5R_OBJECT ///\param plist - IN: Property list - default to PropList::DEFAULT ///\exception H5::ReferenceException -// Programmer Binh-Minh Ribler - Oct, 2006 //-------------------------------------------------------------------------- DataSet::DataSet(const Attribute &attr, const void *ref, H5R_type_t ref_type, const PropList &plist) : H5Object(), AbstractDs(), id(H5I_INVALID_HID) @@ -119,7 +128,6 @@ DataSet::DataSet(const Attribute &attr, const void *ref, H5R_type_t ref_type, co ///\brief Gets a copy of the dataspace of this dataset. ///\return DataSpace instance ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- DataSpace DataSet::getSpace() const @@ -156,7 +164,6 @@ DataSet::p_get_type() const ///\brief Gets the dataset creation property list. ///\return DSetCreatPropList instance ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- DSetCreatPropList DataSet::getCreatePlist() const @@ -200,7 +207,6 @@ DataSet::getAccessPlist() const ///\exception H5::DataSetIException // Note: H5Dget_storage_size returns 0 when there is no data. This // function should have no failure. (from SLU) -// Programmer Binh-Minh Ribler - Mar, 2005 //-------------------------------------------------------------------------- hsize_t DataSet::getStorageSize() const @@ -214,7 +220,6 @@ DataSet::getStorageSize() const ///\brief Gets the size in memory of the dataset's data. ///\return Size of data (in memory) ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - Apr 2009 //-------------------------------------------------------------------------- size_t DataSet::getInMemDataSize() const @@ -272,7 +277,6 @@ DataSet::getInMemDataSize() const ///\brief Returns the address of this dataset in the file. ///\return Address of dataset ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- haddr_t DataSet::getOffset() const @@ -291,7 +295,6 @@ DataSet::getOffset() const ///\brief Determines whether space has been allocated for a dataset. ///\param status - OUT: Space allocation status ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- void DataSet::getSpaceStatus(H5D_space_status_t &status) const @@ -309,7 +312,6 @@ DataSet::getSpaceStatus(H5D_space_status_t &status) const ///\param space - IN: Selection for the memory buffer ///\return Amount of storage ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- hsize_t DataSet::getVlenBufSize(const DataType &type, const DataSpace &space) const @@ -334,7 +336,6 @@ DataSet::getVlenBufSize(const DataType &type, const DataSpace &space) const // misses const's. This wrapper will be removed in future release. // Return Amount of storage // Exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 // Modification // Modified to call its replacement. -BMR, 2014/04/16 // Removed from documentation. -BMR, 2016/03/07 1.8.17 and 1.10.0 @@ -354,7 +355,6 @@ DataSet::getVlenBufSize(const DataType &type, const DataSpace &space) const ///\param xfer_plist - IN: Property list used to create the buffer ///\param buf - IN: Pointer to the buffer to be reclaimed ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- void DataSet::vlenReclaim(const DataType &type, const DataSpace &space, const DSetMemXferPropList &xfer_plist, @@ -380,7 +380,6 @@ DataSet::vlenReclaim(const DataType &type, const DataSpace &space, const DSetMem ///\param xfer_plist - IN: Property list used to create the buffer ///\param buf - IN: Pointer to the buffer to be reclaimed ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 //\parDescription // This function has better prototype for the users than the // other, which might be removed at some point. BMR - 2006/12/20 @@ -413,7 +412,6 @@ DataSet::vlenReclaim(void *buf, const DataType &type, const DataSpace &space, /// This function reads raw data from this dataset into the /// buffer \a buf, converting from file datatype and dataspace /// to memory datatype \a mem_type and dataspace \a mem_space. -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- void DataSet::read(void *buf, const DataType &mem_type, const DataSpace &mem_space, const DataSpace &file_space, @@ -441,7 +439,6 @@ DataSet::read(void *buf, const DataType &mem_type, const DataSpace &mem_space, c ///\param file_space - IN: Dataset's dataspace in the file ///\param xfer_plist - IN: Transfer property list for this I/O operation ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 // Modification // Jul 2009 // Follow the change to Attribute::read and use the following @@ -490,7 +487,6 @@ DataSet::read(H5std_string &strg, const DataType &mem_type, const DataSpace &mem /// \a buf to a dataset, converting from memory datatype /// \a mem_type and dataspace \a mem_space to file datatype /// and dataspace. -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- void DataSet::write(const void *buf, const DataType &mem_type, const DataSpace &mem_space, @@ -512,7 +508,6 @@ DataSet::write(const void *buf, const DataType &mem_type, const DataSpace &mem_s // Function: DataSet::write ///\brief This is an overloaded member function, provided for convenience. /// It takes a reference to a \c H5std_string for the buffer. -// Programmer Binh-Minh Ribler - 2000 // Modification // Jul 2009 // Modified to pass the buffer into H5Dwrite properly depending @@ -568,7 +563,6 @@ DataSet::write(const H5std_string &strg, const DataType &mem_type, const DataSpa ///\exception H5::DataSetIException ///\note This function may not work correctly yet - it's still /// under development. -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- int DataSet::iterateElems(void *buf, const DataType &type, const DataSpace &space, H5D_operator_t op, @@ -594,7 +588,6 @@ DataSet::iterateElems(void *buf, const DataType &type, const DataSpace &space, H ///\par Description /// For information, please refer to the H5Dset_extent API in /// the HDF5 C Reference Manual. -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- void DataSet::extend(const hsize_t *size) const @@ -613,7 +606,6 @@ DataSet::extend(const hsize_t *size) const ///\param buf_type - IN: Datatype of the elements in buffer ///\param space - IN: Dataspace describing memory buffer & containing selection to use ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2014 //-------------------------------------------------------------------------- void DataSet::fillMemBuf(const void *fill, const DataType &fill_type, void *buf, const DataType &buf_type, @@ -639,7 +631,6 @@ DataSet::fillMemBuf(const void *fill, const DataType &fill_type, void *buf, cons // Param buf_type - IN: Datatype of the elements in buffer // Param space - IN: Dataspace describing memory buffer & containing selection to use // Exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 // Modification // Modified to call its replacement. -BMR, 2014/04/16 // Removed from documentation. -BMR, 2016/03/07 1.8.17 and 1.10.0 @@ -658,7 +649,6 @@ DataSet::fillMemBuf(const void *fill, const DataType &fill_type, void *buf, cons ///\param buf_type - IN: Datatype of the elements in buffer ///\param space - IN: Dataspace describing memory buffer & containing selection to use ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- void DataSet::fillMemBuf(void *buf, const DataType &buf_type, const DataSpace &space) const @@ -680,7 +670,6 @@ DataSet::fillMemBuf(void *buf, const DataType &buf_type, const DataSpace &space) // Param buf_type - IN: Datatype of the elements in buffer // Param space - IN: Dataspace describing memory buffer & containing selection to use // Exception H5::DataSetIException -// Programmer Binh-Minh Ribler - 2000 // Modification // Modified to call its replacement. -BMR, 2014/04/16 // Removed from documentation. -BMR, 2016/03/07 1.8.17 and 1.10.0 @@ -700,7 +689,6 @@ DataSet::fillMemBuf(void *buf, const DataType &buf_type, const DataSpace &space) // AbstractDs and Attribute are moved out of H5Object. In // addition, member IdComponent::id is moved into subclasses, and // IdComponent::getId now becomes pure virtual function. -// Programmer Binh-Minh Ribler - May, 2008 //-------------------------------------------------------------------------- hid_t DataSet::getId() const @@ -710,11 +698,10 @@ DataSet::getId() const //-------------------------------------------------------------------------- // Function: DataSet::p_read_fixed_len (private) -// brief Reads a fixed length \a H5std_string from a dataset. -// param mem_type - IN: DataSet datatype (in memory) -// param strg - IN: Buffer for read string -// exception H5::DataSetIException -// Programmer Binh-Minh Ribler - Jul, 2009 +// brief Reads a fixed length \a H5std_string from a dataset. +// param mem_type - IN: DataSet datatype (in memory) +// param strg - IN: Buffer for read string +// exceptio n H5::DataSetIException // Modification // Jul 2009 // Added in follow to the change in Attribute::read @@ -748,11 +735,10 @@ DataSet::p_read_fixed_len(const hid_t mem_type_id, const hid_t mem_space_id, con //-------------------------------------------------------------------------- // Function: DataSet::p_read_variable_len (private) -// brief Reads a variable length \a H5std_string from an dataset. -// param mem_type - IN: DataSet datatype (in memory) -// param strg - IN: Buffer for read string -// exception H5::DataSetIException -// Programmer Binh-Minh Ribler - Jul, 2009 +// brief Reads a variable length \a H5std_string from an dataset. +// param mem_type - IN: DataSet datatype (in memory) +// param strg - IN: Buffer for read string +// exception H5::DataSetIException // Modification // Jul 2009 // Added in follow to the change in Attribute::read @@ -787,7 +773,6 @@ DataSet::p_read_variable_len(const hid_t mem_type_id, const hid_t mem_space_id, // The underlaying reference counting in the C library ensures // that the current valid id of this object is properly closed. // Then the object's id is reset to the new id. -// Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- void DataSet::p_setId(const hid_t new_id) @@ -811,7 +796,6 @@ DataSet::p_setId(const hid_t new_id) // Applications shouldn't need to use it. // param dset - IN/OUT: DataSet object to be changed // param new_id - IN: New id to set -// Programmer Binh-Minh Ribler - 2015 //-------------------------------------------------------------------------- void f_PropList_setId(PropList *plist, hid_t new_id) @@ -826,7 +810,6 @@ f_PropList_setId(PropList *plist, hid_t new_id) ///\brief Closes this dataset. /// ///\exception H5::DataSetIException -// Programmer Binh-Minh Ribler - Mar 9, 2005 //-------------------------------------------------------------------------- void DataSet::close() @@ -844,7 +827,6 @@ DataSet::close() //-------------------------------------------------------------------------- // Function: DataSet destructor ///\brief Properly terminates access to this dataset. -// Programmer Binh-Minh Ribler - 2000 // Modification // - Replaced resetIdComponent() with decRefCount() to use C // library ID reference counting mechanism - BMR, Jun 1, 2004 diff --git a/c++/src/H5DataSet.h b/c++/src/H5DataSet.h index c7454709cfd..333ed7a999f 100644 --- a/c++/src/H5DataSet.h +++ b/c++/src/H5DataSet.h @@ -117,6 +117,9 @@ class H5_DLLCPP DataSet : public H5Object, public AbstractDs { // Copy constructor - same as the original DataSet. DataSet(const DataSet &original); + // Assignment operator + DataSet &operator=(const DataSet &original); + // Creates a copy of an existing DataSet using its id. DataSet(const hid_t existing_id); diff --git a/c++/test/dsets.cpp b/c++/test/dsets.cpp index 472ccb34806..52e3e73ce08 100644 --- a/c++/test/dsets.cpp +++ b/c++/test/dsets.cpp @@ -42,6 +42,8 @@ const H5std_string DSET_SIMPLE_IO_NAME("simple_io"); const H5std_string DSET_TCONV_NAME("tconv"); const H5std_string DSET_COMPRESS_NAME("compressed"); const H5std_string DSET_BOGUS_NAME("bogus"); +const H5std_string DSET_OPERATOR("testing operator="); +const H5std_string DSET_OPERATOR_PATH("/testing operator="); /* Temporary filter IDs used for testing */ const int H5Z_FILTER_BOGUS = 305; @@ -58,9 +60,6 @@ static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, const unsigned * Return Success: 0 * * Failure: -1 - * - * Programmer Binh-Minh Ribler (using C version) - * Friday, January 5, 2001 *------------------------------------------------------------------------- */ static herr_t @@ -184,9 +183,6 @@ test_create(H5File &file) * Return Success: 0 * * Failure: -1 - * - * Programmer Binh-Minh Ribler (using C version) - * Friday, January 5, 2001 *------------------------------------------------------------------------- */ static herr_t @@ -263,9 +259,6 @@ test_simple_io(H5File &file) * Return Success: 0 * * Failure: -1 - * - * Programmer Binh-Minh Ribler - * Thursday, March 22, 2012 *------------------------------------------------------------------------- */ static herr_t @@ -325,9 +318,6 @@ test_datasize(FileAccPropList &fapl) * Return Success: 0 * * Failure: -1 - * - * Programmer Binh-Minh Ribler (using C version) - * Friday, January 5, 2001 *------------------------------------------------------------------------- */ static herr_t @@ -435,9 +425,6 @@ filter_bogus(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[ * Return Success: 0 * * Failure: -1 - * - * Programmer Binh-Minh Ribler (using C version) - * Friday, January 5, 2001 *------------------------------------------------------------------------- */ static herr_t @@ -708,10 +695,6 @@ test_compression(H5File &file) * Return Success: 0 * * Failure: -1 - * - * Programmer Binh-Minh Ribler - * Friday, April 22, 2016 - * *------------------------------------------------------------------------- */ const H5std_string DSET_NBIT_NAME("nbit_dataset"); @@ -823,9 +806,6 @@ test_nbit_compression(H5File &file) * Return Success: 0 * * Failure: -1 - * - * Programmer Binh-Minh Ribler (using C version) - * Saturday, February 17, 2001 *------------------------------------------------------------------------- */ static herr_t @@ -899,9 +879,6 @@ test_multiopen(H5File &file) * Return Success: 0 * * Failure: -1 - * - * Programmer Binh-Minh Ribler (using C version) - * February 17, 2001 *------------------------------------------------------------------------- */ static herr_t @@ -1258,10 +1235,6 @@ test_chunk_cache(const FileAccPropList &fapl) * * Return Success: 0 * Failure: number of errors - * - * Programmer Binh-Minh Ribler - * Friday, March 10, 2017 - * *------------------------------------------------------------------------- */ const int RANK = 2; @@ -1327,6 +1300,60 @@ test_virtual() } } // test_virtual +/*------------------------------------------------------------------------- + * Function: test_operator + * + * Purpose Tests DataSet::operator= + * + * Return Success: 0 + * + * Failure: -1 + *------------------------------------------------------------------------- + */ +static herr_t +test_operator(H5File &file) +{ + SUBTEST("DataSet::operator="); + + try { + // Create a data space + hsize_t dims[2]; + dims[0] = 256; + dims[1] = 512; + DataSpace space(2, dims, NULL); + + // Create a dataset using the default dataset creation properties. + // We're not sure what they are, so we won't check. + DataSet dataset = file.createDataSet(DSET_OPERATOR, PredType::NATIVE_DOUBLE, space); + + // Add a comment to the dataset + file.setComment(DSET_OPERATOR, "Dataset using operator="); + + // Close the dataset + dataset.close(); + + // Re-open the dataset + DataSet another_dataset(file.openDataSet(DSET_OPERATOR)); + + // Try operator= to make another dataset + DataSet copied_dataset = another_dataset; + + H5std_string copied_dataset_name = copied_dataset.getObjName(); + H5std_string another_dataset_name = another_dataset.getObjName(); + + PASSED(); + return 0; + } // try block + + // catch all other exceptions + catch (Exception &E) { + issue_fail_msg("test_operator", __LINE__, __FILE__); + + // clean up and return with failure + return -1; + } +} // test_operator + /*------------------------------------------------------------------------- * Function: test_dset * @@ -1336,9 +1363,6 @@ test_virtual() * * Failure: -1 * - * Programmer Binh-Minh Ribler (using C version) - * Friday, January 5, 2001 - * * Modifications: * Nov 12, 01: * - moved h5_cleanup to outside of try block because @@ -1376,6 +1400,7 @@ test_dset() nerrors += test_multiopen(file) < 0 ? 1 : 0; nerrors += test_types(file) < 0 ? 1 : 0; nerrors += test_virtual() < 0 ? 1 : 0; + nerrors += test_operator(file) < 0 ? 1 : 0; nerrors += test_chunk_cache(fapl) < 0 ? 1 : 0; // Close group "emit diagnostics". diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 312f2b6e7e5..611f59882f1 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -732,7 +732,12 @@ Bug Fixes since HDF5-1.12.0 release C++ APIs -------- - - + - Added DataSet::operator= + + Some compilers complain if the copy constructor is given explicitly + but the assignment operator is implicitly set to default. + + (2021/05/19) Testing From 53e113977a775a9311e7df015937f71b11f76f7b Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Thu, 20 May 2021 00:21:52 +0200 Subject: [PATCH 04/12] Add H5Dchunk_iter method for iterating over chunks (#6) * Add H5Dchunk_iter method for iterating over chunks This method iterates over all chunks in dataset, calling a user-supplied callback with the chunk information and optional user supplied data. The iterator is stopped when ITER_STOP is returned by the user-supplied callback or the iterator is exhausted. Existing methods to get chunk_info performs an iteration each time, so to get many or all chunks causes SUM(i) for i = 0 -> N operations for N chunks, as opposed to N operations when using this iterator for this use case. * H5Dchunk_iter: test iterating all chunks, some chunks and failing iteration. * H5D: move H5Dchunk_iter private methods to specific * trace: add H5D_chunk_iter_op_t and trace H5D.c * chunks-iter: document chunk_iter * chunk-iter: chunk add FUNC_ENTER/FUNC_LEAVE macros * Committing clang-format changes Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- bin/trace | 1 + fortran/src/Makefile.in | 18 +++---- release_docs/RELEASE.txt | 15 ++++++ src/H5D.c | 65 ++++++++++++++++++++++ src/H5Dchunk.c | 98 +++++++++++++++++++++++++++++++++ src/H5Dpkg.h | 1 + src/H5Dpublic.h | 53 ++++++++++++++++++ src/H5VLconnector.h | 4 +- src/H5VLnative_dataset.c | 25 +++++++++ test/chunk_info.c | 113 ++++++++++++++++++++++++++++++++------- 10 files changed, 365 insertions(+), 28 deletions(-) diff --git a/bin/trace b/bin/trace index 2b519e41cbc..8cad5acb904 100755 --- a/bin/trace +++ b/bin/trace @@ -44,6 +44,7 @@ $Source = ""; "H5D_vds_view_t" => "Dv", "H5FD_mpio_xfer_t" => "Dt", "H5FD_splitter_vfd_config_t" => "Dr", + "H5D_chunk_iter_op_t" => "x", "herr_t" => "e", "H5E_direction_t" => "Ed", "H5E_error_t" => "Ee", diff --git a/fortran/src/Makefile.in b/fortran/src/Makefile.in index 0f8472ca1b5..0a123edebb2 100644 --- a/fortran/src/Makefile.in +++ b/fortran/src/Makefile.in @@ -839,9 +839,9 @@ LT_VERS_REVISION = 0 LT_CXX_VERS_INTERFACE = 201 LT_CXX_VERS_REVISION = 0 LT_CXX_VERS_AGE = 1 -LT_F_VERS_INTERFACE = 200 -LT_F_VERS_REVISION = 1 -LT_F_VERS_AGE = 0 +LT_F_VERS_INTERFACE = 201 +LT_F_VERS_REVISION = 0 +LT_F_VERS_AGE = 1 LT_HL_VERS_INTERFACE = 200 LT_HL_VERS_REVISION = 1 LT_HL_VERS_AGE = 0 @@ -851,12 +851,12 @@ LT_HL_CXX_VERS_AGE = 0 LT_HL_F_VERS_INTERFACE = 200 LT_HL_F_VERS_REVISION = 1 LT_HL_F_VERS_AGE = 0 -LT_JAVA_VERS_INTERFACE = 200 -LT_JAVA_VERS_REVISION = 1 -LT_JAVA_VERS_AGE = 0 -LT_TOOLS_VERS_INTERFACE = 200 -LT_TOOLS_VERS_REVISION = 1 -LT_TOOLS_VERS_AGE = 0 +LT_JAVA_VERS_INTERFACE = 201 +LT_JAVA_VERS_REVISION = 0 +LT_JAVA_VERS_AGE = 1 +LT_TOOLS_VERS_INTERFACE = 201 +LT_TOOLS_VERS_REVISION = 0 +LT_TOOLS_VERS_AGE = 1 AM_FCLIBS = $(LIBHDF5) # This is our main target, the fortran library diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 611f59882f1..6a0761eff4c 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -309,6 +309,21 @@ New Features Library: -------- + - H5Dchunk_iter() added for iterating over chunks + + This method iterates over all chunks in a dataset, calling a user-supplied + callback with the chunk information and optional user supplied data. + + The iterator is stopped when ITER_STOP is returned by the user-supplied + callback or the iterator is exhausted. + + Existing methods to get chunk_info perform an iteration each time, so + whereas getting many or all chunks with existing methods caused SUM(i) + for i = 0 -> N operations for N chunks, using this iterator for this + use case will get N chunks with N operations. + + (2021/05/19) + - H5Gcreate1() now rejects size_hint parameters larger than UINT32_MAX The size_hint value is ultimately stored in a uint32_t struct field, diff --git a/src/H5D.c b/src/H5D.c index b69391fad8f..c0ac2a064eb 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -1126,3 +1126,68 @@ H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filte done: FUNC_LEAVE_API(ret_value) } /* end H5Dget_chunk_info_by_coord() */ + +/*------------------------------------------------------------------------- + * Function: H5Dchunk_iter + * + * Purpose: Iterates over all chunks in dataset with given callback and user data. + * + * Parameters: + * hid_t dset_id; IN: Chunked dataset ID + * H5D_chunk_iter_op_t cb IN: User callback function, called for every chunk. + * void *op_data IN/OUT: Optional user data passed on to user callback. + * + * Callback information: + * H5D_chunk_iter_op_t is defined as: + * + * typedef int (*H5D_chunk_iter_op_t)( + * const hsize_t *offset, + * uint32_t filter_mask, + * haddr_t addr, + * uint32_t nbytes, + * void *op_data); + * + * H5D_chunk_iter_op_t parameters: + * hsize_t *offset; IN/OUT: Array of starting logical coordinates of chunk. + * uint32_t filter_mask; IN: Filter mask of chunk. + * haddr_t addr; IN: Offset in file of chunk data. + * uint32_t nbytes; IN: Size in number of bytes of chunk data in file. + * void *op_data; IN/OUT: Pointer to any user-defined data + * associated with the operation. + * + * The return values from an operator are: + * Zero (H5_ITER_CONT) causes the iterator to continue, returning zero when all + * elements have been processed. + * Positive (H5_ITER_STOP) causes the iterator to immediately return that positive + * value, indicating short-circuit success. + * Negative (H5_ITER_ERROR) causes the iterator to immediately return that value, + * indicating failure. + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Gaute Hope + * August 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dchunk_iter(hid_t dset_id, H5D_chunk_iter_op_t cb, void *op_data) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ix*x", dset_id, cb, op_data); + + /* Check arguments */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier") + + /* Call private function to get the chunk info given the chunk's index */ + if (H5VL_dataset_specific(vol_obj, H5VL_DATASET_CHUNK_ITER, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, cb, + op_data) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't iterate over chunks") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dchunk_iter() */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index f335a25886d..2486774b81e 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -245,6 +245,11 @@ typedef struct H5D_chunk_coll_info_t { } H5D_chunk_coll_info_t; #endif /* H5_HAVE_PARALLEL */ +typedef struct H5D_chunk_iter_cb_data_t { + H5D_chunk_iter_op_t cb; /* User defined callback */ + void * op_data; /* User data for user defined callback */ +} H5D_chunk_iter_cb_data_t; + /********************/ /* Local Prototypes */ /********************/ @@ -269,6 +274,7 @@ static herr_t H5D__chunk_dest(H5D_t *dset); static int H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); static int H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); static int H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); +static int H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata); /* "Nonexistent" layout operation callback */ static ssize_t H5D__nonexistent_readvv(const H5D_io_info_t *io_info, size_t chunk_max_nseq, @@ -7451,3 +7457,95 @@ H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned done: FUNC_LEAVE_NOAPI_TAG(ret_value) } /* end H5D__get_chunk_info_by_coord() */ + +/*------------------------------------------------------------------------- + * Function: H5D__chunk_iter + * + * Purpose: Iterate over all the chunks in the dataset with given callbak. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Gaute Hope + * August 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__chunk_iter(const H5D_t *dset, H5D_chunk_iter_op_t cb, void *op_data) +{ + const H5O_layout_t *layout = NULL; /* Dataset layout */ + const H5D_rdcc_t * rdcc = NULL; /* Raw data chunk cache */ + H5D_rdcc_ent_t * ent; /* Cache entry index */ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + /* Check args */ + HDassert(dset); + HDassert(dset->shared); + + /* Get dataset layout and raw data chunk cache */ + layout = &(dset->shared->layout); + rdcc = &(dset->shared->cache.chunk); + HDassert(layout); + HDassert(rdcc); + HDassert(H5D_CHUNKED == layout->type); + + /* Search for cached chunks that haven't been written out */ + for (ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if (H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + /* If the dataset is not written, return without errors */ + if (H5F_addr_defined(idx_info.storage->idx_addr)) { + H5D_chunk_iter_cb_data_t data; + data.cb = cb; + data.op_data = op_data; + + /* Iterate over the allocated chunks calling the iterator callback */ + if ((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__chunk_iter_cb, &data) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to iterate over chunks.") + } /* end if H5F_addr_defined */ + +done: + FUNC_LEAVE_NOAPI_TAG(ret_value) +} /* end H5D__chunk_iter() */ + +/*------------------------------------------------------------------------- + * Function: H5D__chunk_iter_cb + * + * Purpose: Call the user-defined function with the chunk data. The iterator continues if + * the user-defined function returns H5_ITER_CONT, and stops if H5_ITER_STOP is + * returned. + * + * Return: Success: H5_ITER_CONT or H5_ITER_STOP + * Failure: Negative (H5_ITER_ERROR) + * + * Programmer: Gaute Hope + * August 2020 + * + *------------------------------------------------------------------------- + */ +static int +H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata) +{ + int ret_value = 0; + + FUNC_ENTER_STATIC_NOERR + + const H5D_chunk_iter_cb_data_t *data = (H5D_chunk_iter_cb_data_t *)udata; + + ret_value = (data->cb)(chunk_rec->scaled, chunk_rec->filter_mask, chunk_rec->chunk_addr, + chunk_rec->nbytes, data->op_data); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__chunk_iter_cb */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 64f2c7a263f..28561c9ae4a 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -565,6 +565,7 @@ H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_ unsigned *filter_mask, haddr_t *offset, hsize_t *size); H5_DLL herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); +H5_DLL herr_t H5D__chunk_iter(const H5D_t *dset, H5D_chunk_iter_op_t cb, void *op_data); H5_DLL haddr_t H5D__get_offset(const H5D_t *dset); H5_DLL herr_t H5D__vlen_get_buf_size(H5D_t *dset, hid_t type_id, hid_t space_id, hsize_t *size); H5_DLL herr_t H5D__vlen_get_buf_size_gen(H5VL_object_t *vol_obj, hid_t type_id, hid_t space_id, diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index b73e62dc343..1f8598e7b1c 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -160,6 +160,14 @@ typedef herr_t (*H5D_scatter_func_t)(const void **src_buf /*out*/, size_t *src_b typedef herr_t (*H5D_gather_func_t)(const void *dst_buf, size_t dst_buf_bytes_used, void *op_data); //! +//! +/** + * Define the operator function pointer for H5Dchunk_iter() + */ +//! +typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbytes, + void *op_data); + /********************/ /* Public Variables */ /********************/ @@ -601,6 +609,51 @@ H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size); +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Iterate over all chunks + * + * \dset_id + * \param[in] cb User callback function, called for every chunk. + * \param[in] op_data User-defined pointer to data required by op + * + * \return \herr_t + * + * \details H5Dget_chunk_iter iterates over all chunks in the dataset, calling the + * user supplied callback with the details of the chunk and the supplied + * \p op_data. + * + * Callback information: + * H5D_chunk_iter_op_t is defined as: + * + * typedef int (*H5D_chunk_iter_op_t)( + * const hsize_t *offset, + * uint32_t filter_mask, + * haddr_t addr, + * uint32_t nbytes, + * void *op_data); + * + * H5D_chunk_iter_op_t parameters: + * hsize_t *offset; IN/OUT: Array of starting logical coordinates of chunk. + * uint32_t filter_mask; IN: Filter mask of chunk. + * haddr_t addr; IN: Offset in file of chunk data. + * uint32_t nbytes; IN: Size in number of bytes of chunk data in file. + * void *op_data; IN/OUT: Pointer to any user-defined data + * associated with the operation. + * + * The return values from an operator are: + * Zero (H5_ITER_CONT) causes the iterator to continue, returning zero when all + * elements have been processed. + * Positive (H5_ITER_STOP) causes the iterator to immediately return that positive + * value, indicating short-circuit success. + * Negative (H5_ITER_ERROR) causes the iterator to immediately return that value, + * indicating failure. + * + */ +H5_DLL herr_t H5Dchunk_iter(hid_t dset_id, H5D_chunk_iter_op_t cb, void *op_data); + /** * -------------------------------------------------------------------------- * \ingroup H5D diff --git a/src/H5VLconnector.h b/src/H5VLconnector.h index de3f6b3bfb8..7f19824cf8a 100644 --- a/src/H5VLconnector.h +++ b/src/H5VLconnector.h @@ -81,7 +81,9 @@ typedef enum H5VL_dataset_get_t { typedef enum H5VL_dataset_specific_t { H5VL_DATASET_SET_EXTENT, /* H5Dset_extent */ H5VL_DATASET_FLUSH, /* H5Dflush */ - H5VL_DATASET_REFRESH /* H5Drefresh */ + H5VL_DATASET_REFRESH, /* H5Drefresh */ + H5VL_DATASET_WAIT, /* H5Dwait */ + H5VL_DATASET_CHUNK_ITER /* H5Dchunk_iter */ } H5VL_dataset_specific_t; /* Typedef for VOL connector dataset optional VOL operations */ diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index f23267ee91b..21491e706ff 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -346,6 +346,31 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_t specific_type, break; } + case H5VL_DATASET_WAIT: { /* H5Dwait */ + /* The native VOL connector doesn't support asynchronous + * operations, so this is a no-op. + */ + break; + } + + case H5VL_DATASET_CHUNK_ITER: { /* H5Dchunk_iter */ + H5D_chunk_iter_op_t cb = HDva_arg(arguments, H5D_chunk_iter_op_t); + void * op_data = HDva_arg(arguments, void *); + + HDassert(dset->shared); + + /* Make sure the dataset is chunked */ + if (H5D_CHUNKED != dset->shared->layout.type) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + } + + /* Call private function */ + if (H5D__chunk_iter(dset, cb, op_data) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't iterate over chunks") + + break; + } + default: HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "invalid specific operation") } /* end switch */ diff --git a/test/chunk_info.c b/test/chunk_info.c index 7f395579494..b4b91acdb1d 100644 --- a/test/chunk_info.c +++ b/test/chunk_info.c @@ -1482,6 +1482,50 @@ test_chunk_info_version2_btrees(const char *filename, hid_t fapl) return FAIL; } /* test_chunk_info_version2_btrees() */ +typedef struct chunk_iter_info_t { + hsize_t offset[2]; + uint32_t filter_mask; + haddr_t addr; + uint32_t nbytes; +} chunk_iter_info_t; + +static int +iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbytes, void *op_data) +{ + chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; + + (*chunk_info)->offset[0] = offset[0]; + (*chunk_info)->offset[1] = offset[1]; + (*chunk_info)->filter_mask = filter_mask; + (*chunk_info)->addr = addr; + (*chunk_info)->nbytes = nbytes; + + /* printf("offset: [%lld, %lld], addr: %ld, size: %d, filter mask: %d\n", offset[0], offset[1], addr, + * nbytes, filter_mask); */ + + *chunk_info += 1; + + return H5_ITER_CONT; +} + +static int +iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filter_mask, + haddr_t H5_ATTR_UNUSED addr, uint32_t H5_ATTR_UNUSED nbytes, void *op_data) +{ + chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; + *chunk_info += 1; + return H5_ITER_STOP; +} + +static int +iter_cb_fail(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filter_mask, + haddr_t H5_ATTR_UNUSED addr, uint32_t H5_ATTR_UNUSED nbytes, void *op_data) +{ + chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; + *chunk_info += 1; + return H5_ITER_ERROR; +} + /*------------------------------------------------------------------------- * Function: test_basic_query * @@ -1502,24 +1546,26 @@ test_chunk_info_version2_btrees(const char *filename, hid_t fapl) static herr_t test_basic_query(hid_t fapl) { - char filename[FILENAME_BUF_SIZE]; /* File name */ - hid_t basicfile = H5I_INVALID_HID; /* File ID */ - hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ - hid_t dset = H5I_INVALID_HID; /* Dataset ID */ - hid_t cparms = H5I_INVALID_HID; /* Creation plist */ - hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ - hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ - int direct_buf[CHUNK_NX][CHUNK_NY]; /* Data in chunks */ - unsigned flt_msk = 0; /* Filter mask */ - unsigned read_flt_msk = 0; /* Filter mask after direct read */ - hsize_t offset[2]; /* Offset coordinates of a chunk */ - hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ - hsize_t size = 0; /* Size of an allocated/written chunk */ - hsize_t nchunks = 0; /* Number of chunks */ - haddr_t addr = 0; /* Address of an allocated/written chunk */ - hsize_t chk_index = 0; /* Index of a chunk */ - hsize_t ii, jj; /* Array indices */ - herr_t ret; /* Temporary returned value for verifying failure */ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t basicfile = H5I_INVALID_HID; /* File ID */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hid_t cparms = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + int direct_buf[CHUNK_NX][CHUNK_NY]; /* Data in chunks */ + unsigned flt_msk = 0; /* Filter mask */ + unsigned read_flt_msk = 0; /* Filter mask after direct read */ + hsize_t offset[2]; /* Offset coordinates of a chunk */ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + hsize_t chk_index = 0; /* Index of a chunk */ + hsize_t ii, jj; /* Array indices */ + chunk_iter_info_t chunk_infos[2]; /* chunk infos filled up by iterator */ + chunk_iter_info_t *cptr; /* pointer to array of chunks */ + herr_t ret; /* Temporary returned value for verifying failure */ TESTING("basic operations"); @@ -1627,6 +1673,37 @@ test_basic_query(hid_t fapl) if (verify_empty_chunk_info(dset, offset) == FAIL) FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + /* iterate over all chunks */ + cptr = &(chunk_infos[0]); + if (H5Dchunk_iter(dset, &iter_cb, &cptr) < 0) + TEST_ERROR; + + VERIFY(cptr, &(chunk_infos[2]), "Iterator did not iterate all chunks"); + VERIFY(chunk_infos[0].offset[0], 0, "Offset mismatch"); + VERIFY(chunk_infos[0].offset[1], 0, "Offset mismatch"); + VERIFY(chunk_infos[0].filter_mask, 0, "Filter mismatch"); + VERIFY(chunk_infos[0].nbytes, 96, "Size mismatch"); + + VERIFY(chunk_infos[1].offset[0], 1, "Offset mismatch"); + VERIFY(chunk_infos[1].offset[1], 1, "Offset mismatch"); + + /* iterate and stop after one iteration */ + cptr = &(chunk_infos[0]); + if (H5Dchunk_iter(dset, &iter_cb_stop, &cptr) < 0) + TEST_ERROR; + VERIFY(cptr, &(chunk_infos[1]), "Verification of halted iterator failed\n"); + + /* iterate and fail after one iteration */ + cptr = &(chunk_infos[0]); + H5E_BEGIN_TRY + { + ret = H5Dchunk_iter(dset, &iter_cb_fail, &cptr); + } + H5E_END_TRY; + if (ret >= 0) + TEST_ERROR; + VERIFY(cptr, &(chunk_infos[1]), "Verification of halted iterator failed\n"); + /* Release resourse */ if (H5Dclose(dset) < 0) TEST_ERROR From d739e46d55f8c40b9333c57e9ae4f83c03572a27 Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Wed, 19 May 2021 21:02:02 -0500 Subject: [PATCH 05/12] Set initial doxygen version for H5Dchunk_iter(). --- src/H5Dpublic.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 1f8598e7b1c..040e136f67a 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -651,6 +651,8 @@ H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, u * Negative (H5_ITER_ERROR) causes the iterator to immediately return that value, * indicating failure. * + * \since 1.12.1 + * */ H5_DLL herr_t H5Dchunk_iter(hid_t dset_id, H5D_chunk_iter_op_t cb, void *op_data); From 5cee957ebf806682bfd5e6c38f8b62a1aff5fb64 Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Thu, 20 May 2021 08:38:18 -0500 Subject: [PATCH 06/12] Revert "Set initial doxygen version for H5Dchunk_iter()." This reverts commit d739e46d55f8c40b9333c57e9ae4f83c03572a27. --- src/H5Dpublic.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 040e136f67a..1f8598e7b1c 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -651,8 +651,6 @@ H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, u * Negative (H5_ITER_ERROR) causes the iterator to immediately return that value, * indicating failure. * - * \since 1.12.1 - * */ H5_DLL herr_t H5Dchunk_iter(hid_t dset_id, H5D_chunk_iter_op_t cb, void *op_data); From 7226b9f02804729081fd2ac9246270cefe1baa7c Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Thu, 20 May 2021 08:38:32 -0500 Subject: [PATCH 07/12] Revert "Add H5Dchunk_iter method for iterating over chunks (#6)" This reverts commit 53e113977a775a9311e7df015937f71b11f76f7b. --- bin/trace | 1 - fortran/src/Makefile.in | 18 +++---- release_docs/RELEASE.txt | 15 ------ src/H5D.c | 65 ---------------------- src/H5Dchunk.c | 98 --------------------------------- src/H5Dpkg.h | 1 - src/H5Dpublic.h | 53 ------------------ src/H5VLconnector.h | 4 +- src/H5VLnative_dataset.c | 25 --------- test/chunk_info.c | 113 +++++++-------------------------------- 10 files changed, 28 insertions(+), 365 deletions(-) diff --git a/bin/trace b/bin/trace index 8cad5acb904..2b519e41cbc 100755 --- a/bin/trace +++ b/bin/trace @@ -44,7 +44,6 @@ $Source = ""; "H5D_vds_view_t" => "Dv", "H5FD_mpio_xfer_t" => "Dt", "H5FD_splitter_vfd_config_t" => "Dr", - "H5D_chunk_iter_op_t" => "x", "herr_t" => "e", "H5E_direction_t" => "Ed", "H5E_error_t" => "Ee", diff --git a/fortran/src/Makefile.in b/fortran/src/Makefile.in index 0a123edebb2..0f8472ca1b5 100644 --- a/fortran/src/Makefile.in +++ b/fortran/src/Makefile.in @@ -839,9 +839,9 @@ LT_VERS_REVISION = 0 LT_CXX_VERS_INTERFACE = 201 LT_CXX_VERS_REVISION = 0 LT_CXX_VERS_AGE = 1 -LT_F_VERS_INTERFACE = 201 -LT_F_VERS_REVISION = 0 -LT_F_VERS_AGE = 1 +LT_F_VERS_INTERFACE = 200 +LT_F_VERS_REVISION = 1 +LT_F_VERS_AGE = 0 LT_HL_VERS_INTERFACE = 200 LT_HL_VERS_REVISION = 1 LT_HL_VERS_AGE = 0 @@ -851,12 +851,12 @@ LT_HL_CXX_VERS_AGE = 0 LT_HL_F_VERS_INTERFACE = 200 LT_HL_F_VERS_REVISION = 1 LT_HL_F_VERS_AGE = 0 -LT_JAVA_VERS_INTERFACE = 201 -LT_JAVA_VERS_REVISION = 0 -LT_JAVA_VERS_AGE = 1 -LT_TOOLS_VERS_INTERFACE = 201 -LT_TOOLS_VERS_REVISION = 0 -LT_TOOLS_VERS_AGE = 1 +LT_JAVA_VERS_INTERFACE = 200 +LT_JAVA_VERS_REVISION = 1 +LT_JAVA_VERS_AGE = 0 +LT_TOOLS_VERS_INTERFACE = 200 +LT_TOOLS_VERS_REVISION = 1 +LT_TOOLS_VERS_AGE = 0 AM_FCLIBS = $(LIBHDF5) # This is our main target, the fortran library diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 6a0761eff4c..611f59882f1 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -309,21 +309,6 @@ New Features Library: -------- - - H5Dchunk_iter() added for iterating over chunks - - This method iterates over all chunks in a dataset, calling a user-supplied - callback with the chunk information and optional user supplied data. - - The iterator is stopped when ITER_STOP is returned by the user-supplied - callback or the iterator is exhausted. - - Existing methods to get chunk_info perform an iteration each time, so - whereas getting many or all chunks with existing methods caused SUM(i) - for i = 0 -> N operations for N chunks, using this iterator for this - use case will get N chunks with N operations. - - (2021/05/19) - - H5Gcreate1() now rejects size_hint parameters larger than UINT32_MAX The size_hint value is ultimately stored in a uint32_t struct field, diff --git a/src/H5D.c b/src/H5D.c index c0ac2a064eb..b69391fad8f 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -1126,68 +1126,3 @@ H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filte done: FUNC_LEAVE_API(ret_value) } /* end H5Dget_chunk_info_by_coord() */ - -/*------------------------------------------------------------------------- - * Function: H5Dchunk_iter - * - * Purpose: Iterates over all chunks in dataset with given callback and user data. - * - * Parameters: - * hid_t dset_id; IN: Chunked dataset ID - * H5D_chunk_iter_op_t cb IN: User callback function, called for every chunk. - * void *op_data IN/OUT: Optional user data passed on to user callback. - * - * Callback information: - * H5D_chunk_iter_op_t is defined as: - * - * typedef int (*H5D_chunk_iter_op_t)( - * const hsize_t *offset, - * uint32_t filter_mask, - * haddr_t addr, - * uint32_t nbytes, - * void *op_data); - * - * H5D_chunk_iter_op_t parameters: - * hsize_t *offset; IN/OUT: Array of starting logical coordinates of chunk. - * uint32_t filter_mask; IN: Filter mask of chunk. - * haddr_t addr; IN: Offset in file of chunk data. - * uint32_t nbytes; IN: Size in number of bytes of chunk data in file. - * void *op_data; IN/OUT: Pointer to any user-defined data - * associated with the operation. - * - * The return values from an operator are: - * Zero (H5_ITER_CONT) causes the iterator to continue, returning zero when all - * elements have been processed. - * Positive (H5_ITER_STOP) causes the iterator to immediately return that positive - * value, indicating short-circuit success. - * Negative (H5_ITER_ERROR) causes the iterator to immediately return that value, - * indicating failure. - * - * Return: Non-negative on success, negative on failure - * - * Programmer: Gaute Hope - * August 2020 - * - *------------------------------------------------------------------------- - */ -herr_t -H5Dchunk_iter(hid_t dset_id, H5D_chunk_iter_op_t cb, void *op_data) -{ - H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_API(FAIL) - H5TRACE3("e", "ix*x", dset_id, cb, op_data); - - /* Check arguments */ - if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier") - - /* Call private function to get the chunk info given the chunk's index */ - if (H5VL_dataset_specific(vol_obj, H5VL_DATASET_CHUNK_ITER, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, cb, - op_data) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't iterate over chunks") - -done: - FUNC_LEAVE_API(ret_value) -} /* end H5Dchunk_iter() */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 2486774b81e..f335a25886d 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -245,11 +245,6 @@ typedef struct H5D_chunk_coll_info_t { } H5D_chunk_coll_info_t; #endif /* H5_HAVE_PARALLEL */ -typedef struct H5D_chunk_iter_cb_data_t { - H5D_chunk_iter_op_t cb; /* User defined callback */ - void * op_data; /* User data for user defined callback */ -} H5D_chunk_iter_cb_data_t; - /********************/ /* Local Prototypes */ /********************/ @@ -274,7 +269,6 @@ static herr_t H5D__chunk_dest(H5D_t *dset); static int H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); static int H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); static int H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); -static int H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata); /* "Nonexistent" layout operation callback */ static ssize_t H5D__nonexistent_readvv(const H5D_io_info_t *io_info, size_t chunk_max_nseq, @@ -7457,95 +7451,3 @@ H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned done: FUNC_LEAVE_NOAPI_TAG(ret_value) } /* end H5D__get_chunk_info_by_coord() */ - -/*------------------------------------------------------------------------- - * Function: H5D__chunk_iter - * - * Purpose: Iterate over all the chunks in the dataset with given callbak. - * - * Return: Success: Non-negative - * Failure: Negative - * - * Programmer: Gaute Hope - * August 2020 - * - *------------------------------------------------------------------------- - */ -herr_t -H5D__chunk_iter(const H5D_t *dset, H5D_chunk_iter_op_t cb, void *op_data) -{ - const H5O_layout_t *layout = NULL; /* Dataset layout */ - const H5D_rdcc_t * rdcc = NULL; /* Raw data chunk cache */ - H5D_rdcc_ent_t * ent; /* Cache entry index */ - H5D_chk_idx_info_t idx_info; /* Chunked index info */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) - - /* Check args */ - HDassert(dset); - HDassert(dset->shared); - - /* Get dataset layout and raw data chunk cache */ - layout = &(dset->shared->layout); - rdcc = &(dset->shared->cache.chunk); - HDassert(layout); - HDassert(rdcc); - HDassert(H5D_CHUNKED == layout->type); - - /* Search for cached chunks that haven't been written out */ - for (ent = rdcc->head; ent; ent = ent->next) - /* Flush the chunk out to disk, to make certain the size is correct later */ - if (H5D__chunk_flush_entry(dset, ent, FALSE) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") - - /* Compose chunked index info struct */ - idx_info.f = dset->oloc.file; - idx_info.pline = &dset->shared->dcpl_cache.pline; - idx_info.layout = &dset->shared->layout.u.chunk; - idx_info.storage = &dset->shared->layout.storage.u.chunk; - - /* If the dataset is not written, return without errors */ - if (H5F_addr_defined(idx_info.storage->idx_addr)) { - H5D_chunk_iter_cb_data_t data; - data.cb = cb; - data.op_data = op_data; - - /* Iterate over the allocated chunks calling the iterator callback */ - if ((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__chunk_iter_cb, &data) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to iterate over chunks.") - } /* end if H5F_addr_defined */ - -done: - FUNC_LEAVE_NOAPI_TAG(ret_value) -} /* end H5D__chunk_iter() */ - -/*------------------------------------------------------------------------- - * Function: H5D__chunk_iter_cb - * - * Purpose: Call the user-defined function with the chunk data. The iterator continues if - * the user-defined function returns H5_ITER_CONT, and stops if H5_ITER_STOP is - * returned. - * - * Return: Success: H5_ITER_CONT or H5_ITER_STOP - * Failure: Negative (H5_ITER_ERROR) - * - * Programmer: Gaute Hope - * August 2020 - * - *------------------------------------------------------------------------- - */ -static int -H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata) -{ - int ret_value = 0; - - FUNC_ENTER_STATIC_NOERR - - const H5D_chunk_iter_cb_data_t *data = (H5D_chunk_iter_cb_data_t *)udata; - - ret_value = (data->cb)(chunk_rec->scaled, chunk_rec->filter_mask, chunk_rec->chunk_addr, - chunk_rec->nbytes, data->op_data); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__chunk_iter_cb */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 28561c9ae4a..64f2c7a263f 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -565,7 +565,6 @@ H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_ unsigned *filter_mask, haddr_t *offset, hsize_t *size); H5_DLL herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); -H5_DLL herr_t H5D__chunk_iter(const H5D_t *dset, H5D_chunk_iter_op_t cb, void *op_data); H5_DLL haddr_t H5D__get_offset(const H5D_t *dset); H5_DLL herr_t H5D__vlen_get_buf_size(H5D_t *dset, hid_t type_id, hid_t space_id, hsize_t *size); H5_DLL herr_t H5D__vlen_get_buf_size_gen(H5VL_object_t *vol_obj, hid_t type_id, hid_t space_id, diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 1f8598e7b1c..b73e62dc343 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -160,14 +160,6 @@ typedef herr_t (*H5D_scatter_func_t)(const void **src_buf /*out*/, size_t *src_b typedef herr_t (*H5D_gather_func_t)(const void *dst_buf, size_t dst_buf_bytes_used, void *op_data); //! -//! -/** - * Define the operator function pointer for H5Dchunk_iter() - */ -//! -typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbytes, - void *op_data); - /********************/ /* Public Variables */ /********************/ @@ -609,51 +601,6 @@ H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size); -/** - * -------------------------------------------------------------------------- - * \ingroup H5D - * - * \brief Iterate over all chunks - * - * \dset_id - * \param[in] cb User callback function, called for every chunk. - * \param[in] op_data User-defined pointer to data required by op - * - * \return \herr_t - * - * \details H5Dget_chunk_iter iterates over all chunks in the dataset, calling the - * user supplied callback with the details of the chunk and the supplied - * \p op_data. - * - * Callback information: - * H5D_chunk_iter_op_t is defined as: - * - * typedef int (*H5D_chunk_iter_op_t)( - * const hsize_t *offset, - * uint32_t filter_mask, - * haddr_t addr, - * uint32_t nbytes, - * void *op_data); - * - * H5D_chunk_iter_op_t parameters: - * hsize_t *offset; IN/OUT: Array of starting logical coordinates of chunk. - * uint32_t filter_mask; IN: Filter mask of chunk. - * haddr_t addr; IN: Offset in file of chunk data. - * uint32_t nbytes; IN: Size in number of bytes of chunk data in file. - * void *op_data; IN/OUT: Pointer to any user-defined data - * associated with the operation. - * - * The return values from an operator are: - * Zero (H5_ITER_CONT) causes the iterator to continue, returning zero when all - * elements have been processed. - * Positive (H5_ITER_STOP) causes the iterator to immediately return that positive - * value, indicating short-circuit success. - * Negative (H5_ITER_ERROR) causes the iterator to immediately return that value, - * indicating failure. - * - */ -H5_DLL herr_t H5Dchunk_iter(hid_t dset_id, H5D_chunk_iter_op_t cb, void *op_data); - /** * -------------------------------------------------------------------------- * \ingroup H5D diff --git a/src/H5VLconnector.h b/src/H5VLconnector.h index 7f19824cf8a..de3f6b3bfb8 100644 --- a/src/H5VLconnector.h +++ b/src/H5VLconnector.h @@ -81,9 +81,7 @@ typedef enum H5VL_dataset_get_t { typedef enum H5VL_dataset_specific_t { H5VL_DATASET_SET_EXTENT, /* H5Dset_extent */ H5VL_DATASET_FLUSH, /* H5Dflush */ - H5VL_DATASET_REFRESH, /* H5Drefresh */ - H5VL_DATASET_WAIT, /* H5Dwait */ - H5VL_DATASET_CHUNK_ITER /* H5Dchunk_iter */ + H5VL_DATASET_REFRESH /* H5Drefresh */ } H5VL_dataset_specific_t; /* Typedef for VOL connector dataset optional VOL operations */ diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index 21491e706ff..f23267ee91b 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -346,31 +346,6 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_t specific_type, break; } - case H5VL_DATASET_WAIT: { /* H5Dwait */ - /* The native VOL connector doesn't support asynchronous - * operations, so this is a no-op. - */ - break; - } - - case H5VL_DATASET_CHUNK_ITER: { /* H5Dchunk_iter */ - H5D_chunk_iter_op_t cb = HDva_arg(arguments, H5D_chunk_iter_op_t); - void * op_data = HDva_arg(arguments, void *); - - HDassert(dset->shared); - - /* Make sure the dataset is chunked */ - if (H5D_CHUNKED != dset->shared->layout.type) { - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") - } - - /* Call private function */ - if (H5D__chunk_iter(dset, cb, op_data) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't iterate over chunks") - - break; - } - default: HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "invalid specific operation") } /* end switch */ diff --git a/test/chunk_info.c b/test/chunk_info.c index b4b91acdb1d..7f395579494 100644 --- a/test/chunk_info.c +++ b/test/chunk_info.c @@ -1482,50 +1482,6 @@ test_chunk_info_version2_btrees(const char *filename, hid_t fapl) return FAIL; } /* test_chunk_info_version2_btrees() */ -typedef struct chunk_iter_info_t { - hsize_t offset[2]; - uint32_t filter_mask; - haddr_t addr; - uint32_t nbytes; -} chunk_iter_info_t; - -static int -iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbytes, void *op_data) -{ - chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; - - (*chunk_info)->offset[0] = offset[0]; - (*chunk_info)->offset[1] = offset[1]; - (*chunk_info)->filter_mask = filter_mask; - (*chunk_info)->addr = addr; - (*chunk_info)->nbytes = nbytes; - - /* printf("offset: [%lld, %lld], addr: %ld, size: %d, filter mask: %d\n", offset[0], offset[1], addr, - * nbytes, filter_mask); */ - - *chunk_info += 1; - - return H5_ITER_CONT; -} - -static int -iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filter_mask, - haddr_t H5_ATTR_UNUSED addr, uint32_t H5_ATTR_UNUSED nbytes, void *op_data) -{ - chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; - *chunk_info += 1; - return H5_ITER_STOP; -} - -static int -iter_cb_fail(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filter_mask, - haddr_t H5_ATTR_UNUSED addr, uint32_t H5_ATTR_UNUSED nbytes, void *op_data) -{ - chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; - *chunk_info += 1; - return H5_ITER_ERROR; -} - /*------------------------------------------------------------------------- * Function: test_basic_query * @@ -1546,26 +1502,24 @@ iter_cb_fail(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filte static herr_t test_basic_query(hid_t fapl) { - char filename[FILENAME_BUF_SIZE]; /* File name */ - hid_t basicfile = H5I_INVALID_HID; /* File ID */ - hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ - hid_t dset = H5I_INVALID_HID; /* Dataset ID */ - hid_t cparms = H5I_INVALID_HID; /* Creation plist */ - hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ - hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ - int direct_buf[CHUNK_NX][CHUNK_NY]; /* Data in chunks */ - unsigned flt_msk = 0; /* Filter mask */ - unsigned read_flt_msk = 0; /* Filter mask after direct read */ - hsize_t offset[2]; /* Offset coordinates of a chunk */ - hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ - hsize_t size = 0; /* Size of an allocated/written chunk */ - hsize_t nchunks = 0; /* Number of chunks */ - haddr_t addr = 0; /* Address of an allocated/written chunk */ - hsize_t chk_index = 0; /* Index of a chunk */ - hsize_t ii, jj; /* Array indices */ - chunk_iter_info_t chunk_infos[2]; /* chunk infos filled up by iterator */ - chunk_iter_info_t *cptr; /* pointer to array of chunks */ - herr_t ret; /* Temporary returned value for verifying failure */ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t basicfile = H5I_INVALID_HID; /* File ID */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hid_t cparms = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + int direct_buf[CHUNK_NX][CHUNK_NY]; /* Data in chunks */ + unsigned flt_msk = 0; /* Filter mask */ + unsigned read_flt_msk = 0; /* Filter mask after direct read */ + hsize_t offset[2]; /* Offset coordinates of a chunk */ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + hsize_t chk_index = 0; /* Index of a chunk */ + hsize_t ii, jj; /* Array indices */ + herr_t ret; /* Temporary returned value for verifying failure */ TESTING("basic operations"); @@ -1673,37 +1627,6 @@ test_basic_query(hid_t fapl) if (verify_empty_chunk_info(dset, offset) == FAIL) FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); - /* iterate over all chunks */ - cptr = &(chunk_infos[0]); - if (H5Dchunk_iter(dset, &iter_cb, &cptr) < 0) - TEST_ERROR; - - VERIFY(cptr, &(chunk_infos[2]), "Iterator did not iterate all chunks"); - VERIFY(chunk_infos[0].offset[0], 0, "Offset mismatch"); - VERIFY(chunk_infos[0].offset[1], 0, "Offset mismatch"); - VERIFY(chunk_infos[0].filter_mask, 0, "Filter mismatch"); - VERIFY(chunk_infos[0].nbytes, 96, "Size mismatch"); - - VERIFY(chunk_infos[1].offset[0], 1, "Offset mismatch"); - VERIFY(chunk_infos[1].offset[1], 1, "Offset mismatch"); - - /* iterate and stop after one iteration */ - cptr = &(chunk_infos[0]); - if (H5Dchunk_iter(dset, &iter_cb_stop, &cptr) < 0) - TEST_ERROR; - VERIFY(cptr, &(chunk_infos[1]), "Verification of halted iterator failed\n"); - - /* iterate and fail after one iteration */ - cptr = &(chunk_infos[0]); - H5E_BEGIN_TRY - { - ret = H5Dchunk_iter(dset, &iter_cb_fail, &cptr); - } - H5E_END_TRY; - if (ret >= 0) - TEST_ERROR; - VERIFY(cptr, &(chunk_infos[1]), "Verification of halted iterator failed\n"); - /* Release resourse */ if (H5Dclose(dset) < 0) TEST_ERROR From 5df14b4dd7b3da4f3f8618f64209b21da27c909f Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Fri, 21 May 2021 09:44:46 -0500 Subject: [PATCH 08/12] Add flushrefresh errors with check-passthrough-vol to known problems in RELEASE.txt. --- release_docs/RELEASE.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 6a0761eff4c..96a15077cd9 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -920,6 +920,10 @@ The following platforms are not supported but have been tested for this release. Known Problems ============== + "make check-passthough-vol" will fail running testflushrefresh.sh, with + 3 Errors/Segmentation faults. These will not occur when running "make + check". See https://github.com/HDFGroup/hdf5/issues/673 for details. + The t_bigio test fails on several HPC platforms, generally by timeout with OpenMPI 4.0.0 or with this error from spectrum-mpi: *** on communicator MPI_COMM_WORLD From bd0cc9ab94b258e1e958ef52483d3c9227e1ac99 Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Fri, 21 May 2021 16:44:18 -0500 Subject: [PATCH 09/12] Minor RELEASE.txt updates. --- release_docs/RELEASE.txt | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 96a15077cd9..e1d61cea3cc 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -93,7 +93,7 @@ New Features (2021/03/22) - - Add CMake variable HDF5_LIB_INFIX + - Added CMake variable HDF5_LIB_INFIX This infix is added to all library names after 'hdf5'. e.g. the infix '_openmpi' results in the library name 'libhdf5_openmpi.so' @@ -206,7 +206,7 @@ New Features - CMake option to use MSVC naming conventions with MinGW - HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions + HDF5_MSVC_NAMING_CONVENTION option enables use of MSVC naming conventions when using a MinGW toolchain (2020/10/30) @@ -309,21 +309,6 @@ New Features Library: -------- - - H5Dchunk_iter() added for iterating over chunks - - This method iterates over all chunks in a dataset, calling a user-supplied - callback with the chunk information and optional user supplied data. - - The iterator is stopped when ITER_STOP is returned by the user-supplied - callback or the iterator is exhausted. - - Existing methods to get chunk_info perform an iteration each time, so - whereas getting many or all chunks with existing methods caused SUM(i) - for i = 0 -> N operations for N chunks, using this iterator for this - use case will get N chunks with N operations. - - (2021/05/19) - - H5Gcreate1() now rejects size_hint parameters larger than UINT32_MAX The size_hint value is ultimately stored in a uint32_t struct field, From 74cd6bf4161cde608e09cb99f4c2e0d05e44bfcc Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Fri, 21 May 2021 17:09:18 -0500 Subject: [PATCH 10/12] Hdf5 1 12 Remove version from pkgcfg file names (#672) * OESS-98 fix tools test for plugins * sync fork * Merge of changes from dev * Move problem option to bottom of the list until fixed * HDFFV-11106 - fix parsing optional args * HDFFV-11106 add note * grammer fix * Whitespace after clang formatting * Undo format version 11 changes * Update check to working version * Merge workflow and minor changes from develop * Update supported platforms * PR#3 merge from develop * Merge gcc 10 diagnostics option from develop * Merge #318 OSX changes from develop * Merge serval small changes from dev * fix typo * Minor non-space formatting changes * GH #386 copyright corrections for java folder * revert because logic requires false return * Merges from develop #358 patches from vtk #361 fix header guard spelling * Remove case statement for H5I_EVENTSET * Correct call with versioning * Remove tabs * Double underscore change * Merges from develop #340 clang -Wformat-security warnings #360 Fixed uninitialized warnings Remove more underscores from header guards * Merge #380 from develop * Correct date entry * Split format source and commit changes on repo push * remove pre-split setting * Change windows TS to use older VS. * HDFFV-11212 JNI export util and Javadoc * Suggested review changes * Another change found * Committing clang-format changes * Some Javadoc warning fixes * Committing clang-format changes * Updated javadoc fixes * HDFFV-11228/9 merges from develop * remove obsolete debug comment * Fix conflict * HDFFV-11229 merge changes from develop * HDFFV-11229 merge second compare from develop * HDFFV-11229 fix reference file * HDFFV-11229 update autotools test script for two ref files * HDFFV-11229 merge dev changes for long double display in tools * Committing clang-format changes * Update with changes from develop * Add "option" command for clang options * Rework CMake add_custom to use the BYPRODUCTS argument Update pkgconfig scripts for parallel builds. Fix install COPYING file reference. Remove unused round defines. Change CMake default setting of BUILD_CPP to off. * Whitespace changes * Rework CMake add_custom to use the BYPRODUCTS argument * Revert CMake configure checks for round defines * With VS 2015 minimum strdup is supported * Doxygen comments merged from develop * doxygen build updates * Correct version string for map functions * TRILAB-227 and tools debug merge from develop * TRILAB-227 Enable test * Quote subset args * Use MATCHES in compiler id compare, merge from dev * Revert test enable * Add file to list * doxygen version errors * if blocks needed for build modes * Update list of test platforms * Issue #669 remove version from pkgcfg filename * remove version from h5cc script Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- c++/src/CMakeLists.txt | 4 ++-- config/cmake/libh5cc.in | 2 +- fortran/src/CMakeLists.txt | 4 ++-- hl/c++/src/CMakeLists.txt | 4 ++-- hl/fortran/src/CMakeLists.txt | 4 ++-- hl/src/CMakeLists.txt | 4 ++-- src/CMakeLists.txt | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/c++/src/CMakeLists.txt b/c++/src/CMakeLists.txt index 4b1a2d66c46..835d422c71e 100644 --- a/c++/src/CMakeLists.txt +++ b/c++/src/CMakeLists.txt @@ -199,11 +199,11 @@ set (_PKG_CONFIG_REQUIRES_PRIVATE "${HDF5_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}" configure_file ( ${HDF_CONFIG_DIR}/libhdf5.pc.in - ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_CPP_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_CPP_LIB_CORENAME}.pc @ONLY ) install ( - FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_CPP_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_CPP_LIB_CORENAME}.pc DESTINATION ${HDF5_INSTALL_LIB_DIR}/pkgconfig COMPONENT cpplibraries ) diff --git a/config/cmake/libh5cc.in b/config/cmake/libh5cc.in index f7cab986e62..c98d9ca23ba 100644 --- a/config/cmake/libh5cc.in +++ b/config/cmake/libh5cc.in @@ -29,4 +29,4 @@ printf 'dir is %s\n' "$dir" export PKG_CONFIG_PATH=$dir/lib/pkgconfig -@_PKG_CONFIG_COMPILER@ `pkg-config --define-variable=prefix=$dir --cflags --libs @_PKG_CONFIG_LIBNAME@-@_PKG_CONFIG_VERSION@` $@ +@_PKG_CONFIG_COMPILER@ `pkg-config --define-variable=prefix=$dir --cflags --libs @_PKG_CONFIG_LIBNAME@` $@ diff --git a/fortran/src/CMakeLists.txt b/fortran/src/CMakeLists.txt index 1a3aa15dcfd..d849086ebac 100644 --- a/fortran/src/CMakeLists.txt +++ b/fortran/src/CMakeLists.txt @@ -556,11 +556,11 @@ set (_PKG_CONFIG_REQUIRES_PRIVATE "${HDF5_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}" configure_file ( ${HDF_CONFIG_DIR}/libhdf5.pc.in - ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_F90_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_F90_LIB_CORENAME}.pc @ONLY ) install ( - FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_F90_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_F90_LIB_CORENAME}.pc DESTINATION ${HDF5_INSTALL_LIB_DIR}/pkgconfig COMPONENT fortlibraries ) diff --git a/hl/c++/src/CMakeLists.txt b/hl/c++/src/CMakeLists.txt index e4886567d10..c516df15e7e 100644 --- a/hl/c++/src/CMakeLists.txt +++ b/hl/c++/src/CMakeLists.txt @@ -110,11 +110,11 @@ set (_PKG_CONFIG_REQUIRES_PRIVATE "${HDF5_HL_LIB_CORENAME}-${HDF5_PACKAGE_VERSIO configure_file ( ${HDF_CONFIG_DIR}/libhdf5.pc.in - ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_CPP_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_CPP_LIB_CORENAME}.pc @ONLY ) install ( - FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_CPP_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_CPP_LIB_CORENAME}.pc DESTINATION ${HDF5_INSTALL_LIB_DIR}/pkgconfig COMPONENT hlcpplibraries ) diff --git a/hl/fortran/src/CMakeLists.txt b/hl/fortran/src/CMakeLists.txt index 4f0b45167b8..6c97886f9fb 100644 --- a/hl/fortran/src/CMakeLists.txt +++ b/hl/fortran/src/CMakeLists.txt @@ -343,11 +343,11 @@ set (_PKG_CONFIG_REQUIRES_PRIVATE "${HDF5_F90_LIB_CORENAME}-${HDF5_PACKAGE_VERSI configure_file ( ${HDF_CONFIG_DIR}/libhdf5.pc.in - ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_F90_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_F90_LIB_CORENAME}.pc @ONLY ) install ( - FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_F90_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_F90_LIB_CORENAME}.pc DESTINATION ${HDF5_INSTALL_LIB_DIR}/pkgconfig COMPONENT hlfortlibraries ) diff --git a/hl/src/CMakeLists.txt b/hl/src/CMakeLists.txt index 427424ea50b..7678de84001 100644 --- a/hl/src/CMakeLists.txt +++ b/hl/src/CMakeLists.txt @@ -142,11 +142,11 @@ set (_PKG_CONFIG_REQUIRES_PRIVATE "${HDF5_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}" configure_file ( ${HDF_CONFIG_DIR}/libhdf5.pc.in - ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_LIB_CORENAME}.pc @ONLY ) install ( - FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_HL_LIB_CORENAME}.pc DESTINATION ${HDF5_INSTALL_LIB_DIR}/pkgconfig COMPONENT hllibraries ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a367270a1ac..b3c22263923 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1332,11 +1332,11 @@ set (_PKG_CONFIG_REQUIRES_PRIVATE) configure_file ( ${HDF_CONFIG_DIR}/libhdf5.pc.in - ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_LIB_CORENAME}.pc @ONLY ) install ( - FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_LIB_CORENAME}-${HDF5_PACKAGE_VERSION}.pc + FILES ${HDF5_BINARY_DIR}/CMakeFiles/${HDF5_LIB_CORENAME}.pc DESTINATION ${HDF5_INSTALL_LIB_DIR}/pkgconfig COMPONENT libraries ) From 8f953736438a5245e3a52f7d3ba5a5e34b10a30f Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Fri, 21 May 2021 19:48:54 -0500 Subject: [PATCH 11/12] More minor updates. --- release_docs/RELEASE.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index e1d61cea3cc..13ce0a6fbc4 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -206,8 +206,8 @@ New Features - CMake option to use MSVC naming conventions with MinGW - HDF5_MSVC_NAMING_CONVENTION option enables use of MSVC naming conventions - when using a MinGW toolchain + HDF5_MSVC_NAMING_CONVENTION option enables the use of MSVC naming + conventions when using a MinGW toolchain (2020/10/30) @@ -905,9 +905,10 @@ The following platforms are not supported but have been tested for this release. Known Problems ============== - "make check-passthough-vol" will fail running testflushrefresh.sh, with - 3 Errors/Segmentation faults. These will not occur when running "make - check". See https://github.com/HDFGroup/hdf5/issues/673 for details. + testflushrefresh.sh will fail when run with "make check-passthough-vol" + on centos7, with 3 Errors/Segmentation faults. These will not occur when + run with "make check". See https://github.com/HDFGroup/hdf5/issues/673 + for details. The t_bigio test fails on several HPC platforms, generally by timeout with OpenMPI 4.0.0 or with this error from spectrum-mpi: From 1447be2d21ef75831606f2df45651eab85652efe Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Tue, 25 May 2021 17:00:39 -0500 Subject: [PATCH 12/12] Update configure disablhing memory alloc sanity check for release. Last 50 lines of /mnt/scr1/SnapTest/snapshots-hdf5-branchtest4/log/moohan_0525Tue_1021 Last 50 lines of /mnt/scr1/SnapTest/snapshots-hdf5-branchtest4/log/moohan_0525Tue --- configure | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/configure b/configure index 589009d165a..c9958a0a1ba 100755 --- a/configure +++ b/configure @@ -1855,8 +1855,7 @@ Optional Features: allocation sanity checking. This could cause more memory use and somewhat slower allocation. This option is orthogonal to the - --enable-using-memchecker option. [default=yes if - debug build, otherwise no] + --enable-using-memchecker option. [default=no] --enable-parallel Search for MPI-IO and MPI support files --enable-instrument Enable library instrumentation of optimization tracing (only used with parallel builds). @@ -9811,7 +9810,7 @@ else JAVA_TEST=Test.java CLASS_TEST=Test.class cat << \EOF > $JAVA_TEST -/* #line 9814 "configure" */ +/* #line 9813 "configure" */ public class Test { } EOF @@ -10027,7 +10026,7 @@ else JAVA_TEST=Test.java CLASS_TEST=Test.class cat << \EOF > $JAVA_TEST -/* #line 10030 "configure" */ +/* #line 10029 "configure" */ public class Test { } EOF @@ -10061,7 +10060,7 @@ JAVA_TEST=Test.java CLASS_TEST=Test.class TEST=Test cat << \EOF > $JAVA_TEST -/* [#]line 10064 "configure" */ +/* [#]line 10063 "configure" */ public class Test { public static void main (String args[]) { System.exit (0); @@ -10456,7 +10455,7 @@ else cat << \EOF > Test.java -/* #line 10459 "configure" */ +/* #line 10458 "configure" */ import junit.textui.TestRunner; public class Test { @@ -31692,11 +31691,7 @@ fi ## Set default if test "X-$MEMORYALLOCSANITYCHECK" = X- ; then - if test "X-$BUILD_MODE" = "X-debug" ; then - MEMORYALLOCSANITYCHECK=yes - else - MEMORYALLOCSANITYCHECK=no - fi + MEMORYALLOCSANITYCHECK=no fi case "X-$MEMORYALLOCSANITYCHECK" in