Merge hdf5_1_14 changes 07-29 to 08-04 (#3355)

* Fix loading plugin fails with missing directory GH issue #3248 (#3323) * Switch parallel compression to use vector I/O (#3245) (#3327) Updates parallel compression feature to use vector I/O instead of creating and passing down MPI derived types to VFD * Fix incorrect error check in H5Ofill.c for undefined fill values (#3312) (#3328) * Fix H5Otoken_to_str call in h5dump and other minor cleanup (#3314) (#3329) * Fix h5repack for variable-length datatyped datasets (#3331) (#3333) * Merge Made HGOTO_ERROR a do-while loop changes from develop (#3334) * Merge Fixes the last of the -Wextra-semi-stmt warnings #3326 (#3338) * Fix assertion failure in H5D__mpio_collective_filtered_vec_io (#3340) (#3350) * Merges from develop Aug 2-4 (#3354) * Fix CVE-2018-11202 (#3330) A malformed file could result in chunk index memory leaks. Under most conditions (i.e., when the --enable-using-memchecker option is NOT used), this would result in a small memory leak and and infinite loop and abort when shutting down the library. The infinite loop would be due to the "free list" package not being able to clear its resources so the library couldn't shut down. When the "using a memory checker" option is used, the free lists are disabled so there is just a memory leak with no abort on library shutdown. The chunk index resources are now correctly cleaned up when reading misparsed files and valgrind confirms no memory leaks. * Fix CVE-2018-13867 (#3336) * Windows runtime items go into the bin folder (#3320) * Commit changes to generated file generated by autogen.sh: configure, H5Einit.h, and H5Tconv.c. * Committing clang-format changes
HDFGroup · Aug 4, 2023 · f3de9ee · f3de9ee
1 parent 4cab7b0
commit f3de9ee
Show file tree

Hide file tree

Showing 394 changed files with 17,239 additions and 17,614 deletions.
diff --git a/.clang-format b/.clang-format
@@ -85,7 +85,6 @@ StatementMacros:
   - H5_GCC_DIAG_ON
   - H5_LEAVE
   - HGOTO_DONE
-  - HGOTO_ERROR
   - HMPI_DONE_ERROR
   - HMPI_ERROR
   - HMPI_GOTO_ERROR

diff --git a/bin/make_err b/bin/make_err
@@ -243,9 +243,9 @@ sub create_init ($) {
     foreach $name (keys %major) {
         print HEADER " "x(0*$indent),"assert(${name}_g==(-1));\n";
         print HEADER " "x(0*$indent),"if((msg = H5E__create_msg(cls, H5E_MAJOR, \"${major{$name}}\"))==NULL)\n";
-        print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTINIT, FAIL, \"error message initialization failed\")\n";
+        print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTINIT, FAIL, \"error message initialization failed\");\n";
         print HEADER " "x(0*$indent),"if((${name}_g = H5I_register(H5I_ERROR_MSG, msg, FALSE))<0)\n";
-        print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTREGISTER, FAIL, \"can't register error message\")\n";
+        print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTREGISTER, FAIL, \"can't register error message\");\n";
     }
 
     # Iterate over all the minor error sections
@@ -259,9 +259,9 @@ sub create_init ($) {
         for $name ( @{$section_list{$sect_name}}) {
             print HEADER " "x(0*$indent),"assert(${name}_g==(-1));\n";
             print HEADER " "x(0*$indent),"if((msg = H5E__create_msg(cls, H5E_MINOR, \"${minor{$name}}\"))==NULL)\n";
-            print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTINIT, FAIL, \"error message initialization failed\")\n";
+            print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTINIT, FAIL, \"error message initialization failed\");\n";
             print HEADER " "x(0*$indent),"if((${name}_g = H5I_register(H5I_ERROR_MSG, msg, FALSE))<0)\n";
-            print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTREGISTER, FAIL, \"can't register error message\")\n";
+            print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTREGISTER, FAIL, \"can't register error message\");\n";
         }
     }
 

diff --git a/bin/release b/bin/release
@@ -206,7 +206,7 @@ tar2cmakezip()
     cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmziptmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmziptmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-2.0.3-Source.zip $cmziptmpsubdir
-    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_1.zip $cmziptmpsubdir
+    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_2.zip $cmziptmpsubdir
     cp $cmziptmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmziptmpsubdir
     cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmziptmpsubdir
     cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmziptmpsubdir
@@ -298,7 +298,7 @@ tar2cmaketgz()
     cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmgztmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-2.0.3-Source.tar.gz $cmgztmpsubdir
-    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_1.tar.gz $cmgztmpsubdir
+    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_2.tar.gz $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmgztmpsubdir
@@ -377,7 +377,7 @@ tar2hpccmaketgz()
     cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmgztmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-2.0.3-Source.tar.gz $cmgztmpsubdir
-    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_1.tar.gz $cmgztmpsubdir
+    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_2.tar.gz $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir
 

diff --git a/configure b/configure
@@ -4615,14 +4615,21 @@ fi
 ## This allows multiple src-dir builds within one host.
 
 
-## Run post processing on files created by configure.
+## Run post processing on files created by configure
+##
 ## src/H5pubconf.h:
 ## Generate src/H5pubconf.h from src/H5config.h by prepending H5_ to all
 ## macro names. This avoid name conflict between HDF5 macro names and those
 ## generated by another software package that uses the HDF5 library.
+##
 ## src/libhdf5.settings:
-## Remove all lines begun with "#" which are generated by CONDITIONAL's of
-## configure.
+## Remove all lines beginning with "#" which are generated by CONDITIONAL's
+## of configure.
+##
+## src/H5build_settings.c
+## Remove all lines beginning with "#" which are generated by CONDITIONAL's
+## of configure. This uses a check for whitespace after the pound sign
+## to avoid clobbering include statements.
 ac_config_commands="$ac_config_commands pubconf"
 
 
@@ -10364,7 +10371,7 @@ else $as_nop
 JAVA_TEST=Test.java
 CLASS_TEST=Test.class
 cat << \EOF > $JAVA_TEST
-/* #line 10367 "configure" */
+/* #line 10374 "configure" */
 public class Test {
 }
 EOF
@@ -10603,7 +10610,7 @@ else $as_nop
 JAVA_TEST=Test.java
 CLASS_TEST=Test.class
 cat << \EOF > $JAVA_TEST
-/* #line 10606 "configure" */
+/* #line 10613 "configure" */
 public class Test {
 }
 EOF
@@ -10638,7 +10645,7 @@ JAVA_TEST=Test.java
 CLASS_TEST=Test.class
 TEST=Test
 cat << \EOF > $JAVA_TEST
-/* [#]line 10641 "configure" */
+/* [#]line 10648 "configure" */
 public class Test {
 public static void main (String args[]) {
         System.exit (0);
@@ -11063,7 +11070,7 @@ else $as_nop
 
 
 cat << \EOF > Test.java
-/* #line 11066 "configure" */
+/* #line 11073 "configure" */
 import junit.textui.TestRunner;
 public class Test {
 

diff --git a/configure.ac b/configure.ac
@@ -81,14 +81,21 @@ AM_MAINTAINER_MODE([disable])
 ## This allows multiple src-dir builds within one host.
 AC_PREFIX_DEFAULT([`pwd`/hdf5])
 
-## Run post processing on files created by configure.
+## Run post processing on files created by configure
+##
 ## src/H5pubconf.h:
 ## Generate src/H5pubconf.h from src/H5config.h by prepending H5_ to all
 ## macro names. This avoid name conflict between HDF5 macro names and those
 ## generated by another software package that uses the HDF5 library.
+##
 ## src/libhdf5.settings:
-## Remove all lines begun with "#" which are generated by CONDITIONAL's of
-## configure.
+## Remove all lines beginning with "#" which are generated by CONDITIONAL's
+## of configure.
+##
+## src/H5build_settings.c
+## Remove all lines beginning with "#" which are generated by CONDITIONAL's
+## of configure. This uses a check for whitespace after the pound sign
+## to avoid clobbering include statements.
 AC_CONFIG_COMMANDS([pubconf], [
   echo "creating src/H5pubconf.h"
   sed 's/#define /#define H5_/' <src/H5config.h |\

diff --git a/doc/getting-started-with-hdf5-development.md b/doc/getting-started-with-hdf5-development.md
@@ -270,7 +270,7 @@ H5X_do_stuff(/*parameters*/)
 	HDassert(/*parameter check*/);
 
 	if (H5X_other_call() < 0)
-		HGOTO_ERROR(H5E_MAJ, H5E_MIN, FAIL, "badness")
+		HGOTO_ERROR(H5E_MAJ, H5E_MIN, FAIL, "badness");
 
 done:
 	if (ret_value < 0)
@@ -329,12 +329,12 @@ H5Xdo_api_stuff(/*parameters*/)
 	H5TRACE3(/*stuff*/)
 
 	if (/*parameter check*/)
-		HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "badness")
+		HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "badness");
 
 	/* VOL setup */
 
 	if (H5VL_call() < 0)
-		HGOTO_ERROR(H5E_FOO, H5E_BAR, FAIL, "badness")
+		HGOTO_ERROR(H5E_FOO, H5E_BAR, FAIL, "badness");
 
 done:
 	if (ret_value < 0)
@@ -483,7 +483,7 @@ fail. The general format is this:
 
 ```c
 if (function_that_could_fail(foo, bar) < 0)
-    HGOTO_ERROR(H5E_<major>, H5E_<minor>, <bad value>, "tell me about badness")
+    HGOTO_ERROR(H5E_<major>, H5E_<minor>, <bad value>, "tell me about badness");
 ```
 
 `HGOTO_ERROR` is one of a set of macros defined in `H5Eprivate.h`. This macro
@@ -713,7 +713,7 @@ error:
     {
         H5Fclose(fid);
     }
-    H5E_END_TRY;
+    H5E_END_TRY
 
     return FAIL;
 }

diff --git a/java/src/jni/CMakeLists.txt b/java/src/jni/CMakeLists.txt
@@ -106,7 +106,7 @@ if (HDF5_EXPORTED_TARGETS)
           ${HDF5_EXPORTED_TARGETS}
       LIBRARY DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT libraries
       ARCHIVE DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT libraries
-      RUNTIME DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT libraries
+      RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT libraries
       FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT libraries
       INCLUDES DESTINATION include
   )

diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
@@ -59,7 +59,12 @@ New Features
 
     Library:
     --------
-    -
+    - Change the error handling for a not found path in the find plugin process.
+
+      While attempting to load a plugin the HDF5 library will fail if one of the
+      directories in the plugin paths does not exist, even if there are more paths
+      to check. Instead of exiting the function with an error, just logged the error
+      and continue processing the list of paths to check.
 
 
     Parallel Library:
@@ -116,6 +121,37 @@ Bug Fixes since HDF5-1.14.1 release
 ===================================
     Library
     -------
+    - Fixed CVE-2018-13867
+
+      A corrupt file containing an invalid local heap datablock address
+      could trigger an assert failure when the metadata cache attempted
+      to load the datablock from storage.
+
+      The local heap now verifies that the datablock address is valid
+      when the local heap header information is parsed.
+
+    - Fixed CVE-2018-11202
+
+      A malformed file could result in chunk index memory leaks. Under most
+      conditions (i.e., when the --enable-using-memchecker option is NOT
+      used), this would result in a small memory leak and and infinite loop
+      and abort when shutting down the library. The infinite loop would be
+      due to the "free list" package not being able to clear its resources
+      so the library couldn't shut down. When the "using a memory checker"
+      option is used, the free lists are disabled so there is just a memory
+      leak with no abort on library shutdown.
+
+      The chunk index resources are now correctly cleaned up when reading
+      misparsed files and valgrind confirms no memory leaks.
+
+    - Fixed an issue where an assert statement was converted to an
+      incorrect error check statement
+
+      An assert statement in the library dealing with undefined dataset data
+      fill values was converted to an improper error check that would always
+      trigger when a dataset's fill value was set to NULL (undefined). This
+      has now been fixed.
+
     - Fixed an assertion failure when attempting to use the Subfiling IOC
       VFD directly
 
@@ -171,7 +207,21 @@ Bug Fixes since HDF5-1.14.1 release
 
     Tools
     -----
-    -
+    - Fixed an issue in h5repack for variable-length typed datasets
+
+      When repacking datasets into a new file, h5repack tries to determines whether
+      it can use H5Ocopy to copy each dataset into the new file, or if it needs to
+      manually re-create the dataset, then read data from the old dataset and write
+      it to the new dataset. H5repack was previously using H5Ocopy for datasets with
+      variable-length datatypes, but this can be problematic if the global heap
+      addresses involved do not match exactly between the old and new files. These
+      addresses could change for a variety of reasons, such as the command-line options
+      provided to h5repack, how h5repack allocate space in the repacked file, etc.      
+      Since H5Ocopy does not currently perform any translation when these addresses
+      change, datasets that were repacked with H5Ocopy could become unreadable in the
+      new file. H5repack has been fixed to repack variable-length typed datasets without
+      using H5Ocopy to ensure that the new datasets always have the correct global heap
+      addresses.
 
 
     Performance

diff --git a/src/H5.c b/src/H5.c
@@ -115,7 +115,7 @@ H5_default_vfd_init(void)
      * it has of initializing the default VFD.
      */
     if (H5FD_sec2_init() == H5I_INVALID_HID) {
-        HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to load default VFD ID")
+        HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to load default VFD ID");
     }
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -267,7 +267,7 @@ H5_init_library(void)
         for (i = 0; i < NELMTS(initializer); i++) {
             if (initializer[i].func() < 0) {
                 HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL,
-                    "unable to initialize %s interface", initializer[i].descr)
+                    "unable to initialize %s interface", initializer[i].descr);
             }
         }
         /* clang-format on */
@@ -576,7 +576,7 @@ H5garbage_collect(void)
 
     /* Call the garbage collection routines in the library */
     if (H5FL_garbage_coll() < 0)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGC, FAIL, "can't garbage collect objects")
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGC, FAIL, "can't garbage collect objects");
 
 done:
     FUNC_LEAVE_API(ret_value)
@@ -622,7 +622,7 @@ H5set_free_list_limits(int reg_global_lim, int reg_list_lim, int arr_global_lim,
     /* Call the free list function to actually set the limits */
     if (H5FL_set_free_list_limits(reg_global_lim, reg_list_lim, arr_global_lim, arr_list_lim, blk_global_lim,
                                   blk_list_lim, blk_global_lim, blk_list_lim) < 0)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "can't set garbage collection limits")
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "can't set garbage collection limits");
 
 done:
     FUNC_LEAVE_API(ret_value)
@@ -658,7 +658,7 @@ H5get_free_list_sizes(size_t *reg_size /*out*/, size_t *arr_size /*out*/, size_t
 
     /* Call the free list function to actually get the sizes */
     if (H5FL_get_free_list_sizes(reg_size, arr_size, blk_size, fac_size) < 0)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get garbage collection sizes")
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get garbage collection sizes");
 
 done:
     FUNC_LEAVE_API(ret_value)
@@ -1052,11 +1052,11 @@ H5atclose(H5_atclose_func_t func, void *ctx)
 
     /* Check arguments */
     if (NULL == func)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL func pointer")
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL func pointer");
 
     /* Allocate space for the 'atclose' node */
     if (NULL == (new_atclose = H5FL_MALLOC(H5_atclose_node_t)))
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate 'atclose' node")
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate 'atclose' node");
 
     /* Set up 'atclose' node */
     new_atclose->func = func;