Refactor threading and other concurrency support (#4469)

Complete overhaul of the concurrency-related aspects of the library (threading, atomics, locking, etc.), adding private routines in the H5TS package to allow internal algorithms to use all of these capabilities.

Adds many new features & components in the H5TS package that are equivalent to common concurrency data structures and capabilities: "regular" and recursive mutices, condition variables, semaphores, thread barriers, 'once' support, thread pools, atomic variables, thread-local keys, and spawning & joining internal threads.

Now supports C11, pthreads, and Windows threading for all H5TS capabilities, except the recursive readers/writers lock, which is not supported on Windows (because Windows threads don't provide a callback on thread-local variable deletion).

The "global" API lock is switched to use a recursive mutex from the H5TS package, instead of its own variant.

API context code (H5CX package) and error stacks (H5E package) now use the common thread-local info, instead of their own variants.

Subfiling code is switched from using Mercury threading features to the new internal H5TS features.

Removes the mercury threading code.

Adds a configure option (--enable-threads / HDF5_ENABLE_THREADS), enabled by default, to control whether threading is enabled within the library.

.clang-format

@@ -70,13 +70,16 @@ StatementMacros:
   - FUNC_ENTER_API
   - FUNC_ENTER_API_NAMECHECK_ONLY
   - FUNC_ENTER_NOAPI
+  - FUNC_ENTER_NOAPI_NAMECHECK_ONLY
   - FUNC_ENTER_NOAPI_NOERR
   - FUNC_ENTER_NOAPI_NOINIT
   - FUNC_ENTER_NOAPI_NOINIT_NOERR
   - FUNC_ENTER_PACKAGE
   - FUNC_LEAVE_API
   - FUNC_LEAVE_API_NAMECHECK_ONLY
   - FUNC_LEAVE_NOAPI
+  - FUNC_LEAVE_NOAPI_NAMECHECK_ONLY
+  - FUNC_LEAVE_NOAPI_VOID_NAMECHECK_ONLY
   - FUNC_LEAVE_NOAPI_NOFS
   - H5E_END_TRY
   - H5E_PRINTF

ACKNOWLEDGMENTS

@@ -1,18 +1,20 @@
-Acknowledgments - November 2010
--------------------------------
+Acknowledgments
+---------------
 
 We would like to thank the following people who have contributed directly
 or indirectly to HDF5:
 
-Werner Benger, for contributing code used to add support for the Windows 
+Werner Benger, for contributing code used to add support for the Windows
 Threading library included in the 1.8.6 release.
 
 John A. Biddiscombe, Mike Jackson, and Sean McBride for contributing and
 testing CMake code included in the HDF5 1.8.5 distribution.
 
+NVIDIA, for contributing multithreaded concurrency support.
+
 The HDF5 community for helping shape the development of HDF5 by contributing
 bug reports and patches, joining in on forum discussions, and continually
 supporting our software.
 
-Finally, we'd like to thank all organizations whose funding has made HDF5 
+Finally, we'd like to thank all organizations whose funding has made HDF5
 possible.

CMakeLists.txt

@@ -768,6 +768,9 @@ if (HDF5_ENABLE_PARALLEL)
                Reading/Writing >2GB of data in a single parallel I/O operation will be disabled.")
       set (LARGE_PARALLEL_IO OFF)
     endif ()
+
+    # Used by Subfiling VFD feature
+    CHECK_SYMBOL_EXISTS (MPI_Comm_split_type "mpi.h" H5_HAVE_MPI_Comm_split_type)
   else ()
     message (FATAL_ERROR "Parallel libraries not found")
   endif ()
@@ -781,6 +784,38 @@ if (H5_HAVE_PARALLEL)
   endif ()
 endif ()
 
+# Determine if a threading package is available on this system
+option (HDF5_ENABLE_THREADS "Enable thread support" ON)
+set (THREADS_PREFER_PTHREAD_FLAG ON)
+find_package (Threads)
+if (Threads_FOUND)
+  set (H5_HAVE_THREADS 1)
+  set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
+
+  # Determine which threading package to use
+  # Comment out check for C11 threads for now, since it conflicts with the
+  # current --std=c99 compile flags at configuration time.  When we switch to
+  # --std=c11, this can be uncommented.
+  #CHECK_INCLUDE_FILE("threads.h" HAVE_THREADS_H)
+  if (WIN32)
+    # When Win32 is available, we use those threads
+    set (H5_HAVE_WIN_THREADS 1)
+  elseif (HAVE_THREADS_H)
+    # When C11 threads are available, those are the top choice
+    set (H5_HAVE_C11_THREADS 1)
+  elseif (CMAKE_USE_PTHREADS_INIT)
+    set (H5_HAVE_PTHREAD_H 1)
+  else ()
+    message (FATAL_ERROR " **** thread support requires C11 threads, Win32 threads or Pthreads **** ")
+  endif ()
+
+  # Check for compiler support for atomic variables
+  CHECK_INCLUDE_FILE("stdatomic.h" HAVE_STDATOMIC_H)
+  if (HAVE_STDATOMIC_H)
+    set (H5_HAVE_STDATOMIC_H 1)
+  endif()
+endif ()
+
 # Determine whether to build the HDF5 Subfiling VFD
 set (H5FD_SUBFILING_DIR ${HDF5_SRC_DIR}/H5FDsubfiling)
 set (HDF5_SRC_INCLUDE_DIRS
@@ -795,57 +830,21 @@ if (HDF5_ENABLE_SUBFILING_VFD)
   if (NOT HDF5_ENABLE_PARALLEL)
     message (FATAL_ERROR "Subfiling VFD requires a parallel HDF5 build")
   else ()
-    # Check for MPI_Comm_split_type
-    CHECK_SYMBOL_EXISTS (MPI_Comm_split_type "mpi.h" H5_HAVE_MPI_Comm_split_type)
+    # Make sure we found MPI_Comm_split_type previously
     if (NOT H5_HAVE_MPI_Comm_split_type)
       message (FATAL_ERROR "Subfiling VFD requires MPI-3 support for MPI_Comm_split_type")
     endif ()
   endif ()
 
-  if (NOT DEFINED Threads_FOUND)
-    set (THREADS_PREFER_PTHREAD_FLAG ON)
-    find_package (Threads REQUIRED)
-  endif ()
-
-  # For now, make sure we're using pthreads. Once Subfiling can be
-  # supported on Windows, we should allow Win32 threads as well
-  if (NOT ${Threads_FOUND} OR NOT ${CMAKE_USE_PTHREADS_INIT})
-    message (FATAL_ERROR "Subfiling requires pthreads for system thread library")
+  # Subfiling requires thread operations
+  if (NOT Threads_FOUND)
+    message (FATAL_ERROR "Subfiling requires thread operations support")
   endif ()
 
-  CHECK_INCLUDE_FILE("stdatomic.h" HAVE_STDATOMIC_H)
-  if (NOT HAVE_STDATOMIC_H)
-    message (FATAL_ERROR "Subfiling VFD requires atomic operations support. C11 stdatomic.h NOT available.")
-  else()
-    set (H5_HAVE_STDATOMIC_H 1)
-  endif()
-
   set (H5_HAVE_SUBFILING_VFD 1)
   # IOC VFD is currently only built when subfiling is enabled
   set (H5_HAVE_IOC_VFD 1)
 
-  message (STATUS "Setting up to use Mercury components")
-  set (H5FD_SUBFILING_MERCURY_DIR ${H5FD_SUBFILING_DIR}/mercury/src/util)
-  set (HDF5_SRC_INCLUDE_DIRS
-      ${HDF5_SRC_INCLUDE_DIRS}
-      ${H5FD_SUBFILING_MERCURY_DIR}
-  )
-  set (CMAKE_REQUIRED_INCLUDES "${H5FD_SUBFILING_MERCURY_DIR}")
-
-  # Run some configure checks for the Mercury util files
-  set (CMAKE_EXTRA_INCLUDE_FILES pthread.h)
-  set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
-
-  check_type_size(PTHREAD_MUTEX_ADAPTIVE_NP PTHREAD_MUTEX_ADAPTIVE_NP_SIZE)
-  if (HAVE_PTHREAD_MUTEX_ADAPTIVE_NP_SIZE)
-    set (${HDF_PREFIX}_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP 1)
-  endif ()
-
-  check_symbol_exists(pthread_condattr_setclock pthread.h
-    ${HDF_PREFIX}_HAVE_PTHREAD_CONDATTR_SETCLOCK)
-
-  unset (CMAKE_EXTRA_INCLUDE_FILES)
-  unset (CMAKE_REQUIRED_LIBRARIES)
   if(NOT H5_HAVE_SUBFILING_VFD EQUAL 1)
     set (HDF5_ENABLE_SUBFILING_VFD OFF CACHE BOOL "Build Parallel HDF5 Subfiling VFD" FORCE)
   endif()
@@ -942,6 +941,13 @@ if (HDF5_ENABLE_THREADSAFE)
       message (FATAL_ERROR " **** thread-safety option not supported with static library **** ")
     endif ()
   endif ()
+  if (HDF_ENABLE_PARALLEL)
+    if (NOT ALLOW_UNSUPPORTED)
+      message (FATAL_ERROR " **** Parallel and thread-safety options are not supported, override with ALLOW_UNSUPPORTED option **** ")
+    else ()
+      message (VERBOSE " **** Allowing unsupported parallel and thread-safety options **** ")
+    endif ()
+  endif ()
   if (HDF5_BUILD_FORTRAN)
     if (NOT ALLOW_UNSUPPORTED)
       message (FATAL_ERROR " **** Fortran and thread-safety options are not supported, override with ALLOW_UNSUPPORTED option **** ")
@@ -963,19 +969,13 @@ if (HDF5_ENABLE_THREADSAFE)
       message (VERBOSE " **** Allowing unsupported HL and thread-safety options **** ")
     endif ()
   endif ()
-  if (WIN32)
-    # When Win32 is available, we use those threads
-    set (H5_HAVE_WIN_THREADS 1)
-  else ()
-    if (NOT H5_HAVE_PTHREAD_H)
-      message (FATAL_ERROR " **** thread-safe option requires Win32 threads or Pthreads **** ")
-    endif ()
-  endif ()
-  set (THREADS_PREFER_PTHREAD_FLAG ON)
-  find_package (Threads REQUIRED)
-  if (Threads_FOUND)
-    set (H5_HAVE_THREADSAFE 1)
+
+  # Check for threading package
+  if (NOT Threads_FOUND)
+    message (FATAL_ERROR " **** thread-safety option requires a threading package and none was found **** ")
   endif ()
+
+  set (H5_HAVE_THREADSAFE 1)
 endif ()
 
 #-----------------------------------------------------------------------------

HDF5Examples/CMakeLists.txt

@@ -99,27 +99,53 @@ if (H5_HAVE_PARALLEL)
   INCLUDE_DIRECTORIES (${MPI_C_INCLUDE_DIRS})
 endif ()
 
+# Determine if a threading package is available on this system
+option (HDF5_ENABLE_THREADS "Enable thread support" ON)
+set (THREADS_PREFER_PTHREAD_FLAG ON)
+find_package (Threads)
+if (Threads_FOUND)
+  set (H5_HAVE_THREADS 1)
+  set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
+
+  # Determine which threading package to use
+  # Comment out check for C11 threads for now, since it conflicts with the
+  # current --std=c99 compile flags at configuration time.  When we switch to
+  # --std=c11, this can be uncommented.
+  #CHECK_INCLUDE_FILE("threads.h" HAVE_THREADS_H)
+  if (WIN32)
+    # When Win32 is available, we use those threads
+    set (H5_HAVE_WIN_THREADS 1)
+  elseif (HAVE_THREADS_H)
+    # When C11 threads are available, those are the top choice
+    set (H5_HAVE_C11_THREADS 1)
+  elseif (CMAKE_USE_PTHREADS_INIT)
+    set (H5_HAVE_PTHREAD_H 1)
+  else ()
+    message (FATAL_ERROR " **** thread support requires C11 threads, Win32 threads or Pthreads **** ")
+  endif ()
+
+  # Check for compiler support for atomic variables
+  CHECK_INCLUDE_FILE("stdatomic.h" HAVE_STDATOMIC_H)
+  if (HAVE_STDATOMIC_H)
+    set (H5_HAVE_STDATOMIC_H 1)
+  endif()
+endif ()
+
 #-----------------------------------------------------------------------------
 # Option to use threadsafe
-# Note: Currently CMake only allows configuring of threadsafe on WINDOWS.
 #-----------------------------------------------------------------------------
 option (HDF_ENABLE_THREADSAFE "Enable Threadsafety" OFF)
 # Note that HDF_ENABLE_THREADSAFE is the CMake option for determining
 # whether to enable thread-safety in the examples. HDF5_ENABLE_THREADSAFE
 # is the CMake option determining whether HDF5 was configured with
 # thread-safety enabled.
 if (HDF_ENABLE_THREADSAFE AND HDF5_ENABLE_THREADSAFE)
-  if (WIN32)
-    set (H5_HAVE_WIN_THREADS 1)
-    set (H5_HAVE_THREADSAFE 1)
-  endif ()
-  set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
-  set(THREADS_PREFER_PTHREAD_FLAG TRUE)
-  find_package(Threads REQUIRED)
+  # Check for threading package
   if (NOT Threads_FOUND)
-    message (STATUS " **** thread-safe package not found - threads still might work **** ")
+    message (FATAL_ERROR " **** thread-safety option requires a threading package and none was found **** ")
   endif ()
-  set (H5EX_HDF5_LINK_LIBS ${H5EX_HDF5_LINK_LIBS} Threads::Threads)
+
+  set (H5_HAVE_THREADSAFE 1)
 endif ()
 
 set_directory_properties(PROPERTIES INCLUDE_DIRECTORIES 

config/clang-warnings/error-general

@@ -57,8 +57,6 @@
 # Here is a list of tests and examples that have issues with the stricter warnings as error
 #
 # NOTE: Test files are not compatible with these warnings as errors
-#     thread_id.c,
-#       -Werror=unused-function
 #     dsets.c
 #       -Werror=unused-parameter
 #

config/cmake/ConfigureChecks.cmake

@@ -439,7 +439,7 @@ if (MINGW OR NOT WINDOWS)
   foreach (other_test
       HAVE_ATTRIBUTE
       HAVE_BUILTIN_EXPECT
-      SYSTEM_SCOPE_THREADS
+      PTHREAD_BARRIER
       HAVE_SOCKLEN_T
   )
     HDF_FUNCTION_TEST (${other_test})
@@ -582,12 +582,6 @@ if (MINGW OR NOT WINDOWS)
   endif ()
 endif ()
 
-# Check for clock_gettime() CLOCK_MONOTONIC_COARSE
-set (CMAKE_EXTRA_INCLUDE_FILES time.h)
-check_type_size(CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC_COARSE_SIZE)
-if (HAVE_CLOCK_MONOTONIC_COARSE_SIZE)
-  set (${HDF_PREFIX}_HAVE_CLOCK_MONOTONIC_COARSE 1)
-endif ()
 unset (CMAKE_EXTRA_INCLUDE_FILES)
 
 #-----------------------------------------------------------------------------

config/cmake/H5pubconf.h.in

@@ -107,9 +107,6 @@
 /* Define to 1 if you have the `clock_gettime' function. */
 #cmakedefine H5_HAVE_CLOCK_GETTIME @H5_HAVE_CLOCK_GETTIME@
 
-/* Define to 1 if CLOCK_MONOTONIC_COARSE is available */
-#cmakedefine H5_HAVE_CLOCK_MONOTONIC_COARSE @H5_HAVE_CLOCK_MONOTONIC_COARSE@
-
 /* Define to 1 if you have the <curl/curl.h> header file. */
 #cmakedefine H5_HAVE_CURL_CURL_H @H5_HAVE_CURL_H@
 
@@ -261,11 +258,8 @@
 /* Define to 1 if you have the <pthread.h> header file. */
 #cmakedefine H5_HAVE_PTHREAD_H @H5_HAVE_PTHREAD_H@
 
-/* Define to 1 if 'pthread_condattr_setclock()' is available */
-#cmakedefine H5_HAVE_PTHREAD_CONDATTR_SETCLOCK @H5_HAVE_PTHREAD_CONDATTR_SETCLOCK@
-
-/* Define to 1 if PTHREAD_MUTEX_ADAPTIVE_NP is available */
-#cmakedefine H5_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP @H5_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP@
+/* Define to 1 if the compiler supports the pthread_barrier_*() routines */
+#cmakedefine H5_HAVE_PTHREAD_BARRIER @H5_HAVE_PTHREAD_BARRIER@
 
 /* Define to 1 if you have the <pwd.h> header file. */
 #cmakedefine H5_HAVE_PWD_H @H5_HAVE_PWD_H@
@@ -325,6 +319,9 @@
 /* Define to 1 if the compiler supports the __builtin_expect() extension */
 #cmakedefine H5_HAVE_BUILTIN_EXPECT @H5_HAVE_BUILTIN_EXPECT@
 
+/* Define if we have thread support */
+# cmakedefine H5_HAVE_THREADS @H5_HAVE_THREADS@
+
 #if defined(_WIN32) && !defined(H5_BUILT_AS_DYNAMIC_LIB)
 /* Not supported on WIN32 platforms with static linking */
 /* #undef H5_HAVE_THREADSAFE */
@@ -360,6 +357,9 @@
 /* Define to 1 if you have Win32 threads */
 #cmakedefine H5_HAVE_WIN_THREADS @H5_HAVE_WIN_THREADS@
 
+/* Define to 1 if you have C11 threads */
+#cmakedefine H5_HAVE_C11_THREADS @H5_HAVE_C11_THREADS@
+
 /* Define if your system has Windows-style path name. */
 #cmakedefine H5_HAVE_WINDOW_PATH @H5_HAVE_WINDOW_PATH@
 
@@ -589,10 +589,6 @@
 /* Define if strict file format checks are enabled */
 #cmakedefine H5_STRICT_FORMAT_CHECKS @H5_STRICT_FORMAT_CHECKS@
 
-/* Define if your system supports pthread_attr_setscope(&attribute,
-   PTHREAD_SCOPE_SYSTEM) call. */
-#cmakedefine H5_SYSTEM_SCOPE_THREADS @H5_SYSTEM_SCOPE_THREADS@
-
 /* Define using v1.6 public API symbols by default */
 #cmakedefine H5_USE_16_API_DEFAULT @H5_USE_16_API_DEFAULT@
 

config/cmake/HDF5DeveloperBuild.cmake

@@ -163,6 +163,12 @@ if (HDF5_ENABLE_DEBUG_H5FS_ASSERT)
   list (APPEND HDF5_DEBUG_APIS H5FS_DEBUG_ASSERT)
 endif ()
 
+option (HDF5_ENABLE_DEBUG_H5TS "Enable debugging of H5TS module" OFF)
+mark_as_advanced (HDF5_ENABLE_DEBUG_H5TS)
+if (HDF5_ENABLE_DEBUG_H5TS)
+  list (APPEND HDF5_DEBUG_APIS H5TS_DEBUG)
+endif ()
+
 # If HDF5 free list debugging wasn't specifically enabled, disable
 # free lists entirely for developer build modes, as they can
 # make certain types of issues (like references to stale pointers)

config/cmake/HDFTests.c

@@ -64,23 +64,21 @@ SIMPLE_TEST(timezone = 0);
 
 #endif /* HAVE_TIMEZONE */
 
-#ifdef SYSTEM_SCOPE_THREADS
-#include <stdlib.h>
+#ifdef PTHREAD_BARRIER
 #include <pthread.h>
 
 int main(void)
 {
-    pthread_attr_t attribute;
+    pthread_barrier_t barr;
     int ret;
 
-    pthread_attr_init(&attribute);
-    ret = pthread_attr_setscope(&attribute, PTHREAD_SCOPE_SYSTEM);
+    ret = pthread_barrier_init(&barr, NULL, 1);
     if (ret == 0)
         return 0;
     return 1;
 }
 
-#endif /* SYSTEM_SCOPE_THREADS */
+#endif /* PTHREAD_BARRIER */
 
 #ifdef HAVE_SOCKLEN_T
 

config/cmake/hdf5-config.cmake.in

@@ -46,6 +46,7 @@ set (${HDF5_PACKAGE_NAME}_INSTALL_MOD_FORTRAN "@HDF5_INSTALL_MOD_FORTRAN@")
 set (${HDF5_PACKAGE_NAME}_BUILD_HL_LIB          @HDF5_BUILD_HL_LIB@)
 set (${HDF5_PACKAGE_NAME}_BUILD_SHARED_LIBS     @H5_ENABLE_SHARED_LIB@)
 set (${HDF5_PACKAGE_NAME}_BUILD_STATIC_LIBS     @H5_ENABLE_STATIC_LIB@)
+set (${HDF5_PACKAGE_NAME}_ENABLE_THREADS        @HDF5_ENABLE_THREADS@)
 set (${HDF5_PACKAGE_NAME}_ENABLE_THREADSAFE     @HDF5_ENABLE_THREADSAFE@)
 set (${HDF5_PACKAGE_NAME}_ENABLE_PARALLEL       @HDF5_ENABLE_PARALLEL@)
 set (${HDF5_PACKAGE_NAME}_DEFAULT_API_VERSION  "@DEFAULT_API_VERSION@")