Enable dynamic loading of VFDs with HDF5_DRIVER environment variable

.gitattributes

@@ -393,10 +393,10 @@ test/cork.c -text
 test/corrupt_stab_msg.h5 -text
 test/deflate.h5 -text
 test/dynlib4.c -text
-test/family_v16_00000.h5 -text
-test/family_v16_00001.h5 -text
-test/family_v16_00002.h5 -text
-test/family_v16_00003.h5 -text
+test/family_v16-000000.h5 -text
+test/family_v16-000001.h5 -text
+test/family_v16-000002.h5 -text
+test/family_v16-000003.h5 -text
 test/file_image_core_test.h5 -text
 test/filespace_1_6.h5 -text
 test/filespace_1_8.h5 -text

MANIFEST

@@ -1191,10 +1191,10 @@
 ./test/external_common.h
 ./test/external_env.c
 ./test/external_fname.h
-./test/family_v16_00000.h5
-./test/family_v16_00001.h5
-./test/family_v16_00002.h5
-./test/family_v16_00003.h5
+./test/family_v16-000000.h5
+./test/family_v16-000001.h5
+./test/family_v16-000002.h5
+./test/family_v16-000003.h5
 ./test/farray.c
 ./test/fheap.c
 ./test/file_image.c

c++/test/CMakeVFDTests.cmake

@@ -24,11 +24,29 @@ set (VFD_LIST
     split
     multi
     family
+    splitter
+    #log - log VFD currently has file space allocation bugs
 )
 
 if (H5_HAVE_DIRECT)
   set (VFD_LIST ${VFD_LIST} direct)
 endif ()
+if (H5_HAVE_PARALLEL)
+  # MPI I/O VFD is currently incompatible with too many tests in the VFD test set 
+  # set (VFD_LIST ${VFD_LIST} mpio)
+endif ()
+if (H5_HAVE_MIRROR_VFD)
+  set (VFD_LIST ${VFD_LIST} mirror)
+endif ()
+if (H5_HAVE_ROS3_VFD)
+  set (VFD_LIST ${VFD_LIST} ros3)
+endif ()
+if (H5_HAVE_LIBHDFS)
+  set (VFD_LIST ${VFD_LIST} hdfs)
+endif ()
+if (H5_HAVE_WINDOWS)
+  set (VFD_LIST ${VFD_LIST} windows)
+endif ()
 
 ##############################################################################
 ##############################################################################

config/conclude.am

@@ -270,10 +270,27 @@ build-check-p: $(LIB) $(PROGS) $(chk_TESTS)
 	    echo "===Parallel tests in `echo ${PWD} | sed -e s:.*/::` ended `date`===";\
 	fi
 
-VFD_LIST = sec2 stdio core core_paged split multi family
+VFD_LIST = sec2 stdio core core_paged split multi family splitter
+
+# log VFD currently has file space allocation bugs
+# VFD_LIST += log
+
 if DIRECT_VFD_CONDITIONAL
   VFD_LIST += direct
 endif
+if BUILD_PARALLEL_CONDITIONAL
+  # MPI I/O VFD is currently incompatible with too many tests in the VFD test set
+  # VFD_LIST += mpio
+endif
+if MIRROR_VFD_CONDITIONAL
+  VFD_LIST += mirror
+endif
+if ROS3_VFD_CONDITIONAL
+  VFD_LIST += ros3
+endif
+if HDFS_VFD_CONDITIONAL
+  VFD_LIST += hdfs
+endif
 
 # Run test with different Virtual File Driver
 check-vfd: $(LIB) $(PROGS) $(chk_TESTS)

src/H5.c

@@ -261,7 +261,7 @@ H5_init_library(void)
         HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize error interface")
     if (H5VL_init_phase1() < 0)
         HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize vol interface")
-    if (H5P_init() < 0)
+    if (H5P_init_phase1() < 0)
         HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize property list interface")
     if (H5AC_init() < 0)
         HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize metadata caching interface")
@@ -273,6 +273,8 @@ H5_init_library(void)
         HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize dataspace interface")
 
     /* Finish initializing interfaces that depend on the interfaces above */
+    if (H5P_init_phase2() < 0)
+        HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize property list interface")
     if (H5VL_init_phase2() < 0)
         HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize vol interface")
 

src/H5FDcore.c

@@ -149,6 +149,7 @@ static herr_t  H5FD__core_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing
 static herr_t  H5FD__core_lock(H5FD_t *_file, hbool_t rw);
 static herr_t  H5FD__core_unlock(H5FD_t *_file);
 static herr_t  H5FD__core_delete(const char *filename, hid_t fapl_id);
+static inline const H5FD_core_fapl_t *H5FD__core_get_default_config(void);
 
 static const H5FD_class_t H5FD_core_g = {
     H5FD_CORE_VALUE,          /* value                */
@@ -187,6 +188,11 @@ static const H5FD_class_t H5FD_core_g = {
     H5FD_FLMAP_DICHOTOMY      /* fl_map               */
 };
 
+/* Default configurations, if none provided */
+static const H5FD_core_fapl_t H5FD_core_default_config_g       = {H5_MB, TRUE, H5FD_CORE_WRITE_TRACKING_FLAG,
+                                                            H5FD_CORE_WRITE_TRACKING_PAGE_SIZE};
+static const H5FD_core_fapl_t H5FD_core_default_paged_config_g = {H5_MB, TRUE, TRUE, (size_t)4096};
+
 /* Define a free list to manage the region type */
 H5FL_DEFINE(H5FD_core_region_t);
 
@@ -408,6 +414,32 @@ H5FD__core_write_to_bstore(H5FD_core_t *file, haddr_t addr, size_t size)
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__core_write_to_bstore() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__core_get_default_config
+ *
+ * Purpose:     Retrieves a default configuration for this VFD when no
+ *              configuration information has been provided.
+ *
+ * Return:      Valid Core VFD configuration information pointer (can't
+ *                  fail)
+ *
+ *-------------------------------------------------------------------------
+ */
+static inline const H5FD_core_fapl_t *
+H5FD__core_get_default_config(void)
+{
+    char *driver = HDgetenv("HDF5_DRIVER");
+
+    if (driver) {
+        if (!HDstrcmp(driver, "core"))
+            return &H5FD_core_default_config_g;
+        else if (!HDstrcmp(driver, "core_paged"))
+            return &H5FD_core_default_paged_config_g;
+    }
+
+    return &H5FD_core_default_config_g;
+} /* end H5FD__core_get_default_config() */
+
 /*-------------------------------------------------------------------------
  * Function:    H5FD__init_package
  *
@@ -529,7 +561,7 @@ H5Pset_core_write_tracking(hid_t plist_id, hbool_t is_enabled, size_t page_size)
     if (H5FD_CORE != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver")
     if (NULL == (old_fa = (const H5FD_core_fapl_t *)H5P_peek_driver_info(plist)))
-        HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info")
+        old_fa = H5FD__core_get_default_config();
 
     /* Set VFD info values */
     HDmemset(&fa, 0, sizeof(H5FD_core_fapl_t));
@@ -539,7 +571,7 @@ H5Pset_core_write_tracking(hid_t plist_id, hbool_t is_enabled, size_t page_size)
     fa.page_size      = page_size;
 
     /* Set the property values & the driver for the FAPL */
-    if (H5P_set_driver(plist, H5FD_CORE, &fa) < 0)
+    if (H5P_set_driver(plist, H5FD_CORE, &fa, NULL) < 0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set core VFD as driver")
 
 done:
@@ -623,7 +655,7 @@ H5Pset_fapl_core(hid_t fapl_id, size_t increment, hbool_t backing_store)
     fa.page_size      = H5FD_CORE_WRITE_TRACKING_PAGE_SIZE;
 
     /* Set the property values & the driver for the FAPL */
-    if (H5P_set_driver(plist, H5FD_CORE, &fa) < 0)
+    if (H5P_set_driver(plist, H5FD_CORE, &fa, NULL) < 0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set core VFD as driver")
 
 done:
@@ -748,7 +780,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
     if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list")
     if (NULL == (fa = (const H5FD_core_fapl_t *)H5P_peek_driver_info(plist)))
-        HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info")
+        fa = H5FD__core_get_default_config();
 
     /* Build the open flags */
     o_flags = (H5F_ACC_RDWR & flags) ? O_RDWR : O_RDONLY;
@@ -1735,7 +1767,7 @@ H5FD__core_delete(const char *filename, hid_t fapl_id)
     if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
     if (NULL == (fa = (const H5FD_core_fapl_t *)H5P_peek_driver_info(plist)))
-        HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info")
+        fa = H5FD__core_get_default_config();
 
     if (fa->backing_store)
         if (HDremove(filename) < 0)

src/H5FDdirect.c

@@ -120,6 +120,8 @@ typedef struct H5FD_direct_t {
 
 /* Prototypes */
 static herr_t  H5FD__direct_term(void);
+static herr_t  H5FD__direct_populate_config(size_t boundary, size_t block_size, size_t cbuf_size,
+                                            H5FD_direct_fapl_t *fa_out);
 static void *  H5FD__direct_fapl_get(H5FD_t *file);
 static void *  H5FD__direct_fapl_copy(const void *_old_fa);
 static H5FD_t *H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr);
@@ -296,28 +298,10 @@ H5Pset_fapl_direct(hid_t fapl_id, size_t boundary, size_t block_size, size_t cbu
     if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
 
-    HDmemset(&fa, 0, sizeof(H5FD_direct_fapl_t));
-    if (boundary != 0)
-        fa.mboundary = boundary;
-    else
-        fa.mboundary = MBOUNDARY_DEF;
-    if (block_size != 0)
-        fa.fbsize = block_size;
-    else
-        fa.fbsize = FBSIZE_DEF;
-    if (cbuf_size != 0)
-        fa.cbsize = cbuf_size;
-    else
-        fa.cbsize = CBSIZE_DEF;
-
-    /* Set the default to be true for data alignment */
-    fa.must_align = TRUE;
-
-    /* Copy buffer size must be a multiple of file block size */
-    if (fa.cbsize % fa.fbsize != 0)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "copy buffer size must be a multiple of block size")
+    if (H5FD__direct_populate_config(boundary, block_size, cbuf_size, &fa) < 0)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't initialize driver configuration info")
 
-    ret_value = H5P_set_driver(plist, H5FD_DIRECT, &fa);
+    ret_value = H5P_set_driver(plist, H5FD_DIRECT, &fa, NULL);
 
 done:
     FUNC_LEAVE_API(ret_value)
@@ -366,6 +350,53 @@ H5Pget_fapl_direct(hid_t fapl_id, size_t *boundary /*out*/, size_t *block_size /
     FUNC_LEAVE_API(ret_value)
 } /* end H5Pget_fapl_direct() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__direct_populate_config
+ *
+ * Purpose:    Populates a H5FD_direct_fapl_t structure with the provided
+ *             values, supplying defaults where values are not provided.
+ *
+ * Return:    Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__direct_populate_config(size_t boundary, size_t block_size, size_t cbuf_size, H5FD_direct_fapl_t *fa_out)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_STATIC
+
+    HDassert(fa_out);
+
+    HDmemset(fa_out, 0, sizeof(H5FD_direct_fapl_t));
+
+    if (boundary != 0)
+        fa_out->mboundary = boundary;
+    else
+        fa_out->mboundary = MBOUNDARY_DEF;
+
+    if (block_size != 0)
+        fa_out->fbsize = block_size;
+    else
+        fa_out->fbsize = FBSIZE_DEF;
+
+    if (cbuf_size != 0)
+        fa_out->cbsize = cbuf_size;
+    else
+        fa_out->cbsize = CBSIZE_DEF;
+
+    /* Set the default to be true for data alignment */
+    fa_out->must_align = TRUE;
+
+    /* Copy buffer size must be a multiple of file block size */
+    if (fa_out->cbsize % fa_out->fbsize != 0)
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "copy buffer size must be a multiple of block size")
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__direct_get_default_config() */
+
 /*-------------------------------------------------------------------------
  * Function:  H5FD__direct_fapl_get
  *
@@ -450,6 +481,7 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad
     int                       fd   = (-1);
     H5FD_direct_t *           file = NULL;
     const H5FD_direct_fapl_t *fa;
+    H5FD_direct_fapl_t        default_fa;
 #ifdef H5_HAVE_WIN32_API
     HFILE                              filehandle;
     struct _BY_HANDLE_FILE_INFORMATION fileinfo;
@@ -498,8 +530,11 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad
     /* Get the driver specific information */
     if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list")
-    if (NULL == (fa = (const H5FD_direct_fapl_t *)H5P_peek_driver_info(plist)))
-        HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info")
+    if (NULL == (fa = (const H5FD_direct_fapl_t *)H5P_peek_driver_info(plist))) {
+        if (H5FD__direct_populate_config(0, 0, 0, &default_fa) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTSET, NULL, "can't initialize driver configuration info")
+        fa = &default_fa;
+    }
 
     file->fd = fd;
     H5_CHECKED_ASSIGN(file->eof, haddr_t, sb.st_size, h5_stat_size_t);