diff --git a/src/H5Adense.c b/src/H5Adense.c index 80c3c94f733..48004d2aa70 100644 --- a/src/H5Adense.c +++ b/src/H5Adense.c @@ -1104,12 +1104,12 @@ H5A__dense_iterate(H5F_t *f, hid_t loc_id, const H5O_ainfo_t *ainfo, H5_index_t H5_iter_order_t order, hsize_t skip, hsize_t *last_attr, const H5A_attr_iter_op_t *attr_op, void *op_data) { - H5HF_t *fheap = NULL; /* Fractal heap handle */ - H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */ - H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */ - H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */ - haddr_t bt2_addr; /* Address of v2 B-tree to use for lookup */ - herr_t ret_value = FAIL; /* Return value */ + H5HF_t *fheap = NULL; /* Fractal heap handle */ + H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */ + H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */ + H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */ + haddr_t bt2_addr; /* Address of v2 B-tree to use for lookup */ + herr_t ret_value = FAIL; /* Return value */ FUNC_ENTER_PACKAGE @@ -1499,12 +1499,12 @@ herr_t H5A__dense_remove_by_idx(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type, H5_iter_order_t order, hsize_t n) { - H5HF_t *fheap = NULL; /* Fractal heap handle */ - H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */ - H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */ - H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */ - haddr_t bt2_addr; /* Address of v2 B-tree to use for operation */ - herr_t ret_value = SUCCEED; /* Return value */ + H5HF_t *fheap = NULL; /* Fractal heap handle */ + H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */ + H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */ + H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */ + haddr_t bt2_addr; /* Address of v2 B-tree to use for operation */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -1586,7 +1586,7 @@ H5A__dense_remove_by_idx(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "error building table of attributes"); /* Check for skipping too many attributes */ - if (n >= atable.nattrs) + if (n >= atable.num_attrs) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index specified"); /* Delete appropriate attribute from dense storage */ diff --git a/src/H5Aint.c b/src/H5Aint.c index 5ba02163f87..45df0dfbe3e 100644 --- a/src/H5Aint.c +++ b/src/H5Aint.c @@ -51,16 +51,9 @@ typedef struct { H5F_t *f; /* Pointer to file that fractal heap is in */ H5A_attr_table_t *atable; /* Pointer to attribute table to build */ - size_t curr_attr; /* Current attribute to operate on */ bool bogus_crt_idx; /* Whether bogus creation index values need to be set */ } H5A_compact_bt_ud_t; -/* Data exchange structure to use when building table of dense attributes for an object */ -typedef struct { - H5A_attr_table_t *atable; /* Pointer to attribute table to build */ - size_t curr_attr; /* Current attribute to operate on */ -} H5A_dense_bt_ud_t; - /* Data exchange structure to use when copying an attribute from _SRC to _DST */ typedef struct { const H5O_ainfo_t *ainfo; /* dense information */ @@ -1455,30 +1448,31 @@ H5A__compact_build_table_cb(H5O_t H5_ATTR_UNUSED *oh, H5O_mesg_t *mesg /*in,out* assert(mesg); /* Re-allocate the table if necessary */ - if (udata->curr_attr == udata->atable->nattrs) { + if (udata->atable->num_attrs == udata->atable->max_attrs) { H5A_t **new_table; /* New table for attributes */ size_t new_table_size; /* Number of attributes in new table */ /* Allocate larger table */ - new_table_size = MAX(1, 2 * udata->atable->nattrs); + new_table_size = MAX(1, 2 * udata->atable->max_attrs); if (NULL == (new_table = (H5A_t **)H5FL_SEQ_REALLOC(H5A_t_ptr, udata->atable->attrs, new_table_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5_ITER_ERROR, "unable to extend attribute table"); /* Update table information in user data */ - udata->atable->attrs = new_table; - udata->atable->nattrs = new_table_size; + udata->atable->attrs = new_table; + udata->atable->max_attrs = new_table_size; } /* end if */ /* Copy attribute into table */ - if (NULL == (udata->atable->attrs[udata->curr_attr] = H5A__copy(NULL, (const H5A_t *)mesg->native))) + if (NULL == + (udata->atable->attrs[udata->atable->num_attrs] = H5A__copy(NULL, (const H5A_t *)mesg->native))) HGOTO_ERROR(H5E_ATTR, H5E_CANTCOPY, H5_ITER_ERROR, "can't copy attribute"); /* Assign [somewhat arbitrary] creation order value, if requested */ if (udata->bogus_crt_idx) - ((udata->atable->attrs[udata->curr_attr])->shared)->crt_idx = sequence; + ((udata->atable->attrs[udata->atable->num_attrs])->shared)->crt_idx = sequence; - /* Increment current attribute */ - udata->curr_attr++; + /* Increment attribute count */ + udata->atable->num_attrs++; done: FUNC_LEAVE_NOAPI(ret_value) @@ -1501,9 +1495,10 @@ herr_t H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order_t order, H5A_attr_table_t *atable) { - H5A_compact_bt_ud_t udata; /* User data for iteration callback */ - H5O_mesg_operator_t op; /* Wrapper for operator */ - herr_t ret_value = SUCCEED; /* Return value */ + H5A_compact_bt_ud_t udata; /* User data for iteration callback */ + H5O_mesg_operator_t op; /* Wrapper for operator */ + bool iter_set_up = false; /* Is everything set up for iteration */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -1513,13 +1508,13 @@ H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order assert(atable); /* Initialize table */ - atable->attrs = NULL; - atable->nattrs = 0; + atable->attrs = NULL; + atable->num_attrs = 0; + atable->max_attrs = 0; /* Set up user data for iteration */ - udata.f = f; - udata.atable = atable; - udata.curr_attr = 0; + udata.f = f; + udata.atable = atable; udata.bogus_crt_idx = (bool)((oh->version == H5O_VERSION_1 || !(oh->flags & H5O_HDR_ATTR_CRT_ORDER_TRACKED)) ? true : false); @@ -1527,20 +1522,23 @@ H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order /* Iterate over existing attributes, checking for attribute with same name */ op.op_type = H5O_MESG_OP_LIB; op.u.lib_op = H5A__compact_build_table_cb; + iter_set_up = true; if (H5O__msg_iterate_real(f, oh, H5O_MSG_ATTR, &op, &udata) < 0) HGOTO_ERROR(H5E_ATTR, H5E_BADITER, FAIL, "error building attribute table"); - /* Correct # of attributes in table */ - atable->nattrs = udata.curr_attr; - /* Don't sort an empty table. */ - if (atable->nattrs > 0) { + if (atable->num_attrs > 0) /* Sort attribute table in correct iteration order */ if (H5A__attr_sort_table(atable, idx_type, order) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTSORT, FAIL, "error sorting attribute table"); - } /* end if */ done: + if (ret_value < 0) + /* Clean up partially built table on error */ + if (iter_set_up) + if (atable->attrs && H5A__attr_release_table(atable) < 0) + HDONE_ERROR(H5E_ATTR, H5E_CANTFREE, FAIL, "unable to release attribute table"); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5A__compact_build_table() */ @@ -1557,26 +1555,26 @@ H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order static herr_t H5A__dense_build_table_cb(const H5A_t *attr, void *_udata) { - H5A_dense_bt_ud_t *udata = (H5A_dense_bt_ud_t *)_udata; /* 'User data' passed in */ - herr_t ret_value = H5_ITER_CONT; /* Return value */ + H5A_attr_table_t *atable = (H5A_attr_table_t *)_udata; /* 'User data' passed in */ + herr_t ret_value = H5_ITER_CONT; /* Return value */ FUNC_ENTER_PACKAGE /* check arguments */ assert(attr); - assert(udata); - assert(udata->curr_attr < udata->atable->nattrs); + assert(atable); + assert(atable->num_attrs < atable->max_attrs); /* Allocate attribute for entry in the table */ - if (NULL == (udata->atable->attrs[udata->curr_attr] = H5FL_CALLOC(H5A_t))) + if (NULL == (atable->attrs[atable->num_attrs] = H5FL_CALLOC(H5A_t))) HGOTO_ERROR(H5E_ATTR, H5E_CANTALLOC, H5_ITER_ERROR, "can't allocate attribute"); /* Copy attribute information. Share the attribute object in copying. */ - if (NULL == H5A__copy(udata->atable->attrs[udata->curr_attr], attr)) + if (NULL == H5A__copy(atable->attrs[atable->num_attrs], attr)) HGOTO_ERROR(H5E_ATTR, H5E_CANTCOPY, H5_ITER_ERROR, "can't copy attribute"); /* Increment number of attributes stored */ - udata->curr_attr++; + atable->num_attrs++; done: FUNC_LEAVE_NOAPI(ret_value) @@ -1622,22 +1620,18 @@ H5A__dense_build_table(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type, if (H5B2_get_nrec(bt2_name, &nrec) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't retrieve # of records in index"); - /* Set size of table */ - H5_CHECK_OVERFLOW(nrec, /* From: */ hsize_t, /* To: */ size_t); - atable->nattrs = (size_t)nrec; - /* Allocate space for the table entries */ - if (atable->nattrs > 0) { - H5A_dense_bt_ud_t udata; /* User data for iteration callback */ + if (nrec > 0) { H5A_attr_iter_op_t attr_op; /* Attribute operator */ - /* Allocate the table to store the attributes */ - if ((atable->attrs = (H5A_t **)H5FL_SEQ_CALLOC(H5A_t_ptr, atable->nattrs)) == NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed"); + /* Check for overflow on the downcast */ + H5_CHECK_OVERFLOW(nrec, /* From: */ hsize_t, /* To: */ size_t); - /* Set up user data for iteration */ - udata.atable = atable; - udata.curr_attr = 0; + /* Allocate the table to store the attributes */ + if (NULL == (atable->attrs = (H5A_t **)H5FL_SEQ_CALLOC(H5A_t_ptr, (size_t)nrec))) + HGOTO_ERROR(H5E_ATTR, H5E_CANTALLOC, FAIL, "memory allocation failed"); + atable->num_attrs = 0; + atable->max_attrs = (size_t)nrec; /* Build iterator operator */ attr_op.op_type = H5A_ATTR_OP_LIB; @@ -1645,7 +1639,7 @@ H5A__dense_build_table(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type, /* Iterate over the links in the group, building a table of the link messages */ if (H5A__dense_iterate(f, (hid_t)0, ainfo, H5_INDEX_NAME, H5_ITER_NATIVE, (hsize_t)0, NULL, &attr_op, - &udata) < 0) + atable) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, "error building attribute table"); /* Sort attribute table in correct iteration order */ @@ -1791,18 +1785,18 @@ H5A__attr_sort_table(H5A_attr_table_t *atable, H5_index_t idx_type, H5_iter_orde /* Pick appropriate comparison routine */ if (idx_type == H5_INDEX_NAME) { if (order == H5_ITER_INC) - qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_name_inc); + qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_name_inc); else if (order == H5_ITER_DEC) - qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_name_dec); + qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_name_dec); else assert(order == H5_ITER_NATIVE); } /* end if */ else { assert(idx_type == H5_INDEX_CRT_ORDER); if (order == H5_ITER_INC) - qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_corder_inc); + qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_corder_inc); else if (order == H5_ITER_DEC) - qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_corder_dec); + qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_corder_dec); else assert(order == H5_ITER_NATIVE); } /* end else */ @@ -1839,7 +1833,7 @@ H5A__attr_iterate_table(const H5A_attr_table_t *atable, hsize_t skip, hsize_t *l /* Iterate over attribute messages */ H5_CHECKED_ASSIGN(u, size_t, skip, hsize_t); - for (; u < atable->nattrs && !ret_value; u++) { + for (; u < atable->num_attrs && !ret_value; u++) { /* Check which type of callback to make */ switch (attr_op->op_type) { case H5A_ATTR_OP_APP2: { @@ -1906,19 +1900,20 @@ H5A__attr_release_table(H5A_attr_table_t *atable) assert(atable); /* Release attribute info, if any. */ - if (atable->nattrs > 0) { + if (atable->num_attrs > 0) { size_t u; /* Local index variable */ /* Free attribute message information */ - for (u = 0; u < atable->nattrs; u++) + for (u = 0; u < atable->num_attrs; u++) if (atable->attrs[u] && H5A__close(atable->attrs[u]) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTFREE, FAIL, "unable to release attribute"); + + /* Release array */ + atable->attrs = (H5A_t **)H5FL_SEQ_FREE(H5A_t_ptr, atable->attrs); } /* end if */ else assert(atable->attrs == NULL); - atable->attrs = (H5A_t **)H5FL_SEQ_FREE(H5A_t_ptr, atable->attrs); - done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5A__attr_release_table() */ diff --git a/src/H5Apkg.h b/src/H5Apkg.h index 64549c818c0..239d7550c46 100644 --- a/src/H5Apkg.h +++ b/src/H5Apkg.h @@ -145,8 +145,9 @@ typedef struct H5A_bt2_ud_ins_t { /* Data structure to hold table of attributes for an object */ typedef struct { - size_t nattrs; /* # of attributes in table */ - H5A_t **attrs; /* Pointer to array of attribute pointers */ + size_t num_attrs; /* Curr. # of attributes in table */ + size_t max_attrs; /* Max. # of attributes in table */ + H5A_t **attrs; /* Pointer to array of attribute pointers */ } H5A_attr_table_t; /*****************************/ diff --git a/src/H5Dint.c b/src/H5Dint.c index 8f363ebadbe..37c9fe490a8 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -946,6 +946,13 @@ H5D__update_oh_info(H5F_t *file, H5D_t *dset, hid_t dapl_id) if (NULL == (oh = H5O_pin(oloc))) HGOTO_ERROR(H5E_DATASET, H5E_CANTPIN, FAIL, "unable to pin dataset object header"); + /* Check for creating dataset with unusual datatype */ + if (H5T_is_numeric_with_unusual_unused_bits(type) && + !(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "creating dataset with unusual datatype, see documentation for " + "H5Pset_relax_file_integrity_checks for details."); + /* Write the dataspace header message */ if (H5S_append(file, oh, dset->shared->space) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update dataspace header message"); diff --git a/src/H5Dio.c b/src/H5Dio.c index 62bb48c2a26..312c7fd854b 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -395,8 +395,12 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) H5AC_tag(dset_info[i].dset->oloc.addr, &prev_tag); /* Invoke correct "high level" I/O routine */ - if ((*dset_info[i].io_ops.multi_read)(&io_info, &dset_info[i]) < 0) + if ((*dset_info[i].io_ops.multi_read)(&io_info, &dset_info[i]) < 0) { + /* Reset metadata tagging */ + H5AC_tag(prev_tag, NULL); + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data"); + } /* Reset metadata tagging */ H5AC_tag(prev_tag, NULL); diff --git a/src/H5Fint.c b/src/H5Fint.c index 325947656bf..3f5a1379834 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -438,6 +438,8 @@ H5F_get_access_plist(H5F_t *f, bool app_ref) 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set initial metadata cache resize config."); + if (H5P_set(new_plist, H5F_ACS_RFIC_FLAGS_NAME, &(f->shared->rfic_flags)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set RFIC flags value"); /* Prepare the driver property */ driver_prop.driver_id = f->shared->lf->driver_id; @@ -1230,6 +1232,8 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F if (H5P_get(plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, &(f->shared->mdc_initCacheImageCfg)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get initial metadata cache resize config"); + if (H5P_get(plist, H5F_ACS_RFIC_FLAGS_NAME, &(f->shared->rfic_flags)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get RFIC flags value"); /* Get the VFD values to cache */ f->shared->maxaddr = H5FD_get_maxaddr(lf); diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 60de31ebdfd..7e12ff111d0 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -304,6 +304,7 @@ struct H5F_shared_t { bool use_file_locking; /* Whether or not to use file locking */ bool ignore_disabled_locks; /* Whether or not to ignore disabled file locking */ bool closing; /* File is in the process of being closed */ + uint64_t rfic_flags; /* Relaxed file integrity check (RFIC) flags */ /* Cached VOL connector ID & info */ hid_t vol_id; /* ID of VOL connector for the container */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 682e938120c..d2b1b887a7f 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -101,6 +101,7 @@ typedef struct H5F_t H5F_t; #define H5F_VOL_CLS(F) ((F)->shared->vol_cls) #define H5F_VOL_OBJ(F) ((F)->vol_obj) #define H5F_USE_FILE_LOCKING(F) ((F)->shared->use_file_locking) +#define H5F_RFIC_FLAGS(F) ((F)->shared->rfic_flags) #else /* H5F_MODULE */ #define H5F_LOW_BOUND(F) (H5F_get_low_bound(F)) #define H5F_HIGH_BOUND(F) (H5F_get_high_bound(F)) @@ -165,6 +166,7 @@ typedef struct H5F_t H5F_t; #define H5F_VOL_CLS(F) (H5F_get_vol_cls(F)) #define H5F_VOL_OBJ(F) (H5F_get_vol_obj(F)) #define H5F_USE_FILE_LOCKING(F) (H5F_get_use_file_locking(F)) +#define H5F_RFIC_FLAGS(F) (H5F_get_rfic_flags(F)) #endif /* H5F_MODULE */ /* Macros to encode/decode offset/length's for storing in the file */ @@ -282,6 +284,7 @@ typedef struct H5F_t H5F_t; #define H5F_ACS_MPI_PARAMS_COMM_NAME "mpi_params_comm" /* the MPI communicator */ #define H5F_ACS_MPI_PARAMS_INFO_NAME "mpi_params_info" /* the MPI info struct */ #endif /* H5_HAVE_PARALLEL */ +#define H5F_ACS_RFIC_FLAGS_NAME "rfic_flags" /* Relaxed file integrity check (RFIC) flags */ /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ @@ -525,7 +528,8 @@ H5_DLL bool H5F_get_min_dset_ohdr(const H5F_t *f); H5_DLL herr_t H5F_set_min_dset_ohdr(H5F_t *f, bool minimize); H5_DLL const H5VL_class_t *H5F_get_vol_cls(const H5F_t *f); H5_DLL H5VL_object_t *H5F_get_vol_obj(const H5F_t *f); -H5_DLL bool H5F_get_file_locking(const H5F_t *f); +H5_DLL bool H5F_get_use_file_locking(const H5F_t *f); +H5_DLL uint64_t H5F_get_rfic_flags(const H5F_t *f); /* Functions than retrieve values set/cached from the superblock/FCPL */ H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index 45580ac41ec..9dad8852a06 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -242,6 +242,20 @@ typedef struct H5F_retry_info_t { */ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); +/* + * These are the bits that can be passed to the `flags' argument of + * H5Pset_relax_file_integrity_checks(). Use the bit-wise OR operator (|) to + * combine them as needed. + */ +#define H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS \ + (0x0001u) /**< Suppress errors for numeric datatypes with an unusually \ + * high number of unused bits. See documentation for \ + * H5Pset_relax_file_integrity_checks for details. */ +#define H5F_RFIC_ALL \ + (H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS) /**< Suppress all format integrity check errors. See \ + * documentation for H5Pset_relax_file_integrity_checks \ + * for details. */ + /*********************/ /* Public Prototypes */ /*********************/ diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 44a52c8dbfc..89181dad479 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1356,16 +1356,16 @@ H5F__get_cont_info(const H5F_t *f, H5VL_file_cont_info_t *info) } /* end H5F_get_cont_info */ /*------------------------------------------------------------------------- - * Function: H5F_get_file_locking + * Function: H5F_get_use_file_locking * - * Purpose: Get the file locking flag for the file + * Purpose: Get the 'use file locking' flag for the file * * Return: true/false * *------------------------------------------------------------------------- */ bool -H5F_get_file_locking(const H5F_t *f) +H5F_get_use_file_locking(const H5F_t *f) { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1373,7 +1373,7 @@ H5F_get_file_locking(const H5F_t *f) assert(f->shared); FUNC_LEAVE_NOAPI(f->shared->use_file_locking) -} /* end H5F_get_file_locking */ +} /* end H5F_get_use_file_locking */ /*------------------------------------------------------------------------- * Function: H5F_has_vector_select_io @@ -1401,3 +1401,23 @@ H5F_has_vector_select_io(const H5F_t *f, bool is_write) FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_has_vector_select_io */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_rfic_flags + * + * Purpose: Get the relaxed file integrity checks (RFIC) flags for the file + * + * Return: RFIC flags for a file on success (which can be 0), can't fail + * + *------------------------------------------------------------------------- + */ +uint64_t +H5F_get_rfic_flags(const H5F_t *f) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + assert(f); + assert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->rfic_flags) +} /* end H5F_get_rfic_flags */ diff --git a/src/H5HG.c b/src/H5HG.c index 7037376118d..3709c705566 100644 --- a/src/H5HG.c +++ b/src/H5HG.c @@ -559,9 +559,13 @@ H5HG_read(H5F_t *f, H5HG_t *hobj, void *object /*out*/, size_t *buf_size) /* Load the heap */ if (NULL == (heap = H5HG__protect(f, hobj->addr, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, NULL, "unable to protect global heap"); + if (hobj->idx >= heap->nused) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); + if (NULL == heap->obj[hobj->idx].begin) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); - assert(hobj->idx < heap->nused); - assert(heap->obj[hobj->idx].begin); size = heap->obj[hobj->idx].size; p = heap->obj[hobj->idx].begin + H5HG_SIZEOF_OBJHDR(f); @@ -631,8 +635,12 @@ H5HG_link(H5F_t *f, const H5HG_t *hobj, int adjust) HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, FAIL, "unable to protect global heap"); if (adjust != 0) { - assert(hobj->idx < heap->nused); - assert(heap->obj[hobj->idx].begin); + if (hobj->idx >= heap->nused) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); + if (NULL == heap->obj[hobj->idx].begin) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); if ((heap->obj[hobj->idx].nrefs + adjust) < 0) HGOTO_ERROR(H5E_HEAP, H5E_BADRANGE, FAIL, "new link count would be out of range"); if ((heap->obj[hobj->idx].nrefs + adjust) > H5HG_MAXLINK) @@ -678,8 +686,13 @@ H5HG_get_obj_size(H5F_t *f, H5HG_t *hobj, size_t *obj_size) if (NULL == (heap = H5HG__protect(f, hobj->addr, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, FAIL, "unable to protect global heap"); - assert(hobj->idx < heap->nused); - assert(heap->obj[hobj->idx].begin); + /* Sanity check the heap object */ + if (hobj->idx >= heap->nused) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); + if (NULL == heap->obj[hobj->idx].begin) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); /* Set object size */ *obj_size = heap->obj[hobj->idx].size; @@ -722,14 +735,22 @@ H5HG_remove(H5F_t *f, H5HG_t *hobj) if (NULL == (heap = H5HG__protect(f, hobj->addr, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, FAIL, "unable to protect global heap"); - assert(hobj->idx < heap->nused); + /* Sanity check the heap object (split around bugfix below) */ + if (hobj->idx >= heap->nused) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); /* When the application selects the same location to rewrite the VL element by using H5Sselect_elements, * it can happen that the entry has been removed by first rewrite. Here we simply skip the removal of * the entry and let the second rewrite happen (see HDFFV-10635). In the future, it'd be nice to handle * this situation in H5T_conv_vlen in H5Tconv.c instead of this level (HDFFV-10648). */ if (heap->obj[hobj->idx].nrefs == 0 && heap->obj[hobj->idx].size == 0 && !heap->obj[hobj->idx].begin) - HGOTO_DONE(ret_value); + HGOTO_DONE(SUCCEED); + + /* Finish sanity checking the heap object */ + if (NULL == heap->obj[hobj->idx].begin) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}", + hobj->addr, hobj->idx); obj_start = heap->obj[hobj->idx].begin; /* Include object header size */ diff --git a/src/H5Oainfo.c b/src/H5Oainfo.c index 8b82e39e2a6..8b4340d2392 100644 --- a/src/H5Oainfo.c +++ b/src/H5Oainfo.c @@ -138,19 +138,25 @@ H5O__ainfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS ainfo->max_crt_idx = H5O_MAX_CRT_ORDER_IDX; /* Address of fractal heap to store "dense" attributes */ + H5_GCC_CLANG_DIAG_OFF("type-limits") if (H5_IS_BUFFER_OVERFLOW(p, sizeof_addr, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + H5_GCC_CLANG_DIAG_ON("type-limits") H5F_addr_decode(f, &p, &(ainfo->fheap_addr)); /* Address of v2 B-tree to index names of attributes (names are always indexed) */ + H5_GCC_CLANG_DIAG_OFF("type-limits") if (H5_IS_BUFFER_OVERFLOW(p, sizeof_addr, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + H5_GCC_CLANG_DIAG_ON("type-limits") H5F_addr_decode(f, &p, &(ainfo->name_bt2_addr)); /* Address of v2 B-tree to index creation order of links, if there is one */ if (ainfo->index_corder) { + H5_GCC_CLANG_DIAG_OFF("type-limits") if (H5_IS_BUFFER_OVERFLOW(p, sizeof_addr, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + H5_GCC_CLANG_DIAG_ON("type-limits") H5F_addr_decode(f, &p, &(ainfo->corder_bt2_addr)); } else diff --git a/src/H5Oattribute.c b/src/H5Oattribute.c index 88c595ea213..64634aac405 100644 --- a/src/H5Oattribute.c +++ b/src/H5Oattribute.c @@ -211,6 +211,13 @@ H5O__attr_create(const H5O_loc_t *loc, H5A_t *attr) if (NULL == (oh = H5O_pin(loc))) HGOTO_ERROR(H5E_ATTR, H5E_CANTPIN, FAIL, "unable to pin object header"); + /* Check for creating attribute with unusual datatype */ + if (H5T_is_numeric_with_unusual_unused_bits(attr->shared->dt) && + !(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(loc->file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, + "creating attribute with unusual datatype, see documentation for " + "H5Pset_relax_file_integrity_checks for details."); + /* Check if this object already has attribute information */ if (oh->version > H5O_VERSION_1) { bool new_ainfo = false; /* Flag to indicate that the attribute information is new */ @@ -1171,10 +1178,10 @@ herr_t H5O_attr_iterate_real(hid_t loc_id, const H5O_loc_t *loc, H5_index_t idx_type, H5_iter_order_t order, hsize_t skip, hsize_t *last_attr, const H5A_attr_iter_op_t *attr_op, void *op_data) { - H5O_t *oh = NULL; /* Pointer to actual object header */ - H5O_ainfo_t ainfo; /* Attribute information for object */ - H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */ - herr_t ret_value = FAIL; /* Return value */ + H5O_t *oh = NULL; /* Pointer to actual object header */ + H5O_ainfo_t ainfo; /* Attribute information for object */ + H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */ + herr_t ret_value = FAIL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_TAG(loc->addr) @@ -1223,7 +1230,7 @@ H5O_attr_iterate_real(hid_t loc_id, const H5O_loc_t *loc, H5_index_t idx_type, H oh = NULL; /* Check for skipping too many attributes */ - if (skip > 0 && skip >= atable.nattrs) + if (skip > 0 && skip >= atable.num_attrs) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index specified"); /* Iterate over attributes in table */ @@ -1293,8 +1300,8 @@ H5O__attr_iterate(hid_t loc_id, H5_index_t idx_type, H5_iter_order_t order, hsiz static herr_t H5O__attr_remove_update(const H5O_loc_t *loc, H5O_t *oh, H5O_ainfo_t *ainfo) { - H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */ - herr_t ret_value = SUCCEED; /* Return value */ + H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -1530,11 +1537,11 @@ H5O__attr_remove(const H5O_loc_t *loc, const char *name) herr_t H5O__attr_remove_by_idx(const H5O_loc_t *loc, H5_index_t idx_type, H5_iter_order_t order, hsize_t n) { - H5O_t *oh = NULL; /* Pointer to actual object header */ - H5O_ainfo_t ainfo; /* Attribute information for object */ - htri_t ainfo_exists = false; /* Whether the attribute info exists in the file */ - H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_t *oh = NULL; /* Pointer to actual object header */ + H5O_ainfo_t ainfo; /* Attribute information for object */ + htri_t ainfo_exists = false; /* Whether the attribute info exists in the file */ + H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_TAG(loc->addr) @@ -1568,7 +1575,7 @@ H5O__attr_remove_by_idx(const H5O_loc_t *loc, H5_index_t idx_type, H5_iter_order HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, "error building attribute table"); /* Check for skipping too many attributes */ - if (n >= atable.nattrs) + if (n >= atable.num_attrs) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index specified"); /* Set up user data for callback, to remove the attribute by name */ diff --git a/src/H5Odtype.c b/src/H5Odtype.c index 05652df0fe2..674d8d4ea1c 100644 --- a/src/H5Odtype.c +++ b/src/H5Odtype.c @@ -176,6 +176,14 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(*pp, dt->shared->u.atomic.offset); UINT16DECODE(*pp, dt->shared->u.atomic.prec); + + /* Sanity checks */ + if (dt->shared->u.atomic.offset >= (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "integer offset out of bounds"); + if (0 == dt->shared->u.atomic.prec) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "precision is zero"); + if (((dt->shared->u.atomic.offset + dt->shared->u.atomic.prec) - 1) >= (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "integer offset+precision out of bounds"); break; case H5T_FLOAT: @@ -212,6 +220,8 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unknown floating-point normalization"); } dt->shared->u.atomic.u.f.sign = (flags >> 8) & 0xff; + if (dt->shared->u.atomic.u.f.sign >= (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "sign bit position out of bounds"); if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, 2 + 2, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); @@ -224,6 +234,11 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t dt->shared->u.atomic.u.f.esize = *(*pp)++; if (dt->shared->u.atomic.u.f.esize == 0) HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "exponent size can't be zero"); + if (dt->shared->u.atomic.u.f.epos >= (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "exponent starting position out of bounds"); + if (((dt->shared->u.atomic.u.f.epos + dt->shared->u.atomic.u.f.esize) - 1) >= + (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "exponent range out of bounds"); if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, 1 + 1, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); @@ -231,10 +246,30 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t dt->shared->u.atomic.u.f.msize = *(*pp)++; if (dt->shared->u.atomic.u.f.msize == 0) HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "mantissa size can't be zero"); + if (dt->shared->u.atomic.u.f.mpos >= (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "mantissa starting position out of bounds"); + if (((dt->shared->u.atomic.u.f.mpos + dt->shared->u.atomic.u.f.msize) - 1) >= + (dt->shared->size * 8)) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "mantissa range out of bounds"); if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, 4, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(*pp, dt->shared->u.atomic.u.f.ebias); + + /* Sanity check bits don't overlap */ + if (H5_RANGE_OVERLAP(dt->shared->u.atomic.u.f.sign, dt->shared->u.atomic.u.f.sign, + dt->shared->u.atomic.u.f.epos, + ((dt->shared->u.atomic.u.f.epos + dt->shared->u.atomic.u.f.esize) - 1))) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "exponent and sign positions overlap"); + if (H5_RANGE_OVERLAP(dt->shared->u.atomic.u.f.sign, dt->shared->u.atomic.u.f.sign, + dt->shared->u.atomic.u.f.mpos, + ((dt->shared->u.atomic.u.f.mpos + dt->shared->u.atomic.u.f.msize) - 1))) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "mantissa and sign positions overlap"); + if (H5_RANGE_OVERLAP(dt->shared->u.atomic.u.f.epos, + ((dt->shared->u.atomic.u.f.epos + dt->shared->u.atomic.u.f.esize) - 1), + dt->shared->u.atomic.u.f.mpos, + ((dt->shared->u.atomic.u.f.mpos + dt->shared->u.atomic.u.f.msize) - 1))) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "mantissa and exponent positions overlap"); break; case H5T_TIME: @@ -378,9 +413,11 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t /* Decode the field offset */ /* (starting with version 3 of the datatype message, use the minimum # of bytes required) */ if (version >= H5O_DTYPE_VERSION_3) { + H5_GCC_CLANG_DIAG_OFF("type-limits") if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, offset_nbytes, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); + H5_GCC_CLANG_DIAG_ON("type-limits") UINT32DECODE_VAR(*pp, dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset, offset_nbytes); } @@ -452,6 +489,13 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t } if (temp_type->shared->size == 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "type size can't be zero"); + if ((dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset + + temp_type->shared->size) > dt->shared->size) { + if (H5T_close_real(temp_type) < 0) + HDONE_ERROR(H5E_DATATYPE, H5E_CANTRELEASE, FAIL, "can't release datatype info"); + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, + "member type extends outside its parent compound type"); + } /* Upgrade the version if we can and it is necessary */ if (can_upgrade && temp_type->shared->version > version) { @@ -770,6 +814,19 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unknown datatype class found"); } + /* Check for numeric type w/unusual # of unused bits */ + if (H5T_is_numeric_with_unusual_unused_bits(dt)) + /* Throw an error if the object header is not checksummed, unless the + * H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS flag is set with + * H5Pset_relax_file_integrity_checks() to suppress it. + */ + if (!(*ioflags & H5O_DECODEIO_RFIC_UNUBNT)) + HGOTO_ERROR( + H5E_DATATYPE, H5E_BADVALUE, FAIL, + "datatype has unusually large # of unused bits (prec = %zu bits, size = %zu bytes), possibly " + "corrupted file. See documentation for H5Pset_relax_file_integrity_checks for details.", + dt->shared->u.atomic.prec, dt->shared->size); + done: /* Cleanup on error */ if (ret_value < 0) @@ -1307,8 +1364,8 @@ H5O__dtype_encode_helper(uint8_t **pp, const H5T_t *dt) function using malloc() and is returned to the caller. --------------------------------------------------------------------------*/ static void * -H5O__dtype_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, - unsigned *ioflags /*in,out*/, size_t p_size, const uint8_t *p) +H5O__dtype_decode(H5F_t *f, H5O_t *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, unsigned *ioflags /*in,out*/, + size_t p_size, const uint8_t *p) { bool skip; H5T_t *dt = NULL; @@ -1331,6 +1388,17 @@ H5O__dtype_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsign */ skip = (p_size == SIZE_MAX ? true : false); + /* Indicate if the object header has a checksum, or if the + * H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS flag is set */ + if (open_oh) { + if (H5O_SIZEOF_CHKSUM_OH(open_oh) > 0 || + (f && (H5F_RFIC_FLAGS(f) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + *ioflags |= H5O_DECODEIO_RFIC_UNUBNT; + } + else + /* Decode operations from non-object headers are assumed to be checksummed */ + *ioflags |= H5O_DECODEIO_RFIC_UNUBNT; + /* Perform actual decode of message */ if (H5O__dtype_decode_helper(ioflags, &p, dt, skip, p_end) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, NULL, "can't decode type"); diff --git a/src/H5Oefl.c b/src/H5Oefl.c index ebd92a733ba..57e5e6991df 100644 --- a/src/H5Oefl.c +++ b/src/H5Oefl.c @@ -140,6 +140,9 @@ H5O__efl_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED #endif for (size_t u = 0; u < mesg->nused; u++) { + + hsize_t offset = 0; + /* Name */ if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); @@ -156,7 +159,8 @@ H5O__efl_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED /* File offset */ if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); - H5F_DECODE_LENGTH(f, p, mesg->slot[u].offset); + H5F_DECODE_LENGTH(f, p, offset); /* Decode into an hsize_t to avoid sign warnings */ + mesg->slot[u].offset = (HDoff_t)offset; /* Size */ if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) diff --git a/src/H5Oint.c b/src/H5Oint.c index 537563dbbad..a98e22a1821 100644 --- a/src/H5Oint.c +++ b/src/H5Oint.c @@ -2919,3 +2919,23 @@ H5O__reset_info2(H5O_info2_t *oinfo) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5O__reset_info2() */ + +/*------------------------------------------------------------------------- + * Function: H5O_has_chksum + * + * Purpose: Returns true if object header is checksummed + * + * Return: true/false on success, can't fail + * + *------------------------------------------------------------------------- + */ +bool +H5O_has_chksum(const H5O_t *oh) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check args */ + assert(oh); + + FUNC_LEAVE_NOAPI(H5O_SIZEOF_CHKSUM_OH(oh) > 0) +} /* end H5O_has_chksum() */ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index fc0f59ee60d..d14e0009953 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -392,10 +392,16 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU case H5D_CHUNK_IDX_SINGLE: /* Single Chunk Index */ if (mesg->u.chunk.flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) { + uint64_t nbytes = 0; + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f) + 4, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); - H5F_DECODE_LENGTH(f, p, mesg->storage.u.chunk.u.single.nbytes); + + H5F_DECODE_LENGTH(f, p, nbytes); + H5_CHECKED_ASSIGN(mesg->storage.u.chunk.u.single.nbytes, uint32_t, nbytes, + uint64_t); + UINT32DECODE(p, mesg->storage.u.chunk.u.single.filter_mask); } diff --git a/src/H5Olinfo.c b/src/H5Olinfo.c index 830e4e3113c..6b5dc0b89b5 100644 --- a/src/H5Olinfo.c +++ b/src/H5Olinfo.c @@ -140,6 +140,9 @@ H5O__linfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS if (H5_IS_BUFFER_OVERFLOW(p, 8, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); INT64DECODE(p, linfo->max_corder); + if (linfo->max_corder < 0) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, + "invalid max creation order value for message: %" PRId64, linfo->max_corder); } else linfo->max_corder = 0; @@ -156,8 +159,10 @@ H5O__linfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS /* Address of v2 B-tree to index creation order of links, if there is one */ if (linfo->index_corder) { + H5_GCC_CLANG_DIAG_OFF("type-limits") if (H5_IS_BUFFER_OVERFLOW(p, addr_size, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + H5_GCC_CLANG_DIAG_ON("type-limits") H5F_addr_decode(f, &p, &(linfo->corder_bt2_addr)); } else diff --git a/src/H5Opkg.h b/src/H5Opkg.h index 4c719bf340e..8e32f3ae13a 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -150,6 +150,8 @@ /* Input/output flags for decode functions */ #define H5O_DECODEIO_NOCHANGE 0x01u /* IN: do not modify values */ #define H5O_DECODEIO_DIRTY 0x02u /* OUT: message has been changed */ +#define H5O_DECODEIO_RFIC_UNUBNT \ + 0x04u /* IN: Relax file integrity checks for unusual numbers of unused bits in numeric datatypes */ /* Macro to incremend ndecode_dirtied (only if we are debugging) */ #ifndef NDEBUG diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 6f3d39d1e83..968a23caada 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -932,6 +932,7 @@ H5_DLL time_t H5O_get_oh_mtime(const H5O_t *oh); H5_DLL uint8_t H5O_get_oh_version(const H5O_t *oh); H5_DLL herr_t H5O_get_rc_and_type(const H5O_loc_t *oloc, unsigned *rc, H5O_type_t *otype); H5_DLL H5AC_proxy_entry_t *H5O_get_proxy(const H5O_t *oh); +H5_DLL bool H5O_has_chksum(const H5O_t *oh); /* Object header message routines */ H5_DLL herr_t H5O_msg_create(const H5O_loc_t *loc, unsigned type_id, unsigned mesg_flags, diff --git a/src/H5Pencdec.c b/src/H5Pencdec.c index 4d1d8187591..19d453279ee 100644 --- a/src/H5Pencdec.c +++ b/src/H5Pencdec.c @@ -272,6 +272,41 @@ H5P__encode_double(const void *value, void **_pp, size_t *size) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5P__encode_double() */ +/*------------------------------------------------------------------------- + * Function: H5P__encode_uint64_t + * + * Purpose: Generic encoding callback routine for 'uint64_t' properties. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5P__encode_uint64_t(const void *value, void **_pp, size_t *size) +{ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + assert(value); + assert(size); + + if (NULL != *pp) { + /* Encode the size */ + *(*pp)++ = (uint8_t)sizeof(uint64_t); + + /* Encode the value */ + UINT64ENCODE(*pp, *(const unsigned *)value); + } /* end if */ + + /* Set size needed for encoding */ + *size += (1 + sizeof(uint64_t)); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__encode_uint64_t() */ + /*-------------------------------------------------------------------------- NAME H5P__encode_cb @@ -611,6 +646,42 @@ H5P__decode_double(const void **_pp, void *_value) FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__decode_double() */ +/*------------------------------------------------------------------------- + * Function: H5P__decode_uint64_t + * + * Purpose: Generic decoding callback routine for 'uint64_t' properties. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5P__decode_uint64_t(const void **_pp, void *_value) +{ + uint64_t *value = (uint64_t *)_value; /* Property value to return */ + const uint8_t **pp = (const uint8_t **)_pp; + unsigned enc_size; /* Size of encoded property */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + assert(pp); + assert(*pp); + assert(value); + + /* Decode the size */ + enc_size = *(*pp)++; + if (enc_size != sizeof(uint64_t)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "uint64_t value can't be decoded"); + + UINT64DECODE(*pp, *value); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__decode_uint64_t() */ + /*------------------------------------------------------------------------- NAME H5P__decode diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 65c21408a3f..80922bd9d1d 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -333,6 +333,11 @@ #endif #define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC H5P__encode_bool #define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC H5P__decode_bool +/* Definition for 'rfic' flags */ +#define H5F_ACS_RFIC_FLAGS_SIZE sizeof(uint64_t) +#define H5F_ACS_RFIC_FLAGS_DEF 0 +#define H5F_ACS_RFIC_FLAGS_ENC H5P__encode_uint64_t +#define H5F_ACS_RFIC_FLAGS_DEC H5P__decode_uint64_t /******************/ /* Local Typedefs */ @@ -529,6 +534,7 @@ static const bool H5F_def_use_file_locking_g = H5F_ACS_USE_FILE_LOCKING_DEF; /* Default use file locking flag */ static const bool H5F_def_ignore_disabled_file_locks_g = H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */ +static const uint64_t H5F_def_rfic_flags_g = H5F_ACS_RFIC_FLAGS_DEF; /* Default 'rfic' flags */ /*------------------------------------------------------------------------- * Function: H5P__facc_reg_prop @@ -826,6 +832,12 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); + /* Register the 'rfic' flags. */ + if (H5P__register_real(pclass, H5F_ACS_RFIC_FLAGS_NAME, H5F_ACS_RFIC_FLAGS_SIZE, &H5F_def_rfic_flags_g, + NULL, NULL, NULL, H5F_ACS_RFIC_FLAGS_ENC, H5F_ACS_RFIC_FLAGS_DEC, NULL, NULL, NULL, + NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -6264,3 +6276,75 @@ H5P__facc_vol_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_vol_close() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_relax_file_integrity_checks + * + * Purpose: Relax certain file integrity checks that may issue errors + * for valid files that have the potential for incorrect library + * behavior when data is incorrect or corrupted. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_relax_file_integrity_checks(hid_t plist_id, uint64_t flags) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "iUL", plist_id, flags); + + /* Check arguments */ + if (H5P_DEFAULT == plist_id) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't modify default property list"); + if (flags & (uint64_t)~H5F_RFIC_ALL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags"); + + /* Get the property list structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list"); + + /* Set value */ + if (H5P_set(plist, H5F_ACS_RFIC_FLAGS_NAME, &flags) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set relaxed file integrity check flags"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_relax_file_integrity_checks() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_relax_file_integrity_checks + * + * Purpose: Retrieve relaxed file integrity check flags + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_relax_file_integrity_checks(hid_t plist_id, uint64_t *flags /*out*/) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*UL", plist_id, flags); + + if (H5P_DEFAULT == plist_id) + plist_id = H5P_FILE_ACCESS_DEFAULT; + + /* Get the property list structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list"); + + /* Get value */ + if (flags) + if (H5P_get(plist, H5F_ACS_RFIC_FLAGS_NAME, flags) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get relaxed file integrity check flags"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_relax_file_integrity_checks() */ diff --git a/src/H5Ppkg.h b/src/H5Ppkg.h index e249c7f9b2a..63baced637a 100644 --- a/src/H5Ppkg.h +++ b/src/H5Ppkg.h @@ -175,12 +175,14 @@ H5_DLL herr_t H5P__encode_unsigned(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_uint8_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_bool(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_double(const void *value, void **_pp, size_t *size); +H5_DLL herr_t H5P__encode_uint64_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__decode_hsize_t(const void **_pp, void *value); H5_DLL herr_t H5P__decode_size_t(const void **_pp, void *value); H5_DLL herr_t H5P__decode_unsigned(const void **_pp, void *value); H5_DLL herr_t H5P__decode_uint8_t(const void **_pp, void *value); H5_DLL herr_t H5P__decode_bool(const void **_pp, void *value); H5_DLL herr_t H5P__decode_double(const void **_pp, void *value); +H5_DLL herr_t H5P__decode_uint64_t(const void **_pp, void *value); H5_DLL herr_t H5P__encode_coll_md_read_flag_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__decode_coll_md_read_flag_t(const void **_pp, void *value); diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index bf0a3be436c..a73dffb2ca5 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -5634,6 +5634,79 @@ H5_DLL herr_t H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t H5_DLL herr_t H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_per, unsigned min_raw_per); +/** + * \ingroup FAPL + * + * \brief Relax file integrity checks that may issue errors for some valid files + * + * \fapl_id{plist_id} + * \param[in] flags Relaxed integrity checks flag. Valid values are: + * \li #H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS + * suppresses integrity checks for detecting + * unusually high values for the number of unused bits in + * numeric datatype classes (H5T_INTEGER, H5T_FLOAT, and + * H5T_BITFIELD). Integrity checks are triggered when + * the precision for a datatype (i.e. the number of bits + * containing actual data) is less than half of the + * datatype's size and the datatype is greater than + * 1 byte in size. For example, a datatype with a + * precision of 15 bits and a size of 4 bytes (i.e. 32 bits) + * will issue an error, but a datatype with 17 bits of + * precision and a size of 4 bytes will not issue an + * error, nor will a datatype with a precision of 1, 2, or + * 3 bits and a size of 1 byte issue an error. + * \li #H5F_RFIC_ALL relaxes all integrity checks above. + * + * \return \herr_t + * + * \details Incorrectly encoded or corrupted metadata in a native HDF5 + * format file can cause incorrect library behavior when the metadata + * has no checksum. Integrity checks within the library detect these + * circumstances and issue errors when incorrect metadata is found. + * Unfortunately, some of the integrity checks for detecting these + * circumstances may incorrectly issue an error for a valid HDF5 file + * that was intentionally created with these configurations. + * Setting the appropriate flag(s) with this routine will relax the + * file integrity checks for these valid files and suppress errors + * when accessing objects with these configurations. + * + * The library will also issue errors when these configurations are + * used to create objects, preventing applications from unintentionally + * creating them. Setting the appropriate flag with this routine will + * also suppress those errors on creation, although using this routine + * and the appropriate flag(s) will still be required when accessing + * files created with these configurations. + * + * A more complete solution that avoids errors on both object creation + * and access is to use the H5Pset_libver_bounds routine with a low + * bound of at least #H5F_LIBVER_V18 when creating objects with these + * configurations. This will cause the library to checksum a file's + * metadata, allowing accidental data corruption to be correctly + * detected and errors correctly issued without ambiguity. + * + * \since 1.14.4 + * + */ +H5_DLL herr_t H5Pset_relax_file_integrity_checks(hid_t plist_id, uint64_t flags); +/** + * \ingroup FAPL + * + * \brief Retrieve relaxed file integrity check flags + * + * \fapl_id{plist_id} + * \param[out] flags Relaxed file integrity check flags + * + * \return \herr_t + * + * \details H5Pget_relax_file_integrity_checks() retrieves the relaxed file + * integrity check value into \p flags for the file access property + * list specified in \p plist_id. + * + * \since 1.14.4 + * + */ +H5_DLL herr_t H5Pget_relax_file_integrity_checks(hid_t plist_id, uint64_t *flags); + /* Dataset creation property list (DCPL) routines */ /** * \ingroup DCPL diff --git a/src/H5Rint.c b/src/H5Rint.c index 4606a57d3f7..3df70ba48c3 100644 --- a/src/H5Rint.c +++ b/src/H5Rint.c @@ -956,7 +956,10 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref) const uint8_t *p = (const uint8_t *)buf; size_t buf_size = 0, decode_size = 0; uint8_t flags; - herr_t ret_value = SUCCEED; + bool decoded_filename = false; /* Whether filename was decoded, for error handling */ + bool decoded_attrname = false; /* Whether attribute name was decoded, for error handling */ + bool decoded_dataspace = false; /* Whether dataspace was decoded, for error handling */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE @@ -990,6 +993,7 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref) /* Decode file name */ H5R_DECODE(H5R__decode_string, &ref->info.obj.filename, p, buf_size, decode_size, "Cannot decode filename"); + decoded_filename = true; } else ref->info.obj.filename = NULL; @@ -997,22 +1001,28 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref) switch (ref->type) { case H5R_OBJECT2: break; + case H5R_DATASET_REGION2: /* Decode dataspace */ H5R_DECODE(H5R__decode_region, &ref->info.reg.space, p, buf_size, decode_size, "Cannot decode region"); + decoded_dataspace = true; break; + case H5R_ATTR: /* Decode attribute name */ H5R_DECODE(H5R__decode_string, &ref->info.attr.name, p, buf_size, decode_size, "Cannot decode attribute name"); + decoded_attrname = true; break; + case H5R_OBJECT1: case H5R_DATASET_REGION1: case H5R_BADTYPE: case H5R_MAXTYPE: assert("invalid reference type" && 0); HGOTO_ERROR(H5E_REFERENCE, H5E_UNSUPPORTED, FAIL, "internal error (invalid reference type)"); + default: assert("unknown reference type" && 0); HGOTO_ERROR(H5E_REFERENCE, H5E_UNSUPPORTED, FAIL, "internal error (unknown reference type)"); @@ -1031,6 +1041,22 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref) *nbytes = decode_size; done: + if (ret_value < 0) { + if (decoded_filename) { + H5MM_xfree(ref->info.obj.filename); + ref->info.obj.filename = NULL; + } + if (decoded_attrname) { + H5MM_xfree(ref->info.attr.name); + ref->info.attr.name = NULL; + } + if (decoded_dataspace) { + if (H5S_close(ref->info.reg.space) < 0) + HDONE_ERROR(H5E_REFERENCE, H5E_CLOSEERROR, FAIL, "unable to release dataspace"); + ref->info.reg.space = NULL; + } + } + FUNC_LEAVE_NOAPI(ret_value) } /* end H5R__decode() */ @@ -1175,7 +1201,7 @@ H5R__decode_region(const unsigned char *buf, size_t *nbytes, H5S_t **space_ptr) const uint8_t *p_end = p + *nbytes - 1; size_t buf_size = 0; unsigned rank; - H5S_t *space; + H5S_t *space = NULL; herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE @@ -1216,6 +1242,10 @@ H5R__decode_region(const unsigned char *buf, size_t *nbytes, H5S_t **space_ptr) *space_ptr = space; done: + if (ret_value < 0) + if (space && H5S_close(space) < 0) + HDONE_ERROR(H5E_REFERENCE, H5E_CLOSEERROR, FAIL, "unable to release dataspace"); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5R__decode_region() */ diff --git a/src/H5S.c b/src/H5S.c index c58176e3243..722eac8b563 100644 --- a/src/H5S.c +++ b/src/H5S.c @@ -615,6 +615,10 @@ H5S__extent_copy_real(H5S_extent_t *dst, const H5S_extent_t *src, bool copy_max) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOPY, FAIL, "can't copy shared information"); done: + if (ret_value < 0) + if (dst->size) + dst->size = H5FL_ARR_FREE(hsize_t, dst->size); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5S__extent_copy_real() */ diff --git a/src/H5Shyper.c b/src/H5Shyper.c index 344351e05be..6b07529df96 100644 --- a/src/H5Shyper.c +++ b/src/H5Shyper.c @@ -9178,8 +9178,8 @@ H5S__check_spans_overlap(const H5S_hyper_span_info_t *spans1, const H5S_hyper_sp assert(spans2); /* Use low & high bounds to try to avoid spinning through the span lists */ - if (H5S_RANGE_OVERLAP(spans1->low_bounds[0], spans1->high_bounds[0], spans2->low_bounds[0], - spans2->high_bounds[0])) { + if (H5_RANGE_OVERLAP(spans1->low_bounds[0], spans1->high_bounds[0], spans2->low_bounds[0], + spans2->high_bounds[0])) { H5S_hyper_span_t *span1, *span2; /* Hyperslab spans */ /* Walk over spans, comparing them for overlap */ @@ -9187,7 +9187,7 @@ H5S__check_spans_overlap(const H5S_hyper_span_info_t *spans1, const H5S_hyper_sp span2 = spans2->head; while (span1 && span2) { /* Check current two spans for overlap */ - if (H5S_RANGE_OVERLAP(span1->low, span1->high, span2->low, span2->high)) { + if (H5_RANGE_OVERLAP(span1->low, span1->high, span2->low, span2->high)) { /* Check for spans in lowest dimension already */ if (span1->down) { /* Sanity check */ @@ -9765,8 +9765,8 @@ H5S__hyper_regular_and_single_block(H5S_t *space, const hsize_t start[], const h block_end = (start[u] + block[u]) - 1; /* Check for overlap */ - if (!H5S_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u], - block_end)) { + if (!H5_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u], + block_end)) { overlap = false; break; } /* end if */ @@ -9812,8 +9812,8 @@ H5S__hyper_regular_and_single_block(H5S_t *space, const hsize_t start[], const h block_end = (start[u] + block[u]) - 1; /* Check for overlap */ - if (!H5S_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u], - block_end)) { + if (!H5_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u], + block_end)) { overlap = false; break; } /* end if */ @@ -10446,7 +10446,7 @@ H5S_combine_hyperslab(const H5S_t *old_space, H5S_seloper_t op, const hsize_t st } /* end for */ /* Check bound box of both spaces to see if they overlap */ - if (H5S_RANGE_OVERLAP(old_low_bounds[0], old_high_bounds[0], new_low_bounds[0], new_high_bounds[0])) + if (H5_RANGE_OVERLAP(old_low_bounds[0], old_high_bounds[0], new_low_bounds[0], new_high_bounds[0])) overlapped = true; /* Non-overlapping situations can be handled in special ways */ @@ -11415,8 +11415,8 @@ H5S__hyper_proj_int_iterate(H5S_hyper_span_info_t *ss_span_info, const H5S_hyper /* Check for non-overlapping bounds */ check_intersect = true; for (u = 0; u < (udata->ss_rank - depth); u++) - if (!H5S_RANGE_OVERLAP(ss_span_info->low_bounds[u], ss_span_info->high_bounds[u], - sis_span_info->low_bounds[u], sis_span_info->high_bounds[u])) { + if (!H5_RANGE_OVERLAP(ss_span_info->low_bounds[u], ss_span_info->high_bounds[u], + sis_span_info->low_bounds[u], sis_span_info->high_bounds[u])) { check_intersect = false; break; } /* end if */ @@ -11441,7 +11441,7 @@ H5S__hyper_proj_int_iterate(H5S_hyper_span_info_t *ss_span_info, const H5S_hyper /* Main loop */ do { /* Check if spans overlap */ - if (H5S_RANGE_OVERLAP(ss_low, ss_span->high, sis_low, sis_span->high)) { + if (H5_RANGE_OVERLAP(ss_low, ss_span->high, sis_low, sis_span->high)) { high = MIN(ss_span->high, sis_span->high); if (ss_span->down) { /* Add skipped elements if there's a pre-gap */ diff --git a/src/H5Spkg.h b/src/H5Spkg.h index fddd5e3c5fa..e851f548f94 100644 --- a/src/H5Spkg.h +++ b/src/H5Spkg.h @@ -89,15 +89,6 @@ * H5S_UNLIMITED) */ #define H5S_MAX_SIZE ((hsize_t)(hssize_t)(-2)) -/* Macro for checking if two ranges overlap one another */ -/* - * Check for the inverse of whether the ranges are disjoint. If they are - * disjoint, then the low bound of one of the ranges must be greater than the - * high bound of the other. - */ -/* (Assumes that low & high bounds are _inclusive_) */ -#define H5S_RANGE_OVERLAP(L1, H1, L2, H2) (!((L1) > (H2) || (L2) > (H1))) - /* * Dataspace extent information */ diff --git a/src/H5Spoint.c b/src/H5Spoint.c index c95429dcfee..7e66a03a5f6 100644 --- a/src/H5Spoint.c +++ b/src/H5Spoint.c @@ -576,8 +576,14 @@ H5S__point_add(H5S_t *space, H5S_seloper_t op, size_t num_elem, const hsize_t *c for (u = 0; u < num_elem; u++) { unsigned dim; /* Counter for dimensions */ + /* The following allocation relies on the size of an hcoords_t being + * the same as an 'H5S_pnt_node_t *', so fail now if that's not true + */ + HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *)); + /* Allocate space for the new node */ - if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, space->extent.rank))) + /* Note: allocating "rank + 1" to allow for 'next' pointer */ + if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, space->extent.rank + 1))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point node"); /* Initialize fields in node */ @@ -796,7 +802,7 @@ H5S__copy_pnt_list(const H5S_pnt_list_t *src, unsigned rank) assert(rank > 0); /* Allocate room for the head of the point list */ - if (NULL == (dst = H5FL_MALLOC(H5S_pnt_list_t))) + if (NULL == (dst = H5FL_CALLOC(H5S_pnt_list_t))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, NULL, "can't allocate point list node"); curr = src->head; @@ -804,8 +810,14 @@ H5S__copy_pnt_list(const H5S_pnt_list_t *src, unsigned rank) while (curr) { H5S_pnt_node_t *new_node; /* New point information node */ + /* The following allocation relies on the size of an hcoords_t being + * the same as an 'H5S_pnt_node_t *', so fail now if that's not true + */ + HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *)); + /* Create new point */ - if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, rank))) + /* Note: allocating "rank + 1" to allow for 'next' pointer */ + if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, rank + 1))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, NULL, "can't allocate point node"); new_node->next = NULL; @@ -2275,7 +2287,7 @@ H5S__point_project_simple(const H5S_t *base_space, H5S_t *new_space, hsize_t *of HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't release selection"); /* Allocate room for the head of the point list */ - if (NULL == (new_space->select.sel_info.pnt_lst = H5FL_MALLOC(H5S_pnt_list_t))) + if (NULL == (new_space->select.sel_info.pnt_lst = H5FL_CALLOC(H5S_pnt_list_t))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point list node"); /* Check if the new space's rank is < or > base space's rank */ @@ -2294,8 +2306,14 @@ H5S__point_project_simple(const H5S_t *base_space, H5S_t *new_space, hsize_t *of base_node = base_space->select.sel_info.pnt_lst->head; prev_node = NULL; while (base_node) { + /* The following allocation relies on the size of an hcoords_t being + * the same as an 'H5S_pnt_node_t *', so fail now if that's not true + */ + HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *)); + /* Create new point */ - if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank))) + /* Note: allocating "rank + 1" to allow for 'next' pointer */ + if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank + 1))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point node"); new_node->next = NULL; @@ -2336,8 +2354,14 @@ H5S__point_project_simple(const H5S_t *base_space, H5S_t *new_space, hsize_t *of base_node = base_space->select.sel_info.pnt_lst->head; prev_node = NULL; while (base_node) { + /* The following allocation relies on the size of an hcoords_t being + * the same as an 'H5S_pnt_node_t *', so fail now if that's not true + */ + HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *)); + /* Create new point */ - if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank))) + /* Note: allocating "rank + 1" to allow for 'next' pointer */ + if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank + 1))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point node"); new_node->next = NULL; diff --git a/src/H5Sselect.c b/src/H5Sselect.c index d67b8e6f7e0..ef9994ec1b2 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -1945,7 +1945,7 @@ H5S_select_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *en /* Loop over selection bounds and block, checking for overlap */ for (u = 0; u < space->extent.rank; u++) /* If selection bounds & block don't overlap, can leave now */ - if (!H5S_RANGE_OVERLAP(low[u], high[u], start[u], end[u])) + if (!H5_RANGE_OVERLAP(low[u], high[u], start[u], end[u])) HGOTO_DONE(false); } /* end if */ diff --git a/src/H5T.c b/src/H5T.c index 640050b8145..ba93eefd1e4 100644 --- a/src/H5T.c +++ b/src/H5T.c @@ -41,6 +41,7 @@ #include "H5Pprivate.h" /* Property lists */ #include "H5Tpkg.h" /* Datatypes */ #include "H5VLprivate.h" /* Virtual Object Layer */ +#include "H5VMprivate.h" /* Vectors and arrays */ /****************/ /* Local Macros */ @@ -6692,3 +6693,51 @@ H5T__get_path_table_npaths(void) FUNC_LEAVE_NOAPI(ret_value) } + +/*------------------------------------------------------------------------- + * Function: H5T_is_numeric_with_unusual_unused_bits + * + * Purpose: Detect if a datatype is a numeric datatype (int, float, or + * bitfield) with an unusual # of unused bits. This means + * that the precision (i.e. the # of bits used) is less than + * the size of the datatype, at power-of-two boundaries. + * + * Return: true/false on success, can't fail + * + *------------------------------------------------------------------------- + */ +bool +H5T_is_numeric_with_unusual_unused_bits(const H5T_t *dt) +{ + bool ret_value = false; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + assert(dt); + assert(dt->shared); + + /* Is the correct type? */ + if (H5T_INTEGER == dt->shared->type || H5T_FLOAT == dt->shared->type || + H5T_BITFIELD == dt->shared->type) { +#if LDBL_MANT_DIG == 106 + /* This currently won't work for the IBM long double type */ + if (H5T_FLOAT == dt->shared->type && dt->shared->size == 16 && + (dt->shared->u.atomic.prec == 64 || dt->shared->u.atomic.prec == 128)) + HGOTO_DONE(false); +#endif + + /* Has unused bits? */ + if (dt->shared->u.atomic.prec < (dt->shared->size * 8)) { + unsigned surround_bits = + 1U << (1 + H5VM_log2_gen((dt->shared->u.atomic.prec + dt->shared->u.atomic.offset) - 1)); + + /* Unused bits are unusually large? */ + if (dt->shared->size > 1 && ((dt->shared->size * 8) > surround_bits)) + HGOTO_DONE(true); + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5T_is_numeric_with_unusual_unused_bits() */ diff --git a/src/H5Tcommit.c b/src/H5Tcommit.c index 5c4b4be9553..2e23764fa7a 100644 --- a/src/H5Tcommit.c +++ b/src/H5Tcommit.c @@ -424,11 +424,13 @@ H5T__commit_anon(H5F_t *file, H5T_t *type, hid_t tcpl_id) herr_t H5T__commit(H5F_t *file, H5T_t *type, hid_t tcpl_id) { - H5O_loc_t temp_oloc; /* Temporary object header location */ - H5G_name_t temp_path; /* Temporary path */ - bool loc_init = false; /* Have temp_oloc and temp_path been initialized? */ - size_t dtype_size; /* Size of the datatype message */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_t *oh = NULL; /* Pointer to actual object header */ + H5O_loc_t temp_oloc; /* Temporary object header location */ + H5G_name_t temp_path; /* Temporary path */ + bool loc_init = false; /* Have temp_oloc and temp_path been initialized? */ + bool ohdr_created = false; /* Has the object header been created yet? */ + size_t dtype_size; /* Size of the datatype message */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -481,9 +483,23 @@ H5T__commit(H5F_t *file, H5T_t *type, hid_t tcpl_id) */ if (H5O_create(file, dtype_size, (size_t)1, tcpl_id, &temp_oloc) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to create datatype object header"); - if (H5O_msg_create(&temp_oloc, H5O_DTYPE_ID, H5O_MSG_FLAG_CONSTANT | H5O_MSG_FLAG_DONTSHARE, - H5O_UPDATE_TIME, type) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to update type header message"); + ohdr_created = true; + + /* Pin the object header */ + if (NULL == (oh = H5O_pin(&temp_oloc))) + HGOTO_ERROR(H5E_ATTR, H5E_CANTPIN, FAIL, "unable to pin object header"); + + /* Check for creating committed datatype with unusual datatype */ + if (H5T_is_numeric_with_unusual_unused_bits(type) && + !(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, + "creating committed datatype with unusual datatype, see documentation for " + "H5Pset_relax_file_integrity_checks for details."); + + /* Insert the datatype message */ + if (H5O_msg_append_oh(file, oh, H5O_DTYPE_ID, H5O_MSG_FLAG_CONSTANT | H5O_MSG_FLAG_DONTSHARE, + H5O_UPDATE_TIME, type) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to insert type header message"); /* Copy the new object header's location into the datatype, taking ownership of it */ if (H5O_loc_copy_shallow(&(type->oloc), &temp_oloc) < 0) @@ -510,23 +526,39 @@ H5T__commit(H5F_t *file, H5T_t *type, hid_t tcpl_id) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "cannot mark datatype in memory"); done: + if (oh && H5O_unpin(oh) < 0) + HDONE_ERROR(H5E_DATATYPE, H5E_CANTUNPIN, FAIL, "unable to unpin object header"); + if (ret_value < 0) { - if (loc_init) { - H5O_loc_free(&temp_oloc); - H5G_name_free(&temp_path); - } /* end if */ - if ((type->shared->state == H5T_STATE_TRANSIENT || type->shared->state == H5T_STATE_RDONLY) && - (type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED)) { - if (H5O_dec_rc_by_loc(&(type->oloc)) < 0) + /* Close & delete the object header on failure */ + if (ohdr_created) { + H5O_loc_t *oloc_ptr; /* Pointer to object header location */ + + /* Point at correct object header location, depending on state when failure occurred */ + if (loc_init) + oloc_ptr = &temp_oloc; + else + oloc_ptr = &(type->oloc); + if (H5O_dec_rc_by_loc(oloc_ptr) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL, "unable to decrement refcount on newly created object"); - if (H5O_close(&(type->oloc), NULL) < 0) + if (H5O_close(oloc_ptr, NULL) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "unable to release object header"); - if (H5O_delete(file, type->sh_loc.u.loc.oh_addr) < 0) + if (H5O_delete(file, oloc_ptr->addr) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTDELETE, FAIL, "unable to delete object header"); + } + + /* Release the location info, if the datatype doesn't own it */ + if (loc_init) { + H5O_loc_free(&temp_oloc); + H5G_name_free(&temp_path); + } + + /* Reset the shared state for the datatype */ + if ((type->shared->state == H5T_STATE_TRANSIENT || type->shared->state == H5T_STATE_RDONLY) && + (type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED)) type->sh_loc.type = H5O_SHARE_TYPE_UNSHARED; - } /* end if */ - } /* end if */ + } FUNC_LEAVE_NOAPI(ret_value) } /* H5T__commit() */ diff --git a/src/H5Tconv.c b/src/H5Tconv.c index 422e3110497..55b6d7d01da 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -19,18 +19,21 @@ /****************/ #include "H5Tmodule.h" /* This source code file is part of the H5T module */ +#define H5R_FRIEND /* Suppress error about including H5Rpkg */ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5CXprivate.h" /* API Contexts */ -#include "H5Dprivate.h" /* Datasets */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5FLprivate.h" /* Free Lists */ -#include "H5Iprivate.h" /* IDs */ -#include "H5MMprivate.h" /* Memory management */ -#include "H5Tpkg.h" /* Datatypes */ +#include "H5private.h" /* Generic Functions */ +#include "H5CXprivate.h" /* API Contexts */ +#include "H5Dprivate.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ +#include "H5Rpkg.h" /* References */ +#include "H5Tpkg.h" /* Datatypes */ /****************/ /* Local Macros */ @@ -3445,26 +3448,33 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T bool write_to_file = false; /* Flag to indicate writing to file */ htri_t parent_is_vlen; /* Flag to indicate parent is vlen datatype */ size_t bg_seq_len = 0; /* The number of elements in the background sequence */ - H5T_t *tsrc_cpy = NULL; /*temporary copy of source base datatype */ - H5T_t *tdst_cpy = NULL; /*temporary copy of destination base datatype */ - hid_t tsrc_id = H5I_INVALID_HID; /*temporary type atom */ - hid_t tdst_id = H5I_INVALID_HID; /*temporary type atom */ - uint8_t *s = NULL; /*source buffer */ - uint8_t *d = NULL; /*destination buffer */ - uint8_t *b = NULL; /*background buffer */ - ssize_t s_stride, d_stride; /*src and dst strides */ - ssize_t b_stride; /*bkg stride */ - size_t safe; /*how many elements are safe to process in each pass */ - size_t src_base_size; /*source base size*/ - size_t dst_base_size; /*destination base size*/ - void *conv_buf = NULL; /*temporary conversion buffer */ - size_t conv_buf_size = 0; /*size of conversion buffer in bytes */ - void *tmp_buf = NULL; /*temporary background buffer */ - size_t tmp_buf_size = 0; /*size of temporary bkg buffer */ - bool nested = false; /*flag of nested VL case */ - bool need_ids = false; /*whether we need IDs for the datatypes */ - size_t elmtno; /*element number counter */ - herr_t ret_value = SUCCEED; /* Return value */ + H5T_t *tsrc_cpy = NULL; /* Temporary copy of source base datatype */ + H5T_t *tdst_cpy = NULL; /* Temporary copy of destination base datatype */ + hid_t tsrc_id = H5I_INVALID_HID; /* Temporary type atom */ + hid_t tdst_id = H5I_INVALID_HID; /* Temporary type atom */ + uint8_t *s = NULL; /* Source buffer */ + uint8_t *d = NULL; /* Destination buffer */ + uint8_t *b = NULL; /* Background buffer */ + ssize_t s_stride = 0; /* Src stride */ + ssize_t d_stride = 0; /* Dst stride */ + ssize_t b_stride; /* Bkg stride */ + size_t safe = 0; /* How many elements are safe to process in each pass */ + size_t src_base_size; /* Source base size*/ + size_t dst_base_size; /* Destination base size*/ + void *conv_buf = NULL; /* Temporary conversion buffer */ + size_t conv_buf_size = 0; /* Size of conversion buffer in bytes */ + void *tmp_buf = NULL; /* Temporary background buffer */ + size_t tmp_buf_size = 0; /* Size of temporary bkg buffer */ + bool nested = false; /* Flag of nested VL case */ + bool need_ids = false; /* Whether we need IDs for the datatypes */ + size_t elmtno = 0; /* Element number counter */ + size_t orig_d_stride = 0; /* Original destination stride (used for error handling) */ + size_t orig_nelmts = nelmts; /* Original # of elements to convert (used for error handling) */ + bool convert_forward = + true; /* Current direction of conversion (forward or backward, used for error handling) */ + bool conversions_made = + false; /* Flag to indicate conversions have been performed, used for error handling */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -3601,6 +3611,10 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T if (write_to_file && parent_is_vlen && bkg != NULL) nested = true; + /* Save info for unraveling on errors */ + orig_d_stride = (size_t)d_stride; + convert_forward = !(d_stride > s_stride); + /* The outer loop of the type conversion macro, controlling which */ /* direction the buffer is walked */ while (nelmts > 0) { @@ -3782,6 +3796,9 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T } /* end if */ } /* end else */ + /* Indicate that elements have been converted, in case of error */ + conversions_made = true; + /* Advance pointers */ s += s_stride; d += d_stride; @@ -3801,6 +3818,49 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T } /* end switch */ done: + /* Release converted elements on error */ + if (ret_value < 0 && conversions_made) { + size_t dest_count; + + /* Set up for first pass to destroy references */ + if (nelmts < orig_nelmts || (convert_forward && elmtno < safe)) { + dest_count = orig_nelmts - nelmts; + + /* Set pointer to correct location, based on direction chosen */ + if (convert_forward) { + d = (uint8_t *)buf; + dest_count += elmtno; /* Include partial iteration in first pass, for forward conversions */ + } + else + d = (uint8_t *)buf + (nelmts * orig_d_stride); + + /* Destroy vlen elements that have already been converted */ + while (dest_count > 0) { + H5T_vlen_reclaim_elmt(d, dst); /* Ignore errors at this point */ + d += orig_d_stride; + dest_count--; + } + } + + /* Do any remaining partial iteration, if converting backwards */ + if (!convert_forward && elmtno < safe) { + dest_count = elmtno; + + /* Set pointer to correct location */ + if (d_stride > 0) + d = (uint8_t *)buf + ((nelmts - safe) * orig_d_stride); + else + d = (uint8_t *)buf + ((nelmts - elmtno) * orig_d_stride); + + /* Destroy references that have already been converted */ + while (dest_count > 0) { + H5T_vlen_reclaim_elmt(d, dst); /* Ignore errors at this point */ + d += orig_d_stride; + dest_count--; + } + } + } + if (tsrc_id >= 0) { if (H5I_dec_ref(tsrc_id) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL, "can't decrement reference on temporary ID"); @@ -4033,16 +4093,23 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T_conv_ctx_t H5_ATTR_UNUSED *conv_ctx, size_t nelmts, size_t buf_stride, size_t bkg_stride, void *buf, void *bkg) { - uint8_t *s = NULL; /* source buffer */ - uint8_t *d = NULL; /* destination buffer */ - uint8_t *b = NULL; /* background buffer */ - ssize_t s_stride, d_stride; /* src and dst strides */ - ssize_t b_stride; /* bkg stride */ - size_t safe; /* how many elements are safe to process in each pass */ - void *conv_buf = NULL; /* temporary conversion buffer */ - size_t conv_buf_size = 0; /* size of conversion buffer in bytes */ - size_t elmtno; /* element number counter */ - herr_t ret_value = SUCCEED; /* return value */ + uint8_t *s = NULL; /* source buffer */ + uint8_t *d = NULL; /* destination buffer */ + uint8_t *b = NULL; /* background buffer */ + ssize_t s_stride = 0; /* src stride */ + ssize_t d_stride = 0; /* dst stride */ + ssize_t b_stride; /* bkg stride */ + size_t safe = 0; /* how many elements are safe to process in each pass */ + void *conv_buf = NULL; /* temporary conversion buffer */ + size_t conv_buf_size = 0; /* size of conversion buffer in bytes */ + size_t elmtno = 0; /* element number counter */ + size_t orig_d_stride = 0; /* Original destination stride (used for error handling) */ + size_t orig_nelmts = nelmts; /* Original # of elements to convert (used for error handling) */ + bool convert_forward = + true; /* Current direction of conversion (forward or backward, used for error handling) */ + bool conversions_made = + false; /* Flag to indicate conversions have been performed, used for error handling */ + herr_t ret_value = SUCCEED; /* return value */ FUNC_ENTER_PACKAGE @@ -4103,6 +4170,10 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, else b_stride = 0; + /* Save info for unraveling on errors */ + orig_d_stride = (size_t)d_stride; + convert_forward = !(d_stride > s_stride); + /* The outer loop of the type conversion macro, controlling which */ /* direction the buffer is walked */ while (nelmts > 0) { @@ -4202,6 +4273,9 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, } /* end else */ } /* end else */ + /* Indicate that elements have been converted, in case of error */ + conversions_made = true; + /* Advance pointers */ s += s_stride; d += d_stride; @@ -4221,6 +4295,52 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, } /* end switch */ done: + /* Release converted elements on error */ + if (ret_value < 0 && conversions_made) { + H5R_ref_priv_t ref_priv; + size_t dest_count; + + /* Set up for first pass to destroy references */ + if (nelmts < orig_nelmts || (convert_forward && elmtno < safe)) { + dest_count = orig_nelmts - nelmts; + + /* Set pointer to correct location, based on direction chosen */ + if (convert_forward) { + d = (uint8_t *)buf; + dest_count += elmtno; /* Include partial iteration in first pass, for forward conversions */ + } + else + d = (uint8_t *)buf + (nelmts * orig_d_stride); + + /* Destroy references that have already been converted */ + while (dest_count > 0) { + memcpy(&ref_priv, d, sizeof(H5R_ref_priv_t)); + H5R__destroy(&ref_priv); /* Ignore errors at this point */ + d += orig_d_stride; + dest_count--; + } + } + + /* Do any remaining partial iteration, if converting backwards */ + if (!convert_forward && elmtno < safe) { + dest_count = elmtno; + + /* Set pointer to correct location */ + if (d_stride > 0) + d = (uint8_t *)buf + ((nelmts - safe) * orig_d_stride); + else + d = (uint8_t *)buf + ((nelmts - elmtno) * orig_d_stride); + + /* Destroy references that have already been converted */ + while (dest_count > 0) { + memcpy(&ref_priv, d, sizeof(H5R_ref_priv_t)); + H5R__destroy(&ref_priv); /* Ignore errors at this point */ + d += orig_d_stride; + dest_count--; + } + } + } + /* Release the conversion buffer (always allocated, except on errors) */ if (conv_buf) conv_buf = H5FL_BLK_FREE(ref_seq, conv_buf); diff --git a/src/H5Tprivate.h b/src/H5Tprivate.h index 7d8f27615a0..46b2c92fa83 100644 --- a/src/H5Tprivate.h +++ b/src/H5Tprivate.h @@ -139,7 +139,7 @@ H5_DLL herr_t H5T_convert(H5T_path_t *tpath, const H5T_t *src_type, const H5T_t size_t buf_stride, size_t bkg_stride, void *buf, void *bkg); H5_DLL herr_t H5T_reclaim(const H5T_t *type, struct H5S_t *space, void *buf); H5_DLL herr_t H5T_reclaim_cb(void *elem, const H5T_t *dt, unsigned ndim, const hsize_t *point, void *op_data); -H5_DLL herr_t H5T_vlen_reclaim_elmt(void *elem, H5T_t *dt); +H5_DLL herr_t H5T_vlen_reclaim_elmt(void *elem, const H5T_t *dt); H5_DLL htri_t H5T_set_loc(H5T_t *dt, H5VL_object_t *file, H5T_loc_t loc); H5_DLL htri_t H5T_is_sensible(const H5T_t *dt); H5_DLL uint32_t H5T_hash(H5F_t *file, const H5T_t *dt); @@ -157,6 +157,7 @@ H5_DLL bool H5T_already_vol_managed(const H5T_t *dt); H5_DLL htri_t H5T_is_vl_storage(const H5T_t *dt); H5_DLL herr_t H5T_invoke_vol_optional(H5T_t *dt, H5VL_optional_args_t *args, hid_t dxpl_id, void **req, H5VL_object_t **vol_obj_ptr); +H5_DLL bool H5T_is_numeric_with_unusual_unused_bits(const H5T_t *dt); /* Reference specific functions */ H5_DLL H5R_type_t H5T_get_ref_type(const H5T_t *dt); diff --git a/src/H5Tvlen.c b/src/H5Tvlen.c index 6cf11c77234..971dc3c5d22 100644 --- a/src/H5Tvlen.c +++ b/src/H5Tvlen.c @@ -1053,7 +1053,7 @@ H5T__vlen_reclaim(void *elem, const H5T_t *dt, H5T_vlen_alloc_info_t *alloc_info *------------------------------------------------------------------------- */ herr_t -H5T_vlen_reclaim_elmt(void *elem, H5T_t *dt) +H5T_vlen_reclaim_elmt(void *elem, const H5T_t *dt) { H5T_vlen_alloc_info_t vl_alloc_info; /* VL allocation info */ herr_t ret_value = SUCCEED; /* return value */ diff --git a/src/H5Zscaleoffset.c b/src/H5Zscaleoffset.c index fba1c7d6c2e..048344b763c 100644 --- a/src/H5Zscaleoffset.c +++ b/src/H5Zscaleoffset.c @@ -1210,6 +1210,8 @@ H5Z__filter_scaleoffset(unsigned flags, size_t cd_nelmts, const unsigned cd_valu minbits_mask <<= i * 8; minbits |= minbits_mask; } + if (minbits >= p.size * 8) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "minimum number of bits exceeds size of type"); /* retrieval of minval takes into consideration situation where sizeof * unsigned long long (datatype of minval) may change from compression diff --git a/src/H5encode.h b/src/H5encode.h index 5be75d57f77..690aee104d7 100644 --- a/src/H5encode.h +++ b/src/H5encode.h @@ -212,7 +212,7 @@ n = 0; \ (p) += 8; \ for (_i = 0; _i < sizeof(int64_t); _i++) \ - n = (n << 8) | *(--p); \ + n = (int64_t)(((uint64_t)n << 8) | *(--p)); \ (p) += 8; \ } while (0) @@ -224,7 +224,7 @@ n = 0; \ (p) += 8; \ for (_i = 0; _i < sizeof(uint64_t); _i++) \ - n = (n << 8) | *(--p); \ + n = (uint64_t)(((uint64_t)n << 8) | *(--p)); \ (p) += 8; \ } while (0) diff --git a/src/H5private.h b/src/H5private.h index a3b37e5d834..a32421bf2ed 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -318,6 +318,15 @@ /* limit the middle value to be within a range (inclusive) */ #define RANGE(LO, X, HI) MAX(LO, MIN(X, HI)) +/* Macro for checking if two ranges overlap one another */ +/* + * Check for the inverse of whether the ranges are disjoint. If they are + * disjoint, then the low bound of one of the ranges must be greater than the + * high bound of the other. + */ +/* (Assumes that low & high bounds are _inclusive_) */ +#define H5_RANGE_OVERLAP(L1, H1, L2, H2) (!((L1) > (H2) || (L2) > (H1))) + /* absolute value */ #ifndef ABS #define ABS(a) (((a) >= 0) ? (a) : -(a)) @@ -336,9 +345,19 @@ #define H5_EXP2(n) (1 << (n)) /* Check if a read of size bytes starting at ptr would overflow past - * the last valid byte, pointed to by buffer_end. + * the last valid byte, pointed to by buffer_end. Note that 'size' + * is expected to be of type size_t. Providing values of other + * datatypes may cause warnings due to the comparison against + * PTRDIFF_MAX and comparison of < 0 after conversion to ptrdiff_t. + * For the time being, these can be suppressed with + * H5_GCC_CLANG_DIAG_OFF("type-limits")/H5_GCC_CLANG_DIAG_ON("type-limits") */ -#define H5_IS_BUFFER_OVERFLOW(ptr, size, buffer_end) (((ptr) + (size)-1) > (buffer_end)) +#define H5_IS_BUFFER_OVERFLOW(ptr, size, buffer_end) \ + (((ptr) > (buffer_end)) || /* Bad precondition */ \ + (((size_t)(size) <= PTRDIFF_MAX) && \ + ((ptrdiff_t)(size) < 0)) || /* Account for (likely unintentional) negative 'size' */ \ + ((size_t)(size) > \ + (size_t)((((const uint8_t *)buffer_end) - ((const uint8_t *)ptr)) + 1))) /* Typical overflow */ /* Variant of H5_IS_BUFFER_OVERFLOW, used with functions such as H5Tdecode() * that don't take a size parameter, where we need to skip the bounds checks. @@ -451,8 +470,7 @@ (X) >= (Y)) #define H5_addr_cmp(X,Y) (H5_addr_eq((X), (Y)) ? 0 : \ (H5_addr_lt((X), (Y)) ? -1 : 1)) -#define H5_addr_overlap(O1,L1,O2,L2) (((O1) < (O2) && ((O1) + (L1)) > (O2)) || \ - ((O1) >= (O2) && (O1) < ((O2) + (L2)))) +#define H5_addr_overlap(O1,L1,O2,L2) H5_RANGE_OVERLAP(O1, ((O1)+(L1)), O2, ((O2)+(L2))) /* clang-format on */ /* diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index a9b935e80a8..687e72830d0 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -163,6 +163,8 @@ set (HDF5_REFERENCE_TEST_FILES test_filters_le.h5 th5s.h5 tlayouto.h5 + tmisc38a.h5 + tmisc38b.h5 tmtimen.h5 tmtimeo.h5 tsizeslheap.h5 diff --git a/test/testfiles/tmisc38a.h5 b/test/testfiles/tmisc38a.h5 new file mode 100644 index 00000000000..cf516f0a2ab Binary files /dev/null and b/test/testfiles/tmisc38a.h5 differ diff --git a/test/testfiles/tmisc38b.h5 b/test/testfiles/tmisc38b.h5 new file mode 100644 index 00000000000..7f58280ac8d Binary files /dev/null and b/test/testfiles/tmisc38b.h5 differ diff --git a/test/tmisc.c b/test/tmisc.c index e08d76cb7ac..edfdcd406c0 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -337,9 +337,19 @@ typedef struct { See https://nvd.nist.gov/vuln/detail/CVE-2020-10812 */ #define CVE_2020_10812_FILENAME "cve_2020_10812.h5" -#define MISC38_FILE "type_conversion_path_table_issue.h5" -#define MISC39_FILE "set_est_link_info.h5" -#define MISC40_FILE "obj_props_intermediate.h5" +/* Definitions for misc. test #38 */ +#define MISC38A_FILE "tmisc38a.h5" +#define MISC38A_DSETNAME "Fletcher_float_data_be" +#define MISC38B_FILE "tmisc38b.h5" +#define MISC38B_DSETNAME "unusual_datatype" +#define MISC38C_FILE "tmisc38c.h5" +#define MISC38C_DSETNAME "dset_unusual_datatype" +#define MISC38C_TYPENAME "type_unusual_datatype" +#define MISC38C_ATTRNAME "attr_unusual_datatype" + +#define MISC39_FILE "type_conversion_path_table_issue.h5" +#define MISC40_FILE "set_est_link_info.h5" +#define MISC41_FILE "obj_props_intermediate.h5" /**************************************************************** ** @@ -4025,6 +4035,7 @@ test_misc21(void) static void test_misc22(void) { + hid_t fapl; /* File access property list */ hid_t fid, sid, dcpl, dsid, dcpl2; char *buf; hsize_t dims[2] = {MISC22_SPACE_DIM0, MISC22_SPACE_DIM1}, @@ -4059,10 +4070,22 @@ test_misc22(void) buf = (char *)calloc(MISC22_SPACE_DIM0 * MISC22_SPACE_DIM1, 8); CHECK(buf, NULL, "calloc"); + /* Create a file access property list */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + + /* Set property to allow unusual datatypes to be created */ + ret = H5Pset_relax_file_integrity_checks(fapl, H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS); + CHECK(ret, FAIL, "H5Pset_relax_file_integrity_checks"); + /* Create the file */ - fid = H5Fcreate(MISC22_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(MISC22_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); CHECK(fid, FAIL, "H5Fcreate"); + /* Close file access property list */ + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + /* Create the dataspace for the dataset */ sid = H5Screate_simple(MISC22_SPACE_RANK, dims, NULL); CHECK(sid, FAIL, "H5Screate_simple"); @@ -6264,6 +6287,223 @@ test_misc37(void) /**************************************************************** ** ** test_misc38(): +** Test for seg fault issue when opening dataset with corrupted +** object header. +** +****************************************************************/ +static void +test_misc38(void) +{ + const char *testfile = H5_get_srcdir_filename(MISC38A_FILE); /* Corrected test file name */ + const char *testfile2 = H5_get_srcdir_filename(MISC38B_FILE); /* Corrected test file name */ + bool driver_is_default_compatible; + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t tid = H5I_INVALID_HID; /* Datatype ID */ + hid_t gid = H5I_INVALID_HID; /* Group ID */ + hid_t aid = H5I_INVALID_HID; /* Attribute ID */ + size_t type_size; /* Size of dataset's datatype */ + uint64_t rfic_flags; /* Value of RFIC flags property for FAPL & file */ + herr_t ret; + + /* Output message about test being performed */ + MESSAGE(5, ("Fix for detecting numeric datatypes with unusually large numbers of unused bits")); + + ret = h5_driver_is_default_vfd_compatible(H5P_DEFAULT, &driver_is_default_compatible); + CHECK(ret, FAIL, "h5_driver_is_default_vfd_compatible"); + + if (!driver_is_default_compatible) { + printf("-- SKIPPED --\n"); + return; + } + + fid = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fopen"); + + /* This should fail due to the illegal datatype encoding in the corrupted + * object header. + * It should fail gracefully and not seg fault + */ + H5E_BEGIN_TRY + { + did = H5Dopen2(fid, MISC38A_DSETNAME, H5P_DEFAULT); + } + H5E_END_TRY + VERIFY(did, H5I_INVALID_HID, "H5Dopen2"); + + /* Close file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + /* Create a file access property list */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, H5I_INVALID_HID, "H5Pcreate"); + + /* Get property to allow unusual datatypes to be opened */ + rfic_flags = H5F_RFIC_ALL; + ret = H5Pget_relax_file_integrity_checks(fapl, &rfic_flags); + CHECK(ret, FAIL, "H5Pget_relax_file_integrity_checks"); + VERIFY(rfic_flags, 0, "H5Pget_relax_file_integrity_checks"); + + /* Set property to allow unusual datatypes to be opened */ + ret = H5Pset_relax_file_integrity_checks(fapl, H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS); + CHECK(ret, FAIL, "H5Pset_relax_file_integrity_checks"); + + /* Get property to allow unusual datatypes to be opened */ + rfic_flags = 0; + ret = H5Pget_relax_file_integrity_checks(fapl, &rfic_flags); + CHECK(ret, FAIL, "H5Pget_relax_file_integrity_checks"); + VERIFY(rfic_flags, H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS, "H5Pget_relax_file_integrity_checks"); + + /* Open valid file */ + fid = H5Fopen(testfile2, H5F_ACC_RDONLY, fapl); + CHECK(fid, H5I_INVALID_HID, "H5Fopen"); + + /* Close file access property list */ + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* Open dataset w/unusual datatype + * It should succeed and not return an error or seg fault + */ + did = H5Dopen2(fid, MISC38B_DSETNAME, H5P_DEFAULT); + CHECK(did, H5I_INVALID_HID, "H5Dopen2"); + + /* Get the dataset's datatype */ + tid = H5Dget_type(did); + CHECK(tid, H5I_INVALID_HID, "H5Dget_type"); + + type_size = H5Tget_size(tid); + CHECK(type_size, 0, "H5Tget_size"); + VERIFY(type_size, 1000, "H5Tget_size"); + + ret = H5Tclose(tid); + CHECK(ret, FAIL, "H5Tclose"); + + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + + /* Check that property is handled correctly */ + fapl = H5Fget_access_plist(fid); + CHECK(fapl, H5I_INVALID_HID, "H5Fget_access_plist"); + + /* Get property to allow unusual datatypes to be opened */ + rfic_flags = 0; + ret = H5Pget_relax_file_integrity_checks(fapl, &rfic_flags); + CHECK(ret, FAIL, "H5Pget_relax_file_integrity_checks"); + VERIFY(rfic_flags, H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS, "H5Pget_relax_file_integrity_checks"); + + /* Close file access property list */ + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* Close file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + /* Create objects with unusual datatypes and verify correct behavior */ + for (unsigned u = 0; u < 3; u++) { + /* Create a file access property list */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, H5I_INVALID_HID, "H5Pcreate"); + + if (1 == u) { + /* Set property to allow unusual datatypes to be opened */ + ret = H5Pset_relax_file_integrity_checks(fapl, H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS); + CHECK(ret, FAIL, "H5Pset_relax_file_integrity_checks"); + } + else if (2 == u) { + /* Use a later version of the file format, with checksummed object headers */ + ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + CHECK(ret, FAIL, "H5Pset_libver_bounds"); + } + + fid = H5Fcreate(MISC38C_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + CHECK(fid, H5I_INVALID_HID, "H5Fcreate"); + + /* Close file access property list */ + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + sid = H5Screate(H5S_SCALAR); + CHECK(sid, H5I_INVALID_HID, "H5Screate"); + + tid = H5Tcopy(H5T_NATIVE_INT); + CHECK(tid, H5I_INVALID_HID, "H5Tcopy"); + + /* Set type to have unusual size, for precision */ + ret = H5Tset_size(tid, 1000); + CHECK(ret, FAIL, "H5Tset_size"); + + /* Create a dataset with the unusual datatype */ + H5E_BEGIN_TRY + { + did = H5Dcreate2(fid, MISC38C_DSETNAME, tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + } + H5E_END_TRY + if (u > 0) { + CHECK(did, H5I_INVALID_HID, "H5Dcreate2"); + + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + } + else { + VERIFY(did, H5I_INVALID_HID, "H5Dcreate2"); + } + + gid = H5Gopen2(fid, "/", H5P_DEFAULT); + CHECK(gid, H5I_INVALID_HID, "H5Gopen2"); + + /* Create an attribute with the unusual datatype */ + H5E_BEGIN_TRY + { + aid = H5Acreate2(gid, MISC38C_ATTRNAME, tid, sid, H5P_DEFAULT, H5P_DEFAULT); + } + H5E_END_TRY + if (u > 0) { + CHECK(aid, H5I_INVALID_HID, "H5Acreate2"); + + ret = H5Aclose(aid); + CHECK(ret, FAIL, "H5Aclose"); + } + else { + VERIFY(aid, H5I_INVALID_HID, "H5Acreate2"); + } + + ret = H5Gclose(gid); + CHECK(ret, FAIL, "H5Gclose"); + + ret = H5Sclose(sid); + CHECK(ret, FAIL, "H5Sclose"); + + /* Create a committed datatype with the unusual datatype */ + H5E_BEGIN_TRY + { + ret = H5Tcommit2(fid, MISC38C_TYPENAME, tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + } + H5E_END_TRY + if (u > 0) { + CHECK(ret, FAIL, "H5Tcommit2"); + } + else { + VERIFY(ret, FAIL, "H5Tcommit2"); + } + + if (tid != H5I_INVALID_HID) { + ret = H5Tclose(tid); + CHECK(ret, FAIL, "H5Tclose"); + } + + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + } +} /* end test_misc38() */ + +/**************************************************************** +** +** test_misc39(): ** Test for issue where the type conversion path table cache ** would grow continuously when variable-length datatypes ** are involved due to file VOL object comparisons causing @@ -6271,7 +6511,7 @@ test_misc37(void) ** ****************************************************************/ static void -test_misc38(void) +test_misc39(void) { H5VL_object_t *file_vol_obj = NULL; const char *buf[] = {"attr_value"}; @@ -6309,7 +6549,7 @@ test_misc38(void) */ init_npaths = H5T__get_path_table_npaths(); - file_id = H5Fcreate(MISC38_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fcreate(MISC39_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(file_id, H5I_INVALID_HID, "H5Fcreate"); /* Check if native VOL is being used */ @@ -6444,7 +6684,7 @@ test_misc38(void) CHECK_PTR(vlen_rbuf, "vlen varstr read buf allocation"); for (size_t i = 0; i < 10; i++) { - file_id = H5Fopen(MISC38_FILE, H5F_ACC_RDONLY, H5P_DEFAULT); + file_id = H5Fopen(MISC39_FILE, H5F_ACC_RDONLY, H5P_DEFAULT); CHECK(file_id, H5I_INVALID_HID, "H5Fopen"); /* Retrieve file's VOL object field for further use */ @@ -6546,7 +6786,7 @@ test_misc38(void) /**************************************************************** ** -** test_misc39(): Ensure H5Pset_est_link_info() handles large +** test_misc40(): Ensure H5Pset_est_link_info() handles large ** values ** ** H5Pset_est_link_info() values can be set to large values, @@ -6560,7 +6800,7 @@ test_misc38(void) ** ****************************************************************/ static void -test_misc39(void) +test_misc40(void) { hid_t fid = H5I_INVALID_HID; /* File ID */ hid_t gid = H5I_INVALID_HID; /* Group ID */ @@ -6581,7 +6821,7 @@ test_misc39(void) CHECK(ret, FAIL, "H5Pset_libver_bounds"); /* Create the file */ - fid = H5Fcreate(MISC39_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + fid = H5Fcreate(MISC40_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); CHECK(fid, H5I_INVALID_HID, "H5Fcreate"); /* Compose group creation property list */ @@ -6639,16 +6879,16 @@ test_misc39(void) ret = H5Pclose(gcpl); CHECK(ret, FAIL, "H5Pclose"); -} /* end test_misc39() */ +} /* end test_misc40() */ /**************************************************************** ** -** test_misc40(): Test that object creation properties are propagated +** test_misc41(): Test that object creation properties are propagated ** to intermediate groups. ** ****************************************************************/ static void -test_misc40(void) +test_misc41(void) { hid_t lcpl = H5I_INVALID_HID; hid_t gcpl = H5I_INVALID_HID; @@ -6675,7 +6915,7 @@ test_misc40(void) status = H5Pset_create_intermediate_group(lcpl, 1); CHECK(status, FAIL, "H5Pset_create_intermediate_group"); - fid = H5Fcreate(MISC40_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(MISC41_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* @@ -6856,7 +7096,7 @@ test_misc40(void) status = H5Pclose(lcpl); CHECK(status, FAIL, "H5Pclose"); -} /* end test_misc40() */ +} /* end test_misc41() */ /**************************************************************** ** @@ -6930,9 +7170,10 @@ test_misc(void) test_misc35(); /* Test behavior of free-list & allocation statistics API calls */ test_misc36(); /* Exercise H5atclose and H5is_library_terminating */ test_misc37(); /* Test for seg fault failure at file close */ - test_misc38(); /* Test for type conversion path table issue */ - test_misc39(); /* Ensure H5Pset_est_link_info() handles large values */ - test_misc40(); /* Test object properties propagated to intermediate groups */ + test_misc38(); /* Test for seg fault when opening corrupted object header */ + test_misc39(); /* Test for type conversion path table issue */ + test_misc40(); /* Ensure H5Pset_est_link_info() handles large values */ + test_misc41(); /* Test object properties propagated to intermediate groups */ } /* test_misc() */ @@ -6988,9 +7229,10 @@ cleanup_misc(void) #ifndef H5_NO_DEPRECATED_SYMBOLS H5Fdelete(MISC31_FILE, H5P_DEFAULT); #endif /* H5_NO_DEPRECATED_SYMBOLS */ - H5Fdelete(MISC38_FILE, H5P_DEFAULT); + H5Fdelete(MISC38C_FILE, H5P_DEFAULT); H5Fdelete(MISC39_FILE, H5P_DEFAULT); H5Fdelete(MISC40_FILE, H5P_DEFAULT); + H5Fdelete(MISC41_FILE, H5P_DEFAULT); } H5E_END_TRY } /* end cleanup_misc() */ diff --git a/tools/lib/h5tools_str.c b/tools/lib/h5tools_str.c index 47893ec9390..44e9e681371 100644 --- a/tools/lib/h5tools_str.c +++ b/tools/lib/h5tools_str.c @@ -1207,54 +1207,58 @@ h5tools_str_sprint(h5tools_str_t *str, const h5tool_format_t *info, hid_t contai /* * Object references -- show the type and OID of the referenced object. */ - H5O_info2_t oi; - char *obj_tok_str = NULL; - H5TOOLS_DEBUG("H5T_REFERENCE:H5T_STD_REF_OBJ"); obj = H5Rdereference2(container, H5P_DEFAULT, H5R_OBJECT, vp); - H5Oget_info3(obj, &oi, H5O_INFO_BASIC); + if (obj >= 0) { + H5O_info2_t oi; + char *obj_tok_str = NULL; + + H5Oget_info3(obj, &oi, H5O_INFO_BASIC); + + /* Print object type and close object */ + switch (oi.type) { + case H5O_TYPE_GROUP: + h5tools_str_append(str, H5_TOOLS_GROUP); + break; + + case H5O_TYPE_DATASET: + h5tools_str_append(str, H5_TOOLS_DATASET); + break; + + case H5O_TYPE_NAMED_DATATYPE: + h5tools_str_append(str, H5_TOOLS_DATATYPE); + break; + + case H5O_TYPE_MAP: + h5tools_str_append(str, H5_TOOLS_MAP); + break; + + case H5O_TYPE_UNKNOWN: + case H5O_TYPE_NTYPES: + default: + h5tools_str_append(str, "%u-", (unsigned)oi.type); + break; + } - /* Print object type and close object */ - switch (oi.type) { - case H5O_TYPE_GROUP: - h5tools_str_append(str, H5_TOOLS_GROUP); - break; + /* Print OID */ + H5Otoken_to_str(obj, &oi.token, &obj_tok_str); - case H5O_TYPE_DATASET: - h5tools_str_append(str, H5_TOOLS_DATASET); - break; + H5Oclose(obj); - case H5O_TYPE_NAMED_DATATYPE: - h5tools_str_append(str, H5_TOOLS_DATATYPE); - break; + if (info->obj_hidefileno) + h5tools_str_append(str, info->obj_format, obj_tok_str); + else + h5tools_str_append(str, info->obj_format, oi.fileno, obj_tok_str); - case H5O_TYPE_MAP: - h5tools_str_append(str, H5_TOOLS_MAP); - break; + if (obj_tok_str) { + H5free_memory(obj_tok_str); + obj_tok_str = NULL; + } - case H5O_TYPE_UNKNOWN: - case H5O_TYPE_NTYPES: - default: - h5tools_str_append(str, "%u-", (unsigned)oi.type); - break; + h5tools_str_sprint_old_reference(str, container, H5R_OBJECT, vp); } - - /* Print OID */ - H5Otoken_to_str(obj, &oi.token, &obj_tok_str); - - H5Oclose(obj); - - if (info->obj_hidefileno) - h5tools_str_append(str, info->obj_format, obj_tok_str); else - h5tools_str_append(str, info->obj_format, oi.fileno, obj_tok_str); - - if (obj_tok_str) { - H5free_memory(obj_tok_str); - obj_tok_str = NULL; - } - - h5tools_str_sprint_old_reference(str, container, H5R_OBJECT, vp); + h5tools_str_append(str, ""); } /* end else if (H5Tequal(type, H5T_STD_REF_OBJ)) */ } break; diff --git a/tools/test/h5dump/CMakeTests.cmake b/tools/test/h5dump/CMakeTests.cmake index 3a863bdee72..2369f63c746 100644 --- a/tools/test/h5dump/CMakeTests.cmake +++ b/tools/test/h5dump/CMakeTests.cmake @@ -1285,7 +1285,7 @@ ADD_H5_COMP_TEST (tfletcher32 0 0 --enable-error-stack -H -p -d fletcher32 tfilters.h5) # nbit - ADD_H5_COMP_TEST (tnbit 0 10 --enable-error-stack -H -p -d nbit tfilters.h5) + ADD_H5_COMP_TEST (tnbit 0 1 --enable-error-stack -H -p -d nbit tfilters.h5) # scaleoffset ADD_H5_COMP_TEST (tscaleoffset 0 4 --enable-error-stack -H -p -d scaleoffset tfilters.h5) diff --git a/tools/test/h5dump/expected/tnbit.ddl b/tools/test/h5dump/expected/tnbit.ddl index 35c111fb5b3..3801c1bf655 100644 --- a/tools/test/h5dump/expected/tnbit.ddl +++ b/tools/test/h5dump/expected/tnbit.ddl @@ -1,10 +1,10 @@ HDF5 "tfilters.h5" { DATASET "nbit" { - DATATYPE 32-bit little-endian integer 3-bit precision + DATATYPE 32-bit little-endian integer 17-bit precision DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) } STORAGE_LAYOUT { CHUNKED ( 10, 5 ) - SIZE XXXX (10.XXX:1 COMPRESSION) + SIZE XXXX (1.XXX:1 COMPRESSION) } FILTERS { COMPRESSION NBIT diff --git a/tools/test/h5dump/expected/treadintfilter.ddl b/tools/test/h5dump/expected/treadintfilter.ddl index fbad3f67369..7a670a2a25d 100644 --- a/tools/test/h5dump/expected/treadintfilter.ddl +++ b/tools/test/h5dump/expected/treadintfilter.ddl @@ -78,29 +78,29 @@ DATASET "fletcher32" { } } DATASET "nbit" { - DATATYPE 32-bit little-endian integer 3-bit precision + DATATYPE 32-bit little-endian integer 17-bit precision DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) } DATA { - (0,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1, - (1,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3, - (2,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3, - (3,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1, - (4,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1, - (5,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3, - (6,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3, - (7,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1, - (8,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1, - (9,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3, - (10,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3, - (11,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1, - (12,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1, - (13,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3, - (14,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3, - (15,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1, - (16,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1, - (17,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3, - (18,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3, - (19,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1 + (0,0): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + (1,0): 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + (2,0): 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + (3,0): 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + (4,0): 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + (5,0): 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + (6,0): 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + (7,0): 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + (8,0): 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + (9,0): 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + (10,0): 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + (11,0): 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + (12,0): 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + (13,0): 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + (14,0): 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + (15,0): 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + (16,0): 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, + (17,0): 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + (18,0): 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, + (19,0): 190, 191, 192, 193, 194, 195, 196, 197, 198, 199 } } DATASET "scaleoffset" { diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index bf0bfd4a251..c7d5fe66c28 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -5615,7 +5615,7 @@ gent_filters(void) assert(ret >= 0); tid = H5Tcopy(H5T_NATIVE_INT); - H5Tset_precision(tid, H5Tget_size(tid) - 1); + H5Tset_precision(tid, (H5Tget_size(tid) * 4) + 1); ret = make_dset(fid, "nbit", sid, tid, dcpl, buf1); assert(ret >= 0); diff --git a/tools/test/h5dump/testfiles/tfilters.h5 b/tools/test/h5dump/testfiles/tfilters.h5 index 7c33e55aae1..23d68a9b72b 100644 Binary files a/tools/test/h5dump/testfiles/tfilters.h5 and b/tools/test/h5dump/testfiles/tfilters.h5 differ diff --git a/tools/test/h5dump/testh5dump.sh.in b/tools/test/h5dump/testh5dump.sh.in index daba93b3509..0964d7dda6d 100644 --- a/tools/test/h5dump/testh5dump.sh.in +++ b/tools/test/h5dump/testh5dump.sh.in @@ -1430,7 +1430,7 @@ TOOLTEST tshuffle.ddl --enable-error-stack -H -p -d shuffle tfilters.h5 # fletcher32 TOOLTESTC 0 tfletcher32.ddl --enable-error-stack -H -p -d fletcher32 tfilters.h5 # nbit -TOOLTESTC 10 tnbit.ddl --enable-error-stack -H -p -d nbit tfilters.h5 +TOOLTESTC 1 tnbit.ddl --enable-error-stack -H -p -d nbit tfilters.h5 # scaleoffset TOOLTESTC 4 tscaleoffset.ddl --enable-error-stack -H -p -d scaleoffset tfilters.h5 # all diff --git a/tools/test/h5stat/expected/h5stat_filters-F.ddl b/tools/test/h5stat/expected/h5stat_filters-F.ddl index d44445bb2e3..065d0c97477 100644 --- a/tools/test/h5stat/expected/h5stat_filters-F.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-F.ddl @@ -4,12 +4,12 @@ File space information for file metadata (in bytes): Superblock extension: 0 User block: 0 Object headers: (total/unused) - Groups: 48/8 - Datasets(exclude compact data): 4136/1344 + Groups: 40/0 + Datasets(exclude compact data): 4128/1088 Datatypes: 80/0 Groups: B-tree/List: 1200 - Heap: 288 + Heap: 384 Attributes: B-tree/List: 0 Heap: 0 diff --git a/tools/test/h5stat/expected/h5stat_filters-UD.ddl b/tools/test/h5stat/expected/h5stat_filters-UD.ddl index 4efafd13c40..9f6335aaed6 100644 --- a/tools/test/h5stat/expected/h5stat_filters-UD.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-UD.ddl @@ -1,5 +1,5 @@ Filename: h5stat_filters.h5 File space information for datasets' metadata (in bytes): - Object headers (total/unused): 4136/1344 + Object headers (total/unused): 4128/1088 Index for Chunked datasets: 31392 Heap: 72 diff --git a/tools/test/h5stat/expected/h5stat_filters-d.ddl b/tools/test/h5stat/expected/h5stat_filters-d.ddl index 6e6dd6140dd..eee7e1845d4 100644 --- a/tools/test/h5stat/expected/h5stat_filters-d.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-d.ddl @@ -12,7 +12,7 @@ Dataset dimension information: # of datasets with dimension size 100 - 999: 1 Total # of datasets: 1 Dataset storage information: - Total raw data size: 8659 + Total raw data size: 9046 Total external raw data size: 400 Dataset layout information: Dataset layout counts[COMPACT]: 1 diff --git a/tools/test/h5stat/expected/h5stat_filters-dT.ddl b/tools/test/h5stat/expected/h5stat_filters-dT.ddl index b14ca9f9745..e513b3a5f28 100644 --- a/tools/test/h5stat/expected/h5stat_filters-dT.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-dT.ddl @@ -12,7 +12,7 @@ Dataset dimension information: # of datasets with dimension size 100 - 999: 1 Total # of datasets: 1 Dataset storage information: - Total raw data size: 8659 + Total raw data size: 9046 Total external raw data size: 400 Dataset layout information: Dataset layout counts[COMPACT]: 1 diff --git a/tools/test/h5stat/expected/h5stat_filters.ddl b/tools/test/h5stat/expected/h5stat_filters.ddl index 9f9e146f08f..7383f0b31ff 100644 --- a/tools/test/h5stat/expected/h5stat_filters.ddl +++ b/tools/test/h5stat/expected/h5stat_filters.ddl @@ -12,12 +12,12 @@ File space information for file metadata (in bytes): Superblock extension: 0 User block: 0 Object headers: (total/unused) - Groups: 48/8 - Datasets(exclude compact data): 4136/1344 + Groups: 40/0 + Datasets(exclude compact data): 4128/1088 Datatypes: 80/0 Groups: B-tree/List: 1200 - Heap: 288 + Heap: 384 Attributes: B-tree/List: 0 Heap: 0 @@ -50,7 +50,7 @@ Dataset dimension information: # of datasets with dimension size 100 - 999: 1 Total # of datasets: 1 Dataset storage information: - Total raw data size: 8659 + Total raw data size: 9046 Total external raw data size: 400 Dataset layout information: Dataset layout counts[COMPACT]: 1 @@ -91,9 +91,9 @@ Free-space section bins: File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR File space page size: 4096 bytes Summary of file space information: - File metadata: 37312 bytes - Raw data: 8659 bytes + File metadata: 37392 bytes + Raw data: 9046 bytes Amount/Percent of tracked free space: 0 bytes/0.0% - Unaccounted space: 301 bytes -Total space: 46272 bytes + Unaccounted space: 258 bytes +Total space: 46696 bytes External raw data: 400 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_filters.h5 b/tools/test/h5stat/testfiles/h5stat_filters.h5 index 5b5f4bb7a68..23d68a9b72b 100644 Binary files a/tools/test/h5stat/testfiles/h5stat_filters.h5 and b/tools/test/h5stat/testfiles/h5stat_filters.h5 differ