mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-04-24 17:51:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

File format security issues ()

Browse Source
This commit is contained in:
Dana Robinson 2024-03-27 13:12:32 -07:00 committed by GitHub
parent 74498cb211
commit 136739b37a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
55 changed files with 1234 additions and 295 deletions

26
src/H5Adense.c

@ -1104,12 +1104,12 @@ H5A__dense_iterate(H5F_t *f, hid_t loc_id, const H5O_ainfo_t *ainfo, H5_index_t
H5_iter_order_t order, hsize_t skip, hsize_t *last_attr, const H5A_attr_iter_op_t *attr_op,
void *op_data)
{
H5HF_t *fheap = NULL; /* Fractal heap handle */
H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */
H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */
H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */
haddr_t bt2_addr; /* Address of v2 B-tree to use for lookup */
herr_t ret_value = FAIL; /* Return value */
H5HF_t *fheap = NULL; /* Fractal heap handle */
H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */
H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */
H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */
haddr_t bt2_addr; /* Address of v2 B-tree to use for lookup */
herr_t ret_value = FAIL; /* Return value */
FUNC_ENTER_PACKAGE
@ -1499,12 +1499,12 @@ herr_t
H5A__dense_remove_by_idx(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type, H5_iter_order_t order,
hsize_t n)
{
H5HF_t *fheap = NULL; /* Fractal heap handle */
H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */
H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */
H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */
haddr_t bt2_addr; /* Address of v2 B-tree to use for operation */
herr_t ret_value = SUCCEED; /* Return value */
H5HF_t *fheap = NULL; /* Fractal heap handle */
H5HF_t *shared_fheap = NULL; /* Fractal heap handle for shared header messages */
H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */
H5B2_t *bt2 = NULL; /* v2 B-tree handle for index */
haddr_t bt2_addr; /* Address of v2 B-tree to use for operation */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
@ -1586,7 +1586,7 @@ H5A__dense_remove_by_idx(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type
HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "error building table of attributes");
/* Check for skipping too many attributes */
if (n >= atable.nattrs)
if (n >= atable.num_attrs)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index specified");
/* Delete appropriate attribute from dense storage */

109
src/H5Aint.c

@ -51,16 +51,9 @@
typedef struct {
H5F_t *f; /* Pointer to file that fractal heap is in */
H5A_attr_table_t *atable; /* Pointer to attribute table to build */
size_t curr_attr; /* Current attribute to operate on */
bool bogus_crt_idx; /* Whether bogus creation index values need to be set */
} H5A_compact_bt_ud_t;
/* Data exchange structure to use when building table of dense attributes for an object */
typedef struct {
H5A_attr_table_t *atable; /* Pointer to attribute table to build */
size_t curr_attr; /* Current attribute to operate on */
} H5A_dense_bt_ud_t;
/* Data exchange structure to use when copying an attribute from _SRC to _DST */
typedef struct {
const H5O_ainfo_t *ainfo; /* dense information */
@ -1455,30 +1448,31 @@ H5A__compact_build_table_cb(H5O_t H5_ATTR_UNUSED *oh, H5O_mesg_t *mesg /*in,out*
assert(mesg);
/* Re-allocate the table if necessary */
if (udata->curr_attr == udata->atable->nattrs) {
if (udata->atable->num_attrs == udata->atable->max_attrs) {
H5A_t **new_table; /* New table for attributes */
size_t new_table_size; /* Number of attributes in new table */
/* Allocate larger table */
new_table_size = MAX(1, 2 * udata->atable->nattrs);
new_table_size = MAX(1, 2 * udata->atable->max_attrs);
if (NULL == (new_table = (H5A_t **)H5FL_SEQ_REALLOC(H5A_t_ptr, udata->atable->attrs, new_table_size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5_ITER_ERROR, "unable to extend attribute table");
/* Update table information in user data */
udata->atable->attrs = new_table;
udata->atable->nattrs = new_table_size;
udata->atable->attrs = new_table;
udata->atable->max_attrs = new_table_size;
} /* end if */
/* Copy attribute into table */
if (NULL == (udata->atable->attrs[udata->curr_attr] = H5A__copy(NULL, (const H5A_t *)mesg->native)))
if (NULL ==
(udata->atable->attrs[udata->atable->num_attrs] = H5A__copy(NULL, (const H5A_t *)mesg->native)))
HGOTO_ERROR(H5E_ATTR, H5E_CANTCOPY, H5_ITER_ERROR, "can't copy attribute");
/* Assign [somewhat arbitrary] creation order value, if requested */
if (udata->bogus_crt_idx)
((udata->atable->attrs[udata->curr_attr])->shared)->crt_idx = sequence;
((udata->atable->attrs[udata->atable->num_attrs])->shared)->crt_idx = sequence;
/* Increment current attribute */
udata->curr_attr++;
/* Increment attribute count */
udata->atable->num_attrs++;
done:
FUNC_LEAVE_NOAPI(ret_value)
@ -1501,9 +1495,10 @@ herr_t
H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order_t order,
H5A_attr_table_t *atable)
{
H5A_compact_bt_ud_t udata; /* User data for iteration callback */
H5O_mesg_operator_t op; /* Wrapper for operator */
herr_t ret_value = SUCCEED; /* Return value */
H5A_compact_bt_ud_t udata; /* User data for iteration callback */
H5O_mesg_operator_t op; /* Wrapper for operator */
bool iter_set_up = false; /* Is everything set up for iteration */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
@ -1513,13 +1508,13 @@ H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order
assert(atable);
/* Initialize table */
atable->attrs = NULL;
atable->nattrs = 0;
atable->attrs = NULL;
atable->num_attrs = 0;
atable->max_attrs = 0;
/* Set up user data for iteration */
udata.f = f;
udata.atable = atable;
udata.curr_attr = 0;
udata.f = f;
udata.atable = atable;
udata.bogus_crt_idx =
(bool)((oh->version == H5O_VERSION_1 || !(oh->flags & H5O_HDR_ATTR_CRT_ORDER_TRACKED)) ? true
: false);
@ -1527,20 +1522,23 @@ H5A__compact_build_table(H5F_t *f, H5O_t *oh, H5_index_t idx_type, H5_iter_order
/* Iterate over existing attributes, checking for attribute with same name */
op.op_type = H5O_MESG_OP_LIB;
op.u.lib_op = H5A__compact_build_table_cb;
iter_set_up = true;
if (H5O__msg_iterate_real(f, oh, H5O_MSG_ATTR, &op, &udata) < 0)
HGOTO_ERROR(H5E_ATTR, H5E_BADITER, FAIL, "error building attribute table");
/* Correct # of attributes in table */
atable->nattrs = udata.curr_attr;
/* Don't sort an empty table. */
if (atable->nattrs > 0) {
if (atable->num_attrs > 0)
/* Sort attribute table in correct iteration order */
if (H5A__attr_sort_table(atable, idx_type, order) < 0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTSORT, FAIL, "error sorting attribute table");
} /* end if */
done:
if (ret_value < 0)
/* Clean up partially built table on error */
if (iter_set_up)
if (atable->attrs && H5A__attr_release_table(atable) < 0)
HDONE_ERROR(H5E_ATTR, H5E_CANTFREE, FAIL, "unable to release attribute table");
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5A__compact_build_table() */
@ -1557,26 +1555,26 @@ done:
static herr_t
H5A__dense_build_table_cb(const H5A_t *attr, void *_udata)
{
H5A_dense_bt_ud_t *udata = (H5A_dense_bt_ud_t *)_udata; /* 'User data' passed in */
herr_t ret_value = H5_ITER_CONT; /* Return value */
H5A_attr_table_t *atable = (H5A_attr_table_t *)_udata; /* 'User data' passed in */
herr_t ret_value = H5_ITER_CONT; /* Return value */
FUNC_ENTER_PACKAGE
/* check arguments */
assert(attr);
assert(udata);
assert(udata->curr_attr < udata->atable->nattrs);
assert(atable);
assert(atable->num_attrs < atable->max_attrs);
/* Allocate attribute for entry in the table */
if (NULL == (udata->atable->attrs[udata->curr_attr] = H5FL_CALLOC(H5A_t)))
if (NULL == (atable->attrs[atable->num_attrs] = H5FL_CALLOC(H5A_t)))
HGOTO_ERROR(H5E_ATTR, H5E_CANTALLOC, H5_ITER_ERROR, "can't allocate attribute");
/* Copy attribute information. Share the attribute object in copying. */
if (NULL == H5A__copy(udata->atable->attrs[udata->curr_attr], attr))
if (NULL == H5A__copy(atable->attrs[atable->num_attrs], attr))
HGOTO_ERROR(H5E_ATTR, H5E_CANTCOPY, H5_ITER_ERROR, "can't copy attribute");
/* Increment number of attributes stored */
udata->curr_attr++;
atable->num_attrs++;
done:
FUNC_LEAVE_NOAPI(ret_value)
@ -1622,22 +1620,18 @@ H5A__dense_build_table(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type,
if (H5B2_get_nrec(bt2_name, &nrec) < 0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't retrieve # of records in index");
/* Set size of table */
H5_CHECK_OVERFLOW(nrec, /* From: */ hsize_t, /* To: */ size_t);
atable->nattrs = (size_t)nrec;
/* Allocate space for the table entries */
if (atable->nattrs > 0) {
H5A_dense_bt_ud_t udata; /* User data for iteration callback */
if (nrec > 0) {
H5A_attr_iter_op_t attr_op; /* Attribute operator */
/* Allocate the table to store the attributes */
if ((atable->attrs = (H5A_t **)H5FL_SEQ_CALLOC(H5A_t_ptr, atable->nattrs)) == NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
/* Check for overflow on the downcast */
H5_CHECK_OVERFLOW(nrec, /* From: */ hsize_t, /* To: */ size_t);
/* Set up user data for iteration */
udata.atable = atable;
udata.curr_attr = 0;
/* Allocate the table to store the attributes */
if (NULL == (atable->attrs = (H5A_t **)H5FL_SEQ_CALLOC(H5A_t_ptr, (size_t)nrec)))
HGOTO_ERROR(H5E_ATTR, H5E_CANTALLOC, FAIL, "memory allocation failed");
atable->num_attrs = 0;
atable->max_attrs = (size_t)nrec;
/* Build iterator operator */
attr_op.op_type = H5A_ATTR_OP_LIB;
@ -1645,7 +1639,7 @@ H5A__dense_build_table(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type,
/* Iterate over the links in the group, building a table of the link messages */
if (H5A__dense_iterate(f, (hid_t)0, ainfo, H5_INDEX_NAME, H5_ITER_NATIVE, (hsize_t)0, NULL, &attr_op,
&udata) < 0)
atable) < 0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, "error building attribute table");
/* Sort attribute table in correct iteration order */
@ -1791,18 +1785,18 @@ H5A__attr_sort_table(H5A_attr_table_t *atable, H5_index_t idx_type, H5_iter_orde
/* Pick appropriate comparison routine */
if (idx_type == H5_INDEX_NAME) {
if (order == H5_ITER_INC)
qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_name_inc);
qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_name_inc);
else if (order == H5_ITER_DEC)
qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_name_dec);
qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_name_dec);
else
assert(order == H5_ITER_NATIVE);
} /* end if */
else {
assert(idx_type == H5_INDEX_CRT_ORDER);
if (order == H5_ITER_INC)
qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_corder_inc);
qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_corder_inc);
else if (order == H5_ITER_DEC)
qsort(atable->attrs, atable->nattrs, sizeof(H5A_t *), H5A__attr_cmp_corder_dec);
qsort(atable->attrs, atable->num_attrs, sizeof(H5A_t *), H5A__attr_cmp_corder_dec);
else
assert(order == H5_ITER_NATIVE);
} /* end else */
@ -1839,7 +1833,7 @@ H5A__attr_iterate_table(const H5A_attr_table_t *atable, hsize_t skip, hsize_t *l
/* Iterate over attribute messages */
H5_CHECKED_ASSIGN(u, size_t, skip, hsize_t);
for (; u < atable->nattrs && !ret_value; u++) {
for (; u < atable->num_attrs && !ret_value; u++) {
/* Check which type of callback to make */
switch (attr_op->op_type) {
case H5A_ATTR_OP_APP2: {
@ -1906,19 +1900,20 @@ H5A__attr_release_table(H5A_attr_table_t *atable)
assert(atable);
/* Release attribute info, if any. */
if (atable->nattrs > 0) {
if (atable->num_attrs > 0) {
size_t u; /* Local index variable */
/* Free attribute message information */
for (u = 0; u < atable->nattrs; u++)
for (u = 0; u < atable->num_attrs; u++)
if (atable->attrs[u] && H5A__close(atable->attrs[u]) < 0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTFREE, FAIL, "unable to release attribute");
/* Release array */
atable->attrs = (H5A_t **)H5FL_SEQ_FREE(H5A_t_ptr, atable->attrs);
} /* end if */
else
assert(atable->attrs == NULL);
atable->attrs = (H5A_t **)H5FL_SEQ_FREE(H5A_t_ptr, atable->attrs);
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5A__attr_release_table() */

5
src/H5Apkg.h

@ -145,8 +145,9 @@ typedef struct H5A_bt2_ud_ins_t {
/* Data structure to hold table of attributes for an object */
typedef struct {
size_t nattrs; /* # of attributes in table */
H5A_t **attrs; /* Pointer to array of attribute pointers */
size_t num_attrs; /* Curr. # of attributes in table */
size_t max_attrs; /* Max. # of attributes in table */
H5A_t **attrs; /* Pointer to array of attribute pointers */
} H5A_attr_table_t;
/*****************************/

7
src/H5Dint.c

@ -946,6 +946,13 @@ H5D__update_oh_info(H5F_t *file, H5D_t *dset, hid_t dapl_id)
if (NULL == (oh = H5O_pin(oloc)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTPIN, FAIL, "unable to pin dataset object header");
/* Check for creating dataset with unusual datatype */
if (H5T_is_numeric_with_unusual_unused_bits(type) &&
!(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
"creating dataset with unusual datatype, see documentation for "
"H5Pset_relax_file_integrity_checks for details.");
/* Write the dataspace header message */
if (H5S_append(file, oh, dset->shared->space) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update dataspace header message");

6
src/H5Dio.c

@ -395,8 +395,12 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info)
H5AC_tag(dset_info[i].dset->oloc.addr, &prev_tag);
/* Invoke correct "high level" I/O routine */
if ((*dset_info[i].io_ops.multi_read)(&io_info, &dset_info[i]) < 0)
if ((*dset_info[i].io_ops.multi_read)(&io_info, &dset_info[i]) < 0) {
/* Reset metadata tagging */
H5AC_tag(prev_tag, NULL);
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data");
}
/* Reset metadata tagging */
H5AC_tag(prev_tag, NULL);

4
src/H5Fint.c

@ -438,6 +438,8 @@ H5F_get_access_plist(H5F_t *f, bool app_ref)
0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID,
"can't set initial metadata cache resize config.");
if (H5P_set(new_plist, H5F_ACS_RFIC_FLAGS_NAME, &(f->shared->rfic_flags)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set RFIC flags value");
/* Prepare the driver property */
driver_prop.driver_id = f->shared->lf->driver_id;
@ -1230,6 +1232,8 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F
if (H5P_get(plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, &(f->shared->mdc_initCacheImageCfg)) <
0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get initial metadata cache resize config");
if (H5P_get(plist, H5F_ACS_RFIC_FLAGS_NAME, &(f->shared->rfic_flags)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get RFIC flags value");
/* Get the VFD values to cache */
f->shared->maxaddr = H5FD_get_maxaddr(lf);

1
src/H5Fpkg.h

@ -304,6 +304,7 @@ struct H5F_shared_t {
bool use_file_locking; /* Whether or not to use file locking */
bool ignore_disabled_locks; /* Whether or not to ignore disabled file locking */
bool closing; /* File is in the process of being closed */
uint64_t rfic_flags; /* Relaxed file integrity check (RFIC) flags */
/* Cached VOL connector ID & info */
hid_t vol_id; /* ID of VOL connector for the container */

6
src/H5Fprivate.h

@ -101,6 +101,7 @@ typedef struct H5F_t H5F_t;
#define H5F_VOL_CLS(F) ((F)->shared->vol_cls)
#define H5F_VOL_OBJ(F) ((F)->vol_obj)
#define H5F_USE_FILE_LOCKING(F) ((F)->shared->use_file_locking)
#define H5F_RFIC_FLAGS(F) ((F)->shared->rfic_flags)
#else /* H5F_MODULE */
#define H5F_LOW_BOUND(F) (H5F_get_low_bound(F))
#define H5F_HIGH_BOUND(F) (H5F_get_high_bound(F))
@ -165,6 +166,7 @@ typedef struct H5F_t H5F_t;
#define H5F_VOL_CLS(F) (H5F_get_vol_cls(F))
#define H5F_VOL_OBJ(F) (H5F_get_vol_obj(F))
#define H5F_USE_FILE_LOCKING(F) (H5F_get_use_file_locking(F))
#define H5F_RFIC_FLAGS(F) (H5F_get_rfic_flags(F))
#endif /* H5F_MODULE */
/* Macros to encode/decode offset/length's for storing in the file */
@ -282,6 +284,7 @@ typedef struct H5F_t H5F_t;
#define H5F_ACS_MPI_PARAMS_COMM_NAME "mpi_params_comm" /* the MPI communicator */
#define H5F_ACS_MPI_PARAMS_INFO_NAME "mpi_params_info" /* the MPI info struct */
#endif /* H5_HAVE_PARALLEL */
#define H5F_ACS_RFIC_FLAGS_NAME "rfic_flags" /* Relaxed file integrity check (RFIC) flags */
/* ======================== File Mount properties ====================*/
#define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */
@ -525,7 +528,8 @@ H5_DLL bool H5F_get_min_dset_ohdr(const H5F_t *f);
H5_DLL herr_t H5F_set_min_dset_ohdr(H5F_t *f, bool minimize);
H5_DLL const H5VL_class_t *H5F_get_vol_cls(const H5F_t *f);
H5_DLL H5VL_object_t *H5F_get_vol_obj(const H5F_t *f);
H5_DLL bool H5F_get_file_locking(const H5F_t *f);
H5_DLL bool H5F_get_use_file_locking(const H5F_t *f);
H5_DLL uint64_t H5F_get_rfic_flags(const H5F_t *f);
/* Functions than retrieve values set/cached from the superblock/FCPL */
H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f);

14
src/H5Fpublic.h

@ -242,6 +242,20 @@ typedef struct H5F_retry_info_t {
*/
typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata);
/*
* These are the bits that can be passed to the `flags' argument of
* H5Pset_relax_file_integrity_checks(). Use the bit-wise OR operator (|) to
* combine them as needed.
*/
#define H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS \
(0x0001u) /**< Suppress errors for numeric datatypes with an unusually \
* high number of unused bits. See documentation for \
* H5Pset_relax_file_integrity_checks for details. */
#define H5F_RFIC_ALL \
(H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS) /**< Suppress all format integrity check errors. See \
* documentation for H5Pset_relax_file_integrity_checks \
* for details. */
/*********************/
/* Public Prototypes */
/*********************/

28
src/H5Fquery.c

@ -1356,16 +1356,16 @@ done:
} /* end H5F_get_cont_info */
/*-------------------------------------------------------------------------
* Function: H5F_get_file_locking
* Function: H5F_get_use_file_locking
*
* Purpose: Get the file locking flag for the file
* Purpose: Get the 'use file locking' flag for the file
*
* Return: true/false
*
*-------------------------------------------------------------------------
*/
bool
H5F_get_file_locking(const H5F_t *f)
H5F_get_use_file_locking(const H5F_t *f)
{
FUNC_ENTER_NOAPI_NOINIT_NOERR
@ -1373,7 +1373,7 @@ H5F_get_file_locking(const H5F_t *f)
assert(f->shared);
FUNC_LEAVE_NOAPI(f->shared->use_file_locking)
} /* end H5F_get_file_locking */
} /* end H5F_get_use_file_locking */
/*-------------------------------------------------------------------------
* Function: H5F_has_vector_select_io
@ -1401,3 +1401,23 @@ H5F_has_vector_select_io(const H5F_t *f, bool is_write)
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F_has_vector_select_io */
/*-------------------------------------------------------------------------
* Function: H5F_get_rfic_flags
*
* Purpose: Get the relaxed file integrity checks (RFIC) flags for the file
*
* Return: RFIC flags for a file on success (which can be 0), can't fail
*
*-------------------------------------------------------------------------
*/
uint64_t
H5F_get_rfic_flags(const H5F_t *f)
{
FUNC_ENTER_NOAPI_NOINIT_NOERR
assert(f);
assert(f->shared);
FUNC_LEAVE_NOAPI(f->shared->rfic_flags)
} /* end H5F_get_rfic_flags */

37
src/H5HG.c

@ -559,9 +559,13 @@ H5HG_read(H5F_t *f, H5HG_t *hobj, void *object /*out*/, size_t *buf_size)
/* Load the heap */
if (NULL == (heap = H5HG__protect(f, hobj->addr, H5AC__READ_ONLY_FLAG)))
HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, NULL, "unable to protect global heap");
if (hobj->idx >= heap->nused)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}",
hobj->addr, hobj->idx);
if (NULL == heap->obj[hobj->idx].begin)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}",
hobj->addr, hobj->idx);
assert(hobj->idx < heap->nused);
assert(heap->obj[hobj->idx].begin);
size = heap->obj[hobj->idx].size;
p = heap->obj[hobj->idx].begin + H5HG_SIZEOF_OBJHDR(f);
@ -631,8 +635,12 @@ H5HG_link(H5F_t *f, const H5HG_t *hobj, int adjust)
HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, FAIL, "unable to protect global heap");
if (adjust != 0) {
assert(hobj->idx < heap->nused);
assert(heap->obj[hobj->idx].begin);
if (hobj->idx >= heap->nused)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}",
hobj->addr, hobj->idx);
if (NULL == heap->obj[hobj->idx].begin)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}",
hobj->addr, hobj->idx);
if ((heap->obj[hobj->idx].nrefs + adjust) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_BADRANGE, FAIL, "new link count would be out of range");
if ((heap->obj[hobj->idx].nrefs + adjust) > H5HG_MAXLINK)
@ -678,8 +686,13 @@ H5HG_get_obj_size(H5F_t *f, H5HG_t *hobj, size_t *obj_size)
if (NULL == (heap = H5HG__protect(f, hobj->addr, H5AC__READ_ONLY_FLAG)))
HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, FAIL, "unable to protect global heap");
assert(hobj->idx < heap->nused);
assert(heap->obj[hobj->idx].begin);
/* Sanity check the heap object */
if (hobj->idx >= heap->nused)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}",
hobj->addr, hobj->idx);
if (NULL == heap->obj[hobj->idx].begin)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}",
hobj->addr, hobj->idx);
/* Set object size */
*obj_size = heap->obj[hobj->idx].size;
@ -722,14 +735,22 @@ H5HG_remove(H5F_t *f, H5HG_t *hobj)
if (NULL == (heap = H5HG__protect(f, hobj->addr, H5AC__NO_FLAGS_SET)))
HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, FAIL, "unable to protect global heap");
assert(hobj->idx < heap->nused);
/* Sanity check the heap object (split around bugfix below) */
if (hobj->idx >= heap->nused)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap index, heap object = {%" PRIxHADDR ", %zu}",
hobj->addr, hobj->idx);
/* When the application selects the same location to rewrite the VL element by using H5Sselect_elements,
* it can happen that the entry has been removed by first rewrite. Here we simply skip the removal of
* the entry and let the second rewrite happen (see HDFFV-10635). In the future, it'd be nice to handle
* this situation in H5T_conv_vlen in H5Tconv.c instead of this level (HDFFV-10648). */
if (heap->obj[hobj->idx].nrefs == 0 && heap->obj[hobj->idx].size == 0 && !heap->obj[hobj->idx].begin)
HGOTO_DONE(ret_value);
HGOTO_DONE(SUCCEED);
/* Finish sanity checking the heap object */
if (NULL == heap->obj[hobj->idx].begin)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap pointer, heap object = {%" PRIxHADDR ", %zu}",
hobj->addr, hobj->idx);
obj_start = heap->obj[hobj->idx].begin;
/* Include object header size */

6
src/H5Oainfo.c

@ -138,19 +138,25 @@ H5O__ainfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS
ainfo->max_crt_idx = H5O_MAX_CRT_ORDER_IDX;
/* Address of fractal heap to store "dense" attributes */
H5_GCC_CLANG_DIAG_OFF("type-limits")
if (H5_IS_BUFFER_OVERFLOW(p, sizeof_addr, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
H5_GCC_CLANG_DIAG_ON("type-limits")
H5F_addr_decode(f, &p, &(ainfo->fheap_addr));
/* Address of v2 B-tree to index names of attributes (names are always indexed) */
H5_GCC_CLANG_DIAG_OFF("type-limits")
if (H5_IS_BUFFER_OVERFLOW(p, sizeof_addr, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
H5_GCC_CLANG_DIAG_ON("type-limits")
H5F_addr_decode(f, &p, &(ainfo->name_bt2_addr));
/* Address of v2 B-tree to index creation order of links, if there is one */
if (ainfo->index_corder) {
H5_GCC_CLANG_DIAG_OFF("type-limits")
if (H5_IS_BUFFER_OVERFLOW(p, sizeof_addr, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
H5_GCC_CLANG_DIAG_ON("type-limits")
H5F_addr_decode(f, &p, &(ainfo->corder_bt2_addr));
}
else

33
src/H5Oattribute.c

@ -211,6 +211,13 @@ H5O__attr_create(const H5O_loc_t *loc, H5A_t *attr)
if (NULL == (oh = H5O_pin(loc)))
HGOTO_ERROR(H5E_ATTR, H5E_CANTPIN, FAIL, "unable to pin object header");
/* Check for creating attribute with unusual datatype */
if (H5T_is_numeric_with_unusual_unused_bits(attr->shared->dt) &&
!(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(loc->file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS)))
HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL,
"creating attribute with unusual datatype, see documentation for "
"H5Pset_relax_file_integrity_checks for details.");
/* Check if this object already has attribute information */
if (oh->version > H5O_VERSION_1) {
bool new_ainfo = false; /* Flag to indicate that the attribute information is new */
@ -1171,10 +1178,10 @@ herr_t
H5O_attr_iterate_real(hid_t loc_id, const H5O_loc_t *loc, H5_index_t idx_type, H5_iter_order_t order,
hsize_t skip, hsize_t *last_attr, const H5A_attr_iter_op_t *attr_op, void *op_data)
{
H5O_t *oh = NULL; /* Pointer to actual object header */
H5O_ainfo_t ainfo; /* Attribute information for object */
H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */
herr_t ret_value = FAIL; /* Return value */
H5O_t *oh = NULL; /* Pointer to actual object header */
H5O_ainfo_t ainfo; /* Attribute information for object */
H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */
herr_t ret_value = FAIL; /* Return value */
FUNC_ENTER_NOAPI_NOINIT_TAG(loc->addr)
@ -1223,7 +1230,7 @@ H5O_attr_iterate_real(hid_t loc_id, const H5O_loc_t *loc, H5_index_t idx_type, H
oh = NULL;
/* Check for skipping too many attributes */
if (skip > 0 && skip >= atable.nattrs)
if (skip > 0 && skip >= atable.num_attrs)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index specified");
/* Iterate over attributes in table */
@ -1293,8 +1300,8 @@ done:
static herr_t
H5O__attr_remove_update(const H5O_loc_t *loc, H5O_t *oh, H5O_ainfo_t *ainfo)
{
H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */
herr_t ret_value = SUCCEED; /* Return value */
H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
@ -1530,11 +1537,11 @@ done:
herr_t
H5O__attr_remove_by_idx(const H5O_loc_t *loc, H5_index_t idx_type, H5_iter_order_t order, hsize_t n)
{
H5O_t *oh = NULL; /* Pointer to actual object header */
H5O_ainfo_t ainfo; /* Attribute information for object */
htri_t ainfo_exists = false; /* Whether the attribute info exists in the file */
H5A_attr_table_t atable = {0, NULL}; /* Table of attributes */
herr_t ret_value = SUCCEED; /* Return value */
H5O_t *oh = NULL; /* Pointer to actual object header */
H5O_ainfo_t ainfo; /* Attribute information for object */
htri_t ainfo_exists = false; /* Whether the attribute info exists in the file */
H5A_attr_table_t atable = {0, 0, NULL}; /* Table of attributes */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE_TAG(loc->addr)
@ -1568,7 +1575,7 @@ H5O__attr_remove_by_idx(const H5O_loc_t *loc, H5_index_t idx_type, H5_iter_order
HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, "error building attribute table");
/* Check for skipping too many attributes */
if (n >= atable.nattrs)
if (n >= atable.num_attrs)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index specified");
/* Set up user data for callback, to remove the attribute by name */

72
src/H5Odtype.c

@ -176,6 +176,14 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding");
UINT16DECODE(*pp, dt->shared->u.atomic.offset);
UINT16DECODE(*pp, dt->shared->u.atomic.prec);
/* Sanity checks */
if (dt->shared->u.atomic.offset >= (dt->shared->size * 8))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "integer offset out of bounds");
if (0 == dt->shared->u.atomic.prec)
HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "precision is zero");
if (((dt->shared->u.atomic.offset + dt->shared->u.atomic.prec) - 1) >= (dt->shared->size * 8))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "integer offset+precision out of bounds");
break;
case H5T_FLOAT:
@ -212,6 +220,8 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t
HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unknown floating-point normalization");
}
dt->shared->u.atomic.u.f.sign = (flags >> 8) & 0xff;
if (dt->shared->u.atomic.u.f.sign >= (dt->shared->size * 8))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "sign bit position out of bounds");
if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, 2 + 2, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding");
@ -224,6 +234,11 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t
dt->shared->u.atomic.u.f.esize = *(*pp)++;
if (dt->shared->u.atomic.u.f.esize == 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "exponent size can't be zero");
if (dt->shared->u.atomic.u.f.epos >= (dt->shared->size * 8))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "exponent starting position out of bounds");
if (((dt->shared->u.atomic.u.f.epos + dt->shared->u.atomic.u.f.esize) - 1) >=
(dt->shared->size * 8))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "exponent range out of bounds");
if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, 1 + 1, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding");
@ -231,10 +246,30 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t
dt->shared->u.atomic.u.f.msize = *(*pp)++;
if (dt->shared->u.atomic.u.f.msize == 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "mantissa size can't be zero");
if (dt->shared->u.atomic.u.f.mpos >= (dt->shared->size * 8))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "mantissa starting position out of bounds");
if (((dt->shared->u.atomic.u.f.mpos + dt->shared->u.atomic.u.f.msize) - 1) >=
(dt->shared->size * 8))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "mantissa range out of bounds");
if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, 4, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding");
UINT32DECODE(*pp, dt->shared->u.atomic.u.f.ebias);
/* Sanity check bits don't overlap */
if (H5_RANGE_OVERLAP(dt->shared->u.atomic.u.f.sign, dt->shared->u.atomic.u.f.sign,
dt->shared->u.atomic.u.f.epos,
((dt->shared->u.atomic.u.f.epos + dt->shared->u.atomic.u.f.esize) - 1)))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "exponent and sign positions overlap");
if (H5_RANGE_OVERLAP(dt->shared->u.atomic.u.f.sign, dt->shared->u.atomic.u.f.sign,
dt->shared->u.atomic.u.f.mpos,
((dt->shared->u.atomic.u.f.mpos + dt->shared->u.atomic.u.f.msize) - 1)))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "mantissa and sign positions overlap");
if (H5_RANGE_OVERLAP(dt->shared->u.atomic.u.f.epos,
((dt->shared->u.atomic.u.f.epos + dt->shared->u.atomic.u.f.esize) - 1),
dt->shared->u.atomic.u.f.mpos,
((dt->shared->u.atomic.u.f.mpos + dt->shared->u.atomic.u.f.msize) - 1)))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "mantissa and exponent positions overlap");
break;
case H5T_TIME:
@ -378,9 +413,11 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t
/* Decode the field offset */
/* (starting with version 3 of the datatype message, use the minimum # of bytes required) */
if (version >= H5O_DTYPE_VERSION_3) {
H5_GCC_CLANG_DIAG_OFF("type-limits")
if (H5_IS_KNOWN_BUFFER_OVERFLOW(skip, *pp, offset_nbytes, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL,
"ran off end of input buffer while decoding");
H5_GCC_CLANG_DIAG_ON("type-limits")
UINT32DECODE_VAR(*pp, dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset,
offset_nbytes);
}
@ -452,6 +489,13 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t
}
if (temp_type->shared->size == 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "type size can't be zero");
if ((dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset +
temp_type->shared->size) > dt->shared->size) {
if (H5T_close_real(temp_type) < 0)
HDONE_ERROR(H5E_DATATYPE, H5E_CANTRELEASE, FAIL, "can't release datatype info");
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL,
"member type extends outside its parent compound type");
}
/* Upgrade the version if we can and it is necessary */
if (can_upgrade && temp_type->shared->version > version) {
@ -770,6 +814,19 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t
HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unknown datatype class found");
}
/* Check for numeric type w/unusual # of unused bits */
if (H5T_is_numeric_with_unusual_unused_bits(dt))
/* Throw an error if the object header is not checksummed, unless the
* H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS flag is set with
* H5Pset_relax_file_integrity_checks() to suppress it.
*/
if (!(*ioflags & H5O_DECODEIO_RFIC_UNUBNT))
HGOTO_ERROR(
H5E_DATATYPE, H5E_BADVALUE, FAIL,
"datatype has unusually large # of unused bits (prec = %zu bits, size = %zu bytes), possibly "
"corrupted file. See documentation for H5Pset_relax_file_integrity_checks for details.",
dt->shared->u.atomic.prec, dt->shared->size);
done:
/* Cleanup on error */
if (ret_value < 0)
@ -1307,8 +1364,8 @@ done:
function using malloc() and is returned to the caller.
--------------------------------------------------------------------------*/
static void *
H5O__dtype_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags,
unsigned *ioflags /*in,out*/, size_t p_size, const uint8_t *p)
H5O__dtype_decode(H5F_t *f, H5O_t *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, unsigned *ioflags /*in,out*/,
size_t p_size, const uint8_t *p)
{
bool skip;
H5T_t *dt = NULL;
@ -1331,6 +1388,17 @@ H5O__dtype_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsign
*/
skip = (p_size == SIZE_MAX ? true : false);
/* Indicate if the object header has a checksum, or if the
* H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS flag is set */
if (open_oh) {
if (H5O_SIZEOF_CHKSUM_OH(open_oh) > 0 ||
(f && (H5F_RFIC_FLAGS(f) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS)))
*ioflags |= H5O_DECODEIO_RFIC_UNUBNT;
}
else
/* Decode operations from non-object headers are assumed to be checksummed */
*ioflags |= H5O_DECODEIO_RFIC_UNUBNT;
/* Perform actual decode of message */
if (H5O__dtype_decode_helper(ioflags, &p, dt, skip, p_end) < 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, NULL, "can't decode type");

6
src/H5Oefl.c

@ -140,6 +140,9 @@ H5O__efl_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED
#endif
for (size_t u = 0; u < mesg->nused; u++) {
hsize_t offset = 0;
/* Name */
if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
@ -156,7 +159,8 @@ H5O__efl_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED
/* File offset */
if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
H5F_DECODE_LENGTH(f, p, mesg->slot[u].offset);
H5F_DECODE_LENGTH(f, p, offset); /* Decode into an hsize_t to avoid sign warnings */
mesg->slot[u].offset = (HDoff_t)offset;
/* Size */
if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end))

20
src/H5Oint.c

@ -2919,3 +2919,23 @@ H5O__reset_info2(H5O_info2_t *oinfo)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5O__reset_info2() */
/*-------------------------------------------------------------------------
* Function: H5O_has_chksum
*
* Purpose: Returns true if object header is checksummed
*
* Return: true/false on success, can't fail
*
*-------------------------------------------------------------------------
*/
bool
H5O_has_chksum(const H5O_t *oh)
{
FUNC_ENTER_NOAPI_NOINIT_NOERR
/* Check args */
assert(oh);
FUNC_LEAVE_NOAPI(H5O_SIZEOF_CHKSUM_OH(oh) > 0)
} /* end H5O_has_chksum() */

8
src/H5Olayout.c

@ -392,10 +392,16 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
case H5D_CHUNK_IDX_SINGLE: /* Single Chunk Index */
if (mesg->u.chunk.flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) {
uint64_t nbytes = 0;
if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f) + 4, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding");
H5F_DECODE_LENGTH(f, p, mesg->storage.u.chunk.u.single.nbytes);
H5F_DECODE_LENGTH(f, p, nbytes);
H5_CHECKED_ASSIGN(mesg->storage.u.chunk.u.single.nbytes, uint32_t, nbytes,
uint64_t);
UINT32DECODE(p, mesg->storage.u.chunk.u.single.filter_mask);
}

5
src/H5Olinfo.c

@ -140,6 +140,9 @@ H5O__linfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS
if (H5_IS_BUFFER_OVERFLOW(p, 8, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
INT64DECODE(p, linfo->max_corder);
if (linfo->max_corder < 0)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL,
"invalid max creation order value for message: %" PRId64, linfo->max_corder);
}
else
linfo->max_corder = 0;
@ -156,8 +159,10 @@ H5O__linfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS
/* Address of v2 B-tree to index creation order of links, if there is one */
if (linfo->index_corder) {
H5_GCC_CLANG_DIAG_OFF("type-limits")
if (H5_IS_BUFFER_OVERFLOW(p, addr_size, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
H5_GCC_CLANG_DIAG_ON("type-limits")
H5F_addr_decode(f, &p, &(linfo->corder_bt2_addr));
}
else

2
src/H5Opkg.h

@ -150,6 +150,8 @@
/* Input/output flags for decode functions */
#define H5O_DECODEIO_NOCHANGE 0x01u /* IN: do not modify values */
#define H5O_DECODEIO_DIRTY 0x02u /* OUT: message has been changed */
#define H5O_DECODEIO_RFIC_UNUBNT \
0x04u /* IN: Relax file integrity checks for unusual numbers of unused bits in numeric datatypes */
/* Macro to incremend ndecode_dirtied (only if we are debugging) */
#ifndef NDEBUG

1
src/H5Oprivate.h

@ -932,6 +932,7 @@ H5_DLL time_t H5O_get_oh_mtime(const H5O_t *oh);
H5_DLL uint8_t H5O_get_oh_version(const H5O_t *oh);
H5_DLL herr_t H5O_get_rc_and_type(const H5O_loc_t *oloc, unsigned *rc, H5O_type_t *otype);
H5_DLL H5AC_proxy_entry_t *H5O_get_proxy(const H5O_t *oh);
H5_DLL bool H5O_has_chksum(const H5O_t *oh);
/* Object header message routines */
H5_DLL herr_t H5O_msg_create(const H5O_loc_t *loc, unsigned type_id, unsigned mesg_flags,

71
src/H5Pencdec.c

@ -272,6 +272,41 @@ H5P__encode_double(const void *value, void **_pp, size_t *size)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5P__encode_double() */
/*-------------------------------------------------------------------------
* Function: H5P__encode_uint64_t
*
* Purpose: Generic encoding callback routine for 'uint64_t' properties.
*
* Return: Success: Non-negative
* Failure: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
H5P__encode_uint64_t(const void *value, void **_pp, size_t *size)
{
uint8_t **pp = (uint8_t **)_pp;
FUNC_ENTER_PACKAGE_NOERR
/* Sanity checks */
assert(value);
assert(size);
if (NULL != *pp) {
/* Encode the size */
*(*pp)++ = (uint8_t)sizeof(uint64_t);
/* Encode the value */
UINT64ENCODE(*pp, *(const unsigned *)value);
} /* end if */
/* Set size needed for encoding */
*size += (1 + sizeof(uint64_t));
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5P__encode_uint64_t() */
/*--------------------------------------------------------------------------
NAME
H5P__encode_cb
@ -611,6 +646,42 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5P__decode_double() */
/*-------------------------------------------------------------------------
* Function: H5P__decode_uint64_t
*
* Purpose: Generic decoding callback routine for 'uint64_t' properties.
*
* Return: Success: Non-negative
* Failure: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
H5P__decode_uint64_t(const void **_pp, void *_value)
{
uint64_t *value = (uint64_t *)_value; /* Property value to return */
const uint8_t **pp = (const uint8_t **)_pp;
unsigned enc_size; /* Size of encoded property */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* Sanity checks */
assert(pp);
assert(*pp);
assert(value);
/* Decode the size */
enc_size = *(*pp)++;
if (enc_size != sizeof(uint64_t))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "uint64_t value can't be decoded");
UINT64DECODE(*pp, *value);
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5P__decode_uint64_t() */
/*-------------------------------------------------------------------------
NAME
H5P__decode

84
src/H5Pfapl.c

@ -333,6 +333,11 @@
#endif
#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC H5P__encode_bool
#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC H5P__decode_bool
/* Definition for 'rfic' flags */
#define H5F_ACS_RFIC_FLAGS_SIZE sizeof(uint64_t)
#define H5F_ACS_RFIC_FLAGS_DEF 0
#define H5F_ACS_RFIC_FLAGS_ENC H5P__encode_uint64_t
#define H5F_ACS_RFIC_FLAGS_DEC H5P__decode_uint64_t
/******************/
/* Local Typedefs */
@ -529,6 +534,7 @@ static const bool H5F_def_use_file_locking_g =
H5F_ACS_USE_FILE_LOCKING_DEF; /* Default use file locking flag */
static const bool H5F_def_ignore_disabled_file_locks_g =
H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */
static const uint64_t H5F_def_rfic_flags_g = H5F_ACS_RFIC_FLAGS_DEF; /* Default 'rfic' flags */
/*-------------------------------------------------------------------------
* Function: H5P__facc_reg_prop
@ -826,6 +832,12 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass)
H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC, NULL, NULL, NULL, NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class");
/* Register the 'rfic' flags. */
if (H5P__register_real(pclass, H5F_ACS_RFIC_FLAGS_NAME, H5F_ACS_RFIC_FLAGS_SIZE, &H5F_def_rfic_flags_g,
NULL, NULL, NULL, H5F_ACS_RFIC_FLAGS_ENC, H5F_ACS_RFIC_FLAGS_DEC, NULL, NULL, NULL,
NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class");
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5P__facc_reg_prop() */
@ -6264,3 +6276,75 @@ H5P__facc_vol_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size,
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5P__facc_vol_close() */
/*-------------------------------------------------------------------------
* Function: H5Pset_relax_file_integrity_checks
*
* Purpose: Relax certain file integrity checks that may issue errors
* for valid files that have the potential for incorrect library
* behavior when data is incorrect or corrupted.
*
* Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pset_relax_file_integrity_checks(hid_t plist_id, uint64_t flags)
{
H5P_genplist_t *plist; /* Property list pointer */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE2("e", "iUL", plist_id, flags);
/* Check arguments */
if (H5P_DEFAULT == plist_id)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't modify default property list");
if (flags & (uint64_t)~H5F_RFIC_ALL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags");
/* Get the property list structure */
if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list");
/* Set value */
if (H5P_set(plist, H5F_ACS_RFIC_FLAGS_NAME, &flags) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set relaxed file integrity check flags");
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pset_relax_file_integrity_checks() */
/*-------------------------------------------------------------------------
* Function: H5Pget_relax_file_integrity_checks
*
* Purpose: Retrieve relaxed file integrity check flags
*
* Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pget_relax_file_integrity_checks(hid_t plist_id, uint64_t *flags /*out*/)
{
H5P_genplist_t *plist; /* Property list pointer */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE2("e", "i*UL", plist_id, flags);
if (H5P_DEFAULT == plist_id)
plist_id = H5P_FILE_ACCESS_DEFAULT;
/* Get the property list structure */
if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list");
/* Get value */
if (flags)
if (H5P_get(plist, H5F_ACS_RFIC_FLAGS_NAME, flags) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get relaxed file integrity check flags");
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_relax_file_integrity_checks() */

2
src/H5Ppkg.h

@ -175,12 +175,14 @@ H5_DLL herr_t H5P__encode_unsigned(const void *value, void **_pp, size_t *size);
H5_DLL herr_t H5P__encode_uint8_t(const void *value, void **_pp, size_t *size);
H5_DLL herr_t H5P__encode_bool(const void *value, void **_pp, size_t *size);
H5_DLL herr_t H5P__encode_double(const void *value, void **_pp, size_t *size);
H5_DLL herr_t H5P__encode_uint64_t(const void *value, void **_pp, size_t *size);
H5_DLL herr_t H5P__decode_hsize_t(const void **_pp, void *value);
H5_DLL herr_t H5P__decode_size_t(const void **_pp, void *value);
H5_DLL herr_t H5P__decode_unsigned(const void **_pp, void *value);
H5_DLL herr_t H5P__decode_uint8_t(const void **_pp, void *value);
H5_DLL herr_t H5P__decode_bool(const void **_pp, void *value);
H5_DLL herr_t H5P__decode_double(const void **_pp, void *value);
H5_DLL herr_t H5P__decode_uint64_t(const void **_pp, void *value);
H5_DLL herr_t H5P__encode_coll_md_read_flag_t(const void *value, void **_pp, size_t *size);
H5_DLL herr_t H5P__decode_coll_md_read_flag_t(const void **_pp, void *value);

73
src/H5Ppublic.h

@ -5634,6 +5634,79 @@ H5_DLL herr_t H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t
H5_DLL herr_t H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_per,
unsigned min_raw_per);
/**
* \ingroup FAPL
*
* \brief Relax file integrity checks that may issue errors for some valid files
*
* \fapl_id{plist_id}
* \param[in] flags Relaxed integrity checks flag. Valid values are:
* \li #H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS
* suppresses integrity checks for detecting
* unusually high values for the number of unused bits in
* numeric datatype classes (H5T_INTEGER, H5T_FLOAT, and
* H5T_BITFIELD). Integrity checks are triggered when
* the precision for a datatype (i.e. the number of bits
* containing actual data) is less than half of the
* datatype's size and the datatype is greater than
* 1 byte in size. For example, a datatype with a
* precision of 15 bits and a size of 4 bytes (i.e. 32 bits)
* will issue an error, but a datatype with 17 bits of
* precision and a size of 4 bytes will not issue an
* error, nor will a datatype with a precision of 1, 2, or
* 3 bits and a size of 1 byte issue an error.
* \li #H5F_RFIC_ALL relaxes all integrity checks above.
*
* \return \herr_t
*
* \details Incorrectly encoded or corrupted metadata in a native HDF5
* format file can cause incorrect library behavior when the metadata
* has no checksum. Integrity checks within the library detect these
* circumstances and issue errors when incorrect metadata is found.
* Unfortunately, some of the integrity checks for detecting these
* circumstances may incorrectly issue an error for a valid HDF5 file
* that was intentionally created with these configurations.
* Setting the appropriate flag(s) with this routine will relax the
* file integrity checks for these valid files and suppress errors
* when accessing objects with these configurations.
*
* The library will also issue errors when these configurations are
* used to create objects, preventing applications from unintentionally
* creating them. Setting the appropriate flag with this routine will
* also suppress those errors on creation, although using this routine
* and the appropriate flag(s) will still be required when accessing
* files created with these configurations.
*
* A more complete solution that avoids errors on both object creation
* and access is to use the H5Pset_libver_bounds routine with a low
* bound of at least #H5F_LIBVER_V18 when creating objects with these
* configurations. This will cause the library to checksum a file's
* metadata, allowing accidental data corruption to be correctly
* detected and errors correctly issued without ambiguity.
*
* \since 1.14.4
*
*/
H5_DLL herr_t H5Pset_relax_file_integrity_checks(hid_t plist_id, uint64_t flags);
/**
* \ingroup FAPL
*
* \brief Retrieve relaxed file integrity check flags
*
* \fapl_id{plist_id}
* \param[out] flags Relaxed file integrity check flags
*
* \return \herr_t
*
* \details H5Pget_relax_file_integrity_checks() retrieves the relaxed file
* integrity check value into \p flags for the file access property
* list specified in \p plist_id.
*
* \since 1.14.4
*
*/
H5_DLL herr_t H5Pget_relax_file_integrity_checks(hid_t plist_id, uint64_t *flags);
/* Dataset creation property list (DCPL) routines */
/**
* \ingroup DCPL

34
src/H5Rint.c

@ -956,7 +956,10 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref)
const uint8_t *p = (const uint8_t *)buf;
size_t buf_size = 0, decode_size = 0;
uint8_t flags;
herr_t ret_value = SUCCEED;
bool decoded_filename = false; /* Whether filename was decoded, for error handling */
bool decoded_attrname = false; /* Whether attribute name was decoded, for error handling */
bool decoded_dataspace = false; /* Whether dataspace was decoded, for error handling */
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
@ -990,6 +993,7 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref)
/* Decode file name */
H5R_DECODE(H5R__decode_string, &ref->info.obj.filename, p, buf_size, decode_size,
"Cannot decode filename");
decoded_filename = true;
}
else
ref->info.obj.filename = NULL;
@ -997,22 +1001,28 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref)
switch (ref->type) {
case H5R_OBJECT2:
break;
case H5R_DATASET_REGION2:
/* Decode dataspace */
H5R_DECODE(H5R__decode_region, &ref->info.reg.space, p, buf_size, decode_size,
"Cannot decode region");
decoded_dataspace = true;
break;
case H5R_ATTR:
/* Decode attribute name */
H5R_DECODE(H5R__decode_string, &ref->info.attr.name, p, buf_size, decode_size,
"Cannot decode attribute name");
decoded_attrname = true;
break;
case H5R_OBJECT1:
case H5R_DATASET_REGION1:
case H5R_BADTYPE:
case H5R_MAXTYPE:
assert("invalid reference type" && 0);
HGOTO_ERROR(H5E_REFERENCE, H5E_UNSUPPORTED, FAIL, "internal error (invalid reference type)");
default:
assert("unknown reference type" && 0);
HGOTO_ERROR(H5E_REFERENCE, H5E_UNSUPPORTED, FAIL, "internal error (unknown reference type)");
@ -1031,6 +1041,22 @@ H5R__decode(const unsigned char *buf, size_t *nbytes, H5R_ref_priv_t *ref)
*nbytes = decode_size;
done:
if (ret_value < 0) {
if (decoded_filename) {
H5MM_xfree(ref->info.obj.filename);
ref->info.obj.filename = NULL;
}
if (decoded_attrname) {
H5MM_xfree(ref->info.attr.name);
ref->info.attr.name = NULL;
}
if (decoded_dataspace) {
if (H5S_close(ref->info.reg.space) < 0)
HDONE_ERROR(H5E_REFERENCE, H5E_CLOSEERROR, FAIL, "unable to release dataspace");
ref->info.reg.space = NULL;
}
}
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5R__decode() */
@ -1175,7 +1201,7 @@ H5R__decode_region(const unsigned char *buf, size_t *nbytes, H5S_t **space_ptr)
const uint8_t *p_end = p + *nbytes - 1;
size_t buf_size = 0;
unsigned rank;
H5S_t *space;
H5S_t *space = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
@ -1216,6 +1242,10 @@ H5R__decode_region(const unsigned char *buf, size_t *nbytes, H5S_t **space_ptr)
*space_ptr = space;
done:
if (ret_value < 0)
if (space && H5S_close(space) < 0)
HDONE_ERROR(H5E_REFERENCE, H5E_CLOSEERROR, FAIL, "unable to release dataspace");
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5R__decode_region() */

4
src/H5S.c

@ -615,6 +615,10 @@ H5S__extent_copy_real(H5S_extent_t *dst, const H5S_extent_t *src, bool copy_max)
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOPY, FAIL, "can't copy shared information");
done:
if (ret_value < 0)
if (dst->size)
dst->size = H5FL_ARR_FREE(hsize_t, dst->size);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5S__extent_copy_real() */

22
src/H5Shyper.c

@ -9178,8 +9178,8 @@ H5S__check_spans_overlap(const H5S_hyper_span_info_t *spans1, const H5S_hyper_sp
assert(spans2);
/* Use low & high bounds to try to avoid spinning through the span lists */
if (H5S_RANGE_OVERLAP(spans1->low_bounds[0], spans1->high_bounds[0], spans2->low_bounds[0],
spans2->high_bounds[0])) {
if (H5_RANGE_OVERLAP(spans1->low_bounds[0], spans1->high_bounds[0], spans2->low_bounds[0],
spans2->high_bounds[0])) {
H5S_hyper_span_t *span1, *span2; /* Hyperslab spans */
/* Walk over spans, comparing them for overlap */
@ -9187,7 +9187,7 @@ H5S__check_spans_overlap(const H5S_hyper_span_info_t *spans1, const H5S_hyper_sp
span2 = spans2->head;
while (span1 && span2) {
/* Check current two spans for overlap */
if (H5S_RANGE_OVERLAP(span1->low, span1->high, span2->low, span2->high)) {
if (H5_RANGE_OVERLAP(span1->low, span1->high, span2->low, span2->high)) {
/* Check for spans in lowest dimension already */
if (span1->down) {
/* Sanity check */
@ -9765,8 +9765,8 @@ H5S__hyper_regular_and_single_block(H5S_t *space, const hsize_t start[], const h
block_end = (start[u] + block[u]) - 1;
/* Check for overlap */
if (!H5S_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u],
block_end)) {
if (!H5_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u],
block_end)) {
overlap = false;
break;
} /* end if */
@ -9812,8 +9812,8 @@ H5S__hyper_regular_and_single_block(H5S_t *space, const hsize_t start[], const h
block_end = (start[u] + block[u]) - 1;
/* Check for overlap */
if (!H5S_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u],
block_end)) {
if (!H5_RANGE_OVERLAP(space->select.sel_info.hslab->diminfo.opt[u].start, select_end, start[u],
block_end)) {
overlap = false;
break;
} /* end if */
@ -10446,7 +10446,7 @@ H5S_combine_hyperslab(const H5S_t *old_space, H5S_seloper_t op, const hsize_t st
} /* end for */
/* Check bound box of both spaces to see if they overlap */
if (H5S_RANGE_OVERLAP(old_low_bounds[0], old_high_bounds[0], new_low_bounds[0], new_high_bounds[0]))
if (H5_RANGE_OVERLAP(old_low_bounds[0], old_high_bounds[0], new_low_bounds[0], new_high_bounds[0]))
overlapped = true;
/* Non-overlapping situations can be handled in special ways */
@ -11415,8 +11415,8 @@ H5S__hyper_proj_int_iterate(H5S_hyper_span_info_t *ss_span_info, const H5S_hyper
/* Check for non-overlapping bounds */
check_intersect = true;
for (u = 0; u < (udata->ss_rank - depth); u++)
if (!H5S_RANGE_OVERLAP(ss_span_info->low_bounds[u], ss_span_info->high_bounds[u],
sis_span_info->low_bounds[u], sis_span_info->high_bounds[u])) {
if (!H5_RANGE_OVERLAP(ss_span_info->low_bounds[u], ss_span_info->high_bounds[u],
sis_span_info->low_bounds[u], sis_span_info->high_bounds[u])) {
check_intersect = false;
break;
} /* end if */
@ -11441,7 +11441,7 @@ H5S__hyper_proj_int_iterate(H5S_hyper_span_info_t *ss_span_info, const H5S_hyper
/* Main loop */
do {
/* Check if spans overlap */
if (H5S_RANGE_OVERLAP(ss_low, ss_span->high, sis_low, sis_span->high)) {
if (H5_RANGE_OVERLAP(ss_low, ss_span->high, sis_low, sis_span->high)) {
high = MIN(ss_span->high, sis_span->high);
if (ss_span->down) {
/* Add skipped elements if there's a pre-gap */

9
src/H5Spkg.h

@ -89,15 +89,6 @@
* H5S_UNLIMITED) */
#define H5S_MAX_SIZE ((hsize_t)(hssize_t)(-2))
/* Macro for checking if two ranges overlap one another */
/*
* Check for the inverse of whether the ranges are disjoint. If they are
* disjoint, then the low bound of one of the ranges must be greater than the
* high bound of the other.
*/
/* (Assumes that low & high bounds are _inclusive_) */
#define H5S_RANGE_OVERLAP(L1, H1, L2, H2) (!((L1) > (H2) || (L2) > (H1)))
/*
* Dataspace extent information
*/

36
src/H5Spoint.c

@ -576,8 +576,14 @@ H5S__point_add(H5S_t *space, H5S_seloper_t op, size_t num_elem, const hsize_t *c
for (u = 0; u < num_elem; u++) {
unsigned dim; /* Counter for dimensions */
/* The following allocation relies on the size of an hcoords_t being
* the same as an 'H5S_pnt_node_t *', so fail now if that's not true
*/
HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *));
/* Allocate space for the new node */
if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, space->extent.rank)))
/* Note: allocating "rank + 1" to allow for 'next' pointer */
if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, space->extent.rank + 1)))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point node");
/* Initialize fields in node */
@ -796,7 +802,7 @@ H5S__copy_pnt_list(const H5S_pnt_list_t *src, unsigned rank)
assert(rank > 0);
/* Allocate room for the head of the point list */
if (NULL == (dst = H5FL_MALLOC(H5S_pnt_list_t)))
if (NULL == (dst = H5FL_CALLOC(H5S_pnt_list_t)))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, NULL, "can't allocate point list node");
curr = src->head;
@ -804,8 +810,14 @@ H5S__copy_pnt_list(const H5S_pnt_list_t *src, unsigned rank)
while (curr) {
H5S_pnt_node_t *new_node; /* New point information node */
/* The following allocation relies on the size of an hcoords_t being
* the same as an 'H5S_pnt_node_t *', so fail now if that's not true
*/
HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *));
/* Create new point */
if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, rank)))
/* Note: allocating "rank + 1" to allow for 'next' pointer */
if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, rank + 1)))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, NULL, "can't allocate point node");
new_node->next = NULL;
@ -2275,7 +2287,7 @@ H5S__point_project_simple(const H5S_t *base_space, H5S_t *new_space, hsize_t *of
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't release selection");
/* Allocate room for the head of the point list */
if (NULL == (new_space->select.sel_info.pnt_lst = H5FL_MALLOC(H5S_pnt_list_t)))
if (NULL == (new_space->select.sel_info.pnt_lst = H5FL_CALLOC(H5S_pnt_list_t)))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point list node");
/* Check if the new space's rank is < or > base space's rank */
@ -2294,8 +2306,14 @@ H5S__point_project_simple(const H5S_t *base_space, H5S_t *new_space, hsize_t *of
base_node = base_space->select.sel_info.pnt_lst->head;
prev_node = NULL;
while (base_node) {
/* The following allocation relies on the size of an hcoords_t being
* the same as an 'H5S_pnt_node_t *', so fail now if that's not true
*/
HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *));
/* Create new point */
if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank)))
/* Note: allocating "rank + 1" to allow for 'next' pointer */
if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank + 1)))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point node");
new_node->next = NULL;
@ -2336,8 +2354,14 @@ H5S__point_project_simple(const H5S_t *base_space, H5S_t *new_space, hsize_t *of
base_node = base_space->select.sel_info.pnt_lst->head;
prev_node = NULL;
while (base_node) {
/* The following allocation relies on the size of an hcoords_t being
* the same as an 'H5S_pnt_node_t *', so fail now if that's not true
*/
HDcompile_assert(sizeof(hcoords_t) >= sizeof(H5S_pnt_node_t *));
/* Create new point */
if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank)))
/* Note: allocating "rank + 1" to allow for 'next' pointer */
if (NULL == (new_node = (H5S_pnt_node_t *)H5FL_ARR_MALLOC(hcoords_t, new_space->extent.rank + 1)))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate point node");
new_node->next = NULL;

2
src/H5Sselect.c

@ -1945,7 +1945,7 @@ H5S_select_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *en
/* Loop over selection bounds and block, checking for overlap */
for (u = 0; u < space->extent.rank; u++)
/* If selection bounds & block don't overlap, can leave now */
if (!H5S_RANGE_OVERLAP(low[u], high[u], start[u], end[u]))
if (!H5_RANGE_OVERLAP(low[u], high[u], start[u], end[u]))
HGOTO_DONE(false);
} /* end if */

49
src/H5T.c

@ -41,6 +41,7 @@
#include "H5Pprivate.h" /* Property lists */
#include "H5Tpkg.h" /* Datatypes */
#include "H5VLprivate.h" /* Virtual Object Layer */
#include "H5VMprivate.h" /* Vectors and arrays */
/****************/
/* Local Macros */
@ -6692,3 +6693,51 @@ H5T__get_path_table_npaths(void)
FUNC_LEAVE_NOAPI(ret_value)
}
/*-------------------------------------------------------------------------
* Function: H5T_is_numeric_with_unusual_unused_bits
*
* Purpose: Detect if a datatype is a numeric datatype (int, float, or
* bitfield) with an unusual # of unused bits. This means
* that the precision (i.e. the # of bits used) is less than
* the size of the datatype, at power-of-two boundaries.
*
* Return: true/false on success, can't fail
*
*-------------------------------------------------------------------------
*/
bool
H5T_is_numeric_with_unusual_unused_bits(const H5T_t *dt)
{
bool ret_value = false;
FUNC_ENTER_NOAPI_NOINIT_NOERR
/* Sanity check */
assert(dt);
assert(dt->shared);
/* Is the correct type? */
if (H5T_INTEGER == dt->shared->type || H5T_FLOAT == dt->shared->type ||
H5T_BITFIELD == dt->shared->type) {
#if LDBL_MANT_DIG == 106
/* This currently won't work for the IBM long double type */
if (H5T_FLOAT == dt->shared->type && dt->shared->size == 16 &&
(dt->shared->u.atomic.prec == 64 || dt->shared->u.atomic.prec == 128))
HGOTO_DONE(false);
#endif
/* Has unused bits? */
if (dt->shared->u.atomic.prec < (dt->shared->size * 8)) {
unsigned surround_bits =
1U << (1 + H5VM_log2_gen((dt->shared->u.atomic.prec + dt->shared->u.atomic.offset) - 1));
/* Unused bits are unusually large? */
if (dt->shared->size > 1 && ((dt->shared->size * 8) > surround_bits))
HGOTO_DONE(true);
}
}
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5T_is_numeric_with_unusual_unused_bits() */

70
src/H5Tcommit.c

@ -424,11 +424,13 @@ done:
herr_t
H5T__commit(H5F_t *file, H5T_t *type, hid_t tcpl_id)
{
H5O_loc_t temp_oloc; /* Temporary object header location */
H5G_name_t temp_path; /* Temporary path */
bool loc_init = false; /* Have temp_oloc and temp_path been initialized? */
size_t dtype_size; /* Size of the datatype message */
herr_t ret_value = SUCCEED; /* Return value */
H5O_t *oh = NULL; /* Pointer to actual object header */
H5O_loc_t temp_oloc; /* Temporary object header location */
H5G_name_t temp_path; /* Temporary path */
bool loc_init = false; /* Have temp_oloc and temp_path been initialized? */
bool ohdr_created = false; /* Has the object header been created yet? */
size_t dtype_size; /* Size of the datatype message */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
@ -481,9 +483,23 @@ H5T__commit(H5F_t *file, H5T_t *type, hid_t tcpl_id)
*/
if (H5O_create(file, dtype_size, (size_t)1, tcpl_id, &temp_oloc) < 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to create datatype object header");
if (H5O_msg_create(&temp_oloc, H5O_DTYPE_ID, H5O_MSG_FLAG_CONSTANT | H5O_MSG_FLAG_DONTSHARE,
H5O_UPDATE_TIME, type) < 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to update type header message");
ohdr_created = true;
/* Pin the object header */
if (NULL == (oh = H5O_pin(&temp_oloc)))
HGOTO_ERROR(H5E_ATTR, H5E_CANTPIN, FAIL, "unable to pin object header");
/* Check for creating committed datatype with unusual datatype */
if (H5T_is_numeric_with_unusual_unused_bits(type) &&
!(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS)))
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL,
"creating committed datatype with unusual datatype, see documentation for "
"H5Pset_relax_file_integrity_checks for details.");
/* Insert the datatype message */
if (H5O_msg_append_oh(file, oh, H5O_DTYPE_ID, H5O_MSG_FLAG_CONSTANT | H5O_MSG_FLAG_DONTSHARE,
H5O_UPDATE_TIME, type) < 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to insert type header message");
/* Copy the new object header's location into the datatype, taking ownership of it */
if (H5O_loc_copy_shallow(&(type->oloc), &temp_oloc) < 0)
@ -510,23 +526,39 @@ H5T__commit(H5F_t *file, H5T_t *type, hid_t tcpl_id)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "cannot mark datatype in memory");
done:
if (oh && H5O_unpin(oh) < 0)
HDONE_ERROR(H5E_DATATYPE, H5E_CANTUNPIN, FAIL, "unable to unpin object header");
if (ret_value < 0) {
/* Close & delete the object header on failure */
if (ohdr_created) {
H5O_loc_t *oloc_ptr; /* Pointer to object header location */
/* Point at correct object header location, depending on state when failure occurred */
if (loc_init)
oloc_ptr = &temp_oloc;
else
oloc_ptr = &(type->oloc);
if (H5O_dec_rc_by_loc(oloc_ptr) < 0)
HDONE_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL,
"unable to decrement refcount on newly created object");
if (H5O_close(oloc_ptr, NULL) < 0)
HDONE_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "unable to release object header");
if (H5O_delete(file, oloc_ptr->addr) < 0)
HDONE_ERROR(H5E_DATATYPE, H5E_CANTDELETE, FAIL, "unable to delete object header");
}
/* Release the location info, if the datatype doesn't own it */
if (loc_init) {
H5O_loc_free(&temp_oloc);
H5G_name_free(&temp_path);
} /* end if */
}
/* Reset the shared state for the datatype */
if ((type->shared->state == H5T_STATE_TRANSIENT || type->shared->state == H5T_STATE_RDONLY) &&
(type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED)) {
if (H5O_dec_rc_by_loc(&(type->oloc)) < 0)
HDONE_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL,
"unable to decrement refcount on newly created object");
if (H5O_close(&(type->oloc), NULL) < 0)
HDONE_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "unable to release object header");
if (H5O_delete(file, type->sh_loc.u.loc.oh_addr) < 0)
HDONE_ERROR(H5E_DATATYPE, H5E_CANTDELETE, FAIL, "unable to delete object header");
(type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED))
type->sh_loc.type = H5O_SHARE_TYPE_UNSHARED;
} /* end if */
} /* end if */
}
FUNC_LEAVE_NOAPI(ret_value)
} /* H5T__commit() */

196
src/H5Tconv.c

@ -19,18 +19,21 @@
/****************/
#include "H5Tmodule.h" /* This source code file is part of the H5T module */
#define H5R_FRIEND /* Suppress error about including H5Rpkg */
/***********/
/* Headers */
/***********/
#include "H5private.h" /* Generic Functions */
#include "H5CXprivate.h" /* API Contexts */
#include "H5Dprivate.h" /* Datasets */
#include "H5Eprivate.h" /* Error handling */
#include "H5FLprivate.h" /* Free Lists */
#include "H5Iprivate.h" /* IDs */
#include "H5MMprivate.h" /* Memory management */
#include "H5Tpkg.h" /* Datatypes */
#include "H5private.h" /* Generic Functions */
#include "H5CXprivate.h" /* API Contexts */
#include "H5Dprivate.h" /* Datasets */
#include "H5Eprivate.h" /* Error handling */
#include "H5FLprivate.h" /* Free Lists */
#include "H5Iprivate.h" /* IDs */
#include "H5MMprivate.h" /* Memory management */
#include "H5Pprivate.h" /* Property lists */
#include "H5Rpkg.h" /* References */
#include "H5Tpkg.h" /* Datatypes */
/****************/
/* Local Macros */
@ -3445,26 +3448,33 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T
bool write_to_file = false; /* Flag to indicate writing to file */
htri_t parent_is_vlen; /* Flag to indicate parent is vlen datatype */
size_t bg_seq_len = 0; /* The number of elements in the background sequence */
H5T_t *tsrc_cpy = NULL; /*temporary copy of source base datatype */
H5T_t *tdst_cpy = NULL; /*temporary copy of destination base datatype */
hid_t tsrc_id = H5I_INVALID_HID; /*temporary type atom */
hid_t tdst_id = H5I_INVALID_HID; /*temporary type atom */
uint8_t *s = NULL; /*source buffer */
uint8_t *d = NULL; /*destination buffer */
uint8_t *b = NULL; /*background buffer */
ssize_t s_stride, d_stride; /*src and dst strides */
ssize_t b_stride; /*bkg stride */
size_t safe; /*how many elements are safe to process in each pass */
size_t src_base_size; /*source base size*/
size_t dst_base_size; /*destination base size*/
void *conv_buf = NULL; /*temporary conversion buffer */
size_t conv_buf_size = 0; /*size of conversion buffer in bytes */
void *tmp_buf = NULL; /*temporary background buffer */
size_t tmp_buf_size = 0; /*size of temporary bkg buffer */
bool nested = false; /*flag of nested VL case */
bool need_ids = false; /*whether we need IDs for the datatypes */
size_t elmtno; /*element number counter */
herr_t ret_value = SUCCEED; /* Return value */
H5T_t *tsrc_cpy = NULL; /* Temporary copy of source base datatype */
H5T_t *tdst_cpy = NULL; /* Temporary copy of destination base datatype */
hid_t tsrc_id = H5I_INVALID_HID; /* Temporary type atom */
hid_t tdst_id = H5I_INVALID_HID; /* Temporary type atom */
uint8_t *s = NULL; /* Source buffer */
uint8_t *d = NULL; /* Destination buffer */
uint8_t *b = NULL; /* Background buffer */
ssize_t s_stride = 0; /* Src stride */
ssize_t d_stride = 0; /* Dst stride */
ssize_t b_stride; /* Bkg stride */
size_t safe = 0; /* How many elements are safe to process in each pass */
size_t src_base_size; /* Source base size*/
size_t dst_base_size; /* Destination base size*/
void *conv_buf = NULL; /* Temporary conversion buffer */
size_t conv_buf_size = 0; /* Size of conversion buffer in bytes */
void *tmp_buf = NULL; /* Temporary background buffer */
size_t tmp_buf_size = 0; /* Size of temporary bkg buffer */
bool nested = false; /* Flag of nested VL case */
bool need_ids = false; /* Whether we need IDs for the datatypes */
size_t elmtno = 0; /* Element number counter */
size_t orig_d_stride = 0; /* Original destination stride (used for error handling) */
size_t orig_nelmts = nelmts; /* Original # of elements to convert (used for error handling) */
bool convert_forward =
true; /* Current direction of conversion (forward or backward, used for error handling) */
bool conversions_made =
false; /* Flag to indicate conversions have been performed, used for error handling */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
@ -3601,6 +3611,10 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T
if (write_to_file && parent_is_vlen && bkg != NULL)
nested = true;
/* Save info for unraveling on errors */
orig_d_stride = (size_t)d_stride;
convert_forward = !(d_stride > s_stride);
/* The outer loop of the type conversion macro, controlling which */
/* direction the buffer is walked */
while (nelmts > 0) {
@ -3782,6 +3796,9 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T
} /* end if */
} /* end else */
/* Indicate that elements have been converted, in case of error */
conversions_made = true;
/* Advance pointers */
s += s_stride;
d += d_stride;
@ -3801,6 +3818,49 @@ H5T__conv_vlen(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T
} /* end switch */
done:
/* Release converted elements on error */
if (ret_value < 0 && conversions_made) {
size_t dest_count;
/* Set up for first pass to destroy references */
if (nelmts < orig_nelmts || (convert_forward && elmtno < safe)) {
dest_count = orig_nelmts - nelmts;
/* Set pointer to correct location, based on direction chosen */
if (convert_forward) {
d = (uint8_t *)buf;
dest_count += elmtno; /* Include partial iteration in first pass, for forward conversions */
}
else
d = (uint8_t *)buf + (nelmts * orig_d_stride);
/* Destroy vlen elements that have already been converted */
while (dest_count > 0) {
H5T_vlen_reclaim_elmt(d, dst); /* Ignore errors at this point */
d += orig_d_stride;
dest_count--;
}
}
/* Do any remaining partial iteration, if converting backwards */
if (!convert_forward && elmtno < safe) {
dest_count = elmtno;
/* Set pointer to correct location */
if (d_stride > 0)
d = (uint8_t *)buf + ((nelmts - safe) * orig_d_stride);
else
d = (uint8_t *)buf + ((nelmts - elmtno) * orig_d_stride);
/* Destroy references that have already been converted */
while (dest_count > 0) {
H5T_vlen_reclaim_elmt(d, dst); /* Ignore errors at this point */
d += orig_d_stride;
dest_count--;
}
}
}
if (tsrc_id >= 0) {
if (H5I_dec_ref(tsrc_id) < 0)
HDONE_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL, "can't decrement reference on temporary ID");
@ -4033,16 +4093,23 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata,
const H5T_conv_ctx_t H5_ATTR_UNUSED *conv_ctx, size_t nelmts, size_t buf_stride,
size_t bkg_stride, void *buf, void *bkg)
{
uint8_t *s = NULL; /* source buffer */
uint8_t *d = NULL; /* destination buffer */
uint8_t *b = NULL; /* background buffer */
ssize_t s_stride, d_stride; /* src and dst strides */
ssize_t b_stride; /* bkg stride */
size_t safe; /* how many elements are safe to process in each pass */
void *conv_buf = NULL; /* temporary conversion buffer */
size_t conv_buf_size = 0; /* size of conversion buffer in bytes */
size_t elmtno; /* element number counter */
herr_t ret_value = SUCCEED; /* return value */
uint8_t *s = NULL; /* source buffer */
uint8_t *d = NULL; /* destination buffer */
uint8_t *b = NULL; /* background buffer */
ssize_t s_stride = 0; /* src stride */
ssize_t d_stride = 0; /* dst stride */
ssize_t b_stride; /* bkg stride */
size_t safe = 0; /* how many elements are safe to process in each pass */
void *conv_buf = NULL; /* temporary conversion buffer */
size_t conv_buf_size = 0; /* size of conversion buffer in bytes */
size_t elmtno = 0; /* element number counter */
size_t orig_d_stride = 0; /* Original destination stride (used for error handling) */
size_t orig_nelmts = nelmts; /* Original # of elements to convert (used for error handling) */
bool convert_forward =
true; /* Current direction of conversion (forward or backward, used for error handling) */
bool conversions_made =
false; /* Flag to indicate conversions have been performed, used for error handling */
herr_t ret_value = SUCCEED; /* return value */
FUNC_ENTER_PACKAGE
@ -4103,6 +4170,10 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata,
else
b_stride = 0;
/* Save info for unraveling on errors */
orig_d_stride = (size_t)d_stride;
convert_forward = !(d_stride > s_stride);
/* The outer loop of the type conversion macro, controlling which */
/* direction the buffer is walked */
while (nelmts > 0) {
@ -4202,6 +4273,9 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata,
} /* end else */
} /* end else */
/* Indicate that elements have been converted, in case of error */
conversions_made = true;
/* Advance pointers */
s += s_stride;
d += d_stride;
@ -4221,6 +4295,52 @@ H5T__conv_ref(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata,
} /* end switch */
done:
/* Release converted elements on error */
if (ret_value < 0 && conversions_made) {
H5R_ref_priv_t ref_priv;
size_t dest_count;
/* Set up for first pass to destroy references */
if (nelmts < orig_nelmts || (convert_forward && elmtno < safe)) {
dest_count = orig_nelmts - nelmts;
/* Set pointer to correct location, based on direction chosen */
if (convert_forward) {
d = (uint8_t *)buf;
dest_count += elmtno; /* Include partial iteration in first pass, for forward conversions */
}
else
d = (uint8_t *)buf + (nelmts * orig_d_stride);
/* Destroy references that have already been converted */
while (dest_count > 0) {
memcpy(&ref_priv, d, sizeof(H5R_ref_priv_t));
H5R__destroy(&ref_priv); /* Ignore errors at this point */
d += orig_d_stride;
dest_count--;
}
}
/* Do any remaining partial iteration, if converting backwards */
if (!convert_forward && elmtno < safe) {
dest_count = elmtno;
/* Set pointer to correct location */
if (d_stride > 0)
d = (uint8_t *)buf + ((nelmts - safe) * orig_d_stride);
else
d = (uint8_t *)buf + ((nelmts - elmtno) * orig_d_stride);
/* Destroy references that have already been converted */
while (dest_count > 0) {
memcpy(&ref_priv, d, sizeof(H5R_ref_priv_t));
H5R__destroy(&ref_priv); /* Ignore errors at this point */
d += orig_d_stride;
dest_count--;
}
}
}
/* Release the conversion buffer (always allocated, except on errors) */
if (conv_buf)
conv_buf = H5FL_BLK_FREE(ref_seq, conv_buf);

3
src/H5Tprivate.h

@ -139,7 +139,7 @@ H5_DLL herr_t H5T_convert(H5T_path_t *tpath, const H5T_t *src_type, const H5T_t
size_t buf_stride, size_t bkg_stride, void *buf, void *bkg);
H5_DLL herr_t H5T_reclaim(const H5T_t *type, struct H5S_t *space, void *buf);
H5_DLL herr_t H5T_reclaim_cb(void *elem, const H5T_t *dt, unsigned ndim, const hsize_t *point, void *op_data);
H5_DLL herr_t H5T_vlen_reclaim_elmt(void *elem, H5T_t *dt);
H5_DLL herr_t H5T_vlen_reclaim_elmt(void *elem, const H5T_t *dt);
H5_DLL htri_t H5T_set_loc(H5T_t *dt, H5VL_object_t *file, H5T_loc_t loc);
H5_DLL htri_t H5T_is_sensible(const H5T_t *dt);
H5_DLL uint32_t H5T_hash(H5F_t *file, const H5T_t *dt);
@ -157,6 +157,7 @@ H5_DLL bool H5T_already_vol_managed(const H5T_t *dt);
H5_DLL htri_t H5T_is_vl_storage(const H5T_t *dt);
H5_DLL herr_t H5T_invoke_vol_optional(H5T_t *dt, H5VL_optional_args_t *args, hid_t dxpl_id, void **req,
H5VL_object_t **vol_obj_ptr);
H5_DLL bool H5T_is_numeric_with_unusual_unused_bits(const H5T_t *dt);
/* Reference specific functions */
H5_DLL H5R_type_t H5T_get_ref_type(const H5T_t *dt);

2
src/H5Tvlen.c

@ -1053,7 +1053,7 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
H5T_vlen_reclaim_elmt(void *elem, H5T_t *dt)
H5T_vlen_reclaim_elmt(void *elem, const H5T_t *dt)
{
H5T_vlen_alloc_info_t vl_alloc_info; /* VL allocation info */
herr_t ret_value = SUCCEED; /* return value */

2
src/H5Zscaleoffset.c

@ -1210,6 +1210,8 @@ H5Z__filter_scaleoffset(unsigned flags, size_t cd_nelmts, const unsigned cd_valu
minbits_mask <<= i * 8;
minbits |= minbits_mask;
}
if (minbits >= p.size * 8)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "minimum number of bits exceeds size of type");
/* retrieval of minval takes into consideration situation where sizeof
* unsigned long long (datatype of minval) may change from compression

4
src/H5encode.h

@ -212,7 +212,7 @@
n = 0; \
(p) += 8; \
for (_i = 0; _i < sizeof(int64_t); _i++) \
n = (n << 8) | *(--p); \
n = (int64_t)(((uint64_t)n << 8) | *(--p)); \
(p) += 8; \
} while (0)
@ -224,7 +224,7 @@
n = 0; \
(p) += 8; \
for (_i = 0; _i < sizeof(uint64_t); _i++) \
n = (n << 8) | *(--p); \
n = (uint64_t)(((uint64_t)n << 8) | *(--p)); \
(p) += 8; \
} while (0)

26
src/H5private.h

@ -318,6 +318,15 @@
/* limit the middle value to be within a range (inclusive) */
#define RANGE(LO, X, HI) MAX(LO, MIN(X, HI))
/* Macro for checking if two ranges overlap one another */
/*
* Check for the inverse of whether the ranges are disjoint. If they are
* disjoint, then the low bound of one of the ranges must be greater than the
* high bound of the other.
*/
/* (Assumes that low & high bounds are _inclusive_) */
#define H5_RANGE_OVERLAP(L1, H1, L2, H2) (!((L1) > (H2) || (L2) > (H1)))
/* absolute value */
#ifndef ABS
#define ABS(a) (((a) >= 0) ? (a) : -(a))
@ -336,9 +345,19 @@
#define H5_EXP2(n) (1 << (n))
/* Check if a read of size bytes starting at ptr would overflow past
* the last valid byte, pointed to by buffer_end.
* the last valid byte, pointed to by buffer_end. Note that 'size'
* is expected to be of type size_t. Providing values of other
* datatypes may cause warnings due to the comparison against
* PTRDIFF_MAX and comparison of < 0 after conversion to ptrdiff_t.
* For the time being, these can be suppressed with
* H5_GCC_CLANG_DIAG_OFF("type-limits")/H5_GCC_CLANG_DIAG_ON("type-limits")
*/
#define H5_IS_BUFFER_OVERFLOW(ptr, size, buffer_end) (((ptr) + (size)-1) > (buffer_end))
#define H5_IS_BUFFER_OVERFLOW(ptr, size, buffer_end) \
(((ptr) > (buffer_end)) || /* Bad precondition */ \
(((size_t)(size) <= PTRDIFF_MAX) && \
((ptrdiff_t)(size) < 0)) || /* Account for (likely unintentional) negative 'size' */ \
((size_t)(size) > \
(size_t)((((const uint8_t *)buffer_end) - ((const uint8_t *)ptr)) + 1))) /* Typical overflow */
/* Variant of H5_IS_BUFFER_OVERFLOW, used with functions such as H5Tdecode()
* that don't take a size parameter, where we need to skip the bounds checks.
@ -451,8 +470,7 @@
(X) >= (Y))
#define H5_addr_cmp(X,Y) (H5_addr_eq((X), (Y)) ? 0 : \
(H5_addr_lt((X), (Y)) ? -1 : 1))
#define H5_addr_overlap(O1,L1,O2,L2) (((O1) < (O2) && ((O1) + (L1)) > (O2)) || \
((O1) >= (O2) && (O1) < ((O2) + (L2))))
#define H5_addr_overlap(O1,L1,O2,L2) H5_RANGE_OVERLAP(O1, ((O1)+(L1)), O2, ((O2)+(L2)))
/* clang-format on */
/*

2
test/CMakeTests.cmake

@ -163,6 +163,8 @@ set (HDF5_REFERENCE_TEST_FILES
test_filters_le.h5
th5s.h5
tlayouto.h5
tmisc38a.h5
tmisc38b.h5
tmtimen.h5
tmtimeo.h5
tsizeslheap.h5

BIN
test/testfiles/tmisc38a.h5 Normal file

Binary file not shown.

BIN
test/testfiles/tmisc38b.h5 Normal file

Binary file not shown.

280
test/tmisc.c

@ -337,9 +337,19 @@ typedef struct {
See https://nvd.nist.gov/vuln/detail/CVE-2020-10812 */
#define CVE_2020_10812_FILENAME "cve_2020_10812.h5"
#define MISC38_FILE "type_conversion_path_table_issue.h5"
#define MISC39_FILE "set_est_link_info.h5"
#define MISC40_FILE "obj_props_intermediate.h5"
/* Definitions for misc. test #38 */
#define MISC38A_FILE "tmisc38a.h5"
#define MISC38A_DSETNAME "Fletcher_float_data_be"
#define MISC38B_FILE "tmisc38b.h5"
#define MISC38B_DSETNAME "unusual_datatype"
#define MISC38C_FILE "tmisc38c.h5"
#define MISC38C_DSETNAME "dset_unusual_datatype"
#define MISC38C_TYPENAME "type_unusual_datatype"
#define MISC38C_ATTRNAME "attr_unusual_datatype"
#define MISC39_FILE "type_conversion_path_table_issue.h5"
#define MISC40_FILE "set_est_link_info.h5"
#define MISC41_FILE "obj_props_intermediate.h5"
/****************************************************************
**
@ -4025,6 +4035,7 @@ test_misc21(void)
static void
test_misc22(void)
{
hid_t fapl; /* File access property list */
hid_t fid, sid, dcpl, dsid, dcpl2;
char *buf;
hsize_t dims[2] = {MISC22_SPACE_DIM0, MISC22_SPACE_DIM1},
@ -4059,10 +4070,22 @@ test_misc22(void)
buf = (char *)calloc(MISC22_SPACE_DIM0 * MISC22_SPACE_DIM1, 8);
CHECK(buf, NULL, "calloc");
/* Create a file access property list */
fapl = H5Pcreate(H5P_FILE_ACCESS);
CHECK(fapl, FAIL, "H5Pcreate");
/* Set property to allow unusual datatypes to be created */
ret = H5Pset_relax_file_integrity_checks(fapl, H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS);
CHECK(ret, FAIL, "H5Pset_relax_file_integrity_checks");
/* Create the file */
fid = H5Fcreate(MISC22_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
fid = H5Fcreate(MISC22_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
CHECK(fid, FAIL, "H5Fcreate");
/* Close file access property list */
ret = H5Pclose(fapl);
CHECK(ret, FAIL, "H5Pclose");
/* Create the dataspace for the dataset */
sid = H5Screate_simple(MISC22_SPACE_RANK, dims, NULL);
CHECK(sid, FAIL, "H5Screate_simple");
@ -6264,6 +6287,223 @@ test_misc37(void)
/****************************************************************
**
** test_misc38():
** Test for seg fault issue when opening dataset with corrupted
** object header.
**
****************************************************************/
static void
test_misc38(void)
{
const char *testfile = H5_get_srcdir_filename(MISC38A_FILE); /* Corrected test file name */
const char *testfile2 = H5_get_srcdir_filename(MISC38B_FILE); /* Corrected test file name */
bool driver_is_default_compatible;
hid_t fapl = H5I_INVALID_HID; /* File access property list */
hid_t fid = H5I_INVALID_HID; /* File ID */
hid_t did = H5I_INVALID_HID; /* Dataset ID */
hid_t sid = H5I_INVALID_HID; /* Dataspace ID */
hid_t tid = H5I_INVALID_HID; /* Datatype ID */
hid_t gid = H5I_INVALID_HID; /* Group ID */
hid_t aid = H5I_INVALID_HID; /* Attribute ID */
size_t type_size; /* Size of dataset's datatype */
uint64_t rfic_flags; /* Value of RFIC flags property for FAPL & file */
herr_t ret;
/* Output message about test being performed */
MESSAGE(5, ("Fix for detecting numeric datatypes with unusually large numbers of unused bits"));
ret = h5_driver_is_default_vfd_compatible(H5P_DEFAULT, &driver_is_default_compatible);
CHECK(ret, FAIL, "h5_driver_is_default_vfd_compatible");
if (!driver_is_default_compatible) {
printf("-- SKIPPED --\n");
return;
}
fid = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT);
CHECK(fid, FAIL, "H5Fopen");
/* This should fail due to the illegal datatype encoding in the corrupted
* object header.
* It should fail gracefully and not seg fault
*/
H5E_BEGIN_TRY
{
did = H5Dopen2(fid, MISC38A_DSETNAME, H5P_DEFAULT);
}
H5E_END_TRY
VERIFY(did, H5I_INVALID_HID, "H5Dopen2");
/* Close file */
ret = H5Fclose(fid);
CHECK(ret, FAIL, "H5Fclose");
/* Create a file access property list */
fapl = H5Pcreate(H5P_FILE_ACCESS);
CHECK(fapl, H5I_INVALID_HID, "H5Pcreate");
/* Get property to allow unusual datatypes to be opened */
rfic_flags = H5F_RFIC_ALL;
ret = H5Pget_relax_file_integrity_checks(fapl, &rfic_flags);
CHECK(ret, FAIL, "H5Pget_relax_file_integrity_checks");
VERIFY(rfic_flags, 0, "H5Pget_relax_file_integrity_checks");
/* Set property to allow unusual datatypes to be opened */
ret = H5Pset_relax_file_integrity_checks(fapl, H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS);
CHECK(ret, FAIL, "H5Pset_relax_file_integrity_checks");
/* Get property to allow unusual datatypes to be opened */
rfic_flags = 0;
ret = H5Pget_relax_file_integrity_checks(fapl, &rfic_flags);
CHECK(ret, FAIL, "H5Pget_relax_file_integrity_checks");
VERIFY(rfic_flags, H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS, "H5Pget_relax_file_integrity_checks");
/* Open valid file */
fid = H5Fopen(testfile2, H5F_ACC_RDONLY, fapl);
CHECK(fid, H5I_INVALID_HID, "H5Fopen");
/* Close file access property list */
ret = H5Pclose(fapl);
CHECK(ret, FAIL, "H5Pclose");
/* Open dataset w/unusual datatype
* It should succeed and not return an error or seg fault
*/
did = H5Dopen2(fid, MISC38B_DSETNAME, H5P_DEFAULT);
CHECK(did, H5I_INVALID_HID, "H5Dopen2");
/* Get the dataset's datatype */
tid = H5Dget_type(did);
CHECK(tid, H5I_INVALID_HID, "H5Dget_type");
type_size = H5Tget_size(tid);
CHECK(type_size, 0, "H5Tget_size");
VERIFY(type_size, 1000, "H5Tget_size");
ret = H5Tclose(tid);
CHECK(ret, FAIL, "H5Tclose");
ret = H5Dclose(did);
CHECK(ret, FAIL, "H5Dclose");
/* Check that property is handled correctly */
fapl = H5Fget_access_plist(fid);
CHECK(fapl, H5I_INVALID_HID, "H5Fget_access_plist");
/* Get property to allow unusual datatypes to be opened */
rfic_flags = 0;
ret = H5Pget_relax_file_integrity_checks(fapl, &rfic_flags);
CHECK(ret, FAIL, "H5Pget_relax_file_integrity_checks");
VERIFY(rfic_flags, H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS, "H5Pget_relax_file_integrity_checks");
/* Close file access property list */
ret = H5Pclose(fapl);
CHECK(ret, FAIL, "H5Pclose");
/* Close file */
ret = H5Fclose(fid);
CHECK(ret, FAIL, "H5Fclose");
/* Create objects with unusual datatypes and verify correct behavior */
for (unsigned u = 0; u < 3; u++) {
/* Create a file access property list */
fapl = H5Pcreate(H5P_FILE_ACCESS);
CHECK(fapl, H5I_INVALID_HID, "H5Pcreate");
if (1 == u) {
/* Set property to allow unusual datatypes to be opened */
ret = H5Pset_relax_file_integrity_checks(fapl, H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS);
CHECK(ret, FAIL, "H5Pset_relax_file_integrity_checks");
}
else if (2 == u) {
/* Use a later version of the file format, with checksummed object headers */
ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
CHECK(ret, FAIL, "H5Pset_libver_bounds");
}
fid = H5Fcreate(MISC38C_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
CHECK(fid, H5I_INVALID_HID, "H5Fcreate");
/* Close file access property list */
ret = H5Pclose(fapl);
CHECK(ret, FAIL, "H5Pclose");
sid = H5Screate(H5S_SCALAR);
CHECK(sid, H5I_INVALID_HID, "H5Screate");
tid = H5Tcopy(H5T_NATIVE_INT);
CHECK(tid, H5I_INVALID_HID, "H5Tcopy");
/* Set type to have unusual size, for precision */
ret = H5Tset_size(tid, 1000);
CHECK(ret, FAIL, "H5Tset_size");
/* Create a dataset with the unusual datatype */
H5E_BEGIN_TRY
{
did = H5Dcreate2(fid, MISC38C_DSETNAME, tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
H5E_END_TRY
if (u > 0) {
CHECK(did, H5I_INVALID_HID, "H5Dcreate2");
ret = H5Dclose(did);
CHECK(ret, FAIL, "H5Dclose");
}
else {
VERIFY(did, H5I_INVALID_HID, "H5Dcreate2");
}
gid = H5Gopen2(fid, "/", H5P_DEFAULT);
CHECK(gid, H5I_INVALID_HID, "H5Gopen2");
/* Create an attribute with the unusual datatype */
H5E_BEGIN_TRY
{
aid = H5Acreate2(gid, MISC38C_ATTRNAME, tid, sid, H5P_DEFAULT, H5P_DEFAULT);
}
H5E_END_TRY
if (u > 0) {
CHECK(aid, H5I_INVALID_HID, "H5Acreate2");
ret = H5Aclose(aid);
CHECK(ret, FAIL, "H5Aclose");
}
else {
VERIFY(aid, H5I_INVALID_HID, "H5Acreate2");
}
ret = H5Gclose(gid);
CHECK(ret, FAIL, "H5Gclose");
ret = H5Sclose(sid);
CHECK(ret, FAIL, "H5Sclose");
/* Create a committed datatype with the unusual datatype */
H5E_BEGIN_TRY
{
ret = H5Tcommit2(fid, MISC38C_TYPENAME, tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
H5E_END_TRY
if (u > 0) {
CHECK(ret, FAIL, "H5Tcommit2");
}
else {
VERIFY(ret, FAIL, "H5Tcommit2");
}
if (tid != H5I_INVALID_HID) {
ret = H5Tclose(tid);
CHECK(ret, FAIL, "H5Tclose");
}
ret = H5Fclose(fid);
CHECK(ret, FAIL, "H5Fclose");
}
} /* end test_misc38() */
/****************************************************************
**
** test_misc39():
** Test for issue where the type conversion path table cache
** would grow continuously when variable-length datatypes
** are involved due to file VOL object comparisons causing
@ -6271,7 +6511,7 @@ test_misc37(void)
**
****************************************************************/
static void
test_misc38(void)
test_misc39(void)
{
H5VL_object_t *file_vol_obj = NULL;
const char *buf[] = {"attr_value"};
@ -6309,7 +6549,7 @@ test_misc38(void)
*/
init_npaths = H5T__get_path_table_npaths();
file_id = H5Fcreate(MISC38_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
file_id = H5Fcreate(MISC39_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
CHECK(file_id, H5I_INVALID_HID, "H5Fcreate");
/* Check if native VOL is being used */
@ -6444,7 +6684,7 @@ test_misc38(void)
CHECK_PTR(vlen_rbuf, "vlen varstr read buf allocation");
for (size_t i = 0; i < 10; i++) {
file_id = H5Fopen(MISC38_FILE, H5F_ACC_RDONLY, H5P_DEFAULT);
file_id = H5Fopen(MISC39_FILE, H5F_ACC_RDONLY, H5P_DEFAULT);
CHECK(file_id, H5I_INVALID_HID, "H5Fopen");
/* Retrieve file's VOL object field for further use */
@ -6546,7 +6786,7 @@ test_misc38(void)
/****************************************************************
**
** test_misc39(): Ensure H5Pset_est_link_info() handles large
** test_misc40(): Ensure H5Pset_est_link_info() handles large
** values
**
** H5Pset_est_link_info() values can be set to large values,
@ -6560,7 +6800,7 @@ test_misc38(void)
**
****************************************************************/
static void
test_misc39(void)
test_misc40(void)
{
hid_t fid = H5I_INVALID_HID; /* File ID */
hid_t gid = H5I_INVALID_HID; /* Group ID */
@ -6581,7 +6821,7 @@ test_misc39(void)
CHECK(ret, FAIL, "H5Pset_libver_bounds");
/* Create the file */
fid = H5Fcreate(MISC39_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
fid = H5Fcreate(MISC40_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
CHECK(fid, H5I_INVALID_HID, "H5Fcreate");
/* Compose group creation property list */
@ -6639,16 +6879,16 @@ test_misc39(void)
ret = H5Pclose(gcpl);
CHECK(ret, FAIL, "H5Pclose");
} /* end test_misc39() */
} /* end test_misc40() */
/****************************************************************
**
** test_misc40(): Test that object creation properties are propagated
** test_misc41(): Test that object creation properties are propagated
** to intermediate groups.
**
****************************************************************/
static void
test_misc40(void)
test_misc41(void)
{
hid_t lcpl = H5I_INVALID_HID;
hid_t gcpl = H5I_INVALID_HID;
@ -6675,7 +6915,7 @@ test_misc40(void)
status = H5Pset_create_intermediate_group(lcpl, 1);
CHECK(status, FAIL, "H5Pset_create_intermediate_group");
fid = H5Fcreate(MISC40_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
fid = H5Fcreate(MISC41_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
CHECK(fid, FAIL, "H5Fcreate");
/*
@ -6856,7 +7096,7 @@ test_misc40(void)
status = H5Pclose(lcpl);
CHECK(status, FAIL, "H5Pclose");
} /* end test_misc40() */
} /* end test_misc41() */
/****************************************************************
**
@ -6930,9 +7170,10 @@ test_misc(void)
test_misc35(); /* Test behavior of free-list & allocation statistics API calls */
test_misc36(); /* Exercise H5atclose and H5is_library_terminating */
test_misc37(); /* Test for seg fault failure at file close */
test_misc38(); /* Test for type conversion path table issue */
test_misc39(); /* Ensure H5Pset_est_link_info() handles large values */
test_misc40(); /* Test object properties propagated to intermediate groups */
test_misc38(); /* Test for seg fault when opening corrupted object header */
test_misc39(); /* Test for type conversion path table issue */
test_misc40(); /* Ensure H5Pset_est_link_info() handles large values */
test_misc41(); /* Test object properties propagated to intermediate groups */
} /* test_misc() */
@ -6988,9 +7229,10 @@ cleanup_misc(void)
#ifndef H5_NO_DEPRECATED_SYMBOLS
H5Fdelete(MISC31_FILE, H5P_DEFAULT);
#endif /* H5_NO_DEPRECATED_SYMBOLS */
H5Fdelete(MISC38_FILE, H5P_DEFAULT);
H5Fdelete(MISC38C_FILE, H5P_DEFAULT);
H5Fdelete(MISC39_FILE, H5P_DEFAULT);
H5Fdelete(MISC40_FILE, H5P_DEFAULT);
H5Fdelete(MISC41_FILE, H5P_DEFAULT);
}
H5E_END_TRY
} /* end cleanup_misc() */

82
tools/lib/h5tools_str.c

@ -1207,54 +1207,58 @@ h5tools_str_sprint(h5tools_str_t *str, const h5tool_format_t *info, hid_t contai
/*
* Object references -- show the type and OID of the referenced object.
*/
H5O_info2_t oi;
char *obj_tok_str = NULL;
H5TOOLS_DEBUG("H5T_REFERENCE:H5T_STD_REF_OBJ");
obj = H5Rdereference2(container, H5P_DEFAULT, H5R_OBJECT, vp);
H5Oget_info3(obj, &oi, H5O_INFO_BASIC);
if (obj >= 0) {
H5O_info2_t oi;
char *obj_tok_str = NULL;
/* Print object type and close object */
switch (oi.type) {
case H5O_TYPE_GROUP:
h5tools_str_append(str, H5_TOOLS_GROUP);
break;
H5Oget_info3(obj, &oi, H5O_INFO_BASIC);
case H5O_TYPE_DATASET:
h5tools_str_append(str, H5_TOOLS_DATASET);
break;
/* Print object type and close object */
switch (oi.type) {
case H5O_TYPE_GROUP:
h5tools_str_append(str, H5_TOOLS_GROUP);
break;
case H5O_TYPE_NAMED_DATATYPE:
h5tools_str_append(str, H5_TOOLS_DATATYPE);
break;
case H5O_TYPE_DATASET:
h5tools_str_append(str, H5_TOOLS_DATASET);
break;
case H5O_TYPE_MAP:
h5tools_str_append(str, H5_TOOLS_MAP);
break;
case H5O_TYPE_NAMED_DATATYPE:
h5tools_str_append(str, H5_TOOLS_DATATYPE);
break;
case H5O_TYPE_UNKNOWN:
case H5O_TYPE_NTYPES:
default:
h5tools_str_append(str, "%u-", (unsigned)oi.type);
break;
case H5O_TYPE_MAP:
h5tools_str_append(str, H5_TOOLS_MAP);
break;
case H5O_TYPE_UNKNOWN:
case H5O_TYPE_NTYPES:
default:
h5tools_str_append(str, "%u-", (unsigned)oi.type);
break;
}
/* Print OID */
H5Otoken_to_str(obj, &oi.token, &obj_tok_str);
H5Oclose(obj);
if (info->obj_hidefileno)
h5tools_str_append(str, info->obj_format, obj_tok_str);
else
h5tools_str_append(str, info->obj_format, oi.fileno, obj_tok_str);
if (obj_tok_str) {
H5free_memory(obj_tok_str);
obj_tok_str = NULL;
}
h5tools_str_sprint_old_reference(str, container, H5R_OBJECT, vp);
}
/* Print OID */
H5Otoken_to_str(obj, &oi.token, &obj_tok_str);
H5Oclose(obj);
if (info->obj_hidefileno)
h5tools_str_append(str, info->obj_format, obj_tok_str);
else
h5tools_str_append(str, info->obj_format, oi.fileno, obj_tok_str);
if (obj_tok_str) {
H5free_memory(obj_tok_str);
obj_tok_str = NULL;
}
h5tools_str_sprint_old_reference(str, container, H5R_OBJECT, vp);
h5tools_str_append(str, "<unknown>");
} /* end else if (H5Tequal(type, H5T_STD_REF_OBJ)) */
}
break;

2
tools/test/h5dump/CMakeTests.cmake

@ -1285,7 +1285,7 @@
ADD_H5_COMP_TEST (tfletcher32 0 0 --enable-error-stack -H -p -d fletcher32 tfilters.h5)
# nbit
ADD_H5_COMP_TEST (tnbit 0 10 --enable-error-stack -H -p -d nbit tfilters.h5)
ADD_H5_COMP_TEST (tnbit 0 1 --enable-error-stack -H -p -d nbit tfilters.h5)
# scaleoffset
ADD_H5_COMP_TEST (tscaleoffset 0 4 --enable-error-stack -H -p -d scaleoffset tfilters.h5)

4
tools/test/h5dump/expected/tnbit.ddl

@ -1,10 +1,10 @@
HDF5 "tfilters.h5" {
DATASET "nbit" {
DATATYPE 32-bit little-endian integer 3-bit precision
DATATYPE 32-bit little-endian integer 17-bit precision
DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) }
STORAGE_LAYOUT {
CHUNKED ( 10, 5 )
SIZE XXXX (10.XXX:1 COMPRESSION)
SIZE XXXX (1.XXX:1 COMPRESSION)
}
FILTERS {
COMPRESSION NBIT

42
tools/test/h5dump/expected/treadintfilter.ddl

@ -78,29 +78,29 @@ DATASET "fletcher32" {
}
}
DATASET "nbit" {
DATATYPE 32-bit little-endian integer 3-bit precision
DATATYPE 32-bit little-endian integer 17-bit precision
DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) }
DATA {
(0,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1,
(1,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3,
(2,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3,
(3,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1,
(4,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1,
(5,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3,
(6,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3,
(7,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1,
(8,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1,
(9,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3,
(10,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3,
(11,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1,
(12,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1,
(13,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3,
(14,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3,
(15,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1,
(16,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1,
(17,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3,
(18,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3,
(19,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1
(0,0): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
(1,0): 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
(2,0): 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
(3,0): 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
(4,0): 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
(5,0): 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
(6,0): 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
(7,0): 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
(8,0): 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
(9,0): 90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
(10,0): 100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
(11,0): 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
(12,0): 120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
(13,0): 130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
(14,0): 140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
(15,0): 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
(16,0): 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,
(17,0): 170, 171, 172, 173, 174, 175, 176, 177, 178, 179,
(18,0): 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
(19,0): 190, 191, 192, 193, 194, 195, 196, 197, 198, 199
}
}
DATASET "scaleoffset" {

2
tools/test/h5dump/h5dumpgentest.c

@ -5615,7 +5615,7 @@ gent_filters(void)
assert(ret >= 0);
tid = H5Tcopy(H5T_NATIVE_INT);
H5Tset_precision(tid, H5Tget_size(tid) - 1);
H5Tset_precision(tid, (H5Tget_size(tid) * 4) + 1);
ret = make_dset(fid, "nbit", sid, tid, dcpl, buf1);
assert(ret >= 0);

BIN
tools/test/h5dump/testfiles/tfilters.h5

Binary file not shown.

2
tools/test/h5dump/testh5dump.sh.in

@ -1430,7 +1430,7 @@ TOOLTEST tshuffle.ddl --enable-error-stack -H -p -d shuffle tfilters.h5
# fletcher32
TOOLTESTC 0 tfletcher32.ddl --enable-error-stack -H -p -d fletcher32 tfilters.h5
# nbit
TOOLTESTC 10 tnbit.ddl --enable-error-stack -H -p -d nbit tfilters.h5
TOOLTESTC 1 tnbit.ddl --enable-error-stack -H -p -d nbit tfilters.h5
# scaleoffset
TOOLTESTC 4 tscaleoffset.ddl --enable-error-stack -H -p -d scaleoffset tfilters.h5
# all

6
tools/test/h5stat/expected/h5stat_filters-F.ddl

@ -4,12 +4,12 @@ File space information for file metadata (in bytes):
Superblock extension: 0
User block: 0
Object headers: (total/unused)
Groups: 48/8
Datasets(exclude compact data): 4136/1344
Groups: 40/0
Datasets(exclude compact data): 4128/1088
Datatypes: 80/0
Groups:
B-tree/List: 1200
Heap: 288
Heap: 384
Attributes:
B-tree/List: 0
Heap: 0

2
tools/test/h5stat/expected/h5stat_filters-UD.ddl

@ -1,5 +1,5 @@
Filename: h5stat_filters.h5
File space information for datasets' metadata (in bytes):
Object headers (total/unused): 4136/1344
Object headers (total/unused): 4128/1088
Index for Chunked datasets: 31392
Heap: 72

2
tools/test/h5stat/expected/h5stat_filters-d.ddl

@ -12,7 +12,7 @@ Dataset dimension information:
# of datasets with dimension size 100 - 999: 1
Total # of datasets: 1
Dataset storage information:
Total raw data size: 8659
Total raw data size: 9046
Total external raw data size: 400
Dataset layout information:
Dataset layout counts[COMPACT]: 1

2
tools/test/h5stat/expected/h5stat_filters-dT.ddl

@ -12,7 +12,7 @@ Dataset dimension information:
# of datasets with dimension size 100 - 999: 1
Total # of datasets: 1
Dataset storage information:
Total raw data size: 8659
Total raw data size: 9046
Total external raw data size: 400
Dataset layout information:
Dataset layout counts[COMPACT]: 1

16
tools/test/h5stat/expected/h5stat_filters.ddl

@ -12,12 +12,12 @@ File space information for file metadata (in bytes):
Superblock extension: 0
User block: 0
Object headers: (total/unused)
Groups: 48/8
Datasets(exclude compact data): 4136/1344
Groups: 40/0
Datasets(exclude compact data): 4128/1088
Datatypes: 80/0
Groups:
B-tree/List: 1200
Heap: 288
Heap: 384
Attributes:
B-tree/List: 0
Heap: 0
@ -50,7 +50,7 @@ Dataset dimension information:
# of datasets with dimension size 100 - 999: 1
Total # of datasets: 1
Dataset storage information:
Total raw data size: 8659
Total raw data size: 9046
Total external raw data size: 400
Dataset layout information:
Dataset layout counts[COMPACT]: 1
@ -91,9 +91,9 @@ Free-space section bins:
File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR
File space page size: 4096 bytes
Summary of file space information:
File metadata: 37312 bytes
Raw data: 8659 bytes
File metadata: 37392 bytes
Raw data: 9046 bytes
Amount/Percent of tracked free space: 0 bytes/0.0%
Unaccounted space: 301 bytes
Total space: 46272 bytes
Unaccounted space: 258 bytes
Total space: 46696 bytes
External raw data: 400 bytes

BIN
tools/test/h5stat/testfiles/h5stat_filters.h5

Binary file not shown.