mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-03-25 17:00:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Bring over support for retrying metadata cache entry loads, along with all the

Browse Source 
supporting metadata cache callback changes, etc.
This commit is contained in:
Quincey Koziol 2016-11-20 04:24:57 -08:00
parent f6ad126673
commit 3b0c2b24da
42 changed files with 3317 additions and 1515 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
java/src/hdf/hdf5lib/H5.java
View File

@ -2854,6 +2854,7 @@ public class H5 implements java.io.Serializable {
// /////// unimplemented ////////
// ssize_t H5Fget_file_image(hid_t file_id, void * buf_ptr, size_t buf_len);
// herr_t H5Fget_metadata_read_retry_info(hid_t file_id, H5F_retry_info_t *info);
// ssize_t H5Fget_free_sections(hid_t file_id, H5F_mem_t type, size_t nsects, H5F_sect_info_t *sect_info/*out*/);
// /**
@ -5672,6 +5673,41 @@ public class H5 implements java.io.Serializable {
public synchronized static native void H5Pset_elink_file_cache_size(long fapl_id, int efc_size)
throws HDF5LibraryException;
/**
/**
* H5Pget_metadata_read_attempts retrieves the number of read attempts that is set in the file access property list plist_id.
*
* @param plist_id
* IN: File access property list identifier
*
* @return The number of read attempts.
*
* @exception HDF5LibraryException
* - Error from the HDF-5 Library.
*
**/
public synchronized static native long H5Pget_metadata_read_attempts(long plist_id) throws HDF5LibraryException;
/**
* H5Pset_metadata_read_attempts sets the number of reads that the library will try when reading checksummed
* metadata in an HDF5 file opened with SWMR access. When reading such metadata, the library will compare the
* checksum computed for the metadata just read with the checksum stored within the piece of checksum. When
* performing SWMR operations on a file, the checksum check might fail when the library reads data on a system
* that is not atomic. To remedy such situations, the library will repeatedly read the piece of metadata until
* the check passes or finally fails the read when the allowed number of attempts is reached.
*
* @param plist_id
* IN: File access property list identifier
* @param attempts
* IN: The number of read attempts which is a value greater than 0.
*
* @exception HDF5LibraryException
* - Error from the HDF-5 Library.
*
**/
public synchronized static native void H5Pset_metadata_read_attempts(long plist_id, long attempts)
throws HDF5LibraryException;
// Dataset creation property list (DCPL) routines //
/**

34
java/src/jni/h5pImp.c
View File

@ -5542,6 +5542,40 @@ Java_hdf_hdf5lib_H5_H5Piterate
return (jint)status;
} /* end Java_hdf_hdf5lib_H5_H5Piterate */
/*
* Class: hdf_hdf5lib_H5
* Method: H5Pget_metadata_read_attempts
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL
Java_hdf_hdf5lib_H5_H5Pget_1metadata_1read_1attempts
(JNIEnv *env, jclass clss, jlong plist_id)
{
unsigned attempts;
if (H5Pget_metadata_read_attempts((hid_t)plist_id, &attempts) < 0)
h5libraryError(env);
return (jlong) attempts;
} /* end Java_hdf_hdf5lib_H5_H5Pget_1metadata_1read_1attempts */
/*
* Class: hdf_hdf5lib_H5
* Method: H5Pset_metadata_read_attempts
* Signature: (JJ)V
*/
JNIEXPORT void JNICALL
Java_hdf_hdf5lib_H5_H5Pset_1metadata_1read_1attempts
(JNIEnv *env, jclass clss, jlong plist_id, jlong attempts)
{
if (attempts <= 0) {
h5badArgument(env, "H5Pset_metadata_read_attempts: attempts <= 0");
} /* end if */
else {
if(H5Pset_metadata_read_attempts((hid_t)plist_id, (unsigned)attempts) < 0)
h5libraryError(env);
} /* end else */
} /* end Java_hdf_hdf5lib_H5_H5Pset_1metadata_1read_1attempts */
#ifdef __cplusplus
} /* end extern "C" */
#endif /* __cplusplus */

19
java/src/jni/h5pImp.h
View File

@ -1473,6 +1473,25 @@ JNIEXPORT jint JNICALL
Java_hdf_hdf5lib_H5_H5Piterate
(JNIEnv*, jclass, jlong, jintArray, jobject, jobject);
/*
* Class: hdf_hdf5lib_H5
* Method: H5Pget_metadata_read_attempts
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL
Java_hdf_hdf5lib_H5_H5Pget_1metadata_1read_1attempts
(JNIEnv *, jclass, jlong);
/*
* Class: hdf_hdf5lib_H5
* Method: H5Pset_metadata_read_attempts
* Signature: (JJ)V
*/
JNIEXPORT void JNICALL
Java_hdf_hdf5lib_H5_H5Pset_1metadata_1read_1attempts
(JNIEnv *, jclass, jlong, jlong);
#ifdef __cplusplus
} /* end extern "C" */

4
src/H5ACprivate.h
View File

@ -175,7 +175,9 @@ typedef H5C_notify_action_t H5AC_notify_action_t;
#define H5AC__CLASS_SKIP_READS H5C__CLASS_SKIP_READS
#define H5AC__CLASS_SKIP_WRITES H5C__CLASS_SKIP_WRITES
typedef H5C_get_load_size_func_t H5AC_get_load_size_func_t;
typedef H5C_get_initial_load_size_func_t H5AC_get_initial_load_size_func_t;
typedef H5C_get_final_load_size_func_t H5AC_get_final_load_size_func_t;
typedef H5C_verify_chksum_func_t H5AC_verify_chksum_func_t;
typedef H5C_deserialize_func_t H5AC_deserialize_func_t;
typedef H5C_image_len_func_t H5AC_image_len_func_t;

192
src/H5B2cache.c
View File

@ -65,7 +65,8 @@
/********************/
/* Metadata cache callbacks */
static herr_t H5B2__cache_hdr_get_load_size(const void *udata, size_t *image_len);
static herr_t H5B2__cache_hdr_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5B2__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata);
static void *H5B2__cache_hdr_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5B2__cache_hdr_image_len(const void *thing, size_t *image_len);
@ -73,7 +74,8 @@ static herr_t H5B2__cache_hdr_serialize(const H5F_t *f, void *image, size_t len,
void *thing);
static herr_t H5B2__cache_hdr_free_icr(void *thing);
static herr_t H5B2__cache_int_get_load_size(const void *udata, size_t *image_len);
static herr_t H5B2__cache_int_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5B2__cache_int_verify_chksum(const void *image_ptr, size_t len, void *udata);
static void *H5B2__cache_int_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5B2__cache_int_image_len(const void *thing, size_t *image_len);
@ -81,7 +83,8 @@ static herr_t H5B2__cache_int_serialize(const H5F_t *f, void *image, size_t len,
void *thing);
static herr_t H5B2__cache_int_free_icr(void *thing);
static herr_t H5B2__cache_leaf_get_load_size(const void *udata, size_t *image_len);
static herr_t H5B2__cache_leaf_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5B2__cache_leaf_verify_chksum(const void *image_ptr, size_t len, void *udata);
static void *H5B2__cache_leaf_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5B2__cache_leaf_image_len(const void *thing, size_t *image_len);
@ -99,7 +102,9 @@ const H5AC_class_t H5AC_BT2_HDR[1] = {{
"v2 B-tree header", /* Metadata client name (for debugging) */
H5FD_MEM_BTREE, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5B2__cache_hdr_get_load_size, /* 'get_load_size' callback */
H5B2__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5B2__cache_hdr_verify_chksum, /* 'verify_chksum' callback */
H5B2__cache_hdr_deserialize, /* 'deserialize' callback */
H5B2__cache_hdr_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -115,7 +120,9 @@ const H5AC_class_t H5AC_BT2_INT[1] = {{
"v2 B-tree internal node", /* Metadata client name (for debugging) */
H5FD_MEM_BTREE, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5B2__cache_int_get_load_size, /* 'get_load_size' callback */
H5B2__cache_int_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5B2__cache_int_verify_chksum, /* 'verify_chksum' callback */
H5B2__cache_int_deserialize, /* 'deserialize' callback */
H5B2__cache_int_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -131,7 +138,9 @@ const H5AC_class_t H5AC_BT2_LEAF[1] = {{
"v2 B-tree leaf node", /* Metadata client name (for debugging) */
H5FD_MEM_BTREE, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5B2__cache_leaf_get_load_size, /* 'get_load_size' callback */
H5B2__cache_leaf_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5B2__cache_leaf_verify_chksum, /* 'verify_chksum' callback */
H5B2__cache_leaf_deserialize, /* 'deserialize' callback */
H5B2__cache_leaf_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -154,7 +163,7 @@ const H5AC_class_t H5AC_BT2_LEAF[1] = {{
/*-------------------------------------------------------------------------
* Function: H5B2__cache_hdr_get_load_size
* Function: H5B2__cache_hdr_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -167,21 +176,58 @@ const H5AC_class_t H5AC_BT2_LEAF[1] = {{
*-------------------------------------------------------------------------
*/
static herr_t
H5B2__cache_hdr_get_load_size(const void *_udata, size_t *image_len)
H5B2__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5B2_hdr_cache_ud_t *udata = (const H5B2_hdr_cache_ud_t *)_udata; /* User data for callback */
H5B2_hdr_cache_ud_t *udata = (H5B2_hdr_cache_ud_t *)_udata; /* User data for callback */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(udata);
HDassert(udata->f);
HDassert(image_len);
/* Set the image length size */
*image_len = H5B2_HEADER_SIZE_FILE(udata->f);
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5B2__cache_hdr_get_load_size() */
} /* end H5B2__cache_hdr_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5B2__cache_hdr_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
static htri_t
H5B2__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5B2__cache_hdr_verify_chksum() */
/*-------------------------------------------------------------------------
@ -208,7 +254,6 @@ H5B2__cache_hdr_deserialize(const void *_image, size_t H5_ATTR_UNUSED len,
H5B2_subid_t id; /* ID of B-tree class, as found in file */
uint16_t depth; /* Depth of B-tree */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5B2_hdr_t *ret_value = NULL; /* Return value */
@ -254,19 +299,14 @@ H5B2__cache_hdr_deserialize(const void *_image, size_t H5_ATTR_UNUSED len,
UINT16DECODE(image, hdr->root.node_nrec);
H5F_DECODE_LENGTH(udata->f, image, hdr->root.all_nrec);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == hdr->hdr_size);
/* Compute checksum on entire header */
computed_chksum = H5_checksum_metadata(_image, (hdr->hdr_size - H5B2_SIZEOF_CHKSUM), 0);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 B-tree header")
/* Initialize B-tree header info */
cparam.cls = H5B2_client_class_g[id];
if(H5B2__hdr_init(hdr, &cparam, udata->ctx_udata, depth) < 0)
@ -426,7 +466,7 @@ done:
/*-------------------------------------------------------------------------
* Function: H5B2__cache_int_get_load_size
* Function: H5B2__cache_int_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -439,9 +479,9 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5B2__cache_int_get_load_size(const void *_udata, size_t *image_len)
H5B2__cache_int_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5B2_internal_cache_ud_t *udata = (const H5B2_internal_cache_ud_t *)_udata; /* User data for callback */
H5B2_internal_cache_ud_t *udata = (H5B2_internal_cache_ud_t *)_udata; /* User data for callback */
FUNC_ENTER_STATIC_NOERR
@ -454,7 +494,49 @@ H5B2__cache_int_get_load_size(const void *_udata, size_t *image_len)
*image_len = udata->hdr->node_size;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5B2__cache_int_get_load_size() */
} /* end H5B2__cache_int_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5B2__cache_int_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
static htri_t
H5B2__cache_int_verify_chksum(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5B2_internal_cache_ud_t *udata = (H5B2_internal_cache_ud_t *)_udata; /* Pointer to user data */
size_t chk_size; /* Exact size of the node with checksum at the end */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
HDassert(udata);
/* Internal node prefix header + records + child pointer triplets: size with checksum at the end */
chk_size = H5B2_INT_PREFIX_SIZE + (udata->nrec * udata->hdr->rrec_size) + ((size_t)(udata->nrec + 1) * H5B2_INT_POINTER_SIZE(udata->hdr, udata->depth));
/* Get stored and computed checksums */
H5F_get_checksums(image, chk_size, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5B2__cache_int_verify_chksum() */
/*-------------------------------------------------------------------------
@ -481,7 +563,6 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len,
uint8_t *native; /* Pointer to native record info */
H5B2_node_ptr_t *int_node_ptr; /* Pointer to node pointer info */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
unsigned u; /* Local index variable */
H5B2_internal_t *ret_value = NULL; /* Return value */
@ -555,8 +636,7 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len,
int_node_ptr++;
} /* end for */
/* Compute checksum on internal node */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -564,10 +644,6 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len,
/* Sanity check parsing */
HDassert((size_t)(image - (const uint8_t *)_image) <= len);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 internal node")
/* Set return value */
ret_value = internal;
@ -733,7 +809,7 @@ done:
/*-------------------------------------------------------------------------
* Function: H5B2__cache_leaf_get_load_size
* Function: H5B2__cache_leaf_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -746,9 +822,9 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5B2__cache_leaf_get_load_size(const void *_udata, size_t *image_len)
H5B2__cache_leaf_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5B2_leaf_cache_ud_t *udata = (const H5B2_leaf_cache_ud_t *)_udata; /* User data for callback */
H5B2_leaf_cache_ud_t *udata = (H5B2_leaf_cache_ud_t *)_udata; /* User data for callback */
FUNC_ENTER_STATIC_NOERR
@ -761,7 +837,49 @@ H5B2__cache_leaf_get_load_size(const void *_udata, size_t *image_len)
*image_len = udata->hdr->node_size;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5B2__cache_leaf_get_load_size() */
} /* end H5B2__cache_leaf_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5B2__cache_leaf_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
static htri_t
H5B2__cache_leaf_verify_chksum(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5B2_internal_cache_ud_t *udata = (H5B2_internal_cache_ud_t *)_udata; /* Pointer to user data */
size_t chk_size; /* Exact size of the node with checksum at the end */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
HDassert(udata);
/* Leaf node prefix header + records: size with checksum at the end */
chk_size = H5B2_LEAF_PREFIX_SIZE + (udata->nrec * udata->hdr->rrec_size);
/* Get stored and computed checksums */
H5F_get_checksums(image, chk_size, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5B2__cache_leaf_verify_chksum() */
/*-------------------------------------------------------------------------
@ -787,7 +905,6 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len,
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint8_t *native; /* Pointer to native keys */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
unsigned u; /* Local index variable */
H5B2_leaf_t *ret_value = NULL; /* Return value */
@ -841,8 +958,7 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len,
native += udata->hdr->cls->nrec_size;
} /* end for */
/* Compute checksum on leaf node */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -850,10 +966,6 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len,
/* Sanity check parsing */
HDassert((size_t)(image - (const uint8_t *)_image) <= udata->hdr->node_size);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 leaf node")
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) <= len);

54
src/H5Bcache.c
View File

@ -54,13 +54,13 @@
/********************/
/* Metadata cache callbacks */
static herr_t H5B__get_load_size(const void *udata, size_t *image_len);
static void *H5B__deserialize(const void *image, size_t len, void *udata,
static herr_t H5B__cache_get_initial_load_size(void *udata, size_t *image_len);
static void *H5B__cache_deserialize(const void *image, size_t len, void *udata,
hbool_t *dirty);
static herr_t H5B__image_len(const void *thing, size_t *image_len);
static herr_t H5B__serialize(const H5F_t *f, void *image, size_t len,
static herr_t H5B__cache_image_len(const void *thing, size_t *image_len);
static herr_t H5B__cache_serialize(const H5F_t *f, void *image, size_t len,
void *thing);
static herr_t H5B__free_icr(void *thing);
static herr_t H5B__cache_free_icr(void *thing);
/*********************/
@ -73,13 +73,15 @@ const H5AC_class_t H5AC_BT[1] = {{
"v1 B-tree", /* Metadata client name (for debugging) */
H5FD_MEM_BTREE, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5B__get_load_size, /* 'get_load_size' callback */
H5B__deserialize, /* 'deserialize' callback */
H5B__image_len, /* 'image_len' callback */
H5B__cache_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
NULL, /* 'verify_chksum' callback */
H5B__cache_deserialize, /* 'deserialize' callback */
H5B__cache_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
H5B__serialize, /* 'serialize' callback */
H5B__cache_serialize, /* 'serialize' callback */
NULL, /* 'notify' callback */
H5B__free_icr, /* 'free_icr' callback */
H5B__cache_free_icr, /* 'free_icr' callback */
NULL, /* 'fsf_size' callback */
}};
@ -90,7 +92,7 @@ const H5AC_class_t H5AC_BT[1] = {{
/*-------------------------------------------------------------------------
* Function: H5B__get_load_size
* Function: H5B__cache_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -103,9 +105,9 @@ const H5AC_class_t H5AC_BT[1] = {{
*-------------------------------------------------------------------------
*/
static herr_t
H5B__get_load_size(const void *_udata, size_t *image_len)
H5B__cache_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5B_cache_ud_t *udata = (const H5B_cache_ud_t *)_udata; /* User data for callback */
H5B_cache_ud_t *udata = (H5B_cache_ud_t *)_udata; /* User data for callback */
H5B_shared_t *shared; /* Pointer to shared B-tree info */
FUNC_ENTER_STATIC_NOERR
@ -122,11 +124,11 @@ H5B__get_load_size(const void *_udata, size_t *image_len)
*image_len = shared->sizeof_rnode;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5B__get_load_size() */
} /* end H5B__cache_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5B__deserialize
* Function: H5B__cache_deserialize
*
* Purpose: Deserialize the data structure from disk.
*
@ -140,7 +142,7 @@ H5B__get_load_size(const void *_udata, size_t *image_len)
*-------------------------------------------------------------------------
*/
static void *
H5B__deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata,
H5B__cache_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata,
hbool_t H5_ATTR_UNUSED *dirty)
{
H5B_t *bt = NULL; /* Pointer to the deserialized B-tree node */
@ -229,11 +231,11 @@ done:
HDONE_ERROR(H5E_BTREE, H5E_CANTFREE, NULL, "unable to destroy B-tree node")
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5B__deserialize() */
} /* end H5B__cache_deserialize() */
/*-------------------------------------------------------------------------
* Function: H5B__image_len
* Function: H5B__cache_image_len
*
* Purpose: Compute the size of the data structure on disk.
*
@ -246,7 +248,7 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5B__image_len(const void *_thing, size_t *image_len)
H5B__cache_image_len(const void *_thing, size_t *image_len)
{
const H5B_t *bt = (const H5B_t *)_thing; /* Pointer to the B-tree node */
H5B_shared_t *shared; /* Pointer to shared B-tree info */
@ -265,11 +267,11 @@ H5B__image_len(const void *_thing, size_t *image_len)
*image_len = shared->sizeof_rnode;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5B__image_len() */
} /* end H5B__cache_image_len() */
/*-------------------------------------------------------------------------
* Function: H5B__serialize
* Function: H5B__cache_serialize
*
* Purpose: Serialize the data structure for writing to disk.
*
@ -282,7 +284,7 @@ H5B__image_len(const void *_thing, size_t *image_len)
*-------------------------------------------------------------------------
*/
static herr_t
H5B__serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len,
H5B__cache_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len,
void *_thing)
{
H5B_t *bt = (H5B_t *)_thing; /* Pointer to the B-tree node */
@ -351,11 +353,11 @@ H5B__serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len,
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5B__serialize() */
} /* end H5B__cache_serialize() */
/*-------------------------------------------------------------------------
* Function: H5B__free_icr
* Function: H5B__cache_free_icr
*
* Purpose: Destroy/release an "in core representation" of a data structure
*
@ -368,7 +370,7 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5B__free_icr(void *thing)
H5B__cache_free_icr(void *thing)
{
herr_t ret_value = SUCCEED; /* Return value */
@ -383,5 +385,5 @@ H5B__free_icr(void *thing)
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5B__free_icr() */
} /* end H5B__cache_free_icr() */

395
src/H5C.c
View File

@ -167,6 +167,9 @@ static herr_t H5C__mark_flush_dep_dirty(H5C_cache_entry_t * entry);
static herr_t H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry);
static herr_t H5C__verify_len_eoa(H5F_t *f, const H5C_class_t * type,
haddr_t addr, size_t *len, hbool_t actual);
static herr_t H5C__generate_image(const H5F_t *f, H5C_t * cache_ptr, H5C_cache_entry_t *entry_ptr,
hid_t dxpl_id);
@ -6133,6 +6136,70 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5C__flush_single_entry() */
/*-------------------------------------------------------------------------
*
* Function: H5C__verify_len_eoa
*
* Purpose: Verify that 'len' does not exceed eoa when 'actual' is
* false i.e. 'len" is the initial speculative length from
* get_load_size callback with null image pointer.
* If exceed, adjust 'len' accordingly.
*
* Verify that 'len' should not exceed eoa when 'actual' is
* true i.e. 'len' is the actual length from get_load_size
* callback with non-null image pointer.
* If exceed, return error.
*
* Return: FAIL if error is detected, SUCCEED otherwise.
*
* Programmer: Vailin Choi
* 9/6/15
*
*-------------------------------------------------------------------------
*/
static herr_t
H5C__verify_len_eoa(H5F_t *f, const H5C_class_t *type, haddr_t addr,
size_t *len, hbool_t actual)
{
H5FD_mem_t cooked_type; /* Modified type, accounting for switching global heaps */
haddr_t eoa; /* End-of-allocation in the file */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* if type == H5FD_MEM_GHEAP, H5F_block_read() forces
* type to H5FD_MEM_DRAW via its call to H5F__accum_read().
* Thus we do the same for purposes of computing the EOA
* for sanity checks.
*/
cooked_type = (type->mem_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type->mem_type;
/* Get the file's end-of-allocation value */
eoa = H5F_get_eoa(f, cooked_type);
if(!H5F_addr_defined(eoa))
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "invalid EOA address for file")
/* Check for bad address in general */
if(H5F_addr_gt(addr, eoa))
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "address of object past end of allocation")
/* Check if the amount of data to read will be past the EOA */
if(H5F_addr_gt((addr + *len), eoa)) {
if(actual)
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "actual len exceeds EOA.")
else
/* Trim down the length of the metadata */
*len = (size_t)(eoa - addr);
} /* end if */
if(*len <= 0)
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "len not positive after adjustment for EOA.")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5C__verify_len_eoa() */
/*-------------------------------------------------------------------------
*
@ -6163,7 +6230,7 @@ H5C_load_entry(H5F_t * f,
void * udata)
{
hbool_t dirty = FALSE; /* Flag indicating whether thing was dirtied during deserialize */
void * image = NULL; /* Buffer for disk image */
uint8_t * image = NULL; /* Buffer for disk image */
void * thing = NULL; /* Pointer to thing loaded */
H5C_cache_entry_t *entry = NULL; /* Alias for thing loaded, as cache entry */
size_t len; /* Size of image in file */
@ -6176,10 +6243,18 @@ H5C_load_entry(H5F_t * f,
FUNC_ENTER_NOAPI_NOINIT
/* Sanity checks */
HDassert(f);
HDassert(f->shared);
HDassert(f->shared->cache);
HDassert(type);
HDassert(H5F_addr_defined(addr));
HDassert(type->get_initial_load_size);
if(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG)
HDassert(type->get_final_load_size);
else
HDassert(NULL == type->get_final_load_size);
HDassert(type->deserialize);
/* Can't see how skip reads could be usefully combined with
* the speculative read flag. Hence disallow.
@ -6187,59 +6262,19 @@ H5C_load_entry(H5F_t * f,
HDassert(!((type->flags & H5C__CLASS_SKIP_READS) &&
(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG)));
HDassert(H5F_addr_defined(addr));
HDassert(type->get_load_size);
HDassert(type->deserialize);
/* Call the get_load_size callback, to retrieve the initial
* size of image
*/
if(type->get_load_size(udata, &len) < 0)
/* Call the get_initial_load_size callback, to retrieve the initial size of image */
if(type->get_initial_load_size(udata, &len) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "can't retrieve image size")
HDassert(len > 0);
/* Check for possible speculative read off the end of the file */
if(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) {
haddr_t eoa; /* End-of-allocation in the file */
H5FD_mem_t cooked_type;
/* if type == H5FD_MEM_GHEAP, H5F_block_read() forces
* type to H5FD_MEM_DRAW via its call to H5F__accum_read().
* Thus we do the same for purposes of computing the eoa
* for sanity checks.
*/
cooked_type = (type->mem_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type->mem_type;
/* Get the file's end-of-allocation value */
eoa = H5F_get_eoa(f, cooked_type);
HDassert(H5F_addr_defined(eoa));
/* Check for bad address in general */
if ( H5F_addr_gt(addr, eoa) )
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "address of object past end of allocation")
/* Check if the amount of data to read will be past the eoa */
if( H5F_addr_gt((addr + len), eoa) ) {
/* Trim down the length of the metadata */
/* Note that for some cache clients, this will cause an
* assertion failure. JRM -- 8/29/14
*/
len = (size_t)(eoa - addr);
} /* end if */
if ( len <= 0 )
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "len not positive after adjustment for EOA.")
} /* end if */
if(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG)
if(H5C__verify_len_eoa(f, type, addr, &len, FALSE) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "invalid len with respect to EOA")
/* Allocate the buffer for reading the on-disk entry image */
if(NULL == (image = H5MM_malloc(len + H5C_IMAGE_EXTRA_SPACE)))
if(NULL == (image = (uint8_t *)H5MM_malloc(len + H5C_IMAGE_EXTRA_SPACE)))
HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "memory allocation failed for on disk image buffer.")
#if H5C_DO_MEMORY_SANITY_CHECKS
HDmemcpy(image + len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE);
#endif /* H5C_DO_MEMORY_SANITY_CHECKS */
@ -6255,151 +6290,145 @@ H5C_load_entry(H5F_t * f,
/* Get the on-disk entry image */
if(0 == (type->flags & H5C__CLASS_SKIP_READS)) {
#ifdef H5_HAVE_PARALLEL
if(!coll_access || 0 == mpi_rank) {
#endif /* H5_HAVE_PARALLEL */
unsigned tries, max_tries; /* The # of read attempts */
unsigned retries; /* The # of retries */
htri_t chk_ret; /* return from verify_chksum callback */
size_t actual_len = len; /* The actual length, after speculative reads have been resolved */
uint64_t nanosec = 1; /* # of nanoseconds to sleep between retries */
void *new_image; /* Pointer to image */
hbool_t len_changed = TRUE; /* Whether to re-check speculative entries */
if(H5F_block_read(f, type->mem_type, addr, len, dxpl_id, image) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_READERROR, NULL, "Can't read image*")
/* Get the # of read attempts */
max_tries = tries = H5F_GET_READ_ATTEMPTS(f);
#ifdef H5_HAVE_PARALLEL
} /* end if */
/* if the collective metadata read optimization is turned on,
* bcast the metadata read from process 0 to all ranks in the file
* communicator
/*
* This do/while loop performs the following till the metadata checksum
* is correct or the file's number of allowed read attempts are reached.
* --read the metadata
* --determine the actual size of the metadata
* --perform checksum verification
*/
if(coll_access) {
int buf_size;
do {
if(actual_len != len) {
if(NULL == (new_image = H5MM_realloc(image, len + H5C_IMAGE_EXTRA_SPACE)))
HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "image null after H5MM_realloc()")
image = (uint8_t *)new_image;
#if H5C_DO_MEMORY_SANITY_CHECKS
HDmemcpy(image + len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE);
#endif /* H5C_DO_MEMORY_SANITY_CHECKS */
} /* end if */
H5_CHECKED_ASSIGN(buf_size, int, len, size_t);
if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image, buf_size, MPI_BYTE, 0, comm)))
HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code)
} /* end if */
#ifdef H5_HAVE_PARALLEL
if(!coll_access || 0 == mpi_rank) {
#endif /* H5_HAVE_PARALLEL */
} /* end if */
if(H5F_block_read(f, type->mem_type, addr, len, dxpl_id, image) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_READERROR, NULL, "Can't read image*")
#ifdef H5_HAVE_PARALLEL
} /* end if */
/* if the collective metadata read optimization is turned on,
* bcast the metadata read from process 0 to all ranks in the file
* communicator
*/
if(coll_access) {
int buf_size;
H5_CHECKED_ASSIGN(buf_size, int, len, size_t);
if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image, buf_size, MPI_BYTE, 0, comm)))
HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code)
} /* end if */
#endif /* H5_HAVE_PARALLEL */
/* If the entry could be read speculatively and the length is still
* changing, check for updating the actual size
*/
if((type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) && len_changed) {
/* Retrieve the actual length */
actual_len = len;
if(type->get_final_load_size(image, len, udata, &actual_len) < 0)
continue; /* Transfer control to while() and count towards retries */
/* Check for the length changing */
if(actual_len != len) {
/* Verify that the length isn't past the EOA for the file */
if(H5C__verify_len_eoa(f, type, addr, &actual_len, TRUE) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "actual_len exceeds EOA.")
/* Expand buffer to new size */
if(NULL == (new_image = H5MM_realloc(image, actual_len + H5C_IMAGE_EXTRA_SPACE)))
HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "image null after H5MM_realloc()")
image = (uint8_t *)new_image;
#if H5C_DO_MEMORY_SANITY_CHECKS
HDmemcpy(image + actual_len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE);
#endif /* H5C_DO_MEMORY_SANITY_CHECKS */
if(actual_len > len) {
#ifdef H5_HAVE_PARALLEL
if(!coll_access || 0 == mpi_rank) {
#endif /* H5_HAVE_PARALLEL */
/* If the thing's image needs to be bigger for a speculatively
* loaded thing, go get the on-disk image again (the extra portion).
*/
if(H5F_block_read(f, type->mem_type, addr + len, actual_len - len, dxpl_id, image + len) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "can't read image")
#ifdef H5_HAVE_PARALLEL
}
/* If the collective metadata read optimization is turned on,
* Bcast the metadata read from process 0 to all ranks in the file
* communicator */
if(coll_access) {
int buf_size;
H5_CHECKED_ASSIGN(buf_size, int, actual_len - len, size_t);
if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image + len, buf_size, MPI_BYTE, 0, comm)))
HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code)
} /* end if */
#endif /* H5_HAVE_PARALLEL */
} /* end if */
} /* end if (actual_len != len) */
else {
/* The length has stabilized */
len_changed = FALSE;
/* Set the final length */
len = actual_len;
} /* else */
} /* end if */
/* If there's no way to verify the checksum for a piece of metadata
* (usually because there's no checksum in the file), leave now
*/
if(type->verify_chksum == NULL)
break;
/* Verify the checksum for the metadata image */
if((chk_ret = type->verify_chksum(image, actual_len, udata)) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "failure from verify_chksum callback")
if(chk_ret == TRUE)
break;
/* Sleep for some time */
H5_nanosleep(nanosec);
nanosec *= 2; /* Double the sleep time next time */
} while(--tries);
/* Check for too many tries */
if(tries == 0)
HGOTO_ERROR(H5E_CACHE, H5E_READERROR, NULL, "incorrect metadatda checksum after all read attempts")
/* Calculate and track the # of retries */
retries = max_tries - tries;
if(retries) /* Does not track 0 retry */
if(H5F_track_metadata_read_retries(f, (unsigned)type->mem_type, retries) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "cannot track read tries = %u ", retries)
/* Set the final length (in case it wasn't set earlier) */
len = actual_len;
} /* end if !H5C__CLASS_SKIP_READS */
/* Deserialize the on-disk image into the native memory form */
if(NULL == (thing = type->deserialize(image, len, udata, &dirty)))
HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't deserialize image")
/* If the client's cache has an image_len callback, check it */
if(type->image_len) {
size_t new_len; /* New size of on-disk image */
/* set magic and type field in *entry_ptr. While the image_len
* callback shouldn't touch the cache specific fields, it may check
* these fields to ensure that it it has received the expected
* value.
*
* Note that this initialization is repeated below on the off
* chance that we had to re-try the deserialization.
*/
entry = (H5C_cache_entry_t *)thing;
entry->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC;
entry->type = type;
/* Get the actual image size for the thing */
if(type->image_len(thing, &new_len) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "can't retrieve image length")
if(new_len == 0)
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "image length is 0")
if(new_len != len) {
if (type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) {
void *new_image; /* Buffer for disk image */
/* Adjust the size of the image to match new_len */
if(NULL == (new_image = H5MM_realloc(image,
new_len + H5C_IMAGE_EXTRA_SPACE)))
HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "image null after H5MM_realloc()")
image = new_image;
#if H5C_DO_MEMORY_SANITY_CHECKS
HDmemcpy(((uint8_t *)image) + new_len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE);
#endif /* H5C_DO_MEMORY_SANITY_CHECKS */
/* If the thing's image needs to be bigger for a speculatively
* loaded thing, free the thing and retry with new length
*/
if (new_len > len) {
/* Release previous (possibly partially initialized)
* thing. Note that we must set entry->magic to
* H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC and set one or
* two other fields before the call to free_icr
* so as to avoid sanity check failures.
*/
entry->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC;
entry->addr = addr;
if ( type->free_icr(thing) < 0 )
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "free_icr callback failed")
#ifdef H5_HAVE_PARALLEL
if(!coll_access || 0 == mpi_rank) {
#endif /* H5_HAVE_PARALLEL */
/* Go get the on-disk image again */
if(H5F_block_read(f, type->mem_type, addr,
new_len, dxpl_id, image) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't read image")
#ifdef H5_HAVE_PARALLEL
}
/* if the collective metadata read optimization is turned on,
bcast the metadata read from process 0 to all ranks in the file
communicator */
if(coll_access) {
int buf_size;
H5_CHECKED_ASSIGN(buf_size, int, new_len, size_t);
if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image, buf_size, MPI_BYTE, 0, comm)))
HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code)
} /* end if */
#endif /* H5_HAVE_PARALLEL */
/* Deserialize on-disk image into native memory form again */
if(NULL == (thing = type->deserialize(image, new_len, udata, &dirty)))
HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't deserialize image")
#ifndef NDEBUG
{
size_t new_new_len;
/* Get the actual image size for the thing again. Note
* that since this is a new thing, we have to set
* the magic and type fields again so as to avoid
* failing sanity checks.
*/
entry = (H5C_cache_entry_t *)thing;
entry->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC;
entry->type = type;
type->image_len(thing, &new_new_len);
HDassert(new_new_len == new_len);
} /* end block */
#endif /* NDEBUG */
} /* end if (new_len > len) */
/* Retain adjusted size */
len = new_len;
} else { /* throw an error */
HGOTO_ERROR(H5E_CACHE, H5E_UNSUPPORTED, NULL, \
"size of non-speculative object changed")
}
} /* end if (new_len != len) */
} /* end if */
entry = (H5C_cache_entry_t *)thing;
/* In general, an entry should be clean just after it is loaded.
@ -6480,7 +6509,7 @@ done:
if(thing && type->free_icr(thing) < 0)
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "free_icr callback failed")
if(image)
image = H5MM_xfree(image);
image = (uint8_t *)H5MM_xfree(image);
} /* end if */
FUNC_LEAVE_NOAPI(ret_value)

39
src/H5Cepoch.c
View File

@ -63,8 +63,12 @@
* epochs so that they can be evicted from the cache.
*
****************************************************************************/
static herr_t H5C__epoch_marker_get_load_size(const void *udata_ptr,
static herr_t H5C__epoch_marker_get_initial_load_size(void *udata_ptr,
size_t *image_len_ptr);
static herr_t H5C__epoch_marker_get_final_load_size(const void *image_ptr,
size_t image_len_ptr, void *udata_ptr, size_t *actual_len);
static htri_t H5C__epoch_marker_verify_chksum(const void *image_ptr,
size_t len, void *udata_ptr);
static void * H5C__epoch_marker_deserialize(const void * image_ptr,
size_t len, void * udata, hbool_t * dirty_ptr);
static herr_t H5C__epoch_marker_image_len(const void * thing,
@ -101,7 +105,9 @@ const H5C_class_t H5C__epoch_marker_class =
/* name = */ "epoch marker",
/* mem_type = */ H5FD_MEM_DEFAULT, /* value doesn't matter */
/* flags = */ H5C__CLASS_NO_FLAGS_SET,
/* get_load_size = */ H5C__epoch_marker_get_load_size,
/* get_initial_load_size = */ H5C__epoch_marker_get_initial_load_size,
/* get_final_load_size = */ H5C__epoch_marker_get_final_load_size,
/* verify_chksum = */ H5C__epoch_marker_verify_chksum,
/* deserialize = */ H5C__epoch_marker_deserialize,
/* image_len = */ H5C__epoch_marker_image_len,
/* pre_serialize = */ H5C__epoch_marker_pre_serialize,
@ -123,7 +129,7 @@ const H5C_class_t H5C__epoch_marker_class =
static herr_t
H5C__epoch_marker_get_load_size(const void H5_ATTR_UNUSED *udata_ptr,
H5C__epoch_marker_get_initial_load_size(void H5_ATTR_UNUSED *udata_ptr,
size_t H5_ATTR_UNUSED *image_len_ptr)
{
FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */
@ -131,7 +137,32 @@ H5C__epoch_marker_get_load_size(const void H5_ATTR_UNUSED *udata_ptr,
HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn.");
FUNC_LEAVE_NOAPI(FAIL)
} /* end H5C__epoch_marker_get_load_size() */
} /* end H5C__epoch_marker_get_initial_load_size() */
static herr_t
H5C__epoch_marker_get_final_load_size(const void H5_ATTR_UNUSED *image_ptr,
size_t H5_ATTR_UNUSED image_len, void H5_ATTR_UNUSED *udata_ptr,
size_t H5_ATTR_UNUSED *actual_len)
{
FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */
HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn.");
FUNC_LEAVE_NOAPI(FAIL)
} /* end H5C__epoch_marker_final_get_load_size() */
static htri_t
H5C__epoch_marker_verify_chksum(const void H5_ATTR_UNUSED *image_ptr, size_t H5_ATTR_UNUSED len,
void H5_ATTR_UNUSED *udata_ptr)
{
FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */
HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn.");
FUNC_LEAVE_NOAPI(FALSE)
} /* end H5C__epoch_marker_verify_chksum() */
static void *

165
src/H5Cprivate.h
View File

@ -323,50 +323,124 @@ typedef struct H5C_t H5C_t;
* code. When it is set, writes of buffers prepared by the
* serialize callback will be skipped.
*
* GET_LOAD_SIZE: Pointer to the 'get load size' function.
* GET_INITIAL_LOAD_SIZE: Pointer to the 'get initial load size' function.
*
* This function must be able to determine the size of the disk image of
* a metadata cache entry, given the 'udata' that will be passed to the
* 'deserialize' callback.
* This function determines the size based on the information in the
* parameter "udata" or an initial speculative guess. The size is
* returned in the parameter "image_len_ptr".
*
* Note that if either the H5C__CLASS_SPECULATIVE_LOAD_FLAG is set, the disk image size
* returned by this callback is a first guess.
* In other cases, the value returned should be the correct
* size of the entry.
* For an entry with H5C__CLASS_NO_FLAGS_SET:
* This function returns in "image_len_ptr" the on disk size of the
* entry.
*
* For an entry with H5C__CLASS_SPECULATIVE_LOAD_FLAG:
* This function returns in "image_len_ptr" an initial guess of the
* entry's on disk size. This many bytes will be loaded from
* the file and then passed to 'get_final_load_size' callback
* for the actual (final) image length to be determined.
*
* The typedef for the deserialize callback is as follows:
* The typedef for the get_initial_load_size callback is as follows:
*
* typedef herr_t (*H5C_get_load_size_func_t)(void *udata_ptr,
* size_t *image_len_ptr);
* typedef herr_t (*H5C_get_initial_load_size_func_t)(void *udata_ptr,
* size_t *image_len_ptr);
*
* The parameters of the deserialize callback are as follows:
* The parameters of the get_initial_load_size callback are as follows:
*
* udata_ptr: Pointer to user data provided in the protect call, which
* will also be passed through to the deserialize callback.
* will also be passed through to the 'get_final_load_size',
* 'verify_chksum', and 'deserialize' callbacks.
*
* image_len_ptr: Pointer to the location in which the length in bytes
* of the in file image to be deserialized is to be returned.
* image_len_ptr: Pointer to the length in bytes of the in-file image to
* be deserialized is to be returned.
*
* This value is used by the cache to determine the size of
* the disk image for the metadata, in order to read the disk
* image from the file.
*
*
* Processing in the get_load_size function should proceed as follows:
*
* If successful, the function will place the length of the on disk
* image associated with supplied user data in *image_len_ptr, and
* then return SUCCEED.
* If successful, the function will place the length in the *image_len_ptr
* associated with supplied user data and then return SUCCEED.
*
* On failure, the function must return FAIL and push error information
* onto the error stack with the error API routines, without modifying
* the value pointed to by the image_len_ptr.
* the value pointed to by image_len_ptr.
*
*
* GET_FINAL_LOAD_SIZE: Pointer to the 'get final load size' function.
*
* This function determines the final size of a speculatively loaded
* metadata cache entry based on the parameter "image" and the "udata"
* parameters. This callback _must_ be implemented for cache clients
* which set the H5C__CLASS_SPECULATIVE_LOAD_FLAG and must return the
* actual length of on-disk image after being called once.
*
* This function might deserialize the needed metadata information to
* determine the actual size. The size is returned in the parameter
* "actual_len_ptr".
*
* The typedef for the get_load_size callback is as follows:
*
* typedef herr_t (*H5C_get_final_load_size_func_t)(const void *image_ptr,
* size_t image_len,
* void *udata_ptr,
* size_t *actual_len_ptr);
*
* The parameters of the get_load_size callback are as follows:
*
* image_ptr: Pointer to a buffer containing the (possibly partial)
* metadata read in.
*
* image_len: The length in bytes of the (possibly partial) in-file image
* to be queried for an actual length.
*
* udata_ptr: Pointer to user data provided in the protect call, which
* will also be passed through to the 'verify_chksum' and
* 'deserialize' callbacks.
*
* actual_len_ptr: Pointer to the location containing the actual length
* of the metadata entry on disk.
*
* Processing in the get_final_load_size function should proceed as follows:
*
* If successful, the function will place the length in the *actual_len_ptr
* associated with supplied image and/or user data and then return SUCCEED.
*
* On failure, the function must return FAIL and push error information
* onto the error stack with the error API routines, without modifying
* the value pointed to by actual_len_ptr.
*
*
* VERIFY_CHKSUM: Pointer to the verify_chksum function.
*
* This function verifies the checksum computed for the metadata is
* the same as the checksum stored in the metadata.
*
* It computes the checksum based on the metadata stored in the
* parameter "image_ptr" and the actual length of the metadata in the
* parameter "len" which is obtained from the "get_load_size" callback.
*
* The typedef for the verify_chksum callback is as follows:
*
* typedef htri_t (*H5C_verify_chksum_func_t)(const void *image_ptr,
* size_t len,
* void *udata_ptr);
*
* The parameters of the verify_chksum callback are as follows:
*
* image_ptr: Pointer to a buffer containing the metadata read in.
*
* len: The actual length of the metadata.
*
* udata_ptr: Pointer to user data.
*
*
* DESERIALIZE: Pointer to the deserialize function.
*
* This function must be able to read a buffer containing the on disk
* image of a metadata cache entry, allocate and load the equivalent
* in core representation, and return a pointer to that representation.
* This function must be able to deserialize a buffer containing the
* on-disk image of a metadata cache entry, allocate and initialize the
* equivalent in core representation, and return a pointer to that
* representation.
*
* The typedef for the deserialize callback is as follows:
*
@ -399,17 +473,17 @@ typedef struct H5C_t H5C_t;
* Processing in the deserialize function should proceed as follows:
*
* If the image contains valid data, and is of the correct length,
* the deserialize function must allocate space for an in core
* representation of that data, load the contents of the image into
* the space allocated for the in core representation, and return
* the deserialize function must allocate space for an in-core
* representation of that data, deserialize the contents of the image
* into the space allocated for the in-core representation, and return
* a pointer to the in core representation. Observe that an
* instance of H5C_cache_entry_t must be the first item in this
* representation. The cache will initialize it after the callback
* returns.
*
* Note that the structure of the in core representation is otherwise
* Note that the structure of the in-core representation is otherwise
* up to the cache client. All that is required is that the pointer
* returned be sufficient for the clients purposes when it is returned
* returned be sufficient for the client's purposes when it is returned
* on a protect call.
*
* If the deserialize function has to clean up file corruption
@ -421,30 +495,6 @@ typedef struct H5C_t H5C_t;
* buffer length, malloc failure, etc.) the function must return NULL and
* push error information on the error stack with the error API routines.
*
* Exceptions to the above:
*
* If the H5C__CLASS_SPECULATIVE_LOAD_FLAG is set, the buffer supplied
* to the function need not be currect on the first invocation of the
* callback in any single attempt to load the entry.
*
* In this case, if the buffer is larger than necessary, the function
* should load the entry as described above and not flag an error due
* to the oversized buffer. The cache will correct its mis-apprehension
* of the entry size with a subsequent call to the image_len callback.
*
* If the buffer is too small, and this is the first deserialize call
* in the entry load operation, the function should not flag an error.
* Instead, it must compute the correct size of the entry, allocate an
* in core representation and initialize it to the extent that an
* immediate call to the image len callback will return the correct
* image size.
*
* In this case, when the deserialize callback returns, the cache will
* call the image length callback, realize that the supplied buffer was
* too small, discard the returned in core representation, allocate
* and load a new buffer of the correct size from file, and then call
* the deserialize callback again.
*
*
* IMAGE_LEN: Pointer to the image length callback.
*
@ -473,6 +523,7 @@ typedef struct H5C_t H5C_t;
* onto the error stack with the error API routines, and return without
* modifying the values pointed to by the image_len_ptr parameter.
*
*
* PRE_SERIALIZE: Pointer to the pre-serialize callback.
*
* The pre-serialize callback is invoked by the metadata cache before
@ -818,8 +869,10 @@ typedef enum H5C_notify_action_t {
} H5C_notify_action_t;
/* Cache client callback function pointers */
typedef herr_t (*H5C_get_load_size_func_t)(const void *udata_ptr,
size_t *image_len_ptr);
typedef herr_t (*H5C_get_initial_load_size_func_t)(void *udata_ptr, size_t *image_len_ptr);
typedef herr_t (*H5C_get_final_load_size_func_t)(const void *image_ptr,
size_t image_len, void *udata_ptr, size_t *actual_len_ptr);
typedef htri_t (*H5C_verify_chksum_func_t)(const void *image_ptr, size_t len, void *udata_ptr);
typedef void *(*H5C_deserialize_func_t)(const void *image_ptr,
size_t len, void *udata_ptr, hbool_t *dirty_ptr);
typedef herr_t (*H5C_image_len_func_t)(const void *thing, size_t *image_len_ptr);
@ -838,7 +891,9 @@ typedef struct H5C_class_t {
const char * name;
H5FD_mem_t mem_type;
unsigned flags;
H5C_get_load_size_func_t get_load_size;
H5C_get_initial_load_size_func_t get_initial_load_size;
H5C_get_final_load_size_func_t get_final_load_size;
H5C_verify_chksum_func_t verify_chksum;
H5C_deserialize_func_t deserialize;
H5C_image_len_func_t image_len;
H5C_pre_serialize_func_t pre_serialize;

294
src/H5EAcache.c
View File

@ -73,7 +73,8 @@
/********************/
/* Metadata cache (H5AC) callbacks */
static herr_t H5EA__cache_hdr_get_load_size(const void *udata, size_t *image_len);
static herr_t H5EA__cache_hdr_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5EA__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5EA__cache_hdr_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5EA__cache_hdr_image_len(const void *thing, size_t *image_len);
@ -81,7 +82,8 @@ static herr_t H5EA__cache_hdr_serialize(const H5F_t *f, void *image, size_t len,
void *thing);
static herr_t H5EA__cache_hdr_free_icr(void *thing);
static herr_t H5EA__cache_iblock_get_load_size(const void *udata, size_t *image_len);
static herr_t H5EA__cache_iblock_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5EA__cache_iblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5EA__cache_iblock_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5EA__cache_iblock_image_len(const void *thing, size_t *image_len);
@ -90,7 +92,8 @@ static herr_t H5EA__cache_iblock_serialize(const H5F_t *f, void *image, size_t l
static herr_t H5EA__cache_iblock_notify(H5AC_notify_action_t action, void *thing);
static herr_t H5EA__cache_iblock_free_icr(void *thing);
static herr_t H5EA__cache_sblock_get_load_size(const void *udata, size_t *image_len);
static herr_t H5EA__cache_sblock_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5EA__cache_sblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5EA__cache_sblock_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5EA__cache_sblock_image_len(const void *thing, size_t *image_len);
@ -99,7 +102,8 @@ static herr_t H5EA__cache_sblock_serialize(const H5F_t *f, void *image, size_t l
static herr_t H5EA__cache_sblock_notify(H5AC_notify_action_t action, void *thing);
static herr_t H5EA__cache_sblock_free_icr(void *thing);
static herr_t H5EA__cache_dblock_get_load_size(const void *udata, size_t *image_len);
static herr_t H5EA__cache_dblock_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5EA__cache_dblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5EA__cache_dblock_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5EA__cache_dblock_image_len(const void *thing, size_t *image_len);
@ -109,7 +113,8 @@ static herr_t H5EA__cache_dblock_notify(H5AC_notify_action_t action, void *thing
static herr_t H5EA__cache_dblock_free_icr(void *thing);
static herr_t H5EA__cache_dblock_fsf_size(const void *thing, size_t *fsf_size);
static herr_t H5EA__cache_dblk_page_get_load_size(const void *udata, size_t *image_len);
static herr_t H5EA__cache_dblk_page_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5EA__cache_dblk_page_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5EA__cache_dblk_page_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5EA__cache_dblk_page_image_len(const void *thing,
@ -130,7 +135,9 @@ const H5AC_class_t H5AC_EARRAY_HDR[1] = {{
"Extensible Array Header", /* Metadata client name (for debugging) */
H5FD_MEM_EARRAY_HDR, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5EA__cache_hdr_get_load_size, /* 'get_load_size' callback */
H5EA__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5EA__cache_hdr_verify_chksum, /* 'verify_chksum' callback */
H5EA__cache_hdr_deserialize, /* 'deserialize' callback */
H5EA__cache_hdr_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -146,7 +153,9 @@ const H5AC_class_t H5AC_EARRAY_IBLOCK[1] = {{
"Extensible Array Index Block", /* Metadata client name (for debugging) */
H5FD_MEM_EARRAY_IBLOCK, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5EA__cache_iblock_get_load_size, /* 'get_load_size' callback */
H5EA__cache_iblock_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5EA__cache_iblock_verify_chksum, /* 'verify_chksum' callback */
H5EA__cache_iblock_deserialize, /* 'deserialize' callback */
H5EA__cache_iblock_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -162,7 +171,9 @@ const H5AC_class_t H5AC_EARRAY_SBLOCK[1] = {{
"Extensible Array Super Block", /* Metadata client name (for debugging) */
H5FD_MEM_EARRAY_SBLOCK, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5EA__cache_sblock_get_load_size, /* 'get_load_size' callback */
H5EA__cache_sblock_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5EA__cache_sblock_verify_chksum, /* 'verify_chksum' callback */
H5EA__cache_sblock_deserialize, /* 'deserialize' callback */
H5EA__cache_sblock_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -178,7 +189,9 @@ const H5AC_class_t H5AC_EARRAY_DBLOCK[1] = {{
"Extensible Array Data Block", /* Metadata client name (for debugging) */
H5FD_MEM_EARRAY_DBLOCK, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5EA__cache_dblock_get_load_size, /* 'get_load_size' callback */
H5EA__cache_dblock_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5EA__cache_dblock_verify_chksum, /* 'verify_chksum' callback */
H5EA__cache_dblock_deserialize, /* 'deserialize' callback */
H5EA__cache_dblock_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -194,7 +207,9 @@ const H5AC_class_t H5AC_EARRAY_DBLK_PAGE[1] = {{
"Extensible Array Data Block Page", /* Metadata client name (for debugging) */
H5FD_MEM_EARRAY_DBLK_PAGE, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5EA__cache_dblk_page_get_load_size, /* 'get_load_size' callback */
H5EA__cache_dblk_page_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5EA__cache_dblk_page_verify_chksum, /* 'verify_chksum' callback */
H5EA__cache_dblk_page_deserialize, /* 'deserialize' callback */
H5EA__cache_dblk_page_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -217,7 +232,7 @@ const H5AC_class_t H5AC_EARRAY_DBLK_PAGE[1] = {{
/*-------------------------------------------------------------------------
* Function: H5EA__cache_hdr_get_load_size
* Function: H5EA__cache_hdr_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -231,21 +246,54 @@ const H5AC_class_t H5AC_EARRAY_DBLK_PAGE[1] = {{
*/
BEGIN_FUNC(STATIC, NOERR,
herr_t, SUCCEED, -,
H5EA__cache_hdr_get_load_size(const void *_udata, size_t *image_len))
H5EA__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len))
/* Local variables */
const H5EA_hdr_cache_ud_t *udata = (const H5EA_hdr_cache_ud_t *)_udata; /* User data for callback */
H5EA_hdr_cache_ud_t *udata = (H5EA_hdr_cache_ud_t *)_udata; /* User data for callback */
/* Check arguments */
HDassert(udata);
HDassert(udata->f);
HDassert(H5F_addr_defined(udata->addr));
HDassert(image_len);
/* Set the image length size */
*image_len = (size_t)H5EA_HEADER_SIZE_FILE(udata->f);
END_FUNC(STATIC) /* end H5EA__cache_hdr_get_load_size() */
END_FUNC(STATIC) /* end H5EA__cache_hdr_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5EA__cache_hdr_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
BEGIN_FUNC(STATIC, NOERR,
htri_t, TRUE, -,
H5EA__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata))
/* Local variables */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
END_FUNC(STATIC) /* end H5EA__cache_hdr_verify_chksum() */
/*-------------------------------------------------------------------------
@ -273,7 +321,6 @@ H5EA__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
H5EA_hdr_cache_ud_t *udata = (H5EA_hdr_cache_ud_t *)_udata;
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Check arguments */
HDassert(image);
@ -348,17 +395,11 @@ H5EA__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
/* (allow for checksum not decoded yet) */
HDassert((size_t)(image - (const uint8_t *)_image) == (len - H5EA_SIZEOF_CHKSUM));
/* Compute checksum on entire header */
/* (including the filter information, if present) */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
/* Verify checksum */
if(stored_chksum != computed_chksum)
H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array header")
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == len);
@ -510,7 +551,7 @@ END_FUNC(STATIC) /* end H5EA__cache_hdr_free_icr() */
/*-------------------------------------------------------------------------
* Function: H5EA__cache_iblock_get_load_size
* Function: H5EA__cache_iblock_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -524,11 +565,11 @@ END_FUNC(STATIC) /* end H5EA__cache_hdr_free_icr() */
*/
BEGIN_FUNC(STATIC, NOERR,
herr_t, SUCCEED, -,
H5EA__cache_iblock_get_load_size(const void *_udata, size_t *image_len))
H5EA__cache_iblock_get_initial_load_size(void *_udata, size_t *image_len))
/* Local variables */
const H5EA_hdr_t *hdr = (const H5EA_hdr_t *)_udata; /* User data for callback */
H5EA_iblock_t iblock; /* Fake index block for computing size */
H5EA_hdr_t *hdr = (H5EA_hdr_t *)_udata; /* User data for callback */
H5EA_iblock_t iblock; /* Fake index block for computing size */
/* Check arguments */
HDassert(hdr);
@ -544,7 +585,41 @@ H5EA__cache_iblock_get_load_size(const void *_udata, size_t *image_len))
/* Set the image length size */
*image_len = (size_t)H5EA_IBLOCK_SIZE(&iblock);
END_FUNC(STATIC) /* end H5EA__cache_iblock_get_load_size() */
END_FUNC(STATIC) /* end H5EA__cache_iblock_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5EA__cache_iblock_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
BEGIN_FUNC(STATIC, NOERR,
htri_t, TRUE, -,
H5EA__cache_iblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata))
/* Local variables */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
END_FUNC(STATIC) /* end H5EA__cache_iblock_verify_chksum() */
/*-------------------------------------------------------------------------
@ -571,7 +646,6 @@ H5EA__cache_iblock_deserialize(const void *_image, size_t len,
H5EA_hdr_t *hdr = (H5EA_hdr_t *)_udata; /* User data for callback */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
haddr_t arr_addr; /* Address of array header in the file */
size_t u; /* Local index variable */
@ -635,8 +709,7 @@ H5EA__cache_iblock_deserialize(const void *_image, size_t len,
/* Save the index block's size */
iblock->size = len;
/* Compute checksum on index block */
computed_chksum = H5_checksum_metadata((const uint8_t *)_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -644,10 +717,6 @@ H5EA__cache_iblock_deserialize(const void *_image, size_t len,
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == iblock->size);
/* Verify checksum */
if(stored_chksum != computed_chksum)
H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array index block")
/* Set return value */
ret_value = iblock;
@ -863,7 +932,7 @@ END_FUNC(STATIC) /* end H5EA__cache_iblock_free_icr() */
/*-------------------------------------------------------------------------
* Function: H5EA__cache_sblock_get_load_size
* Function: H5EA__cache_sblock_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -877,11 +946,11 @@ END_FUNC(STATIC) /* end H5EA__cache_iblock_free_icr() */
*/
BEGIN_FUNC(STATIC, NOERR,
herr_t, SUCCEED, -,
H5EA__cache_sblock_get_load_size(const void *_udata, size_t *image_len))
H5EA__cache_sblock_get_initial_load_size(void *_udata, size_t *image_len))
/* Local variables */
const H5EA_sblock_cache_ud_t *udata = (const H5EA_sblock_cache_ud_t *)_udata; /* User data */
H5EA_sblock_t sblock; /* Fake super block for computing size */
H5EA_sblock_cache_ud_t *udata = (H5EA_sblock_cache_ud_t *)_udata; /* User data */
H5EA_sblock_t sblock; /* Fake super block for computing size */
/* Check arguments */
HDassert(udata);
@ -917,7 +986,41 @@ H5EA__cache_sblock_get_load_size(const void *_udata, size_t *image_len))
/* Set the image length size */
*image_len = (size_t)H5EA_SBLOCK_SIZE(&sblock);
END_FUNC(STATIC) /* end H5EA__cache_sblock_get_load_size() */
END_FUNC(STATIC) /* end H5EA__cache_sblock_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5EA__cache_sblock_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
BEGIN_FUNC(STATIC, NOERR,
htri_t, TRUE, -,
H5EA__cache_sblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata))
/* Local variables */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
END_FUNC(STATIC) /* end H5EA__cache_sblock_verify_chksum() */
/*-------------------------------------------------------------------------
@ -944,7 +1047,6 @@ H5EA__cache_sblock_deserialize(const void *_image, size_t len,
H5EA_sblock_cache_ud_t *udata = (H5EA_sblock_cache_ud_t *)_udata; /* User data */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
haddr_t arr_addr; /* Address of array header in the file */
size_t u; /* Local index variable */
@ -1005,8 +1107,7 @@ H5EA__cache_sblock_deserialize(const void *_image, size_t len,
/* Save the super block's size */
sblock->size = len;
/* Compute checksum on super block */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -1014,10 +1115,6 @@ H5EA__cache_sblock_deserialize(const void *_image, size_t len,
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == sblock->size);
/* Verify checksum */
if(stored_chksum != computed_chksum)
H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array super block")
/* Set return value */
ret_value = sblock;
@ -1235,7 +1332,7 @@ END_FUNC(STATIC) /* end H5EA__cache_sblock_free_icr() */
/*-------------------------------------------------------------------------
* Function: H5EA__cache_dblock_get_load_size
* Function: H5EA__cache_dblock_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -1249,11 +1346,11 @@ END_FUNC(STATIC) /* end H5EA__cache_sblock_free_icr() */
*/
BEGIN_FUNC(STATIC, NOERR,
herr_t, SUCCEED, -,
H5EA__cache_dblock_get_load_size(const void *_udata, size_t *image_len))
H5EA__cache_dblock_get_initial_load_size(void *_udata, size_t *image_len))
/* Local variables */
const H5EA_dblock_cache_ud_t *udata = (const H5EA_dblock_cache_ud_t *)_udata; /* User data */
H5EA_dblock_t dblock; /* Fake data block for computing size */
H5EA_dblock_cache_ud_t *udata = (H5EA_dblock_cache_ud_t *)_udata; /* User data */
H5EA_dblock_t dblock; /* Fake data block for computing size */
/* Check arguments */
HDassert(udata);
@ -1290,7 +1387,41 @@ H5EA__cache_dblock_get_load_size(const void *_udata, size_t *image_len))
else
*image_len = H5EA_DBLOCK_PREFIX_SIZE(&dblock);
END_FUNC(STATIC) /* end H5EA__cache_dblock_get_load_size() */
END_FUNC(STATIC) /* end H5EA__cache_dblock_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5EA__cache_dblock_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
BEGIN_FUNC(STATIC, NOERR,
htri_t, TRUE, -,
H5EA__cache_dblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata))
/* Local variables */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
END_FUNC(STATIC) /* end H5EA__cache_sblock_verify_chksum() */
/*-------------------------------------------------------------------------
@ -1317,7 +1448,6 @@ H5EA__cache_dblock_deserialize(const void *_image, size_t len,
H5EA_dblock_cache_ud_t *udata = (H5EA_dblock_cache_ud_t *)_udata; /* User data */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
haddr_t arr_addr; /* Address of array header in the file */
/* Check arguments */
@ -1377,8 +1507,7 @@ H5EA__cache_dblock_deserialize(const void *_image, size_t len,
/* (Note: This is not the same as the image length, for paged data blocks) */
dblock->size = H5EA_DBLOCK_SIZE(dblock);
/* Compute checksum on data block */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -1386,10 +1515,6 @@ H5EA__cache_dblock_deserialize(const void *_image, size_t len,
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == len);
/* Verify checksum */
if(stored_chksum != computed_chksum)
H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array data block")
/* Set return value */
ret_value = dblock;
@ -1655,7 +1780,7 @@ END_FUNC(STATIC) /* end H5EA__cache_dblock_fsf_size() */
/*-------------------------------------------------------------------------
* Function: H5EA__cache_dblk_page_get_load_size
* Function: H5EA__cache_dblk_page_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -1669,10 +1794,10 @@ END_FUNC(STATIC) /* end H5EA__cache_dblock_fsf_size() */
*/
BEGIN_FUNC(STATIC, NOERR,
herr_t, SUCCEED, -,
H5EA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len))
H5EA__cache_dblk_page_get_initial_load_size(void *_udata, size_t *image_len))
/* Local variables */
const H5EA_dblk_page_cache_ud_t *udata = (const H5EA_dblk_page_cache_ud_t *)_udata; /* User data */
H5EA_dblk_page_cache_ud_t *udata = (H5EA_dblk_page_cache_ud_t *)_udata; /* User data */
/* Check arguments */
HDassert(udata);
@ -1680,9 +1805,44 @@ H5EA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len))
HDassert(udata->parent);
HDassert(image_len);
/* Set the image length size */
*image_len = (size_t)H5EA_DBLK_PAGE_SIZE(udata->hdr);
END_FUNC(STATIC) /* end H5EA__cache_dblk_page_get_load_size() */
END_FUNC(STATIC) /* end H5EA__cache_dblk_page_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5EA__cache_dblk_page_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
BEGIN_FUNC(STATIC, NOERR,
htri_t, TRUE, -,
H5EA__cache_dblk_page_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata))
/* Local variables */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
END_FUNC(STATIC) /* end H5EA__cache_dblk_page_verify_chksum() */
/*-------------------------------------------------------------------------
@ -1709,7 +1869,6 @@ H5EA__cache_dblk_page_deserialize(const void *_image, size_t len,
H5EA_dblk_page_cache_ud_t *udata = (H5EA_dblk_page_cache_ud_t *)_udata; /* User data for loading data block page */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Sanity check */
HDassert(udata);
@ -1739,8 +1898,7 @@ H5EA__cache_dblk_page_deserialize(const void *_image, size_t len,
/* Set the data block page's size */
dblk_page->size = len;
/* Compute checksum on data block page */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -1748,10 +1906,6 @@ H5EA__cache_dblk_page_deserialize(const void *_image, size_t len,
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == dblk_page->size);
/* Verify checksum */
if(stored_chksum != computed_chksum)
H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array data block page")
/* Set return value */
ret_value = dblk_page;

172
src/H5FAcache.c
View File

@ -71,7 +71,8 @@
/********************/
/* Metadata cache (H5AC) callbacks */
static herr_t H5FA__cache_hdr_get_load_size(const void *udata, size_t *image_len);
static herr_t H5FA__cache_hdr_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5FA__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5FA__cache_hdr_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5FA__cache_hdr_image_len(const void *thing, size_t *image_len);
@ -79,7 +80,8 @@ static herr_t H5FA__cache_hdr_serialize(const H5F_t *f, void *image, size_t len,
void *thing);
static herr_t H5FA__cache_hdr_free_icr(void *thing);
static herr_t H5FA__cache_dblock_get_load_size(const void *udata, size_t *image_len);
static herr_t H5FA__cache_dblock_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5FA__cache_dblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5FA__cache_dblock_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5FA__cache_dblock_image_len(const void *thing, size_t *image_len);
@ -88,7 +90,8 @@ static herr_t H5FA__cache_dblock_serialize(const H5F_t *f, void *image, size_t l
static herr_t H5FA__cache_dblock_free_icr(void *thing);
static herr_t H5FA__cache_dblock_fsf_size(const void *thing, size_t *fsf_size);
static herr_t H5FA__cache_dblk_page_get_load_size(const void *udata, size_t *image_len);
static herr_t H5FA__cache_dblk_page_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5FA__cache_dblk_page_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5FA__cache_dblk_page_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5FA__cache_dblk_page_image_len(const void *thing, size_t *image_len);
@ -107,7 +110,9 @@ const H5AC_class_t H5AC_FARRAY_HDR[1] = {{
"Fixed-array Header", /* Metadata client name (for debugging) */
H5FD_MEM_FARRAY_HDR, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5FA__cache_hdr_get_load_size, /* 'get_load_size' callback */
H5FA__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5FA__cache_hdr_verify_chksum, /* 'verify_chksum' callback */
H5FA__cache_hdr_deserialize, /* 'deserialize' callback */
H5FA__cache_hdr_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -123,7 +128,9 @@ const H5AC_class_t H5AC_FARRAY_DBLOCK[1] = {{
"Fixed Array Data Block", /* Metadata client name (for debugging) */
H5FD_MEM_FARRAY_DBLOCK, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5FA__cache_dblock_get_load_size, /* 'get_load_size' callback */
H5FA__cache_dblock_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5FA__cache_dblock_verify_chksum, /* 'verify_chksum' callback */
H5FA__cache_dblock_deserialize, /* 'deserialize' callback */
H5FA__cache_dblock_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -139,7 +146,9 @@ const H5AC_class_t H5AC_FARRAY_DBLK_PAGE[1] = {{
"Fixed Array Data Block Page", /* Metadata client name (for debugging) */
H5FD_MEM_FARRAY_DBLK_PAGE, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5FA__cache_dblk_page_get_load_size, /* 'get_load_size' callback */
H5FA__cache_dblk_page_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5FA__cache_dblk_page_verify_chksum, /* 'verify_chksum' callback */
H5FA__cache_dblk_page_deserialize, /* 'deserialize' callback */
H5FA__cache_dblk_page_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -162,7 +171,7 @@ const H5AC_class_t H5AC_FARRAY_DBLK_PAGE[1] = {{
/*-------------------------------------------------------------------------
* Function: H5FA__cache_hdr_get_load_size
* Function: H5FA__cache_hdr_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -176,10 +185,10 @@ const H5AC_class_t H5AC_FARRAY_DBLK_PAGE[1] = {{
*/
BEGIN_FUNC(STATIC, NOERR,
herr_t, SUCCEED, -,
H5FA__cache_hdr_get_load_size(const void *_udata, size_t *image_len))
H5FA__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len))
/* Local variables */
const H5FA_hdr_cache_ud_t *udata = (const H5FA_hdr_cache_ud_t *)_udata; /* User data for callback */
H5FA_hdr_cache_ud_t *udata = (H5FA_hdr_cache_ud_t *)_udata; /* User data for callback */
/* Check arguments */
HDassert(udata);
@ -189,7 +198,41 @@ H5FA__cache_hdr_get_load_size(const void *_udata, size_t *image_len))
/* Set the image length size */
*image_len = (size_t)H5FA_HEADER_SIZE_FILE(udata->f);
END_FUNC(STATIC) /* end H5FA__cache_hdr_get_load_size() */
END_FUNC(STATIC) /* end H5FA__cache_hdr_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5FA__cache_hdr_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
BEGIN_FUNC(STATIC, NOERR,
htri_t, TRUE, -,
H5FA__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata))
/* Local variables */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
END_FUNC(STATIC) /* end H5FA__cache_hdr_verify_chksum() */
/*-------------------------------------------------------------------------
@ -217,7 +260,6 @@ H5FA__cache_hdr_deserialize(const void *_image, size_t len,
H5FA_hdr_cache_ud_t *udata = (H5FA_hdr_cache_ud_t *)_udata;
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Check arguments */
HDassert(udata);
@ -281,9 +323,7 @@ H5FA__cache_hdr_deserialize(const void *_image, size_t len,
/* (allow for checksum not decoded yet) */
HDassert((size_t)(image - (const uint8_t *)_image) == (len - H5FA_SIZEOF_CHKSUM));
/* Compute checksum on entire header */
/* (including the filter information, if present) */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -291,10 +331,6 @@ H5FA__cache_hdr_deserialize(const void *_image, size_t len,
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == len);
/* Verify checksum */
if(stored_chksum != computed_chksum)
H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array header")
/* Finish initializing fixed array header */
if(H5FA__hdr_init(hdr, udata->ctx_udata) < 0)
H5E_THROW(H5E_CANTINIT, "initialization failed for fixed array header")
@ -434,7 +470,7 @@ END_FUNC(STATIC) /* end H5FA__cache_hdr_free_icr() */
/*-------------------------------------------------------------------------
* Function: H5FA__cache_dblock_get_load_size
* Function: H5FA__cache_dblock_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -448,7 +484,7 @@ END_FUNC(STATIC) /* end H5FA__cache_hdr_free_icr() */
*/
BEGIN_FUNC(STATIC, NOERR,
herr_t, SUCCEED, -,
H5FA__cache_dblock_get_load_size(const void *_udata, size_t *image_len))
H5FA__cache_dblock_get_initial_load_size(void *_udata, size_t *image_len))
/* Local variables */
H5FA_dblock_cache_ud_t *udata = (H5FA_dblock_cache_ud_t *)_udata; /* User data */
@ -470,7 +506,6 @@ H5FA__cache_dblock_get_load_size(const void *_udata, size_t *image_len))
* dblock->npages
* dblock->dblk_page_init_size
*/
dblock.hdr = udata->hdr;
dblk_page_nelmts = (size_t)1 << udata->hdr->cparam.max_dblk_page_nelmts_bits;
if(udata->hdr->cparam.nelmts > dblk_page_nelmts) {
@ -484,7 +519,41 @@ H5FA__cache_dblock_get_load_size(const void *_udata, size_t *image_len))
else
*image_len = (size_t)H5FA_DBLOCK_PREFIX_SIZE(&dblock);
END_FUNC(STATIC) /* end H5FA__cache_dblock_get_load_size() */
END_FUNC(STATIC) /* end H5FA__cache_dblock_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5FA__cache_dblock_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
BEGIN_FUNC(STATIC, NOERR,
htri_t, TRUE, -,
H5FA__cache_dblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata))
/* Local variables */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
END_FUNC(STATIC) /* end H5FA__cache_dblock_verify_chksum() */
/*-------------------------------------------------------------------------
@ -511,7 +580,6 @@ H5FA__cache_dblock_deserialize(const void *_image, size_t len,
H5FA_dblock_cache_ud_t *udata = (H5FA_dblock_cache_ud_t *)_udata; /* User data for loading data block */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
haddr_t arr_addr; /* Address of array header in the file */
/* Sanity check */
@ -568,8 +636,7 @@ H5FA__cache_dblock_deserialize(const void *_image, size_t len,
/* Set the data block's size */
dblock->size = H5FA_DBLOCK_SIZE(dblock);
/* Compute checksum on data block */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -577,10 +644,6 @@ H5FA__cache_dblock_deserialize(const void *_image, size_t len,
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == len);
/* Verify checksum */
if(stored_chksum != computed_chksum)
H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array data block")
/* Set return value */
ret_value = dblock;
@ -780,7 +843,7 @@ END_FUNC(STATIC) /* end H5FA__cache_dblock_fsf_size() */
/*-------------------------------------------------------------------------
* Function: H5FA__cache_dblk_page_get_load_size
* Function: H5FA__cache_dblk_page_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -794,10 +857,10 @@ END_FUNC(STATIC) /* end H5FA__cache_dblock_fsf_size() */
*/
BEGIN_FUNC(STATIC, NOERR,
herr_t, SUCCEED, -,
H5FA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len))
H5FA__cache_dblk_page_get_initial_load_size(void *_udata, size_t *image_len))
/* Local variables */
const H5FA_dblk_page_cache_ud_t *udata = (const H5FA_dblk_page_cache_ud_t *)_udata; /* User data */
H5FA_dblk_page_cache_ud_t *udata = (H5FA_dblk_page_cache_ud_t *)_udata; /* User data */
/* Check arguments */
HDassert(udata);
@ -805,9 +868,44 @@ H5FA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len))
HDassert(udata->nelmts > 0);
HDassert(image_len);
/* Set the image length size */
*image_len = (size_t)H5FA_DBLK_PAGE_SIZE(udata->hdr, udata->nelmts);
END_FUNC(STATIC) /* end H5FA__cache_dblk_page_get_load_size() */
END_FUNC(STATIC) /* end H5FA__cache_dblk_page_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5FA__cache_dblk_page_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
BEGIN_FUNC(STATIC, NOERR,
htri_t, TRUE, -,
H5FA__cache_dblk_page_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata))
/* Local variables */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
END_FUNC(STATIC) /* end H5FA__cache_dblk_page_verify_chksum() */
/*-------------------------------------------------------------------------
@ -834,7 +932,6 @@ H5FA__cache_dblk_page_deserialize(const void *_image, size_t len,
H5FA_dblk_page_cache_ud_t *udata = (H5FA_dblk_page_cache_ud_t *)_udata; /* User data for loading data block page */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Sanity check */
HDassert(udata);
@ -864,8 +961,7 @@ H5FA__cache_dblk_page_deserialize(const void *_image, size_t len,
/* Set the data block page's size */
dblk_page->size = len;
/* Compute checksum on data block */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -873,10 +969,6 @@ H5FA__cache_dblk_page_deserialize(const void *_image, size_t len,
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == dblk_page->size);
/* Verify checksum */
if(stored_chksum != computed_chksum)
H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array data block page")
/* Set return value */
ret_value = dblk_page;

129
src/H5FScache.c
View File

@ -78,7 +78,8 @@ static herr_t H5FS__sinfo_serialize_sect_cb(void *_item, void H5_ATTR_UNUSED *ke
static herr_t H5FS__sinfo_serialize_node_cb(void *_item, void H5_ATTR_UNUSED *key, void *_udata);
/* Metadata cache callbacks */
static herr_t H5FS__cache_hdr_get_load_size(const void *udata, size_t *image_len);
static herr_t H5FS__cache_hdr_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5FS__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5FS__cache_hdr_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5FS__cache_hdr_image_len(const void *thing, size_t *image_len);
@ -89,7 +90,8 @@ static herr_t H5FS__cache_hdr_serialize(const H5F_t *f, void *image,
size_t len, void *thing);
static herr_t H5FS__cache_hdr_free_icr(void *thing);
static herr_t H5FS__cache_sinfo_get_load_size(const void *udata, size_t *image_len);
static herr_t H5FS__cache_sinfo_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5FS__cache_sinfo_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5FS__cache_sinfo_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5FS__cache_sinfo_image_len(const void *thing, size_t *image_len);
@ -111,7 +113,9 @@ const H5AC_class_t H5AC_FSPACE_HDR[1] = {{
"Free Space Header", /* Metadata client name (for debugging) */
H5FD_MEM_FSPACE_HDR, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5FS__cache_hdr_get_load_size, /* 'get_load_size' callback */
H5FS__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5FS__cache_hdr_verify_chksum, /* 'verify_chksum' callback */
H5FS__cache_hdr_deserialize, /* 'deserialize' callback */
H5FS__cache_hdr_image_len, /* 'image_len' callback */
H5FS__cache_hdr_pre_serialize, /* 'pre_serialize' callback */
@ -127,7 +131,9 @@ const H5AC_class_t H5AC_FSPACE_SINFO[1] = {{
"Free Space Section Info", /* Metadata client name (for debugging) */
H5FD_MEM_FSPACE_SINFO, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5FS__cache_sinfo_get_load_size, /* 'get_load_size' callback */
H5FS__cache_sinfo_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5FS__cache_sinfo_verify_chksum, /* 'verify_chksum' callback */
H5FS__cache_sinfo_deserialize, /* 'deserialize' callback */
H5FS__cache_sinfo_image_len, /* 'image_len' callback */
H5FS__cache_sinfo_pre_serialize, /* 'pre_serialize' callback */
@ -150,7 +156,7 @@ const H5AC_class_t H5AC_FSPACE_SINFO[1] = {{
/*-------------------------------------------------------------------------
* Function: H5FS__cache_hdr_get_load_size
* Function: H5FS__cache_hdr_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -163,9 +169,9 @@ const H5AC_class_t H5AC_FSPACE_SINFO[1] = {{
*-------------------------------------------------------------------------
*/
static herr_t
H5FS__cache_hdr_get_load_size(const void *_udata, size_t *image_len)
H5FS__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5FS_hdr_cache_ud_t *udata = (const H5FS_hdr_cache_ud_t *)_udata; /* User-data for metadata cache callback */
H5FS_hdr_cache_ud_t *udata = (H5FS_hdr_cache_ud_t *)_udata; /* User-data for metadata cache callback */
FUNC_ENTER_STATIC_NOERR
@ -178,7 +184,43 @@ H5FS__cache_hdr_get_load_size(const void *_udata, size_t *image_len)
*image_len = (size_t)H5FS_HEADER_SIZE(udata->f);
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FS__cache_hdr_get_load_size() */
} /* end H5FS__cache_hdr_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5FS__cache_hdr_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
htri_t
H5FS__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FS__cache_hdr_verify_chksum() */
/*-------------------------------------------------------------------------
@ -206,7 +248,6 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
H5FS_hdr_cache_ud_t *udata = (H5FS_hdr_cache_ud_t *)_udata; /* User data for callback */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
unsigned nclasses; /* Number of section classes */
H5FS_t *ret_value = NULL; /* Return value */
@ -279,8 +320,7 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
/* Allocated size of serialized free space sections */
H5F_DECODE_LENGTH(udata->f, image, fspace->alloc_sect_size);
/* Compute checksum on indirect block */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -288,10 +328,6 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) <= len);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block")
/* Set return value */
ret_value = fspace;
@ -721,10 +757,6 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len,
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FS__cache_hdr_serialize() */
/***************************************/
/* no H5FS__cache_hdr_notify() function */
/***************************************/
/*-------------------------------------------------------------------------
* Function: H5FS__cache_hdr_free_icr
@ -768,13 +800,9 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FS__cache_hdr_free_icr() */
/********************************************************/
/* metadata cache callback definitions for section info */
/********************************************************/
/*-------------------------------------------------------------------------
* Function: H5FS__cache_sinfo_get_load_size()
* Function: H5FS__cache_sinfo_get_initial_load_size()
*
* Purpose: Compute the size of the on disk image of the free space
* manager section info, and place this value in *image_len.
@ -788,10 +816,10 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5FS__cache_sinfo_get_load_size(const void *_udata, size_t *image_len)
H5FS__cache_sinfo_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5FS_t *fspace; /* free space manager */
const H5FS_sinfo_cache_ud_t *udata = (const H5FS_sinfo_cache_ud_t *)_udata; /* User data for callback */
const H5FS_t *fspace; /* free space manager */
H5FS_sinfo_cache_ud_t *udata = (H5FS_sinfo_cache_ud_t *)_udata; /* User data for callback */
FUNC_ENTER_STATIC_NOERR
@ -802,10 +830,47 @@ H5FS__cache_sinfo_get_load_size(const void *_udata, size_t *image_len)
HDassert(fspace->sect_size > 0);
HDassert(image_len);
/* Set the image length size */
*image_len = (size_t)(fspace->sect_size);
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FS__cache_sinfo_get_load_size() */
} /* end H5FS__cache_sinfo_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5FS__cache_sinfo_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
htri_t
H5FS__cache_sinfo_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FS__cache_sinfo_verify_chksum() */
/*-------------------------------------------------------------------------
@ -835,7 +900,6 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata,
size_t old_sect_size; /* Old section size */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum */
uint32_t computed_chksum; /* Computed metadata checksum */
void * ret_value = NULL; /* Return value */
FUNC_ENTER_STATIC
@ -942,16 +1006,11 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata,
HDassert(old_tot_space == fspace->tot_space);
} /* end if */
/* Compute checksum on indirect block */
computed_chksum = H5_checksum_metadata((const uint8_t *)_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block")
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == old_sect_size);

108
src/H5Fint.c
View File

@ -686,6 +686,18 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t
*/
f->shared->use_tmp_space = !H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI);
/* Retrieve the # of read attempts here so that sohm in superblock will get the correct # of attempts */
if(H5P_get(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &f->shared->read_attempts) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get the # of read attempts")
/* If no value for read attempts has been set, use the default */
if(!f->shared->read_attempts)
f->shared->read_attempts = H5F_METADATA_READ_ATTEMPTS;
/* Determine the # of bins for metdata read retries */
if(H5F_set_retries(f) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't set retries and retries_nbins")
/* Get the metadata cache log location (if we're logging) */
{
char *mdc_log_location = NULL; /* location of metadata cache log location */
@ -780,6 +792,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
HDassert(f->shared);
if(1 == f->shared->nrefs) {
int actype; /* metadata cache type (enum value) */
H5F_io_info_t fio_info; /* I/O info for operation */
/* Flush at this point since the file will be closed.
@ -897,6 +910,11 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
f->shared->mtab.child = (H5F_mount_t *)H5MM_xfree(f->shared->mtab.child);
f->shared->mtab.nalloc = 0;
/* Clean up the metadata retries array */
for(actype = 0; actype < (int)H5AC_NTYPES; actype++)
if(f->shared->retries[actype])
f->shared->retries[actype] = (uint32_t *)H5MM_xfree(f->shared->retries[actype]);
/* Destroy shared file struct */
f->shared = (H5F_file_t *)H5FL_FREE(H5F_file_t, f->shared);
@ -2111,6 +2129,96 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5F_get_file_image() */
/*-------------------------------------------------------------------------
* Function: H5F_track_metadata_read_retries
*
* Purpose: To track the # of a "retries" (log10) for a metadata item.
* This routine should be used only when:
* "retries" > 0
* f->shared->read_attempts > 1 (does not have retry when 1)
* f->shared->retries_nbins > 0 (calculated based on f->shared->read_attempts)
*
* Return: Success: SUCCEED
* Failure: FAIL
*
* Programmer: Vailin Choi; October 2013
*
*-------------------------------------------------------------------------
*/
herr_t
H5F_track_metadata_read_retries(H5F_t *f, unsigned actype, unsigned retries)
{
unsigned log_ind; /* Index to the array of retries based on log10 of retries */
double tmp; /* Temporary value, to keep compiler quiet */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
/* Sanity check */
HDassert(f);
HDassert(f->shared->read_attempts > 1);
HDassert(f->shared->retries_nbins > 0);
HDassert(retries > 0);
HDassert(retries < f->shared->read_attempts);
HDassert(actype < H5AC_NTYPES);
/* Allocate memory for retries */
if(NULL == f->shared->retries[actype])
if(NULL == (f->shared->retries[actype] = (uint32_t *)H5MM_calloc((size_t)f->shared->retries_nbins * sizeof(uint32_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed")
/* Index to retries based on log10 */
tmp = HDlog10((double)retries);
log_ind = (unsigned)tmp;
HDassert(log_ind < f->shared->retries_nbins);
/* Increment the # of the "retries" */
f->shared->retries[actype][log_ind]++;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5F_track_metadata_read_retries() */
/*-------------------------------------------------------------------------
* Function: H5F_set_retries
*
* Purpose: To initialize data structures for read retries:
* --zero out "retries"
* --set up "retries_nbins" based on read_attempts
*
* Return: Success: SUCCEED
* Failure: FAIL
*
* Programmer: Vailin Choi; November 2013
*
*-------------------------------------------------------------------------
*/
herr_t
H5F_set_retries(H5F_t *f)
{
double tmp; /* Temporary variable */
/* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
FUNC_ENTER_NOAPI_NOINIT_NOERR
/* Sanity check */
HDassert(f);
/* Initialize the tracking for metadata read retries */
HDmemset(f->shared->retries, 0, sizeof(f->shared->retries));
/* Initialize the # of bins for retries */
f->shared->retries_nbins = 0;
if(f->shared->read_attempts > 1) {
tmp = HDlog10((double)(f->shared->read_attempts - 1));
f->shared->retries_nbins = (unsigned)tmp + 1;
}
FUNC_LEAVE_NOAPI(SUCCEED)
} /* H5F_set_retries() */
/*-------------------------------------------------------------------------
* Function: H5F_object_flush_cb

43
src/H5Fio.c
View File

@ -282,3 +282,46 @@ done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5F_evict_tagged_metadata */
/*-------------------------------------------------------------------------
* Function: H5F_get_checksums
*
* Purpose: Decode checksum stored in the buffer
* Calculate checksum for the data in the buffer
*
* Note: Assumes that the checksum is the last data in the buffer
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Vailin Choi
* Sept 2013
*
*-------------------------------------------------------------------------
*/
herr_t
H5F_get_checksums(const uint8_t *buf, size_t buf_size, uint32_t *s_chksum/*out*/, uint32_t *c_chksum/*out*/)
{
FUNC_ENTER_NOAPI_NOINIT_NOERR
/* Check arguments */
HDassert(buf);
HDassert(buf_size);
/* Return the stored checksum */
if(s_chksum) {
const uint8_t *chk_p; /* Pointer into raw data buffer */
/* Offset to the checksum in the buffer */
chk_p = buf + buf_size - H5_SIZEOF_CHKSUM;
/* Decode the checksum stored in the buffer */
UINT32DECODE(chk_p, *s_chksum);
} /* end if */
/* Return the computed checksum for the buffer */
if(c_chksum)
*c_chksum = H5_checksum_metadata(buf, buf_size - H5_SIZEOF_CHKSUM, 0);
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5F_get_chksums() */

8
src/H5Fpkg.h
View File

@ -152,6 +152,9 @@ typedef struct H5F_superblock_cache_ud_t {
unsigned btree_k[H5B_NUM_BTREE_ID]; /* B-tree key values for each type */
haddr_t stored_eof; /* End-of-file in file */
hbool_t drvrinfo_removed; /* Indicate if the driver info was removed */
unsigned super_vers; /* Superblock version obtained in get_load_size callback.
* It will be used later in verify_chksum callback
*/
} H5F_superblock_cache_ud_t;
/* Structure for passing 'user data' to driver info block cache callbacks */
@ -299,6 +302,11 @@ struct H5F_file_t {
/* Metadata accumulator information */
H5F_meta_accum_t accum; /* Metadata accumulator info */
/* Metadata retry info */
unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */
unsigned retries_nbins; /* # of bins for each retries[] */
uint32_t *retries[H5AC_NTYPES]; /* Track # of read retries for metdata items with checksum */
/* Object flush info */
H5F_object_flush_t object_flush; /* Information for object flush callback */
};

18
src/H5Fprivate.h
View File

@ -286,6 +286,7 @@
#define H5F_FILE_ID(F) ((F)->file_id)
#define H5F_PARENT(F) ((F)->parent)
#define H5F_NMOUNTS(F) ((F)->nmounts)
#define H5F_GET_READ_ATTEMPTS(F) ((F)->shared->read_attempts)
#define H5F_DRIVER_ID(F) ((F)->shared->lf->driver_id)
#define H5F_GET_FILENO(F,FILENUM) ((FILENUM) = (F)->shared->lf->fileno)
#define H5F_HAS_FEATURE(F,FL) ((F)->shared->lf->feature_flags & (FL))
@ -335,6 +336,7 @@
#define H5F_FILE_ID(F) (H5F_get_file_id(F))
#define H5F_PARENT(F) (H5F_get_parent(F))
#define H5F_NMOUNTS(F) (H5F_get_nmounts(F))
#define H5F_GET_READ_ATTEMPTS(F) (H5F_get_read_attempts(F))
#define H5F_DRIVER_ID(F) (H5F_get_driver_id(F))
#define H5F_GET_FILENO(F,FILENUM) (H5F_get_fileno((F), &(FILENUM)))
#define H5F_HAS_FEATURE(F,FL) (H5F_has_feature(F,FL))
@ -467,6 +469,7 @@
#define H5F_ACS_MULTI_TYPE_NAME "multi_type" /* Data type in multi file driver */
#define H5F_ACS_LATEST_FORMAT_NAME "latest_format" /* 'Use latest format version' flag */
#define H5F_ACS_WANT_POSIX_FD_NAME "want_posix_fd" /* Internal: query the file descriptor from the core VFD, instead of the memory address */
#define H5F_ACS_METADATA_READ_ATTEMPTS_NAME "metadata_read_attempts" /* # of metadata read attempts */
#define H5F_ACS_OBJECT_FLUSH_CB_NAME "object_flush_cb" /* Object flush callback */
#define H5F_ACS_EFC_SIZE_NAME "efc_size" /* Size of external file cache */
#define H5F_ACS_FILE_IMAGE_INFO_NAME "file_image_info" /* struct containing initial file image and callback info */
@ -518,12 +521,19 @@
/* Default free space section threshold used by free-space managers */
#define H5F_FREE_SPACE_THRESHOLD_DEF 1
/* Metadata read attempt values */
#define H5F_METADATA_READ_ATTEMPTS 1 /* Default # of read attempts for non-SWMR access */
/* Macros to define signatures of all objects in the file */
/* Size of signature information (on disk) */
/* (all on-disk signatures should be this length) */
#define H5_SIZEOF_MAGIC 4
/* Size of checksum information (on disk) */
/* (all on-disk checksums should be this length) */
#define H5_SIZEOF_CHKSUM 4
/* v1 B-tree node signature */
#define H5B_MAGIC "TREE"
@ -652,6 +662,7 @@ H5_DLL unsigned H5F_decr_nopen_objs(H5F_t *f);
H5_DLL hid_t H5F_get_file_id(const H5F_t *f);
H5_DLL H5F_t *H5F_get_parent(const H5F_t *f);
H5_DLL unsigned H5F_get_nmounts(const H5F_t *f);
H5_DLL unsigned H5F_get_read_attempts(const H5F_t *f);
H5_DLL hid_t H5F_get_access_plist(H5F_t *f, hbool_t app_ref);
H5_DLL hid_t H5F_get_id(H5F_t *file, hbool_t app_ref);
H5_DLL herr_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref, size_t *obj_id_count_ptr);
@ -716,6 +727,13 @@ H5_DLL herr_t H5F_block_write(const H5F_t *f, H5FD_mem_t type, haddr_t addr,
H5_DLL herr_t H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id);
H5_DLL herr_t H5F_evict_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id);
/* Functions that verify a piece of metadata with checksum */
H5_DLL herr_t H5F_get_checksums(const uint8_t *buf, size_t chk_size, uint32_t *s_chksum, uint32_t *c_chksum);
/* Routine to track the # of retries */
H5_DLL herr_t H5F_track_metadata_read_retries(H5F_t *f, unsigned actype, unsigned retries);
H5_DLL herr_t H5F_set_retries(H5F_t *f);
/* Routine to invoke callback function upon object flush */
H5_DLL herr_t H5F_object_flush_cb(H5F_t *f, hid_t obj_id);

9
src/H5Fpublic.h
View File

@ -175,6 +175,14 @@ typedef enum H5F_file_space_type_t {
H5F_FILE_SPACE_NTYPES /* must be last */
} H5F_file_space_type_t;
/* Data structure to report the collection of read retries for metadata items with checksum */
/* Used by public routine H5Fget_metadata_read_retry_info() */
#define H5F_NUM_METADATA_READ_RETRY_TYPES 21
typedef struct H5F_retry_info_t {
unsigned nbins;
uint32_t *retries[H5F_NUM_METADATA_READ_RETRY_TYPES];
} H5F_retry_info_t;
/* Callback for H5Pset_object_flush_cb() in a file access property list */
typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata);
@ -216,6 +224,7 @@ H5_DLL herr_t H5Fget_mdc_size(hid_t file_id,
H5_DLL herr_t H5Freset_mdc_hit_rate_stats(hid_t file_id);
H5_DLL ssize_t H5Fget_name(hid_t obj_id, char *name, size_t size);
H5_DLL herr_t H5Fget_info2(hid_t obj_id, H5F_info2_t *finfo);
H5_DLL herr_t H5Fget_metadata_read_retry_info(hid_t file_id, H5F_retry_info_t *info);
H5_DLL ssize_t H5Fget_free_sections(hid_t file_id, H5F_mem_t type,
size_t nsects, H5F_sect_info_t *sect_info/*out*/);
H5_DLL herr_t H5Fclear_elink_file_cache(hid_t file_id);

23
src/H5Fquery.c
View File

@ -320,6 +320,29 @@ H5F_get_nmounts(const H5F_t *f)
FUNC_LEAVE_NOAPI(f->nmounts)
} /* end H5F_get_nmounts() */
/*-------------------------------------------------------------------------
* Function: H5F_get_read_attempts
*
* Purpose: Retrieve the file's 'read_attempts' value
*
* Return: '# of read attempts' on success/abort on failure (shouldn't fail)
*
* Programmer: Vaili Choi; Sept 2013
*
*-------------------------------------------------------------------------
*/
unsigned
H5F_get_read_attempts(const H5F_t *f)
{
/* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
FUNC_ENTER_NOAPI_NOINIT_NOERR
HDassert(f);
FUNC_LEAVE_NOAPI(f->shared->read_attempts)
} /* end H5F_get_read_attempts() */
/*-------------------------------------------------------------------------
* Function: H5F_get_fcpl

569
src/H5Fsuper_cache.c
View File

@ -67,7 +67,10 @@
/********************/
/* Metadata cache (H5AC) callbacks */
static herr_t H5F__cache_superblock_get_load_size(const void *udata, size_t *image_len);
static herr_t H5F__cache_superblock_get_initial_load_size(void *udata, size_t *image_len);
static herr_t H5F__cache_superblock_get_final_load_size(const void *image_ptr,
size_t image_len, void *udata, size_t *actual_len);
static htri_t H5F__cache_superblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5F__cache_superblock_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5F__cache_superblock_image_len(const void *thing, size_t *image_len);
@ -78,7 +81,9 @@ static herr_t H5F__cache_superblock_serialize(const H5F_t *f, void *image, size_
void *thing);
static herr_t H5F__cache_superblock_free_icr(void *thing);
static herr_t H5F__cache_drvrinfo_get_load_size(const void *udata, size_t *image_len);
static herr_t H5F__cache_drvrinfo_get_initial_load_size(void *udata, size_t *image_len);
static herr_t H5F__cache_drvrinfo_get_final_load_size(const void *image_ptr,
size_t image_len, void *udata, size_t *actual_len);
static void *H5F__cache_drvrinfo_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5F__cache_drvrinfo_image_len(const void *thing, size_t *image_len);
@ -97,7 +102,9 @@ const H5AC_class_t H5AC_SUPERBLOCK[1] = {{
"Superblock", /* Metadata client name (for debugging) */
H5FD_MEM_SUPER, /* File space memory type for client */
H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */
H5F__cache_superblock_get_load_size,/* 'get_load_size' callback */
H5F__cache_superblock_get_initial_load_size,/* 'get_initial_load_size' callback */
H5F__cache_superblock_get_final_load_size, /* 'get_final_load_size' callback */
H5F__cache_superblock_verify_chksum, /* 'verify_chksum' callback */
H5F__cache_superblock_deserialize, /* 'deserialize' callback */
H5F__cache_superblock_image_len, /* 'image_len' callback */
H5F__cache_superblock_pre_serialize,/* 'pre_serialize' callback */
@ -113,7 +120,9 @@ const H5AC_class_t H5AC_DRVRINFO[1] = {{
"Driver info block", /* Metadata client name (for debugging) */
H5FD_MEM_SUPER, /* File space memory type for client */
H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */
H5F__cache_drvrinfo_get_load_size, /* 'get_load_size' callback */
H5F__cache_drvrinfo_get_initial_load_size, /* 'get_initial_load_size' callback */
H5F__cache_drvrinfo_get_final_load_size, /* 'get_final_load_size' callback */
NULL, /* 'verify_chksum' callback */
H5F__cache_drvrinfo_deserialize, /* 'deserialize' callback */
H5F__cache_drvrinfo_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -139,7 +148,7 @@ H5FL_EXTERN(H5F_super_t);
/*-------------------------------------------------------------------------
* Function: H5F__cache_superblock_get_load_size
* Function: H5F__cache_superblock_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
*
@ -152,7 +161,7 @@ H5FL_EXTERN(H5F_super_t);
*-------------------------------------------------------------------------
*/
static herr_t
H5F__cache_superblock_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *image_len)
H5F__cache_superblock_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len)
{
FUNC_ENTER_STATIC_NOERR
@ -161,10 +170,138 @@ H5F__cache_superblock_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *im
/* Set the initial image length size */
*image_len = H5F_SUPERBLOCK_FIXED_SIZE + /* Fixed size of superblock */
H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE;
H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5F__cache_superblock_get_load_size() */
} /* end H5F__cache_superblock_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5F__cache_superblock_get_final_load_size
*
* Purpose: Compute the final size of the data structure on disk.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Quincey Koziol
* koziol@lbl.gov
* November 17, 2016
*
*-------------------------------------------------------------------------
*/
static herr_t
H5F__cache_superblock_get_final_load_size(const void *_image, size_t image_len,
void *_udata, size_t *actual_len)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */
unsigned super_vers; /* Superblock version */
uint8_t sizeof_addr; /* Size of offsets in the file (in bytes) */
uint8_t sizeof_size; /* Size of lengths in the file (in bytes) */
size_t variable_size; /* Variable size of superblock */
htri_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Check arguments */
HDassert(image);
HDassert(udata);
HDassert(udata->f);
HDassert(actual_len);
HDassert(*actual_len == image_len);
/* Skip over file signature */
image += H5F_SIGNATURE_LEN;
/* Superblock version */
super_vers = *image++;
if(super_vers > HDF5_SUPERBLOCK_VERSION_LATEST)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad superblock version number")
/* Save the version to be used in verify_chksum callback */
udata->super_vers = super_vers;
/* Sanity check */
HDassert(((size_t)(image - (const uint8_t *)_image)) == H5F_SUPERBLOCK_FIXED_SIZE);
HDassert(image_len >= H5F_SUPERBLOCK_FIXED_SIZE + 6);
/* Determine the size of addresses & size of offsets, for computing the
* variable-sized portion of the superblock.
*/
if(super_vers < HDF5_SUPERBLOCK_VERSION_2) {
sizeof_addr = image[4];
sizeof_size = image[5];
} /* end if */
else {
sizeof_addr = image[0];
sizeof_size = image[1];
} /* end else */
if(sizeof_addr != 2 && sizeof_addr != 4 &&
sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number in an address")
if(sizeof_size != 2 && sizeof_size != 4 &&
sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number for object size")
/* Determine the size of the variable-length part of the superblock */
variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(super_vers, sizeof_addr, sizeof_size);
HDassert(variable_size > 0);
/* Sanity check */
HDassert(image_len == (H5F_SUPERBLOCK_FIXED_SIZE + H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE));
/* Make certain we can read the variable-sized portion of the superblock */
if(H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed")
/* Set the final size for the cache image */
*actual_len = H5F_SUPERBLOCK_FIXED_SIZE + variable_size;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F__cache_superblock_get_final_load_size() */
/*-------------------------------------------------------------------------
* Function: H5F__cache_superblock_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
static htri_t
H5F__cache_superblock_verify_chksum(const void *_image, size_t len, void *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
HDassert(udata);
/* No checksum for version 0 & 1 */
if(udata->super_vers >= HDF5_SUPERBLOCK_VERSION_2) {
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
} /* end if */
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F__cache_superblock_verify_chksum() */
/*-------------------------------------------------------------------------
@ -244,176 +381,161 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata,
variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(super_vers, sizeof_addr, sizeof_size);
HDassert(variable_size > 0);
/* Handle metadata cache retry for variable-sized portion of the superblock */
if(len != (H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) {
/* Sanity check */
HDassert(len == (H5F_SUPERBLOCK_FIXED_SIZE + H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE));
HDassert(len == (H5F_SUPERBLOCK_FIXED_SIZE + variable_size));
/* Check for older version of superblock format */
if(super_vers < HDF5_SUPERBLOCK_VERSION_2) {
uint32_t status_flags; /* File status flags */
unsigned sym_leaf_k; /* Symbol table leaf node's 'K' value */
unsigned snode_btree_k; /* B-tree symbol table internal node 'K' value */
unsigned chunk_btree_k; /* B-tree chunk internal node 'K' value */
/* Freespace version (hard-wired) */
if(HDF5_FREESPACE_VERSION != *image++)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad free space version number")
/* Root group version number (hard-wired) */
if(HDF5_OBJECTDIR_VERSION != *image++)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad object directory version number")
/* Skip over reserved byte */
image++;
/* Shared header version number (hard-wired) */
if(HDF5_SHAREDHEADER_VERSION != *image++)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad shared-header format version number")
/* Size of file addresses */
sizeof_addr = *image++;
if(sizeof_addr != 2 && sizeof_addr != 4 &&
sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address")
sblock->sizeof_addr = sizeof_addr;
udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */
/* Size of file sizes */
sizeof_size = *image++;
if(sizeof_size != 2 && sizeof_size != 4 &&
sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size")
sblock->sizeof_size = sizeof_size;
udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */
/* Skip over reserved byte */
image++;
/* Various B-tree sizes */
UINT16DECODE(image, sym_leaf_k);
if(sym_leaf_k == 0)
HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad symbol table leaf node 1/2 rank")
udata->sym_leaf_k = sym_leaf_k; /* Keep a local copy also */
/* Need 'get' call to set other array values */
UINT16DECODE(image, snode_btree_k);
if(snode_btree_k == 0)
HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad 1/2 rank for btree internal nodes")
udata->btree_k[H5B_SNODE_ID] = snode_btree_k;
/*
* Delay setting the value in the property list until we've checked
* for the indexed storage B-tree internal 'K' value later.
*/
/* File status flags (not really used yet) */
UINT32DECODE(image, status_flags);
HDassert(status_flags <= 255);
sblock->status_flags = (uint8_t)status_flags;
if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock")
/*
* If the superblock version # is greater than 0, read in the indexed
* storage B-tree internal 'K' value
*/
if(super_vers > HDF5_SUPERBLOCK_VERSION_DEF) {
UINT16DECODE(image, chunk_btree_k);
/* Reserved bytes are present only in version 1 */
if(super_vers == HDF5_SUPERBLOCK_VERSION_1)
image += 2; /* reserved */
} /* end if */
else
chunk_btree_k = HDF5_BTREE_CHUNK_IK_DEF;
udata->btree_k[H5B_CHUNK_ID] = chunk_btree_k;
/* Remainder of "variable-sized" portion of superblock */
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->driver_addr/*out*/);
/* Allocate space for the root group symbol table entry */
HDassert(!sblock->root_ent);
if(NULL == (sblock->root_ent = (H5G_entry_t *)H5MM_calloc(sizeof(H5G_entry_t))))
HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for root group symbol table entry")
/* decode the root group symbol table entry */
if(H5G_ent_decode(udata->f, (const uint8_t **)&image, sblock->root_ent) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode root group symbol table entry")
/* Set the root group address to the correct value */
sblock->root_addr = sblock->root_ent->header;
/* This step is for h5repart tool only. If user wants to change file driver
* from family to sec2 while using h5repart, set the driver address to
* undefined to let the library ignore the family driver information saved
* in the superblock.
*/
if(udata->ignore_drvrinfo && H5F_addr_defined(sblock->driver_addr)) {
/* Eliminate the driver info */
sblock->driver_addr = HADDR_UNDEF;
udata->drvrinfo_removed = TRUE;
} /* end if */
/* NOTE: Driver info block is decoded separately, later */
/* Make certain we can read the variabled-sized portion of the superblock */
if(H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "set end of space allocation request failed")
} /* end if */
else {
/* Check for older version of superblock format */
if(super_vers < HDF5_SUPERBLOCK_VERSION_2) {
uint32_t status_flags; /* File status flags */
unsigned sym_leaf_k; /* Symbol table leaf node's 'K' value */
unsigned snode_btree_k; /* B-tree symbol table internal node 'K' value */
unsigned chunk_btree_k; /* B-tree chunk internal node 'K' value */
uint32_t read_chksum; /* Checksum read from file */
/* Freespace version (hard-wired) */
if(HDF5_FREESPACE_VERSION != *image++)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad free space version number")
/* Size of file addresses */
sizeof_addr = *image++;
if(sizeof_addr != 2 && sizeof_addr != 4 &&
sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address")
sblock->sizeof_addr = sizeof_addr;
udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */
/* Root group version number (hard-wired) */
if(HDF5_OBJECTDIR_VERSION != *image++)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad object directory version number")
/* Size of file sizes */
sizeof_size = *image++;
if(sizeof_size != 2 && sizeof_size != 4 &&
sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size")
sblock->sizeof_size = sizeof_size;
udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */
/* Skip over reserved byte */
image++;
/* File status flags (not really used yet) */
sblock->status_flags = *image++;
if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock")
/* Shared header version number (hard-wired) */
if(HDF5_SHAREDHEADER_VERSION != *image++)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad shared-header format version number")
/* Base, superblock extension, end of file & root group object header addresses */
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->root_addr/*out*/);
/* Size of file addresses */
sizeof_addr = *image++;
if(sizeof_addr != 2 && sizeof_addr != 4 &&
sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address")
sblock->sizeof_addr = sizeof_addr;
udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */
/* checksum verification already done in verify_chksum cb */
/* Size of file sizes */
sizeof_size = *image++;
if(sizeof_size != 2 && sizeof_size != 4 &&
sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size")
sblock->sizeof_size = sizeof_size;
udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */
/* Decode checksum */
UINT32DECODE(image, read_chksum);
/* Skip over reserved byte */
image++;
/* Various B-tree sizes */
UINT16DECODE(image, sym_leaf_k);
if(sym_leaf_k == 0)
HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad symbol table leaf node 1/2 rank")
udata->sym_leaf_k = sym_leaf_k; /* Keep a local copy also */
/* Need 'get' call to set other array values */
UINT16DECODE(image, snode_btree_k);
if(snode_btree_k == 0)
HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad 1/2 rank for btree internal nodes")
udata->btree_k[H5B_SNODE_ID] = snode_btree_k;
/*
* Delay setting the value in the property list until we've checked
* for the indexed storage B-tree internal 'K' value later.
*/
/* File status flags (not really used yet) */
UINT32DECODE(image, status_flags);
HDassert(status_flags <= 255);
sblock->status_flags = (uint8_t)status_flags;
if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock")
/*
* If the superblock version # is greater than 0, read in the indexed
* storage B-tree internal 'K' value
*/
if(super_vers > HDF5_SUPERBLOCK_VERSION_DEF) {
UINT16DECODE(image, chunk_btree_k);
/* Reserved bytes are present only in version 1 */
if(super_vers == HDF5_SUPERBLOCK_VERSION_1)
image += 2; /* reserved */
} /* end if */
else
chunk_btree_k = HDF5_BTREE_CHUNK_IK_DEF;
udata->btree_k[H5B_CHUNK_ID] = chunk_btree_k;
/* Remainder of "variable-sized" portion of superblock */
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->driver_addr/*out*/);
/* Allocate space for the root group symbol table entry */
HDassert(!sblock->root_ent);
if(NULL == (sblock->root_ent = (H5G_entry_t *)H5MM_calloc(sizeof(H5G_entry_t))))
HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for root group symbol table entry")
/* decode the root group symbol table entry */
if(H5G_ent_decode(udata->f, (const uint8_t **)&image, sblock->root_ent) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode root group symbol table entry")
/* Set the root group address to the correct value */
sblock->root_addr = sblock->root_ent->header;
/* This step is for h5repart tool only. If user wants to change file driver
* from family to sec2 while using h5repart, set the driver address to
* undefined to let the library ignore the family driver information saved
* in the superblock.
*/
if(udata->ignore_drvrinfo && H5F_addr_defined(sblock->driver_addr)) {
/* Eliminate the driver info */
sblock->driver_addr = HADDR_UNDEF;
udata->drvrinfo_removed = TRUE;
} /* end if */
/* NOTE: Driver info block is decoded separately, later */
} /* end if */
else {
uint32_t computed_chksum; /* Computed checksum */
uint32_t read_chksum; /* Checksum read from file */
/* Size of file addresses */
sizeof_addr = *image++;
if(sizeof_addr != 2 && sizeof_addr != 4 &&
sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address")
sblock->sizeof_addr = sizeof_addr;
udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */
/* Size of file sizes */
sizeof_size = *image++;
if(sizeof_size != 2 && sizeof_size != 4 &&
sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size")
sblock->sizeof_size = sizeof_size;
udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */
/* File status flags (not really used yet) */
sblock->status_flags = *image++;
if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock")
/* Base, superblock extension, end of file & root group object header addresses */
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/);
H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->root_addr/*out*/);
/* Compute checksum for superblock */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* Decode checksum */
UINT32DECODE(image, read_chksum);
/* Verify correct checksum */
if(read_chksum != computed_chksum)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "bad checksum on driver information block")
/* The Driver Information Block may not appear with the version
* 2 super block. Thus we set the driver_addr field of the in
* core representation of the super block HADDR_UNDEF to prevent
* any attempt to load the Driver Information Block.
*/
sblock->driver_addr = HADDR_UNDEF;
} /* end else */
/* The Driver Information Block may not appear with the version
* 2 super block. Thus we set the driver_addr field of the in
* core representation of the super block HADDR_UNDEF to prevent
* any attempt to load the Driver Information Block.
*/
sblock->driver_addr = HADDR_UNDEF;
} /* end else */
/* Sanity check */
@ -759,9 +881,9 @@ done:
/*-------------------------------------------------------------------------
* Function: H5F__cache_drvrinfo_get_load_size
* Function: H5F__cache_drvrinfo_get_initial_load_size
*
* Purpose: Compute the size of the data structure on disk.
* Purpose: Compute the intiial size of the data structure on disk.
*
* Return: Non-negative on success/Negative on failure
*
@ -772,7 +894,7 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5F__cache_drvrinfo_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *image_len)
H5F__cache_drvrinfo_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len)
{
FUNC_ENTER_STATIC_NOERR
@ -783,7 +905,76 @@ H5F__cache_drvrinfo_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *imag
*image_len = H5F_DRVINFOBLOCK_HDR_SIZE; /* Fixed size portion of driver info block */
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5F__cache_drvrinfo_get_load_size() */
} /* end H5F__cache_drvrinfo_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5F__cache_drvrinfo_get_final_load_size
*
* Purpose: Compute the final size of the data structure on disk.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Quincey Koziol
* koziol@lbl.gov
* November 17, 2016
*
*-------------------------------------------------------------------------
*/
static herr_t
H5F__cache_drvrinfo_get_final_load_size(const void *_image, size_t image_len,
void *_udata, size_t *actual_len)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5F_drvrinfo_cache_ud_t *udata = (H5F_drvrinfo_cache_ud_t *)_udata; /* User data */
unsigned drv_vers; /* Version of driver info block */
size_t drvinfo_len; /* Length of encoded buffer */
haddr_t eoa; /* Current EOA for the file */
haddr_t min_eoa; /* Minimum EOA needed for reading the driver info */
htri_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Check arguments */
HDassert(image);
HDassert(udata);
HDassert(udata->f);
HDassert(actual_len);
HDassert(*actual_len == image_len);
/* Version number */
drv_vers = *image++;
if(drv_vers != HDF5_DRIVERINFO_VERSION_0)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad driver information block version number")
image += 3; /* reserved bytes */
/* Driver info size */
UINT32DECODE(image, drvinfo_len);
/* Sanity check */
HDassert(image_len == H5F_DRVINFOBLOCK_HDR_SIZE);
/* Extend the EOA if required so that we can read the complete driver info block */
/* Get current EOA... */
if((eoa = H5FD_get_eoa(udata->f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "driver get_eoa request failed")
/* ... if it is too small, extend it. */
min_eoa = udata->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo_len;
/* If it grew, set it */
if(H5F_addr_gt(min_eoa, eoa))
if(H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed")
/* Set the final size for the cache image */
*actual_len = H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo_len;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F__cache_drvrinfo_get_final_load_size() */
/*-------------------------------------------------------------------------
@ -838,35 +1029,11 @@ H5F__cache_drvrinfo_deserialize(const void *_image, size_t len, void *_udata,
drv_name[8] = '\0';
image += 8; /* advance past name/version */
/* Handle metadata cache retry for variable-sized portion of the driver info block */
if(len != (H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len)) {
/* Sanity check */
HDassert(len == H5F_DRVINFOBLOCK_HDR_SIZE);
HDassert(len == (H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len));
/* extend the eoa if required so that we can read the complete driver info block */
{
haddr_t eoa;
haddr_t min_eoa;
/* get current eoa... */
if ((eoa = H5FD_get_eoa(udata->f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, NULL, "driver get_eoa request failed")
/* ... if it is too small, extend it. */
min_eoa = udata->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len;
if ( H5F_addr_gt(min_eoa, eoa) )
if(H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, \
"set end of space allocation request failed")
}
} /* end if */
else {
/* Validate and decode driver information */
if(H5FD_sb_load(udata->f->shared->lf, drv_name, image) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "unable to decode driver information")
} /* end if */
/* Validate and decode driver information */
if(H5FD_sb_load(udata->f->shared->lf, drv_name, image) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "unable to decode driver information")
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) <= len);
@ -911,7 +1078,7 @@ H5F__cache_drvrinfo_image_len(const void *_thing, size_t *image_len)
/* Set the image length size */
*image_len = (size_t)(H5F_DRVINFOBLOCK_HDR_SIZE + /* Fixed-size portion of driver info block */
drvinfo->len); /* Variable-size portion of driver info block */
drvinfo->len); /* Variable-size portion of driver info block */
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5F__cache_drvrinfo_image_len() */

34
src/H5Gcache.c
View File

@ -63,7 +63,7 @@
/********************/
/* Metadata cache (H5AC) callbacks */
static herr_t H5G__cache_node_get_load_size(const void *udata, size_t *image_len);
static herr_t H5G__cache_node_get_initial_load_size(void *udata, size_t *image_len);
static void *H5G__cache_node_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5G__cache_node_image_len(const void *thing, size_t *image_len);
@ -92,7 +92,9 @@ const H5AC_class_t H5AC_SNODE[1] = {{
"Symbol table node", /* Metadata client name (for debugging) */
H5FD_MEM_BTREE, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5G__cache_node_get_load_size, /* 'get_load_size' callback */
H5G__cache_node_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
NULL, /* 'verify_chksum' callback */
H5G__cache_node_deserialize, /* 'deserialize' callback */
H5G__cache_node_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -111,17 +113,11 @@ H5FL_SEQ_EXTERN(H5G_entry_t);
/*-------------------------------------------------------------------------
* Function: H5G__cache_node_get_load_size()
* Function: H5G__cache_node_get_initial_load_size()
*
* Purpose: Determine the size of the on disk image of the node, and
* Purpose: Determine the size of the on-disk image of the node, and
* return this value in *image_len.
*
* Note that this computation requires access to the file pointer,
* which is not provided in the parameter list for this callback.
* Finesse this issue by passing in the file pointer twice to the
* H5AC_protect() call -- once as the file pointer proper, and
* again as the user data.
*
* Return: Success: SUCCEED
* Failure: FAIL
*
@ -131,9 +127,9 @@ H5FL_SEQ_EXTERN(H5G_entry_t);
*-------------------------------------------------------------------------
*/
static herr_t
H5G__cache_node_get_load_size(const void *_udata, size_t *image_len)
H5G__cache_node_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5F_t *f = (const H5F_t *)_udata; /* User data for callback */
H5F_t *f = (H5F_t *)_udata; /* User data for callback */
FUNC_ENTER_STATIC_NOERR
@ -141,11 +137,11 @@ H5G__cache_node_get_load_size(const void *_udata, size_t *image_len)
HDassert(f);
HDassert(image_len);
/* report image length */
/* Set the image length size */
*image_len = (size_t)(H5G_NODE_SIZE(f));
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5G__cache_node_get_load_size() */
} /* end H5G__cache_node_get_initial_load_size() */
/*-------------------------------------------------------------------------
@ -256,11 +252,6 @@ H5G__cache_node_image_len(const void *_thing, size_t *image_len)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5G__cache_node_image_len() */
/*************************************/
/* no H5G__cache_node_pre_serialize() */
/*************************************/
/*-------------------------------------------------------------------------
* Function: H5G__cache_node_serialize
@ -320,11 +311,6 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5G__cache_node_serialize() */
/***************************************/
/* no H5G__cache_node_notify() function */
/***************************************/
/*-------------------------------------------------------------------------
* Function: H5G__cache_node_free_icr

548
src/H5HFcache.c
View File

@ -73,7 +73,10 @@ static herr_t H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *d
static herr_t H5HF__dtable_decode(H5F_t *f, const uint8_t **pp, H5HF_dtable_t *dtable);
/* Metadata cache (H5AC) callbacks */
static herr_t H5HF__cache_hdr_get_load_size(const void *udata, size_t *image_len);
static herr_t H5HF__cache_hdr_get_initial_load_size(void *udata, size_t *image_len);
static herr_t H5HF__cache_hdr_get_final_load_size(const void *image_ptr,
size_t image_len, void *udata, size_t *actual_len);
static htri_t H5HF__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5HF__cache_hdr_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5HF__cache_hdr_image_len(const void *thing, size_t *image_len);
@ -84,7 +87,8 @@ static herr_t H5HF__cache_hdr_serialize(const H5F_t *f, void *image,
size_t len, void *thing);
static herr_t H5HF__cache_hdr_free_icr(void *thing);
static herr_t H5HF__cache_iblock_get_load_size(const void *udata, size_t *image_len);
static herr_t H5HF__cache_iblock_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5HF__cache_iblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5HF__cache_iblock_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5HF__cache_iblock_image_len(const void *thing, size_t *image_len);
@ -96,7 +100,8 @@ static herr_t H5HF__cache_iblock_serialize(const H5F_t *f, void *image,
static herr_t H5HF__cache_iblock_notify(H5C_notify_action_t action, void *thing);
static herr_t H5HF__cache_iblock_free_icr(void *thing);
static herr_t H5HF__cache_dblock_get_load_size(const void *udata, size_t *image_len);
static herr_t H5HF__cache_dblock_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5HF__cache_dblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5HF__cache_dblock_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5HF__cache_dblock_image_len(const void *thing, size_t *image_len);
@ -131,7 +136,9 @@ const H5AC_class_t H5AC_FHEAP_HDR[1] = {{
"fractal heap header", /* Metadata client name (for debugging) */
H5FD_MEM_FHEAP_HDR, /* File space memory type for client */
H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */
H5HF__cache_hdr_get_load_size, /* 'get_load_size' callback */
H5HF__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */
H5HF__cache_hdr_get_final_load_size, /* 'get_final_load_size' callback */
H5HF__cache_hdr_verify_chksum, /* 'verify_chksum' callback */
H5HF__cache_hdr_deserialize, /* 'deserialize' callback */
H5HF__cache_hdr_image_len, /* 'image_len' callback */
H5HF__cache_hdr_pre_serialize, /* 'pre_serialize' callback */
@ -147,7 +154,9 @@ const H5AC_class_t H5AC_FHEAP_IBLOCK[1] = {{
"fractal heap indirect block", /* Metadata client name (for debugging) */
H5FD_MEM_FHEAP_IBLOCK, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5HF__cache_iblock_get_load_size, /* 'get_load_size' callback */
H5HF__cache_iblock_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5HF__cache_iblock_verify_chksum, /* 'verify_chksum' callback */
H5HF__cache_iblock_deserialize, /* 'deserialize' callback */
H5HF__cache_iblock_image_len, /* 'image_len' callback */
H5HF__cache_iblock_pre_serialize, /* 'pre_serialize' callback */
@ -163,7 +172,9 @@ const H5AC_class_t H5AC_FHEAP_DBLOCK[1] = {{
"fractal heap direct block", /* Metadata client name (for debugging) */
H5FD_MEM_FHEAP_DBLOCK, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5HF__cache_dblock_get_load_size, /* 'get_load_size' callback */
H5HF__cache_dblock_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5HF__cache_dblock_verify_chksum, /* 'verify_chksum' callback */
H5HF__cache_dblock_deserialize, /* 'deserialize' callback */
H5HF__cache_dblock_image_len, /* 'image_len' callback */
H5HF__cache_dblock_pre_serialize, /* 'pre_serialize' callback */
@ -287,21 +298,13 @@ H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HF__dtable_encode() */
/**************************************************/
/* metadata cache callback definitions for header */
/**************************************************/
/*-------------------------------------------------------------------------
* Function: H5HF__cache_hdr_get_load_size()
* Function: H5HF__cache_hdr_get_initial_load_size()
*
* Purpose: Determine the size of the fractal heap header on disk,
* and set *image_len to this value.
*
* This code is based on the old H5HF_cache_hdr_load() routine
* that was used with the version 2 metadata cache. Note the
* use of a dummy header to compute the on disk size of the header.
*
* Note also that the value returned by this function presumes that
* there is no I/O filtering data in the header. If there is, the
* size reported will be too small, and H5C_load_entry()
@ -316,10 +319,10 @@ H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable)
*-------------------------------------------------------------------------
*/
static herr_t
H5HF__cache_hdr_get_load_size(const void *_udata, size_t *image_len)
H5HF__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5HF_hdr_cache_ud_t *udata = (const H5HF_hdr_cache_ud_t *)_udata; /* pointer to user data */
H5HF_hdr_t dummy_hdr; /* dummy header -- to compute size */
H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* Pointer to user data */
H5HF_hdr_t dummy_hdr; /* Dummy header -- to compute size */
FUNC_ENTER_STATIC_NOERR
@ -336,7 +339,99 @@ H5HF__cache_hdr_get_load_size(const void *_udata, size_t *image_len)
*image_len = (size_t)H5HF_HEADER_SIZE(&dummy_hdr);
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HF__cache_hdr_get_load_size() */
} /* end H5HF__cache_hdr_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5HF__cache_hdr_get_final_load_size()
*
* Purpose: Determine the final size of the fractal heap header on disk,
* and set *actual_len to this value.
*
* Return: Success: SUCCEED
* Failure: FAIL
*
* Programmer: Quincey Koziol
* November 18, 2016
*
*-------------------------------------------------------------------------
*/
static herr_t
H5HF__cache_hdr_get_final_load_size(const void *_image, size_t image_len,
void *_udata, size_t *actual_len)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* pointer to user data */
unsigned filter_len; /* Size of I/O filter information (in bytes) */
htri_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Sanity checks */
HDassert(image);
HDassert(udata);
HDassert(actual_len);
HDassert(*actual_len == image_len);
/* Magic number */
if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC))
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "wrong fractal heap header signature")
image += H5_SIZEOF_MAGIC;
/* Version */
if(*image++ != H5HF_HDR_VERSION)
HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong fractal heap header version")
/* General heap information */
image += 2; /* Heap ID length */
UINT16DECODE(image, filter_len); /* I/O filters' encoded length */
/* Check for I/O filter info on this heap */
if(filter_len > 0)
/* Compute the extra heap header size */
*actual_len += (size_t)(H5F_SIZEOF_SIZE(udata->f) /* Size of size for filtered root direct block */
+ (unsigned)4 /* Size of filter mask for filtered root direct block */
+ filter_len); /* Size of encoded I/O filter info */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5HF__cache_hdr_get_final_load_size() */
/*-------------------------------------------------------------------------
* Function: H5HF__cache_hdr_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
static htri_t
H5HF__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5HF__cache_hdr_verify_chksum() */
/*-------------------------------------------------------------------------
@ -347,30 +442,6 @@ H5HF__cache_hdr_get_load_size(const void *_udata, size_t *image_len)
* of the buffer into into the new instance of H5HF_hdr_t, and then
* return a pointer to the new instance.
*
* Since H5HF__cache_hdr_get_load_size() reports header on disk size
* base on the assumption that the header contains no I/O filtering
* data, it is possible that the provided image will be too small.
*
* In this case, we DO NOT flag an error when this is discovered.
* Instead, we make note of the correct image size, and report
* success.
*
* Since H5HF__cache_hdr_image_len() callback is defined,
* H5C_load_entry() will call H5HF__cache_hdr_image_len() and
* obtain the correct image length.
*
* Since the H5AC__CLASS_SPECULATIVE_LOAD_FLAG is set,
* H5C_load_entry() will load an image of the correct size, and
* then call this function again to deserialize it. Before doing
* so, it will also call H5HF__cache_hdr_free_icr() to discard the
* result of the first deserialize call.
*
* Note that the v2 B-tree and free space manager associated
* with the fractal heap (roots stored in the huge_bt2 and fspace
* fields respectively) are not loaded at this time. As best I can
* tell from reviewing the code, they are loaded or created when
* they are accessed.
*
* Return: Success: Pointer to in core representation
* Failure: NULL
*
@ -386,9 +457,7 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
H5HF_hdr_t *hdr = NULL; /* Fractal heap info */
H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* User data for callback */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into into supplied image */
size_t size; /* Header size */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
uint8_t heap_flags; /* Status flags for heap */
void * ret_value = NULL; /* Return value */
@ -404,28 +473,6 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
if(NULL == (hdr = H5HF_hdr_alloc(udata->f)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
/* Compute the 'base' size of the fractal heap header on disk */
size = (size_t)H5HF_HEADER_SIZE(hdr);
/* the size we have just calculated presumes that there is no I/O
* filter information in the header. If there is no filter information,
* the deserialize operation should succeed.
*
* If there is filter information, the first attempt to deserialize
* the header will reveal this. In this case, we will be unable to
* deserialize the header as the supplied image will be too small.
* However, we will make note of the correct size and report success
* anyway.
*
* When H5C_load_entry() calls H5HF__cache_hdr_image_len(), we will report
* the correct size. Since the H5C__CLASS_SPECULATIVE_LOAD_FLAG is set,
* this will prompt H5C_load_entry() to load the correct size image,
* discard the result of the first attempt at deserialization, and
* call this routine a second time to deserialize the correct size
* buffer.
*/
HDassert(size <= len);
/* Magic number */
if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC))
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "wrong fractal heap header signature")
@ -469,32 +516,24 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
if(H5HF__dtable_decode(hdr->f, &image, &(hdr->man_dtable)) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTENCODE, NULL, "unable to encode managed obj. doubling table info")
/* Set the fractal heap header's 'base' size */
hdr->heap_size = (size_t)H5HF_HEADER_SIZE(hdr);
/* Sanity check */
/* (allow for checksum not decoded yet) */
HDassert((size_t)(image - (const uint8_t *)_image) == (size - H5HF_SIZEOF_CHKSUM));
HDassert((size_t)(image - (const uint8_t *)_image) == (hdr->heap_size - H5HF_SIZEOF_CHKSUM));
/* Check for I/O filter information to decode */
if(hdr->filter_len > 0) {
size_t filter_info_size; /* Size of filter information */
H5O_pline_t *pline; /* Pipeline information from the header on disk */
/* Compute the size of the extra filter information */
filter_info_size = (size_t)(hdr->sizeof_size /* Size of size for filtered root direct block */
+ (unsigned)4 /* Size of filter mask for filtered root direct block */
+ hdr->filter_len); /* Size of encoded I/O filter info */
/* Sanity check */
HDassert(len > hdr->heap_size); /* A header with filter info is > than a standard header */
/* Compute the heap header's size */
hdr->heap_size = size + filter_info_size;
if(size == len)
/* we were supplied with too small a buffer -- goto done
* and let H5C_load_entry() retry with a larger buffer
*/
HGOTO_DONE((void *)hdr)
else
if((size + filter_info_size) != len)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "bad image len")
hdr->heap_size += (size_t)(hdr->sizeof_size /* Size of size for filtered root direct block */
+ (unsigned)4 /* Size of filter mask for filtered root direct block */
+ hdr->filter_len); /* Size of encoded I/O filter info */
/* Decode the size of a filtered root direct block */
H5F_DECODE_LENGTH(udata->f, image, hdr->pline_root_direct_size);
@ -506,6 +545,7 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
if(NULL == (pline = (H5O_pline_t *)H5O_msg_decode(hdr->f, udata->dxpl_id, NULL, H5O_PLINE_ID, image)))
HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode I/O pipeline filters")
/* Advance past filter info to checksum */
image += hdr->filter_len;
/* Copy the information into the header's I/O pipeline structure */
@ -515,13 +555,6 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
/* Release the space allocated for the I/O pipeline filters */
H5O_msg_free(H5O_PLINE_ID, pline);
} /* end if */
else
/* Set the heap header's size */
hdr->heap_size = size;
/* Compute checksum on entire header */
/* (including the filter information, if present) */
computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0);
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -529,10 +562,6 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == hdr->heap_size);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap header")
/* Finish initialization of heap header */
if(H5HF_hdr_finish_init(hdr) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, NULL, "can't finish initializing shared fractal heap header")
@ -556,7 +585,7 @@ done:
* disk image.
*
* If the header contains filter information, this size will be
* larger than the value returned by H5HF__cache_hdr_get_load_size().
* larger than the value returned by H5HF__cache_hdr_get_initial_load_size().
*
* Return: Success: SUCCEED
* Failure: FAIL
@ -770,10 +799,6 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5HF__cache_hdr_serialize() */
/***************************************/
/* no H5HF__cache_hdr_notify() function */
/***************************************/
/*-------------------------------------------------------------------------
* Function: H5HF__cache_hdr_free_icr
@ -820,13 +845,9 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5HF__cache_hdr_free_icr() */
/***********************************************************/
/* metadata cache callback definitions for indirect blocks */
/***********************************************************/
/*-------------------------------------------------------------------------
* Function: H5HF__cache_iblock_get_load_size()
* Function: H5HF__cache_iblock_get_initial_load_size()
*
* Purpose: Compute the size of the on disk image of the indirect
* block, and place this value in *image_len.
@ -840,24 +861,60 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5HF__cache_iblock_get_load_size(const void *_udata, size_t *image_len)
H5HF__cache_iblock_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5HF_iblock_cache_ud_t *udata = (const H5HF_iblock_cache_ud_t *)_udata; /* User data for callback */
H5HF_iblock_cache_ud_t *udata = (H5HF_iblock_cache_ud_t *)_udata; /* User data for callback */
FUNC_ENTER_STATIC_NOERR
/* Sanity checks */
HDassert(udata);
HDassert(udata->par_info);
HDassert(udata->par_info->hdr);
HDassert(image_len);
/* Set the image length size */
*image_len = (size_t)H5HF_MAN_INDIRECT_SIZE(udata->par_info->hdr, *udata->nrows);
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HF__cache_iblock_get_load_size() */
} /* end H5HF__cache_iblock_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5HF__cache_iblock_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
static htri_t
H5HF__cache_iblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5HF__cache_iblock_verify_chksum() */
/***********************************************************/
/* metadata cache callback definitions for indirect blocks */
/***********************************************************/
/*-------------------------------------------------------------------------
@ -890,7 +947,6 @@ H5HF__cache_iblock_deserialize(const void *_image, size_t len, void *_udata,
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
haddr_t heap_addr; /* Address of heap header in the file */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
unsigned u; /* Local index variable */
void * ret_value = NULL; /* Return value */
@ -1017,8 +1073,7 @@ H5HF__cache_iblock_deserialize(const void *_image, size_t len, void *_udata,
/* Sanity check */
HDassert(iblock->nchildren); /* indirect blocks w/no children should have been deleted */
/* Compute checksum on indirect block */
computed_chksum = H5_checksum_metadata((const uint8_t *)_image, (size_t)(image - (const uint8_t *)_image), 0);
/* checksum verification already done by verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
@ -1026,10 +1081,6 @@ H5HF__cache_iblock_deserialize(const void *_image, size_t len, void *_udata,
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == iblock->size);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block")
/* Check if we have any indirect block children */
if(iblock->nrows > hdr->man_dtable.max_direct_rows) {
unsigned indir_rows;/* Number of indirect rows in this indirect block */
@ -1481,7 +1532,7 @@ done:
/*-------------------------------------------------------------------------
* Function: H5HF__cache_dblock_get_load_size()
* Function: H5HF__cache_dblock_get_initial_load_size()
*
* Purpose: Determine the size of the direct block on disk image, and
* return it in *image_len.
@ -1495,44 +1546,158 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5HF__cache_dblock_get_load_size(const void *_udata, size_t *image_len)
H5HF__cache_dblock_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5HF_dblock_cache_ud_t *udata = (const H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */
const H5HF_parent_t *par_info; /* Pointer to parent information */
const H5HF_hdr_t *hdr; /* Shared fractal heap information */
size_t size;
const H5HF_parent_t *par_info; /* Pointer to parent information */
const H5HF_hdr_t *hdr; /* Shared fractal heap information */
FUNC_ENTER_STATIC_NOERR
/* Sanity checks */
HDassert(udata);
HDassert(image_len);
/* Convenience variables */
par_info = (const H5HF_parent_t *)(&(udata->par_info));
HDassert(par_info);
hdr = par_info->hdr;
HDassert(hdr);
HDassert(hdr->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
HDassert(hdr->cache_info.type == H5AC_FHEAP_HDR);
/* Check for I/O filters on this heap */
if(hdr->filter_len > 0) {
/* Check for root direct block */
if(par_info->iblock == NULL)
size = hdr->pline_root_direct_size;
/* filtered root direct block */
*image_len = hdr->pline_root_direct_size;
else
size = par_info->iblock->filt_ents[par_info->entry].size;
/* filtered direct block */
*image_len = par_info->iblock->filt_ents[par_info->entry].size;
} /* end if */
else
*image_len = udata->dblock_size;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HF__cache_dblock_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5HF__cache_dblock_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
static htri_t
H5HF__cache_dblock_verify_chksum(const void *_image, size_t len, void *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5HF_dblock_cache_ud_t *udata = (H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */
void *read_buf = NULL; /* Pointer to buffer to read in */
H5HF_hdr_t *hdr; /* Shared fractal heap information */
H5HF_parent_t *par_info; /* Pointer to parent information */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
size_t chk_size; /* The size for validating checksum */
uint8_t *chk_p; /* Pointer to the area for validating checksum */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC
/* Sanity checks */
HDassert(image);
HDassert(udata);
par_info = (H5HF_parent_t *)(&(udata->par_info));
HDassert(par_info);
hdr = par_info->hdr;
HDassert(hdr);
/* Reset callback context info */
udata->decompressed = FALSE;
udata->dblk = NULL;
/* Get out if data block is not checksummed */
if(!(hdr->checksum_dblocks))
HGOTO_DONE(TRUE);
if(hdr->filter_len > 0) {
size_t nbytes; /* Number of bytes used in buffer, after applying reverse filters */
unsigned filter_mask; /* Excluded filters for direct block */
H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */
/* Allocate buffer to perform I/O filtering on and copy image into
* it. Must do this as H5Z_pipeline() may re-size the buffer
* provided to it.
*/
if(NULL == (read_buf = H5MM_malloc(len)))
HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline buffer")
/* Set up parameters for filter pipeline */
nbytes = len;
filter_mask = udata->filter_mask;
HDmemcpy(read_buf, image, len);
/* Push direct block data through I/O filter pipeline */
if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, filter_cb, &nbytes, &len, &read_buf) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, FAIL, "output pipeline failed")
/* Update info about direct block */
udata->decompressed = TRUE;
len = nbytes;
} /* end if */
else
size = udata->dblock_size;
read_buf = (void *)image; /* Casting away const OK - QAK */
*image_len = size;
/* Decode checksum */
chk_size = (size_t)(H5HF_MAN_ABS_DIRECT_OVERHEAD(hdr) - H5HF_SIZEOF_CHKSUM);
chk_p = (uint8_t *)read_buf + chk_size;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HF__cache_dblock_get_load_size() */
/* Metadata checksum */
UINT32DECODE(chk_p, stored_chksum);
/*********************************************************/
/* metadata cache callback definitions for direct blocks */
/*********************************************************/
chk_p -= H5HF_SIZEOF_CHKSUM;
/* Reset checksum field, for computing the checksum */
/* (Casting away const OK - QAK) */
HDmemset(chk_p, 0, (size_t)H5HF_SIZEOF_CHKSUM);
/* Compute checksum on entire direct block */
computed_chksum = H5_checksum_metadata(read_buf, len, 0);
/* Restore the checksum */
UINT32ENCODE(chk_p, stored_chksum)
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_DONE(FALSE);
/* Save the decompressed data to be used later in deserialize callback */
if(hdr->filter_len > 0) {
/* Sanity check */
HDassert(udata->decompressed);
HDassert(len == udata->dblock_size);
/* Allocate block buffer */
if(NULL == (udata->dblk = H5FL_BLK_MALLOC(direct_block, (size_t)len)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed")
/* Copy un-filtered data into block's buffer */
HDmemcpy(udata->dblk, read_buf, len);
} /* end if */
done:
/* Release the read buffer */
if(read_buf && read_buf != image)
H5MM_xfree(read_buf);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5HF__cache_dblock_verify_chksum() */
/*-------------------------------------------------------------------------
@ -1563,14 +1728,15 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata,
H5HF_dblock_cache_ud_t *udata = (H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */
H5HF_parent_t *par_info; /* Pointer to parent information */
H5HF_direct_t *dblock = NULL; /* Direct block info */
const uint8_t *image; /* Pointer into raw data buffer */
const uint8_t *image = _image;/* Pointer into raw data buffer */
void *read_buf = NULL; /* Pointer to buffer to decompress */
haddr_t heap_addr; /* Address of heap header in the file */
void * ret_value = NULL; /* Return value */
FUNC_ENTER_STATIC
/* Sanity checks */
HDassert(_image);
HDassert(image);
HDassert(udata);
par_info = (H5HF_parent_t *)(&(udata->par_info));
HDassert(par_info);
@ -1581,7 +1747,7 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata,
HDassert(dirty);
/* Allocate space for the fractal heap direct block */
if(NULL == (dblock = H5FL_MALLOC(H5HF_direct_t)))
if(NULL == (dblock = H5FL_CALLOC(H5HF_direct_t)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
HDmemset(&dblock->cache_info, 0, sizeof(H5AC_info_t));
@ -1595,61 +1761,62 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata,
/* Set block's internal information */
dblock->size = udata->dblock_size;
dblock->file_size = 0;
/* initialize fields used in serialization */
dblock->write_buf = NULL;
dblock->write_size = 0;
/* Allocate block buffer */
/* XXX: Change to using free-list factories */
if(NULL == (dblock->blk = H5FL_BLK_MALLOC(direct_block, (size_t)dblock->size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
/* Check for I/O filters on this heap */
if(hdr->filter_len > 0) {
H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */
size_t nbytes; /* Number of bytes used in buffer, after applying reverse filters */
void *read_buf; /* Pointer to buffer to read in */
size_t read_size; /* Size of filtered direct block to read */
unsigned filter_mask; /* Excluded filters for direct block */
/* Direct block is already decompressed in verify_chksum callback */
if(udata->decompressed) {
/* Sanity check */
HDassert(udata->dblk);
/* Check for root direct block */
if(par_info->iblock == NULL)
/* Set up parameters to read filtered direct block */
read_size = hdr->pline_root_direct_size;
else
/* Set up parameters to read filtered direct block */
read_size = par_info->iblock->filt_ents[par_info->entry].size;
HDassert(len == read_size);
/* Take ownership of the decompressed direct block */
dblock->blk = udata->dblk;
udata->dblk = NULL;
} /* end if */
else {
H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */
size_t nbytes; /* Number of bytes used in buffer, after applying reverse filters */
unsigned filter_mask; /* Excluded filters for direct block */
/* Allocate buffer to perform I/O filtering on and copy image into
* it. Must do this as H5Z_pipeline() may re-sized the buffer
* provided to it.
*/
if(NULL == (read_buf = H5MM_malloc(read_size)))
HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "memory allocation failed for pipeline buffer")
HDmemcpy(read_buf, _image, len);
/* Sanity check */
HDassert(udata->dblk == NULL);
/* Push direct block data through I/O filter pipeline */
nbytes = read_size;
filter_mask = udata->filter_mask;
if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, filter_cb, &nbytes, &read_size, &read_buf) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, NULL, "output pipeline failed")
/* Allocate buffer to perform I/O filtering on and copy image into
* it. Must do this as H5Z_pipeline() may resize the buffer
* provided to it.
*/
if(NULL == (read_buf = H5MM_malloc(len)))
HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "memory allocation failed for pipeline buffer")
/* Sanity check */
HDassert(nbytes == dblock->size);
/* Copy compressed image into buffer */
HDmemcpy(read_buf, image, len);
/* Copy un-filtered data into block's buffer */
HDmemcpy(dblock->blk, read_buf, dblock->size);
/* Push direct block data through I/O filter pipeline */
nbytes = len;
filter_mask = udata->filter_mask;
if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, filter_cb, &nbytes, &len, &read_buf) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, NULL, "output pipeline failed")
/* Release the read buffer */
H5MM_xfree(read_buf);
/* Sanity check */
HDassert(nbytes == dblock->size);
/* Copy un-filtered data into block's buffer */
HDmemcpy(dblock->blk, read_buf, dblock->size);
} /* end if */
} /* end if */
else {
/* copy image to dblock->blk */
/* Sanity checks */
HDassert(udata->dblk == NULL);
HDassert(!udata->decompressed);
/* Allocate block buffer */
/* XXX: Change to using free-list factories */
if(NULL == (dblock->blk = H5FL_BLK_MALLOC(direct_block, (size_t)dblock->size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
/* Copy image to dblock->blk */
HDassert(dblock->size == len);
HDmemcpy(dblock->blk, _image, dblock->size);
HDmemcpy(dblock->blk, image, dblock->size);
} /* end else */
/* Start decoding direct block */
@ -1684,22 +1851,12 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata,
/* Decode checksum on direct block, if requested */
if(hdr->checksum_dblocks) {
uint32_t stored_chksum; /* Metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
uint32_t stored_chksum; /* Metadata checksum value */
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(image, stored_chksum);
/* Reset checksum field, for computing the checksum */
/* (Casting away const OK - QAK) */
HDmemset((uint8_t *)image - H5HF_SIZEOF_CHKSUM, 0, (size_t)H5HF_SIZEOF_CHKSUM);
/* Compute checksum on entire direct block */
computed_chksum = H5_checksum_metadata(dblock->blk, dblock->size, 0);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap direct block")
} /* end if */
/* Sanity check */
@ -1709,6 +1866,11 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata,
ret_value = (void *)dblock;
done:
/* Release the read buffer */
if(read_buf)
H5MM_xfree(read_buf);
/* Cleanup on error */
if(!ret_value && dblock)
if(H5HF_man_dblock_dest(dblock) < 0)
HDONE_ERROR(H5E_HEAP, H5E_CANTFREE, NULL, "unable to destroy fractal heap direct block")

8
src/H5HFpkg.h
View File

@ -544,6 +544,14 @@ typedef struct H5HF_dblock_cache_ud_t {
* calls to it.
*/
unsigned filter_mask; /* Excluded filters for direct block */
uint8_t *dblk; /* Pointer to the buffer containing the decompressed
* direct block data obtained in verify_chksum callback.
* It will be used later in deserialize callback.
*/
htri_t decompressed; /* Indicate that the direct block has been
* decompressed in verify_chksum callback.
* It will be used later in deserialize callback.
*/
} H5HF_dblock_cache_ud_t;

263
src/H5HGcache.c
View File

@ -62,7 +62,9 @@
/********************/
/* Metadata cache callbacks */
static herr_t H5HG__cache_heap_get_load_size(const void *udata, size_t *image_len);
static herr_t H5HG__cache_heap_get_initial_load_size(void *udata, size_t *image_len);
static herr_t H5HG__cache_heap_get_final_load_size(const void *_image,
size_t image_len, void *udata, size_t *actual_len);
static void *H5HG__cache_heap_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5HG__cache_heap_image_len(const void *thing, size_t *image_len);
@ -81,7 +83,9 @@ const H5AC_class_t H5AC_GHEAP[1] = {{
"global heap", /* Metadata client name (for debugging) */
H5FD_MEM_GHEAP, /* File space memory type for client */
H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */
H5HG__cache_heap_get_load_size, /* 'get_load_size' callback */
H5HG__cache_heap_get_initial_load_size, /* 'get_initial_load_size' callback */
H5HG__cache_heap_get_final_load_size, /* 'get_final_load_size' callback */
NULL, /* 'verify_chksum' callback */
H5HG__cache_heap_deserialize, /* 'deserialize' callback */
H5HG__cache_heap_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -104,13 +108,13 @@ const H5AC_class_t H5AC_GHEAP[1] = {{
/*-------------------------------------------------------------------------
* Function: H5HG__cache_heap_get_load_size()
* Function: H5HG__cache_heap_get_initial_load_size()
*
* Purpose: Return the initial speculative read size to the metadata
* cache. This size will be used in the initial attempt to read
* the global heap. If this read is too small, the cache will
* try again with the correct value obtained from
* H5HG__cache_heap_image_len().
* H5HG__cache_get_final_load_size().
*
* Return: Success: SUCCEED
* Failure: FAIL
@ -121,16 +125,74 @@ const H5AC_class_t H5AC_GHEAP[1] = {{
*-------------------------------------------------------------------------
*/
static herr_t
H5HG__cache_heap_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_len)
H5HG__cache_heap_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len)
{
FUNC_ENTER_STATIC_NOERR
/* Sanity check */
HDassert(image_len);
/* Set the image length size */
*image_len = (size_t)H5HG_MINSIZE;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HG__cache_heap_get_load_size() */
} /* end H5HG__cache_heap_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5HG__cache_heap_get_initial_load_size()
*
* Purpose: Return the final read size for a speculatively ready heap to
* the metadata cache.
*
* Return: Success: SUCCEED
* Failure: FAIL
*
* Programmer: Quincey Koziol
* November 18, 2016
*
*-------------------------------------------------------------------------
*/
static herr_t
H5HG__cache_heap_get_final_load_size(const void *_image, size_t image_len,
void *_udata, size_t *actual_len)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5F_t *f = (H5F_t *)_udata; /* File pointer -- obtained from user data */
size_t heap_size = 0; /* Total size of collection */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Sanity check */
HDassert(image);
HDassert(f);
HDassert(actual_len);
HDassert(*actual_len == image_len);
/* Magic number */
if(HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC))
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad global heap collection signature")
image += H5_SIZEOF_MAGIC;
/* Version */
if(H5HG_VERSION != *image++)
HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in global heap")
/* Reserved */
image += 3;
/* Size */
H5F_DECODE_LENGTH(f, image, heap_size);
HDassert(heap_size >= H5HG_MINSIZE);
HDassert(image_len == H5HG_MINSIZE);
/* Set the final size for the cache image */
*actual_len = heap_size;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5HG__cache_heap_get_final_load_size() */
/*-------------------------------------------------------------------------
@ -140,10 +202,6 @@ H5HG__cache_heap_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_
* heap, deserialize it, load its contents into a newly allocated
* instance of H5HG_heap_t, and return a pointer to the new instance.
*
* Note that this heap client uses speculative reads. If the supplied
* buffer is too small, we simply make note of the correct size, and
* wait for the metadata cache to try again.
*
* Return: Success: Pointer to in core representation
* Failure: NULL
*
@ -159,6 +217,8 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata,
H5F_t *f = (H5F_t *)_udata; /* File pointer -- obtained from user data */
H5HG_heap_t *heap = NULL; /* New global heap */
uint8_t *image; /* Pointer to image to decode */
size_t max_idx = 0; /* Maximum heap object index seen */
size_t nalloc; /* Number of objects allocated */
void *ret_value = NULL; /* Return value */
FUNC_ENTER_STATIC
@ -199,109 +259,98 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata,
HDassert((len == H5HG_MINSIZE) /* first try */ ||
((len == heap->size) && (len > H5HG_MINSIZE))); /* second try */
if(len == heap->size) { /* proceed with the deserialize */
size_t max_idx = 0;
size_t nalloc;
/* Decode each object */
image = heap->chunk + H5HG_SIZEOF_HDR(f);
nalloc = H5HG_NOBJS(f, heap->size);
/* Decode each object */
image = heap->chunk + H5HG_SIZEOF_HDR(f);
nalloc = H5HG_NOBJS(f, heap->size);
/* Calloc the obj array because the file format spec makes no guarantee
* about the order of the objects, and unused slots must be set to zero.
*/
if(NULL == (heap->obj = H5FL_SEQ_CALLOC(H5HG_obj_t, nalloc)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
heap->nalloc = nalloc;
/* Calloc the obj array because the file format spec makes no guarantee
* about the order of the objects, and unused slots must be set to zero.
*/
if(NULL == (heap->obj = H5FL_SEQ_CALLOC(H5HG_obj_t, nalloc)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
heap->nalloc = nalloc;
while(image < (heap->chunk + heap->size)) {
if((image + H5HG_SIZEOF_OBJHDR(f)) > (heap->chunk + heap->size)) {
/*
* The last bit of space is too tiny for an object header, so
* we assume that it's free space.
*/
HDassert(NULL == heap->obj[0].begin);
heap->obj[0].size = (size_t)(((const uint8_t *)heap->chunk + heap->size) - image);
heap->obj[0].begin = image;
image += heap->obj[0].size;
} /* end if */
else {
size_t need;
unsigned idx;
uint8_t *begin = image;
while(image < (heap->chunk + heap->size)) {
if((image + H5HG_SIZEOF_OBJHDR(f)) > (heap->chunk + heap->size)) {
/*
* The last bit of space is too tiny for an object header, so
* we assume that it's free space.
*/
HDassert(NULL == heap->obj[0].begin);
heap->obj[0].size = (size_t)(((const uint8_t *)heap->chunk + heap->size) - image);
heap->obj[0].begin = image;
image += heap->obj[0].size;
UINT16DECODE(image, idx);
/* Check if we need more room to store heap objects */
if(idx >= heap->nalloc) {
size_t new_alloc; /* New allocation number */
H5HG_obj_t *new_obj; /* New array of object descriptions */
/* Determine the new number of objects to index */
new_alloc = MAX(heap->nalloc * 2, (idx + 1));
HDassert(idx < new_alloc);
/* Reallocate array of objects */
if(NULL == (new_obj = H5FL_SEQ_REALLOC(H5HG_obj_t, heap->obj, new_alloc)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
/* Clear newly allocated space */
HDmemset(&new_obj[heap->nalloc], 0, (new_alloc - heap->nalloc) * sizeof(heap->obj[0]));
/* Update heap information */
heap->nalloc = new_alloc;
heap->obj = new_obj;
HDassert(heap->nalloc > heap->nused);
} /* end if */
else {
size_t need;
unsigned idx;
uint8_t *begin = image;
UINT16DECODE(image, idx);
UINT16DECODE(image, heap->obj[idx].nrefs);
image += 4; /*reserved*/
H5F_DECODE_LENGTH(f, image, heap->obj[idx].size);
heap->obj[idx].begin = begin;
/* Check if we need more room to store heap objects */
if(idx >= heap->nalloc) {
size_t new_alloc; /* New allocation number */
H5HG_obj_t *new_obj; /* New array of object descriptions */
/*
* The total storage size includes the size of the object
* header and is zero padded so the next object header is
* properly aligned. The entire obj array was calloc'ed,
* so no need to zero the space here. The last bit of space
* is the free space object whose size is never padded and
* already includes the object header.
*/
if(idx > 0) {
need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(heap->obj[idx].size);
if(idx > max_idx)
max_idx = idx;
} /* end if */
else
need = heap->obj[idx].size;
/* Determine the new number of objects to index */
new_alloc = MAX(heap->nalloc * 2, (idx + 1));
HDassert(idx < new_alloc);
image = begin + need;
} /* end else */
} /* end while */
/* Reallocate array of objects */
if(NULL == (new_obj = H5FL_SEQ_REALLOC(H5HG_obj_t, heap->obj, new_alloc)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
/* Sanity checks */
HDassert(image == heap->chunk + heap->size);
HDassert(H5HG_ISALIGNED(heap->obj[0].size));
/* Clear newly allocated space */
HDmemset(&new_obj[heap->nalloc], 0, (new_alloc - heap->nalloc) * sizeof(heap->obj[0]));
/* Update heap information */
heap->nalloc = new_alloc;
heap->obj = new_obj;
HDassert(heap->nalloc > heap->nused);
} /* end if */
UINT16DECODE(image, heap->obj[idx].nrefs);
image += 4; /*reserved*/
H5F_DECODE_LENGTH(f, image, heap->obj[idx].size);
heap->obj[idx].begin = begin;
/*
* The total storage size includes the size of the object
* header and is zero padded so the next object header is
* properly aligned. The entire obj array was calloc'ed,
* so no need to zero the space here. The last bit of space
* is the free space object whose size is never padded and
* already includes the object header.
*/
if(idx > 0) {
need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(heap->obj[idx].size);
if(idx > max_idx)
max_idx = idx;
} /* end if */
else
need = heap->obj[idx].size;
image = begin + need;
} /* end else */
} /* end while */
HDassert(image == heap->chunk + heap->size);
HDassert(H5HG_ISALIGNED(heap->obj[0].size));
/* Set the next index value to use */
if(max_idx > 0)
heap->nused = max_idx + 1;
else
heap->nused = 1;
HDassert(max_idx < heap->nused);
/* Add the new heap to the CWFS list for the file */
if(H5F_cwfs_add(f, heap) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "unable to add global heap collection to file's CWFS")
} /* end if ( len == heap->size ) */
/* Set the next index value to use */
if(max_idx > 0)
heap->nused = max_idx + 1;
else
/* if len is less than heap size, then the initial speculative
* read was too small. In this case we return without reporting
* failure. H5C_load_entry() will call H5HG__cache_heap_image_len()
* to get the actual read size, and then repeat the read with the
* correct size, and call this function a second time.
*/
HDassert(len < heap->size);
heap->nused = 1;
/* Sanity check */
HDassert(max_idx < heap->nused);
/* Add the new heap to the CWFS list for the file */
if(H5F_cwfs_add(f, heap) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "unable to add global heap collection to file's CWFS")
ret_value = heap;
@ -347,10 +396,6 @@ H5HG__cache_heap_image_len(const void *_thing, size_t *image_len)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HG__cache_heap_image_len() */
/**************************************/
/* no H5HG_cache_heap_pre_serialize() */
/**************************************/
/*-------------------------------------------------------------------------
* Function: H5HG__cache_heap_serialize
@ -390,10 +435,6 @@ H5HG__cache_heap_serialize(const H5F_t *f, void *image, size_t len,
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HG__cache_heap_serialize() */
/****************************************/
/* no H5HG_cache_heap_notify() function */
/****************************************/
/*-------------------------------------------------------------------------
* Function: H5HG__cache_heap_free_icr

40
src/H5HL.c
View File

@ -341,12 +341,10 @@ H5HL_protect(H5F_t *f, hid_t dxpl_id, haddr_t addr, unsigned flags))
HDassert((flags & (unsigned)(~H5AC__READ_ONLY_FLAG)) == 0);
/* Construct the user data for protect callback */
prfx_udata.made_attempt = FALSE;
prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f);
prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f);
prfx_udata.prfx_addr = addr;
prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f);
prfx_udata.loaded = FALSE;
/* Protect the local heap prefix */
if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, flags)))
@ -359,25 +357,14 @@ H5HL_protect(H5F_t *f, hid_t dxpl_id, haddr_t addr, unsigned flags))
/* (for re-entrant situation) */
if(heap->prots == 0) {
/* Check if heap has separate data block */
if(heap->single_cache_obj) {
if(heap->single_cache_obj)
/* Set the flag for pinning the prefix when unprotecting it */
prfx_cache_flags |= H5AC__PIN_ENTRY_FLAG;
} /* end if */
else {
H5HL_cache_dblk_ud_t dblk_udata; /* User data for protecting local heap data block */
/* Construct the user data for protect callback */
dblk_udata.heap = heap;
dblk_udata.loaded = FALSE;
/* Protect the local heap data block */
if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, &dblk_udata, flags)))
if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, heap, flags)))
H5E_THROW(H5E_CANTPROTECT, "unable to load heap data block");
/* Pin the prefix, if the data block was loaded from file */
if(dblk_udata.loaded)
prfx_cache_flags |= H5AC__PIN_ENTRY_FLAG;
/* Set the flag for pinning the data block when unprotecting it */
dblk_cache_flags |= H5AC__PIN_ENTRY_FLAG;
} /* end if */
@ -931,12 +918,10 @@ H5HL_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr))
HDassert(H5F_addr_defined(addr));
/* Construct the user data for protect callback */
prfx_udata.made_attempt = FALSE;
prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f);
prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f);
prfx_udata.prfx_addr = addr;
prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f);
prfx_udata.loaded = FALSE;
/* Protect the local heap prefix */
if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, H5AC__NO_FLAGS_SET)))
@ -946,24 +931,11 @@ H5HL_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr))
heap = prfx->heap;
/* Check if heap has separate data block */
if(!heap->single_cache_obj) {
H5HL_cache_dblk_ud_t dblk_udata; /* User data for protecting local heap data block */
/* Construct the user data for protect callback */
dblk_udata.heap = heap;
dblk_udata.loaded = FALSE;
if(!heap->single_cache_obj)
/* Protect the local heap data block */
if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, &dblk_udata, H5AC__NO_FLAGS_SET)))
if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, heap, H5AC__NO_FLAGS_SET)))
H5E_THROW(H5E_CANTPROTECT, "unable to load heap data block");
/* Pin the prefix, if the data block was loaded from file */
if(dblk_udata.loaded) {
if(FAIL == H5AC_pin_protected_entry(prfx))
H5E_THROW(H5E_CANTPIN, "unable to pin local heap prefix");
} /* end if */
} /* end if */
/* Set the flags for releasing the prefix and data block */
cache_flags |= H5AC__DIRTIED_FLAG | H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG;
@ -1005,12 +977,10 @@ H5HL_get_size(H5F_t *f, hid_t dxpl_id, haddr_t addr, size_t *size))
HDassert(size);
/* Construct the user data for protect callback */
prfx_udata.made_attempt = FALSE;
prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f);
prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f);
prfx_udata.prfx_addr = addr;
prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f);
prfx_udata.loaded = FALSE;
/* Protect the local heap prefix */
if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, H5AC__READ_ONLY_FLAG)))
@ -1056,12 +1026,10 @@ H5HL_heapsize(H5F_t *f, hid_t dxpl_id, haddr_t addr, hsize_t *heap_size))
HDassert(heap_size);
/* Construct the user data for protect callback */
prfx_udata.made_attempt = FALSE;
prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f);
prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f);
prfx_udata.prfx_addr = addr;
prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f);
prfx_udata.loaded = FALSE;
/* Protect the local heap prefix */
if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, H5AC__READ_ONLY_FLAG)))

276
src/H5HLcache.c
View File

@ -71,7 +71,9 @@
/* Metadata cache callbacks */
/* Local heap prefix */
static herr_t H5HL__cache_prefix_get_load_size(const void *udata, size_t *image_len);
static herr_t H5HL__cache_prefix_get_initial_load_size(void *udata, size_t *image_len);
static herr_t H5HL__cache_prefix_get_final_load_size(const void *_image,
size_t image_len, void *udata, size_t *actual_len);
static void *H5HL__cache_prefix_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5HL__cache_prefix_image_len(const void *thing, size_t *image_len);
@ -80,13 +82,13 @@ static herr_t H5HL__cache_prefix_serialize(const H5F_t *f, void *image,
static herr_t H5HL__cache_prefix_free_icr(void *thing);
/* Local heap data block */
static herr_t H5HL__cache_datablock_get_load_size(const void *udata,
size_t *image_len);
static herr_t H5HL__cache_datablock_get_initial_load_size(void *udata, size_t *image_len);
static void *H5HL__cache_datablock_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5HL__cache_datablock_image_len(const void *thing, size_t *image_len);
static herr_t H5HL__cache_datablock_serialize(const H5F_t *f, void *image,
size_t len, void *thing);
static herr_t H5HL__cache_datablock_notify(H5C_notify_action_t action, void *_thing);
static herr_t H5HL__cache_datablock_free_icr(void *thing);
/* Free list de/serialization */
@ -103,7 +105,9 @@ const H5AC_class_t H5AC_LHEAP_PRFX[1] = {{
"local heap prefix", /* Metadata client name (for debugging) */
H5FD_MEM_LHEAP, /* File space memory type for client */
H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */
H5HL__cache_prefix_get_load_size, /* 'get_load_size' callback */
H5HL__cache_prefix_get_initial_load_size, /* 'get_initial_load_size' callback */
H5HL__cache_prefix_get_final_load_size, /* 'get_final_load_size' callback */
NULL, /* 'verify_chksum' callback */
H5HL__cache_prefix_deserialize, /* 'deserialize' callback */
H5HL__cache_prefix_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -118,12 +122,14 @@ const H5AC_class_t H5AC_LHEAP_DBLK[1] = {{
"local heap datablock", /* Metadata client name (for debugging) */
H5FD_MEM_LHEAP, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5HL__cache_datablock_get_load_size,/* 'get_load_size' callback */
H5HL__cache_datablock_get_initial_load_size,/* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
NULL, /* 'verify_chksum' callback */
H5HL__cache_datablock_deserialize, /* 'deserialize' callback */
H5HL__cache_datablock_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
H5HL__cache_datablock_serialize, /* 'serialize' callback */
NULL, /* 'notify' callback */
H5HL__cache_datablock_notify, /* 'notify' callback */
H5HL__cache_datablock_free_icr, /* 'free_icr' callback */
NULL, /* 'fsf_size' callback */
}};
@ -254,18 +260,11 @@ H5HL__fl_serialize(const H5HL_t *heap)
/*-------------------------------------------------------------------------
* Function: H5HL__cache_prefix_get_load_size()
* Function: H5HL__cache_prefix_get_initial_load_size()
*
* Purpose: Return the size of the buffer the metadata cache should
* Purpose: Return the initial size of the buffer the metadata cache should
* load from file and pass to the deserialize routine.
*
* The version 2 metadata cache callbacks included a test to
* ensure that the read did not pass the end of file, but this
* functionality has been moved to H5C_load_entry(). Thus
* all this function does is set *image_len equal to
* H5HL_SPEC_READ_SIZE, leaving it to the metadata cache to
* reduce the size of the read if appropriate.
*
* Return: Success: SUCCEED
* Failure: FAIL
*
@ -275,16 +274,91 @@ H5HL__fl_serialize(const H5HL_t *heap)
*-------------------------------------------------------------------------
*/
static herr_t
H5HL__cache_prefix_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_len)
H5HL__cache_prefix_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len)
{
FUNC_ENTER_STATIC_NOERR
/* Sanity check */
HDassert(image_len);
/* Set the image length size */
*image_len = H5HL_SPEC_READ_SIZE;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HL__cache_prefix_get_load_size() */
} /* end H5HL__cache_prefix_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5HL__cache_prefix_get_final_load_size()
*
* Purpose: Return the final size of the buffer the metadata cache should
* load from file and pass to the deserialize routine.
*
* Return: Success: SUCCEED
* Failure: FAIL
*
* Programmer: Quincey Koziol
* November 18, 2016
*
*-------------------------------------------------------------------------
*/
static herr_t
H5HL__cache_prefix_get_final_load_size(const void *_image, size_t image_len,
void *_udata, size_t *actual_len)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5HL_cache_prfx_ud_t *udata = (H5HL_cache_prfx_ud_t *)_udata; /* User data for callback */
H5HL_t heap; /* Local heap */
htri_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Sanity checks */
HDassert(image);
HDassert(udata);
HDassert(actual_len);
HDassert(*actual_len == image_len);
/* Check magic number */
if(HDmemcmp(image, H5HL_MAGIC, (size_t)H5_SIZEOF_MAGIC))
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad local heap signature")
image += H5_SIZEOF_MAGIC;
/* Version */
if(H5HL_VERSION != *image++)
HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in local heap")
/* Reserved */
image += 3;
/* Store the prefix's address & length */
heap.prfx_addr = udata->prfx_addr; /* NEED */
heap.prfx_size = udata->sizeof_prfx; /* NEED */
/* Heap data size */
H5F_DECODE_LENGTH_LEN(image, heap.dblk_size, udata->sizeof_size); /* NEED */
/* Free list head */
H5F_DECODE_LENGTH_LEN(image, heap.free_block, udata->sizeof_size);
if(heap.free_block != H5HL_FREE_NULL && heap.free_block >= heap.dblk_size)
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap free list");
/* Heap data address */
H5F_addr_decode_len(udata->sizeof_addr, &image, &(heap.dblk_addr)); /* NEED */
/* Set the final size for the cache image */
*actual_len = heap.prfx_size;
/* Check if heap block exists */
if(heap.dblk_size)
/* Check if heap data block is contiguous with header */
if(H5F_addr_eq((heap.prfx_addr + heap.prfx_size), heap.dblk_addr))
/* Note that the heap should be a single object in the cache */
*actual_len += heap.dblk_size;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5HL__cache_prefix_get_final_load_size() */
/*-------------------------------------------------------------------------
@ -353,7 +427,6 @@ H5HL__cache_prefix_deserialize(const void *_image, size_t len, void *_udata,
/* Free list head */
H5F_DECODE_LENGTH_LEN(image, heap->free_block, udata->sizeof_size);
if((heap->free_block != H5HL_FREE_NULL) && (heap->free_block >= heap->dblk_size))
HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap free list")
@ -367,53 +440,29 @@ H5HL__cache_prefix_deserialize(const void *_image, size_t len, void *_udata,
/* Note that the heap should be a single object in the cache */
heap->single_cache_obj = TRUE;
/* Check if the current buffer from the speculative read
* already has the heap data
*/
if(len >= (heap->prfx_size + heap->dblk_size)) {
/* Allocate space for the heap data image */
if(NULL == (heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, heap->dblk_size)))
HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "memory allocation failed")
/* Allocate space for the heap data image */
if(NULL == (heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, heap->dblk_size)))
HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "memory allocation failed")
/* Set image to the start of the data block. This is necessary
* because there may be a gap between the used portion of the
* prefix and the data block due to alignment constraints. */
image = ((const uint8_t *)_image) + heap->prfx_size;
/* Set image to the start of the data block. This is necessary
* because there may be a gap between the used portion of the
* prefix and the data block due to alignment constraints. */
image = ((const uint8_t *)_image) + heap->prfx_size;
/* Copy the heap data from the speculative read buffer */
HDmemcpy(heap->dblk_image, image, heap->dblk_size);
/* Copy the heap data from the speculative read buffer */
HDmemcpy(heap->dblk_image, image, heap->dblk_size);
/* Build free list */
if(H5HL__fl_deserialize(heap) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "can't initialize free list")
} /* end if */
else {
/* the supplied buffer is too small -- We have already made note
* of the correct size, so simply return success. H5C_load_entry()
* will notice the size discrepency, and re-try the load.
*/
/* Make certain that this is the first try ... */
HDassert(!udata->made_attempt);
/* ... and mark the udata so that we know that we have used up
* our first try.
*/
udata->made_attempt = TRUE;
} /* end else */
/* Build free list */
if(H5HL__fl_deserialize(heap) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "can't initialize free list")
} /* end if */
else {
else
/* Note that the heap should _NOT_ be a single
* object in the cache
*/
heap->single_cache_obj = FALSE;
} /* end else */
} /* end if */
/* Set flag to indicate prefix from loaded from file */
udata->loaded = TRUE;
/* Set return value */
ret_value = prfx;
@ -568,10 +617,6 @@ H5HL__cache_prefix_serialize(const H5F_t *f, void *_image, size_t len,
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HL__cache_prefix_serialize() */
/******************************************/
/* no H5HL_cache_prefix_notify() function */
/******************************************/
/*-------------------------------------------------------------------------
* Function: H5HL__cache_prefix_free_icr
@ -619,7 +664,7 @@ done:
/*-------------------------------------------------------------------------
* Function: H5HL__cache_datablock_get_load_size()
* Function: H5HL__cache_datablock_get_initial_load_size()
*
* Purpose: Tell the metadata cache how large a buffer to read from
* file when loading a datablock. In this case, we simply lookup
@ -634,22 +679,22 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5HL__cache_datablock_get_load_size(const void *_udata, size_t *image_len)
H5HL__cache_datablock_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5HL_cache_dblk_ud_t *udata = (const H5HL_cache_dblk_ud_t *)_udata; /* User data for callback */
H5HL_t *heap = (H5HL_t *)_udata; /* User data for callback */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(udata);
HDassert(udata->heap);
HDassert(udata->heap->dblk_size > 0);
HDassert(heap);
HDassert(heap->dblk_size > 0);
HDassert(image_len);
*image_len = udata->heap->dblk_size;
/* Set the image length size */
*image_len = heap->dblk_size;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HL__cache_datablock_get_load_size() */
} /* end H5HL__cache_datablock_get_initial_load_size() */
/*-------------------------------------------------------------------------
@ -671,8 +716,8 @@ static void *
H5HL__cache_datablock_deserialize(const void *image, size_t len, void *_udata,
hbool_t H5_ATTR_UNUSED *dirty)
{
H5HL_dblk_t *dblk = NULL; /* Local heap data block deserialized */
H5HL_cache_dblk_ud_t *udata = (H5HL_cache_dblk_ud_t *)_udata; /* User data for callback */
H5HL_dblk_t *dblk = NULL; /* Local heap data block deserialized */
H5HL_t *heap = (H5HL_t *)_udata; /* User data for callback */
void *ret_value = NULL; /* Return value */
FUNC_ENTER_STATIC
@ -680,34 +725,30 @@ H5HL__cache_datablock_deserialize(const void *image, size_t len, void *_udata,
/* Check arguments */
HDassert(image);
HDassert(len > 0);
HDassert(udata);
HDassert(udata->heap);
HDassert(udata->heap->dblk_size == len);
HDassert(!udata->heap->single_cache_obj);
HDassert(NULL == udata->heap->dblk);
HDassert(heap);
HDassert(heap->dblk_size == len);
HDassert(!heap->single_cache_obj);
HDassert(NULL == heap->dblk);
HDassert(dirty);
/* Allocate space in memory for the heap data block */
if(NULL == (dblk = H5HL__dblk_new(udata->heap)))
if(NULL == (dblk = H5HL__dblk_new(heap)))
HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "memory allocation failed");
/* Check for heap still retaining image */
if(NULL == udata->heap->dblk_image) {
if(NULL == heap->dblk_image) {
/* Allocate space for the heap data image */
if(NULL == (udata->heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, udata->heap->dblk_size)))
if(NULL == (heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, heap->dblk_size)))
HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "can't allocate data block image buffer");
/* copy the datablock from the read buffer */
HDmemcpy(udata->heap->dblk_image, image, len);
HDmemcpy(heap->dblk_image, image, len);
/* Build free list */
if(FAIL == H5HL__fl_deserialize(udata->heap))
if(FAIL == H5HL__fl_deserialize(heap))
HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "can't initialize free list");
} /* end if */
/* Set flag to indicate data block from loaded from file */
udata->loaded = TRUE;
/* Set return value */
ret_value = dblk;
@ -801,9 +842,73 @@ H5HL__cache_datablock_serialize(const H5F_t *f, void *image, size_t len,
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5HL__cache_datablock_serialize() */
/*********************************************/
/* no H5HL_cache_datablock_notify() function */
/*********************************************/
/*-------------------------------------------------------------------------
* Function: H5HL__cache_datablock_notify
*
* Purpose: This function is used to create and destroy pinned
* relationships between datablocks and their prefix parent.
*
* Return: Success: SUCCEED
* Failure: FAIL
*
* Programmer: Quincey Koziol
* November 19, 2016
*
*-------------------------------------------------------------------------
*/
static herr_t
H5HL__cache_datablock_notify(H5C_notify_action_t action, void *_thing)
{
H5HL_dblk_t *dblk = (H5HL_dblk_t *)_thing; /* Pointer to the local heap data block */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Sanity check */
HDassert(dblk);
switch(action) {
case H5AC_NOTIFY_ACTION_AFTER_INSERT:
/* do nothing */
break;
case H5AC_NOTIFY_ACTION_AFTER_LOAD:
/* Sanity checks */
HDassert(dblk->heap);
HDassert(dblk->heap->prfx);
/* Pin the heap's prefix */
if(FAIL == H5AC_pin_protected_entry(dblk->heap->prfx))
HGOTO_ERROR(H5E_HEAP, H5E_CANTPIN, FAIL, "unable to pin local heap prefix")
break;
case H5AC_NOTIFY_ACTION_AFTER_FLUSH:
case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED:
case H5AC_NOTIFY_ACTION_ENTRY_CLEANED:
case H5AC_NOTIFY_ACTION_CHILD_DIRTIED:
case H5AC_NOTIFY_ACTION_CHILD_CLEANED:
/* do nothing */
break;
case H5AC_NOTIFY_ACTION_BEFORE_EVICT:
/* Sanity checks */
HDassert(dblk->heap);
HDassert(dblk->heap->prfx);
/* Unpin the local heap prefix */
if(FAIL == H5AC_unpin_entry(dblk->heap->prfx))
HGOTO_ERROR(H5E_HEAP, H5E_CANTUNPIN, FAIL, "unable to unpin local heap prefix")
break;
default:
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unknown action from metadata cache")
break;
} /* end switch */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5HL__cache_datablock_notify() */
/*-------------------------------------------------------------------------
@ -843,3 +948,4 @@ H5HL__cache_datablock_free_icr(void *_thing)
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5HL__cache_datablock_free_icr() */

10
src/H5HLdblk.c
View File

@ -119,10 +119,9 @@ H5HL__dblk_new(H5HL_t *heap))
CATCH
/* Ensure that the data block memory is deallocated on errors */
if(!ret_value && dblk != NULL) {
if(!ret_value && dblk != NULL)
/* H5FL_FREE always returns NULL so we can't check for errors */
dblk = H5FL_FREE(H5HL_dblk_t, dblk);
}
END_FUNC(PKG) /* end H5HL__dblk_new() */
@ -151,10 +150,6 @@ H5HL__dblk_dest(H5HL_dblk_t *dblk))
/* Unlink data block from heap */
dblk->heap->dblk = NULL;
/* Unpin the local heap prefix */
if(FAIL == H5AC_unpin_entry(dblk->heap->prfx))
H5E_THROW(H5E_CANTUNPIN, "can't unpin local heap prefix")
/* Decrement ref. count on heap data structure */
if(FAIL == H5HL__dec_rc(dblk->heap))
H5E_THROW(H5E_CANTDEC, "can't decrement heap ref. count")
@ -223,7 +218,6 @@ H5HL__dblk_realloc(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, size_t new_heap_size))
/* Resize the heap prefix in the cache */
if(FAIL == H5AC_resize_entry(heap->prfx, (size_t)(heap->prfx_size + new_heap_size)))
H5E_THROW(H5E_CANTRESIZE, "unable to resize heap in cache");
} /* end if */
else {
/* Sanity check */
@ -233,7 +227,6 @@ H5HL__dblk_realloc(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, size_t new_heap_size))
/* Resize the heap data block in the cache */
if(H5AC_resize_entry(heap->dblk, (size_t)new_heap_size) < 0)
H5E_THROW(H5E_CANTRESIZE, "unable to resize heap (data block) in cache");
} /* end else */
} /* end if */
else {
@ -280,3 +273,4 @@ CATCH
} /* end if */
END_FUNC(PKG) /* end H5HL__dblk_realloc() */

16
src/H5HLpkg.h
View File

@ -131,28 +131,12 @@ struct H5HL_prfx_t {
/* Callback information for loading local heap prefix from disk */
typedef struct H5HL_cache_prfx_ud_t {
/* Downwards */
hbool_t made_attempt; /* Whether the deserialize routine */
/* was already attempted */
size_t sizeof_size; /* Size of file sizes */
size_t sizeof_addr; /* Size of file addresses */
haddr_t prfx_addr; /* Address of prefix */
size_t sizeof_prfx; /* Size of heap prefix */
/* Upwards */
hbool_t loaded; /* Whether prefix was loaded */
/* from file */
} H5HL_cache_prfx_ud_t;
/* Callback information for loading local heap data block from disk */
typedef struct H5HL_cache_dblk_ud_t {
/* Downwards */
H5HL_t *heap; /* Local heap */
/* Upwards */
hbool_t loaded; /* Whether data block was loaded from file */
} H5HL_cache_dblk_ud_t;
/******************************/
/* Package Private Prototypes */

4
src/H5HLprfx.c
View File

@ -118,10 +118,9 @@ H5HL__prfx_new(H5HL_t *heap))
CATCH
/* Ensure that the prefix memory is deallocated on errors */
if(!ret_value && prfx != NULL) {
if(!ret_value && prfx != NULL)
/* H5FL_FREE always returns NULL so we can't check for errors */
prfx = H5FL_FREE(H5HL_prfx_t, prfx);
}
END_FUNC(PKG) /* end H5HL__prfx_new() */
@ -164,3 +163,4 @@ CATCH
prfx = H5FL_FREE(H5HL_prfx_t, prfx);
END_FUNC(PKG) /* end H5HL__prfx_dest() */

86
src/H5O.c
View File

@ -1811,11 +1811,12 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags)
/* Construct the user data for protect callback */
udata.made_attempt = FALSE;
udata.v1_pfx_nmesgs = 0;
udata.chunk0_size = 0;
udata.oh = NULL;
udata.common.f = loc->file;
udata.common.dxpl_id = dxpl_id;
udata.common.file_intent = file_intent;
udata.common.merged_null_msgs = 0;
udata.common.mesgs_modified = FALSE;
HDmemset(&cont_msg_info, 0, sizeof(cont_msg_info));
udata.common.cont_msg_info = &cont_msg_info;
udata.common.addr = loc->addr;
@ -1843,10 +1844,12 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags)
chk_udata.common.dxpl_id = dxpl_id;
chk_udata.common.file_intent = file_intent;
chk_udata.common.merged_null_msgs = udata.common.merged_null_msgs;
chk_udata.common.mesgs_modified = udata.common.mesgs_modified;
chk_udata.common.cont_msg_info = &cont_msg_info;
/* Read in continuation messages, until there are no more */
/* (Note that loading chunks could increase the # of continuation
* messages if new ones are found - QAK, 19/11/2016)
*/
curr_msg = 0;
while(curr_msg < cont_msg_info.nmsgs) {
H5O_chunk_proxy_t *chk_proxy; /* Proxy for chunk, to bring it into memory */
@ -1879,16 +1882,12 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags)
/* Pass back out some of the chunk's user data */
udata.common.merged_null_msgs = chk_udata.common.merged_null_msgs;
udata.common.mesgs_modified = chk_udata.common.mesgs_modified;
} /* end if */
/* Check for incorrect # of object header messages, if we've just loaded
* this object header from the file
*/
if(udata.made_attempt) {
/* Check for incorrect # of messages in v1 object header */
if(oh->version == H5O_VERSION_1 &&
(oh->nmesgs + udata.common.merged_null_msgs) != udata.v1_pfx_nmesgs) {
/* Don't enforce the error on an incorrect # of object header messages bug
* unless strict format checking is enabled. This allows for older
* files, created with a version of the library that had a bug in tracking
@ -1896,79 +1895,11 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags)
* erroring out here. -QAK
*/
#ifdef H5_STRICT_FORMAT_CHECKS
/* Check for incorrect # of messages in v1 object header */
if(oh->version == H5O_VERSION_1 &&
(oh->nmesgs + udata.common.merged_null_msgs) != udata.v1_pfx_nmesgs)
HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "corrupt object header - incorrect # of messages")
#else /* H5_STRICT_FORMAT_CHECKS */
/* Mark object header prefix dirty later if we don't have write access */
/* (object header will have been marked dirty during protect, if we
* have write access -QAK)
*/
if((prot_flags & H5AC__READ_ONLY_FLAG) != 0)
oh->prefix_modified = TRUE;
#ifndef NDEBUG
else {
unsigned oh_status = 0; /* Object header entry cache status */
/* Check the object header's status in the metadata cache */
if(H5AC_get_entry_status(loc->file, loc->addr, &oh_status) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, NULL, "unable to check metadata cache status for object header")
/* Make certain that object header is not dirty */
HDassert(!(oh_status & H5AC_ES__IS_DIRTY));
} /* end else */
#endif /* NDEBUG */
#endif /* H5_STRICT_FORMAT_CHECKS */
} /* end if */
/* Check for any messages that were modified while being read in */
if(udata.common.mesgs_modified && (0 == (prot_flags & H5AC__READ_ONLY_FLAG)))
oh->mesgs_modified = TRUE;
/* Reset the field that contained chunk 0's size during speculative load */
oh->chunk0_size = 0;
} /* end if */
/* Take care of loose ends for modifications made while bringing in the
* object header & chunks.
*/
if(0 == (prot_flags & H5AC__READ_ONLY_FLAG)) {
/* Check for the object header prefix being modified somehow */
/* (usually through updating the # of object header messages) */
if(oh->prefix_modified) {
/* Mark the header as dirty now */
if(H5AC_mark_entry_dirty(oh) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTMARKDIRTY, NULL, "unable to mark object header as dirty")
/* Reset flag */
oh->prefix_modified = FALSE;
} /* end if */
/* Check for deferred dirty messages */
if(oh->mesgs_modified) {
unsigned u; /* Local index variable */
/* Loop through all messages, marking their chunks as dirty */
/* (slightly inefficient, since we don't know exactly which messages
* were modified when the object header & chunks were brought in
* from the file, but this only can happen once per load -QAK)
*/
for(u = 0; u < oh->nmesgs; u++) {
/* Mark each chunk with a dirty message as dirty also */
if(oh->mesg[u].dirty) {
H5O_chunk_proxy_t *chk_proxy; /* Chunk that message is in */
/* Protect chunk */
if(NULL == (chk_proxy = H5O_chunk_protect(loc->file, dxpl_id, oh, oh->mesg[u].chunkno)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, NULL, "unable to load object header chunk")
/* Unprotect chunk, marking it dirty */
if(H5O_chunk_unprotect(loc->file, dxpl_id, chk_proxy, TRUE) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, NULL, "unable to unprotect object header chunk")
} /* end if */
} /* end for */
/* Reset flag */
oh->mesgs_modified = FALSE;
} /* end if */
} /* end if */
#ifdef H5O_DEBUG
@ -3645,7 +3576,6 @@ herr_t
H5O__free(H5O_t *oh)
{
unsigned u; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE_NOERR

476
src/H5Ocache.c
View File

@ -68,7 +68,10 @@
/********************/
/* Metadata cache callbacks */
static herr_t H5O__cache_get_load_size(const void *udata, size_t *image_len);
static herr_t H5O__cache_get_initial_load_size(void *udata, size_t *image_len);
static herr_t H5O__cache_get_final_load_size(const void *image_ptr, size_t image_len,
void *udata, size_t *actual_len);
static htri_t H5O__cache_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5O__cache_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5O__cache_image_len(const void *thing, size_t *image_len);
@ -77,7 +80,8 @@ static herr_t H5O__cache_serialize(const H5F_t *f, void *image, size_t len,
static herr_t H5O__cache_notify(H5AC_notify_action_t action, void *_thing);
static herr_t H5O__cache_free_icr(void *thing);
static herr_t H5O__cache_chk_get_load_size(const void *udata, size_t *image_len);
static herr_t H5O__cache_chk_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5O__cache_chk_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5O__cache_chk_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5O__cache_chk_image_len(const void *thing, size_t *image_len);
@ -94,7 +98,6 @@ static herr_t H5O__chunk_serialize(const H5F_t *f, H5O_t *oh, unsigned chunkno);
/* Misc. routines */
static herr_t H5O__add_cont_msg(H5O_cont_msgs_t *cont_msg_info,
const H5O_cont_t *cont);
static herr_t H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata);
/*********************/
@ -107,7 +110,9 @@ const H5AC_class_t H5AC_OHDR[1] = {{
"object header", /* Metadata client name (for debugging) */
H5FD_MEM_OHDR, /* File space memory type for client */
H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */
H5O__cache_get_load_size, /* 'get_load_size' callback */
H5O__cache_get_initial_load_size, /* 'get_initial_load_size' callback */
H5O__cache_get_final_load_size, /* 'get_final_load_size' callback */
H5O__cache_verify_chksum, /* 'verify_chksum' callback */
H5O__cache_deserialize, /* 'deserialize' callback */
H5O__cache_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -123,7 +128,9 @@ const H5AC_class_t H5AC_OHDR_CHK[1] = {{
"object header continuation chunk", /* Metadata client name (for debugging) */
H5FD_MEM_OHDR, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5O__cache_chk_get_load_size, /* 'get_load_size' callback */
H5O__cache_chk_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5O__cache_chk_verify_chksum, /* 'verify_chksum' callback */
H5O__cache_chk_deserialize, /* 'deserialize' callback */
H5O__cache_chk_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -152,48 +159,88 @@ H5FL_SEQ_DEFINE(H5O_cont_t);
/* Local Variables */
/*******************/
/*-------------------------------------------------------------------------
* Function: H5O_decode_prefix
* Function: H5O__cache_get_initial_load_size()
*
* Purpose: To decode the object header prefix.
* The coding is extracted fromt H5O__cache_deserialize() to this routine.
* Purpose: Tell the metadata cache how much data to read from file in
* the first speculative read for the object header.
*
* Return: Non-negative on success/Negative on failure
* Return: Success: SUCCEED
* Failure: FAIL
*
* Programmer: Vailin Choi
* Aug 2015
* Programmer: John Mainzer
* 7/28/14
*
*-------------------------------------------------------------------------
*/
static herr_t
H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata)
H5O__cache_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len)
{
H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */
const uint8_t *p = buf; /* Pointer into buffer to decode */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(f);
HDassert(oh);
HDassert(buf);
HDassert(image_len);
/* Set the image length size */
*image_len = H5O_SPEC_READ_SIZE;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5O__cache_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5O__cache_get_final_load_size()
*
* Purpose: Tell the metadata cache the final size of an object header.
*
* Return: Success: SUCCEED
* Failure: FAIL
*
* Programmer: Quincey Koziol
* November 18, 2016
*
*-------------------------------------------------------------------------
*/
static herr_t
H5O__cache_get_final_load_size(const void *_image, size_t image_len,
void *_udata, size_t *actual_len)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */
H5O_t *oh = NULL; /* Object header read in */
htri_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Check arguments */
HDassert(image);
HDassert(udata);
HDassert(actual_len);
HDassert(*actual_len == image_len);
/* Allocate space for the new object header data structure */
if(NULL == (oh = H5FL_CALLOC(H5O_t)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed")
/* File-specific, non-stored information */
oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f);
oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f);
/* Check for presence of magic number */
/* (indicates version 2 or later) */
if(!HDmemcmp(p, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
if(!HDmemcmp(image, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/* Magic number */
p += H5_SIZEOF_MAGIC;
image += H5_SIZEOF_MAGIC;
/* Version */
oh->version = *p++;
oh->version = *image++;
if(H5O_VERSION_2 != oh->version)
HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number")
/* Flags */
oh->flags = *p++;
oh->flags = *image++;
if(oh->flags & ~H5O_HDR_ALL_FLAGS)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown object header status flag(s)")
@ -204,13 +251,13 @@ H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata)
if(oh->flags & H5O_HDR_STORE_TIMES) {
uint32_t tmp; /* Temporary value */
UINT32DECODE(p, tmp);
UINT32DECODE(image, tmp);
oh->atime = (time_t)tmp;
UINT32DECODE(p, tmp);
UINT32DECODE(image, tmp);
oh->mtime = (time_t)tmp;
UINT32DECODE(p, tmp);
UINT32DECODE(image, tmp);
oh->ctime = (time_t)tmp;
UINT32DECODE(p, tmp);
UINT32DECODE(image, tmp);
oh->btime = (time_t)tmp;
} /* end if */
else
@ -218,8 +265,8 @@ H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata)
/* Attribute fields */
if(oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) {
UINT16DECODE(p, oh->max_compact);
UINT16DECODE(p, oh->min_dense);
UINT16DECODE(image, oh->max_compact);
UINT16DECODE(image, oh->min_dense);
if(oh->max_compact < oh->min_dense)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header attribute phase change values")
} /* end if */
@ -231,30 +278,30 @@ H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata)
/* First chunk size */
switch(oh->flags & H5O_HDR_CHUNK0_SIZE) {
case 0: /* 1 byte size */
oh->chunk0_size = *p++;
udata->chunk0_size = *image++;
break;
case 1: /* 2 byte size */
UINT16DECODE(p, oh->chunk0_size);
UINT16DECODE(image, udata->chunk0_size);
break;
case 2: /* 4 byte size */
UINT32DECODE(p, oh->chunk0_size);
UINT32DECODE(image, udata->chunk0_size);
break;
case 3: /* 8 byte size */
UINT64DECODE(p, oh->chunk0_size);
UINT64DECODE(image, udata->chunk0_size);
break;
default:
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0")
} /* end switch */
if(oh->chunk0_size > 0 && oh->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh))
if(udata->chunk0_size > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh))
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size")
} /* end if */
else {
/* Version */
oh->version = *p++;
oh->version = *image++;
if(H5O_VERSION_1 != oh->version)
HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number")
@ -262,13 +309,13 @@ H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata)
oh->flags = H5O_CRT_OHDR_FLAGS_DEF;
/* Reserved */
p++;
image++;
/* Number of messages */
UINT16DECODE(p, udata->v1_pfx_nmesgs);
UINT16DECODE(image, udata->v1_pfx_nmesgs);
/* Link count */
UINT32DECODE(p, oh->nlink);
UINT32DECODE(image, oh->nlink);
/* Reset unused time fields */
oh->atime = oh->mtime = oh->ctime = oh->btime = 0;
@ -278,51 +325,77 @@ H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata)
oh->min_dense = 0;
/* First chunk size */
UINT32DECODE(p, oh->chunk0_size);
if((udata->v1_pfx_nmesgs > 0 && oh->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) ||
(udata->v1_pfx_nmesgs == 0 && oh->chunk0_size > 0))
UINT32DECODE(image, udata->chunk0_size);
if((udata->v1_pfx_nmesgs > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) ||
(udata->v1_pfx_nmesgs == 0 && udata->chunk0_size > 0))
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size")
/* Reserved, in version 1 (for 8-byte alignment padding) */
p += 4;
image += 4;
} /* end else */
/* Determine object header prefix length */
HDassert((size_t)(p - buf) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh)));
HDassert((size_t)((const uint8_t *)image - (const uint8_t *)_image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh)));
/* Set the final size for the cache image */
*actual_len = udata->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh);
/* Save the object header for later use in 'deserialize' callback */
udata->oh = oh;
oh = NULL;
done:
/* Release the [possibly partially initialized] object header on errors */
if(ret_value < 0 && oh)
if(H5O__free(oh) < 0)
HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header data")
FUNC_LEAVE_NOAPI(ret_value)
} /* H5O_decode_prefix() */
} /* end H5O__cache_get_final_load_size() */
/*-------------------------------------------------------------------------
* Function: H5O__cache_get_load_size()
* Function: H5O__cache_verify_chksum
*
* Purpose: Tell the metadata cache how much data to read from file in
* the first speculative read for the object header. Note that we do
* not have to be concerned about reading past the end of file, as the
* cache will clamp the read to avoid this if needed.
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: SUCCEED
* Failure: FAIL
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: John Mainzer
* 7/28/14
* Programmer: Vailin Choi
* Aug 2015
*
*-------------------------------------------------------------------------
*/
static herr_t
H5O__cache_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_len)
static htri_t
H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image_len);
HDassert(image);
HDassert(udata);
HDassert(udata->oh);
*image_len = H5O_SPEC_READ_SIZE;
/* There is no checksum for version 1 */
if(udata->oh->version != H5O_VERSION_1) {
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5O__cache_get_load_size() */
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
} /* end if */
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5O__cache_verify_chksum() */
/*-------------------------------------------------------------------------
@ -349,10 +422,9 @@ static void *
H5O__cache_deserialize(const void *_image, size_t len, void *_udata,
hbool_t *dirty)
{
H5O_t *oh = NULL; /* Object header read in */
H5O_t *oh; /* Object header read in */
H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into buffer to decode */
size_t buf_size; /* Size of prefix+chunk #0 buffer */
void * ret_value = NULL; /* Return value */
FUNC_ENTER_STATIC
@ -361,38 +433,17 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata,
HDassert(image);
HDassert(len > 0);
HDassert(udata);
HDassert(udata->oh);
HDassert(udata->common.f);
HDassert(udata->common.cont_msg_info);
HDassert(dirty);
/* Allocate space for the object header data structure */
if(NULL == (oh = H5FL_CALLOC(H5O_t)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
/* Retrieve partially deserialized object header from user data */
oh = udata->oh;
/* File-specific, non-stored information */
oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f);
oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f);
/* Decode header prefix */
if(H5O_decode_prefix(udata->common.f, oh, image, udata) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize object header prefix")
/* Compute the size of the buffer used */
buf_size = oh->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh);
/* Check to see if the buffer provided is large enough to contain both
* the prefix and the first chunk. If it isn't, make note of the desired
* size, but otherwise do nothing. H5C_load_entry() will notice the
* discrepency, load the correct size buffer, and retry the deserialize.
*/
if(len >= buf_size) {
/* Parse the first chunk */
if(H5O__chunk_deserialize(oh, udata->common.addr, oh->chunk0_size, (const uint8_t *)_image, &(udata->common), dirty) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize first object header chunk")
} /* end if */
else
HDassert(!udata->made_attempt);
/* Parse the first chunk */
if(H5O__chunk_deserialize(oh, udata->common.addr, udata->chunk0_size, (const uint8_t *)_image, &(udata->common), dirty) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize first object header chunk")
/* Note that we've loaded the object header from the file */
udata->made_attempt = TRUE;
@ -439,10 +490,7 @@ H5O__cache_image_len(const void *_thing, size_t *image_len)
HDassert(image_len);
/* Report the object header's prefix+first chunk length */
if(oh->chunk0_size)
*image_len = (size_t)H5O_SIZEOF_HDR(oh) + oh->chunk0_size;
else
*image_len = oh->chunk[0].size;
*image_len = oh->chunk[0].size;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5O__cache_image_len() */
@ -698,12 +746,11 @@ done:
/*-------------------------------------------------------------------------
* Function: H5O__cache_chk_get_load_size()
* Function: H5O__cache_chk_get_initial_load_size()
*
* Purpose: Tell the metadata cache how large the on disk image of the
* chunk proxy is, so it can load the image into a buffer for the
* deserialize call. In this case, we simply look up the size in
* the user data, and return it in *image_len,
* deserialize call.
*
* Return: Success: SUCCEED
* Failure: FAIL
@ -714,9 +761,9 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5O__cache_chk_get_load_size(const void *_udata, size_t *image_len)
H5O__cache_chk_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5O_chk_cache_ud_t *udata = (const H5O_chk_cache_ud_t *)_udata; /* User data for callback */
const H5O_chk_cache_ud_t *udata = (const H5O_chk_cache_ud_t *)_udata; /* User data for callback */
FUNC_ENTER_STATIC_NOERR
@ -725,10 +772,53 @@ H5O__cache_chk_get_load_size(const void *_udata, size_t *image_len)
HDassert(udata->oh);
HDassert(image_len);
/* Set the image length size */
*image_len = udata->size;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5O__cache_chk_get_load_size() */
} /* end H5O__cache_chk_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5B2__cache_chk_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi
* Aug 2015
*
*-------------------------------------------------------------------------
*/
static htri_t
H5O__cache_chk_verify_chksum(const void *_image, size_t len, void *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5O_chk_cache_ud_t *udata = (H5O_chk_cache_ud_t *)_udata; /* User data for callback */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
/* There is no checksum for version 1 */
if(udata->oh->version != H5O_VERSION_1) {
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
} /* end if */
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5O__cache_chk_verify_chksum() */
/*-------------------------------------------------------------------------
@ -1071,12 +1161,12 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image
{
const uint8_t *chunk_image; /* Pointer into buffer to decode */
uint8_t *eom_ptr; /* Pointer to end of messages for a chunk */
size_t curmesg; /* Current message being decoded in object header */
unsigned merged_null_msgs = 0; /* Number of null messages merged together */
unsigned chunkno; /* Current chunk's index */
#ifndef NDEBUG
unsigned nullcnt; /* Count of null messages (for sanity checking gaps in chunks) */
#endif /* NDEBUG */
hbool_t mesgs_modified = FALSE; /* Whether any messages were modified when the object header was deserialized */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
@ -1102,15 +1192,12 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image
/* Init the chunk data info */
chunkno = (unsigned)oh->nchunks++;
oh->chunk[chunkno].gap = 0;
if(chunkno == 0) {
oh->chunk[chunkno].addr = addr;
if(chunkno == 0)
/* First chunk's 'image' includes room for the object header prefix */
oh->chunk[0].addr = addr;
oh->chunk[0].size = len + (size_t)H5O_SIZEOF_HDR(oh);
} /* end if */
else {
oh->chunk[chunkno].addr = addr;
else
oh->chunk[chunkno].size = len;
} /* end else */
if(NULL == (oh->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, oh->chunk[chunkno].size)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed")
@ -1132,16 +1219,12 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image
chunk_image += H5_SIZEOF_MAGIC;
} /* end if */
/* Save # of messages already inspected */
curmesg = oh->nmesgs;
/* Decode messages from this chunk */
eom_ptr = oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM_OH(oh));
#ifndef NDEBUG
nullcnt = 0;
#endif /* NDEBUG */
while(chunk_image < eom_ptr) {
size_t mesgno; /* Current message to operate on */
size_t mesg_size; /* Size of message read in */
unsigned id; /* ID (type) of current message */
uint8_t flags; /* Flags for current message */
@ -1200,39 +1283,45 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image
H5O_NULL_ID == id && oh->nmesgs > 0 &&
H5O_NULL_ID == oh->mesg[oh->nmesgs - 1].type->id &&
oh->mesg[oh->nmesgs - 1].chunkno == chunkno) {
size_t mesgno; /* Current message to operate on */
/* Combine adjacent null messages */
mesgno = oh->nmesgs - 1;
oh->mesg[mesgno].raw_size += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + mesg_size;
oh->mesg[mesgno].dirty = TRUE;
merged_null_msgs++;
udata->merged_null_msgs++;
} /* end if */
else {
H5O_mesg_t *mesg; /* Pointer to new message */
unsigned ioflags = 0; /* Flags for decode routine */
/* Check if we need to extend message table to hold the new message */
if(oh->nmesgs >= oh->alloc_nmesgs)
if(H5O_alloc_msgs(oh, (size_t)1) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate more space for messages")
/* Get index for message */
mesgno = oh->nmesgs++;
/* Get pointer to message to set up */
mesg = &oh->mesg[oh->nmesgs];
/* Increment # of messages */
oh->nmesgs++;
/* Initialize information about message */
oh->mesg[mesgno].dirty = FALSE;
oh->mesg[mesgno].flags = flags;
oh->mesg[mesgno].crt_idx = crt_idx;
oh->mesg[mesgno].native = NULL;
oh->mesg[mesgno].raw = (uint8_t *)chunk_image; /* Casting away const OK - QAK */
oh->mesg[mesgno].raw_size = mesg_size;
oh->mesg[mesgno].chunkno = chunkno;
mesg->dirty = FALSE;
mesg->flags = flags;
mesg->crt_idx = crt_idx;
mesg->native = NULL;
mesg->raw = (uint8_t *)chunk_image; /* Casting away const OK - QAK */
mesg->raw_size = mesg_size;
mesg->chunkno = chunkno;
/* Point unknown messages at 'unknown' message class */
/* (Usually from future versions of the library) */
if(id >= H5O_UNKNOWN_ID ||
#ifdef H5O_ENABLE_BOGUS
id == H5O_BOGUS_VALID_ID ||
id == H5O_BOGUS_VALID_ID ||
#endif
NULL == H5O_msg_class_g[id]) {
NULL == H5O_msg_class_g[id]) {
H5O_unknown_t *unknown; /* Pointer to "unknown" message info */
@ -1244,10 +1333,10 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image
*unknown = id;
/* Save 'native' form of unknown message */
oh->mesg[mesgno].native = unknown;
mesg->native = unknown;
/* Set message to "unknown" class */
oh->mesg[mesgno].type = H5O_msg_class_g[H5O_UNKNOWN_ID];
mesg->type = H5O_msg_class_g[H5O_UNKNOWN_ID];
/* Check for "fail if unknown" message flags */
if(((udata->file_intent & H5F_ACC_RDWR) &&
@ -1270,17 +1359,66 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image
* the metadata cache in some other "weird" way, like
* using H5Ocopy() - QAK
*/
oh->mesg[mesgno].flags |= H5O_MSG_FLAG_WAS_UNKNOWN;
mesg->flags |= H5O_MSG_FLAG_WAS_UNKNOWN;
/* Mark the message and chunk as dirty */
oh->mesg[mesgno].dirty = TRUE;
udata->mesgs_modified = TRUE;
*dirty = TRUE;
mesg->dirty = TRUE;
mesgs_modified = TRUE;
} /* end if */
} /* end if */
else
/* Set message class for "known" messages */
oh->mesg[mesgno].type = H5O_msg_class_g[id];
mesg->type = H5O_msg_class_g[id];
/* Do some inspection/interpretation of new messages from this chunk */
/* (detect continuation messages, ref. count messages, etc.) */
/* Check if message is a continuation message */
if(H5O_CONT_ID == id) {
H5O_cont_t *cont;
/* Decode continuation message */
cont = (H5O_cont_t *)(H5O_MSG_CONT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, mesg->raw);
H5_CHECKED_ASSIGN(cont->chunkno, unsigned, udata->cont_msg_info->nmsgs + 1, size_t); /* the next continuation message/chunk */
/* Save 'native' form of continuation message */
mesg->native = cont;
/* Add to continuation messages left to interpret */
if(H5O__add_cont_msg(udata->cont_msg_info, cont) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add continuation message")
} /* end if */
/* Check if message is a ref. count message */
else if(H5O_REFCOUNT_ID == id) {
H5O_refcount_t *refcount;
/* Decode ref. count message */
HDassert(oh->version > H5O_VERSION_1);
refcount = (H5O_refcount_t *)(H5O_MSG_REFCOUNT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, mesg->raw);
/* Save 'native' form of ref. count message */
mesg->native = refcount;
/* Set object header values */
oh->has_refcount_msg = TRUE;
oh->nlink = *refcount;
} /* end if */
/* Check if message is a link message */
else if(H5O_LINK_ID == id) {
/* Increment the count of link messages */
oh->link_msgs_seen++;
} /* end if */
/* Check if message is an attribute message */
else if(H5O_ATTR_ID == id) {
/* Increment the count of attribute messages */
oh->attr_msgs_seen++;
} /* end if */
/* Mark the message & chunk as dirty if the message was changed by decoding */
if((ioflags & H5O_DECODEIO_DIRTY) && (udata->file_intent & H5F_ACC_RDWR)) {
mesg->dirty = TRUE;
mesgs_modified = TRUE;
} /* end if */
} /* end else */
/* Advance decode pointer past message */
@ -1305,89 +1443,23 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image
/* Check for correct checksum on chunks, in later versions of the format */
if(oh->version > H5O_VERSION_1) {
uint32_t stored_chksum; /* Checksum from file */
uint32_t computed_chksum; /* Checksum computed in memory */
/* checksum verification already done in verify_chksum cb */
/* Metadata checksum */
UINT32DECODE(chunk_image, stored_chksum);
/* Compute checksum on chunk */
computed_chksum = H5_checksum_metadata(oh->chunk[chunkno].image, (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM), 0);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "incorrect metadata checksum for object header chunk")
} /* end if */
/* Sanity check */
HDassert(chunk_image == oh->chunk[chunkno].image + oh->chunk[chunkno].size);
/* Do some inspection/interpretation of new messages from this chunk */
/* (detect continuation messages, ref. count messages, etc.) */
while(curmesg < oh->nmesgs) {
/* Check if next message to examine is a continuation message */
if(H5O_CONT_ID == oh->mesg[curmesg].type->id) {
H5O_cont_t *cont;
unsigned ioflags = 0; /* Flags for decode routine */
/* Decode continuation message */
cont = (H5O_cont_t *)(H5O_MSG_CONT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, oh->mesg[curmesg].raw);
H5_CHECKED_ASSIGN(cont->chunkno, unsigned, udata->cont_msg_info->nmsgs + 1, size_t); /* the next continuation message/chunk */
/* Save 'native' form of continuation message */
oh->mesg[curmesg].native = cont;
/* Add to continuation messages left to interpret */
if(H5O__add_cont_msg(udata->cont_msg_info, cont) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add continuation message")
/* Mark the message & chunk as dirty if the message was changed by decoding */
if((ioflags & H5O_DECODEIO_DIRTY) && (udata->file_intent & H5F_ACC_RDWR)) {
oh->mesg[curmesg].dirty = TRUE;
udata->mesgs_modified = TRUE;
*dirty = TRUE;
} /* end if */
} /* end if */
/* Check if next message to examine is a ref. count message */
else if(H5O_REFCOUNT_ID == oh->mesg[curmesg].type->id) {
H5O_refcount_t *refcount;
unsigned ioflags = 0; /* Flags for decode routine */
/* Decode ref. count message */
HDassert(oh->version > H5O_VERSION_1);
refcount = (H5O_refcount_t *)(H5O_MSG_REFCOUNT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, oh->mesg[curmesg].raw);
/* Save 'native' form of ref. count message */
oh->mesg[curmesg].native = refcount;
/* Set object header values */
oh->has_refcount_msg = TRUE;
oh->nlink = *refcount;
/* Mark the message & chunk as dirty if the message was changed by decoding */
if((ioflags & H5O_DECODEIO_DIRTY) && (udata->file_intent & H5F_ACC_RDWR)) {
oh->mesg[curmesg].dirty = TRUE;
udata->mesgs_modified = TRUE;
*dirty = TRUE;
} /* end if */
} /* end if */
/* Check if next message to examine is a link message */
else if(H5O_LINK_ID == oh->mesg[curmesg].type->id) {
/* Increment the count of link messages */
oh->link_msgs_seen++;
} /* end if */
/* Check if next message to examine is an attribute message */
else if(H5O_ATTR_ID == oh->mesg[curmesg].type->id) {
/* Increment the count of attribute messages */
oh->attr_msgs_seen++;
} /* end if */
/* Advance to next message */
curmesg++;
} /* end while */
/* Mark the chunk dirty if we've modified messages */
if(mesgs_modified)
*dirty = TRUE;
/* Mark the chunk dirty if we've merged null messages */
if(merged_null_msgs) {
udata->mesgs_modified = TRUE;
if(merged_null_msgs > 0) {
udata->merged_null_msgs += merged_null_msgs;
*dirty = TRUE;
} /* end if */

6
src/H5Opkg.h
View File

@ -292,9 +292,6 @@ struct H5O_t {
/* Chunk management information (not stored) */
size_t rc; /* Reference count of [continuation] chunks using this structure */
size_t chunk0_size; /* Size of serialized first chunk */
hbool_t mesgs_modified; /* Whether any messages were modified when the object header was deserialized */
hbool_t prefix_modified; /* Whether prefix was modified when the object header was deserialized */
/* Object information (stored) */
hbool_t has_refcount_msg; /* Whether the object has a ref. count message */
@ -362,7 +359,6 @@ typedef struct H5O_common_cache_ud_t {
hid_t dxpl_id; /* DXPL for operation */
unsigned file_intent; /* Read/write intent for file */
unsigned merged_null_msgs; /* Number of null messages merged together */
hbool_t mesgs_modified; /* Whether any messages were modified when the object header was deserialized */
H5O_cont_msgs_t *cont_msg_info; /* Pointer to continuation messages to work on */
haddr_t addr; /* Address of the prefix or chunk */
} H5O_common_cache_ud_t;
@ -371,6 +367,8 @@ typedef struct H5O_common_cache_ud_t {
typedef struct H5O_cache_ud_t {
hbool_t made_attempt; /* Whether the deserialize routine was already attempted */
unsigned v1_pfx_nmesgs; /* Number of messages from v1 prefix header */
size_t chunk0_size; /* Size of serialized first chunk */
H5O_t *oh; /* Partially deserialized object header, for later use */
H5O_common_cache_ud_t common; /* Common object header cache callback info */
} H5O_cache_ud_t;

96
src/H5Pfapl.c
View File

@ -178,6 +178,11 @@
#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF 524288
#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_ENC H5P__encode_size_t
#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEC H5P__decode_size_t
/* Definition for # of metadata read attempts */
#define H5F_ACS_METADATA_READ_ATTEMPTS_SIZE sizeof(unsigned)
#define H5F_ACS_METADATA_READ_ATTEMPTS_DEF 0
#define H5F_ACS_METADATA_READ_ATTEMPTS_ENC H5P__encode_unsigned
#define H5F_ACS_METADATA_READ_ATTEMPTS_DEC H5P__decode_unsigned
/* Definition for object flush callback */
#define H5F_ACS_OBJECT_FLUSH_CB_SIZE sizeof(H5F_object_flush_t)
#define H5F_ACS_OBJECT_FLUSH_CB_DEF {NULL, NULL}
@ -327,6 +332,7 @@ static const unsigned H5F_def_efc_size_g = H5F_ACS_EFC_SIZE_DEF;
static const H5FD_file_image_info_t H5F_def_file_image_info_g = H5F_ACS_FILE_IMAGE_INFO_DEF; /* Default file image info and callbacks */
static const hbool_t H5F_def_core_write_tracking_flag_g = H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF; /* Default setting for core VFD write tracking */
static const size_t H5F_def_core_write_tracking_page_size_g = H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF; /* Default core VFD write tracking page size */
static const unsigned H5F_def_metadata_read_attempts_g = H5F_ACS_METADATA_READ_ATTEMPTS_DEF; /* Default setting for the # of metadata read attempts */
static const H5F_object_flush_t H5F_def_object_flush_cb_g = H5F_ACS_OBJECT_FLUSH_CB_DEF; /* Default setting for object flush callback */
static const hbool_t H5F_def_use_mdc_logging_g = H5F_ACS_USE_MDC_LOGGING_DEF; /* Default metadata cache logging flag */
static const char *H5F_def_mdc_log_location_g = H5F_ACS_MDC_LOG_LOCATION_DEF; /* Default mdc log location */
@ -492,6 +498,12 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass)
NULL, NULL, NULL, NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
/* Register the # of read attempts */
if(H5P_register_real(pclass, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, H5F_ACS_METADATA_READ_ATTEMPTS_SIZE, &H5F_def_metadata_read_attempts_g,
NULL, NULL, NULL, H5F_ACS_METADATA_READ_ATTEMPTS_ENC, H5F_ACS_METADATA_READ_ATTEMPTS_DEC,
NULL, NULL, NULL, NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
/* Register object flush callback */
/* (Note: this property should not have an encode/decode callback -QAK) */
if(H5P_register_real(pclass, H5F_ACS_OBJECT_FLUSH_CB_NAME, H5F_ACS_OBJECT_FLUSH_CB_SIZE, &H5F_def_object_flush_cb_g,
@ -3568,6 +3580,90 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_core_write_tracking() */
/*-------------------------------------------------------------------------
* Function: H5Pset_metadata_read_attempts
*
* Purpose: Sets the # of read attempts in the file access property list
* when reading metadata with checksum.
* The # of read attempts set via this routine will only apply
* when opening a file with SWMR access.
* The # of read attempts set via this routine does not have
* any effect when opening a file with non-SWMR access; for this
* case, the # of read attempts will be always be 1.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Vailin Choi; Sept 2013
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts)
{
H5P_genplist_t *plist; /* Property list pointer */
herr_t ret_value = SUCCEED; /* return value */
FUNC_ENTER_API(FAIL)
H5TRACE2("e", "iIu", plist_id, attempts);
/* Cannot set the # of attempts to 0 */
if(attempts == 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "number of metadatata read attempts must be greater than 0");
/* Get the plist structure */
if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
/* Set values */
if(H5P_set(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &attempts) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set # of metadata read attempts")
done:
FUNC_LEAVE_API(ret_value)
} /* H5Pset_metadata_read_attempts() */
/*-------------------------------------------------------------------------
* Function: H5Pget_metadata_read_attempts
*
* Purpose: Returns the # of metadata read attempts set in the file access property list.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Vailin Choi; Sept 2013
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pget_metadata_read_attempts(hid_t plist_id, unsigned *attempts/*out*/)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
H5TRACE2("e", "ix", plist_id, attempts);
/* Get values */
if(attempts) {
H5P_genplist_t *plist; /* Property list pointer */
/* Get the plist structure */
if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
/* Get the # of read attempts set */
if(H5P_get(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, attempts) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get the number of metadata read attempts")
/* If not set, return the default value */
if(*attempts == H5F_ACS_METADATA_READ_ATTEMPTS_DEF) /* 0 */
*attempts = H5F_METADATA_READ_ATTEMPTS;
} /* end if */
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_metadata_read_attempts() */
/*-------------------------------------------------------------------------
* Function: H5Pset_obj_flush_cb

2
src/H5Ppublic.h
View File

@ -351,6 +351,8 @@ H5_DLL herr_t H5Pget_file_image_callbacks(hid_t fapl_id,
H5FD_file_image_callbacks_t *callbacks_ptr);
H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size_t page_size);
H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size);
H5_DLL herr_t H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts);
H5_DLL herr_t H5Pget_metadata_read_attempts(hid_t plist_id, unsigned *attempts);
H5_DLL herr_t H5Pset_object_flush_cb(hid_t plist_id, H5F_flush_cb_t func, void *udata);
H5_DLL herr_t H5Pget_object_flush_cb(hid_t plist_id, H5F_flush_cb_t *func, void **udata);
H5_DLL herr_t H5Pset_mdc_log_options(hid_t plist_id, hbool_t is_enabled, const char *location, hbool_t start_on_access);

133
src/H5SMcache.c
View File

@ -58,7 +58,8 @@
/********************/
/* Metadata cache (H5AC) callbacks */
static herr_t H5SM__cache_table_get_load_size(const void *udata, size_t *image_len);
static herr_t H5SM__cache_table_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5SM__cache_table_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5SM__cache_table_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5SM__cache_table_image_len(const void *thing, size_t *image_len);
@ -66,7 +67,8 @@ static herr_t H5SM__cache_table_serialize(const H5F_t *f, void *image,
size_t len, void *thing);
static herr_t H5SM__cache_table_free_icr(void *thing);
static herr_t H5SM__cache_list_get_load_size(const void *udata, size_t *image_len);
static herr_t H5SM__cache_list_get_initial_load_size(void *udata, size_t *image_len);
static htri_t H5SM__cache_list_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *H5SM__cache_list_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5SM__cache_list_image_len(const void *thing, size_t *image_len);
@ -85,7 +87,9 @@ const H5AC_class_t H5AC_SOHM_TABLE[1] = {{
"shared message table", /* Metadata client name (for debugging) */
H5FD_MEM_SOHM_TABLE, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5SM__cache_table_get_load_size, /* 'get_load_size' callback */
H5SM__cache_table_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5SM__cache_table_verify_chksum, /* 'verify_chksum' callback */
H5SM__cache_table_deserialize, /* 'deserialize' callback */
H5SM__cache_table_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -100,7 +104,9 @@ const H5AC_class_t H5AC_SOHM_LIST[1] = {{
"shared message list", /* Metadata client name (for debugging) */
H5FD_MEM_SOHM_TABLE, /* File space memory type for client */
H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */
H5SM__cache_list_get_load_size, /* 'get_load_size' callback */
H5SM__cache_list_get_initial_load_size, /* 'get_initial_load_size' callback */
NULL, /* 'get_final_load_size' callback */
H5SM__cache_list_verify_chksum, /* 'verify_chksum' callback */
H5SM__cache_list_deserialize, /* 'deserialize' callback */
H5SM__cache_list_image_len, /* 'image_len' callback */
NULL, /* 'pre_serialize' callback */
@ -123,11 +129,10 @@ const H5AC_class_t H5AC_SOHM_LIST[1] = {{
/*-------------------------------------------------------------------------
* Function: H5SM__cache_table_get_load_size()
* Function: H5SM__cache_table_get_initial_load_size()
*
* Purpose: Return the size of the master table of Shared Object Header
* Message indexes on disk. As this cache client doesn't use
* speculative reads, this value should be accurate.
* Message indexes on disk.
*
* Return: Success: SUCCEED
* Failure: FAIL
@ -138,9 +143,9 @@ const H5AC_class_t H5AC_SOHM_LIST[1] = {{
*-------------------------------------------------------------------------
*/
static herr_t
H5SM__cache_table_get_load_size(const void *_udata, size_t *image_len)
H5SM__cache_table_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5SM_table_cache_ud_t *udata = (const H5SM_table_cache_ud_t *)_udata; /* User data for callback */
const H5SM_table_cache_ud_t *udata = (const H5SM_table_cache_ud_t *)_udata; /* User data for callback */
FUNC_ENTER_STATIC_NOERR
@ -149,10 +154,47 @@ H5SM__cache_table_get_load_size(const void *_udata, size_t *image_len)
HDassert(udata->f);
HDassert(image_len);
/* Set the image length size */
*image_len = H5SM_TABLE_SIZE(udata->f);
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5SM__cache_table_get_load_size() */
} /* end H5SM__cache_table_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5SM__cache_table_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
htri_t
H5SM__cache_table_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
/* Get stored and computed checksums */
H5F_get_checksums(image, len, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5SM__cache_table_verify_chksum() */
/*-------------------------------------------------------------------------
@ -180,7 +222,6 @@ H5SM__cache_table_deserialize(const void *_image, size_t len, void *_udata,
H5SM_table_cache_ud_t *udata = (H5SM_table_cache_ud_t *)_udata; /* Pointer to user data */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into input buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
size_t u; /* Counter variable for index headers */
void *ret_value = NULL; /* Return value */
@ -256,19 +297,14 @@ H5SM__cache_table_deserialize(const void *_image, size_t len, void *_udata,
table->indexes[u].list_size = H5SM_LIST_SIZE(f, table->indexes[u].list_max);
} /* end for */
/* checksum verification already done in verify_chksum cb */
/* Read in checksum */
UINT32DECODE(image, stored_chksum);
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) == table->table_size);
/* Compute checksum on entire header */
computed_chksum = H5_checksum_metadata(_image, (table->table_size - H5SM_SIZEOF_CHECKSUM), 0);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_SOHM, H5E_BADVALUE, NULL, "incorrect metadata checksum for shared message table")
/* Set return value */
ret_value = table;
@ -442,7 +478,7 @@ done:
/*-------------------------------------------------------------------------
* Function: H5SM__cache_list_get_load_size()
* Function: H5SM__cache_list_get_initial_load_size()
*
* Purpose: Return the on disk size of list of SOHM messages. In this case,
* we simply look up the size in the user data, and return that value
@ -457,9 +493,9 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
H5SM__cache_list_get_load_size(const void *_udata, size_t *image_len)
H5SM__cache_list_get_initial_load_size(void *_udata, size_t *image_len)
{
const H5SM_list_cache_ud_t *udata = (const H5SM_list_cache_ud_t *)_udata; /* User data for callback */
const H5SM_list_cache_ud_t *udata = (const H5SM_list_cache_ud_t *)_udata; /* User data for callback */
FUNC_ENTER_STATIC_NOERR
@ -469,10 +505,53 @@ H5SM__cache_list_get_load_size(const void *_udata, size_t *image_len)
HDassert(udata->header->list_size > 0);
HDassert(image_len);
/* Set the image length size */
*image_len = udata->header->list_size;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5SM__cache_list_get_load_size() */
} /* end H5SM__cache_list_get_initial_load_size() */
/*-------------------------------------------------------------------------
* Function: H5SM__cache_list_verify_chksum
*
* Purpose: Verify the computed checksum of the data structure is the
* same as the stored chksum.
*
* Return: Success: TRUE/FALSE
* Failure: Negative
*
* Programmer: Vailin Choi; Aug 2015
*
*-------------------------------------------------------------------------
*/
htri_t
H5SM__cache_list_verify_chksum(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata)
{
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
H5SM_list_cache_ud_t *udata = (H5SM_list_cache_ud_t *)_udata; /* User data for callback */
size_t chk_size; /* Exact size of the node with checksum at the end */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check arguments */
HDassert(image);
HDassert(udata);
/* Exact size with checksum at the end */
chk_size = H5SM_LIST_SIZE(udata->f, udata->header->num_messages);
/* Get stored and computed checksums */
H5F_get_checksums(image, chk_size, &stored_chksum, &computed_chksum);
if(stored_chksum != computed_chksum)
ret_value = FALSE;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5SM__cache_list_verify_chksum() */
/*-------------------------------------------------------------------------
@ -499,7 +578,6 @@ H5SM__cache_list_deserialize(const void *_image, size_t len, void *_udata,
H5SM_bt2_ctx_t ctx; /* Message encoding context */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into input buffer */
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
size_t u; /* Counter variable for messages in list */
void *ret_value = NULL; /* Return value */
@ -537,19 +615,14 @@ H5SM__cache_list_deserialize(const void *_image, size_t len, void *_udata,
image += H5SM_SOHM_ENTRY_SIZE(udata->f);
} /* end for */
/* checksum verification already done in verify_chksum cb */
/* Read in checksum */
UINT32DECODE(image, stored_chksum);
/* Sanity check */
HDassert((size_t)(image - (const uint8_t *)_image) <= udata->header->list_size);
/* Compute checksum on entire header */
computed_chksum = H5_checksum_metadata(_image, ((size_t)(image - (const uint8_t *)_image) - H5SM_SIZEOF_CHECKSUM), 0);
/* Verify checksum */
if(stored_chksum != computed_chksum)
HGOTO_ERROR(H5E_SOHM, H5E_BADVALUE, NULL, "incorrect metadata checksum for shared message list")
/* Initialize the rest of the array */
for(u = udata->header->num_messages; u < udata->header->list_max; u++)
list->messages[u].location = H5SM_NO_LOC;

8
src/H5private.h
View File

@ -813,7 +813,8 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation);
/* NOTE: flock(2) is not present on all POSIX systems.
* If it is not present, we try a flock() equivalent based on
* fcntl(2), then fall back to a function that always fails if
* it is not present at all.
* it is not present at all (Windows uses a separate Wflock()
* function).
*/
#if defined(H5_HAVE_FLOCK)
#define HDflock(F,L) flock(F,L)
@ -1114,6 +1115,9 @@ typedef off_t h5_stat_size_t;
#ifndef HDmodf
#define HDmodf(X,Y) modf(X,Y)
#endif /* HDmodf */
#ifndef HDnanosleep
#define HDnanosleep(N, O) nanosleep(N, O)
#endif /* HDnanosleep */
#ifndef HDopen
#ifdef _O_BINARY
#define HDopen(S,F,M) open(S,F|_O_BINARY,M)
@ -2597,6 +2601,8 @@ H5_DLL uint32_t H5_hash_string(const char *str);
/* Time related routines */
H5_DLL time_t H5_make_time(struct tm *tm);
H5_DLL void H5_nanosleep(uint64_t nanosec);
H5_DLL double H5_get_time(void);
/* Functions for building paths, etc. */
H5_DLL herr_t H5_build_extpath(const char *name, char **extpath /*out*/);

81
src/H5system.c
View File

@ -911,6 +911,27 @@ Wflock(int fd, int operation) {
return 0;
} /* end Wflock() */
/*--------------------------------------------------------------------------
* Function: Wnanosleep
*
* Purpose: Sleep for a given # of nanoseconds (Windows version)
*
* Return: SUCCEED/FAIL
*
* Programmer: Dana Robinson
* Fall 2016
*--------------------------------------------------------------------------
*/
void
Wnanosleep(uint64_t nanosec)
{
/* XXX: Currently just a placeholder */
FUNC_ENTER_NOAPI_NOINIT_NOERR
FUNC_LEAVE_NOAPI_VOID
} /* end Wnanosleep() */
#endif /* H5_HAVE_WIN32_API */
@ -1108,3 +1129,63 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5_combine_path() */
/*--------------------------------------------------------------------------
* Function: H5_nanosleep
*
* Purpose: Sleep for a given # of nanoseconds
*
* Return: SUCCEED/FAIL
*
* Programmer: Quincey Koziol
* October 01, 2016
*--------------------------------------------------------------------------
*/
void
H5_nanosleep(uint64_t nanosec)
{
struct timespec sleeptime; /* Struct to hold time to sleep */
FUNC_ENTER_NOAPI_NOINIT_NOERR
/* Set up time to sleep */
sleeptime.tv_sec = 0;
sleeptime.tv_nsec = (long)nanosec;
HDnanosleep(&sleeptime, NULL);
FUNC_LEAVE_NOAPI_VOID
} /* end H5_nanosleep() */
/*--------------------------------------------------------------------------
* Function: H5_get_time
*
* Purpose: Get the current time, as the time of seconds after the UNIX epoch
*
* Return: SUCCEED/FAIL
*
* Programmer: Quincey Koziol
* October 05, 2016
*--------------------------------------------------------------------------
*/
double
H5_get_time(void)
{
#ifdef H5_HAVE_GETTIMEOFDAY
struct timeval curr_time;
#endif /* H5_HAVE_GETTIMEOFDAY */
double ret_value = (double)0.0f;
FUNC_ENTER_NOAPI_NOINIT_NOERR
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&curr_time, NULL);
ret_value = (double)curr_time.tv_sec + ((double)curr_time.tv_usec / (double)1000000.0f);
#endif /* H5_HAVE_GETTIMEOFDAY */
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5_get_time() */

15
src/H5win32defs.h
View File

@ -42,6 +42,7 @@ typedef __int64 h5_stat_size_t;
#define HDlseek(F,O,W) _lseeki64(F,O,W)
#define HDlstat(S,B) _lstati64(S,B)
#define HDmkdir(S,M) _mkdir(S)
#define HDnanosleep(S) Wnanosleep(S)
#define HDoff_t __int64
/* _O_BINARY must be set in Windows to avoid CR-LF <-> LF EOL
* transformations when performing I/O.
@ -65,13 +66,23 @@ typedef __int64 h5_stat_size_t;
*/
#define HDmemset(X,C,Z) memset((void*)(X),C,Z)
#endif /* H5_HAVE_VISUAL_STUDIO */
struct timezone {
int tz_minuteswest;
int tz_dsttime;
};
/* time.h before VS2015 does not include timespec */
#if (_MSC_VER < 1900)
struct timespec
{
time_t tv_sec; // Seconds - >= 0
long tv_nsec; // Nanoseconds - [0, 999999999]
};
#endif /* MSC_VER < 1900 */
#endif /* H5_HAVE_VISUAL_STUDIO */
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */

130
test/cache.c
View File

@ -17147,6 +17147,135 @@ check_protect_ro_rw_err(void)
} /* check_protect_ro_rw_err() */
/*-------------------------------------------------------------------------
* Function: check_protect_retries()
*
* Purpose: To exercise checksum verification retries for an entry with
* a speculative load.
*
* Return:
*
* Programmer:
*
*-------------------------------------------------------------------------
*/
static unsigned
check_protect_retries(void)
{
H5F_t * file_ptr = NULL;
H5C_t *cache_ptr = NULL;
test_entry_t *base_addr = NULL;
test_entry_t *entry_ptr = NULL;
H5C_cache_entry_t * cache_entry_ptr = NULL;
int32_t type;
int32_t idx;
TESTING("protect an entry to verify retries");
pass = TRUE;
/* Set up the cache */
if(pass) {
reset_entries();
file_ptr = setup_cache((size_t)(2 * 1024),
(size_t)(1 * 1024));
/* Set up read attempts for verifying checksum */
file_ptr->shared->read_attempts = 10;
file_ptr->shared->retries_nbins = 1;
}
/* Test only for this type which has a speculative load */
type = VARIABLE_ENTRY_TYPE;
idx = 0;
if(pass) {
cache_ptr = file_ptr->shared->cache;
base_addr = entries[type];
entry_ptr = &(base_addr[idx]);
/* test case (1):
* --actual_len is smaller the initial length from get_load_size()
* --verify_chksum() returns TRUE after max_verify_ct is reached
*
*/
entry_ptr->actual_len = entry_ptr->size/2;
entry_ptr->max_verify_ct = 3;
entry_ptr->verify_ct = 0;
cache_entry_ptr = (H5C_cache_entry_t *)H5C_protect(file_ptr, H5AC_ind_read_dxpl_id,
&(types[type]), entry_ptr->addr, &entry_ptr->addr, H5C__READ_ONLY_FLAG);
if((cache_entry_ptr != (void *)entry_ptr) ||
(!(entry_ptr->header.is_protected)) ||
(!(entry_ptr->header.is_read_only)) ||
(entry_ptr->header.ro_ref_count <= 0) ||
(entry_ptr->header.type != &(types[type])) ||
(entry_ptr->size != entry_ptr->header.size) ||
(entry_ptr->addr != entry_ptr->header.addr) ||
(entry_ptr->verify_ct != entry_ptr->max_verify_ct)) {
pass = FALSE;
failure_mssg = "error from H5C_protect().";
} else {
HDassert((entry_ptr->cache_ptr == NULL) ||
(entry_ptr->cache_ptr == cache_ptr));
entry_ptr->cache_ptr = cache_ptr;
entry_ptr->file_ptr = file_ptr;
entry_ptr->is_protected = TRUE;
entry_ptr->is_read_only = TRUE;
entry_ptr->ro_ref_count++;
}
HDassert(((entry_ptr->header).type)->id == type);
}
if(pass)
unprotect_entry(file_ptr, VARIABLE_ENTRY_TYPE, idx, H5C__NO_FLAGS_SET);
if(pass) {
entry_ptr = &(base_addr[++idx]);
/* test case (2):
* --actual_len is greater the initial length from get_load_size()
* --verify_chksum() returns FALSE even after all tries is reached
* (file_ptr->shared->read_attempts is smaller then max_verify_ct)
*/
entry_ptr->actual_len = entry_ptr->size*2;
entry_ptr->max_verify_ct = 11;
entry_ptr->verify_ct = 0;
cache_entry_ptr = (H5C_cache_entry_t *)H5C_protect(file_ptr, H5AC_ind_read_dxpl_id,
&(types[type]), entry_ptr->addr, &entry_ptr->addr, H5C__READ_ONLY_FLAG);
/* H5C_protect() should fail after all retries fail */
if(cache_entry_ptr != NULL)
pass = FALSE;
}
takedown_cache(file_ptr, FALSE, FALSE);
reset_entries();
if(pass) { PASSED(); } else { H5_FAILED(); }
if(!pass) {
HDfprintf(stdout, "%s: failure_msg = \"%s\".\n",
FUNC, failure_mssg);
}
return (unsigned)!pass;
} /* check_protect_retries() */
/*-------------------------------------------------------------------------
* Function: check_evictions_enabled_err()
@ -36190,6 +36319,7 @@ main(void)
nerrs += check_resize_entry_errs();
nerrs += check_unprotect_ro_dirty_err();
nerrs += check_protect_ro_rw_err();
nerrs += check_protect_retries();
nerrs += check_check_evictions_enabled_err();
nerrs += check_auto_cache_resize(FALSE);
nerrs += check_auto_cache_resize(TRUE);

248
test/cache_common.c
View File

@ -76,17 +76,22 @@ static test_entry_t *notify_entries = NULL, *orig_notify_entries = NULL;
hbool_t orig_entry_arrays_init = FALSE;
static herr_t pico_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t nano_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t micro_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t tiny_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t small_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t medium_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t large_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t huge_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t monster_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t variable_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t notify_get_load_size(const void *udata_ptr, size_t *image_len_ptr);
static herr_t pico_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t nano_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t micro_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t tiny_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t small_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t medium_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t large_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t huge_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t monster_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t variable_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t notify_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr);
static herr_t variable_get_final_load_size(const void *image, size_t image_len,
void *udata, size_t *actual_len);
static htri_t variable_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr);
static void *pico_deserialize(const void *image_ptr, size_t len, void *udata_ptr,
hbool_t *dirty_ptr);
@ -199,8 +204,10 @@ static void mark_flush_dep_dirty(test_entry_t * entry_ptr);
static void mark_flush_dep_clean(test_entry_t * entry_ptr);
/* Generic callback routines */
static herr_t get_load_size(const void *udata_ptr, size_t *image_len_ptr,
static herr_t get_initial_load_size(void *udata_ptr, size_t *image_len_ptr,
int32_t entry_type);
static herr_t get_final_load_size(const void *image, size_t image_len,
void *udata, size_t *actual_len, int32_t entry_type);
static void *deserialize(const void *image_ptr, size_t len, void *udata_ptr,
hbool_t *dirty_ptr, int32_t entry_type);
static herr_t image_len(const void *thing, size_t *image_len_ptr, int32_t entry_type);
@ -306,7 +313,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"pico_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_NO_FLAGS_SET,
pico_get_load_size,
pico_get_initial_load_size,
NULL,
NULL,
pico_deserialize,
pico_image_len,
pico_pre_serialize,
@ -320,7 +329,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"nano_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_NO_FLAGS_SET,
nano_get_load_size,
nano_get_initial_load_size,
NULL,
NULL,
nano_deserialize,
nano_image_len,
nano_pre_serialize,
@ -334,7 +345,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"micro_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_NO_FLAGS_SET,
micro_get_load_size,
micro_get_initial_load_size,
NULL,
NULL,
micro_deserialize,
micro_image_len,
micro_pre_serialize,
@ -348,7 +361,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"tiny_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_NO_FLAGS_SET,
tiny_get_load_size,
tiny_get_initial_load_size,
NULL,
NULL,
tiny_deserialize,
tiny_image_len,
tiny_pre_serialize,
@ -362,7 +377,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"small_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_NO_FLAGS_SET,
small_get_load_size,
small_get_initial_load_size,
NULL,
NULL,
small_deserialize,
small_image_len,
small_pre_serialize,
@ -376,7 +393,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"medium_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_NO_FLAGS_SET,
medium_get_load_size,
medium_get_initial_load_size,
NULL,
NULL,
medium_deserialize,
medium_image_len,
medium_pre_serialize,
@ -390,7 +409,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"large_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_NO_FLAGS_SET,
large_get_load_size,
large_get_initial_load_size,
NULL,
NULL,
large_deserialize,
large_image_len,
large_pre_serialize,
@ -404,7 +425,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"huge_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_NO_FLAGS_SET,
huge_get_load_size,
huge_get_initial_load_size,
NULL,
NULL,
huge_deserialize,
huge_image_len,
huge_pre_serialize,
@ -418,7 +441,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"monster_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_NO_FLAGS_SET,
monster_get_load_size,
monster_get_initial_load_size,
NULL,
NULL,
monster_deserialize,
monster_image_len,
monster_pre_serialize,
@ -432,7 +457,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"variable_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_SPECULATIVE_LOAD_FLAG,
variable_get_load_size,
variable_get_initial_load_size,
variable_get_final_load_size,
variable_verify_chksum,
variable_deserialize,
variable_image_len,
variable_pre_serialize,
@ -446,7 +473,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
"notify_entry",
H5FD_MEM_DEFAULT,
H5C__CLASS_NO_FLAGS_SET,
notify_get_load_size,
notify_get_initial_load_size,
NULL,
NULL,
notify_deserialize,
notify_image_len,
notify_pre_serialize,
@ -570,10 +599,10 @@ check_write_permitted(const H5F_t H5_ATTR_UNUSED *f, hbool_t *write_permitted_pt
/*-------------------------------------------------------------------------
* Function: get_load_size & friends
* Function: get_initial_load_size & friends
*
* Purpose: Query the image size for loading an entry. The helper
* functions funnel into get_load_size proper.
* functions funnel into get_initial_load_size proper.
*
* Return: SUCCEED
*
@ -583,7 +612,7 @@ check_write_permitted(const H5F_t H5_ATTR_UNUSED *f, hbool_t *write_permitted_pt
*-------------------------------------------------------------------------
*/
static herr_t
get_load_size(const void *udata, size_t *image_length, int32_t entry_type)
get_initial_load_size(void *udata, size_t *image_length, int32_t entry_type)
{
test_entry_t *entry;
test_entry_t *base_addr;
@ -609,72 +638,185 @@ get_load_size(const void *udata, size_t *image_length, int32_t entry_type)
*image_length = entry->size;
return(SUCCEED);
} /* get_load_size() */
} /* get_initial_load_size() */
static herr_t
pico_get_load_size(const void *udata, size_t *image_length)
pico_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, PICO_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, PICO_ENTRY_TYPE);
}
static herr_t
nano_get_load_size(const void *udata, size_t *image_length)
nano_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, NANO_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, NANO_ENTRY_TYPE);
}
static herr_t
micro_get_load_size(const void *udata, size_t *image_length)
micro_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, MICRO_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, MICRO_ENTRY_TYPE);
}
static herr_t
tiny_get_load_size(const void *udata, size_t *image_length)
tiny_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, TINY_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, TINY_ENTRY_TYPE);
}
static herr_t
small_get_load_size(const void *udata, size_t *image_length)
small_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, SMALL_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, SMALL_ENTRY_TYPE);
}
static herr_t
medium_get_load_size(const void *udata, size_t *image_length)
medium_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, MEDIUM_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, MEDIUM_ENTRY_TYPE);
}
static herr_t
large_get_load_size(const void *udata, size_t *image_length)
large_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, LARGE_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, LARGE_ENTRY_TYPE);
}
static herr_t
huge_get_load_size(const void *udata, size_t *image_length)
huge_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, HUGE_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, HUGE_ENTRY_TYPE);
}
static herr_t
monster_get_load_size(const void *udata, size_t *image_length)
monster_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, MONSTER_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, MONSTER_ENTRY_TYPE);
}
static herr_t
variable_get_load_size(const void *udata, size_t *image_length)
variable_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, VARIABLE_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, VARIABLE_ENTRY_TYPE);
}
static herr_t
notify_get_load_size(const void *udata, size_t *image_length)
notify_get_initial_load_size(void *udata, size_t *image_length)
{
return get_load_size(udata, image_length, NOTIFY_ENTRY_TYPE);
return get_initial_load_size(udata, image_length, NOTIFY_ENTRY_TYPE);
}
/*-------------------------------------------------------------------------
* Function: get_final_load_size & friends
*
* Purpose: Query the final image size for loading an entry. The helper
* functions funnel into get_final_load_size proper.
*
* Return: SUCCEED
*
* Programmer: Quincey Koziol
* 11/18/16
*
*-------------------------------------------------------------------------
*/
static herr_t
get_final_load_size(const void *image, size_t image_len, void *udata,
size_t *actual_len, int32_t entry_type)
{
test_entry_t *entry;
test_entry_t *base_addr;
haddr_t addr = *(const haddr_t *)udata;
int32_t type;
int32_t idx;
addr_to_type_and_index(addr, &type, &idx);
base_addr = entries[type];
entry = &(base_addr[idx]);
HDassert(entry->type >= 0);
HDassert(entry->type == type);
HDassert(entry->type == entry_type);
HDassert(entry->type < NUMBER_OF_ENTRY_TYPES);
HDassert(entry->index == idx);
HDassert(entry->index >= 0);
HDassert(entry->index <= max_indices[type]);
HDassert(entry == entry->self);
HDassert(entry->addr == addr);
HDassert(type == VARIABLE_ENTRY_TYPE);
/* Simulate SPECULATIVE read with a specified actual_len */
if(entry->actual_len) {
*actual_len = entry->actual_len;
entry->size = entry->actual_len;
} /* end if */
else
*actual_len = entry->size;
return(SUCCEED);
} /* get_final_load_size() */
static herr_t
variable_get_final_load_size(const void *image, size_t image_len,
void *udata, size_t *actual_len)
{
return get_final_load_size(image, image_len, udata, actual_len, VARIABLE_ENTRY_TYPE);
}
/*-------------------------------------------------------------------------
* Function: verify_chksum & friends
* (only done for VARIABLE_ENTRY_TYPE which has a speculative read)
*
* Purpose: Simulate checksum verification:
* --check is ok only after 'max_verify_ct' is reached
* --otherwise check is not ok
*
* Return: TRUE: checksum is ok
* FALSE: checksum is not ok
*
* Programmer:
*
*-------------------------------------------------------------------------
*/
static htri_t
verify_chksum(const void H5_ATTR_UNUSED *image, size_t H5_ATTR_UNUSED len, void *udata, int32_t entry_type)
{
test_entry_t *entry;
test_entry_t *base_addr;
haddr_t addr = *(const haddr_t *)udata;
int32_t type;
int32_t idx;
addr_to_type_and_index(addr, &type, &idx);
base_addr = entries[type];
entry = &(base_addr[idx]);
HDassert(entry->type >= 0);
HDassert(entry->type == type);
HDassert(entry->type == entry_type);
HDassert(entry->type < NUMBER_OF_ENTRY_TYPES);
HDassert(type == VARIABLE_ENTRY_TYPE);
HDassert(entry->index == idx);
HDassert(entry->index >= 0);
HDassert(entry->index <= max_indices[type]);
HDassert(entry == entry->self);
HDassert(entry->addr == addr);
if(++entry->verify_ct >= entry->max_verify_ct)
return(TRUE);
else
return(FALSE);
} /* verify_chksum() */
static htri_t
variable_verify_chksum(const void *image, size_t len, void *udata)
{
return verify_chksum(image, len, udata, VARIABLE_ENTRY_TYPE);
}
@ -2340,6 +2482,10 @@ reset_entries(void)
base_addr[j].notify_after_insert_count = 0;
base_addr[j].notify_before_evict_count = 0;
base_addr[j].actual_len = 0;
base_addr[j].max_verify_ct = 0;
base_addr[j].verify_ct = 0;
addr += (haddr_t)entry_size;
alt_addr += (haddr_t)entry_size;
} /* end for */
@ -3903,6 +4049,10 @@ protect_entry(H5F_t * file_ptr, int32_t type, int32_t idx)
HDfprintf(stdout,
"entry_ptr->addr = %d, entry_ptr->header.addr = %d\n",
(int)(entry_ptr->addr), (int)(entry_ptr->header.addr));
HDfprintf(stdout,
"entry_ptr->verify_ct = %d, entry_ptr->max_verify_ct = %d\n",
entry_ptr->verify_ct, entry_ptr->max_verify_ct);
H5Eprint2(H5E_DEFAULT, stdout);
#endif
pass = FALSE;
failure_mssg = "error in H5C_protect().";

3
test/cache_common.h
View File

@ -360,6 +360,9 @@ typedef struct test_entry_t
unsigned notify_after_insert_count; /* Count of times that entry was inserted in cache */
unsigned notify_before_evict_count; /* Count of times that entry was removed in cache */
unsigned actual_len; /* Simulate the entry's actual size for a speculative load */
unsigned max_verify_ct; /* Maximum # of times to verify an entry's checksum */
unsigned verify_ct; /* Count the # of checksum verification for an entry */
} test_entry_t;
/* The following are cut down test versions of the hash table manipulation

23
test/earray.c
View File

@ -160,9 +160,7 @@ typedef struct earray_test_t {
/* Local prototypes */
/* Metadata cache (H5AC) callbacks */
static herr_t earray_cache_test_get_load_size(const void *udata_ptr,
size_t *image_len_ptr);
static herr_t earray_cache_test_get_initial_load_size(void *udata, size_t *image_len);
static void *earray_cache_test_deserialize(const void *image_ptr, size_t len,
void *udata_ptr, hbool_t *dirty_ptr);
static herr_t earray_cache_test_image_len(const void *thing, size_t *image_len_ptr);
@ -190,7 +188,9 @@ const H5AC_class_t H5AC_EARRAY_TEST[1] = {{
/* name */ "earray test",
/* mem_type */ H5FD_MEM_DEFAULT,
/* flags */ H5AC__CLASS_SKIP_READS | H5AC__CLASS_SKIP_WRITES,
/* get_load_size */ earray_cache_test_get_load_size,
/* get_initial_load_size */ earray_cache_test_get_initial_load_size,
/* get_final_load_size */ NULL,
/* verify_chksum */ NULL,
/* deserialize */ earray_cache_test_deserialize,
/* image_len */ earray_cache_test_image_len,
/* pre_serialize */ NULL,
@ -619,13 +619,12 @@ error:
/*-------------------------------------------------------------------------
* Function: earray_cache_test_get_load_size()
* Function: earray_cache_test_get_initial_load_size()
*
* Purpose: place holder function -- should never be called
*
*
* A generic discussion of metadata cache callbacks of this type
* may be found in H5Cprivate.h:
* may be found in H5Cprivate.h.
*
* Return: Success: SUCCEED
* Failure: FAIL
@ -636,18 +635,18 @@ error:
*-------------------------------------------------------------------------
*/
static herr_t
earray_cache_test_get_load_size(const void *udata_ptr, size_t *image_len_ptr)
earray_cache_test_get_initial_load_size( void *udata, size_t *image_len)
{
HDassert(udata_ptr);
HDassert(image_len_ptr);
HDassert(udata);
HDassert(image_len);
/* Should never be called */
HDassert(0 && "Can't be called!");
*image_len_ptr = 0;
*image_len = 0;
return(SUCCEED);
} /* end earray_cache_test_get_load_size() */
} /* end earray_cache_test_get_initial_load_size() */
/*-------------------------------------------------------------------------

15
testpar/t_cache.c
View File

@ -390,7 +390,7 @@ static hbool_t serve_rw_count_reset_request(struct mssg_t * mssg_ptr);
/* call back functions & related data structures */
static herr_t datum_get_load_size(const void * udata_ptr,
static herr_t datum_get_initial_load_size(void *udata_ptr,
size_t *image_len_ptr);
static void * datum_deserialize(const void * image_ptr,
@ -436,7 +436,9 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] =
/* name */ "datum",
/* mem_type */ H5FD_MEM_DEFAULT,
/* flags */ H5AC__CLASS_SKIP_READS | H5AC__CLASS_SKIP_WRITES,
/* get_load_size */ datum_get_load_size,
/* get_initial_load_size */ datum_get_initial_load_size,
/* get_final_load_size */ NULL,
/* verify_chksum */ NULL,
/* deserialize */ datum_deserialize,
/* image_len */ datum_image_len,
/* pre_serialize */ NULL,
@ -2318,7 +2320,7 @@ serve_rw_count_reset_request(struct mssg_t * mssg_ptr)
/*-------------------------------------------------------------------------
* Function: datum_get_load_size
* Function: datum_get_initial_load_size
*
* Purpose: Query the image size for an entry before deserializing it
*
@ -2330,8 +2332,7 @@ serve_rw_count_reset_request(struct mssg_t * mssg_ptr)
*-------------------------------------------------------------------------
*/
static herr_t
datum_get_load_size(const void * udata_ptr,
size_t *image_len_ptr)
datum_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr)
{
haddr_t addr = *(haddr_t *)udata_ptr;
int idx;
@ -2355,7 +2356,7 @@ datum_get_load_size(const void * udata_ptr,
if ( callbacks_verbose ) {
HDfprintf(stdout,
"%d: get_load_size() idx = %d, addr = %ld, len = %d.\n",
"%d: get_initial_load_size() idx = %d, addr = %ld, len = %d.\n",
world_mpi_rank, idx, (long)addr, (int)entry_ptr->local_len);
fflush(stdout);
}
@ -2364,7 +2365,7 @@ datum_get_load_size(const void * udata_ptr,
*image_len_ptr = entry_ptr->local_len;
return(SUCCEED);
} /* get_load_size() */
} /* get_initial_load_size() */
/*-------------------------------------------------------------------------