mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-11-21 01:04:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r13808] Description:

Browse Source 
Fix possible file corruption when using "new" format object headers and the
size of chunk #0 for an object header transitions between needing 1->2->4->8-
byte encoding for the size and there are "clean" messages in the object header
already.  (Usually triggered by flushing the file while adding attributes to
an object)

Tested on:
    Mac OS X/32 10.4.9 (amazon)
    Linux/32 2.6 (chicago)
    Linux/64 2.6 (chicago2)
This commit is contained in:
Quincey Koziol 2007-05-24 13:36:53 -05:00
parent 415889bef3
commit fa133cfb95
3 changed files with 261 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/H5O.c
View File

@ -720,7 +720,7 @@ H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, hid_t ocpl_id,
/* Allocate disk space for header and first chunk */
if(HADDR_UNDEF == (oh_addr = H5MF_alloc(f, H5FD_MEM_OHDR, dxpl_id, (hsize_t)oh_size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for object header header")
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for object header")
/* Create the chunk list */
oh->nchunks = oh->alloc_nchunks = 1;
@ -1511,23 +1511,27 @@ static herr_t
H5O_obj_type_real(H5O_t *oh, H5O_type_t *obj_type)
{
const H5O_obj_class_t *obj_class; /* Class of object for header */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT(H5O_obj_type_real)
FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_obj_type_real)
/* Sanity check */
HDassert(oh);
HDassert(obj_type);
/* Look up class for object header */
if(NULL == (obj_class = H5O_obj_class_real(oh)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to determine object type")
if(NULL == (obj_class = H5O_obj_class_real(oh))) {
/* Clear error stack from "failed" class lookup */
H5E_clear_stack(NULL);
/* Set object type */
*obj_type = obj_class->type;
/* Set type to "unknown" */
*obj_type = H5O_TYPE_UNKNOWN;
} /* end if */
else {
/* Set object type */
*obj_type = obj_class->type;
} /* end else */
done:
FUNC_LEAVE_NOAPI(ret_value)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5O_obj_type_real() */
@ -1927,7 +1931,7 @@ H5O_get_info(H5O_loc_t *oloc, H5O_info_t *oinfo, hid_t dxpl_id)
/* Iterate over all the chunks, adding any gaps to the free space */
oinfo->hdr.space.total = 0;
for(u = 0, curr_chunk = &oh->chunk[0]; u < oh->nchunks; u++, curr_chunk++) {
/* Accumulate the size of the header on header */
/* Accumulate the size of the header on disk */
oinfo->hdr.space.total += curr_chunk->size;
/* If the chunk has a gap, add it to the free space */

6
src/H5Oalloc.c
View File

@ -594,9 +594,13 @@ H5O_alloc_extend_chunk(H5F_t *f,
/* Spin through existing messages, adjusting them */
for(u = 0; u < oh->nmesgs; u++) {
/* Adjust raw addresses for messages in this chunk to reflect new 'image' address */
if(oh->mesg[u].chunkno == chunkno)
if(oh->mesg[u].chunkno == chunkno) {
oh->mesg[u].raw = oh->chunk[chunkno].image + extra_prfx_size + (oh->mesg[u].raw - old_image);
/* Flag message as dirty */
oh->mesg[u].dirty = TRUE;
} /* endif */
/* Find continuation message which points to this chunk and adjust chunk's size */
/* (Chunk 0 doesn't have a continuation message that points to it and
* it's size is directly encoded in the object header) */

440
test/ohdr.c
View File

@ -67,240 +67,284 @@ main(void)
hid_t dset=-1;
H5F_t *f=NULL;
char filename[1024];
H5O_info_t oinfo; /* Object info */
H5O_loc_t oh_loc;
time_t time_new, ro;
int i;
hbool_t b; /* Index for "new format" loop */
const char *envval = NULL;
/* Reset library */
h5_reset();
fapl = h5_fileaccess();
h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR
if(NULL == (f = H5I_object(file))) FAIL_STACK_ERROR
/* Loop over old & new formats */
for(b = FALSE; b <= TRUE; b++) {
/* Display info about testing */
if(b)
HDputs("Using new file format");
else
HDputs("Using default file format");
/* Set the format to use for the file */
if (H5Pset_latest_format(fapl, b) < 0) FAIL_STACK_ERROR
/* Create the file to operate on */
if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR
if(NULL == (f = H5I_object(file))) FAIL_STACK_ERROR
/*
* Test object header creation
* (using default group creation property list only because it's convenient)
*/
TESTING("object header creation");
HDmemset(&oh_loc, 0, sizeof(oh_loc));
if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, H5P_GROUP_CREATE_DEFAULT, &oh_loc/*out*/) < 0)
FAIL_STACK_ERROR
PASSED();
/* create a new message */
TESTING("message creation");
time_new = 11111111;
if(H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
FAIL_STACK_ERROR
if(NULL == H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro, H5P_DATASET_XFER_DEFAULT))
FAIL_STACK_ERROR
if(ro != time_new)
TEST_ERROR
PASSED();
/*
* Test modification of an existing message.
*/
TESTING("message modification");
time_new = 33333333;
if(H5O_msg_write(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
FAIL_STACK_ERROR
if(NULL == H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro, H5P_DATASET_XFER_DEFAULT))
FAIL_STACK_ERROR
if(ro != time_new)
TEST_ERROR
PASSED();
/*
* Test creation of a bunch of messages one after another to see
* what happens when the object header overflows in core.
* (Use 'old' MTIME message here, because it is large enough to be
* replaced with a continuation message (the new one is too small)
* and the library doesn't understand how to migrate more than one
* message from an object header currently - QAK - 10/8/03)
*/
TESTING("object header overflow in memory");
for(i = 0; i < 40; i++) {
time_new = (i + 1) * 1000 + 1;
if(H5O_msg_create(&oh_loc, H5O_MTIME_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
/*
* Test object header creation
* (using default group creation property list only because it's convenient)
*/
TESTING("object header creation");
HDmemset(&oh_loc, 0, sizeof(oh_loc));
if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, H5P_GROUP_CREATE_DEFAULT, &oh_loc/*out*/) < 0)
FAIL_STACK_ERROR
} /* end for */
if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
FAIL_STACK_ERROR
PASSED();
PASSED();
/*
* Test creation of a bunch of messages one after another to see
* what happens when the object header overflows on disk.
*/
TESTING("object header overflow on disk");
for(i = 0; i < 10; i++) {
time_new = (i + 1) * 1000 + 10;
/* create a new message */
TESTING("message creation");
time_new = 11111111;
if(H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
FAIL_STACK_ERROR
} /* end for */
PASSED();
if(NULL == H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro, H5P_DATASET_XFER_DEFAULT))
FAIL_STACK_ERROR
if(ro != time_new)
TEST_ERROR
PASSED();
/*
* Delete all time messages.
*/
TESTING("message deletion");
if(H5O_msg_remove(&oh_loc, H5O_MTIME_NEW_ID, H5O_ALL, TRUE, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(H5O_msg_remove(&oh_loc, H5O_MTIME_ID, H5O_ALL, TRUE, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro, H5P_DATASET_XFER_DEFAULT))
FAIL_STACK_ERROR
if(H5O_msg_read(&oh_loc, H5O_MTIME_ID, &ro, H5P_DATASET_XFER_DEFAULT))
FAIL_STACK_ERROR
PASSED();
/* release resources */
TESTING("object header closing");
if(H5O_close(&oh_loc) < 0)
FAIL_STACK_ERROR
PASSED();
/* Test reading datasets with undefined object header messages */
puts("Reading objects with unknown header messages");
envval = HDgetenv("HDF5_DRIVER");
if(envval == NULL)
envval = "nomatch";
if(HDstrcmp(envval, "core") && HDstrcmp(envval, "multi") && HDstrcmp(envval, "split") && HDstrcmp(envval, "family")) {
hid_t file2; /* File ID for 'bogus' object file */
char testpath[512] = "";
char testfile[512] = "";
char *srcdir = HDgetenv("srcdir");
/* Build path to all test files */
if(srcdir && ((HDstrlen(srcdir) + 2) < sizeof(testpath))) {
HDstrcpy(testpath, srcdir);
HDstrcat(testpath, "/");
} /* end if */
/* Build path to test file */
if(srcdir && ((HDstrlen(testpath) + HDstrlen(FILE_BOGUS) + 1) < sizeof(testfile)))
HDstrcpy(testfile, testpath);
HDstrcat(testfile, FILE_BOGUS);
TESTING("object with unknown header message and no flags set");
/* Open the file with objects that have unknown header messages (generated with gen_bogus.c) */
if((file2 = H5Fopen(testfile, H5F_ACC_RDONLY, fapl)) < 0)
/*
* Test modification of an existing message.
*/
TESTING("message modification");
time_new = 33333333;
if(H5O_msg_write(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
FAIL_STACK_ERROR
if(NULL == H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro, H5P_DATASET_XFER_DEFAULT))
FAIL_STACK_ERROR
if(ro != time_new)
TEST_ERROR
/* Open the dataset with the unknown header message, but no extra flags */
if((dset = H5Dopen(file2, "/Dataset1")) < 0)
TEST_ERROR
if(H5Dclose(dset) < 0)
/* Make certain that chunk #0 in the object header can be encoded with a 1-byte size */
if(H5O_get_info(&oh_loc, &oinfo, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(oinfo.hdr.space.total >=256)
TEST_ERROR
PASSED();
TESTING("object with unknown header message & 'fail if unknown' flag set");
/* Attempt to open the dataset with the unknown header message, and "fail if unknown" flag */
H5E_BEGIN_TRY {
dset = H5Dopen(file2, "/Dataset2");
} H5E_END_TRY;
if(dset >= 0) {
H5Dclose(dset);
TEST_ERROR
} /* end if */
PASSED();
TESTING("object with unknown header message & 'mark if unknown' flag set");
/* Copy object with "mark if unknown" flag on message into file that can be modified */
if(H5Ocopy(file2, "/Dataset3", file, "/Dataset3", H5P_DEFAULT, H5P_DEFAULT) < 0)
TEST_ERROR
/* Close the file we created (to flush changes to file) */
if(H5Fclose(file) < 0)
TEST_ERROR
/* Re-open the file created, with read-only permissions */
if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR
/* Open the dataset with the "mark if unknown" message */
if((dset = H5Dopen(file, "/Dataset3")) < 0)
TEST_ERROR
/* Check that the "unknown" message was _NOT_ marked */
if(H5O_check_msg_marked_test(dset, FALSE) < 0)
/*
* Test creation of a bunch of messages one after another to see
* what happens when the object header overflows in core.
* (Use 'old' MTIME message here, because it is large enough to be
* replaced with a continuation message (the new one is too small)
* and the library doesn't understand how to migrate more than one
* message from an object header currently - QAK - 10/8/03)
*/
TESTING("object header overflow in memory");
for(i = 0; i < 40; i++) {
time_new = (i + 1) * 1000 + 1;
if(H5O_msg_create(&oh_loc, H5O_MTIME_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
} /* end for */
if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
FAIL_STACK_ERROR
/* Close the dataset */
if(H5Dclose(dset) < 0)
/* Make certain that chunk #0 in the object header will be encoded with a 2-byte size */
if(H5O_get_info(&oh_loc, &oinfo, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(oinfo.hdr.space.total < 256)
TEST_ERROR
/* Close the file we created (to flush change to object header) */
PASSED();
/* Close & re-open file & object header */
/* (makes certain that an object header in the new format that transitions
* between 1-byte chunk #0 size encoding and 2-byte chunk #0 size encoding
* works correctly - QAK)
*/
if(H5O_close(&oh_loc) < 0)
FAIL_STACK_ERROR
if(H5Fclose(file) < 0)
TEST_ERROR
/* Re-open the file created */
FAIL_STACK_ERROR
if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0)
TEST_ERROR
/* Open the dataset with the "mark if unknown" message */
if((dset = H5Dopen(file, "/Dataset3")) < 0)
TEST_ERROR
if(H5Dclose(dset) < 0)
TEST_ERROR
/* Close the file we created (to flush change to object header) */
if(H5Fclose(file) < 0)
TEST_ERROR
/* Re-open the file created */
if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR
/* Re-open the dataset with the "mark if unknown" message */
if((dset = H5Dopen(file, "/Dataset3")) < 0)
TEST_ERROR
/* Check that the "unknown" message was marked */
if(H5O_check_msg_marked_test(dset, TRUE) < 0)
FAIL_STACK_ERROR
if(NULL == (f = H5I_object(file)))
FAIL_STACK_ERROR
oh_loc.file = f;
if(H5O_open(&oh_loc) < 0)
FAIL_STACK_ERROR
/* Close the dataset */
if(H5Dclose(dset) < 0)
TEST_ERROR
/* Close the file with the bogus objects */
if(H5Fclose(file2) < 0)
TEST_ERROR
/*
* Test creation of a bunch of messages one after another to see
* what happens when the object header overflows on disk.
*/
TESTING("object header overflow on disk");
for(i = 0; i < 10; i++) {
time_new = (i + 1) * 1000 + 10;
if(H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
FAIL_STACK_ERROR
} /* end for */
PASSED();
} /* end if */
else {
SKIPPED();
puts(" Test not compatible with current Virtual File Driver");
} /* end else */
/* Close the file we created */
if(H5Fclose(file) < 0)
TEST_ERROR
/*
* Delete all time messages.
*/
TESTING("message deletion");
if(H5O_msg_remove(&oh_loc, H5O_MTIME_NEW_ID, H5O_ALL, TRUE, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(H5O_msg_remove(&oh_loc, H5O_MTIME_ID, H5O_ALL, TRUE, H5P_DATASET_XFER_DEFAULT) < 0)
FAIL_STACK_ERROR
if(H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro, H5P_DATASET_XFER_DEFAULT))
FAIL_STACK_ERROR
if(H5O_msg_read(&oh_loc, H5O_MTIME_ID, &ro, H5P_DATASET_XFER_DEFAULT))
FAIL_STACK_ERROR
PASSED();
/* release resources */
TESTING("object header closing");
if(H5O_close(&oh_loc) < 0)
FAIL_STACK_ERROR
PASSED();
/* Test reading datasets with undefined object header messages */
puts("Reading objects with unknown header messages");
envval = HDgetenv("HDF5_DRIVER");
if(envval == NULL)
envval = "nomatch";
if(HDstrcmp(envval, "core") && HDstrcmp(envval, "multi") && HDstrcmp(envval, "split") && HDstrcmp(envval, "family")) {
hid_t file2; /* File ID for 'bogus' object file */
char testpath[512] = "";
char testfile[512] = "";
char *srcdir = HDgetenv("srcdir");
/* Build path to all test files */
if(srcdir && ((HDstrlen(srcdir) + 2) < sizeof(testpath))) {
HDstrcpy(testpath, srcdir);
HDstrcat(testpath, "/");
} /* end if */
/* Build path to test file */
if(srcdir && ((HDstrlen(testpath) + HDstrlen(FILE_BOGUS) + 1) < sizeof(testfile)))
HDstrcpy(testfile, testpath);
HDstrcat(testfile, FILE_BOGUS);
TESTING("object with unknown header message and no flags set");
/* Open the file with objects that have unknown header messages (generated with gen_bogus.c) */
if((file2 = H5Fopen(testfile, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR
/* Open the dataset with the unknown header message, but no extra flags */
if((dset = H5Dopen(file2, "/Dataset1")) < 0)
TEST_ERROR
if(H5Dclose(dset) < 0)
TEST_ERROR
PASSED();
TESTING("object with unknown header message & 'fail if unknown' flag set");
/* Attempt to open the dataset with the unknown header message, and "fail if unknown" flag */
H5E_BEGIN_TRY {
dset = H5Dopen(file2, "/Dataset2");
} H5E_END_TRY;
if(dset >= 0) {
H5Dclose(dset);
TEST_ERROR
} /* end if */
PASSED();
TESTING("object with unknown header message & 'mark if unknown' flag set");
/* Copy object with "mark if unknown" flag on message into file that can be modified */
if(H5Ocopy(file2, "/Dataset3", file, "/Dataset3", H5P_DEFAULT, H5P_DEFAULT) < 0)
TEST_ERROR
/* Close the file we created (to flush changes to file) */
if(H5Fclose(file) < 0)
TEST_ERROR
/* Re-open the file created, with read-only permissions */
if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR
/* Open the dataset with the "mark if unknown" message */
if((dset = H5Dopen(file, "/Dataset3")) < 0)
TEST_ERROR
/* Check that the "unknown" message was _NOT_ marked */
if(H5O_check_msg_marked_test(dset, FALSE) < 0)
FAIL_STACK_ERROR
/* Close the dataset */
if(H5Dclose(dset) < 0)
TEST_ERROR
/* Close the file we created (to flush change to object header) */
if(H5Fclose(file) < 0)
TEST_ERROR
/* Re-open the file created */
if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0)
TEST_ERROR
/* Open the dataset with the "mark if unknown" message */
if((dset = H5Dopen(file, "/Dataset3")) < 0)
TEST_ERROR
if(H5Dclose(dset) < 0)
TEST_ERROR
/* Close the file we created (to flush change to object header) */
if(H5Fclose(file) < 0)
TEST_ERROR
/* Re-open the file created */
if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR
/* Re-open the dataset with the "mark if unknown" message */
if((dset = H5Dopen(file, "/Dataset3")) < 0)
TEST_ERROR
/* Check that the "unknown" message was marked */
if(H5O_check_msg_marked_test(dset, TRUE) < 0)
FAIL_STACK_ERROR
/* Close the dataset */
if(H5Dclose(dset) < 0)
TEST_ERROR
/* Close the file with the bogus objects */
if(H5Fclose(file2) < 0)
TEST_ERROR
PASSED();
} /* end if */
else {
SKIPPED();
puts(" Test not compatible with current Virtual File Driver");
} /* end else */
/* Close the file we created */
if(H5Fclose(file) < 0)
TEST_ERROR
} /* end for */
puts("All object header tests passed.");
h5_cleanup(FILENAME, fapl);