mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-04-12 17:31:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r22668] Description:

Browse Source 
Have free space manager use temporary address space for storing the
section info, until the file is flushed or closed.

Tested on:
    Mac OSX/64 10.7.4 (amazon) w/debug, gcc 4.7.x, C++, FORTRAN & threadsafe
    (h5committest forthcoming)
This commit is contained in:
Quincey Koziol 2012-08-11 16:27:58 -05:00
parent 3552beb08b
commit d20da589dd
5 changed files with 150 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/H5F.c
View File

@ -1051,13 +1051,15 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
* Only try to flush the file if it was opened with write access, and if
* the caller requested a flush.
*/
if((f->shared->flags & H5F_ACC_RDWR) && flush)
if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush)
if(H5F_flush(f, dxpl_id, TRUE) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache")
/* Release the external file cache */
if(f->shared->efc) {
if(H5F_efc_destroy(f->shared->efc) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't destroy external file cache")
f->shared->efc = NULL;
} /* end if */
@ -1075,6 +1077,14 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
if(H5MF_close(f, dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file free space info")
/* Flush the file again (if requested), as shutting down the
* free space manager may dirty some data structures again.
*/
if(flush)
if(H5F_flush(f, dxpl_id, TRUE) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache")
} /* end if */
/* Unpin the superblock, since we're about to destroy the cache */

129
src/H5FS.c
View File

@ -284,6 +284,50 @@ HDfprintf(stderr, "%s: Deleting free space manager, fs_addr = %a\n", FUNC, fs_ad
cache_udata.cls_init_udata = NULL;
cache_udata.addr = fs_addr;
#ifdef H5FS_DEBUG
{
unsigned fspace_status = 0; /* Free space section info's status in the metadata cache */
/* Sanity check */
HDassert(H5F_addr_defined(fs_addr));
/* Check the free space section info's status in the metadata cache */
if(H5AC_get_entry_status(f, fs_addr, &fspace_status) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "unable to check metadata cache status for free space section info")
HDfprintf(stderr, "%s: fspace_status = %0x: ", FUNC, fspace_status);
if(fspace_status) {
hbool_t printed = FALSE;
if(fspace_status & H5AC_ES__IN_CACHE) {
HDfprintf(stderr, "H5AC_ES__IN_CACHE");
printed = TRUE;
} /* end if */
if(fspace_status & H5AC_ES__IS_DIRTY) {
HDfprintf(stderr, "%sH5AC_ES__IS_DIRTY", (printed ? " | " : ""));
printed = TRUE;
} /* end if */
if(fspace_status & H5AC_ES__IS_PROTECTED) {
HDfprintf(stderr, "%sH5AC_ES__IS_PROTECTED", (printed ? " | " : ""));
printed = TRUE;
} /* end if */
if(fspace_status & H5AC_ES__IS_PINNED) {
HDfprintf(stderr, "%sH5AC_ES__IS_PINNED", (printed ? " | " : ""));
printed = TRUE;
} /* end if */
if(fspace_status & H5AC_ES__IS_FLUSH_DEP_PARENT) {
HDfprintf(stderr, "%sH5AC_ES__IS_FLUSH_DEP_PARENT", (printed ? " | " : ""));
printed = TRUE;
} /* end if */
if(fspace_status & H5AC_ES__IS_FLUSH_DEP_CHILD) {
HDfprintf(stderr, "%sH5AC_ES__IS_FLUSH_DEP_CHILD", (printed ? " | " : ""));
printed = TRUE;
} /* end if */
} /* end if */
HDfprintf(stderr, "\n");
}
#endif /* H5FS_DEBUG */
/* Protect the free space header */
if(NULL == (fspace = (H5FS_t *)H5AC_protect(f, dxpl_id, H5AC_FSPACE_HDR, fs_addr, &cache_udata, H5AC_WRITE)))
HGOTO_ERROR(H5E_FSPACE, H5E_CANTPROTECT, FAIL, "unable to protect free space header")
@ -324,9 +368,14 @@ HDfprintf(stderr, "%s: Done expunging free space section info from cache\n", FUN
#endif /* H5FS_DEBUG */
} /* end if */
else {
#ifdef H5FS_DEBUG
HDfprintf(stderr, "%s: Deleting free space section info from file\n", FUNC);
#endif /* H5FS_DEBUG */
/* Release the space in the file */
if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_addr, fspace->alloc_sect_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to release free space sections")
if(!H5F_IS_TMP_ADDR(f, fspace->sect_addr)) {
if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_addr, fspace->alloc_sect_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to release free space sections")
} /* end if */
} /* end else */
} /* end if */
@ -387,8 +436,14 @@ HDfprintf(stderr, "%s: Real sections to store in file\n", FUNC);
HDassert(fspace->sect_size > 0);
/* Allocate space for the section info in file */
if(HADDR_UNDEF == (fspace->sect_addr = H5MF_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size)))
HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for free space sections")
if(H5F_USE_TMP_SPACE(f)) {
if(HADDR_UNDEF == (fspace->sect_addr = H5MF_alloc_tmp(f, fspace->sect_size)))
HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for free space sections")
} /* end if */
else {
if(HADDR_UNDEF == (fspace->sect_addr = H5MF_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size)))
HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for free space sections")
} /* end if */
fspace->alloc_sect_size = (size_t)fspace->sect_size;
/* Mark free space header as dirty */
@ -430,19 +485,9 @@ HDfprintf(stderr, "%s: Section info allocated though\n", FUNC);
HDfprintf(stderr, "%s: Section info is for file free space\n", FUNC);
#endif /* H5FS_DEBUG */
/* Try to shrink the file or absorb the section info into a block aggregator */
if((status = H5MF_try_shrink(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_addr, fspace->alloc_sect_size)) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTMERGE, FAIL, "can't check for absorbing section info")
else if(status == FALSE) {
/* Section info can't "go away", but it's free. Allow
* header to record it
*/
if(H5F_IS_TMP_ADDR(f, fspace->sect_addr)) {
#ifdef H5FS_DEBUG
HDfprintf(stderr, "%s: Section info can't 'go away', header will own it\n", FUNC);
#endif /* H5FS_DEBUG */
} /* end if */
else {
#ifdef H5FS_DEBUG
HDfprintf(stderr, "%s: Section info went 'go away'\n", FUNC);
HDfprintf(stderr, "%s: Section info in temp. address space went 'go away'\n", FUNC);
#endif /* H5FS_DEBUG */
/* Reset section info in header */
fspace->sect_addr = HADDR_UNDEF;
@ -451,6 +496,30 @@ HDfprintf(stderr, "%s: Section info went 'go away'\n", FUNC);
/* Mark free space header as dirty */
if(H5AC_mark_entry_dirty(fspace) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
} /* end if */
else {
if((status = H5MF_try_shrink(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_addr, fspace->alloc_sect_size)) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTMERGE, FAIL, "can't check for absorbing section info")
else if(status == FALSE) {
/* Section info can't "go away", but it's free. Allow
* header to record it
*/
#ifdef H5FS_DEBUG
HDfprintf(stderr, "%s: Section info can't 'go away', header will own it\n", FUNC);
#endif /* H5FS_DEBUG */
} /* end if */
else {
#ifdef H5FS_DEBUG
HDfprintf(stderr, "%s: Section info went 'go away'\n", FUNC);
#endif /* H5FS_DEBUG */
/* Reset section info in header */
fspace->sect_addr = HADDR_UNDEF;
fspace->alloc_sect_size = 0;
/* Mark free space header as dirty */
if(H5AC_mark_entry_dirty(fspace) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
} /* end else */
} /* end else */
} /* end if */
else {
@ -469,10 +538,12 @@ HDfprintf(stderr, "%s: Section info is NOT for file free space\n", FUNC);
HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
/* Free previous serialized sections disk space */
if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, old_sect_addr, old_alloc_sect_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space sections")
} /* end if */
} /* end else */
if(!H5F_IS_TMP_ADDR(f, old_sect_addr)) {
if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, old_sect_addr, old_alloc_sect_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space sections")
} /* end if */
} /* end else */
} /* end if */
/* Destroy section info */
if(H5FS_sinfo_dest(fspace->sinfo) < 0)
@ -802,11 +873,11 @@ H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id)
HDassert(fspace);
if(!H5F_addr_defined(fspace->sect_addr) && fspace->sinfo && fspace->serial_sect_count > 0) {
/* Allocate space for section info from aggregator/vfd */
/* (The original version called H5MF_alloc(), but that may cause sect_size to change again) */
if(HADDR_UNDEF == (fspace->sect_addr = H5MF_aggr_vfd_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size)))
HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for section info")
/* Allocate space for section info from aggregator/vfd (or temp. address space) */
/* (The original version called H5MF_alloc(), but that may cause sect_size to change again) */
/* (This routine is only called during file close operations, so don't allocate from temp. address space) */
if(HADDR_UNDEF == (fspace->sect_addr = H5MF_aggr_vfd_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size)))
HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for section info")
fspace->alloc_sect_size = fspace->sect_size;
/* Mark free-space header as dirty */
@ -882,8 +953,10 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id)
fspace->alloc_sect_size = 0;
/* Free space for the free-space manager section info */
if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, saved_addr, saved_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to release free space sections")
if(!H5F_IS_TMP_ADDR(f, saved_addr)) {
if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, saved_addr, saved_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to release free space sections")
} /* end if */
/* Mark free-space manager header as dirty */
if(H5FS_dirty(fspace) < 0)
@ -1120,7 +1193,7 @@ herr_t
H5FS_assert(const H5FS_t *fspace)
{
FUNC_ENTER_NOAPI_NOINIT_NOERR
#ifndef QAK
#ifdef QAK
HDfprintf(stderr, "%s: fspace->tot_sect_count = %Hu\n", "H5FS_assert", fspace->tot_sect_count);
#endif /* QAK */

33
src/H5FScache.c
View File

@ -306,7 +306,7 @@ H5FS_cache_hdr_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5F
if(fspace->sinfo->dirty) {
if(fspace->serial_sect_count > 0) {
/* Check if we need to allocate space for section info */
if(!H5F_addr_defined(fspace->sect_addr)) {
if(H5F_IS_TMP_ADDR(f, fspace->sect_addr) || !H5F_addr_defined(fspace->sect_addr)) {
/* Sanity check */
HDassert(fspace->sect_size > 0);
@ -884,6 +884,32 @@ H5FS_cache_sinfo_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H
HDassert((size_t)(p - buf) == sinfo->fspace->sect_size);
HDassert(sinfo->fspace->sect_size <= sinfo->fspace->alloc_sect_size);
/* Check for section info at temporary address */
if(H5F_IS_TMP_ADDR(f, sinfo->fspace->sect_addr)) {
/* Sanity check */
HDassert(sinfo->fspace->sect_size > 0);
HDassert(H5F_addr_eq(sinfo->fspace->sect_addr, addr));
/* Allocate space for the section info in file */
if(HADDR_UNDEF == (addr = H5MF_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, sinfo->fspace->sect_size)))
HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for free space sections")
sinfo->fspace->alloc_sect_size = (size_t)sinfo->fspace->sect_size;
/* Sanity check */
HDassert(!H5F_addr_eq(sinfo->fspace->sect_addr, addr));
/* Let the metadata cache know the section info moved */
if(H5AC_move_entry(f, H5AC_FSPACE_SINFO, sinfo->fspace->sect_addr, addr) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTMOVE, FAIL, "unable to move indirect block")
/* Update the internal address for the section info */
sinfo->fspace->sect_addr = addr;
/* Mark free space header as dirty */
if(H5AC_mark_entry_dirty(sinfo->fspace) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
} /* end if */
/* Write buffer to disk */
if(H5F_block_write(f, H5FD_MEM_FSPACE_SINFO, sinfo->fspace->sect_addr, (size_t)sinfo->fspace->sect_size, dxpl_id, buf) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFLUSH, FAIL, "unable to save free space sections to disk")
@ -936,8 +962,9 @@ H5FS_cache_sinfo_dest(H5F_t *f, H5FS_sinfo_t *sinfo)
/* Release the space on disk */
/* (XXX: Nasty usage of internal DXPL value! -QAK) */
if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, H5AC_dxpl_id, sinfo->cache_info.addr, (hsize_t)sinfo->fspace->alloc_sect_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space section info")
if(!H5F_IS_TMP_ADDR(f, sinfo->cache_info.addr))
if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, H5AC_dxpl_id, sinfo->cache_info.addr, (hsize_t)sinfo->fspace->alloc_sect_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space section info")
} /* end if */
/* Destroy free space info */

6
src/H5FSsection.c
View File

@ -445,8 +445,10 @@ HDfprintf(stderr, "%s: Relinquishing section info ownership\n", FUNC);
HDfprintf(stderr, "%s: Freeing section info on disk, old_sect_addr = %a, old_alloc_sect_size = %Hu\n", FUNC, old_sect_addr, old_alloc_sect_size);
#endif /* H5FS_SINFO_DEBUG */
/* Release space for section info in file */
if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, old_sect_addr, old_alloc_sect_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space sections")
if(!H5F_IS_TMP_ADDR(f, old_sect_addr)) {
if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, old_sect_addr, old_alloc_sect_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space sections")
} /* end if */
} /* end if */
} /* end if */

8
test/tsohm.c
View File

@ -2463,11 +2463,11 @@ static void test_sohm_size2(int close_reopen)
* this happens because it's hard to predict exactly how much space this
* will take.
*/
if((mult_index_med.attrs2 - mult_index_med.attrs1) !=
(list_index_med.attrs2 - list_index_med.attrs1))
if((mult_index_med.attrs2 - mult_index_med.attrs1) >
(list_index_med.attrs2 - list_index_med.attrs1) * OVERHEAD_ALLOWED)
VERIFY(0, 1, "h5_get_file_size");
if((mult_index_btree.attrs2 - mult_index_btree.attrs1) !=
(btree_index.attrs2 - btree_index.attrs1))
if((mult_index_btree.attrs2 - mult_index_btree.attrs1) >
(btree_index.attrs2 - btree_index.attrs1) * OVERHEAD_ALLOWED)
VERIFY(0, 1, "h5_get_file_size");
/* The final file size for both of the multiple index files should be