mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-01-30 15:32:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Bring file shutdown code from cache image branch

Browse Source
This commit is contained in:
Quincey Koziol 2017-01-04 21:36:14 -08:00
parent 6232dd6d62
commit afd7e5b476
27 changed files with 1750 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
src/H5AC.c
View File

@ -1285,6 +1285,44 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5AC_pin_protected_entry() */
/*-------------------------------------------------------------------------
*
* Function: H5AC_prep_for_file_close
*
* Purpose: This function should be called just prior to the cache
* flushes at file close.
*
* The objective of the call is to allow the metadata cache
* to do any preparatory work prior to generation of a
* cache image.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: John Mainzer
* 7/3/15
*
*-------------------------------------------------------------------------
*/
herr_t
H5AC_prep_for_file_close(H5F_t *f, hid_t dxpl_id)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
/* Sanity checks */
HDassert(f);
HDassert(f->shared);
HDassert(f->shared->cache);
if(H5C_prep_for_file_close(f, dxpl_id) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "cache prep for file close failed")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5AC_prep_for_file_close() */
/*-------------------------------------------------------------------------
* Function: H5AC_create_flush_dependency()
@ -2775,6 +2813,54 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5AC_reset_ring() */
/*-------------------------------------------------------------------------
* Function: H5AC_unsettle_entry_ring()
*
* Purpose: Advise the metadata cache that the specified entry's metadata
* cache manager ring is no longer settled (if it was on entry).
*
* If the target metadata cache manager ring is already
* unsettled, do nothing, and return SUCCEED.
*
* If the target metadata cache manager ring is settled, and
* we are not in the process of a file shutdown, mark
* the ring as unsettled, and return SUCCEED.
*
* If the target metadata cache manager is settled, and we
* are in the process of a file shutdown, post an error
* message, and return FAIL.
*
* Note that this function simply passes the call on to
* the metadata cache proper, and returns the result.
*
* Return: Success: Non-negative
* Failure: Negative
*
* Programmer: Quincey Koziol
* September 17, 2016
*
*-------------------------------------------------------------------------
*/
herr_t
H5AC_unsettle_entry_ring(void *_entry)
{
H5AC_info_t *entry = (H5AC_info_t *)_entry; /* Entry to remove */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
/* Sanity checks */
HDassert(entry);
/* Unsettle the entry's ring */
if(H5C_unsettle_entry_ring(entry) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5AC_unsettle_entry_ring() */
/*-------------------------------------------------------------------------
* Function: H5AC_remove_entry()

40
src/H5ACdbg.c
View File

@ -249,3 +249,43 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5AC__open_trace_file() */
/*-------------------------------------------------------------------------
*
* Function: H5AC_cache_is_clean()
*
* Purpose: Debugging function that verifies that all rings in the
* metadata cache are clean from the outermost ring, inwards
* to the inner ring specified.
*
* Returns TRUE if all specified rings are clean, and FALSE
* if not. Throws an assertion failure on error.
*
* Return: TRUE if the indicated ring(s) are clean, and FALSE otherwise.
*
* Programmer: John Mainzer, 6/18/16
*
* Changes: None.
*
*-------------------------------------------------------------------------
*/
#ifndef NDEBUG
hbool_t
H5AC_cache_is_clean(const H5F_t *f, H5AC_ring_t inner_ring)
{
H5C_t *cache_ptr;
hbool_t ret_value; /* Return value */
FUNC_ENTER_NOAPI_NOINIT_NOERR
/* Sanity checks */
HDassert(f);
HDassert(f->shared);
cache_ptr = f->shared->cache;
ret_value = H5C_cache_is_clean(cache_ptr, inner_ring);
FUNC_LEAVE_NOAPI(ret_value)
} /* H5AC_cache_is_clean() */
#endif /* NDEBUG */

5
src/H5ACprivate.h
View File

@ -365,6 +365,7 @@ H5_DLL herr_t H5AC_get_entry_status(const H5F_t *f, haddr_t addr,
H5_DLL herr_t H5AC_insert_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type,
haddr_t addr, void *thing, unsigned int flags);
H5_DLL herr_t H5AC_pin_protected_entry(void *thing);
H5_DLL herr_t H5AC_prep_for_file_close(H5F_t *f, hid_t dxpl_id);
H5_DLL herr_t H5AC_create_flush_dependency(void *parent_thing, void *child_thing);
H5_DLL void * H5AC_protect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type,
haddr_t addr, void *udata, unsigned flags);
@ -404,6 +405,7 @@ H5_DLL herr_t H5AC_get_entry_ring(const H5F_t *f, haddr_t addr, H5AC_ring_t *rin
H5_DLL herr_t H5AC_set_ring(hid_t dxpl_id, H5AC_ring_t ring, H5P_genplist_t **dxpl,
H5AC_ring_t *orig_ring);
H5_DLL herr_t H5AC_reset_ring(H5P_genplist_t *dxpl, H5AC_ring_t orig_ring);
H5_DLL herr_t H5AC_unsettle_entry_ring(void *entry);
H5_DLL herr_t H5AC_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags);
H5_DLL herr_t H5AC_get_tag(const void *thing, /*OUT*/ haddr_t *tag);
@ -423,6 +425,9 @@ H5_DLL herr_t H5AC_add_candidate(H5AC_t * cache_ptr, haddr_t addr);
/* Debugging functions */
H5_DLL herr_t H5AC_stats(const H5F_t *f);
H5_DLL herr_t H5AC_dump_cache(const H5F_t *f);
#ifndef NDEBUG
H5_DLL hbool_t H5AC_cache_is_clean(const H5F_t *f, H5AC_ring_t inner_ring);
#endif /* NDEBUG */ /* end debugging functions */
#endif /* !_H5ACprivate_H */

181
src/H5C.c
View File

@ -440,6 +440,10 @@ H5C_create(size_t max_cache_size,
cache_ptr->entries_inserted_counter = 0;
cache_ptr->entries_relocated_counter = 0;
/* initialize free space manager related fields: */
cache_ptr->rdfsm_settled = FALSE;
cache_ptr->mdfsm_settled = FALSE;
if ( H5C_reset_cache_hit_rate_stats(cache_ptr) != SUCCEED ) {
/* this should be impossible... */
@ -685,6 +689,53 @@ H5C_free_tag_list_cb(void *_item, void H5_ATTR_UNUSED *key, void H5_ATTR_UNUSED
FUNC_LEAVE_NOAPI(0)
} /* H5C_free_tag_list_cb() */
/*-------------------------------------------------------------------------
*
* Function: H5C_prep_for_file_close
*
* Purpose: This function should be called just prior to the cache
* flushes at file close. There should be no protected
* entries in the cache at this point.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: John Mainzer
* 7/3/15
*
*-------------------------------------------------------------------------
*/
herr_t
H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id)
{
H5C_t * cache_ptr;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
/* Sanity checks */
HDassert(f);
HDassert(f->shared);
HDassert(f->shared->cache);
cache_ptr = f->shared->cache;
HDassert(cache_ptr);
HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
/* For now at least, it is possible to receive the
* close warning more than once -- the following
* if statement handles this.
*/
if(cache_ptr->close_warning_received)
HGOTO_DONE(SUCCEED)
cache_ptr->close_warning_received = TRUE;
/* Make certain there aren't any protected entries */
HDassert(cache_ptr->pl_len == 0);
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5C_prep_for_file_close() */
/*-------------------------------------------------------------------------
* Function: H5C_dest
@ -721,6 +772,7 @@ H5C_dest(H5F_t * f, hid_t dxpl_id)
/* Sanity check */
HDassert(cache_ptr);
HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
HDassert(cache_ptr->close_warning_received);
/* Flush and invalidate all cache entries */
if(H5C_flush_invalidate_cache(f, dxpl_id, H5C__NO_FLAGS_SET) < 0 )
@ -904,6 +956,10 @@ done:
*
* JRM -- 8/30/15
*
* Modified function to call the free space manager
* settling functions.
* JRM -- 6/9/16
*
*-------------------------------------------------------------------------
*/
herr_t
@ -983,6 +1039,55 @@ H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags)
*/
ring = H5C_RING_USER;
while(ring < H5C_RING_NTYPES) {
/* only call the free space manager settle routines when close
* warning has been received, and then only when the index is
* non-empty for that ring.
*/
if(cache_ptr->close_warning_received) {
switch(ring) {
case H5C_RING_USER:
break;
case H5C_RING_RDFSM:
if(!cache_ptr->rdfsm_settled) {
hbool_t fsm_settled = FALSE; /* Whether the FSM was actually settled */
/* Settle raw data FSM */
if(H5MF_settle_raw_data_fsm(f, dxpl_id, &fsm_settled) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "RD FSM settle failed")
/* Only set the flag if the FSM was actually settled */
if(fsm_settled)
cache_ptr->rdfsm_settled = TRUE;
} /* end if */
break;
case H5C_RING_MDFSM:
if(!cache_ptr->mdfsm_settled) {
hbool_t fsm_settled = FALSE; /* Whether the FSM was actually settled */
/* Settle metadata FSM */
if(H5MF_settle_meta_data_fsm(f, dxpl_id, &fsm_settled) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "MD FSM settle failed")
/* Only set the flag if the FSM was actually settled */
if(fsm_settled)
cache_ptr->mdfsm_settled = TRUE;
} /* end if */
break;
case H5C_RING_SBE:
break;
case H5C_RING_SB:
break;
default:
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown ring?!?!")
break;
} /* end switch */
} /* end if */
if(H5C_flush_ring(f, dxpl_id, ring, flags) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush ring failed.")
ring++;
@ -3267,6 +3372,82 @@ done:
} /* H5C_unprotect() */
/*-------------------------------------------------------------------------
*
* Function: H5C_unsettle_entry_ring
*
* Purpose: Advise the metadata cache that the specified entry's metadata
* cache manager ring is no longer settled (if it was on entry).
*
* If the target metadata cache manager ring is already
* unsettled, do nothing, and return SUCCEED.
*
* If the target metadata cache manager ring is settled, and
* we are not in the process of a file shutdown, mark
* the ring as unsettled, and return SUCCEED.
*
* If the target metadata cache manager is settled, and we
* are in the process of a file shutdown, post an error
* message, and return FAIL.
*
* Note that this function simply passes the call on to
* the metadata cache proper, and returns the result.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Quincey Koziol
* January 3, 2017
*
*-------------------------------------------------------------------------
*/
herr_t
H5C_unsettle_entry_ring(void *_entry)
{
H5C_cache_entry_t *entry = (H5C_cache_entry_t *)_entry; /* Entry whose ring to unsettle */
H5C_t *cache; /* Cache for file */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
/* Sanity checks */
HDassert(entry);
HDassert(entry->ring != H5C_RING_UNDEFINED);
HDassert((H5C_RING_USER == entry->ring) || (H5C_RING_RDFSM == entry->ring) || (H5C_RING_MDFSM == entry->ring));
cache = entry->cache_ptr;
HDassert(cache);
HDassert(cache->magic == H5C__H5C_T_MAGIC);
switch(entry->ring) {
case H5C_RING_USER:
/* Do nothing */
break;
case H5C_RING_RDFSM:
if(cache->rdfsm_settled) {
if(cache->flush_in_progress || cache->close_warning_received)
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected rdfsm ring unsettle")
cache->rdfsm_settled = FALSE;
} /* end if */
break;
case H5C_RING_MDFSM:
if(cache->mdfsm_settled) {
if(cache->flush_in_progress || cache->close_warning_received)
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected mdfsm ring unsettle")
cache->mdfsm_settled = FALSE;
} /* end if */
break;
default:
HDassert(FALSE); /* this should be un-reachable */
break;
} /* end switch */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5C_unsettle_entry_ring() */
/*-------------------------------------------------------------------------
* Function: H5C_validate_resize_config()

43
src/H5Cdbg.c
View File

@ -952,3 +952,46 @@ H5C__dump_entry(H5C_t *cache_ptr, const H5C_cache_entry_t *entry_ptr,
if(entry_ptr->flush_dep_nchildren)
H5C__dump_children(cache_ptr, entry_ptr, FALSE, "Child", indent);
} /* end H5C__dump_entry() */
/*-------------------------------------------------------------------------
*
* Function: H5C_cache_is_clean()
*
* Purpose: Debugging function that verifies that all rings in the
* metadata cache are clean from the outermost ring, inwards
* to the inner ring specified.
*
* Returns TRUE if all specified rings are clean, and FALSE
* if not. Throws an assertion failure on error.
*
* Return: TRUE if the indicated ring(s) are clean, and FALSE otherwise.
*
* Programmer: John Mainzer, 6/18/16
*
*-------------------------------------------------------------------------
*/
#ifndef NDEBUG
hbool_t
H5C_cache_is_clean(const H5C_t *cache_ptr, H5C_ring_t inner_ring)
{
H5C_ring_t ring = H5C_RING_USER;
hbool_t ret_value = TRUE; /* Return value */
FUNC_ENTER_NOAPI_NOINIT_NOERR
/* Sanity checks */
HDassert(cache_ptr);
HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
HDassert(inner_ring >= H5C_RING_USER);
HDassert(inner_ring <= H5C_RING_SB);
while(ring <= inner_ring) {
if(cache_ptr->dirty_index_ring_size[ring] > 0)
ret_value = FALSE;
ring++;
} /* end while */
FUNC_LEAVE_NOAPI(ret_value)
} /* H5C_cache_is_clean() */
#endif /* NDEBUG */

45
src/H5Cpkg.h
View File

@ -3468,6 +3468,9 @@ typedef struct H5C_tag_info_t {
* clean data so as to avoid case b) above. Again, this is
* a soft limit.
*
* close_warning_received: Boolean flag indicating that a file closing
* warning has been received.
*
*
* In addition to the call back functions required for each entry, the
* cache requires the following call back functions for this instance of
@ -4069,6 +4072,43 @@ typedef struct H5C_tag_info_t {
* entries relocated_counter: Number of entries whose base address has
* been changed since the last time this field was reset.
*
* Free Space Manager Related fields:
*
* The free space managers must be informed when we are about to close
* or flush the file so that they order themselves accordingly. This used
* to be done much later in the close process, but with cache image and
* page buffering, this is no longer viable, as we must finalize the on
* disk image of all metadata much sooner.
*
* This is handled by the H5FS_settle_raw_data_fsm() and
* H5FS_settle_meta_data_fsm() routines. As these calls are expensive,
* the following fields are used to track whether the target free space
* managers are clean.
*
* They are also used in sanity checking, as once a free space manager is
* settled, it should not become unsettled (i.e. be asked to allocate or
* free file space) either ever (in the case of a file close) or until the
* flush is complete.
*
* rdfsm_settled: Boolean flag indicating whether the raw data free space
* manager is settled -- i.e. whether the correct space has
* been allocated for it in the file.
*
* Note that the name of this field is deceptive. In the
* multi file case, the flag applies to all free space
* managers that are not involved in allocating space for
* free space manager metadata.
*
* mdfsm_settled: Boolean flag indicating whether the meta data free space
* manager is settled -- i.e. whether the correct space has
* been allocated for it in the file.
*
* Note that the name of this field is deceptive. In the
* multi file case, the flag applies only to free space
* managers that are involved in allocating space for free
* space managers.
*
*
* Statistics collection fields:
*
* When enabled, these fields are used to collect statistics as described
@ -4349,6 +4389,7 @@ struct H5C_t {
hbool_t write_permitted;
H5C_log_flush_func_t log_flush;
hbool_t evictions_enabled;
hbool_t close_warning_received;
/* Fields for maintaining [hash table] index of entries */
int32_t index_len;
@ -4454,6 +4495,10 @@ struct H5C_t {
int64_t entries_loaded_counter;
int64_t entries_inserted_counter;
int64_t entries_relocated_counter;
/* Free Space Manager Related fields */
hbool_t rdfsm_settled;
hbool_t mdfsm_settled;
#if H5C_COLLECT_CACHE_STATS
/* stats fields */
int64_t hits[H5C__MAX_NUM_TYPE_IDS + 1];

8
src/H5Cprivate.h
View File

@ -1773,6 +1773,7 @@ H5_DLL herr_t H5C_mark_entry_clean(void *thing);
H5_DLL herr_t H5C_move_entry(H5C_t *cache_ptr, const H5C_class_t *type,
haddr_t old_addr, haddr_t new_addr);
H5_DLL herr_t H5C_pin_protected_entry(void *thing);
H5_DLL herr_t H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id);
H5_DLL herr_t H5C_create_flush_dependency(void *parent_thing, void *child_thing);
H5_DLL void * H5C_protect(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type,
haddr_t addr, void *udata, unsigned flags);
@ -1797,7 +1798,8 @@ H5_DLL hbool_t H5C_get_ignore_tags(const H5C_t *cache_ptr);
H5_DLL herr_t H5C_retag_entries(H5C_t * cache_ptr, haddr_t src_tag, haddr_t dest_tag);
H5_DLL herr_t H5C_cork(H5C_t *cache_ptr, haddr_t obj_addr, unsigned action, hbool_t *corked);
H5_DLL herr_t H5C_get_entry_ring(const H5F_t *f, haddr_t addr, H5C_ring_t *ring);
H5_DLL herr_t H5C_remove_entry(void * thing);
H5_DLL herr_t H5C_unsettle_entry_ring(void *thing);
H5_DLL herr_t H5C_remove_entry(void *thing);
#ifdef H5_HAVE_PARALLEL
H5_DLL herr_t H5C_apply_candidate_list(H5F_t *f, hid_t dxpl_id,
@ -1810,5 +1812,9 @@ H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t *f, hid_t dxpl_id, int32_t ce_arra
haddr_t *ce_array_ptr);
#endif /* H5_HAVE_PARALLEL */
#ifndef NDEBUG /* debugging functions */
H5_DLL hbool_t H5C_cache_is_clean(const H5C_t *cache_ptr, H5C_ring_t inner_ring);
#endif /* NDEBUG */
#endif /* !_H5Cprivate_H */

2
src/H5FDfamily.c
View File

@ -1357,7 +1357,7 @@ H5FD_family_lock(H5FD_t *_file, hbool_t rw)
if(u < file->nmembs) {
unsigned v; /* Local index variable */
for(v = 0; v < v; v++) {
for(v = 0; v < u; v++) {
if(H5FD_unlock(file->memb[v]) < 0)
/* Push error, but keep going */
HDONE_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files")

2
src/H5FDprivate.h
View File

@ -151,7 +151,7 @@ H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H
H5_DLL herr_t H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f,
haddr_t addr, hsize_t size);
H5_DLL htri_t H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, struct H5F_t *f,
haddr_t blk_end, hsize_t extra_requested);
hid_t dxpl_id, haddr_t blk_end, hsize_t extra_requested);
H5_DLL haddr_t H5FD_get_eoa(const H5FD_t *file, H5FD_mem_t type);
H5_DLL herr_t H5FD_set_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t addr);
H5_DLL haddr_t H5FD_get_eof(const H5FD_t *file, H5FD_mem_t type);

26
src/H5FDspace.c
View File

@ -243,9 +243,9 @@ H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, H5F_t *f, hsize_t size,
if(!H5F_addr_defined(ret_value))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, HADDR_UNDEF, "real 'alloc' request failed")
/* Mark superblock dirty in cache, so change to EOA will get encoded */
if(H5F_super_dirty(f) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, HADDR_UNDEF, "unable to mark superblock as dirty")
/* Mark EOA info dirty in cache, so change will get encoded */
if(H5F_eoa_dirty(f, dxpl_id) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, HADDR_UNDEF, "unable to mark EOA info as dirty")
done:
FUNC_LEAVE_NOAPI(ret_value)
@ -350,8 +350,8 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, H5F_t *f, haddr_t addr,
hsize_t size)
H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, H5F_t *f,
haddr_t addr, hsize_t size)
{
herr_t ret_value = SUCCEED; /* Return value */
@ -367,9 +367,9 @@ H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, H5F_t *f, haddr_t addr,
if(H5FD_free_real(file, dxpl_id, type, addr, size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "real 'free' request failed")
/* Mark superblock dirty in cache, so change to EOA will get encoded */
if(H5F_super_dirty(f) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
/* Mark EOA info dirty in cache, so change will get encoded */
if(H5F_eoa_dirty(f, dxpl_id) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, FAIL, "unable to mark EOA info as dirty")
done:
FUNC_LEAVE_NOAPI(ret_value)
@ -395,8 +395,8 @@ done:
*-------------------------------------------------------------------------
*/
htri_t
H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, haddr_t blk_end,
hsize_t extra_requested)
H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, hid_t dxpl_id,
haddr_t blk_end, hsize_t extra_requested)
{
haddr_t eoa; /* End of allocated space in file */
htri_t ret_value = FALSE; /* Return value */
@ -423,9 +423,9 @@ H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, haddr_t blk_end,
if(HADDR_UNDEF == H5FD_extend(file, type, FALSE, extra_requested, NULL, NULL))
HGOTO_ERROR(H5E_VFL, H5E_CANTEXTEND, FAIL, "driver extend request failed")
/* Mark superblock dirty in cache, so change to EOA will get encoded */
if(H5F_super_dirty(f) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
/* Mark EOA info dirty in cache, so change will get encoded */
if(H5F_eoa_dirty(f, dxpl_id) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, FAIL, "unable to mark EOA info as dirty")
/* Indicate success */
HGOTO_DONE(TRUE)

163
src/H5FS.c
View File

@ -899,6 +899,169 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FS_alloc_sect() */
/*-------------------------------------------------------------------------
* Function: H5FS_alloc_vfd_alloc_hdr_and_section_info
*
* Purpose: This function is part of a hack to patch over a design
* flaw in the free space managers for file space allocation.
* Specifically, if a free space manager allocates space for
* its own section info, it is possible for it to
* go into an infinite loop as it:
*
* 1) computes the size of the section info
*
* 2) allocates file space for the section info
*
* 3) notices that the size of the section info
* has changed
*
* 4) deallocates the section info file space and
* returns to 1) above.
*
* Similarly, if it allocates space for its own header, it
* can go into an infinte loop as it:
*
* 1) allocates space for the header
*
* 2) notices that the free space manager is empty
* and thus should not have file space allocated
* to its header
*
* 3) frees the space allocated to the header
*
* 4) notices that the free space manager is not
* empty and thus must have file space allocated
* to it, and thus returns to 1) above.
*
* In a nutshell, the solution applied in this hack is to
* deallocate file space for the free space manager(s) that
* handle FSM header and/or section info file space allocations,
* wait until all other allocation/deallocation requests have
* been handled, and then test to see if the free space manager(s)
* in question are empty. If they are, do nothing. If they
* are not, allocate space for them at end of file bypassing the
* usual file space allocation calls, and thus avoiding the
* potential infinite loops.
*
* The purpose of this function is to allocate file space for
* the header and section info of the target free space manager
* directly from the VFD if needed. In this case the function
* also re-inserts the header and section info in the metadata
* cache with this allocation.
*
* Note that if f->shared->alignment > 1, and EOA is not a
* multiple of the alignment, it is possible that performing
* this allocation may generate a fragment of file space in
* addition to the space allocated for the section info.
*
* At present we deal with this by screaming and dying.
* Obviously, this is not acceptatable, but it should do
* for now.
*
*
* Return: Success: non-negative
* Failure: negative
*
* Programmer: John Mainzer
* 6/6/16
*
*-------------------------------------------------------------------------
*/
herr_t
H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id,
H5FS_t *fspace, haddr_t *fs_addr_ptr)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
/* Check arguments */
HDassert(f);
HDassert(fspace);
HDassert(fs_addr_ptr);
/* The section info should be unlocked */
HDassert(fspace->sinfo_lock_count == 0);
/* No space should be allocated */
HDassert(*fs_addr_ptr == HADDR_UNDEF);
HDassert(fspace->addr == HADDR_UNDEF);
HDassert(fspace->sect_addr == HADDR_UNDEF);
HDassert(fspace->alloc_sect_size == 0);
/* Check if any space will be needed */
if(fspace->serial_sect_count > 0) {
haddr_t sect_addr; /* Address of sinfo */
/* The section info is floating, so fspace->sinfo should be defined */
HDassert(fspace->sinfo);
/* Start by allocating file space for the header */
if(HADDR_UNDEF == (fspace->addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_HDR,
(hsize_t)H5FS_HEADER_SIZE(f), FALSE)))
HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr")
/* Cache the new free space header (pinned) */
if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space header to cache")
/* Now allocate file space for the section info */
if(HADDR_UNDEF == (sect_addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_SINFO,
fspace->sect_size, FALSE)))
HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space")
/* Update fspace->alloc_sect_size and fspace->sect_addr to reflect
* the allocation
*/
fspace->alloc_sect_size = fspace->sect_size;
fspace->sect_addr = sect_addr;
/* We have changed the sinfo address -- Mark free space header dirty */
if(H5AC_mark_entry_dirty(fspace) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
/* Insert the new section info into the metadata cache. */
/* Question: Do we need to worry about this insertion causing an
* eviction from the metadata cache? Think on this. If so, add a
* flag to H5AC_insert() to force it to skip the call to make space in
* cache.
*
* On reflection, no.
*
* On a regular file close, any eviction will not change the
* the contents of the free space manger(s), as all entries
* should have correct file space allocated by the time this
* function is called.
*
* In the cache image case, the selection of entries for inclusion
* in the cache image will not take place until after this call.
* (Recall that this call is made during the metadata fsm settle
* routine, which is called during the serialization routine in
* the cache image case. Entries are not selected for inclusion
* in the image until after the cache is serialized.)
*
* JRM -- 11/4/16
*/
if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space sinfo to cache")
/* Since space has been allocated for the section info and the sinfo
* has been inserted into the cache, relinquish owership (i.e. float)
* the section info.
*/
fspace->sinfo = NULL;
/* Set the address of the free space header, on success */
*fs_addr_ptr = fspace->addr;
} /* end if */
done:
FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
} /* H5FS_alloc_vfd_alloc_hdr_and_section_info() */
/*-------------------------------------------------------------------------
* Function: H5FS_free()

60
src/H5FScache.c
View File

@ -88,6 +88,7 @@ static herr_t H5FS__cache_hdr_pre_serialize(H5F_t *f, hid_t dxpl_id,
unsigned *flags);
static herr_t H5FS__cache_hdr_serialize(const H5F_t *f, void *image,
size_t len, void *thing);
static herr_t H5FS__cache_hdr_notify(H5AC_notify_action_t action, void *thing);
static herr_t H5FS__cache_hdr_free_icr(void *thing);
static herr_t H5FS__cache_sinfo_get_initial_load_size(void *udata, size_t *image_len);
@ -121,7 +122,7 @@ const H5AC_class_t H5AC_FSPACE_HDR[1] = {{
H5FS__cache_hdr_image_len, /* 'image_len' callback */
H5FS__cache_hdr_pre_serialize, /* 'pre_serialize' callback */
H5FS__cache_hdr_serialize, /* 'serialize' callback */
NULL, /* 'notify' callback */
H5FS__cache_hdr_notify, /* 'notify' callback */
H5FS__cache_hdr_free_icr, /* 'free_icr' callback */
NULL, /* 'fsf_size' callback */
}};
@ -766,6 +767,63 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len,
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FS__cache_hdr_serialize() */
/*-------------------------------------------------------------------------
* Function: H5FS__cache_hdr_notify
*
* Purpose: Handle cache action notifications
*
* Return: SUCCEED/FAIL
*
* Programmer: Quincey Koziol
* koziol@lbl.gov
* January 3, 2017
*
*-------------------------------------------------------------------------
*/
herr_t
H5FS__cache_hdr_notify(H5AC_notify_action_t action, void *_thing)
{
H5FS_t *fspace = (H5FS_t *)_thing; /* Pointer to the object */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
/* Sanity check */
HDassert(fspace);
/* Determine which action to take */
switch(action) {
case H5AC_NOTIFY_ACTION_AFTER_INSERT:
case H5AC_NOTIFY_ACTION_AFTER_LOAD:
case H5AC_NOTIFY_ACTION_AFTER_FLUSH:
/* do nothing */
break;
case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED:
if(H5AC_unsettle_entry_ring(fspace) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFLUSH, FAIL, "unable to mark FSM ring as unsettled")
break;
case H5AC_NOTIFY_ACTION_ENTRY_CLEANED:
case H5AC_NOTIFY_ACTION_CHILD_DIRTIED:
case H5AC_NOTIFY_ACTION_CHILD_CLEANED:
case H5AC_NOTIFY_ACTION_BEFORE_EVICT:
/* do nothing */
break;
default:
#ifdef NDEBUG
HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, FAIL, "unknown action from metadata cache")
#else /* NDEBUG */
HDassert(0 && "Unknown action?!?");
#endif /* NDEBUG */
} /* end switch */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FS__cache_hdr_notify() */
/*-------------------------------------------------------------------------
* Function: H5FS__cache_hdr_free_icr

2
src/H5FSprivate.h
View File

@ -182,6 +182,8 @@ H5_DLL herr_t H5FS_delete(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr);
H5_DLL herr_t H5FS_close(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace);
H5_DLL herr_t H5FS_alloc_hdr(H5F_t *f, H5FS_t *fspace, haddr_t *fs_addr, hid_t dxpl_id);
H5_DLL herr_t H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id);
H5_DLL herr_t H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id,
H5FS_t *fspace, haddr_t *fs_addr_ptr);
H5_DLL herr_t H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id);
/* Free space section routines */

180
src/H5Fint.c
View File

@ -76,6 +76,8 @@ typedef struct H5F_olist_t {
static int H5F_get_objects_cb(void *obj_ptr, hid_t obj_id, void *key);
static herr_t H5F_build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl,
const char *name, char ** /*out*/ actual_name);/* Declare a free list to manage the H5F_t struct */
static herr_t H5F__flush_phase1(H5F_t *f, hid_t dxpl_id);
static herr_t H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing);
/*********************/
@ -820,15 +822,40 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
int actype; /* metadata cache type (enum value) */
H5F_io_info_t fio_info; /* I/O info for operation */
/* Flush at this point since the file will be closed.
/* Flush at this point since the file will be closed (phase 1).
* Only try to flush the file if it was opened with write access, and if
* the caller requested a flush.
*/
if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush)
if(H5F_flush(f, dxpl_id, TRUE) < 0)
if(H5F__flush_phase1(f, dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache")
/* Notify the metadata cache that the file is about to be closed.
* This allows the cache to set up for creating a metadata cache
* image if this has been requested.
*/
if(H5AC_prep_for_file_close(f, dxpl_id) < 0)
/* Push error, but keep going */
HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "metadata cache prep for close failed")
/* Flush at this point since the file will be closed (phase 2).
* Only try to flush the file if it was opened with write access, and if
* the caller requested a flush.
*/
if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush)
if(H5F__flush_phase2(f, dxpl_id, TRUE) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache")
/* With the shutdown modifications, the contents of the metadata cache
* should be clean at this point, with the possible exception of the
* the superblock and superblock extension.
*
* Verify this.
*/
HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
/* Release the external file cache */
if(f->shared->efc) {
if(H5F_efc_destroy(f->shared->efc) < 0)
@ -837,36 +864,72 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
f->shared->efc = NULL;
} /* end if */
/* With the shutdown modifications, the contents of the metadata cache
* should be clean at this point, with the possible exception of the
* the superblock and superblock extension.
*
* Verify this.
*/
HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
/* Release objects that depend on the superblock being initialized */
if(f->shared->sblock) {
/* Shutdown file free space manager(s) */
/* (We should release the free space information now (before truncating
* the file and before the metadata cache is shut down) since the
* free space manager is holding some data structures in memory
* and also because releasing free space can shrink the file's
* 'eoa' value)
/* (We should release the free space information now (before
* truncating the file and before the metadata cache is shut
* down) since the free space manager is holding some data
* structures in memory and also because releasing free space
* can shrink the file's 'eoa' value)
*
* Update 11/1/16:
*
* With recent library shutdown modifications, the free space
* managers should be settled and written to file at this point
* (assuming they are persistent). In this case, closing the
* free space managers should have no effect on EOA.
*
* -- JRM
*/
if(H5F_ACC_RDWR & H5F_INTENT(f)) {
if(H5MF_close(f, dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file free space info")
/* Flush the file again (if requested), as shutting down the
* free space manager may dirty some data structures again.
/* at this point, only the superblock and superblock
* extension should be dirty.
*/
HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
if(flush) {
/* Clear status_flags */
f->shared->sblock->status_flags &= (uint8_t)(~H5F_SUPER_WRITE_ACCESS);
f->shared->sblock->status_flags &= (uint8_t)(~H5F_SUPER_SWMR_WRITE_ACCESS);
/* Mark superblock dirty in cache, so change will get encoded */
if(H5F_super_dirty(f) < 0)
/* Mark EOA info dirty in cache, so change will get encoded */
if(H5F_eoa_dirty(f, dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
if(H5F_flush(f, dxpl_id, TRUE) < 0)
/* Release any space allocated to space aggregators,
* so that the eoa value corresponds to the end of the
* space written to in the file.
*
* At most, this should change the superblock or the
* superblock extension messages.
*/
if(H5MF_free_aggrs(f, dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache")
HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space")
/* Truncate the file to the current allocated size */
if(H5FD_truncate(f->shared->lf, dxpl_id, TRUE) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed")
/* at this point, only the superblock and superblock
* extension should be dirty.
*/
HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
} /* end if */
} /* end if */
@ -884,6 +947,13 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
HDONE_ERROR(H5E_FSPACE, H5E_CANTUNPIN, FAIL, "unable to unpin superblock")
f->shared->sblock = NULL;
} /* end if */
/* with the possible exception of the superblock and superblock
* extension, the metadata cache should be clean at this point.
*
* Verify this.
*/
HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
/* Remove shared file struct from list of open files */
if(H5F_sfile_remove(f->shared) < 0)
@ -1358,25 +1428,24 @@ done:
/*-------------------------------------------------------------------------
* Function: H5F_flush
* Function: H5F_flush_phase1
*
* Purpose: Flushes cached data.
* Purpose: First phase of flushing cached data.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Robb Matzke
* matzke@llnl.gov
* Aug 29 1997
* Programmer: Quincey Koziol
* koziol@lbl.gov
* Jan 1 2017
*
*-------------------------------------------------------------------------
*/
herr_t
H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
static herr_t
H5F__flush_phase1(H5F_t *f, hid_t dxpl_id)
{
H5F_io_info_t fio_info; /* I/O info for operation */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
FUNC_ENTER_STATIC
/* Sanity check arguments */
HDassert(f);
@ -1396,6 +1465,34 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space")
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F__flush_phase1() */
/*-------------------------------------------------------------------------
* Function: H5F__flush_phase2
*
* Purpose: Second phase of flushing cached data.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Quincey Koziol
* koziol@lbl.gov
* Jan 1 2017
*
*-------------------------------------------------------------------------
*/
static herr_t
H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing)
{
H5F_io_info_t fio_info; /* I/O info for operation */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Sanity check arguments */
HDassert(f);
/* Flush the entire metadata cache */
if(H5AC_flush(f, dxpl_id) < 0)
/* Push error, but keep going*/
@ -1403,6 +1500,7 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
/* Truncate the file to the current allocated size */
if(H5FD_truncate(f->shared->lf, dxpl_id, closing) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed")
/* Flush the entire metadata cache again since the EOA could have changed in the truncate call. */
@ -1413,6 +1511,7 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
/* Set up I/O info for operation */
fio_info.f = f;
if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
/* Push error, but keep going*/
HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
/* Flush out the metadata accumulator */
@ -1425,6 +1524,43 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
/* Push error, but keep going*/
HDONE_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "low level flush failed")
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F__flush_phase2() */
/*-------------------------------------------------------------------------
* Function: H5F_flush
*
* Purpose: Flushes cached data.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Robb Matzke
* matzke@llnl.gov
* Aug 29 1997
*
*-------------------------------------------------------------------------
*/
herr_t
H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
/* Sanity check arguments */
HDassert(f);
/* First phase of flushing data */
if(H5F__flush_phase1(f, dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data")
/* Second phase of flushing data */
if(H5F__flush_phase2(f, dxpl_id, closing) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F_flush() */

8
src/H5Fpkg.h
View File

@ -259,6 +259,14 @@ struct H5F_file_t {
* block is present. At all other times
* it should be NULL.
*/
hbool_t drvinfo_sb_msg_exists; /* Convenience field used to track
* whether the driver info superblock
* extension message has been created
* yet. This field should be TRUE iff the
* superblock extension exists and contains
* a driver info message. Under all other
* circumstances, it must be set to FALSE.
*/
unsigned nrefs; /* Ref count for times file is opened */
unsigned flags; /* Access Permissions for file */
H5F_mtab_t mtab; /* File mount table */

1
src/H5Fprivate.h
View File

@ -756,6 +756,7 @@ H5_DLL herr_t H5F_fake_free(H5F_t *f);
/* Superblock related routines */
H5_DLL herr_t H5F_super_dirty(H5F_t *f);
H5_DLL herr_t H5F_eoa_dirty(H5F_t *f, hid_t dxpl_id);
/* Parallel I/O (i.e. MPI) related routines */
#ifdef H5_HAVE_PARALLEL

155
src/H5Fsuper.c
View File

@ -53,6 +53,7 @@
/* Local Prototypes */
/********************/
static herr_t H5F_super_ext_create(H5F_t *f, hid_t dxpl_id, H5O_loc_t *ext_ptr);
static herr_t H5F__update_super_ext_driver_msg(H5F_t *f, hid_t dxpl_id);
/*********************/
@ -220,6 +221,86 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5F_super_ext_close() */
/*-------------------------------------------------------------------------
* Function: H5F__update_super_ext_driver_msg
*
* Purpose: Update the superblock extension file driver info message if
* we are using a V 2 superblock. Observe that the function
* is a NO-OP if the file driver info message does not exist.
* This is necessary, as the function is called whenever the
* EOA is updated, and were it to create the file driver info
* message, it would find itself in an infinite recursion.
*
* Return: Success: SUCCEED
* Failure: FAIL
*
* Programmer: John Mainzer
* 11/10/15
*
*-------------------------------------------------------------------------
*/
herr_t
H5F__update_super_ext_driver_msg(H5F_t *f, hid_t dxpl_id)
{
H5F_super_t *sblock; /* Pointer to the super block */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Sanity check */
HDassert(f);
HDassert(f->shared);
sblock = f->shared->sblock;
HDassert(sblock);
HDassert(sblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
HDassert(sblock->cache_info.type == H5AC_SUPERBLOCK);
/* Update the driver information message in the superblock extension
* if appropriate.
*/
if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2) {
if(H5F_addr_defined(sblock->ext_addr)) {
/* Check for ignoring the driver info for this file */
if(!H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO)) {
size_t driver_size; /* Size of driver info block (bytes)*/
/* Check for driver info */
H5_CHECKED_ASSIGN(driver_size, size_t, H5FD_sb_size(f->shared->lf), hsize_t);
/* Nothing to do unless there is both driver info and
* the driver info superblock extension message has
* already been created.
*/
if(driver_size > 0) {
H5O_drvinfo_t drvinfo; /* Driver info */
uint8_t dbuf[H5F_MAX_DRVINFOBLOCK_SIZE]; /* Driver info block encoding buffer */
/* Sanity check */
HDassert(driver_size <= H5F_MAX_DRVINFOBLOCK_SIZE);
/* Encode driver-specific data */
if(H5FD_sb_encode(f->shared->lf, drvinfo.name, dbuf) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to encode driver information")
/* Write the message to the superblock extension.
*
* Note that the superblock extension and the
* file driver info message must already exist.
*/
drvinfo.len = driver_size;
drvinfo.buf = dbuf;
if(H5F_super_ext_write_msg(f, dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message")
} /* end if driver_size > 0 */
} /* end if !H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO) */
} /* end if superblock extension exists */
} /* end if sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F__update_super_ext_driver_msg() */
/*-------------------------------------------------------------------------
* Function: H5F__super_read
@ -549,6 +630,9 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
/* Reset driver info message */
H5O_msg_reset(H5O_DRVINFO_ID, &drvinfo);
HDassert(FALSE == f->shared->drvinfo_sb_msg_exists);
f->shared->drvinfo_sb_msg_exists = TRUE;
} /* end else */
} /* end if */
@ -844,8 +928,18 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
/* Compute the size of the driver information block */
H5_CHECKED_ASSIGN(driver_size, size_t, H5FD_sb_size(f->shared->lf), hsize_t);
/* The following code sets driver_size to the valued needed
* for the driver info block, and sets the driver info block
* address regardless of the version of the superblock.
*/
if(driver_size > 0) {
driver_size += H5F_DRVINFOBLOCK_HDR_SIZE;
/* Add in the driver info header, for older superblocks */
/* Superblock versions >= 2 will put the driver info in a message
* and don't need the header -QAK, 1/4/2017
*/
if(super_vers < HDF5_SUPERBLOCK_VERSION_2)
driver_size += H5F_DRVINFOBLOCK_HDR_SIZE;
/*
* The file driver information block begins immediately after the
@ -976,6 +1070,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
info.buf = dbuf;
if(H5O_msg_create(&ext_loc, H5O_DRVINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &info, dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update driver info header message")
HDassert(FALSE == f->shared->drvinfo_sb_msg_exists);
f->shared->drvinfo_sb_msg_exists = TRUE;
} /* end if */
/* Check for non-default free space settings */
@ -1075,6 +1172,54 @@ done:
FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
} /* end H5F__super_init() */
/*-------------------------------------------------------------------------
* Function: H5F_eoa_dirty
*
* Purpose: Mark the file's EOA info dirty
*
* Return: Success: non-negative on success
* Failure: Negative
*
* Programmer: Quincey Koziol
* January 4, 2017
*
*-------------------------------------------------------------------------
*/
herr_t
H5F_eoa_dirty(H5F_t *f, hid_t dxpl_id)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
/* Sanity check */
HDassert(f);
HDassert(f->shared);
HDassert(f->shared->sblock);
/* Mark superblock dirty in cache, so change to EOA will get encoded */
if(H5F_super_dirty(f) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
/* If the driver information block exists, mark it dirty as well
* so that the change in eoa will be reflected there as well if
* appropriate.
*/
if(f->shared->drvinfo) {
if(H5AC_mark_entry_dirty(f->shared->drvinfo) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark drvinfo as dirty")
} /* end if */
/* If the driver info is stored as a message, update that instead */
else if(f->shared->drvinfo_sb_msg_exists) {
if(H5F__update_super_ext_driver_msg(f, dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark drvinfo message as dirty")
} /* end if */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5F_eoa_dirty() */
/*-------------------------------------------------------------------------
* Function: H5F_super_dirty
@ -1105,14 +1250,6 @@ H5F_super_dirty(H5F_t *f)
if(H5AC_mark_entry_dirty(f->shared->sblock) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
/* if the driver information block exists, mark it dirty as well
* so that the change in eoa will be reflected there as well if
* appropriate.
*/
if ( f->shared->drvinfo )
if(H5AC_mark_entry_dirty(f->shared->drvinfo) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark drvinfo as dirty")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5F_super_dirty() */

119
src/H5Fsuper_cache.c
View File

@ -74,9 +74,6 @@ static htri_t H5F__cache_superblock_verify_chksum(const void *image_ptr, size_t
static void *H5F__cache_superblock_deserialize(const void *image, size_t len,
void *udata, hbool_t *dirty);
static herr_t H5F__cache_superblock_image_len(const void *thing, size_t *image_len);
static herr_t H5F__cache_superblock_pre_serialize(H5F_t *f,
hid_t dxpl_id, void *thing, haddr_t addr, size_t len,
haddr_t *new_addr, size_t *new_len, unsigned *flags);
static herr_t H5F__cache_superblock_serialize(const H5F_t *f, void *image, size_t len,
void *thing);
static herr_t H5F__cache_superblock_free_icr(void *thing);
@ -115,7 +112,7 @@ const H5AC_class_t H5AC_SUPERBLOCK[1] = {{
H5F__cache_superblock_verify_chksum, /* 'verify_chksum' callback */
H5F__cache_superblock_deserialize, /* 'deserialize' callback */
H5F__cache_superblock_image_len, /* 'image_len' callback */
H5F__cache_superblock_pre_serialize,/* 'pre_serialize' callback */
NULL, /* 'pre_serialize' callback */
H5F__cache_superblock_serialize, /* 'serialize' callback */
NULL, /* 'notify' callback */
H5F__cache_superblock_free_icr, /* 'free_icr' callback */
@ -651,120 +648,6 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5F__cache_superblock_image_len() */
/*-------------------------------------------------------------------------
* Function: H5FS__cache_superblock_pre_serialize
*
* Purpose: The current use of this function is a cludge to repair an
* oversight in the conversion of the superblock code to use the
* version 3 cache.
*
* In the V2 metadata cache callbacks, the superblock dirver info
* message was updated in the flush routine. Note that this
* operation only applies to version 2 or later superblocks.
*
* Somehow, this functionality was lost in the conversion to use
* the V3 cache, causing failures with the multi file driver
* (and possibly the family file driver as well).
*
* Performing this operation is impossible in the current
* serialize routine, as the dxpl_id is not available. While
* I am pretty sure that this is not the correct place for this
* functionality, as I can see it causing problems with both
* journaling and possibly parallel HDF5 as well, I am placing
* code for the necessary update in the pre_serialize call for
* now for testing purposes. We will almost certainly want to
* change this.
*
* Return: Success: SUCCEED
* Failure: FAIL
*
* Programmer: John Mainzer
* 10/82/14
*
*-------------------------------------------------------------------------
*/
static herr_t
H5F__cache_superblock_pre_serialize(H5F_t *f, hid_t dxpl_id,
void *_thing, haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED len,
haddr_t H5_ATTR_UNUSED *new_addr, size_t H5_ATTR_UNUSED *new_len,
unsigned H5_ATTR_UNUSED *flags)
{
H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
H5F_super_t *sblock = (H5F_super_t *)_thing; /* Pointer to the super block */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
/* Sanity check */
HDassert(f);
HDassert(sblock);
HDassert(sblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
HDassert(sblock->cache_info.type == H5AC_SUPERBLOCK);
HDassert(flags);
if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2) {
/* WARNING: This code almost certainly doesn't belong here. Must
* discuss with Quincey where to put it. Note issues
* for journaling and possibly parallel.
*
* -- JRM
*/
/* Update the driver information message in the superblock extension
* if appropriate.
*/
if(H5F_addr_defined(sblock->ext_addr)) {
size_t driver_size; /* Size of driver info block (bytes)*/
H5O_loc_t ext_loc; /* "Object location" for superblock extension */
HDassert(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2);
/* Open the superblock extension's object header */
if(H5F_super_ext_open((H5F_t *)f, sblock->ext_addr, &ext_loc) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension")
/* Check for ignoring the driver info for this file */
if(!H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO)) {
/* Check for driver info message */
H5_CHECKED_ASSIGN(driver_size, size_t, H5FD_sb_size(f->shared->lf), hsize_t);
if(driver_size > 0) {
H5O_drvinfo_t drvinfo; /* Driver info */
uint8_t dbuf[H5F_MAX_DRVINFOBLOCK_SIZE]; /* Driver info block encoding buffer */
/* Sanity check */
HDassert(driver_size <= H5F_MAX_DRVINFOBLOCK_SIZE);
/* Encode driver-specific data */
if(H5FD_sb_encode(f->shared->lf, drvinfo.name, dbuf) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to encode driver information")
/* Set the ring type in the DXPL */
if(H5AC_set_ring(dxpl_id, H5AC_RING_SBE, &dxpl, &orig_ring) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set ring value")
/* Write driver info information to the superblock extension */
drvinfo.len = driver_size;
drvinfo.buf = dbuf;
if(H5O_msg_write(&ext_loc, H5O_DRVINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &drvinfo, dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message")
} /* end if */
} /* end if */
/* Close the superblock extension object header */
if(H5F_super_ext_close((H5F_t *)f, &ext_loc, dxpl_id, FALSE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension")
} /* end if */
} /* end if */
done:
/* Reset the ring in the DXPL */
if(H5AC_reset_ring(dxpl, orig_ring) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set property value")
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FS_cache_superblock_pre_serialize() */
/*-------------------------------------------------------------------------
* Function: H5F__cache_superblock_serialize

724
src/H5MF.c
View File

@ -863,14 +863,14 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r
reset_ring = TRUE;
/* Check if the block is exactly at the end of the file */
if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, end, extra_requested)) < 0)
if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, dxpl_id, end, extra_requested)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file")
else if(ret_value == FALSE) {
H5F_blk_aggr_t *aggr; /* Aggregator to use */
/* Check for test block able to extend aggregation block */
aggr = (map_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr);
if((ret_value = H5MF_aggr_try_extend(f, aggr, map_type, end, extra_requested)) < 0)
if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block")
else if(ret_value == FALSE) {
@ -1244,6 +1244,632 @@ done:
FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
} /* end H5MF__close_shrink_eoa() */
/*-------------------------------------------------------------------------
* Function: H5MF_settle_raw_data_fsm()
*
* Purpose: Handle any tasks required before the metadata cache
* can serialize or flush the raw data free space manager
* and any metadata free space managers that reside in the
* raw data free space manager ring.
*
* Specifically, any metadata managers that DON'T handle
* space allocation for free space manager header or section
* info will reside in the raw data free space manager ring.
* As of this writing, the plan is to move to only two free space
* managers, one for raw data and one for metadata -- which
* means that only the raw data free space manager will reside
* in the free space manager ring. However, this has not been
* fully implemented yet, so this code must support the
* possibilty of multiple metadata free space managers, at most
* two of which handle free space manager header or section info,
* and thus reside in the metadata free space manager ring.
*
* At present, the task list is:
*
* 1) Reduce the EOA to the extent possible. To do this:
*
* a) Free both aggregators. Space not at EOA will be
* added to the appropriate free space manager.
*
* The raw data aggregator should not be restarted
* after this point. It is possible that the metadata
* aggregator will be.
*
* b) Free all file space currently allocated to free
* space managers.
*
* c) Delete the free space manager superblock
* extension message if allocated.
*
* This done, reduce the EOA by moving it to just before
* the last piece of free memory in the file.
*
* 2) Ensure that space is allocated for the free space
* manager superblock extension message. Must do this
* now, before reallocating file space for free space
* managers, as it is possible that this allocation may
* grab the last section in a FSM -- making it unnecessary
* to re-allocate file space for it.
*
* 3) Scan all free space managers not involved in allocating
* space for free space managers. For each such free space
* manager, test to see if it contains free space. If
* it does, allocate file space for its header and section
* data. If it contains no free space, leave it without
* allocated file space as there is no need to save it to
* file.
*
* Note that all free space managers in this class should
* see no further space allocations / deallocations as
* at this point, all raw data allocations should be
* finalized, as should all metadata allocations not
* involving free space managers.
*
* We will allocate space for free space managers involved
* in the allocation of file space for free space managers
* in H5MF_settle_meta_data_fsm()
* Return: SUCCEED/FAIL
*
* Programmer: John Mainzer
* 5/25/16
*
*-------------------------------------------------------------------------
*/
herr_t
H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
{
H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
hbool_t reset_ring = FALSE; /* Whether the ring was set */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
/* check args */
HDassert(f);
HDassert(f->shared);
HDassert(fsm_settled);
/* Only need to settle things if we are persisting the free space info */
if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
H5O_fsinfo_t fsinfo; /* Free space manager info message */
H5FD_mem_t type; /* Memory type for iteration */
H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */
H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */
hbool_t fsm_opened[H5FD_MEM_NTYPES]; /* State of FSM */
hbool_t fsm_visited[H5FD_MEM_NTYPES]; /* State of FSM */
/* Initialize fsm_opened and fsm_visited */
HDmemset(fsm_opened, 0, sizeof(fsm_opened));
HDmemset(fsm_visited, 0, sizeof(fsm_visited));
/* 1) Reduce the EOA to the extent possible. */
/* a) Free the space in aggregators:
*
* (for space not at EOF, it may be put into free space managers)
*
* Do this now so that the raw data FSM (and any other FSM that isn't
* involved in space allocation for FSMs) will have no further activity.
*
* Note that while the raw data aggregator should not be restarted during
* the close process, this need not be the case for the metadata aggregator.
*/
if(H5MF_free_aggrs(f, dxpl_id) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "can't free aggregators")
/* Set the ring type in the DXPL. In most cases, we will
* need H5AC_RING_MDFSM first, so initialy set the ring in
* the DXPL to that value. We will alter this later if
* needed.
*/
if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value(0)")
reset_ring = TRUE;
curr_ring = H5AC_RING_MDFSM;
/* b) Free the file space (if any) allocated to each free space manager.
*
* Do this to facilitate reduction of the size of the file to the
* extent possible. We will re-allocate space to free space managers
* that have free space to save after this reduction.
*
* In the case of the raw data free space manager, and any other free
* space manager that does not allocate space for free space managers,
* allocations should be complete at this point, as all raw data should
* have space allocated and be flushed to file at this point. Thus we
* can examine such free space managers and only re-allocate space for
* them if they contain free space. Do this later in this function after
* the EOA has been reduced to the extent possible.
*
* For free space managers that allocate file space for free space
* managers (usually just a single metadata free space manager, but for
* now at least, free space managers for different types of metadata
* are possible), the matter is more ticklish due to the self-
* referential nature of the problem. These FSMs are dealt with in
* H5MF_settle_meta_data_fsm().
*/
for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
H5FS_stat_t fs_stat; /* Information for free-space manager */
H5FD_mem_t fsm_type; /* File memory type for FSM */
/* There is potentially a many-to-one mapping from memory types to
* free space managers. Use the fsm_visited[] array to avoid visiting
* a given FSM more than once. Use fsm_opened[] to track which FSMs
* must be closed at the end of this function.
*/
fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type);
if(!fsm_visited[fsm_type]) {
fsm_visited[fsm_type] = TRUE;
/* If there is no active FSM for this type, but such a FSM has
* space allocated in file, open it so that we can free its file
* space.
*/
if(NULL == f->shared->fs_man[fsm_type]) {
if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) {
/* Sanity check */
HDassert(fsm_opened[fsm_type] == FALSE);
/* Start up FSM for the file memory type */
if(H5MF__alloc_open(f, dxpl_id, fsm_type) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
fsm_opened[fsm_type] = TRUE;
} /* end if */
} /* end if */
/* Check for an actual FSM for this type now */
/* (Possibly opened in previous step) */
if(f->shared->fs_man[fsm_type]) {
/* Test to see if we need to switch rings -- do so if required */
if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
|| (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
needed_ring = H5AC_RING_MDFSM;
else
needed_ring = H5AC_RING_RDFSM;
if(needed_ring != curr_ring) {
if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value.")
curr_ring = needed_ring;
} /* end if */
/* Query free space manager info for this type */
if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
/* Check if the free space manager has space in the file */
if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) {
/* Delete the free space manager in the file. Will
* reallocate later if the free space manager contains
* any free space.
*/
if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
f->shared->fs_addr[fsm_type] = HADDR_UNDEF;
} /* end if */
} /* end if */
/* note that we are tracking opened FSM -- we will close them
* at the end of the function.
*/
} /* end if */
} /* end for */
/* c) Delete the free space manager superblock extension message
* if allocated.
*
* Must do this since the routine that writes / creates superblock
* extension messages will choke if the target message is
* unexpectedly either absent or present.
*/
if(H5F_addr_defined(f->shared->sblock->ext_addr))
if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_FSINFO_ID) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension")
/* As the final element in 1), shrink the EOA for the file */
if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
/* 2) Ensure that space is allocated for the free space manager superblock
* extension message. Must do this now, before reallocating file space
* for free space managers, as it is possible that this allocation may
* grab the last section in a FSM -- making it unnecessary to
* re-allocate file space for it.
*
* Do this by writing a free space manager superblock extension message.
*
* Since no free space manager has file space allocated for it, this
* message must be invalid since we can't save addresses of FSMs when
* those addresses are unknown. This is OK -- we will write the correct
* values to the message at free space manager shutdown.
*/
for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
fsinfo.fs_addr[type-1] = HADDR_UNDEF;
fsinfo.strategy = f->shared->fs_strategy;
fsinfo.threshold = f->shared->fs_threshold;
if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
/* 3) Scan all free space managers not involved in allocating
* space for free space managers. For each such free space
* manager, test to see if it contains free space. If
* it does, allocate file space for its header and section
* data. If it contains no free space, leave it without
* allocated file space as there is no need to save it to
* file.
*
* Note that all free space managers in this class should
* see no further space allocations / deallocations as
* at this point, all raw data allocations should be
* finalized, as should all metadata allocations not involving
* free space managers.
*
* We will allocate space for free space managers involved
* in the allocation of file space for free space managers
* in H5MF_settle_meta_data_fsm()
*/
/* Reinitialize fsm_visited */
HDmemset(fsm_visited, 0, sizeof(fsm_visited));
for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
H5FD_mem_t fsm_type; /* File memory type for FSM */
fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type);
/* test to see if we need to switch rings -- do so if required */
if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
|| (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
needed_ring = H5AC_RING_MDFSM;
else
needed_ring = H5AC_RING_RDFSM;
if(needed_ring != curr_ring) {
if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value.")
curr_ring = needed_ring;
} /* end if */
/* Since there can be a many-to-one mapping from memory types
* to free space managers, ensure that we don't visit any FSM
* more than once.
*/
if(!fsm_visited[fsm_type]) {
fsm_visited[fsm_type] = TRUE;
if(f->shared->fs_man[fsm_type]) {
/* Only allocate file space if the target free space manager
* doesn't allocate file space for free space managers. Note
* that this is also the deciding factor as to whether a FSM
* in in the raw data FSM ring.
*/
if((fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
&& (fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) {
H5FS_stat_t fs_stat; /* Information for free-space manager */
/* The current ring should be H5AC_RING_RDFSM */
HDassert(curr_ring == H5AC_RING_RDFSM);
/* Query free space manager info for this type */
if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 )
HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
/* If the free space manager contains section info,
* allocate space for the header and sinfo (note that
* space must not be allocated at present -- verify
* verify this with assertions).
*/
if(fs_stat.serial_sect_count > 0) {
/* Sanity check */
HDassert(!H5F_addr_defined(fs_stat.addr));
/* Allocate FSM header */
if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocated free-space header")
/* Allocate FSM section info */
HDassert(!H5F_addr_defined(fs_stat.sect_addr));
HDassert(fs_stat.alloc_sect_size == 0);
if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 )
HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info")
#ifndef NDEBUG
/* Re-Query free space manager info for this type */
if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
HDassert(H5F_addr_defined(fs_stat.addr));
HDassert(H5F_addr_defined(fs_stat.sect_addr));
HDassert(fs_stat.serial_sect_count > 0);
HDassert(fs_stat.alloc_sect_size > 0);
HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size);
#endif /* NDEBUG */
} /* end if */
else {
HDassert(!H5F_addr_defined(fs_stat.addr));
HDassert(!H5F_addr_defined(fs_stat.sect_addr));
HDassert(fs_stat.serial_sect_count == 0);
HDassert(fs_stat.alloc_sect_size == 0);
} /* end else */
} /* end if */
} /* end if */
/* Close any opened FSMs */
if(fsm_opened[fsm_type]) {
if(H5MF__alloc_close(f, dxpl_id, fsm_type) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't close file free space manager")
fsm_opened[fsm_type] = FALSE;
} /* end if */
} /* end if */
} /* end for */
/* verify that all opened FSMs were closed */
for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
HDassert(!fsm_opened[type]);
/* Indicate that the FSM was settled successfully */
*fsm_settled = TRUE;
} /* end if */
done:
/* Reset the ring in the DXPL */
if(reset_ring)
if(H5AC_reset_ring(dxpl, orig_ring) < 0)
HDONE_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set property value")
FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
} /* H5MF_settle_raw_data_fsm() */
/*-------------------------------------------------------------------------
* Function: H5MF_settle_meta_data_fsm()
*
* Purpose: If the free space manager is persistent, handle any tasks
* required before the metadata cache can serialize or flush
* the metadata free space manager(sI) that handle file space
* allocation for free space managers.
*
* In most cases, there will be only one manager assigned
* to this role. However, since for reason or reason unknown,
* free space manager headers and section info blocks are
* different classes of memory, it is possible that two free
* space managers will be involved.
*
* On entry to this function, the raw data settle routine
* (H5MF_settle_raw_data_fsm()) should have:
*
* 1) Freed the accumulators.
*
* 2) Freed all file space allocated to the free space managers.
*
* 3) Deleted the free space manager superblock extension message
*
* 4) Reduced the EOA to the extent possible.
*
* 5) Re-created the free space manager superblock extension
* message.
*
* 6) Reallocated file space for all non-empty free space
* managers NOT involved in allocation of space for free
* space managers.
*
* Note that these free space managers (if not empty) should
* have been written to file by this point, and that no
* further space allocations involving them should take
* place during file close.
*
* On entry to this routine. the free space manager(s) involved
* in allocation of file space for free space managers should
* still be floating. (i.e. should not have any file space
* allocated to them.)
*
* Similarly, the raw data aggregator should not have been
* restarted. Note that it is probable that reallocation of
* space in 5) and 6) above will have re-started the metadata
* aggregator.
*
*
* In this routine, we proceed as follows:
*
* 1) Verify that the free space manager(s) involved in file
* space allocation for free space managers are still floating.
*
* 2) Free the accumulators.
*
* 3) Reduce the EOA to the extent possible.
*
* 4) Re-allocate space for any free space manager(s) that:
*
* a) are involved in allocation of space for free space
* managers, and
*
* b) contain free space.
*
* It is possible that we could allocate space for one
* of these free space manager(s) only to have the allocation
* result in the free space manager being empty and thus
* obliging us to free the space again. Thus there is the
* potential for an infinte loop if we want to avoid saving
* empty free space managers.
*
* Similarly, it is possible that we could allocate space
* for a section info block, only to discover that this
* allocation has changed the size of the section info --
* forcing us to deallocate and start the loop over again.
*
* To avoid this, simply allocate file space for these
* FSM(s) directly from the VFD layer if allocation is
* indicated. This avoids the issue by bypassing the FSMs
* in this case.
*
* Note that this may increase the size of the file needlessly.
* A better solution would be to modify the FSM code to
* save empty FSMs to file, and to allow section info blocks
* to be oversized. However, given that the FSM code is
* also used by the fractal heaps, and that we are under
* severe time pressure at the moment, the above brute
* force solution is attractive.
*
* Return: SUCCEED/FAIL
*
* Programmer: John Mainzer
* 5/25/16
*
*-------------------------------------------------------------------------
*/
herr_t
H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
{
H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
hbool_t reset_ring = FALSE; /* Whether we set the ring */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
/* Check args */
HDassert(f);
HDassert(f->shared);
HDassert(fsm_settled);
/* Only need to settle things if we are persisting the free space info */
if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
H5FS_t *hdr_fspace; /* Ptr to FSM hdr alloc FSM */
H5FS_t *sinfo_fspace; /* Ptr to FSM sinfo alloc FSM */
H5FS_stat_t fs_stat; /* Information for FSM */
H5FD_mem_t hdr_fsm_alloc_type; /* FSM hdr alloc type */
H5FD_mem_t sinfo_fsm_alloc_type; /* FSM info alloc type */
/* Get the file memory types for the FSM header & section info */
hdr_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR);
sinfo_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO);
/* Sanity checks */
HDassert(hdr_fsm_alloc_type > H5FD_MEM_NOLIST);
HDassert(hdr_fsm_alloc_type < H5FD_MEM_NTYPES);
HDassert(sinfo_fsm_alloc_type > H5FD_MEM_NOLIST);
HDassert(sinfo_fsm_alloc_type < H5FD_MEM_NTYPES);
HDassert(!H5F_addr_defined(f->shared->fs_addr[hdr_fsm_alloc_type]));
HDassert(!H5F_addr_defined(f->shared->fs_addr[sinfo_fsm_alloc_type]));
/* Note that in most cases, hdr_fspace will equal sinfo_fspace. */
hdr_fspace = f->shared->fs_man[hdr_fsm_alloc_type];
sinfo_fspace = f->shared->fs_man[sinfo_fsm_alloc_type];
/* Set the ring in the dxpl appropriately for subsequent calls */
if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value")
reset_ring = TRUE;
#ifndef NDEBUG
/* Verify that hdr_fspace is floating if it exists */
if(hdr_fspace) {
/* Query free space manager info for this type */
if(H5FS_stat_info(f, hdr_fspace, &fs_stat) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "can't get free-space info")
HDassert(!H5F_addr_defined(fs_stat.addr));
HDassert(!H5F_addr_defined(fs_stat.sect_addr));
HDassert(fs_stat.alloc_sect_size == 0);
} /* end if */
/* Verify that sinfo_fspace is floating if it exists */
if((sinfo_fspace) && (hdr_fspace != sinfo_fspace)) {
/* Query free space manager info for this type */
if(H5FS_stat_info(f, sinfo_fspace, &fs_stat) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "can't get free-space info")
HDassert(!H5F_addr_defined(fs_stat.addr));
HDassert(!H5F_addr_defined(fs_stat.sect_addr));
HDassert(fs_stat.alloc_sect_size == 0);
} /* end if */
#endif /* NDEBUG */
/* Free the space in the metadata aggregator. Do this via the
* H5MF_free_aggrs() call. Note that the raw data aggregator must
* have already been freed. Sanity checks for this?
*/
/* (for space not at EOF, it may be put into free space managers) */
if(H5MF_free_aggrs(f, dxpl_id) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "can't free aggregators")
/* Trying shrinking the EOA for the file */
if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
/* ******************* PROBLEM: ********************
*
* If the file has an alignement other than 1, and if
* the EOA is not a multiple of this alignment, allocating sapce
* for the section via the VFD info has the potential of generating
* a fragment that will be added to the free space manager. This
* of course undoes everything we have been doing here.
*
* Need a way around this. Obvious solution is to force the EOA to
* be a multiple of the alignment.
*
* Fortunately, alignment is typically 1, so this is a non-issue in
* most cases. In cases where the alignment is not 1, for now we
* have decided to drop the fragment on the floor.
*
* Eventually, we should fix this by modifying the on disk representations
* of free space managers to allow for empty space, so as to bypass the
* issues created by self-referential free space managers, and make
* this issue moot.
*/
/* HDassert(f->shared->alignment == 1); */
/* The free space manager(s) that handle space allocations for free
* space managers should be settled now, albeit without file space
* allocated to them. To avoid the possibility of changing the sizes
* of their section info blocks, allocate space for them now at the
* end of file via H5FD_alloc().
*
* In the past, this issue of allocating space without touching the
* free space managers has been deal with by calling
* H5MF_aggr_vfd_alloc(), which in turn calls H5MF_aggr_alloc().
* This is problematic since (if I read the code correctly) it will
* re-constitute the metadata aggregator, which will add any left
* over space to one of the free space managers when freed.
*
* This is a non-starter, since the entire objective is to settle the
* free space managers.
*
* Hence the decision to call H5FD_alloc() directly.
*
* As discussed in PROBLEM above, if f->shared->alignment is not 1,
* this has the possibility of generating a fragment of file space
* that would typically be inserted into one of the free space managers.
*
* This is isn't good, but due to schedule pressure, we will just drop
* the fragement on the floor for now.
*/
if(hdr_fspace)
if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, hdr_fspace,
&(f->shared->fs_addr[hdr_fsm_alloc_type])) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't vfd allocate hdr FSM file space")
if(sinfo_fspace && (sinfo_fspace != hdr_fspace))
if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, sinfo_fspace,
&(f->shared->fs_addr[sinfo_fsm_alloc_type])) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't vfd allocate sinfo FSM file space")
/* Indicate that the FSM was settled successfully */
*fsm_settled = TRUE;
} /* end if */
done:
if(reset_ring)
if(H5AC_reset_ring(dxpl, orig_ring) < 0)
HDONE_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set property value")
FUNC_LEAVE_NOAPI(ret_value)
} /* H5MF_settle_meta_data_fsm() */
/*-------------------------------------------------------------------------
* Function: H5MF__close_delete
@ -1462,80 +2088,33 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2
&& f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
H5O_fsinfo_t fsinfo; /* Free space manager info message */
hbool_t update = FALSE; /* To update info for the message */
/* Check to remove free-space manager info message from superblock extension */
if(H5F_addr_defined(f->shared->sblock->ext_addr))
if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_FSINFO_ID) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension")
/* Superblock extension and free space manager message should
* exist at this point -- verify at least the former.
*/
HDassert(H5F_addr_defined(f->shared->sblock->ext_addr));
/* Free free-space manager header and/or section info header */
for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
H5FS_stat_t fs_stat; /* Information for free-space manager */
/* Check for free space manager of this type */
if(f->shared->fs_man[type]) {
/* Switch to "about to be deleted" state */
f->shared->fs_state[type] = H5F_FS_STATE_DELETING;
/* Query the free space manager's information */
if(H5FS_stat_info(f, f->shared->fs_man[type], &fs_stat) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
/* Check if the free space manager has space in the file */
if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) {
/* Delete the free space manager in the file */
/* (will re-allocate later) */
if(H5FS_free(f, f->shared->fs_man[type], dxpl_id) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
f->shared->fs_addr[type] = HADDR_UNDEF;
} /* end if */
} /* end iif */
fsinfo.fs_addr[type-1] = HADDR_UNDEF;
} /* end for */
/* Note that unlike the previous version of this code, we do not
* delete free space managers that have no section to store.
*
* Can't do this, as that would involve freeing file space, which would
* dirty the free space manager in question.
*
* Fortunately, the code doesn't seem to care about this.
*/
/* Gather data for the free space manager superblock extension message.
*
* In passing, verify that all the free space managers are closed.
*/
for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
fsinfo.fs_addr[type - 1] = f->shared->fs_addr[type];
fsinfo.strategy = f->shared->fs_strategy;
fsinfo.threshold = f->shared->fs_threshold;
/* Write free-space manager info message to superblock extension object header */
/* Create the superblock extension object header in advance if needed */
if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
/* Re-allocate free-space manager header and/or section info header */
for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
H5FS_stat_t fs_stat; /* Information for free-space manager */
/* Check for active free space manager of this type */
if(f->shared->fs_man[type]) {
/* Re-query free space manager info for this type */
if(H5FS_stat_info(f, f->shared->fs_man[type], &fs_stat) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
/* Are there sections to persist? */
if(fs_stat.serial_sect_count) {
/* Allocate space for free-space manager header */
if(H5FS_alloc_hdr(f, f->shared->fs_man[type], &f->shared->fs_addr[type], dxpl_id) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "can't allocated free-space header")
/* Allocate space for free-space maanger section info header */
if(H5FS_alloc_sect(f, f->shared->fs_man[type], dxpl_id) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate free-space section info")
HDassert(f->shared->fs_addr[type]);
fsinfo.fs_addr[type-1] = f->shared->fs_addr[type];
update = TRUE;
} /* end if */
} else if(H5F_addr_defined(f->shared->fs_addr[type])) {
fsinfo.fs_addr[type-1] = f->shared->fs_addr[type];
update = TRUE;
} /* end else-if */
} /* end for */
/* Update the free space manager info message in superblock extension object header */
if(update)
if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
/* Write the free space manager message -- message must already exist */
if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
/* Final close of free-space managers */
for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
@ -1564,6 +2143,11 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
f->shared->fs_addr[type] = HADDR_UNDEF;
} /* end for */
/* Verify that we haven't dirtied any metadata cache entries
* from the metadata free space manager ring out.
*/
HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
/* Verify that the aggregators are still shutdown. */
HDassert(f->shared->sdata_aggr.tot_size == 0);
HDassert(f->shared->sdata_aggr.addr == 0);

87
src/H5MFaggr.c
View File

@ -76,6 +76,83 @@ static herr_t H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type,
/*******************/
/*-------------------------------------------------------------------------
* Function: H5MF_vfd_alloc
*
* Purpose: Allocate SIZE bytes of file memory via H5FD_alloc() and return
* the relative address where that contiguous chunk of file memory
* exists.
* The TYPE argument describes the purpose for which the storage
* is being requested.
*
* Return: Success: The file address of new chunk.
* Failure: HADDR_UNDEF
*
* Programmer: Quincey Koziol
* January 2, 2017
*
*-------------------------------------------------------------------------
*/
haddr_t
H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, hsize_t size,
hbool_t keep_fragment)
{
haddr_t eoa; /* Initial EOA for the file */
haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */
hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */
haddr_t ret_value = HADDR_UNDEF; /* Return value */
FUNC_ENTER_NOAPI(HADDR_UNDEF)
#ifdef H5MF_ALLOC_DEBUG
HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size);
#endif /* H5MF_ALLOC_DEBUG */
/* check arguments */
HDassert(f);
HDassert(f->shared);
HDassert(f->shared->lf);
HDassert(size > 0);
/* Get the EOA for the file -- need for sanity check below */
if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type)))
HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa")
/* Check for overlap into temporary allocation space */
if(H5F_addr_gt((eoa + size), f->shared->tmp_addr))
HGOTO_ERROR(H5E_FSPACE, H5E_BADRANGE, HADDR_UNDEF, "hdr file space alloc will overlap into 'temporary' file space")
/* Allocate space for the header */
if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size)))
HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space for hdr")
/* Sanity check for overlapping into file's temporary allocation space */
HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr));
/* If the file alignment is 1, there should be no eoa fragment */
HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1));
/* Check if fragment was generated and we want to keep it */
if(keep_fragment && eoa_frag_size > 0) {
/* Sanity check */
HDassert(H5F_addr_defined(eoa_frag_addr));
/* Put fragment on the free list */
if(H5MF_xfree(f, alloc_type, dxpl_id, eoa_frag_addr, eoa_frag_size) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment")
} /* end if */
done:
#ifdef H5MF_ALLOC_DEBUG
HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size);
#endif /* H5MF_ALLOC_DEBUG */
#ifdef H5MF_ALLOC_DEBUG_DUMP
H5MF_sects_dump(f, dxpl_id, stderr);
#endif /* H5MF_ALLOC_DEBUG_DUMP */
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5MF_vfd_alloc() */
/*-------------------------------------------------------------------------
* Function: H5MF_aggr_vfd_alloc
@ -221,7 +298,7 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz
if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr))
HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space")
if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, aggr->addr + aggr->size, ext_size)) < 0)
if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space")
else if(was_extended) {
/* aggr->size is unchanged */
@ -264,7 +341,7 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC);
if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr))
HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space")
if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, aggr->addr + aggr->size, ext_size)) < 0)
if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space")
else if(was_extended) {
aggr->addr += aggr_frag_size;
@ -404,8 +481,8 @@ HDfprintf(stderr, "%s: ret_value = %a\n", FUNC, ret_value);
*-------------------------------------------------------------------------
*/
htri_t
H5MF_aggr_try_extend(H5F_t *f, H5F_blk_aggr_t *aggr, H5FD_mem_t type,
haddr_t blk_end, hsize_t extra_requested)
H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
H5FD_mem_t type, haddr_t blk_end, hsize_t extra_requested)
{
htri_t ret_value = FALSE; /* Return value */
@ -447,7 +524,7 @@ H5MF_aggr_try_extend(H5F_t *f, H5F_blk_aggr_t *aggr, H5FD_mem_t type,
else {
hsize_t extra = (extra_requested < aggr->alloc_size) ? aggr->alloc_size : extra_requested;
if((ret_value = H5FD_try_extend(f->shared->lf, type, f, (aggr->addr + aggr->size), extra)) < 0)
if((ret_value = H5FD_try_extend(f->shared->lf, type, f, dxpl_id, (aggr->addr + aggr->size), extra)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file")
else if(ret_value == TRUE) {
/* Shift the aggregator block by the extra requested */

2
src/H5MFpkg.h
View File

@ -159,7 +159,7 @@ H5_DLL herr_t H5MF_sect_simple_free(H5FS_section_info_t *sect);
/* Block aggregator routines */
H5_DLL haddr_t H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size);
H5_DLL htri_t H5MF_aggr_try_extend(H5F_t *f, H5F_blk_aggr_t *aggr,
H5_DLL htri_t H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
H5FD_mem_t type, haddr_t abs_blk_end, hsize_t extra_requested);
H5_DLL htri_t H5MF_aggr_can_absorb(const H5F_t *f, const H5F_blk_aggr_t *aggr,
const H5MF_free_section_t *sect, H5MF_shrink_type_t *shrink);

9
src/H5MFprivate.h
View File

@ -58,7 +58,10 @@ H5_DLL herr_t H5MF_try_close(H5F_t *f, hid_t dxpl_id);
/* File space allocation routines */
H5_DLL haddr_t H5MF_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size);
H5_DLL haddr_t H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size);
H5_DLL haddr_t H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id,
hsize_t size);
H5_DLL haddr_t H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type,
hsize_t size, hbool_t keep_fragment);
H5_DLL herr_t H5MF_xfree(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr,
hsize_t size);
H5_DLL herr_t H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type,
@ -75,6 +78,10 @@ H5_DLL haddr_t H5MF_alloc_tmp(H5F_t *f, hsize_t size);
H5_DLL herr_t H5MF_free_aggrs(H5F_t *f, hid_t dxpl_id);
H5_DLL htri_t H5MF_aggrs_try_shrink_eoa(H5F_t *f, hid_t dxpl_id);
/* Settling routines */
H5_DLL herr_t H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled);
H5_DLL herr_t H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled);
/* Debugging routines */
#ifdef H5MF_DEBUGGING
H5_DLL herr_t H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr,

4
src/H5O.c
View File

@ -129,12 +129,12 @@ const H5O_msg_class_t *const H5O_msg_class_g[] = {
H5O_MSG_DRVINFO, /*0x0014 Driver info settings */
H5O_MSG_AINFO, /*0x0015 Attribute information */
H5O_MSG_REFCOUNT, /*0x0016 Object's ref. count */
H5O_MSG_FSINFO, /*0x0017 Free-space manager info message */
H5O_MSG_FSINFO, /*0x0017 Free-space manager info */
H5O_MSG_UNKNOWN, /*0x0018 Placeholder for unknown message */
#ifdef H5O_ENABLE_BOGUS
H5O_MSG_BOGUS_INVALID, /*0x0019 "Bogus invalid" (for testing) */
#else /* H5O_ENABLE_BOGUS */
NULL, /*0x0019 "Bogus invalid" (for testing) */
NULL, /*0x0019 "Bogus invalid" (for testing) */
#endif /* H5O_ENABLE_BOGUS */
};

2
src/H5Oprivate.h
View File

@ -203,7 +203,7 @@ typedef struct H5O_copy_t {
#define H5O_DRVINFO_ID 0x0014 /* Driver info message. */
#define H5O_AINFO_ID 0x0015 /* Attribute info message. */
#define H5O_REFCOUNT_ID 0x0016 /* Reference count message. */
#define H5O_FSINFO_ID 0x0017 /* Free-space manager info message. */
#define H5O_FSINFO_ID 0x0017 /* File space info message. */
#define H5O_UNKNOWN_ID 0x0018 /* Placeholder message ID for unknown message. */
/* (this should never exist in a file) */
/*

19
test/cache.c
View File

@ -15912,6 +15912,11 @@ check_destroy_pinned_err(void)
protect_entry(file_ptr, 0, 0);
unprotect_entry(file_ptr, 0, 0, H5C__PIN_ENTRY_FLAG);
if(H5C_prep_for_file_close(file_ptr, H5P_DATASET_XFER_DEFAULT) < 0 ) {
pass = FALSE;
failure_mssg = "unexpected failure of prep for file close.\n";
} /* end if */
if(H5C_dest(file_ptr, H5AC_ind_read_dxpl_id) >= 0) {
pass = FALSE;
failure_mssg = "destroy succeeded on cache with pinned entry.\n";
@ -15987,6 +15992,20 @@ check_destroy_protected_err(void)
file_ptr = setup_cache((size_t)(2 * 1024),
(size_t)(1 * 1024));
/* Note: normally this call would go just before the series of
* flushes prior to file close -- in particular, all entries
* should be unprotected when this call is made.
*
* Thus H5C_prep_for_file_close() contains an assert to verify
* this. Since this assert would be triggered by the condition
* we are trying to test, put the call to H5C_prep_for_file_close()
* prior to the final protect call.
*/
if(H5C_prep_for_file_close(file_ptr, H5P_DATASET_XFER_DEFAULT) < 0 ) {
pass = FALSE;
failure_mssg = "unexpected failure of prep for file close.\n";
} /* end if */
protect_entry(file_ptr, 0, 0);
if(H5C_dest(file_ptr, H5AC_ind_read_dxpl_id) >= 0) {

6
test/cache_common.c
View File

@ -3408,6 +3408,12 @@ takedown_cache(H5F_t * file_ptr,
H5C_stats(cache_ptr, "test cache", dump_detailed_stats);
}
if ( H5C_prep_for_file_close(file_ptr, H5P_DATASET_XFER_DEFAULT) < 0 ) {
pass = FALSE;
failure_mssg = "unexpected failure of prep for file close.\n";
}
flush_cache(file_ptr, TRUE, FALSE, FALSE);
H5C_dest(file_ptr, H5AC_ind_read_dxpl_id);

29
test/freespace.c
View File

@ -2869,7 +2869,7 @@ test_fs_sect_iterate(hid_t fapl)
if(H5FS_sect_add(f, dxpl_id, frsp, (H5FS_section_info_t *)sect_node,
H5FS_ADD_RETURNED_SPACE, NULL) < 0)
FAIL_STACK_ERROR
}
} /* end for */
if(H5FS_sect_iterate(f, dxpl_id, frsp, TEST_sects_cb, &udata) < 0)
TEST_ERROR
@ -2905,7 +2905,7 @@ error:
H5E_BEGIN_TRY {
if(frsp)
H5FS_close(f, dxpl_id, frsp);
H5Fclose(file);
H5Fclose(file);
H5Pclose(dxpl_id);
} H5E_END_TRY;
return 1;
@ -2915,9 +2915,9 @@ error:
int
main(void)
{
hid_t fapl = -1; /* File access property list for data files */
unsigned nerrors = 0; /* Cumulative error count */
const char *env_h5_drvr = NULL; /* File Driver value from environment */
hid_t fapl = -1; /* File access property list for data files */
unsigned nerrors = 0; /* Cumulative error count */
const char *env_h5_drvr = NULL; /* File Driver value from environment */
/* Get the VFD to use */
env_h5_drvr = HDgetenv("HDF5_DRIVER");
@ -2926,11 +2926,16 @@ main(void)
h5_reset();
fapl = h5_fileaccess();
if((fapl = h5_fileaccess()) < 0) {
nerrors++;
PUTS_ERROR("Can't get VFD-dependent fapl")
} /* end if */
/* make sure alignment is not set for tests to succeed */
if(H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1) < 0)
TEST_ERROR
if(H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1) < 0) {
nerrors++;
PUTS_ERROR("Can't set alignment")
} /* end if */
nerrors += test_fs_create(fapl);
nerrors += test_fs_sect_add(fapl);
@ -2946,16 +2951,16 @@ main(void)
if(nerrors)
goto error;
puts("All free-space tests passed.");
HDputs("All free-space tests passed.");
h5_cleanup(FILENAME, fapl);
return 0;
HDexit(EXIT_SUCCESS);
error:
puts("*** TESTS FAILED ***");
HDputs("*** TESTS FAILED ***");
H5E_BEGIN_TRY {
H5Pclose(fapl);
} H5E_END_TRY;
return 1;
HDexit(EXIT_FAILURE);
} /* main() */