mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-11-21 01:04:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r8664] Purpose:

Browse Source 
Code optimization

Description:
    Allow global heap collections to grow in size (up to a 64K limit) if they
are able to.  This allows them to grow to a more reasonable size than the 4K
minimum size.

Platforms tested:
    Solaris 2.7 (arabica)
    FreeBSD 4.10 (sleipnir) w/parallel
    Too minor to require h5committest
This commit is contained in:
Quincey Koziol 2004-06-12 17:32:19 -05:00
parent ea3103c15e
commit 8b247f8aa3
6 changed files with 426 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
src/H5FD.c
View File

@ -2616,6 +2616,164 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
}
/*-------------------------------------------------------------------------
* Function: H5FD_can_extend
*
* Purpose: Check if a block in the file can be extended.
*
* This is a simple check currently, which only checks for the
* block being at the end of the file. A more sophisticated check
* would also use the free space list to see if there is a block
* appropriately placed to accomodate the space requested.
*
* Return: Success: TRUE(1)/FALSE(0)
*
* Failure: FAIL
*
* Programmer: Quincey Koziol
* Friday, June 11, 2004
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
htri_t
H5FD_can_extend(H5FD_t *file, H5FD_mem_t type, haddr_t addr, hsize_t size, hsize_t UNUSED extra_requested)
{
haddr_t eoa; /* End of address space in the file */
htri_t ret_value=FALSE; /* Return value */
FUNC_ENTER_NOAPI(H5FD_can_extend, FAIL)
/* Retrieve the end of the address space */
if (HADDR_UNDEF==(eoa=H5FD_get_eoa(file)))
HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed")
/* Check if the block is exactly at the end of the file */
if((addr+size)==eoa)
HGOTO_DONE(TRUE)
/* Check if block is inside the metadata or small data accumulator */
else {
if(type!=H5FD_MEM_DRAW) {
if (file->feature_flags & H5FD_FEAT_AGGREGATE_METADATA) {
/* If the metadata block is at the end of the file, and
* the block to test adjoins the beginning of the metadata
* block, then it's extendable
*/
if (file->eoma + file->cur_meta_block_size == eoa &&
(addr+size)==file->eoma)
HGOTO_DONE(TRUE)
} /* end if */
} /* end if */
else {
if (file->feature_flags & H5FD_FEAT_AGGREGATE_SMALLDATA) {
/* If the small data block is at the end of the file, and
* the block to test adjoins the beginning of the small data
* block, then it's extendable
*/
if (file->eosda + file->cur_sdata_block_size == eoa &&
(addr+size)==file->eosda)
HGOTO_DONE(TRUE)
} /* end if */
} /* end else */
} /* end else */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_can_extend() */
/*-------------------------------------------------------------------------
* Function: H5FD_extend
*
* Purpose: Extend a block in the file.
*
* This is simple code currently, which only checks for the
* block being at the end of the file. A more sophisticated check
* would also use the free space list to see if there is a block
* appropriately placed to accomodate the space requested.
*
* Return: Success: TRUE(1)/FALSE(0)
*
* Failure: FAIL
*
* Programmer: Quincey Koziol
* Saturday, June 12, 2004
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5FD_extend(H5FD_t *file, H5FD_mem_t type, haddr_t addr, hsize_t size, hsize_t extra_requested)
{
haddr_t eoa; /* End of address space in the file */
hbool_t update_eoma=FALSE; /* Whether we need to update the eoma */
hbool_t update_eosda=FALSE; /* Whether we need to update the eosda */
herr_t ret_value=SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(H5FD_extend, FAIL)
/* Retrieve the end of the address space */
if (HADDR_UNDEF==(eoa=H5FD_get_eoa(file)))
HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed")
/* Check if the block is exactly at the end of the file */
/* (Check if block is inside the metadata or small data accumulator) */
if((addr+size)!=eoa) {
if(type!=H5FD_MEM_DRAW) {
if (file->feature_flags & H5FD_FEAT_AGGREGATE_METADATA) {
/* If the metadata block is at the end of the file, and
* the block to test adjoins the beginning of the metadata
* block, then it's extendable
*/
if (file->eoma + file->cur_meta_block_size == eoa &&
(addr+size)==file->eoma)
update_eoma=TRUE;
else
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "can't extend block")
} /* end if */
else
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "can't extend block")
} /* end if */
else {
if (file->feature_flags & H5FD_FEAT_AGGREGATE_SMALLDATA) {
/* If the small data block is at the end of the file, and
* the block to test adjoins the beginning of the small data
* block, then it's extendable
*/
if (file->eosda + file->cur_sdata_block_size == eoa &&
(addr+size)==file->eosda)
update_eosda=TRUE;
else
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "can't extend block")
} /* end if */
else
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "can't extend block")
} /* end else */
} /* end else */
/* Check for overflowing the file */
if (H5F_addr_overflow(eoa, extra_requested) || eoa + extra_requested > file->maxaddr)
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "file allocation request failed")
/* Extend the file */
eoa += extra_requested;
if (file->cls->set_eoa(file, eoa) < 0)
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "file allocation request failed")
/* Update the metadata and/or small data block */
assert(!(update_eoma && update_eosda));
if(update_eoma)
file->eoma+=extra_requested;
if(update_eosda)
file->eosda+=extra_requested;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_extend() */
/*-------------------------------------------------------------------------
* Function: H5FDget_eoa

2
src/H5FDfphdf5.h
View File

@ -102,7 +102,7 @@ H5_DLL herr_t H5Pget_fapl_fphdf5(hid_t fapl_id, MPI_Comm *comm,
struct H5P_genplist_t;
H5_DLL hid_t H5FD_fphdf5_init(void);
H5_DLL void H5FD_gass_term(void);
H5_DLL void H5FD_fphdf5_term(void);
H5_DLL unsigned H5FD_fphdf5_file_id(H5FD_t *_file);
H5_DLL hbool_t H5FD_fphdf5_is_sap(H5FD_t *_file);
H5_DLL hbool_t H5FD_fphdf5_is_captain(H5FD_t *_file);

4
src/H5FDprivate.h
View File

@ -70,5 +70,9 @@ H5_DLL herr_t H5FD_flush(H5FD_t *file, hid_t dxpl_id, unsigned closing);
H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum);
H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void** file_handle);
H5_DLL hssize_t H5FD_get_freespace(H5FD_t *file);
H5_DLL htri_t H5FD_can_extend(H5FD_t *file, H5FD_mem_t type, haddr_t addr,
hsize_t size, hsize_t extra_requested);
H5_DLL herr_t H5FD_extend(H5FD_t *file, H5FD_mem_t type, haddr_t addr,
hsize_t size, hsize_t extra_requested);
#endif /* !_H5FDprivate_H */

212
src/H5HG.c
View File

@ -67,6 +67,12 @@
*/
#define H5HG_MINSIZE 4096
/*
* Limit global heap collections to the some reasonable size. This is
* fairly arbitrary...
*/
#define H5HG_MAXSIZE 65536
/*
* Maximum length of the CWFS list, the list of remembered collections that
* have free space.
@ -352,8 +358,30 @@ H5HG_load (H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * udata1,
uint8_t *begin = p;
UINT16DECODE (p, idx);
assert (idx<heap->nalloc);
/* Check if we need more room to store heap objects */
if(idx>=heap->nalloc) {
size_t new_alloc; /* New allocation number */
H5HG_obj_t *new_obj; /* New array of object descriptions */
/* Determine the new number of objects to index */
new_alloc=MAX(heap->nalloc*2,(idx+1));
/* Reallocate array of objects */
if (NULL==(new_obj = H5FL_SEQ_REALLOC (H5HG_obj_t, heap->obj, new_alloc)))
HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed");
/* Reset new objects to zero */
HDmemset(&new_obj[heap->nalloc],0,sizeof(H5HG_obj_t)*(new_alloc-heap->nalloc));
/* Update heap information */
heap->nalloc=new_alloc;
heap->obj=new_obj;
} /* end if */
/* Check that we haven't double-allocated an index */
assert (NULL==heap->obj[idx].begin);
UINT16DECODE (p, heap->obj[idx].nrefs);
p += 4; /*reserved*/
H5F_DECODE_LENGTH (f, p, heap->obj[idx].size);
@ -561,13 +589,14 @@ done:
*-------------------------------------------------------------------------
*/
static unsigned
H5HG_alloc (H5F_t *f, H5HG_heap_t *heap, int cwfsno, size_t size)
H5HG_alloc (H5F_t *f, H5HG_heap_t *heap, size_t size)
{
unsigned idx;
uint8_t *p = NULL;
size_t need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(size);
unsigned ret_value; /* Return value */
FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5HG_alloc);
FUNC_ENTER_NOAPI_NOINIT(H5HG_alloc);
/* Check args */
assert (heap);
@ -577,11 +606,29 @@ H5HG_alloc (H5F_t *f, H5HG_heap_t *heap, int cwfsno, size_t size)
* Find an ID for the new object. ID zero is reserved for the free space
* object.
*/
for (idx=1; idx<heap->nalloc; idx++) {
for (idx=1; idx<heap->nalloc; idx++)
if (NULL==heap->obj[idx].begin)
break;
}
assert (idx < heap->nalloc);
/* Check if we need more room to store heap objects */
if(idx>=heap->nalloc) {
size_t new_alloc; /* New allocation number */
H5HG_obj_t *new_obj; /* New array of object descriptions */
/* Determine the new number of objects to index */
new_alloc=MAX(heap->nalloc*2,(idx+1));
/* Reallocate array of objects */
if (NULL==(new_obj = H5FL_SEQ_REALLOC (H5HG_obj_t, heap->obj, new_alloc)))
HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed");
/* Reset new objects to zero */
HDmemset(&new_obj[heap->nalloc],0,sizeof(H5HG_obj_t)*(new_alloc-heap->nalloc));
/* Update heap information */
heap->nalloc=new_alloc;
heap->obj=new_obj;
} /* end if */
/* Initialize the new object */
heap->obj[idx].nrefs = 0;
@ -596,16 +643,10 @@ H5HG_alloc (H5F_t *f, H5HG_heap_t *heap, int cwfsno, size_t size)
/* Fix the free space object */
if (need==heap->obj[0].size) {
/*
* All free space has been exhausted from this collection. Remove the
* heap from the CWFS list.
* All free space has been exhausted from this collection.
*/
heap->obj[0].size = 0;
heap->obj[0].begin = NULL;
if (cwfsno>=0) {
f->shared->ncwfs -= 1;
HDmemmove (f->shared->cwfs+cwfsno, f->shared->cwfs+cwfsno+1,
(f->shared->ncwfs-cwfsno)*sizeof(H5HG_heap_t*));
}
} else if (heap->obj[0].size-need >= H5HG_SIZEOF_OBJHDR (f)) {
/*
@ -631,10 +672,104 @@ H5HG_alloc (H5F_t *f, H5HG_heap_t *heap, int cwfsno, size_t size)
assert(H5HG_ISALIGNED(heap->obj[0].size));
}
/* Mark the heap as dirty */
heap->cache_info.dirty = 1;
FUNC_LEAVE_NOAPI(idx);
/* Set the return value */
ret_value=idx;
done:
FUNC_LEAVE_NOAPI(ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5HG_extend
*
* Purpose: Extend a heap to hold an object of SIZE bytes.
* SIZE is the exact size of the object data to be
* stored. It will be increased to make room for the object
* header and then rounded up for alignment.
*
* Return: Success: Non-negative
*
* Failure: Negative
*
* Programmer: Quincey Koziol
* Saturday, June 12, 2004
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
H5HG_extend (H5F_t *f, H5HG_heap_t *heap, size_t size)
{
size_t need; /* Actual space needed to store object */
size_t old_size; /* Previous size of the heap's chunk */
uint8_t *new_chunk=NULL; /* Pointer to new chunk information */
uint8_t *p = NULL; /* Pointer to raw heap info */
unsigned u; /* Local index variable */
herr_t ret_value=SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT(H5HG_extend);
/* Check args */
assert (f);
assert (heap);
/* Compute total space need to add to this heap */
need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(size);
/* Decrement the amount needed in the heap by the amount of free space available */
assert(need>heap->obj[0].size);
need -= heap->obj[0].size;
/* Don't do anything less than double the size of the heap */
need = MAX(heap->size,need);
/* Extend the space allocated for this heap on disk */
if(H5MF_extend(f,H5FD_MEM_GHEAP,heap->addr,(hsize_t)heap->size,(hsize_t)need)<0)
HGOTO_ERROR (H5E_HEAP, H5E_NOSPACE, FAIL, "can't extend heap on disk");
/* Re-allocate the heap information in memory */
if (NULL==(new_chunk = H5FL_BLK_REALLOC (heap_chunk, heap->chunk, heap->size+need)))
HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "new heap allocation failed");
/* Adjust the size of the heap */
old_size=heap->size;
heap->size+=need;
/* Encode the new size of the heap */
p = new_chunk + H5HG_SIZEOF_MAGIC + 1 /* version */ + 3 /* reserved */;
H5F_ENCODE_LENGTH (f, p, heap->size);
/* Move the pointers to the existing objects to their new locations */
for (u=0; u<heap->nalloc; u++)
if(heap->obj[u].begin)
heap->obj[u].begin = new_chunk + (heap->obj[u].begin - heap->chunk);
/* Update the heap chunk pointer now */
heap->chunk=new_chunk;
/* Update the free space information for the heap */
heap->obj[0].size+=need;
if(heap->obj[0].begin==NULL)
heap->obj[0].begin=heap->chunk+old_size;
p = heap->obj[0].begin;
UINT16ENCODE(p, 0); /*id*/
UINT16ENCODE(p, 0); /*nrefs*/
UINT32ENCODE(p, 0); /*reserved*/
H5F_ENCODE_LENGTH (f, p, heap->obj[0].size);
assert(H5HG_ISALIGNED(heap->obj[0].size));
/* Mark the heap as dirty */
heap->cache_info.dirty = 1;
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5HG_extend() */
/*-------------------------------------------------------------------------
* Function: H5HG_insert
@ -667,6 +802,7 @@ H5HG_insert (H5F_t *f, hid_t dxpl_id, size_t size, void *obj, H5HG_t *hobj/*out*
int cwfsno;
unsigned idx;
H5HG_heap_t *heap = NULL;
hbool_t found=0; /* Flag to indicate a heap with enough space was found */
herr_t ret_value=SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(H5HG_insert, FAIL);
@ -683,35 +819,53 @@ H5HG_insert (H5F_t *f, hid_t dxpl_id, size_t size, void *obj, H5HG_t *hobj/*out*
need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(size);
for (cwfsno=0; cwfsno<f->shared->ncwfs; cwfsno++) {
if (f->shared->cwfs[cwfsno]->obj[0].size>=need) {
/*
* Found. Move the collection forward in the CWFS list.
*/
heap = f->shared->cwfs[cwfsno];
if (cwfsno>0) {
H5HG_heap_t *tmp = f->shared->cwfs[cwfsno];
f->shared->cwfs[cwfsno] = f->shared->cwfs[cwfsno-1];
f->shared->cwfs[cwfsno-1] = tmp;
--cwfsno;
}
found=1;
break;
}
}
} /* end if */
} /* end for */
/*
* If we didn't find any collection with enough free space the check if
* we can extend any of the collections to make enough room.
*/
if (!found) {
for (cwfsno=0; cwfsno<f->shared->ncwfs; cwfsno++) {
if((f->shared->cwfs[cwfsno]->size+need)<=H5HG_MAXSIZE && H5MF_can_extend(f,H5FD_MEM_GHEAP,f->shared->cwfs[cwfsno]->addr,(hsize_t)f->shared->cwfs[cwfsno]->size,(hsize_t)need)) {
if(H5HG_extend(f,f->shared->cwfs[cwfsno],size)<0)
HGOTO_ERROR (H5E_HEAP, H5E_CANTINIT, FAIL, "unable to extend global heap collection");
found=1;
break;
} /* end if */
} /* end for */
} /* end if */
/*
* If we didn't find any collection with enough free space then allocate a
* new collection large enough for the message plus the collection header.
*/
if (cwfsno>=f->shared->ncwfs) {
if (!found) {
if (NULL==(heap=H5HG_create (f, dxpl_id, need+H5HG_SIZEOF_HDR (f))))
HGOTO_ERROR (H5E_HEAP, H5E_CANTINIT, FAIL, "unable to allocate a global heap collection");
assert (f->shared->ncwfs>0);
assert (f->shared->cwfs[0]==heap);
assert (f->shared->cwfs[0]->obj[0].size >= need);
cwfsno = 0;
}
} /* end if */
else {
/* Found a heap with enough space */
heap = f->shared->cwfs[cwfsno];
/* Move the collection forward in the CWFS list, if it's not already at the front */
if (cwfsno>0) {
H5HG_heap_t *tmp = f->shared->cwfs[cwfsno];
f->shared->cwfs[cwfsno] = f->shared->cwfs[cwfsno-1];
f->shared->cwfs[cwfsno-1] = tmp;
--cwfsno;
} /* end if */
} /* end else */
/* Split the free space to make room for the new object */
idx = H5HG_alloc (f, heap, cwfsno, size);
idx = H5HG_alloc (f, heap, size);
assert (idx>0);
/* Copy data into the heap */

76
src/H5MF.c
View File

@ -209,3 +209,79 @@ H5MF_realloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t old_addr, hsize_t
done:
FUNC_LEAVE_NOAPI(ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5MF_can_extend
*
* Purpose: Check if a block in the file can be extended.
*
* This is a simple check currently, which only checks for the
* block being at the end of the file. A more sophisticated check
* would also use the free space list to see if there is a block
* appropriately placed to accomodate the space requested.
*
* Return: Success: TRUE(1)/FALSE(0)
*
* Failure: FAIL
*
* Programmer: Quincey Koziol
* Friday, June 11, 2004
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
htri_t
H5MF_can_extend(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hsize_t extra_requested)
{
htri_t ret_value; /* Return value */
FUNC_ENTER_NOAPI(H5MF_can_extend, FAIL);
/* Convert old relative address to absolute address */
addr += f->shared->base_addr;
/* Pass the request down to the virtual file layer */
if((ret_value=H5FD_can_extend(f->shared->lf, type, addr, size, extra_requested))<0)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate new file memory");
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5MF_can_extend() */
/*-------------------------------------------------------------------------
* Function: H5MF_extend
*
* Purpose: Extend a block in the file.
*
* Return: Success: TRUE(1)/FALSE(0)
*
* Failure: FAIL
*
* Programmer: Quincey Koziol
* Saturday, June 12, 2004
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
htri_t
H5MF_extend(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hsize_t extra_requested)
{
htri_t ret_value; /* Return value */
FUNC_ENTER_NOAPI(H5MF_extend, FAIL);
/* Convert old relative address to absolute address */
addr += f->shared->base_addr;
/* Pass the request down to the virtual file layer */
if((ret_value=H5FD_extend(f->shared->lf, type, addr, size, extra_requested))<0)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate new file memory");
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5MF_extend() */

4
src/H5MFprivate.h
View File

@ -48,5 +48,9 @@ H5_DLL herr_t H5MF_xfree(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr,
hsize_t size);
H5_DLL haddr_t H5MF_realloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t old_addr,
hsize_t old_size, hsize_t new_size);
H5_DLL htri_t H5MF_can_extend(H5F_t *f, H5FD_mem_t type, haddr_t addr,
hsize_t size, hsize_t extra_requested);
H5_DLL htri_t H5MF_extend(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size,
hsize_t extra_requested);
#endif