mirror of
https://github.com/HDFGroup/hdf5.git
synced 2024-11-27 02:10:55 +08:00
[svn-r11728] Purpose:
To support collective chunk IO, obtain the chunk addresses globally from one process. Description: Implement a call-back function to iterate B-tree to do the job. Check in this code for major check-ins later. Solution: Platforms tested: heping(Linux) Should not be used by any library functions now, so won't affect library and tool tests. No need to test on 3 platforms. Misc. update:
This commit is contained in:
MuQun Yang
parent
e2f0e6a176
commit
f76495cdf5
152
src/H5Distore.c
152
src/H5Distore.c
@ -196,6 +196,13 @@ typedef struct H5D_istore_it_ud4_t {
|
||||
haddr_t addr_dst; /* Address of dest. B-tree */
|
||||
} H5D_istore_it_ud4_t;
|
||||
|
||||
/* B-tree callback info for iteration to obtain chunk address and the index of the chunk for all chunks in the B-tree. */
|
||||
typedef struct H5D_istore_it_ud5_t {
|
||||
H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */
|
||||
hsize_t down_chunks[H5O_LAYOUT_NDIMS];
|
||||
haddr_t *chunk_addr;
|
||||
} H5D_istore_it_ud5_t;
|
||||
|
||||
/********************/
|
||||
/* Local Prototypes */
|
||||
/********************/
|
||||
@ -206,6 +213,8 @@ static herr_t H5D_istore_shared_create (const H5F_t *f, H5O_layout_t *layout);
|
||||
static herr_t H5D_istore_shared_free (void *page);
|
||||
|
||||
/* B-tree iterator callbacks */
|
||||
static int H5D_istore_iter_chunkmap(H5F_t *f, hid_t dxpl_id, const void *left_key, haddr_t addr,
|
||||
const void *right_key, void *_udata);
|
||||
static int H5D_istore_iter_allocated(H5F_t *f, hid_t dxpl_id, const void *left_key, haddr_t addr,
|
||||
const void *right_key, void *_udata);
|
||||
static int H5D_istore_iter_dump(H5F_t *f, hid_t dxpl_id, const void *left_key, haddr_t addr,
|
||||
@ -856,6 +865,53 @@ H5D_istore_iter_allocated (H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_l
|
||||
|
||||
FUNC_LEAVE_NOAPI(H5B_ITER_CONT)
|
||||
} /* H5D_istore_iter_allocated() */
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
* Function: H5D_istore_iter_chunkmap
|
||||
*
|
||||
* Purpose: obtain chunk address and the corresponding index
|
||||
*
|
||||
* Return: Success: Non-negative
|
||||
*
|
||||
* Failure: Negative
|
||||
*
|
||||
* Programmer: Robb Matzke
|
||||
* Wednesday, April 21, 1999
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
/* ARGSUSED */
|
||||
static int
|
||||
H5D_istore_iter_chunkmap (H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_lt_key, haddr_t addr,
|
||||
const void UNUSED *_rt_key, void *_udata)
|
||||
{
|
||||
H5D_istore_it_ud5_t *udata = (H5D_istore_it_ud5_t *)_udata;
|
||||
const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key;
|
||||
unsigned rank;
|
||||
hsize_t chunk_offset[H5O_LAYOUT_NDIMS]; /*logical location of the chunks */
|
||||
hsize_t chunk_index;
|
||||
int ret_value = H5B_ITER_CONT; /* Return value */
|
||||
|
||||
FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_chunkmap);
|
||||
|
||||
rank = udata->common.mesg->u.chunk.ndims - 1;
|
||||
|
||||
if(H5V_chunk_index(rank,lt_key->offset,udata->common.mesg->u.chunk.dim,udata->down_chunks,&chunk_index)<0)
|
||||
HGOTO_ERROR (H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
|
||||
|
||||
#if 0
|
||||
HDfprintf(stdout,"chunk index is %Hu\n",chunk_index);
|
||||
#endif
|
||||
udata->chunk_addr[chunk_index] = addr;
|
||||
#if 0
|
||||
HDfprintf(stdout,"chunk address is %Hu\n",addr);
|
||||
#endif
|
||||
done:
|
||||
|
||||
FUNC_LEAVE_NOAPI(ret_value)
|
||||
} /* H5D_istore_iter_allocated() */
|
||||
|
||||
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
@ -2300,6 +2356,102 @@ H5D_istore_allocated(H5D_t *dset, hid_t dxpl_id)
|
||||
done:
|
||||
FUNC_LEAVE_NOAPI(ret_value)
|
||||
} /* end H5D_istore_allocated() */
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
* Function: H5D_istore_chunkmap
|
||||
*
|
||||
* Purpose: obtain the chunk address and corresponding chunk index
|
||||
*
|
||||
*
|
||||
*
|
||||
* Return: Success: Non-negative on succeed.
|
||||
*
|
||||
* Failure: negative value
|
||||
*
|
||||
* Programmer:
|
||||
*
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
herr_t
|
||||
H5D_istore_chunkmap(const H5D_io_info_t *io_info, hsize_t total_chunks,haddr_t chunk_addr[])
|
||||
{
|
||||
|
||||
H5D_t *dset=io_info->dset; /* Local pointer to dataset info */
|
||||
const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache */
|
||||
H5D_rdcc_ent_t *ent; /*cache entry */
|
||||
H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */
|
||||
H5D_dxpl_cache_t *dxpl_cache=&_dxpl_cache; /* Data transfer property cache */
|
||||
H5D_istore_it_ud5_t udata;
|
||||
|
||||
hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */
|
||||
hsize_t chunks[H5O_LAYOUT_NDIMS]; /*current number of chunks in each dimension */
|
||||
hsize_t down_chunks[H5O_LAYOUT_NDIMS]; /* "down" size of number of elements in each dimension */
|
||||
|
||||
int srank; /*current # of dimensions (signed) */
|
||||
unsigned rank; /*current # of dimensions */
|
||||
unsigned u;
|
||||
|
||||
hid_t dxpl_id = io_info->dxpl_id;
|
||||
herr_t ret_value = SUCCEED; /* Return value */
|
||||
|
||||
FUNC_ENTER_NOAPI(H5D_istore_chunkmap, 0)
|
||||
|
||||
HDassert(dset);
|
||||
|
||||
#if 0
|
||||
/* Fill the DXPL cache values for later use */
|
||||
if (H5D_get_dxpl_cache(dxpl_id,&dxpl_cache)<0)
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't fill dxpl cache")
|
||||
|
||||
/* Construct dataset I/O info */
|
||||
H5D_BUILD_IO_INFO(&io_info,dset,dxpl_cache,dxpl_id,NULL);
|
||||
|
||||
/* Search for cached chunks that haven't been written out */
|
||||
for(ent = rdcc->head; ent; ent = ent->next) {
|
||||
/* Flush the chunk out to disk, to make certain the size is correct later */
|
||||
if (H5D_istore_flush_entry(&io_info, ent, FALSE)<0)
|
||||
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, 0, "cannot flush indexed storage buffer")
|
||||
} /* end for */
|
||||
#endif
|
||||
|
||||
HDmemset(&udata, 0, sizeof udata);
|
||||
udata.common.mesg = &dset->shared->layout;
|
||||
|
||||
/* Go get the rank & dimensions */
|
||||
if((srank = H5S_get_simple_extent_dims(dset->shared->space, curr_dims, NULL)) < 0)
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions");
|
||||
H5_ASSIGN_OVERFLOW(rank,srank,int,unsigned);
|
||||
|
||||
/* Obtain number of chunks for each dimension*/
|
||||
for(u = 0; u < rank; u++) {
|
||||
/* Round up to the next integer # of chunks, to accomodate partial chunks */
|
||||
chunks[u] = ((curr_dims[u]+dset->shared->layout.u.chunk.dim[u])-1) / dset->shared->layout.u.chunk.dim[u];
|
||||
} /* end for */
|
||||
|
||||
/* Get the "down" sizes for each dimension */
|
||||
if(H5V_array_down(rank,chunks,udata.down_chunks)<0)
|
||||
HGOTO_ERROR (H5E_INTERNAL, H5E_BADVALUE, FAIL, "can't compute 'down' sizes")
|
||||
/*
|
||||
for(u = 0; u < rank; u++)
|
||||
udata.down_chunks[u] = down_chunks[u];
|
||||
*/
|
||||
|
||||
assert(total_chunks!=0);
|
||||
udata.chunk_addr=(haddr_t *)malloc(total_chunks*sizeof(haddr_t));
|
||||
HDmemset(udata.chunk_addr,0,total_chunks*sizeof(haddr_t));
|
||||
|
||||
if (H5B_iterate(dset->oloc.file, dxpl_id, H5B_ISTORE, H5D_istore_iter_chunkmap, dset->shared->layout.u.chunk.addr, &udata)<0)
|
||||
HGOTO_ERROR(H5E_IO, H5E_CANTINIT, 0, "unable to iterate over chunk B-tree")
|
||||
|
||||
for(u=0; u<total_chunks;u++)
|
||||
chunk_addr[u] = udata.chunk_addr[u];
|
||||
HDfree(udata.chunk_addr);
|
||||
|
||||
done:
|
||||
FUNC_LEAVE_NOAPI(ret_value)
|
||||
} /* end H5D_istore_allocated() */
|
||||
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
|
||||
@ -259,6 +259,7 @@ H5_DLL herr_t H5D_istore_initialize_by_extent(H5D_io_info_t *io_info);
|
||||
H5_DLL herr_t H5D_istore_update_cache(H5D_t *dset, hid_t dxpl_id);
|
||||
H5_DLL herr_t H5D_istore_dump_btree(H5F_t *f, hid_t dxpl_id, FILE *stream, unsigned ndims,
|
||||
haddr_t addr);
|
||||
H5_DLL herr_t H5D_istore_chunkmap(const H5D_io_info_t *io_info, hsize_t total_chunk, haddr_t chunk_addr[] );
|
||||
#ifdef H5D_ISTORE_DEBUG
|
||||
H5_DLL herr_t H5D_istore_stats (H5D_t *dset, hbool_t headers);
|
||||
#endif /* H5D_ISTORE_DEBUG */
|
||||
|
||||
Loading…
Reference in New Issue
Block a user