mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-03-31 17:10:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r27197] Improve performance of "printf" VDS with selections in the file space - the

Browse Source 
library no longer iterates over all source datasets in the printf mapping, it
now only looks at the ones that could be involved in I/O (the ones whose bounds
overlap with the selection in the unlimited dimension).

Tested: ummon
This commit is contained in:
Neil Fortner 2015-06-12 17:04:35 -05:00
parent 78e128c544
commit 4117cfe2b9
4 changed files with 97 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
src/H5Dvirtual.c
View File

@ -1493,6 +1493,10 @@ H5D__virtual_pre_io(H5D_io_info_t *io_info,
const H5S_t *mem_space, hsize_t *tot_nelmts)
{
hssize_t select_nelmts; /* Number of elements in selection */
hsize_t bounds_start[H5S_MAX_RANK]; /* Selection bounds start */
hsize_t bounds_end[H5S_MAX_RANK]; /* Selection bounds end */
int rank;
hbool_t bounds_init = FALSE; /* Whether bounds_start, bounds_end, and rank are valid */
size_t i, j; /* Local index variables */
herr_t ret_value = SUCCEED; /* Return value */
@ -1519,12 +1523,44 @@ H5D__virtual_pre_io(H5D_io_info_t *io_info,
/* Check for "printf" source dataset resolution */
if(storage->list[i].parsed_source_file_name
|| storage->list[i].parsed_source_dset_name) {
hbool_t partial_block;
HDassert(storage->list[i].unlim_dim_virtual >= 0);
/* Get selection bounds if necessary */
if(!bounds_init) {
/* Get rank of VDS */
if((rank = H5S_GET_EXTENT_NDIMS(io_info->dset->shared->space)) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions")
/* Get selection bounds */
if(H5S_SELECT_BOUNDS(file_space, bounds_start, bounds_end) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds")
/* Adjust bounds_end to represent the extent just enclosing them
* (add 1) */
for(j = 0; j < (size_t)rank; j++)
bounds_end[j]++;
/* Bounds are now initialized */
bounds_init = TRUE;
} /* end if */
/* Get index of first block in virtual selection */
storage->list[i].sub_dset_io_start = (size_t)H5S_hyper_get_first_inc_block(storage->list[i].source_dset.virtual_select, bounds_start[storage->list[i].unlim_dim_virtual], NULL);
/* Get index of first block outside of virtual selection */
storage->list[i].sub_dset_io_end = (size_t)H5S_hyper_get_first_inc_block(storage->list[i].source_dset.virtual_select, bounds_end[storage->list[i].unlim_dim_virtual], &partial_block);
if(partial_block)
storage->list[i].sub_dset_io_end++;
if(storage->list[i].sub_dset_io_end > storage->list[i].sub_dset_nused)
storage->list[i].sub_dset_io_end = storage->list[i].sub_dset_nused;
/* Iterate over sub-source dsets */
for(j = 0; j < storage->list[i].sub_dset_nused; j++) {
/* Check for no clipped selection. This should be an assertion
* once we have I/O scoping for printf working VDSINC */
if(!storage->list[i].sub_dset[j].clipped_virtual_select)
break;
for(j = storage->list[i].sub_dset_io_start;
j < storage->list[i].sub_dset_io_end; j++) {
/* There should always be a clipped virtual selection here */
HDassert(storage->list[i].sub_dset[j].clipped_virtual_select);
/* Project intersection of file space and mapping virtual space
* onto memory space */
@ -1634,7 +1670,8 @@ H5D__virtual_post_io(H5O_storage_virtual_t *storage)
if(storage->list[i].parsed_source_file_name
|| storage->list[i].parsed_source_dset_name) {
/* Iterate over sub-source dsets */
for(j = 0; j < storage->list[i].sub_dset_nused; j++)
for(j = storage->list[i].sub_dset_io_start;
j < storage->list[i].sub_dset_io_end; j++)
/* Close projected memory space */
if(storage->list[i].sub_dset[j].projected_mem_space) {
if(H5S_close(storage->list[i].sub_dset[j].projected_mem_space) < 0)
@ -1761,7 +1798,8 @@ H5D__virtual_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
if(storage->list[i].parsed_source_file_name
|| storage->list[i].parsed_source_dset_name) {
/* Iterate over sub-source dsets */
for(j = 0; j < storage->list[i].sub_dset_nused; j++)
for(j = storage->list[i].sub_dset_io_start;
j < storage->list[i].sub_dset_io_end; j++)
if(H5D__virtual_read_one(io_info, type_info, file_space, &storage->list[i].sub_dset[j]) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset")
} /* end if */
@ -1785,7 +1823,8 @@ H5D__virtual_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
if(storage->list[i].parsed_source_file_name
|| storage->list[i].parsed_source_dset_name) {
/* Iterate over sub-source dsets */
for(j = 0; j < storage->list[i].sub_dset_nused; j++)
for(j = storage->list[i].sub_dset_io_start;
j < storage->list[i].sub_dset_io_end; j++)
if(storage->list[i].sub_dset[j].projected_mem_space)
if(H5S_select_subtract(fill_space, storage->list[i].sub_dset[j].projected_mem_space) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to clip fill selection")
@ -1942,7 +1981,8 @@ H5D__virtual_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
if(storage->list[i].parsed_source_file_name
|| storage->list[i].parsed_source_dset_name) {
/* Iterate over sub-source dsets */
for(j = 0; j < storage->list[i].sub_dset_nused; j++) {
for(j = storage->list[i].sub_dset_io_start;
j < storage->list[i].sub_dset_io_end; j++) {
HDassert(0 && "Checking code coverage..."); //VDSINC
if(H5D__virtual_write_one(io_info, type_info, file_space, &storage->list[i].sub_dset[j]) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to write to source dataset")

2
src/H5Oprivate.h
View File

@ -449,6 +449,8 @@ typedef struct H5O_storage_virtual_ent_t {
H5O_storage_virtual_srcdset_t *sub_dset; /* Array of sub-source dataset info structs */
size_t sub_dset_nalloc; /* Number of slots allocated in sub_dset */
size_t sub_dset_nused; /* Number of slots "used" in sub_dset - essentially the farthest sub dataset in the extent */
size_t sub_dset_io_start; /* First element in sub_dset involved in current I/O op. Field has no meaning and may be uninitialized at all other times */
size_t sub_dset_io_end; /* First element in sub_dset outside of current I/O op. Field has no meaning and may be uninitialized at all other times */
H5O_storage_virtual_name_seg_t *parsed_source_file_name; /* Parsed version of source_dset.file_name */
size_t psfn_static_strlen; /* Length of parsed_source_file_name without block number substitutions */
size_t psfn_nsubs; /* Number of block number substitutions in parsed_source_file_name */

16
src/H5Shyper.c
View File

@ -10180,25 +10180,27 @@ H5S_hyper_get_first_inc_block(const H5S_t *space, hsize_t clip_size,
HDassert(hslab);
HDassert(hslab->unlim_dim >= 0);
HDassert(hslab->opt_unlim_diminfo[hslab->unlim_dim].count == H5S_UNLIMITED);
HDassert(partial);
diminfo = &hslab->opt_unlim_diminfo[hslab->unlim_dim];
/* Check for selection outside of clip_size */
if(diminfo->start >= clip_size) {
ret_value = 0;
partial = FALSE;
if(partial)
partial = FALSE;
} /* end if */
else {
/* Calculate index of first incomplete block */
ret_value = (clip_size - diminfo->start + diminfo->stride
- diminfo->block) / diminfo->stride;
/* Check for partial block */
if((diminfo->stride * ret_value) < (clip_size - diminfo->start))
*partial = TRUE;
else
*partial = FALSE;
if(partial) {
/* Check for partial block */
if((diminfo->stride * ret_value) < (clip_size - diminfo->start))
*partial = TRUE;
else
*partial = FALSE;
} /* end if */
} /* end else */
FUNC_LEAVE_NOAPI(ret_value)

41
test/vds.c
View File

@ -5515,10 +5515,6 @@ test_printf(unsigned config, hid_t fapl)
if(mdims[1] != 20)
TEST_ERROR
/* Close filespace */
if(H5Sclose(filespace) < 0)
TEST_ERROR
/* Read data through virtual dataset */
/* Reset rbuf */
HDmemset(rbuf[0], 0, sizeof(rbuf));
@ -5543,6 +5539,43 @@ test_printf(unsigned config, hid_t fapl)
TEST_ERROR
} /* end for */
/* Now try with different selections */
count[0] = 10;
for(start[1] = (hsize_t)0; start[1] < (hsize_t)5; start[1]++)
for(count[1] = (hsize_t)1; count[1] < (hsize_t)11; count[1]++) {
/* Reset rbuf */
HDmemset(rbuf[0], 0, sizeof(rbuf));
/* Select hyperslab in memory space */
if(H5Sselect_hyperslab(memspace, H5S_SELECT_SET, start, NULL, count, NULL) < 0)
TEST_ERROR
/* Select hyperslab in file space */
if(H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, count, NULL) < 0)
TEST_ERROR
/* Read data */
if(H5Dread(vdset, H5T_NATIVE_INT, memspace, filespace, H5P_DEFAULT, rbuf[0]) < 0)
TEST_ERROR
/* Verify read data */
for(i = 0; i < (int)mdims[0]; i++)
for(j = 0; j < (int)mdims[1]; j++) {
if((j < (int)start[1]) || (j >= (int)(start[1] + count[1]))) {
if(rbuf[i][j] != 0)
TEST_ERROR
} /* end if */
else
if(rbuf[i][j] != erbuf[i][j])
TEST_ERROR
} /* end for */
} /* end for */
start[1] = 0;
/* Close filespace */
if(H5Sclose(filespace) < 0)
TEST_ERROR
/* Close */
if(!(config & TEST_IO_CLOSE_SRC)) {
if(H5Dclose(srcdset[0]) < 0)