mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-21 03:13:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

count argument in H5Sselect_hyperslab

Browse Source 
Argument 'count' in NetCDF is not exactly the same as the 'count' in
H5Sselect_hyperslabs(space_id, op, start, stride, count, block).
When the argument 'stride' is NULL, NetCDF's 'count' should be used in
argument 'block', for example,
   H5Sselect_hyperslabs(space_id, op, start, NULL, ones, count);
where 'one' is an array of all 1s. Although using NULL 'block' below
   H5Sselect_hyperslabs(space_id, op, start, NULL, count, NULL);
has the same effect, HDF5 internally stores the space of a subarray as a
list of single elements, instead of a "block", which can affect the
performance.
This commit is contained in:
wkliao 2022-04-20 00:10:24 -05:00 committed by Ward Fisher
parent 0603c98933
commit 52198b3f12
10 changed files with 121 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
h5_test/tst_h_dimscales2.c
View File

@ -633,6 +633,7 @@ main()
hsize_t h5dimlen[DIMS2], h5dimlenmax[DIMS2], xtend_size[DIMS2] = {1, NUM_VALS};
hsize_t start[DIMS2] = {0, 0};
hsize_t count[DIMS2] = {1, NUM_VALS};
hsize_t ones[DIMS2] = {1, 1};
double value[NUM_VALS];
int dataset_ndims;
int i;
@ -661,7 +662,7 @@ main()
/* Set up the file and memory spaces. */
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, NULL, count, NULL) < 0) ERR;
start, NULL, ones, count) < 0) ERR;
if ((mem_spaceid = H5Screate_simple(DIMS2, count, NULL)) < 0) ERR;
/* Write a slice of data. */
@ -683,7 +684,7 @@ main()
/* Set up the file and memory spaces for a second slice. */
start[0]++;
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, NULL, count, NULL) < 0) ERR;
start, NULL, ones, count) < 0) ERR;
if ((mem_spaceid = H5Screate_simple(DIMS2, count, NULL)) < 0) ERR;
/* Write a second slice of data. */

4
h5_test/tst_h_enums.c
View File

@ -154,7 +154,7 @@ main()
char lang[NUM_LANG][STR_LEN + 1] = {"C", "Fortran", "C++", "MISSING"};
enum langs {CLANG=0, Fortran=1, CPP=2, MISSING=255};
short the_value, fill_value = MISSING, data_point = CLANG;
hsize_t start[1] = {1}, count[1] = {1};
hsize_t start[1] = {1}, count[1] = {1}, one[1] = {1};
int num_members;
size_t size;
hid_t base_hdf_typeid;
@ -197,7 +197,7 @@ main()
if ((mem_spaceid = H5Screate(H5S_SCALAR)) < 0) ERR;
if ((file_spaceid = H5Screate_simple(1, dims, NULL)) < 0) ERR;
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, NULL, count, NULL) < 0) ERR;
start, NULL, one, count) < 0) ERR;
if (H5Dwrite(datasetid, typeid, mem_spaceid, file_spaceid,
H5P_DEFAULT, &data_point) < 0) ERR;

10
h5_test/tst_h_files.c
View File

@ -174,7 +174,7 @@ main()
#define MILLION 1000000
hid_t fileid, write_spaceid, datasetid, mem_spaceid;
hsize_t start[NDIMS], count[NDIMS];
hsize_t start[NDIMS], count[NDIMS], ones[NDIMS];
hsize_t dims[1];
int *data;
int num_steps;
@ -210,8 +210,9 @@ main()
{
/* Select hyperslab for write of one slice. */
start[0] = s * SC;
ones[0] = 1;
if (H5Sselect_hyperslab(write_spaceid, H5S_SELECT_SET,
start, NULL, count, NULL) < 0) ERR;
start, NULL, ones, count) < 0) ERR;
if (H5Dwrite(datasetid, H5T_NATIVE_INT, mem_spaceid, write_spaceid,
H5P_DEFAULT, data) < 0) ERR;
@ -242,7 +243,7 @@ main()
hid_t mem_spaceid, xfer_plistid, native_typeid;
hsize_t *chunksize, dims[1], maxdims[1], *dimsize, *maxdimsize;
hsize_t fdims[MAX_DIMS], fmaxdims[MAX_DIMS];
hsize_t start[MAX_DIMS], count[MAX_DIMS];
hsize_t start[MAX_DIMS], count[MAX_DIMS], ones[MAX_DIMS];
char file_name[STR_LEN + 1];
char dimscale_wo_var[STR_LEN];
void *bufr;
@ -342,7 +343,8 @@ main()
start[1] = 0;
count[0] = 1;
count[1] = 2097153;
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) ERR;
ones[0] = ones[1] = 1;
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, NULL, ones, count) < 0) ERR;
if ((mem_spaceid = H5Screate_simple(NDIMS2, count, NULL)) < 0) ERR;
if ((xfer_plistid = H5Pcreate(H5P_DATASET_XFER)) < 0) ERR;
if ((native_typeid = H5Tget_native_type(H5T_NATIVE_SCHAR, H5T_DIR_DEFAULT)) < 0) ERR;

8
h5_test/tst_h_par.c
View File

@ -66,7 +66,7 @@ main(int argc, char **argv)
printf("*** Creating file for parallel I/O read, and rereading it...");
{
hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid;
hsize_t start[NDIMS], count[NDIMS];
hsize_t start[NDIMS], count[NDIMS], ones[NDIMS];
hsize_t dims[1];
int data[SC1], data_in[SC1];
int num_steps;
@ -126,8 +126,9 @@ main(int argc, char **argv)
/* Select hyperslab for write of one slice. */
start[0] = s * SC1 * p + my_rank * SC1;
count[0] = SC1;
ones[0] = 1;
if (H5Sselect_hyperslab(whole_spaceid, H5S_SELECT_SET,
start, NULL, count, NULL) < 0) ERR;
start, NULL, ones, count) < 0) ERR;
if (H5Dwrite(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid,
xferid, data) < 0) ERR;
@ -185,8 +186,9 @@ main(int argc, char **argv)
/* Select hyperslab for read of one slice. */
start[0] = s * SC1 * p + my_rank * SC1;
count[0] = SC1;
ones[0] = 1;
if (H5Sselect_hyperslab(whole_spaceid1, H5S_SELECT_SET,
start, NULL, count, NULL) < 0)
start, NULL, ones, count) < 0)
{
ERR;
return 2;

8
h5_test/tst_h_par_compress.c
View File

@ -51,7 +51,7 @@ main(int argc, char **argv)
{
hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid;
hid_t plistid;
hsize_t start[NDIMS], count[NDIMS];
hsize_t start[NDIMS], count[NDIMS], ones[NDIMS];
hsize_t dims[1], chunksize = SC1;
int data[SC1], data_in[SC1];
int num_steps;
@ -120,8 +120,9 @@ main(int argc, char **argv)
/* Select hyperslab for write of one slice. */
start[0] = s * SC1 * p + my_rank * SC1;
count[0] = SC1;
ones[0] = 1;
if (H5Sselect_hyperslab(whole_spaceid, H5S_SELECT_SET,
start, NULL, count, NULL) < 0) ERR;
start, NULL, ones, count) < 0) ERR;
if (H5Dwrite(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid,
xferid, data) < 0) ERR;
@ -160,8 +161,9 @@ main(int argc, char **argv)
/* Select hyperslab for read of one slice. */
start[0] = s * SC1 * p + my_rank * SC1;
count[0] = SC1;
ones[0] = 1;
if (H5Sselect_hyperslab(whole_spaceid1, H5S_SELECT_SET,
start, NULL, count, NULL) < 0)
start, NULL, ones, count) < 0)
{
ERR;
return 2;

7
h5_test/tst_h_strings2.c
View File

@ -30,6 +30,7 @@ main()
hid_t file_spaceid, mem_spaceid;
hsize_t dims[1] = {0}, max_dims[1] = {H5S_UNLIMITED}, chunk_dims[1] = {1};
hsize_t xtend_size[NDIMS] = {2}, start[NDIMS] = {1}, count[NDIMS] = {1};
hsize_t ones[NDIMS] = {1};
/* void *fillp;*/
char *data = "A man who carries a cat by the tail learns "
"something he can learn in no other way.";
@ -91,7 +92,7 @@ main()
/* Select space in file to write a record. */
if ((file_spaceid = H5Dget_space(datasetid)) < 0) ERR;
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, NULL, count, NULL) < 0) ERR;
start, NULL, ones, count) < 0) ERR;
/* Select space in memory to read from. */
if ((mem_spaceid = H5Screate_simple(NDIMS, count, NULL)) < 0) ERR;
@ -126,7 +127,7 @@ main()
hid_t typeid, datasetid, plistid;
hid_t file_spaceid, mem_spaceid;
hsize_t dims[1] = {2}, chunk_dims[1] = {1};
hsize_t start[NDIMS] = {1}, count[NDIMS] = {1};
hsize_t start[NDIMS] = {1}, count[NDIMS] = {1}, ones[NDIMS] = {1};
/* void *fillp;*/
char *data = "A man who carries a cat by the tail learns "
"something he can learn in no other way.";
@ -179,7 +180,7 @@ To be good is noble; but to show others how to be good is nobler and no trouble.
/* Select space in file to write a record. */
if ((file_spaceid = H5Dget_space(datasetid)) < 0) ERR;
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, NULL, count, NULL) < 0) ERR;
start, NULL, ones, count) < 0) ERR;
/* Select space in memory to read from. */
if ((mem_spaceid = H5Screate_simple(NDIMS, count, NULL)) < 0)

5
h5_test/tst_h_vars.c
View File

@ -69,7 +69,7 @@ main()
float float_data_out[LAT_LEN][LON_LEN];
hsize_t dims[NDIMS], max_dims[NDIMS];
hsize_t dims_in[NDIMS], max_dims_in[NDIMS];
hsize_t start[MAX_DIMS], count[MAX_DIMS];
hsize_t start[MAX_DIMS], count[MAX_DIMS], ones[MAX_DIMS];
int lat, lon;
/* Set up some phoney data, 1 record's worth. In C, first
@ -153,8 +153,9 @@ main()
start[0] = 1;
start[1] = 0;
start[2] = 0;
ones[0] = ones[1] = ones[2] = 1;
if (H5Sselect_hyperslab(write_spaceid, H5S_SELECT_SET,
start, NULL, count, NULL) < 0) ERR;
start, NULL, ones, count) < 0) ERR;
/* Write second record of data to each dataset. */
if (H5Dwrite(pres_dsid, H5T_IEEE_F32LE, mem_spaceid, write_spaceid,

37
libhdf5/hdf5var.c
View File

@ -1542,11 +1542,12 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
NC_VAR_INFO_T *var;
NC_DIM_INFO_T *dim;
NC_HDF5_VAR_INFO_T *hdf5_var;
herr_t herr;
hid_t file_spaceid = 0, mem_spaceid = 0, xfer_plistid = 0;
long long unsigned xtend_size[NC_MAX_VAR_DIMS];
hsize_t fdims[NC_MAX_VAR_DIMS], fmaxdims[NC_MAX_VAR_DIMS];
hsize_t start[NC_MAX_VAR_DIMS], count[NC_MAX_VAR_DIMS];
hsize_t stride[NC_MAX_VAR_DIMS];
hsize_t stride[NC_MAX_VAR_DIMS], ones[NC_MAX_VAR_DIMS];
int need_to_extend = 0;
#ifdef USE_PARALLEL4
int extend_possible = 0;
@ -1597,6 +1598,7 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
start[i] = startp[i];
count[i] = countp ? countp[i] : var->dim[i]->len;
stride[i] = stridep ? stridep[i] : 1;
ones[i] = 1;
LOG((4, "start[%d] %ld count[%d] %ld stride[%d] %ld", i, start[i], i, count[i], i, stride[i]));
/* Check to see if any counts are zero. */
@ -1647,8 +1649,13 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
}
else
{
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, stride,
count, NULL) < 0)
if (stridep == NULL)
herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start,
NULL, ones, count);
else
herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start,
stride, count, NULL);
if (herr < 0)
BAIL(NC_EHDFERR);
/* Create a space for the memory, just big enough to hold the slab
@ -1773,8 +1780,14 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
BAIL2(NC_EHDFERR);
if ((file_spaceid = H5Dget_space(hdf5_var->hdf_datasetid)) < 0)
BAIL(NC_EHDFERR);
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, stride, count, NULL) < 0)
if (stridep == NULL)
herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, NULL, ones, count);
else
herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, stride, count, NULL);
if (herr < 0)
BAIL(NC_EHDFERR);
}
}
@ -1873,7 +1886,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
hsize_t count[NC_MAX_VAR_DIMS];
hsize_t fdims[NC_MAX_VAR_DIMS], fmaxdims[NC_MAX_VAR_DIMS];
hsize_t start[NC_MAX_VAR_DIMS];
hsize_t stride[NC_MAX_VAR_DIMS];
hsize_t stride[NC_MAX_VAR_DIMS], ones[NC_MAX_VAR_DIMS];
void *fillvalue = NULL;
int no_read = 0, provide_fill = 0;
hssize_t fill_value_size[NC_MAX_VAR_DIMS];
@ -1924,6 +1937,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
start[i] = startp[i];
count[i] = countp[i];
stride[i] = stridep ? stridep[i] : 1;
ones[i] = 1;
/* if any of the count values are zero don't actually read. */
if (count[i] == 0)
@ -2053,9 +2067,16 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
}
else
{
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, stride, count, NULL) < 0)
herr_t herr;
if (stridep == NULL)
herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, NULL, ones, count);
else
herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, stride, count, NULL);
if (herr < 0)
BAIL(NC_EHDFERR);
/* Create a space for the memory, just big enough to hold the slab
we want. */
if ((mem_spaceid = H5Screate_simple(var->ndims, count, NULL)) < 0)

66
libnczarr/zvar.c
View File

@ -1546,7 +1546,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
#endif
size64_t fdims[NC_MAX_VAR_DIMS];
size64_t start[NC_MAX_VAR_DIMS], count[NC_MAX_VAR_DIMS];
size64_t stride[NC_MAX_VAR_DIMS];
size64_t stride[NC_MAX_VAR_DIMS], ones[NC_MAX_VAR_DIMS];
int retval, range_error = 0, i, d2;
void *bufr = NULL;
int bufrd = 0; /* 1 => we allocated bufr */
@ -1591,6 +1591,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
start[0] = 0;
count[0] = 1;
stride[0] = 1;
ones[0] = 1;
} else {
for (i = 0; i < var->ndims; i++)
{
@ -1602,6 +1603,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
start[i] = startp[i];
count[i] = countp ? countp[i] : fdims[i];
stride[i] = stridep ? stridep[i] : 1;
ones[i] = 1;
/* Check to see if any counts are zero. */
if (!count[i])
@ -1633,6 +1635,31 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
}
}
#ifdef LOOK
/* Now you would think that no one would be crazy enough to write
a scalar dataspace with one of the array function calls, but you
would be wrong. So let's check to see if the dataset is
scalar. If it is, we won't try to set up a hyperslab. */
if (H5Sget_simple_extent_type(file_spaceid) == H5S_SCALAR)
{
if ((mem_spaceid = H5Screate(H5S_SCALAR)) < 0)
BAIL(NC_EHDFERR);
}
else
{
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, stride,
ones, count) < 0)
BAIL(NC_EHDFERR);
/* Create a space for the memory, just big enough to hold the slab
we want. */
if ((mem_spaceid = H5Screate_simple(var->ndims, count, NULL)) < 0)
BAIL(NC_EHDFERR);
}
#endif
/* Are we going to convert any data? (No converting of compound or
* opaque or vlen types.) We also need to call this code if we are doing
* quantization. */
@ -1699,6 +1726,30 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
}
}
}
#endif
#ifdef LOOK
/* If we need to extend it, we also need a new file_spaceid
to reflect the new size of the space. */
if (need_to_extend)
{
LOG((4, "extending dataset"));
/* Convert xtend_size back to hsize_t for use with
* H5Dset_extent. */
for (d2 = 0; d2 < var->ndims; d2++)
fdims[d2] = (size64_t)xtend_size[d2];
if (H5Dset_extent(ncz_var->hdf_datasetid, fdims) < 0)
BAIL(NC_EHDFERR);
if (file_spaceid > 0 && H5Sclose(file_spaceid) < 0)
BAIL2(NC_EHDFERR);
if ((file_spaceid = H5Dget_space(ncz_var->hdf_datasetid)) < 0)
BAIL(NC_EHDFERR);
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, stride, ones, count) < 0)
BAIL(NC_EHDFERR);
}
#endif
}
/* Do we need to convert the data? */
@ -1801,8 +1852,8 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
size64_t fdims[NC_MAX_VAR_DIMS]; /* size of the dimensions */
size64_t start[NC_MAX_VAR_DIMS];
size64_t stride[NC_MAX_VAR_DIMS];
int no_read = 0;
int provide_fill = 0;
size64_t ones[NC_MAX_VAR_DIMS];
int no_read = 0, provide_fill = 0;
int fill_value_size[NC_MAX_VAR_DIMS];
int retval, range_error = 0, i, d2;
void *bufr = NULL;
@ -1834,6 +1885,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
start[0] = 0;
count[0] = 1;
stride[0] = 1;
ones[0] = 1;
} else {
for (i = 0; i < var->ndims; i++)
{
@ -1843,6 +1895,12 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
start[i] = startp[i];
count[i] = countp[i];
stride[i] = stridep ? stridep[i] : 1;
ones[i] = 1;
/* if any of the count values are zero don't actually read. */
if (count[i] == 0)
no_read++;
/* Get dimension sizes also */
fdims[i] = var->dim[i]->len;
/* if any of the counts are zero don't actually read. */
@ -1960,7 +2018,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
else
{
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET,
start, stride, count, NULL) < 0)
start, stride, ones, count) < 0)
BAIL(NC_EHDFERR);
/* Create a space for the memory, just big enough to hold the slab
we want. */

3
nc_perf/tst_files3.c
View File

@ -100,6 +100,7 @@ int dump_hdf_file(const float *data, int docompression)
hsize_t dims[NDIMS] = {X_LEN, Y_LEN, Z_LEN};
hsize_t start[NDIMS] = {0, 0, 0};
hsize_t count[NDIMS] = {1, 1, Z_LEN};
hsize_t ones[NDIMS] = {1, 1, 1};
/* create file */
file_id = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC,
@ -138,7 +139,7 @@ int dump_hdf_file(const float *data, int docompression)
for (start[1] = 0; start[1] < Y_LEN; start[1]++)
{
if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, NULL,
count, NULL) < 0) ERR_RET;
ones, count) < 0) ERR_RET;
if (H5Dwrite(dataset_id, H5T_NATIVE_FLOAT, mem_spaceid, file_spaceid,
xfer_plistid, data) < 0) ERR_RET;
}