mirror of
https://github.com/HDFGroup/hdf5.git
synced 2025-03-31 17:10:47 +08:00
[svn-r322] Changes since 19980313
---------------------- ./configure.in ./configure ./test/iopipe.c Added check for system(3) ./config/freebsd2.2.1 ./config/linux Added -DH5D_DEBUG to the debug flags. ./src/H5D.c ./src/H5Dprivate.h ./src/H5P.c ./src/H5Ppublic.h ./src/H5Sprivate.h ./src/H5Ssimp.c ./html/Datasets.html Finally implemented strip mining in the I/O pipeline, placing a user-defined upper bound on the amount of temporary memory used by the pipeline. The default is 1MB type conversion and background buffers allocated/freed on demand. However, the size can be changed and/or application-allocated buffers supplied with H5Pset_buffers() called on the data transfer property list passed to H5Dread() and H5Dwrite(). Minor optimizations to H5Dread() and H5Dwrite(). More coming later... ./test/dsets.c Added calls to H5Pset_buffer() to test application-defined temporary buffers in the I/O pipeline. ./test/Makefile.in Removed `iopipe' from the list of confidence tests. Saying `make timings' in the test directory runs timing tests. I did this because (1) they don't really test anything new, and (2) they can take a long time to run.
This commit is contained in:
parent
48e0751106
commit
31a709a6b2
@ -41,7 +41,7 @@ warn="-Wall -Wshadow -Wpointer-arith -Wcast-qual -Wwrite-strings -Wstrict-protot
|
||||
|
||||
profile="-pg"
|
||||
|
||||
debug="-g -DH5AC_DEBUG -DH5B_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -fverbose-asm"
|
||||
debug="-g -DH5AC_DEBUG -DH5B_DEBUG -DH5D_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -fverbose-asm"
|
||||
|
||||
production="-O3 -DNDEBUG -finline-functions -funroll-loops -malign-double -fomit-frame-pointer"
|
||||
|
||||
|
@ -53,7 +53,7 @@ fi
|
||||
if test "X" = "X$CPPFLAGS"; then
|
||||
warn=
|
||||
profile=
|
||||
debug="-DH5AC_DEBUG -DH5B_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -DH5F_LOW_DFLT=H5F_LOW_SEC2"
|
||||
debug="-DH5AC_DEBUG -DH5B_DEBUG -DH5D_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -DH5F_LOW_DFLT=H5F_LOW_SEC2"
|
||||
production="-DNDEBUG"
|
||||
parallel=""
|
||||
default_mode='$debug $warn $parallel'
|
||||
|
2
configure
vendored
2
configure
vendored
@ -1267,7 +1267,7 @@ fi
|
||||
|
||||
|
||||
|
||||
for ac_func in getpwuid gethostname
|
||||
for ac_func in getpwuid gethostname system
|
||||
do
|
||||
echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
|
||||
echo "configure:1274: checking for $ac_func" >&5
|
||||
|
@ -112,7 +112,7 @@ AC_TYPE_SIZE_T
|
||||
dnl ----------------------------------------------------------------------
|
||||
dnl Check for functions.
|
||||
dnl
|
||||
AC_CHECK_FUNCS(getpwuid gethostname)
|
||||
AC_CHECK_FUNCS(getpwuid gethostname system)
|
||||
|
||||
AC_TRY_COMPILE([#include<sys/types.h>],
|
||||
[off64_t n = 0;],
|
||||
|
412
src/H5D.c
412
src/H5D.c
@ -63,7 +63,9 @@ const H5D_create_t H5D_create_dflt = {
|
||||
|
||||
/* Default dataset transfer property list */
|
||||
const H5D_xfer_t H5D_xfer_dflt = {
|
||||
0, /* Place holder - remove this later */
|
||||
1024*1024, /* Temporary buffer size */
|
||||
NULL, /* Type conversion buffer or NULL */
|
||||
NULL, /* Background buffer or NULL */
|
||||
};
|
||||
|
||||
/* Interface initialization? */
|
||||
@ -813,7 +815,7 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space,
|
||||
}
|
||||
|
||||
/* Create (open for write access) an object header */
|
||||
if (H5O_create(f, 80, &(new_dset->ent)) < 0) {
|
||||
if (H5O_create(f, 96, &(new_dset->ent)) < 0) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL,
|
||||
"unable to create dataset object header");
|
||||
}
|
||||
@ -1087,7 +1089,9 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
|
||||
const H5S_t *file_space, const H5D_xfer_t *xfer_parms,
|
||||
void *buf/*out*/)
|
||||
{
|
||||
size_t nelmts ; /*number of elements */
|
||||
size_t nelmts; /*number of elements */
|
||||
size_t smine_start; /*strip mine start loc */
|
||||
size_t smine_nelmts; /*elements per strip */
|
||||
uint8 *tconv_buf = NULL; /*data type conv buffer */
|
||||
uint8 *bkg_buf = NULL; /*background buffer */
|
||||
H5T_conv_t tconv_func = NULL; /*conversion function */
|
||||
@ -1097,6 +1101,11 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
|
||||
H5T_cdata_t *cdata = NULL; /*type conversion data */
|
||||
herr_t ret_value = FAIL;
|
||||
herr_t status;
|
||||
size_t src_type_size; /*size of source type */
|
||||
size_t dst_type_size; /*size of destination type*/
|
||||
size_t target_size; /*desired buffer size */
|
||||
size_t buffer_size; /*actual buffer size */
|
||||
size_t request_nelmts; /*requested strip mine */
|
||||
|
||||
FUNC_ENTER(H5D_read, FAIL);
|
||||
|
||||
@ -1107,124 +1116,154 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
|
||||
assert(buf);
|
||||
if (!file_space) file_space = dataset->space;
|
||||
if (!mem_space) mem_space = file_space;
|
||||
|
||||
/*
|
||||
* Convert data types to atoms because the conversion functions are
|
||||
* application-level functions.
|
||||
*/
|
||||
if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type))) < 0 ||
|
||||
(dst_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type))) < 0) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
|
||||
"unable to register types for conversion");
|
||||
}
|
||||
nelmts = H5S_get_npoints(mem_space);
|
||||
|
||||
/*
|
||||
* Locate the type conversion function and data space conversion
|
||||
* functions, and set up the element numbering information.
|
||||
* functions, and set up the element numbering information. If a data
|
||||
* type conversion is necessary then register data type atoms.
|
||||
*/
|
||||
if (NULL == (tconv_func = H5T_find(dataset->type, mem_type, &cdata))) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
|
||||
"unable to convert between src and dest data types");
|
||||
} else if (H5T_conv_noop!=tconv_func) {
|
||||
if ((src_id=H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0 ||
|
||||
(dst_id=H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
|
||||
"unable to register types for conversion");
|
||||
}
|
||||
}
|
||||
if (NULL==(sconv_func=H5S_find (mem_space, file_space))) {
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
|
||||
"unable to convert from file to memory data space");
|
||||
}
|
||||
if (sconv_func->init &&
|
||||
(sconv_func->init)(&(dataset->layout), mem_space, file_space,
|
||||
&numbering/*out*/)<=0) {
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
|
||||
"unable to initialize element numbering information");
|
||||
} else {
|
||||
HDmemset (&numbering, 0, sizeof numbering);
|
||||
}
|
||||
if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) {
|
||||
if (nelmts!=H5S_get_npoints (file_space)) {
|
||||
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
|
||||
"src and dest data spaces have different sizes");
|
||||
}
|
||||
|
||||
/*
|
||||
* Compute the size of the request and allocate scratch buffers.
|
||||
*/
|
||||
nelmts = H5S_get_npoints(mem_space);
|
||||
|
||||
/*
|
||||
* If there is no type conversion then try reading directly into the
|
||||
* application's buffer.
|
||||
* application's buffer. This saves at least one mem-to-mem copy.
|
||||
*/
|
||||
if (H5D_OPTIMIZE_PIPE &&
|
||||
H5T_conv_noop==tconv_func &&
|
||||
NULL!=sconv_func->read) {
|
||||
#ifndef NDEBUG
|
||||
fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n");
|
||||
#endif
|
||||
status = (sconv_func->read)(dataset->ent.file, &(dataset->layout),
|
||||
&(dataset->create_parms->efl),
|
||||
H5T_get_size (dataset->type), file_space,
|
||||
mem_space, buf/*out*/);
|
||||
if (status>=0) goto succeed;
|
||||
#ifndef NDEBUG
|
||||
fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n");
|
||||
#ifdef H5D_DEBUG
|
||||
fprintf (stderr, "HDF5-DIAG: input pipe optimization failed "
|
||||
"(falling through)\n");
|
||||
#endif
|
||||
H5E_clear ();
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* This is the general case.
|
||||
* This is the general case. Figure out the strip mine size.
|
||||
*/
|
||||
#ifndef LATER
|
||||
src_type_size = H5T_get_size(dataset->type);
|
||||
dst_type_size = H5T_get_size(mem_type);
|
||||
target_size = xfer_parms->buf_size;
|
||||
request_nelmts = target_size / MAX(src_type_size, dst_type_size);
|
||||
if (request_nelmts<=0) {
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
|
||||
"temporary buffer max size is too small");
|
||||
}
|
||||
if (sconv_func->init) {
|
||||
smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space,
|
||||
file_space, request_nelmts,
|
||||
&numbering/*out*/);
|
||||
if (smine_nelmts<=0) {
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
|
||||
"unable to initialize element numbering information");
|
||||
}
|
||||
} else {
|
||||
smine_nelmts = request_nelmts;
|
||||
HDmemset (&numbering, 0, sizeof numbering);
|
||||
}
|
||||
buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size);
|
||||
|
||||
/*
|
||||
* Note: this prototype version allocates a buffer large enough to
|
||||
* satisfy the entire request; strip mining is not implemented.
|
||||
* Get a temporary buffer for type conversion unless the app has already
|
||||
* supplied one through the xfer properties. Instead of allocating a
|
||||
* buffer which is the exact size, we allocate the target size. The
|
||||
* malloc() is usually less resource-intensive if we allocate/free the
|
||||
* same size over and over.
|
||||
*/
|
||||
{
|
||||
size_t src_size = nelmts * H5T_get_size(dataset->type);
|
||||
size_t dst_size = nelmts * H5T_get_size(mem_type);
|
||||
tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size));
|
||||
if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size);
|
||||
if (NULL==(tconv_buf=xfer_parms->tconv)) {
|
||||
tconv_buf = H5MM_xmalloc (target_size);
|
||||
}
|
||||
if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) {
|
||||
bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size);
|
||||
}
|
||||
|
||||
#ifdef H5D_DEBUG
|
||||
/* Strip mine diagnostics.... */
|
||||
if (smine_nelmts<nelmts) {
|
||||
fprintf (stderr, "HDF5-DIAG: strip mine");
|
||||
if (smine_nelmts!=request_nelmts) {
|
||||
fprintf (stderr, " got %lu of %lu",
|
||||
(unsigned long)smine_nelmts,
|
||||
(unsigned long)request_nelmts);
|
||||
}
|
||||
if (buffer_size!=target_size) {
|
||||
fprintf (stderr, " (%1.1f%% of buffer)",
|
||||
100.0*buffer_size/target_size);
|
||||
}
|
||||
fprintf (stderr, " %1.1f iterations\n",
|
||||
(double)nelmts/smine_nelmts);
|
||||
}
|
||||
#endif
|
||||
/*
|
||||
* Gather the data from disk into the data type conversion buffer. Also
|
||||
* gather data from application to background buffer (this step is not
|
||||
* needed for most conversions, but we leave that as an exercise for
|
||||
* later ;-)
|
||||
*/
|
||||
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
|
||||
&(dataset->create_parms->efl),
|
||||
H5T_get_size (dataset->type), file_space,
|
||||
&numbering, 0, nelmts,
|
||||
tconv_buf/*out*/)!=nelmts) {
|
||||
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed");
|
||||
}
|
||||
if (H5T_BKG_YES==cdata->need_bkg) {
|
||||
if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
|
||||
&numbering, 0, nelmts,
|
||||
bkg_buf/*out*/)!=nelmts) {
|
||||
HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed");
|
||||
|
||||
/* Start strip mining... */
|
||||
for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) {
|
||||
smine_nelmts = MIN (smine_nelmts, nelmts-smine_start);
|
||||
|
||||
/*
|
||||
* Gather the data from disk into the data type conversion
|
||||
* buffer. Also gather data from application to background buffer
|
||||
* if necessary.
|
||||
*/
|
||||
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
|
||||
&(dataset->create_parms->efl),
|
||||
H5T_get_size (dataset->type), file_space,
|
||||
&numbering, smine_start, smine_nelmts,
|
||||
tconv_buf/*out*/)!=smine_nelmts) {
|
||||
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed");
|
||||
}
|
||||
if (H5T_BKG_YES==cdata->need_bkg) {
|
||||
if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
|
||||
&numbering, smine_start, smine_nelmts,
|
||||
bkg_buf/*out*/)!=smine_nelmts) {
|
||||
HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform data type conversion.
|
||||
*/
|
||||
cdata->command = H5T_CONV_CONV;
|
||||
cdata->ncalls++;
|
||||
if ((tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf,
|
||||
bkg_buf)<0) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
|
||||
"data type conversion failed");
|
||||
}
|
||||
cdata->nelmts += smine_nelmts;
|
||||
|
||||
/*
|
||||
* Scatter the data into memory.
|
||||
*/
|
||||
if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space,
|
||||
&numbering, smine_start, smine_nelmts,
|
||||
buf/*out*/)<0) {
|
||||
HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform data type conversion.
|
||||
*/
|
||||
cdata->command = H5T_CONV_CONV;
|
||||
cdata->ncalls++;
|
||||
if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
|
||||
"data type conversion failed");
|
||||
}
|
||||
cdata->nelmts += nelmts;
|
||||
|
||||
/*
|
||||
* Scatter the data into memory.
|
||||
*/
|
||||
if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space,
|
||||
&numbering, 0, nelmts, buf/*out*/)<0) {
|
||||
HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed");
|
||||
}
|
||||
|
||||
|
||||
succeed:
|
||||
ret_value = SUCCEED;
|
||||
@ -1232,8 +1271,12 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
|
||||
done:
|
||||
if (src_id >= 0) H5A_dec_ref(src_id);
|
||||
if (dst_id >= 0) H5A_dec_ref(dst_id);
|
||||
tconv_buf = H5MM_xfree(tconv_buf);
|
||||
bkg_buf = H5MM_xfree (bkg_buf);
|
||||
if (tconv_buf && NULL==xfer_parms->tconv) {
|
||||
H5MM_xfree(tconv_buf);
|
||||
}
|
||||
if (bkg_buf && NULL==xfer_parms->bkg) {
|
||||
H5MM_xfree (bkg_buf);
|
||||
}
|
||||
FUNC_LEAVE(ret_value);
|
||||
}
|
||||
|
||||
@ -1260,7 +1303,9 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
|
||||
const H5S_t *file_space, const H5D_xfer_t *xfer_parms,
|
||||
const void *buf)
|
||||
{
|
||||
size_t nelmts;
|
||||
size_t nelmts; /*total number of elmts */
|
||||
size_t smine_start; /*strip mine start loc */
|
||||
size_t smine_nelmts; /*elements per strip */
|
||||
uint8 *tconv_buf = NULL; /*data type conv buffer */
|
||||
uint8 *bkg_buf = NULL; /*background buffer */
|
||||
H5T_conv_t tconv_func = NULL; /*conversion function */
|
||||
@ -1269,6 +1314,11 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
|
||||
H5S_number_t numbering; /*element numbering info*/
|
||||
H5T_cdata_t *cdata = NULL; /*type conversion data */
|
||||
herr_t ret_value = FAIL, status;
|
||||
size_t src_type_size; /*size of source type */
|
||||
size_t dst_type_size; /*size of destination type*/
|
||||
size_t target_size; /*desired buffer size */
|
||||
size_t buffer_size; /*actual buffer size */
|
||||
size_t request_nelmts; /*requested strip mine */
|
||||
|
||||
FUNC_ENTER(H5D_write, FAIL);
|
||||
|
||||
@ -1279,47 +1329,32 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
|
||||
assert(buf);
|
||||
if (!file_space) file_space = dataset->space;
|
||||
if (!mem_space) mem_space = file_space;
|
||||
nelmts = H5S_get_npoints(mem_space);
|
||||
|
||||
/*
|
||||
* Convert data types to atoms because the conversion functions are
|
||||
* application-level functions.
|
||||
*/
|
||||
if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 ||
|
||||
(dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
|
||||
"unable to register types for conversion");
|
||||
}
|
||||
|
||||
/*
|
||||
* Locate the type conversion function and data space conversion
|
||||
* functions, and set up the element numbering information.
|
||||
* functions, and set up the element numbering information. If a data
|
||||
* type conversion is necessary then register data type atoms.
|
||||
*/
|
||||
if (NULL == (tconv_func = H5T_find(mem_type, dataset->type, &cdata))) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
|
||||
"unable to convert between src and dest data types");
|
||||
} else if (H5T_conv_noop!=tconv_func) {
|
||||
if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 ||
|
||||
(dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
|
||||
"unable to register types for conversion");
|
||||
}
|
||||
}
|
||||
if (NULL==(sconv_func=H5S_find (mem_space, file_space))) {
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
|
||||
"unable to convert from memory to file data space");
|
||||
}
|
||||
if (sconv_func->init &&
|
||||
(sconv_func->init)(&(dataset->layout), mem_space, file_space,
|
||||
&numbering/*out*/)<=0) {
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
|
||||
"unable to initialize element numbering information");
|
||||
} else {
|
||||
HDmemset (&numbering, 0, sizeof numbering);
|
||||
}
|
||||
if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) {
|
||||
if (nelmts!=H5S_get_npoints (file_space)) {
|
||||
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
|
||||
"src and dest data spaces have different sizes");
|
||||
}
|
||||
|
||||
/*
|
||||
* Compute the size of the request and allocate scratch buffers.
|
||||
*/
|
||||
nelmts = H5S_get_npoints(mem_space);
|
||||
|
||||
|
||||
/*
|
||||
* If there is no type conversion then try writing directly from
|
||||
* application buffer to file.
|
||||
@ -1327,87 +1362,140 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
|
||||
if (H5D_OPTIMIZE_PIPE &&
|
||||
H5T_conv_noop==tconv_func &&
|
||||
NULL!=sconv_func->write) {
|
||||
#ifndef NDEBUG
|
||||
fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n");
|
||||
#endif
|
||||
status = (sconv_func->write)(dataset->ent.file, &(dataset->layout),
|
||||
&(dataset->create_parms->efl),
|
||||
H5T_get_size (dataset->type), file_space,
|
||||
mem_space, buf);
|
||||
if (status>=0) goto succeed;
|
||||
#ifndef NDEBUG
|
||||
fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n");
|
||||
#ifdef H5D_DEBUG
|
||||
fprintf (stderr, "HDF5-DIAG: output pipe optimization failed "
|
||||
"(falling through)\n");
|
||||
#endif
|
||||
H5E_clear ();
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* This is the general case.
|
||||
* This is the general case. Figure out the strip mine size.
|
||||
*/
|
||||
#ifndef LATER
|
||||
src_type_size = H5T_get_size(mem_type);
|
||||
dst_type_size = H5T_get_size(dataset->type);
|
||||
target_size = xfer_parms->buf_size;
|
||||
request_nelmts = target_size / MAX (src_type_size, dst_type_size);
|
||||
if (request_nelmts<=0) {
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
|
||||
"temporary buffer max size is too small");
|
||||
}
|
||||
if (sconv_func->init) {
|
||||
smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space,
|
||||
file_space, request_nelmts,
|
||||
&numbering/*out*/);
|
||||
if (smine_nelmts<=0) {
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
|
||||
"unable to initialize element numbering information");
|
||||
}
|
||||
} else {
|
||||
smine_nelmts = request_nelmts;
|
||||
HDmemset (&numbering, 0, sizeof numbering);
|
||||
}
|
||||
buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size);
|
||||
|
||||
/*
|
||||
* Note: This prototype version allocates a buffer large enough to
|
||||
* satisfy the entire request; strip mining is not implemented.
|
||||
* Get a temporary buffer for type conversion unless the app has already
|
||||
* supplied one through the xfer properties. Instead of allocating a
|
||||
* buffer which is the exact size, we allocate the target size. The
|
||||
* malloc() is usually less resource-intensive if we allocate/free the
|
||||
* same size over and over.
|
||||
*/
|
||||
{
|
||||
size_t src_size = nelmts * H5T_get_size(mem_type);
|
||||
size_t dst_size = nelmts * H5T_get_size(dataset->type);
|
||||
tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size));
|
||||
if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size);
|
||||
if (NULL==(tconv_buf=xfer_parms->tconv)) {
|
||||
tconv_buf = H5MM_xmalloc (target_size);
|
||||
}
|
||||
if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) {
|
||||
bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size);
|
||||
}
|
||||
|
||||
#ifdef H5D_DEBUG
|
||||
/* Strip mine diagnostics.... */
|
||||
if (smine_nelmts<nelmts) {
|
||||
fprintf (stderr, "HDF5-DIAG: strip mine");
|
||||
if (smine_nelmts!=request_nelmts) {
|
||||
fprintf (stderr, " got %lu of %lu",
|
||||
(unsigned long)smine_nelmts,
|
||||
(unsigned long)request_nelmts);
|
||||
}
|
||||
if (buffer_size!=target_size) {
|
||||
fprintf (stderr, " (%1.1f%% of buffer)",
|
||||
100.0*buffer_size/target_size);
|
||||
}
|
||||
fprintf (stderr, " %1.1f iterations\n",
|
||||
(double)nelmts/smine_nelmts);
|
||||
}
|
||||
#endif
|
||||
/*
|
||||
* Gather data from application buffer into the data type conversion
|
||||
* buffer. Also gather data from the file into the background buffer
|
||||
* (this step is not needed for most conversions, but we leave that as an
|
||||
* exercise for later ;-)
|
||||
*/
|
||||
if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
|
||||
&numbering, 0, nelmts, tconv_buf/*out*/)!=nelmts) {
|
||||
HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed");
|
||||
}
|
||||
if (H5T_BKG_YES==cdata->need_bkg) {
|
||||
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
|
||||
|
||||
/* Start strip mining... */
|
||||
for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) {
|
||||
smine_nelmts = MIN (smine_nelmts, nelmts-smine_start);
|
||||
|
||||
/*
|
||||
* Gather data from application buffer into the data type conversion
|
||||
* buffer. Also gather data from the file into the background buffer
|
||||
* if necessary.
|
||||
*/
|
||||
if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
|
||||
&numbering, smine_start, smine_nelmts,
|
||||
tconv_buf/*out*/)!=smine_nelmts) {
|
||||
HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed");
|
||||
}
|
||||
if (H5T_BKG_YES==cdata->need_bkg) {
|
||||
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
|
||||
&(dataset->create_parms->efl),
|
||||
H5T_get_size (dataset->type), file_space,
|
||||
&numbering, smine_start, smine_nelmts,
|
||||
bkg_buf/*out*/)!=smine_nelmts) {
|
||||
HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
|
||||
"file gather failed");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform data type conversion.
|
||||
*/
|
||||
cdata->command = H5T_CONV_CONV;
|
||||
cdata->ncalls++;
|
||||
if ((tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf,
|
||||
bkg_buf)<0) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
|
||||
"data type conversion failed");
|
||||
}
|
||||
cdata->nelmts += smine_nelmts;
|
||||
|
||||
/*
|
||||
* Scatter the data out to the file.
|
||||
*/
|
||||
if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
|
||||
&(dataset->create_parms->efl),
|
||||
H5T_get_size (dataset->type), file_space,
|
||||
&numbering, 0, nelmts,
|
||||
bkg_buf/*out*/)!=nelmts) {
|
||||
HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed");
|
||||
&numbering, smine_start, smine_nelmts,
|
||||
tconv_buf)<0) {
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform data type conversion.
|
||||
*/
|
||||
cdata->command = H5T_CONV_CONV;
|
||||
cdata->ncalls++;
|
||||
if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) {
|
||||
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
|
||||
"data type conversion failed");
|
||||
}
|
||||
cdata->nelmts += nelmts;
|
||||
|
||||
/*
|
||||
* Scatter the data out to the file.
|
||||
*/
|
||||
if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
|
||||
&(dataset->create_parms->efl),
|
||||
H5T_get_size (dataset->type), file_space,
|
||||
&numbering, 0, nelmts, tconv_buf)<0) {
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
|
||||
}
|
||||
|
||||
succeed:
|
||||
ret_value = SUCCEED;
|
||||
|
||||
done:
|
||||
if (src_id >= 0) H5A_dec_ref(src_id);
|
||||
if (dst_id >= 0) H5A_dec_ref(dst_id);
|
||||
tconv_buf = H5MM_xfree(tconv_buf);
|
||||
bkg_buf = H5MM_xfree (bkg_buf);
|
||||
if (tconv_buf && NULL==xfer_parms->tconv) {
|
||||
H5MM_xfree(tconv_buf);
|
||||
}
|
||||
if (bkg_buf && NULL==xfer_parms->bkg) {
|
||||
H5MM_xfree (bkg_buf);
|
||||
}
|
||||
FUNC_LEAVE(ret_value);
|
||||
}
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
* Function: H5D_extend
|
||||
|
@ -26,6 +26,15 @@
|
||||
#include <H5Sprivate.h> /*for the H5S_t type */
|
||||
#include <H5Oprivate.h> /*object Headers */
|
||||
|
||||
/*
|
||||
* Feature: Define H5D_DEBUG on the compiler command line if you want to
|
||||
* debug dataset I/O. NDEBUG must not be defined in order for this
|
||||
* to have any effect.
|
||||
*/
|
||||
#ifdef NDEBUG
|
||||
# undef H5D_DEBUG
|
||||
#endif
|
||||
|
||||
#define H5D_RESERVED_ATOMS 0
|
||||
|
||||
/* Set the minimum object header size to create objects with */
|
||||
@ -41,8 +50,11 @@ typedef struct H5D_create_t {
|
||||
|
||||
/* Dataset transfer property list */
|
||||
typedef struct H5D_xfer_t {
|
||||
int _placeholder; /*unused--delete this later */
|
||||
size_t buf_size; /*max temp buffer size */
|
||||
void *tconv; /*type conversion buffer or null */
|
||||
void *bkg; /*background buffer or null */
|
||||
} H5D_xfer_t;
|
||||
|
||||
typedef struct H5D_t H5D_t;
|
||||
extern const H5D_create_t H5D_create_dflt;
|
||||
extern const H5D_xfer_t H5D_xfer_dflt;
|
||||
|
94
src/H5P.c
94
src/H5P.c
@ -1677,6 +1677,100 @@ H5Pget_family (hid_t tid, hid_t *memb_tid)
|
||||
FUNC_LEAVE (SUCCEED);
|
||||
}
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
* Function: H5Pset_buffer
|
||||
*
|
||||
* Purpose: Given a dataset transfer property list, set the maximum size
|
||||
* for the type conversion buffer and background buffer and
|
||||
* optionally supply pointers to application-allocated buffers.
|
||||
* If the buffer size is smaller than the entire amount of data
|
||||
* being transfered between application and file, and a type
|
||||
* conversion buffer or background buffer is required then
|
||||
* strip mining will be used. However, certain restrictions
|
||||
* apply for the size of buffer which can be used for strip
|
||||
* mining. For instance, when strip mining a 100x200x300
|
||||
* hyperslab of a simple data space the buffer must be large
|
||||
* enough to hold a 1x200x300 slab.
|
||||
*
|
||||
* If TCONV and/or BKG are null pointers then buffers will be
|
||||
* allocated and freed during the data transfer.
|
||||
*
|
||||
* Return: Success: SUCCEED
|
||||
*
|
||||
* Failure: FAIL
|
||||
*
|
||||
* Programmer: Robb Matzke
|
||||
* Monday, March 16, 1998
|
||||
*
|
||||
* Modifications:
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
herr_t
|
||||
H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg)
|
||||
{
|
||||
H5D_xfer_t *plist = NULL;
|
||||
|
||||
FUNC_ENTER (H5Pset_buffer, FAIL);
|
||||
|
||||
/* Check arguments */
|
||||
if (H5P_DATASET_XFER != H5Pget_class (plist_id) ||
|
||||
NULL == (plist = H5A_object (plist_id))) {
|
||||
HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
|
||||
"not a dataset transfer property list");
|
||||
}
|
||||
if (size<=0) {
|
||||
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
|
||||
"buffer size must not be zero");
|
||||
}
|
||||
|
||||
/* Update property list */
|
||||
plist->buf_size = size;
|
||||
plist->tconv = tconv;
|
||||
plist->bkg = bkg;
|
||||
|
||||
FUNC_LEAVE (SUCCEED);
|
||||
}
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
* Function: H5Pget_buffer
|
||||
*
|
||||
* Purpose: Reads values previously set with H5Pset_buffer().
|
||||
*
|
||||
* Return: Success: Buffer size.
|
||||
*
|
||||
* Failure: 0
|
||||
*
|
||||
* Programmer: Robb Matzke
|
||||
* Monday, March 16, 1998
|
||||
*
|
||||
* Modifications:
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
size_t
|
||||
H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/)
|
||||
{
|
||||
H5D_xfer_t *plist = NULL;
|
||||
|
||||
FUNC_ENTER (H5Pget_buffer, 0);
|
||||
|
||||
/* Check arguments */
|
||||
if (H5P_DATASET_XFER != H5Pget_class (plist_id) ||
|
||||
NULL == (plist = H5A_object (plist_id))) {
|
||||
HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, 0,
|
||||
"not a dataset transfer property list");
|
||||
}
|
||||
|
||||
/* Return values */
|
||||
if (tconv) *tconv = plist->tconv;
|
||||
if (bkg) *bkg = plist->bkg;
|
||||
|
||||
FUNC_LEAVE (plist->buf_size);
|
||||
}
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
* Function: H5Pset_mpi
|
||||
|
@ -83,8 +83,12 @@ herr_t H5Pget_split (hid_t tid, size_t meta_ext_size, char *meta_ext/*out*/,
|
||||
|
||||
herr_t H5Pset_family (hid_t tid, hid_t memb_tid);
|
||||
herr_t H5Pget_family (hid_t tid, hid_t *memb_tid/*out*/);
|
||||
herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg);
|
||||
size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/);
|
||||
|
||||
#ifdef HAVE_PARALLEL
|
||||
herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info, unsigned access_mode);
|
||||
herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info,
|
||||
unsigned access_mode);
|
||||
herr_t H5Pget_mpi (hid_t tid, MPI_Comm *comm/*out*/, MPI_Info *info/*out*/,
|
||||
unsigned *access_mode/*out*/);
|
||||
#endif
|
||||
|
@ -65,7 +65,8 @@ typedef struct H5S_number_t {
|
||||
typedef struct H5S_tconv_t {
|
||||
/* Initialize element numbering information */
|
||||
size_t (*init)(const struct H5O_layout_t *layout, const H5S_t *mem_space,
|
||||
const H5S_t *file_space, H5S_number_t *numbering/*out*/);
|
||||
const H5S_t *file_space, size_t desired_nelmts,
|
||||
H5S_number_t *numbering/*out*/);
|
||||
|
||||
/* Gather elements from disk to type conversion buffer */
|
||||
size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout,
|
||||
@ -122,7 +123,7 @@ intn H5S_extend (H5S_t *space, const size_t *size);
|
||||
/* Conversion functions for simple data spaces */
|
||||
size_t H5S_simp_init (const struct H5O_layout_t *layout,
|
||||
const H5S_t *mem_space, const H5S_t *file_space,
|
||||
H5S_number_t *numbering/*out*/);
|
||||
size_t desired_nelmts, H5S_number_t *numbering/*out*/);
|
||||
size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
|
||||
const struct H5O_efl_t *efl, size_t elmt_size,
|
||||
const H5S_t *file_space, const H5S_number_t *numbering,
|
||||
|
121
src/H5Ssimp.c
121
src/H5Ssimp.c
@ -37,9 +37,14 @@ static intn interface_initialize_g = FALSE;
|
||||
*/
|
||||
size_t
|
||||
H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
|
||||
const H5S_t *file_space, H5S_number_t *numbering/*out*/)
|
||||
const H5S_t *file_space, size_t desired_nelmts,
|
||||
H5S_number_t *numbering/*out*/)
|
||||
{
|
||||
size_t nelmts;
|
||||
int m_ndims, f_ndims; /*mem, file dimensionality */
|
||||
size_t size[H5O_LAYOUT_NDIMS]; /*size of selected hyperslab */
|
||||
size_t acc;
|
||||
int i;
|
||||
|
||||
FUNC_ENTER (H5S_simp_init, 0);
|
||||
|
||||
@ -49,11 +54,43 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
|
||||
assert (file_space && H5S_SIMPLE==file_space->type);
|
||||
assert (numbering);
|
||||
|
||||
/* Numbering is implied by the hyperslab, C order */
|
||||
/* Numbering is implied by the hyperslab, C order, no data here */
|
||||
HDmemset (numbering, 0, sizeof(H5S_number_t));
|
||||
|
||||
/* Data can be efficiently copied at any size */
|
||||
nelmts = H5S_get_npoints (file_space);
|
||||
/*
|
||||
* The stripmine size is such that only the slowest varying dimension can
|
||||
* be split up. We choose the largest possible strip mine size which is
|
||||
* not larger than the desired size.
|
||||
*/
|
||||
m_ndims = H5S_get_hyperslab (mem_space, NULL, size, NULL);
|
||||
for (i=m_ndims-1, acc=1; i>0; --i) acc *= size[i];
|
||||
nelmts = (desired_nelmts/acc) * acc;
|
||||
if (nelmts<=0) {
|
||||
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
|
||||
"strip mine buffer is too small");
|
||||
}
|
||||
|
||||
/*
|
||||
* The value chosen for mem_space must be the same as the value chosen for
|
||||
* file_space.
|
||||
*/
|
||||
f_ndims = H5S_get_hyperslab (file_space, NULL, size, NULL);
|
||||
if (m_ndims!=f_ndims) {
|
||||
nelmts = H5S_get_npoints (file_space);
|
||||
if (nelmts>desired_nelmts) {
|
||||
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
|
||||
"strip mining not supported across "
|
||||
"dimensionalities");
|
||||
}
|
||||
assert (nelmts==H5S_get_npoints (mem_space));
|
||||
} else {
|
||||
for (i=f_ndims-1, acc=1; i>0; --i) acc *= size[i];
|
||||
acc *= (desired_nelmts/acc);
|
||||
if (nelmts!=acc) {
|
||||
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
|
||||
"unsupported strip mine size for shape change");
|
||||
}
|
||||
}
|
||||
|
||||
FUNC_LEAVE (nelmts);
|
||||
}
|
||||
@ -95,6 +132,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
|
||||
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
|
||||
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
|
||||
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
|
||||
size_t acc; /*accumulator */
|
||||
#ifndef LATER
|
||||
intn file_offset_signed[H5O_LAYOUT_NDIMS];
|
||||
#endif
|
||||
@ -112,12 +150,6 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
|
||||
assert (nelmts>0);
|
||||
assert (buf);
|
||||
|
||||
/*
|
||||
* The prototype doesn't support strip mining.
|
||||
*/
|
||||
assert (0==start);
|
||||
assert (nelmts==H5S_get_npoints (file_space));
|
||||
|
||||
/*
|
||||
* Get hyperslab information to determine what elements are being
|
||||
* selected (there might eventually be other selection methods too).
|
||||
@ -132,6 +164,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
|
||||
"unable to retrieve hyperslab parameters");
|
||||
}
|
||||
#else
|
||||
/* Argument type problems to be fixed later..... -RPM */
|
||||
if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed,
|
||||
hsize, sample))<0) {
|
||||
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0,
|
||||
@ -142,12 +175,23 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
|
||||
file_offset[i] = file_offset_signed[i];
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Check that there is no subsampling of the hyperslab */
|
||||
for (i=0; i<space_ndims; i++) {
|
||||
if (sample[i]!=1) {
|
||||
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0,
|
||||
"hyperslab sampling is not implemented yet");
|
||||
}
|
||||
}
|
||||
|
||||
/* Adjust the slowest varying dimension to take care of strip mining */
|
||||
for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
|
||||
assert (0==start % acc);
|
||||
assert (0==nelmts % acc);
|
||||
file_offset[0] += start / acc;
|
||||
hsize[0] = nelmts / acc;
|
||||
|
||||
/* The fastest varying dimension is for the data point itself */
|
||||
file_offset[space_ndims] = 0;
|
||||
hsize[space_ndims] = elmt_size;
|
||||
HDmemset (zero, 0, layout->ndims*sizeof(size_t));
|
||||
@ -194,6 +238,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
|
||||
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
|
||||
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
|
||||
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
|
||||
size_t acc; /*accumulator */
|
||||
#ifndef LATER
|
||||
intn mem_offset_signed[H5O_LAYOUT_NDIMS];
|
||||
#endif
|
||||
@ -210,12 +255,6 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
|
||||
assert (nelmts>0);
|
||||
assert (buf);
|
||||
|
||||
/*
|
||||
* The prototype doesn't support strip mining.
|
||||
*/
|
||||
assert (0==start);
|
||||
assert (nelmts==H5S_get_npoints (mem_space));
|
||||
|
||||
/*
|
||||
* Retrieve hyperslab information to determine what elements are being
|
||||
* selected (there might be other selection methods in the future). We
|
||||
@ -229,6 +268,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
|
||||
"unable to retrieve hyperslab parameters");
|
||||
}
|
||||
#else
|
||||
/* Argument type problems to be fixed later..... -RPM */
|
||||
if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed,
|
||||
hsize, sample))<0) {
|
||||
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
|
||||
@ -239,6 +279,8 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
|
||||
mem_offset[i] = mem_offset_signed[i];
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Check that there is no subsampling of the hyperslab */
|
||||
for (i=0; i<space_ndims; i++) {
|
||||
if (sample[i]!=1) {
|
||||
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
|
||||
@ -249,6 +291,15 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
|
||||
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
|
||||
"unable to retrieve data space dimensions");
|
||||
}
|
||||
|
||||
/* Adjust the slowest varying dimension to take care of strip mining */
|
||||
for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
|
||||
assert (0==start % acc);
|
||||
assert (0==nelmts % acc);
|
||||
mem_offset[0] += start / acc;
|
||||
hsize[0] = nelmts / acc;
|
||||
|
||||
/* The fastest varying dimension is for the data point itself */
|
||||
mem_offset[space_ndims] = 0;
|
||||
mem_size[space_ndims] = elmt_size;
|
||||
hsize[space_ndims] = elmt_size;
|
||||
@ -299,6 +350,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
|
||||
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
|
||||
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
|
||||
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
|
||||
size_t acc; /*accumulator */
|
||||
#ifndef LATER
|
||||
intn mem_offset_signed[H5O_LAYOUT_NDIMS];
|
||||
#endif
|
||||
@ -315,12 +367,6 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
|
||||
assert (nelmts>0);
|
||||
assert (tconv_buf);
|
||||
|
||||
/*
|
||||
* The prototype doesn't support strip mining.
|
||||
*/
|
||||
assert (0==start);
|
||||
assert (nelmts==H5S_get_npoints (mem_space));
|
||||
|
||||
/*
|
||||
* Retrieve hyperslab information to determine what elements are being
|
||||
* selected (there might be other selection methods in the future). We
|
||||
@ -334,6 +380,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
|
||||
"unable to retrieve hyperslab parameters");
|
||||
}
|
||||
#else
|
||||
/* Argument type problems to be fixed later..... -RPM */
|
||||
if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed,
|
||||
hsize, sample))<0) {
|
||||
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
|
||||
@ -344,6 +391,8 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
|
||||
mem_offset[i] = mem_offset_signed[i];
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Check that there is no subsampling of the hyperslab */
|
||||
for (i=0; i<space_ndims; i++) {
|
||||
if (sample[i]!=1) {
|
||||
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0,
|
||||
@ -354,6 +403,15 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
|
||||
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0,
|
||||
"unable to retrieve data space dimensions");
|
||||
}
|
||||
|
||||
/* Adjust the slowest varying dimension to account for strip mining */
|
||||
for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
|
||||
assert (0==start % acc);
|
||||
assert (0==nelmts % acc);
|
||||
mem_offset[0] += start / acc;
|
||||
hsize[0] = nelmts / acc;
|
||||
|
||||
/* The fastest varying dimension is for the data point itself */
|
||||
mem_offset[space_ndims] = 0;
|
||||
mem_size[space_ndims] = elmt_size;
|
||||
hsize[space_ndims] = elmt_size;
|
||||
@ -404,6 +462,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
|
||||
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
|
||||
size_t zero[H5O_LAYOUT_NDIMS]; /*zero vector */
|
||||
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
|
||||
size_t acc; /*accumulator */
|
||||
#ifndef LATER
|
||||
intn file_offset_signed[H5O_LAYOUT_NDIMS];
|
||||
#endif
|
||||
@ -421,12 +480,6 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
|
||||
assert (nelmts>0);
|
||||
assert (buf);
|
||||
|
||||
/*
|
||||
* The prototype doesn't support strip mining.
|
||||
*/
|
||||
assert (0==start);
|
||||
assert (nelmts==H5S_get_npoints (file_space));
|
||||
|
||||
/*
|
||||
* Get hyperslab information to determine what elements are being
|
||||
* selected (there might eventually be other selection methods too).
|
||||
@ -441,6 +494,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
|
||||
"unable to retrieve hyperslab parameters");
|
||||
}
|
||||
#else
|
||||
/* Argument type problems to be fixed later..... -RPM */
|
||||
if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed,
|
||||
hsize, sample))<0) {
|
||||
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
|
||||
@ -451,12 +505,23 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
|
||||
file_offset[i] = file_offset_signed[i];
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Check that there is no subsampling of the hyperslab */
|
||||
for (i=0; i<space_ndims; i++) {
|
||||
if (sample[i]!=1) {
|
||||
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
|
||||
"hyperslab sampling is not implemented yet");
|
||||
}
|
||||
}
|
||||
|
||||
/* Adjust the slowest varying dimension to account for strip mining */
|
||||
for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
|
||||
assert (0==start % acc);
|
||||
assert (0==nelmts % acc);
|
||||
file_offset[0] += start / acc;
|
||||
hsize[0] = nelmts / acc;
|
||||
|
||||
/* The fastest varying dimension is for the data point itself */
|
||||
file_offset[space_ndims] = 0;
|
||||
hsize[space_ndims] = elmt_size;
|
||||
HDmemset (zero, 0, layout->ndims*sizeof(size_t));
|
||||
|
@ -11,13 +11,14 @@ CPPFLAGS=-I. -I../src @CPPFLAGS@
|
||||
|
||||
# These are our main targets. They should be listed in the order to be
|
||||
# executed, generally most specific tests to least specific tests.
|
||||
PROGS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external \
|
||||
PROGS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external \
|
||||
iopipe
|
||||
TESTS=$(PROGS)
|
||||
TESTS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external
|
||||
TIMINGS=iopipe
|
||||
|
||||
# Temporary files
|
||||
MOSTLYCLEAN=cmpd_dset.h5 dataset.h5 extend.h5 istore.h5 tfile1.h5 tfile2.h5 \
|
||||
tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \
|
||||
tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \
|
||||
extern_1.h5 extern_2.h5 extern_3.h5 extern_1.raw extern_1b.raw \
|
||||
extern_2.raw extern_2b.raw extern_3.raw extern_3b.raw \
|
||||
extern_4.raw extern_4b.raw iopipe.raw iopipe.h5
|
||||
@ -64,6 +65,16 @@ IOPIPE_OBJ=$(IOPIPE_SRC:.c=.o)
|
||||
# Private header files (not to be installed)...
|
||||
PRIVATE_HDR=testhdf5.h
|
||||
|
||||
# Additional targets
|
||||
.PHONY: timings _timings
|
||||
timings _timings: $(TIMINGS)
|
||||
@for timing in $(TIMINGS) dummy; do \
|
||||
if test $$timing != dummy; then \
|
||||
echo "Running $$timing $(TEST_FLAGS)"; \
|
||||
$(RUNTEST) ./$$timing $(TEST_FLAGS) || exit 1; \
|
||||
fi; \
|
||||
done;
|
||||
|
||||
# How to build the programs...
|
||||
testhdf5: $(TESTHDF5_OBJ) ../src/libhdf5.a
|
||||
$(CC) $(CFLAGS) -o $@ $(TESTHDF5_OBJ) ../src/libhdf5.a $(LIBS)
|
||||
|
22
test/dsets.c
22
test/dsets.c
@ -184,11 +184,12 @@ test_create(hid_t file)
|
||||
static herr_t
|
||||
test_simple_io(hid_t file)
|
||||
{
|
||||
hid_t dataset, space;
|
||||
herr_t status;
|
||||
int points[100][200], check[100][200];
|
||||
int i, j, n;
|
||||
size_t dims[2];
|
||||
hid_t dataset, space, xfer;
|
||||
herr_t status;
|
||||
int points[100][200], check[100][200];
|
||||
int i, j, n;
|
||||
size_t dims[2];
|
||||
void *tconv_buf = NULL;
|
||||
|
||||
printf("%-70s", "Testing simple I/O");
|
||||
|
||||
@ -205,6 +206,13 @@ test_simple_io(hid_t file)
|
||||
space = H5Screate_simple(2, dims, NULL);
|
||||
assert(space>=0);
|
||||
|
||||
/* Create a small conversion buffer to test strip mining */
|
||||
tconv_buf = malloc (1000);
|
||||
xfer = H5Pcreate (H5P_DATASET_XFER);
|
||||
assert (xfer>=0);
|
||||
status = H5Pset_buffer (xfer, 1000, tconv_buf, NULL);
|
||||
assert (status>=0);
|
||||
|
||||
/* Create the dataset */
|
||||
dataset = H5Dcreate(file, DSET_SIMPLE_IO_NAME, H5T_NATIVE_INT, space,
|
||||
H5P_DEFAULT);
|
||||
@ -212,12 +220,12 @@ test_simple_io(hid_t file)
|
||||
|
||||
/* Write the data to the dataset */
|
||||
status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
|
||||
H5P_DEFAULT, points);
|
||||
xfer, points);
|
||||
if (status<0) goto error;
|
||||
|
||||
/* Read the dataset back */
|
||||
status = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
|
||||
H5P_DEFAULT, check);
|
||||
xfer, check);
|
||||
if (status<0) goto error;
|
||||
|
||||
/* Check that the values read are the same as the values written */
|
||||
|
@ -19,12 +19,22 @@
|
||||
|
||||
#define RAW_FILE_NAME "iopipe.raw"
|
||||
#define HDF5_FILE_NAME "iopipe.h5"
|
||||
#define HEADING "%-16s"
|
||||
#define PROGRESS '='
|
||||
|
||||
#if 1
|
||||
/* Normal testing */
|
||||
#define REQUEST_SIZE_X 4579
|
||||
#define REQUEST_SIZE_Y 4579
|
||||
#define NREAD_REQUESTS 45
|
||||
#define NWRITE_REQUESTS 45
|
||||
#define HEADING "%-16s"
|
||||
#define PROGRESS '='
|
||||
#else
|
||||
/* Speedy testing */
|
||||
#define REQUEST_SIZE_X 1000
|
||||
#define REQUEST_SIZE_Y 1000
|
||||
#define NREAD_REQUESTS 45
|
||||
#define NWRITE_REQUESTS 45
|
||||
#endif
|
||||
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
@ -83,10 +93,17 @@ print_stats (const char *prefix,
|
||||
static void
|
||||
synchronize (void)
|
||||
{
|
||||
#ifdef HAVE_SYSTEM
|
||||
system ("sync");
|
||||
system ("df >/dev/null");
|
||||
#if 0
|
||||
system ("/sbin/swapout 130");
|
||||
/*
|
||||
* This works well on Linux to get rid of all cached disk buffers. The
|
||||
* number should be approximately the amount of RAM in MB. Do not
|
||||
* include swap space in that amount or the command will fail.
|
||||
*/
|
||||
system ("/sbin/swapout 128");
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user