[svn-r322] Changes since 19980313

----------------------

./configure.in
./configure
./test/iopipe.c
	Added check for system(3)

./config/freebsd2.2.1
./config/linux
	Added -DH5D_DEBUG to the debug flags.

./src/H5D.c
./src/H5Dprivate.h
./src/H5P.c
./src/H5Ppublic.h
./src/H5Sprivate.h
./src/H5Ssimp.c
./html/Datasets.html
	Finally implemented strip mining in the I/O pipeline, placing
	a user-defined upper bound on the amount of temporary memory
	used by the pipeline.  The default is 1MB type conversion and
	background buffers allocated/freed on demand.  However, the
	size can be changed and/or application-allocated buffers
	supplied with H5Pset_buffers() called on the data transfer
	property list passed to H5Dread() and H5Dwrite().

	Minor optimizations to H5Dread() and H5Dwrite().  More coming
	later...

./test/dsets.c
	Added calls to H5Pset_buffer() to test application-defined
	temporary buffers in the I/O pipeline.

./test/Makefile.in
	Removed `iopipe' from the list of confidence tests.  Saying
	`make timings' in the test directory runs timing tests.  I did
	this because (1) they don't really test anything new, and (2)
	they can take a long time to run.
This commit is contained in:
Robb Matzke 1998-03-16 20:29:54 -05:00
parent 48e0751106
commit 31a709a6b2
13 changed files with 511 additions and 211 deletions

View File

@ -41,7 +41,7 @@ warn="-Wall -Wshadow -Wpointer-arith -Wcast-qual -Wwrite-strings -Wstrict-protot
profile="-pg"
debug="-g -DH5AC_DEBUG -DH5B_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -fverbose-asm"
debug="-g -DH5AC_DEBUG -DH5B_DEBUG -DH5D_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -fverbose-asm"
production="-O3 -DNDEBUG -finline-functions -funroll-loops -malign-double -fomit-frame-pointer"

View File

@ -53,7 +53,7 @@ fi
if test "X" = "X$CPPFLAGS"; then
warn=
profile=
debug="-DH5AC_DEBUG -DH5B_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -DH5F_LOW_DFLT=H5F_LOW_SEC2"
debug="-DH5AC_DEBUG -DH5B_DEBUG -DH5D_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -DH5F_LOW_DFLT=H5F_LOW_SEC2"
production="-DNDEBUG"
parallel=""
default_mode='$debug $warn $parallel'

2
configure vendored
View File

@ -1267,7 +1267,7 @@ fi
for ac_func in getpwuid gethostname
for ac_func in getpwuid gethostname system
do
echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
echo "configure:1274: checking for $ac_func" >&5

View File

@ -112,7 +112,7 @@ AC_TYPE_SIZE_T
dnl ----------------------------------------------------------------------
dnl Check for functions.
dnl
AC_CHECK_FUNCS(getpwuid gethostname)
AC_CHECK_FUNCS(getpwuid gethostname system)
AC_TRY_COMPILE([#include<sys/types.h>],
[off64_t n = 0;],

412
src/H5D.c
View File

@ -63,7 +63,9 @@ const H5D_create_t H5D_create_dflt = {
/* Default dataset transfer property list */
const H5D_xfer_t H5D_xfer_dflt = {
0, /* Place holder - remove this later */
1024*1024, /* Temporary buffer size */
NULL, /* Type conversion buffer or NULL */
NULL, /* Background buffer or NULL */
};
/* Interface initialization? */
@ -813,7 +815,7 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space,
}
/* Create (open for write access) an object header */
if (H5O_create(f, 80, &(new_dset->ent)) < 0) {
if (H5O_create(f, 96, &(new_dset->ent)) < 0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL,
"unable to create dataset object header");
}
@ -1087,7 +1089,9 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
const H5S_t *file_space, const H5D_xfer_t *xfer_parms,
void *buf/*out*/)
{
size_t nelmts ; /*number of elements */
size_t nelmts; /*number of elements */
size_t smine_start; /*strip mine start loc */
size_t smine_nelmts; /*elements per strip */
uint8 *tconv_buf = NULL; /*data type conv buffer */
uint8 *bkg_buf = NULL; /*background buffer */
H5T_conv_t tconv_func = NULL; /*conversion function */
@ -1097,6 +1101,11 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
H5T_cdata_t *cdata = NULL; /*type conversion data */
herr_t ret_value = FAIL;
herr_t status;
size_t src_type_size; /*size of source type */
size_t dst_type_size; /*size of destination type*/
size_t target_size; /*desired buffer size */
size_t buffer_size; /*actual buffer size */
size_t request_nelmts; /*requested strip mine */
FUNC_ENTER(H5D_read, FAIL);
@ -1107,124 +1116,154 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
assert(buf);
if (!file_space) file_space = dataset->space;
if (!mem_space) mem_space = file_space;
/*
* Convert data types to atoms because the conversion functions are
* application-level functions.
*/
if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type))) < 0 ||
(dst_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type))) < 0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
"unable to register types for conversion");
}
nelmts = H5S_get_npoints(mem_space);
/*
* Locate the type conversion function and data space conversion
* functions, and set up the element numbering information.
* functions, and set up the element numbering information. If a data
* type conversion is necessary then register data type atoms.
*/
if (NULL == (tconv_func = H5T_find(dataset->type, mem_type, &cdata))) {
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert between src and dest data types");
} else if (H5T_conv_noop!=tconv_func) {
if ((src_id=H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0 ||
(dst_id=H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
"unable to register types for conversion");
}
}
if (NULL==(sconv_func=H5S_find (mem_space, file_space))) {
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert from file to memory data space");
}
if (sconv_func->init &&
(sconv_func->init)(&(dataset->layout), mem_space, file_space,
&numbering/*out*/)<=0) {
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
"unable to initialize element numbering information");
} else {
HDmemset (&numbering, 0, sizeof numbering);
}
if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) {
if (nelmts!=H5S_get_npoints (file_space)) {
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"src and dest data spaces have different sizes");
}
/*
* Compute the size of the request and allocate scratch buffers.
*/
nelmts = H5S_get_npoints(mem_space);
/*
* If there is no type conversion then try reading directly into the
* application's buffer.
* application's buffer. This saves at least one mem-to-mem copy.
*/
if (H5D_OPTIMIZE_PIPE &&
H5T_conv_noop==tconv_func &&
NULL!=sconv_func->read) {
#ifndef NDEBUG
fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n");
#endif
status = (sconv_func->read)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
mem_space, buf/*out*/);
if (status>=0) goto succeed;
#ifndef NDEBUG
fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n");
#ifdef H5D_DEBUG
fprintf (stderr, "HDF5-DIAG: input pipe optimization failed "
"(falling through)\n");
#endif
H5E_clear ();
}
/*
* This is the general case.
* This is the general case. Figure out the strip mine size.
*/
#ifndef LATER
src_type_size = H5T_get_size(dataset->type);
dst_type_size = H5T_get_size(mem_type);
target_size = xfer_parms->buf_size;
request_nelmts = target_size / MAX(src_type_size, dst_type_size);
if (request_nelmts<=0) {
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
"temporary buffer max size is too small");
}
if (sconv_func->init) {
smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space,
file_space, request_nelmts,
&numbering/*out*/);
if (smine_nelmts<=0) {
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
"unable to initialize element numbering information");
}
} else {
smine_nelmts = request_nelmts;
HDmemset (&numbering, 0, sizeof numbering);
}
buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size);
/*
* Note: this prototype version allocates a buffer large enough to
* satisfy the entire request; strip mining is not implemented.
* Get a temporary buffer for type conversion unless the app has already
* supplied one through the xfer properties. Instead of allocating a
* buffer which is the exact size, we allocate the target size. The
* malloc() is usually less resource-intensive if we allocate/free the
* same size over and over.
*/
{
size_t src_size = nelmts * H5T_get_size(dataset->type);
size_t dst_size = nelmts * H5T_get_size(mem_type);
tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size));
if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size);
if (NULL==(tconv_buf=xfer_parms->tconv)) {
tconv_buf = H5MM_xmalloc (target_size);
}
if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) {
bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size);
}
#ifdef H5D_DEBUG
/* Strip mine diagnostics.... */
if (smine_nelmts<nelmts) {
fprintf (stderr, "HDF5-DIAG: strip mine");
if (smine_nelmts!=request_nelmts) {
fprintf (stderr, " got %lu of %lu",
(unsigned long)smine_nelmts,
(unsigned long)request_nelmts);
}
if (buffer_size!=target_size) {
fprintf (stderr, " (%1.1f%% of buffer)",
100.0*buffer_size/target_size);
}
fprintf (stderr, " %1.1f iterations\n",
(double)nelmts/smine_nelmts);
}
#endif
/*
* Gather the data from disk into the data type conversion buffer. Also
* gather data from application to background buffer (this step is not
* needed for most conversions, but we leave that as an exercise for
* later ;-)
*/
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, 0, nelmts,
tconv_buf/*out*/)!=nelmts) {
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed");
}
if (H5T_BKG_YES==cdata->need_bkg) {
if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
&numbering, 0, nelmts,
bkg_buf/*out*/)!=nelmts) {
HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed");
/* Start strip mining... */
for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) {
smine_nelmts = MIN (smine_nelmts, nelmts-smine_start);
/*
* Gather the data from disk into the data type conversion
* buffer. Also gather data from application to background buffer
* if necessary.
*/
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, smine_start, smine_nelmts,
tconv_buf/*out*/)!=smine_nelmts) {
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed");
}
if (H5T_BKG_YES==cdata->need_bkg) {
if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
&numbering, smine_start, smine_nelmts,
bkg_buf/*out*/)!=smine_nelmts) {
HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed");
}
}
/*
* Perform data type conversion.
*/
cdata->command = H5T_CONV_CONV;
cdata->ncalls++;
if ((tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf,
bkg_buf)<0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
"data type conversion failed");
}
cdata->nelmts += smine_nelmts;
/*
* Scatter the data into memory.
*/
if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space,
&numbering, smine_start, smine_nelmts,
buf/*out*/)<0) {
HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed");
}
}
/*
* Perform data type conversion.
*/
cdata->command = H5T_CONV_CONV;
cdata->ncalls++;
if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
"data type conversion failed");
}
cdata->nelmts += nelmts;
/*
* Scatter the data into memory.
*/
if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space,
&numbering, 0, nelmts, buf/*out*/)<0) {
HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed");
}
succeed:
ret_value = SUCCEED;
@ -1232,8 +1271,12 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
done:
if (src_id >= 0) H5A_dec_ref(src_id);
if (dst_id >= 0) H5A_dec_ref(dst_id);
tconv_buf = H5MM_xfree(tconv_buf);
bkg_buf = H5MM_xfree (bkg_buf);
if (tconv_buf && NULL==xfer_parms->tconv) {
H5MM_xfree(tconv_buf);
}
if (bkg_buf && NULL==xfer_parms->bkg) {
H5MM_xfree (bkg_buf);
}
FUNC_LEAVE(ret_value);
}
@ -1260,7 +1303,9 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
const H5S_t *file_space, const H5D_xfer_t *xfer_parms,
const void *buf)
{
size_t nelmts;
size_t nelmts; /*total number of elmts */
size_t smine_start; /*strip mine start loc */
size_t smine_nelmts; /*elements per strip */
uint8 *tconv_buf = NULL; /*data type conv buffer */
uint8 *bkg_buf = NULL; /*background buffer */
H5T_conv_t tconv_func = NULL; /*conversion function */
@ -1269,6 +1314,11 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
H5S_number_t numbering; /*element numbering info*/
H5T_cdata_t *cdata = NULL; /*type conversion data */
herr_t ret_value = FAIL, status;
size_t src_type_size; /*size of source type */
size_t dst_type_size; /*size of destination type*/
size_t target_size; /*desired buffer size */
size_t buffer_size; /*actual buffer size */
size_t request_nelmts; /*requested strip mine */
FUNC_ENTER(H5D_write, FAIL);
@ -1279,47 +1329,32 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
assert(buf);
if (!file_space) file_space = dataset->space;
if (!mem_space) mem_space = file_space;
nelmts = H5S_get_npoints(mem_space);
/*
* Convert data types to atoms because the conversion functions are
* application-level functions.
*/
if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 ||
(dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
"unable to register types for conversion");
}
/*
* Locate the type conversion function and data space conversion
* functions, and set up the element numbering information.
* functions, and set up the element numbering information. If a data
* type conversion is necessary then register data type atoms.
*/
if (NULL == (tconv_func = H5T_find(mem_type, dataset->type, &cdata))) {
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert between src and dest data types");
} else if (H5T_conv_noop!=tconv_func) {
if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 ||
(dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
"unable to register types for conversion");
}
}
if (NULL==(sconv_func=H5S_find (mem_space, file_space))) {
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert from memory to file data space");
}
if (sconv_func->init &&
(sconv_func->init)(&(dataset->layout), mem_space, file_space,
&numbering/*out*/)<=0) {
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
"unable to initialize element numbering information");
} else {
HDmemset (&numbering, 0, sizeof numbering);
}
if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) {
if (nelmts!=H5S_get_npoints (file_space)) {
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"src and dest data spaces have different sizes");
}
/*
* Compute the size of the request and allocate scratch buffers.
*/
nelmts = H5S_get_npoints(mem_space);
/*
* If there is no type conversion then try writing directly from
* application buffer to file.
@ -1327,87 +1362,140 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
if (H5D_OPTIMIZE_PIPE &&
H5T_conv_noop==tconv_func &&
NULL!=sconv_func->write) {
#ifndef NDEBUG
fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n");
#endif
status = (sconv_func->write)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
mem_space, buf);
if (status>=0) goto succeed;
#ifndef NDEBUG
fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n");
#ifdef H5D_DEBUG
fprintf (stderr, "HDF5-DIAG: output pipe optimization failed "
"(falling through)\n");
#endif
H5E_clear ();
}
/*
* This is the general case.
* This is the general case. Figure out the strip mine size.
*/
#ifndef LATER
src_type_size = H5T_get_size(mem_type);
dst_type_size = H5T_get_size(dataset->type);
target_size = xfer_parms->buf_size;
request_nelmts = target_size / MAX (src_type_size, dst_type_size);
if (request_nelmts<=0) {
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
"temporary buffer max size is too small");
}
if (sconv_func->init) {
smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space,
file_space, request_nelmts,
&numbering/*out*/);
if (smine_nelmts<=0) {
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
"unable to initialize element numbering information");
}
} else {
smine_nelmts = request_nelmts;
HDmemset (&numbering, 0, sizeof numbering);
}
buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size);
/*
* Note: This prototype version allocates a buffer large enough to
* satisfy the entire request; strip mining is not implemented.
* Get a temporary buffer for type conversion unless the app has already
* supplied one through the xfer properties. Instead of allocating a
* buffer which is the exact size, we allocate the target size. The
* malloc() is usually less resource-intensive if we allocate/free the
* same size over and over.
*/
{
size_t src_size = nelmts * H5T_get_size(mem_type);
size_t dst_size = nelmts * H5T_get_size(dataset->type);
tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size));
if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size);
if (NULL==(tconv_buf=xfer_parms->tconv)) {
tconv_buf = H5MM_xmalloc (target_size);
}
if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) {
bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size);
}
#ifdef H5D_DEBUG
/* Strip mine diagnostics.... */
if (smine_nelmts<nelmts) {
fprintf (stderr, "HDF5-DIAG: strip mine");
if (smine_nelmts!=request_nelmts) {
fprintf (stderr, " got %lu of %lu",
(unsigned long)smine_nelmts,
(unsigned long)request_nelmts);
}
if (buffer_size!=target_size) {
fprintf (stderr, " (%1.1f%% of buffer)",
100.0*buffer_size/target_size);
}
fprintf (stderr, " %1.1f iterations\n",
(double)nelmts/smine_nelmts);
}
#endif
/*
* Gather data from application buffer into the data type conversion
* buffer. Also gather data from the file into the background buffer
* (this step is not needed for most conversions, but we leave that as an
* exercise for later ;-)
*/
if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
&numbering, 0, nelmts, tconv_buf/*out*/)!=nelmts) {
HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed");
}
if (H5T_BKG_YES==cdata->need_bkg) {
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
/* Start strip mining... */
for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) {
smine_nelmts = MIN (smine_nelmts, nelmts-smine_start);
/*
* Gather data from application buffer into the data type conversion
* buffer. Also gather data from the file into the background buffer
* if necessary.
*/
if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
&numbering, smine_start, smine_nelmts,
tconv_buf/*out*/)!=smine_nelmts) {
HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed");
}
if (H5T_BKG_YES==cdata->need_bkg) {
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, smine_start, smine_nelmts,
bkg_buf/*out*/)!=smine_nelmts) {
HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
"file gather failed");
}
}
/*
* Perform data type conversion.
*/
cdata->command = H5T_CONV_CONV;
cdata->ncalls++;
if ((tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf,
bkg_buf)<0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
"data type conversion failed");
}
cdata->nelmts += smine_nelmts;
/*
* Scatter the data out to the file.
*/
if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, 0, nelmts,
bkg_buf/*out*/)!=nelmts) {
HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed");
&numbering, smine_start, smine_nelmts,
tconv_buf)<0) {
HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
}
}
/*
* Perform data type conversion.
*/
cdata->command = H5T_CONV_CONV;
cdata->ncalls++;
if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
"data type conversion failed");
}
cdata->nelmts += nelmts;
/*
* Scatter the data out to the file.
*/
if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, 0, nelmts, tconv_buf)<0) {
HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
}
succeed:
ret_value = SUCCEED;
done:
if (src_id >= 0) H5A_dec_ref(src_id);
if (dst_id >= 0) H5A_dec_ref(dst_id);
tconv_buf = H5MM_xfree(tconv_buf);
bkg_buf = H5MM_xfree (bkg_buf);
if (tconv_buf && NULL==xfer_parms->tconv) {
H5MM_xfree(tconv_buf);
}
if (bkg_buf && NULL==xfer_parms->bkg) {
H5MM_xfree (bkg_buf);
}
FUNC_LEAVE(ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5D_extend

View File

@ -26,6 +26,15 @@
#include <H5Sprivate.h> /*for the H5S_t type */
#include <H5Oprivate.h> /*object Headers */
/*
* Feature: Define H5D_DEBUG on the compiler command line if you want to
* debug dataset I/O. NDEBUG must not be defined in order for this
* to have any effect.
*/
#ifdef NDEBUG
# undef H5D_DEBUG
#endif
#define H5D_RESERVED_ATOMS 0
/* Set the minimum object header size to create objects with */
@ -41,8 +50,11 @@ typedef struct H5D_create_t {
/* Dataset transfer property list */
typedef struct H5D_xfer_t {
int _placeholder; /*unused--delete this later */
size_t buf_size; /*max temp buffer size */
void *tconv; /*type conversion buffer or null */
void *bkg; /*background buffer or null */
} H5D_xfer_t;
typedef struct H5D_t H5D_t;
extern const H5D_create_t H5D_create_dflt;
extern const H5D_xfer_t H5D_xfer_dflt;

View File

@ -1677,6 +1677,100 @@ H5Pget_family (hid_t tid, hid_t *memb_tid)
FUNC_LEAVE (SUCCEED);
}
/*-------------------------------------------------------------------------
* Function: H5Pset_buffer
*
* Purpose: Given a dataset transfer property list, set the maximum size
* for the type conversion buffer and background buffer and
* optionally supply pointers to application-allocated buffers.
* If the buffer size is smaller than the entire amount of data
* being transfered between application and file, and a type
* conversion buffer or background buffer is required then
* strip mining will be used. However, certain restrictions
* apply for the size of buffer which can be used for strip
* mining. For instance, when strip mining a 100x200x300
* hyperslab of a simple data space the buffer must be large
* enough to hold a 1x200x300 slab.
*
* If TCONV and/or BKG are null pointers then buffers will be
* allocated and freed during the data transfer.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
* Monday, March 16, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg)
{
H5D_xfer_t *plist = NULL;
FUNC_ENTER (H5Pset_buffer, FAIL);
/* Check arguments */
if (H5P_DATASET_XFER != H5Pget_class (plist_id) ||
NULL == (plist = H5A_object (plist_id))) {
HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
"not a dataset transfer property list");
}
if (size<=0) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"buffer size must not be zero");
}
/* Update property list */
plist->buf_size = size;
plist->tconv = tconv;
plist->bkg = bkg;
FUNC_LEAVE (SUCCEED);
}
/*-------------------------------------------------------------------------
* Function: H5Pget_buffer
*
* Purpose: Reads values previously set with H5Pset_buffer().
*
* Return: Success: Buffer size.
*
* Failure: 0
*
* Programmer: Robb Matzke
* Monday, March 16, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
size_t
H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/)
{
H5D_xfer_t *plist = NULL;
FUNC_ENTER (H5Pget_buffer, 0);
/* Check arguments */
if (H5P_DATASET_XFER != H5Pget_class (plist_id) ||
NULL == (plist = H5A_object (plist_id))) {
HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, 0,
"not a dataset transfer property list");
}
/* Return values */
if (tconv) *tconv = plist->tconv;
if (bkg) *bkg = plist->bkg;
FUNC_LEAVE (plist->buf_size);
}
/*-------------------------------------------------------------------------
* Function: H5Pset_mpi

View File

@ -83,8 +83,12 @@ herr_t H5Pget_split (hid_t tid, size_t meta_ext_size, char *meta_ext/*out*/,
herr_t H5Pset_family (hid_t tid, hid_t memb_tid);
herr_t H5Pget_family (hid_t tid, hid_t *memb_tid/*out*/);
herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg);
size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/);
#ifdef HAVE_PARALLEL
herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info, unsigned access_mode);
herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info,
unsigned access_mode);
herr_t H5Pget_mpi (hid_t tid, MPI_Comm *comm/*out*/, MPI_Info *info/*out*/,
unsigned *access_mode/*out*/);
#endif

View File

@ -65,7 +65,8 @@ typedef struct H5S_number_t {
typedef struct H5S_tconv_t {
/* Initialize element numbering information */
size_t (*init)(const struct H5O_layout_t *layout, const H5S_t *mem_space,
const H5S_t *file_space, H5S_number_t *numbering/*out*/);
const H5S_t *file_space, size_t desired_nelmts,
H5S_number_t *numbering/*out*/);
/* Gather elements from disk to type conversion buffer */
size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout,
@ -122,7 +123,7 @@ intn H5S_extend (H5S_t *space, const size_t *size);
/* Conversion functions for simple data spaces */
size_t H5S_simp_init (const struct H5O_layout_t *layout,
const H5S_t *mem_space, const H5S_t *file_space,
H5S_number_t *numbering/*out*/);
size_t desired_nelmts, H5S_number_t *numbering/*out*/);
size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,

View File

@ -37,9 +37,14 @@ static intn interface_initialize_g = FALSE;
*/
size_t
H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
const H5S_t *file_space, H5S_number_t *numbering/*out*/)
const H5S_t *file_space, size_t desired_nelmts,
H5S_number_t *numbering/*out*/)
{
size_t nelmts;
int m_ndims, f_ndims; /*mem, file dimensionality */
size_t size[H5O_LAYOUT_NDIMS]; /*size of selected hyperslab */
size_t acc;
int i;
FUNC_ENTER (H5S_simp_init, 0);
@ -49,11 +54,43 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
assert (file_space && H5S_SIMPLE==file_space->type);
assert (numbering);
/* Numbering is implied by the hyperslab, C order */
/* Numbering is implied by the hyperslab, C order, no data here */
HDmemset (numbering, 0, sizeof(H5S_number_t));
/* Data can be efficiently copied at any size */
nelmts = H5S_get_npoints (file_space);
/*
* The stripmine size is such that only the slowest varying dimension can
* be split up. We choose the largest possible strip mine size which is
* not larger than the desired size.
*/
m_ndims = H5S_get_hyperslab (mem_space, NULL, size, NULL);
for (i=m_ndims-1, acc=1; i>0; --i) acc *= size[i];
nelmts = (desired_nelmts/acc) * acc;
if (nelmts<=0) {
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
"strip mine buffer is too small");
}
/*
* The value chosen for mem_space must be the same as the value chosen for
* file_space.
*/
f_ndims = H5S_get_hyperslab (file_space, NULL, size, NULL);
if (m_ndims!=f_ndims) {
nelmts = H5S_get_npoints (file_space);
if (nelmts>desired_nelmts) {
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
"strip mining not supported across "
"dimensionalities");
}
assert (nelmts==H5S_get_npoints (mem_space));
} else {
for (i=f_ndims-1, acc=1; i>0; --i) acc *= size[i];
acc *= (desired_nelmts/acc);
if (nelmts!=acc) {
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
"unsupported strip mine size for shape change");
}
}
FUNC_LEAVE (nelmts);
}
@ -95,6 +132,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
size_t acc; /*accumulator */
#ifndef LATER
intn file_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@ -112,12 +150,6 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
assert (nelmts>0);
assert (buf);
/*
* The prototype doesn't support strip mining.
*/
assert (0==start);
assert (nelmts==H5S_get_npoints (file_space));
/*
* Get hyperslab information to determine what elements are being
* selected (there might eventually be other selection methods too).
@ -132,6 +164,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
"unable to retrieve hyperslab parameters");
}
#else
/* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0,
@ -142,12 +175,23 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
file_offset[i] = file_offset_signed[i];
}
#endif
/* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0,
"hyperslab sampling is not implemented yet");
}
}
/* Adjust the slowest varying dimension to take care of strip mining */
for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
assert (0==start % acc);
assert (0==nelmts % acc);
file_offset[0] += start / acc;
hsize[0] = nelmts / acc;
/* The fastest varying dimension is for the data point itself */
file_offset[space_ndims] = 0;
hsize[space_ndims] = elmt_size;
HDmemset (zero, 0, layout->ndims*sizeof(size_t));
@ -194,6 +238,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
size_t acc; /*accumulator */
#ifndef LATER
intn mem_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@ -210,12 +255,6 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
assert (nelmts>0);
assert (buf);
/*
* The prototype doesn't support strip mining.
*/
assert (0==start);
assert (nelmts==H5S_get_npoints (mem_space));
/*
* Retrieve hyperslab information to determine what elements are being
* selected (there might be other selection methods in the future). We
@ -229,6 +268,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
"unable to retrieve hyperslab parameters");
}
#else
/* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@ -239,6 +279,8 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
mem_offset[i] = mem_offset_signed[i];
}
#endif
/* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
@ -249,6 +291,15 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
"unable to retrieve data space dimensions");
}
/* Adjust the slowest varying dimension to take care of strip mining */
for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
assert (0==start % acc);
assert (0==nelmts % acc);
mem_offset[0] += start / acc;
hsize[0] = nelmts / acc;
/* The fastest varying dimension is for the data point itself */
mem_offset[space_ndims] = 0;
mem_size[space_ndims] = elmt_size;
hsize[space_ndims] = elmt_size;
@ -299,6 +350,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
size_t acc; /*accumulator */
#ifndef LATER
intn mem_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@ -315,12 +367,6 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
assert (nelmts>0);
assert (tconv_buf);
/*
* The prototype doesn't support strip mining.
*/
assert (0==start);
assert (nelmts==H5S_get_npoints (mem_space));
/*
* Retrieve hyperslab information to determine what elements are being
* selected (there might be other selection methods in the future). We
@ -334,6 +380,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
"unable to retrieve hyperslab parameters");
}
#else
/* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@ -344,6 +391,8 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
mem_offset[i] = mem_offset_signed[i];
}
#endif
/* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0,
@ -354,6 +403,15 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0,
"unable to retrieve data space dimensions");
}
/* Adjust the slowest varying dimension to account for strip mining */
for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
assert (0==start % acc);
assert (0==nelmts % acc);
mem_offset[0] += start / acc;
hsize[0] = nelmts / acc;
/* The fastest varying dimension is for the data point itself */
mem_offset[space_ndims] = 0;
mem_size[space_ndims] = elmt_size;
hsize[space_ndims] = elmt_size;
@ -404,6 +462,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero vector */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
size_t acc; /*accumulator */
#ifndef LATER
intn file_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@ -421,12 +480,6 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
assert (nelmts>0);
assert (buf);
/*
* The prototype doesn't support strip mining.
*/
assert (0==start);
assert (nelmts==H5S_get_npoints (file_space));
/*
* Get hyperslab information to determine what elements are being
* selected (there might eventually be other selection methods too).
@ -441,6 +494,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
"unable to retrieve hyperslab parameters");
}
#else
/* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@ -451,12 +505,23 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
file_offset[i] = file_offset_signed[i];
}
#endif
/* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"hyperslab sampling is not implemented yet");
}
}
/* Adjust the slowest varying dimension to account for strip mining */
for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
assert (0==start % acc);
assert (0==nelmts % acc);
file_offset[0] += start / acc;
hsize[0] = nelmts / acc;
/* The fastest varying dimension is for the data point itself */
file_offset[space_ndims] = 0;
hsize[space_ndims] = elmt_size;
HDmemset (zero, 0, layout->ndims*sizeof(size_t));

View File

@ -11,13 +11,14 @@ CPPFLAGS=-I. -I../src @CPPFLAGS@
# These are our main targets. They should be listed in the order to be
# executed, generally most specific tests to least specific tests.
PROGS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external \
PROGS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external \
iopipe
TESTS=$(PROGS)
TESTS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external
TIMINGS=iopipe
# Temporary files
MOSTLYCLEAN=cmpd_dset.h5 dataset.h5 extend.h5 istore.h5 tfile1.h5 tfile2.h5 \
tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \
tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \
extern_1.h5 extern_2.h5 extern_3.h5 extern_1.raw extern_1b.raw \
extern_2.raw extern_2b.raw extern_3.raw extern_3b.raw \
extern_4.raw extern_4b.raw iopipe.raw iopipe.h5
@ -64,6 +65,16 @@ IOPIPE_OBJ=$(IOPIPE_SRC:.c=.o)
# Private header files (not to be installed)...
PRIVATE_HDR=testhdf5.h
# Additional targets
.PHONY: timings _timings
timings _timings: $(TIMINGS)
@for timing in $(TIMINGS) dummy; do \
if test $$timing != dummy; then \
echo "Running $$timing $(TEST_FLAGS)"; \
$(RUNTEST) ./$$timing $(TEST_FLAGS) || exit 1; \
fi; \
done;
# How to build the programs...
testhdf5: $(TESTHDF5_OBJ) ../src/libhdf5.a
$(CC) $(CFLAGS) -o $@ $(TESTHDF5_OBJ) ../src/libhdf5.a $(LIBS)

View File

@ -184,11 +184,12 @@ test_create(hid_t file)
static herr_t
test_simple_io(hid_t file)
{
hid_t dataset, space;
herr_t status;
int points[100][200], check[100][200];
int i, j, n;
size_t dims[2];
hid_t dataset, space, xfer;
herr_t status;
int points[100][200], check[100][200];
int i, j, n;
size_t dims[2];
void *tconv_buf = NULL;
printf("%-70s", "Testing simple I/O");
@ -205,6 +206,13 @@ test_simple_io(hid_t file)
space = H5Screate_simple(2, dims, NULL);
assert(space>=0);
/* Create a small conversion buffer to test strip mining */
tconv_buf = malloc (1000);
xfer = H5Pcreate (H5P_DATASET_XFER);
assert (xfer>=0);
status = H5Pset_buffer (xfer, 1000, tconv_buf, NULL);
assert (status>=0);
/* Create the dataset */
dataset = H5Dcreate(file, DSET_SIMPLE_IO_NAME, H5T_NATIVE_INT, space,
H5P_DEFAULT);
@ -212,12 +220,12 @@ test_simple_io(hid_t file)
/* Write the data to the dataset */
status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
H5P_DEFAULT, points);
xfer, points);
if (status<0) goto error;
/* Read the dataset back */
status = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
H5P_DEFAULT, check);
xfer, check);
if (status<0) goto error;
/* Check that the values read are the same as the values written */

View File

@ -19,12 +19,22 @@
#define RAW_FILE_NAME "iopipe.raw"
#define HDF5_FILE_NAME "iopipe.h5"
#define HEADING "%-16s"
#define PROGRESS '='
#if 1
/* Normal testing */
#define REQUEST_SIZE_X 4579
#define REQUEST_SIZE_Y 4579
#define NREAD_REQUESTS 45
#define NWRITE_REQUESTS 45
#define HEADING "%-16s"
#define PROGRESS '='
#else
/* Speedy testing */
#define REQUEST_SIZE_X 1000
#define REQUEST_SIZE_Y 1000
#define NREAD_REQUESTS 45
#define NWRITE_REQUESTS 45
#endif
/*-------------------------------------------------------------------------
@ -83,10 +93,17 @@ print_stats (const char *prefix,
static void
synchronize (void)
{
#ifdef HAVE_SYSTEM
system ("sync");
system ("df >/dev/null");
#if 0
system ("/sbin/swapout 130");
/*
* This works well on Linux to get rid of all cached disk buffers. The
* number should be approximately the amount of RAM in MB. Do not
* include swap space in that amount or the command will fail.
*/
system ("/sbin/swapout 128");
#endif
#endif
}