mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-03-31 17:10:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r305] Changes since 19980304

Browse Source 
----------------------

./MANIFEST
	New files

./html/extern1.gif		[NEW]
./html/extern1.obj		[NEW]
./html/extern2.gif		[NEW]
./html/extern2.obj		[NEW]
./html/Datasets.html
	Documented external raw data storage.

./html/Datasets.html
./html/chunk1.gif		[NEW]
./html/chunk1.obj		[NEW]
	Added a picture and better description for chunking since it
	seems that people are thinking chunks are fixed-size at the
	file level instead of at the array level.

./src/H5D.c
./src/H5Dpublic.h
./test/external.c
	Changed H5Dget_create_parms() to H5Dget_create_plist().

./src/H5D.c
./src/H5Farray.c
./src/H5Fprivate.h
./src/H5Oefl.c
./src/H5Oprivate.h
./src/H5Sprivate.h
./src/H5Ssimp.c
./test/external.c
./test/istore.c
	External raw data file I/O for contiguous storage is now
	working.  The library supports segments of various sizes which
	can be scattered throughout multiple files in any order.

./test/Makefile.in
	Remove more temporary files.
This commit is contained in:
Robb Matzke 1998-03-05 10:48:16 -05:00
parent 2b323ed454
commit 4badc161ae
13 changed files with 490 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
MANIFEST
View File

@ -52,9 +52,15 @@
./html/MemoryManagement.html
./html/ObjectHeader.txt
./html/Properties.html
./html/chunk1.gif
./html/chunk1.obj
./html/compat.html
./html/dataset_p1.gif
./html/dataset_p1.obj
./html/extern1.gif
./html/extern1.obj
./html/extern2.gif
./html/extern2.obj
./html/group_p1.gif
./html/group_p1.obj
./html/group_p2.gif

25
src/H5D.c
View File

@ -413,7 +413,7 @@ H5Dget_type (hid_t dataset_id)
}
/*-------------------------------------------------------------------------
* Function: H5Dget_create_parms
* Function: H5Dget_create_plist
*
* Purpose: Returns a copy of the dataset creation property list.
*
@ -431,13 +431,13 @@ H5Dget_type (hid_t dataset_id)
*-------------------------------------------------------------------------
*/
hid_t
H5Dget_create_parms (hid_t dataset_id)
H5Dget_create_plist (hid_t dataset_id)
{
H5D_t *dataset = NULL;
H5D_create_t *copied_parms = NULL;
hid_t ret_value = FAIL;
FUNC_ENTER (H5Dget_create_parms, FAIL);
FUNC_ENTER (H5Dget_create_plist, FAIL);
/* Check args */
if (H5_DATASET!=H5A_group (dataset_id) ||
@ -962,9 +962,9 @@ H5D_open(H5F_t *f, const char *name)
}
/* Get the external file list message, which might not exist */
if (!H5F_addr_defined (&(dataset->layout.addr)) &&
NULL==H5O_read (&(dataset->ent), H5O_EFL, 0,
&(dataset->create_parms->efl))) {
if (NULL==H5O_read (&(dataset->ent), H5O_EFL, 0,
&(dataset->create_parms->efl)) &&
!H5F_addr_defined (&(dataset->layout.addr))) {
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL,
"storage address is undefined an no external file list");
}
@ -1128,10 +1128,6 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"src and dest data spaces have different sizes");
}
if (dataset->create_parms->efl.nused>0) {
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"reading externally-stored data is not implemented yet");
}
/*
* Compute the size of the request and allocate scratch buffers.
@ -1157,6 +1153,7 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
* later ;-)
*/
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, 0, nelmts,
tconv_buf/*out*/)!=nelmts) {
@ -1275,11 +1272,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"src and dest data spaces have different sizes");
}
if (dataset->create_parms->efl.nused>0) {
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"writing externally-stored data is not implemente yet");
}
/*
* Compute the size of the request and allocate scratch buffers.
*/
@ -1310,6 +1303,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
}
if (H5T_BKG_YES==cdata->need_bkg) {
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, 0, nelmts,
bkg_buf/*out*/)!=nelmts) {
@ -1332,6 +1326,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
* Scatter the data out to the file.
*/
if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, 0, nelmts, tconv_buf)<0) {
HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");

2
src/H5Dpublic.h
View File

@ -41,7 +41,7 @@ hid_t H5Dopen (hid_t file_id, const char *name);
herr_t H5Dclose (hid_t dataset_id);
hid_t H5Dget_space (hid_t dataset_id);
hid_t H5Dget_type (hid_t dataset_id);
hid_t H5Dget_create_parms (hid_t dataset_id);
hid_t H5Dget_create_plist (hid_t dataset_id);
herr_t H5Dread (hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id,
hid_t file_space_id, hid_t xfer_parms_id, void *buf/*out*/);
herr_t H5Dwrite (hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id,

69
src/H5Farray.c
View File

@ -89,8 +89,9 @@ H5F_arr_create (H5F_t *f, struct H5O_layout_t *layout/*in,out*/)
*
* Purpose: Reads a hyperslab of a file byte array into a hyperslab of
* a byte array in memory. The data is read from file F and the
* array's size and storage information is in LAYOUT. The
* hyperslab offset is FILE_OFFSET[] in the file and
* array's size and storage information is in LAYOUT. External
* files are described according to the external file list, EFL.
* The hyperslab offset is FILE_OFFSET[] in the file and
* MEM_OFFSET[] in memory (offsets are relative to the origin of
* the array) and the size of the hyperslab is HSLAB_SIZE[]. The
* total size of the file array is implied in the LAYOUT
@ -111,6 +112,7 @@ H5F_arr_create (H5F_t *f, struct H5O_layout_t *layout/*in,out*/)
*/
herr_t
H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
const struct H5O_efl_t *efl,
const size_t _hslab_size[], const size_t mem_size[],
const size_t mem_offset[], const size_t file_offset[],
void *_buf/*out*/)
@ -165,7 +167,11 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
*/
H5V_vector_cpy (ndims, idx, hslab_size);
nelmts = H5V_vector_reduce_product (ndims, hslab_size);
addr = layout->addr;
if (efl && efl->nused>0) {
H5F_addr_reset (&addr);
} else {
addr = layout->addr;
}
H5F_addr_inc (&addr, file_start);
buf += mem_start;
@ -176,7 +182,12 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
for (z=0; z<nelmts; z++) {
/* Read from file */
if (H5F_block_read (f, &addr, elmt_size, buf)<0) {
if (efl && efl->nused>0) {
if (H5O_efl_read (f, efl, &addr, elmt_size, buf)<0) {
HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL,
"external data read failed");
}
} else if (H5F_block_read (f, &addr, elmt_size, buf)<0) {
HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL,
"block read failed");
}
@ -195,8 +206,14 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
case H5D_CHUNKED:
/*
* This method is unable to copy into a proper hyperslab.
* This method is unable to access external raw data files or to copy
* into a proper hyperslab.
*/
if (efl && efl->nused>0) {
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL,
"chunking and external files are mutually "
"exclusive");
}
for (i=0; i<layout->ndims; i++) {
if (0!=mem_offset[i] || hslab_size[i]!=mem_size[i]) {
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL,
@ -224,13 +241,14 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
* Purpose: Copies a hyperslab of a memory array to a hyperslab of a
* file array. The data is written to file F and the file
* array's size and storage information is implied by LAYOUT.
* The hyperslab offset is FILE_OFFSET[] in the file and
* MEM_OFFSET[] in memory (offsets are relative to the origin of
* the array) and the size of the hyperslab is HSLAB_SIZE[].
* The total size of the file array is implied by the LAYOUT
* argument and the total size of the memory array is
* MEM_SIZE[]. The dimensionality of these vectors is implied
* by the LAYOUT argument.
* The data is stored in external files according to the
* external file list, EFL. The hyperslab offset is
* FILE_OFFSET[] in the file and MEM_OFFSET[] in memory (offsets
* are relative to the origin of the array) and the size of the
* hyperslab is HSLAB_SIZE[]. The total size of the file array
* is implied by the LAYOUT argument and the total size of the
* memory array is MEM_SIZE[]. The dimensionality of these
* vectors is implied by the LAYOUT argument.
*
* Return: Success: SUCCEED
*
@ -245,9 +263,9 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
*/
herr_t
H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
const size_t _hslab_size[], const size_t mem_size[],
const size_t mem_offset[], const size_t file_offset[],
const void *_buf)
const struct H5O_efl_t *efl, const size_t _hslab_size[],
const size_t mem_size[], const size_t mem_offset[],
const size_t file_offset[], const void *_buf)
{
const uint8 *buf = (const uint8 *)_buf; /*cast for arithmetic */
ssize_t file_stride[H5O_LAYOUT_NDIMS]; /*strides through file */
@ -300,7 +318,11 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
*/
H5V_vector_cpy (ndims, idx, hslab_size);
nelmts = H5V_vector_reduce_product (ndims, hslab_size);
addr = layout->addr;
if (efl && efl->nused>0) {
H5F_addr_reset (&addr);
} else {
addr = layout->addr;
}
H5F_addr_inc (&addr, file_start);
buf += mem_start;
@ -311,7 +333,12 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
for (z=0; z<nelmts; z++) {
/* Write to file */
if (H5F_block_write (f, &addr, elmt_size, buf)<0) {
if (efl && efl->nused>0) {
if (H5O_efl_write (f, efl, &addr, elmt_size, buf)<0) {
HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL,
"external data write failed");
}
} else if (H5F_block_write (f, &addr, elmt_size, buf)<0) {
HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
@ -331,8 +358,14 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
case H5D_CHUNKED:
/*
* This method is unable to copy from a proper hyperslab.
* This method is unable to access external raw daa files or to copy
* from a proper hyperslab.
*/
if (efl && efl->nused>0) {
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL,
"chunking and external files are mutually "
"exclusive");
}
for (i=0; i<layout->ndims; i++) {
if (0!=mem_offset[i] || hslab_size[i]!=mem_size[i]) {
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL,

2
src/H5Flow.c
View File

@ -628,7 +628,7 @@ H5F_addr_defined(const haddr_t *addr)
*-------------------------------------------------------------------------
*/
void
H5F_addr_reset(haddr_t *addr /*out */ )
H5F_addr_reset(haddr_t *addr/*out*/)
{
assert(addr);
addr->offset = 0;

13
src/H5Fprivate.h
View File

@ -449,6 +449,7 @@ typedef struct H5F_t {
}
struct H5O_layout_t; /*forward decl for prototype arguments */
struct H5O_efl_t; /*forward decl for prototype arguments */
/* library variables */
extern const H5F_create_t H5F_create_dflt;
@ -467,13 +468,13 @@ herr_t H5F_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent,
/* Functions that operate on array storage */
herr_t H5F_arr_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/);
herr_t H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
const size_t _hslab_size[], const size_t mem_size[],
const size_t mem_offset[], const size_t file_offset[],
void *_buf/*out*/);
const struct H5O_efl_t *efl, const size_t _hslab_size[],
const size_t mem_size[], const size_t mem_offset[],
const size_t file_offset[], void *_buf/*out*/);
herr_t H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
const size_t _hslab_size[], const size_t mem_size[],
const size_t mem_offset[], const size_t file_offset[],
const void *_buf);
const struct H5O_efl_t *efl, const size_t _hslab_size[],
const size_t mem_size[], const size_t mem_offset[],
const size_t file_offset[], const void *_buf);
/* Functions that operate on indexed storage */
herr_t H5F_istore_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/);

163
src/H5Oefl.c
View File

@ -348,6 +348,169 @@ H5O_efl_total_size (H5O_efl_t *efl)
FUNC_LEAVE (ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5O_efl_read
*
* Purpose: Reads data from an external file list. It is an error to
* read past the logical end of file, but reading past the end
* of any particular member of the external file list results in
* zeros.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
* Wednesday, March 4, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5O_efl_read (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size,
uint8 *buf)
{
int i, fd=-1;
size_t to_read, cur, skip;
ssize_t n;
herr_t ret_value = FAIL;
FUNC_ENTER (H5O_efl_read, FAIL);
/* Check args */
assert (efl && efl->nused>0);
assert (addr && H5F_addr_defined (addr));
assert (buf || 0==size);
/* Find the first efl member from which to read */
for (i=0, cur=0; i<efl->nused; i++) {
if (addr->offset < cur+efl->slot[i].size) {
skip = addr->offset - cur;
break;
}
cur += efl->slot[i].size;
}
/* Read the data */
while (size) {
if (i>=efl->nused) {
HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL,
"read past logical end of file");
}
if ((fd=open (efl->slot[i].name, O_RDONLY))<0) {
HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL,
"unable to open external raw data file");
}
if (lseek (fd, efl->slot[i].offset+skip, SEEK_SET)<0) {
HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL,
"unable to seek in external raw data file");
}
to_read = MIN(efl->slot[i].size-skip, size);
if ((n=read (fd, buf, to_read))<0) {
HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL,
"read error in external raw data file");
} else if (n<to_read) {
HDmemset (buf+n, 0, to_read-n);
}
close (fd);
fd = -1;
size -= to_read;
buf += to_read;
skip = 0;
i++;
}
ret_value = SUCCEED;
done:
if (fd>=0) close (fd);
FUNC_LEAVE (ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5O_efl_write
*
* Purpose: Writes data to an external file list. It is an error to
* write past the logical end of file, but writing past the end
* of any particular member of the external file list just
* extends that file.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
* Wednesday, March 4, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5O_efl_write (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size,
const uint8 *buf)
{
int i, fd=-1;
size_t to_write, cur, skip;
ssize_t n;
herr_t ret_value = FAIL;
FUNC_ENTER (H5O_efl_write, FAIL);
/* Check args */
assert (efl && efl->nused>0);
assert (addr && H5F_addr_defined (addr));
assert (buf || 0==size);
/* Find the first efl member in which to write */
for (i=0, cur=0; i<efl->nused; i++) {
if (addr->offset < cur+efl->slot[i].size) {
skip = addr->offset - cur;
break;
}
cur += efl->slot[i].size;
}
/* Write the data */
while (size) {
if (i>=efl->nused) {
HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL,
"write past logical end of file");
}
if ((fd=open (efl->slot[i].name, O_RDWR))<0) {
if (access (efl->slot[i].name, F_OK)<0) {
HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL,
"external raw data file does not exist");
} else {
HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL,
"unable to open external raw data file");
}
}
if (lseek (fd, efl->slot[i].offset+skip, SEEK_SET)<0) {
HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL,
"unable to seek in external raw data file");
}
to_write = MIN(efl->slot[i].size-skip, size);
if ((n=write (fd, buf, to_write))!=to_write) {
HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL,
"write error in external raw data file");
}
close (fd);
fd = -1;
size -= to_write;
buf += to_write;
skip = 0;
i++;
}
ret_value = SUCCEED;
done:
if (fd>=0) close (fd);
FUNC_LEAVE (ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5O_efl_debug

4
src/H5Oprivate.h
View File

@ -216,5 +216,9 @@ herr_t H5O_debug (H5F_t *f, const haddr_t *addr, FILE * stream, intn indent,
/* EFL operators */
size_t H5O_efl_total_size (H5O_efl_t *efl);
herr_t H5O_efl_read (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr,
size_t size, uint8 *buf);
herr_t H5O_efl_write (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr,
size_t size, const uint8 *buf);
#endif

24
src/H5Sprivate.h
View File

@ -69,9 +69,9 @@ typedef struct H5S_tconv_t {
/* Gather elements from disk to type conversion buffer */
size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout,
size_t elmt_size, const H5S_t *file_space,
const H5S_number_t *numbering, size_t start, size_t nelmts,
void *tconv_buf/*out*/);
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, void *tconv_buf/*out*/);
/* Scatter elements from type conversion buffer to application buffer */
herr_t (*mscat)(const void *tconv_buf, size_t elmt_size,
@ -85,9 +85,9 @@ typedef struct H5S_tconv_t {
/* Scatter elements from type conversion buffer to disk */
herr_t (*fscat)(H5F_t *f, const struct H5O_layout_t *layout,
size_t elmt_size, const H5S_t *file_space,
const H5S_number_t *numbering, size_t start, size_t nelmts,
const void *tconv_buf);
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, const void *tconv_buf);
} H5S_conv_t;
H5S_t *H5S_copy (const H5S_t *src);
@ -112,9 +112,9 @@ size_t H5S_simp_init (const struct H5O_layout_t *layout,
const H5S_t *mem_space, const H5S_t *file_space,
H5S_number_t *numbering/*out*/);
size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
size_t elmt_size, const H5S_t *file_space,
const H5S_number_t *numbering, size_t start,
size_t nelmts, void *tconv_buf/*out*/);
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, void *tconv_buf/*out*/);
herr_t H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
const H5S_t *mem_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, void *buf/*out*/);
@ -122,7 +122,7 @@ size_t H5S_simp_mgath (const void *buf, size_t elmt_size,
const H5S_t *mem_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, void *tconv_buf/*out*/);
herr_t H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
size_t elmt_size, const H5S_t *file_space,
const H5S_number_t *numbering, size_t start,
size_t nelmts, const void *tconv_buf);
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, const void *tconv_buf);
#endif

18
src/H5Ssimp.c
View File

@ -63,7 +63,8 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
*
* Purpose: Gathers data points from file F and accumulates them in the
* type conversion buffer BUF. The LAYOUT argument describes
* how the data is stored on disk. ELMT_SIZE is the size in
* how the data is stored on disk and EFL describes how the data
* is organized in external files. ELMT_SIZE is the size in
* bytes of a datum which this function treats as opaque.
* FILE_SPACE describes the data space of the dataset on disk
* and the elements that have been selected for reading (via
@ -85,6 +86,7 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
*/
size_t
H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
const struct H5O_efl_t *efl,
size_t elmt_size, const H5S_t *file_space,
const H5S_number_t *numbering, size_t start, size_t nelmts,
void *buf/*out*/)
@ -153,7 +155,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
/*
* Gather from file.
*/
if (H5F_arr_read (f, layout, hsize, hsize, zero, file_offset,
if (H5F_arr_read (f, layout, efl, hsize, hsize, zero, file_offset,
buf/*out*/)<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_READERROR, 0, "read error");
}
@ -375,10 +377,10 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
* Purpose: Scatters dataset elements from the type conversion buffer BUF
* to the file F where the data points are arranged according to
* the file data space FILE_SPACE and stored according to
* LAYOUT. Each element is ELMT_SIZE bytes and has a unique
* number according to NUMBERING. The caller is requesting that
* NELMTS elements are coppied beginning with element number
* START.
* LAYOUT and EFL. Each element is ELMT_SIZE bytes and has a
* unique number according to NUMBERING. The caller is
* requesting that NELMTS elements are coppied beginning with
* element number START.
*
* Return: Success: SUCCEED
*
@ -393,6 +395,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
*/
herr_t
H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
const struct H5O_efl_t *efl,
size_t elmt_size, const H5S_t *file_space,
const H5S_number_t *numbering, size_t start, size_t nelmts,
const void *buf)
@ -461,7 +464,8 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
/*
* Scatter to file.
*/
if (H5F_arr_write (f, layout, hsize, hsize, zero, file_offset, buf)<0) {
if (H5F_arr_write (f, layout, efl, hsize, hsize, zero,
file_offset, buf)<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_WRITEERROR, FAIL, "write error");
}

4
test/Makefile.in
View File

@ -16,7 +16,9 @@ TESTS=$(PROGS)
# Temporary files
MOSTLYCLEAN=cmpd_dset.h5 dataset.h5 extend.h5 istore.h5 tfile1.h5 tfile2.h5 \
tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5
tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \
extern_1.h5 extern_2.h5 extern_1.raw extern_1b.raw extern_2.raw \
extern_2b.raw extern_3.raw extern_3b.raw extern_4.raw extern_4b.raw
# Source and object files for programs... The PROG_SRC list contains all the
# source files and is used for things like dependencies, archiving, etc. The

244
test/external.c
View File

@ -12,8 +12,11 @@
#include <hdf5.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
static int nerrors_g = 0;
/*-------------------------------------------------------------------------
* Function: display_error_cb
@ -36,9 +39,54 @@ display_error_cb (void *client_data)
{
puts ("*FAILED*");
H5Eprint (stdout);
nerrors_g++;
return 0;
}
/*-------------------------------------------------------------------------
* Function: same_contents
*
* Purpose: Determines whether two files are exactly the same.
*
* Return: Success: nonzero if same, zero if different.
*
* Failure: zero
*
* Programmer: Robb Matzke
* Wednesday, March 4, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static int
same_contents (const char *name1, const char *name2)
{
int fd1, fd2;
ssize_t n1, n2;
char buf1[1024], buf2[1024];
fd1 = open (name1, O_RDONLY);
fd2 = open (name2, O_RDONLY);
assert (fd1>=0 && fd2>=0);
while (1) {
n1 = read (fd1, buf1, sizeof(buf1));
n2 = read (fd2, buf2, sizeof(buf2));
assert (n1==n2);
if (n1<=0 && n2<=0) break;
if (memcmp (buf1, buf2, n1)) {
close (fd1);
close (fd2);
return 0;
}
}
close (fd1);
close (fd2);
return 1;
}
/*-------------------------------------------------------------------------
* Function: test_1
@ -46,9 +94,7 @@ display_error_cb (void *client_data)
* Purpose: Describes various external datasets in an HDF5 file without
* actually creating the external raw files.
*
* Return: Success: 0
*
* Failure: -1
* Return: void
*
* Programmer: Robb Matzke
* Tuesday, March 3, 1998
@ -57,7 +103,7 @@ display_error_cb (void *client_data)
*
*-------------------------------------------------------------------------
*/
static herr_t
static void
test_1 (void)
{
hid_t file, plist, space, dset, grp;
@ -78,8 +124,6 @@ test_1 (void)
grp = H5Gcreate (file, "emit-diagnostics", 8);
H5Gclose (grp);
printf ("Testing external storage descriptions...\n");
/*
* A single external file for a non-extendible dataset.
*/
@ -127,6 +171,7 @@ test_1 (void)
if (dset>=0) {
puts ("*FAILED*");
printf (" Small external file succeeded instead of failing\n");
nerrors_g++;
H5Dclose (dset);
break;
}
@ -188,6 +233,7 @@ test_1 (void)
puts ("*FAILED*");
printf (" Small external file succeeded instead of failing\n");
H5Dclose (dset);
nerrors_g++;
break;
}
puts (" PASSED");
@ -234,7 +280,7 @@ test_1 (void)
dset = H5Dopen (file, "dset1");
assert (dset>=0);
plist = H5Dget_create_parms (dset);
plist = H5Dget_create_plist (dset);
assert (plist>=0);
n = H5Pget_external_count (plist);
@ -242,6 +288,7 @@ test_1 (void)
if (1!=n) {
puts ("*FAILED*");
printf (" Returned external count is wrong.\n");
nerrors_g++;
break;
}
strcpy (name+sizeof(name)-4, "...");
@ -253,10 +300,12 @@ test_1 (void)
} else if (file_offset!=0) {
puts ("*FAILED*");
printf (" Wrong file offset.\n");
nerrors_g++;
break;
} else if (file_size!=400) {
puts ("*FAILED*");
printf (" Wrong file size.\n");
nerrors_g++;
break;
}
puts (" PASSED");
@ -279,7 +328,7 @@ test_1 (void)
dset = H5Dopen (file, "dset5");
assert (dset>=0);
plist = H5Dget_create_parms (dset);
plist = H5Dget_create_plist (dset);
assert (plist>=0);
n = H5Pget_external_count (plist);
@ -287,6 +336,7 @@ test_1 (void)
if (1!=n) {
puts ("*FAILED*");
printf (" Returned external count is wrong.\n");
nerrors_g++;
break;
}
strcpy (name+sizeof(name)-4, "...");
@ -298,10 +348,12 @@ test_1 (void)
} else if (file_offset!=0) {
puts ("*FAILED*");
printf (" Wrong file offset.\n");
nerrors_g++;
break;
} else if (H5F_UNLIMITED!=file_size) {
puts ("*FAILED*");
printf (" Wrong file size.\n");
nerrors_g++;
break;
}
puts (" PASSED");
@ -358,6 +410,7 @@ test_1 (void)
if (status>=0) {
puts ("*FAILED*");
puts (" H5Pset_external() succeeded when it should have failed");
nerrors_g++;
break;
}
@ -367,6 +420,8 @@ test_1 (void)
if (1!=n) {
puts ("*FAILED*");
puts (" Wrong external file count returned.");
nerrors_g++;
break;
}
puts (" PASSED");
} while (0);
@ -392,6 +447,7 @@ test_1 (void)
if (status>=0) {
puts ("*FAILED*");
puts (" H5Pset_external() succeeded when it should have failed");
nerrors_g++;
break;
}
puts (" PASSED");
@ -402,7 +458,6 @@ test_1 (void)
/* END OF TESTS */
H5Fclose (file);
return 0;
}
@ -411,9 +466,7 @@ test_1 (void)
*
* Purpose: Tests reading from an external file set.
*
* Return: Success: 0
*
* Failure: -1
* Return: void
*
* Programmer: Robb Matzke
* Wednesday, March 4, 1998
@ -422,7 +475,7 @@ test_1 (void)
*
*-------------------------------------------------------------------------
*/
static herr_t
static void
test_2 (void)
{
hid_t file, plist, space, dset, grp;
@ -434,7 +487,6 @@ test_2 (void)
size_t size;
/* Write the data to external files */
printf ("Writing external data...\n");
for (i=0; i<4; i++) {
for (j=0; j<25; j++) {
part[j] = i*25+j;
@ -443,6 +495,8 @@ test_2 (void)
sprintf (fname, "extern_%d.raw", i+1);
fd = open (fname, O_RDWR|O_CREAT|O_TRUNC, 0666);
assert (fd>=0);
n = lseek (fd, i*10, SEEK_SET);
assert (n==i*10);
n = write (fd, part, sizeof(part));
assert (n==sizeof(part));
close (fd);
@ -457,18 +511,17 @@ test_2 (void)
assert (file>=0);
grp = H5Gcreate (file, "emit-diagnostics", 8);
H5Gclose (grp);
printf ("Testing external data reading...\n");
/* Create the external file list */
plist = H5Pcreate (H5P_DATASET_CREATE);
assert (plist>=0);
status = H5Pset_external (plist, "extern_1.raw", 0, sizeof(part));
assert (status>=0);
status = H5Pset_external (plist, "extern_2.raw", 0, sizeof(part));
status = H5Pset_external (plist, "extern_2.raw", 10, sizeof(part));
assert (status>=0);
status = H5Pset_external (plist, "extern_3.raw", 0, sizeof(part));
status = H5Pset_external (plist, "extern_3.raw", 20, sizeof(part));
assert (status>=0);
status = H5Pset_external (plist, "extern_4.raw", 0, sizeof(part));
status = H5Pset_external (plist, "extern_4.raw", 30, sizeof(part));
assert (status>=0);
/* Create the data space */
@ -487,6 +540,8 @@ test_2 (void)
/* Read from the dataset */
printf ("%-70s", "...reading entire dataset");
fflush (stdout);
memset (whole, 0, sizeof(whole));
status = H5Dread (dset, H5T_NATIVE_INT, space, space,
H5P_DEFAULT, whole);
if (status<0) {
@ -498,6 +553,53 @@ test_2 (void)
if (whole[i]!=i) {
puts ("*FAILED*");
puts (" Incorrect value(s) read.");
nerrors_g++;
break;
}
}
puts (" PASSED");
} while (0);
/*
* Read the middle of the dataset
*/
do {
hid_t hs_space;
int hs_start = 30;
size_t hs_count = 25;
/* Read from the dataset */
printf ("%-70s", "...reading partial dataset");
fflush (stdout);
hs_space = H5Scopy (space);
assert (hs_space>=0);
status = H5Sset_hyperslab (hs_space, &hs_start, &hs_count, NULL);
assert (status>=0);
memset (whole, 0, sizeof(whole));
status = H5Dread (dset, H5T_NATIVE_INT, hs_space, hs_space,
H5P_DEFAULT, whole);
H5Sclose (hs_space);
if (status<0) {
puts (" Failed to read dataset");
break;
}
#if 0
for (i=0; i<100; i++) {
printf (" #%02d %3d %s\n",
i, whole[i], whole[i]==i?"":" <------------------------");
}
#endif
for (i=hs_start; i<hs_start+hs_count; i++) {
if (whole[i]!=i) {
puts ("*FAILED*");
puts (" Incorrect value(s) read.");
nerrors_g++;
break;
}
}
@ -508,9 +610,96 @@ test_2 (void)
H5Pclose (plist);
H5Sclose (space);
H5Fclose (file);
return 0;
}
/*-------------------------------------------------------------------------
* Function: test_3
*
* Purpose: Tests writing to an external file set.
*
* Return: void
*
* Programmer: Robb Matzke
* Wednesday, March 4, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static void
test_3 (void)
{
hid_t file, plist, space, dset;
herr_t status;
int i;
int part[25], whole[100];
size_t size;
/*
* Open the file from test_2().
*/
file = H5Fcreate ("extern_2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
assert (file>=0);
/* Create the external file list */
plist = H5Pcreate (H5P_DATASET_CREATE);
assert (plist>=0);
status = H5Pset_external (plist, "extern_1b.raw", 0, sizeof(part));
assert (status>=0);
status = H5Pset_external (plist, "extern_2b.raw", 10, sizeof(part));
assert (status>=0);
status = H5Pset_external (plist, "extern_3b.raw", 20, sizeof(part));
assert (status>=0);
status = H5Pset_external (plist, "extern_4b.raw", 30, sizeof(part));
assert (status>=0);
/* Touch the files so they exist */
system ("touch extern_1b.raw extern_2b.raw extern_3b.raw extern_4b.raw");
/* Create the data space */
size = 100;
space = H5Screate_simple (1, &size, NULL);
assert (space>=0);
/* Create the dataset */
dset = H5Dcreate (file, "dset1", H5T_NATIVE_INT, space, plist);
assert (dset>=0);
/*
* Write the entire dataset and compare with the original
*/
do {
/* Write to the dataset */
printf ("%-70s", "...writing entire dataset");
fflush (stdout);
for (i=0; i<100; i++) whole[i] = i;
status = H5Dwrite (dset, H5T_NATIVE_INT, space, space,
H5P_DEFAULT, whole);
if (status<0) break;
for (i=0; i<4; i++) {
char name1[64], name2[64];
sprintf (name1, "extern_%d.raw", i+1);
sprintf (name2, "extern_%db.raw", i+1);
if (!same_contents (name1, name2)) {
puts ("*FAIL*");
puts (" Output differs from expected value.");
nerrors_g++;
break;
}
}
puts (" PASSED");
} while (0);
H5Dclose (dset);
H5Pclose (plist);
H5Sclose (space);
H5Fclose (file);
}
/*-------------------------------------------------------------------------
* Function: main
@ -531,20 +720,17 @@ test_2 (void)
int
main (void)
{
herr_t status;
int nerrors=0;
H5Eset_auto (display_error_cb, NULL);
status = test_1 ();
nerrors += (status<0 ? 1 : 0);
test_1 ();
test_2 ();
test_3 ();
status = test_2 ();
nerrors += (status<0 ? 1 : 0);
if (0==nerrors) {
if (0==nerrors_g) {
printf ("All external storage tests passed.\n");
} else {
printf ("%d TEST%s FAILED.\n", nerrors_g, 1==nerrors_g?"":"s");
}
exit (nerrors?1:0);
exit (nerrors_g?1:0);
}

10
test/istore.c
View File

@ -316,7 +316,7 @@ test_extend(H5F_t *f, const char *prefix,
memset(buf, 128 + ctr, nelmts);
/* Write to disk */
if (H5F_arr_write(f, &layout, size, size, zero, offset, buf) < 0) {
if (H5F_arr_write(f, &layout, NULL, size, size, zero, offset, buf)<0) {
puts("*FAILED*");
if (!isatty(1)) {
AT();
@ -326,7 +326,8 @@ test_extend(H5F_t *f, const char *prefix,
}
/* Read from disk */
memset(check, 0xff, nelmts);
if (H5F_arr_read(f, &layout, size, size, zero, offset, check) < 0) {
if (H5F_arr_read(f, &layout, NULL, size, size, zero, offset,
check)<0) {
puts("*FAILED*");
if (!isatty(1)) {
AT();
@ -359,7 +360,8 @@ test_extend(H5F_t *f, const char *prefix,
/* Now read the entire array back out and check it */
memset(buf, 0xff, nx * ny * nz);
if (H5F_arr_read(f, &layout, whole_size, whole_size, zero, zero, buf)<0) {
if (H5F_arr_read(f, &layout, NULL, whole_size, whole_size, zero, zero,
buf)<0) {
puts("*FAILED*");
if (!isatty(1)) {
AT();
@ -478,7 +480,7 @@ test_sparse(H5F_t *f, const char *prefix, size_t nblocks,
memset(buf, 128 + ctr, nx * ny * nz);
/* write to disk */
if (H5F_arr_write(f, &layout, size, size, zero, offset, buf) < 0) {
if (H5F_arr_write(f, &layout, NULL, size, size, zero, offset, buf)<0) {
puts("*FAILED*");
if (!isatty(1)) {
AT();