mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-11-27 02:10:55 +08:00
Code Issues Packages Projects Releases Wiki Activity
8751379484
hdf5/test/istore.c
Quincey Koziol 43e3b45021 [svn-r6825] Purpose: 
New feature/enhancement

Description:
    Chunked datasets are handled poorly in several circumstances involving
certain selections and chunks that are too large for the chunk cache and/or
chunks with filters, causing the chunk to be read from disk multiple times.

Solution:
    Rearrange raw data I/O infrastructure to handle chunked datasets in a much
more friendly way by creating a selection in memory and on disk for each chunk
in a chunked dataset and performing all of the I/O on that chunk at one time.

    There are still some scalability (the current code attempts to
create a selection for all the chunks in the dataset, instead of just the
chunks that are accessed, requiring portions of the istore.c and fillval.c
tests to be commented out) and performance issues, but checking this in will
allow the changes to be tested by a much wider audience while I address the
remaining issues.


Platforms tested:
    h5committested, FreeBSD 4.8 (sleipnir) serial & parallel, Linux 2.4 (eirene)
2003-05-07 16:52:24 -05:00

699 lines
19 KiB
C
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the files COPYING and Copyright.html. COPYING can be found at the root *
* of the source code distribution tree; Copyright.html can be found at the *
* root level of an installed copy of the electronic HDF5 document set and *
* is linked from the top-level documents page. It can also be found at *
* http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have *
* access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* Programmer: Robb Matzke <matzke@llnl.gov>
* Wednesday, October 15, 1997
*
* Purpose: Tests various aspects of indexed raw data storage.
*/
#define H5F_PACKAGE /*suppress error about including H5Fpkg */
#include "h5test.h"
#include "H5private.h"
#include "H5Dprivate.h"
#include "H5Eprivate.h"
#include "H5Iprivate.h"
#include "H5Pprivate.h"
#include "H5Fpkg.h"
#include "H5Gprivate.h"
#include "H5MMprivate.h"
#include "H5Oprivate.h"
#include "H5Pprivate.h"
#include "H5Vprivate.h"
const char *FILENAME[] = {
"istore",
NULL
};
#define TEST_SMALL 0x0001
#define TEST_MEDIUM 0x0002
#define TEST_LARGE 0x0004
/* The datatype of the dataset operated on by this test */
#define TEST_DATATYPE H5T_NATIVE_UCHAR
#define TEST_CHUNK_SIZE 50
#define TEST_SPARSE_SIZE 1000000
hsize_t chunk_dims[H5O_LAYOUT_NDIMS];
hssize_t zero[H5O_LAYOUT_NDIMS];
/*-------------------------------------------------------------------------
* Function: print_array
*
* Purpose: Prints the values in an array
*
* Return: void
*
* Programmer: Robb Matzke
* Friday, October 10, 1997
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static void
print_array(uint8_t *array, size_t nx, size_t ny, size_t nz)
{
size_t i, j, k;
for (i = 0; i < nx; i++) {
if (nz > 1) {
fprintf(stderr,"i=%lu:\n", (unsigned long)i);
} else {
fprintf(stderr,"%03lu:", (unsigned long)i);
}
for (j = 0; j < ny; j++) {
if (nz > 1)
fprintf(stderr,"%03lu:", (unsigned long)j);
for (k = 0; k < nz; k++) {
fprintf(stderr," %3d", *array++);
}
if (nz > 1)
fprintf(stderr,"\n");
}
fprintf(stderr,"\n");
}
}
/*-------------------------------------------------------------------------
* Function: new_object
*
* Purpose: Creates a new object that refers to a indexed storage of raw
* data. No raw data is stored.
*
* Return: Success: ID of dataset
*
* Failure: -1
*
* Programmer: Robb Matzke
* Wednesday, October 15, 1997
*
* Modifications:
* Converted to use datasets instead of directly messing with
* the istore routines, etc. since the new raw data architecture
* performs hyperslab operations at a higher level than the
* istore routines did and the new istore routines can't handle
* I/O on more than one chunk at a time. QAK - 2003/04/16
*
*-------------------------------------------------------------------------
*/
static hid_t
new_object(hid_t f, const char *name, int ndims, hsize_t dims[], hsize_t cdims[])
{
hid_t dataset; /* Dataset ID */
hid_t space; /* Dataspace ID */
hid_t dcpl; /* Dataset creation property list ID */
/* Create the dataset creation property list */
if ((dcpl=H5Pcreate(H5P_DATASET_CREATE))<0) TEST_ERROR;
/* Set the chunk dimensions */
if(H5Pset_chunk(dcpl, ndims, cdims) < 0) TEST_ERROR;
/* Create the dataspace */
if((space = H5Screate_simple(ndims, dims, NULL))<0) TEST_ERROR;
/* Create the dataset */
if((dataset = H5Dcreate (f, name, TEST_DATATYPE, space, dcpl))<0) TEST_ERROR;
/* Clean up */
/* Close property lists */
if(H5Pclose(dcpl)<0) TEST_ERROR;
/* Close dataspace */
if(H5Sclose(space)<0) TEST_ERROR;
return dataset;
error:
return -1;
}
/*-------------------------------------------------------------------------
* Function: test_create
*
* Purpose: Creates a named object that refers to indexed storage of raw
* data. No raw data is stored.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
* Wednesday, October 15, 1997
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
test_create(hid_t f, const char *prefix)
{
hid_t dataset; /* Dataset ID */
hsize_t dims[H5O_LAYOUT_NDIMS+1]; /* Dimensions of dataset */
char name[256]; /* Dataset name */
unsigned u; /* Local index variable */
TESTING("istore create");
dims[0]=TEST_CHUNK_SIZE;
for (u = 1; u <= H5S_MAX_RANK; u++) {
/* Initialize the dimension size in this new dimension */
dims[u]=TEST_CHUNK_SIZE;
/* Create chunked dataset of this dimensionality */
HDsnprintf(name, sizeof name, "%s_%02u", prefix, u);
if ((dataset=new_object(f, name, (int)u, dims, chunk_dims)) < 0)
return FAIL;
/* Close dataset created */
if(H5Dclose(dataset)<0)
return FAIL;
}
PASSED();
return SUCCEED;
}
/*-------------------------------------------------------------------------
* Function: test_extend
*
* Purpose: Creates an empty object and then writes to it in such a way
* as to always extend the object's domain without creating
* holes and without causing the object to become concave.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
* Wednesday, October 15, 1997
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
test_extend(hid_t f, const char *prefix,
size_t nx, size_t ny, size_t nz)
{
hid_t dataset; /* Dataset ID */
hid_t fspace; /* Dataset's file dataspace */
hid_t mspace; /* Dataset's memory dataspace */
hsize_t i, j, k, ctr;
int ndims;
uint8_t *buf = NULL, *check = NULL, *whole = NULL;
char dims[64], s[256], name[256];
hssize_t offset[3];
hssize_t max_corner[3];
hsize_t size[3];
hsize_t whole_size[3];
hsize_t nelmts;
if (!nz) {
if (!ny) {
ndims = 1;
ny = nz = 1;
sprintf(dims, "%lu", (unsigned long) nx);
} else {
ndims = 2;
nz = 1;
sprintf(dims, "%lux%lu", (unsigned long) nx, (unsigned long) ny);
}
} else {
ndims = 3;
sprintf(dims, "%lux%lux%lu",
(unsigned long) nx, (unsigned long) ny, (unsigned long) nz);
}
sprintf(s, "Testing istore extend: %s", dims);
TESTING(s);
buf = H5MM_malloc(nx * ny * nz);
check = H5MM_malloc(nx * ny * nz);
whole = H5MM_calloc(nx*ny*nz);
whole_size[0] = nx;
whole_size[1] = ny;
whole_size[2] = nz;
max_corner[0] = 0;
max_corner[1] = 0;
max_corner[2] = 0;
/* Build the new empty object */
sprintf(name, "%s_%s", prefix, dims);
if ((dataset=new_object(f, name, ndims, whole_size, whole_size)) < 0) {
fprintf(stderr," Cannot create %u-d object `%s'\n", ndims, name);
goto error;
}
/* Get dataset's dataspace */
if((fspace=H5Dget_space(dataset))<0) TEST_ERROR;
for (ctr = 0;
H5V_vector_lt_s((unsigned)ndims, max_corner, (hssize_t*)whole_size);
ctr++) {
/* Size and location */
if (0 == ctr) {
offset[0] = offset[1] = offset[2] = 0;
size[0] = size[1] = size[2] = 1;
nelmts = 1;
} else {
for (i=0, nelmts=1; i<(size_t)ndims; i++) {
if (ctr % ndims == i) {
offset[i] = max_corner[i];
size[i] = MIN(1, whole_size[i] - offset[i]);
} else {
offset[i] = 0;
size[i] = max_corner[i];
}
nelmts *= size[i];
}
}
#if 0
if (0 == ctr)
fprintf(stderr,"\n");
fprintf(stderr," Insert: ctr=%d, corner=(%d", ctr, offset[0]);
if (ndims > 1)
fprintf(stderr,",%d", offset[1]);
if (ndims > 2)
fprintf(stderr,",%d", offset[2]);
fprintf(stderr,"), size=(%d", size[0]);
if (ndims > 1)
fprintf(stderr,",%d", size[1]);
if (ndims > 2)
fprintf(stderr,",%d", size[2]);
fprintf(stderr,"), %d element%s", nelmts, 1 == nelmts ? "" : "s");
if (0 == nelmts)
fprintf(stderr," *SKIPPED*");
fprintf(stderr,"\n");
#endif
/* Fill the source array */
if (0 == nelmts) continue;
HDmemset(buf, (signed)(128+ctr), (size_t)nelmts);
/* Create dataspace for selection in memory */
if((mspace=H5Screate_simple(1,&nelmts,NULL))<0) TEST_ERROR;
/* Select region in file dataspace */
if(H5Sselect_hyperslab(fspace,H5S_SELECT_SET,offset,NULL,size,NULL)<0) TEST_ERROR;
/* Write to disk */
if (H5Dwrite(dataset, TEST_DATATYPE, mspace, fspace, H5P_DEFAULT, buf)<0) {
H5_FAILED();
fprintf(stderr," Write failed: ctr=%lu\n", (unsigned long)ctr);
goto error;
}
/* Read from disk */
HDmemset(check, 0xff, (size_t)nelmts);
if (H5Dread(dataset, TEST_DATATYPE, mspace, fspace, H5P_DEFAULT, check)<0) {
H5_FAILED();
fprintf(stderr," Read failed: ctr=%lu\n", (unsigned long)ctr);
goto error;
}
if (HDmemcmp(buf, check, (size_t)nelmts)) {
H5_FAILED();
fprintf(stderr," Read check failed: ctr=%lu\n", (unsigned long)ctr);
fprintf(stderr," Wrote:\n");
print_array(buf, (size_t)size[0], (size_t)size[1],
(size_t)size[2]);
fprintf(stderr," Read:\n");
print_array(check, (size_t)size[0], (size_t)size[1],
(size_t)size[2]);
goto error;
}
/* Close memory dataspace */
if(H5Sclose(mspace)<0) TEST_ERROR;
/* Write to `whole' buffer for later checking */
H5V_hyper_copy((unsigned)ndims, size,
whole_size, offset, whole, /*dst*/
size, H5V_ZERO, buf); /*src*/
/* Update max corner */
for (i=0; i<(size_t)ndims; i++) {
max_corner[i] = MAX(max_corner[i], offset[i]+(hssize_t)size[i]);
}
}
/* Now read the entire array back out and check it */
HDmemset(buf, 0xff, nx * ny * nz);
if (H5Dread(dataset, TEST_DATATYPE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)<0) {
H5_FAILED();
fprintf(stderr," Read failed for whole array.\n");
goto error;
}
for (i=0; i<nx; i++) {
for (j=0; j<ny; j++) {
for (k=0; k<nz; k++) {
if (whole[i*ny*nz + j*nz + k] != buf[i*ny*nz + j*nz + k]) {
H5_FAILED();
fprintf(stderr," Check failed at i=%lu", (unsigned long)i);
if (ndims > 1) {
fprintf(stderr,", j=%lu", (unsigned long)j);
}
if (ndims > 2) {
fprintf(stderr,", k=%lu", (unsigned long)k);
}
fprintf(stderr,"\n Check array is:\n");
print_array(whole, nx, ny, nz);
fprintf(stderr," Value read is:\n");
print_array(buf, nx, ny, nz);
goto error;
}
}
}
}
/* Close dataset's dataspace */
if(H5Sclose(fspace)<0) TEST_ERROR;
/* Close dataset */
if(H5Dclose(dataset)<0) TEST_ERROR;
/* Free memory used */
H5MM_xfree(buf);
H5MM_xfree(check);
H5MM_xfree(whole);
PASSED();
return SUCCEED;
error:
H5MM_xfree(buf);
H5MM_xfree(check);
H5MM_xfree(whole);
return FAIL;
}
/*-------------------------------------------------------------------------
* Function: test_sparse
*
* Purpose: Creates a sparse matrix consisting of NBLOCKS randomly placed
* blocks each of size NX,NY,NZ.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
* Wednesday, October 22, 1997
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
test_sparse(hid_t f, const char *prefix, size_t nblocks,
size_t nx, size_t ny, size_t nz)
{
hid_t dataset; /* Dataset ID */
hid_t fspace; /* Dataset's file dataspace */
hid_t mspace; /* Dataset's memory dataspace */
int ndims;
hsize_t ctr;
char dims[64], s[256], name[256];
hssize_t offset[3];
hsize_t size[3], total = 0;
uint8_t *buf = NULL;
hsize_t whole_size[3]; /* Size of dataset's dataspace */
size_t u; /* Local index variable */
if (!nz) {
if (!ny) {
ndims = 1;
ny = nz = 1;
sprintf(dims, "%lu", (unsigned long) nx);
} else {
ndims = 2;
nz = 1;
sprintf(dims, "%lux%lu", (unsigned long) nx, (unsigned long) ny);
}
} else {
ndims = 3;
sprintf(dims, "%lux%lux%lu",
(unsigned long) nx, (unsigned long) ny, (unsigned long) nz);
}
sprintf(s, "Testing istore sparse: %s", dims);
TESTING(s);
buf = H5MM_malloc(nx * ny * nz);
/* Set dimensions of dataset */
for (u=0; u<(size_t)ndims; u++)
whole_size[u]=TEST_SPARSE_SIZE;
/* Set dimensions of selection */
size[0] = nx;
size[1] = ny;
size[2] = nz;
/* Build the new empty object */
sprintf(name, "%s_%s", prefix, dims);
if ((dataset=new_object(f, name, ndims, whole_size, chunk_dims)) < 0) {
printf(" Cannot create %u-d object `%s'\n", ndims, name);
goto error;
}
/* Get dataset's dataspace */
if((fspace=H5Dget_space(dataset))<0) TEST_ERROR;
/* Create dataspace for memory buffer */
if((mspace=H5Screate_simple(ndims,size,NULL))<0) TEST_ERROR;
for (ctr=0; ctr<nblocks; ctr++) {
offset[0] = HDrandom() % (TEST_SPARSE_SIZE-nx);
offset[1] = HDrandom() % (TEST_SPARSE_SIZE-ny);
offset[2] = HDrandom() % (TEST_SPARSE_SIZE-nz);
HDmemset(buf, (signed)(128+ctr), nx * ny * nz);
/* Select region in file dataspace */
if(H5Sselect_hyperslab(fspace,H5S_SELECT_SET,offset,NULL,size,NULL)<0) TEST_ERROR;
/* write to disk */
if (H5Dwrite(dataset, TEST_DATATYPE, mspace, fspace, H5P_DEFAULT, buf)<0) {
H5_FAILED();
printf(" Write failed: ctr=%lu\n", (unsigned long)ctr);
printf(" offset=(%lu", (unsigned long) (offset[0]));
if (ndims > 1)
printf(",%lu", (unsigned long) (offset[1]));
if (ndims > 2)
printf(",%lu", (unsigned long) (offset[2]));
printf("), size=(%lu", (unsigned long) (size[0]));
if (ndims > 1)
printf(",%lu", (unsigned long) (size[1]));
if (ndims > 2)
printf(",%lu", (unsigned long) (size[2]));
printf(")\n");
goto error;
}
total += nx * ny * nz;
#if 0
printf("ctr: ctr=%d, total=%lu\n", ctr, (unsigned long) total);
#endif
/* We don't test reading yet.... */
}
/* Close memory dataspace */
if(H5Sclose(mspace)<0) TEST_ERROR;
/* Close dataset's dataspace */
if(H5Sclose(fspace)<0) TEST_ERROR;
/* Close dataset */
if(H5Dclose(dataset)<0) TEST_ERROR;
H5MM_xfree(buf);
PASSED();
return SUCCEED;
error:
H5MM_xfree(buf);
return FAIL;
}
/*-------------------------------------------------------------------------
* Function: main
*
* Purpose: Tests indexed storage stuff.
*
* Return: Success: exit(0)
*
* Failure: exit(non-zero)
*
* Programmer: Robb Matzke
* Wednesday, October 15, 1997
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
int
main(int argc, char *argv[])
{
hid_t fapl=-1, file=-1, fcpl=-1;
herr_t status;
int nerrors = 0;
unsigned size_of_test;
unsigned u; /* Local index variable */
char filename[1024];
/* Parse arguments or assume all tests (`small', `medium' and `large') */
if (1 == argc) {
size_of_test = TEST_SMALL | TEST_MEDIUM | TEST_LARGE;
} else {
int i;
for (i = 1, size_of_test = 0; i < argc; i++) {
if (!strcmp(argv[i], "small")) {
size_of_test |= TEST_SMALL;
} else if (!strcmp(argv[i], "medium")) {
size_of_test |= TEST_MEDIUM;
} else if (!strcmp(argv[i], "large")) {
size_of_test |= TEST_LARGE;
} else {
printf("unrecognized argument: %s\n", argv[i]);
#if 0
exit(1);
#endif
}
}
}
printf("Test sizes: ");
if (size_of_test & TEST_SMALL)
printf(" SMALL");
if (size_of_test & TEST_MEDIUM)
printf(" MEDIUM");
if (size_of_test & TEST_LARGE)
printf(" LARGE");
printf("\n");
/* Set the random # seed */
HDsrandom((unsigned long)time(NULL));
/* Reset library */
h5_reset();
fapl = h5_fileaccess();
/* Use larger file addresses... */
fcpl = H5Pcreate(H5P_FILE_CREATE);
H5Pset_sizes(fcpl, 8, 0);
/* Create the test file */
h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
if ((file=H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl))<0) {
printf("Cannot create file %s; test aborted\n", filename);
exit(1);
}
/*
* For testing file families, fool the library into thinking it already
* allocated a whole bunch of data.
*/
if (H5FD_FAMILY==H5Pget_driver(fapl)) {
haddr_t addr;
H5F_t *f;
addr = 8 * ((uint64_t)1<<30); /*8 GB */
f=H5I_object(file);
if (H5FDset_eoa(f->shared->lf, addr)<0) {
printf("Cannot create large file family\n");
exit(1);
}
}
/* Initialize chunk dimensions */
for (u = 0; u < H5O_LAYOUT_NDIMS; u++)
chunk_dims[u]=TEST_CHUNK_SIZE;
/*
* Creation test: Creates empty objects with various raw data sizes
* and alignments.
*/
status = test_create(file, "create");
nerrors += status < 0 ? 1 : 0;
if (size_of_test & TEST_SMALL) {
status = test_extend(file, "extend", 10, 0, 0);
nerrors += status < 0 ? 1 : 0;
status = test_extend(file, "extend", 10, 10, 0);
nerrors += status < 0 ? 1 : 0;
status = test_extend(file, "extend", 10, 10, 10);
nerrors += status < 0 ? 1 : 0;
}
if (size_of_test & TEST_MEDIUM) {
status = test_extend(file, "extend", 10000, 0, 0);
nerrors += status < 0 ? 1 : 0;
status = test_extend(file, "extend", 2500, 10, 0);
nerrors += status < 0 ? 1 : 0;
status = test_extend(file, "extend", 10, 400, 10);
nerrors += status < 0 ? 1 : 0;
}
#ifdef QAK
if (size_of_test & TEST_SMALL) {
status = test_sparse(file, "sparse", 100, 5, 0, 0);
nerrors += status < 0 ? 1 : 0;
status = test_sparse(file, "sparse", 100, 3, 4, 0);
nerrors += status < 0 ? 1 : 0;
status = test_sparse(file, "sparse", 100, 2, 3, 4);
nerrors += status < 0 ? 1 : 0;
}
if (size_of_test & TEST_MEDIUM) {
status = test_sparse(file, "sparse", 1000, 30, 0, 0);
nerrors += status < 0 ? 1 : 0;
status = test_sparse(file, "sparse", 2000, 7, 3, 0);
nerrors += status < 0 ? 1 : 0;
status = test_sparse(file, "sparse", 2000, 4, 2, 3);
nerrors += status < 0 ? 1 : 0;
}
if (size_of_test & TEST_LARGE) {
status = test_sparse(file, "sparse", 800, 50, 50, 50);
nerrors += status < 0 ? 1 : 0;
}
#else /* QAK */
HDfprintf(stderr,"Uncomment tests!\n");
#endif /* QAK */
/* Close the test file and exit */
H5Pclose(fcpl);
H5Fclose(file);
if (nerrors) {
printf("***** %d I-STORE TEST%s FAILED! *****\n",
nerrors, 1 == nerrors ? "" : "S");
exit(1);
}
printf("All i-store tests passed.\n");
h5_cleanup(FILENAME, fapl);
return 0;
}
Reference in New Issue View Git Blame Copy Permalink