mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-11-27 02:10:55 +08:00
Code Issues Packages Projects Releases Wiki Activity
d0fe57624f
hdf5/test/istore.c

681 lines
21 KiB
C
Raw Blame History

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://www.hdfgroup.org/licenses. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Purpose: Tests various aspects of indexed raw data storage.
*/
#define H5F_FRIEND /*suppress error about including H5Fpkg */
#include "h5test.h"
#include "H5Dprivate.h"
#include "H5Iprivate.h"
#include "H5Pprivate.h"
#include "H5Fpkg.h"
#include "H5Gprivate.h"
#include "H5Oprivate.h"
#include "H5VMprivate.h"
static const char *FILENAME[] = {"istore", NULL};
#define TEST_SMALL 0x0001
#define TEST_MEDIUM 0x0002
#define TEST_LARGE 0x0004
/* The datatype of the dataset operated on by this test */
#define TEST_DATATYPE H5T_NATIVE_UCHAR
#define TEST_CHUNK_SIZE 50
#define TEST_SPARSE_SIZE 1000000
hsize_t chunk_dims[H5O_LAYOUT_NDIMS];
hsize_t zero[H5O_LAYOUT_NDIMS];
/*-------------------------------------------------------------------------
* Function: is_sparse
*
* Purpose: Determines if the file system of the current working
* directory supports holes.
*
* Return: Success: Non-zero if holes are supported; zero
* otherwise.
*
* Failure: zero
*
*-------------------------------------------------------------------------
*/
static int
is_sparse(void)
{
int fd;
h5_stat_t sb;
if ((fd = HDopen("x.h5", O_RDWR | O_TRUNC | O_CREAT, H5_POSIX_CREATE_MODE_RW)) < 0)
return 0;
if (HDlseek(fd, (1024 * 1024), SEEK_SET) != 1024 * 1024)
return 0;
if (5 != HDwrite(fd, "hello", (size_t)5))
return 0;
if (HDclose(fd) < 0)
return 0;
memset(&sb, 0, sizeof(h5_stat_t));
if (HDstat("x.h5", &sb) < 0)
return 0;
if (HDremove("x.h5") < 0)
return 0;
#ifdef H5_HAVE_STAT_ST_BLOCKS
return ((unsigned long)sb.st_blocks * 512 < (unsigned long)sb.st_size);
#else
return (0);
#endif
}
/*-------------------------------------------------------------------------
* Function: print_array
*
* Purpose: Prints the values in an array
*
* Return: void
*
*-------------------------------------------------------------------------
*/
static void
print_array(uint8_t *array, size_t nx, size_t ny, size_t nz)
{
size_t i, j, k;
for (i = 0; i < nx; i++) {
if (nz > 1) {
fprintf(stderr, "i=%lu:\n", (unsigned long)i);
}
else {
fprintf(stderr, "%03lu:", (unsigned long)i);
}
for (j = 0; j < ny; j++) {
if (nz > 1)
fprintf(stderr, "%03lu:", (unsigned long)j);
for (k = 0; k < nz; k++) {
fprintf(stderr, " %3d", *array++);
}
if (nz > 1)
fprintf(stderr, "\n");
}
fprintf(stderr, "\n");
}
}
/*-------------------------------------------------------------------------
* Function: new_object
*
* Purpose: Creates a new object that refers to a indexed storage of raw
* data. No raw data is stored.
*
* Return: Success: ID of dataset
*
* Failure: -1
*
*-------------------------------------------------------------------------
*/
static hid_t
new_object(hid_t f, const char *name, int ndims, hsize_t dims[], hsize_t cdims[])
{
hid_t dataset; /* Dataset ID */
hid_t space; /* Dataspace ID */
hid_t dcpl; /* Dataset creation property list ID */
/* Create the dataset creation property list */
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
/* Set the chunk dimensions */
if (H5Pset_chunk(dcpl, ndims, cdims) < 0)
TEST_ERROR;
/* Create the dataspace */
if ((space = H5Screate_simple(ndims, dims, NULL)) < 0)
TEST_ERROR;
/* Create the dataset */
if ((dataset = H5Dcreate2(f, name, TEST_DATATYPE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Clean up */
/* Close property lists */
if (H5Pclose(dcpl) < 0)
TEST_ERROR;
/* Close dataspace */
if (H5Sclose(space) < 0)
TEST_ERROR;
return dataset;
error:
return -1;
}
/*-------------------------------------------------------------------------
* Function: test_create
*
* Purpose: Creates a named object that refers to indexed storage of raw
* data. No raw data is stored.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
test_create(hid_t f, const char *prefix)
{
hid_t dataset; /* Dataset ID */
hsize_t dims[H5O_LAYOUT_NDIMS + 1]; /* Dimensions of dataset */
hsize_t my_chunk_dims[H5O_LAYOUT_NDIMS + 1]; /* Dimensions of chunks */
char name[256]; /* Dataset name */
unsigned u; /* Local index variable */
TESTING("istore create");
dims[0] = my_chunk_dims[0] = 1;
for (u = 1; u <= H5S_MAX_RANK; u++) {
/* Initialize the dimension size in this new dimension */
dims[u] = my_chunk_dims[u] = 2;
/* Create chunked dataset of this dimensionality */
snprintf(name, sizeof name, "%s_%02u", prefix, u);
if ((dataset = new_object(f, name, (int)u, dims, my_chunk_dims)) < 0)
return FAIL;
/* Close dataset created */
if (H5Dclose(dataset) < 0)
return FAIL;
}
PASSED();
return SUCCEED;
}
/*-------------------------------------------------------------------------
* Function: test_extend
*
* Purpose: Creates an empty object and then writes to it in such a way
* as to always extend the object's domain without creating
* holes and without causing the object to become concave.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
test_extend(hid_t f, const char *prefix, size_t nx, size_t ny, size_t nz)
{
hid_t dataset; /* Dataset ID */
hid_t fspace; /* Dataset's file dataspace */
hid_t mspace; /* Dataset's memory dataspace */
size_t i, j, k, ctr;
int ndims;
uint8_t *buf = NULL, *check = NULL, *whole = NULL;
char dims[64], s[256], name[256];
hsize_t offset[3];
hsize_t max_corner[3];
hsize_t size[3];
hsize_t whole_size[3];
hsize_t nelmts;
if (!nz) {
if (!ny) {
ndims = 1;
ny = nz = 1;
snprintf(dims, sizeof(dims), "%lu", (unsigned long)nx);
}
else {
ndims = 2;
nz = 1;
snprintf(dims, sizeof(dims), "%lux%lu", (unsigned long)nx, (unsigned long)ny);
}
}
else {
ndims = 3;
snprintf(dims, sizeof(dims), "%lux%lux%lu", (unsigned long)nx, (unsigned long)ny, (unsigned long)nz);
}
snprintf(s, sizeof(s), "istore extend: %s", dims);
TESTING(s);
buf = (uint8_t *)malloc(nx * ny * nz);
check = (uint8_t *)malloc(nx * ny * nz);
whole = (uint8_t *)calloc((size_t)1, nx * ny * nz);
whole_size[0] = nx;
whole_size[1] = ny;
whole_size[2] = nz;
max_corner[0] = 0;
max_corner[1] = 0;
max_corner[2] = 0;
/* Build the new empty object */
snprintf(name, sizeof(name), "%s_%s", prefix, dims);
if ((dataset = new_object(f, name, ndims, whole_size, whole_size)) < 0) {
fprintf(stderr, " Cannot create %u-d object `%s'\n", ndims, name);
goto error;
}
/* Get dataset's dataspace */
if ((fspace = H5Dget_space(dataset)) < 0)
TEST_ERROR;
for (ctr = 0; H5VM_vector_lt_u((unsigned)ndims, max_corner, whole_size); ctr++) {
/* Size and location */
if (0 == ctr) {
offset[0] = offset[1] = offset[2] = 0;
size[0] = size[1] = size[2] = 1;
nelmts = 1;
}
else {
for (i = 0, nelmts = 1; i < (size_t)ndims; i++) {
if (ctr % (size_t)ndims == i) {
offset[i] = max_corner[i];
size[i] = MIN(1, whole_size[i] - offset[i]);
}
else {
offset[i] = 0;
size[i] = max_corner[i];
}
nelmts *= size[i];
}
}
/* Fill the source array */
if (0 == nelmts)
continue;
memset(buf, (signed)(128 + ctr), (size_t)nelmts);
/* Create dataspace for selection in memory */
if ((mspace = H5Screate_simple(1, &nelmts, NULL)) < 0)
TEST_ERROR;
/* Select region in file dataspace */
if (H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, size, NULL) < 0)
TEST_ERROR;
/* Write to disk */
if (H5Dwrite(dataset, TEST_DATATYPE, mspace, fspace, H5P_DEFAULT, buf) < 0) {
H5_FAILED();
fprintf(stderr, " Write failed: ctr=%lu\n", (unsigned long)ctr);
goto error;
}
/* Read from disk */
memset(check, 0xff, (size_t)nelmts);
if (H5Dread(dataset, TEST_DATATYPE, mspace, fspace, H5P_DEFAULT, check) < 0) {
H5_FAILED();
fprintf(stderr, " Read failed: ctr=%lu\n", (unsigned long)ctr);
goto error;
}
if (memcmp(buf, check, (size_t)nelmts) != 0) {
H5_FAILED();
fprintf(stderr, " Read check failed: ctr=%lu\n", (unsigned long)ctr);
fprintf(stderr, " Wrote:\n");
print_array(buf, (size_t)size[0], (size_t)size[1], (size_t)size[2]);
fprintf(stderr, " Read:\n");
print_array(check, (size_t)size[0], (size_t)size[1], (size_t)size[2]);
goto error;
}
/* Close memory dataspace */
if (H5Sclose(mspace) < 0)
TEST_ERROR;
/* Write to `whole' buffer for later checking */
H5VM_hyper_copy((unsigned)ndims, size, whole_size, offset, whole, /*dst*/
size, H5VM_ZERO, buf); /*src*/
/* Update max corner */
for (i = 0; i < (size_t)ndims; i++)
max_corner[i] = MAX(max_corner[i], offset[i] + size[i]);
}
/* Now read the entire array back out and check it */
memset(buf, 0xff, nx * ny * nz);
if (H5Dread(dataset, TEST_DATATYPE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) {
H5_FAILED();
fprintf(stderr, " Read failed for whole array.\n");
goto error;
}
for (i = 0; i < nx; i++) {
for (j = 0; j < ny; j++) {
for (k = 0; k < nz; k++) {
if (whole[i * ny * nz + j * nz + k] != buf[i * ny * nz + j * nz + k]) {
H5_FAILED();
fprintf(stderr, " Check failed at i=%lu", (unsigned long)i);
if (ndims > 1) {
fprintf(stderr, ", j=%lu", (unsigned long)j);
}
if (ndims > 2) {
fprintf(stderr, ", k=%lu", (unsigned long)k);
}
fprintf(stderr, "\n Check array is:\n");
print_array(whole, nx, ny, nz);
fprintf(stderr, " Value read is:\n");
print_array(buf, nx, ny, nz);
goto error;
}
}
}
}
/* Close dataset's dataspace */
if (H5Sclose(fspace) < 0)
TEST_ERROR;
/* Close dataset */
if (H5Dclose(dataset) < 0)
TEST_ERROR;
/* Free memory used */
free(buf);
free(check);
free(whole);
PASSED();
return SUCCEED;
error:
free(buf);
free(check);
free(whole);
return FAIL;
}
/*-------------------------------------------------------------------------
* Function: test_sparse
*
* Purpose: Creates a sparse matrix consisting of NBLOCKS randomly placed
* blocks each of size NX,NY,NZ.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
test_sparse(hid_t f, const char *prefix, size_t nblocks, size_t nx, size_t ny, size_t nz, int skip_test)
{
hid_t dataset; /* Dataset ID */
hid_t fspace; /* Dataset's file dataspace */
hid_t mspace; /* Dataset's memory dataspace */
int ndims;
hsize_t ctr;
char dims[64], s[256], name[256];
hsize_t offset[3];
hsize_t size[3];
uint8_t *buf = NULL;
hsize_t whole_size[3]; /* Size of dataset's dataspace */
size_t u; /* Local index variable */
if (!nz) {
if (!ny) {
ndims = 1;
ny = nz = 1;
snprintf(dims, sizeof(dims), "%lu", (unsigned long)nx);
}
else {
ndims = 2;
nz = 1;
snprintf(dims, sizeof(dims), "%lux%lu", (unsigned long)nx, (unsigned long)ny);
}
}
else {
ndims = 3;
snprintf(dims, sizeof(dims), "%lux%lux%lu", (unsigned long)nx, (unsigned long)ny, (unsigned long)nz);
}
snprintf(s, sizeof(s), "istore sparse: %s", dims);
TESTING(s);
if (skip_test) {
SKIPPED();
return SUCCEED;
}
buf = (uint8_t *)malloc(nx * ny * nz);
memset(buf, 128, nx * ny * nz);
/* Set dimensions of dataset */
for (u = 0; u < (size_t)ndims; u++)
whole_size[u] = TEST_SPARSE_SIZE;
/* Set dimensions of selection */
size[0] = nx;
size[1] = ny;
size[2] = nz;
/* Build the new empty object */
snprintf(name, sizeof(name), "%s_%s", prefix, dims);
if ((dataset = new_object(f, name, ndims, whole_size, chunk_dims)) < 0) {
printf(" Cannot create %u-d object `%s'\n", ndims, name);
goto error;
}
/* Get dataset's dataspace */
if ((fspace = H5Dget_space(dataset)) < 0)
TEST_ERROR;
/* Create dataspace for memory buffer */
if ((mspace = H5Screate_simple(ndims, size, NULL)) < 0)
TEST_ERROR;
for (ctr = 0; ctr < nblocks; ctr++) {
offset[0] = (hsize_t)(rand() % (int)(TEST_SPARSE_SIZE - nx));
offset[1] = (hsize_t)(rand() % (int)(TEST_SPARSE_SIZE - ny));
offset[2] = (hsize_t)(rand() % (int)(TEST_SPARSE_SIZE - nz));
/* Select region in file dataspace */
if (H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, size, NULL) < 0)
TEST_ERROR;
/* write to disk */
if (H5Dwrite(dataset, TEST_DATATYPE, mspace, fspace, H5P_DEFAULT, buf) < 0) {
H5_FAILED();
printf(" Write failed: ctr=%lu\n", (unsigned long)ctr);
printf(" offset=(%lu", (unsigned long)(offset[0]));
if (ndims > 1)
printf(",%lu", (unsigned long)(offset[1]));
if (ndims > 2)
printf(",%lu", (unsigned long)(offset[2]));
printf("), size=(%lu", (unsigned long)(size[0]));
if (ndims > 1)
printf(",%lu", (unsigned long)(size[1]));
if (ndims > 2)
printf(",%lu", (unsigned long)(size[2]));
printf(")\n");
goto error;
}
/* We don't test reading yet.... */
}
/* Close memory dataspace */
if (H5Sclose(mspace) < 0)
TEST_ERROR;
/* Close dataset's dataspace */
if (H5Sclose(fspace) < 0)
TEST_ERROR;
/* Close dataset */
if (H5Dclose(dataset) < 0)
TEST_ERROR;
free(buf);
PASSED();
return SUCCEED;
error:
free(buf);
return FAIL;
}
/*-------------------------------------------------------------------------
* Function: main
*
* Purpose: Tests indexed storage
*
* Return: EXIT_SUCCESS/EXIT_FAILURE
*
*-------------------------------------------------------------------------
*/
int
main(int argc, char *argv[])
{
hid_t fapl = H5I_INVALID_HID, file = H5I_INVALID_HID, fcpl = H5I_INVALID_HID;
herr_t status;
int nerrors = 0;
unsigned size_of_test;
unsigned u; /* Local index variable */
char filename[1024];
int skip_test = 0;
int has_sparse_support = 0;
/* Parse arguments or assume these tests (`small', `medium' ) */
if (1 == argc) {
size_of_test = TEST_SMALL | TEST_MEDIUM | TEST_LARGE;
}
else {
int i;
for (i = 1, size_of_test = 0; i < argc; i++) {
if (!strcmp(argv[i], "small")) {
size_of_test |= TEST_SMALL;
}
else if (!strcmp(argv[i], "medium")) {
size_of_test |= TEST_MEDIUM;
}
else if (!strcmp(argv[i], "large")) {
size_of_test |= TEST_LARGE;
}
else {
printf("unrecognized argument: %s\n", argv[i]);
}
}
}
printf("Test sizes: ");
if (size_of_test & TEST_SMALL)
printf(" SMALL");
if (size_of_test & TEST_MEDIUM)
printf(" MEDIUM");
if (size_of_test & TEST_LARGE)
printf(" LARGE");
printf("\n");
/* Set the random # seed */
srand((unsigned)time(NULL));
/* Check to see if the file system supports POSIX-style sparse files.
* Windows NTFS does not, so we want to avoid tests which create
* very large files.
*/
has_sparse_support = is_sparse();
/* Reset library */
h5_test_init();
fapl = h5_fileaccess();
/* Use larger file addresses... */
fcpl = H5Pcreate(H5P_FILE_CREATE);
H5Pset_sizes(fcpl, (size_t)8, (size_t)0);
/* Create the test file */
h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) {
printf("Cannot create file %s; test aborted\n", filename);
exit(EXIT_FAILURE);
}
/* Initialize chunk dimensions */
for (u = 0; u < H5O_LAYOUT_NDIMS; u++)
chunk_dims[u] = TEST_CHUNK_SIZE;
/*
* Creation test: Creates empty objects with various raw data sizes
* and alignments.
*/
status = test_create(file, "create");
nerrors += status < 0 ? 1 : 0;
if (size_of_test & TEST_SMALL) {
status = test_extend(file, "extend", (size_t)10, (size_t)0, (size_t)0);
nerrors += status < 0 ? 1 : 0;
status = test_extend(file, "extend", (size_t)10, (size_t)10, (size_t)0);
nerrors += status < 0 ? 1 : 0;
status = test_extend(file, "extend", (size_t)10, (size_t)10, (size_t)10);
nerrors += status < 0 ? 1 : 0;
}
if (size_of_test & TEST_MEDIUM) {
status = test_extend(file, "extend", (size_t)10000, (size_t)0, (size_t)0);
nerrors += status < 0 ? 1 : 0;
status = test_extend(file, "extend", (size_t)2500, (size_t)10, (size_t)0);
nerrors += status < 0 ? 1 : 0;
status = test_extend(file, "extend", (size_t)10, (size_t)400, (size_t)10);
nerrors += status < 0 ? 1 : 0;
}
skip_test = 0;
if (size_of_test & TEST_SMALL) {
status = test_sparse(file, "sparse", (size_t)100, (size_t)5, (size_t)0, (size_t)0, skip_test);
nerrors += status < 0 ? 1 : 0;
status = test_sparse(file, "sparse", (size_t)100, (size_t)3, (size_t)4, (size_t)0, skip_test);
nerrors += status < 0 ? 1 : 0;
status = test_sparse(file, "sparse", (size_t)100, (size_t)2, (size_t)3, (size_t)4, skip_test);
nerrors += status < 0 ? 1 : 0;
}
if (size_of_test & TEST_MEDIUM) {
status = test_sparse(file, "sparse", (size_t)1000, (size_t)30, (size_t)0, (size_t)0, skip_test);
nerrors += status < 0 ? 1 : 0;
status = test_sparse(file, "sparse", (size_t)2000, (size_t)7, (size_t)3, (size_t)0, skip_test);
nerrors += status < 0 ? 1 : 0;
status = test_sparse(file, "sparse", (size_t)2000, (size_t)4, (size_t)2, (size_t)3, skip_test);
nerrors += status < 0 ? 1 : 0;
}
skip_test = !has_sparse_support;
if (size_of_test & TEST_LARGE) {
/* This test is skipped if there is no POSIX-style sparse file support
* e.g.: Windows NTFS filesystems
*/
status = test_sparse(file, "sparse", (size_t)800, (size_t)50, (size_t)50, (size_t)50, skip_test);
if (skip_test)
printf(" The current VFD does not support sparse files on this platform.\n");
nerrors += status < 0 ? 1 : 0;
}
/* Close the test file and exit */
H5Pclose(fcpl);
H5Fclose(file);
/* Verify symbol table messages are cached */
nerrors += (h5_verify_cached_stabs(FILENAME, fapl) < 0 ? 1 : 0);
if (nerrors) {
printf("***** %d I-STORE TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S");
exit(EXIT_FAILURE);
}
printf("All i-store tests passed.\n");
h5_cleanup(FILENAME, fapl);
exit(EXIT_SUCCESS);
}
Reference in New Issue View Git Blame Copy Permalink