[svn-r29722] Description:

Bring over more dataset tests from the revise_chunks branch.

Tested on:
    MacOSX/64 10.11.4 (amazon) w/serial, parallel & production
    (h5committest forthcoming)
This commit is contained in:
Quincey Koziol 2016-04-17 02:26:51 -05:00
parent ce8905cc90
commit 36062736a5
8 changed files with 1990 additions and 146 deletions

View File

@ -1,15 +0,0 @@
autotools_rework
The purpose of this branch is to make changes to the autotools files,
particularly configure.ac. This will remove a lot of old cruft and update
the input files. This effort was started in January 2015 and should be
completed within the year.
This branch tracks the trunk.
Dana Robinson and Allen Byrne are the owners of this branch.
This branch should be closed when the bulk of the autotools work is
complete and has been merged into the trunk and 1.8 branch. This
work should be completed by mid-2015.

View File

@ -28,7 +28,6 @@
./Makefile.dist
./Makefile.am
./README.txt
./BRANCH.txt
./acsite.m4
./autogen.sh
./configure.ac
@ -868,6 +867,8 @@
./test/be_extlink2.h5
./test/big.c
./test/bittests.c
./test/btree_idx_1_6.h5
./test/btree_idx_1_8.h5
./test/btree2.c
./test/cache.c
./test/cache_api.c

View File

@ -65,6 +65,23 @@ typedef herr_t (*H5VM_opvv_func_t)(hsize_t dst_off, hsize_t src_off,
} /* end if */ \
}
/* Given a coordinate offset array (COORDS) of type TYPE, move the value at
* offset 0 to offset of the unlimied dimension (UNLIM_DIM), sliding any
* intermediate values up one position. Undoes the "swizzle_coords" operation.
*/
#define H5VM_unswizzle_coords(TYPE,COORDS,UNLIM_DIM) { \
/* COORDS must be an array of type TYPE */ \
HDassert(sizeof(COORDS[0]) == sizeof(TYPE)); \
\
/* Nothing to do when unlimited dimension is at position 0 */ \
if(0 != (UNLIM_DIM)) { \
TYPE _tmp = (COORDS)[0]; \
\
HDmemmove(&(COORDS)[0], &(COORDS)[1], sizeof(TYPE) * (UNLIM_DIM)); \
(COORDS)[UNLIM_DIM] = _tmp; \
} /* end if */ \
}
/* A null pointer is equivalent to a zero vector */
#define H5VM_ZERO NULL

View File

@ -128,7 +128,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse
max_compact_dataset.h5 simple.h5 set_local.h5 random_chunks.h5 \
huge_chunks.h5 chunk_cache.h5 big_chunk.h5 chunk_fast.h5 chunk_expand.h5 \
chunk_fixed.h5 copy_dcpl_newfile.h5 partial_chunks.h5 layout_extend.h5 \
zero_chunk.h5 storage_size.h5 \
zero_chunk.h5 dls_01_strings.h5 storage_size.h5 \
extend.h5 istore.h5 extlinks*.h5 frspace.h5 links*.h5 \
sys_file1 tfile[1-7].h5 th5s[1-4].h5 lheap.h5 fheap.h5 ohdr.h5 \
stab.h5 extern_[1-5].h5 extern_[1-4][rw].raw gheap[0-4].h5 \

BIN
test/btree_idx_1_6.h5 Normal file

Binary file not shown.

BIN
test/btree_idx_1_8.h5 Normal file

Binary file not shown.

File diff suppressed because it is too large Load Diff

View File

@ -81,8 +81,17 @@ const char *FILENAME[] = {
test_random_rank4_dump(NDIM_SETS, dim_log, cdims, J, K, L, M); \
goto error; \
} /* end RAND4_FAIL_DUMP */
#define RAND4_VL_NITER 40
#define RAND4_VL_SPARSE_SWITCH 5
static int do_ranks( hid_t fapl );
typedef enum rank4_index_t {
RANK4_INDEX_BTREE = 0, /* Use b-tree (1/2) as chunk index */
RANK4_INDEX_FARRAY, /* Use fixed array as chunk index */
RANK4_INDEX_EARRAY, /* Use extensible array as chunk index */
RANK4_NINDICES, /* Must be last */
} rank4_index_t;
static int do_ranks( hid_t fapl, hbool_t new_format );
static int do_layouts( hid_t fapl );
static int test_rank1( hid_t fapl,
@ -104,7 +113,14 @@ static int test_random_rank4( hid_t fapl,
hid_t dcpl,
hbool_t do_fillvalue,
hbool_t disable_edge_filters,
hbool_t do_sparse);
hbool_t do_sparse,
rank4_index_t index_type);
static int test_random_rank4_vl( hid_t fapl,
hid_t dcpl,
hbool_t do_fillvalue,
hbool_t disable_edge_filters,
hbool_t do_sparse,
rank4_index_t index_type);
static int test_external( hid_t fapl );
static int test_layouts( H5D_layout_t layout, hid_t fapl );
@ -178,7 +194,7 @@ int main( void )
H5F_LIBVER_LATEST) < 0) TEST_ERROR
/* Tests which use chunked datasets */
nerrors += do_ranks( my_fapl ) < 0 ? 1 : 0;
nerrors += do_ranks( my_fapl, new_format ) < 0 ? 1 : 0;
} /* end for */
/* Tests which do not use chunked datasets */
@ -215,11 +231,12 @@ error:
* test with several ranks
*-------------------------------------------------------------------------
*/
static int do_ranks( hid_t fapl )
static int do_ranks( hid_t fapl, hbool_t new_format )
{
hbool_t do_fillvalue = FALSE;
hbool_t disable_edge_filters = FALSE;
rank4_index_t index_type;
hid_t dcpl = -1;
int fillvalue = FILL_VALUE;
unsigned config;
@ -329,17 +346,55 @@ static int do_ranks( hid_t fapl )
if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_IFSET) < 0)
TEST_ERROR
if(test_random_rank4(fapl, dcpl, do_fillvalue, disable_edge_filters, FALSE) < 0) {
DO_RANKS_PRINT_CONFIG("Randomized rank 4")
goto error;
} /* end if */
if(!(config & CONFIG_EARLY_ALLOC))
if(test_random_rank4(fapl, dcpl, do_fillvalue, disable_edge_filters, TRUE) < 0) {
DO_RANKS_PRINT_CONFIG("Randomized rank 4 with sparse allocation")
/* Iterate over different index types, but only if using the new format
*/
for(index_type = RANK4_INDEX_BTREE; index_type < RANK4_NINDICES;
index_type++) {
/* Standard test */
if(test_random_rank4(fapl, dcpl, do_fillvalue, disable_edge_filters,
FALSE, index_type) < 0) {
DO_RANKS_PRINT_CONFIG("Randomized rank 4")
printf(" Index: %s\n", index_type == RANK4_INDEX_BTREE
? "btree" : (index_type == RANK4_INDEX_FARRAY ? "farray"
: "earray"));
goto error;
} /* end if */
/* VL test */
if(test_random_rank4_vl(fapl, dcpl, do_fillvalue,
disable_edge_filters, FALSE, index_type) < 0) {
DO_RANKS_PRINT_CONFIG("Randomized rank 4 variable length")
printf(" Index: %s\n", index_type == RANK4_INDEX_BTREE
? "btree" : (index_type == RANK4_INDEX_FARRAY ? "farray"
: "earray"));
goto error;
} /* end if */
/* Sparse allocation test (regular and VL) */
if(!(config & CONFIG_EARLY_ALLOC)) {
if(test_random_rank4(fapl, dcpl, do_fillvalue,
disable_edge_filters, TRUE, index_type) < 0) {
DO_RANKS_PRINT_CONFIG("Randomized rank 4 with sparse allocation")
printf(" Index: %s\n", index_type == RANK4_INDEX_BTREE
? "btree" : (index_type == RANK4_INDEX_FARRAY
? "farray" : "earray"));
goto error;
} /* end if */
if(test_random_rank4_vl(fapl, dcpl, do_fillvalue,
disable_edge_filters, TRUE, index_type) < 0) {
DO_RANKS_PRINT_CONFIG("Randomized rank 4 variable length with sparse allocation")
printf(" Index: %s\n", index_type == RANK4_INDEX_BTREE
? "btree" : (index_type == RANK4_INDEX_FARRAY
? "farray" : "earray"));
goto error;
} /* end if */
} /* end if */
/* Break out if using the old format */
if(!new_format)
break;
} /* end for */
/* Close dcpl */
if(H5Pclose(dcpl) < 0)
TEST_ERROR
@ -2513,14 +2568,16 @@ error:
*-------------------------------------------------------------------------
*/
static int test_random_rank4( hid_t fapl, hid_t dcpl, hbool_t do_fillvalue,
hbool_t disable_edge_filters, hbool_t do_sparse )
hbool_t disable_edge_filters, hbool_t do_sparse,
rank4_index_t index_type )
{
hid_t file = -1;
hid_t dset = -1;
hid_t fspace = -1;
hid_t mspace = -1;
hid_t my_dcpl = -1;
hsize_t dims[4]; /* Dataset's dimensions */
hsize_t dims[4] = {10, 10, 10, 10}; /* Dataset's dimensions */
hsize_t max_dims[4] = {10, 10, 10, 10}; /* Maximum dimensions */
hsize_t old_dims[4]; /* Old dataset dimensions */
hsize_t min_unwritten_dims[4]; /* Minimum dimensions since last write */
hsize_t *valid_dims = old_dims; /* Dimensions of region still containing written data */
@ -2532,26 +2589,46 @@ static int test_random_rank4( hid_t fapl, hid_t dcpl, hbool_t do_fillvalue,
static hsize_t dim_log[RAND4_NITER+1][4]; /* Log of dataset dimensions */
hbool_t zero_dim = FALSE; /* Whether a dimension is 0 */
hbool_t writing = TRUE; /* Whether we're writing to the dset */
unsigned scalar_iter; /* Iteration to shrink dset to 1x1x1x1 */
volatile unsigned i, j, k, l, m; /* Local indices */
char filename[NAME_BUF_SIZE];
/*!FIXME Skip the test if a fixed array index is requested, as resizing
* fixed arrays is broken now. Extensible arrays are also broken. Remove
* these lines as appropriate when these problems are fixed. */
/* Fixed Array index type is now fixed */
if(index_type == RANK4_INDEX_EARRAY)
return 0;
/* create a new file */
h5_fixname(FILENAME[4], fapl, filename, sizeof filename);
if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR
/* Set maximum dimensions as appropriate for index type */
if(index_type == RANK4_INDEX_BTREE)
for(i=0; i<4; i++)
max_dims[i] = H5S_UNLIMITED;
else if(index_type == RANK4_INDEX_EARRAY)
max_dims[1] = H5S_UNLIMITED;
/* Generate random chunk dimensions, 2-4 */
for(i=0; i<4; i++)
cdims[i] = (hsize_t)((HDrandom() % 3) + 2);
/* Generate initial dataset size, 1-10 */
/* Pick iteration to shrink dataset to 1x1x1x1 */
scalar_iter = (unsigned)(HDrandom() % RAND4_NITER);
/* Generate initial dataset size, 1-10, unless using fixed array index or
* scalar_iter is 0 */
for(i=0; i<4; i++) {
dims[i] = (hsize_t)((HDrandom() % 10) + 1);
dims[i] = (hsize_t)(index_type != RANK4_INDEX_FARRAY
? (0 == scalar_iter ? 1 : ((HDrandom() % 10) + 1)) : 10);
dim_log[0][i] = dims[i];
} /* end for */
/* Create dataset */
if((fspace = H5Screate_simple(4, dims, mdims)) < 0)
if((fspace = H5Screate_simple(4, dims, max_dims)) < 0)
TEST_ERROR
if((my_dcpl = H5Pcopy(dcpl)) < 0)
TEST_ERROR
@ -2590,11 +2667,13 @@ static int test_random_rank4( hid_t fapl, hid_t dcpl, hbool_t do_fillvalue,
RAND4_FAIL_DUMP(i+1, -1, -1, -1, -1)
} /* end if */
/* Generate new dataset size, 0-10 (0 much less likely) */
/* Generate new dataset size, 0-10 (0 much less likely). If i is
* scalar_iter, set all dims to 1. */
zero_dim = FALSE;
for(j=0; j<4; j++) {
old_dims[j] = dims[j];
if((dims[j] = (hsize_t)(HDrandom() % 11)) == 0)
if((dims[j] = (hsize_t)(i == scalar_iter ? 1 : (HDrandom() % 11)))
== 0)
if((dims[j] = (hsize_t)(HDrandom() % 11)) == 0)
zero_dim = TRUE;
dim_log[i+1][j] = dims[j];
@ -2681,6 +2760,289 @@ error:
return -1;
} /* end test_random_rank4 */
/*-------------------------------------------------------------------------
* Function: test_random_rank4_vl
*
* Purpose: Test expanding and shrinking a rank 4 dataset with
* variable length data in a randomized fashion. Verifies
* that data is preserved (and filled, if do_fillvalue is
* true) as expected.
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Neil Fortner
* Tueday, June 29, 2010
*
*-------------------------------------------------------------------------
*/
static int test_random_rank4_vl( hid_t fapl, hid_t dcpl, hbool_t do_fillvalue,
hbool_t disable_edge_filters, hbool_t do_sparse,
rank4_index_t index_type )
{
hid_t file = -1;
hid_t dset = -1;
hid_t type = -1;
hid_t fspace = -1;
hid_t mspace = -1;
hid_t my_dcpl = -1;
hsize_t dims[4] = {10, 10, 10, 10}; /* Dataset's dimensions */
hsize_t max_dims[4] = {10, 10, 10, 10}; /* Maximum dimensions */
hsize_t old_dims[4]; /* Old dataset dimensions */
hsize_t min_unwritten_dims[4]; /* Minimum dimensions since last write */
hsize_t *valid_dims = old_dims; /* Dimensions of region still containing written data */
hsize_t cdims[4]; /* Chunk dimensions */
const hsize_t mdims[4] = {10, 10, 10, 10}; /* Memory buffer dimensions */
const hsize_t start[4] = {0, 0, 0, 0}; /* Start for hyperslab operations on memory */
static hvl_t rbuf[10][10][10][10]; /* Read buffer */
static hvl_t wbuf[10][10][10][10]; /* Write buffer */
static hsize_t dim_log[RAND4_NITER+1][4]; /* Log of dataset dimensions */
hbool_t zero_dim = FALSE; /* Whether a dimension is 0 */
hbool_t writing = TRUE; /* Whether we're writing to the dset */
hvl_t fill_value; /* Fill value */
unsigned scalar_iter; /* Iteration to shrink dset to 1x1x1x1 */
volatile unsigned i, j, k, l, m; /* Local indices */
char filename[NAME_BUF_SIZE];
/*!FIXME Skip the test if a fixed array index is requested, as resizing
* fixed arrays is broken now. Extensible arrays are also broken. Remove
* these lines as appropriate when these problems are fixed. */
if(index_type == RANK4_INDEX_FARRAY || index_type == RANK4_INDEX_EARRAY)
return 0;
/* Initialize fill value buffers so they aren't freed in case of an error */
fill_value.len = 0;
fill_value.p = NULL;
for(i=0; i<dims[0]; i++)
for(j=0; j<dims[1]; j++)
for(k=0; k<dims[2]; k++)
for(l=0; l<dims[3]; l++) {
rbuf[i][j][k][l].len = 0;
rbuf[i][j][k][l].p = NULL;
wbuf[i][j][k][l].len = 0;
wbuf[i][j][k][l].p = NULL;
} /* end for */
/* Allocate space for VL write buffers, since these never need to be
* reallocated */
for(i=0; i<dims[0]; i++)
for(j=0; j<dims[1]; j++)
for(k=0; k<dims[2]; k++)
for(l=0; l<dims[3]; l++) {
wbuf[i][j][k][l].len = 2;
if(NULL == (wbuf[i][j][k][l].p = HDmalloc(2 * sizeof(int))))
TEST_ERROR;
} /* end for */
/* create a new file */
h5_fixname(FILENAME[4], fapl, filename, sizeof filename);
if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR
/* Create VL type */
if((type = H5Tvlen_create(H5T_NATIVE_INT)) < 0)
TEST_ERROR
/* Set maximum dimensions as appropriate for index type */
if(index_type == RANK4_INDEX_BTREE)
for(i=0; i<4; i++)
max_dims[i] = H5S_UNLIMITED;
else if(index_type == RANK4_INDEX_EARRAY)
max_dims[1] = H5S_UNLIMITED;
/* Generate random chunk dimensions, 2-4 */
for(i=0; i<4; i++)
cdims[i] = (hsize_t)((HDrandom() % 3) + 2);
/* Pick iteration to shrink dataset to 1x1x1x1 */
scalar_iter = (unsigned)(HDrandom() % RAND4_NITER);
/* Generate initial dataset size, 1-10, unless using fixed array index or
* scalar_iter is 0 */
for(i=0; i<4; i++) {
dims[i] = (hsize_t)(index_type != RANK4_INDEX_FARRAY
? (0 == scalar_iter ? 1 : ((HDrandom() % 10) + 1)) : 10);
dim_log[0][i] = dims[i];
} /* end for */
/* Make a copy of the dcpl */
if((my_dcpl = H5Pcopy(dcpl)) < 0)
TEST_ERROR
/* Create VL fill value, if requested */
if(do_fillvalue) {
fill_value.len = 2;
if(NULL == (fill_value.p = HDmalloc(2 * sizeof(int))))
TEST_ERROR
((int *)fill_value.p)[0] = 1;
((int *)fill_value.p)[1] = 2;
if(H5Pset_fill_value(my_dcpl, type, &fill_value) < 0)
TEST_ERROR
} /* end if */
/* Create dataset */
if((fspace = H5Screate_simple(4, dims, max_dims)) < 0)
TEST_ERROR
if(H5Pset_chunk(my_dcpl, 4, cdims) < 0)
TEST_ERROR
if(disable_edge_filters)
if(H5Pset_chunk_opts(my_dcpl, H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS) < 0)
TEST_ERROR
if((dset = H5Dcreate2(file, "dset", type, fspace, H5P_DEFAULT, my_dcpl,
H5P_DEFAULT)) < 0)
TEST_ERROR
if(H5Sclose(fspace) < 0)
TEST_ERROR
/* Create memory space, and set initial selection */
if((mspace = H5Screate_simple(4, mdims, NULL)) < 0)
TEST_ERROR
if(H5Sselect_hyperslab(mspace, H5S_SELECT_SET, start, NULL, dims, NULL)
< 0)
TEST_ERROR
/* Main loop */
for(i=0; i<RAND4_VL_NITER; i++) {
/* Generate random write buffer */
if(writing && !zero_dim) {
for(j=0; j<dims[0]; j++)
for(k=0; k<dims[1]; k++)
for(l=0; l<dims[2]; l++)
for(m=0; m<dims[3]; m++) {
((int *)wbuf[j][k][l][m].p)[0] = HDrandom();
((int *)wbuf[j][k][l][m].p)[1] = HDrandom();
} /* end for */
/* Write data */
if(H5Dwrite(dset, type, mspace, H5S_ALL, H5P_DEFAULT, wbuf) < 0)
RAND4_FAIL_DUMP(i+1, -1, -1, -1, -1)
} /* end if */
/* Generate new dataset size, 0-10 (0 much less likely). If i is
* scalar_iter, set all dims to 1. */
zero_dim = FALSE;
for(j=0; j<4; j++) {
old_dims[j] = dims[j];
if((dims[j] = (hsize_t)(i == scalar_iter ? 1 : (HDrandom() % 11)))
== 0)
if((dims[j] = (hsize_t)(HDrandom() % 11)) == 0)
zero_dim = TRUE;
dim_log[i+1][j] = dims[j];
} /* end for */
/* If writing is disabled, update min_unwritten_dims */
if(!writing)
for(j=0; j<4; j++)
if(old_dims[j] < min_unwritten_dims[j])
min_unwritten_dims[j] = old_dims[j];
/* Resize dataset */
if(H5Dset_extent(dset, dims) < 0)
RAND4_FAIL_DUMP(i+2, -1, -1, -1, -1)
if(!zero_dim) {
/* Read data from resized dataset */
if(H5Sselect_hyperslab(mspace, H5S_SELECT_SET, start, NULL, dims,
NULL) < 0)
RAND4_FAIL_DUMP(i+2, -1, -1, -1, -1)
if(H5Dread(dset, type, mspace, H5S_ALL, H5P_DEFAULT, rbuf) < 0)
RAND4_FAIL_DUMP(i+2, -1, -1, -1, -1)
/* Verify correctness of read data */
if(do_fillvalue) {
for(j=0; j<dims[0]; j++)
for(k=0; k<dims[1]; k++)
for(l=0; l<dims[2]; l++)
for(m=0; m<dims[3]; m++)
if(j >= valid_dims[0] || k >= valid_dims[1]
|| l >= valid_dims[2]
|| m >= valid_dims[3]) {
if(((int *)fill_value.p)[0]
!= ((int *)rbuf[j][k][l][m].p)[0]
|| ((int *)fill_value.p)[1]
!= ((int *)rbuf[j][k][l][m].p)[1])
RAND4_FAIL_DUMP(i+2, (int)j, (int)k, (int)l, (int)m)
} /* end if */
else
if(((int *)wbuf[j][k][l][m].p)[0]
!= ((int *)rbuf[j][k][l][m].p)[0]
|| ((int *)wbuf[j][k][l][m].p)[1]
!= ((int *)rbuf[j][k][l][m].p)[1])
RAND4_FAIL_DUMP(i+2, (int)j, (int)k, (int)l, (int)m)
} /* end if */
else {
for(j=0; j<MIN(dims[0],valid_dims[0]); j++)
for(k=0; k<MIN(dims[1],valid_dims[1]); k++)
for(l=0; l<MIN(dims[2],valid_dims[2]); l++)
for(m=0; m<MIN(dims[3],valid_dims[3]); m++)
if(((int *)wbuf[j][k][l][m].p)[0]
!= ((int *)rbuf[j][k][l][m].p)[0]
|| ((int *)wbuf[j][k][l][m].p)[1]
!= ((int *)rbuf[j][k][l][m].p)[1])
RAND4_FAIL_DUMP(i+2, (int)j, (int)k, (int)l, (int)m)
} /* end else */
/* Free read buffer */
if(H5Dvlen_reclaim(type, mspace, H5P_DEFAULT, rbuf) < 0)
TEST_ERROR
} /* end if */
/* Handle the switch between writing and not writing */
if(do_sparse && !(i % RAND4_VL_SPARSE_SWITCH)) {
writing = !writing;
if(!writing) {
for(j=0; j<4; j++)
min_unwritten_dims[j] = old_dims[j];
valid_dims = min_unwritten_dims;
} /* end if */
else
valid_dims = old_dims;
} /* end if */
} /* end for */
/* Close */
if(H5Sselect_all(mspace) < 0)
TEST_ERROR
if(H5Dvlen_reclaim(type, mspace, H5P_DEFAULT, wbuf) < 0)
TEST_ERROR
free(fill_value.p);
if(H5Sclose(mspace) < 0)
TEST_ERROR
if(H5Pclose(my_dcpl) < 0)
TEST_ERROR
if(H5Dclose(dset) < 0)
TEST_ERROR
if(H5Tclose(type) < 0)
TEST_ERROR
if(H5Fclose(file) < 0)
TEST_ERROR
return 0;
error:
H5E_BEGIN_TRY {
for(i=0; i<dims[0]; i++)
for(j=0; j<dims[1]; j++)
for(k=0; k<dims[2]; k++)
for(l=0; l<dims[3]; l++) {
if(rbuf[i][j][k][l].p)
HDfree(rbuf[i][j][k][l].p);
if(wbuf[i][j][k][l].p)
HDfree(wbuf[i][j][k][l].p);
} /* end for */
if(fill_value.p)
HDfree(fill_value.p);
H5Sclose(fspace);
H5Sclose(mspace);
H5Pclose(dcpl);
H5Dclose(dset);
H5Tclose(type);
H5Fclose(file);
} H5E_END_TRY
return -1;
} /* end test_random_rank4_vl */
/*
* test_random_rank4_dump: Dump debugging info from test_random_rank4 to screen
* after failure.