Refactor h5repackgentest to have more modular internals, for easier

addition of new tests.
Update manifest and test script for new hdf5 external-storage .h5 files.
This commit is contained in:
Jacob Smith 2019-09-23 16:21:37 -05:00
parent 7d3e170763
commit aa797af31b
22 changed files with 278 additions and 154 deletions

View File

@ -2552,16 +2552,26 @@
./tools/test/h5repack/testfiles/h5repack_early.h5
./tools/test/h5repack/testfiles/h5repack_ext.bin
./tools/test/h5repack/testfiles/h5repack_ext.h5
./tools/test/h5repack/testfiles/h5repack_f32le.h5
./tools/test/h5repack/testfiles/h5repack_f32le_ex-0.dat
./tools/test/h5repack/testfiles/h5repack_f32le_ex.h5
./tools/test/h5repack/testfiles/h5repack_fill.h5
./tools/test/h5repack/testfiles/h5repack_filters.h5
./tools/test/h5repack/testfiles/h5repack_fletcher.h5
./tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5
./tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5
./tools/test/h5repack/testfiles/h5repack_hlink.h5
./tools/test/h5repack/testfiles/h5repack_int32le_1d.h5
./tools/test/h5repack/testfiles/h5repack_int32le_1d_ex-0.dat
./tools/test/h5repack/testfiles/h5repack_int32le_1d_ex-1.dat
./tools/test/h5repack/testfiles/h5repack_int32le_1d_ex.h5
./tools/test/h5repack/testfiles/h5repack_int32le_2d.h5
./tools/test/h5repack/testfiles/h5repack_int32le_2d_ex-0.dat
./tools/test/h5repack/testfiles/h5repack_int32le_2d_ex.h5
./tools/test/h5repack/testfiles/h5repack_int32le_3d.h5
./tools/test/h5repack/testfiles/h5repack_int32le_3d_ex-0.dat
./tools/test/h5repack/testfiles/h5repack_int32le_3d_ex.h5
./tools/test/h5repack/testfiles/h5repack.info
./tools/test/h5repack/testfiles/h5repack_int32le.h5
./tools/test/h5repack/testfiles/h5repack_int32le_ex-0.dat
./tools/test/h5repack/testfiles/h5repack_int32le_ex.h5
./tools/test/h5repack/testfiles/h5repack_layout.h5
./tools/test/h5repack/testfiles/h5repack_layouto.h5
./tools/test/h5repack/testfiles/h5repack_layout2.h5
@ -2584,10 +2594,6 @@
./tools/test/h5repack/testfiles/h5repack_uint8be_ex-1.dat
./tools/test/h5repack/testfiles/h5repack_uint8be_ex-2.dat
./tools/test/h5repack/testfiles/h5repack_uint8be_ex-3.dat
./tools/test/h5repack/testfiles/h5repack_uint8be_ex-4.dat
./tools/test/h5repack/testfiles/h5repack_uint8be_ex-5.dat
./tools/test/h5repack/testfiles/h5repack_uint8be_ex-6.dat
./tools/test/h5repack/testfiles/h5repack_uint8be_ex-7.dat
./tools/test/h5repack/testfiles/h5repack_uint8be_ex.h5
./tools/test/h5repack/testfiles/ublock.bin
./tools/test/h5repack/testfiles/crtorder.tordergr.h5.ddl

View File

@ -115,8 +115,14 @@ $SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5
$SRC_H5REPACK_TESTFILES/h5repack_none.h5
$SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5
$SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_ex.h5
$SRC_H5REPACK_TESTFILES/h5repack_f32le.h5
$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d.h5
$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex.h5
$SRC_H5REPACK_TESTFILES/h5repack_uint8be.h5
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex.h5
$SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
@ -164,7 +170,11 @@ $SRC_H5REPACK_TESTFILES/ublock.bin
$SRC_H5REPACK_TESTFILES/h5repack.info
$SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl
$SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl
$SRC_H5REPACK_TESTFILES/h5repack_int32le_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-1.dat
$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl
$SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst
@ -173,10 +183,6 @@ $SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-1.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-2.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-3.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-4.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-5.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-6.dat
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-7.dat
$SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst
$SRC_H5REPACK_TESTFILES/1_vds.h5-vds_dset_chunk20x10x5-v.ddl
$SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl
@ -647,7 +653,7 @@ DIFFFAIL()
# -----------------------------------------------------------------------------
VERIFY_EXTERNAL_CONSOLIDATION()
{
for name in int32le uint8be
for name in f32le int32le_1d int32le_2d int32le_3d uint8be
do
basename="h5repack_${name}"
base_ext="${basename}_ex"
@ -672,7 +678,7 @@ VERIFY_EXTERNAL_CONSOLIDATION()
DIFFTEST $repacked $canonical
DIFFTEST $repacked $external
# invalidate external file by moving its data file; verify
# invalidate external file by moving its first data file; verify
( cd $TESTDIR
mv $ex_dat $ex_dat_mv
)

View File

@ -32,8 +32,11 @@
#include "H5private.h"
#define MAX_NAME_SIZE 256
#define FILE_INT32LE "h5repack_int32le"
#define FILE_INT32LE_1 "h5repack_int32le_1d"
#define FILE_INT32LE_2 "h5repack_int32le_2d"
#define FILE_INT32LE_3 "h5repack_int32le_3d"
#define FILE_UINT8BE "h5repack_uint8be"
#define FILE_F32LE "h5repack_f32le"
#define H5REPACKGENTEST_OOPS ret_value = -1; goto done;
@ -45,37 +48,43 @@
if ((space) != H5I_INVALID_HID) { (void)H5Sclose((space)); } \
}
struct external_def {
hsize_t type_size;
unsigned n_elts_per_file;
unsigned n_elts_total;
};
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Helper function to create and write a dataset to file.
* Returns 0 on success, -1 on failure.
*/
static int
make_dataset(
hid_t file,
__make_dataset(
hid_t file_id,
const char *dset_name,
hid_t mem_type_id,
hid_t space,
hid_t dcpl,
hid_t space_id,
hid_t dcpl_id,
void *wdata)
{
hid_t dset = H5I_INVALID_HID;
hid_t dset_id = H5I_INVALID_HID;
int ret_value = 0;
dset = H5Dcreate2(
file,
"dset",
dset_id = H5Dcreate2(
file_id,
dset_name,
mem_type_id,
space,
space_id,
H5P_DEFAULT,
dcpl,
dcpl_id,
H5P_DEFAULT);
if (dset == H5I_INVALID_HID) {
if (dset_id == H5I_INVALID_HID) {
H5REPACKGENTEST_OOPS;
}
if (H5Dwrite(
dset,
dset_id,
mem_type_id,
H5S_ALL,
H5S_ALL,
@ -87,47 +96,194 @@ make_dataset(
}
done:
if (dset != H5I_INVALID_HID) { (void)H5Dclose(dset); }
if (dset_id != H5I_INVALID_HID) { (void)H5Dclose(dset_id); }
return ret_value;
} /* end make_dataset() */
} /* end __make_dataset() */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Helper function to populate the DCPL external storage list.
* Creates external files for the DCPL, with each file name following the
* convention "<filename>_ex-<num>.dat". Will append `n_external_files` to
* the filename list, with each file having space for `n_elts` items of the
* type (of size `elt_size`). The numeric inputs are not sanity-checked.
* Returns 0 on success, -1 on failure.
*/
static int
set_dcpl_external_list(
__set_dcpl_external_list(
hid_t dcpl,
const char *filename,
hsize_t n_external_files,
hsize_t n_elts,
hsize_t elt_size)
unsigned n_elts_per_file,
unsigned n_elts_total,
hsize_t elt_size)
{
hsize_t i = 0;
char name[MAX_NAME_SIZE];
char name[MAX_NAME_SIZE];
unsigned n_external_files = 0;
unsigned i = 0;
if (NULL == filename || '\0' == *filename) {
return -1;
}
n_external_files = n_elts_total / n_elts_per_file;
if (n_elts_total != (n_external_files * n_elts_per_file)) {
return -1;
}
for (i = 0; i < n_external_files; i++) {
snprintf(name, MAX_NAME_SIZE, "%s_ex-%llu.dat", filename, i);
if (H5Pset_external(dcpl, name, 0, n_elts * elt_size) < 0) {
if (snprintf(name, MAX_NAME_SIZE, "%s_ex-%u.dat", filename, i) >= MAX_NAME_SIZE) {
return -1;
}
if (H5Pset_external(dcpl, name, 0, n_elts_per_file * elt_size) < 0) {
return -1;
}
}
return 0;
} /* end set_dcpl_external_list() */
} /* end __set_dcpl_external_list() */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Generalized utility function to write a file with the specified data and
* dataset configuration. If `ext` is provided, will attempt to use external
* storage.
* Returns 0 on success, -1 on failure.
*/
static int
generate_int32le(hbool_t external)
__make_file(
const char *basename,
struct external_def *ext,
hid_t type_id,
hsize_t rank,
hsize_t *dims,
void *wdata)
{
char filename[MAX_NAME_SIZE];
hid_t file = H5I_INVALID_HID;
hid_t dcpl = H5P_DEFAULT;
hid_t space = H5I_INVALID_HID;
hsize_t dims[] = {8, 8, 8};
int32_t wdata[512]; /* 8^3, from dims */
hid_t file_id = H5I_INVALID_HID;
hid_t dcpl_id = H5P_DEFAULT;
hid_t space_id = H5I_INVALID_HID;
int ret_value = 0;
if (snprintf(filename,
MAX_NAME_SIZE,
"%s%s.h5",
basename,
(NULL != ext) ? "_ex" : "")
>= MAX_NAME_SIZE)
{
H5REPACKGENTEST_OOPS;
}
if (NULL != ext) {
dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
if (dcpl_id == H5I_INVALID_HID) {
H5REPACKGENTEST_OOPS;
}
if (__set_dcpl_external_list(
dcpl_id,
basename,
ext->n_elts_per_file,
ext->n_elts_total,
ext->type_size)
< 0)
{
H5REPACKGENTEST_OOPS;
}
}
space_id = H5Screate_simple(rank, dims, NULL);
if (space_id == H5I_INVALID_HID) {
H5REPACKGENTEST_OOPS;
}
file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
if (file_id == H5I_INVALID_HID) {
H5REPACKGENTEST_OOPS;
}
if (__make_dataset(
file_id,
"dset",
type_id,
space_id,
dcpl_id,
wdata)
< 0)
{
H5REPACKGENTEST_OOPS;
}
done:
H5REPACKGENTEST_COMMON_CLEANUP(dcpl_id, file_id, space_id);
return ret_value;
} /* end __make_file() */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Returns 0 on success, -1 on failure.
*/
static int
generate_int32le_1d(hbool_t external)
{
int32_t wdata[12];
hsize_t dims[] = {12};
struct external_def *def_ptr = NULL;
struct external_def def = { (hsize_t)sizeof(int32_t), 6, 12 };
int32_t n = 0;
int ret_value = 0;
/* Generate values
*/
for (n = 0; n < 12; n++) {
wdata[n] = n-6;
}
def_ptr = (TRUE == external) ? (&def) : NULL;
if (__make_file(FILE_INT32LE_1, def_ptr, H5T_STD_I32LE, 1, dims, wdata) < 0) {
ret_value = -1;
}
return ret_value;
} /* end generate_int32le_1d() */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Returns 0 on success, -1 on failure.
*/
static int
generate_int32le_2d(hbool_t external)
{
int32_t wdata[64];
hsize_t dims[] = {8, 8};
struct external_def *def_ptr = NULL;
struct external_def def = { (hsize_t)sizeof(int32_t), 64, 64 };
int32_t n = 0;
int ret_value = 0;
/* Generate values
*/
for (n = 0; n < 64; n++) {
wdata[n] = n-32;
}
def_ptr = (TRUE == external) ? (&def) : NULL;
if (__make_file(FILE_INT32LE_2, def_ptr, H5T_STD_I32LE, 2, dims, wdata) < 0) {
ret_value = -1;
}
return ret_value;
} /* end generate_int32le_2d() */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Returns 0 on success, -1 on failure.
*/
static int
generate_int32le_3d(hbool_t external)
{
hsize_t dims[] = {8, 8, 8};
int32_t wdata[512]; /* 8^3, from dims */
struct external_def *def_ptr = NULL;
struct external_def def = { (hsize_t)sizeof(int32_t), 512, 512 };
int32_t n = 0;
int i = 0;
int j = 0;
@ -136,160 +292,115 @@ generate_int32le(hbool_t external)
/* generate values, alternating positive and negative
*/
for (i=0, n=0; i < 8; i++) {
for (j=0; j < 8; j++) {
for (k=0; k < 8; k++, n++) {
for (i=0, n=0; i < dims[0]; i++) {
for (j=0; j < dims[1]; j++) {
for (k=0; k < dims[2]; k++, n++) {
wdata[n] = (k + j*512 + i*4096) * ((n&1) ? (-1) : (1));
}
}
}
snprintf(filename,
MAX_NAME_SIZE,
"%s%s.h5",
FILE_INT32LE,
(external) ? "_ex" : "");
if (external) {
dcpl = H5Pcreate(H5P_DATASET_CREATE);
if (dcpl == H5I_INVALID_HID) {
H5REPACKGENTEST_OOPS;
}
if (set_dcpl_external_list(
dcpl,
FILE_INT32LE,
1,
512,
sizeof(int32_t))
< 0)
{
H5REPACKGENTEST_OOPS;
}
def_ptr = (TRUE == external) ? (&def) : NULL;
if (__make_file(FILE_INT32LE_3, def_ptr, H5T_STD_I32LE, 3, dims, wdata) < 0) {
ret_value = -1;
}
space = H5Screate_simple(3, dims, NULL);
if (space == H5I_INVALID_HID) {
H5REPACKGENTEST_OOPS;
}
file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
if (file == H5I_INVALID_HID) {
H5REPACKGENTEST_OOPS;
}
if (make_dataset(
file,
"dset",
H5T_STD_I32LE,
space,
dcpl,
wdata)
< 0)
{
H5REPACKGENTEST_OOPS;
}
done:
H5REPACKGENTEST_COMMON_CLEANUP(dcpl, file, space);
return ret_value;
} /* end generate_int32le() */
} /* end generate_int32le_3d() */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Returns 0 on success, -1 on failure.
*/
static int
generate_uint8be(hbool_t external)
{
char filename[MAX_NAME_SIZE];
hid_t file = H5I_INVALID_HID;
hid_t dcpl = H5P_DEFAULT;
hid_t space = H5I_INVALID_HID;
hsize_t dims[] = {8, 8, 8};
uint8_t wdata[512]; /* 8^3, from dims */
hsize_t dims[] = {4, 8, 8};
uint8_t wdata[256]; /* 4*8*8, from dims */
struct external_def *def_ptr = NULL;
struct external_def def = { (hsize_t)sizeof(uint8_t), 64, 256 };
uint8_t n = 0;
int i = 0;
int j = 0;
int k = 0;
int ret_value = 0;
/* Generate values, alternating positive and negative
* The latter half of the dataset is "overflow garbage" (TODO?)
/* Generate values, ping-pong from ends of range
*/
for (i=0, n=0; i < 8; i++) {
for (j=0; j < 8; j++) {
for (k=0; k < 8; k++, n++) {
for (i=0, n=0; i < dims[0]; i++) {
for (j=0; j < dims[1]; j++) {
for (k=0; k < dims[2]; k++, n++) {
wdata[n] = n * ((n&1) ? (-1) : (1));
}
}
}
snprintf(filename,
MAX_NAME_SIZE,
"%s%s.h5",
FILE_UINT8BE,
(external) ? "_ex" : "");
if (external) {
dcpl = H5Pcreate(H5P_DATASET_CREATE);
if (dcpl == H5I_INVALID_HID) {
H5REPACKGENTEST_OOPS;
}
if (set_dcpl_external_list(
dcpl,
FILE_UINT8BE,
8,
64,
sizeof(uint8_t))
< 0)
{
H5REPACKGENTEST_OOPS;
}
def_ptr = (TRUE == external) ? (&def) : NULL;
if (__make_file(FILE_UINT8BE, def_ptr, H5T_STD_U8BE, 3, dims, wdata) < 0) {
ret_value = -1;
}
space = H5Screate_simple(3, dims, NULL);
if (space == H5I_INVALID_HID) {
H5REPACKGENTEST_OOPS;
}
file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
if (file == H5I_INVALID_HID) {
H5REPACKGENTEST_OOPS;
}
if (make_dataset(
file,
"dset",
H5T_STD_U8BE,
space,
dcpl,
wdata)
< 0)
{
H5REPACKGENTEST_OOPS;
}
done:
H5REPACKGENTEST_COMMON_CLEANUP(dcpl, file, space);
return ret_value;
} /* end generate_uint8be() */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Returns 0 on success, -1 on failure.
*/
static int
generate_f32le(hbool_t external)
{
hsize_t dims[] = {12, 6};
float wdata[72]; /* 12*6, from dims */
struct external_def *def_ptr = NULL;
struct external_def def = {
(hsize_t)sizeof(float),
72,
72
};
float n = 0;
int i = 0;
int j = 0;
int k = 0;
int ret_value = 0;
/* Generate values */
for (i=0, k=0, n=0; i < dims[0]; i++) {
for (j=0; j < dims[1]; j++, k++, n++) {
wdata[k] = n * 801.1 * ((k % 5 == 1) ? (-1) : (1));
}
}
def_ptr = (TRUE == external) ? (&def) : NULL;
if (__make_file(FILE_F32LE, def_ptr, H5T_IEEE_F32LE, 2, dims, wdata) < 0) {
ret_value = -1;
}
return ret_value;
} /* end generate_f32le() */
/* ----------------------------------------------------------------------------
* Create files.
* Return 0 on success, nonzero on failure.
*/
int
main(void)
{
int i = 0; /* loop variable */
int i = 0;
int ret_value = 0;
for (i=0; i < 2; i++) {
hbool_t external = (i&1) ? TRUE : FALSE;
if (ret_value == 0) { ret_value -= generate_int32le(external); }
if (ret_value == 0) { ret_value -= generate_int32le_1d(external); }
if (ret_value == 0) { ret_value -= generate_int32le_2d(external); }
if (ret_value == 0) { ret_value -= generate_int32le_3d(external); }
if (ret_value == 0) { ret_value -= generate_uint8be(external); }
if (ret_value == 0) { ret_value -= generate_f32le(external); }
} /* end for external data storage or not */
if (ret_value != 0) {
/* TODO: print warning message? */
HDprintf("A problem occurred!\n");
}
return ret_value;

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

Binary file not shown.