mirror of
https://github.com/HDFGroup/hdf5.git
synced 2025-02-11 16:01:00 +08:00
aa797af31b
addition of new tests. Update manifest and test script for new hdf5 external-storage .h5 files.
410 lines
12 KiB
C
410 lines
12 KiB
C
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
|
||
* Copyright by The HDF Group. *
|
||
* Copyright by the Board of Trustees of the University of Illinois. *
|
||
* All rights reserved. *
|
||
* *
|
||
* This file is part of HDF5. The full HDF5 copyright notice, including *
|
||
* terms governing use, modification, and redistribution, is contained in *
|
||
* the COPYING file, which can be found at the root of the source code *
|
||
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
|
||
* If you do not have access to either file, you may request a copy from *
|
||
* help@hdfgroup.org. *
|
||
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
|
||
|
||
/* For generating files for verifying h5repack with external storage. . .
|
||
*
|
||
* Each case file should follow the format of:
|
||
* + h5repack_<NAME>.h5
|
||
* + h5repack_<NAME>_ex.h5
|
||
* + h5repack_<NAME>_ex-<N>.dat
|
||
* ...where NAME idenfities the type, and N is a positive decimal number;
|
||
* multiple external files (*.dat) are allowed per file, but they must
|
||
* follow the pattern and be in contiguous numerical sequence starting at 0.
|
||
*
|
||
* Each file typename must be added to the listing for
|
||
* `VERIFY_EXTERNAL_CONSOLIDATION` in h5repack.sh
|
||
*
|
||
* There is no restriction on the name, number, or structure of datasets and
|
||
* groups in HDF5 file.
|
||
*/
|
||
|
||
#include "hdf5.h"
|
||
#include "H5private.h"
|
||
|
||
#define MAX_NAME_SIZE 256
|
||
#define FILE_INT32LE_1 "h5repack_int32le_1d"
|
||
#define FILE_INT32LE_2 "h5repack_int32le_2d"
|
||
#define FILE_INT32LE_3 "h5repack_int32le_3d"
|
||
#define FILE_UINT8BE "h5repack_uint8be"
|
||
#define FILE_F32LE "h5repack_f32le"
|
||
|
||
#define H5REPACKGENTEST_OOPS ret_value = -1; goto done;
|
||
|
||
#define H5REPACKGENTEST_COMMON_CLEANUP(dcpl, file, space) { \
|
||
if ((dcpl) != H5P_DEFAULT && (dcpl) != H5I_INVALID_HID) { \
|
||
(void)H5Pclose((dcpl)); \
|
||
} \
|
||
if ((file) != H5I_INVALID_HID) { (void)H5Fclose((file)); } \
|
||
if ((space) != H5I_INVALID_HID) { (void)H5Sclose((space)); } \
|
||
}
|
||
|
||
struct external_def {
|
||
hsize_t type_size;
|
||
unsigned n_elts_per_file;
|
||
unsigned n_elts_total;
|
||
};
|
||
|
||
|
||
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||
* Helper function to create and write a dataset to file.
|
||
* Returns 0 on success, -1 on failure.
|
||
*/
|
||
static int
|
||
__make_dataset(
|
||
hid_t file_id,
|
||
const char *dset_name,
|
||
hid_t mem_type_id,
|
||
hid_t space_id,
|
||
hid_t dcpl_id,
|
||
void *wdata)
|
||
{
|
||
hid_t dset_id = H5I_INVALID_HID;
|
||
int ret_value = 0;
|
||
|
||
dset_id = H5Dcreate2(
|
||
file_id,
|
||
dset_name,
|
||
mem_type_id,
|
||
space_id,
|
||
H5P_DEFAULT,
|
||
dcpl_id,
|
||
H5P_DEFAULT);
|
||
if (dset_id == H5I_INVALID_HID) {
|
||
H5REPACKGENTEST_OOPS;
|
||
}
|
||
|
||
if (H5Dwrite(
|
||
dset_id,
|
||
mem_type_id,
|
||
H5S_ALL,
|
||
H5S_ALL,
|
||
H5P_DEFAULT,
|
||
wdata)
|
||
< 0)
|
||
{
|
||
H5REPACKGENTEST_OOPS;
|
||
}
|
||
|
||
done:
|
||
if (dset_id != H5I_INVALID_HID) { (void)H5Dclose(dset_id); }
|
||
return ret_value;
|
||
} /* end __make_dataset() */
|
||
|
||
|
||
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||
* Helper function to populate the DCPL external storage list.
|
||
* Creates external files for the DCPL, with each file name following the
|
||
* convention "<filename>_ex-<num>.dat". Will append `n_external_files` to
|
||
* the filename list, with each file having space for `n_elts` items of the
|
||
* type (of size `elt_size`). The numeric inputs are not sanity-checked.
|
||
* Returns 0 on success, -1 on failure.
|
||
*/
|
||
static int
|
||
__set_dcpl_external_list(
|
||
hid_t dcpl,
|
||
const char *filename,
|
||
unsigned n_elts_per_file,
|
||
unsigned n_elts_total,
|
||
hsize_t elt_size)
|
||
{
|
||
char name[MAX_NAME_SIZE];
|
||
unsigned n_external_files = 0;
|
||
unsigned i = 0;
|
||
|
||
if (NULL == filename || '\0' == *filename) {
|
||
return -1;
|
||
}
|
||
|
||
n_external_files = n_elts_total / n_elts_per_file;
|
||
if (n_elts_total != (n_external_files * n_elts_per_file)) {
|
||
return -1;
|
||
}
|
||
|
||
for (i = 0; i < n_external_files; i++) {
|
||
if (snprintf(name, MAX_NAME_SIZE, "%s_ex-%u.dat", filename, i) >= MAX_NAME_SIZE) {
|
||
return -1;
|
||
}
|
||
if (H5Pset_external(dcpl, name, 0, n_elts_per_file * elt_size) < 0) {
|
||
return -1;
|
||
}
|
||
}
|
||
return 0;
|
||
} /* end __set_dcpl_external_list() */
|
||
|
||
|
||
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||
* Generalized utility function to write a file with the specified data and
|
||
* dataset configuration. If `ext` is provided, will attempt to use external
|
||
* storage.
|
||
* Returns 0 on success, -1 on failure.
|
||
*/
|
||
static int
|
||
__make_file(
|
||
const char *basename,
|
||
struct external_def *ext,
|
||
hid_t type_id,
|
||
hsize_t rank,
|
||
hsize_t *dims,
|
||
void *wdata)
|
||
{
|
||
char filename[MAX_NAME_SIZE];
|
||
hid_t file_id = H5I_INVALID_HID;
|
||
hid_t dcpl_id = H5P_DEFAULT;
|
||
hid_t space_id = H5I_INVALID_HID;
|
||
int ret_value = 0;
|
||
|
||
if (snprintf(filename,
|
||
MAX_NAME_SIZE,
|
||
"%s%s.h5",
|
||
basename,
|
||
(NULL != ext) ? "_ex" : "")
|
||
>= MAX_NAME_SIZE)
|
||
{
|
||
H5REPACKGENTEST_OOPS;
|
||
}
|
||
|
||
if (NULL != ext) {
|
||
dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
|
||
if (dcpl_id == H5I_INVALID_HID) {
|
||
H5REPACKGENTEST_OOPS;
|
||
}
|
||
if (__set_dcpl_external_list(
|
||
dcpl_id,
|
||
basename,
|
||
ext->n_elts_per_file,
|
||
ext->n_elts_total,
|
||
ext->type_size)
|
||
< 0)
|
||
{
|
||
H5REPACKGENTEST_OOPS;
|
||
}
|
||
}
|
||
|
||
space_id = H5Screate_simple(rank, dims, NULL);
|
||
if (space_id == H5I_INVALID_HID) {
|
||
H5REPACKGENTEST_OOPS;
|
||
}
|
||
|
||
file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
|
||
if (file_id == H5I_INVALID_HID) {
|
||
H5REPACKGENTEST_OOPS;
|
||
}
|
||
|
||
if (__make_dataset(
|
||
file_id,
|
||
"dset",
|
||
type_id,
|
||
space_id,
|
||
dcpl_id,
|
||
wdata)
|
||
< 0)
|
||
{
|
||
H5REPACKGENTEST_OOPS;
|
||
}
|
||
|
||
done:
|
||
H5REPACKGENTEST_COMMON_CLEANUP(dcpl_id, file_id, space_id);
|
||
return ret_value;
|
||
} /* end __make_file() */
|
||
|
||
|
||
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||
* Returns 0 on success, -1 on failure.
|
||
*/
|
||
static int
|
||
generate_int32le_1d(hbool_t external)
|
||
{
|
||
int32_t wdata[12];
|
||
hsize_t dims[] = {12};
|
||
struct external_def *def_ptr = NULL;
|
||
struct external_def def = { (hsize_t)sizeof(int32_t), 6, 12 };
|
||
int32_t n = 0;
|
||
int ret_value = 0;
|
||
|
||
/* Generate values
|
||
*/
|
||
for (n = 0; n < 12; n++) {
|
||
wdata[n] = n-6;
|
||
}
|
||
|
||
def_ptr = (TRUE == external) ? (&def) : NULL;
|
||
if (__make_file(FILE_INT32LE_1, def_ptr, H5T_STD_I32LE, 1, dims, wdata) < 0) {
|
||
ret_value = -1;
|
||
}
|
||
|
||
return ret_value;
|
||
} /* end generate_int32le_1d() */
|
||
|
||
|
||
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||
* Returns 0 on success, -1 on failure.
|
||
*/
|
||
static int
|
||
generate_int32le_2d(hbool_t external)
|
||
{
|
||
int32_t wdata[64];
|
||
hsize_t dims[] = {8, 8};
|
||
struct external_def *def_ptr = NULL;
|
||
struct external_def def = { (hsize_t)sizeof(int32_t), 64, 64 };
|
||
int32_t n = 0;
|
||
int ret_value = 0;
|
||
|
||
/* Generate values
|
||
*/
|
||
for (n = 0; n < 64; n++) {
|
||
wdata[n] = n-32;
|
||
}
|
||
|
||
def_ptr = (TRUE == external) ? (&def) : NULL;
|
||
if (__make_file(FILE_INT32LE_2, def_ptr, H5T_STD_I32LE, 2, dims, wdata) < 0) {
|
||
ret_value = -1;
|
||
}
|
||
|
||
return ret_value;
|
||
} /* end generate_int32le_2d() */
|
||
|
||
|
||
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||
* Returns 0 on success, -1 on failure.
|
||
*/
|
||
static int
|
||
generate_int32le_3d(hbool_t external)
|
||
{
|
||
hsize_t dims[] = {8, 8, 8};
|
||
int32_t wdata[512]; /* 8^3, from dims */
|
||
struct external_def *def_ptr = NULL;
|
||
struct external_def def = { (hsize_t)sizeof(int32_t), 512, 512 };
|
||
int32_t n = 0;
|
||
int i = 0;
|
||
int j = 0;
|
||
int k = 0;
|
||
int ret_value = 0;
|
||
|
||
/* generate values, alternating positive and negative
|
||
*/
|
||
for (i=0, n=0; i < dims[0]; i++) {
|
||
for (j=0; j < dims[1]; j++) {
|
||
for (k=0; k < dims[2]; k++, n++) {
|
||
wdata[n] = (k + j*512 + i*4096) * ((n&1) ? (-1) : (1));
|
||
}
|
||
}
|
||
}
|
||
|
||
def_ptr = (TRUE == external) ? (&def) : NULL;
|
||
if (__make_file(FILE_INT32LE_3, def_ptr, H5T_STD_I32LE, 3, dims, wdata) < 0) {
|
||
ret_value = -1;
|
||
}
|
||
|
||
return ret_value;
|
||
} /* end generate_int32le_3d() */
|
||
|
||
|
||
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||
* Returns 0 on success, -1 on failure.
|
||
*/
|
||
static int
|
||
generate_uint8be(hbool_t external)
|
||
{
|
||
hsize_t dims[] = {4, 8, 8};
|
||
uint8_t wdata[256]; /* 4*8*8, from dims */
|
||
struct external_def *def_ptr = NULL;
|
||
struct external_def def = { (hsize_t)sizeof(uint8_t), 64, 256 };
|
||
uint8_t n = 0;
|
||
int i = 0;
|
||
int j = 0;
|
||
int k = 0;
|
||
int ret_value = 0;
|
||
|
||
/* Generate values, ping-pong from ends of range
|
||
*/
|
||
for (i=0, n=0; i < dims[0]; i++) {
|
||
for (j=0; j < dims[1]; j++) {
|
||
for (k=0; k < dims[2]; k++, n++) {
|
||
wdata[n] = n * ((n&1) ? (-1) : (1));
|
||
}
|
||
}
|
||
}
|
||
|
||
def_ptr = (TRUE == external) ? (&def) : NULL;
|
||
if (__make_file(FILE_UINT8BE, def_ptr, H5T_STD_U8BE, 3, dims, wdata) < 0) {
|
||
ret_value = -1;
|
||
}
|
||
|
||
return ret_value;
|
||
} /* end generate_uint8be() */
|
||
|
||
|
||
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||
* Returns 0 on success, -1 on failure.
|
||
*/
|
||
static int
|
||
generate_f32le(hbool_t external)
|
||
{
|
||
hsize_t dims[] = {12, 6};
|
||
float wdata[72]; /* 12*6, from dims */
|
||
struct external_def *def_ptr = NULL;
|
||
struct external_def def = {
|
||
(hsize_t)sizeof(float),
|
||
72,
|
||
72
|
||
};
|
||
float n = 0;
|
||
int i = 0;
|
||
int j = 0;
|
||
int k = 0;
|
||
int ret_value = 0;
|
||
|
||
/* Generate values */
|
||
for (i=0, k=0, n=0; i < dims[0]; i++) {
|
||
for (j=0; j < dims[1]; j++, k++, n++) {
|
||
wdata[k] = n * 801.1 * ((k % 5 == 1) ? (-1) : (1));
|
||
}
|
||
}
|
||
|
||
def_ptr = (TRUE == external) ? (&def) : NULL;
|
||
if (__make_file(FILE_F32LE, def_ptr, H5T_IEEE_F32LE, 2, dims, wdata) < 0) {
|
||
ret_value = -1;
|
||
}
|
||
|
||
return ret_value;
|
||
} /* end generate_f32le() */
|
||
|
||
|
||
/* ----------------------------------------------------------------------------
|
||
* Create files.
|
||
* Return 0 on success, nonzero on failure.
|
||
*/
|
||
int
|
||
main(void)
|
||
{
|
||
int i = 0;
|
||
int ret_value = 0;
|
||
|
||
for (i=0; i < 2; i++) {
|
||
hbool_t external = (i&1) ? TRUE : FALSE;
|
||
if (ret_value == 0) { ret_value -= generate_int32le_1d(external); }
|
||
if (ret_value == 0) { ret_value -= generate_int32le_2d(external); }
|
||
if (ret_value == 0) { ret_value -= generate_int32le_3d(external); }
|
||
if (ret_value == 0) { ret_value -= generate_uint8be(external); }
|
||
if (ret_value == 0) { ret_value -= generate_f32le(external); }
|
||
} /* end for external data storage or not */
|
||
|
||
if (ret_value != 0) {
|
||
HDprintf("A problem occurred!\n");
|
||
}
|
||
|
||
return ret_value;
|
||
} /* end main() */
|
||
|
||
|