mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-11-21 01:04:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

New references for Dimension Scale APIs (#1139)

Browse Source 
* Enable usage of new-style references with dimension scale APIs

* Add API to check if an object ID represents a native connector object

* Modified code to use new function H5DSwith_new_ref to determine if new references should be used with
Dimension Scales. The new function return TRUE if non-native connector is used or if H5_DIMENSION_SCALES_WITH_NEW_REF
varible is define at configure time (--enable-dimension-scales-with-new-ref).

Tested on jelly.

ToDo: generate testing file on BE system and enable the test; add flag to CMake; test netCDF-4 with the new references.

* Adding new test files generated on BE system (hedgehog) created by 32 and 64-bit library.

test_ds chokes on test_ds_le_new_ref.h5  on BE system; test passes for test_ds_be_new_ref-32bit.h5
for the 32-bit library and fails for the 64-bit library, and vice versa. I am checking the files for further
investigation; but current implementation of the new references is not portable between LE and BE systems,
and 32 and 64-bit systems.

* Minor fixes for testing issues

* Update test_ds.c

Enabled broken test; tests pass now.

* Update RELEASE.txt

Documented new option to use new references with the HDF5 dimension scales APIs (H5DS*).

* Update MANIFEST for new 32-bit new-style references test file for H5DS APIs

* Update 'dimension scales w/ new-style refs' feature based on review

Co-authored-by: Elena <epourmal@hdfgroup.org>
This commit is contained in:
jhendersonHDF 2021-11-05 07:33:02 -05:00 committed by GitHub
parent 55ee1fd655
commit 93ac9b0aeb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 827 additions and 426 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
MANIFEST
View File

@ -3066,7 +3066,10 @@
./hl/test/sepia.pal
./hl/test/test_ds.c
./hl/test/test_ds_be.h5
./hl/test/test_ds_be_new_ref.h5
./hl/test/test_ds_be_new_ref-32bit.h5
./hl/test/test_ds_le.h5
./hl/test/test_ds_le_new_ref.h5
./hl/test/test_dset_append.c
./hl/test/test_file_image.c
./hl/test/test_h5do_compat.c

3
config/cmake/H5pubconf.h.in
View File

@ -406,6 +406,9 @@
/* Define if the high-level library headers should be included in hdf5.h */
#cmakedefine H5_INCLUDE_HL @H5_INCLUDE_HL@
/* Define if new-style references should be used with dimension scales */
#cmakedefine H5_DIMENSION_SCALES_WITH_NEW_REF @H5_DIMENSION_SCALES_WITH_NEW_REF@
/* Define if your system can convert long double to (unsigned) long long
values correctly. */
#cmakedefine H5_LDOUBLE_TO_LLONG_ACCURATE @H5_LDOUBLE_TO_LLONG_ACCURATE@

1
config/cmake/libhdf5.settings.cmake.in
View File

@ -68,6 +68,7 @@ Features:
Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@
Large Parallel I/O: @LARGE_PARALLEL_IO@
High-level library: @HDF5_BUILD_HL_LIB@
Dimension scales w/ new references: @DIMENSION_SCALES_WITH_NEW_REF@
Build HDF5 Tests: @BUILD_TESTING@
Build HDF5 Tools: @HDF5_BUILD_TOOLS@
Build High-level HDF5 Tools: @HDF5_BUILD_HL_TOOLS@

32
configure.ac
View File

@ -875,6 +875,38 @@ else
fi
## ----------------------------------------------------------------------
## Enable new references for dimension scales
##
AC_SUBST([DIMENSION_SCALES_WITH_NEW_REF])
AC_MSG_CHECKING([whether to use new references with dimension scales]);
AC_ARG_ENABLE([dimension-scales-with-new-ref],
[AS_HELP_STRING([--enable-dimension-scales-with-new-ref],
[Use new references when creating dimension scales.
[default=no]
])],
[DIMENSION_SCALES_WITH_NEW_REF=$enableval])
## Set the default value to use old references.
if test "X-$DIMENSION_SCALES_WITH_NEW_REF" = X- ; then
DIMENSION_SCALES_WITH_NEW_REF=no
fi
case "X-$DIMENSION_SCALES_WITH_NEW_REF" in
X-yes)
AC_MSG_RESULT([yes])
AC_DEFINE([DIMENSION_SCALES_WITH_NEW_REF], [1],
[Define if new references for dimension scales were requested])
;;
X-no)
AC_MSG_RESULT([no])
;;
*)
AC_MSG_ERROR([Unrecognized value: $DIMENSION_SCALES_WITH_NEW_REF])
;;
esac
## ----------------------------------------------------------------------
## Check which archiving tool to use. This needs to be done before
## the AM_PROG_LIBTOOL macro.

8
hl/CMakeLists.txt
View File

@ -1,6 +1,14 @@
cmake_minimum_required (VERSION 3.12)
project (HDF5_HL C)
#-----------------------------------------------------------------------------
# Option to use new-style references with dimension scale APIs
#-----------------------------------------------------------------------------
option (HDF5_DIMENSION_SCALES_NEW_REF "Use new-style references with dimension scale APIs" OFF)
if (HDF5_DIMENSION_SCALES_NEW_REF)
set (H5_DIMENSION_SCALES_WITH_NEW_REF 1)
endif ()
#-----------------------------------------------------------------------------
# List Source files
#-----------------------------------------------------------------------------

604
hl/src/H5DS.c
View File

File diff suppressed because it is too large Load Diff

8
hl/src/H5DSprivate.h
View File

@ -20,12 +20,18 @@
/* public LT prototypes */
#include "H5DSpublic.h"
/* attribute type of a DS dataset */
/* attribute type of a DS dataset when old references are used*/
typedef struct ds_list_t {
hobj_ref_t ref; /* object reference */
unsigned int dim_idx; /* dimension index of the dataset */
} ds_list_t;
/* attribute type of a DS dataset when new references are used*/
typedef struct nds_list_t {
H5R_ref_t ref;
unsigned int dim_idx; /* dimension index of the dataset */
} nds_list_t;
/*-------------------------------------------------------------------------
* private functions
*-------------------------------------------------------------------------

2
hl/src/H5DSpublic.h
View File

@ -25,6 +25,8 @@ typedef herr_t (*H5DS_iterate_t)(hid_t dset, unsigned dim, hid_t scale, void *vi
extern "C" {
#endif
H5_HLDLL hbool_t H5DSwith_new_ref(hid_t obj_id);
H5_HLDLL herr_t H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx);
H5_HLDLL herr_t H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx);

62
hl/src/H5LT.c
View File

@ -1855,46 +1855,6 @@ H5LTset_attribute_double(hid_t loc_id, const char *obj_name, const char *attr_na
return 0;
}
/*-------------------------------------------------------------------------
* Function: find_attr
*
* Purpose: operator function used by H5LT_find_attribute
*
* Programmer: Pedro Vicente
*
* Date: June 21, 2001
*
* Comments:
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
find_attr(H5_ATTR_UNUSED hid_t loc_id, const char *name, H5_ATTR_UNUSED const H5A_info_t *ainfo,
void *op_data)
{
int ret = H5_ITER_CONT;
/* check the arguments */
if (name == NULL)
return H5_ITER_CONT;
/* Shut compiler up */
(void)loc_id;
(void)ainfo;
/* Define a positive value for return value if the attribute was found. This will
* cause the iterator to immediately return that positive value,
* indicating short-circuit success
*/
if (HDstrncmp(name, (char *)op_data, MAX(HDstrlen((char *)op_data), HDstrlen(name))) == 0)
ret = H5_ITER_STOP;
return ret;
}
/*-------------------------------------------------------------------------
* Function: H5LTfind_attribute
*
@ -1926,32 +1886,22 @@ H5LTfind_attribute(hid_t loc_id, const char *attr_name)
*
* Date: June 21, 2001
*
* Comments:
* The function uses H5Aiterate2 with the operator function find_attr
*
* Return:
* Success: The return value of the first operator that
* returns non-zero, or zero if all members were
* processed with no operator returning non-zero.
* Success: Positive if the attribute exists attached to the
* object loc_id. Zero if the attribute does not
* exist attached to the object loc_id.
*
* Failure: Negative if something goes wrong within the
* library, or the negative value returned by one
* of the operators.
* library.
*
*-------------------------------------------------------------------------
*/
/* H5Aiterate wants a non-const pointer but we have a const pointer in the API
* call. It's safe to ignore this because we control the callback, don't
* modify the op_data buffer (i.e.: attr_name) during the traversal, and the
* library never modifies that buffer.
*/
H5_GCC_CLANG_DIAG_OFF("cast-qual")
herr_t
H5LT_find_attribute(hid_t loc_id, const char *attr_name)
{
return H5Aiterate2(loc_id, H5_INDEX_NAME, H5_ITER_INC, NULL, find_attr, (void *)attr_name);
htri_t attr_exists = H5Aexists(loc_id, attr_name);
return (attr_exists < 0) ? (herr_t)-1 : (attr_exists) ? (herr_t)1 : (herr_t)0;
}
H5_GCC_CLANG_DIAG_ON("cast-qual")
/*-------------------------------------------------------------------------
* Function: H5LTget_attribute_ndims

2
hl/test/CMakeTests.cmake
View File

@ -31,7 +31,9 @@ set (HL_REFERENCE_TEST_FILES
dslat.txt
dslon.txt
test_ds_be.h5
test_ds_be_new_ref.h5
test_ds_le.h5
test_ds_le_new_ref.h5
test_ld.h5
)

4
hl/test/gen_test_ds.c
View File

@ -63,7 +63,11 @@ static int test_foreign_scaleattached(const char *filename);
#define SCALE_4_NAME "scalename_4_"
#define FILENAME "test_ds_"
#ifdef H5_DIMENSION_SCALES_WITH_NEW_REF
#define FILEEXT "_new_ref.h5"
#else
#define FILEEXT ".h5"
#endif
/*-------------------------------------------------------------------------
* the main program

41
hl/test/test_ds.c
View File

@ -137,8 +137,13 @@ static int test_attach_detach(void);
#define DIM0_LABEL "Latitude"
#define DIM1_LABEL "Longitude"
#ifdef H5_DIMENSION_SCALES_WITH_NEW_REF
#define FOREIGN_FILE1 "test_ds_le_new_ref.h5"
#define FOREIGN_FILE2 "test_ds_be_new_ref.h5"
#else
#define FOREIGN_FILE1 "test_ds_le.h5"
#define FOREIGN_FILE2 "test_ds_be.h5"
#endif
#define FILENAME "test_ds"
#define FILEEXT ".h5"
@ -161,10 +166,15 @@ static int test_attach_detach(void);
int
main(void)
{
hid_t file_id = H5I_INVALID_HID;
int nerrors = 0;
/* create file to be used in following tests */
if (create_test_file("1") < 0) {
if ((file_id = create_test_file("1")) < 0) {
nerrors = 1;
goto error;
}
if (H5Fclose(file_id) < 0) {
nerrors = 1;
goto error;
}
@ -179,7 +189,11 @@ main(void)
nerrors += test_long_scalenames("1") < 0 ? 1 : 0;
nerrors += test_float_scalenames("1") < 0 ? 1 : 0;
nerrors += test_numberofscales("1") < 0 ? 1 : 0;
if (create_test_file("2") < 0) {
if ((file_id = create_test_file("2")) < 0) {
nerrors = 1;
goto error;
}
if (H5Fclose(file_id) < 0) {
nerrors = 1;
goto error;
}
@ -257,7 +271,6 @@ create_char_dataset(hid_t fid, const char *dsidx, int fulldims)
int rank = 3;
int rankds = 1;
hsize_t dims[3] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE};
char buf[DIM1_SIZE * DIM2_SIZE * DIM3_SIZE];
hsize_t s1_dim[1] = {DIM1_SIZE};
hsize_t s2_dim[1] = {DIM2_SIZE};
hsize_t s3_dim[1] = {DIM3_SIZE};
@ -275,7 +288,7 @@ create_char_dataset(hid_t fid, const char *dsidx, int fulldims)
HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
/* make a dataset */
if (H5LTmake_dataset_char(fid, name, rank, dims, buf) >= 0) {
if (H5LTmake_dataset_char(fid, name, rank, dims, NULL) >= 0) {
if (fulldims == 0) {
/* make a DS dataset for the first dimension */
if (create_DS1_char_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@ -312,7 +325,6 @@ create_short_dataset(hid_t fid, const char *dsidx, int fulldims)
int rank = 3;
int rankds = 1;
hsize_t dims[3] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE};
short buf[DIM1_SIZE * DIM2_SIZE * DIM3_SIZE];
hsize_t s1_dim[1] = {DIM1_SIZE};
hsize_t s2_dim[1] = {DIM2_SIZE};
hsize_t s3_dim[1] = {DIM3_SIZE};
@ -330,7 +342,7 @@ create_short_dataset(hid_t fid, const char *dsidx, int fulldims)
HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
/* make a dataset */
if (H5LTmake_dataset_short(fid, name, rank, dims, buf) >= 0) {
if (H5LTmake_dataset_short(fid, name, rank, dims, NULL) >= 0) {
if (fulldims == 0) {
/* make a DS dataset for the first dimension */
if (create_DS1_short_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@ -367,7 +379,6 @@ create_int_dataset(hid_t fid, const char *dsidx, int fulldims)
int rank = RANK;
int rankds = 1;
hsize_t dims[RANK] = {DIM1_SIZE, DIM2_SIZE};
int buf[DIM1_SIZE * DIM2_SIZE];
hsize_t s1_dim[1] = {DIM1_SIZE};
hsize_t s2_dim[1] = {DIM2_SIZE};
int s1_wbuf[DIM1_SIZE] = {10, 20, 30};
@ -380,7 +391,7 @@ create_int_dataset(hid_t fid, const char *dsidx, int fulldims)
HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
/* make a dataset */
if (H5LTmake_dataset_int(fid, name, rank, dims, buf) >= 0) {
if (H5LTmake_dataset_int(fid, name, rank, dims, NULL) >= 0) {
if (fulldims == 0) {
/* make a DS dataset for the first dimension */
if (create_DS1_int_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@ -409,7 +420,6 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi
int rank = 4;
int rankds = 1;
hsize_t dims[4] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE, DIM4_SIZE};
long * buf = NULL;
hsize_t s1_dim[1] = {DIM1_SIZE};
hsize_t s2_dim[1] = {DIM2_SIZE};
hsize_t s3_dim[1] = {DIM3_SIZE};
@ -429,12 +439,8 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi
long s43_wbuf[DIM4_SIZE] = {180, 180};
long s44_wbuf[DIM4_SIZE] = {280, 280};
/* Allocate buffer */
if (NULL == (buf = (long *)HDmalloc(sizeof(long) * DIM1_SIZE * DIM2_SIZE * DIM3_SIZE * DIM4_SIZE)))
goto error;
/* make a dataset */
if (H5LTmake_dataset_long(fid, dsname, rank, dims, buf) >= 0) {
if (H5LTmake_dataset_long(fid, dsname, rank, dims, NULL) >= 0) {
if (fulldims == 0) {
/* make a DS dataset for the first dimension */
if (create_DS1_long_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)
@ -471,13 +477,9 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi
else
goto error;
HDfree(buf);
return SUCCEED;
error:
HDfree(buf);
return FAIL;
}
@ -487,7 +489,6 @@ create_float_dataset(hid_t fid, const char *dsidx, int fulldims)
int rank = RANK;
int rankds = 1;
hsize_t dims[RANK] = {DIM1_SIZE, DIM2_SIZE};
float buf[DIM1_SIZE * DIM2_SIZE];
hsize_t s1_dim[1] = {DIM1_SIZE};
hsize_t s2_dim[1] = {DIM2_SIZE};
float s1_wbuf[DIM1_SIZE] = {10, 20, 30};
@ -500,7 +501,7 @@ create_float_dataset(hid_t fid, const char *dsidx, int fulldims)
HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx);
/* make a dataset */
if (H5LTmake_dataset_float(fid, name, rank, dims, buf) >= 0) {
if (H5LTmake_dataset_float(fid, name, rank, dims, NULL) >= 0) {
if (fulldims == 0) {
/* make a DS dataset for the first dimension */
if (create_DS1_float_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0)

BIN
hl/test/test_ds_be_new_ref-32bit.h5 Normal file
View File

Binary file not shown.

BIN
hl/test/test_ds_be_new_ref.h5 Normal file
View File

Binary file not shown.

BIN
hl/test/test_ds_le_new_ref.h5 Normal file
View File

Binary file not shown.

10
release_docs/RELEASE.txt
View File

@ -47,6 +47,16 @@ New Features
Configuration:
-------------
- Added new configure options to enable dimension scales APIs (H5DS*) to
use new object references with the native VOL connector (aka native HDF5
library). New references are always used for non-native terminal VOL
connectors (e.g., DAOS).
Autotools --enable-dimension-scales-with-new-ref
CMake HDF5_DIMENSION_SCALES_NEW_REF=ON
(EIP - 2021/10/25, HDFFV-11180)
- Refactored the utils folder.
Added subfolder test and moved the 'swmr_check_compat_vfd.c file'

31
src/H5VL.c
View File

@ -655,6 +655,37 @@ done:
FUNC_LEAVE_API(ret_value)
} /* H5VLobject() */
/*---------------------------------------------------------------------------
* Function: H5VLobject_is_native
*
* Purpose: Determines whether an object ID represents a native VOL
* connector object.
*
* Return: Success: TRUE/FALSE
* Failure: FAIL
*
*---------------------------------------------------------------------------
*/
hbool_t
H5VLobject_is_native(hid_t obj_id)
{
H5VL_object_t *vol_obj = NULL;
hbool_t ret_value = FALSE;
FUNC_ENTER_API(FALSE)
H5TRACE1("b", "i", obj_id);
/* Get the location object for the ID */
if (NULL == (vol_obj = H5VL_vol_object(obj_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid object identifier")
if (H5VL_object_is_native(vol_obj, &ret_value) < 0)
HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "can't determine if object is a native connector object")
done:
FUNC_LEAVE_API(ret_value)
} /* H5VLobject_is_native() */
/*-------------------------------------------------------------------------
* Function: H5VLget_file_type
*

11
src/H5VLpublic.h
View File

@ -356,6 +356,17 @@ H5_DLL herr_t H5VLunregister_connector(hid_t connector_id);
* \since 1.12.0
*/
H5_DLL herr_t H5VLquery_optional(hid_t obj_id, H5VL_subclass_t subcls, int opt_type, uint64_t *flags);
/**
* \ingroup H5VL
* \brief Determines whether an object ID represents a native
* VOL connector object.
*
* \param[in] obj_id Object identifier
* \return \hbool_t
*
* \since 1.12.1
*/
H5_DLL hbool_t H5VLobject_is_native(hid_t obj_id);
#ifdef __cplusplus
}

1
src/libhdf5.settings.in
View File

@ -71,6 +71,7 @@ Features:
Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@
Large Parallel I/O: @LARGE_PARALLEL_IO@
High-level library: @HDF5_HL@
Dimension scales w/ new references: @DIMENSION_SCALES_WITH_NEW_REF@
Build HDF5 Tests: @HDF5_TESTS@
Build HDF5 Tools: @HDF5_TOOLS@
Threadsafety: @THREADSAFE@