mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-02-17 16:10:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

HDFFV-8745 merge from develop

Browse Source
This commit is contained in:
Allen Byrne 2018-01-09 09:20:25 -06:00
parent 49479589d3
commit f548e878b0
16 changed files with 3787 additions and 3157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
MANIFEST
View File

@ -2382,6 +2382,7 @@
./tools/test/h5diff/testfiles/h5diff_attr1.h5
./tools/test/h5diff/testfiles/h5diff_attr2.h5
./tools/test/h5diff/testfiles/h5diff_attr3.h5
./tools/test/h5diff/testfiles/h5diff_attr_v_level1.h5
./tools/test/h5diff/testfiles/h5diff_attr_v_level2.h5
./tools/test/h5diff/testfiles/h5diff_basic1.h5
@ -2423,12 +2424,8 @@
./tools/test/h5diff/testfiles/compounds_array_vlen2.h5
./tools/test/h5diff/testfiles/non_comparables1.h5
./tools/test/h5diff/testfiles/non_comparables2.h5
./tools/test/h5diff/testfiles/tmptest2.he5
./tools/test/h5diff/testfiles/tmpSingleSiteBethe.reference.h5
./tools/test/h5diff/testfiles/h5diff_tmp1.txt
./tools/test/h5diff/testfiles/tmptest.he5
./tools/test/h5diff/testfiles/h5diff_tmp2.txt
./tools/test/h5diff/testfiles/tmpSingleSiteBethe.output.h5
./tools/test/h5diff/testfiles/h5diff_8625.txt
./tools/test/h5diff/testfiles/h5diff_8639.txt
./tools/test/h5diff/testfiles/tudfilter.h5
./tools/test/h5diff/testfiles/tudfilter2.h5
./tools/test/h5diff/testfiles/h5diff_ud.txt

15
release_docs/RELEASE.txt
View File

@ -493,6 +493,21 @@ Bug Fixes since HDF5-1.10.1 release
Tools
-----
- h5diff
h5diff seg faulted if comparing VL strings against fixed strings.
Reworked solution for HDFFV-8625 and HDFFV-8639. Implemented the check
for string objects of same type in the diff_can_type function by
adding an if(tclass1 == H5T_STRING) block. This if block moves the
same check that was added for attributes to this function, which is
used by all object types. This function also handles complex type
structures.
Also added a new test file in h5diffgenttest for testing this issue
and removed the temporary files used in the test scripts.
(ADB - 2018/01/04, HDFFV-8745)
- h5repack
h5repack failed to copy a dataset with existing filter.

20
tools/lib/h5diff_attr.c
View File

@ -385,28 +385,8 @@ hsize_t diff_attr(hid_t loc1_id,
/* get the datatypes */
if((ftype1_id = H5Aget_type(attr1_id)) < 0)
HGOTO_ERROR(1, H5E_tools_min_id_g, "H5Aget_type first attribute failed");
vstrtype1 = H5Tis_variable_str(ftype1_id);
if((ftype2_id = H5Aget_type(attr2_id)) < 0)
HGOTO_ERROR(1, H5E_tools_min_id_g, "H5Aget_type second attribute failed");
vstrtype2 = H5Tis_variable_str(ftype2_id);
/* no compare if either one but not both are variable string type */
if (vstrtype1 != vstrtype2) {
if((opts->m_verbose || opts->m_list_not_cmp))
parallel_print("Not comparable: one of attribute <%s/%s> or <%s/%s> is of variable length type\n",
path1, name1, path2, name2);
opts->not_cmp = 1;
if (H5Tclose(ftype1_id) < 0)
HGOTO_ERROR(1, H5E_tools_min_id_g, "H5Tclose first attribute ftype failed");
if (H5Tclose(ftype2_id) < 0)
HGOTO_ERROR(1, H5E_tools_min_id_g, "H5Tclose second attribute ftype failed");
if (H5Aclose(attr1_id) < 0)
HGOTO_ERROR(1, H5E_tools_min_id_g, "H5Aclose first attribute failed");
if (H5Aclose(attr2_id) < 0)
HGOTO_ERROR(1, H5E_tools_min_id_g, "H5Aclose second attribute failed");
continue;
}
if((mtype1_id = H5Tget_native_type(ftype1_id, H5T_DIR_DEFAULT)) < 0)
HGOTO_ERROR(1, H5E_tools_min_id_g, "H5Tget_native_type first attribute ftype failed");

50
tools/lib/h5diff_dset.c
View File

@ -192,7 +192,8 @@ hsize_t diff_datasetid(hid_t did1,
hid_t sm_space = -1; /*stripmine data space */
size_t need; /* bytes needed for malloc */
int i;
unsigned int vl_data = 0; /*contains VL datatypes */
unsigned int vl_data1 = 0; /*contains VL datatypes */
unsigned int vl_data2 = 0; /*contains VL datatypes */
h5difftrace("diff_datasetid start\n");
/* Get the dataspace handle */
@ -320,7 +321,9 @@ hsize_t diff_datasetid(hid_t did1,
* VLEN memory buffer later
*/
if(TRUE == h5tools_detect_vlen(m_tid1))
vl_data = TRUE;
vl_data1 = TRUE;
if(TRUE == h5tools_detect_vlen(m_tid2))
vl_data2 = TRUE;
h5diffdebug2("h5tools_detect_vlen - errstat:%d\n", opts->err_stat);
/*------------------------------------------------------------------------
@ -411,11 +414,12 @@ hsize_t diff_datasetid(hid_t did1,
h5diffdebug2("diff_array nfound:%d\n", nfound);
/* reclaim any VL memory, if necessary */
if(vl_data) {
h5difftrace("check vl_data\n");
h5diffdebug2("check vl_data1:%d\n", vl_data1);
if(vl_data1)
H5Dvlen_reclaim(m_tid1, sid1, H5P_DEFAULT, buf1);
h5diffdebug2("check vl_data2:%d\n", vl_data2);
if(vl_data2)
H5Dvlen_reclaim(m_tid2, sid2, H5P_DEFAULT, buf2);
} /* end if */
if(buf1 != NULL) {
HDfree(buf1);
buf1 = NULL;
@ -505,10 +509,10 @@ hsize_t diff_datasetid(hid_t did1,
dadims, opts, name1, name2, dam_tid, did1, did2);
/* reclaim any VL memory, if necessary */
if(vl_data) {
if(vl_data1)
H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf1);
H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf2);
} /* end if */
if(vl_data2)
H5Dvlen_reclaim(m_tid2, sm_space, H5P_DEFAULT, sm_buf2);
/* calculate the next hyperslab offset */
for(i = rank1, carry = 1; i > 0 && carry; --i) {
@ -545,29 +549,29 @@ done:
/* free */
if(buf1 != NULL) {
/* reclaim any VL memory, if necessary */
if(vl_data)
if(vl_data1)
H5Dvlen_reclaim(m_tid1, sid1, H5P_DEFAULT, buf1);
HDfree(buf1);
buf1 = NULL;
}
if(buf2 != NULL) {
/* reclaim any VL memory, if necessary */
if(vl_data)
if(vl_data2)
H5Dvlen_reclaim(m_tid2, sid2, H5P_DEFAULT, buf2);
HDfree(buf2);
buf2 = NULL;
}
if(sm_buf1 != NULL) {
/* reclaim any VL memory, if necessary */
if(vl_data)
if(vl_data1)
H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf1);
HDfree(sm_buf1);
sm_buf1 = NULL;
}
if(sm_buf2 != NULL) {
/* reclaim any VL memory, if necessary */
if(vl_data)
H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf2);
if(vl_data2)
H5Dvlen_reclaim(m_tid2, sm_space, H5P_DEFAULT, sm_buf2);
HDfree(sm_buf2);
sm_buf2 = NULL;
}
@ -576,6 +580,7 @@ done:
H5E_BEGIN_TRY {
H5Sclose(sid1);
H5Sclose(sid2);
H5Sclose(sm_space);
H5Tclose(f_tid1);
H5Tclose(f_tid2);
H5Tclose(m_tid1);
@ -767,12 +772,31 @@ int diff_can_type(hid_t f_tid1, /* file data type */
}
}
if(tclass1 == H5T_STRING) {
hid_t vstrtype1 = -1;
hid_t vstrtype2 = -1;
h5difftrace("diff_can_type end - H5T_STRING\n");
vstrtype1 = H5Tis_variable_str(f_tid1);
vstrtype2 = H5Tis_variable_str(f_tid2);
/* no compare if either one but not both are variable string type */
if (vstrtype1 != vstrtype2) {
if((opts->m_verbose || opts->m_list_not_cmp))
parallel_print("Not comparable: <%s> or <%s> is of mixed string type\n",
obj1_name, obj2_name);
opts->not_cmp = 1;
HGOTO_DONE(0);
}
}
if(tclass1 == H5T_COMPOUND) {
int nmembs1;
int nmembs2;
int j;
hid_t memb_type1 = -1;
hid_t memb_type2 = -1;
h5difftrace("diff_can_type end - H5T_COMPOUND\n");
nmembs1 = H5Tget_nmembers(f_tid1);
nmembs2 = H5Tget_nmembers(f_tid2);

23
tools/test/h5diff/CMakeTests.cmake
View File

@ -26,6 +26,7 @@
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_dtypes.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_attr1.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_attr2.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_attr3.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_dset1.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_dset2.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_dset3.h5
@ -63,10 +64,6 @@
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/compounds_array_vlen2.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/non_comparables1.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/non_comparables2.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/tmptest.he5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/tmptest2.he5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/tmpSingleSiteBethe.reference.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/tmpSingleSiteBethe.output.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/tudfilter.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/tudfilter2.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/diff_strings1.h5
@ -293,8 +290,8 @@
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_800.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_801.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_90.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_tmp1.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_tmp2.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_8625.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_8639.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_ud.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_udfail.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_v1.txt
@ -1021,6 +1018,10 @@
h5diff_800.out.err
h5diff_801.out
h5diff_801.out.err
h5diff_8625.out
h5diff_8625.out.err
h5diff_8639.out
h5diff_8639.out.err
h5diff_90.out
h5diff_90.out.err
h5diff_v1.out
@ -1281,10 +1282,6 @@ ADD_H5_TEST (h5diff_631 0 -v --use-system-epsilon ${FILE1} ${FILE1} g1/fp18 g1/f
# 7. attributes
# ##############################################################################
ADD_H5_TEST (h5diff_70 1 -v ${FILE5} ${FILE6})
# temporary test to verify HDF5-8625
ADD_H5_TEST (h5diff_tmp1 0 tmptest2.he5 tmptest.he5)
# temporary test to verify HDF5-8639
ADD_H5_TEST (h5diff_tmp2 1 tmpSingleSiteBethe.output.h5 tmpSingleSiteBethe.reference.h5)
# ##################################################
# attrs with verbose option level
@ -1613,7 +1610,11 @@ ADD_H5_TEST (h5diff_487 1 -v --exclude-path "/group1/dset" h5diff_exclude3-1.h5
# ##############################################################################
# # diff various multiple vlen and fixed strings in a compound type dataset
# ##############################################################################
ADD_H5_TEST (h5diff_530 0 -v ${COMP_VL_STRS_FILE} ${COMP_VL_STRS_FILE} /group /group_copy)
ADD_H5_TEST (h5diff_530 0 -v ${COMP_VL_STRS_FILE} ${COMP_VL_STRS_FILE} /group /group_copy)
# test to verify HDFFV-8625
ADD_H5_TEST (h5diff_8625 0 -v --enable-error-stack ${COMP_VL_STRS_FILE} ${COMP_VL_STRS_FILE} /group/Compound_dset1 /group_copy/Compound_dset3)
# test to verify HDFFV-8639
ADD_H5_TEST (h5diff_8639 0 -v h5diff_attr3.h5 h5diff_attr2.h5 /g1)
ADD_H5_TEST (h5diff_vlstr 0 -v tvlstr.h5 tvlstr2.h5)
# ##############################################################################

6710
tools/test/h5diff/h5diffgentest.c
View File

File diff suppressed because it is too large Load Diff

10
tools/test/h5diff/testfiles/h5diff_8625.txt Normal file
View File

@ -0,0 +1,10 @@
dataset: </group/Compound_dset1> and </group_copy/Compound_dset3>
Warning: different storage datatype
Warning: different storage datatype
Not comparable: </group/Compound_dset1> or </group_copy/Compound_dset3> is of mixed string type
0 differences found
--------------------------------
Some objects are not comparable
--------------------------------
Use -c for a list of objects without details of differences.
EXIT CODE: 0

72
tools/test/h5diff/testfiles/h5diff_8639.txt Normal file
View File

@ -0,0 +1,72 @@
group1 group2
---------------------------------------
x x
group : </g1> and </g1>
0 differences found
Warning: different storage datatype
Not comparable: <VLstring> or <VLstring> is of mixed string type
Warning: different storage datatype
Not comparable: <VLstring2D> or <VLstring2D> is of mixed string type
Warning: different storage datatype
Not comparable: <VLstring3D> or <VLstring3D> is of mixed string type
attribute: <array of </g1>> and <array of </g1>>
0 differences found
attribute: <array2D of </g1>> and <array2D of </g1>>
0 differences found
attribute: <array3D of </g1>> and <array3D of </g1>>
0 differences found
attribute: <bitfield of </g1>> and <bitfield of </g1>>
0 differences found
attribute: <bitfield2D of </g1>> and <bitfield2D of </g1>>
0 differences found
attribute: <bitfield3D of </g1>> and <bitfield3D of </g1>>
0 differences found
attribute: <compound of </g1>> and <compound of </g1>>
0 differences found
attribute: <compound2D of </g1>> and <compound2D of </g1>>
0 differences found
attribute: <compound3D of </g1>> and <compound3D of </g1>>
0 differences found
attribute: <enum of </g1>> and <enum of </g1>>
0 differences found
attribute: <enum2D of </g1>> and <enum2D of </g1>>
0 differences found
attribute: <enum3D of </g1>> and <enum3D of </g1>>
0 differences found
attribute: <float of </g1>> and <float of </g1>>
0 differences found
attribute: <float2D of </g1>> and <float2D of </g1>>
0 differences found
attribute: <float3D of </g1>> and <float3D of </g1>>
0 differences found
attribute: <integer of </g1>> and <integer of </g1>>
0 differences found
attribute: <integer2D of </g1>> and <integer2D of </g1>>
0 differences found
attribute: <integer3D of </g1>> and <integer3D of </g1>>
0 differences found
attribute: <opaque of </g1>> and <opaque of </g1>>
0 differences found
attribute: <opaque2D of </g1>> and <opaque2D of </g1>>
0 differences found
attribute: <opaque3D of </g1>> and <opaque3D of </g1>>
0 differences found
Warning: different storage datatype
Not comparable: <string> or <string> is of mixed string type
Warning: different storage datatype
Not comparable: <string2D> or <string2D> is of mixed string type
Warning: different storage datatype
Not comparable: <string3D> or <string3D> is of mixed string type
attribute: <vlen of </g1>> and <vlen of </g1>>
0 differences found
attribute: <vlen2D of </g1>> and <vlen2D of </g1>>
0 differences found
attribute: <vlen3D of </g1>> and <vlen3D of </g1>>
0 differences found
--------------------------------
Some objects are not comparable
--------------------------------
Use -c for a list of objects without details of differences.
EXIT CODE: 0

BIN
tools/test/h5diff/testfiles/h5diff_attr1.h5
View File

Binary file not shown.

BIN
tools/test/h5diff/testfiles/h5diff_attr2.h5
View File

Binary file not shown.

BIN
tools/test/h5diff/testfiles/h5diff_attr3.h5 Normal file
View File

Binary file not shown.

5
tools/test/h5diff/testfiles/h5diff_tmp1.txt
View File

@ -1,5 +0,0 @@
--------------------------------
Some objects are not comparable
--------------------------------
Use -c for a list of objects.
EXIT CODE: 0

13
tools/test/h5diff/testfiles/h5diff_tmp2.txt
View File

@ -1,13 +0,0 @@
dataset: </G/down/Data> and </G/down/Data>
1599 differences found
dataset: </G/down/Tail/array> and </G/down/Tail/array>
8 differences found
dataset: </G/up/Data> and </G/up/Data>
1845 differences found
dataset: </G/up/Tail/array> and </G/up/Tail/array>
10 differences found
--------------------------------
Some objects are not comparable
--------------------------------
Use -c for a list of objects.
EXIT CODE: 1

BIN
tools/test/h5diff/testfiles/tmptest.he5
View File

Binary file not shown.

BIN
tools/test/h5diff/testfiles/tmptest2.he5
View File

Binary file not shown.

17
tools/test/h5diff/testh5diff.sh.in
View File

@ -82,6 +82,7 @@ $SRC_H5DIFF_TESTFILES/h5diff_types.h5
$SRC_H5DIFF_TESTFILES/h5diff_dtypes.h5
$SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
$SRC_H5DIFF_TESTFILES/h5diff_attr2.h5
$SRC_H5DIFF_TESTFILES/h5diff_attr3.h5
$SRC_H5DIFF_TESTFILES/h5diff_dset1.h5
$SRC_H5DIFF_TESTFILES/h5diff_dset2.h5
$SRC_H5DIFF_TESTFILES/h5diff_hyper1.h5
@ -118,10 +119,6 @@ $SRC_H5DIFF_TESTFILES/h5diff_attr_v_level2.h5
$SRC_H5DIFF_TESTFILES/h5diff_enum_invalid_values.h5
$SRC_H5DIFF_TESTFILES/non_comparables1.h5
$SRC_H5DIFF_TESTFILES/non_comparables2.h5
$SRC_H5DIFF_TESTFILES/tmptest.he5
$SRC_H5DIFF_TESTFILES/tmptest2.he5
$SRC_H5DIFF_TESTFILES/tmpSingleSiteBethe.reference.h5
$SRC_H5DIFF_TESTFILES/tmpSingleSiteBethe.output.h5
$SRC_H5DIFF_TESTFILES/diff_strings1.h5
$SRC_H5DIFF_TESTFILES/diff_strings2.h5
$SRC_TOOLS_TESTFILES/tvlstr.h5
@ -337,8 +334,8 @@ $SRC_H5DIFF_TESTFILES/h5diff_709.txt
$SRC_H5DIFF_TESTFILES/h5diff_710.txt
$SRC_H5DIFF_TESTFILES/h5diff_80.txt
$SRC_H5DIFF_TESTFILES/h5diff_90.txt
$SRC_H5DIFF_TESTFILES/h5diff_tmp1.txt
$SRC_H5DIFF_TESTFILES/h5diff_tmp2.txt
$SRC_H5DIFF_TESTFILES/h5diff_8625.txt
$SRC_H5DIFF_TESTFILES/h5diff_8639.txt
$SRC_H5DIFF_TESTFILES/h5diff_v1.txt
$SRC_H5DIFF_TESTFILES/h5diff_v2.txt
$SRC_H5DIFF_TESTFILES/h5diff_v3.txt
@ -820,10 +817,6 @@ TOOLTEST h5diff_631.txt -v --use-system-epsilon h5diff_basic1.h5 h5diff_basic1.h
# 7. attributes
# ##############################################################################
TOOLTEST h5diff_70.txt -v h5diff_attr1.h5 h5diff_attr2.h5
# temporary test to verify HDF5-8625
TOOLTEST h5diff_tmp1.txt tmptest2.he5 tmptest.he5
# temporary test to verify HDF5-8639
TOOLTEST h5diff_tmp2.txt tmpSingleSiteBethe.output.h5 tmpSingleSiteBethe.reference.h5
# ##################################################
# attrs with verbose option level
@ -1145,6 +1138,10 @@ TOOLTEST h5diff_487.txt -v --exclude-path "/group1/dset" h5diff_exclude3-1.h5 h5
# # diff various multiple vlen and fixed strings in a compound type dataset
# ##############################################################################
TOOLTEST h5diff_530.txt -v h5diff_comp_vl_strs.h5 h5diff_comp_vl_strs.h5 /group /group_copy
# test to verify HDFFV-8625
TOOLTEST h5diff_8625.txt -v --enable-error-stack h5diff_comp_vl_strs.h5 h5diff_comp_vl_strs.h5 /group/Compound_dset1 /group_copy/Compound_dset3
# test to verify HDFFV-8639
TOOLTEST h5diff_8639.txt -v h5diff_attr3.h5 h5diff_attr2.h5 /g1
TOOLTEST h5diff_vlstr.txt -v tvlstr.h5 tvlstr2.h5
# ##############################################################################