mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-03-31 17:10:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

HDFFV-10297 - Fix test errors - five remaining

Browse Source
This commit is contained in:
Allen Byrne 2017-10-09 12:59:28 -05:00
parent a07d6b52fb
commit daddf169b0
14 changed files with 330 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tools/lib/h5diff_attr.c
View File

@ -472,8 +472,8 @@ hsize_t diff_attr(hid_t loc1_id,
for(j = 0; j < rank1; j++)
nelmts1 *= dims1[j];
buf1 = (void *)HDmalloc((size_t)(nelmts1 * msize1));
buf2 = (void *)HDmalloc((size_t)(nelmts1 * msize2));
buf1 = (void *)HDcalloc((size_t)(nelmts1), msize1);
buf2 = (void *)HDcalloc((size_t)(nelmts1), msize2);
if(buf1 == NULL || buf2 == NULL) {
parallel_print("cannot read into memory\n");
goto error;

188
tools/lib/h5tools_filters.c
View File

@ -20,9 +20,9 @@
*/
static void print_warning(const char *dname, const char *fname)
{
fprintf(stderr,
"warning: dataset <%s> cannot be read, %s filter is not available\n",
dname, fname);
fprintf(stderr,
"warning: dataset <%s> cannot be read, %s filter is not available\n",
dname, fname);
}
/*-------------------------------------------------------------------------
@ -34,100 +34,95 @@ static void print_warning(const char *dname, const char *fname)
* 2) the internal filters might be turned off
*
* Return: 1, can read, 0, cannot, -1 error
*
* Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
*
* Date: March 1, 2004
*
*-------------------------------------------------------------------------
*/
int h5tools_canreadf(const char* name, /* object name, serves also as boolean print */
hid_t dcpl_id) /* dataset creation property list */
{
int nfilters; /* number of filters */
H5Z_filter_t filtn; /* filter identification number */
int i; /* index */
int udfilter_avail; /* index */
int nfilters; /* number of filters */
H5Z_filter_t filtn; /* filter identification number */
int i; /* index */
int udfilter_avail; /* index */
/* get information about filters */
if ((nfilters = H5Pget_nfilters(dcpl_id)) < 0)
return -1;
/* get information about filters */
if ((nfilters = H5Pget_nfilters(dcpl_id)) < 0)
return -1;
/* if we do not have filters, we can read the dataset safely */
if (!nfilters)
return 1;
/* if we do not have filters, we can read the dataset safely */
if (!nfilters)
return 1;
/* check availability of filters */
for (i = 0; i < nfilters; i++) {
if ((filtn = H5Pget_filter2(dcpl_id, (unsigned) i, 0, 0, 0, (size_t) 0, 0, NULL)) < 0)
return -1;
/* check availability of filters */
for (i = 0; i < nfilters; i++) {
if ((filtn = H5Pget_filter2(dcpl_id, (unsigned) i, 0, 0, 0, (size_t) 0, 0, NULL)) < 0)
return -1;
switch (filtn) {
/*-------------------------------------------------------------------------
* user defined filter
*-------------------------------------------------------------------------
*/
default:
if ((udfilter_avail = H5Zfilter_avail(filtn)) < 0)
return -1;
else if (udfilter_avail == 0) {
if (name)
print_warning(name, "user defined");
return 0;
}
break;
switch (filtn) {
/*-------------------------------------------------------------------------
* user defined filter
*-------------------------------------------------------------------------
*/
default:
if ((udfilter_avail = H5Zfilter_avail(filtn)) < 0)
return -1;
else if (udfilter_avail == 0) {
if (name)
print_warning(name, "user defined");
return 0;
}
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_DEFLATE 1 , deflation like gzip
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_DEFLATE:
/*-------------------------------------------------------------------------
* H5Z_FILTER_DEFLATE 1 , deflation like gzip
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_DEFLATE:
#ifndef H5_HAVE_FILTER_DEFLATE
if (name)
print_warning(name,"deflate");
return 0;
if (name)
print_warning(name,"deflate");
return 0;
#endif
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SZIP 4 , szip compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SZIP:
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SZIP 4 , szip compression
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SZIP:
#ifndef H5_HAVE_FILTER_SZIP
if (name)
print_warning(name,"SZIP");
return 0;
if (name)
print_warning(name,"SZIP");
return 0;
#endif
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SHUFFLE 2 , shuffle the data
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SHUFFLE:
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_FLETCHER32:
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_NBIT
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_NBIT:
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SCALEOFFSET
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SCALEOFFSET:
break;
}/*switch*/
}/*for*/
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SHUFFLE 2 , shuffle the data
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SHUFFLE:
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_FLETCHER32:
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_NBIT
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_NBIT:
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SCALEOFFSET
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SCALEOFFSET:
break;
}/*switch*/
}/*for*/
return 1;
return 1;
}
/*-------------------------------------------------------------------------
@ -138,31 +133,26 @@ int h5tools_canreadf(const char* name, /* object name, serves also as boolean pr
* except SZIP, which may be configured decoder-only.
*
* Return: 1, can write, 0, cannot, -1 error
*
* Programmer:
*
* Date: October 5, 2004
*
*-------------------------------------------------------------------------
*/
H5_ATTR_CONST int
h5tools_can_encode(H5Z_filter_t filtn) {
switch (filtn) {
/* user defined filter */
default:
/* user defined filter */
default:
return 0;
case H5Z_FILTER_DEFLATE:
case H5Z_FILTER_DEFLATE:
#ifndef H5_HAVE_FILTER_DEFLATE
return 0;
#endif
break;
case H5Z_FILTER_SZIP:
case H5Z_FILTER_SZIP:
#ifndef H5_HAVE_FILTER_SZIP
return 0;
#else
{
{
unsigned int filter_config_flags;
if (H5Zget_filter_info(filtn, &filter_config_flags) < 0)
@ -171,36 +161,36 @@ h5tools_can_encode(H5Z_filter_t filtn) {
& (H5Z_FILTER_CONFIG_ENCODE_ENABLED | H5Z_FILTER_CONFIG_DECODE_ENABLED)) == 0) {
/* filter present but neither encode nor decode is supported (???) */
return -1;
}
}
else if ((filter_config_flags
& (H5Z_FILTER_CONFIG_ENCODE_ENABLED | H5Z_FILTER_CONFIG_DECODE_ENABLED)) == H5Z_FILTER_CONFIG_DECODE_ENABLED) {
/* decoder only: read but not write */
return 0;
}
}
else if ((filter_config_flags
& (H5Z_FILTER_CONFIG_ENCODE_ENABLED | H5Z_FILTER_CONFIG_DECODE_ENABLED)) == H5Z_FILTER_CONFIG_ENCODE_ENABLED) {
/* encoder only: write but not read (???) */
return -1;
}
}
else if ((filter_config_flags
& (H5Z_FILTER_CONFIG_ENCODE_ENABLED | H5Z_FILTER_CONFIG_DECODE_ENABLED))
== (H5Z_FILTER_CONFIG_ENCODE_ENABLED | H5Z_FILTER_CONFIG_DECODE_ENABLED)) {
return 1;
}
}
}
#endif
break;
case H5Z_FILTER_SHUFFLE:
case H5Z_FILTER_SHUFFLE:
break;
case H5Z_FILTER_FLETCHER32:
case H5Z_FILTER_FLETCHER32:
break;
case H5Z_FILTER_NBIT:
case H5Z_FILTER_NBIT:
break;
case H5Z_FILTER_SCALEOFFSET:
case H5Z_FILTER_SCALEOFFSET:
break;
}/*switch*/

87
tools/src/h5repack/h5repack.c
View File

@ -693,6 +693,7 @@ static int check_objects(const char* fname, pack_opt_t *options) {
hid_t did;
hid_t sid;
unsigned int i;
unsigned int uf;
trav_table_t *travt = NULL;
/* nothing to do */
@ -740,52 +741,54 @@ static int check_objects(const char* fname, pack_opt_t *options) {
if (options->verbose)
printf("...Found\n");
if (options->op_tbl->objs[i].filter->filtn < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "invalid filter");
/* check for extra filter conditions */
switch (options->op_tbl->objs[i].filter->filtn) {
/* chunk size must be smaller than pixels per block */
case H5Z_FILTER_SZIP:
{
int j;
hsize_t csize = 1;
unsigned ppb = options->op_tbl->objs[i].filter->cd_values[0];
hsize_t dims[H5S_MAX_RANK];
int rank;
for (uf = 0; uf < options->op_tbl->objs[i].nfilters; uf++) {
if (options->op_tbl->objs[i].filter[uf].filtn < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "invalid filter");
/* check for extra filter conditions */
switch (options->op_tbl->objs[i].filter[uf].filtn) {
/* chunk size must be smaller than pixels per block */
case H5Z_FILTER_SZIP:
{
int j;
hsize_t csize = 1;
unsigned ppb = options->op_tbl->objs[i].filter[uf].cd_values[0];
hsize_t dims[H5S_MAX_RANK];
int rank;
if (options->op_tbl->objs[i].chunk.rank > 0) {
rank = options->op_tbl->objs[i].chunk.rank;
for (j = 0; j < rank; j++)
csize *= options->op_tbl->objs[i].chunk.chunk_lengths[j];
}
else {
if ((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed");
if ((sid = H5Dget_space(did)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed");
if ((rank = H5Sget_simple_extent_ndims(sid)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_ndims failed");
HDmemset(dims, 0, sizeof dims);
if (H5Sget_simple_extent_dims(sid, dims, NULL) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_dims failed");
for (j = 0; j < rank; j++)
csize *= dims[j];
if (H5Sclose(sid) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed");
if (H5Dclose(did) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed");
}
if (options->op_tbl->objs[i].chunk.rank > 0) {
rank = options->op_tbl->objs[i].chunk.rank;
for (j = 0; j < rank; j++)
csize *= options->op_tbl->objs[i].chunk.chunk_lengths[j];
}
else {
if ((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen2 failed");
if ((sid = H5Dget_space(did)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space failed");
if ((rank = H5Sget_simple_extent_ndims(sid)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_ndims failed");
HDmemset(dims, 0, sizeof dims);
if (H5Sget_simple_extent_dims(sid, dims, NULL) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_dims failed");
for (j = 0; j < rank; j++)
csize *= dims[j];
if (H5Sclose(sid) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed");
if (H5Dclose(did) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed");
}
if (csize < ppb) {
printf(" <warning: SZIP settings, chunk size is smaller than pixels per block>\n");
HGOTO_DONE(0);
if (csize < ppb) {
printf(" <warning: SZIP settings, chunk size is smaller than pixels per block>\n");
HGOTO_DONE(0);
}
}
break;
default:
break;
}
break;
default:
break;
}
} /* i */
} /* for uf */
} /* for i */
done:
H5E_BEGIN_TRY {

9
tools/src/h5repack/h5repack_copy.c
View File

@ -698,6 +698,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
unsigned crt_order_flags; /* group creation order flag */
unsigned i;
unsigned u;
unsigned uf;
int is_ref = 0;
htri_t is_named;
hbool_t limit_maxdims;
@ -803,8 +804,10 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
if (options->op_tbl->objs) {
for (u = 0; u < options->op_tbl->nelems; u++) {
if (HDstrcmp(travt->objs[i].name, options->op_tbl->objs[u].path) == 0)
if (options->op_tbl->objs[u].filter->filtn > 0)
req_filter = 1;
for (uf = 0; uf < options->op_tbl->objs[uf].nfilters; uf++) {
if (options->op_tbl->objs[u].filter[uf].filtn > 0)
req_filter = 1;
}
}
}
@ -1293,7 +1296,7 @@ done:
HDfree(hslab_buf);
/* Finalize (link) the stack of named datatypes (if any) */
if (0 == ret_value)
if (0 == ret_value && named_dt_head != NULL)
named_datatype_free(&named_dt_head, 0);
else
H5E_BEGIN_TRY {

28
tools/src/h5repack/h5repack_filters.c
View File

@ -469,9 +469,33 @@ int apply_filters(const char* name, /* object name from traverse list */
if (H5Zfilter_avail(filtobj.filter[i].filtn) <= 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "%d filter unavailable", filtobj.filter[i].filtn);
}/*for*/
} /* for */
} /* nfilters */
/*-------------------------------------------------------------------------
* layout
*-------------------------------------------------------------------------
*/
if (obj.layout >= 0) {
/* a layout was defined */
if (H5Pset_layout(dcpl_id, obj.layout) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_layout failed");
if (H5D_CHUNKED == obj.layout) {
if (H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_chunk failed");
}
else if (H5D_COMPACT == obj.layout) {
if (H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_alloc_time failed");
}
/* remove filters for the H5D_CONTIGUOUS case */
else if (H5D_CONTIGUOUS == obj.layout) {
if (H5Premove_filter(dcpl_id, H5Z_FILTER_ALL) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Premove_filter failed");
}
}
/*nfilters*/
done:
return ret_value;

4
tools/test/h5repack/testfiles/deflate_limit.h5repack_layout.h5.ddl
View File

@ -76,8 +76,8 @@ GROUP "/" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) }
STORAGE_LAYOUT {
CHUNKED ( 20, 10 )
SIZE 1283 (2.494:1 COMPRESSION)
CHUNKED ( 40, 20 )
SIZE 1150 (2.783:1 COMPRESSION)
}
FILTERS {
COMPRESSION DEFLATE { LEVEL 1 }

1
tools/test/h5repack/testfiles/h5repack_filters.h5-gzip_verbose_filters.tst
View File

@ -2,6 +2,7 @@ No all objects to modify layout
No all objects to apply filter
</dset_deflate> with GZIP filter
</dset_deflate>...Found
Making new file ...
-----------------------------------------
Type Filter (Compression) Name
-----------------------------------------

2
tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl
View File

@ -92,7 +92,7 @@ GROUP "/" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) }
STORAGE_LAYOUT {
CHUNKED ( 20, 10 )
CHUNKED ( 40, 20 )
SIZE 3200 (1.000:1 COMPRESSION)
}
FILTERS {

2
tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl
View File

@ -92,7 +92,7 @@ GROUP "/" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) }
STORAGE_LAYOUT {
CHUNKED ( 20, 10 )
CHUNKED ( 40, 20 )
SIZE 3200 (1.000:1 COMPRESSION)
}
FILTERS {

131
tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.ddl
View File

@ -1,4 +1,135 @@
HDF5 "out-plugin_zero.h5repack_layout.h5" {
GROUP "/" {
DATASET "dset1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) }
STORAGE_LAYOUT {
CONTIGUOUS
SIZE 3200
OFFSET 2048
}
FILTERS {
NONE
}
FILLVALUE {
FILL_TIME H5D_FILL_TIME_IFSET
VALUE H5D_FILL_VALUE_DEFAULT
}
ALLOCATION_TIME {
H5D_ALLOC_TIME_LATE
}
}
DATASET "dset2" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) }
STORAGE_LAYOUT {
CONTIGUOUS
SIZE 3200
OFFSET 5248
}
FILTERS {
NONE
}
FILLVALUE {
FILL_TIME H5D_FILL_TIME_IFSET
VALUE H5D_FILL_VALUE_DEFAULT
}
ALLOCATION_TIME {
H5D_ALLOC_TIME_LATE
}
}
DATASET "dset3" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) }
STORAGE_LAYOUT {
CONTIGUOUS
SIZE 3200
OFFSET 8448
}
FILTERS {
NONE
}
FILLVALUE {
FILL_TIME H5D_FILL_TIME_IFSET
VALUE H5D_FILL_VALUE_DEFAULT
}
ALLOCATION_TIME {
H5D_ALLOC_TIME_LATE
}
}
DATASET "dset4" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) }
STORAGE_LAYOUT {
CONTIGUOUS
SIZE 3200
OFFSET 13696
}
FILTERS {
NONE
}
FILLVALUE {
FILL_TIME H5D_FILL_TIME_IFSET
VALUE H5D_FILL_VALUE_DEFAULT
}
ALLOCATION_TIME {
H5D_ALLOC_TIME_LATE
}
}
DATASET "dset_chunk" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) }
STORAGE_LAYOUT {
CHUNKED ( 20, 10 )
SIZE 3200
}
FILTERS {
NONE
}
FILLVALUE {
FILL_TIME H5D_FILL_TIME_IFSET
VALUE H5D_FILL_VALUE_DEFAULT
}
ALLOCATION_TIME {
H5D_ALLOC_TIME_INCR
}
}
DATASET "dset_compact" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) }
STORAGE_LAYOUT {
COMPACT
SIZE 3200
}
FILTERS {
NONE
}
FILLVALUE {
FILL_TIME H5D_FILL_TIME_IFSET
VALUE H5D_FILL_VALUE_DEFAULT
}
ALLOCATION_TIME {
H5D_ALLOC_TIME_EARLY
}
}
DATASET "dset_contiguous" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) }
STORAGE_LAYOUT {
CONTIGUOUS
SIZE 3200
OFFSET 26184
}
FILTERS {
NONE
}
FILLVALUE {
FILL_TIME H5D_FILL_TIME_IFSET
VALUE H5D_FILL_VALUE_DEFAULT
}
ALLOCATION_TIME {
H5D_ALLOC_TIME_LATE
}
}
}
}

5
tools/test/h5repack/testfiles/plugin_none.h5repack_layout.UD.h5.tst
View File

@ -1,6 +1,7 @@
Objects to modify layout are...
Objects to apply filter are...
No all objects to modify layout
All objects to apply filter are...
Uncompress all
Making new file ...
-----------------------------------------
Type Filter (Compression) Name
-----------------------------------------

5
tools/test/h5repack/testfiles/plugin_test.h5repack_layout.h5.tst
View File

@ -1,6 +1,7 @@
Objects to modify layout are...
Objects to apply filter are...
No all objects to modify layout
All objects to apply filter are...
User Defined 257
Making new file ...
-----------------------------------------
Type Filter (Compression) Name
-----------------------------------------

5
tools/test/h5repack/testfiles/plugin_version_test.h5repack_layout.h5.tst
View File

@ -1,6 +1,7 @@
Objects to modify layout are...
Objects to apply filter are...
No all objects to modify layout
All objects to apply filter are...
User Defined 260
Making new file ...
-----------------------------------------
Type Filter (Compression) Name
-----------------------------------------

19
tools/test/h5repack/testfiles/plugin_zero.h5repack_layout.h5.tst
View File

@ -1,7 +1,22 @@
Objects to modify layout are...
Objects to apply filter are...
No all objects to modify layout
All objects to apply filter are...
User Defined 250
Making new file ...
-----------------------------------------
Type Filter (Compression) Name
-----------------------------------------
group /
warning: could not create dataset </dset1>. Applying original settings
dset /dset1
warning: could not create dataset </dset2>. Applying original settings
dset /dset2
warning: could not create dataset </dset3>. Applying original settings
dset /dset3
warning: could not create dataset </dset4>. Applying original settings
dset /dset4
warning: could not create dataset </dset_chunk>. Applying original settings
dset /dset_chunk
warning: could not create dataset </dset_compact>. Applying original settings
dset /dset_compact
warning: could not create dataset </dset_contiguous>. Applying original settings
dset /dset_contiguous