mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-03-31 17:10:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r8969] Purpose:

Browse Source 
Bug fix.

Description:
    Address two problems:
        - The computation of the scanline in the szip filter was being
            performed in the "can apply" callback routine instead of the
            "set local" routine.
        - The routine which allocated all the chunks for an entire dataset
            (which is invoked when the allocation time is early or late,
            rather than incremental) wasn't recording a failed filter in
            the information for the chunk, causing the library to believe
            that the chunk had the filter applied when it really hadn't.

Solution:
    - Move the scanline computation to the "set local" callback.

    - Record the filter mask with each chunk created when allocating them.

Platforms tested:
    FreeBSD 4.10 (sleipnir) w/szip
    Too obscure to require h5committest
This commit is contained in:
Quincey Koziol 2004-07-29 22:34:15 -05:00
parent abf5d5cf60
commit c4015e03e5
4 changed files with 45 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
release_docs/RELEASE.txt
View File

@ -170,6 +170,9 @@ Bug Fixes since HDF5-1.6.0 release
Library
-------
- Fixed obscure bug where a filter which failed during chunk allocation
could allow library to write uncompressed data to disk but think
the data was compressed. QAK - 2004/07/29
- Fixed bug where I/O to an extendible chunked dataset with zero-sized
dimensions would cause library to fail an assertion.
QAK - 2004/07/27

4
src/H5Distore.c
View File

@ -2323,6 +2323,7 @@ H5D_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5D_t *dset,
{
hssize_t chunk_offset[H5O_LAYOUT_NDIMS]; /* Offset of current chunk */
hsize_t chunk_size; /* Size of chunk in bytes */
unsigned filter_mask=0; /* Filter mask for chunks that have them */
H5O_pline_t pline; /* I/O pipeline information */
H5O_fill_t fill; /* Fill value information */
H5D_fill_time_t fill_time; /* When to write fill values */
@ -2444,7 +2445,6 @@ H5D_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5D_t *dset,
/* Check if there are filters which need to be applied to the chunk */
if (pline.nused>0) {
unsigned filter_mask=0;
size_t buf_size=(size_t)chunk_size;
size_t nbytes=(size_t)chunk_size;
@ -2483,7 +2483,7 @@ H5D_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5D_t *dset,
if(!chunk_exists) {
/* Initialize the chunk information */
udata.mesg = &dset->layout;
udata.key.filter_mask = 0;
udata.key.filter_mask = filter_mask;
udata.addr = HADDR_UNDEF;
H5_CHECK_OVERFLOW(chunk_size,hsize_t,size_t);
udata.key.nbytes = (size_t)chunk_size;

84
src/H5Zszip.c
View File

@ -96,29 +96,14 @@ H5Z_class_t H5Z_SZIP[1] = {{
static herr_t
H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
{
unsigned flags; /* Filter flags */
size_t cd_nelmts=H5Z_SZIP_USER_NPARMS; /* Number of filter parameters */
unsigned cd_values[H5Z_SZIP_TOTAL_NPARMS]; /* Filter parameters */
hsize_t dims[H5O_LAYOUT_NDIMS]; /* Dataspace (i.e. chunk) dimensions */
int ndims; /* Number of chunk dimensions */
hssize_t npoints; /* Number of points in the dataspace */
unsigned dtype_size; /* Datatype's size (in bits) */
H5T_order_t dtype_order; /* Datatype's endianness order */
hsize_t scanline; /* Size of dataspace's fastest changing dimension */
herr_t ret_value=TRUE; /* Return value */
FUNC_ENTER_NOAPI(H5Z_can_apply_szip, FAIL)
/* Get the filter's current parameters */
#ifdef H5_WANT_H5_V1_6_COMPAT
if(H5Pget_filter_by_id(dcpl_id,H5Z_FILTER_SZIP,&flags,&cd_nelmts, cd_values,0,NULL)<0)
#else
if(H5Pget_filter_by_id(dcpl_id,H5Z_FILTER_SZIP,&flags,&cd_nelmts, cd_values,0,NULL,NULL)<0)
#endif
HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get szip parameters")
/* Get datatype's size, for checking the "bits-per-pixel" */
if((dtype_size=(sizeof(unsigned char)*H5Tget_size(type_id)))==0)
if((dtype_size=(8*H5Tget_size(type_id)))==0)
HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size")
/* Range check datatype's size */
@ -133,36 +118,6 @@ H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
/* (Note: this may not handle non-atomic datatypes well) */
if(dtype_order != H5T_ORDER_LE && dtype_order != H5T_ORDER_BE)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "invalid datatype endianness order")
/* Get dimensions for dataspace */
if ((ndims=H5Sget_simple_extent_dims(space_id, dims, NULL))<0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get dataspace dimensions")
/* Get "local" parameter for this dataset's "pixels-per-scanline" */
/* (Use the chunk's fastest changing dimension size) */
assert(ndims>0);
scanline=dims[ndims-1];
/* Adjust scanline if it is smaller than number of pixels per block or
if it is bigger than maximum pixels per scanline, or there are more than
SZ_MAX_BLOCKS_PER_SCANLINE blocks per scanline */
/* Check the pixels per block against the 'scanline' size */
if(scanline<cd_values[H5Z_SZIP_PARM_PPB]) {
/* Get number of elements for the dataspace; use
total number of elements in the chunk to define the new 'scanline' size */
if ((npoints=H5Sget_simple_extent_npoints(space_id))<0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get number of points in the dataspace")
if(npoints<cd_values[H5Z_SZIP_PARM_PPB])
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "pixels per block greater than total number of elements in the chunk")
scanline = MIN((cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), npoints);
HGOTO_DONE(TRUE);
}
if(scanline <= SZ_MAX_PIXELS_PER_SCANLINE)
scanline = MIN((cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), scanline);
else
scanline = cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE;
done:
FUNC_LEAVE_NOAPI(ret_value)
@ -193,6 +148,7 @@ H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
hsize_t dims[H5O_LAYOUT_NDIMS]; /* Dataspace (i.e. chunk) dimensions */
int ndims; /* Number of (chunk) dimensions */
H5T_order_t dtype_order; /* Datatype's endianness order */
hsize_t scanline; /* Size of dataspace's fastest changing dimension */
herr_t ret_value=SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(H5Z_set_local_szip, FAIL)
@ -205,18 +161,44 @@ H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
#endif
HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get szip parameters")
/* Get dimensions for dataspace */
if ((ndims=H5Sget_simple_extent_dims(space_id, dims, NULL))<0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get dataspace dimensions")
/* Set "local" parameter for this dataset's "bits-per-pixel" */
if((cd_values[H5Z_SZIP_PARM_BPP]=(8*sizeof(unsigned char)*H5Tget_size(type_id)))==0)
HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size")
/* Get dimensions for dataspace */
if ((ndims=H5Sget_simple_extent_dims(space_id, dims, NULL))<0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get dataspace dimensions")
/* Set "local" parameter for this dataset's "pixels-per-scanline" */
/* (Use the chunk's fastest changing dimension size) */
assert(ndims>0);
H5_ASSIGN_OVERFLOW(cd_values[H5Z_SZIP_PARM_PPS],dims[ndims-1],hsize_t,unsigned);
scanline=dims[ndims-1];
/* Adjust scanline if it is smaller than number of pixels per block or
if it is bigger than maximum pixels per scanline, or there are more than
SZ_MAX_BLOCKS_PER_SCANLINE blocks per scanline */
/* Check the pixels per block against the 'scanline' size */
if(scanline<cd_values[H5Z_SZIP_PARM_PPB]) {
hssize_t npoints; /* Number of points in the dataspace */
/* Get number of elements for the dataspace; use
total number of elements in the chunk to define the new 'scanline' size */
if ((npoints=H5Sget_simple_extent_npoints(space_id))<0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get number of points in the dataspace")
if(npoints<cd_values[H5Z_SZIP_PARM_PPB])
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "pixels per block greater than total number of elements in the chunk")
scanline = MIN((cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), npoints);
}
else {
if(scanline <= SZ_MAX_PIXELS_PER_SCANLINE)
scanline = MIN((cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), scanline);
else
scanline = cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE;
} /* end else */
/* Assign the final value to the scanline */
H5_ASSIGN_OVERFLOW(cd_values[H5Z_SZIP_PARM_PPS],scanline,hsize_t,unsigned);
/* Get datatype's endianness order */
if((dtype_order=H5Tget_order(type_id))==H5T_ORDER_ERROR)

12
test/tmisc.c
View File

@ -240,8 +240,8 @@ unsigned m13_rdata[MISC13_DIM1][MISC13_DIM2]; /* Data read from dataset
#define MISC21_SPACE_RANK 2
#define MISC21_SPACE_DIM0 7639
#define MISC21_SPACE_DIM1 6308
#define MISC21_CHUNK_DIM0 1024
#define MISC21_CHUNK_DIM1 1024
#define MISC21_CHUNK_DIM0 2048
#define MISC21_CHUNK_DIM1 2048
/****************************************************************
**
@ -3493,6 +3493,7 @@ test_misc20(void)
** don't exactly match the dataspace.
**
****************************************************************/
#ifdef H5_HAVE_FILTER_SZIP
static void
test_misc21(void)
{
@ -3520,7 +3521,7 @@ test_misc21(void)
/* Set custom DCPL properties */
ret = H5Pset_chunk (dcpl, MISC21_SPACE_RANK, chunk_size);
CHECK(ret, FAIL, "H5Pset_chunk");
ret = H5Pset_deflate (dcpl, 6);
ret = H5Pset_szip (dcpl, H5_SZIP_NN_OPTION_MASK, 8);
CHECK(ret, FAIL, "H5Pset_deflate");
ret = H5Pset_alloc_time (dcpl, H5D_ALLOC_TIME_LATE);
CHECK(ret, FAIL, "H5Pset_alloc_time");
@ -3549,6 +3550,7 @@ test_misc21(void)
HDfree(buf);
} /* end test_misc21() */
#endif /* H5_HAVE_FILTER_SZIP */
/****************************************************************
**
@ -3581,9 +3583,9 @@ test_misc(void)
test_misc18(); /* Test new object header information in H5G_stat_t struct */
test_misc19(); /* Test incrementing & decrementing ref count on IDs */
test_misc20(); /* Test problems with truncated dimensions in version 2 of storage layout message */
#ifdef H5_HAVE_FILTER_DEFLATE
#ifdef H5_HAVE_FILTER_SZIP
test_misc21(); /* Test that "late" allocation time is treated the same as "incremental", for chunked datasets w/a filters */
#endif /* H5_HAVE_FILTER_DEFLATE */
#endif /* H5_HAVE_FILTER_SZIP */
} /* test_misc() */