mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-11-27 02:10:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r22277] Purpose:

Browse Source 
Fix for HDFFV-7993 - h5repack fails with error "chunk size must be <= maximum dimension size for fixed-sized dimensions"

Description:
  Fixed a failure when change the chunk size of a specified chunked dataset with unlimited max dims.
  Also took care of converting to contiguous and compact from the dataset.
  Test cases were added and tagged with jira#.

Tested:
  jam (linux32-LE), koala (linux64-LE), ostrich (linuxppc64-BE), tejeda (mac32-LE), linew (solaris-BE),  Windows (32-LE cmake), Cmake (jam)
This commit is contained in:
Jonathan Kim 2012-04-11 15:57:10 -05:00
parent 5044553875
commit 10b62203a3
11 changed files with 235 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MANIFEST
View File

@ -1944,6 +1944,7 @@
./tools/h5repack/testfiles/h5repack_layout.h5
./tools/h5repack/testfiles/h5repack_layouto.h5
./tools/h5repack/testfiles/h5repack_layout2.h5
./tools/h5repack/testfiles/h5repack_layout3.h5
./tools/h5repack/testfiles/h5repack_early.h5
./tools/h5repack/testfiles/h5repack_szip.h5
./tools/h5repack/testfiles/h5repack_deflate.h5

2
release_docs/RELEASE.txt
View File

@ -696,6 +696,8 @@ Bug Fixes since HDF5-1.8.0 release
Tools
-----
- h5repack: Fixed a failure when change the chunk size of a specified
chunked dataset with unlimited max dims. HDFFV-7993 (JKM 2012/04/11)
- h5diff: Fixed failure for comparing same named object with different
object types in comparing groups. Prior to the fix, h5diff resulted
in error. After the fix, h5diff detects such case as non-comparable

31
tools/h5repack/CMakeLists.txt
View File

@ -96,6 +96,7 @@ IF (BUILD_TESTING)
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layouto.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout2.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout3.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_named_dtypes.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_nbit.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_objs.h5
@ -322,10 +323,12 @@ IF (BUILD_TESTING)
-E remove
./testfiles/h5repack_filters.h5.out
./testfiles/h5repack_filters.h5.out.err
./testfiles/h5repack_layout2.h5-v.out
./testfiles/h5repack_layout2.h5-v.out.err
./testfiles/h5repack_layout.h5-v.out
./testfiles/h5repack_layout.h5-v.out.err
./testfiles/h5repack_layout2.h5-v.out
./testfiles/h5repack_layout2.h5-v.out.err
./testfiles/h5repack_layout3.h5-v.out
./testfiles/h5repack_layout3.h5-v.out.err
./testfiles/out.tfamily%05d.h5
./testfiles/out.h5diff_attr1.h5
./testfiles/out.h5repack_attr.h5
@ -336,9 +339,10 @@ IF (BUILD_TESTING)
./testfiles/out.h5repack_filters.h5
./testfiles/out.h5repack_fletcher.h5
./testfiles/out.h5repack_hlink.h5
./testfiles/out.h5repack_layout2.h5
./testfiles/out.h5repack_layout.h5
./testfiles/out.h5repack_layouto.h5
./testfiles/out.h5repack_layout2.h5
./testfiles/out.h5repack_layout3.h5
./testfiles/out.h5repack_named_dtypes.h5
./testfiles/out.h5repack_nbit.h5
./testfiles/out.h5repack_objs.h5
@ -369,9 +373,10 @@ IF (BUILD_TESTING)
h5repack_fletcher_out.h5
h5repack_hlink.h5
h5repack_hlink_out.h5
h5repack_layout2.h5
h5repack_layout.h5
h5repack_layout_out.h5
h5repack_layout2.h5
h5repack_layout3.h5
h5repack_named_dtypes.h5
h5repack_named_dtypes_out.h5
h5repack_nbit.h5
@ -777,6 +782,24 @@ IF (BUILD_TESTING)
ADD_H5_TEST (contig_small_fixed_compa "TEST" ${FILE18} -l chunked_small_fixed:COMPA)
ADD_H5_VERIFY_TEST (contig_small_fixed_compa "TEST" 0 ${FILE18} chunked_small_fixed COMPACT)
#---------------------------------------------------------------------------
# Test file contains chunked datasets (need multiple dsets) with
# unlimited max dims. (HDFFV-7933)
# Use first dset to test.
#---------------------------------------------------------------------------
# chunk to chunk - specify chunk dim bigger than any current dim
ADD_H5_TEST (chunk2chunk "TEST" h5repack_layout3.h5 -l chunk_unlimit1:CHUNK=100x300)
ADD_H5_VERIFY_TEST (chunk2chunk "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CHUNK)
# chunk to contiguous
ADD_H5_TEST (chunk2conti "TEST" h5repack_layout3.h5 -l chunk_unlimit1:CONTI)
ADD_H5_VERIFY_TEST (chunk2conti "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CONTI)
# chunk to compact - convert big dataset (should be > 64k) for this purpose,
# should remain as original layout (chunk)
ADD_H5_TEST (chunk2compa "TEST" h5repack_layout3.h5 -l chunk_unlimit1:COMPA)
ADD_H5_VERIFY_TEST (chunk2compa "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CHUNK)
# Native option
# Do not use FILE1, as the named dtype will be converted to native, and h5diff will
# report a difference.

9
tools/h5repack/h5repack.c
View File

@ -206,10 +206,12 @@ int h5repack_addlayout(const char* str,
if (obj_list==NULL)
return -1;
/* set global layout option */
/* set layout option */
options->layout_g = pack.layout;
/* no individual dataset specified */
if (options->all_layout==1 )
{
options->layout_g=pack.layout;
if (pack.layout==H5D_CHUNKED)
{
/* -2 means the NONE option, remove chunking
@ -228,6 +230,7 @@ int h5repack_addlayout(const char* str,
}
}
/* individual dataset specified */
if (options->all_layout==0)
options_add_layout(obj_list,
n_objs,
@ -504,7 +507,7 @@ int copy_attr(hid_t loc_in,
if (type_class == H5T_COMPOUND) {
int nmembers = H5Tget_nmembers(wtype_id) ;
for (j=0; j<nmembers; j++) {
hid_t mtid = H5Tget_member_type( wtype_id, j );
hid_t mtid = H5Tget_member_type( wtype_id, (unsigned) j );
H5T_class_t mtclass = H5Tget_class(mtid);
H5Tclose(mtid);

1
tools/h5repack/h5repack.h
View File

@ -28,6 +28,7 @@
#define MAX_VAR_DIMS 32 /* max per variable dimensions */
#define FORMAT_OBJ " %-27s %s\n" /* obj type, name */
#define FORMAT_OBJ_ATTR " %-27s %s\n" /* obj type, name */
#define MAX_COMPACT_DSIZE 64512 /* max data size for compact layout. -1k for header size */
/*-------------------------------------------------------------------------
* data structures for command line options

51
tools/h5repack/h5repack.sh.in
View File

@ -94,6 +94,7 @@ $SRC_H5REPACK_TESTFILES/h5repack_hlink.h5
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5
$SRC_H5REPACK_TESTFILES/h5repack_layouto.h5
$SRC_H5REPACK_TESTFILES/h5repack_layout2.h5
$SRC_H5REPACK_TESTFILES/h5repack_layout3.h5
$SRC_H5REPACK_TESTFILES/h5repack_named_dtypes.h5
$SRC_H5REPACK_TESTFILES/h5repack_nbit.h5
$SRC_H5REPACK_TESTFILES/h5repack_objs.h5
@ -165,7 +166,7 @@ TESTING() {
# beginning with the word "Verifying".
#
VERIFY() {
MESSAGE "Verifying h5diff output $*"
MESSAGE "Verifying $*"
}
# Print a message that a test has been skipped (because a required filter
@ -179,7 +180,7 @@ SKIP() {
#
DIFFTEST()
{
VERIFY $@
VERIFY h5diff output $@
$RUNSERIAL $H5DIFF_BIN -q "$@"
RET=$?
if [ $RET != 0 ] ; then
@ -235,7 +236,7 @@ VERIFY_LAYOUT_DSET()
#---------------------------------
# check the layout from a dataset
VERIFY "Layout"
VERIFY "a dataset layout"
$RUNSERIAL $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile
$GREP $expectlayout $layoutfile > /dev/null
if [ $? -eq 0 ]; then
@ -260,7 +261,7 @@ VERIFY_LAYOUT_ALL()
#---------------------------------
# check the layout from a dataset
# check if the other layouts still exsit
VERIFY "Layout "
VERIFY "layouts"
# if CONTIGUOUS
if [ $expectlayout = "CONTIGUOUS" ]; then
$RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
@ -360,7 +361,8 @@ TOOLTEST1()
rm -f $outfile
}
# Call h5repack and compare output to a text file for -v option
# This is same as TOOLTEST() with comparing display output
# from -v option
#
TOOLTESTV()
{
@ -374,10 +376,20 @@ TOOLTESTV()
outfile=$TESTDIR/out.$1
shift
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile >$actual 2>$actual_err
cp $actual $actual_sav
RET=$?
if [ $RET != 0 ] ; then
echo "*FAILED*"
nerrors="`expr $nerrors + 1`"
else
echo " PASSED"
DIFFTEST $infile $outfile
fi
# display output compare
STDOUT_FILTER $actual
cat $actual_err >> $actual
VERIFY output from $H5REPACK $@
if cmp -s $expect $actual; then
echo " PASSED"
else
@ -387,7 +399,8 @@ TOOLTESTV()
test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /'
fi
rm -f $actual $actual_err $actual_sav
rm -f $actual $actual_err
rm -f $outfile
}
# TOOLTEST_META:
@ -559,7 +572,8 @@ arg="h5repack_filters.h5 -v -f /dset_deflate:GZIP=9"
if test $USE_FILTER_DEFLATE != "yes" ; then
SKIP $arg
else
TOOLTEST $arg
# compare output
TOOLTESTV $arg
fi
###########################################################
@ -778,6 +792,24 @@ VERIFY_LAYOUT_DSET h5repack_layout2.h5 contig_small COMPACT
TOOLTEST_MAIN h5repack_layout2.h5 -l chunked_small_fixed:COMPA
VERIFY_LAYOUT_DSET h5repack_layout2.h5 chunked_small_fixed COMPACT
#---------------------------------------------------------------------------
# Test file contains chunked datasets (need multiple dsets) with
# unlimited max dims. (HDFFV-7933)
# Use first dset to test.
#---------------------------------------------------------------------------
# chunk to chunk - specify chunk dim bigger than any current dim
TOOLTEST_MAIN h5repack_layout3.h5 -l chunk_unlimit1:CHUNK=100x300
VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CHUNK
# chunk to contiguous
TOOLTEST_MAIN h5repack_layout3.h5 -l chunk_unlimit1:CONTI
VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CONTI
# chunk to compact - convert big dataset (should be > 64k) for this purpose,
# should remain as original layout (chunk)
TOOLTEST_MAIN h5repack_layout3.h5 -l chunk_unlimit1:COMPA
VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CHUNK
# Native option
# Do not use FILE1, as the named dtype will be converted to native, and h5diff will
# report a difference.
@ -856,8 +888,7 @@ TOOLTEST h5repack_attr_refs.h5
# Note: this test is experimental for sharing test file among tools
TOOLTEST h5diff_attr1.h5
# tests for metadata block size option ('-M')
TOOLTEST_META h5repack_layout.h5 -M 8192
# tests for metadata block size option
TOOLTEST_META h5repack_layout.h5 --metadata_block_size=8192
if test $nerrors -eq 0 ; then

46
tools/h5repack/h5repack_copy.c
View File

@ -778,6 +778,8 @@ int do_copy_objects(hid_t fidin,
unsigned u;
int is_ref=0;
htri_t is_named;
hbool_t limit_maxdims;
hsize_t size_dset;
/*-------------------------------------------------------------------------
@ -794,8 +796,10 @@ int do_copy_objects(hid_t fidin,
for ( i = 0; i < travt->nobjs; i++)
{
/* init variables per obj */
buf = NULL;
limit_maxdims = FALSE;
switch ( travt->objs[i].type )
{
@ -970,6 +974,9 @@ int do_copy_objects(hid_t fidin,
if((msize = H5Tget_size(wtype_id)) == 0)
goto error;
/* size of current dset */
size_dset = nelmts * msize;
/*-------------------------------------------------------------------------
* check if the dataset creation property list has filters that
* are not registered in the current configuration
@ -999,7 +1006,7 @@ int do_copy_objects(hid_t fidin,
*/
if (options->layout_g != H5D_COMPACT)
{
if ( nelmts*msize < options->min_comp )
if ( size_dset < options->min_comp )
apply_s=0;
}
@ -1016,9 +1023,38 @@ int do_copy_objects(hid_t fidin,
goto error;
}
/* unset the unlimimted dimensions, which cannot be applied to layout other than chunked. */
if (options->layout_g != H5D_CHUNKED) {
H5Sset_extent_simple( f_space_id, rank, dims, NULL );
/*-------------------------------------------------
* Unset the unlimited max dims if convert to other
* than chunk layouts, because unlimited max dims
* only can be applied to chunk layout.
* Also perform only for targeted dataset
* Also check for size limit to convert to compact
*-------------------------------------------------*/
if (options->layout_g != H5D_CHUNKED)
{
/* any dataset is specified */
if (options->op_tbl->nelems > 0)
{
/* if current obj match specified obj */
if (options_get_object (travt->objs[i].name, options->op_tbl))
limit_maxdims = TRUE;
}
else /* no dataset is specified */
{
limit_maxdims = TRUE;
}
/* if convert to COMPACT */
if (options->layout_g == H5D_COMPACT)
{
/* should be smaller than 64K */
if ( size_dset > MAX_COMPACT_DSIZE )
limit_maxdims = FALSE;
}
/* unset unlimited max dims */
if (limit_maxdims)
H5Sset_extent_simple( f_space_id, rank, dims, NULL );
}
/*-------------------------------------------------------------------------

2
tools/h5repack/h5repack_main.c
View File

@ -466,6 +466,8 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options)
options->fs_threshold = (hsize_t)HDatol( opt_arg );
break;
default:
break;
} /* switch */

13
tools/h5repack/h5repack_opttable.c
View File

@ -388,11 +388,22 @@ pack_info_t* options_get_object( const char *path,
pack_opttbl_t *table )
{
unsigned int i;
const char tbl_path[MAX_NC_NAME];
for ( i = 0; i < table->nelems; i++)
{
/* make full path (start with "/") to compare correctly */
if (HDstrncmp(table->objs[i].path, "/", 1))
{
HDstrcpy(tbl_path, "/");
HDstrcat(tbl_path, table->objs[i].path);
}
else
HDstrcpy(tbl_path, table->objs[i].path);
/* found it */
if (HDstrcmp(table->objs[i].path,path)==0)
if (HDstrcmp(tbl_path, path)==0)
{
return (&table->objs[i]);
}

102
tools/h5repack/h5repacktst.c
View File

@ -124,6 +124,7 @@ static int make_hlinks(hid_t loc_id);
static int make_early(void);
static int make_layout(hid_t loc_id);
static int make_layout2(hid_t loc_id);
static int make_layout3(hid_t loc_id);
#ifdef H5_HAVE_FILTER_SZIP
static int make_szip(hid_t loc_id);
#endif /* H5_HAVE_FILTER_SZIP */
@ -1711,6 +1712,19 @@ int make_testfiles(void)
if(H5Fclose(fid) < 0)
return -1;
/*-------------------------------------------------------------------------
* for test layout conversions form chunk with unlimited max dims
*-------------------------------------------------------------------------
*/
if((fid = H5Fcreate("h5repack_layout3.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
return -1;
if(make_layout3(fid) < 0)
goto out;
if(H5Fclose(fid) < 0)
return -1;
/*-------------------------------------------------------------------------
* create a file for the H5D_ALLOC_TIME_EARLY test
*-------------------------------------------------------------------------
@ -3074,6 +3088,94 @@ out:
} /* make_layout2() */
/*-------------------------------------------------------------------------
* Function: make_layout3
*
* Purpose: make chunked datasets with unlimited max dim and chunk dim is
* bigger than current dim. (HDFFV-7933)
* Test for converting chunk to chunk , chunk to conti and chunk
* to compact.
* - The chunk to chunk changes layout bigger than any current dim
* again.
* - The chunk to compact test dataset bigger than 64K, should
* remain original layout.*
*
*-------------------------------------------------------------------------
*/
#define DIM1_L3 300
#define DIM2_L3 200
static
int make_layout3(hid_t loc_id)
{
hid_t dcpl=-1; /* dataset creation property list */
hid_t sid=-1; /* dataspace ID */
hsize_t dims[RANK]={DIM1_L3,DIM2_L3};
hsize_t maxdims[RANK]={H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk_dims[RANK]={DIM1_L3*2,5};
int buf[DIM1_L3][DIM2_L3];
int i, j, n;
for (i=n=0; i<DIM1_L3; i++)
{
for (j=0; j<DIM2_L3; j++)
{
buf[i][j]=n++;
}
}
/*-------------------------------------------------------------------------
* make several dataset with several layout options
*-------------------------------------------------------------------------
*/
/* create a space */
if((sid = H5Screate_simple(RANK, dims, maxdims)) < 0)
return -1;
/* create a dataset creation property list; the same DCPL is used for all dsets */
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
{
goto out;
}
/*-------------------------------------------------------------------------
* H5D_CHUNKED
*-------------------------------------------------------------------------
*/
if(H5Pset_chunk(dcpl, RANK, chunk_dims) < 0)
goto out;
if (make_dset(loc_id,"chunk_unlimit1",sid,dcpl,buf) < 0)
{
goto out;
}
if(H5Pset_chunk(dcpl, RANK, chunk_dims) < 0)
goto out;
if (make_dset(loc_id,"chunk_unlimit2",sid,dcpl,buf) < 0)
{
goto out;
}
/*-------------------------------------------------------------------------
* close space and dcpl
*-------------------------------------------------------------------------
*/
if(H5Sclose(sid) < 0)
goto out;
if(H5Pclose(dcpl) < 0)
goto out;
return 0;
out:
H5E_BEGIN_TRY
{
H5Pclose(dcpl);
H5Sclose(sid);
} H5E_END_TRY;
return -1;
}
/*-------------------------------------------------------------------------
* Function: make a file with an integer dataset with a fill value
*

BIN
tools/h5repack/testfiles/h5repack_layout3.h5 Normal file
View File

Binary file not shown.