mirror of
https://github.com/HDFGroup/hdf5.git
synced 2024-12-27 08:01:04 +08:00
8d1fe44164
* H5repack tests should fail if a corrupted file causes h5repack to segfault/core dump. * Add release note for HDFV-10590, CVE-2018-17432.
1845 lines
55 KiB
Bash
1845 lines
55 KiB
Bash
#! /bin/sh
|
|
#
|
|
# Copyright by The HDF Group.
|
|
# Copyright by the Board of Trustees of the University of Illinois.
|
|
# All rights reserved.
|
|
#
|
|
# This file is part of HDF5. The full HDF5 copyright notice, including
|
|
# terms governing use, modification, and redistribution, is contained in
|
|
# the COPYING file, which can be found at the root of the source code
|
|
# distribution tree, or in https://www.hdfgroup.org/licenses.
|
|
# If you do not have access to either file, you may request a copy from
|
|
# help@hdfgroup.org.
|
|
#
|
|
# Tests for the h5repack tool
|
|
#
|
|
# Modification:
|
|
# Pedro Vicente Nunes, 11/15/2006
|
|
# Added $FILEN variables for file names
|
|
#
|
|
|
|
srcdir=@srcdir@
|
|
|
|
USE_FILTER_SZIP="@USE_FILTER_SZIP@"
|
|
USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"
|
|
|
|
TESTNAME=h5repack
|
|
EXIT_SUCCESS=0
|
|
EXIT_FAILURE=1
|
|
|
|
H5REPACK=../../src/h5repack/h5repack # The tool name
|
|
H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary
|
|
|
|
H5DIFF=../../src/h5diff/h5diff # The h5diff tool name
|
|
H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary
|
|
|
|
H5DUMP=../../src/h5dump/h5dump # The h5dump tool name
|
|
H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary
|
|
|
|
H5STAT=../../src/h5stat/h5stat # The h5stat tool name
|
|
H5STAT_BIN=`pwd`/$H5STAT # The path of the h5stat tool binary
|
|
|
|
RM='rm -rf'
|
|
CMP='cmp'
|
|
DIFF='diff -c'
|
|
GREP='grep'
|
|
CP='cp'
|
|
DIRNAME='dirname'
|
|
LS='ls'
|
|
AWK='awk'
|
|
|
|
# Reset the VOL connector to only use the native connector, because of the
|
|
# error stack checking. QAK - 2019/03/09
|
|
ENVCMD="env HDF5_VOL_CONNECTOR=native"
|
|
|
|
H5DETECTSZIP=testh5repack_detect_szip
|
|
H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP
|
|
|
|
|
|
nerrors=0
|
|
verbose=yes
|
|
|
|
# source dirs
|
|
SRC_TOOLS="$srcdir/../.."
|
|
SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles"
|
|
|
|
# testfiles source dirs for tools
|
|
SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES"
|
|
SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES"
|
|
SRC_H5DIFF_TESTFILES="$SRC_TOOLS/test/h5diff/testfiles"
|
|
SRC_H5COPY_TESTFILES="$SRC_TOOLS/test/h5copy/testfiles"
|
|
SRC_H5REPACK_TESTFILES="$SRC_TOOLS/test/h5repack/testfiles"
|
|
SRC_H5JAM_TESTFILES="$SRC_TOOLS/test/h5jam/testfiles"
|
|
SRC_H5STAT_TESTFILES="$SRC_TOOLS/test/h5stat/testfiles"
|
|
SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/test/h5import/testfiles"
|
|
|
|
TESTDIR=./testpack
|
|
test -d $TESTDIR || mkdir $TESTDIR
|
|
|
|
######################################################################
|
|
# test files
|
|
# --------------------------------------------------------------------
|
|
# All the test files copy from source directory to test directory
|
|
# NOTE: Keep this framework to add/remove test files.
|
|
# Any test files from other tools can be used in this framework.
|
|
# This list are also used for checking exist.
|
|
# Comment '#' without space can be used.
|
|
# --------------------------------------------------------------------
|
|
LIST_HDF5_TEST_FILES="
|
|
$SRC_H5REPACK_TESTFILES/bounds_latest_latest.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_attr.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_attr_refs.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_deflate.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_early.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_ext.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_f32le.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_fill.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_filters.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_fletcher.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_hlink.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_layouto.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_layout2.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_layout3.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_named_dtypes.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum_deflated.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_nbit.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_objs.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_refs.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_soffset.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_szip.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_uint8be.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex.h5
|
|
########fsm#files########
|
|
$SRC_H5REPACK_TESTFILES/h5repack_aggr.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_nopersist.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_none.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5
|
|
########h5diff/testfile########
|
|
$SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
|
|
########test#HDFFV-10590########
|
|
$SRC_H5REPACK_TESTFILES/h5repack_CVE-2018-17432.h5
|
|
$SRC_H5REPACK_TESTFILES/h5repack_CVE-2018-14460.h5
|
|
########tools/testfiles#for#external#links########
|
|
$SRC_TOOLS_TESTFILES/tsoftlinks.h5
|
|
$SRC_TOOLS_TESTFILES/textlinkfar.h5
|
|
$SRC_TOOLS_TESTFILES/textlinksrc.h5
|
|
$SRC_TOOLS_TESTFILES/textlinktar.h5
|
|
$SRC_TOOLS_TESTFILES/textlink.h5
|
|
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src.h5
|
|
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_trg.h5
|
|
########tools/testfiles########
|
|
$SRC_TOOLS_TESTFILES/tfamily00000.h5
|
|
$SRC_TOOLS_TESTFILES/tfamily00001.h5
|
|
$SRC_TOOLS_TESTFILES/tfamily00002.h5
|
|
$SRC_TOOLS_TESTFILES/tfamily00003.h5
|
|
$SRC_TOOLS_TESTFILES/tfamily00004.h5
|
|
$SRC_TOOLS_TESTFILES/tfamily00005.h5
|
|
$SRC_TOOLS_TESTFILES/tfamily00006.h5
|
|
$SRC_TOOLS_TESTFILES/tfamily00007.h5
|
|
$SRC_TOOLS_TESTFILES/tfamily00008.h5
|
|
$SRC_TOOLS_TESTFILES/tfamily00009.h5
|
|
$SRC_TOOLS_TESTFILES/tfamily00010.h5
|
|
$SRC_TOOLS_TESTFILES/tordergr.h5
|
|
########reference#conversion#files########
|
|
########tools/testfiles/vds########
|
|
$SRC_TOOLS_TESTFILES/vds/1_a.h5
|
|
$SRC_TOOLS_TESTFILES/vds/1_b.h5
|
|
$SRC_TOOLS_TESTFILES/vds/1_c.h5
|
|
$SRC_TOOLS_TESTFILES/vds/1_d.h5
|
|
$SRC_TOOLS_TESTFILES/vds/1_e.h5
|
|
$SRC_TOOLS_TESTFILES/vds/1_f.h5
|
|
$SRC_TOOLS_TESTFILES/vds/1_vds.h5
|
|
$SRC_TOOLS_TESTFILES/vds/2_a.h5
|
|
$SRC_TOOLS_TESTFILES/vds/2_b.h5
|
|
$SRC_TOOLS_TESTFILES/vds/2_c.h5
|
|
$SRC_TOOLS_TESTFILES/vds/2_d.h5
|
|
$SRC_TOOLS_TESTFILES/vds/2_e.h5
|
|
$SRC_TOOLS_TESTFILES/vds/2_vds.h5
|
|
$SRC_TOOLS_TESTFILES/vds/3_1_vds.h5
|
|
$SRC_TOOLS_TESTFILES/vds/3_2_vds.h5
|
|
$SRC_TOOLS_TESTFILES/vds/4_0.h5
|
|
$SRC_TOOLS_TESTFILES/vds/4_1.h5
|
|
$SRC_TOOLS_TESTFILES/vds/4_2.h5
|
|
$SRC_TOOLS_TESTFILES/vds/4_vds.h5
|
|
$SRC_TOOLS_TESTFILES/vds/5_a.h5
|
|
$SRC_TOOLS_TESTFILES/vds/5_b.h5
|
|
$SRC_TOOLS_TESTFILES/vds/5_c.h5
|
|
$SRC_TOOLS_TESTFILES/vds/5_vds.h5
|
|
"
|
|
|
|
LIST_OTHER_TEST_FILES="
|
|
$SRC_H5REPACK_TESTFILES/h5repack-help.txt
|
|
$SRC_H5REPACK_TESTFILES/h5repack_ext.bin
|
|
$SRC_H5REPACK_TESTFILES/ublock.bin
|
|
$SRC_H5REPACK_TESTFILES/h5repack.info
|
|
########dat#files########
|
|
$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex-0.dat
|
|
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-0.dat
|
|
$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-1.dat
|
|
$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex-0.dat
|
|
$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex-0.dat
|
|
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-0.dat
|
|
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-1.dat
|
|
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-2.dat
|
|
$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-3.dat
|
|
###############
|
|
$SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl
|
|
########fsm#files########
|
|
$SRC_H5REPACK_TESTFILES/SP.h5repack_fsm_aggr_nopersist.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/S.h5repack_fsm_aggr_persist.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/STG.h5repack_none.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/GS.h5repack_paged_nopersist.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/SP.h5repack_paged_persist.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/SPT.h5repack_aggr.h5.ddl
|
|
########vds#files########
|
|
$SRC_H5REPACK_TESTFILES/1_vds.h5-vds_dset_chunk20x10x5-v.ddl
|
|
$SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl
|
|
$SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl
|
|
$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl
|
|
$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl
|
|
########refs#files########
|
|
$SRC_H5REPACK_TESTFILES/attrregion.tattrreg.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/dataregion.tdatareg.h5.ddl
|
|
########external#links#files########
|
|
$SRC_H5REPACK_TESTFILES/textlinkfar-base.textlinkfar.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlinksrc-base.textlinksrc.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlinktar-base.textlinktar.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlink-base.textlink.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/tsoftlinks-base.tsoftlinks.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-base.h5copy_extlinks_src.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlinkfar-merge.textlinkfar.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/textlinksrc-merge.textlinksrc.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/textlinktar-merge.textlinktar.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/textlink-merge.textlink.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/tsoftlinks-merge.tsoftlinks.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/textlinkfar-prune.textlinkfar.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlinksrc-prune.textlinksrc.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlinktar-prune.textlinktar.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlink-prune.textlink.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/tsoftlinks-prune.tsoftlinks.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-prune.h5copy_extlinks_src.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlinkfar-mergeprune.textlinkfar.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlinksrc-mergeprune.textlinksrc.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlinktar-mergeprune.textlinktar.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/textlink-mergeprune.textlink.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/tsoftlinks-mergeprune.tsoftlinks.h5.ddl
|
|
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5.ddl
|
|
########tst#files########
|
|
$SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst
|
|
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst
|
|
$SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst
|
|
########external#links#tst#files########
|
|
$SRC_H5REPACK_TESTFILES/tsoftlinks-merge.tsoftlinks.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/textlinkfar-merge.textlinkfar.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/textlinksrc-merge.textlinksrc.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/textlinktar-merge.textlinktar.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/textlink-merge.textlink.h5.tst
|
|
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst
|
|
"
|
|
|
|
#
|
|
# copy test files and expected output files from source dirs to test dir
|
|
#
|
|
COPY_TESTFILES="$LIST_HDF5_TEST_FILES $LIST_OTHER_TEST_FILES"
|
|
|
|
COPY_TESTFILES_TO_TESTDIR()
|
|
{
|
|
# copy test files. Used -f to make sure get a new copy
|
|
for tstfile in $COPY_TESTFILES
|
|
do
|
|
# ignore '#' comment
|
|
echo $tstfile | tr -d ' ' | grep '^#' > /dev/null
|
|
RET=$?
|
|
if [ $RET -eq 1 ]; then
|
|
# skip cp if srcdir is same as destdir
|
|
# this occurs when build/test performed in source dir and
|
|
# make cp fail
|
|
SDIR=`$DIRNAME $tstfile`
|
|
INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'`
|
|
INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'`
|
|
if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then
|
|
$CP -f $tstfile $TESTDIR
|
|
if [ $? -ne 0 ]; then
|
|
echo "Error: FAILED to copy $tstfile ."
|
|
|
|
# Comment out this to CREATE expected file
|
|
exit $EXIT_FAILURE
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
CLEAN_TESTFILES_AND_TESTDIR()
|
|
{
|
|
# skip rm if srcdir is same as destdir
|
|
# this occurs when build/test performed in source dir and
|
|
# make cp fail
|
|
SDIR=$SRC_H5REPACK_TESTFILES
|
|
INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'`
|
|
INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'`
|
|
if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then
|
|
$RM $TESTDIR
|
|
fi
|
|
}
|
|
|
|
# Print a $* message left justified in a field of 70 characters
|
|
#
|
|
MESSAGE() {
|
|
SPACES=" "
|
|
echo "$* $SPACES" | cut -c1-70 | tr -d '\012'
|
|
}
|
|
|
|
# Print a line-line message left justified in a field of 70 characters
|
|
# beginning with the word "Testing".
|
|
#
|
|
TESTING() {
|
|
MESSAGE "Testing $*"
|
|
}
|
|
|
|
# Print a line-line message left justified in a field of 70 characters
|
|
# beginning with the word "Verifying".
|
|
#
|
|
VERIFY() {
|
|
MESSAGE "Verifying $*"
|
|
}
|
|
|
|
# Print a message that a test has been skipped (because a required filter
|
|
# was unavailable)
|
|
SKIP() {
|
|
TESTING $H5REPACK $@
|
|
echo " -SKIP-"
|
|
}
|
|
|
|
##############################################################################
|
|
##############################################################################
|
|
### T H E T E S T S M A C R O S ###
|
|
##############################################################################
|
|
##############################################################################
|
|
|
|
# Call the h5diff tool
|
|
#
|
|
DIFFTEST()
|
|
{
|
|
VERIFY h5diff output $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5DIFF_BIN -q "$@"
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
fi
|
|
|
|
}
|
|
|
|
# Call h5repack
|
|
#
|
|
|
|
|
|
# call TOOLTEST_MAIN and delete $output file
|
|
TOOLTEST()
|
|
{
|
|
echo $@
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
DIFFTEST $infile $outfile
|
|
fi
|
|
rm -f $outfile
|
|
}
|
|
|
|
#------------------------------------------
|
|
# Verifying layouts of a dataset
|
|
VERIFY_LAYOUT_DSET()
|
|
{
|
|
layoutfile=layout-$1.$2
|
|
dset=$3
|
|
expectlayout=$4
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
shift
|
|
shift
|
|
shift
|
|
shift
|
|
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
DIFFTEST $infile $outfile
|
|
fi
|
|
|
|
#---------------------------------
|
|
# check the layout from a dataset
|
|
VERIFY "a dataset layout"
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile
|
|
)
|
|
$GREP $expectlayout $TESTDIR/$layoutfile > /dev/null
|
|
if [ $? -eq 0 ]; then
|
|
echo " PASSED"
|
|
else
|
|
echo " FAILED"
|
|
nerrors="`expr $nerrors + 1`"
|
|
fi
|
|
|
|
# clean up tmp files
|
|
# rm -f $outfile
|
|
# rm -f $layoutfile
|
|
}
|
|
|
|
#------------------------------------------
|
|
# Verifying layouts of a dataset
|
|
VERIFY_LAYOUT_VDS()
|
|
{
|
|
layoutfile=layout-$1.$2
|
|
dset=$3
|
|
expectlayout=$4
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
|
|
expect="$TESTDIR/$2-$1-v.ddl"
|
|
actual="$TESTDIR/$2-$1-v.out"
|
|
actual_err="$TESTDIR/$2-$1-v.err"
|
|
|
|
shift
|
|
shift
|
|
shift
|
|
shift
|
|
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
fi
|
|
|
|
#---------------------------------
|
|
# check the layout from a dataset
|
|
VERIFY "a dataset layout"
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5DUMP_BIN -d $dset -p $outfile
|
|
) >$actual 2>$actual_err
|
|
|
|
cat $actual_err >> $actual
|
|
|
|
if [ ! -f $expect ]; then
|
|
# Create the expect file if it doesn't yet exist.
|
|
echo " CREATED"
|
|
cp $actual $expect
|
|
echo " Expected result (*.ddl) missing"
|
|
nerrors="`expr $nerrors + 1`"
|
|
elif $CMP $expect $actual > /dev/null 2>&1 ; then
|
|
echo " PASSED"
|
|
else
|
|
echo "*FAILED*"
|
|
echo " Expected result (*.ddl) differs from actual result (*.out)"
|
|
nerrors="`expr $nerrors + 1`"
|
|
test yes = "$verbose" && $DIFF $caseless $expect $actual |sed 's/^/ /'
|
|
fi
|
|
|
|
# Clean up output file
|
|
if test -z "$HDF5_NOCLEANUP"; then
|
|
rm -f $actual $actual_err
|
|
fi
|
|
|
|
# clean up tmp files
|
|
rm -f $outfile
|
|
}
|
|
|
|
#----------------------------------------
|
|
# Verifying layouts from entire file
|
|
VERIFY_LAYOUT_ALL()
|
|
{
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
layoutfile=layout-$1.$2
|
|
expectlayout=$3
|
|
shift
|
|
shift
|
|
shift
|
|
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
DIFFTEST $infile $outfile
|
|
fi
|
|
|
|
|
|
#---------------------------------
|
|
# check the layout from a dataset
|
|
# check if the other layouts still exsit
|
|
VERIFY "layouts"
|
|
(
|
|
cd $TESTDIR
|
|
echo
|
|
# if CONTIGUOUS
|
|
if [ $expectlayout = "CONTIGUOUS" ]; then
|
|
TESTING $H5DUMP_BIN -pH $outfile
|
|
(
|
|
$RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
|
|
)
|
|
$GREP "COMPACT" $layoutfile > /dev/null
|
|
if [ $? -eq 0 ]; then
|
|
echo " FAILED"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
$GREP "CHUNKED" $layoutfile > /dev/null
|
|
if [ $? -eq 0 ]; then
|
|
echo " FAILED"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
fi
|
|
fi
|
|
else
|
|
# if COMPACT
|
|
if [ $expectlayout = "COMPACT" ]; then
|
|
TESTING $H5DUMP_BIN -pH $outfile
|
|
(
|
|
$RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
|
|
)
|
|
$GREP "CHUNKED" $layoutfile > /dev/null
|
|
if [ $? -eq 0 ]; then
|
|
echo " FAILED"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
$GREP "CONTIGUOUS" $layoutfile > /dev/null
|
|
if [ $? -eq 0 ]; then
|
|
echo " FAILED"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
fi
|
|
fi
|
|
else
|
|
# if CHUNKED
|
|
if [ $expectlayout = "CHUNKED" ]; then
|
|
TESTING $H5DUMP_BIN -pH $outfile
|
|
(
|
|
$RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
|
|
)
|
|
$GREP "CONTIGUOUS" $layoutfile > /dev/null
|
|
if [ $? -eq 0 ]; then
|
|
echo " FAILED"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
$GREP "COMPACT" $layoutfile > /dev/null
|
|
if [ $? -eq 0 ]; then
|
|
echo " FAILED"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
)
|
|
|
|
# clean up tmp files
|
|
rm -f $outfile
|
|
rm -f $layoutfile
|
|
}
|
|
|
|
#--------------------------------------------------------------
|
|
# Verifying superblock of a file with different version bounds
|
|
VERIFY_SUPERBLOCK()
|
|
{
|
|
lowbound=$1
|
|
highbound=$2
|
|
superblock=$3
|
|
infile=$4
|
|
layoutfile=verbounds-low$lowbound-high$highbound.$4
|
|
outfile=out-verbounds-low$lowbound-high$highbound.$infile
|
|
expectedsuperblock="SUPERBLOCK_VERSION $superblock"
|
|
|
|
shift
|
|
shift
|
|
shift
|
|
shift
|
|
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN -j $lowbound -k $highbound $infile $outfile
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
DIFFTEST $infile $outfile
|
|
fi
|
|
|
|
#---------------------------------
|
|
# check the layout from a dataset
|
|
VERIFY "superblock"
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5DUMP_BIN -H -B $outfile > $layoutfile
|
|
)
|
|
$GREP "$expectedsuperblock" $TESTDIR/$layoutfile > /dev/null
|
|
if [ $? -eq 0 ]; then
|
|
echo " PASSED"
|
|
else
|
|
echo " FAILED"
|
|
nerrors="`expr $nerrors + 1`"
|
|
fi
|
|
|
|
# clean up tmp files
|
|
rm -f $outfile
|
|
rm -f $layoutfile
|
|
|
|
} # end of VERIFY_SUPERBLOCK
|
|
|
|
#------------------------------------------------------------------
|
|
# Verifying that a latest file cannot be opened with earlier bounds
|
|
VERIFY_INVALIDBOUNDS()
|
|
{
|
|
lowbound=$1
|
|
highbound=$2
|
|
infile=$3
|
|
outfile=out-verbounds-low$lowbound-high$highbound.$infile
|
|
|
|
#--------------------------------------
|
|
# check for failure with invalid bounds
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN -j $lowbound -k $highbound $infile $outfile
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo " PASSED"
|
|
else
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
fi
|
|
|
|
# clean up tmp files
|
|
rm -f $outfile
|
|
|
|
} # end of VERIFY_INVALIDBOUNDS
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Expect h5diff to fail
|
|
# Use only by VERIFY_EXTERNAL_CONSOLIDATION
|
|
# -----------------------------------------------------------------------------
|
|
DIFFFAIL()
|
|
{
|
|
VERIFY h5diff unequal $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5DIFF_BIN -q "$@"
|
|
)
|
|
RET=$?
|
|
if [ $RET -eq 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Catchall test for repacking with external files
|
|
# Loops over all (internally-listed) cases and applies the given arguments
|
|
# to h5repack.
|
|
# Each test file family must be listed in the for loop at the start of the
|
|
# 'function'.
|
|
# Expects case files to be of format:
|
|
# + h5repack_<NAME>.h5
|
|
# + h5repack_<NAME>_ex.h5
|
|
# + h5repack_<NAME>_ex-<N>.dat
|
|
# ...where NAME is part of the internal list, and N is a positive decimal
|
|
# number; multiple external files (*.dat) are allowed per file, but they must
|
|
# follow the pattern and be in contiguous numerical sequence starting at 0.
|
|
# -----------------------------------------------------------------------------
|
|
VERIFY_EXTERNAL_CONSOLIDATION()
|
|
{
|
|
for name in f32le int32le_1d int32le_2d int32le_3d uint8be
|
|
do
|
|
basename="h5repack_${name}"
|
|
base_ext="${basename}_ex"
|
|
base_rpkg="${basename}_rp"
|
|
canonical="${basename}.h5"
|
|
external="${base_ext}.h5"
|
|
ex_dat="${base_ext}-0.dat"
|
|
ex_dat_mv="__${ext_dat}"
|
|
repacked="${base_rpkg}.h5"
|
|
prev_nerrors=$nerrors
|
|
|
|
TESTING external consolidation on $name with h5repack $@
|
|
echo " . . ."
|
|
|
|
# baseline comparison of known files
|
|
DIFFTEST $canonical $external
|
|
|
|
# repack the external file, verify integrity
|
|
( cd $TESTDIR
|
|
$H5REPACK_BIN $@ $external $repacked
|
|
)
|
|
DIFFTEST $repacked $canonical
|
|
DIFFTEST $repacked $external
|
|
|
|
# invalidate external file by moving its first data file; verify
|
|
( cd $TESTDIR
|
|
mv $ex_dat $ex_dat_mv
|
|
)
|
|
DIFFFAIL $canonical $external
|
|
DIFFFAIL $repacked $external
|
|
DIFFTEST $canonical $repacked
|
|
|
|
# reset test state for re-runs with different h5repack arguments
|
|
( cd $TESTDIR
|
|
mv $ex_dat_mv $ex_dat
|
|
rm $repacked
|
|
)
|
|
|
|
# flag if any of the above checks failed
|
|
if [ $prev_nerrors != $nerrors ]
|
|
then
|
|
echo ". . . *FAILED*"
|
|
nerrors="`expr $prev_nerrors + 1`"
|
|
else
|
|
echo ". . . PASSED"
|
|
fi
|
|
done
|
|
} # end VERIFY_EXTERNAL_CONSOLIDATION
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# same as TOOLTEST, but it uses the old syntax -i input_file -o output_file
|
|
# -----------------------------------------------------------------------------
|
|
TOOLTEST0()
|
|
{
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@"
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
DIFFTEST $infile $outfile
|
|
fi
|
|
rm -f $outfile
|
|
}
|
|
|
|
|
|
# same as TOOLTEST, but it uses without -i -o options
|
|
# used to test the family driver, where these files reside
|
|
#
|
|
TOOLTEST1()
|
|
{
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
DIFFTEST $infile $outfile
|
|
fi
|
|
rm -f $outfile
|
|
}
|
|
|
|
# This is same as TOOLTEST() with comparing display output
|
|
# from -v option
|
|
#
|
|
TOOLTESTV()
|
|
{
|
|
expect="$TESTDIR/$2-$1.tst"
|
|
actual="$TESTDIR/`basename $2 .ddl`.out"
|
|
actual_err="$TESTDIR/`basename $2 .ddl`.err"
|
|
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
) >$actual 2>$actual_err
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
DIFFTEST $infile $outfile
|
|
fi
|
|
|
|
# display output compare
|
|
STDOUT_FILTER $actual
|
|
cat $actual_err >> $actual
|
|
|
|
VERIFY output from $H5REPACK $@
|
|
if cmp -s $expect $actual; then
|
|
echo " PASSED"
|
|
else
|
|
echo "*FAILED*"
|
|
echo " Expected result (*.tst) differs from actual result (*.out)"
|
|
nerrors="`expr $nerrors + 1`"
|
|
test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /'
|
|
fi
|
|
|
|
rm -f $actual $actual_err
|
|
rm -f $outfile
|
|
}
|
|
|
|
# Same as TOOLTEST, but expects h5repack fails
|
|
#
|
|
TOOLTEST_FAIL()
|
|
{
|
|
infile=$1
|
|
outfile=$2
|
|
expect="$TESTDIR/$2-$1.ddl"
|
|
actual="$TESTDIR/$2-$1.out"
|
|
actual_err="$TESTDIR/$2-$1.err"
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
) >&$actual
|
|
RET=$?
|
|
|
|
# Normally h5repack of files tested with this function are expected
|
|
# to return not 0, but if the command results in "Segmentation fault"
|
|
# or "core dumped" it is a failure regardless of the return value.
|
|
failure=`grep -e 'Segmentation fault' -e 'core dumped' $actual`
|
|
if [ "$failure" != "" ]; then
|
|
nerrors="`expr $nerrors + 1`"
|
|
echo " FAILED"
|
|
echo " $failure"
|
|
else
|
|
if [ $RET == 0 ] ; then
|
|
nerrors="`expr $nerrors + 1`"
|
|
echo " FAILED"
|
|
else
|
|
echo " PASSED"
|
|
fi
|
|
fi
|
|
rm -f $outfile
|
|
}
|
|
|
|
# This is same as TOOLTESTV() with comparing display output
|
|
# with actual filename swapped
|
|
#
|
|
TOOLTESTSV()
|
|
{
|
|
expect="$TESTDIR/$1.$2.tst"
|
|
actual="$TESTDIR/`basename $2 .ddl`.out"
|
|
actual_err="$TESTDIR/`basename $2 .ddl`.err"
|
|
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
) >$actual 2>$actual_err
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
VERIFY h5diff equal $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5DIFF_BIN -v $infile $outfile
|
|
) >$actual 2>$actual_err
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
fi
|
|
fi
|
|
|
|
# display output compare
|
|
STDOUT_FILTER $actual
|
|
cat $actual_err >> $actual
|
|
|
|
VERIFY output from $H5REPACK $@
|
|
if cmp -s $expect $actual; then
|
|
echo " PASSED"
|
|
else
|
|
echo "*FAILED*"
|
|
echo " Expected result (*.tst) differs from actual result (*.out)"
|
|
nerrors="`expr $nerrors + 1`"
|
|
test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /'
|
|
fi
|
|
|
|
rm -f $actual $actual_err
|
|
rm -f $outfile
|
|
}
|
|
|
|
# This is same as TOOLTESTSV() but expects a diff fail
|
|
#
|
|
TOOLFAILSV()
|
|
{
|
|
expect="$TESTDIR/$1.$2.tst"
|
|
actual="$TESTDIR/`basename $2 .ddl`.out"
|
|
actual_err="$TESTDIR/`basename $2 .ddl`.err"
|
|
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
) >$actual 2>$actual_err
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
VERIFY h5diff unequal $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5DIFF_BIN -v $infile $outfile
|
|
) >$actual 2>$actual_err
|
|
RET=$?
|
|
if [ $RET == 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
fi
|
|
fi
|
|
|
|
# display output compare
|
|
STDOUT_FILTER $actual
|
|
cat $actual_err >> $actual
|
|
|
|
VERIFY output from $H5REPACK $@
|
|
if cmp -s $expect $actual; then
|
|
echo " PASSED"
|
|
else
|
|
echo "*FAILED*"
|
|
echo " Expected result (*.tst) differs from actual result (*.out)"
|
|
nerrors="`expr $nerrors + 1`"
|
|
test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /'
|
|
fi
|
|
|
|
rm -f $actual $actual_err
|
|
rm -f $outfile
|
|
}
|
|
|
|
# same as TOOLTESTV but filters error stack output and compares to an error file
|
|
# Extract file name, line number, version and thread IDs because they may be different
|
|
# ADD_H5ERR_MASK_TEST
|
|
TOOLTESTM() {
|
|
|
|
expect="$TESTDIR/$2-$1.tst"
|
|
actual="$TESTDIR/`basename $2 .tst`.out"
|
|
actual_err="$TESTDIR/`basename $2 .tst`.oerr"
|
|
actual_sav=${actual}-sav
|
|
actual_err_sav=${actual_err}-sav
|
|
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
) >$actual 2>$actual_err
|
|
|
|
# save actual and actual_err in case they are needed later.
|
|
cp $actual $actual_sav
|
|
cp $actual_err $actual_err_sav
|
|
|
|
# Extract file name, line number, version and thread IDs because they may be different
|
|
sed -e 's/thread [0-9]*/thread (IDs)/' -e 's/: .*\.c /: (file name) /' \
|
|
-e 's/line [0-9]*/line (number)/' \
|
|
-e 's/v[1-9]*\.[0-9]*\./version (number)\./' \
|
|
-e 's/[1-9]*\.[0-9]*\.[0-9]*[^)]*/version (number)/' \
|
|
-e 's/H5Eget_auto[1-2]*/H5Eget_auto(1 or 2)/' \
|
|
-e 's/H5Eset_auto[1-2]*/H5Eset_auto(1 or 2)/' \
|
|
$actual_err > $actual
|
|
|
|
if [ ! -f $expect ]; then
|
|
# Create the expect file if it doesn't yet exist.
|
|
echo " CREATED"
|
|
cp $actual $expect
|
|
echo " Expected result (*.tst) missing"
|
|
nerrors="`expr $nerrors + 1`"
|
|
elif $CMP $expect $actual; then
|
|
echo " PASSED"
|
|
else
|
|
echo "*FAILED*"
|
|
echo " Expected result (*.tst) differs from actual result (*.out)"
|
|
nerrors="`expr $nerrors + 1`"
|
|
test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /'
|
|
fi
|
|
|
|
# Clean up output file
|
|
if test -z "$HDF5_NOCLEANUP"; then
|
|
rm -f $actual $actual_err $actual_sav $actual_err_sav
|
|
fi
|
|
|
|
}
|
|
|
|
# This is same as TOOLTESTV() with comparing h5dump output
|
|
# from -pH option
|
|
#
|
|
TOOLTEST_DUMP()
|
|
{
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
expect="$TESTDIR/$1.$2.ddl"
|
|
actual="$TESTDIR/out-$1.$2.out"
|
|
actual_err="$TESTDIR/out-$1.$2.err"
|
|
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
) >$actual 2>$actual_err
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
VERIFY h5dump output $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5DUMP_BIN -q creation_order -pH $outfile
|
|
) >$actual 2>$actual_err
|
|
cat $actual_err >> $actual
|
|
|
|
RET=$?
|
|
|
|
fi
|
|
|
|
if cmp -s $expect $actual; then
|
|
echo " PASSED"
|
|
else
|
|
echo "*FAILED*"
|
|
echo " Expected result (*.ddl) differs from actual result (*.out)"
|
|
nerrors="`expr $nerrors + 1`"
|
|
test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /'
|
|
fi
|
|
|
|
rm -f $actual $actual_err
|
|
rm -f $outfile
|
|
}
|
|
|
|
# This is similar to TOOLTEST_DUMP().
|
|
# Test h5repack with options added for paged aggregation.
|
|
# h5stat is used on the repacked file and the expected output
|
|
# is compared for correctness.
|
|
#
|
|
TOOLTEST_STAT()
|
|
{
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
expect="$TESTDIR/$1.$2.ddl"
|
|
actual="$TESTDIR/out-$1.$2.out"
|
|
actual_err="$TESTDIR/out-$1.$2.err"
|
|
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
) >$actual 2>$actual_err
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
VERIFY h5stat output $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5STAT_BIN -S -s $outfile
|
|
) >$actual 2>$actual_err
|
|
cat $actual_err >> $actual
|
|
|
|
RET=$?
|
|
|
|
fi
|
|
|
|
if cmp -s $expect $actual; then
|
|
echo " PASSED"
|
|
else
|
|
echo "*FAILED*"
|
|
echo " Expected result (*.ddl) differs from actual result (*.out)"
|
|
nerrors="`expr $nerrors + 1`"
|
|
test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /'
|
|
fi
|
|
|
|
rm -f $actual $actual_err
|
|
rm -f $outfile
|
|
}
|
|
|
|
# TOOLTEST_META:
|
|
# Test metadata block size option.
|
|
# Reason to create a function here is to localize all special steps related to
|
|
# metadata block size option in one place. This is a quick solution. More
|
|
# thought out solution needed when there is more time.
|
|
#
|
|
# $1: test input file
|
|
# $2:$: metadata options (either "-M size" or "--metadata_block_size=size")
|
|
#
|
|
# Algorithm:
|
|
# Run it once without the metadata option ($2-$);
|
|
# Save the result output file;
|
|
# Run it second time with the metadata option;
|
|
# Verify the output file of second run is larger than the one of 1st run.
|
|
TOOLTEST_META()
|
|
{
|
|
infile=$2
|
|
outfile=out-$1.$2
|
|
shift
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN $infile $outfile
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
DIFFTEST $infile $outfile
|
|
fi
|
|
# get the size of the first output file
|
|
size1=`wc -c $TESTDIR/$outfile | cut -d' ' -f1`
|
|
|
|
# 2nd run with metadata option
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
|
|
)
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
DIFFTEST $infile $outfile
|
|
fi
|
|
# get the size of the second output file
|
|
size2=`wc -c $TESTDIR/$outfile | cut -d' ' -f1`
|
|
|
|
# verify sizes.
|
|
MESSAGE "Verify the sizes of both output files ($size1 vs $size2)"
|
|
if [ $size1 -lt $size2 ]; then
|
|
# pass
|
|
echo " PASSED"
|
|
else
|
|
#fail
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
fi
|
|
|
|
rm -f $outfile
|
|
}
|
|
|
|
# ADD_HELP_TEST
|
|
TOOLTEST_HELP() {
|
|
|
|
expect="$TESTDIR/$1"
|
|
actual="$TESTDIR/`basename $1 .txt`.out"
|
|
actual_err="$TESTDIR/`basename $1 .txt`.err"
|
|
shift
|
|
|
|
# Run test.
|
|
TESTING $H5REPACK $@
|
|
(
|
|
cd $TESTDIR
|
|
$RUNSERIAL $H5REPACK_BIN "$@"
|
|
) >$actual 2>$actual_err
|
|
cat $actual_err >> $actual
|
|
|
|
if [ ! -f $expect ]; then
|
|
# Create the expect data file if it doesn't yet exist.
|
|
echo " CREATED"
|
|
cp $actual $expect-CREATED
|
|
echo " Expected result (*.txt) missing"
|
|
nerrors="`expr $nerrors + 1`"
|
|
elif cmp -s $expect $actual; then
|
|
echo " PASSED"
|
|
else
|
|
echo "*FAILED*"
|
|
echo " Expected output (*.txt) differs from actual output (*.out)"
|
|
nerrors="`expr $nerrors + 1`"
|
|
fi
|
|
|
|
# Clean up output file
|
|
if test -z "$HDF5_NOCLEANUP"; then
|
|
rm -f $actual $actual_err
|
|
fi
|
|
|
|
}
|
|
|
|
# This is different from $srcdir/../../../bin/output_filter.sh
|
|
STDOUT_FILTER() {
|
|
result_file=$1
|
|
tmp_file=/tmp/h5test_tmp_$$
|
|
# Filter name of files.
|
|
cp $result_file $tmp_file
|
|
sed -e '/^Opening file/d' -e '/^Making file/d' \
|
|
< $tmp_file > $result_file
|
|
# cleanup
|
|
rm -f $tmp_file
|
|
}
|
|
|
|
#
|
|
# The tests
|
|
# We use the files generated by h5repacktst
|
|
# Each run generates "<file>.out.h5" and the tool h5diff is used to
|
|
# compare the input and output files
|
|
#
|
|
# the tests are the same as the program h5repacktst, but run from the CLI
|
|
#
|
|
|
|
# See which filters are usable (and skip tests for filters we
|
|
# don't have). Do this by searching H5pubconf.h to see which
|
|
# filters are defined.
|
|
|
|
# detect whether the encoder is present.
|
|
USE_FILTER_SZIP_ENCODER="no";
|
|
if test $USE_FILTER_SZIP = "yes"; then
|
|
USE_FILTER_SZIP_ENCODER=`$RUNSERIAL $H5DETECTSZIP_BIN`
|
|
fi
|
|
|
|
##############################################################################
|
|
### T H E T E S T S
|
|
##############################################################################
|
|
|
|
# prepare for test
|
|
COPY_TESTFILES_TO_TESTDIR
|
|
|
|
TOOLTEST_HELP h5repack-help.txt -h
|
|
|
|
# copy files (these files have no filters)
|
|
TOOLTEST fill h5repack_fill.h5
|
|
TOOLTEST objs h5repack_objs.h5
|
|
TOOLTEST attr h5repack_attr.h5
|
|
TOOLTEST hlink h5repack_hlink.h5
|
|
TOOLTEST layout h5repack_layout.h5
|
|
TOOLTEST early h5repack_early.h5
|
|
|
|
# nested 8bit enum in both deflated and non-deflated datafiles
|
|
if [ $USE_FILTER_DEFLATE != "yes" ]; then
|
|
TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum.h5
|
|
else
|
|
TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum_deflated.h5
|
|
fi
|
|
|
|
# use h5repack_layout.h5 to write some filters (this file has no filters)
|
|
|
|
# gzip with individual object
|
|
arg="h5repack_layout.h5 -f dset1:GZIP=1 -l dset1:CHUNK=20x10"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST gzip_individual $arg
|
|
fi
|
|
|
|
# gzip for all
|
|
arg="h5repack_layout.h5 -f GZIP=1"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST gzip_all $arg
|
|
fi
|
|
|
|
# szip with individual object
|
|
arg="h5repack_layout.h5 -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST szip_individual $arg
|
|
fi
|
|
|
|
# szip for all
|
|
arg="h5repack_layout.h5 -f SZIP=8,NN"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST szip_all $arg
|
|
fi
|
|
|
|
# shuffle with individual object
|
|
arg="h5repack_layout.h5 -f dset2:SHUF -l dset2:CHUNK=20x10"
|
|
TOOLTEST shuffle_individual $arg
|
|
|
|
|
|
# shuffle for all
|
|
arg="h5repack_layout.h5 -f SHUF"
|
|
TOOLTEST shuffle_all $arg
|
|
|
|
# fletcher32 with individual object
|
|
arg="h5repack_layout.h5 -f dset2:FLET -l dset2:CHUNK=20x10"
|
|
TOOLTEST fletcher_individual $arg
|
|
|
|
# fletcher32 for all
|
|
arg="h5repack_layout.h5 -f FLET"
|
|
TOOLTEST fletcher_all $arg
|
|
|
|
# all filters
|
|
arg="h5repack_layout.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST all_filters $arg
|
|
fi
|
|
|
|
# verbose gzip with individual object
|
|
arg="h5repack_filters.h5 -v -f /dset_deflate:GZIP=9"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
# compare output
|
|
TOOLTESTV gzip_verbose_filters $arg
|
|
fi
|
|
|
|
###########################################################
|
|
# the following tests assume the input files have filters
|
|
###########################################################
|
|
|
|
# szip copy
|
|
arg="h5repack_szip.h5"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST szip_copy $arg
|
|
fi
|
|
|
|
# szip remove
|
|
arg="h5repack_szip.h5 --filter=dset_szip:NONE"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST szip_remove $arg
|
|
fi
|
|
|
|
# deflate copy
|
|
arg="h5repack_deflate.h5"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST deflate_copy $arg
|
|
fi
|
|
|
|
# deflate remove
|
|
arg="h5repack_deflate.h5 -f dset_deflate:NONE"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST deflate_remove $arg
|
|
fi
|
|
|
|
# shuffle copy
|
|
arg="h5repack_shuffle.h5"
|
|
TOOLTEST shuffle_copy $arg
|
|
|
|
# shuffle remove
|
|
arg="h5repack_shuffle.h5 -f dset_shuffle:NONE"
|
|
TOOLTEST shuffle_remove $arg
|
|
|
|
# fletcher32 copy
|
|
arg="h5repack_fletcher.h5"
|
|
TOOLTEST fletcher_copy $arg
|
|
|
|
# fletcher32 remove
|
|
arg="h5repack_fletcher.h5 -f dset_fletcher32:NONE"
|
|
TOOLTEST fletcher_remove $arg
|
|
|
|
# nbit copy
|
|
arg="h5repack_nbit.h5"
|
|
TOOLTEST nbit_copy $arg
|
|
|
|
# nbit remove
|
|
arg="h5repack_nbit.h5 -f dset_nbit:NONE"
|
|
TOOLTEST nbit_remove $arg
|
|
|
|
# nbit add
|
|
arg="h5repack_nbit.h5 -f dset_int31:NBIT"
|
|
TOOLTEST nbit_add $arg
|
|
|
|
# scaleoffset copy
|
|
arg="h5repack_soffset.h5"
|
|
TOOLTEST scale_copy $arg
|
|
|
|
# scaleoffset add
|
|
arg="h5repack_soffset.h5 -f dset_none:SOFF=31,IN"
|
|
TOOLTEST scale_add $arg
|
|
|
|
# scaleoffset remove
|
|
arg="h5repack_soffset.h5 -f dset_scaleoffset:NONE"
|
|
TOOLTEST scale_remove $arg
|
|
|
|
# remove all filters
|
|
arg="h5repack_filters.h5 -f NONE"
|
|
if test $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST remove_all $arg
|
|
fi
|
|
|
|
#filter conversions
|
|
|
|
arg="h5repack_deflate.h5 -f dset_deflate:SZIP=8,NN"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST deflate_convert $arg
|
|
fi
|
|
|
|
arg="h5repack_szip.h5 -f dset_szip:GZIP=1"
|
|
if test $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST szip_convert $arg
|
|
fi
|
|
|
|
|
|
#limit
|
|
arg="h5repack_layout.h5 -f GZIP=1 -m 1024"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST_DUMP deflate_limit $arg
|
|
fi
|
|
|
|
#file
|
|
arg="h5repack_layout.h5 -e h5repack.info"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST deflate_file $arg
|
|
fi
|
|
|
|
#crtorder
|
|
arg="tordergr.h5 -L"
|
|
TOOLTEST_DUMP crtorder $arg
|
|
|
|
###################################################################################################
|
|
# Testing paged aggregation related options:
|
|
# -G pagesize
|
|
# -P 1 or 0
|
|
# -S strategy
|
|
# -T threshold
|
|
#
|
|
# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_":
|
|
# (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */
|
|
# (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */
|
|
# (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */
|
|
# (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */
|
|
# (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */
|
|
# (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */
|
|
#
|
|
#####################################################################################################
|
|
#
|
|
arg="h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1"
|
|
TOOLTEST_STAT SP $arg
|
|
#
|
|
arg="h5repack_fsm_aggr_persist.h5 -S AGGR"
|
|
TOOLTEST_STAT S $arg
|
|
#
|
|
arg="h5repack_none.h5 -S PAGE -T 10 -G 2048"
|
|
TOOLTEST_STAT STG $arg
|
|
#
|
|
arg="h5repack_paged_nopersist.h5 -G 512 -S AGGR"
|
|
TOOLTEST_STAT GS $arg
|
|
#
|
|
arg="h5repack_paged_persist.h5 -S NONE -P 1"
|
|
TOOLTEST_STAT SP $arg
|
|
#
|
|
arg="h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5"
|
|
TOOLTEST_STAT SPT $arg
|
|
|
|
|
|
#########################################################
|
|
# layout options (these files have no filters)
|
|
#########################################################
|
|
VERIFY_LAYOUT_DSET dset2_chunk_20x10 h5repack_layout.h5 dset2 CHUNKED --layout dset2:CHUNK=20x10
|
|
VERIFY_LAYOUT_ALL chunk_20x10 h5repack_layout.h5 CHUNKED -l CHUNK=20x10
|
|
VERIFY_LAYOUT_DSET dset2_conti h5repack_layout.h5 dset2 CONTIGUOUS -l dset2:CONTI
|
|
VERIFY_LAYOUT_ALL conti h5repack_layout.h5 CONTIGUOUS -l CONTI
|
|
VERIFY_LAYOUT_DSET dset2_compa h5repack_layout.h5 dset2 COMPACT -l dset2:COMPA
|
|
VERIFY_LAYOUT_ALL compa h5repack_layout.h5 COMPACT -l COMPA
|
|
TOOLTESTM dset2_chunk_20x10-errstk h5repack_layout.h5 --layout=dset2:CHUNK=20x10x5 --enable-error-stack
|
|
|
|
################################################################
|
|
# layout conversions (file has no filters)
|
|
###############################################################
|
|
VERIFY_LAYOUT_DSET dset_compa_conti h5repack_layout.h5 dset_compact CONTIGUOUS -l dset_compact:CONTI
|
|
VERIFY_LAYOUT_DSET dset_compa_chunk h5repack_layout.h5 dset_compact CHUNKED -l dset_compact:CHUNK=2x5
|
|
VERIFY_LAYOUT_DSET dset_compa_compa h5repack_layout.h5 dset_compact COMPACT -l dset_compact:COMPA
|
|
VERIFY_LAYOUT_DSET dset_conti_compa h5repack_layout.h5 dset_contiguous COMPACT -l dset_contiguous:COMPA
|
|
VERIFY_LAYOUT_DSET dset_conti_chunk h5repack_layout.h5 dset_contiguous CHUNKED -l dset_contiguous:CHUNK=3x6
|
|
VERIFY_LAYOUT_DSET dset_conti_conti h5repack_layout.h5 dset_contiguous CONTIGUOUS -l dset_contiguous:CONTI
|
|
VERIFY_LAYOUT_DSET chunk_compa h5repack_layout.h5 dset_chunk COMPACT -l dset_chunk:COMPA
|
|
VERIFY_LAYOUT_DSET chunk_conti h5repack_layout.h5 dset_chunk CONTIGUOUS -l dset_chunk:CONTI
|
|
VERIFY_LAYOUT_DSET chunk_18x13 h5repack_layout.h5 dset_chunk CHUNKED -l dset_chunk:CHUNK=18x13
|
|
|
|
# test convert small size dataset ( < 1k) to compact layout without -m
|
|
VERIFY_LAYOUT_DSET contig_small_compa h5repack_layout2.h5 contig_small COMPACT -l contig_small:COMPA
|
|
VERIFY_LAYOUT_DSET contig_small_fixed_compa h5repack_layout2.h5 chunked_small_fixed COMPACT -l chunked_small_fixed:COMPA
|
|
|
|
#---------------------------------------------------------------------------
|
|
# Test file contains chunked datasets (need multiple dsets) with
|
|
# unlimited max dims. (HDFFV-7933)
|
|
# Use first dset to test.
|
|
#---------------------------------------------------------------------------
|
|
# chunk to chunk - specify chunk dim bigger than any current dim
|
|
VERIFY_LAYOUT_DSET chunk2chunk h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:CHUNK=100x300
|
|
|
|
# chunk to contiguous
|
|
VERIFY_LAYOUT_DSET chunk2conti h5repack_layout3.h5 chunk_unlimit1 CONTI -l chunk_unlimit1:CONTI
|
|
|
|
# chunk to compact - convert big dataset (should be > 64k) for this purpose,
|
|
# should remain as original layout (chunk)
|
|
VERIFY_LAYOUT_DSET chunk2compa h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:COMPA
|
|
|
|
#--------------------------------------------------------------------------
|
|
# Test -f for some specific cases. Chunked dataset with unlimited max dims.
|
|
# (HDFFV-8012)
|
|
#--------------------------------------------------------------------------
|
|
# - should not fail
|
|
# - should not change max dims from unlimit
|
|
|
|
# chunk dim is bigger than dataset dim. ( dset size < 64k )
|
|
VERIFY_LAYOUT_DSET error1 h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED -f chunk_unlimit1:NONE
|
|
# chunk dim is bigger than dataset dim. ( dset size > 64k )
|
|
|
|
VERIFY_LAYOUT_DSET error2 h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED -f chunk_unlimit2:NONE
|
|
|
|
# chunk dims are smaller than dataset dims. ( dset size < 64k )
|
|
#TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit3:NONE
|
|
VERIFY_LAYOUT_DSET error3 h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED -f chunk_unlimit3:NONE
|
|
|
|
# file input - should not fail
|
|
TOOLTEST error4 h5repack_layout3.h5 -f NONE
|
|
|
|
#--------------------------------------------------------------------------
|
|
# Test base: Convert CHUNK to CONTI for a chunked dataset with small dataset
|
|
# (dset size < 64K) and with unlimited max dims on a condition as follow.
|
|
# (HDFFV-8214)
|
|
#--------------------------------------------------------------------------
|
|
# chunk dim is bigger than dataset dim. should succeed.
|
|
VERIFY_LAYOUT_DSET ckdim_biger h5repack_layout3.h5 chunk_unlimit2 CONTI -l chunk_unlimit2:CONTI
|
|
# chunk dim is smaller than dataset dim. should succeed.
|
|
VERIFY_LAYOUT_DSET ckdim_smaller h5repack_layout3.h5 chunk_unlimit3 CONTI -l chunk_unlimit3:CONTI
|
|
|
|
|
|
# Native option
|
|
# Do not use FILE1, as the named dtype will be converted to native, and h5diff will
|
|
# report a difference.
|
|
TOOLTEST native_fill h5repack_fill.h5 -n
|
|
TOOLTEST native_attr h5repack_attr.h5 -n
|
|
|
|
|
|
# latest file format with long switches. use FILE4=h5repack_layout.h5 (no filters)
|
|
arg="h5repack_layout.h5 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
VERIFY_LAYOUT_ALL layout_long_switches h5repack_layout.h5 CHUNKED --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]
|
|
fi
|
|
|
|
# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters)
|
|
arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
VERIFY_LAYOUT_ALL layout_short_switches h5repack_layout.h5 CHUNKED -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]
|
|
fi
|
|
|
|
# several global filters
|
|
arg="h5repack_layout.h5 --filter GZIP=1 --filter SHUF"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST global_filters $arg
|
|
fi
|
|
|
|
# syntax of -i infile -o outfile
|
|
# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters)
|
|
arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST0 old_style_layout_short_switches $arg
|
|
fi
|
|
|
|
# add a userblock to file
|
|
arg="h5repack_objs.h5 -u ublock.bin -b 2048"
|
|
TOOLTEST add_userblock $arg
|
|
|
|
# add alignment
|
|
arg="h5repack_objs.h5 -t 1 -a 1 "
|
|
TOOLTEST add_alignment $arg
|
|
|
|
# Check repacking file with old version of layout message (should get upgraded
|
|
# to new version and be readable, etc.)
|
|
TOOLTEST upgrade_layout h5repack_layouto.h5
|
|
|
|
# test for datum size > H5TOOLS_MALLOCSIZE
|
|
arg="h5repack_objs.h5 -f GZIP=1"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST gt_mallocsize $arg
|
|
fi
|
|
|
|
# Check repacking file with committed datatypes in odd configurations
|
|
TOOLTEST committed_dt h5repack_named_dtypes.h5
|
|
|
|
# tests family driver (file is located in common testfiles folder, uses TOOLTEST1
|
|
TOOLTEST1 family tfamily%05d.h5
|
|
|
|
# test various references (bug 1814 and 1726)
|
|
TOOLTEST bug1814 h5repack_refs.h5
|
|
|
|
# test attribute with various references (bug1797 / HDFFV-5932)
|
|
# the references in attribute of compund or vlen datatype
|
|
TOOLTEST HDFFV-5932 h5repack_attr_refs.h5
|
|
|
|
# Add test for memory leak in attirbute. This test is verified by CTEST.
|
|
# 1. leak from vlen string
|
|
# 2. leak from compound type without reference member
|
|
# (HDFFV-7840, )
|
|
# Note: this test is experimental for sharing test file among tools
|
|
TOOLTEST HDFFV-7840 h5diff_attr1.h5
|
|
|
|
# test HDFFV-10590
|
|
arg="h5repack_CVE-2018-17432.h5 h5repack_CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6"
|
|
TOOLTEST_FAIL $arg
|
|
|
|
# test HDFFV-11223
|
|
arg="h5repack_CVE-2018-14460.h5 h5repack_CVE-2018-14460_out.h5"
|
|
TOOLTEST_FAIL $arg
|
|
|
|
# tests for metadata block size option
|
|
TOOLTEST_META meta_short h5repack_layout.h5 -M 8192
|
|
TOOLTEST_META meta_long h5repack_layout.h5 --metadata_block_size=8192
|
|
|
|
# VDS tests
|
|
|
|
################################################################
|
|
# layout conversions
|
|
###############################################################
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP vds_dset_chunk_20x10x5
|
|
else
|
|
VERIFY_LAYOUT_VDS vds_dset_chunk20x10x5 1_vds.h5 vds_dset CHUNKED --layout vds_dset:CHUNK=20x10x5
|
|
fi
|
|
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP vds_chunk2x5x8
|
|
else
|
|
VERIFY_LAYOUT_VDS vds_chunk2x5x8 3_1_vds.h5 vds_dset CHUNKED -l vds_dset:CHUNK=2x5x8
|
|
fi
|
|
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP vds_chunk3x6x9
|
|
else
|
|
VERIFY_LAYOUT_VDS vds_chunk3x6x9 2_vds.h5 vds_dset CHUNKED -l vds_dset:CHUNK=3x6x9
|
|
fi
|
|
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP vds_compa 4_vds.h5
|
|
else
|
|
VERIFY_LAYOUT_VDS vds_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA
|
|
fi
|
|
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP vds_conti 4_vds.h5
|
|
else
|
|
VERIFY_LAYOUT_VDS vds_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI
|
|
fi
|
|
|
|
################################################################
|
|
# reference new api conversions
|
|
###############################################################
|
|
#TOOLTEST_DUMP attrregion tattrreg.h5
|
|
#TOOLTEST_DUMP dataregion tdatareg.h5
|
|
|
|
##############################################################################
|
|
### V E R S I O N B O U N D S T E S T S
|
|
##############################################################################
|
|
# -j 0 -k 2, superblock will be 0
|
|
VERIFY_SUPERBLOCK 0 2 0 h5repack_layout.h5 -j 0 -k 2 h5repack_layout.h5
|
|
# -j 1 -k 2, superblock will be 2
|
|
VERIFY_SUPERBLOCK 1 2 2 h5repack_layout.h5 -j 1 -k 2 h5repack_layout.h5
|
|
# -j 2 -k 2, superblock will be 3
|
|
VERIFY_SUPERBLOCK 2 2 3 h5repack_layout.h5 -j 2 -k 2 h5repack_layout.h5
|
|
# -j 0 -k 1, file cannot be opened
|
|
VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5
|
|
|
|
##############################################################################
|
|
### E X T E R N A L S T O R A G E T E S T S
|
|
##############################################################################
|
|
VERIFY_EXTERNAL_CONSOLIDATION -l CONTI
|
|
|
|
##############################################################################
|
|
### E X T E R N A L L I N K T E S T S
|
|
##############################################################################
|
|
### HDFFV-11128 needs fixed to enable the following test
|
|
#TOOLTEST_DUMP h5copy_extlinks_src-base h5copy_extlinks_src.h5 --enable-error-stack
|
|
TOOLTEST_DUMP tsoftlinks-base tsoftlinks.h5 --enable-error-stack
|
|
TOOLTEST_DUMP textlink-base textlink.h5 --enable-error-stack
|
|
TOOLTEST_DUMP textlinkfar-base textlinkfar.h5 --enable-error-stack
|
|
TOOLTEST_DUMP textlinksrc-base textlinksrc.h5 --enable-error-stack
|
|
TOOLTEST_DUMP textlinktar-base textlinktar.h5 --enable-error-stack
|
|
|
|
TOOLTESTSV h5copy_extlinks_src-merge h5copy_extlinks_src.h5 --merge
|
|
TOOLFAILSV tsoftlinks-merge tsoftlinks.h5 --merge
|
|
TOOLTESTSV textlink-merge textlink.h5 --merge
|
|
### HDFFV-11128 needs fixed to enable the following test
|
|
#TOOLFAILSV textlinkfar-merge textlinkfar.h5 --merge
|
|
### HDFFV-11128 needs fixed to enable the following test
|
|
#TOOLFAILSV textlinksrc-merge textlinksrc.h5 --merge
|
|
### HDFFV-11128 needs fixed to enable the following test
|
|
#TOOLFAILSV textlinktar-merge textlinktar.h5 --merge
|
|
|
|
TOOLTEST_DUMP h5copy_extlinks_src-prune h5copy_extlinks_src.h5 --prune --enable-error-stack
|
|
TOOLTEST_DUMP tsoftlinks-prune tsoftlinks.h5 --prune --enable-error-stack
|
|
TOOLTEST_DUMP textlink-prune textlink.h5 --prune --enable-error-stack
|
|
TOOLTEST_DUMP textlinkfar-prune textlinkfar.h5 --prune --enable-error-stack
|
|
TOOLTEST_DUMP textlinksrc-prune textlinksrc.h5 --prune --enable-error-stack
|
|
TOOLTEST_DUMP textlinktar-prune textlinktar.h5 --prune --enable-error-stack
|
|
|
|
TOOLTEST_DUMP h5copy_extlinks_src-mergeprune h5copy_extlinks_src.h5 --merge --prune --enable-error-stack
|
|
TOOLTEST_DUMP tsoftlinks-mergeprune tsoftlinks.h5 --merge --prune --enable-error-stack
|
|
TOOLTEST_DUMP textlink-mergeprune textlink.h5 --merge --prune --enable-error-stack
|
|
### HDFFV-11128 needs fixed to enable the following test
|
|
#TOOLTEST_DUMP textlinkfar-mergeprune textlinkfar.h5 --merge --prune --enable-error-stack
|
|
### HDFFV-11128 needs fixed to enable the following test
|
|
#TOOLTEST_DUMP textlinksrc-mergeprune textlinksrc.h5 --merge --prune --enable-error-stack
|
|
### HDFFV-11128 needs fixed to enable the following test
|
|
#TOOLTEST_DUMP textlinktar-mergeprune textlinktar.h5 --merge --prune --enable-error-stack
|
|
|
|
# Clean up temporary files/directories
|
|
CLEAN_TESTFILES_AND_TESTDIR
|
|
|
|
if test $nerrors -eq 0 ; then
|
|
echo "All $TESTNAME tests passed."
|
|
exit $EXIT_SUCCESS
|
|
else
|
|
echo "$TESTNAME tests failed with $nerrors errors."
|
|
exit $EXIT_FAILURE
|
|
fi
|
|
|