#! /bin/sh # # Copyright by The HDF Group. # All rights reserved. # # This file is part of HDF5. The full HDF5 copyright notice, including # terms governing use, modification, and redistribution, is contained in # the COPYING file, which can be found at the root of the source code # distribution tree, or in https://www.hdfgroup.org/licenses. # If you do not have access to either file, you may request a copy from # help@hdfgroup.org. # # Tests for the h5repack tool # srcdir=@srcdir@ USE_FILTER_SZIP="@USE_FILTER_SZIP@" USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@" TESTNAME=h5repack EXIT_SUCCESS=0 EXIT_FAILURE=1 H5REPACK=../../src/h5repack/h5repack # The tool name H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary H5DIFF=../../src/h5diff/h5diff # The h5diff tool name H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary H5DUMP=../../src/h5dump/h5dump # The h5dump tool name H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary H5STAT=../../src/h5stat/h5stat # The h5stat tool name H5STAT_BIN=`pwd`/$H5STAT # The path of the h5stat tool binary RM='rm -rf' CMP='cmp' DIFF='diff -c' GREP='grep' CP='cp' DIRNAME='dirname' LS='ls' AWK='awk' # Reset the VOL connector to only use the native connector, because of the # error stack checking. QAK - 2019/03/09 ENVCMD="env HDF5_VOL_CONNECTOR=native" H5DETECTSZIP=testh5repack_detect_szip H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP nerrors=0 verbose=yes # source dirs SRC_TOOLS="$srcdir/../.." # testfiles source dirs for tools SRC_H5LS_TESTFILES="$SRC_TOOLS/test/h5ls/testfiles" SRC_H5DUMP_TESTFILES="$SRC_TOOLS/test/h5dump/testfiles" SRC_H5DIFF_TESTFILES="$SRC_TOOLS/test/h5diff/testfiles" SRC_H5COPY_TESTFILES="$SRC_TOOLS/test/h5copy/testfiles" SRC_H5REPACK_TESTFILES="$SRC_TOOLS/test/h5repack/testfiles" SRC_H5JAM_TESTFILES="$SRC_TOOLS/test/h5jam/testfiles" SRC_H5STAT_TESTFILES="$SRC_TOOLS/test/h5stat/testfiles" SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/test/h5import/testfiles" SRC_H5REPACK_OUTFILES="$SRC_TOOLS/test/h5repack/expected" TESTDIR=./tmp test -d $TESTDIR || mkdir $TESTDIR ###################################################################### # test files # -------------------------------------------------------------------- # All the test files copy from source directory to test directory # NOTE: Keep this framework to add/remove test files. # Any test files from other tools can be used in this framework. # This list are also used for checking exist. # Comment '#' without space can be used. # -------------------------------------------------------------------- LIST_HDF5_TEST_FILES=" $SRC_H5REPACK_TESTFILES/bounds_latest_latest.h5 $SRC_H5REPACK_TESTFILES/h5repack_attr.h5 $SRC_H5REPACK_TESTFILES/h5repack_attr_refs.h5 $SRC_H5REPACK_TESTFILES/h5repack_deflate.h5 $SRC_H5REPACK_TESTFILES/h5repack_early.h5 $SRC_H5REPACK_TESTFILES/h5repack_ext.h5 $SRC_H5REPACK_TESTFILES/h5repack_f32le.h5 $SRC_H5REPACK_TESTFILES/h5repack_f32le_ex.h5 $SRC_H5REPACK_TESTFILES/h5repack_fill.h5 $SRC_H5REPACK_TESTFILES/h5repack_filters.h5 $SRC_H5REPACK_TESTFILES/h5repack_fletcher.h5 $SRC_H5REPACK_TESTFILES/h5repack_hlink.h5 $SRC_H5REPACK_TESTFILES/h5repack_int32le_1d.h5 $SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex.h5 $SRC_H5REPACK_TESTFILES/h5repack_int32le_2d.h5 $SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex.h5 $SRC_H5REPACK_TESTFILES/h5repack_int32le_3d.h5 $SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex.h5 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5 $SRC_H5REPACK_TESTFILES/h5repack_layouto.h5 $SRC_H5REPACK_TESTFILES/h5repack_layout2.h5 $SRC_H5REPACK_TESTFILES/h5repack_layout3.h5 $SRC_H5REPACK_TESTFILES/h5repack_named_dtypes.h5 $SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum.h5 $SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum_deflated.h5 $SRC_H5REPACK_TESTFILES/h5repack_nbit.h5 $SRC_H5REPACK_TESTFILES/h5repack_objs.h5 $SRC_H5REPACK_TESTFILES/h5repack_refs.h5 $SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5 $SRC_H5REPACK_TESTFILES/h5repack_soffset.h5 $SRC_H5REPACK_TESTFILES/h5repack_szip.h5 $SRC_H5REPACK_TESTFILES/h5repack_uint8be.h5 $SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex.h5 ########fsm#files######## $SRC_H5REPACK_TESTFILES/h5repack_aggr.h5 $SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_nopersist.h5 $SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5 $SRC_H5REPACK_TESTFILES/h5repack_none.h5 $SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5 $SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5 ########h5diff/testfile######## $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 ########test#HDFFV-10590######## $SRC_H5REPACK_TESTFILES/h5repack_CVE-2018-17432.h5 $SRC_H5REPACK_TESTFILES/h5repack_CVE-2018-14460.h5 ########tools/testfiles#for#external#links######## $SRC_H5DUMP_TESTFILES/tsoftlinks.h5 $SRC_H5DUMP_TESTFILES/textlinkfar.h5 $SRC_H5DUMP_TESTFILES/textlinksrc.h5 $SRC_H5DUMP_TESTFILES/textlinktar.h5 $SRC_H5DUMP_TESTFILES/textlink.h5 $SRC_H5COPY_TESTFILES/h5copy_extlinks_src.h5 $SRC_H5COPY_TESTFILES/h5copy_extlinks_trg.h5 ########tools/testfiles######## $SRC_H5DUMP_TESTFILES/tfamily00000.h5 $SRC_H5DUMP_TESTFILES/tfamily00001.h5 $SRC_H5DUMP_TESTFILES/tfamily00002.h5 $SRC_H5DUMP_TESTFILES/tfamily00003.h5 $SRC_H5DUMP_TESTFILES/tfamily00004.h5 $SRC_H5DUMP_TESTFILES/tfamily00005.h5 $SRC_H5DUMP_TESTFILES/tfamily00006.h5 $SRC_H5DUMP_TESTFILES/tfamily00007.h5 $SRC_H5DUMP_TESTFILES/tfamily00008.h5 $SRC_H5DUMP_TESTFILES/tfamily00009.h5 $SRC_H5DUMP_TESTFILES/tfamily00010.h5 $SRC_H5DUMP_TESTFILES/tordergr.h5 ########reference#conversion#files######## ########tools/testfiles/vds######## $SRC_H5DUMP_TESTFILES/vds/1_a.h5 $SRC_H5DUMP_TESTFILES/vds/1_b.h5 $SRC_H5DUMP_TESTFILES/vds/1_c.h5 $SRC_H5DUMP_TESTFILES/vds/1_d.h5 $SRC_H5DUMP_TESTFILES/vds/1_e.h5 $SRC_H5DUMP_TESTFILES/vds/1_f.h5 $SRC_H5DUMP_TESTFILES/vds/1_vds.h5 $SRC_H5DUMP_TESTFILES/vds/2_a.h5 $SRC_H5DUMP_TESTFILES/vds/2_b.h5 $SRC_H5DUMP_TESTFILES/vds/2_c.h5 $SRC_H5DUMP_TESTFILES/vds/2_d.h5 $SRC_H5DUMP_TESTFILES/vds/2_e.h5 $SRC_H5DUMP_TESTFILES/vds/2_vds.h5 $SRC_H5DUMP_TESTFILES/vds/3_1_vds.h5 $SRC_H5DUMP_TESTFILES/vds/3_2_vds.h5 $SRC_H5DUMP_TESTFILES/vds/4_0.h5 $SRC_H5DUMP_TESTFILES/vds/4_1.h5 $SRC_H5DUMP_TESTFILES/vds/4_2.h5 $SRC_H5DUMP_TESTFILES/vds/4_vds.h5 $SRC_H5DUMP_TESTFILES/vds/5_a.h5 $SRC_H5DUMP_TESTFILES/vds/5_b.h5 $SRC_H5DUMP_TESTFILES/vds/5_c.h5 $SRC_H5DUMP_TESTFILES/vds/5_vds.h5 ########tools/testfiles######## $SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.h5 $SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.h5.onion $SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.h5 $SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.h5.onion $SRC_H5DUMP_TESTFILES/tst_onion_objs.h5 $SRC_H5DUMP_TESTFILES/tst_onion_objs.h5.onion " LIST_OTHER_TEST_FILES=" $SRC_H5REPACK_OUTFILES/h5repack-help.txt $SRC_H5REPACK_TESTFILES/h5repack_ext.bin $SRC_H5REPACK_TESTFILES/ublock.bin $SRC_H5REPACK_TESTFILES/h5repack.info ########dat#files######## $SRC_H5REPACK_TESTFILES/h5repack_f32le_ex-0.dat $SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-0.dat $SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-1.dat $SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex-0.dat $SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex-0.dat $SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-0.dat $SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-1.dat $SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-2.dat $SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-3.dat ############### $SRC_H5REPACK_OUTFILES/crtorder.tordergr.h5.ddl $SRC_H5REPACK_OUTFILES/deflate_limit.h5repack_layout.h5.ddl $SRC_H5REPACK_OUTFILES/h5repack_layout.h5.ddl $SRC_H5REPACK_OUTFILES/h5repack_layout.h5-plugin_test.ddl ########fsm#files######## $SRC_H5REPACK_OUTFILES/SP.h5repack_fsm_aggr_nopersist.h5.ddl $SRC_H5REPACK_OUTFILES/S.h5repack_fsm_aggr_persist.h5.ddl $SRC_H5REPACK_OUTFILES/STG.h5repack_none.h5.ddl $SRC_H5REPACK_OUTFILES/GS.h5repack_paged_nopersist.h5.ddl $SRC_H5REPACK_OUTFILES/SP.h5repack_paged_persist.h5.ddl $SRC_H5REPACK_OUTFILES/SPT.h5repack_aggr.h5.ddl ########vds#files######## $SRC_H5REPACK_OUTFILES/1_vds.h5-vds_dset_chunk20x10x5-v.ddl $SRC_H5REPACK_OUTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl $SRC_H5REPACK_OUTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl $SRC_H5REPACK_OUTFILES/4_vds.h5-vds_compa-v.ddl $SRC_H5REPACK_OUTFILES/4_vds.h5-vds_conti-v.ddl ########refs#files######## $SRC_H5REPACK_OUTFILES/attrregion.tattrreg.h5.ddl $SRC_H5REPACK_OUTFILES/dataregion.tdatareg.h5.ddl ########external#links#files######## $SRC_H5REPACK_OUTFILES/textlinkfar-base.textlinkfar.h5.ddl $SRC_H5REPACK_OUTFILES/textlinksrc-base.textlinksrc.h5.ddl $SRC_H5REPACK_OUTFILES/textlinktar-base.textlinktar.h5.ddl $SRC_H5REPACK_OUTFILES/textlink-base.textlink.h5.ddl $SRC_H5REPACK_OUTFILES/tsoftlinks-base.tsoftlinks.h5.ddl $SRC_H5REPACK_OUTFILES/h5copy_extlinks_src-base.h5copy_extlinks_src.h5.ddl $SRC_H5REPACK_OUTFILES/textlinkfar-merge.textlinkfar.h5.tst $SRC_H5REPACK_OUTFILES/textlinksrc-merge.textlinksrc.h5.tst $SRC_H5REPACK_OUTFILES/textlinktar-merge.textlinktar.h5.tst $SRC_H5REPACK_OUTFILES/textlink-merge.textlink.h5.tst $SRC_H5REPACK_OUTFILES/tsoftlinks-merge.tsoftlinks.h5.tst $SRC_H5REPACK_OUTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst $SRC_H5REPACK_OUTFILES/textlinkfar-prune.textlinkfar.h5.ddl $SRC_H5REPACK_OUTFILES/textlinksrc-prune.textlinksrc.h5.ddl $SRC_H5REPACK_OUTFILES/textlinktar-prune.textlinktar.h5.ddl $SRC_H5REPACK_OUTFILES/textlink-prune.textlink.h5.ddl $SRC_H5REPACK_OUTFILES/tsoftlinks-prune.tsoftlinks.h5.ddl $SRC_H5REPACK_OUTFILES/h5copy_extlinks_src-prune.h5copy_extlinks_src.h5.ddl $SRC_H5REPACK_OUTFILES/textlinkfar-mergeprune.textlinkfar.h5.ddl $SRC_H5REPACK_OUTFILES/textlinksrc-mergeprune.textlinksrc.h5.ddl $SRC_H5REPACK_OUTFILES/textlinktar-mergeprune.textlinktar.h5.ddl $SRC_H5REPACK_OUTFILES/textlink-mergeprune.textlink.h5.ddl $SRC_H5REPACK_OUTFILES/tsoftlinks-mergeprune.tsoftlinks.h5.ddl $SRC_H5REPACK_OUTFILES/h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5.ddl ########tst#files######## $SRC_H5REPACK_OUTFILES/h5repack_filters.h5-gzip_verbose_filters.tst $SRC_H5REPACK_OUTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst $SRC_H5REPACK_OUTFILES/plugin_test.h5repack_layout.h5.tst ########external#links#tst#files######## $SRC_H5REPACK_OUTFILES/tsoftlinks-merge.tsoftlinks.h5.tst $SRC_H5REPACK_OUTFILES/textlinkfar-merge.textlinkfar.h5.tst $SRC_H5REPACK_OUTFILES/textlinksrc-merge.textlinksrc.h5.tst $SRC_H5REPACK_OUTFILES/textlinktar-merge.textlinktar.h5.tst $SRC_H5REPACK_OUTFILES/textlink-merge.textlink.h5.tst $SRC_H5REPACK_OUTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst ########onion#files######## $SRC_H5REPACK_OUTFILES/onion.tst_onion_dset_1d.h5.ddl $SRC_H5REPACK_OUTFILES/onion.tst_onion_dset_ext.h5.ddl $SRC_H5REPACK_OUTFILES/onion.tst_onion_objs.h5.ddl " # # copy test files and expected output files from source dirs to test dir # COPY_TESTFILES="$LIST_HDF5_TEST_FILES $LIST_OTHER_TEST_FILES" COPY_TESTFILES_TO_TESTDIR() { # copy test files. Used -f to make sure get a new copy for tstfile in $COPY_TESTFILES do # ignore '#' comment echo $tstfile | tr -d ' ' | grep '^#' > /dev/null RET=$? if [ $RET -eq 1 ]; then # skip cp if srcdir is same as destdir # this occurs when build/test performed in source dir and # make cp fail SDIR=`$DIRNAME $tstfile` INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then $CP -f $tstfile $TESTDIR if [ $? -ne 0 ]; then echo "Error: FAILED to copy $tstfile ." # Comment out this to CREATE expected file exit $EXIT_FAILURE fi fi fi done } CLEAN_TESTFILES_AND_TESTDIR() { # skip rm if srcdir is same as destdir # this occurs when build/test performed in source dir and # make cp fail SDIR=$SRC_H5REPACK_TESTFILES INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then $RM $TESTDIR fi } # Print a $* message left justified in a field of 70 characters # MESSAGE() { SPACES=" " echo "$* $SPACES" | cut -c1-70 | tr -d '\012' } # Print a line-line message left justified in a field of 70 characters # beginning with the word "Testing". # TESTING() { MESSAGE "Testing $*" } # Print a line-line message left justified in a field of 70 characters # beginning with the word "Verifying". # VERIFY() { MESSAGE "Verifying $*" } # Print a message that a test has been skipped (because a required filter # was unavailable) SKIP() { TESTING $H5REPACK $@ echo " -SKIP-" } ############################################################################## ############################################################################## ### T H E T E S T S M A C R O S ### ############################################################################## ############################################################################## # Call the h5diff tool # DIFFTEST() { VERIFY h5diff output $@ ( cd $TESTDIR $RUNSERIAL $H5DIFF_BIN -q "$@" ) RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi } # Call h5repack # # call TOOLTEST_MAIN and delete $output file TOOLTEST() { echo $@ infile=$2 outfile=out-$1.$2 shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi rm -f $outfile } #------------------------------------------ # Verifying layouts of a dataset VERIFY_LAYOUT_DSET() { layoutfile=layout-$1.$2 dset=$3 expectlayout=$4 infile=$2 outfile=out-$1.$2 shift shift shift shift TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi #--------------------------------- # check the layout from a dataset VERIFY "a dataset layout" ( cd $TESTDIR $RUNSERIAL $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile ) $GREP $expectlayout $TESTDIR/$layoutfile > /dev/null if [ $? -eq 0 ]; then echo " PASSED" else echo " FAILED" nerrors="`expr $nerrors + 1`" fi # clean up tmp files # rm -f $outfile # rm -f $layoutfile } #------------------------------------------ # Verifying layouts of a dataset VERIFY_LAYOUT_VDS() { layoutfile=layout-$1.$2 dset=$3 expectlayout=$4 infile=$2 outfile=out-$1.$2 expect="$TESTDIR/$2-$1-v.ddl" actual="$TESTDIR/$2-$1-v.out" actual_err="$TESTDIR/$2-$1-v.err" shift shift shift shift TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi #--------------------------------- # check the layout from a dataset VERIFY "a dataset layout" ( cd $TESTDIR $RUNSERIAL $H5DUMP_BIN -d $dset -p $outfile ) >$actual 2>$actual_err cat $actual_err >> $actual if [ ! -f $expect ]; then # Create the expect file if it doesn't yet exist. echo " CREATED" cp $actual $expect echo " Expected result (*.ddl) missing" nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual > /dev/null 2>&1 ; then echo " PASSED" else echo "*FAILED*" echo " Expected result (*.ddl) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" test yes = "$verbose" && $DIFF $caseless $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then rm -f $actual $actual_err fi # clean up tmp files rm -f $outfile } #---------------------------------------- # Verifying layouts from entire file VERIFY_LAYOUT_ALL() { infile=$2 outfile=out-$1.$2 layoutfile=layout-$1.$2 expectlayout=$3 shift shift shift TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi #--------------------------------- # check the layout from a dataset # check if the other layouts still exist VERIFY "layouts" ( cd $TESTDIR echo # if CONTIGUOUS if [ $expectlayout = "CONTIGUOUS" ]; then TESTING $H5DUMP_BIN -pH $outfile ( $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile ) $GREP "COMPACT" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" nerrors="`expr $nerrors + 1`" else $GREP "CHUNKED" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi fi else # if COMPACT if [ $expectlayout = "COMPACT" ]; then TESTING $H5DUMP_BIN -pH $outfile ( $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile ) $GREP "CHUNKED" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" nerrors="`expr $nerrors + 1`" else $GREP "CONTIGUOUS" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi fi else # if CHUNKED if [ $expectlayout = "CHUNKED" ]; then TESTING $H5DUMP_BIN -pH $outfile ( $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile ) $GREP "CONTIGUOUS" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" nerrors="`expr $nerrors + 1`" else $GREP "COMPACT" $layoutfile > /dev/null if [ $? -eq 0 ]; then echo " FAILED" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi fi fi fi fi ) # clean up tmp files rm -f $outfile rm -f $layoutfile } #-------------------------------------------------------------- # Verifying superblock of a file with different version bounds VERIFY_SUPERBLOCK() { lowbound=$1 highbound=$2 superblock=$3 infile=$4 layoutfile=verbounds-low$lowbound-high$highbound.$4 outfile=out-verbounds-low$lowbound-high$highbound.$infile expectedsuperblock="SUPERBLOCK_VERSION $superblock" shift shift shift shift TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN -j $lowbound -k $highbound $infile $outfile ) RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi #--------------------------------- # check the layout from a dataset VERIFY "superblock" ( cd $TESTDIR $RUNSERIAL $H5DUMP_BIN -H -B $outfile > $layoutfile ) $GREP "$expectedsuperblock" $TESTDIR/$layoutfile > /dev/null if [ $? -eq 0 ]; then echo " PASSED" else echo " FAILED" nerrors="`expr $nerrors + 1`" fi # clean up tmp files rm -f $outfile rm -f $layoutfile } # end of VERIFY_SUPERBLOCK #------------------------------------------------------------------ # Verifying that a latest file cannot be opened with earlier bounds VERIFY_INVALIDBOUNDS() { lowbound=$1 highbound=$2 infile=$3 outfile=out-verbounds-low$lowbound-high$highbound.$infile #-------------------------------------- # check for failure with invalid bounds TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN -j $lowbound -k $highbound $infile $outfile ) RET=$? if [ $RET != 0 ] ; then echo " PASSED" else echo "*FAILED*" nerrors="`expr $nerrors + 1`" fi # clean up tmp files rm -f $outfile } # end of VERIFY_INVALIDBOUNDS # ----------------------------------------------------------------------------- # Expect h5diff to fail # Use only by VERIFY_EXTERNAL_CONSOLIDATION # ----------------------------------------------------------------------------- DIFFFAIL() { VERIFY h5diff unequal $@ ( cd $TESTDIR $RUNSERIAL $H5DIFF_BIN -q "$@" ) RET=$? if [ $RET -eq 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi } # ----------------------------------------------------------------------------- # Catchall test for repacking with external files # Loops over all (internally-listed) cases and applies the given arguments # to h5repack. # Each test file family must be listed in the for loop at the start of the # 'function'. # Expects case files to be of format: # + h5repack_.h5 # + h5repack__ex.h5 # + h5repack__ex-.dat # ...where NAME is part of the internal list, and N is a positive decimal # number; multiple external files (*.dat) are allowed per file, but they must # follow the pattern and be in contiguous numerical sequence starting at 0. # ----------------------------------------------------------------------------- VERIFY_EXTERNAL_CONSOLIDATION() { for name in f32le int32le_1d int32le_2d int32le_3d uint8be do basename="h5repack_${name}" base_ext="${basename}_ex" base_rpkg="${basename}_rp" canonical="${basename}.h5" external="${base_ext}.h5" ex_dat="${base_ext}-0.dat" ex_dat_mv="__${ext_dat}" repacked="${base_rpkg}.h5" prev_nerrors=$nerrors TESTING external consolidation on $name with h5repack $@ echo " . . ." # baseline comparison of known files DIFFTEST $canonical $external # repack the external file, verify integrity ( cd $TESTDIR $H5REPACK_BIN $@ $external $repacked ) DIFFTEST $repacked $canonical DIFFTEST $repacked $external # invalidate external file by moving its first data file; verify ( cd $TESTDIR mv $ex_dat $ex_dat_mv ) DIFFFAIL $canonical $external DIFFFAIL $repacked $external DIFFTEST $canonical $repacked # reset test state for re-runs with different h5repack arguments ( cd $TESTDIR mv $ex_dat_mv $ex_dat rm $repacked ) # flag if any of the above checks failed if [ $prev_nerrors != $nerrors ] then echo ". . . *FAILED*" nerrors="`expr $prev_nerrors + 1`" else echo ". . . PASSED" fi done } # end VERIFY_EXTERNAL_CONSOLIDATION # ----------------------------------------------------------------------------- # same as TOOLTEST, but it uses the old syntax -i input_file -o output_file # ----------------------------------------------------------------------------- TOOLTEST0() { infile=$2 outfile=out-$1.$2 shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@" ) RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi rm -f $outfile } # same as TOOLTEST, but it uses without -i -o options # used to test the family driver, where these files reside # TOOLTEST1() { infile=$2 outfile=out-$1.$2 shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi rm -f $outfile } # This is same as TOOLTEST() with comparing display output # from -v option # TOOLTESTV() { expect="$TESTDIR/$2-$1.tst" actual="$TESTDIR/`basename $2 .ddl`.out" actual_err="$TESTDIR/`basename $2 .ddl`.err" infile=$2 outfile=out-$1.$2 shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) >$actual 2>$actual_err RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi # display output compare STDOUT_FILTER $actual tmp_file=/tmp/h5test_tmp_$$ cp $actual $tmp_file sed -e 's/GZIP (0\.[0-9][0-9][0-9]:1)/GZIP (0.XXX:1)/' \ < $tmp_file > $actual rm -f $tmp_file cat $actual_err >> $actual VERIFY output from $H5REPACK $@ if cmp -s $expect $actual; then echo " PASSED" else echo "*FAILED*" echo " Expected result (*.tst) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' fi rm -f $actual $actual_err rm -f $outfile } # Same as TOOLTEST, but expects h5repack fails # TOOLTEST_FAIL() { infile=$1 outfile=$2 expect="$TESTDIR/$2-$1.ddl" actual="$TESTDIR/$2-$1.out" actual_err="$TESTDIR/$2-$1.err" shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) >&$actual RET=$? # Normally h5repack of files tested with this function are expected # to return not 0, but if the command results in "Segmentation fault" # or "core dumped" it is a failure regardless of the return value. failure=`grep -e 'Segmentation fault' -e 'core dumped' $actual` if [ "$failure" != "" ]; then nerrors="`expr $nerrors + 1`" echo " FAILED" echo " $failure" else if [ $RET == 0 ] ; then nerrors="`expr $nerrors + 1`" echo " FAILED" else echo " PASSED" fi fi rm -f $outfile } # This is same as TOOLTESTV() with comparing display output # with actual filename swapped # TOOLTESTSV() { expect="$TESTDIR/$1.$2.tst" actual="$TESTDIR/`basename $2 .ddl`.out" actual_err="$TESTDIR/`basename $2 .ddl`.err" infile=$2 outfile=out-$1.$2 shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) >$actual 2>$actual_err RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" VERIFY h5diff equal $@ ( cd $TESTDIR $RUNSERIAL $H5DIFF_BIN -v $infile $outfile ) >$actual 2>$actual_err RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi fi # display output compare STDOUT_FILTER $actual cat $actual_err >> $actual VERIFY output from $H5REPACK $@ if cmp -s $expect $actual; then echo " PASSED" else echo "*FAILED*" echo " Expected result (*.tst) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' fi rm -f $actual $actual_err rm -f $outfile } # This is same as TOOLTESTSV() but expects a diff fail # TOOLFAILSV() { expect="$TESTDIR/$1.$2.tst" actual="$TESTDIR/`basename $2 .ddl`.out" actual_err="$TESTDIR/`basename $2 .ddl`.err" infile=$2 outfile=out-$1.$2 shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) >$actual 2>$actual_err RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" VERIFY h5diff unequal $@ ( cd $TESTDIR $RUNSERIAL $H5DIFF_BIN -v $infile $outfile ) >$actual 2>$actual_err RET=$? if [ $RET == 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi fi # display output compare STDOUT_FILTER $actual cat $actual_err >> $actual VERIFY output from $H5REPACK $@ if cmp -s $expect $actual; then echo " PASSED" else echo "*FAILED*" echo " Expected result (*.tst) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' fi rm -f $actual $actual_err rm -f $outfile } # same as TOOLTESTV but filters error stack output and compares to an error file # Extract file name, line number, version and thread IDs because they may be different # ADD_H5ERR_MASK_TEST TOOLTESTM() { expect="$TESTDIR/$2-$1.tst" actual="$TESTDIR/`basename $2 .tst`.out" actual_err="$TESTDIR/`basename $2 .tst`.oerr" actual_sav=${actual}-sav actual_err_sav=${actual_err}-sav infile=$2 outfile=out-$1.$2 shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. cp $actual $actual_sav cp $actual_err $actual_err_sav # Extract file name, line number, version and thread IDs because they may be different sed -e 's/thread [0-9]*/thread (IDs)/' -e 's/: .*\.c /: (file name) /' \ -e 's/line [0-9]*/line (number)/' \ -e 's/v[1-9]*\.[0-9]*\./version (number)\./' \ -e 's/[1-9]*\.[0-9]*\.[0-9]*[^)]*/version (number)/' \ -e 's/H5Eget_auto[1-2]*/H5Eget_auto(1 or 2)/' \ -e 's/H5Eset_auto[1-2]*/H5Eset_auto(1 or 2)/' \ $actual_err > $actual if [ ! -f $expect ]; then # Create the expect file if it doesn't yet exist. echo " CREATED" cp $actual $expect echo " Expected result (*.tst) missing" nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then echo " PASSED" else echo "*FAILED*" echo " Expected result (*.tst) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then rm -f $actual $actual_err $actual_sav $actual_err_sav fi } # This is same as TOOLTESTV() with comparing h5dump output # from -pH option # TOOLTEST_DUMP() { infile=$2 outfile=out-$1.$2 expect="$TESTDIR/$1.$2.ddl" actual="$TESTDIR/out-$1.$2.out" actual_err="$TESTDIR/out-$1.$2.err" shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) >$actual 2>$actual_err RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" VERIFY h5dump output $@ ( cd $TESTDIR $RUNSERIAL $H5DUMP_BIN -q creation_order -pH $outfile ) >$actual 2>$actual_err cat $actual_err >> $actual RET=$? fi if cmp -s $expect $actual; then echo " PASSED" else echo "*FAILED*" echo " Expected result (*.ddl) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' fi rm -f $actual $actual_err rm -f $outfile } # This is same as TOOLTESTV() with comparing h5dump output # from -pH option and MASK out compression # TOOLTEST_DUMP_MASK() { infile=$2 outfile=out-$1.$2 expect="$TESTDIR/$1.$2.ddl" actual="$TESTDIR/out-$1.$2.out" actual_err="$TESTDIR/out-$1.$2.err" shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) >$actual 2>$actual_err RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" VERIFY h5dump output $@ ( cd $TESTDIR $RUNSERIAL $H5DUMP_BIN -q creation_order -pH $outfile ) >$actual 2>$actual_err cat $actual_err >> $actual RET=$? fi tmp_file=/tmp/h5test_tmp_$$ cp $actual $tmp_file sed -e 's/SIZE [0-9][0-9][0-9][0-9] (2\.[0-9][0-9][0-9]:1 COMPRESSION)/SIZE XXXX (2.XXX:1 COMPRESSION)/' \ < $tmp_file > $actual rm -f $tmp_file cat $actual_err >> $actual if cmp -s $expect $actual; then echo " PASSED" else echo "*FAILED*" echo " Expected result (*.ddl) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' fi rm -f $actual $actual_err rm -f $outfile } # This is same as TOOLTEST_DUMP() with comparing h5dump output # without any option # TOOLTEST_DUMP_NO_OPT() { infile=$2 outfile=out-$1.$2 expect="$TESTDIR/$1.$2.ddl" actual="$TESTDIR/out-$1.$2.out" actual_err="$TESTDIR/out-$1.$2.err" shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) >$actual 2>$actual_err RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" VERIFY h5dump output $@ ( cd $TESTDIR $RUNSERIAL $H5DUMP_BIN $outfile ) >$actual 2>$actual_err cat $actual_err >> $actual RET=$? fi if cmp -s $expect $actual; then echo " PASSED" else echo "*FAILED*" echo " Expected result (*.ddl) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' fi rm -f $actual $actual_err rm -f $outfile } # This is similar to TOOLTEST_DUMP(). # Test h5repack with options added for paged aggregation. # h5stat is used on the repacked file and the expected output # is compared for correctness. # TOOLTEST_STAT() { infile=$2 outfile=out-$1.$2 expect="$TESTDIR/$1.$2.ddl" actual="$TESTDIR/out-$1.$2.out" actual_err="$TESTDIR/out-$1.$2.err" shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) >$actual 2>$actual_err RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" VERIFY h5stat output $@ ( cd $TESTDIR $RUNSERIAL $H5STAT_BIN -S -s $outfile ) >$actual 2>$actual_err cat $actual_err >> $actual RET=$? fi if cmp -s $expect $actual; then echo " PASSED" else echo "*FAILED*" echo " Expected result (*.ddl) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' fi rm -f $actual $actual_err rm -f $outfile } # TOOLTEST_META: # Test metadata block size option. # Reason to create a function here is to localize all special steps related to # metadata block size option in one place. This is a quick solution. More # thought out solution needed when there is more time. # # $1: test input file # $2:$: metadata options (either "-M size" or "--metadata_block_size=size") # # Algorithm: # Run it once without the metadata option ($2-$); # Save the result output file; # Run it second time with the metadata option; # Verify the output file of second run is larger than the one of 1st run. TOOLTEST_META() { infile=$2 outfile=out-$1.$2 shift shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN $infile $outfile ) RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi # get the size of the first output file size1=`wc -c $TESTDIR/$outfile | cut -d' ' -f1` # 2nd run with metadata option # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile ) RET=$? if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" DIFFTEST $infile $outfile fi # get the size of the second output file size2=`wc -c $TESTDIR/$outfile | cut -d' ' -f1` # verify sizes. MESSAGE "Verify the sizes of both output files ($size1 vs $size2)" if [ $size1 -lt $size2 ]; then # pass echo " PASSED" else #fail echo "*FAILED*" nerrors="`expr $nerrors + 1`" fi rm -f $outfile } # ADD_HELP_TEST TOOLTEST_HELP() { expect="$TESTDIR/$1" actual="$TESTDIR/`basename $1 .txt`.out" actual_err="$TESTDIR/`basename $1 .txt`.err" shift # Run test. TESTING $H5REPACK $@ ( cd $TESTDIR $RUNSERIAL $H5REPACK_BIN "$@" ) >$actual 2>$actual_err cat $actual_err >> $actual if [ ! -f $expect ]; then # Create the expect data file if it doesn't yet exist. echo " CREATED" cp $actual $expect-CREATED echo " Expected result (*.txt) missing" nerrors="`expr $nerrors + 1`" elif cmp -s $expect $actual; then echo " PASSED" else echo "*FAILED*" echo " Expected output (*.txt) differs from actual output (*.out)" nerrors="`expr $nerrors + 1`" fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then rm -f $actual $actual_err fi } # This is different from $srcdir/../../../bin/output_filter.sh STDOUT_FILTER() { result_file=$1 tmp_file=/tmp/h5test_tmp_$$ # Filter name of files. cp $result_file $tmp_file sed -e '/^Opening file/d' -e '/^Making file/d' \ < $tmp_file > $result_file # cleanup rm -f $tmp_file } # # The tests # We use the files generated by h5repacktst # Each run generates ".out.h5" and the tool h5diff is used to # compare the input and output files # # the tests are the same as the program h5repacktst, but run from the CLI # # See which filters are usable (and skip tests for filters we # don't have). Do this by searching H5pubconf.h to see which # filters are defined. # detect whether the encoder is present. USE_FILTER_SZIP_ENCODER="no"; if test $USE_FILTER_SZIP = "yes"; then USE_FILTER_SZIP_ENCODER=`$RUNSERIAL $H5DETECTSZIP_BIN` fi ############################################################################## ### T H E T E S T S ############################################################################## # prepare for test COPY_TESTFILES_TO_TESTDIR TOOLTEST_HELP h5repack-help.txt -h # copy files (these files have no filters) TOOLTEST fill h5repack_fill.h5 TOOLTEST objs h5repack_objs.h5 TOOLTEST attr h5repack_attr.h5 TOOLTEST hlink h5repack_hlink.h5 TOOLTEST layout h5repack_layout.h5 TOOLTEST early h5repack_early.h5 # nested 8bit enum in both deflated and non-deflated datafiles if [ $USE_FILTER_DEFLATE != "yes" ]; then TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum.h5 else TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum_deflated.h5 fi # use h5repack_layout.h5 to write some filters (this file has no filters) # gzip with individual object arg="h5repack_layout.h5 -f dset1:GZIP=1 -l dset1:CHUNK=20x10" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST gzip_individual $arg fi # gzip for all arg="h5repack_layout.h5 -f GZIP=1" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST gzip_all $arg fi # szip with individual object arg="h5repack_layout.h5 -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else TOOLTEST szip_individual $arg fi # szip for all arg="h5repack_layout.h5 -f SZIP=8,NN" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else TOOLTEST szip_all $arg fi # shuffle with individual object arg="h5repack_layout.h5 -f dset2:SHUF -l dset2:CHUNK=20x10" TOOLTEST shuffle_individual $arg # shuffle for all arg="h5repack_layout.h5 -f SHUF" TOOLTEST shuffle_all $arg # fletcher32 with individual object arg="h5repack_layout.h5 -f dset2:FLET -l dset2:CHUNK=20x10" TOOLTEST fletcher_individual $arg # fletcher32 for all arg="h5repack_layout.h5 -f FLET" TOOLTEST fletcher_all $arg # all filters arg="h5repack_layout.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST all_filters $arg fi # verbose gzip with individual object arg="h5repack_filters.h5 -v -f /dset_deflate:GZIP=9" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else # compare output TOOLTESTV gzip_verbose_filters $arg fi ########################################################### # the following tests assume the input files have filters ########################################################### # szip copy arg="h5repack_szip.h5" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else TOOLTEST szip_copy $arg fi # szip remove arg="h5repack_szip.h5 --filter=dset_szip:NONE" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then SKIP $arg else TOOLTEST szip_remove $arg fi # deflate copy arg="h5repack_deflate.h5" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST deflate_copy $arg fi # deflate remove arg="h5repack_deflate.h5 -f dset_deflate:NONE" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST deflate_remove $arg fi # shuffle copy arg="h5repack_shuffle.h5" TOOLTEST shuffle_copy $arg # shuffle remove arg="h5repack_shuffle.h5 -f dset_shuffle:NONE" TOOLTEST shuffle_remove $arg # fletcher32 copy arg="h5repack_fletcher.h5" TOOLTEST fletcher_copy $arg # fletcher32 remove arg="h5repack_fletcher.h5 -f dset_fletcher32:NONE" TOOLTEST fletcher_remove $arg # nbit copy arg="h5repack_nbit.h5" TOOLTEST nbit_copy $arg # nbit remove arg="h5repack_nbit.h5 -f dset_nbit:NONE" TOOLTEST nbit_remove $arg # nbit add arg="h5repack_nbit.h5 -f dset_int31:NBIT" TOOLTEST nbit_add $arg # scaleoffset copy arg="h5repack_soffset.h5" TOOLTEST scale_copy $arg # scaleoffset add arg="h5repack_soffset.h5 -f dset_none:SOFF=31,IN" TOOLTEST scale_add $arg # scaleoffset remove arg="h5repack_soffset.h5 -f dset_scaleoffset:NONE" TOOLTEST scale_remove $arg # remove all filters arg="h5repack_filters.h5 -f NONE" if test $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" ; then SKIP $arg else TOOLTEST remove_all $arg fi #filter conversions arg="h5repack_deflate.h5 -f dset_deflate:SZIP=8,NN" if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST deflate_convert $arg fi arg="h5repack_szip.h5 -f dset_szip:GZIP=1" if test $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST szip_convert $arg fi #limit arg="h5repack_layout.h5 -f GZIP=1 -m 1024" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST_DUMP_MASK deflate_limit $arg fi #file arg="h5repack_layout.h5 -e h5repack.info" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST deflate_file $arg fi #crtorder arg="tordergr.h5 -L" TOOLTEST_DUMP crtorder $arg ################################################################################################### # Testing paged aggregation related options: # -G pagesize # -P 1 or 0 # -S strategy # -T threshold # # The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_": # (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */ # (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */ # (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */ # (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */ # (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */ # (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */ # ##################################################################################################### # arg="h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1" TOOLTEST_STAT SP $arg # arg="h5repack_fsm_aggr_persist.h5 -S AGGR" TOOLTEST_STAT S $arg # arg="h5repack_none.h5 -S PAGE -T 10 -G 2048" TOOLTEST_STAT STG $arg # arg="h5repack_paged_nopersist.h5 -G 512 -S AGGR" TOOLTEST_STAT GS $arg # arg="h5repack_paged_persist.h5 -S NONE -P 1" TOOLTEST_STAT SP $arg # arg="h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5" TOOLTEST_STAT SPT $arg ######################################################### # layout options (these files have no filters) ######################################################### VERIFY_LAYOUT_DSET dset2_chunk_20x10 h5repack_layout.h5 dset2 CHUNKED --layout dset2:CHUNK=20x10 VERIFY_LAYOUT_ALL chunk_20x10 h5repack_layout.h5 CHUNKED -l CHUNK=20x10 VERIFY_LAYOUT_DSET dset2_conti h5repack_layout.h5 dset2 CONTIGUOUS -l dset2:CONTI VERIFY_LAYOUT_ALL conti h5repack_layout.h5 CONTIGUOUS -l CONTI VERIFY_LAYOUT_DSET dset2_compa h5repack_layout.h5 dset2 COMPACT -l dset2:COMPA VERIFY_LAYOUT_ALL compa h5repack_layout.h5 COMPACT -l COMPA TOOLTESTM dset2_chunk_20x10-errstk h5repack_layout.h5 --layout=dset2:CHUNK=20x10x5 --enable-error-stack ################################################################ # layout conversions (file has no filters) ############################################################### VERIFY_LAYOUT_DSET dset_compa_conti h5repack_layout.h5 dset_compact CONTIGUOUS -l dset_compact:CONTI VERIFY_LAYOUT_DSET dset_compa_chunk h5repack_layout.h5 dset_compact CHUNKED -l dset_compact:CHUNK=2x5 VERIFY_LAYOUT_DSET dset_compa_compa h5repack_layout.h5 dset_compact COMPACT -l dset_compact:COMPA VERIFY_LAYOUT_DSET dset_conti_compa h5repack_layout.h5 dset_contiguous COMPACT -l dset_contiguous:COMPA VERIFY_LAYOUT_DSET dset_conti_chunk h5repack_layout.h5 dset_contiguous CHUNKED -l dset_contiguous:CHUNK=3x6 VERIFY_LAYOUT_DSET dset_conti_conti h5repack_layout.h5 dset_contiguous CONTIGUOUS -l dset_contiguous:CONTI VERIFY_LAYOUT_DSET chunk_compa h5repack_layout.h5 dset_chunk COMPACT -l dset_chunk:COMPA VERIFY_LAYOUT_DSET chunk_conti h5repack_layout.h5 dset_chunk CONTIGUOUS -l dset_chunk:CONTI VERIFY_LAYOUT_DSET chunk_18x13 h5repack_layout.h5 dset_chunk CHUNKED -l dset_chunk:CHUNK=18x13 # test convert small size dataset ( < 1k) to compact layout without -m VERIFY_LAYOUT_DSET contig_small_compa h5repack_layout2.h5 contig_small COMPACT -l contig_small:COMPA VERIFY_LAYOUT_DSET contig_small_fixed_compa h5repack_layout2.h5 chunked_small_fixed COMPACT -l chunked_small_fixed:COMPA #--------------------------------------------------------------------------- # Test file contains chunked datasets (need multiple dsets) with # unlimited max dims. (HDFFV-7933) # Use first dset to test. #--------------------------------------------------------------------------- # chunk to chunk - specify chunk dim bigger than any current dim VERIFY_LAYOUT_DSET chunk2chunk h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:CHUNK=100x300 # chunk to contiguous VERIFY_LAYOUT_DSET chunk2conti h5repack_layout3.h5 chunk_unlimit1 CONTI -l chunk_unlimit1:CONTI # chunk to compact - convert big dataset (should be > 64k) for this purpose, # should remain as original layout (chunk) VERIFY_LAYOUT_DSET chunk2compa h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:COMPA #-------------------------------------------------------------------------- # Test -f for some specific cases. Chunked dataset with unlimited max dims. # (HDFFV-8012) #-------------------------------------------------------------------------- # - should not fail # - should not change max dims from unlimit # chunk dim is bigger than dataset dim. ( dset size < 64k ) VERIFY_LAYOUT_DSET error1 h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED -f chunk_unlimit1:NONE # chunk dim is bigger than dataset dim. ( dset size > 64k ) VERIFY_LAYOUT_DSET error2 h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED -f chunk_unlimit2:NONE # chunk dims are smaller than dataset dims. ( dset size < 64k ) #TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit3:NONE VERIFY_LAYOUT_DSET error3 h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED -f chunk_unlimit3:NONE # file input - should not fail TOOLTEST error4 h5repack_layout3.h5 -f NONE #-------------------------------------------------------------------------- # Test base: Convert CHUNK to CONTI for a chunked dataset with small dataset # (dset size < 64K) and with unlimited max dims on a condition as follow. # (HDFFV-8214) #-------------------------------------------------------------------------- # chunk dim is bigger than dataset dim. should succeed. VERIFY_LAYOUT_DSET ckdim_biger h5repack_layout3.h5 chunk_unlimit2 CONTI -l chunk_unlimit2:CONTI # chunk dim is smaller than dataset dim. should succeed. VERIFY_LAYOUT_DSET ckdim_smaller h5repack_layout3.h5 chunk_unlimit3 CONTI -l chunk_unlimit3:CONTI # Native option # Do not use FILE1, as the named dtype will be converted to native, and h5diff will # report a difference. TOOLTEST native_fill h5repack_fill.h5 -n TOOLTEST native_attr h5repack_attr.h5 -n # latest file format with long switches. use FILE4=h5repack_layout.h5 (no filters) arg="h5repack_layout.h5 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else VERIFY_LAYOUT_ALL layout_long_switches h5repack_layout.h5 CHUNKED --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype] fi # latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else VERIFY_LAYOUT_ALL layout_short_switches h5repack_layout.h5 CHUNKED -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype] fi # several global filters arg="h5repack_layout.h5 --filter GZIP=1 --filter SHUF" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST global_filters $arg fi # syntax of -i infile -o outfile # latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST0 old_style_layout_short_switches $arg fi # add a userblock to file arg="h5repack_objs.h5 -u ublock.bin -b 2048" TOOLTEST add_userblock $arg # add alignment arg="h5repack_objs.h5 -t 1 -a 1 " TOOLTEST add_alignment $arg # Check repacking file with old version of layout message (should get upgraded # to new version and be readable, etc.) TOOLTEST upgrade_layout h5repack_layouto.h5 # test for datum size > H5TOOLS_MALLOCSIZE arg="h5repack_objs.h5 -f GZIP=1" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else TOOLTEST gt_mallocsize $arg fi # Check repacking file with committed datatypes in odd configurations TOOLTEST committed_dt h5repack_named_dtypes.h5 # tests family driver (file is located in common testfiles folder, uses TOOLTEST1 TOOLTEST1 family tfamily%05d.h5 # test various references (bug 1814 and 1726) TOOLTEST bug1814 h5repack_refs.h5 # test attribute with various references (bug1797 / HDFFV-5932) # the references in attribute of compound or vlen datatype TOOLTEST HDFFV-5932 h5repack_attr_refs.h5 # Add test for memory leak in attribute. This test is verified by CTEST. # 1. leak from vlen string # 2. leak from compound type without reference member # (HDFFV-7840, ) # Note: this test is experimental for sharing test file among tools TOOLTEST HDFFV-7840 h5diff_attr1.h5 # test HDFFV-10590 arg="h5repack_CVE-2018-17432.h5 h5repack_CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6" TOOLTEST_FAIL $arg # test HDFFV-11223 arg="h5repack_CVE-2018-14460.h5 h5repack_CVE-2018-14460_out.h5" TOOLTEST_FAIL $arg # tests for metadata block size option TOOLTEST_META meta_short h5repack_layout.h5 -M 8192 TOOLTEST_META meta_long h5repack_layout.h5 --metadata_block_size=8192 # VDS tests ################################################################ # layout conversions ############################################################### if test $USE_FILTER_DEFLATE != "yes" ; then SKIP vds_dset_chunk_20x10x5 else VERIFY_LAYOUT_VDS vds_dset_chunk20x10x5 1_vds.h5 vds_dset CHUNKED --layout vds_dset:CHUNK=20x10x5 fi if test $USE_FILTER_DEFLATE != "yes" ; then SKIP vds_chunk2x5x8 else VERIFY_LAYOUT_VDS vds_chunk2x5x8 3_1_vds.h5 vds_dset CHUNKED -l vds_dset:CHUNK=2x5x8 fi if test $USE_FILTER_DEFLATE != "yes" ; then SKIP vds_chunk3x6x9 else VERIFY_LAYOUT_VDS vds_chunk3x6x9 2_vds.h5 vds_dset CHUNKED -l vds_dset:CHUNK=3x6x9 fi if test $USE_FILTER_DEFLATE != "yes" ; then SKIP vds_compa 4_vds.h5 else VERIFY_LAYOUT_VDS vds_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA fi if test $USE_FILTER_DEFLATE != "yes" ; then SKIP vds_conti 4_vds.h5 else VERIFY_LAYOUT_VDS vds_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI fi ################################################################ # reference new api conversions ############################################################### #TOOLTEST_DUMP attrregion tattrreg.h5 #TOOLTEST_DUMP dataregion tdatareg.h5 ############################################################################## ### V E R S I O N B O U N D S T E S T S ############################################################################## # -j 0 -k 2, superblock will be 0 VERIFY_SUPERBLOCK 0 2 0 h5repack_layout.h5 -j 0 -k 2 h5repack_layout.h5 # -j 1 -k 2, superblock will be 2 VERIFY_SUPERBLOCK 1 2 2 h5repack_layout.h5 -j 1 -k 2 h5repack_layout.h5 # -j 2 -k 2, superblock will be 3 VERIFY_SUPERBLOCK 2 2 3 h5repack_layout.h5 -j 2 -k 2 h5repack_layout.h5 # -j 0 -k 1, file cannot be opened VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5 ############################################################################## ### E X T E R N A L S T O R A G E T E S T S ############################################################################## VERIFY_EXTERNAL_CONSOLIDATION -l CONTI ############################################################################## ### E X T E R N A L L I N K T E S T S ############################################################################## ### HDFFV-11128 needs fixed to enable the following test #TOOLTEST_DUMP h5copy_extlinks_src-base h5copy_extlinks_src.h5 --enable-error-stack TOOLTEST_DUMP tsoftlinks-base tsoftlinks.h5 --enable-error-stack TOOLTEST_DUMP textlink-base textlink.h5 --enable-error-stack TOOLTEST_DUMP textlinkfar-base textlinkfar.h5 --enable-error-stack TOOLTEST_DUMP textlinksrc-base textlinksrc.h5 --enable-error-stack TOOLTEST_DUMP textlinktar-base textlinktar.h5 --enable-error-stack TOOLTESTSV h5copy_extlinks_src-merge h5copy_extlinks_src.h5 --merge TOOLFAILSV tsoftlinks-merge tsoftlinks.h5 --merge TOOLTESTSV textlink-merge textlink.h5 --merge ### HDFFV-11128 needs fixed to enable the following test #TOOLFAILSV textlinkfar-merge textlinkfar.h5 --merge ### HDFFV-11128 needs fixed to enable the following test #TOOLFAILSV textlinksrc-merge textlinksrc.h5 --merge ### HDFFV-11128 needs fixed to enable the following test #TOOLFAILSV textlinktar-merge textlinktar.h5 --merge TOOLTEST_DUMP h5copy_extlinks_src-prune h5copy_extlinks_src.h5 --prune --enable-error-stack TOOLTEST_DUMP tsoftlinks-prune tsoftlinks.h5 --prune --enable-error-stack TOOLTEST_DUMP textlink-prune textlink.h5 --prune --enable-error-stack TOOLTEST_DUMP textlinkfar-prune textlinkfar.h5 --prune --enable-error-stack TOOLTEST_DUMP textlinksrc-prune textlinksrc.h5 --prune --enable-error-stack TOOLTEST_DUMP textlinktar-prune textlinktar.h5 --prune --enable-error-stack TOOLTEST_DUMP h5copy_extlinks_src-mergeprune h5copy_extlinks_src.h5 --merge --prune --enable-error-stack TOOLTEST_DUMP tsoftlinks-mergeprune tsoftlinks.h5 --merge --prune --enable-error-stack TOOLTEST_DUMP textlink-mergeprune textlink.h5 --merge --prune --enable-error-stack ### HDFFV-11128 needs fixed to enable the following test #TOOLTEST_DUMP textlinkfar-mergeprune textlinkfar.h5 --merge --prune --enable-error-stack ### HDFFV-11128 needs fixed to enable the following test #TOOLTEST_DUMP textlinksrc-mergeprune textlinksrc.h5 --merge --prune --enable-error-stack ### HDFFV-11128 needs fixed to enable the following test #TOOLTEST_DUMP textlinktar-mergeprune textlinktar.h5 --merge --prune --enable-error-stack TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_1d.h5 --src-vfd-name onion --src-vfd-info 1 TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_ext.h5 --src-vfd-name onion --src-vfd-info 1 TOOLTEST_DUMP_NO_OPT onion tst_onion_objs.h5 --src-vfd-name onion --src-vfd-info 1 # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR if test $nerrors -eq 0 ; then echo "All $TESTNAME tests passed." exit $EXIT_SUCCESS else echo "$TESTNAME tests failed with $nerrors errors." exit $EXIT_FAILURE fi