mirror of
https://github.com/HDFGroup/hdf5.git
synced 2024-12-27 08:01:04 +08:00
fedd036348
feature Description: h5repack support for scaleoffset compression Checking in early to help debug the filter. Solution: Added messages and command line to handle new scale offset filter. Note: TESTS ARE DISABLED FOR NOW. The filter is not complete, repack tests may fail due to know problems. PLEASE DO NOT MESS WITH THE SCALEOFFSET TESTS AT THIS TIME. They will be enabled when the filter is ready. Platforms tested: verbena,copper,shanti Misc. update: MANIFEST
422 lines
9.7 KiB
Bash
Executable File
422 lines
9.7 KiB
Bash
Executable File
#! /bin/sh
|
|
#
|
|
# Copyright by the Board of Trustees of the University of Illinois.
|
|
# All rights reserved.
|
|
#
|
|
# This file is part of HDF5. The full HDF5 copyright notice, including
|
|
# terms governing use, modification, and redistribution, is contained in
|
|
# the files COPYING and Copyright.html. COPYING can be found at the root
|
|
# of the source code distribution tree; Copyright.html can be found at the
|
|
# root level of an installed copy of the electronic HDF5 document set and
|
|
# is linked from the top-level documents page. It can also be found at
|
|
# http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have
|
|
# access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu.
|
|
#
|
|
# Tests for the h5repack tool
|
|
|
|
USE_FILTER_SZIP="@USE_FILTER_SZIP@"
|
|
USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"
|
|
USE_FILTER_SHUFFLE="@USE_FILTER_SHUFFLE@"
|
|
USE_FILTER_FLETCHER32="@USE_FILTER_FLETCHER32@"
|
|
USE_FILTER_NBIT="@USE_FILTER_NBIT@"
|
|
#
|
|
# Leave these tests off for now. Enable when filter is ready
|
|
:
|
|
#USE_FILTER_SCALEOFFSET="@USE_FILTER_SCALEOFFSET@"
|
|
USE_FILTER_SCALEOFFSET="no"
|
|
|
|
|
|
H5REPACK=h5repack # The tool name
|
|
H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary
|
|
|
|
H5DIFF=../h5diff/h5diff # The h5diff tool name
|
|
H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary
|
|
|
|
H5DETECTSZIP=testh5repack_detect_szip # The tool name
|
|
H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP # The path of the tool binary
|
|
|
|
nerrors=0
|
|
verbose=yes
|
|
|
|
# The build (current) directory might be different than the source directory.
|
|
#
|
|
if test -z "$srcdir"; then
|
|
srcdir=.
|
|
fi
|
|
|
|
test -d ../testfiles || mkdir ../testfiles
|
|
|
|
# Print a line-line message left justified in a field of 70 characters
|
|
# beginning with the word "Testing".
|
|
#
|
|
TESTING() {
|
|
SPACES=" "
|
|
echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
|
|
}
|
|
|
|
# Print a line-line message left justified in a field of 70 characters
|
|
# beginning with the word "Verifying".
|
|
#
|
|
VERIFY() {
|
|
SPACES=" "
|
|
echo "Testing h5diff output $* $SPACES" | cut -c1-70 | tr -d '\012'
|
|
}
|
|
|
|
# Print a message that a test has been skipped (because a required filter
|
|
# was unavailable)
|
|
SKIP() {
|
|
TESTING $H5REPACK $@
|
|
echo " -SKIP-"
|
|
}
|
|
|
|
# Call the h5diff tool
|
|
#
|
|
DIFFTEST()
|
|
{
|
|
VERIFY $@
|
|
if [ "`uname -s`" = "TFLOPS O/S" ]; then
|
|
$RUNSERIAL $H5DIFF_BIN $@ -q
|
|
else
|
|
$RUNSERIAL $H5DIFF_BIN "$@" -q
|
|
fi
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
fi
|
|
|
|
}
|
|
|
|
# Call h5repack
|
|
#
|
|
TOOLTEST()
|
|
{
|
|
# Run test.
|
|
# Tflops interprets "$@" as "" when no parameter is given (e.g., the
|
|
# case of missing file name). Changed it to use $@ till Tflops fixes it.
|
|
TESTING $H5REPACK $@
|
|
|
|
infile=$srcdir/../testfiles/$1
|
|
path=`pwd`
|
|
outfile=$path/out.$1
|
|
shift
|
|
if [ "`uname -s`" = "TFLOPS O/S" ]; then
|
|
$RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile $@
|
|
else
|
|
$RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@"
|
|
fi
|
|
|
|
RET=$?
|
|
if [ $RET != 0 ] ; then
|
|
echo "*FAILED*"
|
|
nerrors="`expr $nerrors + 1`"
|
|
else
|
|
echo " PASSED"
|
|
DIFFTEST $infile $outfile
|
|
fi
|
|
rm -f $outfile
|
|
}
|
|
|
|
#
|
|
# The tests
|
|
# We use the files generated by h5repacktst
|
|
# Each run generates "<file>.out.h5" and the tool h5diff is used to
|
|
# compare the input and output files
|
|
#
|
|
# the tests are the same as the program h5repacktst, but run from the CLI
|
|
#
|
|
|
|
# See which filters are usable (and skip tests for filters we
|
|
# don't have). Do this by searching H5pubconf.h to see which
|
|
# filters are defined.
|
|
|
|
# detect whether the encoder is present.
|
|
USE_FILTER_SZIP_ENCODER="no";
|
|
if test $USE_FILTER_SZIP = "yes"; then
|
|
USE_FILTER_SZIP_ENCODER=`$RUNSERIAL $H5DETECTSZIP_BIN`
|
|
fi
|
|
|
|
# copy files (these files have no filters; test2.h5 has copied references that h5diff detects)
|
|
TOOLTEST test0.h5
|
|
TOOLTEST test1.h5
|
|
TOOLTEST test3.h5
|
|
TOOLTEST test4.h5
|
|
TOOLTEST test5.h5
|
|
|
|
# use test4.h5 to write some filters (this file has no filters)
|
|
|
|
# gzip with individual object
|
|
arg="test4.h5 -f dset1:GZIP=1 -l dset1:CHUNK=20x10"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# gzip for all
|
|
arg="test4.h5 -f GZIP=1"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# szip with individual object
|
|
arg="test4.h5 -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# szip for all
|
|
arg="test4.h5 -f SZIP=8,NN"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# shuffle with individual object
|
|
arg="test4.h5 -f dset2:SHUF -l dset2:CHUNK=20x10"
|
|
if test $USE_FILTER_SHUFFLE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
|
|
# shuffle for all
|
|
arg="test4.h5 -f SHUF"
|
|
if test $USE_FILTER_SHUFFLE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# fletcher32 with individual object
|
|
arg="test4.h5 -f dset2:FLET -l dset2:CHUNK=20x10"
|
|
if test $USE_FILTER_FLETCHER32 != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# fletcher32 for all
|
|
arg="test4.h5 -f FLET"
|
|
if test $USE_FILTER_FLETCHER32 != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# all filters
|
|
arg="test4.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
###########################################################
|
|
# the following tests assume the input files have filters
|
|
###########################################################
|
|
|
|
# szip copy
|
|
arg="test_szip.h5"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# szip remove
|
|
arg="test_szip.h5 -f dset_szip:NONE"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# deflate copy
|
|
arg="test_deflate.h5"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# deflate remove
|
|
arg="test_deflate.h5 -f dset_deflate:NONE"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# shuffle copy
|
|
arg="test_shuffle.h5"
|
|
if test $USE_FILTER_SHUFFLE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# shuffle remove
|
|
arg="test_shuffle.h5 -f dset_shuffle:NONE"
|
|
if test $USE_FILTER_SHUFFLE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# fletcher32 copy
|
|
arg="test_fletcher32.h5"
|
|
if test $USE_FILTER_FLETCHER32 != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# fletcher32 remove
|
|
arg="test_fletcher32.h5 -f dset_fletcher32:NONE"
|
|
if test $USE_FILTER_FLETCHER32 != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# nbit copy
|
|
arg="test_nbit.h5"
|
|
if test $USE_FILTER_NBIT != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# nbit remove
|
|
arg="test_nbit.h5 -f dset_nbit:NONE"
|
|
if test $USE_FILTER_NBIT != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# nbit add
|
|
arg="test_nbit.h5 -f dset_int31:NBIT"
|
|
if test $USE_FILTER_NBIT != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# scaleoffset add
|
|
arg="test_scaleoffset.h5 -f dset_none:S+O=31"
|
|
if test $USE_FILTER_SCALEOFFSET != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# scaleoffset copy
|
|
arg="test_scaleoffset.h5"
|
|
if test $USE_FILTER_SCALEOFFSET != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# scaleoffset remove
|
|
arg="test_scaleoffset.h5 -f dset_scaleoffset:NONE"
|
|
if test $USE_FILTER_SCALEOFFSET != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
# remove all filters
|
|
arg="test_all.h5 -f NONE"
|
|
if test $USE_FILTER_FLETCHER32 != "yes" -o $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SHUFFLE != "yes" -o $USE_FILTER_NBIT != "yes" -o $USE_FILTER_SCALEOFFSET != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
#filter conversions
|
|
|
|
arg="test_deflate.h5 -f dset_deflate:SZIP=8,NN"
|
|
if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
arg="test_szip.h5 -f dset_szip:GZIP=1"
|
|
if test $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
|
|
#limit
|
|
arg="test4.h5 -f GZIP=1 -m 1024"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
#file
|
|
arg="test4.h5 -e ../testfiles/h5repack_info.txt"
|
|
if test $USE_FILTER_DEFLATE != "yes" ; then
|
|
SKIP $arg
|
|
else
|
|
TOOLTEST $arg
|
|
fi
|
|
|
|
|
|
#########################################################
|
|
# layout options (these files have no filters)
|
|
#########################################################
|
|
|
|
TOOLTEST test4.h5 -l dset2:CHUNK=20x10
|
|
TOOLTEST test4.h5 -l CHUNK=20x10
|
|
TOOLTEST test4.h5 -l dset2:CONTI
|
|
TOOLTEST test4.h5 -l CONTI
|
|
TOOLTEST test4.h5 -l dset2:COMPA
|
|
TOOLTEST test4.h5 -l COMPA
|
|
|
|
|
|
################################################################
|
|
# layout conversions (file has no filters)
|
|
###############################################################
|
|
|
|
arg1="test4.h5 -l dset_compact:CONTI"
|
|
arg2="test4.h5 -l dset_compact:CHUNK=2x5"
|
|
arg3="test4.h5 -l dset_compact:COMPA"
|
|
arg4="test4.h5 -l dset_contiguous:COMPA"
|
|
arg5="test4.h5 -l dset_contiguous:CHUNK=3x6"
|
|
arg6="test4.h5 -l dset_contiguous:CONTI"
|
|
arg7="test4.h5 -l dset_chunk:COMPA"
|
|
arg8="test4.h5 -l dset_chunk:CONTI"
|
|
arg9="test4.h5 -l dset_chunk:CHUNK=18x13"
|
|
TOOLTEST $arg1
|
|
TOOLTEST $arg2
|
|
TOOLTEST $arg3
|
|
TOOLTEST $arg4
|
|
TOOLTEST $arg5
|
|
TOOLTEST $arg6
|
|
TOOLTEST $arg7
|
|
TOOLTEST $arg8
|
|
TOOLTEST $arg9
|
|
|
|
|
|
if test $nerrors -eq 0 ; then
|
|
echo "All $H5REPACK tests passed."
|
|
fi
|
|
|
|
exit $nerrors
|