hdf5/tools/test/h5format_convert/testh5fc.sh.in
lrknox 8452976de5 tools/test/h5diff/testh5diff.sh.in:
Add code to delete copies of test .h5 files copied from
         tools/testfiles/vds to tools/test/h5diff/testfiles when running
         "make check" for an in-source build.

    tools/test/misc/testh5repart.sh.in:
         Change the name of the temporary test directory from "testfiles"
         which is also used for 2 h5mkgrp test files under source control to
         "testrepart" to avoid deleting source controlled files when running
         "make check" in the source directory.

    Fix test scripts that remove source-controlled files during "make check" when run in-source.
    Fix test scripts that don't remove test files because they add files to test directories but don't remove them if the build directories are the same as the source directory.

    Fix any test source file or Makefile.am files for tests that leave data files are removed by neither "make check" nor "make distclean".
2017-01-12 22:20:11 -06:00

509 lines
16 KiB
Bash

#! /bin/sh
#
# Copyright by The HDF Group.
# Copyright by the Board of Trustees of the University of Illinois.
# All rights reserved.
#
# This file is part of HDF5. The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the files COPYING and Copyright.html. COPYING can be found at the root
# of the source code distribution tree; Copyright.html can be found at the
# root level of an installed copy of the electronic HDF5 document set and
# is linked from the top-level documents page. It can also be found at
# http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have
# access to either file, you may request a copy from help@hdfgroup.org.
#
# Tests for the h5format_convert tool
#
#
srcdir=@srcdir@
# Determine which filters are available
USE_FILTER_SZIP="@USE_FILTER_SZIP@"
USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"
USE_FILTER_SHUFFLE="@USE_FILTER_SHUFFLE@"
USE_FILTER_FLETCHER32="@USE_FILTER_FLETCHER32@"
USE_FILTER_NBIT="@USE_FILTER_NBIT@"
USE_FILTER_SCALEOFFSET="@USE_FILTER_SCALEOFFSET@"
TESTNAME=h5format_convert
EXIT_SUCCESS=0
EXIT_FAILURE=1
FORMCONV=../../src/h5format_convert/h5format_convert # The tool name
FORMCONV_BIN=`pwd`/$FORMCONV # The path of the tool binary
CHK_IDX=h5fc_chk_idx # The program name
CHK_IDX_BIN=`pwd`/$CHK_IDX # The program to verify the chunk indexing type is v1 B-tree
RM='rm -rf'
CMP='cmp -s'
DIFF='diff -c'
CP='cp'
H5DUMP=../../src/h5dump/h5dump # The h5dump tool name
H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary
DIRNAME='dirname'
LS='ls'
AWK='awk'
nerrors=0
verbose=yes
# source dirs
SRC_TOOLS="$srcdir/../.."
SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles"
# testfiles source dirs for tools
SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES"
SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES"
SRC_H5DIFF_TESTFILES="$SRC_TOOLS/test/h5diff/testfiles"
SRC_H5COPY_TESTFILES="$SRC_TOOLS/test/h5copy/testfiles"
SRC_H5REPACK_TESTFILES="$SRC_TOOLS/test/h5repack/testfiles"
SRC_H5JAM_TESTFILES="$SRC_TOOLS/test/h5jam/testfiles"
SRC_H5STAT_TESTFILES="$SRC_TOOLS/test/h5stat/testfiles"
SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/test/h5import/testfiles"
SRC_H5FORMCONV_TESTFILES="$SRC_TOOLS/test/h5format_convert/testfiles"
TESTDIR=./testfiles
test -d $TESTDIR || mkdir $TESTDIR
# Copy the testfile to a temporary file for testing as h5format_convert is changing the file in place
TMPOUTFILE=outtmp.h5
TMPFILE=tmp.h5
TMPCHKFILE=chktmp.h5
TMPDMPFILE=dmptmp.h5
######################################################################
# test files
# --------------------------------------------------------------------
# All the test files copy from source directory to test directory
# NOTE: Keep this framework to add/remove test files.
# Any test files from other tools can be used in this framework.
# This list are also used for checking exist.
# Comment '#' without space can be used.
# --------------------------------------------------------------------
LIST_HDF5_TEST_FILES="
$SRC_H5FORMCONV_TESTFILES/h5fc_non_v3.h5
$SRC_H5FORMCONV_TESTFILES/h5fc_edge_v3.h5
$SRC_H5FORMCONV_TESTFILES/h5fc_ext_none.h5
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext_none.h5
$SRC_H5FORMCONV_TESTFILES/h5fc_ext1_i.h5
$SRC_H5FORMCONV_TESTFILES/h5fc_ext1_s.h5
$SRC_H5FORMCONV_TESTFILES/h5fc_ext1_f.h5
$SRC_H5FORMCONV_TESTFILES/h5fc_ext2_if.h5
$SRC_H5FORMCONV_TESTFILES/h5fc_ext2_is.h5
$SRC_H5FORMCONV_TESTFILES/h5fc_ext2_sf.h5
$SRC_H5FORMCONV_TESTFILES/h5fc_ext3_isf.h5
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext1_i.h5
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext1_s.h5
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext1_f.h5
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext2_if.h5
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext2_is.h5
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext2_sf.h5
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext3_isf.h5
$SRC_H5FORMCONV_TESTFILES/h5fc_err_level.h5
"
LIST_OTHER_TEST_FILES="
$SRC_H5FORMCONV_TESTFILES/h5fc_help.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_nooption.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_nonexistfile.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_d_file.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_dname.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_nonexistdset_file.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_v_non_chunked.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_v_bt1.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_v_ndata_bt1.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_v_all.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_v_n_1d.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_v_n_all.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_ext1_i.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_ext1_s.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_ext1_f.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_ext2_if.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_ext2_is.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_ext2_sf.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_ext3_isf.ddl
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext1_i.ddl
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext1_s.ddl
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext1_f.ddl
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext2_if.ddl
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext2_is.ddl
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext2_sf.ddl
$SRC_H5FORMCONV_TESTFILES/old_h5fc_ext3_isf.ddl
$SRC_H5FORMCONV_TESTFILES/h5fc_v_err.ddl
"
#
# copy test files and expected output files from source dirs to test dir
#
COPY_TESTFILES="$LIST_HDF5_TEST_FILES $LIST_OTHER_TEST_FILES"
COPY_TESTFILES_TO_TESTDIR()
{
# copy test files. Used -f to make sure get a new copy
for tstfile in $COPY_TESTFILES
do
# ignore '#' comment
echo $tstfile | tr -d ' ' | grep '^#' > /dev/null
RET=$?
if [ $RET -eq 1 ]; then
# skip cp if srcdir is same as destdir
# this occurs when build/test performed in source dir and
# make cp fail
SDIR=`$DIRNAME $tstfile`
INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'`
INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'`
if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then
$CP -f $tstfile $TESTDIR
if [ $? -ne 0 ]; then
echo "Error: FAILED to copy $tstfile ."
# Comment out this to CREATE expected file
exit $EXIT_FAILURE
fi
fi
fi
done
}
CLEAN_TESTFILES_AND_TESTDIR()
{
# skip rm if srcdir is same as destdir
# this occurs when build/test performed in source dir and
# make cp fail
SDIR=$SRC_H5FORMCONV_TESTFILES
INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'`
INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'`
if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then
$RM $TESTDIR
else
$RM $TESTDIR/$TMPFILE
$RM $TESTDIR/$TMPOUTFILE
$RM $TESTDIR/$TMPCHKFILE
$RM $TESTDIR/$TMPDMPFILE
fi
}
# Print a line-line message left justified in a field of 80 characters
# beginning with the word "Testing".
#
TESTING() {
SPACES=" "
echo "Testing $* $SPACES" | cut -c1-80 | tr -d '\012'
}
# Run a test and print PASS or *FAIL*. If a test fails then increment
# the `nerrors' global variable and (if $verbose is set) display the
# difference between the actual output and the expected output. The
# expected output is given as the first argument to this function and
# the actual output file is calculated by replacing the `.ddl' with
# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a
# non-zero value.
#
# $1: expected output
# $2: the test file name
# --fname might be empty or fname does not exist
# --fname is copied to a temporary file for testing
# $3 to at most $6--options to the tool such as:
# -d dname or --dname=dname
# -v or --verbose
# -n or --noop
TOOLTEST_OUT() {
# Prepare expected and actual output
expect="$TESTDIR/$1"
actual="$TESTDIR/`basename $1 .ddl`.out"
actual_err="$TESTDIR/`basename $1 .ddl`.err"
actual_sav=${actual}-sav
actual_err_sav=${actual_err}-sav
# Prepare the test file
$RM $TESTDIR/$TMPOUTFILE
TFILE=$2
if [ ! -z "$2" ] && [ -e $TESTDIR/$2 ] ; then
$CP $TESTDIR/$2 $TESTDIR/$TMPOUTFILE
TFILE=$TMPOUTFILE
fi
# Run test.
TESTING $FORMCONV $3 $4 $5 $6 $2
(
cd $TESTDIR
$RUNSERIAL $FORMCONV_BIN $3 $4 $5 $6 $TFILE
) >$actual 2>$actual_err
cp $actual $actual_sav
cp $actual_err $actual_err_sav
cat $actual_err >> $actual
# Compare output
COMPARE_OUT $expect $actual
# Clean up output file
if test -z "$HDF5_NOCLEANUP"; then
$RM $actual $actual_err
$RM $actual $actual_err $actual_sav $actual_err_sav
fi
}
# To check that the tool exits success, no output
# Assume all short options
# $1 is the test file name
# --fname exists
# --fname is copied to a temporary file for testing
# $2 is the temporary file name
# $3 to at most $5--options to the tool such as:
# -d dname
# -n
TOOLTEST() {
TESTING $FORMCONV $3 $4 $5 $1
$RM $TESTDIR/$2
$CP $TESTDIR/$1 $TESTDIR/$2
$RUNSERIAL $FORMCONV_BIN $3 $4 $5 $TESTDIR/$2
exitcode=$?
if [ $exitcode -ne 0 ]; then
echo "*FAILED*"
echo " The tool exits failure"
nerrors="`expr $nerrors + 1`"
else
echo " PASSED"
fi
}
CHECKING() {
SPACES=" "
echo "Verifying $* $SPACES" | cut -c1-80 | tr -d '\012'
}
# Assume $TESTDIR/$TMPFILE is the converted test file
# $1 dataset name
IDX_CHECK() {
CHECKING $1
$RUNSERIAL $CHK_IDX_BIN $TESTDIR/$TMPCHKFILE $1
ret=$?
if [ $ret -eq 0 ]; then
echo " PASSED"
else
echo "*FAILED*"
echo " The chunk indexing type is not correct"
nerrors="`expr $nerrors + 1`"
fi
}
# $1 is the expected output
# $2 is the output from testing
COMPARE_OUT() {
if $CMP $1 $2; then
echo " PASSED"
else
echo "*FAILED*"
echo " Expected result (*.ddl) differs from actual result (*.out)"
nerrors="`expr $nerrors + 1`"
test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /'
fi
}
# Assume $TESTDIR/$TMPFILE is the converted test file
# $1 is the test file for verifying h5dump output
# $2 is the expected output from h5dump
H5DUMP_CHECK() {
CHECKING h5dump output for $1
expect="$TESTDIR/$2"
actual="$TESTDIR/`basename $2 .ddl`.out"
actual_err="$TESTDIR/`basename $2 .ddl`.err"
$RUNSERIAL $H5DUMP_BIN -BH $TESTDIR/$TMPDMPFILE > $actual 2>$actual_err
cat $actual_err >> $actual
# Compare output
COMPARE_OUT $expect $actual
# Clean up output file
if test -z "$HDF5_NOCLEANUP"; then
$RM $actual $actual_err
fi
}
# Print a "SKIP" message
SKIP() {
TESTING $STAT $@
echo " -SKIP-"
}
##############################################################################
##############################################################################
### T H E T E S T S ###
##############################################################################
##############################################################################
# prepare for test
COPY_TESTFILES_TO_TESTDIR
#
#
#
# h5format_convert --help
# h5format_convert (no options)
# h5format_convert nonexist.h5 (no options, file does not exist)
TOOLTEST_OUT h5fc_help.ddl '' --help
TOOLTEST_OUT h5fc_nooption.ddl ''
TOOLTEST_OUT h5fc_nonexistfile.ddl nonexist.h5
#
#
# h5format_convert -d old_h5fc_ext_none.h5 (just -d option, file exists)
# h5format_convert --dname old_h5fc_ext_none.h5 (just --dname option, file exists)
# h5format_convert --dname (just --dname option)
# h5format_convert --dname=nonexist old_h5fc_ext_none.h5 (dataset does not exist, file exists)
TOOLTEST_OUT h5fc_d_file.ddl old_h5fc_ext_none.h5 -d
TOOLTEST_OUT h5fc_d_file.ddl old_h5fc_ext_none.h5 --dname
TOOLTEST_OUT h5fc_dname.ddl '' --dname
TOOLTEST_OUT h5fc_nonexistdset_file.ddl old_h5fc_ext_none.h5 --dname=nonexist
#
#
#
# h5format_convert -d /DSET_CONTIGUOUS -v old_h5fc_ext_none.h5 (verbose, contiguous dataset)
# h5format_convert -d /GROUP/DSET_BT2 --verbose old_h5fc_ext_none.h5 (verbose, bt1 dataset)
# h5format_convert -d /DSET_NDATA_BT2 -v -n old_h5fc_ext_none.h5 (verbose, noop, bt1+nodata dataset)
# h5format_convert -v old_h5fc_ext_none.h5 (verbose, all datasets)
TOOLTEST_OUT h5fc_v_non_chunked.ddl old_h5fc_ext_none.h5 -d /DSET_CONTIGUOUS -v
TOOLTEST_OUT h5fc_v_bt1.ddl old_h5fc_ext_none.h5 -d /GROUP/DSET_BT2 --verbose
TOOLTEST_OUT h5fc_v_ndata_bt1.ddl old_h5fc_ext_none.h5 -d /DSET_NDATA_BT2 -v -n
TOOLTEST_OUT h5fc_v_all.ddl old_h5fc_ext_none.h5 -v
#
#
#
# h5format_convert -d /DSET_EA -v -n h5fc_ext_none.h5 (verbose, noop, one ea dataset)
# h5format_convert -v -n h5fc_non_v3.h5 (verbose, noop, all datasets)
TOOLTEST_OUT h5fc_v_n_1d.ddl h5fc_ext_none.h5 -d /DSET_EA -v -n
TOOLTEST_OUT h5fc_v_n_all.ddl h5fc_non_v3.h5 -v -n
#
#
#
# h5format_convert -v h5fc_err_level.h5 (error encountered in converting the dataset)
TOOLTEST_OUT h5fc_v_err.ddl h5fc_err_level.h5 -v
#
#
#
# No output from tests
# 1) Use the tool to convert the dataset
# 2) Verify the chunk indexing type is correct
# h5format_convert -d /DSET_EA h5fc_ext_none.h5
# h5format_convert -d /GROUP/DSET_NDATA_EA h5fc_ext_none.h5
# h5format_convert -d /GROUP/DSET_BT2 h5fc_ext_none.h5
# h5format_convert -d /DSET_NDATA_BT2 h5fc_ext_none.h5
# h5format_convert -d /DSET_FA h5fc_ext_none.h5
# h5format_convert -d /GROUP/DSET_FA h5fc_ext_none.h5
# h5format_convert -d /DSET_NONE h5fc_ext_none.h5
# h5format_convert -d /GROUP/DSET_NDATA_NONE h5fc_ext_none.h5
TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /DSET_EA
IDX_CHECK /DSET_EA
#
TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /GROUP/DSET_NDATA_EA
IDX_CHECK /GROUP/DSET_NDATA_EA
#
TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /GROUP/DSET_BT2
IDX_CHECK /GROUP/DSET_BT2
#
TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /DSET_NDATA_BT2
IDX_CHECK /DSET_NDATA_BT2
#
TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /DSET_FA
IDX_CHECK /DSET_FA
#
TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /GROUP/DSET_NDATA_FA
IDX_CHECK /GROUP/DSET_NDATA_FA
#
TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /DSET_NONE
IDX_CHECK /DSET_NONE
#
TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /GROUP/DSET_NDATA_NONE
IDX_CHECK /GROUP/DSET_NDATA_NONE
#
#
#
# No output from tests: just check exit code
# h5format_convert -d /DSET_NDATA_BT2 old_h5fc_ext_none.h5 (v1-btree dataset)
# h5format_convert -d /DSET_CONTIGUOUS h5fc_non_v3.h5 (non-chunked dataset)
TOOLTEST old_h5fc_ext_none.h5 $TMPFILE -d /DSET_NDATA_BT2
TOOLTEST h5fc_non_v3.h5 $TMPFILE -d /DSET_CONTIGUOUS
#
#
#
# No output from tests: just check exit code
# h5format_convert -d /GROUP/DSET_BT2 -n h5fc_non_v3.h5 (noop, one dataset)
# h5format_convert -n h5fc_non_v3.h5 (noop, all datasets)
TOOLTEST h5fc_non_v3.h5 $TMPFILE -d /GROUP/DSET_BT2 -n
TOOLTEST h5fc_non_v3.h5 $TMPFILE -n
#
#
#
# No output from tests: just check exit code
# h5format_convert h5fc_non_v3.h5
# 1) convert all datasets
# 2) verify indexing types
TOOLTEST h5fc_non_v3.h5 $TMPCHKFILE
IDX_CHECK /DSET_NDATA_EA
IDX_CHECK /DSET_NDATA_BT2
IDX_CHECK /GROUP/DSET_BT2
IDX_CHECK /GROUP/DSET_EA
#
#
#
# No output from test: just check exit code
# h5format_convert h5fc_edge_v3.h5
# 1) convert the chunked dataset (filter, no-filter-edge-chunk)
# 2) verify the indexing type
TOOLTEST h5fc_edge_v3.h5 $TMPCHKFILE
IDX_CHECK /DSET_EDGE
#
#
# The following test files have messages in the superblock extension.
# Verify h5dump output for correctness after conversion
TOOLTEST h5fc_ext1_i.h5 $TMPDMPFILE
H5DUMP_CHECK h5fc_ext1_i.h5 h5fc_ext1_i.ddl
TOOLTEST h5fc_ext1_s.h5 $TMPDMPFILE
H5DUMP_CHECK h5fc_ext1_s.h5 h5fc_ext1_s.ddl
TOOLTEST h5fc_ext1_f.h5 $TMPDMPFILE
H5DUMP_CHECK h5fc_ext1_f.h5 h5fc_ext1_f.ddl
#
TOOLTEST h5fc_ext2_if.h5 $TMPDMPFILE
H5DUMP_CHECK h5fc_ext2_if.h5 h5fc_ext2_if.ddl
TOOLTEST h5fc_ext2_is.h5 $TMPDMPFILE
H5DUMP_CHECK h5fc_ext2_is.h5 h5fc_ext2_is.ddl
TOOLTEST h5fc_ext2_sf.h5 $TMPDMPFILE
H5DUMP_CHECK h5fc_ext2_sf.h5 h5fc_ext2_sf.ddl
#
TOOLTEST h5fc_ext3_isf.h5 $TMPDMPFILE
H5DUMP_CHECK h5fc_ext3_isf.h5 h5fc_ext3_isf.ddl
#
#
#
TOOLTEST old_h5fc_ext1_i.h5 $TMPDMPFILE
H5DUMP_CHECK old_h5fc_ext1_i.h5 old_h5fc_ext1_i.ddl
TOOLTEST old_h5fc_ext1_s.h5 $TMPDMPFILE
H5DUMP_CHECK old_h5fc_ext1_s.h5 old_h5fc_ext1_s.ddl
TOOLTEST old_h5fc_ext1_f.h5 $TMPDMPFILE
H5DUMP_CHECK old_h5fc_ext1_f.h5 old_h5fc_ext1_f.ddl
#
TOOLTEST old_h5fc_ext2_if.h5 $TMPDMPFILE
H5DUMP_CHECK old_h5fc_ext2_if.h5 old_h5fc_ext2_if.ddl
TOOLTEST old_h5fc_ext2_is.h5 $TMPDMPFILE
H5DUMP_CHECK old_h5fc_ext2_is.h5 old_h5fc_ext2_is.ddl
TOOLTEST old_h5fc_ext2_sf.h5 $TMPDMPFILE
H5DUMP_CHECK old_h5fc_ext2_sf.h5 old_h5fc_ext2_sf.ddl
#
TOOLTEST old_h5fc_ext3_isf.h5 $TMPDMPFILE
H5DUMP_CHECK old_h5fc_ext3_isf.h5 old_h5fc_ext3_isf.ddl
#
# Clean up temporary files/directories
CLEAN_TESTFILES_AND_TESTDIR
if test $nerrors -eq 0 ; then
echo "All $TESTNAME tests passed."
exit $EXIT_SUCCESS
else
echo "$TESTNAME tests failed with $nerrors errors."
exit $EXIT_FAILURE
fi