H5repack tests should fail if a corrupted file causes h5repack to (#1138)

* H5repack tests should fail if a corrupted file causes h5repack to
segfault/core dump.

* Add release note for HDFV-10590, CVE-2018-17432.

release_docs/RELEASE.txt

@@ -1132,6 +1132,17 @@ Bug Fixes since HDF5-1.12.0 release
 
         (NAF - 2021/01/22)
 
+    - Fixed CVE-2018-17432
+
+        The tool h5repack produced a segfault on a corrupted file which had
+        invalid rank for scalar or NULL datatype.
+
+        The problem was fixed by modifying the dataspace encode and decode
+        functions to detect and report invalid rank. h5repack now fails
+        with an error message for the corrupted file.
+
+        (BMR - 2020/10/26, HDFFV-10590)
+
     - Creation of dataset with optional filter
 
         When the combination of type, space, etc doesn't work for filter

tools/test/h5repack/CMakeTests.cmake

@@ -1544,7 +1544,7 @@
 # the references in attribute of compund or vlen datatype
   ADD_H5_TEST (HDFFV-5932 "TEST" ${FILE_ATTR_REF})
 
-# Add test for memory leak in attirbute. This test is verified by CTEST.
+# Add test for memory leak in attribute. This test is verified by CTEST.
 # 1. leak from vlen string
 # 2. leak from compound type without reference member
 # (HDFFV-7840, )
@@ -1552,12 +1552,12 @@
   ADD_H5_TEST (HDFFV-7840 "TEST" h5diff_attr1.h5)
 
 # test CVE-2018-17432 fix
-  set (arg h5repack_CVE-2018-17432.h5 h5repack__CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6)
+  set (arg h5repack_CVE-2018-17432.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6)
   set (TESTTYPE "TEST")
   ADD_H5_FILTER_TEST (HDFFV-10590 "" ${TESTTYPE} 1 ${arg})
 
 # test CVE-2018-14460 fix
-  set (arg h5repack_CVE-2018-14460.h5 h5repack_CVE-2018-14460_out.h5)
+  set (arg h5repack_CVE-2018-14460.h5)
   set (TESTTYPE "TEST")
   ADD_H5_FILTER_TEST (HDFFV-11223 "" ${TESTTYPE} 1 ${arg})
 

tools/test/h5repack/h5repack.sh.in

@@ -885,13 +885,24 @@ TOOLTEST_FAIL()
     (
         cd $TESTDIR
         $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
-    ) >$actual
+    ) >&$actual
     RET=$?
-    if [ $RET == 0 ] ; then
+
+    # Normally h5repack of files tested with this function are expected
+    # to return not 0, but if the command results in "Segmentation fault"
+    # or "core dumped" it is a failure regardless of the return value.
+    failure=`grep -e 'Segmentation fault' -e 'core dumped' $actual`
+    if [ "$failure" != "" ]; then
         nerrors="`expr $nerrors + 1`"
         echo " FAILED"
+        echo "    $failure"
     else
-        echo " PASSED"
+        if [ $RET == 0 ] ; then
+            nerrors="`expr $nerrors + 1`"
+            echo " FAILED"
+        else
+            echo " PASSED"
+        fi
     fi
     rm -f $outfile
 }