Merge pull request #620 in HDFFV/hdf5 from ~VCHOI/my_hdf5_fork:develop to develop

* commit '59e94f5009a1f738d9924be942448f8cf6355b00':
  Add h5repack tests for paged aggregation Add tests to h5repack.sh.in to verify options added for paged aggregation work as expected.

MANIFEST

@@ -2411,6 +2411,7 @@
 
 #test files for h5repack
 ./tools/test/h5repack/testfiles/README
+./tools/test/h5repack/testfiles/h5repack_aggr.h5
 ./tools/test/h5repack/testfiles/h5repack_attr.h5
 ./tools/test/h5repack/testfiles/h5repack_attr_refs.h5
 ./tools/test/h5repack/testfiles/h5repack_deflate.h5
@@ -2420,6 +2421,8 @@
 ./tools/test/h5repack/testfiles/h5repack_fill.h5
 ./tools/test/h5repack/testfiles/h5repack_filters.h5
 ./tools/test/h5repack/testfiles/h5repack_fletcher.h5
+./tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5
+./tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5
 ./tools/test/h5repack/testfiles/h5repack_hlink.h5
 ./tools/test/h5repack/testfiles/h5repack.info
 ./tools/test/h5repack/testfiles/h5repack_layout.h5
@@ -2431,7 +2434,10 @@
 ./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum_deflated.h5
 ./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum.h5
 ./tools/test/h5repack/testfiles/h5repack_nbit.h5
+./tools/test/h5repack/testfiles/h5repack_none.h5
 ./tools/test/h5repack/testfiles/h5repack_objs.h5
+./tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5
+./tools/test/h5repack/testfiles/h5repack_paged_persist.h5
 ./tools/test/h5repack/testfiles/h5repack_refs.h5
 ./tools/test/h5repack/testfiles/h5repack_shuffle.h5
 ./tools/test/h5repack/testfiles/h5repack_soffset.h5
@@ -2450,6 +2456,12 @@
 ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl
 ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl
 ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.tst
+./tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl
+./tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl
+./tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl
+./tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl
+./tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl
+./tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl
 ./tools/test/h5repack/testfiles/1_vds.h5-vds_dset_chunk20x10x5-v.ddl
 ./tools/test/h5repack/testfiles/2_vds.h5-vds_chunk3x6x9-v.ddl
 ./tools/test/h5repack/testfiles/3_1_vds.h5-vds_chunk2x5x8-v.ddl

tools/test/h5repack/h5repack.sh.in

@@ -36,6 +36,9 @@ H5DIFF_BIN=`pwd`/$H5DIFF                # The path of the h5diff tool binary
 H5DUMP=../../src/h5dump/h5dump          # The h5dump tool name
 H5DUMP_BIN=`pwd`/$H5DUMP                # The path of the h5dump tool binary
 
+H5STAT=../../src/h5stat/h5stat          # The h5stat tool name
+H5STAT_BIN=`pwd`/$H5STAT                # The path of the h5stat tool binary
+
 RM='rm -rf'
 CMP='cmp'
 DIFF='diff -c'
@@ -101,6 +104,12 @@ $SRC_H5REPACK_TESTFILES/h5repack_refs.h5
 $SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5
 $SRC_H5REPACK_TESTFILES/h5repack_soffset.h5
 $SRC_H5REPACK_TESTFILES/h5repack_szip.h5
+$SRC_H5REPACK_TESTFILES/h5repack_aggr.h5
+$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_nopersist.h5
+$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5
+$SRC_H5REPACK_TESTFILES/h5repack_none.h5
+$SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5
+$SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5
 $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
 $SRC_TOOLS_TESTFILES/tfamily00000.h5
 $SRC_TOOLS_TESTFILES/tfamily00001.h5
@@ -156,6 +165,12 @@ $SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl
 $SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl
 $SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl
 $SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl
+$SRC_H5REPACK_TESTFILES/SP.h5repack_fsm_aggr_nopersist.h5.ddl
+$SRC_H5REPACK_TESTFILES/S.h5repack_fsm_aggr_persist.h5.ddl
+$SRC_H5REPACK_TESTFILES/STG.h5repack_none.h5.ddl
+$SRC_H5REPACK_TESTFILES/GS.h5repack_paged_nopersist.h5.ddl
+$SRC_H5REPACK_TESTFILES/SP.h5repack_paged_persist.h5.ddl
+$SRC_H5REPACK_TESTFILES/SPT.h5repack_aggr.h5.ddl
 "
 
 #
@@ -707,6 +722,58 @@ TOOLTEST_DUMP()
    rm -f $outfile
 }
 
+# This is similar to TOOLTEST_DUMP().
+# Test h5repack with options added for paged aggregation.
+# h5stat is used on the repacked file and the expected output
+# is compared for correctness.
+#
+TOOLTEST_STAT()
+{
+    infile=$2
+    outfile=out-$1.$2
+    expect="$TESTDIR/$1.$2.ddl"
+    actual="$TESTDIR/out-$1.$2.out"
+    actual_err="$TESTDIR/out-$1.$2.err"
+
+    shift
+    shift
+
+    # Run test.
+    TESTING $H5REPACK $@
+    (
+        cd $TESTDIR
+        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
+    ) >$actual 2>$actual_err
+    RET=$?
+    if [ $RET != 0 ] ; then
+    echo "*FAILED*"
+    nerrors="`expr $nerrors + 1`"
+    else
+     echo " PASSED"
+     VERIFY h5stat output $@
+     (
+        cd $TESTDIR
+        $RUNSERIAL $H5STAT_BIN -S -s $outfile
+     ) >$actual 2>$actual_err
+     cat $actual_err >> $actual
+
+     RET=$?
+
+    fi
+
+    if cmp -s $expect $actual; then
+     echo " PASSED"
+    else
+     echo "*FAILED*"
+     echo "    Expected result (*.ddl) differs from actual result (*.out)"
+     nerrors="`expr $nerrors + 1`"
+     test yes = "$verbose" && diff -c $expect $actual |sed 's/^/    /'
+    fi
+
+   rm -f $actual $actual_err
+   rm -f $outfile
+}
+
 # TOOLTEST_META:
 # Test metadata block size option.
 # Reason to create a function here is to localize all special steps related to
@@ -1057,6 +1124,42 @@ fi
 arg="tordergr.h5 -L"
 TOOLTEST_DUMP crtorder $arg
 
+###################################################################################################
+# Testing paged aggregation related options:
+#   -G pagesize
+#   -P 1 or 0
+#   -S strategy
+#   -T threshold
+#
+# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_":
+#   (1) "fsm_aggr_nopersist.h5"  /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */
+#   (2) "fsm_aggr_persist.h5"    /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */
+#   (3) "paged_nopersist.h5"     /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */
+#   (4) "paged_persist.h5"       /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */
+#   (5) "aggr.h5"                /* H5F_FSPACE_STRATEGY_AGGR */
+#   (6) "none.h5"                /* H5F_FSPACE_STRATEGY_NONE */
+#
+#####################################################################################################
+#
+arg="h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1"
+TOOLTEST_STAT SP $arg
+#
+arg="h5repack_fsm_aggr_persist.h5 -S AGGR"
+TOOLTEST_STAT S $arg
+#
+arg="h5repack_none.h5 -S PAGE -T 10 -G 2048"
+TOOLTEST_STAT STG $arg
+#
+arg="h5repack_paged_nopersist.h5 -G 512 -S AGGR"
+TOOLTEST_STAT GS $arg
+#
+arg="h5repack_paged_persist.h5 -S NONE -P 1"
+TOOLTEST_STAT SP $arg
+#
+arg="h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5"
+TOOLTEST_STAT SPT $arg
+
+
 #########################################################
 # layout options (these files have no filters)
 #########################################################

tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl

@@ -0,0 +1,15 @@
+Filename: out-GS.h5repack_paged_nopersist.h5
+Free-space persist: FALSE
+Free-space section threshold: 1 bytes
+Small size free-space sections (< 10 bytes):
+	Total # of small size sections: 0
+Free-space section bins:
+	Total # of sections: 0
+File space management strategy: H5F_FSPACE_STRATEGY_AGGR
+File space page size: 512 bytes
+Summary of file space information:
+  File metadata: 1272 bytes
+  Raw data: 400 bytes
+  Amount/Percent of tracked free space: 0 bytes/0.0%
+  Unaccounted space: 776 bytes
+Total space: 2448 bytes

tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl

@@ -0,0 +1,15 @@
+Filename: out-S.h5repack_fsm_aggr_persist.h5
+Free-space persist: FALSE
+Free-space section threshold: 1 bytes
+Small size free-space sections (< 10 bytes):
+	Total # of small size sections: 0
+Free-space section bins:
+	Total # of sections: 0
+File space management strategy: H5F_FSPACE_STRATEGY_AGGR
+File space page size: 4096 bytes
+Summary of file space information:
+  File metadata: 1272 bytes
+  Raw data: 400 bytes
+  Amount/Percent of tracked free space: 0 bytes/0.0%
+  Unaccounted space: 776 bytes
+Total space: 2448 bytes

tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl

@@ -0,0 +1,16 @@
+Filename: out-SP.h5repack_fsm_aggr_nopersist.h5
+Free-space persist: TRUE
+Free-space section threshold: 1 bytes
+Small size free-space sections (< 10 bytes):
+	Total # of small size sections: 0
+Free-space section bins:
+	# of sections of size 1000 - 9999: 2
+	Total # of sections: 2
+File space management strategy: H5F_FSPACE_STRATEGY_PAGE
+File space page size: 4096 bytes
+Summary of file space information:
+  File metadata: 1602 bytes
+  Raw data: 400 bytes
+  Amount/Percent of tracked free space: 6307 bytes/38.5%
+  Unaccounted space: 8075 bytes
+Total space: 16384 bytes

tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl

@@ -0,0 +1,15 @@
+Filename: out-SP.h5repack_paged_persist.h5
+Free-space persist: FALSE
+Free-space section threshold: 1 bytes
+Small size free-space sections (< 10 bytes):
+	Total # of small size sections: 0
+Free-space section bins:
+	Total # of sections: 0
+File space management strategy: H5F_FSPACE_STRATEGY_NONE
+File space page size: 4096 bytes
+Summary of file space information:
+  File metadata: 1272 bytes
+  Raw data: 400 bytes
+  Amount/Percent of tracked free space: 0 bytes/0.0%
+  Unaccounted space: 0 bytes
+Total space: 1672 bytes

tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl

@@ -0,0 +1,16 @@
+Filename: out-SPT.h5repack_aggr.h5
+Free-space persist: TRUE
+Free-space section threshold: 5 bytes
+Small size free-space sections (< 10 bytes):
+	Total # of small size sections: 0
+Free-space section bins:
+	# of sections of size 100 - 999: 1
+	Total # of sections: 1
+File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR
+File space page size: 4096 bytes
+Summary of file space information:
+  File metadata: 1485 bytes
+  Raw data: 400 bytes
+  Amount/Percent of tracked free space: 680 bytes/26.5%
+  Unaccounted space: 0 bytes
+Total space: 2565 bytes

tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl

@@ -0,0 +1,15 @@
+Filename: out-STG.h5repack_none.h5
+Free-space persist: FALSE
+Free-space section threshold: 10 bytes
+Small size free-space sections (< 10 bytes):
+	Total # of small size sections: 0
+Free-space section bins:
+	Total # of sections: 0
+File space management strategy: H5F_FSPACE_STRATEGY_PAGE
+File space page size: 2048 bytes
+Summary of file space information:
+  File metadata: 1272 bytes
+  Raw data: 400 bytes
+  Amount/Percent of tracked free space: 0 bytes/0.0%
+  Unaccounted space: 2424 bytes
+Total space: 4096 bytes