mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-02-11 16:01:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://bitbucket.hdfgroup.org/scm/hdffv/hdf5 into develop

Browse Source
This commit is contained in:
Jordan Henderson 2017-08-11 14:25:33 -05:00
commit c34ee0b2a5
92 changed files with 5766 additions and 1439 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
MANIFEST
View File

@ -799,8 +799,11 @@
./src/H5PBpkg.h
./src/H5PBprivate.h
./src/H5PL.c
./src/H5PLint.c
./src/H5PLmodule.h
./src/H5PLpath.c
./src/H5PLpkg.h
./src/H5PLplugin_cache.c
./src/H5PLprivate.h
./src/H5PLpublic.h
./src/H5PLextern.h
@ -1222,6 +1225,7 @@
./testpar/COPYING
./testpar/Makefile.am
./testpar/t_bigio.c
./testpar/t_cache.c
./testpar/t_cache_image.c
./testpar/t_chunk_alloc.c
@ -2090,6 +2094,8 @@
./tools/testfiles/tdset2.h5.xml
./tools/testfiles/tempty.h5.xml
./tools/testfiles/tenum.h5.xml
./tools/testfiles/test35.nc
./tools/testfiles/test35.nc.xml
./tools/testfiles/tfpformat.h5.xml
./tools/testfiles/tgroup.h5.xml
./tools/testfiles/thlink.h5.xml
@ -2399,6 +2405,7 @@
./tools/test/h5diff/testfiles/h5diff_udfail.txt
./tools/test/h5diff/testfiles/diff_strings1.h5
./tools/test/h5diff/testfiles/diff_strings2.h5
./tools/test/h5diff/testfiles/h5diff_vlstr.txt
#vds
./tools/test/h5diff/testfiles/h5diff_v1.txt
./tools/test/h5diff/testfiles/h5diff_v2.txt
@ -2406,6 +2413,7 @@
#test files for h5repack
./tools/test/h5repack/testfiles/README
./tools/test/h5repack/testfiles/h5repack_aggr.h5
./tools/test/h5repack/testfiles/h5repack_attr.h5
./tools/test/h5repack/testfiles/h5repack_attr_refs.h5
./tools/test/h5repack/testfiles/h5repack_deflate.h5
@ -2415,6 +2423,8 @@
./tools/test/h5repack/testfiles/h5repack_fill.h5
./tools/test/h5repack/testfiles/h5repack_filters.h5
./tools/test/h5repack/testfiles/h5repack_fletcher.h5
./tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5
./tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5
./tools/test/h5repack/testfiles/h5repack_hlink.h5
./tools/test/h5repack/testfiles/h5repack.info
./tools/test/h5repack/testfiles/h5repack_layout.h5
@ -2426,7 +2436,10 @@
./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum_deflated.h5
./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum.h5
./tools/test/h5repack/testfiles/h5repack_nbit.h5
./tools/test/h5repack/testfiles/h5repack_none.h5
./tools/test/h5repack/testfiles/h5repack_objs.h5
./tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5
./tools/test/h5repack/testfiles/h5repack_paged_persist.h5
./tools/test/h5repack/testfiles/h5repack_refs.h5
./tools/test/h5repack/testfiles/h5repack_shuffle.h5
./tools/test/h5repack/testfiles/h5repack_soffset.h5
@ -2445,6 +2458,12 @@
./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl
./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl
./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.tst
./tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl
./tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl
./tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl
./tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl
./tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl
./tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl
./tools/test/h5repack/testfiles/1_vds.h5-vds_dset_chunk20x10x5-v.ddl
./tools/test/h5repack/testfiles/2_vds.h5-vds_chunk3x6x9-v.ddl
./tools/test/h5repack/testfiles/3_1_vds.h5-vds_chunk2x5x8-v.ddl

4
bin/trace
View File

@ -76,7 +76,7 @@ $Source = "";
"off_t" => "o",
"H5O_type_t" => "Ot",
"H5P_class_t" => "p",
"hobj_ref_t" => "r",
"hobj_ref_t" => "r",
"H5R_type_t" => "Rt",
"char" => "s",
"unsigned char" => "s",
@ -124,7 +124,7 @@ $Source = "";
"H5G_iterate_t" => "x",
"H5G_info_t" => "x",
"H5I_free_t" => "x",
"H5I_search_func_t" => "x",
"H5I_search_func_t" => "x",
"H5L_class_t" => "x",
"H5L_elink_traverse_t" => "x",
"H5L_iterate_t" => "x",

10
config/cmake_ext_mod/HDFLibMacros.cmake
View File

@ -74,14 +74,14 @@ macro (EXTERNAL_JPEG_LIBRARY compress_type jpeg_pic)
HDF_IMPORT_SET_LIB_OPTIONS (jpeg-static "jpeg" STATIC "")
add_dependencies (JPEG jpeg-static)
set (JPEG_STATIC_LIBRARY "jpeg-static")
set (JPEG_LIBRARIES ${JPEG_static_LIBRARY})
set (JPEG_LIBRARIES ${JPEG_STATIC_LIBRARY})
if (BUILD_SHARED_LIBS)
# Create imported target jpeg-shared
add_library(jpeg-shared SHARED IMPORTED)
HDF_IMPORT_SET_LIB_OPTIONS (jpeg-shared "jpeg" SHARED "")
add_dependencies (JPEG jpeg-shared)
set (JPEG_SHARED_LIBRARY "jpeg-shared")
set (JPEG_LIBRARIES ${JPEG_LIBRARIES} ${JPEG_shared_LIBRARY})
set (JPEG_LIBRARIES ${JPEG_LIBRARIES} ${JPEG_SHARED_LIBRARY})
endif ()
set (JPEG_INCLUDE_DIR_GEN "${BINARY_DIR}")
@ -167,14 +167,14 @@ macro (EXTERNAL_SZIP_LIBRARY compress_type encoding)
HDF_IMPORT_SET_LIB_OPTIONS (szip-static "szip" STATIC "")
add_dependencies (SZIP szip-static)
set (SZIP_STATIC_LIBRARY "szip-static")
set (SZIP_LIBRARIES ${SZIP_static_LIBRARY})
set (SZIP_LIBRARIES ${SZIP_STATIC_LIBRARY})
if (BUILD_SHARED_LIBS)
# Create imported target szip-shared
add_library(szip-shared SHARED IMPORTED)
HDF_IMPORT_SET_LIB_OPTIONS (szip-shared "szip" SHARED "")
add_dependencies (SZIP szip-shared)
set (SZIP_SHARED_LIBRARY "szip-shared")
set (SZIP_LIBRARIES ${SZIP_LIBRARIES} ${SZIP_shared_LIBRARY})
set (SZIP_LIBRARIES ${SZIP_LIBRARIES} ${SZIP_SHARED_LIBRARY})
endif ()
set (SZIP_INCLUDE_DIR_GEN "${BINARY_DIR}")
@ -262,7 +262,7 @@ macro (EXTERNAL_ZLIB_LIBRARY compress_type)
HDF_IMPORT_SET_LIB_OPTIONS (zlib-static ${ZLIB_LIB_NAME} STATIC "")
add_dependencies (ZLIB zlib-static)
set (ZLIB_STATIC_LIBRARY "zlib-static")
set (ZLIB_LIBRARIES ${ZLIB_static_LIBRARY})
set (ZLIB_LIBRARIES ${ZLIB_STATIC_LIBRARY})
if (BUILD_SHARED_LIBS)
# Create imported target zlib-shared
add_library(zlib-shared SHARED IMPORTED)

63
config/linux-gnulibc1
View File

@ -84,7 +84,20 @@ else
$RM $tmpfile
fc_version_info=`$FC -V | grep Absoft`
;;
# The NAG compiler
nagfor*|nagftn*)
RM='rm -f'
tmpfile=/tmp/cmpver.$$
$FC -V >& $tmpfile
if test -s "$tmpfile"; then
if( grep -s 'NAG Fortran' $tmpfile > /dev/null) then
FC_BASENAME=nagfor
fi
fi
fc_version_info=`grep "NAG Fortran" $tmpfile`
echo "compiler '$FC' is $fc_version_info"
$RM $tmpfile
;;
*)
;;
esac
@ -108,10 +121,10 @@ case $FC_BASENAME in
#
f95)
# Set required flag for compiling C stubs
H5_CFLAGS="$H5_CFLAGS"
H5_CFLAGS="$H5_CFLAGS"
F9XSUFFIXFLAG=""
# We force compiler to use upper case for external names
# We force compiler to use upper case for external names
# (just in case since this should be a default EIP)
H5_FCFLAGS="$H5_FCFLAGS"
FSEARCH_DIRS=""
@ -134,6 +147,37 @@ case $FC_BASENAME in
DEBUG_OPT_FCFLAGS=
NO_OPT_FCFLAGS=
f9x_flags_set=yes
;;
#
# NAG compiler
#
nagfor)
F9XSUFFIXFLAG=""
# We force compiler to use upper case for external names
# (just in case since this should be a default EIP)
H5_FCFLAGS="$H5_FCFLAGS"
FSEARCH_DIRS=""
# Production
PROD_FCFLAGS=
# Debug
DEBUG_FCFLAGS="-C"
# Symbols
SYMBOLS_FCFLAGS="-g"
NO_SYMBOLS_FCFLAGS="-s"
# Profiling
PROFILE_FCFLAGS="-pg"
# Optimization
HIGH_OPT_FCFLAGS="-O"
DEBUG_OPT_FCFLAGS="-O0"
NO_OPT_FCFLAGS="-O0"
f9x_flags_set=yes
;;
@ -230,6 +274,19 @@ case $FC in
*pgf90*)
fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'pgf90'`
;;
*nagfor*|*nagftn*)
RM='rm -f'
tmpfile=/tmp/cmpver.$$
$FC -V >& $tmpfile
if test -s "$tmpfile"; then
if( grep -s 'NAG Fortran' $tmpfile > /dev/null) then
FC_BASENAME=nagfor
fi
fi
fc_version_info=`grep "NAG Fortran" $tmpfile`
$RM $tmpfile
echo "compiler '$FC' is $fc_version_info"
;;
*)
echo "No match to get fc_version_info for $FC"

39
configure.ac
View File

@ -3338,6 +3338,27 @@ else
STATIC_SHARED="none"
fi
if test "X$HDF_FORTRAN" = "Xyes"; then
### libtool does not pass the correct argument linking (-Wl,-Wl,,) for the NAG Fortran compiler
### on Linux (other OSs have not been tested).
### Therefore, detect if we are using the NAG Fortran compiler, and replace the wl="-Wl," for Fortran to
### wl="-Wl,-Wl,," in the libtool file. (HDFFV-10037)
case "`uname`" in
Linux*)
fortran_linux_linker_option="-Wl,"
if test "X$FC_BASENAME" = "Xnagfor"; then
fortran_linux_linker_option="-Wl,-Wl,,"
fi
## Set the correct linker option for use in h5fc.in markup
AC_SUBST([fortran_linux_linker_option])
;;
esac
fi
## ----------------------------------------------------------------------
## Set a macro if shared library is enabled.
##
@ -3473,6 +3494,10 @@ AC_CONFIG_COMMANDS([.classes], [], [$MKDIR_P java/src/.classes;
AC_OUTPUT
chmod 755 tools/src/misc/h5cc
if test "X$HDF_CXX" = "Xyes"; then
chmod 755 c++/src/h5c++
fi
if test "X$HDF_FORTRAN" = "Xyes"; then
chmod 755 fortran/src/h5fc
@ -3484,10 +3509,18 @@ if test "X$HDF_FORTRAN" = "Xyes"; then
cat libtool | awk '/wl=\"/{c++;if(c==3){sub("wl=\"\"","wl=\"-Wl,\"");c=0}}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool
;;
esac
fi
if test "X$HDF_CXX" = "Xyes"; then
chmod 755 c++/src/h5c++
### libtool does not pass the correct argument linking (-Wl,-Wl,,) for the NAG Fortran compiler
### on Linux (other OSs have not been tested).
### Therefore, detect if we are using the NAG Fortran compiler, and replace the wl="-Wl," for Fortran to
### wl="-Wl,-Wl,," in the libtool file. (HDFFV-10037)
case "`uname`" in
Linux*)
if test "X$FC_BASENAME" = "Xnagfor"; then
cat libtool | awk '/NAG_Fortran/{flag=1}flag&&/wl=/{$NF="wl=\"-Wl,Wl,,\"";flag=0}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool
fi
;;
esac
fi
## HDF5 configure code created by autotools with gcc 4.9.2 is adding problematic

4
fortran/examples/Makefile.am
View File

@ -66,10 +66,10 @@ h5_crtatt.chkexe_: h5_rdwt.chkexe_
# Additional dependencies for the examples are listed below
if BUILD_PARALLEL_CONDITIONAL
$(EXTRA_PROG): $(H5FC_PP)
$(H5FC_PP) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90;
$(H5FC_PP) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90;
else
$(EXTRA_PROG): $(H5FC)
$(H5FC) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90;
$(H5FC) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90;
endif
# Tell automake how to install examples

2
fortran/src/h5fc.in
View File

@ -307,7 +307,7 @@ if test "x$do_link" = "xyes"; then
link_args="$link_args -L${libdir}"
case "$host_os" in
linux*) flag="-Wl,-rpath -Wl," ;;
linux*) flag="@fortran_linux_linker_option@-rpath -Wl," ;;
hpux*) flag="-Wl,+b -Wl," ;;
freebsd*|solaris*) flag="-R" ;;
rs6000*|aix*) flag="-L" ;;

4
hl/fortran/examples/Makefile.am
View File

@ -43,10 +43,10 @@ FORTRAN_API=yes
# Tell automake how to build examples using h5fc
if BUILD_PARALLEL_CONDITIONAL
$(EXTRA_PROG): $(H5FC_PP)
$(H5FC_PP) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90;
$(H5FC_PP) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90;
else
$(EXTRA_PROG): $(H5FC)
$(H5FC) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90;
$(H5FC) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90;
endif
# Tell automake how to install examples

11
hl/tools/h5watch/CMakeTests.cmake
View File

@ -56,18 +56,11 @@ set (H5WATCH_TEST_FILES
# make test dir
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles")
add_custom_target(h5watch-files ALL COMMENT "Copying files needed by h5watch tests")
foreach (h5watch_file ${H5WATCH_TEST_FILES})
set (dest "${PROJECT_BINARY_DIR}/testfiles/${h5watch_file}")
#message (STATUS " Copying ${h5watch_file}")
add_custom_command (
TARGET h5watch-files
POST_BUILD
COMMAND ${CMAKE_COMMAND}
ARGS -E copy_if_different ${HDF5_HL_TOOLS_DIR}/testfiles/${h5watch_file} ${dest}
)
HDFTEST_COPY_FILE("${HDF5_HL_TOOLS_DIR}/testfiles/${h5watch_file}" "${PROJECT_BINARY_DIR}/testfiles/${h5watch_file}" "H5WATCH_files")
endforeach ()
add_custom_target(H5WATCH_files ALL COMMENT "Copying files needed by H5WATCH tests" DEPENDS ${H5WATCH_files_list})
##############################################################################
##############################################################################

1
java/src/jni/h5Constants.c
View File

@ -26,6 +26,7 @@ extern "C" {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmissing-prototypes"
#pragma GCC diagnostic ignored "-Wunused-parameter"
JNIEXPORT jlong JNICALL
Java_hdf_hdf5lib_HDF5Constants_H5_1QUARTER_1HADDR_1MAX(JNIEnv *env, jclass cls) { return (hsize_t)HADDR_MAX/4; }

88
java/test/TestH5PL.java
View File

@ -70,26 +70,74 @@ public class TestH5PL {
@Test
public void TestH5PLpaths() {
try {
int original_entries = H5.H5PLsize();
H5.H5PLappend("path_one");
int plugin_entries = H5.H5PLsize();
assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+1) == plugin_entries);
H5.H5PLprepend("path_two");
plugin_entries = H5.H5PLsize();
assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+2) == plugin_entries);
H5.H5PLinsert("path_three", original_entries);
plugin_entries = H5.H5PLsize();
assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+3) == plugin_entries);
String first_path = H5.H5PLget(original_entries);
assertTrue("First path was : "+first_path + " ",first_path.compareToIgnoreCase("path_three")==0);
H5.H5PLreplace("path_four", original_entries);
first_path = H5.H5PLget(original_entries);
assertTrue("First path changed to : "+first_path + " ",first_path.compareToIgnoreCase("path_four")==0);
H5.H5PLremove(original_entries);
first_path = H5.H5PLget(original_entries);
assertTrue("First path now : "+first_path + " ",first_path.compareToIgnoreCase("path_two")==0);
plugin_entries = H5.H5PLsize();
assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+2) == plugin_entries);
// Get the original number of paths
int nStartPaths = H5.H5PLsize();
int nPaths; /* # paths from H5PLSize() */
int nTruePaths = nStartPaths; /* What the # paths should be */
int index; /* Path table index */
String path; /* Path from H5PLget() */
// APPEND a path and ensure it was added correctly
String pathAppend = "path_append";
H5.H5PLappend(pathAppend);
nPaths = H5.H5PLsize();
nTruePaths++;
assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths);
index = nTruePaths - 1;
path = H5.H5PLget(index);
assertTrue("Path should be " + pathAppend + " but was " + path, path.compareToIgnoreCase(pathAppend) == 0);
// PREPEND a path and ensure it was added correctly
String pathPrepend = "path_prepend";
H5.H5PLprepend(pathPrepend);
nPaths = H5.H5PLsize();
nTruePaths++;
assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths);
index = 0;
path = H5.H5PLget(index);
assertTrue("Path should be " + pathPrepend + " but was " + path, path.compareToIgnoreCase(pathPrepend) == 0);
// INSERT a path and ensure it was added correctly
// Inserting at the index == # of start paths ensures we're in the middle
String pathInsert = "path_insert";
index = nStartPaths;
H5.H5PLinsert(pathInsert, index);
nPaths = H5.H5PLsize();
nTruePaths++;
assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths);
path = H5.H5PLget(index);
assertTrue("Path should be " + pathInsert + " but was " + path, path.compareToIgnoreCase(pathInsert) == 0);
// REPLACE the path we just added and ensure it updated correctly
String pathReplace = "path_replace";
index = nStartPaths;
H5.H5PLreplace(pathReplace, index);
nPaths = H5.H5PLsize();
assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths);
path = H5.H5PLget(index);
assertTrue("Path should be " + pathReplace + " but was " + path, path.compareToIgnoreCase(pathReplace) == 0);
// REMOVE the path we just replaced and check that the table was compacted
// The (index+1) path should move down to fill the space when the path is removed.
index = nStartPaths;
String pathRemove = H5.H5PLget(index + 1);
H5.H5PLremove(index);
nPaths = H5.H5PLsize();
nTruePaths--;
assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths);
path = H5.H5PLget(index);
assertTrue("Path should be " + pathRemove + " but was " + path, path.compareToIgnoreCase(pathRemove) == 0);
}
catch (Throwable err) {
err.printStackTrace();

55
release_docs/RELEASE.txt
View File

@ -62,7 +62,29 @@ New Features
Parallel Library:
-----------------
-
- Large MPI-IO transfers
Previous releases of PHDF5 would fail when attempting to
read or write greater than 2GB of data in a single IO operation.
This issue stems principally from an MPI API whose definitions
utilize 32 bit integers to describe the number of data elements
and datatype that MPI should use to effect a data transfer.
Historically, HDF5 has invoked MPI-IO with the number of
elements in a contiguous buffer represented as the length
of that buffer in bytes.
Resolving the issue and thus enabling larger MPI-IO transfers
is accomplished first, by detecting when a user IO request would
exceed the 2GB limit as described above. Once a transfer request
is identified as requiring special handling, PHDF5 now creates a
derived datatype consisting of a vector of fixed sized blocks
which is in turn wrapped within a single MPI_Type_struct to
contain the vector and any remaining data. The newly created
datatype is then used in place of MPI_BYTE and can be used to
fulfill the original user request without encountering API
errors.
(RAW – 2017/07/11, HDFFV-8839)
Fortran Library:
----------------
@ -105,7 +127,15 @@ Bug Fixes since HDF5-1.10.1 release
Configuration
-------------
-
- cmake
Too many commands for POST_BUILD step caused command line to be
too big on windows.
Changed foreach of copy command to use a custom command with the
use of the HDFTEST_COPY_FILE macro.
(ADB - 2017/07/12, HDFFV-10254)
Performance
-------------
@ -117,6 +147,25 @@ Bug Fixes since HDF5-1.10.1 release
Tools
-----
- h5dump
h5dump segfaulted on output of XML file.
Function that escape'd strings used the full buffer length
instead of just the length of the replacement string in a
strncpy call. Using the correct length fixed the issue.
(ADB - 2017/08/01, HDFFV-10256)
- h5diff
h5diff segfaulted on compare of a NULL variable length string.
Improved h5diff compare of strings by adding a check for
NULL strings and setting the lengths to zero.
(ADB - 2017/07/25, HDFFV-10246)
- h5import
h5import crashed trying to import data from a subset of a dataset.
@ -127,7 +176,7 @@ Bug Fixes since HDF5-1.10.1 release
The import from h5dump function expects the binary files to use native
types (FILE '-b' option) in the binary file.
(ADB - 2017/06/15, HDFFV-102191)
(ADB - 2017/06/15, HDFFV-10219)
- h5repack

3
src/CMakeLists.txt
View File

@ -516,6 +516,9 @@ IDE_GENERATED_PROPERTIES ("H5PB" "${H5PB_HDRS}" "${H5PB_SOURCES}" )
set (H5PL_SOURCES
${HDF5_SRC_DIR}/H5PL.c
${HDF5_SRC_DIR}/H5PLint.c
${HDF5_SRC_DIR}/H5PLpath.c
${HDF5_SRC_DIR}/H5PLplugin_cache.c
)
set (H5PL_HDRS

7
src/H5Dchunk.c
View File

@ -1016,11 +1016,16 @@ H5D__chunk_init(H5F_t *f, hid_t dxpl_id, const H5D_t *dset, hid_t dapl_id)
unsigned u; /* Local index value */
for(u = 0; u < dset->shared->ndims; u++) {
hsize_t scaled_power2up; /* Scaled value, rounded to next power of 2 */
/* Initial scaled dimension sizes */
rdcc->scaled_dims[u] = dset->shared->curr_dims[u] / dset->shared->layout.u.chunk.dim[u];
if( !(scaled_power2up = H5VM_power2up(rdcc->scaled_dims[u])) )
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2")
/* Inital 'power2up' values for scaled dimensions */
rdcc->scaled_power2up[u] = H5VM_power2up(rdcc->scaled_dims[u]);
rdcc->scaled_power2up[u] = scaled_power2up;
/* Number of bits required to encode scaled dimension size */
rdcc->scaled_encode_bits[u] = H5VM_log2_gen(rdcc->scaled_power2up[u]);

5
src/H5Ddeprec.c
View File

@ -329,8 +329,11 @@ H5D__extend(H5D_t *dataset, const hsize_t *size, hid_t dxpl_id)
dataset->shared->cache.chunk.scaled_dims[u] > dataset->shared->cache.chunk.nslots))
update_chunks = TRUE;
if( !(scaled_power2up = H5VM_power2up(scaled)) )
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2")
/* Check if the number of bits required to encode the scaled size value changed */
if(dataset->shared->cache.chunk.scaled_power2up[u] != (scaled_power2up = H5VM_power2up(scaled))) {
if(dataset->shared->cache.chunk.scaled_power2up[u] != scaled_power2up) {
/* Update the 'power2up' & 'encode_bits' values for the current dimension */
dataset->shared->cache.chunk.scaled_power2up[u] = scaled_power2up;
dataset->shared->cache.chunk.scaled_encode_bits[u] = H5VM_log2_gen(scaled_power2up);

14
src/H5Dint.c
View File

@ -745,8 +745,13 @@ H5D__cache_dataspace_info(const H5D_t *dset)
dset->shared->ndims = (unsigned)sndims;
/* Compute the inital 'power2up' values */
for(u = 0; u < dset->shared->ndims; u++)
dset->shared->curr_power2up[u] = H5VM_power2up(dset->shared->curr_dims[u]);
for(u = 0; u < dset->shared->ndims; u++) {
hsize_t scaled_power2up; /* Scaled value, rounded to next power of 2 */
if( !(scaled_power2up = H5VM_power2up(dset->shared->curr_dims[u])) )
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2")
dset->shared->curr_power2up[u] = scaled_power2up;
}
done:
FUNC_LEAVE_NOAPI(ret_value)
@ -2805,8 +2810,11 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size, hid_t dxpl_id)
dset->shared->cache.chunk.scaled_dims[u] > dset->shared->cache.chunk.nslots))
update_chunks = TRUE;
if( !(scaled_power2up = H5VM_power2up(scaled)) )
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2")
/* Check if the number of bits required to encode the scaled size value changed */
if(dset->shared->cache.chunk.scaled_power2up[u] != (scaled_power2up = H5VM_power2up(scaled))) {
if(dset->shared->cache.chunk.scaled_power2up[u] != scaled_power2up) {
/* Update the 'power2up' & 'encode_bits' values for the current dimension */
dset->shared->cache.chunk.scaled_power2up[u] = scaled_power2up;
dset->shared->cache.chunk.scaled_encode_bits[u] = H5VM_log2_gen(scaled_power2up);

4
src/H5Fprivate.h
View File

@ -549,9 +549,11 @@
#define H5F_FILE_SPACE_PAGE_SIZE_DEF 4096
/* For paged aggregation: minimum value for file space page size */
#define H5F_FILE_SPACE_PAGE_SIZE_MIN 512
/* For paged aggregation: maxiumum value for file space page size: 1 gigabyte */
#define H5F_FILE_SPACE_PAGE_SIZE_MAX 1024*1024*1024
/* For paged aggregation: drop free-space with size <= this threshold for small meta section */
#define H5F_FILE_SPACE_PGEND_META_THRES 10
#define H5F_FILE_SPACE_PGEND_META_THRES 0
/* Default for threshold for alignment (can be set via H5Pset_alignment()) */
#define H5F_ALIGN_DEF 1

2
src/H5MF.c
View File

@ -1519,7 +1519,7 @@ H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr,
H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */
H5F_mem_page_t fs_type; /* Free space type */
hbool_t reset_ring = FALSE; /* Whether the ring was set */
htri_t ret_value = FAIL; /* Return value */
htri_t ret_value = FALSE; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
#ifdef H5MF_ALLOC_DEBUG

34
src/H5MFsection.c
View File

@ -130,27 +130,27 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{
/* Class info for "small" free space sections */
H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1] = {{
/* Class variables */
H5MF_FSPACE_SECT_SMALL, /* Section type */
0, /* Extra serialized size */
H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */
NULL, /* Class private info */
H5MF_FSPACE_SECT_SMALL, /* Section type */
0, /* Extra serialized size */
H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */
NULL, /* Class private info */
/* Class methods */
NULL, /* Initialize section class */
NULL, /* Terminate section class */
NULL, /* Initialize section class */
NULL, /* Terminate section class */
/* Object methods */
H5MF_sect_small_add, /* Add section */
NULL, /* Serialize section */
H5MF_sect_small_add, /* Add section */
NULL, /* Serialize section */
H5MF_sect_deserialize, /* Deserialize section */
H5MF_sect_small_can_merge, /* Can sections merge? */
H5MF_sect_small_merge, /* Merge sections */
H5MF_sect_small_can_shrink, /* Can section shrink container?*/
H5MF_sect_small_shrink, /* Shrink container w/section */
H5MF_sect_free, /* Free section */
H5MF_sect_valid, /* Check validity of section */
H5MF_sect_split, /* Split section node for alignment */
NULL, /* Dump debugging for section */
H5MF_sect_small_can_merge, /* Can sections merge? */
H5MF_sect_small_merge, /* Merge sections */
NULL, /* Can section shrink container?*/
NULL, /* Shrink container w/section */
H5MF_sect_free, /* Free section */
H5MF_sect_valid, /* Check validity of section */
H5MF_sect_split, /* Split section node for alignment */
NULL, /* Dump debugging for section */
}};
/* Class info for "large" free space sections */
@ -674,7 +674,7 @@ HDfprintf(stderr, "%s: Entering, section {%a, %Hu}\n", FUNC, (*sect)->sect_info.
HDfprintf(stderr, "%s: section is dropped\n", FUNC);
#endif /* H5MF_ALLOC_DEBUG_MORE */
} /* end if */
/* Adjust the section if it is not at page end but its size + pgend threshold is at page end */
/* Adjust the section if it is not at page end but its size + prem is at page end */
else
if(prem <= H5F_PGEND_META_THRES(udata->f)) {
(*sect)->sect_info.size += prem;

981
src/H5PL.c
View File

File diff suppressed because it is too large Load Diff

4
src/H5PLextern.h
View File

@ -11,9 +11,9 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Raymond Lu <songyulu@hdfgroup.org>
* 13 February 2013
* Purpose: Header file for writing external HDF5 plugins.
*/
#ifndef _H5PLextern_H
#define _H5PLextern_H

384
src/H5PLint.c Normal file
View File

@ -0,0 +1,384 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Purpose: Internal routines for managing plugins.
*
*/
/****************/
/* Module Setup */
/****************/
#include "H5PLmodule.h" /* This source code file is part of the H5PL module */
/***********/
/* Headers */
/***********/
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5MMprivate.h" /* Memory management */
#include "H5PLpkg.h" /* Plugin */
#include "H5Zprivate.h" /* Filter pipeline */
/****************/
/* Local Macros */
/****************/
/******************/
/* Local Typedefs */
/******************/
/********************/
/* Local Prototypes */
/********************/
/*********************/
/* Package Variables */
/*********************/
/* Package initialization variable */
hbool_t H5_PKG_INIT_VAR = FALSE;
/*****************************/
/* Library Private Variables */
/*****************************/
/*******************/
/* Local Variables */
/*******************/
/* Bitmask that controls whether classes of plugins
* (e.g.: filters, VOL drivers) can be loaded.
*/
static unsigned int H5PL_plugin_control_mask_g = H5PL_ALL_PLUGIN;
/* This flag will be set to FALSE if the HDF5_PLUGIN_PRELOAD
* environment variable was set to H5PL_NO_PLUGIN at
* package initialization.
*/
static hbool_t H5PL_allow_plugins_g = TRUE;
/*-------------------------------------------------------------------------
* Function: H5PL__get_plugin_control_mask
*
* Purpose: Gets the internal plugin control mask value.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__get_plugin_control_mask(unsigned int *mask /*out*/)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE_NOERR
/* Check args - Just assert on package functions */
HDassert(mask);
/* Return the mask */
*mask = H5PL_plugin_control_mask_g;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__get_plugin_control_mask() */
/*-------------------------------------------------------------------------
* Function: H5PL__set_plugin_control_mask
*
* Purpose: Sets the internal plugin control mask value.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__set_plugin_control_mask(unsigned int mask)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE_NOERR
/* Only allow setting this if plugins have not been disabled.
* XXX: Note that we don't consider this an error, but instead
* silently ignore it. We may want to consider this behavior
* more carefully.
*/
if (H5PL_allow_plugins_g)
H5PL_plugin_control_mask_g = mask;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__set_plugin_control_mask() */
/*-------------------------------------------------------------------------
* Function: H5PL__init_package
*
* Purpose: Initialize any package-specific data and call any init
* routines for the package.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__init_package(void)
{
char *env_var = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
/* Check the environment variable to determine if the user wants
* to ignore plugins. The special symbol H5PL_NO_PLUGIN (defined in
* H5PLpublic.h) means we don't want to load plugins.
*/
if (NULL != (env_var = HDgetenv("HDF5_PLUGIN_PRELOAD")))
if (!HDstrcmp(env_var, H5PL_NO_PLUGIN)) {
H5PL_plugin_control_mask_g = 0;
H5PL_allow_plugins_g = FALSE;
}
/* Create the table of previously-loaded plugins */
if (H5PL__create_plugin_cache() < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin cache")
/* Create the table of search paths for dynamic libraries */
if (H5PL__create_path_table() < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin search path table")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__init_package() */
/*-------------------------------------------------------------------------
* Function: H5PL_term_package
*
* Purpose: Terminate the H5PL interface: release all memory, reset all
* global variables to initial values. This only happens if all
* types have been destroyed from other interfaces.
*
* Return: Success: Positive if any action was taken that might
* affect some other interface; zero otherwise
* Failure: Negative
*
*-------------------------------------------------------------------------
*/
int
H5PL_term_package(void)
{
hbool_t already_closed = FALSE;
int ret_value = 0;
FUNC_ENTER_NOAPI_NOINIT
if (H5_PKG_INIT_VAR) {
/* Close the plugin cache.
* We need to bump the return value if we did any real work here.
*/
if (H5PL__close_plugin_cache(&already_closed) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing plugin cache")
if (!already_closed)
ret_value++;
/* Close the search path table and free the paths */
if (H5PL__close_path_table() < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing search path table")
/* Mark the interface as uninitialized */
if (0 == ret_value)
H5_PKG_INIT_VAR = FALSE;
} /* end if */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL_term_package() */
/*-------------------------------------------------------------------------
* Function: H5PL_load
*
* Purpose: Given the plugin type and identifier, this function searches
* for and, if found, loads a dynamic plugin library.
*
* The function searches first in the cached plugins and then
* in the paths listed in the path table.
*
* Return: Success: A pointer to the plugin info
* Failure: NULL
*
*-------------------------------------------------------------------------
*/
const void *
H5PL_load(H5PL_type_t type, int id)
{
H5PL_search_params_t search_params;
hbool_t found = FALSE; /* Whether the plugin was found */
const void *plugin_info = NULL;
const void *ret_value = NULL;
FUNC_ENTER_NOAPI(NULL)
/* Check if plugins can be loaded for this plugin type */
switch (type) {
case H5PL_TYPE_FILTER:
if ((H5PL_plugin_control_mask_g & H5PL_FILTER_PLUGIN) == 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id)
break;
case H5PL_TYPE_ERROR:
case H5PL_TYPE_NONE:
default:
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id)
}
/* Set up the search parameters */
search_params.type = type;
search_params.id = id;
/* Search in the table of already loaded plugin libraries */
if(H5PL__find_plugin_in_cache(&search_params, &found, &plugin_info) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in plugin cache failed")
/* If not found, try iterating through the path table to find an appropriate plugin */
if (!found)
if (H5PL__find_plugin_in_path_table(&search_params, &found, &plugin_info) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in path table failed")
/* Set the return value we found the plugin */
if (found)
ret_value = plugin_info;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL_load() */
/*-------------------------------------------------------------------------
* Function: H5PL__open
*
* Purpose: Opens a plugin.
*
* The success parameter will be set to TRUE and the plugin_info
* parameter will be filled in on success. Otherwise, they
* will be FALSE and NULL, respectively.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
/* NOTE: We turn off -Wpedantic in gcc to quiet a warning about converting
* object pointers to function pointers, which is undefined in ANSI C.
* This is basically unavoidable due to the nature of dlsym() and *is*
* defined in POSIX, so it's fine.
*
* This pragma only needs to surround the assignment of the
* get_plugin_info function pointer, but early (4.4.7, at least) gcc
* only allows diagnostic pragmas to be toggled outside of functions.
*/
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
herr_t
H5PL__open(const char *path, H5PL_type_t type, int id, hbool_t *success, const void **plugin_info)
{
H5PL_HANDLE handle = NULL;
H5PL_get_plugin_info_t get_plugin_info = NULL;
htri_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
/* Check args - Just assert on package functions */
HDassert(path);
HDassert(success);
HDassert(plugin_info);
/* Initialize out parameters */
*success = FALSE;
*plugin_info = NULL;
/* There are different reasons why a library can't be open, e.g. wrong architecture.
* If we can't open the library, just return.
*/
if (NULL == (handle = H5PL_OPEN_DLIB(path))) {
H5PL_CLR_ERROR; /* clear error */
HGOTO_DONE(SUCCEED);
}
/* Return a handle for the function H5PLget_plugin_info in the dynamic library.
* The plugin library is suppose to define this function.
*/
if (NULL != (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) {
const H5Z_class2_t *info;
/* Get the plugin info */
if (NULL == (info = (const H5Z_class2_t *)(*get_plugin_info)()))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info")
/* Check if the filter IDs match */
if (info->id == id) {
/* Store the plugin in the cache */
if (H5PL__add_plugin(type, id, handle))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to add new plugin to plugin cache")
/* Set output parameters */
*success = TRUE;
*plugin_info = (const void *)info;
}
}
done:
if (!success && handle)
if (H5PL__close(handle) < 0)
HDONE_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library")
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__open() */
#pragma GCC diagnostic pop
/*-------------------------------------------------------------------------
* Function: H5PL__close
*
* Purpose: Closes the handle for dynamic library
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__close(H5PL_HANDLE handle)
{
FUNC_ENTER_PACKAGE_NOERR
H5PL_CLOSE_LIB(handle);
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5PL__close() */

10
src/H5PLmodule.h
View File

@ -11,13 +11,11 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Quincey Koziol <koziol@hdfgroup.org>
* Saturday, September 12, 2015
*
* Purpose: This file contains declarations which define macros for the
* H5PL package. Including this header means that the source file
* is part of the H5PL package.
* Purpose: This file contains declarations which define macros for the
* H5PL package. Including this header means that the source file
* is part of the H5PL package.
*/
#ifndef _H5PLmodule_H
#define _H5PLmodule_H

776
src/H5PLpath.c Normal file
View File

@ -0,0 +1,776 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Purpose: Code to implement a path table which stores plugin search paths.
*
* The path table is implemented as a dynamic, global array which
* will grow as new paths are inserted. The capacity of the path
* table never shrinks (though given the low number of paths
* expected and the low likelihood of paths being removed, this
* seems unlikely to be a problem). Inserts and removals rework
* the array so that there are no 'holes' in the in-use part
* of the array.
*
* Note that it's basically up to the user to manage the indexes
* when a complicated series of insert, overwrite, and, remove
* operations take place.
*/
/****************/
/* Module Setup */
/****************/
#include "H5PLmodule.h" /* This source code file is part of the H5PL module */
/***********/
/* Headers */
/***********/
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5MMprivate.h" /* Memory management */
#include "H5PLpkg.h" /* Plugin */
/****************/
/* Local Macros */
/****************/
/* Initial capacity of the path table */
#define H5PL_INITIAL_PATH_CAPACITY 16
/* The amount to add to the capacity when the table is full */
#define H5PL_PATH_CAPACITY_ADD 16
/******************/
/* Local Typedefs */
/******************/
/********************/
/* Local Prototypes */
/********************/
static herr_t H5PL__insert_at(const char *path, unsigned int index);
static herr_t H5PL__make_space_at(unsigned int index);
static herr_t H5PL__replace_at(const char *path, unsigned int index);
static herr_t H5PL__expand_path_table(void);
static herr_t H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info);
/*********************/
/* Package Variables */
/*********************/
/*****************************/
/* Library Private Variables */
/*****************************/
/*******************/
/* Local Variables */
/*******************/
/* Stored plugin paths to search */
static char **H5PL_paths_g = NULL;
/* The number of stored paths */
static unsigned H5PL_num_paths_g = 0;
/* The capacity of the path table */
static unsigned H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY;
/*-------------------------------------------------------------------------
* Function: H5PL__insert_at()
*
* Purpose: Insert a path at a particular index in the path table.
* Does not clobber! Will move existing paths up to make
* room. Use H5PL__replace_at(index) if you want to clobber.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
H5PL__insert_at(const char *path, unsigned int index)
{
char *path_copy = NULL; /* copy of path string (for storing) */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Check args - Just assert on package functions */
HDassert(path);
HDassert(HDstrlen(path));
/* Expand the table if it is full */
if (H5PL_num_paths_g == H5PL_path_capacity_g)
if (H5PL__expand_path_table() < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand path table")
/* Copy the path for storage so the caller can dispose of theirs */
if (NULL == (path_copy = H5MM_strdup(path)))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path")
#ifdef H5_HAVE_WIN32_API
/* Clean up Microsoft Windows environment variables in the path string */
if(H5_expand_windows_env_vars(&path_copy))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string")
#endif /* H5_HAVE_WIN32_API */
/* If the table entry is in use, make some space */
if (H5PL_paths_g[index])
if (H5PL__make_space_at(index) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "unable to make space in the table for the new entry")
/* Insert the copy of the search path into the table at the specified index */
H5PL_paths_g[index] = path_copy;
H5PL_num_paths_g++;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__insert_at() */
/*-------------------------------------------------------------------------
* Function: H5PL__make_space_at()
*
* Purpose: Free up a slot in the path table, moving existing path
* entries as necessary.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
H5PL__make_space_at(unsigned int index)
{
unsigned u; /* iterator */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC_NOERR
/* Check args - Just assert on package functions */
HDassert(index < H5PL_path_capacity_g);
/* Copy the paths back to make a space */
for (u = H5PL_num_paths_g; u > index; u--)
H5PL_paths_g[u] = H5PL_paths_g[u-1];
H5PL_paths_g[index] = NULL;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__make_space_at() */
/*-------------------------------------------------------------------------
* Function: H5PL__replace_at()
*
* Purpose: Replace a path at a particular index in the path table.
* The path in the table must exist and will be freed by this
* function.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
H5PL__replace_at(const char *path, unsigned int index)
{
char *path_copy = NULL; /* copy of path string (for storing) */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* Check args - Just assert on package functions */
HDassert(path);
HDassert(HDstrlen(path));
/* Check that the table entry is in use */
if (!H5PL_paths_g[index])
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, FAIL, "path entry at index %u in the table is NULL", index)
/* Copy the path for storage so the caller can dispose of theirs */
if (NULL == (path_copy = H5MM_strdup(path)))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path")
#ifdef H5_HAVE_WIN32_API
/* Clean up Microsoft Windows environment variables in the path string */
if (H5_expand_windows_env_vars(&path_copy))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string")
#endif /* H5_HAVE_WIN32_API */
/* Free the existing path entry */
H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]);
/* Copy the search path into the table at the specified index */
H5PL_paths_g[index] = path_copy;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__replace_at() */
/*-------------------------------------------------------------------------
* Function: H5PL__create_path_table
*
* Purpose: Create the collection of paths that will be searched
* when loading plugins.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__create_path_table(void)
{
char *env_var= NULL; /* Path string from environment variable */
char *paths = NULL; /* Delimited paths string. Either from the
* environment variable or the default.
*/
char *next_path = NULL; /* A path tokenized from the paths string */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* Allocate memory for the path table */
H5PL_num_paths_g = 0;
H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY;
if (NULL == (H5PL_paths_g = (char **)H5MM_calloc((size_t)H5PL_path_capacity_g * sizeof(char *))))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path table")
/* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it
* or from the default paths if it isn't set.
*/
env_var = HDgetenv("HDF5_PLUGIN_PATH");
if (NULL == env_var)
paths = H5MM_strdup(H5PL_DEFAULT_PATH);
else
paths = H5MM_strdup(env_var);
if (NULL == paths)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path copy")
/* Separate the paths and store them */
/* XXX: strtok() is not thread-safe */
next_path = HDstrtok(paths, H5PL_PATH_SEPARATOR);
while (next_path) {
/* Insert the path into the table */
if (H5PL__append_path(next_path) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't insert path: %s", next_path)
/* Get the next path from the environment string */
next_path = HDstrtok(NULL, H5PL_PATH_SEPARATOR);
} /* end while */
done:
if (paths)
paths = (char *)H5MM_xfree(paths);
/* Try to clean up on errors */
if (FAIL == ret_value) {
if (H5PL_paths_g)
H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g);
H5PL_path_capacity_g = 0;
}
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__create_path_table() */
/*-------------------------------------------------------------------------
* Function: H5PL__close_path_table
*
* Purpose: Close the collection of paths that will be searched
* when loading plugins.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__close_path_table(void)
{
unsigned u; /* iterator */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE_NOERR
/* Free paths */
for (u = 0; u < H5PL_num_paths_g; u++)
if (H5PL_paths_g[u])
H5PL_paths_g[u] = (char *)H5MM_xfree(H5PL_paths_g[u]);
/* Free path table */
H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g);
/* Reset values */
H5PL_num_paths_g = 0;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__close_path_table() */
/*-------------------------------------------------------------------------
* Function: H5PL__get_num_paths
*
* Purpose: Gets the number of plugin paths that have been stored.
*
* Return: Success: The number of paths
* Failture: Can't fail
*-------------------------------------------------------------------------
*/
unsigned
H5PL__get_num_paths(void)
{
FUNC_ENTER_PACKAGE_NOERR
FUNC_LEAVE_NOAPI(H5PL_num_paths_g)
} /* end H5PL__get_num_paths() */
/*-------------------------------------------------------------------------
* Function: H5PL__expand_path_table
*
* Purpose: Expand the path table when it's full.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
H5PL__expand_path_table(void)
{
herr_t ret_value = SUCCEED;
FUNC_ENTER_STATIC
/* Update the capacity */
H5PL_path_capacity_g += H5PL_PATH_CAPACITY_ADD;
/* Resize the array */
if(NULL == (H5PL_paths_g = (char **)H5MM_realloc(H5PL_paths_g, (size_t)H5PL_path_capacity_g * sizeof(char *))))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for path table failed")
/* Initialize the new memory */
HDmemset(H5PL_paths_g + H5PL_num_paths_g, 0, (size_t)H5PL_PATH_CAPACITY_ADD * sizeof(char *));
done:
/* Set the path capacity back if there were problems */
if (FAIL == ret_value)
H5PL_path_capacity_g -= H5PL_PATH_CAPACITY_ADD;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__expand_path_table() */
/*-------------------------------------------------------------------------
* Function: H5PL__append_path
*
* Purpose: Insert a path at the end of the table.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__append_path(const char *path)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* Check args - Just assert on package functions */
HDassert(path);
HDassert(HDstrlen(path));
/* Insert the path at the end of the table */
if (H5PL__insert_at(path, H5PL_num_paths_g) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to append search path")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__append_path() */
/*-------------------------------------------------------------------------
* Function: H5PL__prepend_path
*
* Purpose: Insert a path at the beginning of the table.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__prepend_path(const char *path)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* Check args - Just assert on package functions */
HDassert(path);
HDassert(HDstrlen(path));
/* Insert the path at the beginning of the table */
if (H5PL__insert_at(path, 0) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__prepend_path() */
/*-------------------------------------------------------------------------
* Function: H5PL__replace_path
*
* Purpose: Replace a path at particular index in the table.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__replace_path(const char *path, unsigned int index)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* Check args - Just assert on package functions */
HDassert(path);
HDassert(HDstrlen(path));
HDassert(index < H5PL_path_capacity_g);
/* Insert the path at the requested index */
if (H5PL__replace_at(path, index) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__replace_path() */
/*-------------------------------------------------------------------------
* Function: H5PL__insert_path
*
* Purpose: Insert a path at particular index in the table, moving
* any existing paths back to make space.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__insert_path(const char *path, unsigned int index)
{
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* Check args - Just assert on package functions */
HDassert(path);
HDassert(HDstrlen(path));
HDassert(index < H5PL_path_capacity_g);
/* Insert the path at the requested index */
if (H5PL__insert_at(path, index) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__insert_path() */
/*-------------------------------------------------------------------------
* Function: H5PL__remove_path
*
* Purpose: Remove a path at particular index in the table, freeing
* the path string and moving the paths down to close the gap.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__remove_path(unsigned int index)
{
unsigned u; /* iterator */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* Check args - Just assert on package functions */
HDassert(index < H5PL_path_capacity_g);
/* Check if the path at that index is set */
if (!H5PL_paths_g[index])
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "search path at index %u is NULL", index)
/* Delete the path */
H5PL_num_paths_g--;
H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]);
/* Shift the paths down to close the gap */
for (u = index; u < H5PL_num_paths_g; u++)
H5PL_paths_g[u] = H5PL_paths_g[u+1];
/* Set the (former) last path to NULL */
H5PL_paths_g[H5PL_num_paths_g] = NULL;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__remove_path() */
/*-------------------------------------------------------------------------
* Function: H5PL__get_path
*
* Purpose: Get a pointer to a path at particular index in the table.
*
* Return: Success: A pointer to a path string stored in the table
* Failure: NULL
*
*-------------------------------------------------------------------------
*/
const char *
H5PL__get_path(unsigned int index)
{
char *ret_value = NULL; /* Return value */
FUNC_ENTER_PACKAGE
/* Get the path at the requested index */
if (index >= H5PL_num_paths_g)
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "path index %u is out of range in table", index)
return H5PL_paths_g[index];
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__replace_path() */
/*-------------------------------------------------------------------------
* Function: H5PL__find_plugin_in_path_table
*
* Purpose: Attempts to find a matching plugin in the file system
* using the paths stored in the path table.
*.
* The 'found' parameter will be set appropriately.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info)
{
unsigned int u; /* iterator */
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
/* Check args - Just assert on package functions */
HDassert(search_params);
HDassert(found);
HDassert(plugin_info);
/* Initialize output parameters */
*found = FALSE;
*plugin_info = NULL;
/* Loop over the paths in the table, checking for an appropriate plugin */
for (u = 0; u < H5PL_num_paths_g; u++) {
/* Search for the plugin in this path */
if (H5PL__find_plugin_in_path(search_params, found, H5PL_paths_g[u], plugin_info) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in path %s encountered an error", H5PL_paths_g[u])
/* Break out if found */
if (*found) {
if (!plugin_info)
HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, FAIL, "plugin info should not be NULL")
break;
}
}
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__find_plugin_in_path_table() */
/*-------------------------------------------------------------------------
* Function: H5PL__find_plugin_in_path
*
* Purpose: Given a path, this function opens the directory and envokes
* another function to go through all files to find the right
* plugin library. Two function definitions are for Unix and
* Windows.
*
* The found parameter will be set to TRUE and the info
* parameter will be filled in on success.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
#ifndef H5_HAVE_WIN32_API
static herr_t
H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info)
{
char *path = NULL;
DIR *dirp = NULL; /* Directory stream */
struct dirent *dp = NULL; /* Directory entry */
herr_t ret_value = SUCCEED;
FUNC_ENTER_STATIC
/* Check args - Just assert on package functions */
HDassert(search_params);
HDassert(found);
HDassert(dir);
HDassert(plugin_info);
/* Initialize the found parameter */
*found = FALSE;
/* Open the directory */
if (!(dirp = HDopendir(dir)))
HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir)
/* Iterate through all entries in the directory */
while (NULL != (dp = HDreaddir(dirp))) {
/* The library we are looking for should be called libxxx.so... on Unix
* or libxxx.xxx.dylib on Mac.
*/
#ifndef __CYGWIN__
if (!HDstrncmp(dp->d_name, "lib", (size_t)3) &&
(HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) {
#else
if (!HDstrncmp(dp->d_name, "cyg", (size_t)3) &&
HDstrstr(dp->d_name, ".dll") ) {
#endif
h5_stat_t my_stat;
size_t len;
/* Allocate & initialize the path name */
len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(dp->d_name) + 1 /*\0*/;
if (NULL == (path = (char *)H5MM_calloc(len)))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
HDsnprintf(path, len, "%s/%s", dir, dp->d_name);
/* Get info for directory entry */
if (HDstat(path, &my_stat) == -1)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file %s -- error was: %s", path, HDstrerror(errno))
/* If it is a directory, skip it */
if (S_ISDIR(my_stat.st_mode))
continue;
/* attempt to open the dynamic library as a filter library */
if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed")
if (*found)
HGOTO_DONE(SUCCEED)
path = (char *)H5MM_xfree(path);
} /* end if */
} /* end while */
done:
if (dirp)
if (HDclosedir(dirp) < 0)
HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno))
path = (char *)H5MM_xfree(path);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__find_plugin_in_path() */
#else /* H5_HAVE_WIN32_API */
static herr_t
H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info)
{
WIN32_FIND_DATAA fdFile;
HANDLE hFind = INVALID_HANDLE_VALUE;
char *path = NULL;
char service[2048];
herr_t ret_value = SUCCEED;
FUNC_ENTER_STATIC
/* Check args - Just assert on package functions */
HDassert(search_params);
HDassert(found);
HDassert(dir);
HDassert(plugin_info);
/* Initialize the found parameter */
*found = FALSE;
/* Specify a file mask. *.* = We want everything! */
HDsprintf(service, "%s\\*.dll", dir);
if ((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE)
HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory")
/* Loop over all the files */
do {
/* Ignore '.' and '..' */
if (HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) {
/* XXX: Probably just continue here and move the code below over one tab */
size_t len;
/* Allocate & initialize the path name */
len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(fdFile.cFileName) + 1;
if (NULL == (path = (char *)H5MM_calloc(len)))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path")
HDsnprintf(path, len, "%s\\%s", dir, fdFile.cFileName);
/* Ignore directories */
if (fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
continue;
/* attempt to open the dynamic library as a filter library */
if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed")
if (*found)
HGOTO_DONE(SUCCEED)
path = (char *)H5MM_xfree(path);
}
} while (FindNextFileA(hFind, &fdFile));
done:
if (hFind != INVALID_HANDLE_VALUE)
FindClose(hFind);
if (path)
path = (char *)H5MM_xfree(path);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__find_plugin_in_path() */
#endif /* H5_HAVE_WIN32_API */

113
src/H5PLpkg.h
View File

@ -11,6 +11,12 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Purpose: This file contains declarations which are visible only within
* the H5PL package. Source files outside the H5PL package should
* include H5PLprivate.h instead.
*/
#if !(defined H5PL_FRIEND || defined H5PL_MODULE)
#error "Do not include this file outside the H5PL package!"
#endif
@ -27,13 +33,92 @@
/* Package Private Macros */
/**************************/
#define H5PL_MAX_PATH_NUM 16
/* Whether to pre-load pathnames for plugin libraries */
#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR
/****************************/
/* Macros for supporting */
/* both Windows and POSIX */
/****************************/
/*******************/
/* Windows support */
/*******************/
/*
* SPECIAL WINDOWS NOTE
*
* Some of the Win32 API functions expand to fooA or fooW depending on
* whether UNICODE or _UNICODE are defined. You MUST explicitly use
* the A version of the functions to force char * behavior until we
* work out a scheme for proper Windows Unicode support.
*
* If you do not do this, people will be unable to incorporate our
* source code into their own CMake builds if they define UNICODE.
*/
#ifdef H5_HAVE_WIN32_API
/* The path separator on this platform */
# define H5PL_PATH_SEPARATOR ";"
/* Handle for dynamic library */
# define H5PL_HANDLE HINSTANCE
/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */
# define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH)
/* Get the address of a symbol in dynamic library */
# define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N)
/* Close dynamic library */
# define H5PL_CLOSE_LIB(H) FreeLibrary(H)
/* Clear error - nothing to do */
# define H5PL_CLR_ERROR
/* maximum size for expanding env vars */
# define H5PL_EXPAND_BUFFER_SIZE 32767
typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void);
#else /* H5_HAVE_WIN32_API */
/*****************/
/* POSIX support */
/*****************/
/* The path separator on this platform */
# define H5PL_PATH_SEPARATOR ":"
/* Handle for dynamic library */
# define H5PL_HANDLE void *
/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */
# define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY)
/* Get the address of a symbol in dynamic library */
# define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N)
/* Close dynamic library */
# define H5PL_CLOSE_LIB(H) dlclose(H)
/* Clear error */
# define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror())
typedef const void *(*H5PL_get_plugin_info_t)(void);
#endif /* H5_HAVE_WIN32_API */
/****************************/
/* Package Private Typedefs */
/****************************/
/* Data used to search for plugins */
typedef struct H5PL_search_params_t {
H5PL_type_t type;
int id;
} H5PL_search_params_t;
/*****************************/
/* Package Private Variables */
@ -44,5 +129,31 @@
/* Package Private Prototypes */
/******************************/
/* Accessors to global variables and flags */
H5_DLL herr_t H5PL__get_plugin_control_mask(unsigned int *mask /*out*/);
H5_DLL herr_t H5PL__set_plugin_control_mask(unsigned int mask);
/* Plugin search and manipulation */
H5_DLL herr_t H5PL__open(const char *libname, H5PL_type_t type, int id, hbool_t *success /*out*/, const void **plugin_info /*out*/);
H5_DLL herr_t H5PL__close(H5PL_HANDLE handle);
/* Plugin cache calls */
H5_DLL herr_t H5PL__create_plugin_cache(void);
H5_DLL herr_t H5PL__close_plugin_cache(hbool_t *already_closed /*out*/);
H5_DLL herr_t H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle);
H5_DLL herr_t H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/);
/* Plugin search path calls */
H5_DLL herr_t H5PL__create_path_table(void);
H5_DLL herr_t H5PL__close_path_table(void);
H5_DLL unsigned H5PL__get_num_paths(void);
H5_DLL herr_t H5PL__append_path(const char *path);
H5_DLL herr_t H5PL__prepend_path(const char *path);
H5_DLL herr_t H5PL__replace_path(const char *path, unsigned int index);
H5_DLL herr_t H5PL__insert_path(const char *path, unsigned int index);
H5_DLL herr_t H5PL__remove_path(unsigned int index);
H5_DLL const char *H5PL__get_path(unsigned int index);
H5_DLL herr_t H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/);
#endif /* _H5PLpkg_H */

307
src/H5PLplugin_cache.c Normal file
View File

@ -0,0 +1,307 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Purpose: Code to implement a plugin cache which stores information
* about plugins which have already been loaded.
*
* The plugin cache is implemented as a dynamic, global array which
* will grow as new plugins are added. The capacity of the cache
* never shrinks since plugins stay in memory once loaded.
*
* Note that this functionality has absolutely nothing to do with
* the metadata or chunk caches.
*/
/****************/
/* Module Setup */
/****************/
#include "H5PLmodule.h" /* This source code file is part of the H5PL module */
/***********/
/* Headers */
/***********/
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5MMprivate.h" /* Memory management */
#include "H5PLpkg.h" /* Plugin */
#include "H5Zprivate.h" /* Filter pipeline */
/****************/
/* Local Macros */
/****************/
/* Initial capacity of the plugin cache */
#define H5PL_INITIAL_CACHE_CAPACITY 16
/* The amount to add to the capacity when the cache is full */
#define H5PL_CACHE_CAPACITY_ADD 16
/******************/
/* Local Typedefs */
/******************/
/* Type for the list of info for opened plugin libraries */
typedef struct H5PL_plugin_t {
H5PL_type_t type; /* Plugin type */
int id; /* ID for the plugin */
H5PL_HANDLE handle; /* Plugin handle */
} H5PL_plugin_t;
/********************/
/* Local Prototypes */
/********************/
static herr_t H5PL__expand_cache(void);
/*********************/
/* Package Variables */
/*********************/
/*****************************/
/* Library Private Variables */
/*****************************/
/*******************/
/* Local Variables */
/*******************/
/* Cache for storing opened plugin libraries */
static H5PL_plugin_t *H5PL_cache_g = NULL;
/* The number of stored plugins */
static unsigned int H5PL_num_plugins_g = 0;
/* The capacity of the plugin cache */
static unsigned int H5PL_cache_capacity_g = 0;
/*-------------------------------------------------------------------------
* Function: H5PL__create_plugin_cache
*
* Purpose: Create the cache that will store plugins that have already
* been loaded.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__create_plugin_cache(void)
{
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
/* Allocate memory for the plugin cache */
H5PL_num_plugins_g = 0;
H5PL_cache_capacity_g = H5PL_INITIAL_CACHE_CAPACITY;
if (NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_calloc((size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t))))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for plugin cache")
done:
/* Try to clean up on errors */
if (FAIL == ret_value) {
if (H5PL_cache_g)
H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g);
H5PL_cache_capacity_g = 0;
}
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__create_plugin_cache() */
/*-------------------------------------------------------------------------
* Function: H5PL__close_plugin_cache
*
* Purpose: Close the cache of plugins that have already been loaded,
* closing all the plugins contained inside.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__close_plugin_cache(hbool_t *already_closed /*out*/)
{
unsigned int u; /* iterator */
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE_NOERR
/* Close opened dynamic libraries */
if (H5PL_cache_g) {
/* Close any cached plugins */
for (u = 0; u < H5PL_num_plugins_g; u++)
H5PL__close((H5PL_cache_g[u]).handle);
/* Free the cache array */
H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g);
H5PL_num_plugins_g = 0;
H5PL_cache_capacity_g = 0;
/* Note that actually closed the table (needed by package close call) */
*already_closed = FALSE;
}
else
*already_closed = TRUE;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__close_plugin_cache() */
/*-------------------------------------------------------------------------
* Function: H5PL__expand_cache
*
* Purpose: Expand the plugin cache when it's full.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
H5PL__expand_cache(void)
{
herr_t ret_value = SUCCEED;
FUNC_ENTER_STATIC
/* Update the capacity */
H5PL_cache_capacity_g += H5PL_CACHE_CAPACITY_ADD;
/* Resize the array */
if(NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_realloc(H5PL_cache_g, (size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t))))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for plugin cache failed")
/* Initialize the new memory */
HDmemset(H5PL_cache_g + H5PL_num_plugins_g, 0, (size_t)H5PL_CACHE_CAPACITY_ADD * sizeof(H5PL_plugin_t));
done:
/* Set the cache capacity back if there were problems */
if (FAIL == ret_value)
H5PL_cache_capacity_g -= H5PL_CACHE_CAPACITY_ADD;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__expand_cache() */
/*-------------------------------------------------------------------------
* Function: H5PL__add_plugin
*
* Purpose: Add a plugin to the plugin cached.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle)
{
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
/* Expand the cache if it is too small */
if (H5PL_num_plugins_g >= H5PL_cache_capacity_g)
if (H5PL__expand_cache() < 0)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand plugin cache")
/* Store the plugin info and bump the # of plugins */
H5PL_cache_g[H5PL_num_plugins_g].type = type;
H5PL_cache_g[H5PL_num_plugins_g].id = id;
H5PL_cache_g[H5PL_num_plugins_g].handle = handle;
H5PL_num_plugins_g++;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__add_plugin() */
/*-------------------------------------------------------------------------
* Function: H5PL__find_plugin_in_cache
*
* Purpose: Attempts to find a matching plugin from the cache.
*
* The 'found' parameter will be set appropriately.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
/* See the other use of H5PL_GET_LIB_FUNC() for an explanation
* for why we disable -Wpedantic here.
*/
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
herr_t
H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info)
{
unsigned int u; /* iterator */
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
/* Check args - Just assert on package functions */
HDassert(search_params);
HDassert(found);
HDassert(plugin_info);
/* Initialize output parameters */
*found = FALSE;
*plugin_info = NULL;
/* Loop over all the plugins, looking for one that matches */
for (u = 0; u < H5PL_num_plugins_g; u++) {
/* If the plugin type (filter, etc.) and ID match, query the plugin for its info */
if ((search_params->type == (H5PL_cache_g[u]).type) && (search_params->id == (H5PL_cache_g[u]).id)) {
H5PL_get_plugin_info_t get_plugin_info_function;
const H5Z_class2_t *filter_info;
/* Get the "get plugin info" function from the plugin. */
if (NULL == (get_plugin_info_function = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_cache_g[u]).handle, "H5PLget_plugin_info")))
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info")
/* Call the "get plugin info" function */
if (NULL == (filter_info = (const H5Z_class2_t *)(*get_plugin_info_function)()))
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info")
/* Set output parameters */
*found = TRUE;
*plugin_info = filter_info;
/* No need to continue processing */
break;
} /* end if */
} /* end for */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5PL__find_plugin_in_cache() */
#pragma GCC diagnostic pop

4
src/H5PLprivate.h
View File

@ -10,8 +10,8 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* Programmer: Raymond Lu <songyulu@hdfgroup.org>
* 13 February 2013
/*
* This file contains private information about the H5PL module
*/
#ifndef _H5PLprivate_H

34
src/H5PLpublic.h
View File

@ -10,8 +10,8 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* Programmer: Raymond Lu <songyulu@hdfgroup.org>
* 13 February 2013
/*
* This file contains public declarations for the H5PL module.
*/
#ifndef _H5PLpublic_H
@ -24,31 +24,35 @@
/* Public Typedefs */
/*******************/
/* Special string to indicate no plugin loading.
*/
#define H5PL_NO_PLUGIN "::"
/* Plugin type used by the plugin library */
typedef enum H5PL_type_t {
H5PL_TYPE_ERROR = -1, /*error */
H5PL_TYPE_FILTER = 0, /*filter */
H5PL_TYPE_NONE = 1 /*this must be last! */
H5PL_TYPE_ERROR = -1, /* Error */
H5PL_TYPE_FILTER = 0, /* Filter */
H5PL_TYPE_NONE = 1 /* This must be last! */
} H5PL_type_t;
/* Common dynamic plugin type flags used by the set/get_loading_state functions */
#define H5PL_FILTER_PLUGIN 0x0001
#define H5PL_ALL_PLUGIN 0xFFFF
#define H5PL_FILTER_PLUGIN 0x0001
#define H5PL_ALL_PLUGIN 0xFFFF
#ifdef __cplusplus
extern "C" {
#endif
/* plugin state */
H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_type);
H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_type/*out*/);
H5_DLL herr_t H5PLappend(const char *plugin_path);
H5_DLL herr_t H5PLprepend(const char *plugin_path);
H5_DLL herr_t H5PLreplace(const char *plugin_path, unsigned int index);
H5_DLL herr_t H5PLinsert(const char *plugin_path, unsigned int index);
H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_control_mask);
H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_control_mask /*out*/);
H5_DLL herr_t H5PLappend(const char *search_path);
H5_DLL herr_t H5PLprepend(const char *search_path);
H5_DLL herr_t H5PLreplace(const char *search_path, unsigned int index);
H5_DLL herr_t H5PLinsert(const char *search_path, unsigned int index);
H5_DLL herr_t H5PLremove(unsigned int index);
H5_DLL ssize_t H5PLget(unsigned int index, char *pathname/*out*/, size_t size);
H5_DLL herr_t H5PLsize(unsigned int *listsize/*out*/);
H5_DLL ssize_t H5PLget(unsigned int index, char *path_buf /*out*/, size_t buf_size);
H5_DLL herr_t H5PLsize(unsigned int *num_paths /*out*/);
#ifdef __cplusplus
}

3
src/H5Pfcpl.c
View File

@ -1472,6 +1472,9 @@ H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size)
if(fsp_size < H5F_FILE_SPACE_PAGE_SIZE_MIN)
HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to less than 512")
if(fsp_size > H5F_FILE_SPACE_PAGE_SIZE_MAX)
HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to more than 1GB")
/* Set the value*/
if(H5P_set(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsp_size) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space page size")

504
src/H5Smpio.c
View File

@ -33,7 +33,7 @@
#include "H5Oprivate.h" /* Object headers */
#include "H5Pprivate.h" /* Property lists */
#include "H5Spkg.h" /* Dataspaces */
#include "H5VMprivate.h" /* Vector and array functions */
#include "H5VMprivate.h" /* Vector and array functions */
#ifdef H5_HAVE_PARALLEL
@ -55,9 +55,42 @@ static herr_t H5S_mpio_span_hyper_type(const H5S_t *space, size_t elmt_size,
MPI_Datatype *new_type, int *count, hbool_t *is_derived_type);
static herr_t H5S_obtain_datatype(const hsize_t down[], H5S_hyper_span_t* span,
const MPI_Datatype *elmt_type, MPI_Datatype *span_type, size_t elmt_size);
static herr_t H5S_mpio_create_large_type (hsize_t, MPI_Aint, MPI_Datatype , MPI_Datatype *);
#define H5S_MPIO_INITIAL_ALLOC_COUNT 256
#define TWO_GIG_LIMIT 2147483648
#ifndef H5S_MAX_MPI_COUNT
#define H5S_MAX_MPI_COUNT 536870911 /* (2^29)-1 */
#endif
static hsize_t bigio_count = H5S_MAX_MPI_COUNT;
/*-------------------------------------------------------------------------
* Function: H5S_mpio_set_bigio_count
*
* Purpose: Allow us to programatically change the switch point
* when we utilize derived datatypes. This is of
* particular interest for allowing nightly testing
*
* Return: the current/previous value of bigio_count.
*
* Programmer: Richard Warren, March 10, 2017
*
*-------------------------------------------------------------------------
*/
hsize_t
H5S_mpio_set_bigio_count(hsize_t new_count)
{
hsize_t orig_count = bigio_count;
if ((new_count > 0) && (new_count < TWO_GIG_LIMIT)) {
bigio_count = new_count;
}
return orig_count;
}
/*-------------------------------------------------------------------------
* Function: H5S_mpio_all_type
@ -72,6 +105,11 @@ static herr_t H5S_obtain_datatype(const hsize_t down[], H5S_hyper_span_t* span,
* *is_derived_type 0 if MPI primitive type, 1 if derived
*
* Programmer: rky 980813
* Modifications:
* Mohamad Chaarawi
* Adding support for large datatypes (beyond the limit of a
* 32 bit integer.
*
*
*-------------------------------------------------------------------------
*/
@ -95,11 +133,22 @@ H5S_mpio_all_type(const H5S_t *space, size_t elmt_size,
H5_CHECKED_ASSIGN(nelmts, hsize_t, snelmts, hssize_t);
total_bytes = (hsize_t)elmt_size * nelmts;
/* fill in the return values */
*new_type = MPI_BYTE;
H5_CHECKED_ASSIGN(*count, int, total_bytes, hsize_t);
*is_derived_type = FALSE;
/* Verify that the size can be expressed as a 32 bit integer */
if(bigio_count >= total_bytes) {
/* fill in the return values */
*new_type = MPI_BYTE;
H5_CHECKED_ASSIGN(*count, int, total_bytes, hsize_t);
*is_derived_type = FALSE;
}
else {
/* Create a LARGE derived datatype for this transfer */
if (H5S_mpio_create_large_type (total_bytes, 0, MPI_BYTE, new_type) < 0) {
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,
"couldn't create a large datatype from the all selection")
}
*count = 1;
*is_derived_type = TRUE;
}
done:
FUNC_LEAVE_NOAPI(ret_value)
@ -167,27 +216,103 @@ H5S_mpio_create_point_datatype (size_t elmt_size, hsize_t num_points,
HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
elmt_type_created = TRUE;
/* Check whether standard or BIGIO processing will be employeed */
if(bigio_count >= num_points) {
#if MPI_VERSION >= 3
/* Create an MPI datatype for the whole point selection */
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed_block((int)num_points, 1, disp, elmt_type, new_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code)
/* Create an MPI datatype for the whole point selection */
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed_block((int)num_points, 1, disp, elmt_type, new_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code)
#else
/* Allocate block sizes for MPI datatype call */
if(NULL == (blocks = (int *)H5MM_malloc(sizeof(int) * num_points)))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks")
/* Allocate block sizes for MPI datatype call */
if(NULL == (blocks = (int *)H5MM_malloc(sizeof(int) * num_points)))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks")
for(u = 0; u < num_points; u++)
blocks[u] = 1;
for(u = 0; u < num_points; u++)
blocks[u] = 1;
/* Create an MPI datatype for the whole point selection */
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_points, blocks, disp, elmt_type, new_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code)
/* Create an MPI datatype for the whole point selection */
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_points, blocks, disp, elmt_type, new_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code)
#endif
/* Commit MPI datatype for later use */
if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
/* Commit MPI datatype for later use */
if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
}
else {
/* use LARGE_DATATYPE::
* We'll create an hindexed_block type for every 2G point count and then combine
* those and any remaining points into a single large datatype.
*/
int total_types, i;
int remaining_points;
int num_big_types;
hsize_t leftover;
int *inner_blocks;
MPI_Aint *inner_disps;
MPI_Datatype *inner_types = NULL;
/* Calculate how many Big MPI datatypes are needed to represent the buffer */
num_big_types = (int)(num_points/bigio_count);
leftover = (hsize_t)num_points - (hsize_t)num_big_types * (hsize_t)bigio_count;
H5_CHECKED_ASSIGN(remaining_points, int, leftover, hsize_t);
total_types = (int)(remaining_points) ? (num_big_types + 1) : num_big_types;
/* Allocate array if MPI derived types needed */
if(NULL == (inner_types = (MPI_Datatype *)H5MM_malloc((sizeof(MPI_Datatype) * (size_t)total_types))))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks")
if(NULL == (inner_blocks = (int *)H5MM_malloc(sizeof(int) * (size_t)total_types)))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks")
if(NULL == (inner_disps = (MPI_Aint *)H5MM_malloc(sizeof(MPI_Aint) * (size_t)total_types)))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks")
for(i=0 ; i<num_big_types ; i++) {
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed_block(bigio_count,
1,
&disp[i*bigio_count],
elmt_type,
&inner_types[i]))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed_block failed", mpi_code);
}
inner_blocks[i] = 1;
inner_disps[i] = 0;
}
if(remaining_points) {
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed_block(remaining_points,
1,
&disp[num_big_types*bigio_count],
elmt_type,
&inner_types[num_big_types]))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed_block failed", mpi_code);
}
inner_blocks[num_big_types] = 1;
inner_disps[num_big_types] = 0;
}
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_struct(total_types,
inner_blocks,
inner_disps,
inner_types,
new_type))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct", mpi_code);
}
for(i=0 ; i<total_types ; i++)
MPI_Type_free(&inner_types[i]);
H5MM_free(inner_types);
H5MM_free(inner_blocks);
H5MM_free(inner_disps);
/* Commit MPI datatype for later use */
if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
}
done:
if(elmt_type_created)
MPI_Type_free(&elmt_type);
@ -481,7 +606,10 @@ done:
* *is_derived_type 0 if MPI primitive type, 1 if derived
*
* Programmer: rky 980813
*
* Modifications:
* Mohamad Chaarawi
* Adding support for large datatypes (beyond the limit of a
* 32 bit integer.
*-------------------------------------------------------------------------
*/
static herr_t
@ -636,8 +764,25 @@ H5S_mpio_hyper_type(const H5S_t *space, size_t elmt_size,
HDfprintf(H5DEBUG(S), "d[%d].xtent=%Hu \n", i, d[i].xtent);
}
#endif
if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &inner_type)))
/* LARGE_DATATYPE::
* Check if the number of elements to form the inner type fits into a 32 bit integer.
* If yes then just create the innertype with MPI_Type_contiguous.
* Otherwise create a compound datatype by iterating as many times as needed
* for the innertype to be created.
*/
if(bigio_count >= elmt_size) {
/* Use a single MPI datatype that has a 32 bit size */
if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &inner_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
}
else {
/* Create the compound datatype for this operation (> 2GB) */
if (H5S_mpio_create_large_type (elmt_size, 0, MPI_BYTE, &inner_type) < 0) {
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,
"couldn't ccreate a large inner datatype in hyper selection")
}
}
/*******************************************************
* Construct the type by walking the hyperslab dims
@ -645,30 +790,93 @@ H5S_mpio_hyper_type(const H5S_t *space, size_t elmt_size,
*******************************************************/
for(i = ((int)rank) - 1; i >= 0; --i) {
#ifdef H5S_DEBUG
if(H5DEBUG(S))
HDfprintf(H5DEBUG(S), "%s: Dimension i=%d \n"
"start=%Hd count=%Hu block=%Hu stride=%Hu, xtent=%Hu max_xtent=%d\n",
FUNC, i, d[i].start, d[i].count, d[i].block, d[i].strid, d[i].xtent, max_xtent[i]);
if(H5DEBUG(S))
HDfprintf(H5DEBUG(S), "%s: Dimension i=%d \n"
"start=%Hd count=%Hu block=%Hu stride=%Hu, xtent=%Hu max_xtent=%d\n",
FUNC, i, d[i].start, d[i].count, d[i].block, d[i].strid, d[i].xtent, max_xtent[i]);
#endif
#ifdef H5S_DEBUG
if(H5DEBUG(S))
HDfprintf(H5DEBUG(S), "%s: i=%d Making vector-type \n", FUNC,i);
if(H5DEBUG(S))
HDfprintf(H5DEBUG(S), "%s: i=%d Making vector-type \n", FUNC,i);
#endif
/****************************************
* Build vector type of the selection.
****************************************/
mpi_code = MPI_Type_vector((int)(d[i].count), /* count */
(int)(d[i].block), /* blocklength */
(int)(d[i].strid), /* stride */
inner_type, /* old type */
&outer_type); /* new type */
if (bigio_count >= d[i].count &&
bigio_count >= d[i].block &&
bigio_count >= d[i].strid) {
MPI_Type_free(&inner_type);
if(mpi_code != MPI_SUCCESS)
HMPI_GOTO_ERROR(FAIL, "couldn't create MPI vector type", mpi_code)
/* All the parameters fit into 32 bit integers so create the vector type normally */
mpi_code = MPI_Type_vector((int)(d[i].count), /* count */
(int)(d[i].block), /* blocklength */
(int)(d[i].strid), /* stride */
inner_type, /* old type */
&outer_type); /* new type */
/****************************************
MPI_Type_free(&inner_type);
if(mpi_code != MPI_SUCCESS)
HMPI_GOTO_ERROR(FAIL, "couldn't create MPI vector type", mpi_code)
}
else {
/* Things get a bit more complicated and require LARGE_DATATYPE processing
* There are two MPI datatypes that need to be created:
* 1) an internal contiguous block; and
* 2) a collection of elements where an element is a contiguous block(1).
* Remember that the input arguments to the MPI-IO functions use integer
* values to represent element counts. We ARE allowed however, in the
* more recent MPI implementations to use constructed datatypes whereby
* the total number of bytes in a transfer could be :
* (2GB-1)number_of_blocks * the_datatype_extent.
*/
MPI_Aint stride_in_bytes, inner_extent;
MPI_Datatype block_type;
/* create a contiguous datatype inner_type x number of BLOCKS.
* Again we need to check that the number of BLOCKS can fit into
* a 32 bit integer */
if (bigio_count < d[i].block) {
if (H5S_mpio_create_large_type(d[i].block, 0, inner_type,
&block_type) < 0) {
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,
"couldn't ccreate a large block datatype in hyper selection")
}
}
else {
if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)d[i].block,
inner_type,
&block_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
}
MPI_Type_extent (inner_type, &inner_extent);
stride_in_bytes = inner_extent * (MPI_Aint)d[i].strid;
/* If the element count is larger than what a 32 bit integer can hold,
* we call the large type creation function to handle that
*/
if (bigio_count < d[i].count) {
if (H5S_mpio_create_large_type (d[i].count, stride_in_bytes, block_type,
&outer_type) < 0) {
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,
"couldn't create a large outer datatype in hyper selection")
}
}
/* otherwise a regular create_hvector will do */
else {
mpi_code = MPI_Type_create_hvector((int)d[i].count, /* count */
1, /* blocklength */
stride_in_bytes, /* stride in bytes*/
block_type, /* old type */
&outer_type); /* new type */
if(MPI_SUCCESS != mpi_code)
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code)
}
MPI_Type_free(&block_type);
MPI_Type_free(&inner_type);
}
/****************************************
* Then build the dimension type as (start, vector type, xtent).
****************************************/
/* calculate start and extent values of this dimension */
@ -752,6 +960,10 @@ done:
*
* Programmer: kyang
*
* Modifications:
* Mohamad Chaarawi
* Adding support for large datatypes (beyond the limit of a
* 32 bit integer.
*-------------------------------------------------------------------------
*/
static herr_t
@ -774,8 +986,17 @@ H5S_mpio_span_hyper_type(const H5S_t *space, size_t elmt_size,
HDassert(space->select.sel_info.hslab->span_lst->head);
/* Create the base type for an element */
if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &elmt_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
if (bigio_count >= elmt_size) {
if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &elmt_type))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
}
}
else {
if (H5S_mpio_create_large_type (elmt_size, 0, MPI_BYTE, &elmt_type) < 0) {
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,
"couldn't create a large element datatype in span_hyper selection")
}
}
elmt_type_is_derived = TRUE;
/* Compute 'down' sizes for each dimension */
@ -821,14 +1042,15 @@ static herr_t
H5S_obtain_datatype(const hsize_t *down, H5S_hyper_span_t *span,
const MPI_Datatype *elmt_type, MPI_Datatype *span_type, size_t elmt_size)
{
size_t alloc_count; /* Number of span tree nodes allocated at this level */
size_t outercount; /* Number of span tree nodes at this level */
size_t alloc_count = 0; /* Number of span tree nodes allocated at this level */
size_t outercount = 0; /* Number of span tree nodes at this level */
MPI_Datatype *inner_type = NULL;
hbool_t inner_types_freed = FALSE; /* Whether the inner_type MPI datatypes have been freed */
hbool_t span_type_valid = FALSE; /* Whether the span_type MPI datatypes is valid */
hbool_t large_block = FALSE; /* Wether the block length is larger than 32 bit integer */
int *blocklen = NULL;
MPI_Aint *disp = NULL;
H5S_hyper_span_t *tspan; /* Temporary pointer to span tree node */
H5S_hyper_span_t *tspan = NULL; /* Temporary pointer to span tree node */
int mpi_code; /* MPI return status code */
herr_t ret_value = SUCCEED; /* Return value */
@ -870,14 +1092,70 @@ H5S_obtain_datatype(const hsize_t *down, H5S_hyper_span_t *span,
disp[outercount] = (MPI_Aint)elmt_size * tspan->low;
H5_CHECK_OVERFLOW(tspan->nelem, hsize_t, int)
blocklen[outercount] = (int)tspan->nelem;
tspan = tspan->next;
if (bigio_count < blocklen[outercount]) {
large_block = TRUE; /* at least one block type is large, so set this flag to true */
}
outercount++;
} /* end while */
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)outercount, blocklen, disp, *elmt_type, span_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code)
span_type_valid = TRUE;
/* Everything fits into integers, so cast them and use hindexed */
if (bigio_count >= outercount && large_block == FALSE) {
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)outercount, blocklen, disp, *elmt_type, span_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code)
span_type_valid = TRUE;
}
else { /* LARGE_DATATYPE:: Something doesn't fit into a 32 bit integer */
size_t i;
for (i=0 ; i<outercount ; i++) {
MPI_Datatype temp_type = MPI_DATATYPE_NULL, outer_type = MPI_DATATYPE_NULL;
/* create the block type from elmt_type while checking the 32 bit int limit */
if (blocklen[i] > bigio_count) {
if (H5S_mpio_create_large_type (blocklen[i], 0, *elmt_type, &temp_type) < 0) {
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,
"couldn't create a large element datatype in span_hyper selection")
}
}
else {
if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)blocklen[i],
*elmt_type,
&temp_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
}
/* combine the current datatype that is created with this current block type */
if (0 == i) { /* first iteration, there is no combined datatype yet */
*span_type = temp_type;
}
else {
int bl[2] = {1,1};
MPI_Aint ds[2] = {disp[i-1],disp[i]};
MPI_Datatype dt[2] = {*span_type, temp_type};
if (MPI_SUCCESS != (mpi_code = MPI_Type_create_struct (2, /* count */
bl, /* blocklength */
ds, /* stride in bytes*/
dt, /* old type */
&outer_type))){ /* new type */
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code)
}
*span_type = outer_type;
}
if (outer_type != MPI_DATATYPE_NULL)
MPI_Type_free(&outer_type);
/* temp_type shouldn't be freed here...
* Note that we have simply copied it above (not MPI_Type_dup)
* into the 'span_type' argument of the caller.
* The caller needs to deal with it there!
*/
}
} /* end (LARGE_DATATYPE::) */
} /* end if */
else {
size_t u; /* Local index variable */
@ -1091,5 +1369,139 @@ H5S_mpio_space_type(const H5S_t *space, size_t elmt_size, MPI_Datatype *new_type
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5S_mpio_space_type() */
/*-------------------------------------------------------------------------
* Function: H5S_mpio_create_large_type
*
* Purpose: Create a large datatype of size larger than what a 32 bit integer
* can hold.
*
* Return: non-negative on success, negative on failure.
*
* *new_type the new datatype created
*
* Programmer: Mohamad Chaarawi
*
*-------------------------------------------------------------------------
*/
static herr_t H5S_mpio_create_large_type (hsize_t num_elements,
MPI_Aint stride_bytes,
MPI_Datatype old_type,
MPI_Datatype *new_type)
{
int num_big_types; /* num times the 2G datatype will be repeated */
int remaining_bytes; /* the number of bytes left that can be held in an int value */
hsize_t leftover;
int block_len[2];
int mpi_code; /* MPI return code */
MPI_Datatype inner_type, outer_type, leftover_type, type[2];
MPI_Aint disp[2], old_extent;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
/* Calculate how many Big MPI datatypes are needed to represent the buffer */
num_big_types = (int)(num_elements/bigio_count);
leftover = num_elements - num_big_types * (hsize_t)bigio_count;
H5_CHECKED_ASSIGN(remaining_bytes, int, leftover, hsize_t);
/* Create a contiguous datatype of size equal to the largest
* number that a 32 bit integer can hold x size of old type.
* If the displacement is 0, then the type is contiguous, otherwise
* use type_hvector to create the type with the displacement provided
*/
if (0 == stride_bytes) {
if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(bigio_count,
old_type,
&inner_type))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
}
}
else {
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector (bigio_count,
1,
stride_bytes,
old_type,
&inner_type))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code)
}
}
/* Create a contiguous datatype of the buffer (minus the remaining < 2GB part)
* If a stride is present, use hvector type
*/
if (0 == stride_bytes) {
if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(num_big_types,
inner_type,
&outer_type))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
}
}
else {
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector (num_big_types,
1,
stride_bytes,
inner_type,
&outer_type))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code)
}
}
MPI_Type_free(&inner_type);
/* If there is a remaining part create a contiguous/vector datatype and then
* use a struct datatype to encapsulate everything.
*/
if(remaining_bytes) {
if (stride_bytes == 0) {
if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous (remaining_bytes,
old_type,
&leftover_type))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
}
}
else {
if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector
((int)(num_elements - (hsize_t)num_big_types*bigio_count),
1,
stride_bytes,
old_type,
&leftover_type))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code)
}
}
MPI_Type_extent (old_type, &old_extent);
/* Set up the arguments for MPI_Type_struct constructor */
type[0] = outer_type;
type[1] = leftover_type;
block_len[0] = 1;
block_len[1] = 1;
disp[0] = 0;
disp[1] = (old_extent+stride_bytes)*num_big_types*(MPI_Aint)bigio_count;
if(MPI_SUCCESS != (mpi_code =
MPI_Type_create_struct(2, block_len, disp, type, new_type))) {
HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code)
}
MPI_Type_free(&outer_type);
MPI_Type_free(&leftover_type);
}
else {
/* There are no remaining bytes so just set the new type to
* the outer type created */
*new_type = outer_type;
}
if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type)))
HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code)
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5S_mpio_create_large_type */
#endif /* H5_HAVE_PARALLEL */

6
src/H5VMprivate.h
View File

@ -460,7 +460,11 @@ H5VM_power2up(hsize_t n)
{
hsize_t ret_value = 1; /* Return value */
while(ret_value < n)
/* Returns 0 when n exceeds 2^63 */
if(n >= (hsize_t)1 << ((sizeof(hsize_t) * CHAR_BIT) - 1))
ret_value = 0;
while(ret_value && ret_value < n)
ret_value <<= 1;
return(ret_value);

46
src/H5system.c
View File

@ -1237,3 +1237,49 @@ H5_get_time(void)
} /* end H5_get_time() */
#ifdef H5_HAVE_WIN32_API
#define H5_WIN32_ENV_VAR_BUFFER_SIZE 32767
/*-------------------------------------------------------------------------
* Function: H5_expand_windows_env_vars()
*
* Purpose: Replaces windows environment variables of the form %foo%
* with user-specific values.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
herr_t
H5_expand_windows_env_vars(char **env_var)
{
long n_chars = 0;
char *temp_buf = NULL;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
/* Allocate buffer for expanded environment variable string */
if (NULL == (temp_buf = (char *)H5MM_calloc((size_t)H5_WIN32_ENV_VAR_BUFFER_SIZE)))
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path")
/* Expand the environment variable string */
if ((n_chars = ExpandEnvironmentStringsA(*env_var, temp_buf, H5_WIN32_ENV_VAR_BUFFER_SIZE)) > H5_WIN32_ENV_VAR_BUFFER_SIZE)
HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long")
if (0 == n_chars)
HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path")
*env_var = (char *)H5MM_xfree(*env_var);
*env_var = temp_buf;
done:
if (FAIL == ret_value && temp_buf)
temp_buf = (char *)H5MM_xfree(temp_buf);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5_expand_windows_env_vars() */
#endif /* H5_HAVE_WIN32_API */

1
src/H5win32defs.h
View File

@ -119,6 +119,7 @@ extern "C" {
H5_DLL int c99_snprintf(char* str, size_t size, const char* format, ...);
H5_DLL int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap);
H5_DLL int Wnanosleep(const struct timespec *req, struct timespec *rem);
H5_DLL herr_t H5_expand_windows_env_vars(char **env_var);
/* Round functions only needed for VS2012 and earlier.
* They are always built to ensure they don't go stale and

2
src/Makefile.am
View File

@ -97,7 +97,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Pgcpl.c H5Pint.c \
H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \
H5PB.c \
H5PL.c \
H5PL.c H5PLint.c H5PLpath.c H5PLplugin_cache.c \
H5R.c H5Rdeprec.c \
H5UC.c \
H5RS.c \

97
test/dsets.c
View File

@ -51,14 +51,15 @@ const char *FILENAME[] = {
"copy_dcpl_newfile",/* 13 */
"partial_chunks", /* 14 */
"layout_extend", /* 15 */
"zero_chunk", /* 16 */
"zero_chunk", /* 16 */
"chunk_single", /* 17 */
"swmr_non_latest", /* 18 */
"earray_hdr_fd", /* 19 */
"farray_hdr_fd", /* 20 */
"bt2_hdr_fd", /* 21 */
"storage_size", /* 22 */
"storage_size", /* 22 */
"dls_01_strings", /* 23 */
"power2up", /* 24 */
NULL
};
#define FILENAME_BUF_SIZE 1024
@ -11343,6 +11344,97 @@ error:
return -1;
} /* end test_storage_size() */
/*-------------------------------------------------------------------------
* Function: test_power2up
*
* Purpose: Tests that the H5VM_power2up(n) function does not result in an
* infinite loop when input n exceeds 2^63. (HDFFV-10217)
* H5VM_power2up() is used to calculate the next power of 2 for
* a dataset's scaled dimension sizes.
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Vailin Choi; June 2017
*
*-------------------------------------------------------------------------
*/
static herr_t
test_power2up(hid_t fapl)
{
char filename[FILENAME_BUF_SIZE];
hid_t fid = -1; /* File ID */
hid_t dcpl = -1; /* Dataset creation property list */
hid_t sid = -1; /* Dataspace ID */
hid_t did = -1; /* Dataset ID */
hsize_t dims[2]; /* Dataset dimension sizes */
hsize_t max_dims[2]; /* Maximum dimension sizes */
hsize_t chunk_dims[2]; /* Chunk dimensions */
hsize_t ext_dims[2]; /* Extended dimension sizes */
herr_t status; /* Error status */
TESTING("the next power of 2");
h5_fixname(FILENAME[24], fapl, filename, sizeof filename);
/* Create file */
if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR
/* Set dims[1] to ((2^63) -1) */
dims[0] = 0;
dims[1] = ((hsize_t)1 << ((sizeof(hsize_t) * CHAR_BIT) -1)) - 1;
max_dims[0] = max_dims[1] = H5S_UNLIMITED;
sid = H5Screate_simple(2, dims, max_dims);
/* Create dataset creation property list */
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR
/* Set chunk size */
chunk_dims[0] = chunk_dims[1] = 1;
if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0)
TEST_ERROR
/* Create chunked dataset */
if((did = H5Dcreate2(fid, "dset", H5T_NATIVE_INT64, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR
ext_dims[0] = 1;
ext_dims[1] = dims[1] + 5;
/* Extend to (2^63)+ */
H5E_BEGIN_TRY {
status = H5Dset_extent(did, ext_dims);
} H5E_END_TRY;
if(status >= 0)
TEST_ERROR
/* Closing */
if(H5Dclose(did) < 0)
TEST_ERROR
if(H5Sclose(sid) < 0)
TEST_ERROR
if(H5Pclose(dcpl) < 0)
TEST_ERROR
if(H5Fclose(fid) < 0)
TEST_ERROR
PASSED();
return 0;
error:
H5E_BEGIN_TRY {
H5Pclose(dcpl);
H5Dclose(did);
H5Sclose(sid);
H5Pclose(dcpl);
H5Fclose(fid);
} H5E_END_TRY;
return -1;
} /* end test_power2up() */
/*-------------------------------------------------------------------------
* Function: test_scatter
@ -12928,6 +13020,7 @@ main(void)
nerrors += (test_large_chunk_shrink(my_fapl) < 0 ? 1 : 0);
nerrors += (test_zero_dim_dset(my_fapl) < 0 ? 1 : 0);
nerrors += (test_storage_size(my_fapl) < 0 ? 1 : 0);
nerrors += (test_power2up(my_fapl) < 0 ? 1 : 0);
nerrors += (test_swmr_non_latest(envval, my_fapl) < 0 ? 1 : 0);
nerrors += (test_earray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0);

16
test/fheap.c
View File

@ -16377,6 +16377,16 @@ main(void)
unsigned nerrors = 0; /* Cumulative error count */
unsigned num_pb_fs = 1; /* The number of settings to test for page buffering and file space handling */
int ExpressMode; /* Express testing level */
const char *envval; /* Environment variable */
hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */
/* Don't run this test using certain file drivers */
envval = HDgetenv("HDF5_DRIVER");
if(envval == NULL)
envval = "nomatch";
/* Current VFD that does not support contigous address space */
contig_addr_vfd = (hbool_t)(HDstrcmp(envval, "split") && HDstrcmp(envval, "multi"));
/* Reset library */
h5_reset();
@ -16428,6 +16438,12 @@ main(void)
shared_wobj_g[u] = (unsigned char)u;
for(v = 0; v < num_pb_fs; v++) {
/* Skip test when:
a) multi/split drivers and
b) persisting free-space or using paged aggregation strategy
because the library will fail file creation (temporary) for the above conditions */
if(!contig_addr_vfd && v)
break;
if((fcpl = H5Pcopy(def_fcpl)) < 0)
TEST_ERROR

497
test/plugin.c
View File

@ -10,9 +10,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Raymond Lu
* 13 February 2013
*
* Purpose: Tests the plugin module (H5PL)
*/
@ -660,13 +657,10 @@ error:
/*-------------------------------------------------------------------------
* Function: test_groups_with_filters
*
* Purpose: Tests opening group with dynamically loaded filters
* Purpose: Tests opening group with dynamically loaded filters
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
* 1 April 2013
* Return: Success: 0
* Failure: -1
*
*-------------------------------------------------------------------------
*/
@ -710,261 +704,388 @@ error:
/*-------------------------------------------------------------------------
* Function: test_filter_path_apis
* Function: test_path_api_calls
*
* Purpose: Tests accessing the path table for dynamically loaded filters
* Purpose: Tests the H5PL API calls that manipulate the plugin search
* paths.
*
* Return: Success: 0
* Failure: -1
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
test_filter_path_apis(void)
test_path_api_calls(void)
{
herr_t ret_value = -1;
unsigned int i;
unsigned int ndx;
unsigned int n_starting_paths;
unsigned int u;
unsigned int n_paths;
herr_t ret;
ssize_t pathlen = -1;
char pathname[256];
char tempname[256];
ssize_t path_len = -1;
char path[256];
char temp_name[256];
HDputs("Testing access to the filter path table");
if(H5Zfilter_avail(H5Z_FILTER_DYNLIB1) != TRUE) TEST_ERROR
if(H5Zfilter_avail(H5Z_FILTER_DYNLIB1) != TRUE)
TEST_ERROR
H5PLsize(&ndx);
/* Set the number of paths to create for this test.
*
* This should be set high enough to ensure that at least one array
* expansion will take place. See H5PLpath.c for details.
*/
n_starting_paths = 42;
/* Check that initialization is correct */
TESTING(" initialize");
if(H5PLsize(&n_paths) < 0)
TEST_ERROR
if(n_paths != 2)
TEST_ERROR
TESTING(" remove");
/* Remove all existing paths*/
for(i=ndx; i > 0; i--)
if(H5PLremove(i-1) < 0) {
HDfprintf(stderr," at %d: %s\n", i, pathname);
TEST_ERROR
} /* end if */
/* Verify the table is empty */
H5PLsize(&ndx);
if(ndx > 0) TEST_ERROR
PASSED();
TESTING(" remove (exceed min)");
/* Exceed the min path removal */
/****************/
/* H5PLremove() */
/****************/
/* Remove all the current paths */
TESTING(" remove");
/* Get the current size */
if(H5PLsize(&n_paths) < 0)
TEST_ERROR
/* Remove all existing paths */
for(u = n_paths; u > 0; u--)
if(H5PLremove(u-1) < 0) {
HDfprintf(stderr," at %u: %s\n", u, path);
TEST_ERROR
}
/* Verify the table is empty */
if(H5PLsize(&n_paths) < 0)
TEST_ERROR
if(n_paths > 0)
TEST_ERROR
PASSED();
TESTING(" remove (index 0 in empty table)");
/* Try to remove index zero in an empty list (SHOULD FAIL) */
H5E_BEGIN_TRY {
ret = H5PLremove(0);
} H5E_END_TRY
if(ret >= 0) TEST_ERROR
if(ret >= 0)
TEST_ERROR
PASSED();
/****************/
/* H5PLappend() */
/****************/
TESTING(" append");
/* Create multiple paths to fill table */
for(i=0; i < H5PL_MAX_PATH_NUM; i++) {
HDsprintf(pathname, "a_path_%d", i);
if(H5PLappend(pathname) < 0) {
HDfprintf(stderr," at %d: %s\n", i, pathname);
/* Add a bunch of paths to the path table */
for(u = 0; u < n_starting_paths; u++) {
HDsprintf(path, "a_path_%u", u);
if(H5PLappend(path) < 0) {
HDfprintf(stderr," at %u: %s\n", u, path);
TEST_ERROR
}
}
/* Verify the table is full */
H5PLsize(&ndx);
if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR
PASSED();
TESTING(" append (exceed)");
/* Exceed the max path append */
/**********************/
/* H5PLremove() again */
/**********************/
TESTING(" remove (index too high)");
/* Try to remove a path where the index is beyond the table capacity (SHOULD FAIL) */
H5E_BEGIN_TRY {
HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM);
ret = H5PLappend(pathname);
ret = H5PLremove(n_starting_paths);
} H5E_END_TRY
if(ret >= 0) TEST_ERROR
if(ret >= 0)
TEST_ERROR
PASSED();
TESTING(" remove (exceed max)");
/* Exceed the max path removal */
H5E_BEGIN_TRY {
ret = H5PLremove(H5PL_MAX_PATH_NUM);
} H5E_END_TRY
if(ret >= 0) TEST_ERROR
PASSED();
/*************/
/* H5PLget() */
/*************/
TESTING(" get (path name)");
if((pathlen = H5PLget(0, NULL, 0)) <= 0) {
/* Get the path length by passing in NULL */
if((path_len = H5PLget(0, NULL, 0)) <= 0) {
HDfprintf(stderr," get path 0 length failed\n");
TEST_ERROR
}
if(pathlen != 8) TEST_ERROR
if(path_len != 8)
TEST_ERROR
if((pathlen = H5PLget(0, pathname, 256)) <= 0) {
HDfprintf(stderr," get 0 len: %d : %s\n", pathlen, pathname);
/* Get the path */
if((path_len = H5PLget(0, path, 256)) <= 0) {
HDfprintf(stderr," get 0 len: %u : %s\n", path_len, path);
TEST_ERROR
}
if(HDstrcmp(pathname, "a_path_0") != 0) {
HDfprintf(stderr," get 0: %s\n", pathname);
if(HDstrcmp(path, "a_path_0") != 0) {
HDfprintf(stderr," get 0: %s\n", path);
TEST_ERROR
}
PASSED();
TESTING(" get (bounds)");
if((pathlen = H5PLget(1, pathname, 256)) <= 0) TEST_ERROR
if(HDstrcmp(pathname, "a_path_1") != 0) {
HDfprintf(stderr," get 1: %s\n", pathname);
TESTING(" get (high and low indices)");
/* Get path at index 1 */
if((path_len = H5PLget(1, path, 256)) <= 0)
TEST_ERROR
if(HDstrcmp(path, "a_path_1") != 0) {
HDfprintf(stderr," get 1: %s\n", path);
TEST_ERROR
}
if((pathlen = H5PLget(H5PL_MAX_PATH_NUM - 1, pathname, 256)) <= 0) TEST_ERROR
HDsprintf(tempname, "a_path_%d", H5PL_MAX_PATH_NUM - 1);
if(HDstrcmp(pathname, tempname) != 0) {
HDfprintf(stderr," get %d: %s\n", H5PL_MAX_PATH_NUM - 1, pathname);
/* Get path at the last index */
if((path_len = H5PLget(n_starting_paths - 1, path, 256)) <= 0)
TEST_ERROR
HDsprintf(temp_name, "a_path_%u", n_starting_paths - 1);
if(HDstrcmp(path, temp_name) != 0) {
HDfprintf(stderr," get %u: %s\n", n_starting_paths - 1, path);
TEST_ERROR
}
PASSED();
TESTING(" get (bounds exceed)");
TESTING(" get (index too high)");
/* Get path at the last + 1 index (SHOULD FAIL) */
H5E_BEGIN_TRY {
pathlen = H5PLget(H5PL_MAX_PATH_NUM, NULL, 0);
path_len = H5PLget(n_starting_paths, NULL, 0);
} H5E_END_TRY
if(pathlen > 0) TEST_ERROR
if(path_len > 0)
TEST_ERROR
PASSED();
TESTING(" remove (verify for prepend)");
/* Remove one path*/
if(H5PLremove(8) < 0) TEST_ERROR
/*****************/
/* H5PLprepend() */
/*****************/
/* We'll remove a path at an arbitrary index and then
* prepend a new path.
*/
TESTING(" remove (arbitrary index 1)");
/* Remove one path */
if(H5PLremove(8) < 0)
TEST_ERROR
/* Verify that the entries were moved */
if((pathlen = H5PLget(8, pathname, 256)) <= 0) TEST_ERROR
if(HDstrcmp(pathname, "a_path_9") != 0) {
HDfprintf(stderr," get 8: %s\n", pathname);
if((path_len = H5PLget(8, path, 256)) <= 0)
TEST_ERROR
if(HDstrcmp(path, "a_path_9") != 0) {
HDfprintf(stderr," get 8: %s\n", path);
TEST_ERROR
}
/* Verify the table shrank */
if(H5PLsize(&n_paths) < 0)
TEST_ERROR
if(n_paths != n_starting_paths - 1)
TEST_ERROR
PASSED();
/* Verify the table is not full */
H5PLsize(&ndx);
if (ndx != H5PL_MAX_PATH_NUM - 1) TEST_ERROR
TESTING(" prepend");
/* Prepend one path*/
HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 1);
if(H5PLprepend(pathname) < 0) {
HDfprintf(stderr," prepend %d: %s\n", H5PL_MAX_PATH_NUM + 1, pathname);
/* Prepend one path */
HDsprintf(path, "a_path_%d", n_starting_paths + 1);
if(H5PLprepend(path) < 0) {
HDfprintf(stderr," prepend %u: %s\n", n_starting_paths + 1, path);
TEST_ERROR
}
/* Verify the table is full */
H5PLsize(&ndx);
if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR
/* Verify the table increased */
if(H5PLsize(&n_paths) < 0)
TEST_ERROR
if(n_paths != n_starting_paths)
TEST_ERROR
/* Verify that the entries were moved */
if(H5PLget(8, pathname, 256) <= 0) TEST_ERROR
if(HDstrcmp(pathname, "a_path_7") != 0) {
HDfprintf(stderr," get 8: %s\n", pathname);
if(H5PLget(8, path, 256) <= 0)
TEST_ERROR
if(HDstrcmp(path, "a_path_7") != 0) {
HDfprintf(stderr," get 8: %s\n", path);
TEST_ERROR
}
if(H5PLget(0, pathname, 256) <= 0) TEST_ERROR
HDsprintf(tempname, "a_path_%d", H5PL_MAX_PATH_NUM + 1);
if(HDstrcmp(pathname, tempname) != 0) {
HDfprintf(stderr," get 0: %s\n", pathname);
/* Verify that the path was inserted at index zero */
if(H5PLget(0, path, 256) <= 0)
TEST_ERROR
HDsprintf(temp_name, "a_path_%d", n_starting_paths + 1);
if(HDstrcmp(path, temp_name) != 0) {
HDfprintf(stderr," get 0: %s\n", path);
TEST_ERROR
}
PASSED();
TESTING(" prepend (exceed)");
/* Exceed the max path prepend */
H5E_BEGIN_TRY {
HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 2);
ret = H5PLprepend(pathname);
} H5E_END_TRY
if(ret >= 0) TEST_ERROR
PASSED();
/*****************/
/* H5PLreplace() */
/*****************/
TESTING(" replace");
/* Replace one path*/
HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 4);
if(H5PLreplace(pathname, 1) < 0) {
HDfprintf(stderr," replace 1: %s\n", pathname);
/* Replace one path at index 1 */
HDsprintf(path, "a_path_%u", n_starting_paths + 4);
if(H5PLreplace(path, 1) < 0) {
HDfprintf(stderr," replace 1: %s\n", path);
TEST_ERROR
}
/* Verify the table is full */
H5PLsize(&ndx);
if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR
/* Verify the table size remained the same */
if(H5PLsize(&n_paths) < 0)
TEST_ERROR
if(n_paths != n_starting_paths)
TEST_ERROR
/* Verify that the entries were not moved */
if(H5PLget(0, pathname, 256) <= 0) TEST_ERROR
HDsprintf(tempname, "a_path_%d", H5PL_MAX_PATH_NUM + 1);
if(HDstrcmp(pathname, tempname) != 0) {
HDfprintf(stderr," get 0: %s\n", pathname);
/* Verify that the entries were not moved by
* inspecting the paths at indices +/- 1.
*/
/* Check path at index 0 */
if(H5PLget(0, path, 256) <= 0)
TEST_ERROR
HDsprintf(temp_name, "a_path_%u", n_starting_paths + 1);
if(HDstrcmp(path, temp_name) != 0) {
HDfprintf(stderr," get 0: %s\n", path);
TEST_ERROR
}
if(H5PLget(2, pathname, 256) <= 0) TEST_ERROR
if(HDstrcmp(pathname, "a_path_1") != 0) {
HDfprintf(stderr," get 2: %s\n", pathname);
/* Check path at index 2 */
if(H5PLget(2, path, 256) <= 0)
TEST_ERROR
if(HDstrcmp(path, "a_path_1") != 0) {
HDfprintf(stderr," get 2: %s\n", path);
TEST_ERROR
}
PASSED();
TESTING(" remove (verify for insert)");
/* Remove one path*/
if(H5PLremove(4) < 0) TEST_ERROR
/****************/
/* H5PLinsert() */
/****************/
/* We'll remove a path at an arbitrary index and then
* insert a new path.
*/
TESTING(" remove (arbitrary index 2)");
/* Remove one path */
if(H5PLremove(4) < 0)
TEST_ERROR
/* Verify that the entries were moved */
if(H5PLget(4, pathname, 256) <= 0) TEST_ERROR
if(HDstrcmp(pathname, "a_path_4") != 0) {
HDfprintf(stderr," get 4: %s\n", pathname);
if(H5PLget(4, path, 256) <= 0)
TEST_ERROR
if(HDstrcmp(path, "a_path_4") != 0) {
HDfprintf(stderr," get 4: %s\n", path);
TEST_ERROR
}
/* Verify the table size */
if(H5PLsize(&n_paths) < 0)
TEST_ERROR
if(n_paths != n_starting_paths - 1)
TEST_ERROR
PASSED();
/* Verify the table is not full */
H5PLsize(&ndx);
if(ndx != 15) TEST_ERROR
TESTING(" insert");
/* Insert one path*/
HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 5);
if(H5PLinsert(pathname, 3) < 0) {
HDfprintf(stderr," insert 3: %s\n", pathname);
/* Insert one path at index 3*/
HDsprintf(path, "a_path_%d", n_starting_paths + 5);
if(H5PLinsert(path, 3) < 0) {
HDfprintf(stderr," insert 3: %s\n", path);
TEST_ERROR
}
/* Verify that the entries were moved */
if(H5PLget(4, pathname, 256) <= 0) TEST_ERROR
if(HDstrcmp(pathname, "a_path_2") != 0) {
HDfprintf(stderr," get 4: %s\n", pathname);
if(H5PLget(4, path, 256) <= 0)
TEST_ERROR
if(HDstrcmp(path, "a_path_2") != 0) {
HDfprintf(stderr," get 4: %s\n", path);
TEST_ERROR
}
PASSED();
/* Verify the table is full */
H5PLsize(&ndx);
if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR
TESTING(" insert (exceed)");
/* Exceed the max path insert */
H5E_BEGIN_TRY {
HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 6);
ret = H5PLinsert(pathname, 12);
} H5E_END_TRY
if(ret >= 0) TEST_ERROR
/* Verify the table size increased */
if(H5PLsize(&n_paths) < 0)
TEST_ERROR
if(n_paths != n_starting_paths)
TEST_ERROR
PASSED();
ret_value = 0;
/****************/
/* H5PLremove() */
/****************/
/* Remove all the current paths */
TESTING(" remove (all)");
/* Get the current size */
if(H5PLsize(&n_paths) < 0)
TEST_ERROR
/* Remove all existing paths */
for(u = n_paths; u > 0; u--)
if(H5PLremove(u-1) < 0) {
HDfprintf(stderr," at %u: %s\n", u, path);
TEST_ERROR
}
/* Verify the table is empty */
if(H5PLsize(&n_paths) < 0)
TEST_ERROR
if(n_paths > 0)
TEST_ERROR
PASSED();
return SUCCEED;
error:
return ret_value;
}
return FAIL;
} /* end test_path_api_calls() */
/*-------------------------------------------------------------------------
* Function: main
*
* Purpose: Tests the plugin module (H5PL)
* Purpose: Tests the plugin module (H5PL)
*
* Return: Success: exit(EXIT_SUCCESS)
*
* Failure: exit(EXIT_FAILURE)
*
* Programmer: Raymond Lu
* 14 March 2013
* Return: EXIT_SUCCESS/EXIT_FAILURE
*
*-------------------------------------------------------------------------
*/
@ -984,37 +1105,44 @@ main(void)
/* Testing setup */
h5_reset();
fapl = h5_fileaccess();
if ((fapl = h5_fileaccess()) < 0)
TEST_ERROR
/* Turn off the chunk cache, so all the chunks are immediately written to disk */
if(H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0) TEST_ERROR
if (H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0)
TEST_ERROR
rdcc_nbytes = 0;
if(H5Pset_cache(fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0) TEST_ERROR
if (H5Pset_cache(fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0)
TEST_ERROR
/* Copy the file access property list */
if((fapl2 = H5Pcopy(fapl)) < 0) TEST_ERROR
if ((fapl2 = H5Pcopy(fapl)) < 0)
TEST_ERROR
/* Set the "use the latest version of the format" bounds for creating objects in the file */
if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) TEST_ERROR
if (H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
TEST_ERROR
h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
h5_fixname(FILENAME[0], fapl, filename, sizeof(filename));
/* Test with old & new format groups */
for(new_format = FALSE; new_format <= TRUE; new_format++) {
for (new_format = FALSE; new_format <= TRUE; new_format++) {
hid_t my_fapl;
/* Set the FAPL for the type of format */
if(new_format) {
if (new_format) {
HDputs("\nTesting with new file format:");
my_fapl = fapl2;
} /* end if */
}
else {
HDputs("Testing with old file format:");
my_fapl = fapl;
} /* end else */
}
/* Create the file for this test */
if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) TEST_ERROR
if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0)
TEST_ERROR
/* Test dynamically loaded filters for chunked dataset */
nerrors += (test_filters_for_datasets(file) < 0 ? 1 : 0);
@ -1022,12 +1150,15 @@ main(void)
/* Test dynamically loaded filters for groups */
nerrors += (test_filters_for_groups(file) < 0 ? 1 : 0);
if(H5Fclose(file) < 0) TEST_ERROR
if (H5Fclose(file) < 0)
TEST_ERROR
} /* end for */
/* Close FAPL */
if(H5Pclose(fapl2) < 0) TEST_ERROR
if(H5Pclose(fapl) < 0) TEST_ERROR
if (H5Pclose(fapl2) < 0)
TEST_ERROR
if (H5Pclose(fapl) < 0)
TEST_ERROR
/* Restore the default error handler (set in h5_reset()) */
h5_restore_err();
@ -1036,10 +1167,12 @@ main(void)
/* Close the library so that all loaded plugin libraries are unloaded */
h5_reset();
fapl = h5_fileaccess();
if ((fapl = h5_fileaccess()) < 0)
TEST_ERROR
/* Reopen the file for testing data reading */
if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR
if ((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR
/* Read the data with filters */
nerrors += (test_read_with_filters(file) < 0 ? 1 : 0);
@ -1052,29 +1185,33 @@ main(void)
/* Close the library so that all loaded plugin libraries are unloaded */
h5_reset();
fapl = h5_fileaccess();
if ((fapl = h5_fileaccess()) < 0)
TEST_ERROR
/* Reopen the file for testing data reading */
if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR
if ((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR
/* Read the data with disabled filters */
nerrors += (test_noread_with_filters(file) < 0 ? 1 : 0);
if(H5Fclose(file) < 0) TEST_ERROR
if (H5Fclose(file) < 0)
TEST_ERROR
/* Test the APIs for access to the filter plugin path table */
nerrors += (test_filter_path_apis() < 0 ? 1 : 0);
nerrors += (test_path_api_calls() < 0 ? 1 : 0);
if(nerrors) TEST_ERROR
if (nerrors)
TEST_ERROR
HDprintf("All plugin tests passed.\n");
h5_cleanup(FILENAME, fapl);
return 0;
HDexit(EXIT_SUCCESS);
error:
nerrors = MAX(1, nerrors);
HDprintf("***** %d PLUGIN TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S");
return 1;
}
HDexit(EXIT_FAILURE);
} /* end main() */

17
test/tfile.c
View File

@ -109,6 +109,7 @@
#define TEST_THRESHOLD10 10 /* Free space section threshold */
#define FSP_SIZE_DEF 4096 /* File space page size default */
#define FSP_SIZE512 512 /* File space page size */
#define FSP_SIZE1G 1024*1024*1024 /* File space page size */
/* Declaration for test_libver_macros2() */
#define FILE6 "tfile6.h5" /* Test file */
@ -3581,6 +3582,9 @@ test_filespace_info(const char *env_h5_drvr)
* Setting value less than 512 will return an error;
* --setting file space page size to 0
* --setting file space page size to 511
*
* File space page size has a maximum size of 1 gigabyte.
* Setting value greater than 1 gigabyte will return an error.
*/
/* Create file creation property list template */
fcpl = H5Pcreate(H5P_FILE_CREATE);
@ -3598,6 +3602,12 @@ test_filespace_info(const char *env_h5_drvr)
} H5E_END_TRY;
VERIFY(ret, FAIL, "H5Pset_file_space_page_size");
/* Setting to 1GB+1: should fail */
H5E_BEGIN_TRY {
ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE1G+1);
} H5E_END_TRY;
VERIFY(ret, FAIL, "H5Pset_file_space_page_size");
/* Setting to 512: should succeed */
ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE512);
CHECK(ret, FAIL, "H5Pset_file_space_page_size");
@ -3605,6 +3615,13 @@ test_filespace_info(const char *env_h5_drvr)
CHECK(ret, FAIL, "H5Pget_file_space_page_size");
VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size");
/* Setting to 1GB: should succeed */
ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE1G);
CHECK(ret, FAIL, "H5Pset_file_space_page_size");
ret = H5Pget_file_space_page_size(fcpl, &fsp_size);
CHECK(ret, FAIL, "H5Pget_file_space_page_size");
VERIFY(fsp_size, FSP_SIZE1G, "H5Pget_file_space_page_size");
/* Close property list */
H5Pclose(fcpl);

282
test/tmisc.c
View File

@ -30,7 +30,6 @@
#include "testhdf5.h"
#include "H5srcdir.h"
#include "H5Dpkg.h" /* Datasets */
#include "H5Iprivate.h" /* IDs, can be removed when H5I_REFERENCE is gone */
/* Definitions for misc. test #1 */
#define MISC1_FILE "tmisc1.h5"
@ -179,17 +178,12 @@ typedef struct
#define MISC13_GROUP1_NAME "Group1"
#define MISC13_GROUP2_NAME "Group2"
#define MISC13_DTYPE_NAME "Datatype"
#define MISC13_RANK 2
#define MISC13_DIM1 20
#define MISC13_DIM2 30
#define MISC13_RANK 1
#define MISC13_DIM1 600
#define MISC13_CHUNK_DIM1 10
#define MISC13_CHUNK_DIM2 15
#define MISC13_USERBLOCK_SIZE 512
#define MISC13_COPY_BUF_SIZE 4096
unsigned m13_data[MISC13_DIM1][MISC13_DIM2]; /* Data to write to dataset */
unsigned m13_rdata[MISC13_DIM1][MISC13_DIM2]; /* Data read from dataset */
/* Definitions for misc. test #14 */
#define MISC14_FILE "tmisc14.h5"
#define MISC14_DSET1_NAME "Dataset1"
@ -2084,38 +2078,36 @@ test_misc12(void)
/* Various routines for misc. 13 test */
static void
init_data(void)
misc13_init_data(unsigned *original_data)
{
unsigned u,v; /* Local index variables */
unsigned u;
for(u=0; u<MISC13_DIM1; u++)
for(v=0; v<MISC13_DIM2; v++)
m13_data[u][v]=(u*MISC13_DIM2)+v;
for(u = 0; u < MISC13_DIM1; u++)
original_data[u] = u;
}
static int
verify_data(void)
static hbool_t
misc13_verify_data_match(const unsigned *original_data, const unsigned *read_data)
{
unsigned u,v; /* Local index variables */
unsigned u;
for(u=0; u<MISC13_DIM1; u++)
for(v=0; v<MISC13_DIM2; v++)
if(m13_data[u][v]!=m13_rdata[u][v])
return(-1);
return(0);
for(u = 0; u < MISC13_DIM1; u++)
if(original_data[u] != read_data[u])
return FALSE;
return TRUE;
}
static void
create_dataset(hid_t loc_id, const char *name, hid_t dcpl)
misc13_create_dataset(hid_t loc_id, const char *name, hid_t dcpl, const unsigned *data)
{
hid_t dsid; /* Dataset ID */
hid_t sid; /* Dataspace ID */
hsize_t dims[MISC13_RANK]; /* Dataset dimensions */
herr_t ret; /* Generic return value */
hid_t dsid = -1; /* Dataset ID */
hid_t sid = -1; /* Dataspace ID */
hsize_t dims[MISC13_RANK]; /* Dataset dimensions */
herr_t ret; /* Generic return value */
/* Create dataspace for use with dataset */
dims[0] = MISC13_DIM1;
dims[1] = MISC13_DIM2;
sid = H5Screate_simple(MISC13_RANK, dims, NULL);
CHECK(sid, FAIL, "H5Screate_simple");
@ -2124,7 +2116,7 @@ create_dataset(hid_t loc_id, const char *name, hid_t dcpl)
CHECK(dsid, FAIL, "H5Dcreate2");
/* Write some data to dataset */
ret = H5Dwrite(dsid, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, m13_data);
ret = H5Dwrite(dsid, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
CHECK(ret, FAIL, "H5Dwrite");
/* Close the contiguous dataset */
@ -2134,64 +2126,74 @@ create_dataset(hid_t loc_id, const char *name, hid_t dcpl)
/* Close the dataspace */
ret = H5Sclose(sid);
CHECK(ret, FAIL, "H5Sclose");
}
} /* end misc13_create_dataset() */
static void
verify_dataset(hid_t loc_id, const char *name)
misc13_verify_dataset(hid_t loc_id, const char *name, const unsigned *data)
{
hid_t dsid; /* Dataset ID */
herr_t ret; /* Generic return value */
unsigned *read_data = NULL; /* Data to write to dataset */
hid_t dsid = -1; /* Dataset ID */
herr_t ret; /* Generic return value */
/* Create a data buffer for the dataset read */
read_data = (unsigned *)HDcalloc(MISC13_DIM1, sizeof(unsigned));
CHECK(read_data, NULL, "HDcalloc");
/* Open the contiguous dataset in the root group */
dsid = H5Dopen2(loc_id, name, H5P_DEFAULT);
CHECK(dsid, FAIL, "H5Dopen2");
/* Read the data */
ret = H5Dread(dsid, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, m13_rdata);
ret = H5Dread(dsid, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, read_data);
CHECK(ret, FAIL, "H5Dread");
/* Verify that the data is correct */
ret=verify_data();
CHECK(ret, FAIL, "verify_data");
/* Verify that the data are correct */
ret = misc13_verify_data_match(data, read_data);
CHECK(ret, FAIL, "misc13_verify_data_match");
/* Close the contiguous dataset */
ret = H5Dclose(dsid);
CHECK(ret, FAIL, "H5Dclose");
}
/* Free the dataset read buffer */
HDfree(read_data);
} /* end misc13_verify_dataset() */
static void
create_hdf_file(const char *name)
misc13_create_hdf_file(const char *name, const unsigned *data)
{
hid_t fid; /* File ID */
hid_t gid,gid2; /* Group IDs */
hid_t tid; /* Datatype ID */
hid_t dcpl; /* Dataset creation property list ID */
hsize_t chunk_dims[MISC13_RANK]; /* Chunk dimensions */
herr_t ret; /* Generic return value */
hid_t fid = -1; /* File ID */
hid_t gid1 = -1; /* Group ID (level 1) */
hid_t gid2 = -1; /* Group ID (level 2) */
hid_t tid = -1; /* Datatype ID */
hid_t dcplid = -1; /* Dataset creation property list ID */
hsize_t chunk_dims[MISC13_RANK]; /* Chunk dimensions */
herr_t ret; /* Generic return value */
/* Create file */
fid=H5Fcreate(name, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
fid = H5Fcreate(name, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
CHECK(fid, FAIL, "H5Fcreate");
/* Create DCPL for use with datasets */
dcpl = H5Pcreate(H5P_DATASET_CREATE);
CHECK(dcpl, FAIL, "H5Pcreate");
dcplid = H5Pcreate(H5P_DATASET_CREATE);
CHECK(dcplid, FAIL, "H5Pcreate");
/* Set the DCPL to be chunked */
ret = H5Pset_layout(dcpl, H5D_CHUNKED);
ret = H5Pset_layout(dcplid, H5D_CHUNKED);
CHECK(ret, FAIL, "H5Pset_layout");
/* Use chunked storage for this DCPL */
chunk_dims[0] = MISC13_CHUNK_DIM1;
chunk_dims[1] = MISC13_CHUNK_DIM2;
ret = H5Pset_chunk(dcpl, MISC13_RANK, chunk_dims);
ret = H5Pset_chunk(dcplid, MISC13_RANK, chunk_dims);
CHECK(ret, FAIL, "H5Pset_chunk");
/* Create contiguous dataset in root group */
create_dataset(fid, MISC13_DSET1_NAME, H5P_DEFAULT);
misc13_create_dataset(fid, MISC13_DSET1_NAME, H5P_DEFAULT, data);
/* Create chunked dataset in root group */
create_dataset(fid, MISC13_DSET2_NAME, dcpl);
misc13_create_dataset(fid, MISC13_DSET2_NAME, dcplid, data);
/* Create a datatype to commit to the file */
tid = H5Tcopy(H5T_NATIVE_INT);
@ -2206,11 +2208,11 @@ create_hdf_file(const char *name)
CHECK(ret, FAIL, "H5Tclose");
/* Create a group in the root group */
gid = H5Gcreate2(fid, MISC13_GROUP1_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
CHECK(gid, FAIL, "H5Gcreate2");
gid1 = H5Gcreate2(fid, MISC13_GROUP1_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
CHECK(gid1, FAIL, "H5Gcreate2");
/* Create another group in the new group */
gid2 = H5Gcreate2(gid, MISC13_GROUP2_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
gid2 = H5Gcreate2(gid1, MISC13_GROUP2_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
CHECK(gid2, FAIL, "H5Gcreate2");
/* Close the second group */
@ -2218,17 +2220,17 @@ create_hdf_file(const char *name)
CHECK(ret, FAIL, "H5Gclose");
/* Create contiguous dataset in new group */
create_dataset(gid, MISC13_DSET1_NAME, H5P_DEFAULT);
misc13_create_dataset(gid1, MISC13_DSET1_NAME, H5P_DEFAULT, data);
/* Create chunked dataset in new group */
create_dataset(gid, MISC13_DSET2_NAME, dcpl);
misc13_create_dataset(gid1, MISC13_DSET2_NAME, dcplid, data);
/* Create a datatype to commit to the new group */
tid = H5Tcopy(H5T_NATIVE_INT);
CHECK(tid, FAIL, "H5Tcopy");
/* Create a named datatype in the new group */
ret = H5Tcommit2(gid, MISC13_DTYPE_NAME, tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
ret = H5Tcommit2(gid1, MISC13_DTYPE_NAME, tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
CHECK(ret, FAIL, "H5Tcommit2");
/* Close named datatype */
@ -2236,25 +2238,26 @@ create_hdf_file(const char *name)
CHECK(ret, FAIL, "H5Tclose");
/* Close the first group */
ret = H5Gclose(gid);
ret = H5Gclose(gid1);
CHECK(ret, FAIL, "H5Gclose");
/* Close the DCPL */
ret = H5Pclose(dcpl);
ret = H5Pclose(dcplid);
CHECK(ret, FAIL, "H5Pclose");
/* Close the file */
ret = H5Fclose(fid);
HDassert(ret >= 0);
CHECK(ret, FAIL, "H5Fclose");
}
} /* end misc13_create_hdf_file() */
static void
insert_user_block(const char *old_name, const char *new_name,const char *str,size_t size)
misc13_insert_user_block(const char *old_name, const char *new_name, const char *str, size_t size)
{
FILE *new_fp, *old_fp; /* Pointers to new & old files */
void *user_block; /* Pointer to user block to write to file */
void *copy_buf; /* Pointer to buffer for copying data */
FILE *new_fp = NULL; /* Pointers to new & old files */
FILE *old_fp = NULL;
void *user_block = NULL; /* Pointer to user block to write to file */
void *copy_buf = NULL; /* Pointer to buffer for copying data */
size_t written; /* Amount of data written to new file */
size_t read_in; /* Amount of data read in from old file */
int ret; /* Generic status value */
@ -2264,10 +2267,10 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz
CHECK(user_block, NULL, "HDcalloc");
/* Copy in the user block data */
HDmemcpy(user_block,str,strlen(str));
HDmemcpy(user_block, str, strlen(str));
/* Open the new file */
new_fp=HDfopen(new_name,"wb");
new_fp = HDfopen(new_name,"wb");
CHECK(new_fp, NULL, "HDfopen");
/* Write the user block to the new file */
@ -2275,7 +2278,7 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz
VERIFY(written, size, "HDfwrite");
/* Open the old file */
old_fp=HDfopen(old_name,"rb");
old_fp = HDfopen(old_name,"rb");
CHECK(old_fp, NULL, "HDfopen");
/* Allocate space for the copy buffer */
@ -2287,14 +2290,14 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz
/* Write the data to the new file */
written = HDfwrite(copy_buf, (size_t)1, read_in, new_fp);
VERIFY(written, read_in, "HDfwrite");
} /* end while */
}
/* Close the old file */
ret=HDfclose(old_fp);
ret = HDfclose(old_fp);
VERIFY(ret, 0, "HDfclose");
/* Close the new file */
ret=HDfclose(new_fp);
ret = HDfclose(new_fp);
VERIFY(ret, 0, "HDfclose");
/* Free the copy buffer */
@ -2302,81 +2305,84 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz
/* Free the user block */
HDfree(user_block);
}
} /* end misc13_insert_user_block() */
static void
verify_file(const char *name, hsize_t blk_size, unsigned check_new_data)
misc13_verify_file(const char *name, const unsigned *data, hsize_t userblock_size,
hbool_t check_for_new_dataset)
{
hid_t fid; /* File ID */
hid_t gid,gid2; /* Group IDs */
hid_t tid; /* Datatype ID */
hid_t fcpl; /* File creation property list ID */
hsize_t userblock; /* Userblock size retrieved from FCPL */
herr_t ret; /* Generic return value */
hid_t fid = -1; /* File ID */
hid_t gid1 = -1; /* Group IDs */
hid_t gid2 = -1; /* Group IDs */
hid_t tid = -1; /* Datatype ID */
hid_t fcplid = -1; /* File creation property list ID */
hsize_t ub_size_out; /* Userblock size retrieved from FCPL */
herr_t ret; /* Generic return value */
/* Open the file */
fid=H5Fopen(name, H5F_ACC_RDONLY, H5P_DEFAULT);
fid = H5Fopen(name, H5F_ACC_RDONLY, H5P_DEFAULT);
CHECK(fid, FAIL, "H5Fopen");
/* Get the file's FCPL */
fcpl=H5Fget_create_plist(fid);
CHECK(fcpl, FAIL, "H5Fget_create_plist");
fcplid = H5Fget_create_plist(fid);
CHECK(fcplid, FAIL, "H5Fget_create_plist");
/* Get the user block size for the file */
ret=H5Pget_userblock(fcpl,&userblock);
ret = H5Pget_userblock(fcplid, &ub_size_out);
CHECK(ret, FAIL, "H5Pget_userblock");
/* Check the userblock size */
VERIFY(userblock, blk_size, "H5Pget_userblock");
VERIFY(userblock_size, ub_size_out, "H5Pget_userblock");
/* Close the FCPL */
ret = H5Pclose(fcpl);
ret = H5Pclose(fcplid);
CHECK(ret, FAIL, "H5Pclose");
/* Verify the contiguous dataset in the root group */
verify_dataset(fid,MISC13_DSET1_NAME);
misc13_verify_dataset(fid, MISC13_DSET1_NAME, data);
/* Verify the chunked dataset in the root group */
verify_dataset(fid,MISC13_DSET2_NAME);
misc13_verify_dataset(fid, MISC13_DSET2_NAME, data);
/* Verify the "new" contiguous dataset in the root group, if asked */
if(check_new_data)
verify_dataset(fid,MISC13_DSET3_NAME);
if(check_for_new_dataset)
misc13_verify_dataset(fid, MISC13_DSET3_NAME, data);
/* Open the named datatype in the root group */
tid = H5Topen2(fid, MISC13_DTYPE_NAME, H5P_DEFAULT);
CHECK(tid, FAIL, "H5Topen2");
/* Verify the type is correct */
VERIFY(H5Tequal(tid,H5T_NATIVE_INT), TRUE, "H5Tequal");
VERIFY(H5Tequal(tid, H5T_NATIVE_INT), TRUE, "H5Tequal");
/* Close named datatype */
ret=H5Tclose(tid);
ret = H5Tclose(tid);
CHECK(ret, FAIL, "H5Tclose");
/* Open the first group */
gid = H5Gopen2(fid, MISC13_GROUP1_NAME, H5P_DEFAULT);
CHECK(gid, FAIL, "H5Gopen2");
gid1 = H5Gopen2(fid, MISC13_GROUP1_NAME, H5P_DEFAULT);
CHECK(gid1, FAIL, "H5Gopen2");
/* Verify the contiguous dataset in the first group */
verify_dataset(gid,MISC13_DSET1_NAME);
misc13_verify_dataset(gid1, MISC13_DSET1_NAME, data);
/* Verify the chunked dataset in the first group */
verify_dataset(gid,MISC13_DSET2_NAME);
misc13_verify_dataset(gid1, MISC13_DSET2_NAME, data);
/* Open the named datatype in the first group */
tid = H5Topen2(gid,MISC13_DTYPE_NAME, H5P_DEFAULT);
tid = H5Topen2(gid1, MISC13_DTYPE_NAME, H5P_DEFAULT);
CHECK(tid, FAIL, "H5Topen2");
/* Verify the type is correct */
VERIFY(H5Tequal(tid,H5T_NATIVE_INT), TRUE, "H5Tequal");
VERIFY(H5Tequal(tid, H5T_NATIVE_INT), TRUE, "H5Tequal");
/* Close named datatype */
ret=H5Tclose(tid);
ret = H5Tclose(tid);
CHECK(ret, FAIL, "H5Tclose");
/* Open the second group */
gid2 = H5Gopen2(gid, MISC13_GROUP2_NAME, H5P_DEFAULT);
gid2 = H5Gopen2(gid1, MISC13_GROUP2_NAME, H5P_DEFAULT);
CHECK(gid2, FAIL, "H5Gopen2");
/* Close the second group */
@ -2384,31 +2390,33 @@ verify_file(const char *name, hsize_t blk_size, unsigned check_new_data)
CHECK(ret, FAIL, "H5Gclose");
/* Close the first group */
ret = H5Gclose(gid);
ret = H5Gclose(gid1);
CHECK(ret, FAIL, "H5Gclose");
/* Close the file */
ret = H5Fclose(fid);
CHECK(ret, FAIL, "H5Fclose");
}
} /* end misc13_verify_file() */
static void
add_to_new_file(const char *name)
misc13_add_to_new_file(const char *name, const unsigned *data)
{
hid_t fid; /* File ID */
hid_t fid = -1; /* File ID */
herr_t ret; /* Generic return value */
/* Open the file */
fid=H5Fopen(name, H5F_ACC_RDWR, H5P_DEFAULT);
fid = H5Fopen(name, H5F_ACC_RDWR, H5P_DEFAULT);
CHECK(fid, FAIL, "H5Fopen");
/* Create new contiguous dataset in root group */
create_dataset(fid, MISC13_DSET3_NAME, H5P_DEFAULT);
misc13_create_dataset(fid, MISC13_DSET3_NAME, H5P_DEFAULT, data);
/* Close the file */
ret = H5Fclose(fid);
CHECK(ret, FAIL, "H5Fclose");
}
} /* end misc13_add_to_new_file() */
/****************************************************************
**
@ -2419,26 +2427,44 @@ add_to_new_file(const char *name)
static void
test_misc13(void)
{
unsigned *data = NULL; /* Data to write to dataset */
hsize_t userblock_size; /* Correct size of userblock */
hbool_t check_for_new_dataset; /* Whether to check for the post-userblock-creation dataset */
/* Create a data buffer for the datasets */
data = (unsigned *)HDcalloc(MISC13_DIM1, sizeof(unsigned));
CHECK(data, NULL, "HDcalloc");
/* Initialize data to write */
init_data();
misc13_init_data(data);
/* Create first file, with no user block */
create_hdf_file(MISC13_FILE_1);
misc13_create_hdf_file(MISC13_FILE_1, data);
/* Verify file contents are correct */
verify_file(MISC13_FILE_1,(hsize_t)0,0);
userblock_size = 0;
check_for_new_dataset = FALSE;
misc13_verify_file(MISC13_FILE_1, data, userblock_size, check_for_new_dataset);
/* Create a new file by inserting a user block in front of the first file */
insert_user_block(MISC13_FILE_1, MISC13_FILE_2, "Test String", (size_t)MISC13_USERBLOCK_SIZE);
misc13_insert_user_block(MISC13_FILE_1, MISC13_FILE_2, "Test String", (size_t)MISC13_USERBLOCK_SIZE);
/* Verify file contents are still correct */
verify_file(MISC13_FILE_2,(hsize_t)MISC13_USERBLOCK_SIZE,0);
userblock_size = MISC13_USERBLOCK_SIZE;
check_for_new_dataset = FALSE;
misc13_verify_file(MISC13_FILE_2, data, userblock_size, check_for_new_dataset);
/* Make certain we can modify the new file */
add_to_new_file(MISC13_FILE_2);
misc13_add_to_new_file(MISC13_FILE_2, data);
/* Verify file contents are still correct */
verify_file(MISC13_FILE_2,(hsize_t)MISC13_USERBLOCK_SIZE,1);
userblock_size = MISC13_USERBLOCK_SIZE;
check_for_new_dataset = TRUE;
misc13_verify_file(MISC13_FILE_2, data, userblock_size, check_for_new_dataset);
/* Free the dataset buffer */
HDfree(data);
} /* end test_misc13() */
/****************************************************************
@ -3448,34 +3474,6 @@ test_misc19(void)
HDfree(vfd_cls);
/* Check H5I operations on references */
/* Reference IDs are not used by the library so there's no
* way of testing if incr/decr, etc. work. Instead, just
* do a quick smoke check to ensure that a couple of basic
* calls return sane values.
*
* H5I_REFERENCE has been declared deprecated with a tentative
* removal version of HDF5 1.12.0.
*
* Delete this entire block when H5I_REFERENCE no longer exists.
*
* The H5Iprivate.h header was included to support H5I_nmembers()
* so that can also probably be removed as well.
*/
{
htri_t tf; /* Boolean generic return */
int64_t num_members; /* Number of members in type */
tf = H5Itype_exists(H5I_REFERENCE);
VERIFY(tf, TRUE, "H5Itype_exists");
num_members = 999;
num_members = H5I_nmembers(H5I_REFERENCE);
VERIFY(num_members, 0, "H5Inmembers");
} /* end block */
} /* end test_misc19() */
/****************************************************************

1
testpar/CMakeLists.txt
View File

@ -43,6 +43,7 @@ ENDMACRO (ADD_H5P_EXE file)
set (H5P_TESTS
t_mpi
t_bigio
t_cache
t_pflush1
t_pflush2

2
testpar/Makefile.am
View File

@ -23,7 +23,7 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test
# Test programs. These are our main targets.
#
TEST_PROG_PARA=t_mpi testphdf5 t_cache t_cache_image t_pflush1 t_pflush2 t_pshutdown t_prestart t_init_term t_shapesame t_filters_parallel
TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pflush1 t_pflush2 t_pshutdown t_prestart t_init_term t_shapesame t_filters_parallel
check_PROGRAMS = $(TEST_PROG_PARA)

2153
testpar/t_bigio.c Normal file
View File

File diff suppressed because it is too large Load Diff

2
tools/lib/h5diff.c
View File

@ -11,8 +11,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdlib.h>
#include "H5private.h"
#include "h5tools.h"
#include "h5tools_utils.h"

20
tools/lib/h5diff_array.c
View File

@ -673,21 +673,33 @@ static hsize_t diff_datum(void *_mem1,
h5difftrace("diff_datum H5T_STRING variable\n");
/* Get pointer to first string */
s1 = *(char**) mem1;
size1 = HDstrlen(s1);
if(s1)
size1 = HDstrlen(s1);
else
size1 = 0;
/* Get pointer to second string */
s2 = *(char**) mem2;
size2 = HDstrlen(s2);
if(s2)
size2 = HDstrlen(s2);
else
size2 = 0;
}
else if (H5T_STR_NULLTERM == pad) {
h5difftrace("diff_datum H5T_STRING null term\n");
/* Get pointer to first string */
s1 = (char*) mem1;
size1 = HDstrlen(s1);
if(s1)
size1 = HDstrlen(s1);
else
size1 = 0;
if (size1 > size_mtype)
size1 = size_mtype;
/* Get pointer to second string */
s2 = (char*) mem2;
size2 = HDstrlen(s2);
if(s2)
size2 = HDstrlen(s2);
else
size2 = 0;
if (size2 > size_mtype)
size2 = size_mtype;
}

6
tools/lib/h5tools.c
View File

@ -12,16 +12,10 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Robb Matzke <matzke@llnl.gov>
* Thursday, July 23, 1998
*
* Purpose: A library for displaying the values of a dataset in a human
* readable format.
*/
#include <stdio.h>
#include <stdlib.h>
#include "h5tools.h"
#include "h5tools_dump.h"
#include "h5tools_ref.h"

6
tools/lib/h5tools_dump.c
View File

@ -12,16 +12,10 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Robb Matzke <matzke@llnl.gov>
* Thursday, July 23, 1998
*
* Purpose: A library for displaying the values of a dataset in a human
* readable format.
*/
#include <stdio.h>
#include <stdlib.h>
#include "h5tools.h"
#include "h5tools_dump.h"
#include "h5tools_ref.h"

2
tools/lib/h5tools_ref.c
View File

@ -11,8 +11,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdio.h>
#include <stdlib.h>
#include "h5tools_ref.h"
#include "H5private.h"
#include "H5SLprivate.h"

8
tools/lib/h5tools_str.c
View File

@ -12,16 +12,8 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Bill Wendling <wendling@ncsa.uiuc.edu>
* Monday, 19. February 2001
*
* Purpose: These are string functions for us to use and abuse.
*/
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "H5private.h"
#include "h5tools.h" /* for h5tool_format_t structure */
#include "h5tools_ref.h"

10
tools/lib/h5tools_utils.c
View File

@ -11,21 +11,11 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Bill Wendling <wendling@ncsa.uiuc.edu>
* Tuesday, 6. March 2001
*/
/*
* Portions of this work are derived from _Obfuscated C and Other Mysteries_,
* by Don Libes, copyright (c) 1993 by John Wiley & Sons, Inc.
*/
#include <ctype.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include "h5tools.h"
#include "h5tools_utils.h"
#include "H5private.h"

7
tools/lib/io_timer.c
View File

@ -22,16 +22,9 @@
* This is a module of useful timing functions for performance testing.
*/
#include <stdio.h>
#include <stdlib.h>
#include "H5private.h"
#include "hdf5.h"
#ifdef H5_HAVE_PARALLEL
#include <mpi.h>
#endif
#include "io_timer.h"
/*

4
tools/lib/ph5diff.h
View File

@ -42,9 +42,5 @@ struct diffs_found
int not_cmp;
};
#ifdef H5_HAVE_PARALLEL
#include <mpi.h>
#endif
#endif /* _PH5DIFF_H__ */

2
tools/src/h5copy/h5copy.c
View File

@ -14,8 +14,6 @@
#include "H5private.h"
#include "h5tools.h"
#include "h5tools_utils.h"
#include <string.h>
#include <stdlib.h>
/* Name of tool */
#define PROGRAMNAME "h5copy"

2
tools/src/h5diff/h5diff_common.c
View File

@ -11,8 +11,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdlib.h>
#include <string.h>
#include "H5private.h"
#include "h5diff.h"
#include "h5diff_common.h"

3
tools/src/h5diff/h5diff_main.c
View File

@ -11,9 +11,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdlib.h>
#include <assert.h>
#include <memory.h>
#include "H5private.h"
#include "h5diff.h"
#include "h5diff_common.h"

3
tools/src/h5diff/ph5diff_main.c
View File

@ -11,9 +11,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "H5private.h"
#include "h5diff.h"
#include "ph5diff.h"

2
tools/src/h5dump/h5dump.c
View File

@ -10,8 +10,6 @@
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdio.h>
#include <stdlib.h>
#include "h5dump.h"
#include "h5dump_ddl.h"

2
tools/src/h5dump/h5dump_ddl.c
View File

@ -10,8 +10,6 @@
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdio.h>
#include <stdlib.h>
#include "H5private.h"
#include "h5tools.h"

10
tools/src/h5dump/h5dump_xml.c
View File

@ -10,8 +10,6 @@
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdio.h>
#include <stdlib.h>
#include "H5private.h"
#include "h5tools.h"
@ -782,20 +780,20 @@ xml_escape_the_string(const char *str, int slen)
esc_len = 1;
}
else if (*cp == '\'') {
HDstrncpy(ncp, apos, ncp_len);
esc_len = HDstrlen(apos);
HDstrncpy(ncp, apos, esc_len);
}
else if (*cp == '<') {
HDstrncpy(ncp, lt, ncp_len);
esc_len = HDstrlen(lt);
HDstrncpy(ncp, lt, esc_len);
}
else if (*cp == '>') {
HDstrncpy(ncp, gt, ncp_len);
esc_len = HDstrlen(gt);
HDstrncpy(ncp, gt, esc_len);
}
else if (*cp == '&') {
HDstrncpy(ncp, amp, ncp_len);
esc_len = HDstrlen(amp);
HDstrncpy(ncp, amp, esc_len);
}
else {
*ncp = *cp;

4
tools/src/h5repack/h5repack.c
View File

@ -11,10 +11,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include "H5private.h"
#include "h5repack.h"
#include "h5tools.h"

2
tools/src/h5stat/h5stat.c
View File

@ -11,8 +11,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdlib.h>
#include <string.h>
#include "H5private.h" /* Generic Functions */
#include "h5tools.h"
#include "h5tools_utils.h"

2
tools/src/misc/h5mkgrp.c
View File

@ -15,8 +15,6 @@
#include "H5private.h"
#include "h5tools.h"
#include "h5tools_utils.h"
#include <string.h>
#include <stdlib.h>
/* Name of tool */
#define PROGRAMNAME "h5mkgrp"

32
tools/src/misc/h5repart.c
View File

@ -25,40 +25,10 @@
/* See H5private.h for how to include system headers */
#include "hdf5.h"
#include "H5private.h"
#ifdef H5_STDC_HEADERS
# include <ctype.h>
# include <errno.h>
# include <fcntl.h>
# include <stdio.h>
# include <stdlib.h>
# include <string.h>
#endif
#ifdef H5_HAVE_UNISTD_H
# include <sys/types.h>
# include <unistd.h>
#endif
#ifdef H5_HAVE_SYS_STAT_H
# include <sys/stat.h>
#endif
#ifndef FALSE
# define FALSE 0
#endif
#ifndef TRUE
# define TRUE 1
#endif
# define NAMELEN 4096
#define NAMELEN 4096
#define GB *1024*1024*1024
#ifndef MIN
# define MIN(X,Y) ((X)<(Y)?(X):(Y))
#endif
#ifndef MIN3
# define MIN3(X,Y,Z) MIN(MIN(X,Y),Z)
#endif
/*Make these 2 private properties(defined in H5Fprivate.h) available to h5repart.
*The first one updates the member file size in the superblock. The second one
*change file driver from family to sec2. */

1
tools/test/h5copy/h5copygentest.c
View File

@ -14,7 +14,6 @@
/*
* Generate the binary hdf5 file for the h5copy tests
*/
#include <stdlib.h>
#include "hdf5.h"
#include "H5private.h"

11
tools/test/h5diff/CMakeTests.cmake
View File

@ -95,6 +95,8 @@
${HDF5_TOOLS_DIR}/testfiles/vds/5_b.h5
${HDF5_TOOLS_DIR}/testfiles/vds/5_c.h5
${HDF5_TOOLS_DIR}/testfiles/vds/5_vds.h5
# tools/testfiles
${HDF5_TOOLS_DIR}/testfiles/tvlstr.h5
)
set (LIST_OTHER_TEST_FILES
@ -286,6 +288,7 @@
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_v1.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_v2.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_v3.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_vlstr.txt
)
set (LIST_WIN_TEST_FILES
@ -311,6 +314,11 @@
HDFTEST_COPY_FILE("${h5_tstfiles}" "${PROJECT_BINARY_DIR}/PAR/testfiles/${fname}" "h5diff_files")
endif ()
endforeach ()
# copy second version of tvlstr.h5
HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/tvlstr.h5" "${PROJECT_BINARY_DIR}/testfiles/tvlstr2.h5" "h5diff_files")
if (H5_HAVE_PARALLEL)
HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/tvlstr.h5" "${PROJECT_BINARY_DIR}/PAR/testfiles/tvlstr2.h5" "h5diff_files")
endif ()
#
@ -888,6 +896,8 @@
h5diff_v2.out.err
h5diff_v3.out
h5diff_v3.out.err
h5diff_vlstr.out
h5diff_vlstr.out.err
)
set_tests_properties (H5DIFF-clearall-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles")
if (NOT "${last_test}" STREQUAL "")
@ -1465,6 +1475,7 @@ ADD_H5_TEST (h5diff_487 1 -v --exclude-path "/group1/dset" h5diff_exclude3-1.h5
# # diff various multiple vlen and fixed strings in a compound type dataset
# ##############################################################################
ADD_H5_TEST (h5diff_530 0 -v ${COMP_VL_STRS_FILE} ${COMP_VL_STRS_FILE} /group /group_copy)
ADD_H5_TEST (h5diff_vlstr 0 -v tvlstr.h5 tvlstr2.h5)
# ##############################################################################
# # Test container types (array,vlen) with multiple nested compound types

16
tools/test/h5diff/testfiles/h5diff_vlstr.txt Normal file
View File

@ -0,0 +1,16 @@
file1 file2
---------------------------------------
x x /
x x /Dataset1
x x /vl_string_type
group : </> and </>
0 differences found
attribute: <test_scalar of </>> and <test_scalar of </>>
0 differences found
dataset: </Dataset1> and </Dataset1>
0 differences found
datatype: </vl_string_type> and </vl_string_type>
0 differences found
EXIT CODE: 0

5
tools/test/h5diff/testh5diff.sh.in
View File

@ -124,6 +124,7 @@ $SRC_H5DIFF_TESTFILES/tmpSingleSiteBethe.reference.h5
$SRC_H5DIFF_TESTFILES/tmpSingleSiteBethe.output.h5
$SRC_H5DIFF_TESTFILES/diff_strings1.h5
$SRC_H5DIFF_TESTFILES/diff_strings2.h5
$SRC_TOOLS_TESTFILES/tvlstr.h5
"
LIST_HDF5_VDS_TEST_FILES="
@ -341,6 +342,7 @@ $SRC_H5DIFF_TESTFILES/h5diff_tmp2.txt
$SRC_H5DIFF_TESTFILES/h5diff_v1.txt
$SRC_H5DIFF_TESTFILES/h5diff_v2.txt
$SRC_H5DIFF_TESTFILES/h5diff_v3.txt
$SRC_H5DIFF_TESTFILES/h5diff_vlstr.txt
"
#
@ -566,6 +568,8 @@ SKIP() {
##############################################################################
# prepare for test
COPY_TESTFILES_TO_TESTDIR
# second copy of tvlstr.h5
$CP -f $SRC_TOOLS_TESTFILES/tvlstr.h5 $TESTDIR/tvlstr2.h5
# ############################################################################
# # Common usage
@ -1141,6 +1145,7 @@ TOOLTEST h5diff_487.txt -v --exclude-path "/group1/dset" h5diff_exclude3-1.h5 h5
# # diff various multiple vlen and fixed strings in a compound type dataset
# ##############################################################################
TOOLTEST h5diff_530.txt -v h5diff_comp_vl_strs.h5 h5diff_comp_vl_strs.h5 /group /group_copy
TOOLTEST h5diff_vlstr.txt -v tvlstr.h5 tvlstr2.h5
# ##############################################################################
# # Test container types (array,vlen) with multiple nested compound types

7
tools/test/h5dump/CMakeTestsXML.cmake
View File

@ -50,6 +50,7 @@
${HDF5_TOOLS_DIR}/testfiles/tname-quot.h5
${HDF5_TOOLS_DIR}/testfiles/tname-sp.h5
${HDF5_TOOLS_DIR}/testfiles/tnamed_dtype_attr.h5
${HDF5_TOOLS_DIR}/testfiles/test35.nc
${HDF5_TOOLS_DIR}/testfiles/tnestedcomp.h5
${HDF5_TOOLS_DIR}/testfiles/tnodata.h5
${HDF5_TOOLS_DIR}/testfiles/tobjref.h5
@ -99,6 +100,7 @@
${HDF5_TOOLS_DIR}/testfiles/tempty-ns.h5.xml
${HDF5_TOOLS_DIR}/testfiles/tempty-ns-2.h5.xml
${HDF5_TOOLS_DIR}/testfiles/tenum.h5.xml
${HDF5_TOOLS_DIR}/testfiles/test35.nc.xml
${HDF5_TOOLS_DIR}/testfiles/textlink.h5.xml
${HDF5_TOOLS_DIR}/testfiles/tfpformat.h5.xml
${HDF5_TOOLS_DIR}/testfiles/tgroup.h5.xml
@ -258,6 +260,8 @@
tempty.h5.out.err
tenum.h5.out
tenum.h5.out.err
test35.nc.out
test35.nc.out.err
textlink.h5.out
textlink.h5.out.err
tfpformat.h5.out
@ -430,3 +434,6 @@
# tests for floating point user defined printf format
ADD_XML_H5_TEST (tfpformat.h5 0 -u -m %.7f tfpformat.h5)
# test for HDFFV-10256 issue
ADD_XML_H5_TEST (test35.nc 0 test35.nc)

1
tools/test/h5dump/h5dumpgentest.c
View File

@ -20,7 +20,6 @@
* trying it on a new platform, ...), you need to verify the correctness
* of the expected output and update the corresponding *.ddl files.
*/
#include <limits.h>
#include "hdf5.h"
#include "H5private.h"

5
tools/test/h5dump/testh5dumpxml.sh.in
View File

@ -80,6 +80,7 @@ $SRC_H5DUMP_TESTFILES/tdset.h5
$SRC_H5DUMP_TESTFILES/tdset2.h5
$SRC_H5DUMP_TESTFILES/tempty.h5
$SRC_H5DUMP_TESTFILES/tenum.h5
$SRC_H5DUMP_TESTFILES/test35.nc
$SRC_H5DUMP_TESTFILES/textlink.h5
$SRC_H5DUMP_TESTFILES/tfpformat.h5
$SRC_H5DUMP_TESTFILES/tgroup.h5
@ -144,6 +145,7 @@ $SRC_H5DUMP_TESTFILES/tempty-nons-uri.h5.xml
$SRC_H5DUMP_TESTFILES/tempty-ns.h5.xml
$SRC_H5DUMP_TESTFILES/tempty-ns-2.h5.xml
$SRC_H5DUMP_TESTFILES/tenum.h5.xml
$SRC_H5DUMP_TESTFILES/test35.nc.xml
$SRC_H5DUMP_TESTFILES/textlink.h5.xml
$SRC_H5DUMP_TESTFILES/tfpformat.h5.xml
$SRC_H5DUMP_TESTFILES/tgroup.h5.xml
@ -383,6 +385,9 @@ TOOLTEST torderattr4.h5.xml --xml -H --sort_by=creation_order --sort_order=desce
# tests for floating point user defined printf format
TOOLTEST tfpformat.h5.xml -u -m %.7f tfpformat.h5
# test for HDFFV-10256 issue
TOOLTEST test35.nc.xml --xml test35.nc
# Clean up temporary files/directories
CLEAN_TESTFILES_AND_TESTDIR

1
tools/test/h5import/h5importtest.c
View File

@ -11,7 +11,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdio.h>
#include "H5private.h"
#ifdef H5_HAVE_WIN32_API

2
tools/test/h5jam/h5jamgentest.c
View File

@ -20,8 +20,6 @@
* trying it on a new platform, ...), you need to verify the correctness
* of the expected output and update the corresponding *.ddl files.
*/
#include <assert.h>
#include <limits.h>
#include "hdf5.h"
#include "H5private.h"

6
tools/test/h5jam/tellub.c
View File

@ -11,12 +11,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdio.h>
#ifdef H5_HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "hdf5.h"
#include "H5private.h"
#include "h5tools.h"

97
tools/test/h5repack/CMakeTests.cmake
View File

@ -76,6 +76,12 @@
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_shuffle.h5
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_soffset.h5
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_szip.h5
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_aggr.h5
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fsm_aggr_nopersist.h5
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fsm_aggr_persist.h5
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_none.h5
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_paged_nopersist.h5
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_paged_persist.h5
# h5diff/testfile
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_attr1.h5
# tools/testfiles
@ -139,6 +145,12 @@
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_conti-v.ddl
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5-plugin_zero.tst
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/crtorder.tordergr.h5.ddl
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/STG.h5repack_none.h5.ddl
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/GS.h5repack_paged_nopersist.h5.ddl
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SP.h5repack_paged_persist.h5.ddl
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SPT.h5repack_aggr.h5.ddl
)
foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES})
@ -238,7 +250,8 @@
if (HDF5_ENABLE_USING_MEMCHECKER)
add_test (
NAME H5REPACK_CMP-${testname}
COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile})
COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}
)
else ()
add_test (
NAME H5REPACK_CMP-${testname}
@ -272,7 +285,8 @@
if (HDF5_ENABLE_USING_MEMCHECKER)
add_test (
NAME H5REPACK_MASK-${testname}
COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile})
COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}
)
else (HDF5_ENABLE_USING_MEMCHECKER)
add_test (
NAME H5REPACK_MASK-${testname}
@ -305,7 +319,8 @@
# If using memchecker add tests without using scripts
add_test (
NAME H5REPACK_DMP-${testname}
COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile})
COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}
)
if (NOT "${last_test}" STREQUAL "")
set_tests_properties (H5REPACK_DMP-${testname} PROPERTIES DEPENDS ${last_test})
endif ()
@ -326,6 +341,40 @@
endif ()
endmacro ()
macro (ADD_H5_STAT_TEST testname testtype resultcode statarg resultfile)
if ("${testtype}" STREQUAL "SKIP")
if (NOT HDF5_ENABLE_USING_MEMCHECKER)
add_test (
NAME H5REPACK_STAT-${testname}-SKIPPED
COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${statarg}.${resultfile}"
)
endif ()
else ()
# If using memchecker add tests without using scripts
add_test (
NAME H5REPACK_STAT-${testname}
COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${statarg}.${resultfile}
)
if (NOT "${last_test}" STREQUAL "")
set_tests_properties (H5REPACK_STAT-${testname} PROPERTIES DEPENDS ${last_test})
endif ()
if (NOT HDF5_ENABLE_USING_MEMCHECKER)
add_test (
NAME H5REPACK_STAT-h5stat-${testname}
COMMAND "${CMAKE_COMMAND}"
-D "TEST_PROGRAM=$<TARGET_FILE:h5stat>"
-D "TEST_ARGS:STRING=-S;-s;out-${statarg}.${resultfile}"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles"
-D "TEST_OUTPUT=${resultfile}-${testname}.out"
-D "TEST_EXPECT=${resultcode}"
-D "TEST_REFERENCE=${statarg}.${resultfile}.ddl"
-P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake"
)
set_tests_properties (H5REPACK_STAT-h5stat-${testname} PROPERTIES DEPENDS "H5REPACK_STAT-${testname}")
endif ()
endif ()
endmacro ()
macro (ADD_H5_VERIFY_TEST testname testtype resultcode testfile testdset testfilter)
if ("${testtype}" STREQUAL "SKIP")
if (NOT HDF5_ENABLE_USING_MEMCHECKER)
@ -972,6 +1021,48 @@
set (TESTTYPE "TEST")
ADD_H5_DMP_TEST (crtorder ${TESTTYPE} 0 ${arg})
###################################################################################################
# Testing paged aggregation related options:
# -G pagesize
# -P 1 or 0
# -S strategy
# -T threshold
#
# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_":
# (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */
# (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */
# (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */
# (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */
# (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */
# (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */
#
#####################################################################################################
#
set (arg h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1)
set (TESTTYPE "TEST")
ADD_H5_STAT_TEST (SP_PAGE ${TESTTYPE} 0 SP ${arg})
set (arg h5repack_fsm_aggr_persist.h5 -S AGGR)
set (TESTTYPE "TEST")
ADD_H5_STAT_TEST (S_AGGR ${TESTTYPE} 0 S ${arg})
set (arg h5repack_none.h5 -S PAGE -T 10 -G 2048)
set (TESTTYPE "TEST")
ADD_H5_STAT_TEST (STG_PAGE ${TESTTYPE} 0 STG ${arg})
set (arg h5repack_paged_nopersist.h5 -G 512 -S AGGR)
set (TESTTYPE "TEST")
ADD_H5_STAT_TEST (GS_AGGR ${TESTTYPE} 0 GS ${arg})
set (arg h5repack_paged_persist.h5 -S NONE -P 1)
set (TESTTYPE "TEST")
ADD_H5_STAT_TEST (SP_NONE ${TESTTYPE} 0 SP ${arg})
set (arg h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5)
set (TESTTYPE "TEST")
ADD_H5_STAT_TEST (SPT_FSM_AGGR ${TESTTYPE} 0 SPT ${arg})
#########################################################
# layout options (these files have no filters)
#########################################################

103
tools/test/h5repack/h5repack.sh.in
View File

@ -36,6 +36,9 @@ H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary
H5DUMP=../../src/h5dump/h5dump # The h5dump tool name
H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary
H5STAT=../../src/h5stat/h5stat # The h5stat tool name
H5STAT_BIN=`pwd`/$H5STAT # The path of the h5stat tool binary
RM='rm -rf'
CMP='cmp'
DIFF='diff -c'
@ -101,6 +104,12 @@ $SRC_H5REPACK_TESTFILES/h5repack_refs.h5
$SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5
$SRC_H5REPACK_TESTFILES/h5repack_soffset.h5
$SRC_H5REPACK_TESTFILES/h5repack_szip.h5
$SRC_H5REPACK_TESTFILES/h5repack_aggr.h5
$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_nopersist.h5
$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5
$SRC_H5REPACK_TESTFILES/h5repack_none.h5
$SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5
$SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5
$SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
$SRC_TOOLS_TESTFILES/tfamily00000.h5
$SRC_TOOLS_TESTFILES/tfamily00001.h5
@ -156,6 +165,12 @@ $SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl
$SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl
$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl
$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl
$SRC_H5REPACK_TESTFILES/SP.h5repack_fsm_aggr_nopersist.h5.ddl
$SRC_H5REPACK_TESTFILES/S.h5repack_fsm_aggr_persist.h5.ddl
$SRC_H5REPACK_TESTFILES/STG.h5repack_none.h5.ddl
$SRC_H5REPACK_TESTFILES/GS.h5repack_paged_nopersist.h5.ddl
$SRC_H5REPACK_TESTFILES/SP.h5repack_paged_persist.h5.ddl
$SRC_H5REPACK_TESTFILES/SPT.h5repack_aggr.h5.ddl
"
#
@ -707,6 +722,58 @@ TOOLTEST_DUMP()
rm -f $outfile
}
# This is similar to TOOLTEST_DUMP().
# Test h5repack with options added for paged aggregation.
# h5stat is used on the repacked file and the expected output
# is compared for correctness.
#
TOOLTEST_STAT()
{
infile=$2
outfile=out-$1.$2
expect="$TESTDIR/$1.$2.ddl"
actual="$TESTDIR/out-$1.$2.out"
actual_err="$TESTDIR/out-$1.$2.err"
shift
shift
# Run test.
TESTING $H5REPACK $@
(
cd $TESTDIR
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
) >$actual 2>$actual_err
RET=$?
if [ $RET != 0 ] ; then
echo "*FAILED*"
nerrors="`expr $nerrors + 1`"
else
echo " PASSED"
VERIFY h5stat output $@
(
cd $TESTDIR
$RUNSERIAL $H5STAT_BIN -S -s $outfile
) >$actual 2>$actual_err
cat $actual_err >> $actual
RET=$?
fi
if cmp -s $expect $actual; then
echo " PASSED"
else
echo "*FAILED*"
echo " Expected result (*.ddl) differs from actual result (*.out)"
nerrors="`expr $nerrors + 1`"
test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /'
fi
rm -f $actual $actual_err
rm -f $outfile
}
# TOOLTEST_META:
# Test metadata block size option.
# Reason to create a function here is to localize all special steps related to
@ -1057,6 +1124,42 @@ fi
arg="tordergr.h5 -L"
TOOLTEST_DUMP crtorder $arg
###################################################################################################
# Testing paged aggregation related options:
# -G pagesize
# -P 1 or 0
# -S strategy
# -T threshold
#
# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_":
# (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */
# (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */
# (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */
# (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */
# (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */
# (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */
#
#####################################################################################################
#
arg="h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1"
TOOLTEST_STAT SP $arg
#
arg="h5repack_fsm_aggr_persist.h5 -S AGGR"
TOOLTEST_STAT S $arg
#
arg="h5repack_none.h5 -S PAGE -T 10 -G 2048"
TOOLTEST_STAT STG $arg
#
arg="h5repack_paged_nopersist.h5 -G 512 -S AGGR"
TOOLTEST_STAT GS $arg
#
arg="h5repack_paged_persist.h5 -S NONE -P 1"
TOOLTEST_STAT SP $arg
#
arg="h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5"
TOOLTEST_STAT SPT $arg
#########################################################
# layout options (these files have no filters)
#########################################################

15
tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl Normal file
View File

@ -0,0 +1,15 @@
Filename: out-GS.h5repack_paged_nopersist.h5
Free-space persist: FALSE
Free-space section threshold: 1 bytes
Small size free-space sections (< 10 bytes):
Total # of small size sections: 0
Free-space section bins:
Total # of sections: 0
File space management strategy: H5F_FSPACE_STRATEGY_AGGR
File space page size: 512 bytes
Summary of file space information:
File metadata: 1272 bytes
Raw data: 400 bytes
Amount/Percent of tracked free space: 0 bytes/0.0%
Unaccounted space: 776 bytes
Total space: 2448 bytes

15
tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl Normal file
View File

@ -0,0 +1,15 @@
Filename: out-S.h5repack_fsm_aggr_persist.h5
Free-space persist: FALSE
Free-space section threshold: 1 bytes
Small size free-space sections (< 10 bytes):
Total # of small size sections: 0
Free-space section bins:
Total # of sections: 0
File space management strategy: H5F_FSPACE_STRATEGY_AGGR
File space page size: 4096 bytes
Summary of file space information:
File metadata: 1272 bytes
Raw data: 400 bytes
Amount/Percent of tracked free space: 0 bytes/0.0%
Unaccounted space: 776 bytes
Total space: 2448 bytes

16
tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl Normal file
View File

@ -0,0 +1,16 @@
Filename: out-SP.h5repack_fsm_aggr_nopersist.h5
Free-space persist: TRUE
Free-space section threshold: 1 bytes
Small size free-space sections (< 10 bytes):
Total # of small size sections: 0
Free-space section bins:
# of sections of size 1000 - 9999: 2
Total # of sections: 2
File space management strategy: H5F_FSPACE_STRATEGY_PAGE
File space page size: 4096 bytes
Summary of file space information:
File metadata: 1602 bytes
Raw data: 400 bytes
Amount/Percent of tracked free space: 6307 bytes/38.5%
Unaccounted space: 8075 bytes
Total space: 16384 bytes

15
tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl Normal file
View File

@ -0,0 +1,15 @@
Filename: out-SP.h5repack_paged_persist.h5
Free-space persist: FALSE
Free-space section threshold: 1 bytes
Small size free-space sections (< 10 bytes):
Total # of small size sections: 0
Free-space section bins:
Total # of sections: 0
File space management strategy: H5F_FSPACE_STRATEGY_NONE
File space page size: 4096 bytes
Summary of file space information:
File metadata: 1272 bytes
Raw data: 400 bytes
Amount/Percent of tracked free space: 0 bytes/0.0%
Unaccounted space: 0 bytes
Total space: 1672 bytes

16
tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl Normal file
View File

@ -0,0 +1,16 @@
Filename: out-SPT.h5repack_aggr.h5
Free-space persist: TRUE
Free-space section threshold: 5 bytes
Small size free-space sections (< 10 bytes):
Total # of small size sections: 0
Free-space section bins:
# of sections of size 100 - 999: 1
Total # of sections: 1
File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR
File space page size: 4096 bytes
Summary of file space information:
File metadata: 1485 bytes
Raw data: 400 bytes
Amount/Percent of tracked free space: 680 bytes/26.5%
Unaccounted space: 0 bytes
Total space: 2565 bytes

15
tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl Normal file
View File

@ -0,0 +1,15 @@
Filename: out-STG.h5repack_none.h5
Free-space persist: FALSE
Free-space section threshold: 10 bytes
Small size free-space sections (< 10 bytes):
Total # of small size sections: 0
Free-space section bins:
Total # of sections: 0
File space management strategy: H5F_FSPACE_STRATEGY_PAGE
File space page size: 2048 bytes
Summary of file space information:
File metadata: 1272 bytes
Raw data: 400 bytes
Amount/Percent of tracked free space: 0 bytes/0.0%
Unaccounted space: 2424 bytes
Total space: 4096 bytes

BIN
tools/test/h5repack/testfiles/h5repack_aggr.h5 Normal file
View File

Binary file not shown.

BIN
tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 Normal file
View File

Binary file not shown.

BIN
tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 Normal file
View File

Binary file not shown.

BIN
tools/test/h5repack/testfiles/h5repack_none.h5 Normal file
View File

Binary file not shown.

BIN
tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 Normal file
View File

Binary file not shown.

BIN
tools/test/h5repack/testfiles/h5repack_paged_persist.h5 Normal file
View File

Binary file not shown.

1
tools/test/h5repack/testh5repack_detect_szip.c
View File

@ -11,7 +11,6 @@
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdio.h>
#include "h5repack.h"
#include "h5tools.h"
#include "h5tools_utils.h"

95
tools/test/misc/repart_test.c
View File

@ -12,19 +12,16 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Raymond Lu <slu@ncsa.uiuc.edu>
* June 1, 2005
*
* Purpose: This program tests family files after being repartitioned
* by h5repart. It simply tries to reopen the files with
* correct family driver and member size.
*/
#include "hdf5.h"
#include "H5private.h"
#define KB 1024
#define FAMILY_H5REPART_SIZE1 20000
#define FAMILY_H5REPART_SIZE2 (5*KB)
#define MAX(a,b) (a>b ? a:b)
const char *FILENAME[] = {
"fst_family%05d.h5",
@ -42,52 +39,54 @@ herr_t test_sec2_h5repart_opens(void);
*
* Purpose: Tries to reopen family files.
*
* Return: Success: exit(0)
* Return: SUCCEED/FAIL
*
* Failure: exit(1)
*
* Programmer: Raymond Lu
* June 1, 2005
*
* Modifications:
*-------------------------------------------------------------------------
*/
herr_t
test_family_h5repart_opens(void)
{
hid_t file=(-1), fapl=(-1);
hid_t fid = -1;
hid_t fapl_id = -1;
/* open 1st file(single member file) with correct family size(20000 byte) */
if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0)
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if(H5Pset_fapl_family(fapl, (hsize_t)FAMILY_H5REPART_SIZE1, H5P_DEFAULT)<0)
if (H5Pset_fapl_family(fapl_id, (hsize_t)FAMILY_H5REPART_SIZE1, H5P_DEFAULT) < 0)
goto error;
if((file=H5Fopen(FILENAME[0], H5F_ACC_RDWR, fapl))<0)
if ((fid = H5Fopen(FILENAME[0], H5F_ACC_RDWR, fapl_id))<0)
goto error;
if(H5Fclose(file)<0)
if (H5Fclose(fid) < 0)
goto error;
/* open 2nd file(multiple member files) with correct family size(5KB) */
if(H5Pset_fapl_family(fapl, (hsize_t)FAMILY_H5REPART_SIZE2, H5P_DEFAULT)<0)
if (H5Pset_fapl_family(fapl_id, (hsize_t)FAMILY_H5REPART_SIZE2, H5P_DEFAULT) < 0)
goto error;
if((file=H5Fopen(FILENAME[1], H5F_ACC_RDWR, fapl))<0)
if ((fid = H5Fopen(FILENAME[1], H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if(H5Fclose(file)<0)
if (H5Pclose(fapl_id) < 0)
goto error;
return 0;
if (H5Fclose(fid) < 0)
goto error;
return SUCCEED;
error:
H5E_BEGIN_TRY {
H5Fclose(file);
H5Pclose(fapl_id);
H5Fclose(fid);
} H5E_END_TRY;
return -1;
}
return FAIL;
} /* end test_family_h5repart_opens() */
/*-------------------------------------------------------------------------
@ -95,36 +94,32 @@ error:
*
* Purpose: Tries to reopen a sec2 file.
*
* Return: Success: exit(0)
* Return: SUCCEED/FAIL
*
* Failure: exit(1)
*
* Programmer: Raymond Lu
* June 21, 2005
*
* Modifications:
*-------------------------------------------------------------------------
*/
herr_t
test_sec2_h5repart_opens(void)
{
hid_t file=(-1);
hid_t fid = -1;
/* open the sec2 file */
if((file=H5Fopen(FILENAME[2], H5F_ACC_RDWR, H5P_DEFAULT))<0)
if ((fid = H5Fopen(FILENAME[2], H5F_ACC_RDWR, H5P_DEFAULT)) < 0)
goto error;
if(H5Fclose(file)<0)
if (H5Fclose(fid) < 0)
goto error;
return 0;
return SUCCEED;
error:
H5E_BEGIN_TRY {
H5Fclose(file);
H5Fclose(fid);
} H5E_END_TRY;
return -1;
}
return FAIL;
} /* end test_sec2_h5repart_opens() */
/*-------------------------------------------------------------------------
@ -132,32 +127,26 @@ error:
*
* Purpose: Tests h5repart-ed family files
*
* Return: Success: exit(0)
*
* Failure: exit(1)
*
* Programmer: Raymond Lu
* June 1, 2005
*
* Modifications:
* Return: EXIT_SUCCESS/EXIT_FAILURE
*
*-------------------------------------------------------------------------
*/
int
main(void)
{
int nerrors=0;
int nerrors = 0;
nerrors += test_family_h5repart_opens()<0 ?1:0;
nerrors += test_sec2_h5repart_opens()<0 ?1:0;
nerrors += test_family_h5repart_opens() < 0 ? 1 : 0;
nerrors += test_sec2_h5repart_opens() < 0 ? 1 : 0;
if (nerrors) goto error;
if (nerrors)
goto error;
return 0;
HDexit(EXIT_SUCCESS);
error:
nerrors = MAX(1, nerrors);
printf("***** %d FAMILY FILE TEST%s FAILED! *****\n",
HDprintf("***** %d FAMILY FILE TEST%s FAILED! *****\n",
nerrors, 1 == nerrors ? "" : "S");
return 1;
}
HDexit(EXIT_FAILURE);
} /* end main() */

3
tools/test/misc/talign.c
View File

@ -15,9 +15,6 @@
* Small program to illustrate the "misalignment" of members within a compound
* datatype, in a datatype fixed by H5Tget_native_type().
*/
#include <string.h>
#include <stdlib.h>
/*#include <unistd.h> *//* Required for unlink() */
#include "hdf5.h"
#include "H5private.h"

1
tools/test/perform/CMakeTests.cmake
View File

@ -146,6 +146,7 @@ else ()
-P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake"
)
endif ()
set_tests_properties (PERFORM_zip_perf PROPERTIES DEPENDS PERFORM_zip_perf_help)
if (H5_HAVE_PARALLEL)
add_test (NAME PERFORM_h5perf COMMAND ${MPIEXEC} ${MPIEXEC_PREFLAGS} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_POSTFLAGS} $<TARGET_FILE:h5perf>)

BIN
tools/testfiles/test35.nc Normal file
View File

Binary file not shown.

20
tools/testfiles/test35.nc.xml Normal file
View File

File diff suppressed because one or more lines are too long