mirror of
https://github.com/HDFGroup/hdf5.git
synced 2025-04-12 17:31:09 +08:00
Add support for parallel tools based on the 3rd party library mpiFileUtils (libMFU) … (#1177)
Adds tool h5dwalk and configure options to enable building it. Co-authored-by: Richard Warren <Richard.Warren@hdfgroup.org> Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Richard.Warren <richard.warren@jelly.ad.hdfgroup.org> Co-authored-by: Larry Knox <lrknox@hdfgroup.org>
This commit is contained in:
rawarren
GitHub
parent
9cdc6d58bd
commit
720ddb20f3
@ -1007,6 +1007,31 @@ endif ()
|
||||
# Option to build HDF5 Utilities
|
||||
#-----------------------------------------------------------------------------
|
||||
if (EXISTS "${HDF5_SOURCE_DIR}/utils" AND IS_DIRECTORY "${HDF5_SOURCE_DIR}/utils")
|
||||
option (HDF5_BUILD_PARALLEL_TOOLS "Build Parallel HDF5 Tools" OFF)
|
||||
if (HDF5_BUILD_PARALLEL_TOOLS AND HDF5_ENABLE_PARALLEL)
|
||||
set (CMAKE_PREFIX_PATH "$HDF_RESOURCES_DIR")
|
||||
find_package(MFU REQUIRED)
|
||||
if (MFU_FOUND)
|
||||
message(STATUS "LL_PATH=${LL_PATH}")
|
||||
set (H5_HAVE_LIBMFU 1)
|
||||
set (H5_HAVE_MFU_H 1)
|
||||
set (CMAKE_REQUIRED_INCLUDES "${MFU_INCLUDE_DIR}")
|
||||
set (MFU_LIBRARY_DEBUG "$MFU_LIBRARY")
|
||||
set (MFU_LIBRARY_RELEASE "$MFU_LIBRARY")
|
||||
endif ()
|
||||
find_package(CIRCLE REQUIRED)
|
||||
if (CIRCLE_FOUND)
|
||||
set (H5_HAVE_LIBCIRCLE 1)
|
||||
set (H5_HAVE_CIRCLE_H 1)
|
||||
set (CMAKE_REQUIRED_INCLUDES "${CIRCLE_INCLUDE_DIR}")
|
||||
endif ()
|
||||
find_package(DTCMP REQUIRED)
|
||||
if (DTCMP_FOUND)
|
||||
set (H5_HAVE_LIBDTCMP 1)
|
||||
set (H5_HAVE_DTCMP_H 1)
|
||||
set (CMAKE_REQUIRED_INCLUDES "${DTCMP_INCLUDE_DIR}")
|
||||
endif ()
|
||||
endif ()
|
||||
add_subdirectory (utils)
|
||||
endif ()
|
||||
|
||||
|
||||
20
MANIFEST
20
MANIFEST
@ -202,6 +202,10 @@
|
||||
./config/intel-warnings/win-general
|
||||
./config/intel-warnings/ifort-general
|
||||
|
||||
./config/cmake/FindMFU.cmake
|
||||
./config/cmake/FindDTCMP.cmake
|
||||
./config/cmake/FindCIRCLE.cmake
|
||||
|
||||
./config/site-specific/BlankForm
|
||||
|
||||
./doc/branches-explained.md
|
||||
@ -3001,6 +3005,22 @@
|
||||
./utils/test/Makefile.am
|
||||
./utils/test/swmr_check_compat_vfd.c
|
||||
|
||||
# parallel tools (h5dwalk) and tests
|
||||
./utils/tools/CMakeLists.txt
|
||||
./utils/tools/Makefile.am
|
||||
./utils/tools/h5dwalk/CMakeLists.txt
|
||||
./utils/tools/h5dwalk/Makefile.am
|
||||
./utils/tools/h5dwalk/h5dwalk.1
|
||||
./utils/tools/h5dwalk/h5dwalk.c
|
||||
./utils/tools/test/CMakeLists.txt
|
||||
./utils/tools/test/Makefile.am
|
||||
./utils/tools/test/h5dwalk/CMakeLists.txt
|
||||
./utils/tools/test/h5dwalk/CMakeTests.cmake
|
||||
./utils/tools/test/h5dwalk/Makefile.am
|
||||
./utils/tools/test/h5dwalk/copy_demo_files.sh.in
|
||||
./utils/tools/test/h5dwalk/help.h5dwalk
|
||||
./utils/tools/test/h5dwalk/testh5dwalk.sh.in
|
||||
|
||||
# high level libraries
|
||||
./hl/Makefile.am
|
||||
./hl/examples/Makefile.am
|
||||
|
||||
@ -88,9 +88,9 @@ else
|
||||
TOOLS_DIR=
|
||||
endif
|
||||
|
||||
SUBDIRS = src $(TESTSERIAL_DIR) $(TESTPARALLEL_DIR) bin utils $(TOOLS_DIR) . \
|
||||
SUBDIRS = src $(TESTSERIAL_DIR) $(TESTPARALLEL_DIR) bin $(TOOLS_DIR) utils . \
|
||||
$(CXX_DIR) $(FORTRAN_DIR) $(JAVA_DIR) $(HDF5_HL_DIR)
|
||||
DIST_SUBDIRS = src test testpar utils tools . c++ fortran hl examples java
|
||||
DIST_SUBDIRS = src test testpar tools utils . c++ fortran hl examples java
|
||||
|
||||
# Some files generated during configure that should be cleaned
|
||||
DISTCLEANFILES=config/stamp1 config/stamp2
|
||||
|
||||
49
config/cmake/FindCIRCLE.cmake
Normal file
49
config/cmake/FindCIRCLE.cmake
Normal file
@ -0,0 +1,49 @@
|
||||
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
|
||||
# file Copyright.txt or https://cmake.org/licensing for details.
|
||||
|
||||
#[=======================================================================[.rst:
|
||||
FindCIRCLE
|
||||
--------
|
||||
|
||||
Find the native CIRCLE includes and library
|
||||
|
||||
This module defines
|
||||
|
||||
::
|
||||
|
||||
CIRCLE_INCLUDE_DIR, where to find CIRCLE.h, etc.
|
||||
CIRCLE_LIBRARIES, the libraries required to use CIRCLE.
|
||||
CIRCLE_FOUND, If false, do not try to use CIRCLE.
|
||||
|
||||
also defined, but not for general use are
|
||||
|
||||
::
|
||||
|
||||
CIRCLE_LIBRARY, where to find the CIRCLE library.
|
||||
#]=======================================================================]
|
||||
|
||||
if(DEFINED ENV{MFU_ROOT})
|
||||
set(ENV{MFU_INCLUDE} "$ENV{MFU_ROOT}/include")
|
||||
set(ENV{MFU_LIB} "$ENV{MFU_ROOT}/lib")
|
||||
set(ENV{MFU_LIB64} "$ENV{MFU_ROOT}/lib64")
|
||||
else()
|
||||
message("CIRCLE_LIBRARY: If you have problems building this library,\nconsider setting the MFU_ROOT environment variable to indicate\nwhere to find the support libraries and header files!")
|
||||
endif()
|
||||
|
||||
find_path(CIRCLE_INCLUDE_DIR
|
||||
NAMES libcircle.h
|
||||
HINTS ENV MFU_INCLUDE)
|
||||
|
||||
find_library(CIRCLE_LIBRARY
|
||||
NAMES circle
|
||||
HINTS ENV MFU_LIB ENV MFU_LIB64
|
||||
)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(CIRCLE REQUIRED_VARS CIRCLE_LIBRARY CIRCLE_INCLUDE_DIR)
|
||||
|
||||
if(CIRCLE_FOUND)
|
||||
set(CIRCLE_LIBRARIES ${CIRCLE_LIBRARY} )
|
||||
endif()
|
||||
|
||||
mark_as_advanced(CIRCLE_INCLUDE_DIR CIRCLE_LIBRARY)
|
||||
48
config/cmake/FindDTCMP.cmake
Normal file
48
config/cmake/FindDTCMP.cmake
Normal file
@ -0,0 +1,48 @@
|
||||
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
|
||||
# file Copyright.txt or https://cmake.org/licensing for details.
|
||||
|
||||
#[=======================================================================[.rst:
|
||||
FindDTCMP
|
||||
--------
|
||||
|
||||
Find the native DTCMP includes and library
|
||||
|
||||
This module defines
|
||||
|
||||
::
|
||||
|
||||
DTCMP_INCLUDE_DIR, where to find DTCMP.h, etc.
|
||||
DTCMP_LIBRARIES, the libraries required to use DTCMP.
|
||||
DTCMP_FOUND, If false, do not try to use DTCMP.
|
||||
|
||||
also defined, but not for general use are
|
||||
|
||||
::
|
||||
|
||||
DTCMP_LIBRARY, where to find the DTCMP library.
|
||||
#]=======================================================================]
|
||||
|
||||
if(DEFINED ENV{MFU_ROOT})
|
||||
set(ENV{MFU_INCLUDE} "$ENV{MFU_ROOT}/include")
|
||||
set(ENV{MFU_LIB} "$ENV{MFU_ROOT}/lib")
|
||||
set(ENV{MFU_LIB64} "$ENV{MFU_ROOT}/lib64")
|
||||
else()
|
||||
message("DTCMP_LIBRARY: If you have problems building this library,\nconsider setting the MFU_ROOT environment variable to indicate\nwhere to find the support libraries and header files!")
|
||||
endif()
|
||||
|
||||
find_path(DTCMP_INCLUDE_DIR
|
||||
NAMES dtcmp.h
|
||||
HINTS ENV MFU_INCLUDE)
|
||||
|
||||
find_library(DTCMP_LIBRARY
|
||||
NAMES dtcmp
|
||||
HINTS ENV MFU_LIB ENV MFU_LIB64)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(DTCMP REQUIRED_VARS DTCMP_LIBRARY DTCMP_INCLUDE_DIR)
|
||||
|
||||
if(DTCMP_FOUND)
|
||||
set(DTCMP_LIBRARIES ${DTCMP_LIBRARY} )
|
||||
endif()
|
||||
|
||||
mark_as_advanced(DTCMP_INCLUDE_DIR DTCMP_LIBRARY)
|
||||
100
config/cmake/FindMFU.cmake
Normal file
100
config/cmake/FindMFU.cmake
Normal file
@ -0,0 +1,100 @@
|
||||
#
|
||||
# Copyright by The HDF Group.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is part of HDF5. The full HDF5 copyright notice, including
|
||||
# terms governing use, modification, and redistribution, is contained in
|
||||
# the COPYING file, which can be found at the root of the source code
|
||||
# distribution tree, or in https://www.hdfgroup.org/licenses.
|
||||
# If you do not have access to either file, you may request a copy from
|
||||
# help@hdfgroup.org.
|
||||
#
|
||||
#########################################################################
|
||||
|
||||
# - Derived from the FindTiff.cmake and FindJPEG.cmake that is included with cmake
|
||||
# FindMFU
|
||||
|
||||
# Find the native MFU includes and library
|
||||
|
||||
# Imported targets
|
||||
##################
|
||||
|
||||
# This module defines the following :prop_tgt:`IMPORTED` targets:
|
||||
#
|
||||
# MFU::MFU
|
||||
# The MFU library, if found.
|
||||
#
|
||||
# Result variables
|
||||
###################
|
||||
|
||||
# This module will set the following variables in your project:
|
||||
|
||||
# MFU_FOUND, true if the MFU headers and libraries were found.
|
||||
# MFU_INCLUDE_DIR, the directory containing the MFU headers.
|
||||
# MFU_INCLUDE_DIRS, the directory containing the MFU headers.
|
||||
# MFU_LIBRARIES, libraries to link against to use MFU.
|
||||
|
||||
# Cache variables
|
||||
#################
|
||||
|
||||
# The following variables may also be set:
|
||||
|
||||
# MFU_LIBRARY, where to find the MFU library.
|
||||
# message (STATUS "Finding MFU library and headers..." )
|
||||
#########################################################################
|
||||
|
||||
|
||||
|
||||
FIND_PATH(MFU_INCLUDE_DIR
|
||||
NAMES mfu.h
|
||||
HINTS "$ENV{MFU_ROOT}/include"
|
||||
)
|
||||
FIND_LIBRARY(MFU_LIBRARY
|
||||
NAMES mfu
|
||||
HINTS "$ENV{MFU_ROOT}/lib64"
|
||||
)
|
||||
|
||||
if(NOT MFU_LIBRARY)
|
||||
set(mfu_names ${MFU_NAMES} mfu libmfu)
|
||||
find_library(MFU_LIBRARY NAMES ${mfu_names})
|
||||
include(SelectLibraryConfigurations)
|
||||
select_library_configurations(MFU)
|
||||
endif()
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(MFU
|
||||
REQUIRED_VARS MFU_LIBRARY MFU_INCLUDE_DIR)
|
||||
|
||||
if(MFU_FOUND)
|
||||
set(MFU_LIBRARIES "${MFU_LIBRARY}")
|
||||
set(MFU_INCLUDE_DIRS "${MFU_INCLUDE_DIR}")
|
||||
set(LL_PATH "$ENV{MFU_ROOT}/lib64:$ENV{MFU_ROOT}/lib")
|
||||
if(NOT TARGET MFU::MFU)
|
||||
add_library(MFU::MFU UNKNOWN IMPORTED)
|
||||
if(MFU_INCLUDE_DIRS)
|
||||
set_target_properties(MFU::MFU PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${MFU_INCLUDE_DIRS}")
|
||||
endif()
|
||||
if(EXISTS "${MFU_LIBRARY}")
|
||||
set_target_properties(MFU::MFU PROPERTIES
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
|
||||
IMPORTED_LOCATION "${MFU_LIBRARY}")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Report the results.
|
||||
if (NOT MFU_FOUND)
|
||||
set (MFU_DIR_MESSAGE
|
||||
"Mfu was not found. Make sure MFU_LIBRARY and MFU_INCLUDE_DIR are set or set the MFU_INSTALL environment variable."
|
||||
)
|
||||
if (NOT MFU_FIND_QUIETLY)
|
||||
if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0")
|
||||
message (VERBOSE "${MFU_DIR_MESSAGE}")
|
||||
endif ()
|
||||
else ()
|
||||
if (MFU_FIND_REQUIRED)
|
||||
message (FATAL_ERROR "Mfu was NOT found and is Required by this project")
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
142
configure.ac
142
configure.ac
@ -1574,6 +1574,7 @@ case "X-$withval" in
|
||||
;;
|
||||
esac
|
||||
|
||||
|
||||
## ----------------------------------------------------------------------
|
||||
## Make the external filters list available to *.in files
|
||||
## At this point it's unset (no external filters by default) but it
|
||||
@ -1682,6 +1683,7 @@ fi
|
||||
## command-line switch. The value is an include path and/or a library path.
|
||||
## If the library path is specified then it must be preceded by a comma.
|
||||
##
|
||||
AC_SUBST([LL_PATH])
|
||||
AC_SUBST([USE_FILTER_SZIP]) USE_FILTER_SZIP="no"
|
||||
AC_ARG_WITH([szlib],
|
||||
[AS_HELP_STRING([--with-szlib=DIR],
|
||||
@ -1766,7 +1768,7 @@ if test "x$HAVE_SZLIB" = "xyes" -a "x$HAVE_SZLIB_H" = "xyes"; then
|
||||
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$szlib_lib"
|
||||
fi
|
||||
|
||||
AC_SUBST([LL_PATH]) LL_PATH="$LD_LIBRARY_PATH"
|
||||
LL_PATH="$LD_LIBRARY_PATH"
|
||||
|
||||
AC_CACHE_VAL([hdf5_cv_szlib_can_encode],
|
||||
[AC_RUN_IFELSE(
|
||||
@ -3031,6 +3033,138 @@ if test -n "$PARALLEL"; then
|
||||
|
||||
fi
|
||||
|
||||
## ----------------------------------------------------------------------
|
||||
## Build parallel tools if parallel tools, parallel, and build tools options
|
||||
## are all enabled.
|
||||
##
|
||||
AC_SUBST([PARALLEL_TOOLS])
|
||||
|
||||
## Default is no parallel tools
|
||||
PARALLEL_TOOLS=no
|
||||
|
||||
AC_MSG_CHECKING([parallel tools])
|
||||
AC_ARG_ENABLE([parallel-tools],
|
||||
[AS_HELP_STRING([--enable-parallel-tools],
|
||||
[Enable building parallel tools.
|
||||
[default=no]])],
|
||||
[PARALLEL_TOOLS=$enableval])
|
||||
|
||||
if test "X${PARALLEL_TOOLS}" = "Xyes"; then
|
||||
if test "X${HDF5_TOOLS}" != "Xyes"; then
|
||||
AC_MSG_ERROR([--enable-tools is required for --enable-parallel-tools])
|
||||
fi
|
||||
if test "X${PARALLEL}" != "Xyes"; then
|
||||
AC_MSG_ERROR([--enable-parallel is required for --enable-parallel-tools])
|
||||
fi
|
||||
fi
|
||||
|
||||
case "X-$PARALLEL_TOOLS" in
|
||||
X-|X-no)
|
||||
AC_MSG_RESULT([no])
|
||||
;;
|
||||
X-yes)
|
||||
AC_MSG_RESULT([yes])
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
|
||||
if test "X${PARALLEL_TOOLS}" = "Xyes"; then
|
||||
## Is the mpiFileUtils library (libmfu) required and available?
|
||||
##
|
||||
AC_SUBST([H5DWALK_LDFLAGS])
|
||||
AC_SUBST([H5DWALK_LIBS])
|
||||
AC_SUBST([H5DWALK_CPPFLAGS])
|
||||
|
||||
H5DWALK_LDFLAGS="${H5DWALK_LDFLAGS}"
|
||||
|
||||
## Default is not present
|
||||
HAVE_LIBMFU=no
|
||||
|
||||
AC_ARG_WITH([libmfu],
|
||||
[AS_HELP_STRING([--with-libmfu=DIR],
|
||||
[Use the libmfu library [default=no]])],,
|
||||
[withval=no])
|
||||
|
||||
case "X-$withval" in
|
||||
X-yes)
|
||||
HAVE_LIBMFU="yes"
|
||||
AC_CHECK_HEADERS([mfu.h],, [unset HAVE_LIBMFU])
|
||||
if test "x$HAVE_LIBMFU" = "xyes"; then
|
||||
AC_CHECK_LIB([mfu], [mfu_init], [H5DWALK_LIBS="-lmfu"], [unset HAVE_LIBMFU])
|
||||
fi
|
||||
if test -z "$HAVE_LIBMFU" -a -n "$HDF5_CONFIG_ABORT"; then
|
||||
AC_MSG_ERROR([couldn't find libmfu library])
|
||||
fi
|
||||
;;
|
||||
X-|X-no|X-none)
|
||||
HAVE_LIBMFU="no"
|
||||
AC_MSG_CHECKING([for libmfu library])
|
||||
AC_MSG_RESULT([suppressed])
|
||||
;;
|
||||
*)
|
||||
HAVE_LIBMFU="yes"
|
||||
case "$withval" in
|
||||
*,*)
|
||||
libmfu_inc="`echo $withval |cut -f1 -d,`"
|
||||
libmfu_lib="`echo $withval |cut -f2 -d, -s`"
|
||||
;;
|
||||
*)
|
||||
if test -n "$withval"; then
|
||||
libmfu_inc="$withval/include"
|
||||
libmfu_lib="$withval/lib64"
|
||||
libcircle_lib="$withval/lib"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
saved_CPPFLAGS="$CPPFLAGS"
|
||||
saved_AM_CPPFLAGS="$AM_CPPFLAGS"
|
||||
saved_LDFLAGS="$LDFLAGS"
|
||||
saved_AM_LDFLAGS="$AM_LDFLAGS"
|
||||
|
||||
## For these checks we need the libmfu locations added to CPPFLAGS,
|
||||
## AM_CPPFLAGS, LDFLAGS, and AM_LDFLAGS. The third param should set them
|
||||
## back to these saved values. If the checks pass, then normally these four
|
||||
## flag variables would be updated, but in this case we put the changes in
|
||||
## variables specific to H5DWALK since they aren't used elsewhere.
|
||||
if test -n "$libmfu_inc"; then
|
||||
CPPFLAGS="$CPPFLAGS -I$libmfu_inc"
|
||||
AM_CPPFLAGS="$AM_CPPFLAGS -I$libmfu_inc"
|
||||
fi
|
||||
|
||||
if test -n "$libmfu_lib"; then
|
||||
LDFLAGS="$LDFLAGS -L$libmfu_lib -L$libcircle_lib"
|
||||
AM_LDFLAGS="$AM_LDFLAGS -L$libmfu_lib -L$libcircle_lib"
|
||||
fi
|
||||
|
||||
if test "x$HAVE_LIBMFU" = "xyes"; then
|
||||
AC_CHECK_LIB([mfu], [mfu_init],[H5DWALK_LIBS="-lmfu"], [CPPFLAGS="$saved_CPPFLAGS"; AM_CPPFLAGS="$saved_AM_CPPFLAGS"; LDFLAGS="$saved_LDFLAGS"; AM_LDFLAGS="$saved_AM_LDFLAGS"; unset HAVE_LIBMFU])
|
||||
if test -n "$HAVE_LIBMFU"; then
|
||||
AC_CHECK_HEADERS([mfu.h],[H5DWALK_CPPFLAGS="-I$libmfu_inc"],[CPPFLAGS="$saved_CPPFLAGS"; AM_CPPFLAGS="$saved_AM_CPPFLAGS"; LDFLAGS="$saved_LDFLAGS"; AM_LDFLAGS="$saved_AM_LDFLAGS"; unset HAVE_LIBMFU])
|
||||
fi
|
||||
fi
|
||||
|
||||
if test -z "$HAVE_LIBMFU" -a -n "$HDF5_CONFIG_ABORT"; then
|
||||
AC_MSG_ERROR([couldn't find libmfu library])
|
||||
else
|
||||
H5DWALK_LDFLAGS="-L$libmfu_lib -L$libcircle_lib"
|
||||
fi
|
||||
|
||||
if test -z "$LD_LIBRARY_PATH"; then
|
||||
export LD_LIBRARY_PATH="$libmfu_lib:$libcircle_lib"
|
||||
else
|
||||
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$libmfu_lib:$libcircle_lib"
|
||||
fi
|
||||
|
||||
LL_PATH="$LD_LIBRARY_PATH"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
##
|
||||
AM_CONDITIONAL([PARALLEL_TOOLS_CONDITIONAL], [test "X$HAVE_LIBMFU" = "Xyes"])
|
||||
|
||||
## ----------------------------------------------------------------------
|
||||
## Check if the map API is enabled by --enable-map-api
|
||||
##
|
||||
@ -4037,6 +4171,12 @@ AC_CONFIG_FILES([src/libhdf5.settings
|
||||
utils/Makefile
|
||||
utils/mirror_vfd/Makefile
|
||||
utils/test/Makefile
|
||||
utils/tools/Makefile
|
||||
utils/tools/h5dwalk/Makefile
|
||||
utils/tools/test/Makefile
|
||||
utils/tools/test/h5dwalk/Makefile
|
||||
utils/tools/test/h5dwalk/copy_demo_files.sh
|
||||
utils/tools/test/h5dwalk/testh5dwalk.sh
|
||||
tools/Makefile
|
||||
tools/lib/Makefile
|
||||
tools/libtest/Makefile
|
||||
|
||||
@ -47,6 +47,12 @@ New Features
|
||||
|
||||
Configuration:
|
||||
-------------
|
||||
- Added new configure option to support building parallel tools.
|
||||
See Tools below (autotools - CMake):
|
||||
--enable-parallel-tools HDF5_BUILD_PARALLEL_TOOLS
|
||||
|
||||
(RAW - 2021/10/25)
|
||||
|
||||
- Added new configure options to enable dimension scales APIs (H5DS*) to
|
||||
use new object references with the native VOL connector (aka native HDF5
|
||||
library). New references are always used for non-native terminal VOL
|
||||
@ -929,6 +935,19 @@ New Features
|
||||
|
||||
Tools:
|
||||
------
|
||||
- Added a new (unix ONLY) parallel meta tool 'h5dwalk', which utilizes the
|
||||
mpifileutils (https://hpc.github.io/mpifileutils) open source utility
|
||||
library to enable parallel execution of other HDF5 tools.
|
||||
This approach can greatly enhance the serial hdf5 tool performance over large
|
||||
collections of files by utilizing MPI parallelism to distribute an application
|
||||
load over many independent MPI ranks and files.
|
||||
|
||||
An introduction to the mpifileutils library and initial 'User Guide' for
|
||||
the new 'h5dwalk" tool can be found at:
|
||||
https://github.com/HDFGroup/hdf5doc/tree/master/RFCs/HDF5/tools/parallel_tools
|
||||
|
||||
(RAW - 2021/10/25)
|
||||
|
||||
- Refactored the perform tools and removed depends on test library.
|
||||
|
||||
Moved the perf and h5perf tools from tools/test/perform to
|
||||
|
||||
@ -9,3 +9,8 @@ option (HDF5_BUILD_UTILS "Build HDF5 Utils" ON)
|
||||
if (HDF5_BUILD_UTILS)
|
||||
add_subdirectory (mirror_vfd)
|
||||
endif ()
|
||||
|
||||
#-- Add the h5dwalk and test executables
|
||||
if (HDF5_BUILD_PARALLEL_TOOLS AND HDF5_ENABLE_PARALLEL)
|
||||
add_subdirectory(tools)
|
||||
endif()
|
||||
|
||||
@ -32,7 +32,13 @@ else
|
||||
MIRROR_VFD_DIR=
|
||||
endif
|
||||
|
||||
if BUILD_TOOLS_CONDITIONAL
|
||||
TOOLS_DIR =tools
|
||||
else
|
||||
TOOLS_DIR=
|
||||
endif
|
||||
|
||||
# All subdirectories
|
||||
SUBDIRS=$(MIRROR_VFD_DIR) $(TESTUTIL_DIR)
|
||||
SUBDIRS=$(MIRROR_VFD_DIR) $(TESTUTIL_DIR) $(TOOLS_DIR)
|
||||
|
||||
include $(top_srcdir)/config/conclude.am
|
||||
|
||||
12
utils/tools/CMakeLists.txt
Normal file
12
utils/tools/CMakeLists.txt
Normal file
@ -0,0 +1,12 @@
|
||||
cmake_minimum_required (VERSION 3.12)
|
||||
project (HDF5_UTILS_TOOLS C)
|
||||
|
||||
|
||||
if (HDF5_BUILD_PARALLEL_TOOLS)
|
||||
add_subdirectory (h5dwalk)
|
||||
endif()
|
||||
|
||||
#-- Add the tests
|
||||
if (BUILD_TESTING)
|
||||
add_subdirectory (test)
|
||||
endif()
|
||||
38
utils/tools/Makefile.am
Normal file
38
utils/tools/Makefile.am
Normal file
@ -0,0 +1,38 @@
|
||||
#
|
||||
# Copyright by The HDF Group.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is part of HDF5. The full HDF5 copyright notice, including
|
||||
# terms governing use, modification, and redistribution, is contained in
|
||||
# the COPYING file, which can be found at the root of the source code
|
||||
# distribution tree, or in https://www.hdfgroup.org/licenses.
|
||||
# If you do not have access to either file, you may request a copy from
|
||||
# help@hdfgroup.org.
|
||||
##
|
||||
## Makefile.am
|
||||
## Run automake to generate a Makefile.in from this file.
|
||||
##
|
||||
#
|
||||
# Tools HDF5 Makefile(.in)
|
||||
#
|
||||
|
||||
include $(top_srcdir)/config/commence.am
|
||||
|
||||
if PARALLEL_TOOLS_CONDITIONAL
|
||||
H5DWALK=h5dwalk
|
||||
else
|
||||
H5DWALK=
|
||||
endif
|
||||
|
||||
if BUILD_TESTS_CONDITIONAL
|
||||
TESTSERIAL_DIR =test
|
||||
else
|
||||
TESTSERIAL_DIR=
|
||||
endif
|
||||
|
||||
CONFIG=ordered
|
||||
|
||||
# All subdirectories
|
||||
SUBDIRS=$(H5DWALK) $(TESTSERIAL_DIR)
|
||||
|
||||
include $(top_srcdir)/config/conclude.am
|
||||
66
utils/tools/h5dwalk/CMakeLists.txt
Normal file
66
utils/tools/h5dwalk/CMakeLists.txt
Normal file
@ -0,0 +1,66 @@
|
||||
cmake_minimum_required (VERSION 3.12)
|
||||
project (HDF5_UTILS_TOOLS_H5DWALK C)
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# Add the h5dwalk and test executables
|
||||
# --------------------------------------------------------------------
|
||||
if (NOT ONLY_SHARED_LIBS)
|
||||
add_executable (h5dwalk ${HDF5_UTILS_TOOLS_H5DWALK_SOURCE_DIR}/h5dwalk.c)
|
||||
# add_custom_target(generate_demo ALL
|
||||
# DEPENDS "${HDF5_TOOLS_DIR}/test/demo_destfiles.test"
|
||||
# )
|
||||
target_include_directories (h5dwalk PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
|
||||
target_compile_options(h5dwalk PRIVATE "${HDF5_CMAKE_C_FLAGS}")
|
||||
TARGET_C_PROPERTIES (h5dwalk STATIC)
|
||||
target_link_libraries (h5dwalk PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} ${MFU_LIBRARY} "$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_LIBRARIES}>")
|
||||
set_target_properties (h5dwalk PROPERTIES FOLDER tools)
|
||||
set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5dwalk")
|
||||
|
||||
set (H5_DEP_EXECUTABLES h5dwalk)
|
||||
endif ()
|
||||
|
||||
if (BUILD_SHARED_LIBS)
|
||||
add_executable (h5dwalk-shared ${HDF5_UTILS_TOOLS_H5DWALK_SOURCE_DIR}/h5dwalk.c)
|
||||
target_include_directories (h5dwalk-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
|
||||
target_compile_options(h5dwalk-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}")
|
||||
TARGET_C_PROPERTIES (h5dwalk-shared SHARED)
|
||||
target_link_libraries (h5dwalk-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} ${MFU_LIBRARY} "$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_LIBRARIES}>")
|
||||
set_target_properties (h5dwalk-shared PROPERTIES FOLDER tools)
|
||||
set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5dwalk-shared")
|
||||
|
||||
set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} h5dwalk-shared)
|
||||
endif ()
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Add Target to clang-format
|
||||
#-----------------------------------------------------------------------------
|
||||
if (HDF5_ENABLE_FORMATTERS)
|
||||
if (NOT ONLY_SHARED_LIBS)
|
||||
clang_format (HDF5_H5DWALK_SRC_FORMAT h5dwalk)
|
||||
else ()
|
||||
clang_format (HDF5_H5DWALK_SRC_FORMAT h5dwalk-shared)
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
##############################################################################
|
||||
##############################################################################
|
||||
### I N S T A L L A T I O N ###
|
||||
##############################################################################
|
||||
##############################################################################
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Rules for Installation of tools using make Install target
|
||||
#-----------------------------------------------------------------------------
|
||||
if (HDF5_EXPORTED_TARGETS)
|
||||
foreach (exec ${H5_DEP_EXECUTABLES})
|
||||
INSTALL_PROGRAM_PDB (${exec} ${HDF5_INSTALL_BIN_DIR} toolsapplications)
|
||||
endforeach ()
|
||||
|
||||
install (
|
||||
TARGETS
|
||||
${H5_DEP_EXECUTABLES}
|
||||
EXPORT
|
||||
${HDF5_EXPORTED_TARGETS}
|
||||
RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications
|
||||
)
|
||||
endif ()
|
||||
37
utils/tools/h5dwalk/Makefile.am
Normal file
37
utils/tools/h5dwalk/Makefile.am
Normal file
@ -0,0 +1,37 @@
|
||||
#
|
||||
# Copyright by The HDF Group.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is part of HDF5. The full HDF5 copyright notice, including
|
||||
# terms governing use, modification, and redistribution, is contained in
|
||||
# the COPYING file, which can be found at the root of the source code
|
||||
# distribution tree, or in https://www.hdfgroup.org/licenses.
|
||||
# If you do not have access to either file, you may request a copy from
|
||||
# help@hdfgroup.org.
|
||||
##
|
||||
## Makefile.am
|
||||
## Run automake to generate a Makefile.in from this file.
|
||||
#
|
||||
# HDF5 Library Makefile(.in)
|
||||
#
|
||||
|
||||
include $(top_srcdir)/config/commence.am
|
||||
|
||||
# Include src directory
|
||||
AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib $(H5DWALK_CPPFLAGS)
|
||||
|
||||
# These are our main targets, the tools
|
||||
# h5dwalk_SOURCES=h5dwalk.c $(TOOLSOURCES)
|
||||
bin_PROGRAMS=h5dwalk
|
||||
#bin_SCRIPTS=install-examples
|
||||
|
||||
# Add h5stat specific linker flags here
|
||||
h5dwalk_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) $(H5DWALK_LDFLAGS)
|
||||
|
||||
# Tell automake to clean h5redeploy script
|
||||
CLEANFILES=
|
||||
|
||||
# All programs rely on hdf5 library and h5tools library
|
||||
h5dwalk_LDADD=$(LIBH5TOOLS) $(LIBHDF5) $(H5DWALK_LIBS)
|
||||
|
||||
include $(top_srcdir)/config/conclude.am
|
||||
42
utils/tools/h5dwalk/h5dwalk.1
Normal file
42
utils/tools/h5dwalk/h5dwalk.1
Normal file
@ -0,0 +1,42 @@
|
||||
.TH "h5dwalk" 1
|
||||
.SH NAME
|
||||
h5dwalk \- Provides a means of extending HDF5 tools by using parallelism on groups of files.
|
||||
.SH SYNOPSIS
|
||||
h5dwalk [OPTIONS] -T h5tool [H5TOOL_options...]
|
||||
.SH DESCRIPTION
|
||||
h5dwalk utilizes the mpiFileUtils library to invoke a selected HDF5 tool on a collection of files. The mpiFileUtils library provides the facilities to walk directory trees and provide a selection of files contained therein. This selection can be filtered in various ways. At present, h5dwalk filters the original file selection to include only HDF5 formatted files. The resulting collection or collections can be utilized as the file inputs to the selected h5tool.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.B \-h
|
||||
or
|
||||
.B \-\-help
|
||||
Print a usage message and exit.
|
||||
.TP
|
||||
.B \-i
|
||||
or
|
||||
.B \-\-input filename
|
||||
Read command input from a file. Not yet implemented.
|
||||
.TP
|
||||
.B \-o
|
||||
or
|
||||
.B \-\-output filename
|
||||
Captures the hdf5 tool output into a named file.
|
||||
.TP
|
||||
.B \-l
|
||||
or
|
||||
.B \-\-log [file]
|
||||
Captures hdf5 tool output into a individual log files. If an optional file (directory) is specified, then output from all tool instances will be written in the given file directory. Without the optional filename, each tool instance ouput will be captured in a new log file whose name is associated with the hdf5 tool that was run and is written in the current working directory.
|
||||
.TP
|
||||
.B \-E
|
||||
or
|
||||
.B \-\-error [file]
|
||||
Show all HDF5 error reporting. Behavior is similar to --log, i.e. errors can either be logged in a single named file or in individual tool specific files. Not yet implemented.
|
||||
.TP
|
||||
.B \-T
|
||||
or
|
||||
.B \-\-tool hdf5_tool
|
||||
Specifies the hdf5 tool that should be invoked for each file in a collection of files. The collection consists of individual HDF5 files found by walking a specified directory tree which is used in place of the normal tool filename argument. The '-T' option should appear on the command line just prior to the HDF5 tool argument options.
|
||||
.TP
|
||||
.SH "SEE ALSO"
|
||||
\&\fIh5dump\fR\|(1), \fIh5diff\fR\|(1), \fIh5repart\fR\|(1), \fIh5diff\fR\|(1),
|
||||
\&\fIh5import\fR\|(1), \fIgif2h5\fR\|(1), \fIh52gif\fR\|(1), \fIh5perf\fR\|(1)
|
||||
1712
utils/tools/h5dwalk/h5dwalk.c
Normal file
1712
utils/tools/h5dwalk/h5dwalk.c
Normal file
File diff suppressed because it is too large
Load Diff
8
utils/tools/test/CMakeLists.txt
Normal file
8
utils/tools/test/CMakeLists.txt
Normal file
@ -0,0 +1,8 @@
|
||||
cmake_minimum_required (VERSION 3.12)
|
||||
project (HDF5_TOOLS_TEST C)
|
||||
|
||||
#-- Add the h5diff tests
|
||||
if (HDF5_BUILD_PARALLEL_TOOLS)
|
||||
add_subdirectory (h5dwalk)
|
||||
endif()
|
||||
|
||||
32
utils/tools/test/Makefile.am
Normal file
32
utils/tools/test/Makefile.am
Normal file
@ -0,0 +1,32 @@
|
||||
#
|
||||
# Copyright by The HDF Group.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is part of HDF5. The full HDF5 copyright notice, including
|
||||
# terms governing use, modification, and redistribution, is contained in
|
||||
# the COPYING file, which can be found at the root of the source code
|
||||
# distribution tree, or in https://www.hdfgroup.org/licenses.
|
||||
# If you do not have access to either file, you may request a copy from
|
||||
# help@hdfgroup.org.
|
||||
##
|
||||
## Makefile.am
|
||||
## Run automake to generate a Makefile.in from this file.
|
||||
##
|
||||
#
|
||||
# Tools HDF5 Makefile(.in)
|
||||
#
|
||||
|
||||
include $(top_srcdir)/config/commence.am
|
||||
|
||||
if PARALLEL_TOOLS_CONDITIONAL
|
||||
H5DWALK=h5dwalk
|
||||
else
|
||||
H5DWALK=
|
||||
endif
|
||||
|
||||
CONFIG=ordered
|
||||
|
||||
# All subdirectories
|
||||
SUBDIRS=$(H5DWALK)
|
||||
|
||||
include $(top_srcdir)/config/conclude.am
|
||||
15
utils/tools/test/h5dwalk/CMakeLists.txt
Normal file
15
utils/tools/test/h5dwalk/CMakeLists.txt
Normal file
@ -0,0 +1,15 @@
|
||||
cmake_minimum_required (VERSION 3.12)
|
||||
project (HDF5_TOOLS_TEST_H5DWALK)
|
||||
|
||||
if (HDF5_BUILD_PARALLEL_TOOLS)
|
||||
add_custom_command(
|
||||
OUTPUT ${HDF5_TOOLS_DIR}/test/demo_destfiles.test
|
||||
COMMAND bash -c ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh
|
||||
ARGS ${HDF5_TOOLS_DIR}/test ${CMAKE_BINARY_DIR}/bin
|
||||
DEPENDS ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh
|
||||
)
|
||||
endif ()
|
||||
|
||||
if (HDF5_TEST_TOOLS AND HDF5_TEST_SERIAL)
|
||||
include (CMakeTests.cmake)
|
||||
endif ()
|
||||
56
utils/tools/test/h5dwalk/CMakeTests.cmake
Normal file
56
utils/tools/test/h5dwalk/CMakeTests.cmake
Normal file
@ -0,0 +1,56 @@
|
||||
#
|
||||
# Copyright by The HDF Group.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is part of HDF5. The full HDF5 copyright notice, including
|
||||
# terms governing use, modification, and redistribution, is contained in
|
||||
# the COPYING file, which can be found at the root of the source code
|
||||
# distribution tree, or in https://www.hdfgroup.org/licenses.
|
||||
# If you do not have access to either file, you may request a copy from
|
||||
# help@hdfgroup.org.
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
##############################################################################
|
||||
### T E S T I N G ###
|
||||
##############################################################################
|
||||
##############################################################################
|
||||
|
||||
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles")
|
||||
|
||||
|
||||
##############################################################################
|
||||
##############################################################################
|
||||
### T H E T E S T S M A C R O S ###
|
||||
##############################################################################
|
||||
##############################################################################
|
||||
|
||||
macro (ADD_H5_TEST resultfile resultcode)
|
||||
# If using memchecker add tests without using scripts
|
||||
if (HDF5_ENABLE_USING_MEMCHECKER)
|
||||
message("Entered ADD_H5_TEST - 0")
|
||||
add_test (NAME H5DWALK-${resultfile} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5dwalk${tgt_file_ext}> ${ARGN})
|
||||
set_tests_properties (H5DWALK-${resultfile} PROPERTIES
|
||||
WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles")
|
||||
if ("${resultcode}" STREQUAL "1")
|
||||
set_tests_properties (H5DWALK-${resultfile} PROPERTIES WILL_FAIL "true")
|
||||
endif ()
|
||||
else ()
|
||||
# Remove any output file left over from previous test run
|
||||
add_test (
|
||||
NAME H5DWALK-${resultfile}
|
||||
COMMAND "${CMAKE_COMMAND}"
|
||||
-D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}"
|
||||
-D "TEST_PROGRAM=$<TARGET_FILE:h5dwalk${tgt_file_ext}>"
|
||||
-D "TEST_ARGS=${ARGN}"
|
||||
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles"
|
||||
-D "TEST_OUTPUT=${resultfile}.out"
|
||||
-D "TEST_EXPECT=${resultcode}"
|
||||
-D "TEST_REFERENCE=${resultfile}.h5dwalk"
|
||||
-D "TEST_LIBRARY_DIRECTORY=${LL_PATH}"
|
||||
-P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake"
|
||||
)
|
||||
endif ()
|
||||
endmacro ()
|
||||
|
||||
ADD_H5_TEST(help-1 0 -h)
|
||||
43
utils/tools/test/h5dwalk/Makefile.am
Normal file
43
utils/tools/test/h5dwalk/Makefile.am
Normal file
@ -0,0 +1,43 @@
|
||||
#
|
||||
# Copyright by The HDF Group.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is part of HDF5. The full HDF5 copyright notice, including
|
||||
# terms governing use, modification, and redistribution, is contained in
|
||||
# the COPYING file, which can be found at the root of the source code
|
||||
# distribution tree, or in https://www.hdfgroup.org/licenses.
|
||||
# If you do not have access to either file, you may request a copy from
|
||||
# help@hdfgroup.org.
|
||||
##
|
||||
## Makefile.am
|
||||
## Run automake to generate a Makefile.in from this file.
|
||||
#
|
||||
# HDF5 Library Makefile(.in)
|
||||
#
|
||||
|
||||
include $(top_srcdir)/config/commence.am
|
||||
|
||||
# Include src directory
|
||||
AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib
|
||||
|
||||
install-examples:
|
||||
@echo "Creating demo files" && \
|
||||
. copy_demo_files.sh $(top_srcdir)/src
|
||||
|
||||
bin_SCRIPTS:install-examples
|
||||
|
||||
#test script and program
|
||||
TEST_SCRIPT=testh5dwalk.sh copy_demo_files.sh
|
||||
check_SCRIPTS=$(TEST_SCRIPT)
|
||||
SCRIPT_DEPEND=../../h5dwalk/h5dwalk$(EXEEXT)
|
||||
|
||||
# Tell automake to clean h5redeploy script
|
||||
CLEANFILES=
|
||||
|
||||
# These were generated by configure. Remove them only when distclean.
|
||||
DISTCLEANFILES=testh5dwalk.sh copy_demo_files.sh
|
||||
|
||||
# All programs rely on hdf5 library and h5tools library
|
||||
LDADD=$(LIBH5TOOLS) $(LIBHDF5)
|
||||
|
||||
include $(top_srcdir)/config/conclude.am
|
||||
86
utils/tools/test/h5dwalk/copy_demo_files.sh.in
Normal file
86
utils/tools/test/h5dwalk/copy_demo_files.sh.in
Normal file
@ -0,0 +1,86 @@
|
||||
#! /bin/sh
|
||||
#
|
||||
# Copyright by The HDF Group.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is part of HDF5. The full HDF5 copyright notice, including
|
||||
# terms governing use, modification, and redistribution, is contained in
|
||||
# the COPYING file, which can be found at the root of the source code
|
||||
# distribution tree, or in https://www.hdfgroup.org/licenses.
|
||||
# If you do not have access to either file, you may request a copy from
|
||||
# help@hdfgroup.org.
|
||||
#
|
||||
srcdir=@srcdir@
|
||||
TOP_BUILDDIR=..
|
||||
|
||||
# Determine if backward compatibility options enabled
|
||||
DEPRECATED_SYMBOLS="yes"
|
||||
|
||||
EXIT_SUCCESS=0
|
||||
EXIT_FAILURE=1
|
||||
|
||||
CP='cp'
|
||||
|
||||
THIS_DIR=`pwd`
|
||||
SRC_TOOLS_DIR=$srcdir/../../../../tools
|
||||
|
||||
nerrors=0
|
||||
verbose=yes
|
||||
exit_code=$EXIT_SUCCESS
|
||||
|
||||
|
||||
# Add Testing files into the local testfiles directory::
|
||||
TESTDIR=./testfiles
|
||||
test -d $TESTDIR || mkdir $TESTDIR
|
||||
|
||||
echo "HDF5 \"$THIS_DIR/testfiles/h5diff_basic1.h5\" {" > "$THIS_DIR"/testfiles/h5diff_basic1.h5_h5dump.txt
|
||||
echo "FILE_CONTENTS {
|
||||
group /
|
||||
group /g1
|
||||
dataset /g1/d1
|
||||
dataset /g1/d2
|
||||
dataset /g1/dset1
|
||||
dataset /g1/dset10
|
||||
dataset /g1/dset11
|
||||
dataset /g1/dset12
|
||||
dataset /g1/dset3
|
||||
dataset /g1/dset5
|
||||
dataset /g1/dset6
|
||||
dataset /g1/dset7
|
||||
dataset /g1/dset8
|
||||
dataset /g1/dset9
|
||||
dataset /g1/fp1
|
||||
dataset /g1/fp15
|
||||
dataset /g1/fp16
|
||||
dataset /g1/fp17
|
||||
dataset /g1/fp18
|
||||
dataset /g1/fp18_COPY
|
||||
dataset /g1/fp19
|
||||
dataset /g1/fp19_COPY
|
||||
dataset /g1/fp2
|
||||
dataset /g1/fp20
|
||||
dataset /g1/fp20_COPY
|
||||
dataset /g1/ld
|
||||
}
|
||||
}" >> "$THIS_DIR"/testfiles/h5diff_basic1.h5_h5dump.txt
|
||||
|
||||
# Create the help-1.txt output file for '-h' validation
|
||||
echo "
|
||||
Usage: h5dwalk [options] <path> ...
|
||||
|
||||
Options:
|
||||
-i, --input <file> - read list from file
|
||||
-o, --output <file> - write output summary to the named file.
|
||||
-E, --error <file> - write processed errors to file in text format
|
||||
-l, --log_text <dir> - write individual tool outputs to a file. Logs can be written to an optional named directory.
|
||||
-T, --tool <executable> - name of the HDF5 tool to invoke
|
||||
-h, --help - print usage
|
||||
|
||||
For more information see https://mpifileutils.readthedocs.io.
|
||||
" > "$THIS_DIR"/testfiles/help-1.txt
|
||||
|
||||
# Make a copy of the help-1.txt output file for --help validation
|
||||
$CP "$srcdir"/help.h5dwalk "$THIS_DIR"/testfiles/help-1.txt
|
||||
$CP "$srcdir"/help.h5dwalk "$THIS_DIR"/testfiles/help-2.txt
|
||||
# Make a copy of a simple HDF5 datafile which will be used as input for h5dump -n (see the expected output above)
|
||||
$CP "$SRC_TOOLS_DIR"/test/h5diff/testfiles/h5diff_basic1.h5 "$THIS_DIR"/testfiles
|
||||
13
utils/tools/test/h5dwalk/help.h5dwalk
Normal file
13
utils/tools/test/h5dwalk/help.h5dwalk
Normal file
@ -0,0 +1,13 @@
|
||||
|
||||
Usage: h5dwalk [options] <path> ...
|
||||
|
||||
Options:
|
||||
-i, --input <file> - read list from file
|
||||
-o, --output <file> - write output summary to the named file.
|
||||
-E, --error <file> - write processed errors to file in text format
|
||||
-l, --log_text <dir> - write individual tool outputs to a file. Logs can be written to an optional named directory.
|
||||
-T, --tool <executable> - name of the HDF5 tool to invoke
|
||||
-h, --help - print usage
|
||||
|
||||
For more information see https://mpifileutils.readthedocs.io.
|
||||
|
||||
249
utils/tools/test/h5dwalk/testh5dwalk.sh.in
Normal file
249
utils/tools/test/h5dwalk/testh5dwalk.sh.in
Normal file
@ -0,0 +1,249 @@
|
||||
#! /bin/sh
|
||||
#
|
||||
# Copyright by The HDF Group.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is part of HDF5. The full HDF5 copyright notice, including
|
||||
# terms governing use, modification, and redistribution, is contained in
|
||||
# the COPYING file, which can be found at the root of the source code
|
||||
# distribution tree, or in https://www.hdfgroup.org/licenses.
|
||||
# If you do not have access to either file, you may request a copy from
|
||||
# help@hdfgroup.org.
|
||||
#
|
||||
|
||||
# Tests for the h5dwalk tool
|
||||
|
||||
#
|
||||
#
|
||||
|
||||
srcdir=@srcdir@
|
||||
|
||||
# Determine which filters are available
|
||||
USE_FILTER_SZIP="@USE_FILTER_SZIP@"
|
||||
USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"
|
||||
|
||||
|
||||
TESTNAME=h5dwalk
|
||||
|
||||
EXIT_SUCCESS=0
|
||||
EXIT_FAILURE=1
|
||||
|
||||
THIS_DIR="`pwd`"
|
||||
ROOTDIR="`cd ../../../..; pwd`"
|
||||
cd "$THIS_DIR"
|
||||
TOP_DIR="$ROOTDIR"
|
||||
|
||||
|
||||
H5DWALK=../../h5dwalk/h5dwalk
|
||||
H5DWALK_BIN="$TOP_DIR/utils/tools/h5dwalk/h5dwalk"
|
||||
|
||||
|
||||
H5DUMP="$TOP_DIR/src/h5dump/h5dump"
|
||||
H5DUMP_BIN="$TOP_DIR/tools/src/h5dump/h5dump"
|
||||
|
||||
RM='rm -rf'
|
||||
CMP='cmp -s'
|
||||
DIFF='diff -c'
|
||||
CP='cp'
|
||||
DIRNAME='dirname'
|
||||
LS='ls'
|
||||
AWK='awk'
|
||||
WC='wc'
|
||||
|
||||
nerrors=0
|
||||
verbose=yes
|
||||
|
||||
export LD_LIBRARY_PATH=@LL_PATH@
|
||||
|
||||
# source dirs
|
||||
SRC_TOOLS="$TOP_DIR/tools/test"
|
||||
SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles"
|
||||
|
||||
# testfiles source dirs for tools
|
||||
SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES"
|
||||
SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES"
|
||||
SRC_H5DIFF_TESTFILES="$SRC_TOOLS/h5diff/testfiles"
|
||||
SRC_H5COPY_TESTFILES="$SRC_TOOLS/h5copy/testfiles"
|
||||
SRC_H5REPACK_TESTFILES="$SRC_TOOLS/h5repack/testfiles"
|
||||
SRC_H5JAM_TESTFILES="$SRC_TOOLS/h5jam/testfiles"
|
||||
SRC_H5DWALK_TESTFILES="$SRC_TOOLS/h5dwalk/testfiles"
|
||||
SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/h5import/testfiles"
|
||||
|
||||
TESTDIR=./testfiles
|
||||
test -d $TESTDIR || mkdir $TESTDIR
|
||||
|
||||
echo "SRC_H5DIFF_TESTFILES = $SRC_H5DIFF_TESTFILES"
|
||||
echo "Creating demo files"
|
||||
. ./copy_demo_files.sh
|
||||
|
||||
|
||||
|
||||
CLEAN_TESTFILES_AND_TESTDIR()
|
||||
{
|
||||
echo "cleaning logfiles"
|
||||
$RM $TESTDIR/*log*
|
||||
}
|
||||
|
||||
# Print a line-line message left justified in a field of 70 characters
|
||||
# beginning with the word "Testing".
|
||||
#
|
||||
TESTING() {
|
||||
SPACES=" "
|
||||
echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
|
||||
}
|
||||
|
||||
# Run a test and print PASS or *FAIL*. If a test fails then increment
|
||||
# the `nerrors' global variable and (if $verbose is set) display the
|
||||
# difference between the actual output and the expected output. The
|
||||
# expected output is given as the first argument to this function and
|
||||
# the actual output file is calculated by replacing the `.ddl' with
|
||||
# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a
|
||||
# non-zero value.
|
||||
#
|
||||
TOOLTEST() {
|
||||
expect="$TESTDIR/$1"
|
||||
expect_err="$TESTDIR/`basename $1`.err"
|
||||
actual="$TESTDIR/`basename $1`.out"
|
||||
actual_err="$TESTDIR/`basename $1`.out.err"
|
||||
actual_sav=${actual}-sav
|
||||
actual_err_sav=${actual_err}-sav
|
||||
shift
|
||||
|
||||
# Run test.
|
||||
|
||||
TESTING $H5DWALK $@
|
||||
(
|
||||
cd $TESTDIR
|
||||
$RUNSERIAL $H5DWALK_BIN $@
|
||||
) 1> $actual 2> $actual_err
|
||||
|
||||
# save actual and actual_err in case they are needed later.
|
||||
cp $actual $actual_sav
|
||||
cp $actual_err $actual_err_sav
|
||||
|
||||
if [ ! -f $expect ]; then
|
||||
# Compare error files if the expect file doesn't exist.
|
||||
if $CMP $expect_err $actual_err; then
|
||||
echo " PASSED"
|
||||
else
|
||||
echo "*FAILED*"
|
||||
echo " Expected result (*.err) differs from actual result (*.out.err)"
|
||||
nerrors="`expr $nerrors + 1`"
|
||||
test yes = "$verbose" && $DIFF $expect_err $actual_err |sed 's/^/ /'
|
||||
fi
|
||||
elif $CMP $expect $actual; then
|
||||
echo " PASSED"
|
||||
else
|
||||
echo "*FAILED*"
|
||||
echo " Expected result (*.ddl) differs from actual result (*.out)"
|
||||
nerrors="`expr $nerrors + 1`"
|
||||
test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# Clean up output file
|
||||
if test -z "$HDF5_NOCLEANUP"; then
|
||||
rm -f $actual $actual_err $actual_sav $actual_err_sav
|
||||
fi
|
||||
}
|
||||
|
||||
TOOL_LOGTEST() {
|
||||
expect="$TESTDIR/`basename $1`.txt"
|
||||
expect_err="$TESTDIR/`basename $1`.err"
|
||||
actual="$TESTDIR/`basename $1`.log"
|
||||
actual_err="$TESTDIR/`basename $1`.out.err"
|
||||
actual_sav=${actual}-sav
|
||||
actual_err_sav=${actual_err}-sav
|
||||
shift
|
||||
|
||||
echo "running logtest"
|
||||
|
||||
# Run test.
|
||||
TESTING $H5DWALK $@
|
||||
(
|
||||
cd $TESTDIR
|
||||
$RUNSERIAL $H5DWALK_BIN $@
|
||||
|
||||
) 1> $actual 2> $actual_err
|
||||
expect_len="`wc -l < $expect`"
|
||||
|
||||
if [ ! -f $actual ]; then
|
||||
echo "*FAILED*"
|
||||
echo " The expected .log file is missing"
|
||||
echo " Perhaps the test failed to run?"
|
||||
else
|
||||
actual_len="`wc -l < $actual`"
|
||||
if [ $actual_len -eq $expect_len ]; then
|
||||
echo " PASSED"
|
||||
else
|
||||
echo "*FAILED*"
|
||||
echo " The generated .log file length does not match the expected length. $actual_len != $expected_len"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Clean up output file
|
||||
if test -z "$HDF5_NOCLEANUP"; then
|
||||
rm -f $actual $actual_err $actual_sav $actual_err_sav
|
||||
fi
|
||||
}
|
||||
|
||||
TOOL_CHK_LOGLEN() {
|
||||
expect=$1
|
||||
shift
|
||||
|
||||
echo "running tool_chk_loglen"
|
||||
|
||||
# Run test.
|
||||
TESTING $H5DWALK $@
|
||||
(
|
||||
cd $TESTDIR
|
||||
$RUNSERIAL $H5DWALK_BIN $@
|
||||
)
|
||||
|
||||
expect_len="`wc -l < $expect`"
|
||||
if [ "$expect_len" -gt 0 ]; then
|
||||
echo " PASSED"
|
||||
else
|
||||
echo "*FAILED*"
|
||||
echo " The generated .log file is empty!."
|
||||
fi
|
||||
|
||||
# Clean up output file
|
||||
if test -z "$HDF5_NOCLEANUP"; then
|
||||
rm -f $expect
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
# Print a "SKIP" message
|
||||
SKIP() {
|
||||
TESTING $H5DWALK $@6
|
||||
echo " -SKIP-"
|
||||
}
|
||||
|
||||
|
||||
|
||||
##############################################################################
|
||||
##############################################################################
|
||||
### T H E T E S T S ###
|
||||
##############################################################################
|
||||
##############################################################################
|
||||
|
||||
TOOLTEST help-1.txt -h
|
||||
TOOLTEST help-2.txt --help
|
||||
TOOL_LOGTEST h5diff_basic1.h5_h5dump -l -T $H5DUMP_BIN -n ./h5diff_basic1.h5
|
||||
TOOL_CHK_LOGLEN showme-h5dump.log -o `pwd`/showme-h5dump.log -T $H5DUMP_BIN -n `pwd`
|
||||
|
||||
|
||||
#
|
||||
#
|
||||
# Clean up temporary files/directories
|
||||
CLEAN_TESTFILES_AND_TESTDIR
|
||||
|
||||
if test $nerrors -eq 0 ; then
|
||||
echo "All $TESTNAME tests passed."
|
||||
exit $EXIT_SUCCESS
|
||||
else
|
||||
echo "$TESTNAME tests failed with $nerrors errors."
|
||||
exit $EXIT_FAILURE
|
||||
fi
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user