mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-02-17 16:10:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add Splitter VFD to library.

Browse Source 
* "Simultaneous and equivalent" Read-Write and Write-Only channels for
  file I/O.
* Only supports drivers with the H5FD_FEAT_DEFAULT_VFD_COMPATIBLE flag for
  now, preventing issues with multi-file drivers.

Add Mirror VFD to library.

* Write-only operations over a network.
* Uses TCP/IP sockets.
* Server and auxiliary server-shutdown programs provided in a new directory,
  `utils/mirror_vfd`.
* Automated testing via loopback ("remote" of localhost).
This commit is contained in:
Jacob Smith 2020-03-13 17:13:17 -05:00
parent 7613f7e1aa
commit b65405439d
45 changed files with 11479 additions and 562 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CMakeInstallation.cmake
View File

@ -554,6 +554,13 @@ The HDF5 data model, file format, API, library, and tools are open and distribut
)
endif ()
cpack_add_component (utilsapplications
DISPLAY_NAME "HDF5 Utility Applications"
DEPENDS libraries
GROUP Applications
INSTALL_TYPES Full Developer User
)
if (HDF5_BUILD_TOOLS)
cpack_add_component (toolsapplications
DISPLAY_NAME "HDF5 Tools Applications"

15
CMakeLists.txt
View File

@ -137,6 +137,7 @@ set (HDF5_CPP_LIB_CORENAME "hdf5_cpp")
set (HDF5_HL_LIB_CORENAME "hdf5_hl")
set (HDF5_HL_CPP_LIB_CORENAME "hdf5_hl_cpp")
set (HDF5_TOOLS_LIB_CORENAME "hdf5_tools")
set (HDF5_UTILS_LIB_CORENAME "hdf5_utils")
set (HDF5_F90_LIB_CORENAME "hdf5_fortran")
set (HDF5_F90_C_LIB_CORENAME "hdf5_f90cstub")
set (HDF5_F90_TEST_LIB_CORENAME "hdf5_test_fortran")
@ -156,6 +157,7 @@ set (HDF5_CPP_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_CPP_LIB_COREN
set (HDF5_HL_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_HL_LIB_CORENAME}")
set (HDF5_HL_CPP_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_HL_CPP_LIB_CORENAME}")
set (HDF5_TOOLS_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_TOOLS_LIB_CORENAME}")
set (HDF5_UTILS_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_UTILS_LIB_CORENAME}")
set (HDF5_F90_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_F90_LIB_CORENAME}")
set (HDF5_F90_C_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_F90_C_LIB_CORENAME}")
set (HDF5_F90_TEST_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_F90_TEST_LIB_CORENAME}")
@ -175,6 +177,7 @@ set (HDF5_CPP_LIB_TARGET "${HDF5_CPP_LIB_CORENAME}-static")
set (HDF5_HL_LIB_TARGET "${HDF5_HL_LIB_CORENAME}-static")
set (HDF5_HL_CPP_LIB_TARGET "${HDF5_HL_CPP_LIB_CORENAME}-static")
set (HDF5_TOOLS_LIB_TARGET "${HDF5_TOOLS_LIB_CORENAME}-static")
set (HDF5_UTILS_LIB_TARGET "${HDF5_UTILS_LIB_CORENAME}-static")
set (HDF5_F90_LIB_TARGET "${HDF5_F90_LIB_CORENAME}-static")
set (HDF5_F90_C_LIB_TARGET "${HDF5_F90_C_LIB_CORENAME}-static")
set (HDF5_F90_TEST_LIB_TARGET "${HDF5_F90_TEST_LIB_CORENAME}-static")
@ -190,6 +193,7 @@ set (HDF5_CPP_LIBSH_TARGET "${HDF5_CPP_LIB_CORENAME}-shared")
set (HDF5_HL_LIBSH_TARGET "${HDF5_HL_LIB_CORENAME}-shared")
set (HDF5_HL_CPP_LIBSH_TARGET "${HDF5_HL_CPP_LIB_CORENAME}-shared")
set (HDF5_TOOLS_LIBSH_TARGET "${HDF5_TOOLS_LIB_CORENAME}-shared")
set (HDF5_UTILS_LIBSH_TARGET "${HDF5_UTILS_LIB_CORENAME}-shared")
set (HDF5_F90_LIBSH_TARGET "${HDF5_F90_LIB_CORENAME}-shared")
set (HDF5_F90_C_LIBSH_TARGET "${HDF5_F90_C_LIB_CORENAME}-shared")
set (HDF5_F90_TEST_LIBSH_TARGET "${HDF5_F90_TEST_LIB_CORENAME}-shared")
@ -212,6 +216,7 @@ set (HDF5_HL_TOOLS_DIR ${HDF5_SOURCE_DIR}/hl/tools)
set (HDF5_TOOLS_DIR ${HDF5_SOURCE_DIR}/tools)
set (HDF5_TOOLS_SRC_DIR ${HDF5_SOURCE_DIR}/tools/src)
set (HDF5_PERFORM_SRC_DIR ${HDF5_SOURCE_DIR}/tools/src/perform)
set (HDF5_UTILS_DIR ${HDF5_SOURCE_DIR}/utils)
set (HDF5_F90_SRC_DIR ${HDF5_SOURCE_DIR}/fortran)
set (HDF5_JAVA_JNI_SRC_DIR ${HDF5_SOURCE_DIR}/java/src/jni)
set (HDF5_JAVA_HDF5_SRC_DIR ${HDF5_SOURCE_DIR}/java/src/hdf)
@ -919,6 +924,16 @@ if (BUILD_TESTING)
endif ()
endif ()
#-----------------------------------------------------------------------------
# Option to build HDF5 Utilities
#-----------------------------------------------------------------------------
if (EXISTS "${HDF5_SOURCE_DIR}/utils" AND IS_DIRECTORY "${HDF5_SOURCE_DIR}/utils")
option (HDF5_BUILD_UTILS "Build HDF5 Utils" ON)
if (HDF5_BUILD_UTILS)
add_subdirectory (utils)
endif ()
endif ()
#-----------------------------------------------------------------------------
# Option to build HDF5 Tools
#-----------------------------------------------------------------------------

21
MANIFEST
View File

@ -654,6 +654,8 @@
./src/H5FDint.c
./src/H5FDlog.c
./src/H5FDlog.h
./src/H5FDmirror.c
./src/H5FDmirror.h
./src/H5FDmodule.h
./src/H5FDmpi.c
./src/H5FDmpi.h
@ -671,6 +673,8 @@
./src/H5FDsec2.c
./src/H5FDsec2.h
./src/H5FDspace.c
./src/H5FDsplitter.c
./src/H5FDsplitter.h
./src/H5FDstdio.c
./src/H5FDstdio.h
./src/H5FDtest.c
@ -1113,6 +1117,7 @@
./test/memleak_H5O_dtype_decode_helper_H5Odtype.h5
./test/mergemsg.h5
./test/mf.c
./test/mirror_vfd.c
./test/mount.c
./test/mtime.c
./test/multi_file_v16-r.h5
@ -1173,6 +1178,7 @@
./test/testhdf5.c
./test/testhdf5.h
./test/testlibinfo.sh.in
./test/test_mirror.sh.in
./test/test_usecases.sh.in
./test/test_vol_plugin.sh.in
./test/testmeta.c
@ -1216,6 +1222,7 @@
./test/unlink.c
./test/unregister.c
./test/use_append_chunk.c
./test/use_append_chunk_mirror.c
./test/use_append_mchunks.c
./test/use_common.c
./test/use_disable_mdc_flushes.c
@ -2772,6 +2779,18 @@
./tools/test/perform/sio_standalone.h
./tools/test/perform/zip_perf.c
# Utils directory
./utils/COPYING
./utils/Makefile.am
# Mirror VFD utilities
./utils/mirror_vfd/Makefile.am
./utils/mirror_vfd/mirror_remote.c
./utils/mirror_vfd/mirror_remote.h
./utils/mirror_vfd/mirror_server.c
./utils/mirror_vfd/mirror_server_halten_sie.c
./utils/mirror_vfd/mirror_writer.c
# high level libraries
./hl/COPYING
./hl/Makefile.am
@ -3507,6 +3526,8 @@
./tools/test/misc/vds/CMakeLists.txt
./tools/test/perform/CMakeLists.txt
./tools/test/perform/CMakeTests.cmake
./utils/CMakeLists.txt
./utils/mirror_vfd/CMakeLists.txt
# CMake-specific User Scripts
./config/cmake/CTestScript.cmake

6
Makefile.am
View File

@ -85,9 +85,9 @@ else
TOOLS_DIR=
endif
SUBDIRS = src $(TESTSERIAL_DIR) $(TESTPARALLEL_DIR) bin $(TOOLS_DIR) . \
SUBDIRS = src $(TESTSERIAL_DIR) $(TESTPARALLEL_DIR) bin utils $(TOOLS_DIR) . \
$(CXX_DIR) $(FORTRAN_DIR) $(JAVA_DIR) $(HDF5_HL_DIR)
DIST_SUBDIRS = src test testpar tools . c++ fortran hl examples java
DIST_SUBDIRS = src test testpar utils tools . c++ fortran hl examples java
# Some files generated during configure that should be cleaned
DISTCLEANFILES=config/stamp1 config/stamp2
@ -190,7 +190,7 @@ trace:
# Run tests with different Virtual File Drivers.
# Currently, only invoke check-vfd in the test directory.
check-vfd:
for d in src test; do \
for d in src utils test; do \
if test $$d != .; then \
(cd $$d && $(MAKE) $(AM_MAKEFLAGS) $@) || exit 1; \
fi; \

2
bin/trace
View File

@ -43,6 +43,7 @@ $Source = "";
"H5D_space_status_t" => "Ds",
"H5D_vds_view_t" => "Dv",
"H5FD_mpio_xfer_t" => "Dt",
"H5FD_splitter_vfd_config_t" => "Dr",
"herr_t" => "e",
"H5E_direction_t" => "Ed",
"H5E_error_t" => "Ee",
@ -158,6 +159,7 @@ $Source = "";
"H5FD_ros3_fapl_t" => "x",
"H5FD_hdfs_fapl_t" => "x",
"H5FD_file_image_callbacks_t" => "x",
"H5FD_mirror_fapl_t" => "x",
"H5G_iterate_t" => "x",
"H5G_info_t" => "x",
"H5I_free_t" => "x",

11
config/cmake/ConfigureChecks.cmake
View File

@ -171,6 +171,17 @@ option (HDF5_ENABLE_ROS3_VFD "Build the ROS3 Virtual File Driver" OFF)
endif ()
endif ()
# ----------------------------------------------------------------------
# Check whether we can build the Mirror VFD
# Header-check flags set in config/cmake_ext_mod/ConfigureChecks.cmake
# ----------------------------------------------------------------------
if ( ${HDF_PREFIX}_HAVE_NETINET_IN_H AND
${HDF_PREFIX}_HAVE_NETDB_H AND
${HDF_PREFIX}_HAVE_ARPA_INET_H AND
${HDF_PREFIX}_HAVE_SYS_SOCKET_H)
set (${HDF_PREFIX}_HAVE_MIRROR_VFD 1)
endif()
#-----------------------------------------------------------------------------
# Check if C has __float128 extension
#-----------------------------------------------------------------------------

12
config/cmake/H5pubconf.h.in
View File

@ -104,6 +104,9 @@
/* Define if the compiler understands the __func__ keyword */
#cmakedefine H5_HAVE_C99_FUNC @H5_HAVE_C99_FUNC@
/* Define to 1 if you have the <arpa/inet.h> header file. */
#cmakedefine H5_HAVE_ARPA_INET_H @H5_HAVE_ARPA_INET_H@
/* Define to 1 if you have the `clock_gettime' function. */
#cmakedefine H5_HAVE_CLOCK_GETTIME @H5_HAVE_CLOCK_GETTIME@
@ -273,6 +276,9 @@
/* Define to 1 if you have the <memory.h> header file. */
#cmakedefine H5_HAVE_MEMORY_H @H5_HAVE_MEMORY_H@
/* Define if we can build the Mirror VFD */
#cmakedefine H5_HAVE_MIRROR_VFD @H5_HAVE_MIRROR_VFD@
/* Define if we have MPE support */
#cmakedefine H5_HAVE_MPE @H5_HAVE_MPE@
@ -285,6 +291,12 @@
/* Define if MPI_Info_c2f and MPI_Info_f2c exists */
#cmakedefine H5_HAVE_MPI_MULTI_LANG_Info @H5_HAVE_MPI_MULTI_LANG_Info@
/* Define to 1 if you have the <netdb.h> header file. */
#cmakedefine H5_HAVE_NETDB_H @H5_HAVE_NETDB_H@
/* Define to 1 if you have the <netinet/in.h> header file. */
#cmakedefine H5_HAVE_NETINET_IN_H @H5_HAVE_NETINET_IN_H@
/* Define to 1 if you have the <openssl/evp.h> header file. */
#cmakedefine H5_HAVE_OPENSSL_EVP_H @H5_HAVE_OPENSSL_EVP_H@

1
config/cmake/libhdf5.settings.cmake.in
View File

@ -76,6 +76,7 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@
I/O filters (external): @EXTERNAL_FILTERS@
MPE: @H5_HAVE_LIBLMPE@
Direct VFD: @H5_HAVE_DIRECT@
Mirror VFD: @H5_HAVE_MIRROR_VFD@
(Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@
(Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@
dmalloc: @H5_HAVE_LIBDMALLOC@

3
config/cmake_ext_mod/ConfigureChecks.cmake
View File

@ -165,6 +165,8 @@ CHECK_INCLUDE_FILE_CONCAT ("memory.h" ${HDF_PREFIX}_HAVE_MEMORY_H)
CHECK_INCLUDE_FILE_CONCAT ("dlfcn.h" ${HDF_PREFIX}_HAVE_DLFCN_H)
CHECK_INCLUDE_FILE_CONCAT ("inttypes.h" ${HDF_PREFIX}_HAVE_INTTYPES_H)
CHECK_INCLUDE_FILE_CONCAT ("netinet/in.h" ${HDF_PREFIX}_HAVE_NETINET_IN_H)
CHECK_INCLUDE_FILE_CONCAT ("netdb.h" ${HDF_PREFIX}_HAVE_NETDB_H)
CHECK_INCLUDE_FILE_CONCAT ("arpa/inet.h" ${HDF_PREFIX}_HAVE_ARPA_INET_H)
# _Bool type support
CHECK_INCLUDE_FILE_CONCAT (stdbool.h ${HDF_PREFIX}_HAVE_STDBOOL_H)
@ -677,3 +679,4 @@ endif ()
# the cache value is set in it's config file)
#
set (${HDF_PREFIX}_CONVERT_DENORMAL_FLOAT 1)

34
configure.ac
View File

@ -1086,6 +1086,7 @@ AC_CHECK_HEADERS([stddef.h setjmp.h features.h])
AC_CHECK_HEADERS([dirent.h])
AC_CHECK_HEADERS([stdint.h], [C9x=yes])
AC_CHECK_HEADERS([stdbool.h])
AC_CHECK_HEADERS([netdb.h netinet/in.h arpa/inet.h])
## Darwin
AC_CHECK_HEADERS([mach/mach_time.h])
@ -2837,6 +2838,36 @@ fi
## Direct VFD files are not built if not required.
AM_CONDITIONAL([DIRECT_VFD_CONDITIONAL], [test "X$DIRECT_VFD" = "Xyes"])
## ----------------------------------------------------------------------
## Check whether the Mirror VFD can be built.
## Auto-enabled if the required libraries are present.
##
AC_SUBST([MIRROR_VFD])
## Default is no Mirror VFD
MIRROR_VFD=yes
AC_CHECK_HEADERS([arpa/inet.h],, [unset MIRROR_VFD])
AC_CHECK_HEADERS([netinet/in.h],, [unset MIRROR_VFD])
AC_CHECK_HEADERS([netdb.h],, [unset MIRROR_VFD])
AC_CHECK_HEADERS([sys/socket.h],, [unset MIRROR_VFD])
AC_MSG_CHECKING([if the Mirror virtual file driver (VFD) can be built])
if test "X$MIRROR_VFD" = "Xyes"; then
AC_DEFINE([HAVE_MIRROR_VFD], [1],
[Define whether the Mirror virtual file driver (VFD) will be compiled])
AC_MSG_RESULT([yes])
else
AC_MSG_RESULT([no])
MIRROR_VFD=no
AC_MSG_ERROR([The Mirror VFD was cannot be built.
Missing any of: arpa/inet.h, netinet/in.h, netdb.h,
sys/socket.h.])
fi
## Mirror VFD files built only if able.
AM_CONDITIONAL([MIRROR_VFD_CONDITIONAL], [test "X$MIRROR_VFD" = "Xyes"])
## ----------------------------------------------------------------------
## Check if Read-Only S3 virtual file driver is enabled by --enable-ros3-vfd
##
@ -3716,10 +3747,13 @@ AC_CONFIG_FILES([src/libhdf5.settings
test/testvds_env.sh
test/testvdsswmr.sh
test/test_filter_plugin.sh
test/test_mirror.sh
test/test_usecases.sh
test/test_vol_plugin.sh
testpar/Makefile
testpar/testpflush.sh
utils/Makefile
utils/mirror_vfd/Makefile
tools/Makefile
tools/lib/Makefile
tools/libtest/Makefile

14
release_docs/RELEASE.txt
View File

@ -270,6 +270,20 @@ New Features
Library:
--------
- Add Mirror VFD
Use TCP/IP sockets to perform write-only (W/O) file I/O on a remote
machine. Must be used in conjunction with the Splitter VFD.
(JOS - 2020/03/13, TBD)
- Add Splitter VFD
Maintain separate R/W and W/O channels for "concurrent" file writes
to two files using a single HDF5 file handle.
(JOS - 2020/03/13, TBD)
- Refactored public exposure of haddr_t type in favor of "object tokens"
To better accommodate HDF5 VOL connectors where "object addresses in a file"

4
src/CMakeLists.txt
View File

@ -231,6 +231,7 @@ set (H5FD_SOURCES
${HDF5_SRC_DIR}/H5FDhdfs.c
${HDF5_SRC_DIR}/H5FDint.c
${HDF5_SRC_DIR}/H5FDlog.c
${HDF5_SRC_DIR}/H5FDmirror.c
${HDF5_SRC_DIR}/H5FDmpi.c
${HDF5_SRC_DIR}/H5FDmpio.c
${HDF5_SRC_DIR}/H5FDmulti.c
@ -238,6 +239,7 @@ set (H5FD_SOURCES
${HDF5_SRC_DIR}/H5FDs3comms.c
${HDF5_SRC_DIR}/H5FDsec2.c
${HDF5_SRC_DIR}/H5FDspace.c
${HDF5_SRC_DIR}/H5FDsplitter.c
${HDF5_SRC_DIR}/H5FDstdio.c
${HDF5_SRC_DIR}/H5FDtest.c
${HDF5_SRC_DIR}/H5FDwindows.c
@ -249,6 +251,7 @@ set (H5FD_HDRS
${HDF5_SRC_DIR}/H5FDfamily.h
${HDF5_SRC_DIR}/H5FDhdfs.h
${HDF5_SRC_DIR}/H5FDlog.h
${HDF5_SRC_DIR}/H5FDmirror.h
${HDF5_SRC_DIR}/H5FDmpi.h
${HDF5_SRC_DIR}/H5FDmpio.h
${HDF5_SRC_DIR}/H5FDmulti.h
@ -256,6 +259,7 @@ set (H5FD_HDRS
${HDF5_SRC_DIR}/H5FDros3.h
${HDF5_SRC_DIR}/H5FDs3comms.h
${HDF5_SRC_DIR}/H5FDsec2.h
${HDF5_SRC_DIR}/H5FDsplitter.h
${HDF5_SRC_DIR}/H5FDstdio.h
${HDF5_SRC_DIR}/H5FDwindows.h
)

1991
src/H5FDmirror.c Normal file
View File

File diff suppressed because it is too large Load Diff

371
src/H5FDmirror.h Normal file
View File

@ -0,0 +1,371 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Purpose: Public, shared definitions for Mirror VFD & remote Writer.
*/
#ifndef H5FDmirror_H
#define H5FDmirror_H
#ifdef H5_HAVE_MIRROR_VFD
#define H5FD_MIRROR (H5FD_mirror_init())
#ifdef __cplusplus
extern "C" {
#endif
/* ============================================================================
* Mirror VFD use and operation.
* ============================================================================
*/
/* ---------------------------------------------------------------------------
* Structure: H5FD_mirror_fapl_t
*
* Used to pass configuraiton information to the Mirror VFD.
* Populate components as appropriate and pass structure pointer to
* `H5Pset_fapl_mirror()`.
*
* `magic` (uint32_t)
* Semi-unique number to sanity-check pointers to this structure type.
* MUST equal H5FD_MIRROR_FAPL_MAGIC to be considered valid.
*
* `version` (uint32_t)
* Indicates expected components of the structure.
*
* `handshake_port (int)
* Port number to expect to reach the "Mirror Server" on the remote host.
*
* `remote_ip` (char[])
* IP address string of "Mirror Server" remote host.
* ---------------------------------------------------------------------------
*/
#define H5FD_MIRROR_FAPL_MAGIC 0xF8DD514C
#define H5FD_MIRROR_CURR_FAPL_T_VERSION 1
#define H5FD_MIRROR_MAX_IP_LEN 32
typedef struct H5FD_mirror_fapl_t {
uint32_t magic;
uint32_t version;
int handshake_port;
char remote_ip[H5FD_MIRROR_MAX_IP_LEN + 1];
} H5FD_mirror_fapl_t;
H5_DLL hid_t H5FD_mirror_init(void);
H5_DLL herr_t H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_out);
H5_DLL herr_t H5Pset_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa);
/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
* IPC - Mirror VFD and Remote Worker application.
* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
*/
/* The maximum allowed size for a receiving buffer when accepting bytes to
* write. Writes larger than this size are performed by multiple accept-write
* steps by the Writer. */
#define H5FD_MIRROR_DATA_BUFFER_MAX H5_GB /* 1 Gigabyte */
#define H5FD_MIRROR_XMIT_CURR_VERSION 1
#define H5FD_MIRROR_XMIT_MAGIC 0x87F8005B
#define H5FD_MIRROR_OP_OPEN 1
#define H5FD_MIRROR_OP_CLOSE 2
#define H5FD_MIRROR_OP_WRITE 3
#define H5FD_MIRROR_OP_TRUNCATE 4
#define H5FD_MIRROR_OP_REPLY 5
#define H5FD_MIRROR_OP_SET_EOA 6
#define H5FD_MIRROR_OP_LOCK 7
#define H5FD_MIRROR_OP_UNLOCK 8
#define H5FD_MIRROR_STATUS_OK 0
#define H5FD_MIRROR_STATUS_ERROR 1
#define H5FD_MIRROR_STATUS_MESSAGE_MAX 256 /* Dedicated error message size */
/* Maximum length of a path/filename string, including the NULL-terminator.
* Must not be smaller than H5FD_SPLITTER_PATH_MAX. */
#define H5FD_MIRROR_XMIT_FILEPATH_MAX 4097
/* Define the exact sizes of the various xmit blobs as sent over the wire.
* This is used to minimize the number of bytes transmitted as well as to
* sanity-check received bytes.
* Any modifications to the xmit structures and/or the encode/decode functions
* must be reflected here.
* */
#define H5FD_MIRROR_XMIT_HEADER_SIZE 14
#define H5FD_MIRROR_XMIT_EOA_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 9)
#define H5FD_MIRROR_XMIT_LOCK_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 8)
#define H5FD_MIRROR_XMIT_OPEN_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 20 + H5FD_MIRROR_XMIT_FILEPATH_MAX)
#define H5FD_MIRROR_XMIT_REPLY_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 4 + H5FD_MIRROR_STATUS_MESSAGE_MAX)
#define H5FD_MIRROR_XMIT_WRITE_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 17)
/* Maximum length of any xmit. */
#define H5FD_MIRROR_XMIT_BUFFER_MAX MAX2( MAX3(H5FD_MIRROR_XMIT_HEADER_SIZE, \
H5FD_MIRROR_XMIT_EOA_SIZE, \
H5FD_MIRROR_XMIT_LOCK_SIZE), \
MAX3(H5FD_MIRROR_XMIT_OPEN_SIZE, \
H5FD_MIRROR_XMIT_REPLY_SIZE, \
H5FD_MIRROR_XMIT_WRITE_SIZE) ) \
/* ---------------------------------------------------------------------------
* Structure: H5FD_mirror_xmit_t
*
* Common structure 'header' for all mirror VFD/worker IPC.
* Must be the first component of a derived operation xmit structure,
* such as file-open or write command.
*
* `magic` (uint32_t)
* A "unique" number identifying the structure and endianness of
* transmitting maching.
* Must be set to H5FD_MIRROR_XMIT_MAGIC native to the VFD "sender".
*
* `version` (uint8_t)
* Number used to identify the structure membership.
* Allows sane modifications to this structure in the future.
* Must be set to H5FD_MIRROR_XMIT_CURR_VERSION.
*
* `session_token` (uint32_t)
* A "unique" number identifying the session between VFD sender and
* remote receiver/worker/writer. Exists to help sanity-check.
*
* `xmit_count` (uint32_t)
* Which transmission this is since the session began.
* Used to sanity-check transmission errors.
* First xmit (file-open) must be 0.
*
* `op` (uint8_t)
* Number identifying which operation to perform.
* Corresponds with the extended structure outside of this xmit header.
* Possible values are all defined H5FD_MIRROR_OP_* constants.
*
* ---------------------------------------------------------------------------
*/
typedef struct H5FD_mirror_xmit_t {
uint32_t magic;
uint8_t version;
uint32_t session_token;
uint32_t xmit_count;
uint8_t op;
} H5FD_mirror_xmit_t;
/* ---------------------------------------------------------------------------
* Structure: H5FD_mirror_xmit_eoa_t
*
* Structure containing eoa-set information from VFD sender.
*
* `pub` (H5FD_mirror_xmit_t)
* Common transmission header, containing session information.
* Must be first.
*
* `type` (uint8_t)
* System-independent alias for H5F[D]_mem_t.
* Specifies datatype to be written.
*
* `eoa_addr` (uint64_t)
* New address for eoa.
* (Natively 'haddr_t', always a 64-bit field)
*
* ---------------------------------------------------------------------------
*/
typedef struct H5FD_mirror_xmit_eoa_t {
H5FD_mirror_xmit_t pub;
uint8_t type;
uint64_t eoa_addr;
} H5FD_mirror_xmit_eoa_t;
/* ---------------------------------------------------------------------------
* Structure: H5FD_mirror_xmit_lock_t
*
* Structure containing eoa-set information from VFD sender.
*
* `pub` (H5FD_mirror_xmit_t)
* Common transmission header, containing session information.
* Must be first.
*
* `rw` (uint64_t)
* The Read/Write mode flag passed into H5FDlock().
* (Natively `hbool_t`, an 'int') TODO: native int may be 64-bit?
*
* ---------------------------------------------------------------------------
*/
typedef struct H5FD_mirror_xmit_lock_t {
H5FD_mirror_xmit_t pub;
uint64_t rw;
} H5FD_mirror_xmit_lock_t;
/* ---------------------------------------------------------------------------
* Structure: H5FD_mirror_xmit_open_t
*
* Structure containing file-open information from the VFD sender.
*
* `pub` (H5FD_mirror_xmit_t)
* Common transmission header, containing session information.
* Must be first.
*
* `flags` (uint32_t)
* VFL-layer file-open flags passed directly to H5FDopen().
* (Natively 'unsigned [int]') TODO: native int may be 64-bit?
*
* `maxaddr` (uint64_t)
* VFL-layer maximum allowed address space for the file to open passed
* directly to H5FDopen().
* (Natively 'haddr_t', always a 64-bit field)
*
* `size_t_blob` (uint64_t)
* A number indicating how large a size_t is on the sending system.
* Must be set to (uint64_t)((size_t)(-1))
* (maximum possible value of size_t, cast to uint64_t).
* The receiving system inspects this value -- if the local (remote)
* size_t is smaller than that of the Sender, issues a warning.
* Not an error, as:
* 1. It is assumed that underlying file systems/drivers have become
* smart enough to handle file sizes that otherwise might be
* constrained.
* 2. The Mirror Writer ingests bytes to write multiple 'slices' if the
* size is greater than H5FD_MIRROR_DATA_BUFFER_MAX, regardless of
* any size_t storage size disparity.
*
* `filename` (char[])
* String giving the filename and path of file to open.
*
* ---------------------------------------------------------------------------
*/
typedef struct H5FD_mirror_xmit_open_t {
H5FD_mirror_xmit_t pub;
uint32_t flags;
uint64_t maxaddr;
uint64_t size_t_blob;
char filename[H5FD_MIRROR_XMIT_FILEPATH_MAX];
} H5FD_mirror_xmit_open_t;
/* ---------------------------------------------------------------------------
* Structure: H5FD_mirror_xmit_reply_t
*
* Structure used by the remote receiver/worker/writer to respond to
* a command from the VFD sender.
*
* `pub` (H5FD_mirror_xmit_t)
* Common transmission header, containing session information.
* Must be first.
*
* `status` (uint32_t)
* Number indicating whether the command was successful or if an
* occured.
* Allowed values are H5FD_MIRROR_STATUS_OK and
* H5FD_MIRROR_STATUS_ERROR.
*
* `message` (char[])
* Error message. Populated if and only if there was a problem.
* It is possible that a message may reach the end of the alloted
* space without a NULL terminator -- the onus is on the programmer to
* handle this situation.
*
* ---------------------------------------------------------------------------
*/
typedef struct H5FD_mirror_xmit_reply_t {
H5FD_mirror_xmit_t pub;
uint32_t status;
char message[H5FD_MIRROR_STATUS_MESSAGE_MAX];
} H5FD_mirror_xmit_reply_t;
/* ---------------------------------------------------------------------------
* Structure: H5FD_mirror_xmit_write_t
*
* Structure containing data-write information from VFD sender.
*
* The data to be written is transmitted in subsequent, packets
* and may be broken up into more than one transmission buffer.
* The VFD sender and remote receiver/worker/writer must coordinate
* the receipt of data.
*
* `pub` (H5FD_mirror_xmit_t)
* Common transmission header, containing session information.
* Must be first.
*
* `type` (uint8_t)
* Specifies datatype to be written.
* (Natively 'H5FD_mem_t', an enumerated type in H5Fpublic.h)
*
* `offset` (uint64_t)
* Start location of write in file.
* (Natively 'haddr_t', always a 64-bit field)
*
* `size` (uint64_t)
* Size of the data to be written, in bytes.
* (Natively 'size_t', accommodate the largest possible as 64-bits)
*
* ---------------------------------------------------------------------------
*/
typedef struct H5FD_mirror_xmit_write_t {
H5FD_mirror_xmit_t pub;
uint8_t type;
uint64_t offset;
uint64_t size;
} H5FD_mirror_xmit_write_t;
/* Encode/decode routines are required to "pack" the xmit data into a known
* byte format for transmission over the wire.
*
* All component numbers must be stored in "network" word order (Big-Endian).
*
* All components must be packed in the order given in the structure definition.
*
* All components must be packed with zero padding between.
*/
H5_DLL size_t H5FD__mirror_xmit_decode_uint16(uint16_t *out, const unsigned char *buf);
H5_DLL size_t H5FD__mirror_xmit_decode_uint32(uint32_t *out, const unsigned char *buf);
H5_DLL size_t H5FD__mirror_xmit_decode_uint64(uint64_t *out, const unsigned char *buf);
H5_DLL size_t H5FD__mirror_xmit_decode_uint8(uint8_t *out, const unsigned char *buf);
H5_DLL size_t H5FD__mirror_xmit_encode_uint16(unsigned char *dest, uint16_t v);
H5_DLL size_t H5FD__mirror_xmit_encode_uint32(unsigned char *dest, uint32_t v);
H5_DLL size_t H5FD__mirror_xmit_encode_uint64(unsigned char *dest, uint64_t v);
H5_DLL size_t H5FD__mirror_xmit_encode_uint8(unsigned char *dest, uint8_t v);
H5_DLL size_t H5FD_mirror_xmit_decode_header(H5FD_mirror_xmit_t *out, const unsigned char *buf);
H5_DLL size_t H5FD_mirror_xmit_decode_lock(H5FD_mirror_xmit_lock_t *out, const unsigned char *buf);
H5_DLL size_t H5FD_mirror_xmit_decode_open(H5FD_mirror_xmit_open_t *out, const unsigned char *buf);
H5_DLL size_t H5FD_mirror_xmit_decode_reply(H5FD_mirror_xmit_reply_t *out, const unsigned char *buf);
H5_DLL size_t H5FD_mirror_xmit_decode_set_eoa(H5FD_mirror_xmit_eoa_t *out, const unsigned char *buf);
H5_DLL size_t H5FD_mirror_xmit_decode_write(H5FD_mirror_xmit_write_t *out, const unsigned char *buf);
H5_DLL size_t H5FD_mirror_xmit_encode_header(unsigned char *dest, const H5FD_mirror_xmit_t *x);
H5_DLL size_t H5FD_mirror_xmit_encode_lock(unsigned char *dest, const H5FD_mirror_xmit_lock_t *x);
H5_DLL size_t H5FD_mirror_xmit_encode_open(unsigned char *dest, const H5FD_mirror_xmit_open_t *x);
H5_DLL size_t H5FD_mirror_xmit_encode_reply(unsigned char *dest, const H5FD_mirror_xmit_reply_t *x);
H5_DLL size_t H5FD_mirror_xmit_encode_set_eoa(unsigned char *dest, const H5FD_mirror_xmit_eoa_t *x);
H5_DLL size_t H5FD_mirror_xmit_encode_write(unsigned char *dest, const H5FD_mirror_xmit_write_t *x);
H5_DLL hbool_t H5FD_mirror_xmit_is_close(const H5FD_mirror_xmit_t *xmit);
H5_DLL hbool_t H5FD_mirror_xmit_is_lock(const H5FD_mirror_xmit_lock_t *xmit);
H5_DLL hbool_t H5FD_mirror_xmit_is_open(const H5FD_mirror_xmit_open_t *xmit);
H5_DLL hbool_t H5FD_mirror_xmit_is_reply(const H5FD_mirror_xmit_reply_t *xmit);
H5_DLL hbool_t H5FD_mirror_xmit_is_set_eoa(const H5FD_mirror_xmit_eoa_t *xmit);
H5_DLL hbool_t H5FD_mirror_xmit_is_write(const H5FD_mirror_xmit_write_t *xmit);
H5_DLL hbool_t H5FD_mirror_xmit_is_xmit(const H5FD_mirror_xmit_t *xmit);
#ifdef __cplusplus
}
#endif
#else /* H5_HAVE_MIRROR_VFD */
#define H5FD_MIRROR (H5I_INAVLID_HID)
#endif /* H5_HAVE_MIRROR_VFD */
#endif /* H5FDmirror_H */

2
src/H5FDpublic.h
View File

@ -255,6 +255,8 @@ typedef enum H5F_mem_t H5FD_mem_t;
* that creates a file which is compatible with the default VFD.
* Generally, this means that the VFD creates a single file that follows
* the canonical HDF5 file format.
* Regarding the Splitter VFD specifically, only drivers with this flag
* enabled may be used as the Write-Only (W/O) channel driver.
*/
#define H5FD_FEAT_DEFAULT_VFD_COMPATIBLE 0x00008000

6
src/H5FDsec2.c
View File

@ -521,6 +521,12 @@ H5FD_sec2_query(const H5FD_t *_file, unsigned long *flags /* out */)
FUNC_ENTER_NOAPI_NOINIT_NOERR
/* Set the VFL feature flags that this driver supports */
/* Notice: the Mirror VFD Writer currently uses only the Sec2 driver as
* the underying driver -- as such, the Mirror VFD implementation copies
* these feature flags as its own. Any modifications made here must be
* reflected in H5FDmirror.c
* -- JOS 2020-01-13
*/
if(flags) {
*flags = 0;
*flags |= H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */

1467
src/H5FDsplitter.c Normal file
View File

File diff suppressed because it is too large Load Diff

99
src/H5FDsplitter.h Normal file
View File

@ -0,0 +1,99 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Purpose: The public header file for the "splitter" driver.
*/
#ifndef H5FDsplitter_H
#define H5FDsplitter_H
#define H5FD_SPLITTER (H5FD_splitter_init())
/* The version of the H5FD_splitter_vfd_config_t structure used */
#define H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION 1
/* Maximum length of a filename/path string in the Write-Only channel,
* including the NULL-terminator.
*/
#define H5FD_SPLITTER_PATH_MAX 4096
/* Semi-unique constant used to help identify structure pointers */
#define H5FD_SPLITTER_MAGIC 0x2B916880
/* ----------------------------------------------------------------------------
* Structure: H5FD_spliiter_vfd_config_t
*
* One-stop shopping for configuring a Splitter VFD (rather than many
* paramaters passed into H5Pset/get functions).
*
* magic (int32_t)
* Semi-unique number, used to sanity-check that a given pointer is
* likely (or not) to be this structure type. MUST be first.
* If magic is not H5FD_SPLITTER_MAGIC, the structure (and/or pointer to)
* must be considered invalid.
*
* version (unsigned int)
* Version number of this structure -- informs component membership.
* If not H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION, the structure (and/or
* pointer to) must be considered invalid.
*
* rw_fapl_id (hid_t)
* Library-given identification number of the Read/Write channel driver
* File Access Property List.
* The driver must support read/write access.
* Must be set to H5P_DEFAULT or a valid FAPL ID.
*
* wo_fapl_id (hid_t)
* Library-given identification number of the Read/Write channel driver
* File Access Property List.
* The driver feature flags must include H5FD_FEAT_DEFAULT_VFD_COMPAITBLE.
* Must be set to H5P_DEFAULT or a valid FAPL ID.
*
* wo_file_path (char[H5FD_SPLITTER_PATH_MAX + 1])
* String buffer for the Write-Only channel target file.
* Must be null-terminated, cannot be empty.
*
* log_file_path (char[H5FD_SPLITTER_PATH_MAX + 1])
* String buffer for the Splitter VFD logging output.
* Must be null-terminated.
* If null, no logfile is created.
*
* ignore_wo_errors (hbool_t)
* Toggle flag for how judiciously to respond to errors on the Write-Only
* channel.
*
* ----------------------------------------------------------------------------
*/
typedef struct H5FD_splitter_vfd_config_t {
int32_t magic;
unsigned int version;
hid_t rw_fapl_id;
hid_t wo_fapl_id;
char wo_path[H5FD_SPLITTER_PATH_MAX + 1];
char log_file_path[H5FD_SPLITTER_PATH_MAX + 1];
hbool_t ignore_wo_errs;
} H5FD_splitter_vfd_config_t;
#ifdef __cplusplus
extern "C" {
#endif
H5_DLL hid_t H5FD_splitter_init(void);
H5_DLL herr_t H5Pset_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_ptr);
H5_DLL herr_t H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_ptr);
#ifdef __cplusplus
}
#endif
#endif

66
src/H5private.h
View File

@ -364,6 +364,22 @@
#endif
#endif /* __cplusplus */
/*
* Networking headers used by the mirror VFD and related tests and utilities.
*/
#ifdef H5_HAVE_ARPA_INET_H
# include <arpa/inet.h>
#endif
#ifdef H5_HAVE_NETDB_H
# include <netdb.h>
#endif
#ifdef H5_HAVE_NETINET_IN_H
# include <netinet/in.h>
#endif
#ifdef H5_HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
/*
* Status return values for the `herr_t' type.
* Since some unix/c routines use 0 and -1 (or more precisely, non-negative
@ -646,6 +662,9 @@ typedef struct {
#ifndef HDabs
#define HDabs(X) abs(X)
#endif /* HDabs */
#ifndef HDaccept
#define HDaccept(A,B,C) accept((A),(B),(C)) /* mirror VFD */
#endif /* HDaccept */
#ifndef HDaccess
#define HDaccess(F,M) access(F, M)
#endif /* HDaccess */
@ -692,9 +711,15 @@ typedef struct {
#ifndef HDatoll
#define HDatoll(S) atoll(S)
#endif /* HDatol */
#ifndef HDbind
#define HDbind(A,B,C) bind((A),(B),(C)) /* mirror VFD */
#endif /* HDbind */
#ifndef HDbsearch
#define HDbsearch(K,B,N,Z,F) bsearch(K,B,N,Z,F)
#endif /* HDbsearch */
#ifndef HDbzero
#define HDbzero(A,B) bzero((A),(B)) /* mirror VFD */
#endif /* HDbzero */
#ifndef HDcalloc
#define HDcalloc(N,Z) calloc(N,Z)
#endif /* HDcalloc */
@ -734,6 +759,9 @@ typedef struct {
#ifndef HDclosedir
#define HDclosedir(D) closedir(D)
#endif /* HDclosedir */
#ifndef HDconnect
#define HDconnect(A,B,C) connect((A),(B),(C)) /* mirror VFD */
#endif /* HDconnect */
#ifndef HDcos
#define HDcos(X) cos(X)
#endif /* HDcos */
@ -978,9 +1006,12 @@ typedef off_t h5_stat_size_t;
#ifndef HDgetgroups
#define HDgetgroups(Z,G) getgroups(Z,G)
#endif /* HDgetgroups */
#ifndef HDgethostbyaddr
#define HDgethostbyaddr(A,B,C) gethostbyaddr((A),(B),(C)) /* mirror VFD */
#endif /* HDgethostbyaddr */
#ifndef HDgethostname
#define HDgethostname(N,L) gethostname(N,L)
#endif /* HDgetlogin */
#endif /* HDgethostname */
#ifndef HDgetlogin
#define HDgetlogin() getlogin()
#endif /* HDgetlogin */
@ -1014,6 +1045,18 @@ typedef off_t h5_stat_size_t;
#ifndef HDgmtime
#define HDgmtime(T) gmtime(T)
#endif /* HDgmtime */
#ifndef HDhtonl
#define HDhtonl(X) htonl((X)) /* mirror VFD */
#endif /* HDhtonl */
#ifndef HDhtons
#define HDhtons(X) htons((X)) /* mirror VFD */
#endif /* HDhtons */
#ifndef HDinet_addr
#define HDinet_addr(C) inet_addr((C)) /* mirror VFD */
#endif /* HDinet_addr */
#ifndef HDinet_ntoa
#define HDinet_ntoa(C) inet_ntoa((C)) /* mirror VFD */
#endif /* HDinet_ntoa */
#ifndef HDisalnum
#define HDisalnum(C) isalnum((int)(C)) /*cast for solaris warning*/
#endif /* HDisalnum */
@ -1068,6 +1111,9 @@ typedef off_t h5_stat_size_t;
#ifndef HDlink
#define HDlink(OLD,NEW) link(OLD,NEW)
#endif /* HDlink */
#ifndef HDlisten
#define HDlisten(A,B) listen((A),(B)) /* mirror VFD */
#endif /* HDlisten */
#ifndef HDllround
#define HDllround(V) llround(V)
#endif /* HDround */
@ -1149,6 +1195,12 @@ typedef off_t h5_stat_size_t;
#ifndef HDnanosleep
#define HDnanosleep(N, O) nanosleep(N, O)
#endif /* HDnanosleep */
#ifndef HDntohl
#define HDntohl(A) ntohl((A)) /* mirror VFD */
#endif /* HDntohl */
#ifndef HDntohs
#define HDntohs(A) ntohs((A)) /* mirror VFD */
#endif /* HDntohs */
#ifndef HDopen
#define HDopen(F,...) open(F,__VA_ARGS__)
#endif /* HDopen */
@ -1296,12 +1348,21 @@ typedef off_t h5_stat_size_t;
#ifndef HDsetsid
#define HDsetsid() setsid()
#endif /* HDsetsid */
#ifndef HDsetsockopt
#define HDsetsockopt(A,B,C,D,E) setsockopt((A),(B),(C),(D),(E)) /* mirror VFD */
#endif /* HDsetsockopt */
#ifndef HDsetuid
#define HDsetuid(U) setuid(U)
#endif /* HDsetuid */
#ifndef HDsetvbuf
#define HDsetvbuf(F,S,M,Z) setvbuf(F,S,M,Z)
#endif /* HDsetvbuf */
#ifndef HDshutdown
#define HDshutdown(A, B) shutdown((A),(B)) /* mirror VFD */
#endif /* HDshutdown */
#ifndef HDsigaction
#define HDsigaction(S,A,O) sigaction((S),(A),(O))
#endif /* HDsigaction */
#ifndef HDsigaddset
#define HDsigaddset(S,N) sigaddset(S,N)
#endif /* HDsigaddset */
@ -1347,6 +1408,9 @@ typedef off_t h5_stat_size_t;
#ifndef HDsnprintf
#define HDsnprintf snprintf /*varargs*/
#endif /* HDsnprintf */
#ifndef HDsocket
#define HDsocket(A,B,C) socket((A),(B),(C)) /* mirror VFD */
#endif /* HDsocket */
#ifndef HDsprintf
#define HDsprintf sprintf /*varargs*/
#endif /* HDsprintf */

9
src/Makefile.am
View File

@ -62,7 +62,8 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5lib_settings.c H5system.c \
H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \
H5FAint.c H5FAstat.c H5FAtest.c \
H5FD.c H5FDcore.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c \
H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c H5FDtest.c \
H5FDmirror.c H5FDmulti.c H5FDros3.c H5FDsec2.c H5FDspace.c \
H5FDsplitter.c H5FDstdio.c H5FDtest.c \
H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \
H5FSstat.c H5FStest.c \
H5G.c H5Gbtree2.c H5Gcache.c H5Gcompact.c H5Gdense.c H5Gdeprec.c \
@ -135,9 +136,9 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers
H5Apublic.h H5ACpublic.h \
H5Cpublic.h H5Dpublic.h \
H5Epubgen.h H5Epublic.h H5ESpublic.h H5Fpublic.h \
H5FDpublic.h H5FDcore.h H5FDdirect.h \
H5FDfamily.h H5FDhdfs.h H5FDlog.h H5FDmpi.h H5FDmpio.h \
H5FDmulti.h H5FDros3.h H5FDsec2.h H5FDstdio.h H5FDwindows.h \
H5FDpublic.h H5FDcore.h H5FDdirect.h H5FDfamily.h H5FDhdfs.h \
H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \
H5FDsec2.h H5FDsplitter.h H5FDstdio.h H5FDwindows.h \
H5Gpublic.h H5Ipublic.h H5Lpublic.h \
H5Mpublic.h H5MMpublic.h H5Opublic.h H5Ppublic.h \
H5PLextern.h H5PLpublic.h \

2
src/hdf5.h
View File

@ -46,10 +46,12 @@
#include "H5FDfamily.h" /* File families */
#include "H5FDhdfs.h" /* Hadoop HDFS */
#include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */
#include "H5FDmirror.h" /* Mirror VFD and IPC definitions */
#include "H5FDmpi.h" /* MPI-based file drivers */
#include "H5FDmulti.h" /* Usage-partitioned file family */
#include "H5FDros3.h" /* R/O S3 "file" I/O */
#include "H5FDsec2.h" /* POSIX unbuffered file I/O */
#include "H5FDsplitter.h" /* Twin-channel (R/W & R/O) I/O passthrough */
#include "H5FDstdio.h" /* Standard C buffered I/O */
#ifdef H5_HAVE_WINDOWS
#include "H5FDwindows.h" /* Win32 I/O */

1
src/libhdf5.settings.in
View File

@ -80,6 +80,7 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@
MPE: @MPE@
Map (H5M) API: @MAP_API@
Direct VFD: @DIRECT_VFD@
Mirror VFD: @MIRROR_VFD@
(Read-Only) S3 VFD: @ROS3_VFD@
(Read-Only) HDFS VFD: @HAVE_LIBHDFS@
dmalloc: @HAVE_DMALLOC@

19
test/CMakeLists.txt
View File

@ -211,6 +211,11 @@ set (cache_image_SOURCES
${HDF5_TEST_SOURCE_DIR}/genall5.c
)
set(mirror_vfd_SOURCES
${HDF5_TEST_SOURCE_DIR}/mirror_vfd.c
${HDF5_TEST_SOURCE_DIR}/genall5.c
)
set (ttsafe_SOURCES
${HDF5_TEST_SOURCE_DIR}/ttsafe.c
${HDF5_TEST_SOURCE_DIR}/ttsafe_dcreate.c
@ -273,6 +278,7 @@ set (H5_TESTS
ros3
s3comms
hdfs
mirror_vfd
ntypes
dangle
dtransform
@ -310,6 +316,7 @@ set (H5_TESTS_MULTIPLE
testhdf5
cache_image
ttsafe
mirror_vfd
)
# Only build single source tests here
foreach (h5_test ${H5_TESTS})
@ -389,6 +396,18 @@ else ()
endif ()
set_target_properties (ttsafe PROPERTIES FOLDER test)
#-- Adding test for mirror_vfd
add_executable (mirror_vfd ${mirror_vfd_SOURCES})
target_include_directories (mirror_vfd PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
if (NOT BUILD_SHARED_LIBS)
TARGET_C_PROPERTIES (mirror_vfd STATIC)
target_link_libraries (mirror_vfd PRIVATE ${HDF5_TEST_LIB_TARGET})
else ()
TARGET_C_PROPERTIES (mirror_vfd SHARED)
target_link_libraries (mirror_vfd PRIVATE ${HDF5_TEST_LIBSH_TARGET})
endif ()
set_target_properties (mirror_vfd PROPERTIES FOLDER test)
##############################################################################
### A D D I T I O N A L T E S T S ###
##############################################################################

4
test/CMakeTests.cmake
View File

@ -446,6 +446,10 @@ set (test_CLEANFILES
vds_swmr_src_*.h5
tmp_vds_env/vds_src_2.h5
direct_chunk.h5
splitter*.h5
splitter.log
mirror_rw/*
mirror_wo/*
native_vol_test.h5
)

19
test/Makefile.am
View File

@ -33,9 +33,11 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_builddir)/src
# testvdsswmr.sh: vds_swmr*
# testabort_fail.sh: filenotclosed.c and del_many_dense_attrs.c
# test_filter_plugin.sh: filter_plugin.c
# test_mirror.sh: mirror_vfd ../utils/mirror_vfd/*
# test_usecases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes
TEST_SCRIPT = testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh testexternal_env.sh \
testswmr.sh testvds_env.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh testabort_fail.sh
testswmr.sh testvds_env.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh testabort_fail.sh \
test_mirror.sh
SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \
external_env$(EXEEXT) filenotclosed$(EXEEXT) del_many_dense_attrs$(EXEEXT) \
flushrefresh$(EXEEXT) use_append_chunk$(EXEEXT) use_append_mchunks$(EXEEXT) use_disable_mdc_flushes$(EXEEXT) \
@ -78,16 +80,18 @@ TEST_PROG= testhdf5 \
# swmr_* files (besides swmr.c) are used by testswmr.sh.
# vds_swmr_* files are used by testvdsswmr.sh
# vds_env is used by testvds_env.sh
# mirror_vfd is used by test_mirror.sh
# 'make check' doesn't run them directly, so they are not included in TEST_PROG.
# Also build testmeta, which is used for timings test. It builds quickly,
# and this lets automake keep all its test programs in one place.
check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \
testmeta accum_swmr_reader atomic_writer atomic_reader external_env \
links_env filenotclosed del_many_dense_attrs flushrefresh \
use_append_chunk use_append_mchunks use_disable_mdc_flushes \
use_append_chunk use_append_chunk_mirror use_append_mchunks use_disable_mdc_flushes \
swmr_generator swmr_start_write swmr_reader swmr_writer swmr_remove_reader \
swmr_remove_writer swmr_addrem_writer swmr_sparse_reader swmr_sparse_writer \
swmr_check_compat_vfd vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer
swmr_check_compat_vfd vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer \
mirror_vfd
if HAVE_SHARED_CONDITIONAL
check_PROGRAMS+= filter_plugin vol_plugin
endif
@ -144,6 +148,7 @@ LDADD=libh5test.la $(LIBHDF5)
ttsafe_SOURCES=ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \
ttsafe_acreate.c
cache_image_SOURCES=cache_image.c genall5.c
mirror_vfd_SOURCES=mirror_vfd.c genall5.c
VFD_LIST = sec2 stdio core core_paged split multi family
if DIRECT_VFD_CONDITIONAL
@ -213,7 +218,8 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse
flushrefresh_VERIFICATION_DONE atomic_data accum_swmr_big.h5 ohdr_swmr.h5 \
test_swmr*.h5 cache_logging.h5 cache_logging.out vds_swmr.h5 vds_swmr_src_*.h5 \
swmr[0-2].h5 swmr_writer.out swmr_writer.log.* swmr_reader.out.* swmr_reader.log.* \
tbogus.h5.copy cache_image_test.h5 direct_chunk.h5 native_vol_test.h5
tbogus.h5.copy cache_image_test.h5 direct_chunk.h5 native_vol_test.h5 \
splitter*.h5 splitter.log
# Sources for testhdf5 executable
testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \
@ -223,12 +229,13 @@ testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \
# Sources for Use Cases
use_append_chunk_SOURCES=use_append_chunk.c use_common.c
use_append_chunk_mirror_SOURCES=use_append_chunk_mirror.c use_common.c
use_append_mchunks_SOURCES=use_append_mchunks.c use_common.c
use_disable_mdc_flushes_SOURCES=use_disable_mdc_flushes.c
# Temporary files.
DISTCLEANFILES=testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh test_filter_plugin.sh \
testexternal_env.sh testswmr.sh testvds_env.sh testvdsswmr.sh test_usecases.sh testflushrefresh.sh testabort_fail.sh \
test_vol_plugin.sh
testexternal_env.sh testswmr.sh testvds_env.sh testvdsswmr.sh test_usecases.sh testflushrefresh.sh \
testabort_fail.sh test_vol_plugin.sh test_mirror.sh
include $(top_srcdir)/config/conclude.am

75
test/h5test.c
View File

@ -2042,6 +2042,80 @@ h5_get_version_string(H5F_libver_t libver)
return(LIBVER_NAMES[libver]);
} /* end of h5_get_version_string */
/*-------------------------------------------------------------------------
* Function: h5_compare_file_bytes()
*
* Purpose: Helper function to compare two files byte-for-byte.
*
* Return: Success: 0, if files are identical
* Failure: -1, if files differ
*
* Programmer: Binh-Minh Ribler
* October, 2018
*-------------------------------------------------------------------------
*/
int
h5_compare_file_bytes(char *f1name, char *f2name)
{
FILE *f1ptr = NULL; /* two file pointers */
FILE *f2ptr = NULL;
hsize_t f1size = 0; /* size of the files */
hsize_t f2size = 0;
char f1char = 0; /* one char from each file */
char f2char = 0;
hsize_t ii = 0;
int ret_value = 0; /* for error handling */
/* Open files for reading */
f1ptr = fopen(f1name, "r");
if (f1ptr == NULL) {
HDfprintf(stderr, "Unable to fopen() %s\n", f1name);
ret_value = -1;
goto done;
}
f2ptr = fopen(f2name, "r");
if (f2ptr == NULL) {
HDfprintf(stderr, "Unable to fopen() %s\n", f2name);
ret_value = -1;
goto done;
}
/* Get the file sizes and verify that they equal */
fseek(f1ptr , 0 , SEEK_END);
f1size = ftell(f1ptr);
fseek(f2ptr , 0 , SEEK_END);
f2size = ftell(f2ptr);
if (f1size != f2size) {
HDfprintf(stderr, "Files differ in size, %llu vs. %llu\n", f1size, f2size);
ret_value = -1;
goto done;
}
/* Compare each byte and fail if a difference is found */
rewind(f1ptr);
rewind(f2ptr);
for (ii = 0; ii < f1size; ii++) {
fread(&f1char, 1, 1, f1ptr);
fread(&f2char, 1, 1, f2ptr);
if (f1char != f2char) {
HDfprintf(stderr, "Mismatch @ 0x%llX: 0x%X != 0x%X\n", ii, f1char, f2char);
ret_value = -1;
goto done;
}
}
done:
if (f1ptr) {
fclose(f1ptr);
}
if (f2ptr) {
fclose(f2ptr);
}
return(ret_value);
} /* end h5_compare_file_bytes() */
/*-------------------------------------------------------------------------
* Function: H5_get_srcdir_filename
*
@ -2094,3 +2168,4 @@ const char *H5_get_srcdir(void)
else
return(NULL);
} /* end H5_get_srcdir() */

1
test/h5test.h
View File

@ -150,6 +150,7 @@ H5TEST_DLL herr_t h5_verify_cached_stabs(const char *base_name[], hid_t fapl);
H5TEST_DLL H5FD_class_t *h5_get_dummy_vfd_class(void);
H5TEST_DLL H5VL_class_t *h5_get_dummy_vol_class(void);
H5TEST_DLL const char *h5_get_version_string(H5F_libver_t libver);
H5TEST_DLL int h5_compare_file_bytes(char *fname1, char *fname2);
/* Functions that will replace components of a FAPL */
H5TEST_DLL herr_t h5_get_vfd_fapl(hid_t fapl_id);

2736
test/mirror_vfd.c Normal file
View File

File diff suppressed because it is too large Load Diff

100
test/test_mirror.sh.in Normal file
View File

@ -0,0 +1,100 @@
#! /bin/bash
#
# Copyright by The HDF Group.
# All rights reserved.
#
# This file is part of HDF5. The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the COPYING file, which can be found at the root of the source code
# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
#
# Tests for the Mirror VFD feature.
#
# Created:
# Jacob Smith, 2019-12-30
###############################################################################
## test parameters
###############################################################################
nerrors=0
SERVER_VERBOSITY="--verbosity=1"
SERVER_PORT="--port=3000"
###############################################################################
## Main
###############################################################################
## TODO: arguments for main port, port range, verbosity?
# Parse options (none accepted at this time)
while [ $# -gt 0 ]; do
case "$1" in
*) # unknown option
echo "$0: Unknown option ($1)"
exit 1
;;
esac
done
RUN_DIR=mirror_vfd_test
MIRROR_UTILS=../utils/mirror_vfd # TODO: presupposes from test/
mkdir $RUN_DIR
# Copy program files into dedicated test directory
for FILE in $MIRROR_UTILS/mirror_* ; do
case "$FILE" in
*.o) continue ;; # Don't copy .o files
esac
cp $FILE $RUN_DIR
done
cp mirror_vfd $RUN_DIR
# With the --disable-shared option, program files are built in their main
# directories; otherwise they are built in dir/.libs with a corresponding
# wrapper script. Copy these libs builds if appropriate.
if [ -f $MIRROR_UTILS/.libs/mirror_server ] ; then
RUN_LIBS=$RUN_DIR/.libs
mkdir $RUN_LIBS
for FILE in $MIRROR_UTILS/.libs/mirror_* ; do
case "$FILE" in
*.o) continue ;; # Don't copy .o files
esac
cp $FILE $RUN_LIBS
done
cp .libs/mirror_vfd $RUN_LIBS
fi
cd $RUN_DIR
echo "Launching Mirror Server"
SERVER_ARGS="$SERVER_PORT $SERVER_VERBOSITY"
./mirror_server $SERVER_ARGS &
./mirror_vfd
nerrors=$?
echo "Stopping Mirror Server"
./mirror_server_halten_sie $SERVER_PORT
###############################################################################
## Report and exit
###############################################################################
cd ..
if test $nerrors -eq 0 ; then
echo "Mirror VFD tests passed."
if test -z "$HDF5_NOCLEANUP" ; then
rm -rf $RUN_DIR
fi
exit 0
else
echo "Mirror VFD tests FAILED."
exit 1
fi

52
test/use.h
View File

@ -16,48 +16,46 @@
#include "h5test.h"
/* Macro definitions */
#define Hgoto_error(val) {ret_value=val; goto done;}
#define Hgoto_done {goto done;}
#define Chunksize_DFT 256 /* chunksize default */
#define ErrorReportMax 10 /* max number of errors reported */
#define Hgoto_error(val) {ret_value=val; goto done;}
#define Hgoto_done {goto done;}
#define Chunksize_DFT 256 /* chunksize default */
#define ErrorReportMax 10 /* max number of errors reported */
/* these two definitions must match each other */
#define UC_DATATYPE H5T_NATIVE_SHORT /* use case HDF5 data type */
#define UC_CTYPE short /* use case C data type */
#define UC_RANK 3 /* use case dataset rank */
#define UC_DATATYPE H5T_NATIVE_SHORT /* use case HDF5 data type */
#define UC_CTYPE short /* use case C data type */
#define UC_RANK 3 /* use case dataset rank */
/* Name of message file that is sent by the writer */
#define WRITER_MESSAGE "USE_WRITER_MESSAGE"
/* type declarations */
typedef enum part_t {
UC_READWRITE =0, /* both writer and reader */
UC_WRITER, /* writer only */
UC_READER /* reader only */
UC_READWRITE = 0, /* both writer and reader */
UC_WRITER, /* writer only */
UC_READER /* reader only */
} part_t;
typedef struct options_t {
hsize_t chunksize; /* chunks are chunksize^2 planes */
hsize_t chunkplanes; /* number of planes per chunk, default 1 */
hsize_t chunksize; /* chunks are chunksize^2 planes */
hsize_t chunkplanes; /* number of planes per chunk, default 1 */
hsize_t chunkdims[UC_RANK]; /* chunk dims is (chunkplan, chunksize, chunksize) */
hsize_t dims[UC_RANK]; /* dataset initial dims */
hsize_t max_dims[UC_RANK]; /* dataset max dims */
hsize_t nplanes; /* number of planes to write, default proportional to chunksize */
char *filename; /* use case data filename */
part_t launch; /* launch writer, reader or both */
hbool_t use_swmr; /* use swmr open (1) or not */
int iterations; /* iterations, default 1 */
hsize_t nplanes; /* number of planes to write, default proportional to chunksize */
char *filename; /* use case data filename */
part_t launch; /* launch writer, reader or both */
hbool_t use_swmr; /* use swmr open (1) or not */
int iterations; /* iterations, default 1 */
hid_t fapl_id; /* instance-specific FAPL ID */
char *progname; /* Program name (used in usage and dset name) */
} options_t;
/* global variables declarations */
extern options_t UC_opts; /* Use Case Options */
extern const char *progname_g; /* Program name */
/* prototype declarations */
int parse_option(int argc, char * const argv[]);
int setup_parameters(int argc, char * const argv[]);
void show_parameters(void);
int parse_option(int argc, char * const argv[], options_t * opts);
int setup_parameters(int argc, char * const argv[], options_t * opts);
void show_parameters(options_t * opts);
void usage(const char *prog);
int create_uc_file(void);
int write_uc_file(hbool_t tosend, hid_t fid);
int read_uc_file(hbool_t towait);
int create_uc_file(options_t * opts);
int write_uc_file(hbool_t tosend, hid_t file_id, options_t * opts);
int read_uc_file(hbool_t towait, options_t * opts);

227
test/use_append_chunk.c
View File

@ -11,7 +11,7 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Use Case 1.7 Appending a single chunk
* Use Case 1.7 Appending a single chunk
* Description:
* Appending a single chunk of raw data to a dataset along an unlimited
* dimension within a pre-created file and reading the new data back.
@ -24,35 +24,36 @@
* Level:
* User Level
* Guarantees:
* o Readers will see the modified dimension sizes after the Writer
* finishes HDF5 metadata updates and issues H5Fflush or H5Oflush calls.
* o Readers will see newly appended data after the Writer finishes
* the flush operation.
*
* o Readers will see the modified dimension sizes after the Writer
* finishes HDF5 metadata updates and issues H5Fflush or H5Oflush calls.
* o Readers will see newly appended data after the Writer finishes
* the flush operation.
*
* Preconditions:
* o Readers are not allowed to modify the file. o All datasets
* that are modified by the Writer exist when the Writer opens the file.
* o All datasets that are modified by the Writer exist when a Reader
* opens the file. o Data is written by a hyperslab contained in
* one chunk.
*
* o Readers are not allowed to modify the file.
* o All datasets that are modified by the Writer exist when the Writer
* opens the file.
* o All datasets that are modified by the Writer exist when a Reader
* opens the file.
* o Data is written by a hyperslab contained in one chunk.
*
* Main Success Scenario:
* 1. An application creates a file with required objects (groups,
* datasets, and attributes).
* 2. The Writer application opens the file and datasets in the file
* and starts adding data along the unlimited dimension using a hyperslab
* selection that corresponds to an HDF5 chunk.
* 3. A Reader opens the file and a dataset in a file, and queries
* the sizes of the dataset; if the extent of the dataset has changed,
* reads the appended data back.
*
* 1. An application creates a file with required objects (groups,
* datasets, and attributes).
* 2. The Writer application opens the file and datasets in the file
* and starts adding data along the unlimited dimension using a hyperslab
* selection that corresponds to an HDF5 chunk.
* 3. A Reader opens the file and a dataset in a file, and queries
* the sizes of the dataset; if the extent of the dataset has changed,
* reads the appended data back.
*
* Discussion points:
* 1. Since the new data is written to the file, and metadata update
* operation of adding pointer to the newly written chunk is atomic and
* happens after the chunk is on the disk, only two things may happen
* to the Reader:
* o The Reader will not see new data.
* o The Reader will see all new data written by Writer.
* 1. Since the new data is written to the file, and metadata update
* operation of adding pointer to the newly written chunk is atomic and
* happens after the chunk is on the disk, only two things may happen
* to the Reader:
* o The Reader will not see new data.
* o The Reader will see all new data written by Writer.
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* Created: Albert Cheng, 2013/5/28 */
@ -68,46 +69,43 @@
#include "use.h"
/* Global Variable definitions */
options_t UC_opts; /* Use Case Options */
const char *progname_g="use_append_chunk"; /* program name */
#define USE_APPEND_CHUNK_PROGNAME "use_append_chunk"
static options_t UC_opts; /* Use Case Options */
/* Setup parameters for the use case.
* Return: 0 succeed; -1 fail.
*/
int setup_parameters(int argc, char * const argv[])
int
setup_parameters(int argc, char * const argv[], options_t * opts)
{
/* use case defaults */
HDmemset(&UC_opts, 0, sizeof(options_t));
UC_opts.chunksize = Chunksize_DFT;
UC_opts.use_swmr = TRUE; /* use swmr open */
UC_opts.iterations = 1;
UC_opts.chunkplanes = 1;
HDmemset(opts, 0, sizeof(options_t));
opts->chunksize = Chunksize_DFT;
opts->use_swmr = TRUE; /* use swmr open */
opts->iterations = 1;
opts->chunkplanes = 1;
opts->progname = USE_APPEND_CHUNK_PROGNAME;
/* parse options */
if (parse_option(argc, argv) < 0)
return(-1);
if (parse_option(argc, argv, opts) < 0)
return(-1);
/* set chunk dims */
UC_opts.chunkdims[0] = UC_opts.chunkplanes;
UC_opts.chunkdims[1] = UC_opts.chunkdims[2] = UC_opts.chunksize;
opts->chunkdims[0] = opts->chunkplanes;
opts->chunkdims[1] = opts->chunkdims[2] = opts->chunksize;
/* set dataset initial and max dims */
UC_opts.dims[0] = 0;
UC_opts.max_dims[0] = H5S_UNLIMITED;
UC_opts.dims[1] = UC_opts.dims[2] = UC_opts.max_dims[1] = UC_opts.max_dims[2] = UC_opts.chunksize;
opts->dims[0] = 0;
opts->max_dims[0] = H5S_UNLIMITED;
opts->dims[1] = opts->dims[2] = opts->max_dims[1] = opts->max_dims[2] = opts->chunksize;
/* set nplanes */
if (UC_opts.nplanes == 0)
UC_opts.nplanes = (hsize_t)UC_opts.chunksize;
if (opts->nplanes == 0)
opts->nplanes = (hsize_t)opts->chunksize;
/* show parameters and return */
show_parameters();
show_parameters(opts);
return(0);
}
} /* setup_parameters() */
/* Overall Algorithm:
/* Overall Algorithm:
* Parse options from user;
* Generate/pre-created test files needed and close it;
* fork: child process becomes the reader process;
@ -119,22 +117,20 @@ main(int argc, char *argv[])
{
pid_t childpid=0;
pid_t mypid, tmppid;
int child_status;
int child_status;
int child_wait_option=0;
int ret_value = 0;
int child_ret_value;
hbool_t send_wait = FALSE;
hid_t fapl = -1; /* File access property list */
hid_t fid = -1; /* File ID */
char *name; /* Test file name */
/* initialization */
if (setup_parameters(argc, argv) < 0){
if (setup_parameters(argc, argv, &UC_opts) < 0) {
Hgoto_error(1);
}
/* Determine the need to send/wait message file*/
if(UC_opts.launch == UC_READWRITE) {
if (UC_opts.launch == UC_READWRITE) {
HDunlink(WRITER_MESSAGE);
send_wait = TRUE;
}
@ -144,38 +140,63 @@ main(int argc, char *argv[])
/* UC_WRITER: create datafile, skip reader, launch writer. */
/* UC_READER: skip create, launch reader, exit. */
/* ==============================================================*/
/* ============*/
/* =========== */
/* Create file */
/* ============*/
if (UC_opts.launch != UC_READER){
/* =========== */
if (UC_opts.launch != UC_READER) {
HDprintf("Creating skeleton data file for test...\n");
if (create_uc_file() < 0){
if ((UC_opts.fapl_id = h5_fileaccess()) < 0) {
HDfprintf(stderr, "can't create creation FAPL\n");
Hgoto_error(1);
}
if (H5Pset_libver_bounds(UC_opts.fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) {
HDfprintf(stderr, "can't set creation FAPL libver bounds\n");
Hgoto_error(1);
}
if (create_uc_file(&UC_opts) < 0) {
HDfprintf(stderr, "***encounter error\n");
Hgoto_error(1);
}else
} else {
HDprintf("File created.\n");
}
/* Close FAPL to prevent issues with forking later */
if (H5Pclose(UC_opts.fapl_id) < 0) {
HDfprintf(stderr, "can't close creation FAPL\n");
Hgoto_error(1);
}
UC_opts.fapl_id = H5I_INVALID_HID;
}
if (UC_opts.launch==UC_READWRITE){
/* fork process */
if((childpid = HDfork()) < 0) {
/* ============ */
/* Fork process */
/* ============ */
if (UC_opts.launch == UC_READWRITE) {
if ((childpid = HDfork()) < 0) {
HDperror("fork");
Hgoto_error(1);
};
};
}
}
mypid = HDgetpid();
/* ============= */
/* launch reader */
/* ============= */
if (UC_opts.launch != UC_WRITER){
if (UC_opts.launch != UC_WRITER) {
/* child process launch the reader */
if(0 == childpid) {
if (0 == childpid) {
HDprintf("%d: launch reader process\n", mypid);
if (read_uc_file(send_wait) < 0){
if ((UC_opts.fapl_id = h5_fileaccess()) < 0) {
HDfprintf(stderr, "can't create read FAPL\n");
HDexit(EXIT_FAILURE);
}
if (read_uc_file(send_wait, &UC_opts) < 0) {
HDfprintf(stderr, "read_uc_file encountered error\n");
HDexit(EXIT_FAILURE);
}
if (H5Pclose(UC_opts.fapl_id) < 0) {
HDfprintf(stderr, "can't close read FAPL\n");
HDexit(EXIT_FAILURE);
}
HDexit(EXIT_SUCCESS);
}
}
@ -186,65 +207,63 @@ main(int argc, char *argv[])
/* this process continues to launch the writer */
HDprintf("%d: continue as the writer process\n", mypid);
name = UC_opts.filename;
/* Set file access proeprty list */
if((fapl = h5_fileaccess()) < 0)
if ((fapl = h5_fileaccess()) < 0) {
HDfprintf(stderr, "can't create write FAPL\n");
Hgoto_error(1);
}
if(UC_opts.use_swmr)
if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
if (UC_opts.use_swmr) {
if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) {
HDfprintf(stderr, "can't set write FAPL libver bounds\n");
Hgoto_error(1);
}
}
/* Open the file */
if((fid = H5Fopen(name, H5F_ACC_RDWR | (UC_opts.use_swmr ? H5F_ACC_SWMR_WRITE : 0), fapl)) < 0) {
if ((fid = H5Fopen(UC_opts.filename, H5F_ACC_RDWR | (UC_opts.use_swmr ? H5F_ACC_SWMR_WRITE : 0), fapl)) < 0) {
HDfprintf(stderr, "H5Fopen failed\n");
Hgoto_error(1);
}
if(write_uc_file(send_wait, fid) < 0) {
if (write_uc_file(send_wait, fid, &UC_opts) < 0) {
HDfprintf(stderr, "write_uc_file encountered error\n");
Hgoto_error(1);
}
if (H5Fclose(fid) < 0) {
HDfprintf(stderr, "Failed to close write\n");
Hgoto_error(1);
}
if (H5Pclose(fapl) < 0) {
HDfprintf(stderr, "can't close write FAPL\n");
Hgoto_error(1);
}
/* ================================================ */
/* If readwrite, collect exit code of child process */
/* ================================================ */
if (UC_opts.launch == UC_READWRITE){
if ((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0){
if (UC_opts.launch == UC_READWRITE) {
if ((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) {
HDperror("waitpid");
Hgoto_error(1);
}
/* Close the file */
if(H5Fclose(fid) < 0) {
HDfprintf(stderr, "Failed to close file id\n");
Hgoto_error(1);
}
/* Close the property list */
if(H5Pclose(fapl) < 0) {
HDfprintf(stderr, "Failed to close the property list\n");
Hgoto_error(1);
}
if (WIFEXITED(child_status)){
if ((child_ret_value=WEXITSTATUS(child_status)) != 0){
if (WIFEXITED(child_status)) {
if ((child_ret_value = WEXITSTATUS(child_status)) != 0) {
HDprintf("%d: child process exited with non-zero code (%d)\n",
mypid, child_ret_value);
Hgoto_error(2);
}
} else {
HDprintf("%d: child process terminated abnormally\n", mypid);
Hgoto_error(2);
}
} else {
HDprintf("%d: child process terminated abnormally\n", mypid);
Hgoto_error(2);
}
}
done:
/* Print result and exit */
if (ret_value != 0){
if (ret_value != 0) {
HDprintf("Error(s) encountered\n");
}else{
} else {
HDprintf("All passed\n");
}

403
test/use_append_chunk_mirror.c Normal file
View File

@ -0,0 +1,403 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* HACKED VERSION
* Demonstrate SWMR with a mirrored file.
*
* Must be built with SERVER_IP as the IP address of the target system
* with a running mirror server, and SERVER_PORT as the primary server port.
*
* In addition to the local file, 'shinano.h5' will be created on the remote
* system, mirroring the local file. The file location will be local to
* Server's/Writer's invocation directory.
*
* Template for demonstration purposes:
*
* # Launch mirror server on remote machine (in foreground to easily stop)
* REMOTE(1)$ ./mirror_server /path/to/mirror_worker
*
* # Launch chunk writer with plenty of chunks.
* LOCAL(1)$ ./use_append_chunk_mirror -l w -n 10000
*
* # Wait one second for files to be created.
*
* # Launch chunk readers on both files.
* LOCAL(2)$ ./use_append_chunk_mirror -l r -n 10000
* REMOTE(2)$ ./use_append_chunk_mirror -l r -n 10000 -f shinano.h5
*
* # Hard-stop the server.
* REMOTE(1)$ ^C
* # alt, softer shutdown using echo and nc
* echo "SHUTDOWN" | nc localhost 3000
*/
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Use Case 1.7 Appending a single chunk
* Description:
* Appending a single chunk of raw data to a dataset along an unlimited
* dimension within a pre-created file and reading the new data back.
* Goal:
* Read data appended by the Writer to a pre-existing dataset in a
* file. The dataset has one or more unlimited dimensions. The data is
* appended by a hyperslab that is contained in one chunk (for example,
* appending 2-dim planes along the slowest changing dimension in the
* 3-dim dataset).
* Level:
* User Level
* Guarantees:
* o Readers will see the modified dimension sizes after the Writer
* finishes HDF5 metadata updates and issues H5Fflush or H5Oflush calls.
* o Readers will see newly appended data after the Writer finishes
* the flush operation.
*
* Preconditions:
* o Readers are not allowed to modify the file.
* o All datasets that are modified by the Writer exist when the Writer
* opens the file.
* o All datasets that are modified by the Writer exist when a Reader
* opens the file.
* o Data is written by a hyperslab contained in one chunk.
*
* Main Success Scenario:
* 1. An application creates a file with required objects (groups,
* datasets, and attributes).
* 2. The Writer application opens the file and datasets in the file
* and starts adding data along the unlimited dimension using a hyperslab
* selection that corresponds to an HDF5 chunk.
* 3. A Reader opens the file and a dataset in a file, and queries
* the sizes of the dataset; if the extent of the dataset has changed,
* reads the appended data back.
*
* Discussion points:
* 1. Since the new data is written to the file, and metadata update
* operation of adding pointer to the newly written chunk is atomic and
* happens after the chunk is on the disk, only two things may happen
* to the Reader:
* o The Reader will not see new data.
* o The Reader will see all new data written by Writer.
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* Created: Jacob Smith, 2019 */
#include "use.h"
/* This test uses many POSIX things that are not available on
* Windows. We're using a check for fork(2) here as a proxy for
* all POSIX/Unix/Linux things until this test can be made
* more platform-independent.
*/
#ifdef H5_HAVE_FORK
#ifdef H5_HAVE_MIRROR_VFD
#define THIS_PROGNAME "use_append_chunk_mirror"
#define CONNECT_WITH_JELLY 0
#if CONNECT_WITH_JELLY
#define SERVER_IP "10.10.10.248" /* hard-coded IP address */
#else
#define SERVER_IP "127.0.0.1" /* localhost */
#endif /* CONNECT_WITH_JELLY */
#define SERVER_PORT 3000 /* hard-coded port number */
#define MIRROR_FILE_NAME "shinano.h5" /* hard-coded duplicate/mirror filename */
static options_t UC_opts; /* Use Case Options */
/* Setup parameters for the use case.
* Return: 0 succeed; -1 fail.
*/
int
setup_parameters(int argc, char * const argv[], options_t * opts)
{
/* use case defaults */
HDmemset(opts, 0, sizeof(options_t));
opts->chunksize = Chunksize_DFT;
opts->use_swmr = TRUE;
opts->iterations = 1;
opts->chunkplanes = 1;
opts->progname = THIS_PROGNAME;
if (parse_option(argc, argv, opts) < 0)
return(-1);
opts->chunkdims[0] = opts->chunkplanes;
opts->chunkdims[1] = opts->chunkdims[2] = opts->chunksize;
opts->dims[0] = 0;
opts->max_dims[0] = H5S_UNLIMITED;
opts->dims[1] = opts->dims[2] = opts->max_dims[1] = opts->max_dims[2] = opts->chunksize;
if (opts->nplanes == 0)
opts->nplanes = (hsize_t)opts->chunksize;
show_parameters(opts);
return(0);
} /* setup_parameters() */
/* Overall Algorithm:
* Parse options from user;
* Generate/pre-created test files needed and close it;
* fork: child process becomes the reader process;
* while parent process continues as the writer process;
* both run till ending conditions are met.
*/
int
main(int argc, char *argv[])
{
pid_t childpid=0;
pid_t mypid, tmppid;
int child_status;
int child_wait_option=0;
int ret_value = 0;
int child_ret_value;
hbool_t send_wait = FALSE;
hid_t fid = -1; /* File ID */
H5FD_mirror_fapl_t mirr_fa;
H5FD_splitter_vfd_config_t split_fa;
hid_t mirr_fapl_id = H5I_INVALID_HID;
if (setup_parameters(argc, argv, &UC_opts) < 0) {
Hgoto_error(1);
}
mirr_fa.magic = H5FD_MIRROR_FAPL_MAGIC;
mirr_fa.version = H5FD_MIRROR_CURR_FAPL_T_VERSION;
mirr_fa.handshake_port = SERVER_PORT;
HDstrncpy(mirr_fa.remote_ip, SERVER_IP, H5FD_MIRROR_MAX_IP_LEN);
split_fa.wo_fapl_id = H5I_INVALID_HID;
split_fa.rw_fapl_id = H5I_INVALID_HID;
split_fa.magic = H5FD_SPLITTER_MAGIC;
split_fa.version = H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION;
split_fa.log_file_path[0] = '\0'; /* none */
split_fa.ignore_wo_errs = FALSE;
HDstrncpy(split_fa.wo_path, MIRROR_FILE_NAME, H5FD_SPLITTER_PATH_MAX);
/* Determine the need to send/wait message file*/
if (UC_opts.launch == UC_READWRITE) {
HDunlink(WRITER_MESSAGE);
send_wait = TRUE;
}
/* ==============================================================*/
/* UC_READWRITE: create datafile, launch both reader and writer. */
/* UC_WRITER: create datafile, skip reader, launch writer. */
/* UC_READER: skip create, launch reader, exit. */
/* ==============================================================*/
/* =========== */
/* Create file */
/* =========== */
if (UC_opts.launch != UC_READER) {
HDprintf("Creating skeleton data file for test...\n");
/* Prepare mirror child driver */
mirr_fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if (mirr_fapl_id == H5I_INVALID_HID) {
HDfprintf(stderr, "can't create creation mirror FAPL\n");
Hgoto_error(1);
}
if (H5Pset_fapl_mirror(mirr_fapl_id, &mirr_fa) < 0) {
HDfprintf(stderr, "can't set creation mirror FAPL\n");
H5Eprint2(H5E_DEFAULT, stdout);
Hgoto_error(1);
}
/* Prepare parent "splitter" driver in UC_opts */
split_fa.wo_fapl_id = mirr_fapl_id;
split_fa.rw_fapl_id = H5P_DEFAULT;
UC_opts.fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if (UC_opts.fapl_id == H5I_INVALID_HID) {
HDfprintf(stderr, "can't create creation FAPL\n");
Hgoto_error(1);
}
if (H5Pset_fapl_splitter(UC_opts.fapl_id, &split_fa) < 0) {
HDfprintf(stderr, "can't set creation FAPL\n");
H5Eprint2(H5E_DEFAULT, stdout);
Hgoto_error(1);
}
if (H5Pset_libver_bounds(UC_opts.fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) {
HDfprintf(stderr, "can't set creation FAPL libver bounds\n");
Hgoto_error(1);
}
/* Create file */
if (create_uc_file(&UC_opts) < 0) {
HDfprintf(stderr, "***encounter error\n");
Hgoto_error(1);
} else {
HDprintf("File created.\n");
}
/* Close FAPLs to prevent issues with forking later */
if (H5Pclose(UC_opts.fapl_id) < 0) {
HDfprintf(stderr, "can't close creation FAPL\n");
Hgoto_error(1);
}
UC_opts.fapl_id = H5I_INVALID_HID;
if (H5Pclose(mirr_fapl_id) < 0) {
HDfprintf(stderr, "can't close creation mirror FAPL\n");
Hgoto_error(1);
}
mirr_fapl_id = H5I_INVALID_HID;
}
/* ============ */
/* Fork process */
/* ============ */
if (UC_opts.launch == UC_READWRITE) {
if ((childpid = HDfork()) < 0) {
HDperror("fork");
Hgoto_error(1);
}
}
mypid = HDgetpid();
/* ============= */
/* launch reader */
/* ============= */
if (UC_opts.launch != UC_WRITER) {
/* child process -- launch the reader */
/* reader only opens the one file -- separate reader needed for mirrored file 'shinano.h5' */
if (0 == childpid) {
HDprintf("%d: launch reader process\n", mypid);
UC_opts.fapl_id = H5P_DEFAULT;
if (read_uc_file(send_wait, &UC_opts) < 0) {
HDfprintf(stderr, "read_uc_file encountered error (%d)\n", mypid);
HDexit(1);
}
HDexit(0);
}
}
/* ============= */
/* launch writer */
/* ============= */
/* this process continues to launch the writer */
HDprintf("%d: continue as the writer process\n", mypid);
/* Prepare mirror child driver */
mirr_fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if (mirr_fapl_id == H5I_INVALID_HID) {
HDfprintf(stderr, "can't create creation mirror FAPL\n");
Hgoto_error(1);
}
if (H5Pset_fapl_mirror(mirr_fapl_id, &mirr_fa) < 0) {
HDfprintf(stderr, "can't set creation mirror FAPL\n");
H5Eprint2(H5E_DEFAULT, stdout);
Hgoto_error(1);
}
/* Prepare parent "splitter" driver in UC_opts */
split_fa.wo_fapl_id = mirr_fapl_id;
split_fa.rw_fapl_id = H5P_DEFAULT;
UC_opts.fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if (UC_opts.fapl_id == H5I_INVALID_HID) {
HDfprintf(stderr, "can't create creation FAPL\n");
Hgoto_error(1);
}
if (H5Pset_fapl_splitter(UC_opts.fapl_id, &split_fa) < 0) {
HDfprintf(stderr, "can't set creation FAPL\n");
H5Eprint2(H5E_DEFAULT, stdout);
Hgoto_error(1);
}
if (UC_opts.use_swmr) {
if (H5Pset_libver_bounds(UC_opts.fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) {
HDfprintf(stderr, "can't set write FAPL libver bounds\n");
Hgoto_error(1);
}
}
if ((fid = H5Fopen(UC_opts.filename, H5F_ACC_RDWR | (UC_opts.use_swmr ? H5F_ACC_SWMR_WRITE : 0), UC_opts.fapl_id)) < 0) {
HDfprintf(stderr, "H5Fopen failed\n");
Hgoto_error(1);
}
if (write_uc_file(send_wait, fid, &UC_opts) < 0) {
HDfprintf(stderr, "write_uc_file encountered error\n");
Hgoto_error(1);
}
if (H5Fclose(fid) < 0) {
HDfprintf(stderr, "Failed to close write\n");
Hgoto_error(1);
}
if (H5Pclose(UC_opts.fapl_id) < 0) {
HDfprintf(stderr, "can't close write FAPL\n");
Hgoto_error(1);
}
if (H5Pclose(mirr_fapl_id) < 0) {
HDfprintf(stderr, "can't close write mirror FAPL\n");
Hgoto_error(1);
}
/* ================================================ */
/* If readwrite, collect exit code of child process */
/* ================================================ */
if (UC_opts.launch == UC_READWRITE) {
if ((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) {
HDperror("waitpid");
Hgoto_error(1);
}
if (WIFEXITED(child_status)) {
if ((child_ret_value = WEXITSTATUS(child_status)) != 0) {
HDprintf("%d: child process exited with non-zero code (%d)\n",
mypid, child_ret_value);
Hgoto_error(2);
}
} else {
HDprintf("%d: child process terminated abnormally\n", mypid);
Hgoto_error(2);
}
}
done:
if (ret_value != 0) {
HDprintf("Error(s) encountered\n");
} else {
HDprintf("All passed\n");
}
return(ret_value);
}
#else /* H5_HAVE_MIRROR_VFD */
int
main(void)
{
HDfprintf(stderr, "Mirror VFD is not built. Skipping.\n");
return EXIT_SUCCESS;
} /* end main() */
#endif /* H5_HAVE_MIRROR_VFD */
#else /* H5_HAVE_FORK */
int
main(void)
{
HDfprintf(stderr, "Non-POSIX platform. Skipping.\n");
return EXIT_SUCCESS;
} /* end main() */
#endif /* H5_HAVE_FORK */

178
test/use_append_mchunks.c
View File

@ -29,14 +29,14 @@
* finishes HDF5 metadata updates and issues H5Fflush or H5Oflush calls.
* o Readers will see newly appended data after the Writer finishes
* the flush operation.
*
*
* Preconditions:
* o Readers are not allowed to modify the file.
* o All datasets that are modified by the Writer exist when the
* Writer opens the file.
* o All datasets that are modified by the Writer exist when a Reader
* opens the file.
*
*
* Main Success Scenario:
* 1. An application creates a file with required objects (groups,
* datasets, and attributes).
@ -45,7 +45,7 @@
* spans several chunks.
* 3. A Reader opens the file and a dataset in a file; if the size of
* the unlimited dimension has changed, reads the appended data back.
*
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* Created: Albert Cheng, 2013/6/1 */
@ -61,46 +61,45 @@
#include "use.h"
/* Global Variable definitions */
options_t UC_opts; /* Use Case Options */
const char *progname_g="use_append_mchunks"; /* program name */
#define USE_APPEND_MCHUNKS_PROGNAME "use_append_mchunks"
static options_t UC_opts; /* Use Case Options */
/* Setup parameters for the use case.
* Return: 0 succeed; -1 fail.
*/
int setup_parameters(int argc, char * const argv[])
int
setup_parameters(int argc, char * const argv[], options_t * opts)
{
/* use case defaults */
HDmemset(&UC_opts, 0, sizeof(options_t));
UC_opts.chunksize = Chunksize_DFT;
UC_opts.use_swmr = 1; /* use swmr open */
UC_opts.iterations = 1;
UC_opts.chunkplanes = 1;
HDmemset(opts, 0, sizeof(options_t));
opts->chunksize = Chunksize_DFT;
opts->use_swmr = 1; /* use swmr open */
opts->iterations = 1;
opts->chunkplanes = 1;
opts->progname = USE_APPEND_MCHUNKS_PROGNAME;
opts->fapl_id = H5I_INVALID_HID;
/* parse options */
if (parse_option(argc, argv) < 0){
return(-1);
if (parse_option(argc, argv, opts) < 0) {
return(-1);
}
/* set chunk dims */
UC_opts.chunkdims[0] = (hsize_t)UC_opts.chunkplanes;
UC_opts.chunkdims[1] = UC_opts.chunkdims[2] = (hsize_t)UC_opts.chunksize;
/* set dataset initial and max dims */
UC_opts.dims[0] = 0;
UC_opts.max_dims[0] = H5S_UNLIMITED;
UC_opts.dims[1] = UC_opts.dims[2] = UC_opts.max_dims[1] = UC_opts.max_dims[2] = 2 * (hsize_t)UC_opts.chunksize;
opts->chunkdims[0] = (hsize_t)opts->chunkplanes;
opts->chunkdims[1] = opts->chunkdims[2] = (hsize_t)opts->chunksize;
/* set nplanes */
if (UC_opts.nplanes == 0)
UC_opts.nplanes = 2 * (hsize_t)UC_opts.chunksize;
opts->dims[0] = 0;
opts->max_dims[0] = H5S_UNLIMITED;
opts->dims[1] = opts->dims[2] = opts->max_dims[1] = opts->max_dims[2] = 2 * (hsize_t)opts->chunksize;
/* show parameters and return */
show_parameters();
if (opts->nplanes == 0)
opts->nplanes = 2 * (hsize_t)opts->chunksize;
show_parameters(opts);
return(0);
}
} /* end setup_parameters() */
/* Overall Algorithm:
/* Overall Algorithm:
* Parse options from user;
* Generate/pre-created test files needed and close it;
* fork: child process becomes the reader process;
@ -112,22 +111,20 @@ main(int argc, char *argv[])
{
pid_t childpid=0;
pid_t mypid, tmppid;
int child_status;
int child_status;
int child_wait_option=0;
int ret_value = 0;
int child_ret_value;
hbool_t send_wait = 0;
hid_t fapl = -1; /* File access property list */
hid_t fid = -1; /* File ID */
char *name; /* Test file name */
/* initialization */
if (setup_parameters(argc, argv) < 0){
if (setup_parameters(argc, argv, &UC_opts) < 0) {
Hgoto_error(1);
}
/* Determine the need to send/wait message file*/
if(UC_opts.launch == UC_READWRITE) {
if (UC_opts.launch == UC_READWRITE) {
HDunlink(WRITER_MESSAGE);
send_wait = 1;
}
@ -137,38 +134,63 @@ main(int argc, char *argv[])
/* UC_WRITER: create datafile, skip reader, launch writer. */
/* UC_READER: skip create, launch reader, exit. */
/* ==============================================================*/
/* ============*/
/* =========== */
/* Create file */
/* ============*/
if (UC_opts.launch != UC_READER){
/* =========== */
if (UC_opts.launch != UC_READER) {
HDprintf("Creating skeleton data file for test...\n");
if (create_uc_file() < 0){
HDfprintf(stderr, "***encounter error\n");
Hgoto_error(1);
}else
HDprintf("File created.\n");
if ((UC_opts.fapl_id = h5_fileaccess()) < 0) {
HDfprintf(stderr, "can't create creation FAPL\n");
Hgoto_error(1);
}
if (H5Pset_libver_bounds(UC_opts.fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) {
HDfprintf(stderr, "can't set creation FAPL libver bounds\n");
Hgoto_error(1);
}
if (create_uc_file(&UC_opts) < 0) {
HDfprintf(stderr, "***encounter error\n");
Hgoto_error(1);
} else {
HDprintf("File created.\n");
}
/* Close FAPL to prevent issues with forking later */
if (H5Pclose(UC_opts.fapl_id) < 0) {
HDfprintf(stderr, "can't close creation FAPL\n");
Hgoto_error(1);
}
UC_opts.fapl_id = H5I_INVALID_HID;
}
if (UC_opts.launch==UC_READWRITE){
/* fork process */
if((childpid = fork()) < 0) {
/* ============ */
/* Fork process */
/* ============ */
if (UC_opts.launch==UC_READWRITE) {
if ((childpid = fork()) < 0) {
perror("fork");
Hgoto_error(1);
};
};
}
}
mypid = getpid();
/* ============= */
/* launch reader */
/* ============= */
if (UC_opts.launch != UC_WRITER){
if (UC_opts.launch != UC_WRITER) {
/* child process launch the reader */
if(0 == childpid) {
if (0 == childpid) {
HDprintf("%d: launch reader process\n", mypid);
if (read_uc_file(send_wait) < 0){
if ((UC_opts.fapl_id = h5_fileaccess()) < 0) {
HDfprintf(stderr, "can't create read FAPL\n");
HDexit(EXIT_FAILURE);
}
if (read_uc_file(send_wait, &UC_opts) < 0) {
HDfprintf(stderr, "read_uc_file encountered error\n");
HDexit(EXIT_FAILURE);
}
if (H5Pclose(UC_opts.fapl_id) < 0) {
HDfprintf(stderr, "can't close read FAPL\n");
HDexit(EXIT_FAILURE);
}
HDexit(EXIT_SUCCESS);
}
}
@ -179,51 +201,50 @@ main(int argc, char *argv[])
/* this process continues to launch the writer */
HDprintf("%d: continue as the writer process\n", mypid);
name = UC_opts.filename;
/* Set the file access property list */
if((fapl = h5_fileaccess()) < 0)
if ((fapl = h5_fileaccess()) < 0) {
HDfprintf(stderr, "can't get write FAPL\n");
Hgoto_error(1);
}
if(UC_opts.use_swmr)
if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
if (UC_opts.use_swmr) {
if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) {
HDfprintf(stderr, "can't set write FAPL libver bounds\n");
Hgoto_error(1);
}
}
/* Open the file */
if((fid = H5Fopen(name, H5F_ACC_RDWR | (UC_opts.use_swmr ? H5F_ACC_SWMR_WRITE : 0), fapl)) < 0) {
if ((fid = H5Fopen(UC_opts.filename, H5F_ACC_RDWR | (UC_opts.use_swmr ? H5F_ACC_SWMR_WRITE : 0), fapl)) < 0) {
HDfprintf(stderr, "H5Fopen failed\n");
Hgoto_error(1);
}
if(write_uc_file(send_wait, fid) < 0) {
if (write_uc_file(send_wait, fid, &UC_opts) < 0) {
HDfprintf(stderr, "write_uc_file encountered error\n");
Hgoto_error(1);
}
if (H5Fclose(fid) < 0) {
HDfprintf(stderr, "Failed to close file id\n");
Hgoto_error(1);
}
if (H5Pclose(fapl) < 0) {
HDfprintf(stderr, "can't close write FAPL\n");
Hgoto_error(1);
}
/* ================================================ */
/* If readwrite, collect exit code of child process */
/* ================================================ */
if (UC_opts.launch == UC_READWRITE){
if ((tmppid = waitpid(childpid, &child_status, child_wait_option)) < 0){
if (UC_opts.launch == UC_READWRITE) {
if ((tmppid = waitpid(childpid, &child_status, child_wait_option)) < 0) {
perror("waitpid");
Hgoto_error(1);
}
/* Close the file */
if(H5Fclose(fid) < 0) {
HDfprintf(stderr, "Failed to close file id\n");
Hgoto_error(1);
}
/* Close the property list */
if(H5Pclose(fapl) < 0) {
HDfprintf(stderr, "Failed to close the property list\n");
Hgoto_error(1);
}
if (WIFEXITED(child_status)){
if ((child_ret_value=WEXITSTATUS(child_status)) != 0){
if (WIFEXITED(child_status)) {
if ((child_ret_value=WEXITSTATUS(child_status)) != 0) {
HDprintf("%d: child process exited with non-zero code (%d)\n",
mypid, child_ret_value);
Hgoto_error(1);
@ -233,17 +254,16 @@ main(int argc, char *argv[])
Hgoto_error(2);
}
}
done:
/* Print result and exit */
if (ret_value != 0){
if (ret_value != 0) {
HDprintf("Error(s) encountered\n");
}else{
} else {
HDprintf("All passed\n");
}
return(ret_value);
}
} /* end main() */
#else /* H5_HAVE_FORK */

651
test/use_common.c
View File

@ -23,8 +23,13 @@
#define H5D_FRIEND /*suppress error about including H5Dpkg */
#define H5D_TESTING
#include "H5Dpkg.h"
#include "H5Dpkg.h" /* TODO : used to verify chunk index type (yes?) -- is there a way to do this sanity-check using the public API instead? */
/* ----------------------------------------------------------------------------
* Print a common/shared usage message.
* Receives program name to show default test file name (<program_name>.h5).
* ----------------------------------------------------------------------------
*/
void
usage(const char *prog)
{
@ -41,10 +46,13 @@ usage(const char *prog)
HDfprintf(stderr, "\n");
} /* end usage() */
/* Setup Use Case parameters by parsing command line options.
* Setup default values if not set by options. */
/* ----------------------------------------------------------------------------
* Setup Use Case parameters by parsing command line options.
* Includes default values for unspecified options.
* ----------------------------------------------------------------------------
*/
int
parse_option(int argc, char * const argv[])
parse_option(int argc, char * const argv[], options_t * opts)
{
int ret_value=0;
int c;
@ -56,302 +64,307 @@ parse_option(int argc, char * const argv[])
/* suppress getopt from printing error */
opterr = 0;
while (1){
c = getopt (argc, argv, nagg_options);
if (-1 == c)
break;
switch (c) {
case 'h':
usage(progname_g);
HDexit(EXIT_SUCCESS);
break;
case 'f': /* usecase data file name */
UC_opts.filename = optarg;
break;
case 'i': /* iterations */
if ((UC_opts.iterations = HDatoi(optarg)) <= 0) {
HDfprintf(stderr, "bad iterations number %s, must be a positive integer\n", optarg);
usage(progname_g);
Hgoto_error(-1);
};
break;
case 'l': /* launch reader or writer only */
switch (*optarg) {
case 'r': /* reader only */
UC_opts.launch = UC_READER;
break;
case 'w': /* writer only */
UC_opts.launch = UC_WRITER;
break;
default:
HDfprintf(stderr, "launch value(%c) should be w or r only.\n", *optarg);
usage(progname_g);
Hgoto_error(-1);
break;
}
break;
case 'n': /* number of planes to write/read */
if ((UC_opts.nplanes = HDstrtoul(optarg, NULL, 0)) <= 0) {
HDfprintf(stderr, "bad number of planes %s, must be a positive integer\n", optarg);
usage(progname_g);
Hgoto_error(-1);
};
break;
case 's': /* use swmr file open mode */
use_swmr = HDatoi(optarg);
if (use_swmr != 0 && use_swmr != 1) {
HDfprintf(stderr, "swmr value should be 0(no) or 1(yes)\n");
usage(progname_g);
while (1) {
c = getopt(argc, argv, nagg_options);
if (-1 == c)
break;
switch (c) {
case 'h':
usage(opts->progname);
exit(EXIT_SUCCESS);
break;
case 'f': /* usecase data file name */
opts->filename = optarg;
break;
case 'i': /* iterations */
if ((opts->iterations = HDatoi(optarg)) <= 0) {
HDfprintf(stderr, "bad iterations number %s, must be a positive integer\n", optarg);
usage(opts->progname);
Hgoto_error(-1);
}
break;
case 'l': /* launch reader or writer only */
switch (*optarg) {
case 'r': /* reader only */
opts->launch = UC_READER;
break;
case 'w': /* writer only */
opts->launch = UC_WRITER;
break;
default:
HDfprintf(stderr, "launch value(%c) should be w or r only.\n", *optarg);
usage(opts->progname);
Hgoto_error(-1);
break;
} /* end switch (reader/writer-only mode toggle) */
break;
case 'n': /* number of planes to write/read */
if ((opts->nplanes = HDstrtoul(optarg, NULL, 0)) <= 0) {
HDfprintf(stderr, "bad number of planes %s, must be a positive integer\n", optarg);
usage(opts->progname);
Hgoto_error(-1);
}
break;
case 's': /* use swmr file open mode */
use_swmr = HDatoi(optarg);
if (use_swmr != 0 && use_swmr != 1) {
HDfprintf(stderr, "swmr value should be 0(no) or 1(yes)\n");
usage(opts->progname);
Hgoto_error(-1);
}
opts->use_swmr = (hbool_t)use_swmr;
break;
case 'y': /* Number of planes per chunk */
if ((opts->chunkplanes = HDstrtoul(optarg, NULL, 0)) <= 0) {
HDfprintf(stderr, "bad number of planes per chunk %s, must be a positive integer\n", optarg);
usage(opts->progname);
Hgoto_error(-1);
}
break;
case 'z': /* size of chunk=(z,z) */
if ((opts->chunksize = HDstrtoull(optarg, NULL, 0)) <= 0) {
HDfprintf(stderr, "bad chunksize %s, must be a positive integer\n", optarg);
usage(opts->progname);
Hgoto_error(-1);
}
break;
case '?':
HDfprintf(stderr, "getopt returned '%c'.\n", c);
Hgoto_error(-1);
}
UC_opts.use_swmr = (hbool_t)use_swmr;
break;
case 'y': /* Number of planes per chunk */
if ((UC_opts.chunkplanes = HDstrtoul(optarg, NULL, 0)) <= 0) {
HDfprintf(stderr, "bad number of planes per chunk %s, must be a positive integer\n", optarg);
usage(progname_g);
Hgoto_error(-1);
};
break;
case 'z': /* size of chunk=(z,z) */
if ((UC_opts.chunksize = HDstrtoull(optarg, NULL, 0)) <= 0) {
HDfprintf(stderr, "bad chunksize %s, must be a positive integer\n", optarg);
usage(progname_g);
Hgoto_error(-1);
};
break;
case '?':
HDfprintf(stderr, "getopt returned '%c'.\n", c);
Hgoto_error(-1);
default:
HDfprintf(stderr, "getopt returned unexpected value.\n");
HDfprintf(stderr, "Unexpected value is %d\n", c);
Hgoto_error(-1);
}
}
default:
HDfprintf(stderr, "getopt returned unexpected value.\n");
HDfprintf(stderr, "Unexpected value is %d\n", c);
Hgoto_error(-1);
} /* end switch (argument symbol) */
} /* end while (there are still arguments) */
/* set test file name if not given */
if (!UC_opts.filename){
/* default data file name is <progname>.h5 */
if ((UC_opts.filename=(char*)HDmalloc(HDstrlen(progname_g)+4))==NULL) {
HDfprintf(stderr, "malloc: failed\n");
Hgoto_error(-1);
};
HDstrcpy(UC_opts.filename, progname_g);
HDstrcat(UC_opts.filename, ".h5");
if (!opts->filename) {
/* default data file name is <progname>.h5 */
if ((opts->filename=(char*)HDmalloc(HDstrlen(opts->progname)+4))==NULL) {
HDfprintf(stderr, "malloc: failed\n");
Hgoto_error(-1);
}
HDstrcpy(opts->filename, opts->progname);
HDstrcat(opts->filename, ".h5");
}
done:
/* All done. */
return(ret_value);
}
} /* end parse_option() */
/* Show parameters used for this use case */
void show_parameters(void){
/* ----------------------------------------------------------------------------
* Show parameters used for this use case.
* ----------------------------------------------------------------------------
*/
void
show_parameters(options_t * opts)
{
HDprintf("===Parameters used:===\n");
printf("chunk dims=(%llu, %llu, %llu)\n", (unsigned long long)UC_opts.chunkdims[0],
(unsigned long long)UC_opts.chunkdims[1], (unsigned long long)UC_opts.chunkdims[2]);
printf("dataset max dims=(%llu, %llu, %llu)\n", (unsigned long long)UC_opts.max_dims[0],
(unsigned long long)UC_opts.max_dims[1], (unsigned long long)UC_opts.max_dims[2]);
HDprintf("number of planes to write=%llu\n", (unsigned long long)UC_opts.nplanes);
HDprintf("using SWMR mode=%s\n", UC_opts.use_swmr ? "yes(1)" : "no(0)");
HDprintf("data filename=%s\n", UC_opts.filename);
HDprintf("chunk dims=(%llu, %llu, %llu)\n", (unsigned long long)opts->chunkdims[0],
(unsigned long long)opts->chunkdims[1], (unsigned long long)opts->chunkdims[2]);
HDprintf("dataset max dims=(%llu, %llu, %llu)\n", (unsigned long long)opts->max_dims[0],
(unsigned long long)opts->max_dims[1], (unsigned long long)opts->max_dims[2]);
HDprintf("number of planes to write=%llu\n", (unsigned long long)opts->nplanes);
HDprintf("using SWMR mode=%s\n", opts->use_swmr ? "yes(1)" : "no(0)");
HDprintf("data filename=%s\n", opts->filename);
HDprintf("launch part=");
switch (UC_opts.launch){
case UC_READWRITE:
printf("Reader/Writer\n");
break;
case UC_WRITER:
printf("Writer\n");
break;
case UC_READER:
printf("Reader\n");
break;
default:
/* should not happen */
printf("Illegal part(%d)\n", UC_opts.launch);
};
HDprintf("number of iterations=%d (not used yet)\n", UC_opts.iterations);
switch (opts->launch) {
case UC_READWRITE:
HDprintf("Reader/Writer\n");
break;
case UC_WRITER:
HDprintf("Writer\n");
break;
case UC_READER:
HDprintf("Reader\n");
break;
default:
/* should not happen */
HDprintf("Illegal part(%d)\n", opts->launch);
}
HDprintf("number of iterations=%d (not used yet)\n", opts->iterations);
HDprintf("===Parameters shown===\n");
}
} /* end show_parameters() */
/* Create the skeleton use case file for testing.
/* ----------------------------------------------------------------------------
* Create the skeleton use case file for testing.
* It has one 3d dataset using chunked storage.
* The dataset is (unlimited, chunksize, chunksize).
* Dataset type is 2 bytes integer.
* It starts out "empty", i.e., first dimension is 0.
*
* Return: 0 succeed; -1 fail.
* ----------------------------------------------------------------------------
*/
int create_uc_file(void)
int
create_uc_file(options_t * opts)
{
hsize_t dims[3]; /* Dataset starting dimensions */
hid_t fid; /* File ID for new HDF5 file */
hid_t dcpl; /* Dataset creation property list */
hid_t sid; /* Dataspace ID */
hid_t dsid; /* Dataset ID */
hid_t fapl; /* File access property list */
hsize_t dims[3]; /* Dataset starting dimensions */
hid_t fid; /* File ID for new HDF5 file */
hid_t dcpl; /* Dataset creation property list */
hid_t sid; /* Dataspace ID */
hid_t dsid; /* Dataset ID */
H5D_chunk_index_t idx_type; /* Chunk index type */
/* Create the file */
if((fapl = h5_fileaccess()) < 0)
return -1;
if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
return -1;
if((fid = H5Fcreate(UC_opts.filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
if ((fid = H5Fcreate(opts->filename, H5F_ACC_TRUNC, H5P_DEFAULT, opts->fapl_id)) < 0)
return -1;
/* Set up dimension sizes */
dims[0] = 0;
dims[1] = dims[2] = UC_opts.max_dims[1];
dims[1] = dims[2] = opts->max_dims[1];
/* Create dataspace for creating datasets */
if((sid = H5Screate_simple(3, dims, UC_opts.max_dims)) < 0)
if ((sid = H5Screate_simple(3, dims, opts->max_dims)) < 0)
return -1;
/* Create dataset creation property list */
if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
return -1;
if(H5Pset_chunk(dcpl, 3, UC_opts.chunkdims) < 0)
if (H5Pset_chunk(dcpl, 3, opts->chunkdims) < 0)
return -1;
/* create dataset of progname */
if((dsid = H5Dcreate2(fid, progname_g, UC_DATATYPE, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
return -1;
if ((dsid = H5Dcreate2(fid, opts->progname, UC_DATATYPE, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
return -1;
/* Check that the chunk index type is not version 1 B-tree.
* Version 1 B-trees are not supported under SWMR.
*/
if(H5D__layout_idx_type_test(dsid, &idx_type) < 0)
if (H5D__layout_idx_type_test(dsid, &idx_type) < 0)
return -1;
if(idx_type == H5D_CHUNK_IDX_BTREE) {
if (idx_type == H5D_CHUNK_IDX_BTREE) {
HDfprintf(stderr, "ERROR: Chunk index is version 1 B-tree: aborting.\n");
return -1;
}
/* Close everything */
if(H5Dclose(dsid) < 0)
return -1;
if(H5Pclose(fapl) < 0)
if (H5Dclose(dsid) < 0)
return -1;
if(H5Pclose(dcpl) < 0)
if (H5Pclose(dcpl) < 0)
return -1;
if(H5Sclose(sid) < 0)
if (H5Sclose(sid) < 0)
return -1;
if(H5Fclose(fid) < 0)
if (H5Fclose(fid) < 0)
return -1;
return 0;
}
} /* end create_uc_file() */
/* Append planes, each of (1,2*chunksize,2*chunksize) to the dataset.
/* ----------------------------------------------------------------------------
* Append planes, each of (1,2*chunksize,2*chunksize) to the dataset.
* In other words, 4 chunks are appended to the dataset at a time.
* Fill each plan with the plane number and then write it at the nth plane.
* Increase the plane number and repeat till the end of dataset, when it
* reaches chunksize long. End product is a (2*chunksize)^3 cube.
*
* Return: 0 succeed; -1 fail.
* ----------------------------------------------------------------------------
*/
int write_uc_file(hbool_t tosend, hid_t fid)
int
write_uc_file(hbool_t tosend, hid_t file_id, options_t * opts)
{
hid_t dsid; /* dataset ID */
hid_t dcpl; /* Dataset creation property list */
UC_CTYPE *buffer, *bufptr; /* data buffer */
hsize_t cz=UC_opts.chunksize; /* Chunk size */
hid_t f_sid; /* dataset file space id */
hid_t m_sid; /* memory space id */
int rank; /* rank */
hsize_t chunk_dims[3]; /* Chunk dimensions */
hsize_t dims[3]; /* Dataspace dimensions */
hsize_t memdims[3]; /* Memory space dimensions */
hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */
hsize_t i, j, k;
hid_t dsid; /* dataset ID */
hid_t dcpl; /* Dataset creation property list */
UC_CTYPE *buffer, *bufptr; /* data buffer */
hsize_t cz=opts->chunksize; /* Chunk size */
hid_t f_sid; /* dataset file space id */
hid_t m_sid; /* memory space id */
int rank; /* rank */
hsize_t chunk_dims[3]; /* Chunk dimensions */
hsize_t dims[3]; /* Dataspace dimensions */
hsize_t memdims[3]; /* Memory space dimensions */
hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */
hsize_t i, j, k;
if(tosend)
if (TRUE == tosend) {
/* Send a message that H5Fopen is complete--releasing the file lock */
h5_send_message(WRITER_MESSAGE, NULL, NULL);
}
/* Open the dataset of the program name */
if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){
if ((dsid = H5Dopen2(file_id, opts->progname, H5P_DEFAULT)) < 0) {
HDfprintf(stderr, "H5Dopen2 failed\n");
return -1;
}
/* Find chunksize used */
if ((dcpl = H5Dget_create_plist(dsid)) < 0){
if ((dcpl = H5Dget_create_plist(dsid)) < 0) {
HDfprintf(stderr, "H5Dget_create_plist failed\n");
return -1;
}
if (H5D_CHUNKED != H5Pget_layout(dcpl)){
if (H5D_CHUNKED != H5Pget_layout(dcpl)) {
HDfprintf(stderr, "storage layout is not chunked\n");
return -1;
}
if ((rank = H5Pget_chunk(dcpl, 3, chunk_dims)) != 3){
if ((rank = H5Pget_chunk(dcpl, 3, chunk_dims)) != 3) {
HDfprintf(stderr, "storage rank is not 3\n");
return -1;
}
/* verify chunk_dims against set paramenters */
if (chunk_dims[0]!=UC_opts.chunkdims[0] || chunk_dims[1] != cz || chunk_dims[2] != cz){
if (chunk_dims[0]!=opts->chunkdims[0] || chunk_dims[1] != cz || chunk_dims[2] != cz) {
HDfprintf(stderr, "chunk size is not as expected. Got dims=(%llu,%llu,%llu)\n",
(unsigned long long)chunk_dims[0], (unsigned long long)chunk_dims[1],
(unsigned long long)chunk_dims[0], (unsigned long long)chunk_dims[1],
(unsigned long long)chunk_dims[2]);
return -1;
}
/* allocate space for data buffer 1 X dims[1] X dims[2] of UC_CTYPE */
memdims[0]=1;
memdims[1] = UC_opts.dims[1];
memdims[2] = UC_opts.dims[2];
memdims[1] = opts->dims[1];
memdims[2] = opts->dims[2];
if ((buffer=(UC_CTYPE*)HDmalloc((size_t)memdims[1]*(size_t)memdims[2]*sizeof(UC_CTYPE)))==NULL) {
HDfprintf(stderr, "malloc: failed\n");
return -1;
};
}
/*
* Get dataset rank and dimension.
*/
f_sid = H5Dget_space(dsid); /* Get filespace handle first. */
rank = H5Sget_simple_extent_ndims(f_sid);
if (rank != UC_RANK){
if (rank != UC_RANK) {
HDfprintf(stderr, "rank(%d) of dataset does not match\n", rank);
return -1;
}
if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){
if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0) {
HDfprintf(stderr, "H5Sget_simple_extent_dims got error\n");
return -1;
}
HDprintf("dataset rank %d, dimensions %llu x %llu x %llu\n",
rank, (unsigned long long)(dims[0]), (unsigned long long)(dims[1]),
rank, (unsigned long long)(dims[0]), (unsigned long long)(dims[1]),
(unsigned long long)(dims[2]));
/* verify that file space dims are as expected and are consistent with memory space dims */
if (dims[0] != 0 || dims[1] != memdims[1] || dims[2] != memdims[2]){
if (dims[0] != 0 || dims[1] != memdims[1] || dims[2] != memdims[2]) {
HDfprintf(stderr, "dataset is not empty. Got dims=(%llu,%llu,%llu)\n",
(unsigned long long)dims[0], (unsigned long long)dims[1],
(unsigned long long)dims[0], (unsigned long long)dims[1],
(unsigned long long)dims[2]);
return -1;
}
/* setup mem-space for buffer */
if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){
if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0) {
HDfprintf(stderr, "H5Screate_simple for memory failed\n");
return -1;
};
}
/* write planes */
count[0]=1;
count[1]=dims[1];
count[2]=dims[2];
for (i=0; i<UC_opts.nplanes; i++){
for (i=0; i < opts->nplanes; i++) {
/* fill buffer with value i+1 */
bufptr = buffer;
for (j=0; j<dims[1]; j++)
for (k=0; k<dims[2]; k++)
for (j=0; j < dims[1]; j++) {
for (k=0; k < dims[2]; k++) {
*bufptr++ = (UC_CTYPE)i;
}
}
/* Cork the dataset's metadata in the cache, if SWMR is enabled */
if(UC_opts.use_swmr) {
if(H5Odisable_mdc_flushes(dsid) < 0) {
if (opts->use_swmr) {
if (H5Odisable_mdc_flushes(dsid) < 0) {
HDfprintf(stderr, "H5Odisable_mdc_flushes failed\n");
return -1;
}
@ -359,64 +372,65 @@ int write_uc_file(hbool_t tosend, hid_t fid)
/* extend the dataset by one for new plane */
dims[0]=i+1;
if(H5Dset_extent(dsid, dims) < 0){
if (H5Dset_extent(dsid, dims) < 0) {
HDfprintf(stderr, "H5Dset_extent failed\n");
return -1;
}
/* Get the dataset's dataspace */
if((f_sid = H5Dget_space(dsid)) < 0){
if ((f_sid = H5Dget_space(dsid)) < 0) {
HDfprintf(stderr, "H5Dset_extent failed\n");
return -1;
}
start[0]=i;
/* Choose the next plane to write */
if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){
if (H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) {
HDfprintf(stderr, "Failed H5Sselect_hyperslab\n");
return -1;
}
/* Write plane to the dataset */
if(H5Dwrite(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){
if (H5Dwrite(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0) {
HDfprintf(stderr, "Failed H5Dwrite\n");
return -1;
}
/* Uncork the dataset's metadata from the cache, if SWMR is enabled */
if(UC_opts.use_swmr)
if(H5Oenable_mdc_flushes(dsid) < 0) {
if (opts->use_swmr) {
if (H5Oenable_mdc_flushes(dsid) < 0) {
HDfprintf(stderr, "H5Oenable_mdc_flushes failed\n");
return -1;
}
}
/* flush file to make the just written plane available. */
if(H5Dflush(dsid) < 0) {
if (H5Dflush(dsid) < 0) {
HDfprintf(stderr, "Failed to H5Fflush file\n");
return -1;
}
}
} /* end for each plane to write */
/* Done writing. Free/Close all resources including data file */
HDfree(buffer);
if (H5Dclose(dsid) < 0){
if (H5Dclose(dsid) < 0) {
HDfprintf(stderr, "Failed to close datasete\n");
return -1;
}
if (H5Sclose(m_sid) < 0){
if (H5Sclose(m_sid) < 0) {
HDfprintf(stderr, "Failed to close memory space\n");
return -1;
}
if (H5Sclose(f_sid) < 0){
if (H5Sclose(f_sid) < 0) {
HDfprintf(stderr, "Failed to close file space\n");
return -1;
}
return 0;
}
} /* end write_uc_file() */
/* Read planes from the dataset.
/* ----------------------------------------------------------------------------
* Read planes from the dataset.
* It expects the dataset is being changed (growing).
* It checks the unlimited dimension (1st one). When it increases,
* it will read in the new planes, one by one, and verify the data correctness.
@ -425,61 +439,51 @@ int write_uc_file(hbool_t tosend, hid_t fid)
* that is the expected end of data, the reader exits.
*
* Return: 0 succeed; -1 fail.
* ----------------------------------------------------------------------------
*/
int read_uc_file(hbool_t towait)
int
read_uc_file(hbool_t towait, options_t * opts)
{
hid_t fapl; /* file access property list ID */
hid_t fid; /* File ID for new HDF5 file */
hid_t dsid; /* dataset ID */
char *name;
UC_CTYPE *buffer, *bufptr; /* read data buffer */
hid_t f_sid; /* dataset file space id */
hid_t m_sid; /* memory space id */
int rank; /* rank */
hsize_t dims[3]; /* Dataspace dimensions */
hsize_t memdims[3]; /* Memory space dimensions */
hsize_t nplane=0, nplane_old=0; /* nth plane, last nth plane */
hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */
hsize_t j, k;
int nreadererr=0;
int nerrs;
int nonewplane;
hid_t fid; /* File ID for new HDF5 file */
hid_t dsid; /* dataset ID */
UC_CTYPE *buffer, *bufptr; /* read data buffer */
hid_t f_sid; /* dataset file space id */
hid_t m_sid; /* memory space id */
int rank; /* rank */
hsize_t dims[3]; /* Dataspace dimensions */
hsize_t memdims[3]; /* Memory space dimensions */
hsize_t nplane=0, nplanes_seen=0; /* nth plane, last nth plane */
hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */
hsize_t j, k;
int nreadererr=0;
int nerrs;
int loops_waiting_for_plane;
/* Before reading, wait for the message that H5Fopen is complete--file lock is released */
if(towait && h5_wait_message(WRITER_MESSAGE) < 0) {
if (towait && h5_wait_message(WRITER_MESSAGE) < 0) {
HDfprintf(stderr, "Cannot find writer message file...failed\n");
return -1;
}
name = UC_opts.filename;
/* Open the file */
if((fapl = h5_fileaccess()) < 0)
return -1;
if((fid = H5Fopen(name, H5F_ACC_RDONLY | (UC_opts.use_swmr ? H5F_ACC_SWMR_READ : 0), fapl)) < 0){
HDfprintf(stderr, "Opening to read %s\n", opts->filename);
if ((fid = H5Fopen(opts->filename, H5F_ACC_RDONLY | (opts->use_swmr ? H5F_ACC_SWMR_READ : 0), opts->fapl_id)) < 0) {
HDfprintf(stderr, "H5Fopen failed\n");
return -1;
}
if (H5Pclose(fapl) < 0){
HDfprintf(stderr, "Failed to property list\n");
return -1;
}
/* Open the dataset of the program name */
if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){
if ((dsid = H5Dopen2(fid, opts->progname, H5P_DEFAULT)) < 0) {
HDfprintf(stderr, "H5Dopen2 failed\n");
return -1;
}
/* allocate space for data buffer 1 X dims[1] X dims[2] of UC_CTYPE */
memdims[0]=1;
memdims[1] = UC_opts.dims[1];
memdims[2] = UC_opts.dims[2];
/* Allocate space for data buffer 1 X dims[1] X dims[2] of UC_CTYPE */
memdims[0] = 1;
memdims[1] = opts->dims[1];
memdims[2] = opts->dims[2];
if ((buffer=(UC_CTYPE*)HDmalloc((size_t)memdims[1]*(size_t)memdims[2]*sizeof(UC_CTYPE)))==NULL) {
HDfprintf(stderr, "malloc: failed\n");
return -1;
};
}
/*
* Get dataset rank and dimension.
@ -487,11 +491,11 @@ int read_uc_file(hbool_t towait)
*/
f_sid = H5Dget_space(dsid); /* Get filespace handle first. */
rank = H5Sget_simple_extent_ndims(f_sid);
if (rank != UC_RANK){
if (rank != UC_RANK) {
HDfprintf(stderr, "rank(%d) of dataset does not match\n", rank);
return -1;
}
if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){
if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0) {
HDfprintf(stderr, "H5Sget_simple_extent_dims got error\n");
return -1;
}
@ -499,7 +503,7 @@ int read_uc_file(hbool_t towait)
rank, (unsigned long long)(dims[0]), (unsigned long long)(dims[1]),
(unsigned long long)(dims[2]));
/* verify that file space dims are as expected and are consistent with memory space dims */
if (dims[1] != memdims[1] || dims[2] != memdims[2]){
if (dims[1] != memdims[1] || dims[2] != memdims[2]) {
HDfprintf(stderr, "dataset dimension is not as expected. Got dims=(%llu,%llu,%llu)\n",
(unsigned long long)dims[0], (unsigned long long)dims[1],
(unsigned long long)dims[2]);
@ -509,120 +513,101 @@ int read_uc_file(hbool_t towait)
return -1;
}
/* setup mem-space for buffer */
if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){
/* Setup mem-space for buffer */
if ((m_sid=H5Screate_simple(rank, memdims, NULL)) < 0) {
HDfprintf(stderr, "H5Screate_simple for memory failed\n");
return -1;
};
/* Read 1 plane at a time whenever the dataset grows larger
* (along dim[0]) */
count[0]=1;
count[1]=dims[1];
count[2]=dims[2];
/* quit when all nplanes have been read */
nonewplane=0;
while (nplane_old < UC_opts.nplanes ){
/* print progress message according to if new planes are availalbe */
if (nplane_old < dims[0]) {
if (nonewplane){
/* end the previous message */
HDprintf("\n");
nonewplane=0;
}
HDprintf("reading planes %llu to %llu\n", (unsigned long long)nplane_old,
(unsigned long long)dims[0]);
}else{
if (nonewplane){
HDprintf(".");
if (nonewplane>=30){
HDfprintf(stderr, "waited too long for new plane, quit.\n");
return -1;
}
}else{
/* print mesg only the first time; dots still no new plane */
HDprintf("no new planes to read ");
}
nonewplane++;
/* pause for a second */
HDsleep(1);
}
for (nplane=nplane_old; nplane < dims[0]; nplane++){
/* read planes between last old nplanes and current extent */
/* Get the dataset's dataspace */
if((f_sid = H5Dget_space(dsid)) < 0){
HDfprintf(stderr, "H5Dget_space failed\n");
return -1;
}
start[0]=nplane;
/* Choose the next plane to read */
if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){
HDfprintf(stderr, "H5Sselect_hyperslab failed\n");
return -1;
/* Read 1 plane at a time whenever the dataset grows larger (along dim[0]) */
count[0] = 1;
count[1] = dims[1];
count[2] = dims[2];
/* quit when all nplanes have been read */
loops_waiting_for_plane=0;
while (nplanes_seen < opts->nplanes) {
/* print progress message according to if new planes are availalbe */
if (nplanes_seen < dims[0]) {
if (loops_waiting_for_plane) {
/* end the previous message */
HDprintf("\n");
loops_waiting_for_plane=0;
}
HDprintf("reading planes %llu to %llu\n", (unsigned long long)nplanes_seen,
(unsigned long long)dims[0]);
}
/* Read the plane from the dataset */
if(H5Dread(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){
HDfprintf(stderr, "H5Dread failed\n");
return -1;
}
/* compare read data with expected data value which is nplane */
bufptr = buffer;
nerrs=0;
for (j=0; j<dims[1]; j++){
for (k=0; k<dims[2]; k++){
if ((hsize_t)*bufptr++ != nplane){
if (++nerrs < ErrorReportMax){
HDfprintf(stderr,
"found error %llu plane(%llu,%llu), expected %llu, got %d\n",
(unsigned long long)nplane, (unsigned long long)j,
(unsigned long long)k, (unsigned long long)nplane, (int)*(bufptr-1));
}
else {
if (loops_waiting_for_plane) {
HDprintf(".");
if (loops_waiting_for_plane>=30) {
HDfprintf(stderr, "waited too long for new plane, quit.\n");
return -1;
}
}
else {
/* print mesg only the first time; dots still no new plane */
HDprintf("waiting for new planes to read ");
}
loops_waiting_for_plane++;
/* pause for a second */
HDsleep(1);
}
if (nerrs){
nreadererr++;
HDfprintf(stderr, "found %d unexpected values in plane %llu\n", nerrs,
for (nplane=nplanes_seen; nplane < dims[0]; nplane++) {
/* read planes between last old nplanes and current extent */
/* Get the dataset's dataspace */
if ((f_sid = H5Dget_space(dsid)) < 0) {
HDfprintf(stderr, "H5Dget_space failed\n");
return -1;
}
start[0]=nplane;
/* Choose the next plane to read */
if (H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) {
HDfprintf(stderr, "H5Sselect_hyperslab failed\n");
return -1;
}
/* Read the plane from the dataset */
if (H5Dread(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0) {
HDfprintf(stderr, "H5Dread failed\n");
return -1;
}
/* compare read data with expected data value which is nplane */
bufptr = buffer;
nerrs=0;
for (j=0; j < dims[1]; j++) {
for (k=0; k < dims[2]; k++) {
if ((hsize_t)*bufptr++ != nplane) {
if (++nerrs < ErrorReportMax) {
HDfprintf(stderr,
"found error %llu plane(%llu,%llu), expected %llu, got %d\n",
(unsigned long long)nplane, (unsigned long long)j,
(unsigned long long)k, (unsigned long long)nplane, (int)*(bufptr-1));
} /* end if should print error */
} /* end if value mismatch */
} /* end for plane second dimension */
} /* end for plane first dimension */
if (nerrs) {
nreadererr++;
HDfprintf(stderr, "found %d unexpected values in plane %llu\n", nerrs,
(unsigned long long)nplane);
}
} /* end for each plane added since last read */
nplanes_seen=dims[0];
/* check if dataset has grown since last time (update dims) */
H5Drefresh(dsid);
f_sid = H5Dget_space(dsid); /* Get filespace handle first. */
if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0) {
HDfprintf(stderr, "H5Sget_simple_extent_dims got error\n");
return -1;
}
}
/* Have read all current planes */
nplane_old=dims[0];
} /* end while (expecting more planes to read) */
/* check if dataset has grown since last time */
#if 0
/* close dsid and file, then reopen them */
if (H5Dclose(dsid) < 0){
HDfprintf(stderr, "H5Dclose failed\n");
return -1;
}
if (H5Fclose(fid) < 0){
HDfprintf(stderr, "H5Fclose failed\n");
return -1;
}
if((fid = H5Fopen(name, H5F_ACC_RDONLY | (UC_opts.use_swmr ? H5F_ACC_SWMR_READ : 0), H5P_DEFAULT)) < 0){
HDfprintf(stderr, "H5Fopen failed\n");
return -1;
}
if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){
HDfprintf(stderr, "H5Dopen2 failed\n");
return -1;
}
#else
H5Drefresh(dsid);
#endif
f_sid = H5Dget_space(dsid); /* Get filespace handle first. */
if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){
HDfprintf(stderr, "H5Sget_simple_extent_dims got error\n");
return -1;
}
}
/* Close the file */
if(H5Fclose(fid) < 0) {
if (H5Fclose(fid) < 0) {
HDfprintf(stderr, "H5Fclose failed\n");
return -1;
}
@ -631,7 +616,7 @@ int read_uc_file(hbool_t towait)
return -1;
else
return 0;
} /* read_uc_file() */
} /* end read_uc_file() */
#endif /* H5_HAVE_FORK */

1097
test/vfd.c
View File

File diff suppressed because it is too large Load Diff

4
utils/CMakeLists.txt Normal file
View File

@ -0,0 +1,4 @@
cmake_minimum_required (VERSION 3.10)
project (HDF5_UTILS C)
add_subdirectory (mirror_vfd)

12
utils/COPYING Normal file
View File

@ -0,0 +1,12 @@
Copyright by The HDF Group.
All rights reserved.
The files and subdirectories in this directory are part of HDF5.
The full HDF5 copyright notice, including terms governing use,
modification, and redistribution, is contained in the COPYING file
which can be found at the root of the source code distribution tree
or in https://support.hdfgroup.org/ftp/HDF5/releases. If you do
not have access to either file, you may request a copy from
help@hdfgroup.org.

26
utils/Makefile.am Normal file
View File

@ -0,0 +1,26 @@
#
# Copyright by The HDF Group.
# All rights reserved.
#
# This file is part of HDF5. The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the COPYING file, which can be found at the root of the source code
# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
##
## Makefile.am
## Run automake to generate a Makefile.in from this file.
##
#
# Utils HDF5 Makefile(.in)
#
include $(top_srcdir)/config/commence.am
CONFIG=ordered
# All subdirectories
SUBDIRS=mirror_vfd
include $(top_srcdir)/config/conclude.am

64
utils/mirror_vfd/CMakeLists.txt Normal file
View File

@ -0,0 +1,64 @@
cmake_minimum_required (VERSION 3.10)
project (HDF5_UTILS_MIRRORVFD C)
#-----------------------------------------------------------------------------
# Add the mirror_server executable
#-----------------------------------------------------------------------------
set (mirror_server_SOURCES
${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_remote.c
${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_server.c
${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_writer.c)
add_executable (mirror_server ${mirror_server_SOURCES})
target_include_directories (mirror_server PRIVATE "${HDF5_UITLS_DIR};${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
if (NOT BUILD_SHARED_LIBS)
TARGET_C_PROPERTIES (mirror_server STATIC)
target_link_libraries (mirror_server PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET})
else ()
TARGET_C_PROPERTIES (mirror_server SHARED)
target_link_libraries (mirror_server PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET})
endif ()
set_target_properties (mirror_server PROPERTIES FOLDER utils)
set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};mirror_server")
set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} mirror_server)
#-----------------------------------------------------------------------------
# Add the mirror_server_halten_sie executable
#-----------------------------------------------------------------------------
set (mirror_server_halt_SOURCES ${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_server_halten_sie.c)
add_executable (mirror_server_halt ${mirror_server_halt_SOURCES})
target_include_directories (mirror_server_halt PRIVATE "${HDF5_UITLS_DIR};${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
if (NOT BUILD_SHARED_LIBS)
TARGET_C_PROPERTIES (mirror_server_halt STATIC)
target_link_libraries (mirror_server_halt PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET})
else ()
TARGET_C_PROPERTIES (mirror_server_halt SHARED)
target_link_libraries (mirror_server_halt PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET})
endif ()
set_target_properties (mirror_server_halt PROPERTIES FOLDER utils)
set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};mirror_server_halt")
set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} mirror_server_halt)
##############################################################################
##############################################################################
### I N S T A L L A T I O N ###
##############################################################################
##############################################################################
#-----------------------------------------------------------------------------
# Rules for Installation of tools using make Install target
#-----------------------------------------------------------------------------
if (HDF5_EXPORTED_TARGETS)
foreach (exec ${H5_DEP_EXECUTABLES})
INSTALL_PROGRAM_PDB (${exec} ${HDF5_INSTALL_BIN_DIR} utilsapplications)
endforeach ()
install (
TARGETS
${H5_DEP_EXECUTABLES}
EXPORT
${HDF5_EXPORTED_TARGETS}
RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT utilsapplications
)
endif ()

30
utils/mirror_vfd/Makefile.am Normal file
View File

@ -0,0 +1,30 @@
#
# Copyright by The HDF Group.
# All rights reserved.
#
# This file is part of HDF5. The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the COPYING file, which can be found at the root of the source code
# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
##
## Makefile.am
## Run automake to generate a Makefile.in from this file.
#
# HDF5 Library Makefile(.in)
#
include $(top_srcdir)/config/commence.am
AM_CPPFLAGS+=-I$(top_srcdir)/src
bin_PROGRAMS = mirror_server mirror_server_halten_sie
mirror_server_SOURCES = mirror_server.c mirror_writer.c mirror_remote.c
#mirror_writer_SOURCES = mirror_writer.c mirror_remote.c
# All programs depend on the hdf5 library
LDADD=$(LIBHDF5)
include $(top_srcdir)/config/conclude.am

225
utils/mirror_vfd/mirror_remote.c Normal file
View File

@ -0,0 +1,225 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* Common operations for "remote" processes for the Mirror VFD.
*
* Jacob Smith, 2020-03-06
*/
#include "mirror_remote.h"
#ifdef H5_HAVE_MIRROR_VFD
/* ---------------------------------------------------------------------------
* Function: mirror_log
*
* Purpose: Write message to the logging stream/file.
* If logging info pointer is NULL, uses logging defaults.
* ----------------------------------------------------------------------------
*/
void
mirror_log(struct mirror_log_info *info,
unsigned int level,
const char *format,
...)
{
FILE *stream = MIRROR_LOG_DEFAULT_STREAM;
unsigned int verbosity = MIRROR_LOG_DEFAULT_VERBOSITY;
hbool_t custom = FALSE;
if (info != NULL && info->magic == MIRROR_LOG_INFO_MAGIC) {
stream = info->stream;
verbosity = info->verbosity;
custom = TRUE;
}
if (level == V_NONE) {
return;
}
else
if (level <= verbosity) {
if (custom == TRUE && info->prefix[0] != '\0') {
HDfprintf(stream, "%s", info->prefix);
}
switch (level) {
case (V_ERR) :
HDfprintf(stream, "ERROR ");
break;
case (V_WARN) :
HDfprintf(stream, "WARNING ");
break;
default:
break;
}
if (format != NULL) {
va_list args;
HDva_start(args, format);
HDvfprintf(stream, format, args);
HDva_end(args);
}
HDfprintf(stream, "\n");
HDfflush(stream);
} /* end if sufficiently verbose to print */
} /* end mirror_log() */
/* ---------------------------------------------------------------------------
* Function: session_log_bytes
*
* Purpose: "Pretty-print" raw binary data to logging stream/file.
* If info pointer is NULL, uses logging defaults.
* ----------------------------------------------------------------------------
*/
void
mirror_log_bytes(struct mirror_log_info *info,
unsigned int level,
size_t n_bytes,
const unsigned char *buf)
{
FILE *stream = MIRROR_LOG_DEFAULT_STREAM;
unsigned int verbosity = MIRROR_LOG_DEFAULT_VERBOSITY;
if (buf == NULL) {
return;
}
if (info != NULL && info->magic == MIRROR_LOG_INFO_MAGIC) {
stream = info->stream;
verbosity = info->verbosity;
}
if (level <= verbosity) {
size_t bytes_written = 0;
const unsigned char *b = NULL;
/* print whole lines */
while ((n_bytes - bytes_written) >= 32) {
b = buf + bytes_written; /* point to region in buffer */
HDfprintf(stream,
"%04zX %02X%02X%02X%02X %02X%02X%02X%02X" \
" %02X%02X%02X%02X %02X%02X%02X%02X" \
" %02X%02X%02X%02X %02X%02X%02X%02X" \
" %02X%02X%02X%02X %02X%02X%02X%02X\n",
bytes_written,
b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7],
b[8], b[9], b[10],b[11], b[12],b[13],b[14],b[15],
b[16],b[17],b[18],b[19], b[20],b[21],b[22],b[23],
b[24],b[25],b[26],b[27], b[28],b[29],b[30],b[31]);
bytes_written += 32;
}
/* start partial line */
if (n_bytes > bytes_written) {
HDfprintf(stream, "%04zX ", bytes_written);
}
/* partial line blocks */
while ((n_bytes - bytes_written) >= 4) {
HDfprintf(stream, " %02X%02X%02X%02X",
buf[bytes_written], buf[bytes_written+1],
buf[bytes_written+2], buf[bytes_written+3]);
bytes_written += 4;
}
/* block separator before partial block */
if (n_bytes > bytes_written) {
HDfprintf(stream, " ");
}
/* partial block individual bytes */
while (n_bytes > bytes_written) {
HDfprintf(stream, "%02X", buf[bytes_written++]);
}
/* end partial line */
HDfprintf(stream, "\n");
} /* end if suitably verbose to log */
} /* end mirror_log_bytes() */
/* ---------------------------------------------------------------------------
* Function: mirror_log_init
*
* Purpose: Prepare a loginfo_t structure for use.
*
* Return: Success: Pointer to newly-ceated info.
* Failure: NULL. Either unable to allocate or cannot open file.
* ----------------------------------------------------------------------------
*/
loginfo_t *
mirror_log_init(char *path, char *prefix, unsigned int verbosity)
{
loginfo_t *info = NULL;
info = (loginfo_t *)HDmalloc(sizeof(loginfo_t));
if (info != NULL) {
info->magic = MIRROR_LOG_INFO_MAGIC;
info->verbosity = verbosity;
info->stream = MIRROR_LOG_DEFAULT_STREAM;
info->prefix[0] = '\0';
if (prefix && *prefix) {
HDstrncpy(info->prefix, prefix, MIRROR_LOG_PREFIX_MAX);
}
if (path && *path) {
FILE *f = NULL;
f = HDfopen(path, "w");
if (NULL == f) {
HDfprintf(MIRROR_LOG_DEFAULT_STREAM,
"WARN custom logging path could not be opened: %s\n",
path);
info->magic += 1;
HDfree(info);
}
else {
info->stream = f;
}
}
} /* end if able to allocate */
return info;
} /* end mirror_log_init() */
/* ---------------------------------------------------------------------------
* Function: mirror_log_term
*
* Purpose: Shut down and clean up a loginfo_t structure.
*
* Return: Success: SUCCEED. Resources released.
* Failure: FAIL. Indeterminite state.
* ----------------------------------------------------------------------------
*/
herr_t
mirror_log_term(loginfo_t *info)
{
if (info == NULL || info->magic != MIRROR_LOG_INFO_MAGIC) {
return FAIL;
}
if (info->stream != stderr || info->stream != stdout) {
if (HDfclose(info->stream) < 0) {
return FAIL;
}
}
info->magic += 1;
HDfree(info);
return SUCCEED;
} /* end mirror_log_term() */
#endif /* H5_HAVE_MIRROR_VFD */

51
utils/mirror_vfd/mirror_remote.h Normal file
View File

@ -0,0 +1,51 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* Common definitions for "remote" processes for the Mirror VFD.
*
* Jacob Smith, 2020-03-06
*/
#include "hdf5.h"
#include "H5private.h"
#ifdef H5_HAVE_MIRROR_VFD
#define V_NONE 0
#define V_ERR 1
#define V_WARN 2
#define V_INFO 3
#define V_ALL 4
#define MIRROR_LOG_DEFAULT_STREAM stdout
#define MIRROR_LOG_DEFAULT_VERBOSITY V_WARN
#define MIRROR_LOG_PREFIX_MAX 79
#define MIRROR_LOG_INFO_MAGIC 0x569D589A
typedef struct mirror_log_info {
uint32_t magic;
FILE *stream;
unsigned int verbosity;
char prefix[MIRROR_LOG_PREFIX_MAX+1];
} loginfo_t;
void mirror_log(loginfo_t *info, unsigned int level,
const char *format, ...);
void mirror_log_bytes(loginfo_t *info, unsigned int level,
size_t n_bytes, const unsigned char *buf);
loginfo_t *mirror_log_init(char *path, char *prefix, unsigned int verbosity);
int mirror_log_term(loginfo_t *loginfo);
herr_t run_writer(int socketfd, H5FD_mirror_xmit_open_t *xmit_open);
#endif /* H5_HAVE_MIRROR_VFD */

645
utils/mirror_vfd/mirror_server.c Normal file
View File

@ -0,0 +1,645 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* "Server" application to associate a Mirror VFD Driver with a Writer.
*
* Server waits on a dedicated port for a Driver to attempt to connect.
* When connection is made, reads a message from the Driver.
* If message is "SHUTDOWN", Server closes connection and terminates.
* Else, if it receives an encoded OPEN xmit (from Driver), the Server forks
* itself; the child becomes a dedicated Writer and maintains connection with
* the Driver instance, and the parent process remains a Server and returns
* to listening for incoming requests.
* Else, the message is not recognized and is ignored.
*
*
*
* mirror_server [args]
*
* Primary server for coordinating mirror VFD connections with the remote
* process.
*
* args:
* --help, -h Print help message and exit.
* --port=N Positive integer for primary listening port.
* --verbosity=N Debugging verbosity
* 0: none
* 1: errors
* 2: details
* 3: all
* --logpath=S File path to direct debugging output, if any.
* Default of none prints output to stdout.
*
*/
#include "mirror_remote.h"
#ifdef H5_HAVE_MIRROR_VFD
#define MAXBUF 2048 /* max buffer length. */
#define LISTENQ 80 /* max pending mirrorS requests */
#define DEFAULT_PORT 3000 /* default listening port */
#define MAX_PORT_LOOPS 20 /* max iteratations through port range */
#define PORT_LOOP_RETRY_DELAY 1 /* seconds to wait between port scans */
/* semi-unique "magic" numbers to sanity-check structure pointers */
#define OP_ARGS_MAGIC 0xCF074379u
#define SERVER_RUN_MAGIC 0x741B459Au
/* ---------------------------------------------------------------------------
* Structure: struct op_args
*
* Purpose: Convenience structure for holding arguments from command-line.
*
* `magic` (uint32_t)
* Semi-unique number to help validate a pointer to this struct type.
* Must be OP_ARGS_MAGIC to be considered valid.
*
* `help` (int)
* Flag that the help argument was present in the command line.
*
* `main_port` (int)
* Flag that the help argument was present in the command line.
*
* `verbosity` (int)
* Number between 0 (none) and 4 (all) that controls how much detail
* the program prints as a course of logging.
*
* `log_prepend_serv` (int)
* Flag that the logging messages should have 'S- ' at the start of each
* line.
*
* `log_prepend_type` (int)
* Flag that the logging messages should have the assocaited verbosity
* level present in the line (e.g., "WARN", "ERROR", or "INFO").
*
* `log_path` (char *)
* Path string from the command line, giving the absolute path
* for the file for logging output. Can be empty.
*
* `writer_log_path` (char *)
* Path string from the command line, giving the absolute path
* for the file for writer's logging output. Can be empty.
*
* ---------------------------------------------------------------------------
*/
struct op_args {
uint32_t magic;
int help;
int main_port;
int verbosity;
int log_prepend_serv;
int log_prepend_type;
char log_path[PATH_MAX+1];
char writer_log_path[PATH_MAX+1];
};
/* ---------------------------------------------------------------------------
* Structure: struct server_run
*
* Purpose: Convenience structure for holding information about a server
* in operation.
*
* `magic` (uint32_t)
* Semi-unique number to help validate a pointer to this struct type.
* Must be SERVER_RUN_MAGIC to be considered valid.
*
* `log_stream` (FILE *)
* File handle where logging output is directed.
* By default, is stdout.
*
* `opts` (struct opt_args)
* Contained structure, holds the server's configuration.
*
* `listenfd` (int)
* File descriptor of the listening socket.
*
* ---------------------------------------------------------------------------
*/
struct server_run {
uint32_t magic;
struct op_args opts;
struct mirror_log_info *loginfo;
int listenfd;
};
/* ---------------------------------------------------------------------------
* Function: usage
*
* Purpose: Print the usage message to stdout.
* ---------------------------------------------------------------------------
*/
static void
usage(void)
{
HDfprintf(stdout,
"mirror_server [options]\n" \
"\n" \
"Application for providing Mirror Writer process to " \
" Mirror VFD on file-open.\n" \
"Listens on a dedicated socket; forks as a Writer upon receipt" \
" of a valid OPEN xmit.\n" \
"\n" \
"Options:\n" \
"--help [-h] : Print this help message and quit.\n" \
"--logpath=PATH : File path for logging output " \
"(default none, to stdout).\n" \
"--port=PORT : Primary port (default %d).\n" \
"--verbosity=NUM : Debug printing level " \
"0..4, (default %d).\n",
DEFAULT_PORT,
MIRROR_LOG_DEFAULT_VERBOSITY);
} /* end usage() */
/* ---------------------------------------------------------------------------
* Function: parse_args
*
* Purpose: Read command line options and store results in args_out
* structure. Fails in event of unrecognized option.
*
* Return: 0 on success, -1 on failure.
* ---------------------------------------------------------------------------
*/
static int
parse_args(int argc, char **argv, struct op_args *args_out)
{
int i;
/* preset default values
*/
args_out->main_port = DEFAULT_PORT;
args_out->help = 0;
args_out->log_prepend_serv = 1;
args_out->log_prepend_type = 1;
args_out->verbosity = MIRROR_LOG_DEFAULT_VERBOSITY;
/* preset empty strings */
HDbzero(args_out->log_path, PATH_MAX+1);
HDbzero(args_out->writer_log_path, PATH_MAX+1);
if (argv == NULL || *argv == NULL) {
mirror_log(NULL, V_ERR, "invalid argv pointer");
return -1;
}
/* Loop over arguments after program name and writer_path */
for (i=2; i < argc; i++) {
if (!HDstrncmp(argv[i], "-h", 3) || !HDstrncmp(argv[i], "--help", 7)) {
mirror_log(NULL, V_INFO, "found help argument");
args_out->help = 1;
return 0;
} /* end if help */
else
if (!HDstrncmp(argv[i], "--port=", 7)) {
mirror_log(NULL, V_INFO, "parsing 'main_port' (%s)", argv[i]+7);
args_out->main_port = HDatoi(argv[i]+7);
} /* end if port */
else
if (!HDstrncmp(argv[i], "--verbosity=", 12)) {
mirror_log(NULL, V_INFO, "parsing 'verbosity' (%s)", argv[i]+12);
args_out->verbosity = HDatoi(argv[i]+12);
} /* end if verbosity */
else
if (!HDstrncmp(argv[i], "--logpath=", 10)) {
mirror_log(NULL, V_INFO, "parsing 'logpath' (%s)", argv[i]+10);
HDstrncpy(args_out->log_path, argv[i]+10, PATH_MAX);
} /* end if logpath */
else {
mirror_log(NULL, V_ERR, "unrecognized argument: %s", argv[i]);
return -1;
} /* end if unrecognized argument */
} /* end for each arg after the path to writer "receiver process" */
mirror_log(NULL, V_INFO, "all args parsed");
return 0;
} /* end parse_args() */
/* ---------------------------------------------------------------------------
* Function: prepare_listening_socket
*
* Purpose: Configure and open a socket.
* In event of error, attempts to undo its processes.
*
* Return: Success: non-negative (the file descriptor of the socket)
* Failure: -1
* ---------------------------------------------------------------------------
*/
static int
prepare_listening_socket(struct server_run *run)
{
struct sockaddr_in server_addr;
int _true = 1; /* needed for setsockopt() */
int ret_value = -1;
int ret = 0; /* for checking return value of function calls */
if (run == NULL || run->magic != SERVER_RUN_MAGIC) {
mirror_log(NULL, V_ERR, "invalid server_run pointer");
return -1;
}
mirror_log(run->loginfo, V_INFO, "preparing socket");
server_addr.sin_family = AF_INET;
server_addr.sin_addr.s_addr = HDhtonl(INADDR_ANY);
server_addr.sin_port = HDhtons((uint16_t)run->opts.main_port);
mirror_log(run->loginfo, V_INFO, "socket()");
ret_value = HDsocket(AF_INET, SOCK_STREAM, 0);
if (ret_value < 0) {
mirror_log(run->loginfo, V_ERR, "listening socket:%d", ret_value);
goto error;
}
mirror_log(run->loginfo, V_ALL, "setsockopt()");
HDsetsockopt(ret_value, SOL_SOCKET, SO_REUSEADDR, &_true, sizeof(int));
mirror_log(run->loginfo, V_INFO, "bind()");
ret = HDbind(ret_value, (struct sockaddr *)&server_addr,
sizeof(server_addr));
if (ret < 0) {
mirror_log(run->loginfo, V_ERR, "bind() %s", HDstrerror(errno));
goto error;
}
mirror_log(run->loginfo, V_INFO, "listen()");
ret = HDlisten(ret_value, LISTENQ);
if (ret < 0) {
mirror_log(run->loginfo, V_ERR, "H5FD server listen:%d", ret);
goto error;
}
return ret_value;
error:
if (ret_value >= 0) {
HDshutdown(ret_value, SHUT_RDWR);
HDclose(ret_value);
}
return -1;
} /* end prepare_listening_socket() */
/* ---------------------------------------------------------------------------
* Function: init_server_run
*
* Purpose: Set up server_run struct with default and specified values.
*
* Return: Zero (0) if successful, -1 if an error occurred.
* ---------------------------------------------------------------------------
*/
static struct server_run *
init_server_run(int argc, char **argv)
{
struct server_run *run;
run = (struct server_run *)HDmalloc(sizeof(struct server_run));
if (run == NULL) {
mirror_log(NULL, V_ERR, "can't allocate server_run struct");
return NULL;
}
run->magic = (uint32_t)SERVER_RUN_MAGIC;
run->opts.magic = (uint32_t)OP_ARGS_MAGIC;
run->listenfd = -1;
if (parse_args(argc, argv, &(run->opts)) < 0) {
mirror_log(NULL, V_ERR, "can't parse arguments");
usage();
goto error;
}
if (run->opts.help) {
usage();
return run; /* early exit */
}
run->loginfo = mirror_log_init(run->opts.log_path, "s- ",
run->opts.verbosity);
run->listenfd = prepare_listening_socket(run);
if (run->listenfd < 0) {
mirror_log(NULL, V_ERR, "can't prepare listening socket");
goto error;
}
return run;
error:
if (run != NULL) {
HDfree(run);
}
return NULL;
} /* end init_server_run() */
/* ---------------------------------------------------------------------------
* Function: term_server_run
*
* Purpose: Close opened items in a sever_run and release the pointer.
*
* Return: Zero (0) if successful, -1 if an error occurred.
* ---------------------------------------------------------------------------
*/
static int
term_server_run(struct server_run *run)
{
if (run == NULL || run->magic != SERVER_RUN_MAGIC) {
mirror_log(NULL, V_ERR, "invalid server_run pointer");
return -1;
}
mirror_log(run->loginfo, V_INFO, "shutting down");
if (run->listenfd >= 0) {
HDshutdown(run->listenfd, SHUT_RDWR); /* TODO: error-checking? */
HDclose(run->listenfd); /* TODO: error-checking? */
run->listenfd = -1;
}
if (mirror_log_term(run->loginfo) < 0) {
mirror_log(NULL, V_ERR, "can't close logging stream");
return -1; /* doesn't solve the problem, but informs of error */
}
run->loginfo = NULL;
(run->magic)++;
(run->opts.magic)++;
HDfree(run);
return 0;
} /* end term_server_run() */
/* ---------------------------------------------------------------------------
* Function: accept_connection
*
* Purpose: Main working loop; process requests as they are received.
* Does nothing if the run option help is set.
*
* Return: -1 on error, else a non-negative file descriptor of the socket.
* ---------------------------------------------------------------------------
*/
static int
accept_connection(struct server_run *run)
{
struct sockaddr_in client_addr; /**/
socklen_t clilen; /**/
struct hostent *host_port = NULL; /**/
char *hostaddrp; /**/
int connfd = -1; /* connection file descriptor */
if (run == NULL || run->magic != SERVER_RUN_MAGIC) {
mirror_log(NULL, V_ERR, "invalid server_run pointer");
return -1;
}
/*------------------------------*/
/* accept a connection on a socket */
clilen = sizeof(client_addr);
connfd = HDaccept(run->listenfd, (struct sockaddr *)&client_addr, &clilen);
if (connfd < 0) {
mirror_log(run->loginfo, V_ERR, "accept:%d", connfd);
goto error;
}
mirror_log(run->loginfo, V_INFO, "connection achieved");
/*------------------------------*/
/* get client address information */
host_port = HDgethostbyaddr(
(const char *)&client_addr.sin_addr.s_addr,
sizeof(client_addr.sin_addr.s_addr),
AF_INET);
if (host_port == NULL) {
mirror_log(run->loginfo, V_ERR, "gethostbyaddr()");
goto error;
}
/* function has the string space statically scoped -- OK until next call */
hostaddrp = HDinet_ntoa(client_addr.sin_addr);
/* TODO? proper error-checking */
mirror_log(run->loginfo, V_INFO,
"server connected with %s (%s)",
host_port->h_name,
hostaddrp);
return connfd;
error:
if (connfd >= 0) {
close(connfd);
}
return -1;
} /* end accept_connection() */
/* ---------------------------------------------------------------------------
* Function: wait_for_child
*
* Purpose: Signal handler to reap zombie processes.
* ---------------------------------------------------------------------------
*/
static void
wait_for_child(int sig)
{
while (HDwaitpid(-1, NULL, WNOHANG) > 0);
} /* end wait_for_child() */
/* ---------------------------------------------------------------------------
* Function: handle_requests
*
* Purpose: Main working loop; process requests as they are received.
* Does nothing if the run option `help` is set.
*
* Return: -1 on error, else 0 for successful operation.
* ---------------------------------------------------------------------------
*/
static int
handle_requests(struct server_run *run)
{
int connfd = -1; /**/
char mybuf[H5FD_MIRROR_XMIT_OPEN_SIZE]; /**/
int ret; /* general-purpose error-checking */
int pid; /* process ID of fork */
struct sigaction sa;
int ret_value = 0;
if (run == NULL || run->magic != SERVER_RUN_MAGIC) {
mirror_log(NULL, V_ERR, "invalid server_run pointer");
return -1;
}
if (run->opts.help) {
return 0;
}
if (run->listenfd < 0) {
mirror_log(NULL, V_ERR, "invalid listening socket");
return -1;
}
/* Set up the signal handler */
sa.sa_handler = wait_for_child;
HDsigemptyset(&sa.sa_mask);
sa.sa_flags = SA_RESTART;
if (HDsigaction(SIGCHLD, &sa, NULL) == -1) {
perror("sigaction");
return 1;
}
/* Keep listening for attempts to connect.
*/
while (1) { /* infinite loop, exited via break or goto */
mirror_log(run->loginfo, V_INFO, "server waiting for connections...");
connfd = -1;
connfd = accept_connection(run);
if (connfd < 0) {
mirror_log(run->loginfo, V_ERR, "unable to receive connection");
goto error;
}
/* Read handshake from port connection.
*/
ret = (int)HDread(connfd, &mybuf, H5FD_MIRROR_XMIT_OPEN_SIZE);
if (-1 == ret) {
mirror_log(run->loginfo, V_ERR, "read:%d", ret);
goto error;
}
mirror_log(run->loginfo, V_INFO, "received %d bytes", ret);
mirror_log(run->loginfo, V_ALL, "```");
mirror_log_bytes(run->loginfo, V_ALL, ret,
(const unsigned char *)mybuf);
mirror_log(run->loginfo, V_ALL, "```");
/* Respond to handshake message.
*/
if (!HDstrncmp("SHUTDOWN", mybuf, 8)) {
/* Stop operation if told to stop */
mirror_log(run->loginfo, V_INFO, "received SHUTDOWN!", ret);
HDclose(connfd);
connfd = -1;
goto done;
} /* end if explicit "SHUTDOWN" directive */
else
if (H5FD_MIRROR_XMIT_OPEN_SIZE == ret) {
H5FD_mirror_xmit_open_t xopen;
mirror_log(run->loginfo, V_INFO,
"probable OPEN xmit received");
H5FD_mirror_xmit_decode_open(&xopen, (const unsigned char *)mybuf);
if (FALSE == H5FD_mirror_xmit_is_open(&xopen)) {
mirror_log(run->loginfo, V_WARN,
"expected OPEN xmit was malformed");
HDclose(connfd);
continue;
}
mirror_log(run->loginfo, V_INFO,
"probable OPEN xmit confirmed");
pid = HDfork();
if (pid < 0) { /* fork error */
mirror_log(run->loginfo, V_ERR, "cannot fork");
goto error;
} /* end if fork error */
else
if (pid == 0) { /* child process (writer side of fork) */
mirror_log(run->loginfo, V_INFO,
"executing writer");
if (run_writer(connfd, &xopen) < 0) {
HDprintf("can't run writer\n");
}
else {
HDprintf("writer OK\n");
}
HDclose(connfd);
HDexit(EXIT_SUCCESS);
} /* end if writer side of fork */
else { /* parent process (server side of fork) */
mirror_log(run->loginfo, V_INFO, "tidying up from handshake");
HDclose(connfd);
} /* end if server side of fork */
} /* end else-if valid request for service */
else {
/* Ignore unrecognized messages */
HDclose(connfd);
continue;
} /* end else (not a valid message, to be ignored) */
} /* end while listening for new connections */
done:
if (connfd >= 0) {
mirror_log(run->loginfo, V_WARN, "connfd still open upon cleanup");
HDclose(connfd);
}
return ret_value;
error:
if (connfd >= 0) {
HDclose(connfd);
}
return -1;
} /* end handle_requests() */
/* ------------------------------------------------------------------------- */
int
main(int argc, char **argv)
{
struct server_run *run;
run = init_server_run(argc, argv);
if (NULL == run) {
mirror_log(NULL, V_ERR, "can't initialize run");
HDexit(EXIT_FAILURE);
}
if (handle_requests(run) < 0) {
mirror_log(run->loginfo, V_ERR, "problem handling requests");
}
if (term_server_run(run) < 0) {
mirror_log(NULL, V_ERR, "problem closing server run");
HDexit(EXIT_FAILURE);
}
HDexit(EXIT_SUCCESS);
} /* end main() */
#else /* H5_HAVE_MIRROR_VFD */
int
main(int argc, char **argv)
{
HDprintf("Mirror VFD was not built -- cannot launch server.\n");
HDexit(EXIT_FAILURE);
}
#endif /* H5_HAVE_MIRROR_VFD */

214
utils/mirror_vfd/mirror_server_halten_sie.c Normal file
View File

@ -0,0 +1,214 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Purpose: Stop the mirror server
* Exists for cross-platform, optionally remote shutdown.
*/
#include "H5private.h" /* System compatability call-wrapper macros */
#ifdef H5_HAVE_MIRROR_VFD
#define MSHS_OPTS_MAGIC 0x613B1C15u /* sanity-checking constant */
#define MSHS_IP_STR_SIZE 20
#define MSHS_DEFAULT_IP "127.0.0.1"
#define MSHS_DEFAULT_PORTNO 3000
/* ----------------------------------------------------------------------------
* Structure: struct mshs_opts
*
* Purpose: Convenience structure to hold options as parsed from the
* command line.
*
* `magic` (uint32_t)
* Semi-unique constant to help verify pointer integrity.
*
* `help` (int)
* Flag that the help argument was present.
*
* `portno` (int)
* Port number, as received from arguments.
*
* `ip` (char *)
* IP address string as received from arguments.
*
* ----------------------------------------------------------------------------
*/
struct mshs_opts {
uint32_t magic;
int help;
int portno;
char ip[MSHS_IP_STR_SIZE + 1];
};
/* ----------------------------------------------------------------------------
* Function: usage
*
* Purpose: Print usage message to stdout.
* ----------------------------------------------------------------------------
*/
static void
usage(void)
{
HDprintf("mirror_server_halten_sie [options]\n" \
"System-independent Mirror Server shutdown program.\n" \
"Sends shutdown message to Mirror Server at given IP:port\n" \
"\n" \
"Options:\n" \
" -h | --help Print this usage message and exit.\n" \
" --ip=ADDR IP Address of remote server (defaut %s)\n" \
" --port=PORT Handshake port of remote server (default %d)\n",
MSHS_DEFAULT_IP,
MSHS_DEFAULT_PORTNO);
} /* end usage() */
/* ----------------------------------------------------------------------------
* Function: parse_args
*
* Purpose: Parse command-line arguments, populating the options struct
* pointer as appropriate.
* Default values will be set for unspecified options.
*
* Return: 0 on success, negative (-1) if error.
* ----------------------------------------------------------------------------
*/
static int
parse_args(int argc, char **argv, struct mshs_opts *opts)
{
int i = 0;
opts->magic = MSHS_OPTS_MAGIC;
opts->help = 0;
opts->portno = MSHS_DEFAULT_PORTNO;
HDstrncpy(opts->ip, MSHS_DEFAULT_IP, MSHS_IP_STR_SIZE);
for (i=1; i < argc; i++) { /* start with first possible option argument */
if (!HDstrncmp(argv[i], "-h", 3) || !HDstrncmp(argv[i], "--help", 7)) {
opts->help = 1;
}
else
if (!HDstrncmp(argv[i], "--ip=", 5)) {
HDstrncpy(opts->ip, argv[i]+5, MSHS_IP_STR_SIZE);
}
else
if (!HDstrncmp(argv[i], "--port=", 7)) {
opts->portno = HDatoi(argv[i]+7);
}
else {
HDprintf("Unrecognized option: '%s'\n", argv[i]);
usage();
opts->magic++; /* invalidate for sanity */
return -1;
}
} /* end for each argument from command line */
/* auto-replace 'localhost' with numeric IP */
if (!HDstrncmp(opts->ip, "localhost", 10)) { /* include null terminator */
HDstrncpy(opts->ip, "127.0.0.1", MSHS_IP_STR_SIZE);
}
return 0;
} /* end parse_args() */
/* ----------------------------------------------------------------------------
* Function: send_shutdown
*
* Purpose: Create socket and send shutdown signal to remote server.
*
* Return: 0 on success, negative (-1) if error.
* ----------------------------------------------------------------------------
*/
static int
send_shutdown(struct mshs_opts *opts)
{
int live_socket;
struct sockaddr_in target_addr;
if (opts->magic != MSHS_OPTS_MAGIC) {
HDprintf("invalid options structure\n");
return -1;
}
live_socket = HDsocket(AF_INET, SOCK_STREAM, 0);
if (live_socket < 0) {
HDprintf("ERROR socket()\n");
return -1;
}
target_addr.sin_family = AF_INET;
target_addr.sin_port = HDhtons((uint16_t)opts->portno);
target_addr.sin_addr.s_addr = HDinet_addr(opts->ip);
HDmemset(target_addr.sin_zero, '\0', sizeof(target_addr.sin_zero));
if (HDconnect(live_socket, (struct sockaddr *)&target_addr,
(socklen_t)sizeof(target_addr))
< 0)
{
HDprintf("ERROR connect() (%d)\n%s\n", errno, HDstrerror(errno));
return -1;
}
if (HDwrite(live_socket, "SHUTDOWN", 9) == -1) {
HDprintf("ERROR write() (%d)\n%s\n", errno, HDstrerror(errno));
return -1;
}
if (HDclose(live_socket) < 0) {
HDprintf("ERROR close() can't close socket\n");
return -1;
}
return 0;
} /* end send_shutdown() */
/* ------------------------------------------------------------------------- */
int
main(int argc, char **argv)
{
struct mshs_opts opts;
if (parse_args(argc, argv, &opts) < 0) {
HDprintf("Unable to parse arguments\n");
return EXIT_FAILURE;
}
if (opts.help) {
usage();
return EXIT_SUCCESS;
}
if (send_shutdown(&opts) < 0) {
HDprintf("Unable to send shutdown command\n");
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
} /* end main() */
#else /* H5_HAVE_MIRROR_VFD */
/* ------------------------------------------------------------------------- */
int
main(int argc, char **argv)
{
HDprintf("Mirror VFD not built -- unable to perform shutdown.\n");
return EXIT_FAILURE;
}
#endif /* H5_HAVE_MIRROR_VFD */

1064
utils/mirror_vfd/mirror_writer.c Normal file
View File

File diff suppressed because it is too large Load Diff