mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-02-23 16:20:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add support for native zlib-ng in the library and compression references (#4935)

Browse Source 
* Correct compile defs for zip_perf program
This commit is contained in:
Allen Byrne 2024-10-23 16:20:15 -05:00 committed by GitHub
parent ed4419cc52
commit 1b72a1bca0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 136 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CMakeFilters.cmake
View File

@ -101,7 +101,11 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT)
endif ()
set(H5_ZLIB_FOUND ${ZLIB_FOUND})
if (H5_ZLIB_FOUND)
set (H5_ZLIB_HEADER "zlib.h")
if (HDF5_USE_ZLIB_NG)
set (H5_ZLIB_HEADER "zlib-ng.h")
else ()
set (H5_ZLIB_HEADER "zlib.h")
endif ()
set (H5_ZLIB_INCLUDE_DIR_GEN ${ZLIB_INCLUDE_DIR})
set (H5_ZLIB_INCLUDE_DIRS ${H5_ZLIB_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR})
# The FindZLIB.cmake module does not set an OUTPUT_NAME
@ -125,6 +129,9 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT)
if (H5_ZLIB_FOUND)
set (H5_HAVE_FILTER_DEFLATE 1)
set (H5_HAVE_ZLIB_H 1)
if (HDF5_USE_ZLIB_NG)
set (H5_HAVE_ZLIBNG_H 1)
endif ()
set (H5_HAVE_LIBZ 1)
if (H5_HAVE_FILTER_DEFLATE)
set (EXTERNAL_FILTERS "${EXTERNAL_FILTERS} DEFLATE")
@ -135,6 +142,7 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT)
set (HDF5_ENABLE_Z_LIB_SUPPORT OFF CACHE BOOL "" FORCE)
message (WARNING " ZLib support in HDF5 was enabled but not found")
endif ()
message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
endif ()
#-----------------------------------------------------------------------------

3
config/cmake/H5pubconf.h.in
View File

@ -366,6 +366,9 @@
/* Define to 1 if you have the <zlib.h> header file. */
#cmakedefine H5_HAVE_ZLIB_H @H5_HAVE_ZLIB_H@
/* Define to 1 if you have the <zlib.h> header file. */
#cmakedefine H5_HAVE_ZLIBNG_H @H5_HAVE_ZLIBNG_H@
/* Define to 1 if you have the `_getvideoconfig' function. */
#cmakedefine H5_HAVE__GETVIDEOCONFIG @H5_HAVE__GETVIDEOCONFIG@

5
config/cmake/HDFLibMacros.cmake
View File

@ -40,6 +40,11 @@ macro (EXTERNAL_ZLIB_LIBRARY compress_type)
add_library(${HDF_PACKAGE_NAMESPACE}zlib-static ALIAS zlib-static)
set (H5_ZLIB_STATIC_LIBRARY "${HDF_PACKAGE_NAMESPACE}zlib-static")
set (H5_ZLIB_LIBRARIES ${H5_ZLIB_STATIC_LIBRARY})
if (HDF5_USE_ZLIB_NG)
set (H5_ZLIB_HEADER "zlib-ng.h")
else ()
set (H5_ZLIB_HEADER "zlib.h")
endif ()
set (H5_ZLIB_INCLUDE_DIR_GEN "${hdf5_zlib_BINARY_DIR}")
set (H5_ZLIB_INCLUDE_DIR "${hdf5_zlib_SOURCE_DIR}")

4
config/cmake/ZLIBNG/CMakeLists.txt
View File

@ -1,5 +1,5 @@
cmake_minimum_required(VERSION 3.18)
project(ZLIB VERSION ${ZLIB_HEADER_VERSION} LANGUAGES C)
project(ZLIB LANGUAGES C)
message(STATUS "Using CMake version ${CMAKE_VERSION}")
@ -98,7 +98,7 @@ endif()
# Options parsing
#
option(WITH_GZFILEOP "Compile with support for gzFile related functions" ON)
option(ZLIB_COMPAT "Compile with zlib compatible API" ON)
option(ZLIB_COMPAT "Compile with zlib compatible API" OFF)
option(ZLIB_ENABLE_TESTS "Build test binaries" OFF)
option(ZLIBNG_ENABLE_TESTS "Test zlib-ng specific API" OFF)
option(WITH_GTEST "Build gtest_zlib" OFF)

6
release_docs/RELEASE.txt
View File

@ -47,6 +47,12 @@ New Features
Configuration:
-------------
- Added support for native zlib-ng compression.
Changed the zlib-ng CMake logic to prefer the native zlib-ng library. Added
#ifdef around the compression function calls. Added including the correct
header file with the same #ifdef.
- Renamed remaining HDF5 library CMake options except for CMake BUILD* variables
DEFAULT_API_VERSION to HDF5_DEFAULT_API_VERSION

6
src/CMakeLists.txt
View File

@ -727,8 +727,10 @@ set (H5Z_SOURCES
${HDF5_SRC_DIR}/H5Ztrans.c
)
if (H5_ZLIB_HEADER)
SET_PROPERTY(SOURCE ${HDF5_SRC_DIR}/H5Zdeflate.c PROPERTY
COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}")
message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
set_source_files_properties(${HDF5_SRC_DIR}/H5Zdeflate.c
PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
)
endif ()

43
src/H5Zdeflate.c
View File

@ -74,8 +74,12 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
if (flags & H5Z_FLAG_REVERSE) {
/* Input; uncompress */
z_stream z_strm; /* zlib parameters */
size_t nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */
#if defined(H5_HAVE_ZLIBNG_H)
zng_stream z_strm; /* zlib parameters */
#else
z_stream z_strm; /* zlib parameters */
#endif
size_t nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */
/* Allocate space for the compressed buffer */
if (NULL == (outbuf = H5MM_malloc(nalloc)))
@ -89,13 +93,22 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
H5_CHECKED_ASSIGN(z_strm.avail_out, unsigned, nalloc, size_t);
/* Initialize the uncompression routines */
#if defined(H5_HAVE_ZLIBNG_H)
if (Z_OK != zng_inflateInit(&z_strm))
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed");
#else
if (Z_OK != inflateInit(&z_strm))
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed");
#endif
/* Loop to uncompress the buffer */
do {
/* Uncompress some data */
#if defined(H5_HAVE_ZLIBNG_H)
status = zng_inflate(&z_strm, Z_SYNC_FLUSH);
#else
status = inflate(&z_strm, Z_SYNC_FLUSH);
#endif
/* Check if we are done uncompressing data */
if (Z_STREAM_END == status)
@ -103,7 +116,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
/* Check for error */
if (Z_OK != status) {
#if defined(H5_HAVE_ZLIBNG_H)
(void)zng_inflateEnd(&z_strm);
#else
(void)inflateEnd(&z_strm);
#endif
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed");
}
else {
@ -114,7 +131,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
/* Allocate a buffer twice as big */
nalloc *= 2;
if (NULL == (new_outbuf = H5MM_realloc(outbuf, nalloc))) {
#if defined(H5_HAVE_ZLIBNG_H)
(void)zng_inflateEnd(&z_strm);
#else
(void)inflateEnd(&z_strm);
#endif
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
"memory allocation failed for deflate uncompression");
} /* end if */
@ -137,7 +158,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
ret_value = z_strm.total_out;
/* Finish uncompressing the stream */
#if defined(H5_HAVE_ZLIBNG_H)
(void)zng_inflateEnd(&z_strm);
#else
(void)inflateEnd(&z_strm);
#endif
} /* end if */
else {
/*
@ -147,9 +172,13 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
*/
const Bytef *z_src = (const Bytef *)(*buf);
Bytef *z_dst; /*destination buffer */
uLongf z_dst_nbytes = (uLongf)compressBound(nbytes);
uLong z_src_nbytes = (uLong)nbytes;
int aggression; /* Compression aggression setting */
#if defined(H5_HAVE_ZLIBNG_H)
uLongf z_dst_nbytes = (uLongf)zng_compressBound(nbytes);
#else
uLongf z_dst_nbytes = (uLongf)compressBound(nbytes);
#endif
uLong z_src_nbytes = (uLong)nbytes;
int aggression; /* Compression aggression setting */
/* Set the compression aggression level */
H5_CHECKED_ASSIGN(aggression, int, cd_values[0], unsigned);
@ -160,7 +189,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
z_dst = (Bytef *)outbuf;
/* Perform compression from the source to the destination buffer */
#if defined(H5_HAVE_ZLIBNG_H)
status = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
#else
status = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
#endif
/* Check for various zlib errors */
if (Z_BUF_ERROR == status)

13
test/CMakeLists.txt
View File

@ -483,6 +483,12 @@ endforeach ()
##############################################################################
######### Also special handling of link libs #############
#-- Adding test for chunk_info
if (H5_ZLIB_HEADER)
message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
set_source_files_properties(${HDF5_TEST_SOURCE_DIR}/chunk_info.c
PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
)
endif ()
add_executable (chunk_info ${HDF5_TEST_SOURCE_DIR}/chunk_info.c)
target_compile_options(chunk_info PRIVATE "${HDF5_CMAKE_C_FLAGS}")
target_compile_definitions(chunk_info PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}")
@ -504,6 +510,13 @@ if (HDF5_ENABLE_FORMATTERS)
endif ()
#-- Adding test for direct_chunk
if (H5_ZLIB_HEADER)
message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
set_source_files_properties(${HDF5_TEST_SOURCE_DIR}/direct_chunk.c
PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
)
endif ()
add_executable (direct_chunk ${HDF5_TEST_SOURCE_DIR}/direct_chunk.c)
target_compile_options(direct_chunk PRIVATE "${HDF5_CMAKE_C_FLAGS}")
target_compile_definitions(direct_chunk PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}")

11
test/chunk_info.c
View File

@ -40,7 +40,12 @@
#include "h5test.h"
#ifdef H5_HAVE_FILTER_DEFLATE
#include "zlib.h"
#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER)
#define H5_ZLIB_HEADER "zlib.h"
#endif
#if defined(H5_ZLIB_HEADER)
#include H5_ZLIB_HEADER /* "zlib.h" */
#endif
#endif
/* Test file names, using H5F_libver_t as indices */
@ -530,7 +535,11 @@ test_get_chunk_info_highest_v18(hid_t fapl)
z_dst = (Bytef *)inbuf;
/* Perform compression from the source to the destination buffer */
#if defined(H5_HAVE_ZLIBNG_H)
ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
#else
ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
#endif
/* Set the chunk size to the compressed chunk size */
chunk_size = (hsize_t)z_dst_nbytes;

22
test/direct_chunk.c
View File

@ -12,12 +12,14 @@
#include "h5test.h"
#ifdef H5_HAVE_FILTER_DEFLATE
#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER)
#define H5_ZLIB_HEADER "zlib.h"
#endif
#if defined(H5_ZLIB_HEADER)
#include H5_ZLIB_HEADER /* "zlib.h" */
#endif
#endif
#define FILE_NAME "direct_chunk.h5"
@ -205,7 +207,11 @@ test_direct_chunk_write(hid_t file)
z_dst = (Bytef *)outbuf;
/* Perform compression from the source to the destination buffer */
#if defined(H5_HAVE_ZLIBNG_H)
ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
#else
ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
#endif
/* Check for various zlib errors */
if (Z_BUF_ERROR == ret) {
@ -284,7 +290,11 @@ test_direct_chunk_write(hid_t file)
z_dst = (Bytef *)outbuf;
/* Perform compression from the source to the destination buffer */
#if defined(H5_HAVE_ZLIBNG_H)
ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
#else
ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
#endif
/* Check for various zlib errors */
if (Z_BUF_ERROR == ret) {
@ -1522,8 +1532,12 @@ test_direct_chunk_read_no_cache(hid_t file)
if (filter_mask != 0)
goto error;
/* Perform decompression from the source to the destination buffer */
/* Perform decompression from the source to the destination buffer */
#if defined(H5_HAVE_ZLIBNG_H)
ret = zng_uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
#else
ret = uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
#endif
/* Check for various zlib errors */
if (Z_BUF_ERROR == ret) {
@ -1713,8 +1727,12 @@ test_direct_chunk_read_cache(hid_t file, bool flush)
if (filter_mask != 0)
goto error;
/* Perform decompression from the source to the destination buffer */
/* Perform decompression from the source to the destination buffer */
#if defined(H5_HAVE_ZLIBNG_H)
ret = zng_uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
#else
ret = uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
#endif
/* Check for various zlib errors */
if (Z_BUF_ERROR == ret) {

6
tools/test/perform/CMakeLists.txt
View File

@ -112,6 +112,12 @@ endif ()
set (zip_perf_SOURCES
${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/zip_perf.c
)
if (H5_ZLIB_HEADER)
message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
if (HDF5_USE_ZLIB_NG)
add_compile_definitions(H5_HAVE_ZLIBNG_H=1 H5_ZLIB_HEADER="${H5_ZLIB_HEADER}")
endif ()
endif ()
add_executable (zip_perf ${zip_perf_SOURCES})
target_include_directories (zip_perf PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
if (NOT BUILD_SHARED_LIBS)

11
tools/test/perform/direct_write_perf.c
View File

@ -18,7 +18,12 @@
#include "hdf5.h"
#ifdef H5_HAVE_FILTER_DEFLATE
#include <zlib.h>
#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER)
#define H5_ZLIB_HEADER "zlib.h"
#endif
#if defined(H5_ZLIB_HEADER)
#include H5_ZLIB_HEADER /* "zlib.h" */
#endif
#if !defined(WIN32) && !defined(__MINGW32__)
@ -242,7 +247,11 @@ create_file(hid_t fapl_id)
z_dst = (Bytef *)outbuf[i];
/* Perform compression from the source to the destination buffer */
#if defined(H5_HAVE_ZLIBNG_H)
ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
#else
ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
#endif
data_size[i] = (size_t)z_dst_nbytes;
total_size += data_size[i];

12
tools/test/perform/zip_perf.c
View File

@ -24,8 +24,12 @@
#include "h5tools_utils.h"
#ifdef H5_HAVE_FILTER_DEFLATE
#include <zlib.h>
#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER)
#define H5_ZLIB_HEADER "zlib.h"
#endif
#if defined(H5_ZLIB_HEADER)
#include H5_ZLIB_HEADER /* "zlib.h" */
#endif
#define ONE_KB 1024
#define ONE_MB (ONE_KB * ONE_KB)
@ -163,7 +167,11 @@ write_file(Bytef *source, uLongf sourceLen)
static void
compress_buffer(Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)
{
#if defined(H5_HAVE_ZLIBNG_H)
int rc = zng_compress2(dest, destLen, source, sourceLen, compress_level);
#else
int rc = compress2(dest, destLen, source, sourceLen, compress_level);
#endif
if (rc != Z_OK) {
/* compress2 failed - cleanup and tell why */