mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-03-31 17:50:26 +08:00
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294 Ed Hartnett suggested that the netcdf library installation process be extended to install the standard filters into a user specified location. The user can then set HDF5_PLUGIN_PATH to that location. This PR provides that capability using: ```` configure option: --with-plugin-dir=<absolute directory path> cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path> ```` Currently, the following plugins are always installed, if available: bzip2, zstd, blosc. If NCZarr is enabled, then additional plugins are installed: fletcher32, shuffle, deflate, szip. Additionally, the necessary codec support is installed for each of the above filters that is installed. ## Changes: 1. Cleanup handling of built-in bzip2. 2. Add documentation to docs/filters.md 3. Re-factor the NCZarr codec libraries 4. Add a test, although it can only be exercised after the library is installed, so it cannot be used during normal testing. 5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
This commit is contained in:
Dennis Heimbigner
parent
2856ee751d
commit
126b3f9423
2
.github/workflows/run_tests_osx.yml
vendored
2
.github/workflows/run_tests_osx.yml
vendored
@ -7,7 +7,7 @@
|
||||
name: Run macOS-based netCDF Tests
|
||||
|
||||
|
||||
on: [pull_request]
|
||||
on: [pull_request,push]
|
||||
|
||||
jobs:
|
||||
|
||||
|
||||
2
.github/workflows/run_tests_ubuntu.yml
vendored
2
.github/workflows/run_tests_ubuntu.yml
vendored
@ -4,7 +4,7 @@
|
||||
|
||||
name: Run Ubuntu/Linux netCDF Tests
|
||||
|
||||
on: [ pull_request ]
|
||||
on: [pull_request,push]
|
||||
|
||||
jobs:
|
||||
|
||||
|
||||
2
.github/workflows/run_tests_win_mingw.yml
vendored
2
.github/workflows/run_tests_win_mingw.yml
vendored
@ -7,7 +7,7 @@
|
||||
name: Run MSYS2, MinGW64-based Tests
|
||||
|
||||
|
||||
on: [pull_request]
|
||||
on: [pull_request,push]
|
||||
|
||||
jobs:
|
||||
|
||||
|
||||
@ -1114,9 +1114,12 @@ string(TOLOWER "${filter}" downfilter)
|
||||
IF(${filter}_FOUND)
|
||||
INCLUDE_DIRECTORIES(${filter}_INCLUDE_DIRS})
|
||||
SET(ENABLE_${upfilter} TRUE)
|
||||
SET(STD_FILTERS "${STD_FILTERS},${downfilter}")
|
||||
SET(HAVE_${upfilter} ON)
|
||||
SET(STD_FILTERS "${STD_FILTERS} ${downfilter}")
|
||||
MESSAGE(">>> Standard Filter: ${downfilter}")
|
||||
ELSE()
|
||||
SET(ENABLE_${upfilter} FALSE)
|
||||
SET(HAVE_${upfilter} OFF)
|
||||
ENDIF()
|
||||
endmacro(set_std_filter)
|
||||
|
||||
@ -1125,20 +1128,36 @@ FIND_PACKAGE(Bz2)
|
||||
FIND_PACKAGE(Blosc)
|
||||
FIND_PACKAGE(Zstd)
|
||||
|
||||
IF(Zstd_FOUND)
|
||||
SET(HAVE_ZSTD yes)
|
||||
ELSE()
|
||||
SET(HAVE_ZSTD no)
|
||||
ENDIF()
|
||||
|
||||
# Accumulate standard filters
|
||||
set(STD_FILTERS "deflate") # Always have deflate */
|
||||
set(STD_FILTERS "deflate") # Always have deflate*/
|
||||
set_std_filter(SZIP)
|
||||
set_std_filter(Blosc)
|
||||
set_std_filter(Zstd)
|
||||
set_std_filter(Bz2)
|
||||
IF(NOT Bz2_FOUND)
|
||||
set(STD_FILTERS "${STD_FILTERS},bzip2") # Always have bzip2 */
|
||||
IF(Bz2_FOUND)
|
||||
set_std_filter(Bz2)
|
||||
ELSE()
|
||||
# The reason we use a local version is to support a more comples test case
|
||||
MESSAGE(WARNING "libbz2 not found using built-in version")
|
||||
SET(HAVE_LOCAL_BZ2 ON)
|
||||
SET(HAVE_BZ2 ON)
|
||||
set(STD_FILTERS "${STD_FILTERS} bz2")
|
||||
ENDIF()
|
||||
|
||||
# If user wants, then install selected plugins
|
||||
SET(PLUGIN_INSTALL_DIR "" CACHE STRING "Whether and where we should install plugins")
|
||||
SET(ENABLE_PLUGIN_INSTALL OFF)
|
||||
if(DEFINED PLUGIN_INSTALL_DIR OR DEFINED CACHE{PLUGIN_INSTALL_DIR})
|
||||
IF(PLUGIN_INSTALL_DIR STREQUAL "")
|
||||
MESSAGE(WARNING "No plugin directory value specified; option ignored.")
|
||||
UNSET(PLUGIN_INSTALL_DIR)
|
||||
UNSET(PLUGIN_INSTALL_DIR CACHE)
|
||||
SET(PLUGIN_INSTALL_DIR_SETTING "N.A.")
|
||||
ELSE()
|
||||
SET(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}")
|
||||
SET(ENABLE_PLUGIN_INSTALL ON)
|
||||
ENDIF()
|
||||
ELSE()
|
||||
SET(PLUGIN_INSTALL_DIR_SETTING "N.A.")
|
||||
ENDIF()
|
||||
|
||||
# See if we have libzip
|
||||
@ -2528,6 +2547,10 @@ IF(ENABLE_NCZARR)
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nczarr_test/findplugin.sh @ONLY NEWLINE_STYLE LF)
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_PLUGINS)
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/plugins/findplugin.sh @ONLY NEWLINE_STYLE LF)
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_EXAMPLES)
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF)
|
||||
ENDIF()
|
||||
|
||||
64
cmake/modules/FindBzip2.cmake
Normal file
64
cmake/modules/FindBzip2.cmake
Normal file
@ -0,0 +1,64 @@
|
||||
# Searches for an installation of the bzip2 library. On success, it sets the following variables:
|
||||
#
|
||||
# Bzip2_FOUND Set to true to indicate the bzip2 library was found
|
||||
# Bzip2_INCLUDE_DIRS The directory containing the header file bzip2/bzip2.h
|
||||
# Bzip2_LIBRARIES The libraries needed to use the bzip2 library
|
||||
#
|
||||
# To specify an additional directory to search, set Bzip2_ROOT.
|
||||
#
|
||||
# Author: Siddhartha Chaudhuri, 2009
|
||||
#
|
||||
|
||||
# Look for the header, first in the user-specified location and then in the system locations
|
||||
SET(Bzip2_INCLUDE_DOC "The directory containing the header file bzip2.h")
|
||||
FIND_PATH(Bzip2_INCLUDE_DIRS NAMES bzip2.h bzip2/bzip2.h PATHS ${Bzip2_ROOT} ${Bzip2_ROOT}/include DOC ${Bzip2_INCLUDE_DOC} NO_DEFAULT_PATH)
|
||||
IF(NOT Bzip2_INCLUDE_DIRS) # now look in system locations
|
||||
FIND_PATH(Bzip2_INCLUDE_DIRS NAMES bzlib.h DOC ${Bzip2_INCLUDE_DOC})
|
||||
ENDIF(NOT Bzip2_INCLUDE_DIRS)
|
||||
|
||||
SET(Bzip2_FOUND FALSE)
|
||||
|
||||
IF(Bzip2_INCLUDE_DIRS)
|
||||
SET(Bzip2_LIBRARY_DIRS ${Bzip2_INCLUDE_DIRS})
|
||||
|
||||
IF("${Bzip2_LIBRARY_DIRS}" MATCHES "/include$")
|
||||
# Strip off the trailing "/include" in the path.
|
||||
GET_FILENAME_COMPONENT(Bzip2_LIBRARY_DIRS ${Bzip2_LIBRARY_DIRS} PATH)
|
||||
ENDIF("${Bzip2_LIBRARY_DIRS}" MATCHES "/include$")
|
||||
|
||||
IF(EXISTS "${Bzip2_LIBRARY_DIRS}/lib")
|
||||
SET(Bzip2_LIBRARY_DIRS ${Bzip2_LIBRARY_DIRS}/lib)
|
||||
ENDIF(EXISTS "${Bzip2_LIBRARY_DIRS}/lib")
|
||||
|
||||
# Find Bzip2 libraries
|
||||
FIND_LIBRARY(Bzip2_DEBUG_LIBRARY NAMES bzip2d bzip2_d libbzip2d libbzip2_d libbzip2
|
||||
PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug
|
||||
PATHS ${Bzip2_LIBRARY_DIRS} NO_DEFAULT_PATH)
|
||||
FIND_LIBRARY(Bzip2_RELEASE_LIBRARY NAMES bzip2 libbzip2
|
||||
PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release
|
||||
PATHS ${Bzip2_LIBRARY_DIRS} NO_DEFAULT_PATH)
|
||||
|
||||
SET(Bzip2_LIBRARIES )
|
||||
IF(Bzip2_DEBUG_LIBRARY AND Bzip2_RELEASE_LIBRARY)
|
||||
SET(Bzip2_LIBRARIES debug ${Bzip2_DEBUG_LIBRARY} optimized ${Bzip2_RELEASE_LIBRARY})
|
||||
ELSEIF(Bzip2_DEBUG_LIBRARY)
|
||||
SET(Bzip2_LIBRARIES ${Bzip2_DEBUG_LIBRARY})
|
||||
ELSEIF(Bzip2_RELEASE_LIBRARY)
|
||||
SET(Bzip2_LIBRARIES ${Bzip2_RELEASE_LIBRARY})
|
||||
ENDIF(Bzip2_DEBUG_LIBRARY AND Bzip2_RELEASE_LIBRARY)
|
||||
|
||||
IF(Bzip2_LIBRARIES)
|
||||
SET(Bzip2_FOUND TRUE)
|
||||
ENDIF(Bzip2_LIBRARIES)
|
||||
ENDIF(Bzip2_INCLUDE_DIRS)
|
||||
|
||||
IF(Bzip2_FOUND)
|
||||
# IF(NOT Bzip2_FIND_QUIETLY)
|
||||
MESSAGE(STATUS "Found Bzip2: headers at ${Bzip2_INCLUDE_DIRS}, libraries at ${Bzip2_LIBRARY_DIRS}")
|
||||
MESSAGE(STATUS " library is ${Bzip2_LIBRARIES}")
|
||||
# ENDIF(NOT Bzip2_FIND_QUIETLY)
|
||||
ELSE(Bzip2_FOUND)
|
||||
IF(Bzip2_FIND_REQUIRED)
|
||||
MESSAGE(FATAL_ERROR "Bzip2 library not found")
|
||||
ENDIF(Bzip2_FIND_REQUIRED)
|
||||
ENDIF(Bzip2_FOUND)
|
||||
@ -173,7 +173,7 @@ are set when opening a binary file on Windows. */
|
||||
#cmakedefine HAVE_ATEXIT 1
|
||||
|
||||
/* Define to 1 if bzip2 library available. */
|
||||
#cmakedefine HAVE_BZIP2 1
|
||||
#cmakedefine HAVE_BZ2 1
|
||||
|
||||
/* Define to 1 if zstd library available. */
|
||||
#cmakedefine HAVE_ZSTD 1
|
||||
|
||||
57
configure.ac
57
configure.ac
@ -667,24 +667,21 @@ fi
|
||||
AC_MSG_CHECKING([whether libzstd library is available])
|
||||
AC_MSG_RESULT([${have_zstd}])
|
||||
|
||||
# See if we have libbzip2 or libbz2
|
||||
AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no])
|
||||
if test "x$have_bzip2" = "xyes" ; then
|
||||
# See if we have libbz2
|
||||
AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bz2=yes],[have_bz2=no])
|
||||
if test "x$have_bz2" = "xyes" ; then
|
||||
AC_SEARCH_LIBS([BZ2_bzCompress],[bz2 bz2.dll cygbz2.dll], [], [])
|
||||
AC_DEFINE([HAVE_BZIP2], [1], [if true, bz2 library is installed])
|
||||
AC_DEFINE([HAVE_BZ2], [1], [if true, bz2 library is installed])
|
||||
fi
|
||||
AC_MSG_CHECKING([whether libbz2 library is available])
|
||||
AC_MSG_RESULT([${have_bzip2}])
|
||||
AC_MSG_RESULT([${have_bz2}])
|
||||
|
||||
if test "x$have_bzip2" = "xno" ; then
|
||||
AC_CHECK_LIB([bzip2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no])
|
||||
if test "x$have_bzip2" = "xyes" ; then
|
||||
AC_SEARCH_LIBS([BZ2_bzCompress],[bzip2 bzip2.dll cygbzip2.dll], [], [])
|
||||
AC_DEFINE([HAVE_BZIP2], [1], [if true, bzip2 library is installed])
|
||||
fi
|
||||
AC_MSG_CHECKING([whether libbzip2 library is available])
|
||||
AC_MSG_RESULT([${have_bzip2}])
|
||||
if test "x$have_bz2" = "xno" ; then
|
||||
have_local_bz2=yes
|
||||
else
|
||||
have_local_bz2=no
|
||||
fi
|
||||
AM_CONDITIONAL(HAVE_LOCAL_BZ2, [test "x$have_local_bz2" = xyes])
|
||||
|
||||
# Note that szip management is tricky.
|
||||
# This is because we have three things to consider:
|
||||
@ -1753,7 +1750,7 @@ AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
|
||||
AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes])
|
||||
AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes])
|
||||
AM_CONDITIONAL(HAVE_BLOSC, [test "x$have_blosc" = xyes])
|
||||
AM_CONDITIONAL(HAVE_BZIP2, [test "x$have_bzip2" = xyes])
|
||||
AM_CONDITIONAL(HAVE_BZ2, [test "x$have_bz2" = xyes])
|
||||
AM_CONDITIONAL(HAVE_ZSTD, [test "x$have_zstd" = xyes])
|
||||
|
||||
# If the machine doesn't have a long long, and we want netCDF-4, then
|
||||
@ -1864,26 +1861,45 @@ AC_SUBST(HAS_SZLIB_WRITE, [$have_sz])
|
||||
AC_SUBST(HAS_ZSTD,[$have_zstd])
|
||||
|
||||
# Always available
|
||||
std_filters="deflate,bzip2"
|
||||
std_filters="deflate bz2"
|
||||
if test "x$enable_szlib" = xyes ; then
|
||||
std_filters="${std_filters},szip"
|
||||
std_filters="${std_filters} szip"
|
||||
fi
|
||||
# We need to include szip iff HDF5 && HDF5_HAS_SZIP || !HDF5 && NCZARR && libsz
|
||||
if test "x$enable_hdf5" = xyes && test "x$enable_hdf5_szip" = xyes ; then
|
||||
std_filters="${std_filters},szip"
|
||||
std_filters="${std_filters} szip"
|
||||
fi
|
||||
if test "x$enable_hdf5" = xno && test "x$have_sz" = xyes ; then
|
||||
std_filters="${std_filters},szip"
|
||||
std_filters="${std_filters} szip"
|
||||
fi
|
||||
if test "x$have_blosc" = xyes ; then
|
||||
std_filters="${std_filters},blosc"
|
||||
std_filters="${std_filters} blosc"
|
||||
fi
|
||||
if test "x$have_zstd" = xyes ; then
|
||||
std_filters="${std_filters},zst"
|
||||
std_filters="${std_filters} zstd"
|
||||
fi
|
||||
|
||||
AC_SUBST(STD_FILTERS,[$std_filters])
|
||||
|
||||
# If user wants, then install selected standard filters
|
||||
AC_MSG_CHECKING([whether and where we should install plugins])
|
||||
AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=<absolute directory>],
|
||||
[Install selected standard filters in specified directory])])
|
||||
AC_MSG_RESULT([$with_plugin_dir])
|
||||
if test "x$with_plugin_dir" = x ; then
|
||||
AC_MSG_WARN([No plugin directory value specified; option ignored.])
|
||||
with_plugin_dir=
|
||||
with_plugin_dir_setting="N.A."
|
||||
enable_plugin_dir=no
|
||||
else
|
||||
with_plugin_dir_setting="$with_plugin_dir"
|
||||
enable_plugin_dir=yes
|
||||
fi
|
||||
AM_CONDITIONAL([ENABLE_PLUGIN_DIR], [test "x$enable_plugin_dir" = xyes])
|
||||
AC_SUBST([PLUGIN_INSTALL_DIR], [$with_plugin_dir])
|
||||
# Better value for libnetcdf.settings
|
||||
AC_SUBST([PLUGIN_INSTALL_DIR_SETTING], [$with_plugin_dir_setting])
|
||||
|
||||
# Access netcdf specific version of config.h
|
||||
AH_BOTTOM([#include "ncconfigure.h"])
|
||||
|
||||
@ -1973,6 +1989,7 @@ AC_MSG_NOTICE([generating header files and makefiles])
|
||||
AC_CONFIG_FILES(test_common.sh:test_common.in)
|
||||
AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in)
|
||||
AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in)
|
||||
AC_CONFIG_FILES(plugins/findplugin.sh:nc_test4/findplugin.in)
|
||||
AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in)
|
||||
AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in)
|
||||
AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)
|
||||
|
||||
@ -932,28 +932,34 @@ This is for internal use only.
|
||||
|
||||
# Appendix F. Pre-built Filters
|
||||
|
||||
As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory.
|
||||
They may be in that directory or the "plugins/.libs" subdirectory.
|
||||
It may be possible for users to utilize some of those libraries to provide filter support for general use.
|
||||
One simple way to reuse these libraries is to make the environment variable "HDF5_PLUGIN_PATH" refer to the plugin directory or the subdirectory, whichever contains the shared libraries.
|
||||
This may not be possible if the user already has other filter shared libraries in use.
|
||||
As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory
|
||||
— in that directory or the "plugins/.libs" subdirectory.
|
||||
|
||||
The second way is to copy the necessary shared libraries from the plugins directory to the user's HDF5_PLUGIN_PATH directory.
|
||||
If the user is using HDF5, then the following filters are probably usable:
|
||||
An option exists to allow some of those filters to be installed into a user-specified directory.
|
||||
Presumably this directory is part of the value of the HDF5_PLUGIN_PATH environment variable.
|
||||
The ./configure option is ````--with-plugin-dir=<absolute-directory-path>````.
|
||||
The corresponding CMake option is ````-DPLUGIN_INSTALL_DIR=<absolute-directory-path>````.
|
||||
|
||||
* ''libh5bzip2.so'' -- an HDF5 filter for bzip2 compression
|
||||
* ''libh5blosc.so'' -- an HDF5 filter for blosc compression
|
||||
If this option is specified, then as part of the "install" build action,
|
||||
a specified set of filter shared libraries will be copied into the specified directory.
|
||||
Any existing library of the same name will be overwritten. If the specified directory
|
||||
itself does not exist, then it will be created.
|
||||
|
||||
If the user is using NCZarr filters, then you can install the following additional shared libraries:
|
||||
Currently, if the following filters are available, they will be installed;
|
||||
* ''libh5bzip2.so'' -- an HDF5 filter wrapper for bzip2 compression
|
||||
* ''libh5blosc.so'' -- an HDF5 filter wrapper for blosc compression
|
||||
* ''libh5zstd.so'' -- an HDF5 filter wrapper for zstandardcompression
|
||||
|
||||
If the user is using NCZarr filters, then the following additional filters will be installed.
|
||||
* libh5shuffle.so -- shuffle filter
|
||||
* libh5fletcher32.so -- fletcher32 checksum
|
||||
* libh5deflate.so -- deflate compression
|
||||
* libh5szip.so -- szip compression
|
||||
* libnczdefaults.so -- provide NCZarr support for shuffle, fletcher32, and deflate.
|
||||
* libnczszip.so -- provide NCZarr support for szip.
|
||||
|
||||
The shuffle, fletcher32, and deflate filters in this case will be ignored by HDF5 and only used by the NCZarr code.
|
||||
But in order to use them, it needs additional Codec capabilities provided by the libnczdefauts.so shared library.
|
||||
Note also that if you disable HDF5 support, but leave NCZarr support enabled, then all of the above filters
|
||||
The shuffle, fletcher32, deflate, and szip filters in this case will be ignored by HDF5 and only used by the NCZarr code.
|
||||
Note that if you disable HDF5 support, but leave NCZarr support enabled, then all of the above filters
|
||||
should continue to work.
|
||||
|
||||
# Point of Contact {#filters_poc}
|
||||
|
||||
@ -7,7 +7,9 @@
|
||||
|
||||
# Un comment to use a more verbose test driver
|
||||
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
#TESTS_ENVIRONMENT = export SETX=1;
|
||||
|
||||
# Put together AM_CPPFLAGS and AM_LDFLAGS.
|
||||
include $(top_srcdir)/lib_flags.am
|
||||
|
||||
@ -10,26 +10,25 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../../test_common.sh
|
||||
|
||||
echo "*** Running filter example for netCDF-4."
|
||||
|
||||
set -e
|
||||
|
||||
if test -f ${builddir}/findplugin.sh ; then
|
||||
echo "*** running test_filter example..."
|
||||
. ${builddir}/findplugin.sh
|
||||
|
||||
# Locate the plugin path and the library names; argument order is critical
|
||||
# Find bzip2 and capture
|
||||
findplugin h5bzip2
|
||||
BZIP2PATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
|
||||
BZIP2PATH="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
|
||||
|
||||
# Verify
|
||||
if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
|
||||
export HDF5_PLUGIN_PATH
|
||||
|
||||
echo "*** running filter_example..."
|
||||
rm -f ./bzip2.nc
|
||||
export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
|
||||
echo "*** running filter_example..."
|
||||
${execdir}/filter_example
|
||||
#rm -f ./bzip2.nc
|
||||
|
||||
fi # Filter enabled
|
||||
rm -f ./bzip2.nc
|
||||
|
||||
echo "*** Filter example successful!"
|
||||
exit 0
|
||||
|
||||
@ -34,17 +34,46 @@
|
||||
/* Defined flags for filter invocation (not stored); powers of two */
|
||||
#define NCZ_FILTER_DECODE 0x00000001
|
||||
|
||||
/* External Discovery Function */
|
||||
/* External Discovery Functions */
|
||||
|
||||
/*
|
||||
Obtain a pointer to an instance of NCZ_codec_class_t.
|
||||
|
||||
NCZ_get_codec_info(void) -- returns pointer to instance of NCZ_codec_class_t.
|
||||
Instance an be recast based on version+sort to the plugin type specific info.
|
||||
So the void* return value is typically actually of type NCZ_codec_class_t*.
|
||||
|
||||
Signature: typedef const void* (*NCZ_get_codec_info_proto)(void);
|
||||
|
||||
The current object returned by NCZ_get_codec_info is a
|
||||
pointer to an instance of NCZ_codec_t.
|
||||
|
||||
The key to this struct are the several function pointers that do
|
||||
initialize/finalize and conversion between codec JSON and HDF5
|
||||
parameters. The function pointers defined in NCZ_codec_t
|
||||
manipulate HDF5 parameters and NumCodec JSON.
|
||||
|
||||
Obtain a pointer to an instance of NCZ_codec_class_t.
|
||||
|
||||
NCZ_get_codec_info(void) -- returns pointer to instance of NCZ_codec_class_t.
|
||||
Instance an be recast based on version+sort to the plugin type specific info.
|
||||
So the void* return value is typically actually of type NCZ_codec_class_t*.
|
||||
*/
|
||||
typedef const void* (*NCZ_get_codec_info_proto)(void);
|
||||
|
||||
/*
|
||||
Obtain a pointer to a NULL terminated vector of NCZ_codec_class_t*.
|
||||
|
||||
NCZ_codec_info_defaults(void) -- returns pointer to a vector of pointers to instances of NCZ_codec_class_t. The vector is NULL terminated.
|
||||
So the void* return value is typically actually of type NCZ_codec_class_t**.
|
||||
|
||||
Signature: typedef const void* (*NCZ_codec_info_defaults_proto)(void);
|
||||
|
||||
This entry point is used to return the codec information for
|
||||
multiple filters that otherwise do not have codec information defined.
|
||||
*/
|
||||
typedef const void* (*NCZ_codec_info_defaults_proto)(void);
|
||||
|
||||
/* The current object returned by NCZ_get_plugin_info is a
|
||||
pointer to an instance of NCZ_codec_t.
|
||||
|
||||
@ -86,7 +115,7 @@ int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp);
|
||||
* Convert a set of visible parameters to a set of working parameters using extra environmental information.
|
||||
Also allows for changes to the visible parameters. Invoked before filter is actually used.
|
||||
|
||||
int (*NCZ_build_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
|
||||
int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
|
||||
|
||||
@param ncid -- (in) ncid of the variable's group
|
||||
@param varid -- (in) varid of the variable
|
||||
@ -96,8 +125,15 @@ int (*NCZ_build_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** v
|
||||
@params wparamsp -- (out) vector of working parameters
|
||||
@return -- a netcdf-c error code.
|
||||
|
||||
* Convert a set of working parameters to a set of visible parameters using extra environmental information, if needed.
|
||||
Invoked before filter metadata is written.
|
||||
* Convert an HDF5 vector of visible parameters to a JSON representation.
|
||||
|
||||
int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp);
|
||||
|
||||
@param nparams -- (in) the length of the HDF5 unsigned vector
|
||||
@param params -- (in) pointer to the HDF5 unsigned vector.
|
||||
@param codecp -- (out) store the string representation of the codec; caller must free.
|
||||
@return -- a netcdf-c error code.
|
||||
|
||||
*/
|
||||
|
||||
/*
|
||||
|
||||
@ -211,11 +211,11 @@ NC_mktmp(const char* base)
|
||||
int fd = -1;
|
||||
char* tmp = NULL;
|
||||
size_t len;
|
||||
#define MAXTRIES 4
|
||||
#ifdef HAVE_MKSTEMP
|
||||
mode_t mask;
|
||||
#else
|
||||
#ifndef HAVE_MKSTEMP
|
||||
int tries;
|
||||
#define MAXTRIES 4
|
||||
#else
|
||||
mode_t mask;
|
||||
#endif
|
||||
|
||||
len = strlen(base)+6+1;
|
||||
|
||||
@ -52,9 +52,11 @@
|
||||
#include "netcdf_filter_build.h"
|
||||
#include "netcdf_aux.h"
|
||||
|
||||
#undef DEBUG
|
||||
#undef DEBUGF
|
||||
#undef DEBUGL
|
||||
#if 0
|
||||
#define DEBUG
|
||||
#define DEBUGF
|
||||
#define DEBUGL
|
||||
#endif
|
||||
|
||||
/* If set, then triage potential shared libraries based on extension */
|
||||
#define NAMEOPT
|
||||
@ -177,7 +179,7 @@ NCJtrace(const NCjson* j)
|
||||
const char*
|
||||
printplugin(const NCZ_Plugin* plugin)
|
||||
{
|
||||
char* plbuf = malloc(4096);
|
||||
static char plbuf[4096];
|
||||
char plbuf2[2000];
|
||||
char plbuf1[2000];
|
||||
|
||||
@ -194,8 +196,7 @@ printplugin(const NCZ_Plugin* plugin)
|
||||
static char*
|
||||
printparams(size_t nparams, const unsigned* params)
|
||||
{
|
||||
char* ppbuf = malloc(4096);
|
||||
|
||||
static char ppbuf[4096];
|
||||
if(nparams == 0)
|
||||
snprintf(ppbuf,4096,"{0,%p}",params);
|
||||
else
|
||||
@ -1378,20 +1379,27 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
|
||||
|
||||
/* See what we have */
|
||||
{
|
||||
H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type");
|
||||
H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info");
|
||||
NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info");
|
||||
NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults");
|
||||
const H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type");
|
||||
const H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info");
|
||||
const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info");
|
||||
const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults");
|
||||
|
||||
if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL)
|
||||
{stat = NC_ENOFILTER; goto done;}
|
||||
|
||||
if(cpd != NULL) {
|
||||
/* Deal with defaults first */
|
||||
NCZ_codec_t** cp = NULL;
|
||||
const NCZ_codec_t** cp = NULL;
|
||||
nclistpush(default_libs,lib);
|
||||
for(cp=cpd();*cp;cp++) {
|
||||
cp = (const NCZ_codec_t**)cpd();
|
||||
#ifdef DEBUGL
|
||||
fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp);
|
||||
#endif
|
||||
for(;*cp;cp++) {
|
||||
struct CodecAPI* c0;
|
||||
#ifdef DEBUGL
|
||||
fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id);
|
||||
#endif
|
||||
c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI));
|
||||
if(c0 == NULL) {stat = NC_ENOMEM; goto done1;}
|
||||
c0->codec = *cp;
|
||||
|
||||
@ -25,16 +25,6 @@
|
||||
#define plugin_dir_win "%s/hdf5/lib/plugin"
|
||||
#define win32_root_env "ALLUSERSPROFILE"
|
||||
|
||||
/*
|
||||
Return a NULL terminated vector of pointers to instances of ''NCZ_codec_t''.
|
||||
The value returned is actually of type ''NCZ_codec_t**'',
|
||||
but is of type ''void*'' to allow for extensions.
|
||||
The list of returned items are used to try to provide defaults
|
||||
for any HDF5 filters that have no corresponding Codec.
|
||||
This is for internal use only.
|
||||
*/
|
||||
typedef void* (*NCZ_codec_info_defaults_proto)(void);
|
||||
|
||||
/* Opaque */
|
||||
struct NCZ_Filter;
|
||||
|
||||
|
||||
@ -9,6 +9,7 @@ Configured On: @CONFIG_DATE@
|
||||
Host System: @host_cpu@-@host_vendor@-@host_os@
|
||||
Build Directory: @abs_top_builddir@
|
||||
Install Prefix: @prefix@
|
||||
Plugin Install Prefix: @PLUGIN_INSTALL_DIR_SETTING@
|
||||
|
||||
# Compiling Options
|
||||
-----------------
|
||||
@ -49,4 +50,4 @@ Logging: @HAS_LOGGING@
|
||||
SZIP Write Support: @HAS_SZLIB_WRITE@
|
||||
Standard Filters: @STD_FILTERS@
|
||||
ZSTD Support: @HAS_ZSTD@
|
||||
Benchmarks: @HAS_BENCHMARKS@
|
||||
Benchmarks: @HAS_BENCHMARKS@
|
||||
|
||||
@ -109,9 +109,15 @@ ref_filteredvv.cdl ref_multi.cdl \
|
||||
ref_ncgenF.cdl ref_nccopyF.cdl \
|
||||
ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
|
||||
ref_filter_order_create.txt ref_filter_order_read.txt \
|
||||
ref_any.cdl tst_specific_filters.sh \
|
||||
ref_any.cdl tst_specific_filters.sh \
|
||||
tst_virtual_datasets.c
|
||||
|
||||
# The tst_filterinstall test can only be run after an install
|
||||
# occurred with --with-plugin-dir enabled. So there is no point
|
||||
#in running it via make check. It is kept here so it can be
|
||||
# manually invoked if desired
|
||||
EXTRA_DIST += tst_filterinstall.sh
|
||||
|
||||
CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
|
||||
tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt \
|
||||
tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl tst_elena_*.cdl \
|
||||
|
||||
@ -9,12 +9,10 @@
|
||||
# $2 is 1 if we are running under cmake
|
||||
# $3 is 1 if we are running using Visual Studio, blank otherwise
|
||||
# $4 is the build type; only used if $3 is 1
|
||||
# Optional Input:
|
||||
# HDF5_PLUGIN_PATH environment variable
|
||||
# Outputs:
|
||||
# return code is 0 is success, 1 if failed
|
||||
# Variable HDF5_PLUGIN_LIB is set to the library file name
|
||||
# Variable HDF5_PLUGIN_PATH is setthe absolute path to the
|
||||
# Variable HDF5_PLUGIN_DIR is set to the absolute path to the
|
||||
# directory containing the plugin library file
|
||||
# Local variables are prefixed with FP_
|
||||
#
|
||||
@ -27,7 +25,7 @@ findplugin() {
|
||||
FP_NAME="$1"
|
||||
|
||||
FP_PLUGIN_LIB=
|
||||
FP_PLUGIN_PATH=
|
||||
FP_PLUGIN_DIR=
|
||||
|
||||
# Figure out the plugin file name
|
||||
# Test for visual studio before cygwin since both might be true
|
||||
@ -43,14 +41,6 @@ else # Presumably some form on *nix"
|
||||
FP_PLUGIN_LIB="lib${FP_NAME}.so"
|
||||
fi
|
||||
|
||||
# If HDF5_PLUGIN_PATH is defined, then it overrides everything else.
|
||||
if test "x$HDF5_PLUGIN_PATH" != x ; then
|
||||
HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB"
|
||||
# HDF5_PLUGIN_PATH already set
|
||||
HDF5_PLUGIN_PATH=`${NCPATHCVT} $HDF5_PLUGIN_PATH`
|
||||
return 0;
|
||||
fi
|
||||
|
||||
# Figure out the path to where the lib is stored
|
||||
# This can probably be simplified
|
||||
|
||||
@ -59,40 +49,38 @@ cd ${TOPBUILDDIR}/plugins
|
||||
FP_PLUGINS=`pwd`
|
||||
cd ${CURWD}
|
||||
|
||||
# Case 1: Cmake with Visual Studio
|
||||
# Do not know where to look for a dylib
|
||||
# Case 1: Cmake with Visual Studio
|
||||
if test "x$FP_ISCMAKE" != x -a "x${FP_ISMSVC}" != x ; then
|
||||
# Case 1a: ignore the build type directory
|
||||
if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
|
||||
FP_PLUGIN_PATH="${FP_PLUGINS}"
|
||||
FP_PLUGIN_DIR="${FP_PLUGINS}"
|
||||
fi
|
||||
else # Case 2: automake
|
||||
# Case 2a: look in .libs
|
||||
if test -f "${FP_PLUGINS}/.libs/${FP_PLUGIN_LIB}" ; then
|
||||
FP_PLUGIN_PATH="${FP_PLUGINS}/.libs"
|
||||
FP_PLUGIN_DIR="${FP_PLUGINS}/.libs"
|
||||
else # Case 2: look in FP_PLUGINS directly
|
||||
if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
|
||||
FP_PLUGIN_PATH="${FP_PLUGINS}"
|
||||
FP_PLUGIN_DIR="${FP_PLUGINS}"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify
|
||||
if test "x$FP_PLUGIN_PATH" = x ; then
|
||||
echo "***Fail: Could not locate a usable HDF5_PLUGIN_PATH"
|
||||
if test "x$FP_PLUGIN_DIR" = x ; then
|
||||
echo "***Fail: Could not locate a usable HDF5_PLUGIN_DIR"
|
||||
return 1
|
||||
fi
|
||||
if ! test -f "$FP_PLUGIN_PATH/$FP_PLUGIN_LIB" ; then
|
||||
if ! test -f "$FP_PLUGIN_DIR/$FP_PLUGIN_LIB" ; then
|
||||
echo "***Fail: Could not locate a usable HDF5_PLUGIN_LIB"
|
||||
return 1
|
||||
fi
|
||||
|
||||
FP_PLUGIN_PATH=`${NCPATHCVT} $FP_PLUGIN_PATH`
|
||||
FP_PLUGIN_DIR=`${NCPATHCVT} $FP_PLUGIN_DIR`
|
||||
|
||||
# Set the final output variables
|
||||
HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB"
|
||||
HDF5_PLUGIN_PATH="$FP_PLUGIN_PATH"
|
||||
HDF5_PLUGIN_DIR="$FP_PLUGIN_DIR"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@ -43,36 +43,37 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
|
||||
|
||||
# Hide/unhide the noop filter
|
||||
hidenoop() {
|
||||
rm -fr ${HDF5_PLUGIN_PATH}/save
|
||||
mkdir ${HDF5_PLUGIN_PATH}/save
|
||||
mv ${NOOPPATH} ${HDF5_PLUGIN_PATH}/save
|
||||
rm -fr ${HDF5_PLUGIN_DIR}/save
|
||||
mkdir ${HDF5_PLUGIN_DIR}/save
|
||||
mv ${NOOPDIR} ${HDF5_PLUGIN_DIR}/save
|
||||
}
|
||||
|
||||
unhidenoop() {
|
||||
mv ${HDF5_PLUGIN_PATH}/save/${NOOPLIB} ${HDF5_PLUGIN_PATH}
|
||||
rm -fr ${HDF5_PLUGIN_PATH}/save
|
||||
mv ${HDF5_PLUGIN_DIR}/save/${NOOPLIB} ${HDF5_PLUGIN_DIR}
|
||||
rm -fr ${HDF5_PLUGIN_DIR}/save
|
||||
}
|
||||
|
||||
# Locate the plugin path and the library names; argument order is critical
|
||||
# Locate the plugin dir and the library names; argument order is critical
|
||||
# Find bzip2 and capture
|
||||
findplugin h5bzip2
|
||||
BZIP2LIB="${HDF5_PLUGIN_LIB}"
|
||||
BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}"
|
||||
BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}"
|
||||
# Find misc and capture
|
||||
findplugin h5misc
|
||||
MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
|
||||
MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
|
||||
# Find noop and capture
|
||||
findplugin h5noop
|
||||
NOOPLIB="${HDF5_PLUGIN_LIB}"
|
||||
NOOPPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
|
||||
NOOPDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
|
||||
|
||||
echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
|
||||
export HDF5_PLUGIN_PATH
|
||||
echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}"
|
||||
export HDF5_PLUGIN_DIR
|
||||
export HDF5_PLUGIN_PATH="$HDF5_PLUGIN_DIR"
|
||||
|
||||
# Verify
|
||||
if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
|
||||
if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi
|
||||
if ! test -f ${NOOPPATH} ; then echo "Unable to locate ${NOOPPATH}"; exit 1; fi
|
||||
if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi
|
||||
if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi
|
||||
if ! test -f ${NOOPDIR} ; then echo "Unable to locate ${NOOPDIR}"; exit 1; fi
|
||||
|
||||
# See if we have szip
|
||||
if avail szip; then HAVE_SZIP=1; else HAVE_SZIP=0; fi
|
||||
|
||||
111
nc_test4/tst_filterinstall.sh
Executable file
111
nc_test4/tst_filterinstall.sh
Executable file
@ -0,0 +1,111 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test the filter install
|
||||
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
. ./test_nczarr.sh
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
# Use this plugin path
|
||||
export HDF5_PLUGIN_PATH="${FEATURE_PLUGIN_INSTALL_DIR}"
|
||||
|
||||
# Function to remove selected -s attributes from file;
|
||||
# These attributes might be platform dependent
|
||||
sclean() {
|
||||
cat $1 \
|
||||
| sed -e '/:_IsNetcdf4/d' \
|
||||
| sed -e '/:_Endianness/d' \
|
||||
| sed -e '/_NCProperties/d' \
|
||||
| sed -e '/_SuperblockVersion/d' \
|
||||
| sed -e '/_Format/d' \
|
||||
| sed -e '/global attributes:/d' \
|
||||
| cat > $2
|
||||
}
|
||||
|
||||
# Function to extract _Filter attribute from a file
|
||||
# These attributes might be platform dependent
|
||||
getfilterattr() {
|
||||
V="$1"
|
||||
sed -e '/${V}.*:_Filter/p' -ed <$2 >$3
|
||||
}
|
||||
|
||||
# Function to extract _Codecs attribute from a file
|
||||
# These attributes might be platform dependent
|
||||
getcodecsattr() {
|
||||
V="$1"
|
||||
sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3
|
||||
}
|
||||
|
||||
trimleft() {
|
||||
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
|
||||
}
|
||||
|
||||
setfilter() {
|
||||
FF="$1"
|
||||
FSRC="$2"
|
||||
FDST="$3"
|
||||
FIH5="$4"
|
||||
FICX="$5"
|
||||
FFH5="$6"
|
||||
FFCX="$7"
|
||||
if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi
|
||||
if test "x$FFCX" = x ; then FFCX="$FICX" ; fi
|
||||
rm -f $FDST
|
||||
cat ${srcdir}/$FSRC \
|
||||
| sed -e "s/ref_any/${FF}/" \
|
||||
| sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \
|
||||
| sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \
|
||||
| sed -e 's/"/\\"/g' -e 's/@/"/g' \
|
||||
| cat > $FDST
|
||||
}
|
||||
|
||||
# Execute the specified tests
|
||||
|
||||
runfilter() {
|
||||
zext=$1
|
||||
zfilt="$2"
|
||||
zparams="$3"
|
||||
zcodec="$4"
|
||||
echo "*** Testing processing of filter $zfilt for map $zext"
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
fileargs "tmp_filt_${zfilt}"
|
||||
deletemap $zext $file
|
||||
else
|
||||
file="tmp_filt_${zfilt}.nc"
|
||||
rm -f $file
|
||||
fi
|
||||
setfilter $zfilt ref_any.cdl "tmp_filt_${zfilt}.cdl" "$zparams" "$zcodec"
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
${NCGEN} -4 -lb -o $fileurl "tmp_filt_${zfilt}.cdl"
|
||||
${NCDUMP} -n $zfilt -sF $fileurl > "tmp_filt_${zfilt}.tmp"
|
||||
else
|
||||
${NCGEN} -4 -lb -o $file "tmp_filt_${zfilt}.cdl"
|
||||
${NCDUMP} -n $zfilt -sF $file > "tmp_filt_${zfilt}.tmp"
|
||||
fi
|
||||
sclean "tmp_filt_${zfilt}.tmp" "tmp_filt_${zfilt}.dump"
|
||||
}
|
||||
|
||||
testbzip2() {
|
||||
zext=$1
|
||||
if ! avail bzip2; then return 0; fi
|
||||
runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]'
|
||||
diff -b -w "tmp_filt_bzip2.cdl" "tmp_filt_bzip2.dump"
|
||||
}
|
||||
|
||||
testset() {
|
||||
# Which test cases to exercise
|
||||
testbzip2 $1
|
||||
}
|
||||
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
testset file
|
||||
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi
|
||||
else
|
||||
testset nc
|
||||
fi
|
||||
@ -28,9 +28,6 @@ echo "findplugin.sh loaded"
|
||||
# Assume all test filters are in same plugin dir
|
||||
findplugin h5bzip2
|
||||
|
||||
echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
|
||||
export HDF5_PLUGIN_PATH
|
||||
|
||||
# Function to remove selected -s attributes from file;
|
||||
# These attributes might be platform dependent
|
||||
sclean() {
|
||||
@ -167,14 +164,14 @@ testzstd() {
|
||||
testset() {
|
||||
# Which test cases to exercise
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
# testfletcher32 $1
|
||||
testfletcher32 $1
|
||||
testshuffle $1
|
||||
fi
|
||||
# testdeflate $1
|
||||
# testszip $1
|
||||
# testbzip2 $1
|
||||
# testblosc $1
|
||||
# testzstd $1
|
||||
testdeflate $1
|
||||
testszip $1
|
||||
testbzip2 $1
|
||||
testblosc $1
|
||||
testzstd $1
|
||||
}
|
||||
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
|
||||
@ -10,7 +10,7 @@ include $(top_srcdir)/lib_flags.am
|
||||
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
#TEST_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
#TESTS_ENVIRONMENT += export SETX=1;
|
||||
#TESTS_ENVIRONMENT = export SETX=1;
|
||||
|
||||
# Note which tests depend on other tests. Necessary for make -j check.
|
||||
TEST_EXTENSIONS = .sh
|
||||
|
||||
@ -154,7 +154,7 @@ EXTRA_DIST += ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes
|
||||
|
||||
CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl
|
||||
|
||||
BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh
|
||||
BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh run_filterinstall.sh
|
||||
test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c
|
||||
rm -f $@
|
||||
echo "#define TESTNCZARR" > $@
|
||||
@ -174,6 +174,15 @@ run_specific_filters.sh: $(top_srcdir)/nc_test4/tst_specific_filters.sh
|
||||
chmod a+x $@
|
||||
rm -f run_specific_filters.tmp
|
||||
|
||||
run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh
|
||||
rm -f $@ run_filterinstall.tmp
|
||||
echo "#!/bin/bash" > run_filterinstall.tmp
|
||||
echo "TESTNCZARR=1" >> run_filterinstall.tmp
|
||||
cat $(top_srcdir)/nc_test4/tst_filterinstall.sh >> run_filterinstall.tmp
|
||||
tr -d '\r' < run_filterinstall.tmp > $@
|
||||
chmod a+x $@
|
||||
rm -f run_filterinstall.tmp
|
||||
|
||||
# Remove directories
|
||||
clean-local:
|
||||
rm -fr tmp*.file results.file results.s3 results.zip
|
||||
|
||||
@ -51,21 +51,21 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
|
||||
|
||||
# Find misc and capture
|
||||
findplugin h5misc
|
||||
MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
|
||||
MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
|
||||
|
||||
# Find noop and capture
|
||||
findplugin h5noop
|
||||
NOOPLIB="${HDF5_PLUGIN_LIB}"
|
||||
NOOPPATH="${HDF5_PLUGIN_PATH}/${NOOPLIB}"
|
||||
NOOPDIR="${HDF5_PLUGIN_DIR}/${NOOPLIB}"
|
||||
|
||||
# Find bzip2 and capture
|
||||
findplugin h5bzip2
|
||||
BZIP2LIB="${HDF5_PLUGIN_LIB}"
|
||||
BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}"
|
||||
BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}"
|
||||
|
||||
# Verify
|
||||
if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
|
||||
if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi
|
||||
if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi
|
||||
if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi
|
||||
|
||||
# Execute the specified tests
|
||||
|
||||
@ -144,11 +144,11 @@ ${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt
|
||||
# Remove irrelevant -s output
|
||||
sclean ./tmp_known_$zext.txt tmp_known_$zext.dump
|
||||
# Now hide the filter code
|
||||
mv ${NOOPPATH} ./${NOOPLIB}.save
|
||||
mv ${NOOPDIR} ./${NOOPLIB}.save
|
||||
# dump and clean noop.nc header when filter is not avail
|
||||
${NCDUMP} -hs $fileurl > ./tmp_unk_$zext.txt
|
||||
# Restore the filter code
|
||||
mv ./${NOOPLIB}.save ${NOOPPATH}
|
||||
mv ./${NOOPLIB}.save ${NOOPDIR}
|
||||
# Verify that the filter is no longer defined
|
||||
UNK=`sed -e '/var:_Filter/p' -e d ./tmp_unk_$zext.txt`
|
||||
test "x$UNK" = x
|
||||
|
||||
@ -133,12 +133,12 @@ if test "x$FP_USEPLUGINS" = xyes; then
|
||||
echo "findplugin.sh loaded"
|
||||
|
||||
# Locate the plugin path and the library names; argument order is critical
|
||||
# Find bzip2 and capture
|
||||
# Find misc in order to determine HDF5_PLUGIN+PATH.
|
||||
# Assume all test filters are in same plugin dir
|
||||
findplugin h5misc
|
||||
|
||||
echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
|
||||
export HDF5_PLUGIN_PATH
|
||||
echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}"
|
||||
export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
|
||||
fi # USEPLUGINS
|
||||
|
||||
resetrc() {
|
||||
|
||||
@ -10,12 +10,6 @@ FILE(READ H5Znoop.c NOOP_SOURCE)
|
||||
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "#define NOOP_INSTANCE 1\n")
|
||||
FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "${NOOP_SOURCE}")
|
||||
|
||||
SET(h5bzip2_SOURCES H5Zbzip2.c)
|
||||
|
||||
IF(NOT HAVE_BZIP2)
|
||||
SET(h5bzip2_SOURCES ${h5bzip2_SOURCES} blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c)
|
||||
ENDIF()
|
||||
|
||||
SET(h5misc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h)
|
||||
|
||||
SET(h5noop_SOURCES H5Znoop.c H5Zutil.c h5noop.h)
|
||||
@ -29,11 +23,10 @@ SET(h5fletcher32_SOURCES H5Zfletcher32.c H5checksum.c)
|
||||
SET(h5deflate_SOURCES H5Zdeflate.c)
|
||||
|
||||
SET(nczmisc_SOURCES NCZmisc.c)
|
||||
SET(nczdefaults_SOURCES NCZdefaults.c)
|
||||
|
||||
IF(ENABLE_FILTER_TESTING)
|
||||
IF(BUILD_UTILITIES)
|
||||
SET(nczhdf5filters_SOURCES NCZhdf5filters.c)
|
||||
SET(nczstdfilters_SOURCES NCZstdfilters.c)
|
||||
|
||||
IF(ENABLE_PLUGINS)
|
||||
# LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined
|
||||
|
||||
SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
|
||||
@ -64,7 +57,8 @@ buildplugin(h5fletcher32 "h5fletcher32")
|
||||
buildplugin(h5deflate "h5deflate")
|
||||
|
||||
buildplugin(nczmisc "nczmisc")
|
||||
buildplugin(nczdefaults "nczdefaults" netcdf)
|
||||
buildplugin(nczhdf5filters "nczhdf5filters" netcdf)
|
||||
buildplugin(nczstdfilters "nczstdfilters" netcdf)
|
||||
|
||||
IF(ENABLE_BLOSC)
|
||||
SET(h5blosc_SOURCES H5Zblosc.c)
|
||||
@ -80,17 +74,54 @@ ENDIF()
|
||||
IF(HAVE_SZ)
|
||||
SET(h5szip_SOURCES H5Zszip.c H5Zszip.h)
|
||||
buildplugin(h5szip "h5szip" ${Szip_LIBRARIES})
|
||||
SET(nczszip_SOURCES NCZszip.c)
|
||||
buildplugin(nczszip "nczszip" netcdf)
|
||||
ENDIF()
|
||||
|
||||
IF(HAVE_LOCAL_BZ2)
|
||||
SET(h5bzip2_SOURCES H5Zbzip2.c blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h)
|
||||
buildplugin(h5bzip2 "h5bzip2")
|
||||
ELSE()
|
||||
SET(h5bzip2_SOURCES H5Zbzip2.c)
|
||||
buildplugin(h5bzip2 "h5bzip2" ${Bzip2_LIBRARIES})
|
||||
# Note we use name h5bzip2 instead of bzip2 to avoid logical
|
||||
# target name clash with examples/C/hdf5plugins
|
||||
SET_TARGET_PROPERTIES(h5bzip2 PROPERTIES OUTPUT_NAME "bzip2")
|
||||
ENDIF()
|
||||
|
||||
ENDIF(BUILD_UTILITIES)
|
||||
ENDIF(ENABLE_FILTER_TESTING)
|
||||
|
||||
# Installation
|
||||
IF(ENABLE_PLUGIN_INSTALL)
|
||||
|
||||
MACRO(installplugin PLUG)
|
||||
IF(MSVC)
|
||||
SET(BUILD_PLUGIN_LIB "${PLUG}.dll")
|
||||
ELSE()
|
||||
SET(BUILD_PLUGIN_LIB "lib${PLUG}.so")
|
||||
ENDIF()
|
||||
MESSAGE(STATUS "Installing: ${BUILD_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}")
|
||||
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${BUILD_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR})
|
||||
ENDMACRO()
|
||||
|
||||
install(DIRECTORY DESTINATION ${PLUGIN_INSTALL_DIR})
|
||||
IF(Bzip2_FOUND)
|
||||
installplugin(h5bzip2)
|
||||
ENDIF()
|
||||
IF(Zstd_FOUND)
|
||||
installplugin(h5zstd)
|
||||
ENDIF()
|
||||
IF(Blosc_FOUND)
|
||||
installplugin(h5blosc)
|
||||
ENDIF()
|
||||
IF(ENABLE_NCZARR)
|
||||
installplugin(h5fletcher32)
|
||||
installplugin(h5shuffle)
|
||||
installplugin(h5deflate)
|
||||
installplugin(nczhdf5filters)
|
||||
installplugin(nczstdfilters)
|
||||
IF(Szip_FOUND)
|
||||
installplugin(h5szip)
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
|
||||
ENDIF(ENABLE_PLUGIN_INSTALL)
|
||||
|
||||
ENDIF(ENABLE_PLUGINS)
|
||||
|
||||
# Copy some test files from current source dir to out-of-tree build dir.
|
||||
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
|
||||
|
||||
@ -125,8 +125,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#define PUSH_ERR(f,m,s,...) fprintf(stderr,"%s\n",s)
|
||||
#endif /*USE_HDF5*/
|
||||
|
||||
static int h5z_blosc_initialized = 0;
|
||||
|
||||
static size_t blosc_filter(unsigned flags, size_t cd_nelmts,
|
||||
const unsigned cd_values[], size_t nbytes,
|
||||
size_t* buf_size, void** buf);
|
||||
@ -162,7 +160,7 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space)
|
||||
hid_t super_type;
|
||||
H5T_class_t classt;
|
||||
|
||||
r = GET_FILTER(dcpl, FILTER_BLOSC, &flags, &nelements, values, 0, NULL);
|
||||
r = GET_FILTER(dcpl, H5Z_FILTER_BLOSC, &flags, &nelements, values, 0, NULL);
|
||||
if (r < 0) return -1;
|
||||
|
||||
if (nelements < 4) nelements = 4; /* First 4 slots reserved. */
|
||||
@ -209,7 +207,7 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space)
|
||||
fprintf(stderr, "Blosc: Computed buffer size %d\n", bufsize);
|
||||
#endif
|
||||
|
||||
r = H5Pmodify_filter(dcpl, FILTER_BLOSC, flags, nelements, values);
|
||||
r = H5Pmodify_filter(dcpl, H5Z_FILTER_BLOSC, flags, nelements, values);
|
||||
if (r < 0) return -1;
|
||||
|
||||
return 1;
|
||||
@ -365,7 +363,7 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts,
|
||||
const H5Z_class2_t blosc_H5Filter[1] = {
|
||||
{
|
||||
H5Z_CLASS_T_VERS,
|
||||
(H5Z_filter_t)(FILTER_BLOSC),
|
||||
(H5Z_filter_t)(H5Z_FILTER_BLOSC),
|
||||
1, /* encoder_present flag (set to true) */
|
||||
1, /* decoder_present flag (set to true) */
|
||||
"blosc",
|
||||
@ -379,235 +377,3 @@ const H5Z_class2_t blosc_H5Filter[1] = {
|
||||
|
||||
H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_FILTER; }
|
||||
const void* H5PLget_plugin_info(void) { return blosc_H5Filter; }
|
||||
|
||||
|
||||
/* Provide the codec support for the HDF5 blosc library */
|
||||
|
||||
/* NCZarr Filter Objects */
|
||||
|
||||
#define DEFAULT_LEVEL 9
|
||||
#define DEFAULT_BLOCKSIZE 1
|
||||
#define DEFAULT_TYPESIZE 1
|
||||
#define DEFAULT_COMPCODE BLOSC_LZ4
|
||||
|
||||
/* Forward */
|
||||
static void NCZ_blosc_codec_finalize(void);
|
||||
static int NCZ_blosc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
||||
static int NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
||||
static int NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
|
||||
|
||||
/* Structure for NCZ_PLUGIN_CODEC */
|
||||
static NCZ_codec_t NCZ_blosc_codec = {/* NCZ_codec_t codec fields */
|
||||
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
||||
NCZ_CODEC_HDF5, /* Struct sort */
|
||||
"blosc", /* Standard name/id of the codec */
|
||||
FILTER_BLOSC, /* HDF5 alias for blosc */
|
||||
NULL, /*NCZ_blosc_codec_initialize*/
|
||||
NCZ_blosc_codec_finalize,
|
||||
NCZ_blosc_codec_to_hdf5,
|
||||
NCZ_blosc_hdf5_to_codec,
|
||||
NCZ_blosc_modify_parameters,
|
||||
};
|
||||
|
||||
/* External Export API */
|
||||
const void*
|
||||
NCZ_get_codec_info(void)
|
||||
{
|
||||
if(!h5z_blosc_initialized) {
|
||||
h5z_blosc_initialized = 1;
|
||||
blosc_init();
|
||||
}
|
||||
return (void*)&NCZ_blosc_codec;
|
||||
}
|
||||
|
||||
/* NCZarr Interface Functions */
|
||||
|
||||
/* Create the true parameter set:
|
||||
Visible parameters:
|
||||
param[0] -- reserved
|
||||
param[1] -- reserved
|
||||
param[2] -- reserved
|
||||
param[3] -- variable chunksize in bytes | 0 (=>default)
|
||||
param[4] -- compression level
|
||||
param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
|
||||
param[6] -- compressor index
|
||||
|
||||
Working parameters:
|
||||
param[0] -- filter revision
|
||||
param[1] -- blosc version
|
||||
param[2] -- variable type size in bytes
|
||||
param[3] -- variable chunksize in bytes
|
||||
param[4] -- compression level
|
||||
param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
|
||||
param[6] -- compressor index
|
||||
*/
|
||||
|
||||
static void
|
||||
NCZ_blosc_codec_finalize(void)
|
||||
{
|
||||
if(h5z_blosc_initialized) {
|
||||
blosc_destroy();
|
||||
h5z_blosc_initialized = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
|
||||
{
|
||||
int i,stat = NC_NOERR;
|
||||
nc_type vtype;
|
||||
int storage, ndims;
|
||||
size_t* chunklens = NULL;
|
||||
size_t typesize, chunksize;
|
||||
char vname[NC_MAX_NAME+1];
|
||||
unsigned* params = NULL;
|
||||
size_t nparams;
|
||||
size_t vnparams = *vnparamsp;
|
||||
unsigned* vparams = *vparamsp;
|
||||
|
||||
if(vnparams < 7)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
nparams = 7;
|
||||
|
||||
if(vparams == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
|
||||
if(wnparamsp == NULL || wparamsp == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
|
||||
vnparams = *vnparamsp;
|
||||
vparams = *vparamsp;
|
||||
|
||||
/* Get variable info */
|
||||
if((stat = nc_inq_var(ncid,varid,vname,&vtype,&ndims,NULL,NULL))) goto done;
|
||||
|
||||
if(ndims == 0) {stat = NC_EFILTER; goto done;}
|
||||
|
||||
/* Get the typesize */
|
||||
if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
|
||||
|
||||
/* Compute chunksize */
|
||||
if((chunklens = (size_t*)calloc(ndims,sizeof(size_t)))==NULL) goto done;
|
||||
if((stat = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
|
||||
if(storage != NC_CHUNKED) {stat = NC_EFILTER; goto done;}
|
||||
chunksize = typesize;
|
||||
for(i=0;i<ndims;i++) chunksize *= chunklens[i];
|
||||
|
||||
if((params = (unsigned*)malloc(vnparams*sizeof(unsigned)))==NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
memcpy(params,vparams,vnparams*sizeof(unsigned));
|
||||
|
||||
params[0] = FILTER_BLOSC_VERSION;
|
||||
params[1] = BLOSC_VERSION_FORMAT;
|
||||
params[2] = (unsigned)typesize;
|
||||
|
||||
params[3] = chunksize;
|
||||
params[4] = params[4];
|
||||
params[5] = params[5];
|
||||
params[6] = params[6];
|
||||
|
||||
*wnparamsp = nparams;
|
||||
nullfree(*wparamsp);
|
||||
*wparamsp = params; params = NULL;
|
||||
|
||||
done:
|
||||
nullfree(chunklens);
|
||||
nullfree(params);
|
||||
FUNC_LEAVE_NOAPI(stat)
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_blosc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCjson* jcodec = NULL;
|
||||
NCjson* jtmp = NULL;
|
||||
unsigned* params = NULL;
|
||||
struct NCJconst jc = {0,0,0,NULL};
|
||||
int compcode;
|
||||
|
||||
/* parse the JSON */
|
||||
if(NCJparse(codec_json,0,&jcodec)) {stat = NC_EFILTER; goto done;}
|
||||
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
|
||||
/* Verify the codec ID */
|
||||
if(NCJdictget(jcodec,"id",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;}
|
||||
if(strcmp(NCJstring(jtmp),NCZ_blosc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
|
||||
|
||||
if((params = (unsigned*)calloc(7,sizeof(unsigned)))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
|
||||
/* Get compression level*/
|
||||
if(NCJdictget(jcodec,"clevel",&jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(jtmp) {
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
|
||||
} else
|
||||
jc.ival = DEFAULT_LEVEL;
|
||||
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
|
||||
params[4] = (unsigned)jc.ival;
|
||||
|
||||
/* Get blocksize */
|
||||
if(NCJdictget(jcodec,"blocksize",&jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(jtmp) {
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
|
||||
} else
|
||||
jc.ival = DEFAULT_BLOCKSIZE;
|
||||
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
|
||||
params[3] = (unsigned)jc.ival;
|
||||
|
||||
/* Get shuffle */
|
||||
if(NCJdictget(jcodec,"shuffle",&jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(jtmp) {
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
|
||||
} else
|
||||
jc.ival = BLOSC_NOSHUFFLE;
|
||||
params[5] = (unsigned)jc.ival;
|
||||
|
||||
/* Get compname */
|
||||
if(NCJdictget(jcodec,"cname",&jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(jtmp) {
|
||||
if(NCJcvt(jtmp,NCJ_STRING,&jc)) {stat = NC_EFILTER; goto done;}
|
||||
if(jc.sval == NULL || strlen(jc.sval) == 0) {stat = NC_EFILTER; goto done;}
|
||||
if((compcode = blosc_compname_to_compcode(jc.sval)) < 0) {stat = NC_EFILTER; goto done;}
|
||||
} else
|
||||
compcode = DEFAULT_COMPCODE;
|
||||
params[6] = (unsigned)compcode;
|
||||
|
||||
if(nparamsp) *nparamsp = 7;
|
||||
if(paramsp) {*paramsp = params; params = NULL;}
|
||||
|
||||
done:
|
||||
if(jc.sval) {
|
||||
free(jc.sval);
|
||||
}
|
||||
if(params) {
|
||||
free(params);
|
||||
}
|
||||
NCJreclaim(jcodec);
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
char json[1024];
|
||||
const char* compname = NULL;
|
||||
|
||||
if(nparams == 0 || params == NULL)
|
||||
{stat = NC_EINVAL; goto done;}
|
||||
|
||||
/* Get the sub-compressor name */
|
||||
if(blosc_compcode_to_compname((int)params[6],&compname) < 0) {stat = NC_EFILTER; goto done;}
|
||||
|
||||
snprintf(json,sizeof(json),
|
||||
"{\"id\": \"blosc\",\"clevel\": %u,\"blocksize\": %u,\"cname\": \"%s\",\"shuffle\": %d}",
|
||||
params[4],params[3],compname,params[5]);
|
||||
|
||||
if(codecp) {
|
||||
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
}
|
||||
|
||||
done:
|
||||
return stat;
|
||||
}
|
||||
|
||||
@ -29,13 +29,6 @@ extern "C" {
|
||||
#include "netcdf_filter_build.h"
|
||||
#include "blosc.h"
|
||||
|
||||
/* Filter revision number, starting at 1 */
|
||||
/* #define FILTER_BLOSC_VERSION 1 */
|
||||
#define FILTER_BLOSC_VERSION 2 /* multiple compressors since Blosc 1.3 */
|
||||
|
||||
/* Filter ID registered with the HDF Group */
|
||||
#define FILTER_BLOSC 32001
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#ifdef DLL_EXPORT /* define when building the library */
|
||||
#define DECLSPEC __declspec(dllexport)
|
||||
@ -46,6 +39,15 @@ extern "C" {
|
||||
#define DECLSPEC extern
|
||||
#endif
|
||||
|
||||
/* Filter revision number, starting at 1 */
|
||||
/* #define FILTER_BLOSC_VERSION 1 */
|
||||
#define FILTER_BLOSC_VERSION 2 /* multiple compressors since Blosc 1.3 */
|
||||
|
||||
#define DEFAULT_LEVEL 9
|
||||
#define DEFAULT_BLOCKSIZE 1
|
||||
#define DEFAULT_TYPESIZE 1
|
||||
#define DEFAULT_COMPCODE BLOSC_LZ4
|
||||
|
||||
/* HDF5 Plugin API */
|
||||
DECLSPEC H5PL_type_t H5PLget_plugin_type(void);
|
||||
DECLSPEC const void* H5PLget_plugin_info(void);
|
||||
|
||||
@ -214,92 +214,3 @@ H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts,
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**************************************************/
|
||||
/* NCZarr Filter Objects */
|
||||
|
||||
/* Provide the codec support for the HDF5 bzip library */
|
||||
|
||||
static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
||||
static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
||||
|
||||
static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */
|
||||
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
||||
NCZ_CODEC_HDF5, /* Struct sort */
|
||||
"bz2", /* Standard name/id of the codec */
|
||||
H5Z_FILTER_BZIP2, /* HDF5 alias for bzip2 */
|
||||
NULL, /*NCZ_bzip2_codec_initialize*/
|
||||
NULL, /*NCZ_bzip2_codec_finalize*/
|
||||
NCZ_bzip2_codec_to_hdf5,
|
||||
NCZ_bzip2_hdf5_to_codec,
|
||||
NULL, /*NCZ_bzip2_modify_parameters*/
|
||||
};
|
||||
|
||||
/* External Export API */
|
||||
DLLEXPORT
|
||||
const void*
|
||||
NCZ_get_codec_info(void)
|
||||
{
|
||||
return (void*)&NCZ_bzip2_codec;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_bzip2_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCjson* jcodec = NULL;
|
||||
NCjson* jtmp = NULL;
|
||||
unsigned* params = NULL;
|
||||
struct NCJconst jc;
|
||||
|
||||
if(nparamsp == NULL || paramsp == NULL)
|
||||
{stat = NC_EINTERNAL; goto done;}
|
||||
|
||||
if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
|
||||
/* parse the JSON */
|
||||
if(NCJparse(codec_json,0,&jcodec))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
|
||||
/* Verify the codec ID */
|
||||
if(NCJdictget(jcodec,"id",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(strcmp(NCJstring(jtmp),NCZ_bzip2_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
|
||||
|
||||
/* Get Level */
|
||||
if(NCJdictget(jcodec,"level",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
|
||||
params[0] = (unsigned)jc.ival;
|
||||
*nparamsp = 1;
|
||||
*paramsp = params; params = NULL;
|
||||
|
||||
done:
|
||||
if(params) free(params);
|
||||
NCJreclaim(jcodec);
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
unsigned level = 0;
|
||||
char json[1024];
|
||||
|
||||
if(nparams == 0 || params == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
|
||||
level = params[0];
|
||||
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level);
|
||||
if(codecp) {
|
||||
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
}
|
||||
|
||||
done:
|
||||
return stat;
|
||||
}
|
||||
|
||||
|
||||
@ -166,92 +166,3 @@ cleanupAndFail:
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**************************************************/
|
||||
/* NCZarr Filter Objects */
|
||||
|
||||
/* Provide the codec support for the HDF5 zstandard library */
|
||||
|
||||
static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
||||
static int NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
||||
|
||||
static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */
|
||||
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
||||
NCZ_CODEC_HDF5, /* Struct sort */
|
||||
"zstd", /* Standard name/id of the codec */
|
||||
H5Z_FILTER_ZSTD, /* HDF5 alias for zstd */
|
||||
NULL, /*NCZ_zstd_codec_initialize*/
|
||||
NULL, /*NCZ_zstd_codec_finalize*/
|
||||
NCZ_zstd_codec_to_hdf5,
|
||||
NCZ_zstd_hdf5_to_codec,
|
||||
NULL, /*NCZ_zstd_modify_parameters*/
|
||||
};
|
||||
|
||||
/* External Export API */
|
||||
DLLEXPORT
|
||||
const void*
|
||||
NCZ_get_codec_info(void)
|
||||
{
|
||||
return (void*)&NCZ_zstd_codec;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_zstd_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCjson* jcodec = NULL;
|
||||
NCjson* jtmp = NULL;
|
||||
unsigned* params = NULL;
|
||||
struct NCJconst jc;
|
||||
|
||||
if(nparamsp == NULL || paramsp == NULL)
|
||||
{stat = NC_EINTERNAL; goto done;}
|
||||
|
||||
if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
|
||||
/* parse the JSON */
|
||||
if(NCJparse(codec_json,0,&jcodec))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
|
||||
/* Verify the codec ID */
|
||||
if(NCJdictget(jcodec,"id",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
|
||||
|
||||
/* Get Level */
|
||||
if(NCJdictget(jcodec,"level",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
|
||||
params[0] = (unsigned)jc.ival;
|
||||
*nparamsp = 1;
|
||||
*paramsp = params; params = NULL;
|
||||
|
||||
done:
|
||||
if(params) free(params);
|
||||
NCJreclaim(jcodec);
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
unsigned level = 0;
|
||||
char json[1024];
|
||||
|
||||
if(nparams == 0 || params == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
|
||||
level = params[0];
|
||||
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level);
|
||||
if(codecp) {
|
||||
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
}
|
||||
|
||||
done:
|
||||
return stat;
|
||||
}
|
||||
|
||||
|
||||
@ -14,34 +14,22 @@ if ISMINGW
|
||||
LDADD = ${top_builddir}/liblib/libnetcdf.la
|
||||
endif
|
||||
|
||||
# BZIP2 version 1.0.8 (https://sourceware.org/bzip2/)
|
||||
BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h
|
||||
|
||||
EXTRA_DIST = CMakeLists.txt
|
||||
|
||||
# The HDF5 filter wrappers
|
||||
EXTRA_DIST += \
|
||||
EXTRA_DIST += stdinstall.sh \
|
||||
H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \
|
||||
H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \
|
||||
H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h H5Zzstd.c H5Zzstd.h
|
||||
|
||||
# The Codec filter wrappers
|
||||
EXTRA_DIST += NCZdefaults.c NCZszip.c
|
||||
EXTRA_DIST += NCZhdf5filters.c NCZstdfilters.c
|
||||
# The Filter implementations
|
||||
EXTRA_DIST += H5checksum.c
|
||||
|
||||
|
||||
EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE
|
||||
|
||||
if ENABLE_FILTER_TESTING
|
||||
|
||||
lib_LTLIBRARIES += libh5bzip2.la
|
||||
libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h
|
||||
if ! HAVE_BZIP2
|
||||
libh5bzip2_la_SOURCES += ${BZIP2SRC}
|
||||
endif
|
||||
|
||||
noinst_LTLIBRARIES += libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la
|
||||
noinst_LTLIBRARIES += libnczdefaults.la
|
||||
|
||||
if ENABLE_NCZARR_FILTERS
|
||||
noinst_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la
|
||||
@ -51,27 +39,39 @@ libh5deflate_la_SOURCES = H5Zdeflate.c
|
||||
|
||||
# Need our version of szip if libsz available and we are not using HDF5
|
||||
if HAVE_SZ
|
||||
noinst_LTLIBRARIES += libh5szip.la libnczszip.la
|
||||
noinst_LTLIBRARIES += libh5szip.la
|
||||
libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h
|
||||
libnczszip_la_SOURCES = NCZszip.c
|
||||
endif
|
||||
|
||||
libnczdefaults_la_SOURCES = NCZdefaults.c
|
||||
|
||||
endif # ENABLE_NCZARR_FILTERS
|
||||
|
||||
if ENABLE_PLUGINS
|
||||
|
||||
libnczstdfilters_la_SOURCES = NCZstdfilters.c
|
||||
|
||||
libnczhdf5filters_la_SOURCES = NCZhdf5filters.c
|
||||
|
||||
if HAVE_BLOSC
|
||||
noinst_LTLIBRARIES += libh5blosc.la
|
||||
libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h
|
||||
endif
|
||||
|
||||
if HAVE_ZSTD
|
||||
noinst_LTLIBRARIES += libh5zstd.la
|
||||
libh5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h
|
||||
endif
|
||||
|
||||
libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h
|
||||
noinst_LTLIBRARIES += libnczhdf5filters.la
|
||||
noinst_LTLIBRARIES += libnczstdfilters.la
|
||||
|
||||
if HAVE_BLOSC
|
||||
noinst_LTLIBRARIES += libh5blosc.la
|
||||
endif
|
||||
if HAVE_ZSTD
|
||||
noinst_LTLIBRARIES += libh5zstd.la
|
||||
endif
|
||||
|
||||
endif #ENABLE_PLUGINS
|
||||
|
||||
libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h
|
||||
libnczmisc_la_SOURCES = NCZmisc.c
|
||||
|
||||
# The noop filter is to allow testing of multifilters and filter order
|
||||
@ -79,6 +79,14 @@ libnczmisc_la_SOURCES = NCZmisc.c
|
||||
libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h
|
||||
libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h
|
||||
|
||||
# Bzip2 is used to test more complex filters
|
||||
libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h
|
||||
BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h
|
||||
EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE
|
||||
if HAVE_LOCAL_BZ2
|
||||
libh5bzip2_la_SOURCES += ${BZIP2SRC}
|
||||
endif
|
||||
|
||||
endif #ENABLE_FILTER_TESTING
|
||||
|
||||
BUILT_SOURCES = H5Znoop1.c
|
||||
@ -87,6 +95,9 @@ H5Znoop1.c: Makefile H5Znoop.c
|
||||
echo '#define NOOP_INSTANCE 1' > $@
|
||||
cat ${srcdir}/H5Znoop.c >> $@
|
||||
|
||||
noinst_LTLIBRARIES += libh5bzip2.la
|
||||
|
||||
# Record where bzip2 came from; may be out of date
|
||||
BZIP2VER = 1.0.8
|
||||
BZIP2DIR = bzip2-${BZIP2VER}
|
||||
BZIP2URL = https://sourceware.org/pub/bzip2/${BZIP2DIR}.tar.gz
|
||||
@ -96,3 +107,7 @@ bzip2::
|
||||
tar -zxf ${BZIP2DIR}.tar.gz
|
||||
cd ${BZIP2DIR}; cp ${BZIP2SRC} ..; cp LICENSE ../BZIP2_LICENSE ; cd ..
|
||||
rm -fr ./${BZIP2DIR}
|
||||
|
||||
# Custom install
|
||||
install-exec-hook:
|
||||
sh ./stdinstall.sh
|
||||
|
||||
@ -21,6 +21,11 @@ Author: Dennis Heimbigner
|
||||
#include "netcdf_filter_build.h"
|
||||
#include "netcdf_json.h"
|
||||
|
||||
#ifdef HAVE_SZ
|
||||
#include <szlib.h>
|
||||
#include "H5Zszip.h"
|
||||
#endif
|
||||
|
||||
#define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */
|
||||
#define H5Z_FILTER_SHUFFLE 2 /*shuffle the data */
|
||||
#define H5Z_FILTER_FLETCHER32 3 /*fletcher32 checksum of EDC */
|
||||
@ -40,6 +45,12 @@ static int NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparam
|
||||
static int NCZ_deflate_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
||||
static int NCZ_deflate_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
||||
|
||||
#ifdef HAVE_SZ
|
||||
static int NCZ_szip_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
||||
static int NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
||||
static int NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
|
||||
#endif
|
||||
|
||||
/**************************************************/
|
||||
|
||||
static NCZ_codec_t NCZ_shuffle_codec = {
|
||||
@ -302,10 +313,224 @@ done:
|
||||
|
||||
/**************************************************/
|
||||
|
||||
#ifdef HAVE_SZ
|
||||
|
||||
static NCZ_codec_t NCZ_szip_codec = {
|
||||
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
||||
NCZ_CODEC_HDF5, /* Struct sort */
|
||||
"szip", /* Standard name/id of the codec */
|
||||
H5Z_FILTER_SZIP, /* HDF5 alias for szip */
|
||||
NULL, /*NCZ_szip_codec_initialize*/
|
||||
NULL, /*NCZ_szip_codec_finalize*/
|
||||
NCZ_szip_codec_to_hdf5,
|
||||
NCZ_szip_hdf5_to_codec,
|
||||
NCZ_szip_modify_parameters,
|
||||
};
|
||||
|
||||
static int
|
||||
NCZ_szip_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
unsigned* params = NULL;
|
||||
size_t nparams = 2; /* No. of visible parameters */
|
||||
NCjson* json = NULL;
|
||||
NCjson* jtmp = NULL;
|
||||
struct NCJconst jc = {0,0,0,NULL};
|
||||
|
||||
if(nparamsp == NULL || paramsp == NULL)
|
||||
{stat = NC_EINTERNAL; goto done;}
|
||||
|
||||
if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
|
||||
if(NCJparse(codec_json,0,&json))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
|
||||
if(NCJdictget(json,"mask",&jtmp) || jtmp == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
params[H5Z_SZIP_PARM_MASK] = (unsigned)jc.ival;
|
||||
|
||||
jtmp = NULL;
|
||||
if(NCJdictget(json,"pixels-per-block",&jtmp) || jtmp == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
params[H5Z_SZIP_PARM_PPB] = (unsigned)jc.ival;
|
||||
|
||||
*nparamsp = nparams;
|
||||
*paramsp = params; params = NULL;
|
||||
|
||||
done:
|
||||
NCJreclaim(json);
|
||||
nullfree(params);
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
char json[2048];
|
||||
|
||||
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": %u, \"pixels-per-block\": %u}",
|
||||
NCZ_szip_codec.codecid,
|
||||
params[H5Z_SZIP_PARM_MASK],
|
||||
params[H5Z_SZIP_PARM_PPB]);
|
||||
if(codecp) {
|
||||
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
}
|
||||
|
||||
done:
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
|
||||
{
|
||||
int i,ret_value = NC_NOERR;
|
||||
nc_type vtype;
|
||||
size_t typesize, scanline, dtype_precision, npoints;
|
||||
int ndims, storage, dtype_order;
|
||||
int dimids[NC_MAX_VAR_DIMS];
|
||||
char vname[NC_MAX_NAME+1];
|
||||
size_t chunklens[NC_MAX_VAR_DIMS];
|
||||
unsigned* params = NULL;
|
||||
unsigned* vparams = NULL;
|
||||
size_t wnparams = 4;
|
||||
|
||||
if(wnparamsp == NULL || wparamsp == NULL)
|
||||
{ret_value = NC_EFILTER; goto done;}
|
||||
if(vnparamsp == NULL || vparamsp == NULL)
|
||||
{ret_value = NC_EFILTER; goto done;}
|
||||
if(*vnparamsp > 0 && *vparamsp == NULL)
|
||||
{ret_value = NC_EFILTER; goto done;}
|
||||
|
||||
vparams = *vparamsp;
|
||||
|
||||
/* Get variable info */
|
||||
if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done;
|
||||
|
||||
/* Get the typesize */
|
||||
if((ret_value = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
|
||||
|
||||
/* Get datatype's precision, in case is less than full bits */
|
||||
dtype_precision = typesize*8;
|
||||
|
||||
if(dtype_precision > 24) {
|
||||
if(dtype_precision <= 32)
|
||||
dtype_precision = 32;
|
||||
else if(dtype_precision <= 64)
|
||||
dtype_precision = 64;
|
||||
} /* end if */
|
||||
|
||||
if(ndims == 0) {ret_value = NC_EFILTER; goto done;}
|
||||
|
||||
/* Set "local" parameter for this dataset's "pixels-per-scanline" */
|
||||
if((ret_value = nc_inq_dimlen(ncid,dimids[ndims-1],&scanline))) goto done;
|
||||
|
||||
/* Get number of elements for the dataspace; use
|
||||
total number of elements in the chunk to define the new 'scanline' size */
|
||||
/* Compute chunksize */
|
||||
if((ret_value = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
|
||||
if(storage != NC_CHUNKED) {ret_value = NC_EFILTER; goto done;}
|
||||
npoints = 1;
|
||||
for(i=0;i<ndims;i++) npoints *= chunklens[i];
|
||||
|
||||
/* Get datatype's endianness order */
|
||||
if((ret_value = nc_inq_var_endian(ncid,varid,&dtype_order))) goto done;
|
||||
|
||||
if((params = (unsigned*)malloc(wnparams*sizeof(unsigned)))==NULL)
|
||||
{ret_value = NC_ENOMEM; goto done;}
|
||||
params[H5Z_SZIP_PARM_MASK] = vparams[H5Z_SZIP_PARM_MASK];
|
||||
params[H5Z_SZIP_PARM_PPB] = vparams[H5Z_SZIP_PARM_PPB];
|
||||
|
||||
/* Set "local" parameter for this dataset's "bits-per-pixel" */
|
||||
params[H5Z_SZIP_PARM_BPP] = dtype_precision;
|
||||
|
||||
/* Adjust scanline if it is smaller than number of pixels per block or
|
||||
if it is bigger than maximum pixels per scanline, or there are more than
|
||||
SZ_MAX_BLOCKS_PER_SCANLINE blocks per scanline */
|
||||
if(scanline < vparams[H5Z_SZIP_PARM_PPB]) {
|
||||
if(npoints < vparams[H5Z_SZIP_PARM_PPB])
|
||||
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "pixels per block greater than total number of elements in the chunk")
|
||||
scanline = MIN((vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), npoints);
|
||||
} else {
|
||||
if(scanline <= SZ_MAX_PIXELS_PER_SCANLINE)
|
||||
scanline = MIN((vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), scanline);
|
||||
else
|
||||
scanline = vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE;
|
||||
} /* end else */
|
||||
/* Assign the final value to the scanline */
|
||||
params[H5Z_SZIP_PARM_PPS] = (unsigned)scanline;
|
||||
|
||||
/* Set the correct mask flags */
|
||||
|
||||
/* From H5Pdcpl.c#H5Pset_szip */
|
||||
params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~H5_SZIP_CHIP_OPTION_MASK);
|
||||
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_ALLOW_K13_OPTION_MASK;
|
||||
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_RAW_OPTION_MASK;
|
||||
params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~(H5_SZIP_LSB_OPTION_MASK | H5_SZIP_MSB_OPTION_MASK));
|
||||
|
||||
/* From H5Zszip.c#H5Z__set_local_szip */
|
||||
params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~(H5_SZIP_LSB_OPTION_MASK | H5_SZIP_MSB_OPTION_MASK));
|
||||
switch(dtype_order) {
|
||||
case NC_ENDIAN_LITTLE: /* Little-endian byte order */
|
||||
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_LSB_OPTION_MASK;
|
||||
break;
|
||||
case NC_ENDIAN_BIG: /* Big-endian byte order */
|
||||
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_MSB_OPTION_MASK;
|
||||
break;
|
||||
default:
|
||||
HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
|
||||
} /* end switch */
|
||||
|
||||
*wnparamsp = wnparams;
|
||||
nullfree(*wparamsp);
|
||||
*wparamsp = params; params = NULL;
|
||||
|
||||
done:
|
||||
nullfree(params);
|
||||
FUNC_LEAVE_NOAPI(ret_value)
|
||||
}
|
||||
|
||||
#if 0
|
||||
static int
|
||||
NCZ_szip_visible_parameters(int ncid, int varid, size_t nparamsin, const unsigned int* paramsin, size_t* nparamsp, unsigned** paramsp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
unsigned* params = NULL;
|
||||
|
||||
if(nparamsp == NULL || paramsp == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
|
||||
if((params = (unsigned*)malloc(2*sizeof(unsigned)))==NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
|
||||
params[H5Z_SZIP_PARM_MASK] = paramsin[H5Z_SZIP_PARM_MASK];
|
||||
params[H5Z_SZIP_PARM_PPB] = paramsin[H5Z_SZIP_PARM_PPB];
|
||||
|
||||
nullfree(*paramsp);
|
||||
*paramsp = params; params = NULL;
|
||||
|
||||
done:
|
||||
nullfree(params);
|
||||
return stat;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*HAVE_SZ*/
|
||||
|
||||
/**************************************************/
|
||||
|
||||
NCZ_codec_t* NCZ_default_codecs[] = {
|
||||
&NCZ_shuffle_codec,
|
||||
&NCZ_fletcher32_codec,
|
||||
&NCZ_zlib_codec,
|
||||
#ifdef HAVE_SZ
|
||||
&NCZ_szip_codec,
|
||||
#endif
|
||||
NULL
|
||||
};
|
||||
|
||||
460
plugins/NCZstdfilters.c
Normal file
460
plugins/NCZstdfilters.c
Normal file
@ -0,0 +1,460 @@
|
||||
/* Copyright 2003-2018, University Corporation for Atmospheric
|
||||
* Research. See the COPYRIGHT file for copying and redistribution
|
||||
* conditions.
|
||||
*/
|
||||
|
||||
/*
|
||||
Author: Dennis Heimbigner
|
||||
*/
|
||||
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "netcdf.h"
|
||||
#include "netcdf_filter.h"
|
||||
#include "netcdf_filter_build.h"
|
||||
#include "netcdf_json.h"
|
||||
|
||||
/* Provide Codec information for the standard filters */
|
||||
|
||||
#ifndef H5Z_FILTER_BZIP2
|
||||
#define H5Z_FILTER_BZIP2 1
|
||||
#define H5Z_FILTER_ZSTD 2
|
||||
#define H5Z_FILTER_BLOSC 3
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_BLOSC
|
||||
#include "H5Zblosc.h"
|
||||
#endif
|
||||
|
||||
/**************************************************/
|
||||
/* NCZarr Filter Objects */
|
||||
|
||||
/* Forward */
|
||||
|
||||
#ifdef HAVE_BZ2
|
||||
static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
||||
static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_ZSTD
|
||||
static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
||||
static int NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_BLOSC
|
||||
static int NCZ_blosc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
||||
static int NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
||||
static int NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
|
||||
static void NCZ_blosc_codec_finalize(void);
|
||||
#endif
|
||||
|
||||
/**************************************************/
|
||||
/* Provide the codec support for bzip2 filter */
|
||||
|
||||
#ifdef HAVE_BZ2
|
||||
static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */
|
||||
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
||||
NCZ_CODEC_HDF5, /* Struct sort */
|
||||
"bz2", /* Standard name/id of the codec */
|
||||
H5Z_FILTER_BZIP2, /* HDF5 alias for bzip2 */
|
||||
NULL, /*NCZ_bzip2_codec_initialize*/
|
||||
NULL, /*NCZ_bzip2_codec_finalize*/
|
||||
NCZ_bzip2_codec_to_hdf5,
|
||||
NCZ_bzip2_hdf5_to_codec,
|
||||
NULL, /*NCZ_bzip2_modify_parameters*/
|
||||
};
|
||||
|
||||
/* External Export API */
|
||||
DLLEXPORT
|
||||
const void*
|
||||
NCZ_get_codec_info(void)
|
||||
{
|
||||
return (void*)&NCZ_bzip2_codec;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_bzip2_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCjson* jcodec = NULL;
|
||||
NCjson* jtmp = NULL;
|
||||
unsigned* params = NULL;
|
||||
struct NCJconst jc;
|
||||
|
||||
if(nparamsp == NULL || paramsp == NULL)
|
||||
{stat = NC_EINTERNAL; goto done;}
|
||||
|
||||
if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
|
||||
/* parse the JSON */
|
||||
if(NCJparse(codec_json,0,&jcodec))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
|
||||
/* Verify the codec ID */
|
||||
if(NCJdictget(jcodec,"id",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(strcmp(NCJstring(jtmp),NCZ_bzip2_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
|
||||
|
||||
/* Get Level */
|
||||
if(NCJdictget(jcodec,"level",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
|
||||
params[0] = (unsigned)jc.ival;
|
||||
*nparamsp = 1;
|
||||
*paramsp = params; params = NULL;
|
||||
|
||||
done:
|
||||
if(params) free(params);
|
||||
NCJreclaim(jcodec);
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
unsigned level = 0;
|
||||
char json[1024];
|
||||
|
||||
if(nparams == 0 || params == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
|
||||
level = params[0];
|
||||
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level);
|
||||
if(codecp) {
|
||||
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
}
|
||||
|
||||
done:
|
||||
return stat;
|
||||
}
|
||||
#endif
|
||||
|
||||
/**************************************************/
|
||||
/* Provide the codec support for zstandard filter */
|
||||
|
||||
#ifdef HAVE_ZSTD
|
||||
|
||||
static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */
|
||||
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
||||
NCZ_CODEC_HDF5, /* Struct sort */
|
||||
"zstd", /* Standard name/id of the codec */
|
||||
H5Z_FILTER_ZSTD, /* HDF5 alias for zstd */
|
||||
NULL, /*NCZ_zstd_codec_initialize*/
|
||||
NULL, /*NCZ_zstd_codec_finalize*/
|
||||
NCZ_zstd_codec_to_hdf5,
|
||||
NCZ_zstd_hdf5_to_codec,
|
||||
NULL, /*NCZ_zstd_modify_parameters*/
|
||||
};
|
||||
|
||||
static int
|
||||
NCZ_zstd_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCjson* jcodec = NULL;
|
||||
NCjson* jtmp = NULL;
|
||||
unsigned* params = NULL;
|
||||
struct NCJconst jc;
|
||||
|
||||
if(nparamsp == NULL || paramsp == NULL)
|
||||
{stat = NC_EINTERNAL; goto done;}
|
||||
|
||||
if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
|
||||
/* parse the JSON */
|
||||
if(NCJparse(codec_json,0,&jcodec))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
|
||||
/* Verify the codec ID */
|
||||
if(NCJdictget(jcodec,"id",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
|
||||
|
||||
/* Get Level */
|
||||
if(NCJdictget(jcodec,"level",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
|
||||
params[0] = (unsigned)jc.ival;
|
||||
*nparamsp = 1;
|
||||
*paramsp = params; params = NULL;
|
||||
|
||||
done:
|
||||
if(params) free(params);
|
||||
NCJreclaim(jcodec);
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
unsigned level = 0;
|
||||
char json[1024];
|
||||
|
||||
if(nparams == 0 || params == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
|
||||
level = params[0];
|
||||
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level);
|
||||
if(codecp) {
|
||||
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
}
|
||||
|
||||
done:
|
||||
return stat;
|
||||
}
|
||||
#endif
|
||||
|
||||
/**************************************************/
|
||||
/* Provide the codec support for blosc filter */
|
||||
|
||||
#ifdef HAVE_BLOSC
|
||||
|
||||
/* Structure for NCZ_PLUGIN_CODEC */
|
||||
static NCZ_codec_t NCZ_blosc_codec = {/* NCZ_codec_t codec fields */
|
||||
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
||||
NCZ_CODEC_HDF5, /* Struct sort */
|
||||
"blosc", /* Standard name/id of the codec */
|
||||
H5Z_FILTER_BLOSC, /* HDF5 alias for blosc */
|
||||
NULL, /*NCZ_blosc_codec_initialize*/
|
||||
NCZ_blosc_codec_finalize,
|
||||
NCZ_blosc_codec_to_hdf5,
|
||||
NCZ_blosc_hdf5_to_codec,
|
||||
NCZ_blosc_modify_parameters,
|
||||
};
|
||||
|
||||
/* NCZarr Interface Functions */
|
||||
|
||||
/* Create the true parameter set:
|
||||
Visible parameters:
|
||||
param[0] -- reserved
|
||||
param[1] -- reserved
|
||||
param[2] -- reserved
|
||||
param[3] -- variable chunksize in bytes | 0 (=>default)
|
||||
param[4] -- compression level
|
||||
param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
|
||||
param[6] -- compressor index
|
||||
|
||||
Working parameters:
|
||||
param[0] -- filter revision
|
||||
param[1] -- blosc version
|
||||
param[2] -- variable type size in bytes
|
||||
param[3] -- variable chunksize in bytes
|
||||
param[4] -- compression level
|
||||
param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
|
||||
param[6] -- compressor index
|
||||
*/
|
||||
|
||||
void blosc_destroy(void);
|
||||
|
||||
static int ncz_blosc_initialized = 0;
|
||||
|
||||
static void
|
||||
NCZ_blosc_codec_finalize(void)
|
||||
{
|
||||
if(ncz_blosc_initialized) {
|
||||
blosc_destroy();
|
||||
ncz_blosc_initialized = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
|
||||
{
|
||||
int i,stat = NC_NOERR;
|
||||
nc_type vtype;
|
||||
int storage, ndims;
|
||||
size_t* chunklens = NULL;
|
||||
size_t typesize, chunksize;
|
||||
char vname[NC_MAX_NAME+1];
|
||||
unsigned* params = NULL;
|
||||
size_t nparams;
|
||||
size_t vnparams = *vnparamsp;
|
||||
unsigned* vparams = *vparamsp;
|
||||
|
||||
if(vnparams < 7)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
nparams = 7;
|
||||
|
||||
if(vparams == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
|
||||
if(wnparamsp == NULL || wparamsp == NULL)
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
|
||||
vnparams = *vnparamsp;
|
||||
vparams = *vparamsp;
|
||||
|
||||
/* Get variable info */
|
||||
if((stat = nc_inq_var(ncid,varid,vname,&vtype,&ndims,NULL,NULL))) goto done;
|
||||
|
||||
if(ndims == 0) {stat = NC_EFILTER; goto done;}
|
||||
|
||||
/* Get the typesize */
|
||||
if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
|
||||
|
||||
/* Compute chunksize */
|
||||
if((chunklens = (size_t*)calloc(ndims,sizeof(size_t)))==NULL) goto done;
|
||||
if((stat = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
|
||||
if(storage != NC_CHUNKED) {stat = NC_EFILTER; goto done;}
|
||||
chunksize = typesize;
|
||||
for(i=0;i<ndims;i++) chunksize *= chunklens[i];
|
||||
|
||||
if((params = (unsigned*)malloc(vnparams*sizeof(unsigned)))==NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
memcpy(params,vparams,vnparams*sizeof(unsigned));
|
||||
|
||||
params[0] = FILTER_BLOSC_VERSION;
|
||||
params[1] = BLOSC_VERSION_FORMAT;
|
||||
params[2] = (unsigned)typesize;
|
||||
|
||||
params[3] = chunksize;
|
||||
params[4] = params[4];
|
||||
params[5] = params[5];
|
||||
params[6] = params[6];
|
||||
|
||||
*wnparamsp = nparams;
|
||||
nullfree(*wparamsp);
|
||||
*wparamsp = params; params = NULL;
|
||||
|
||||
done:
|
||||
nullfree(chunklens);
|
||||
nullfree(params);
|
||||
FUNC_LEAVE_NOAPI(stat)
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_blosc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCjson* jcodec = NULL;
|
||||
NCjson* jtmp = NULL;
|
||||
unsigned* params = NULL;
|
||||
struct NCJconst jc = {0,0,0,NULL};
|
||||
int compcode;
|
||||
|
||||
/* parse the JSON */
|
||||
if(NCJparse(codec_json,0,&jcodec)) {stat = NC_EFILTER; goto done;}
|
||||
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
|
||||
/* Verify the codec ID */
|
||||
if(NCJdictget(jcodec,"id",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;}
|
||||
if(strcmp(NCJstring(jtmp),NCZ_blosc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
|
||||
|
||||
if((params = (unsigned*)calloc(7,sizeof(unsigned)))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
|
||||
/* Get compression level*/
|
||||
if(NCJdictget(jcodec,"clevel",&jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(jtmp) {
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
|
||||
} else
|
||||
jc.ival = DEFAULT_LEVEL;
|
||||
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
|
||||
params[4] = (unsigned)jc.ival;
|
||||
|
||||
/* Get blocksize */
|
||||
if(NCJdictget(jcodec,"blocksize",&jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(jtmp) {
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
|
||||
} else
|
||||
jc.ival = DEFAULT_BLOCKSIZE;
|
||||
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
|
||||
params[3] = (unsigned)jc.ival;
|
||||
|
||||
/* Get shuffle */
|
||||
if(NCJdictget(jcodec,"shuffle",&jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(jtmp) {
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
|
||||
} else
|
||||
jc.ival = BLOSC_NOSHUFFLE;
|
||||
params[5] = (unsigned)jc.ival;
|
||||
|
||||
/* Get compname */
|
||||
if(NCJdictget(jcodec,"cname",&jtmp)) {stat = NC_EFILTER; goto done;}
|
||||
if(jtmp) {
|
||||
if(NCJcvt(jtmp,NCJ_STRING,&jc)) {stat = NC_EFILTER; goto done;}
|
||||
if(jc.sval == NULL || strlen(jc.sval) == 0) {stat = NC_EFILTER; goto done;}
|
||||
if((compcode = blosc_compname_to_compcode(jc.sval)) < 0) {stat = NC_EFILTER; goto done;}
|
||||
} else
|
||||
compcode = DEFAULT_COMPCODE;
|
||||
params[6] = (unsigned)compcode;
|
||||
|
||||
if(nparamsp) *nparamsp = 7;
|
||||
if(paramsp) {*paramsp = params; params = NULL;}
|
||||
|
||||
done:
|
||||
if(jc.sval) {
|
||||
free(jc.sval);
|
||||
}
|
||||
if(params) {
|
||||
free(params);
|
||||
}
|
||||
NCJreclaim(jcodec);
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
char json[1024];
|
||||
const char* compname = NULL;
|
||||
|
||||
if(nparams == 0 || params == NULL)
|
||||
{stat = NC_EINVAL; goto done;}
|
||||
|
||||
/* Get the sub-compressor name */
|
||||
if(blosc_compcode_to_compname((int)params[6],&compname) < 0) {stat = NC_EFILTER; goto done;}
|
||||
|
||||
snprintf(json,sizeof(json),
|
||||
"{\"id\": \"blosc\",\"clevel\": %u,\"blocksize\": %u,\"cname\": \"%s\",\"shuffle\": %d}",
|
||||
params[4],params[3],compname,params[5]);
|
||||
|
||||
if(codecp) {
|
||||
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
}
|
||||
|
||||
done:
|
||||
return stat;
|
||||
}
|
||||
#endif
|
||||
|
||||
/**************************************************/
|
||||
|
||||
NCZ_codec_t* NCZ_stdfilters_codecs[] = {
|
||||
#ifdef HAVE_BZ2
|
||||
&NCZ_bzip2_codec,
|
||||
#endif
|
||||
#ifdef HAVE_ZSTD
|
||||
&NCZ_zstd_codec,
|
||||
#endif
|
||||
#ifdef HAVE_BLOSC
|
||||
&NCZ_blosc_codec,
|
||||
#endif
|
||||
NULL
|
||||
};
|
||||
|
||||
/* External Export API */
|
||||
DLLEXPORT
|
||||
const void*
|
||||
NCZ_codec_info_defaults(void)
|
||||
{
|
||||
return (void*)&NCZ_stdfilters_codecs;
|
||||
}
|
||||
|
||||
44
plugins/stdinstall.in
Executable file
44
plugins/stdinstall.in
Executable file
@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
set -x
|
||||
|
||||
INSTALLDIR=@PLUGIN_INSTALL_DIR@
|
||||
|
||||
# Load the findplugins function
|
||||
. ${builddir}/findplugin.sh
|
||||
echo "findplugin.sh loaded"
|
||||
|
||||
installplugin() {
|
||||
PLUG="$1"
|
||||
# Locate the plugin path and the library name; argument order is critical
|
||||
findplugin ${PLUG}
|
||||
if ! test -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" ; then
|
||||
echo "Not exists: ${HDF5_PLUGIN_DIR}/$HDF5_PLUGIN_LIB ; ignored"
|
||||
return
|
||||
fi
|
||||
if ! test -d "${INSTALLDIR}" ; then
|
||||
echo "Not exists: ${INSTALLDIR} ; creating"
|
||||
mkdir ${INSTALLDIR}
|
||||
fi
|
||||
echo "Installing: $HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB into $INSTALLDIR"
|
||||
cp -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" $INSTALLDIR
|
||||
}
|
||||
|
||||
if test "x$USEPLUGINS" != x ; then
|
||||
if test "x$INSTALLDIR" != x ; then
|
||||
installplugin h5bzip2
|
||||
installplugin h5zstd
|
||||
installplugin h5blosc
|
||||
if test "x$FEATURE_NCZARR" ; then
|
||||
installplugin h5fletcher32
|
||||
installplugin h5shuffle
|
||||
installplugin h5deflate
|
||||
installplugin h5szip
|
||||
installplugin nczdefaults
|
||||
installplugin nczszip
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
@ -28,9 +28,11 @@ FEATURE_PARALLEL=@HAS_PARALLEL@
|
||||
|
||||
# Define selected features of the build
|
||||
FEATURE_HDF5=@HAS_HDF5@
|
||||
FEATURE_NCZARR=@HAS_NCZARR@
|
||||
FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@
|
||||
FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@
|
||||
FEATURE_FILTERTESTS=@DO_FILTER_TESTS@
|
||||
FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@
|
||||
|
||||
set -e
|
||||
|
||||
|
||||
@ -108,12 +108,12 @@ static Test PATHTESTS[] = {
|
||||
char* macros[128];
|
||||
|
||||
/*Forward */
|
||||
#ifdef DEBUG
|
||||
static const char* kind2string(int kind);
|
||||
#endif
|
||||
static char* expand(const char* s);
|
||||
static void setmacros(void);
|
||||
static void reclaimmacros(void);
|
||||
#ifdef DEBUG
|
||||
static const char* kind2string(int kind);
|
||||
#endif
|
||||
|
||||
int
|
||||
main(int argc, char** argv)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user