From 126b3f9423648b80ed4b89f8f2c7418bec06ee73 Mon Sep 17 00:00:00 2001
From: Dennis Heimbigner <dmh@ucar.edu>
Date: Fri, 29 Apr 2022 14:31:55 -0600
Subject: [PATCH] Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
---
.github/workflows/run_tests_osx.yml | 2 +-
.github/workflows/run_tests_ubuntu.yml | 2 +-
.github/workflows/run_tests_win_mingw.yml | 2 +-
CMakeLists.txt | 45 +-
cmake/modules/FindBzip2.cmake | 64 +++
config.h.cmake.in | 2 +-
configure.ac | 57 ++-
docs/filters.md | 32 +-
examples/C/Makefile.am | 2 +
examples/C/run_filter.sh | 13 +-
include/netcdf_filter_build.h | 44 +-
libdispatch/dutil.c | 8 +-
libnczarr/zfilter.c | 32 +-
libnczarr/zfilter.h | 10 -
libnetcdf.settings.in | 3 +-
nc_test4/Makefile.am | 8 +-
nc_test4/findplugin.in | 32 +-
nc_test4/tst_filter.sh | 29 +-
nc_test4/tst_filterinstall.sh | 111 +++++
nc_test4/tst_specific_filters.sh | 15 +-
ncdap_test/Makefile.am | 2 +-
nczarr_test/Makefile.am | 11 +-
nczarr_test/run_filter.sh | 14 +-
nczarr_test/test_nczarr.sh | 6 +-
plugins/CMakeLists.txt | 67 ++-
plugins/H5Zblosc.c | 240 +---------
plugins/H5Zblosc.h | 16 +-
plugins/H5Zbzip2.c | 89 ----
plugins/H5Zzstd.c | 89 ----
plugins/Makefile.am | 59 ++-
plugins/{NCZdefaults.c => NCZhdf5filters.c} | 225 ++++++++++
plugins/NCZstdfilters.c | 460 ++++++++++++++++++++
plugins/stdinstall.in | 44 ++
test_common.in | 2 +
unit_test/test_pathcvt.c | 6 +-
35 files changed, 1234 insertions(+), 609 deletions(-)
create mode 100644 cmake/modules/FindBzip2.cmake
create mode 100755 nc_test4/tst_filterinstall.sh
rename plugins/{NCZdefaults.c => NCZhdf5filters.c} (53%)
create mode 100644 plugins/NCZstdfilters.c
create mode 100755 plugins/stdinstall.in
diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml
index 8e9d65a6f..aa35ae879 100644
--- a/.github/workflows/run_tests_osx.yml
+++ b/.github/workflows/run_tests_osx.yml
@@ -7,7 +7,7 @@
name: Run macOS-based netCDF Tests
-on: [pull_request]
+on: [pull_request,push]
jobs:
diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml
index 927ea68e4..826ac4358 100644
--- a/.github/workflows/run_tests_ubuntu.yml
+++ b/.github/workflows/run_tests_ubuntu.yml
@@ -4,7 +4,7 @@
name: Run Ubuntu/Linux netCDF Tests
-on: [ pull_request ]
+on: [pull_request,push]
jobs:
diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml
index 913920a3a..aaf999e6a 100644
--- a/.github/workflows/run_tests_win_mingw.yml
+++ b/.github/workflows/run_tests_win_mingw.yml
@@ -7,7 +7,7 @@
name: Run MSYS2, MinGW64-based Tests
-on: [pull_request]
+on: [pull_request,push]
jobs:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e89e351c5..191b988b8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1114,9 +1114,12 @@ string(TOLOWER "${filter}" downfilter)
IF(${filter}_FOUND)
INCLUDE_DIRECTORIES(${filter}_INCLUDE_DIRS})
SET(ENABLE_${upfilter} TRUE)
- SET(STD_FILTERS "${STD_FILTERS},${downfilter}")
+ SET(HAVE_${upfilter} ON)
+ SET(STD_FILTERS "${STD_FILTERS} ${downfilter}")
+ MESSAGE(">>> Standard Filter: ${downfilter}")
ELSE()
SET(ENABLE_${upfilter} FALSE)
+ SET(HAVE_${upfilter} OFF)
ENDIF()
endmacro(set_std_filter)
@@ -1125,20 +1128,36 @@ FIND_PACKAGE(Bz2)
FIND_PACKAGE(Blosc)
FIND_PACKAGE(Zstd)
-IF(Zstd_FOUND)
- SET(HAVE_ZSTD yes)
-ELSE()
- SET(HAVE_ZSTD no)
-ENDIF()
-
# Accumulate standard filters
-set(STD_FILTERS "deflate") # Always have deflate */
+set(STD_FILTERS "deflate") # Always have deflate*/
set_std_filter(SZIP)
set_std_filter(Blosc)
set_std_filter(Zstd)
-set_std_filter(Bz2)
-IF(NOT Bz2_FOUND)
-set(STD_FILTERS "${STD_FILTERS},bzip2") # Always have bzip2 */
+IF(Bz2_FOUND)
+ set_std_filter(Bz2)
+ELSE()
+ # The reason we use a local version is to support a more comples test case
+ MESSAGE(WARNING "libbz2 not found using built-in version")
+ SET(HAVE_LOCAL_BZ2 ON)
+ SET(HAVE_BZ2 ON)
+ set(STD_FILTERS "${STD_FILTERS} bz2")
+ENDIF()
+
+# If user wants, then install selected plugins
+SET(PLUGIN_INSTALL_DIR "" CACHE STRING "Whether and where we should install plugins")
+SET(ENABLE_PLUGIN_INSTALL OFF)
+if(DEFINED PLUGIN_INSTALL_DIR OR DEFINED CACHE{PLUGIN_INSTALL_DIR})
+ IF(PLUGIN_INSTALL_DIR STREQUAL "")
+ MESSAGE(WARNING "No plugin directory value specified; option ignored.")
+ UNSET(PLUGIN_INSTALL_DIR)
+ UNSET(PLUGIN_INSTALL_DIR CACHE)
+ SET(PLUGIN_INSTALL_DIR_SETTING "N.A.")
+ ELSE()
+ SET(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}")
+ SET(ENABLE_PLUGIN_INSTALL ON)
+ ENDIF()
+ELSE()
+ SET(PLUGIN_INSTALL_DIR_SETTING "N.A.")
ENDIF()
# See if we have libzip
@@ -2528,6 +2547,10 @@ IF(ENABLE_NCZARR)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nczarr_test/findplugin.sh @ONLY NEWLINE_STYLE LF)
ENDIF()
+IF(ENABLE_PLUGINS)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/plugins/findplugin.sh @ONLY NEWLINE_STYLE LF)
+ENDIF()
+
IF(ENABLE_EXAMPLES)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF)
ENDIF()
diff --git a/cmake/modules/FindBzip2.cmake b/cmake/modules/FindBzip2.cmake
new file mode 100644
index 000000000..9b6c9471d
--- /dev/null
+++ b/cmake/modules/FindBzip2.cmake
@@ -0,0 +1,64 @@
+# Searches for an installation of the bzip2 library. On success, it sets the following variables:
+#
+# Bzip2_FOUND Set to true to indicate the bzip2 library was found
+# Bzip2_INCLUDE_DIRS The directory containing the header file bzip2/bzip2.h
+# Bzip2_LIBRARIES The libraries needed to use the bzip2 library
+#
+# To specify an additional directory to search, set Bzip2_ROOT.
+#
+# Author: Siddhartha Chaudhuri, 2009
+#
+
+# Look for the header, first in the user-specified location and then in the system locations
+SET(Bzip2_INCLUDE_DOC "The directory containing the header file bzip2.h")
+FIND_PATH(Bzip2_INCLUDE_DIRS NAMES bzip2.h bzip2/bzip2.h PATHS ${Bzip2_ROOT} ${Bzip2_ROOT}/include DOC ${Bzip2_INCLUDE_DOC} NO_DEFAULT_PATH)
+IF(NOT Bzip2_INCLUDE_DIRS) # now look in system locations
+ FIND_PATH(Bzip2_INCLUDE_DIRS NAMES bzlib.h DOC ${Bzip2_INCLUDE_DOC})
+ENDIF(NOT Bzip2_INCLUDE_DIRS)
+
+SET(Bzip2_FOUND FALSE)
+
+IF(Bzip2_INCLUDE_DIRS)
+ SET(Bzip2_LIBRARY_DIRS ${Bzip2_INCLUDE_DIRS})
+
+ IF("${Bzip2_LIBRARY_DIRS}" MATCHES "/include$")
+ # Strip off the trailing "/include" in the path.
+ GET_FILENAME_COMPONENT(Bzip2_LIBRARY_DIRS ${Bzip2_LIBRARY_DIRS} PATH)
+ ENDIF("${Bzip2_LIBRARY_DIRS}" MATCHES "/include$")
+
+ IF(EXISTS "${Bzip2_LIBRARY_DIRS}/lib")
+ SET(Bzip2_LIBRARY_DIRS ${Bzip2_LIBRARY_DIRS}/lib)
+ ENDIF(EXISTS "${Bzip2_LIBRARY_DIRS}/lib")
+
+ # Find Bzip2 libraries
+ FIND_LIBRARY(Bzip2_DEBUG_LIBRARY NAMES bzip2d bzip2_d libbzip2d libbzip2_d libbzip2
+ PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug
+ PATHS ${Bzip2_LIBRARY_DIRS} NO_DEFAULT_PATH)
+ FIND_LIBRARY(Bzip2_RELEASE_LIBRARY NAMES bzip2 libbzip2
+ PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release
+ PATHS ${Bzip2_LIBRARY_DIRS} NO_DEFAULT_PATH)
+
+ SET(Bzip2_LIBRARIES )
+ IF(Bzip2_DEBUG_LIBRARY AND Bzip2_RELEASE_LIBRARY)
+ SET(Bzip2_LIBRARIES debug ${Bzip2_DEBUG_LIBRARY} optimized ${Bzip2_RELEASE_LIBRARY})
+ ELSEIF(Bzip2_DEBUG_LIBRARY)
+ SET(Bzip2_LIBRARIES ${Bzip2_DEBUG_LIBRARY})
+ ELSEIF(Bzip2_RELEASE_LIBRARY)
+ SET(Bzip2_LIBRARIES ${Bzip2_RELEASE_LIBRARY})
+ ENDIF(Bzip2_DEBUG_LIBRARY AND Bzip2_RELEASE_LIBRARY)
+
+ IF(Bzip2_LIBRARIES)
+ SET(Bzip2_FOUND TRUE)
+ ENDIF(Bzip2_LIBRARIES)
+ENDIF(Bzip2_INCLUDE_DIRS)
+
+IF(Bzip2_FOUND)
+# IF(NOT Bzip2_FIND_QUIETLY)
+ MESSAGE(STATUS "Found Bzip2: headers at ${Bzip2_INCLUDE_DIRS}, libraries at ${Bzip2_LIBRARY_DIRS}")
+ MESSAGE(STATUS " library is ${Bzip2_LIBRARIES}")
+# ENDIF(NOT Bzip2_FIND_QUIETLY)
+ELSE(Bzip2_FOUND)
+ IF(Bzip2_FIND_REQUIRED)
+ MESSAGE(FATAL_ERROR "Bzip2 library not found")
+ ENDIF(Bzip2_FIND_REQUIRED)
+ENDIF(Bzip2_FOUND)
diff --git a/config.h.cmake.in b/config.h.cmake.in
index b77c1bab2..97004f62c 100644
--- a/config.h.cmake.in
+++ b/config.h.cmake.in
@@ -173,7 +173,7 @@ are set when opening a binary file on Windows. */
#cmakedefine HAVE_ATEXIT 1
/* Define to 1 if bzip2 library available. */
-#cmakedefine HAVE_BZIP2 1
+#cmakedefine HAVE_BZ2 1
/* Define to 1 if zstd library available. */
#cmakedefine HAVE_ZSTD 1
diff --git a/configure.ac b/configure.ac
index eb1abf660..74ae16734 100644
--- a/configure.ac
+++ b/configure.ac
@@ -667,24 +667,21 @@ fi
AC_MSG_CHECKING([whether libzstd library is available])
AC_MSG_RESULT([${have_zstd}])
-# See if we have libbzip2 or libbz2
-AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no])
-if test "x$have_bzip2" = "xyes" ; then
+# See if we have libbz2
+AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bz2=yes],[have_bz2=no])
+if test "x$have_bz2" = "xyes" ; then
AC_SEARCH_LIBS([BZ2_bzCompress],[bz2 bz2.dll cygbz2.dll], [], [])
- AC_DEFINE([HAVE_BZIP2], [1], [if true, bz2 library is installed])
+ AC_DEFINE([HAVE_BZ2], [1], [if true, bz2 library is installed])
fi
AC_MSG_CHECKING([whether libbz2 library is available])
-AC_MSG_RESULT([${have_bzip2}])
+AC_MSG_RESULT([${have_bz2}])
-if test "x$have_bzip2" = "xno" ; then
-AC_CHECK_LIB([bzip2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no])
-if test "x$have_bzip2" = "xyes" ; then
- AC_SEARCH_LIBS([BZ2_bzCompress],[bzip2 bzip2.dll cygbzip2.dll], [], [])
- AC_DEFINE([HAVE_BZIP2], [1], [if true, bzip2 library is installed])
-fi
-AC_MSG_CHECKING([whether libbzip2 library is available])
-AC_MSG_RESULT([${have_bzip2}])
+if test "x$have_bz2" = "xno" ; then
+ have_local_bz2=yes
+else
+ have_local_bz2=no
fi
+AM_CONDITIONAL(HAVE_LOCAL_BZ2, [test "x$have_local_bz2" = xyes])
# Note that szip management is tricky.
# This is because we have three things to consider:
@@ -1753,7 +1750,7 @@ AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes])
AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes])
AM_CONDITIONAL(HAVE_BLOSC, [test "x$have_blosc" = xyes])
-AM_CONDITIONAL(HAVE_BZIP2, [test "x$have_bzip2" = xyes])
+AM_CONDITIONAL(HAVE_BZ2, [test "x$have_bz2" = xyes])
AM_CONDITIONAL(HAVE_ZSTD, [test "x$have_zstd" = xyes])
# If the machine doesn't have a long long, and we want netCDF-4, then
@@ -1864,26 +1861,45 @@ AC_SUBST(HAS_SZLIB_WRITE, [$have_sz])
AC_SUBST(HAS_ZSTD,[$have_zstd])
# Always available
-std_filters="deflate,bzip2"
+std_filters="deflate bz2"
if test "x$enable_szlib" = xyes ; then
-std_filters="${std_filters},szip"
+std_filters="${std_filters} szip"
fi
# We need to include szip iff HDF5 && HDF5_HAS_SZIP || !HDF5 && NCZARR && libsz
if test "x$enable_hdf5" = xyes && test "x$enable_hdf5_szip" = xyes ; then
- std_filters="${std_filters},szip"
+ std_filters="${std_filters} szip"
fi
if test "x$enable_hdf5" = xno && test "x$have_sz" = xyes ; then
-std_filters="${std_filters},szip"
+std_filters="${std_filters} szip"
fi
if test "x$have_blosc" = xyes ; then
-std_filters="${std_filters},blosc"
+std_filters="${std_filters} blosc"
fi
if test "x$have_zstd" = xyes ; then
-std_filters="${std_filters},zst"
+std_filters="${std_filters} zstd"
fi
AC_SUBST(STD_FILTERS,[$std_filters])
+# If user wants, then install selected standard filters
+AC_MSG_CHECKING([whether and where we should install plugins])
+AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=<absolute directory>],
+ [Install selected standard filters in specified directory])])
+AC_MSG_RESULT([$with_plugin_dir])
+if test "x$with_plugin_dir" = x ; then
+ AC_MSG_WARN([No plugin directory value specified; option ignored.])
+ with_plugin_dir=
+ with_plugin_dir_setting="N.A."
+ enable_plugin_dir=no
+else
+ with_plugin_dir_setting="$with_plugin_dir"
+ enable_plugin_dir=yes
+fi
+AM_CONDITIONAL([ENABLE_PLUGIN_DIR], [test "x$enable_plugin_dir" = xyes])
+AC_SUBST([PLUGIN_INSTALL_DIR], [$with_plugin_dir])
+# Better value for libnetcdf.settings
+AC_SUBST([PLUGIN_INSTALL_DIR_SETTING], [$with_plugin_dir_setting])
+
# Access netcdf specific version of config.h
AH_BOTTOM([#include "ncconfigure.h"])
@@ -1973,6 +1989,7 @@ AC_MSG_NOTICE([generating header files and makefiles])
AC_CONFIG_FILES(test_common.sh:test_common.in)
AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in)
+AC_CONFIG_FILES(plugins/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in)
AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)
diff --git a/docs/filters.md b/docs/filters.md
index 303a2644d..e0335151f 100644
--- a/docs/filters.md
+++ b/docs/filters.md
@@ -932,28 +932,34 @@ This is for internal use only.
# Appendix F. Pre-built Filters
-As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory.
-They may be in that directory or the "plugins/.libs" subdirectory.
-It may be possible for users to utilize some of those libraries to provide filter support for general use.
-One simple way to reuse these libraries is to make the environment variable "HDF5_PLUGIN_PATH" refer to the plugin directory or the subdirectory, whichever contains the shared libraries.
-This may not be possible if the user already has other filter shared libraries in use.
+As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory
+— in that directory or the "plugins/.libs" subdirectory.
-The second way is to copy the necessary shared libraries from the plugins directory to the user's HDF5_PLUGIN_PATH directory.
-If the user is using HDF5, then the following filters are probably usable:
+An option exists to allow some of those filters to be installed into a user-specified directory.
+Presumably this directory is part of the value of the HDF5_PLUGIN_PATH environment variable.
+The ./configure option is ````--with-plugin-dir=<absolute-directory-path>````.
+The corresponding CMake option is ````-DPLUGIN_INSTALL_DIR=<absolute-directory-path>````.
-* ''libh5bzip2.so'' -- an HDF5 filter for bzip2 compression
-* ''libh5blosc.so'' -- an HDF5 filter for blosc compression
+If this option is specified, then as part of the "install" build action,
+a specified set of filter shared libraries will be copied into the specified directory.
+Any existing library of the same name will be overwritten. If the specified directory
+itself does not exist, then it will be created.
-If the user is using NCZarr filters, then you can install the following additional shared libraries:
+Currently, if the following filters are available, they will be installed;
+* ''libh5bzip2.so'' -- an HDF5 filter wrapper for bzip2 compression
+* ''libh5blosc.so'' -- an HDF5 filter wrapper for blosc compression
+* ''libh5zstd.so'' -- an HDF5 filter wrapper for zstandardcompression
+If the user is using NCZarr filters, then the following additional filters will be installed.
* libh5shuffle.so -- shuffle filter
* libh5fletcher32.so -- fletcher32 checksum
* libh5deflate.so -- deflate compression
+* libh5szip.so -- szip compression
* libnczdefaults.so -- provide NCZarr support for shuffle, fletcher32, and deflate.
+* libnczszip.so -- provide NCZarr support for szip.
-The shuffle, fletcher32, and deflate filters in this case will be ignored by HDF5 and only used by the NCZarr code.
-But in order to use them, it needs additional Codec capabilities provided by the libnczdefauts.so shared library.
-Note also that if you disable HDF5 support, but leave NCZarr support enabled, then all of the above filters
+The shuffle, fletcher32, deflate, and szip filters in this case will be ignored by HDF5 and only used by the NCZarr code.
+Note that if you disable HDF5 support, but leave NCZarr support enabled, then all of the above filters
should continue to work.
# Point of Contact {#filters_poc}
diff --git a/examples/C/Makefile.am b/examples/C/Makefile.am
index e9f087ada..34dbfbec4 100644
--- a/examples/C/Makefile.am
+++ b/examples/C/Makefile.am
@@ -7,7 +7,9 @@
# Un comment to use a more verbose test driver
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+#TESTS_ENVIRONMENT = export SETX=1;
# Put together AM_CPPFLAGS and AM_LDFLAGS.
include $(top_srcdir)/lib_flags.am
diff --git a/examples/C/run_filter.sh b/examples/C/run_filter.sh
index 37dc8f806..90e0935fe 100755
--- a/examples/C/run_filter.sh
+++ b/examples/C/run_filter.sh
@@ -10,26 +10,25 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../../test_common.sh
echo "*** Running filter example for netCDF-4."
+
set -e
-if test -f ${builddir}/findplugin.sh ; then
echo "*** running test_filter example..."
. ${builddir}/findplugin.sh
# Locate the plugin path and the library names; argument order is critical
# Find bzip2 and capture
findplugin h5bzip2
-BZIP2PATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
+BZIP2PATH="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
+
# Verify
if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
-export HDF5_PLUGIN_PATH
-echo "*** running filter_example..."
rm -f ./bzip2.nc
+export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
+echo "*** running filter_example..."
${execdir}/filter_example
-#rm -f ./bzip2.nc
-
-fi # Filter enabled
+rm -f ./bzip2.nc
echo "*** Filter example successful!"
exit 0
diff --git a/include/netcdf_filter_build.h b/include/netcdf_filter_build.h
index 6c6f8e948..11c80f809 100644
--- a/include/netcdf_filter_build.h
+++ b/include/netcdf_filter_build.h
@@ -34,17 +34,46 @@
/* Defined flags for filter invocation (not stored); powers of two */
#define NCZ_FILTER_DECODE 0x00000001
-/* External Discovery Function */
+/* External Discovery Functions */
/*
Obtain a pointer to an instance of NCZ_codec_class_t.
+NCZ_get_codec_info(void) -- returns pointer to instance of NCZ_codec_class_t.
+ Instance an be recast based on version+sort to the plugin type specific info.
+So the void* return value is typically actually of type NCZ_codec_class_t*.
+
+Signature: typedef const void* (*NCZ_get_codec_info_proto)(void);
+
+The current object returned by NCZ_get_codec_info is a
+ pointer to an instance of NCZ_codec_t.
+
+The key to this struct are the several function pointers that do
+initialize/finalize and conversion between codec JSON and HDF5
+parameters. The function pointers defined in NCZ_codec_t
+manipulate HDF5 parameters and NumCodec JSON.
+
+Obtain a pointer to an instance of NCZ_codec_class_t.
+
NCZ_get_codec_info(void) -- returns pointer to instance of NCZ_codec_class_t.
Instance an be recast based on version+sort to the plugin type specific info.
So the void* return value is typically actually of type NCZ_codec_class_t*.
*/
typedef const void* (*NCZ_get_codec_info_proto)(void);
+/*
+Obtain a pointer to a NULL terminated vector of NCZ_codec_class_t*.
+
+NCZ_codec_info_defaults(void) -- returns pointer to a vector of pointers to instances of NCZ_codec_class_t. The vector is NULL terminated.
+So the void* return value is typically actually of type NCZ_codec_class_t**.
+
+Signature: typedef const void* (*NCZ_codec_info_defaults_proto)(void);
+
+This entry point is used to return the codec information for
+multiple filters that otherwise do not have codec information defined.
+*/
+typedef const void* (*NCZ_codec_info_defaults_proto)(void);
+
/* The current object returned by NCZ_get_plugin_info is a
pointer to an instance of NCZ_codec_t.
@@ -86,7 +115,7 @@ int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp);
* Convert a set of visible parameters to a set of working parameters using extra environmental information.
Also allows for changes to the visible parameters. Invoked before filter is actually used.
-int (*NCZ_build_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
+int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
@param ncid -- (in) ncid of the variable's group
@param varid -- (in) varid of the variable
@@ -96,8 +125,15 @@ int (*NCZ_build_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** v
@params wparamsp -- (out) vector of working parameters
@return -- a netcdf-c error code.
-* Convert a set of working parameters to a set of visible parameters using extra environmental information, if needed.
-Invoked before filter metadata is written.
+* Convert an HDF5 vector of visible parameters to a JSON representation.
+
+int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp);
+
+@param nparams -- (in) the length of the HDF5 unsigned vector
+@param params -- (in) pointer to the HDF5 unsigned vector.
+@param codecp -- (out) store the string representation of the codec; caller must free.
+@return -- a netcdf-c error code.
+
*/
/*
diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c
index b8f48e17c..d4b37a344 100644
--- a/libdispatch/dutil.c
+++ b/libdispatch/dutil.c
@@ -211,11 +211,11 @@ NC_mktmp(const char* base)
int fd = -1;
char* tmp = NULL;
size_t len;
-#define MAXTRIES 4
-#ifdef HAVE_MKSTEMP
- mode_t mask;
-#else
+#ifndef HAVE_MKSTEMP
int tries;
+#define MAXTRIES 4
+#else
+ mode_t mask;
#endif
len = strlen(base)+6+1;
diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c
index 58bb0984d..963a97718 100644
--- a/libnczarr/zfilter.c
+++ b/libnczarr/zfilter.c
@@ -52,9 +52,11 @@
#include "netcdf_filter_build.h"
#include "netcdf_aux.h"
-#undef DEBUG
-#undef DEBUGF
-#undef DEBUGL
+#if 0
+#define DEBUG
+#define DEBUGF
+#define DEBUGL
+#endif
/* If set, then triage potential shared libraries based on extension */
#define NAMEOPT
@@ -177,7 +179,7 @@ NCJtrace(const NCjson* j)
const char*
printplugin(const NCZ_Plugin* plugin)
{
- char* plbuf = malloc(4096);
+ static char plbuf[4096];
char plbuf2[2000];
char plbuf1[2000];
@@ -194,8 +196,7 @@ printplugin(const NCZ_Plugin* plugin)
static char*
printparams(size_t nparams, const unsigned* params)
{
- char* ppbuf = malloc(4096);
-
+ static char ppbuf[4096];
if(nparams == 0)
snprintf(ppbuf,4096,"{0,%p}",params);
else
@@ -1378,20 +1379,27 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
/* See what we have */
{
- H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type");
- H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info");
- NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info");
- NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults");
+ const H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type");
+ const H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info");
+ const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info");
+ const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults");
if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL)
{stat = NC_ENOFILTER; goto done;}
if(cpd != NULL) {
/* Deal with defaults first */
- NCZ_codec_t** cp = NULL;
+ const NCZ_codec_t** cp = NULL;
nclistpush(default_libs,lib);
- for(cp=cpd();*cp;cp++) {
+ cp = (const NCZ_codec_t**)cpd();
+#ifdef DEBUGL
+ fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp);
+#endif
+ for(;*cp;cp++) {
struct CodecAPI* c0;
+#ifdef DEBUGL
+ fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id);
+#endif
c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI));
if(c0 == NULL) {stat = NC_ENOMEM; goto done1;}
c0->codec = *cp;
diff --git a/libnczarr/zfilter.h b/libnczarr/zfilter.h
index 4aa7de80a..38d8b0c3d 100644
--- a/libnczarr/zfilter.h
+++ b/libnczarr/zfilter.h
@@ -25,16 +25,6 @@
#define plugin_dir_win "%s/hdf5/lib/plugin"
#define win32_root_env "ALLUSERSPROFILE"
-/*
-Return a NULL terminated vector of pointers to instances of ''NCZ_codec_t''.
-The value returned is actually of type ''NCZ_codec_t**'',
-but is of type ''void*'' to allow for extensions.
-The list of returned items are used to try to provide defaults
-for any HDF5 filters that have no corresponding Codec.
-This is for internal use only.
-*/
-typedef void* (*NCZ_codec_info_defaults_proto)(void);
-
/* Opaque */
struct NCZ_Filter;
diff --git a/libnetcdf.settings.in b/libnetcdf.settings.in
index d9cc1af96..6dea12d5c 100644
--- a/libnetcdf.settings.in
+++ b/libnetcdf.settings.in
@@ -9,6 +9,7 @@ Configured On: @CONFIG_DATE@
Host System: @host_cpu@-@host_vendor@-@host_os@
Build Directory: @abs_top_builddir@
Install Prefix: @prefix@
+Plugin Install Prefix: @PLUGIN_INSTALL_DIR_SETTING@
# Compiling Options
-----------------
@@ -49,4 +50,4 @@ Logging: @HAS_LOGGING@
SZIP Write Support: @HAS_SZLIB_WRITE@
Standard Filters: @STD_FILTERS@
ZSTD Support: @HAS_ZSTD@
-Benchmarks: @HAS_BENCHMARKS@
\ No newline at end of file
+Benchmarks: @HAS_BENCHMARKS@
diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am
index 8ce7e97de..9524339f3 100644
--- a/nc_test4/Makefile.am
+++ b/nc_test4/Makefile.am
@@ -109,9 +109,15 @@ ref_filteredvv.cdl ref_multi.cdl \
ref_ncgenF.cdl ref_nccopyF.cdl \
ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
ref_filter_order_create.txt ref_filter_order_read.txt \
-ref_any.cdl tst_specific_filters.sh \
+ref_any.cdl tst_specific_filters.sh \
tst_virtual_datasets.c
+# The tst_filterinstall test can only be run after an install
+# occurred with --with-plugin-dir enabled. So there is no point
+#in running it via make check. It is kept here so it can be
+# manually invoked if desired
+EXTRA_DIST += tst_filterinstall.sh
+
CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt \
tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl tst_elena_*.cdl \
diff --git a/nc_test4/findplugin.in b/nc_test4/findplugin.in
index fb3068630..a8f2a4e3b 100644
--- a/nc_test4/findplugin.in
+++ b/nc_test4/findplugin.in
@@ -9,12 +9,10 @@
# $2 is 1 if we are running under cmake
# $3 is 1 if we are running using Visual Studio, blank otherwise
# $4 is the build type; only used if $3 is 1
-# Optional Input:
-# HDF5_PLUGIN_PATH environment variable
# Outputs:
# return code is 0 is success, 1 if failed
# Variable HDF5_PLUGIN_LIB is set to the library file name
-# Variable HDF5_PLUGIN_PATH is setthe absolute path to the
+# Variable HDF5_PLUGIN_DIR is set to the absolute path to the
# directory containing the plugin library file
# Local variables are prefixed with FP_
#
@@ -27,7 +25,7 @@ findplugin() {
FP_NAME="$1"
FP_PLUGIN_LIB=
-FP_PLUGIN_PATH=
+FP_PLUGIN_DIR=
# Figure out the plugin file name
# Test for visual studio before cygwin since both might be true
@@ -43,14 +41,6 @@ else # Presumably some form on *nix"
FP_PLUGIN_LIB="lib${FP_NAME}.so"
fi
-# If HDF5_PLUGIN_PATH is defined, then it overrides everything else.
-if test "x$HDF5_PLUGIN_PATH" != x ; then
- HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB"
- # HDF5_PLUGIN_PATH already set
- HDF5_PLUGIN_PATH=`${NCPATHCVT} $HDF5_PLUGIN_PATH`
- return 0;
-fi
-
# Figure out the path to where the lib is stored
# This can probably be simplified
@@ -59,40 +49,38 @@ cd ${TOPBUILDDIR}/plugins
FP_PLUGINS=`pwd`
cd ${CURWD}
-# Case 1: Cmake with Visual Studio
-# Do not know where to look for a dylib
# Case 1: Cmake with Visual Studio
if test "x$FP_ISCMAKE" != x -a "x${FP_ISMSVC}" != x ; then
# Case 1a: ignore the build type directory
if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
- FP_PLUGIN_PATH="${FP_PLUGINS}"
+ FP_PLUGIN_DIR="${FP_PLUGINS}"
fi
else # Case 2: automake
# Case 2a: look in .libs
if test -f "${FP_PLUGINS}/.libs/${FP_PLUGIN_LIB}" ; then
- FP_PLUGIN_PATH="${FP_PLUGINS}/.libs"
+ FP_PLUGIN_DIR="${FP_PLUGINS}/.libs"
else # Case 2: look in FP_PLUGINS directly
if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
- FP_PLUGIN_PATH="${FP_PLUGINS}"
+ FP_PLUGIN_DIR="${FP_PLUGINS}"
fi
fi
fi
# Verify
-if test "x$FP_PLUGIN_PATH" = x ; then
- echo "***Fail: Could not locate a usable HDF5_PLUGIN_PATH"
+if test "x$FP_PLUGIN_DIR" = x ; then
+ echo "***Fail: Could not locate a usable HDF5_PLUGIN_DIR"
return 1
fi
-if ! test -f "$FP_PLUGIN_PATH/$FP_PLUGIN_LIB" ; then
+if ! test -f "$FP_PLUGIN_DIR/$FP_PLUGIN_LIB" ; then
echo "***Fail: Could not locate a usable HDF5_PLUGIN_LIB"
return 1
fi
-FP_PLUGIN_PATH=`${NCPATHCVT} $FP_PLUGIN_PATH`
+FP_PLUGIN_DIR=`${NCPATHCVT} $FP_PLUGIN_DIR`
# Set the final output variables
HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB"
-HDF5_PLUGIN_PATH="$FP_PLUGIN_PATH"
+HDF5_PLUGIN_DIR="$FP_PLUGIN_DIR"
return 0
}
diff --git a/nc_test4/tst_filter.sh b/nc_test4/tst_filter.sh
index 44bcf211a..b47807909 100755
--- a/nc_test4/tst_filter.sh
+++ b/nc_test4/tst_filter.sh
@@ -43,36 +43,37 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
# Hide/unhide the noop filter
hidenoop() {
- rm -fr ${HDF5_PLUGIN_PATH}/save
- mkdir ${HDF5_PLUGIN_PATH}/save
- mv ${NOOPPATH} ${HDF5_PLUGIN_PATH}/save
+ rm -fr ${HDF5_PLUGIN_DIR}/save
+ mkdir ${HDF5_PLUGIN_DIR}/save
+ mv ${NOOPDIR} ${HDF5_PLUGIN_DIR}/save
}
unhidenoop() {
- mv ${HDF5_PLUGIN_PATH}/save/${NOOPLIB} ${HDF5_PLUGIN_PATH}
- rm -fr ${HDF5_PLUGIN_PATH}/save
+ mv ${HDF5_PLUGIN_DIR}/save/${NOOPLIB} ${HDF5_PLUGIN_DIR}
+ rm -fr ${HDF5_PLUGIN_DIR}/save
}
-# Locate the plugin path and the library names; argument order is critical
+# Locate the plugin dir and the library names; argument order is critical
# Find bzip2 and capture
findplugin h5bzip2
BZIP2LIB="${HDF5_PLUGIN_LIB}"
-BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}"
+BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}"
# Find misc and capture
findplugin h5misc
-MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
+MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
# Find noop and capture
findplugin h5noop
NOOPLIB="${HDF5_PLUGIN_LIB}"
-NOOPPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
+NOOPDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
-echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
-export HDF5_PLUGIN_PATH
+echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}"
+export HDF5_PLUGIN_DIR
+export HDF5_PLUGIN_PATH="$HDF5_PLUGIN_DIR"
# Verify
-if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
-if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi
-if ! test -f ${NOOPPATH} ; then echo "Unable to locate ${NOOPPATH}"; exit 1; fi
+if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi
+if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi
+if ! test -f ${NOOPDIR} ; then echo "Unable to locate ${NOOPDIR}"; exit 1; fi
# See if we have szip
if avail szip; then HAVE_SZIP=1; else HAVE_SZIP=0; fi
diff --git a/nc_test4/tst_filterinstall.sh b/nc_test4/tst_filterinstall.sh
new file mode 100755
index 000000000..465a42141
--- /dev/null
+++ b/nc_test4/tst_filterinstall.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+
+# Test the filter install
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+if test "x$TESTNCZARR" = x1 ; then
+. ./test_nczarr.sh
+fi
+
+set -e
+
+# Use this plugin path
+export HDF5_PLUGIN_PATH="${FEATURE_PLUGIN_INSTALL_DIR}"
+
+# Function to remove selected -s attributes from file;
+# These attributes might be platform dependent
+sclean() {
+ cat $1 \
+ | sed -e '/:_IsNetcdf4/d' \
+ | sed -e '/:_Endianness/d' \
+ | sed -e '/_NCProperties/d' \
+ | sed -e '/_SuperblockVersion/d' \
+ | sed -e '/_Format/d' \
+ | sed -e '/global attributes:/d' \
+ | cat > $2
+}
+
+# Function to extract _Filter attribute from a file
+# These attributes might be platform dependent
+getfilterattr() {
+V="$1"
+sed -e '/${V}.*:_Filter/p' -ed <$2 >$3
+}
+
+# Function to extract _Codecs attribute from a file
+# These attributes might be platform dependent
+getcodecsattr() {
+V="$1"
+sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3
+}
+
+trimleft() {
+sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
+}
+
+setfilter() {
+ FF="$1"
+ FSRC="$2"
+ FDST="$3"
+ FIH5="$4"
+ FICX="$5"
+ FFH5="$6"
+ FFCX="$7"
+ if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi
+ if test "x$FFCX" = x ; then FFCX="$FICX" ; fi
+ rm -f $FDST
+ cat ${srcdir}/$FSRC \
+ | sed -e "s/ref_any/${FF}/" \
+ | sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \
+ | sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \
+ | sed -e 's/"/\\"/g' -e 's/@/"/g' \
+ | cat > $FDST
+}
+
+# Execute the specified tests
+
+runfilter() {
+zext=$1
+zfilt="$2"
+zparams="$3"
+zcodec="$4"
+echo "*** Testing processing of filter $zfilt for map $zext"
+if test "x$TESTNCZARR" = x1 ; then
+fileargs "tmp_filt_${zfilt}"
+deletemap $zext $file
+else
+file="tmp_filt_${zfilt}.nc"
+rm -f $file
+fi
+setfilter $zfilt ref_any.cdl "tmp_filt_${zfilt}.cdl" "$zparams" "$zcodec"
+if test "x$TESTNCZARR" = x1 ; then
+${NCGEN} -4 -lb -o $fileurl "tmp_filt_${zfilt}.cdl"
+${NCDUMP} -n $zfilt -sF $fileurl > "tmp_filt_${zfilt}.tmp"
+else
+${NCGEN} -4 -lb -o $file "tmp_filt_${zfilt}.cdl"
+${NCDUMP} -n $zfilt -sF $file > "tmp_filt_${zfilt}.tmp"
+fi
+sclean "tmp_filt_${zfilt}.tmp" "tmp_filt_${zfilt}.dump"
+}
+
+testbzip2() {
+ zext=$1
+ if ! avail bzip2; then return 0; fi
+ runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]'
+ diff -b -w "tmp_filt_bzip2.cdl" "tmp_filt_bzip2.dump"
+}
+
+testset() {
+# Which test cases to exercise
+ testbzip2 $1
+}
+
+if test "x$TESTNCZARR" = x1 ; then
+ testset file
+ if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi
+ if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi
+else
+ testset nc
+fi
diff --git a/nc_test4/tst_specific_filters.sh b/nc_test4/tst_specific_filters.sh
index 874aabcd0..43235d286 100755
--- a/nc_test4/tst_specific_filters.sh
+++ b/nc_test4/tst_specific_filters.sh
@@ -28,9 +28,6 @@ echo "findplugin.sh loaded"
# Assume all test filters are in same plugin dir
findplugin h5bzip2
-echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
-export HDF5_PLUGIN_PATH
-
# Function to remove selected -s attributes from file;
# These attributes might be platform dependent
sclean() {
@@ -167,14 +164,14 @@ testzstd() {
testset() {
# Which test cases to exercise
if test "x$TESTNCZARR" = x1 ; then
-# testfletcher32 $1
+ testfletcher32 $1
testshuffle $1
fi
-# testdeflate $1
-# testszip $1
-# testbzip2 $1
-# testblosc $1
-# testzstd $1
+ testdeflate $1
+ testszip $1
+ testbzip2 $1
+ testblosc $1
+ testzstd $1
}
if test "x$TESTNCZARR" = x1 ; then
diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am
index b8de229ca..25b19cde2 100644
--- a/ncdap_test/Makefile.am
+++ b/ncdap_test/Makefile.am
@@ -10,7 +10,7 @@ include $(top_srcdir)/lib_flags.am
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#TEST_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
-#TESTS_ENVIRONMENT += export SETX=1;
+#TESTS_ENVIRONMENT = export SETX=1;
# Note which tests depend on other tests. Necessary for make -j check.
TEST_EXTENSIONS = .sh
diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am
index c7c15565d..656334486 100644
--- a/nczarr_test/Makefile.am
+++ b/nczarr_test/Makefile.am
@@ -154,7 +154,7 @@ EXTRA_DIST += ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes
CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl
-BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh
+BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh run_filterinstall.sh
test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c
rm -f $@
echo "#define TESTNCZARR" > $@
@@ -174,6 +174,15 @@ run_specific_filters.sh: $(top_srcdir)/nc_test4/tst_specific_filters.sh
chmod a+x $@
rm -f run_specific_filters.tmp
+run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh
+ rm -f $@ run_filterinstall.tmp
+ echo "#!/bin/bash" > run_filterinstall.tmp
+ echo "TESTNCZARR=1" >> run_filterinstall.tmp
+ cat $(top_srcdir)/nc_test4/tst_filterinstall.sh >> run_filterinstall.tmp
+ tr -d '\r' < run_filterinstall.tmp > $@
+ chmod a+x $@
+ rm -f run_filterinstall.tmp
+
# Remove directories
clean-local:
rm -fr tmp*.file results.file results.s3 results.zip
diff --git a/nczarr_test/run_filter.sh b/nczarr_test/run_filter.sh
index fce92ba60..da653a985 100755
--- a/nczarr_test/run_filter.sh
+++ b/nczarr_test/run_filter.sh
@@ -51,21 +51,21 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
# Find misc and capture
findplugin h5misc
-MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
+MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
# Find noop and capture
findplugin h5noop
NOOPLIB="${HDF5_PLUGIN_LIB}"
-NOOPPATH="${HDF5_PLUGIN_PATH}/${NOOPLIB}"
+NOOPDIR="${HDF5_PLUGIN_DIR}/${NOOPLIB}"
# Find bzip2 and capture
findplugin h5bzip2
BZIP2LIB="${HDF5_PLUGIN_LIB}"
-BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}"
+BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}"
# Verify
-if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
-if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi
+if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi
+if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi
# Execute the specified tests
@@ -144,11 +144,11 @@ ${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt
# Remove irrelevant -s output
sclean ./tmp_known_$zext.txt tmp_known_$zext.dump
# Now hide the filter code
-mv ${NOOPPATH} ./${NOOPLIB}.save
+mv ${NOOPDIR} ./${NOOPLIB}.save
# dump and clean noop.nc header when filter is not avail
${NCDUMP} -hs $fileurl > ./tmp_unk_$zext.txt
# Restore the filter code
-mv ./${NOOPLIB}.save ${NOOPPATH}
+mv ./${NOOPLIB}.save ${NOOPDIR}
# Verify that the filter is no longer defined
UNK=`sed -e '/var:_Filter/p' -e d ./tmp_unk_$zext.txt`
test "x$UNK" = x
diff --git a/nczarr_test/test_nczarr.sh b/nczarr_test/test_nczarr.sh
index 489bc0512..e63b82a42 100755
--- a/nczarr_test/test_nczarr.sh
+++ b/nczarr_test/test_nczarr.sh
@@ -133,12 +133,12 @@ if test "x$FP_USEPLUGINS" = xyes; then
echo "findplugin.sh loaded"
# Locate the plugin path and the library names; argument order is critical
-# Find bzip2 and capture
+# Find misc in order to determine HDF5_PLUGIN+PATH.
# Assume all test filters are in same plugin dir
findplugin h5misc
-echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
-export HDF5_PLUGIN_PATH
+echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}"
+export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
fi # USEPLUGINS
resetrc() {
diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt
index 71a6af7c4..e606b392a 100644
--- a/plugins/CMakeLists.txt
+++ b/plugins/CMakeLists.txt
@@ -10,12 +10,6 @@ FILE(READ H5Znoop.c NOOP_SOURCE)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "#define NOOP_INSTANCE 1\n")
FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "${NOOP_SOURCE}")
-SET(h5bzip2_SOURCES H5Zbzip2.c)
-
-IF(NOT HAVE_BZIP2)
-SET(h5bzip2_SOURCES ${h5bzip2_SOURCES} blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c)
-ENDIF()
-
SET(h5misc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h)
SET(h5noop_SOURCES H5Znoop.c H5Zutil.c h5noop.h)
@@ -29,11 +23,10 @@ SET(h5fletcher32_SOURCES H5Zfletcher32.c H5checksum.c)
SET(h5deflate_SOURCES H5Zdeflate.c)
SET(nczmisc_SOURCES NCZmisc.c)
-SET(nczdefaults_SOURCES NCZdefaults.c)
-
-IF(ENABLE_FILTER_TESTING)
-IF(BUILD_UTILITIES)
+SET(nczhdf5filters_SOURCES NCZhdf5filters.c)
+SET(nczstdfilters_SOURCES NCZstdfilters.c)
+IF(ENABLE_PLUGINS)
# LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined
SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
@@ -64,7 +57,8 @@ buildplugin(h5fletcher32 "h5fletcher32")
buildplugin(h5deflate "h5deflate")
buildplugin(nczmisc "nczmisc")
-buildplugin(nczdefaults "nczdefaults" netcdf)
+buildplugin(nczhdf5filters "nczhdf5filters" netcdf)
+buildplugin(nczstdfilters "nczstdfilters" netcdf)
IF(ENABLE_BLOSC)
SET(h5blosc_SOURCES H5Zblosc.c)
@@ -80,17 +74,54 @@ ENDIF()
IF(HAVE_SZ)
SET(h5szip_SOURCES H5Zszip.c H5Zszip.h)
buildplugin(h5szip "h5szip" ${Szip_LIBRARIES})
- SET(nczszip_SOURCES NCZszip.c)
- buildplugin(nczszip "nczszip" netcdf)
ENDIF()
+IF(HAVE_LOCAL_BZ2)
+SET(h5bzip2_SOURCES H5Zbzip2.c blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h)
+buildplugin(h5bzip2 "h5bzip2")
+ELSE()
+SET(h5bzip2_SOURCES H5Zbzip2.c)
buildplugin(h5bzip2 "h5bzip2" ${Bzip2_LIBRARIES})
-# Note we use name h5bzip2 instead of bzip2 to avoid logical
-# target name clash with examples/C/hdf5plugins
-SET_TARGET_PROPERTIES(h5bzip2 PROPERTIES OUTPUT_NAME "bzip2")
+ENDIF()
-ENDIF(BUILD_UTILITIES)
-ENDIF(ENABLE_FILTER_TESTING)
+
+# Installation
+IF(ENABLE_PLUGIN_INSTALL)
+
+MACRO(installplugin PLUG)
+IF(MSVC)
+ SET(BUILD_PLUGIN_LIB "${PLUG}.dll")
+ELSE()
+ SET(BUILD_PLUGIN_LIB "lib${PLUG}.so")
+ENDIF()
+MESSAGE(STATUS "Installing: ${BUILD_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}")
+install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${BUILD_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR})
+ENDMACRO()
+
+install(DIRECTORY DESTINATION ${PLUGIN_INSTALL_DIR})
+IF(Bzip2_FOUND)
+installplugin(h5bzip2)
+ENDIF()
+IF(Zstd_FOUND)
+installplugin(h5zstd)
+ENDIF()
+IF(Blosc_FOUND)
+installplugin(h5blosc)
+ENDIF()
+IF(ENABLE_NCZARR)
+ installplugin(h5fletcher32)
+ installplugin(h5shuffle)
+ installplugin(h5deflate)
+ installplugin(nczhdf5filters)
+ installplugin(nczstdfilters)
+IF(Szip_FOUND)
+ installplugin(h5szip)
+ENDIF()
+ENDIF()
+
+ENDIF(ENABLE_PLUGIN_INSTALL)
+
+ENDIF(ENABLE_PLUGINS)
# Copy some test files from current source dir to out-of-tree build dir.
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
diff --git a/plugins/H5Zblosc.c b/plugins/H5Zblosc.c
index 06afc4c71..3b36d568c 100755
--- a/plugins/H5Zblosc.c
+++ b/plugins/H5Zblosc.c
@@ -125,8 +125,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define PUSH_ERR(f,m,s,...) fprintf(stderr,"%s\n",s)
#endif /*USE_HDF5*/
-static int h5z_blosc_initialized = 0;
-
static size_t blosc_filter(unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t* buf_size, void** buf);
@@ -162,7 +160,7 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space)
hid_t super_type;
H5T_class_t classt;
- r = GET_FILTER(dcpl, FILTER_BLOSC, &flags, &nelements, values, 0, NULL);
+ r = GET_FILTER(dcpl, H5Z_FILTER_BLOSC, &flags, &nelements, values, 0, NULL);
if (r < 0) return -1;
if (nelements < 4) nelements = 4; /* First 4 slots reserved. */
@@ -209,7 +207,7 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space)
fprintf(stderr, "Blosc: Computed buffer size %d\n", bufsize);
#endif
- r = H5Pmodify_filter(dcpl, FILTER_BLOSC, flags, nelements, values);
+ r = H5Pmodify_filter(dcpl, H5Z_FILTER_BLOSC, flags, nelements, values);
if (r < 0) return -1;
return 1;
@@ -365,7 +363,7 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts,
const H5Z_class2_t blosc_H5Filter[1] = {
{
H5Z_CLASS_T_VERS,
- (H5Z_filter_t)(FILTER_BLOSC),
+ (H5Z_filter_t)(H5Z_FILTER_BLOSC),
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"blosc",
@@ -379,235 +377,3 @@ const H5Z_class2_t blosc_H5Filter[1] = {
H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_FILTER; }
const void* H5PLget_plugin_info(void) { return blosc_H5Filter; }
-
-
-/* Provide the codec support for the HDF5 blosc library */
-
-/* NCZarr Filter Objects */
-
-#define DEFAULT_LEVEL 9
-#define DEFAULT_BLOCKSIZE 1
-#define DEFAULT_TYPESIZE 1
-#define DEFAULT_COMPCODE BLOSC_LZ4
-
-/* Forward */
-static void NCZ_blosc_codec_finalize(void);
-static int NCZ_blosc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
-static int NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
-static int NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
-
-/* Structure for NCZ_PLUGIN_CODEC */
-static NCZ_codec_t NCZ_blosc_codec = {/* NCZ_codec_t codec fields */
- NCZ_CODEC_CLASS_VER, /* Struct version number */
- NCZ_CODEC_HDF5, /* Struct sort */
- "blosc", /* Standard name/id of the codec */
- FILTER_BLOSC, /* HDF5 alias for blosc */
- NULL, /*NCZ_blosc_codec_initialize*/
- NCZ_blosc_codec_finalize,
- NCZ_blosc_codec_to_hdf5,
- NCZ_blosc_hdf5_to_codec,
- NCZ_blosc_modify_parameters,
-};
-
-/* External Export API */
-const void*
-NCZ_get_codec_info(void)
-{
- if(!h5z_blosc_initialized) {
- h5z_blosc_initialized = 1;
- blosc_init();
- }
- return (void*)&NCZ_blosc_codec;
-}
-
-/* NCZarr Interface Functions */
-
-/* Create the true parameter set:
-Visible parameters:
-param[0] -- reserved
-param[1] -- reserved
-param[2] -- reserved
-param[3] -- variable chunksize in bytes | 0 (=>default)
-param[4] -- compression level
-param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
-param[6] -- compressor index
-
-Working parameters:
-param[0] -- filter revision
-param[1] -- blosc version
-param[2] -- variable type size in bytes
-param[3] -- variable chunksize in bytes
-param[4] -- compression level
-param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
-param[6] -- compressor index
-*/
-
-static void
-NCZ_blosc_codec_finalize(void)
-{
- if(h5z_blosc_initialized) {
- blosc_destroy();
- h5z_blosc_initialized = 0;
- }
-}
-
-static int
-NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
-{
- int i,stat = NC_NOERR;
- nc_type vtype;
- int storage, ndims;
- size_t* chunklens = NULL;
- size_t typesize, chunksize;
- char vname[NC_MAX_NAME+1];
- unsigned* params = NULL;
- size_t nparams;
- size_t vnparams = *vnparamsp;
- unsigned* vparams = *vparamsp;
-
- if(vnparams < 7)
- {stat = NC_EFILTER; goto done;}
- nparams = 7;
-
- if(vparams == NULL)
- {stat = NC_EFILTER; goto done;}
-
- if(wnparamsp == NULL || wparamsp == NULL)
- {stat = NC_EFILTER; goto done;}
-
- vnparams = *vnparamsp;
- vparams = *vparamsp;
-
- /* Get variable info */
- if((stat = nc_inq_var(ncid,varid,vname,&vtype,&ndims,NULL,NULL))) goto done;
-
- if(ndims == 0) {stat = NC_EFILTER; goto done;}
-
- /* Get the typesize */
- if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
-
- /* Compute chunksize */
- if((chunklens = (size_t*)calloc(ndims,sizeof(size_t)))==NULL) goto done;
- if((stat = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
- if(storage != NC_CHUNKED) {stat = NC_EFILTER; goto done;}
- chunksize = typesize;
- for(i=0;i<ndims;i++) chunksize *= chunklens[i];
-
- if((params = (unsigned*)malloc(vnparams*sizeof(unsigned)))==NULL)
- {stat = NC_ENOMEM; goto done;}
- memcpy(params,vparams,vnparams*sizeof(unsigned));
-
- params[0] = FILTER_BLOSC_VERSION;
- params[1] = BLOSC_VERSION_FORMAT;
- params[2] = (unsigned)typesize;
-
- params[3] = chunksize;
- params[4] = params[4];
- params[5] = params[5];
- params[6] = params[6];
-
- *wnparamsp = nparams;
- nullfree(*wparamsp);
- *wparamsp = params; params = NULL;
-
-done:
- nullfree(chunklens);
- nullfree(params);
- FUNC_LEAVE_NOAPI(stat)
-}
-
-static int
-NCZ_blosc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
-{
- int stat = NC_NOERR;
- NCjson* jcodec = NULL;
- NCjson* jtmp = NULL;
- unsigned* params = NULL;
- struct NCJconst jc = {0,0,0,NULL};
- int compcode;
-
- /* parse the JSON */
- if(NCJparse(codec_json,0,&jcodec)) {stat = NC_EFILTER; goto done;}
- if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
- /* Verify the codec ID */
- if(NCJdictget(jcodec,"id",&jtmp))
- {stat = NC_EFILTER; goto done;}
- if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;}
- if(strcmp(NCJstring(jtmp),NCZ_blosc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
-
- if((params = (unsigned*)calloc(7,sizeof(unsigned)))==NULL) {stat = NC_ENOMEM; goto done;}
-
- /* Get compression level*/
- if(NCJdictget(jcodec,"clevel",&jtmp)) {stat = NC_EFILTER; goto done;}
- if(jtmp) {
- if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
- } else
- jc.ival = DEFAULT_LEVEL;
- if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
- params[4] = (unsigned)jc.ival;
-
- /* Get blocksize */
- if(NCJdictget(jcodec,"blocksize",&jtmp)) {stat = NC_EFILTER; goto done;}
- if(jtmp) {
- if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
- } else
- jc.ival = DEFAULT_BLOCKSIZE;
- if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
- params[3] = (unsigned)jc.ival;
-
- /* Get shuffle */
- if(NCJdictget(jcodec,"shuffle",&jtmp)) {stat = NC_EFILTER; goto done;}
- if(jtmp) {
- if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
- } else
- jc.ival = BLOSC_NOSHUFFLE;
- params[5] = (unsigned)jc.ival;
-
- /* Get compname */
- if(NCJdictget(jcodec,"cname",&jtmp)) {stat = NC_EFILTER; goto done;}
- if(jtmp) {
- if(NCJcvt(jtmp,NCJ_STRING,&jc)) {stat = NC_EFILTER; goto done;}
- if(jc.sval == NULL || strlen(jc.sval) == 0) {stat = NC_EFILTER; goto done;}
- if((compcode = blosc_compname_to_compcode(jc.sval)) < 0) {stat = NC_EFILTER; goto done;}
- } else
- compcode = DEFAULT_COMPCODE;
- params[6] = (unsigned)compcode;
-
- if(nparamsp) *nparamsp = 7;
- if(paramsp) {*paramsp = params; params = NULL;}
-
-done:
- if(jc.sval) {
- free(jc.sval);
- }
- if(params) {
- free(params);
- }
- NCJreclaim(jcodec);
- return stat;
-}
-
-static int
-NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
-{
- int stat = NC_NOERR;
- char json[1024];
- const char* compname = NULL;
-
- if(nparams == 0 || params == NULL)
- {stat = NC_EINVAL; goto done;}
-
- /* Get the sub-compressor name */
- if(blosc_compcode_to_compname((int)params[6],&compname) < 0) {stat = NC_EFILTER; goto done;}
-
- snprintf(json,sizeof(json),
- "{\"id\": \"blosc\",\"clevel\": %u,\"blocksize\": %u,\"cname\": \"%s\",\"shuffle\": %d}",
- params[4],params[3],compname,params[5]);
-
- if(codecp) {
- if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
- }
-
-done:
- return stat;
-}
diff --git a/plugins/H5Zblosc.h b/plugins/H5Zblosc.h
index 6e834a293..911f24c2b 100755
--- a/plugins/H5Zblosc.h
+++ b/plugins/H5Zblosc.h
@@ -29,13 +29,6 @@ extern "C" {
#include "netcdf_filter_build.h"
#include "blosc.h"
-/* Filter revision number, starting at 1 */
-/* #define FILTER_BLOSC_VERSION 1 */
-#define FILTER_BLOSC_VERSION 2 /* multiple compressors since Blosc 1.3 */
-
-/* Filter ID registered with the HDF Group */
-#define FILTER_BLOSC 32001
-
#ifdef _MSC_VER
#ifdef DLL_EXPORT /* define when building the library */
#define DECLSPEC __declspec(dllexport)
@@ -46,6 +39,15 @@ extern "C" {
#define DECLSPEC extern
#endif
+/* Filter revision number, starting at 1 */
+/* #define FILTER_BLOSC_VERSION 1 */
+#define FILTER_BLOSC_VERSION 2 /* multiple compressors since Blosc 1.3 */
+
+#define DEFAULT_LEVEL 9
+#define DEFAULT_BLOCKSIZE 1
+#define DEFAULT_TYPESIZE 1
+#define DEFAULT_COMPCODE BLOSC_LZ4
+
/* HDF5 Plugin API */
DECLSPEC H5PL_type_t H5PLget_plugin_type(void);
DECLSPEC const void* H5PLget_plugin_info(void);
diff --git a/plugins/H5Zbzip2.c b/plugins/H5Zbzip2.c
index fa9ed8a6c..6f1128c48 100644
--- a/plugins/H5Zbzip2.c
+++ b/plugins/H5Zbzip2.c
@@ -214,92 +214,3 @@ H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts,
return 0;
}
-/**************************************************/
-/* NCZarr Filter Objects */
-
-/* Provide the codec support for the HDF5 bzip library */
-
-static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
-static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
-
-static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */
- NCZ_CODEC_CLASS_VER, /* Struct version number */
- NCZ_CODEC_HDF5, /* Struct sort */
- "bz2", /* Standard name/id of the codec */
- H5Z_FILTER_BZIP2, /* HDF5 alias for bzip2 */
- NULL, /*NCZ_bzip2_codec_initialize*/
- NULL, /*NCZ_bzip2_codec_finalize*/
- NCZ_bzip2_codec_to_hdf5,
- NCZ_bzip2_hdf5_to_codec,
- NULL, /*NCZ_bzip2_modify_parameters*/
-};
-
-/* External Export API */
-DLLEXPORT
-const void*
-NCZ_get_codec_info(void)
-{
- return (void*)&NCZ_bzip2_codec;
-}
-
-static int
-NCZ_bzip2_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
-{
- int stat = NC_NOERR;
- NCjson* jcodec = NULL;
- NCjson* jtmp = NULL;
- unsigned* params = NULL;
- struct NCJconst jc;
-
- if(nparamsp == NULL || paramsp == NULL)
- {stat = NC_EINTERNAL; goto done;}
-
- if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
- {stat = NC_ENOMEM; goto done;}
-
- /* parse the JSON */
- if(NCJparse(codec_json,0,&jcodec))
- {stat = NC_EFILTER; goto done;}
- if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
- /* Verify the codec ID */
- if(NCJdictget(jcodec,"id",&jtmp))
- {stat = NC_EFILTER; goto done;}
- if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
- if(strcmp(NCJstring(jtmp),NCZ_bzip2_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
-
- /* Get Level */
- if(NCJdictget(jcodec,"level",&jtmp))
- {stat = NC_EFILTER; goto done;}
- if(NCJcvt(jtmp,NCJ_INT,&jc))
- {stat = NC_EFILTER; goto done;}
- if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
- params[0] = (unsigned)jc.ival;
- *nparamsp = 1;
- *paramsp = params; params = NULL;
-
-done:
- if(params) free(params);
- NCJreclaim(jcodec);
- return stat;
-}
-
-static int
-NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
-{
- int stat = NC_NOERR;
- unsigned level = 0;
- char json[1024];
-
- if(nparams == 0 || params == NULL)
- {stat = NC_EFILTER; goto done;}
-
- level = params[0];
- snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level);
- if(codecp) {
- if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
- }
-
-done:
- return stat;
-}
-
diff --git a/plugins/H5Zzstd.c b/plugins/H5Zzstd.c
index 9d56425c7..58759ce5a 100644
--- a/plugins/H5Zzstd.c
+++ b/plugins/H5Zzstd.c
@@ -166,92 +166,3 @@ cleanupAndFail:
return 0;
}
-/**************************************************/
-/* NCZarr Filter Objects */
-
-/* Provide the codec support for the HDF5 zstandard library */
-
-static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
-static int NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
-
-static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */
- NCZ_CODEC_CLASS_VER, /* Struct version number */
- NCZ_CODEC_HDF5, /* Struct sort */
- "zstd", /* Standard name/id of the codec */
- H5Z_FILTER_ZSTD, /* HDF5 alias for zstd */
- NULL, /*NCZ_zstd_codec_initialize*/
- NULL, /*NCZ_zstd_codec_finalize*/
- NCZ_zstd_codec_to_hdf5,
- NCZ_zstd_hdf5_to_codec,
- NULL, /*NCZ_zstd_modify_parameters*/
-};
-
-/* External Export API */
-DLLEXPORT
-const void*
-NCZ_get_codec_info(void)
-{
- return (void*)&NCZ_zstd_codec;
-}
-
-static int
-NCZ_zstd_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
-{
- int stat = NC_NOERR;
- NCjson* jcodec = NULL;
- NCjson* jtmp = NULL;
- unsigned* params = NULL;
- struct NCJconst jc;
-
- if(nparamsp == NULL || paramsp == NULL)
- {stat = NC_EINTERNAL; goto done;}
-
- if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
- {stat = NC_ENOMEM; goto done;}
-
- /* parse the JSON */
- if(NCJparse(codec_json,0,&jcodec))
- {stat = NC_EFILTER; goto done;}
- if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
- /* Verify the codec ID */
- if(NCJdictget(jcodec,"id",&jtmp))
- {stat = NC_EFILTER; goto done;}
- if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
- if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
-
- /* Get Level */
- if(NCJdictget(jcodec,"level",&jtmp))
- {stat = NC_EFILTER; goto done;}
- if(NCJcvt(jtmp,NCJ_INT,&jc))
- {stat = NC_EFILTER; goto done;}
- if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
- params[0] = (unsigned)jc.ival;
- *nparamsp = 1;
- *paramsp = params; params = NULL;
-
-done:
- if(params) free(params);
- NCJreclaim(jcodec);
- return stat;
-}
-
-static int
-NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
-{
- int stat = NC_NOERR;
- unsigned level = 0;
- char json[1024];
-
- if(nparams == 0 || params == NULL)
- {stat = NC_EFILTER; goto done;}
-
- level = params[0];
- snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level);
- if(codecp) {
- if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
- }
-
-done:
- return stat;
-}
-
diff --git a/plugins/Makefile.am b/plugins/Makefile.am
index b367c98d0..ded5b3076 100644
--- a/plugins/Makefile.am
+++ b/plugins/Makefile.am
@@ -14,34 +14,22 @@ if ISMINGW
LDADD = ${top_builddir}/liblib/libnetcdf.la
endif
-# BZIP2 version 1.0.8 (https://sourceware.org/bzip2/)
-BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h
-
EXTRA_DIST = CMakeLists.txt
# The HDF5 filter wrappers
-EXTRA_DIST += \
+EXTRA_DIST += stdinstall.sh \
H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \
H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \
H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h H5Zzstd.c H5Zzstd.h
+
# The Codec filter wrappers
-EXTRA_DIST += NCZdefaults.c NCZszip.c
+EXTRA_DIST += NCZhdf5filters.c NCZstdfilters.c
# The Filter implementations
EXTRA_DIST += H5checksum.c
-
-EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE
-
if ENABLE_FILTER_TESTING
-lib_LTLIBRARIES += libh5bzip2.la
-libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h
-if ! HAVE_BZIP2
-libh5bzip2_la_SOURCES += ${BZIP2SRC}
-endif
-
noinst_LTLIBRARIES += libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la
-noinst_LTLIBRARIES += libnczdefaults.la
if ENABLE_NCZARR_FILTERS
noinst_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la
@@ -51,27 +39,39 @@ libh5deflate_la_SOURCES = H5Zdeflate.c
# Need our version of szip if libsz available and we are not using HDF5
if HAVE_SZ
-noinst_LTLIBRARIES += libh5szip.la libnczszip.la
+noinst_LTLIBRARIES += libh5szip.la
libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h
-libnczszip_la_SOURCES = NCZszip.c
endif
-libnczdefaults_la_SOURCES = NCZdefaults.c
-
endif # ENABLE_NCZARR_FILTERS
+if ENABLE_PLUGINS
+
+libnczstdfilters_la_SOURCES = NCZstdfilters.c
+
+libnczhdf5filters_la_SOURCES = NCZhdf5filters.c
+
if HAVE_BLOSC
-noinst_LTLIBRARIES += libh5blosc.la
libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h
endif
if HAVE_ZSTD
-noinst_LTLIBRARIES += libh5zstd.la
libh5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h
endif
-libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h
+noinst_LTLIBRARIES += libnczhdf5filters.la
+noinst_LTLIBRARIES += libnczstdfilters.la
+if HAVE_BLOSC
+noinst_LTLIBRARIES += libh5blosc.la
+endif
+if HAVE_ZSTD
+noinst_LTLIBRARIES += libh5zstd.la
+endif
+
+endif #ENABLE_PLUGINS
+
+libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h
libnczmisc_la_SOURCES = NCZmisc.c
# The noop filter is to allow testing of multifilters and filter order
@@ -79,6 +79,14 @@ libnczmisc_la_SOURCES = NCZmisc.c
libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h
libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h
+# Bzip2 is used to test more complex filters
+libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h
+BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h
+EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE
+if HAVE_LOCAL_BZ2
+libh5bzip2_la_SOURCES += ${BZIP2SRC}
+endif
+
endif #ENABLE_FILTER_TESTING
BUILT_SOURCES = H5Znoop1.c
@@ -87,6 +95,9 @@ H5Znoop1.c: Makefile H5Znoop.c
echo '#define NOOP_INSTANCE 1' > $@
cat ${srcdir}/H5Znoop.c >> $@
+noinst_LTLIBRARIES += libh5bzip2.la
+
+# Record where bzip2 came from; may be out of date
BZIP2VER = 1.0.8
BZIP2DIR = bzip2-${BZIP2VER}
BZIP2URL = https://sourceware.org/pub/bzip2/${BZIP2DIR}.tar.gz
@@ -96,3 +107,7 @@ bzip2::
tar -zxf ${BZIP2DIR}.tar.gz
cd ${BZIP2DIR}; cp ${BZIP2SRC} ..; cp LICENSE ../BZIP2_LICENSE ; cd ..
rm -fr ./${BZIP2DIR}
+
+# Custom install
+install-exec-hook:
+ sh ./stdinstall.sh
diff --git a/plugins/NCZdefaults.c b/plugins/NCZhdf5filters.c
similarity index 53%
rename from plugins/NCZdefaults.c
rename to plugins/NCZhdf5filters.c
index 48dc3439b..a8336ad73 100644
--- a/plugins/NCZdefaults.c
+++ b/plugins/NCZhdf5filters.c
@@ -21,6 +21,11 @@ Author: Dennis Heimbigner
#include "netcdf_filter_build.h"
#include "netcdf_json.h"
+#ifdef HAVE_SZ
+#include <szlib.h>
+#include "H5Zszip.h"
+#endif
+
#define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */
#define H5Z_FILTER_SHUFFLE 2 /*shuffle the data */
#define H5Z_FILTER_FLETCHER32 3 /*fletcher32 checksum of EDC */
@@ -40,6 +45,12 @@ static int NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparam
static int NCZ_deflate_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_deflate_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
+#ifdef HAVE_SZ
+static int NCZ_szip_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
+static int NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
+static int NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
+#endif
+
/**************************************************/
static NCZ_codec_t NCZ_shuffle_codec = {
@@ -302,10 +313,224 @@ done:
/**************************************************/
+#ifdef HAVE_SZ
+
+static NCZ_codec_t NCZ_szip_codec = {
+ NCZ_CODEC_CLASS_VER, /* Struct version number */
+ NCZ_CODEC_HDF5, /* Struct sort */
+ "szip", /* Standard name/id of the codec */
+ H5Z_FILTER_SZIP, /* HDF5 alias for szip */
+ NULL, /*NCZ_szip_codec_initialize*/
+ NULL, /*NCZ_szip_codec_finalize*/
+ NCZ_szip_codec_to_hdf5,
+ NCZ_szip_hdf5_to_codec,
+ NCZ_szip_modify_parameters,
+};
+
+static int
+NCZ_szip_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
+{
+ int stat = NC_NOERR;
+ unsigned* params = NULL;
+ size_t nparams = 2; /* No. of visible parameters */
+ NCjson* json = NULL;
+ NCjson* jtmp = NULL;
+ struct NCJconst jc = {0,0,0,NULL};
+
+ if(nparamsp == NULL || paramsp == NULL)
+ {stat = NC_EINTERNAL; goto done;}
+
+ if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL)
+ {stat = NC_ENOMEM; goto done;}
+
+ if(NCJparse(codec_json,0,&json))
+ {stat = NC_EFILTER; goto done;}
+
+ if(NCJdictget(json,"mask",&jtmp) || jtmp == NULL)
+ {stat = NC_EFILTER; goto done;}
+ if(NCJcvt(jtmp,NCJ_INT,&jc))
+ {stat = NC_EFILTER; goto done;}
+ params[H5Z_SZIP_PARM_MASK] = (unsigned)jc.ival;
+
+ jtmp = NULL;
+ if(NCJdictget(json,"pixels-per-block",&jtmp) || jtmp == NULL)
+ {stat = NC_EFILTER; goto done;}
+ if(NCJcvt(jtmp,NCJ_INT,&jc))
+ {stat = NC_EFILTER; goto done;}
+ params[H5Z_SZIP_PARM_PPB] = (unsigned)jc.ival;
+
+ *nparamsp = nparams;
+ *paramsp = params; params = NULL;
+
+done:
+ NCJreclaim(json);
+ nullfree(params);
+ return stat;
+}
+
+static int
+NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
+{
+ int stat = NC_NOERR;
+ char json[2048];
+
+ snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": %u, \"pixels-per-block\": %u}",
+ NCZ_szip_codec.codecid,
+ params[H5Z_SZIP_PARM_MASK],
+ params[H5Z_SZIP_PARM_PPB]);
+ if(codecp) {
+ if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
+ }
+
+done:
+ return stat;
+}
+
+static int
+NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
+{
+ int i,ret_value = NC_NOERR;
+ nc_type vtype;
+ size_t typesize, scanline, dtype_precision, npoints;
+ int ndims, storage, dtype_order;
+ int dimids[NC_MAX_VAR_DIMS];
+ char vname[NC_MAX_NAME+1];
+ size_t chunklens[NC_MAX_VAR_DIMS];
+ unsigned* params = NULL;
+ unsigned* vparams = NULL;
+ size_t wnparams = 4;
+
+ if(wnparamsp == NULL || wparamsp == NULL)
+ {ret_value = NC_EFILTER; goto done;}
+ if(vnparamsp == NULL || vparamsp == NULL)
+ {ret_value = NC_EFILTER; goto done;}
+ if(*vnparamsp > 0 && *vparamsp == NULL)
+ {ret_value = NC_EFILTER; goto done;}
+
+ vparams = *vparamsp;
+
+ /* Get variable info */
+ if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done;
+
+ /* Get the typesize */
+ if((ret_value = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
+
+ /* Get datatype's precision, in case is less than full bits */
+ dtype_precision = typesize*8;
+
+ if(dtype_precision > 24) {
+ if(dtype_precision <= 32)
+ dtype_precision = 32;
+ else if(dtype_precision <= 64)
+ dtype_precision = 64;
+ } /* end if */
+
+ if(ndims == 0) {ret_value = NC_EFILTER; goto done;}
+
+ /* Set "local" parameter for this dataset's "pixels-per-scanline" */
+ if((ret_value = nc_inq_dimlen(ncid,dimids[ndims-1],&scanline))) goto done;
+
+ /* Get number of elements for the dataspace; use
+ total number of elements in the chunk to define the new 'scanline' size */
+ /* Compute chunksize */
+ if((ret_value = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
+ if(storage != NC_CHUNKED) {ret_value = NC_EFILTER; goto done;}
+ npoints = 1;
+ for(i=0;i<ndims;i++) npoints *= chunklens[i];
+
+ /* Get datatype's endianness order */
+ if((ret_value = nc_inq_var_endian(ncid,varid,&dtype_order))) goto done;
+
+ if((params = (unsigned*)malloc(wnparams*sizeof(unsigned)))==NULL)
+ {ret_value = NC_ENOMEM; goto done;}
+ params[H5Z_SZIP_PARM_MASK] = vparams[H5Z_SZIP_PARM_MASK];
+ params[H5Z_SZIP_PARM_PPB] = vparams[H5Z_SZIP_PARM_PPB];
+
+ /* Set "local" parameter for this dataset's "bits-per-pixel" */
+ params[H5Z_SZIP_PARM_BPP] = dtype_precision;
+
+ /* Adjust scanline if it is smaller than number of pixels per block or
+ if it is bigger than maximum pixels per scanline, or there are more than
+ SZ_MAX_BLOCKS_PER_SCANLINE blocks per scanline */
+ if(scanline < vparams[H5Z_SZIP_PARM_PPB]) {
+ if(npoints < vparams[H5Z_SZIP_PARM_PPB])
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "pixels per block greater than total number of elements in the chunk")
+ scanline = MIN((vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), npoints);
+ } else {
+ if(scanline <= SZ_MAX_PIXELS_PER_SCANLINE)
+ scanline = MIN((vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), scanline);
+ else
+ scanline = vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE;
+ } /* end else */
+ /* Assign the final value to the scanline */
+ params[H5Z_SZIP_PARM_PPS] = (unsigned)scanline;
+
+ /* Set the correct mask flags */
+
+ /* From H5Pdcpl.c#H5Pset_szip */
+ params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~H5_SZIP_CHIP_OPTION_MASK);
+ params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_ALLOW_K13_OPTION_MASK;
+ params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_RAW_OPTION_MASK;
+ params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~(H5_SZIP_LSB_OPTION_MASK | H5_SZIP_MSB_OPTION_MASK));
+
+ /* From H5Zszip.c#H5Z__set_local_szip */
+ params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~(H5_SZIP_LSB_OPTION_MASK | H5_SZIP_MSB_OPTION_MASK));
+ switch(dtype_order) {
+ case NC_ENDIAN_LITTLE: /* Little-endian byte order */
+ params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_LSB_OPTION_MASK;
+ break;
+ case NC_ENDIAN_BIG: /* Big-endian byte order */
+ params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_MSB_OPTION_MASK;
+ break;
+ default:
+ HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
+ } /* end switch */
+
+ *wnparamsp = wnparams;
+ nullfree(*wparamsp);
+ *wparamsp = params; params = NULL;
+
+done:
+ nullfree(params);
+ FUNC_LEAVE_NOAPI(ret_value)
+}
+
+#if 0
+static int
+NCZ_szip_visible_parameters(int ncid, int varid, size_t nparamsin, const unsigned int* paramsin, size_t* nparamsp, unsigned** paramsp)
+{
+ int stat = NC_NOERR;
+ unsigned* params = NULL;
+
+ if(nparamsp == NULL || paramsp == NULL)
+ {stat = NC_EFILTER; goto done;}
+
+ if((params = (unsigned*)malloc(2*sizeof(unsigned)))==NULL)
+ {stat = NC_ENOMEM; goto done;}
+
+ params[H5Z_SZIP_PARM_MASK] = paramsin[H5Z_SZIP_PARM_MASK];
+ params[H5Z_SZIP_PARM_PPB] = paramsin[H5Z_SZIP_PARM_PPB];
+
+ nullfree(*paramsp);
+ *paramsp = params; params = NULL;
+
+done:
+ nullfree(params);
+ return stat;
+}
+#endif
+
+#endif /*HAVE_SZ*/
+
+/**************************************************/
+
NCZ_codec_t* NCZ_default_codecs[] = {
&NCZ_shuffle_codec,
&NCZ_fletcher32_codec,
&NCZ_zlib_codec,
+#ifdef HAVE_SZ
+&NCZ_szip_codec,
+#endif
NULL
};
diff --git a/plugins/NCZstdfilters.c b/plugins/NCZstdfilters.c
new file mode 100644
index 000000000..eda186516
--- /dev/null
+++ b/plugins/NCZstdfilters.c
@@ -0,0 +1,460 @@
+/* Copyright 2003-2018, University Corporation for Atmospheric
+ * Research. See the COPYRIGHT file for copying and redistribution
+ * conditions.
+ */
+
+/*
+Author: Dennis Heimbigner
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+#include "netcdf.h"
+#include "netcdf_filter.h"
+#include "netcdf_filter_build.h"
+#include "netcdf_json.h"
+
+/* Provide Codec information for the standard filters */
+
+#ifndef H5Z_FILTER_BZIP2
+#define H5Z_FILTER_BZIP2 1
+#define H5Z_FILTER_ZSTD 2
+#define H5Z_FILTER_BLOSC 3
+#endif
+
+#ifdef HAVE_BLOSC
+#include "H5Zblosc.h"
+#endif
+
+/**************************************************/
+/* NCZarr Filter Objects */
+
+/* Forward */
+
+#ifdef HAVE_BZ2
+static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
+static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
+#endif
+
+#ifdef HAVE_ZSTD
+static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
+static int NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
+#endif
+
+#ifdef HAVE_BLOSC
+static int NCZ_blosc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
+static int NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
+static int NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
+static void NCZ_blosc_codec_finalize(void);
+#endif
+
+/**************************************************/
+/* Provide the codec support for bzip2 filter */
+
+#ifdef HAVE_BZ2
+static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */
+ NCZ_CODEC_CLASS_VER, /* Struct version number */
+ NCZ_CODEC_HDF5, /* Struct sort */
+ "bz2", /* Standard name/id of the codec */
+ H5Z_FILTER_BZIP2, /* HDF5 alias for bzip2 */
+ NULL, /*NCZ_bzip2_codec_initialize*/
+ NULL, /*NCZ_bzip2_codec_finalize*/
+ NCZ_bzip2_codec_to_hdf5,
+ NCZ_bzip2_hdf5_to_codec,
+ NULL, /*NCZ_bzip2_modify_parameters*/
+};
+
+/* External Export API */
+DLLEXPORT
+const void*
+NCZ_get_codec_info(void)
+{
+ return (void*)&NCZ_bzip2_codec;
+}
+
+static int
+NCZ_bzip2_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
+{
+ int stat = NC_NOERR;
+ NCjson* jcodec = NULL;
+ NCjson* jtmp = NULL;
+ unsigned* params = NULL;
+ struct NCJconst jc;
+
+ if(nparamsp == NULL || paramsp == NULL)
+ {stat = NC_EINTERNAL; goto done;}
+
+ if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
+ {stat = NC_ENOMEM; goto done;}
+
+ /* parse the JSON */
+ if(NCJparse(codec_json,0,&jcodec))
+ {stat = NC_EFILTER; goto done;}
+ if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
+ /* Verify the codec ID */
+ if(NCJdictget(jcodec,"id",&jtmp))
+ {stat = NC_EFILTER; goto done;}
+ if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
+ if(strcmp(NCJstring(jtmp),NCZ_bzip2_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
+
+ /* Get Level */
+ if(NCJdictget(jcodec,"level",&jtmp))
+ {stat = NC_EFILTER; goto done;}
+ if(NCJcvt(jtmp,NCJ_INT,&jc))
+ {stat = NC_EFILTER; goto done;}
+ if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
+ params[0] = (unsigned)jc.ival;
+ *nparamsp = 1;
+ *paramsp = params; params = NULL;
+
+done:
+ if(params) free(params);
+ NCJreclaim(jcodec);
+ return stat;
+}
+
+static int
+NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
+{
+ int stat = NC_NOERR;
+ unsigned level = 0;
+ char json[1024];
+
+ if(nparams == 0 || params == NULL)
+ {stat = NC_EFILTER; goto done;}
+
+ level = params[0];
+ snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level);
+ if(codecp) {
+ if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
+ }
+
+done:
+ return stat;
+}
+#endif
+
+/**************************************************/
+/* Provide the codec support for zstandard filter */
+
+#ifdef HAVE_ZSTD
+
+static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */
+ NCZ_CODEC_CLASS_VER, /* Struct version number */
+ NCZ_CODEC_HDF5, /* Struct sort */
+ "zstd", /* Standard name/id of the codec */
+ H5Z_FILTER_ZSTD, /* HDF5 alias for zstd */
+ NULL, /*NCZ_zstd_codec_initialize*/
+ NULL, /*NCZ_zstd_codec_finalize*/
+ NCZ_zstd_codec_to_hdf5,
+ NCZ_zstd_hdf5_to_codec,
+ NULL, /*NCZ_zstd_modify_parameters*/
+};
+
+static int
+NCZ_zstd_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
+{
+ int stat = NC_NOERR;
+ NCjson* jcodec = NULL;
+ NCjson* jtmp = NULL;
+ unsigned* params = NULL;
+ struct NCJconst jc;
+
+ if(nparamsp == NULL || paramsp == NULL)
+ {stat = NC_EINTERNAL; goto done;}
+
+ if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
+ {stat = NC_ENOMEM; goto done;}
+
+ /* parse the JSON */
+ if(NCJparse(codec_json,0,&jcodec))
+ {stat = NC_EFILTER; goto done;}
+ if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
+ /* Verify the codec ID */
+ if(NCJdictget(jcodec,"id",&jtmp))
+ {stat = NC_EFILTER; goto done;}
+ if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
+ if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
+
+ /* Get Level */
+ if(NCJdictget(jcodec,"level",&jtmp))
+ {stat = NC_EFILTER; goto done;}
+ if(NCJcvt(jtmp,NCJ_INT,&jc))
+ {stat = NC_EFILTER; goto done;}
+ if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
+ params[0] = (unsigned)jc.ival;
+ *nparamsp = 1;
+ *paramsp = params; params = NULL;
+
+done:
+ if(params) free(params);
+ NCJreclaim(jcodec);
+ return stat;
+}
+
+static int
+NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
+{
+ int stat = NC_NOERR;
+ unsigned level = 0;
+ char json[1024];
+
+ if(nparams == 0 || params == NULL)
+ {stat = NC_EFILTER; goto done;}
+
+ level = params[0];
+ snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level);
+ if(codecp) {
+ if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
+ }
+
+done:
+ return stat;
+}
+#endif
+
+/**************************************************/
+/* Provide the codec support for blosc filter */
+
+#ifdef HAVE_BLOSC
+
+/* Structure for NCZ_PLUGIN_CODEC */
+static NCZ_codec_t NCZ_blosc_codec = {/* NCZ_codec_t codec fields */
+ NCZ_CODEC_CLASS_VER, /* Struct version number */
+ NCZ_CODEC_HDF5, /* Struct sort */
+ "blosc", /* Standard name/id of the codec */
+ H5Z_FILTER_BLOSC, /* HDF5 alias for blosc */
+ NULL, /*NCZ_blosc_codec_initialize*/
+ NCZ_blosc_codec_finalize,
+ NCZ_blosc_codec_to_hdf5,
+ NCZ_blosc_hdf5_to_codec,
+ NCZ_blosc_modify_parameters,
+};
+
+/* NCZarr Interface Functions */
+
+/* Create the true parameter set:
+Visible parameters:
+param[0] -- reserved
+param[1] -- reserved
+param[2] -- reserved
+param[3] -- variable chunksize in bytes | 0 (=>default)
+param[4] -- compression level
+param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
+param[6] -- compressor index
+
+Working parameters:
+param[0] -- filter revision
+param[1] -- blosc version
+param[2] -- variable type size in bytes
+param[3] -- variable chunksize in bytes
+param[4] -- compression level
+param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
+param[6] -- compressor index
+*/
+
+void blosc_destroy(void);
+
+static int ncz_blosc_initialized = 0;
+
+static void
+NCZ_blosc_codec_finalize(void)
+{
+ if(ncz_blosc_initialized) {
+ blosc_destroy();
+ ncz_blosc_initialized = 0;
+ }
+}
+
+static int
+NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
+{
+ int i,stat = NC_NOERR;
+ nc_type vtype;
+ int storage, ndims;
+ size_t* chunklens = NULL;
+ size_t typesize, chunksize;
+ char vname[NC_MAX_NAME+1];
+ unsigned* params = NULL;
+ size_t nparams;
+ size_t vnparams = *vnparamsp;
+ unsigned* vparams = *vparamsp;
+
+ if(vnparams < 7)
+ {stat = NC_EFILTER; goto done;}
+ nparams = 7;
+
+ if(vparams == NULL)
+ {stat = NC_EFILTER; goto done;}
+
+ if(wnparamsp == NULL || wparamsp == NULL)
+ {stat = NC_EFILTER; goto done;}
+
+ vnparams = *vnparamsp;
+ vparams = *vparamsp;
+
+ /* Get variable info */
+ if((stat = nc_inq_var(ncid,varid,vname,&vtype,&ndims,NULL,NULL))) goto done;
+
+ if(ndims == 0) {stat = NC_EFILTER; goto done;}
+
+ /* Get the typesize */
+ if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
+
+ /* Compute chunksize */
+ if((chunklens = (size_t*)calloc(ndims,sizeof(size_t)))==NULL) goto done;
+ if((stat = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
+ if(storage != NC_CHUNKED) {stat = NC_EFILTER; goto done;}
+ chunksize = typesize;
+ for(i=0;i<ndims;i++) chunksize *= chunklens[i];
+
+ if((params = (unsigned*)malloc(vnparams*sizeof(unsigned)))==NULL)
+ {stat = NC_ENOMEM; goto done;}
+ memcpy(params,vparams,vnparams*sizeof(unsigned));
+
+ params[0] = FILTER_BLOSC_VERSION;
+ params[1] = BLOSC_VERSION_FORMAT;
+ params[2] = (unsigned)typesize;
+
+ params[3] = chunksize;
+ params[4] = params[4];
+ params[5] = params[5];
+ params[6] = params[6];
+
+ *wnparamsp = nparams;
+ nullfree(*wparamsp);
+ *wparamsp = params; params = NULL;
+
+done:
+ nullfree(chunklens);
+ nullfree(params);
+ FUNC_LEAVE_NOAPI(stat)
+}
+
+static int
+NCZ_blosc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
+{
+ int stat = NC_NOERR;
+ NCjson* jcodec = NULL;
+ NCjson* jtmp = NULL;
+ unsigned* params = NULL;
+ struct NCJconst jc = {0,0,0,NULL};
+ int compcode;
+
+ /* parse the JSON */
+ if(NCJparse(codec_json,0,&jcodec)) {stat = NC_EFILTER; goto done;}
+ if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
+ /* Verify the codec ID */
+ if(NCJdictget(jcodec,"id",&jtmp))
+ {stat = NC_EFILTER; goto done;}
+ if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;}
+ if(strcmp(NCJstring(jtmp),NCZ_blosc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
+
+ if((params = (unsigned*)calloc(7,sizeof(unsigned)))==NULL) {stat = NC_ENOMEM; goto done;}
+
+ /* Get compression level*/
+ if(NCJdictget(jcodec,"clevel",&jtmp)) {stat = NC_EFILTER; goto done;}
+ if(jtmp) {
+ if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
+ } else
+ jc.ival = DEFAULT_LEVEL;
+ if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
+ params[4] = (unsigned)jc.ival;
+
+ /* Get blocksize */
+ if(NCJdictget(jcodec,"blocksize",&jtmp)) {stat = NC_EFILTER; goto done;}
+ if(jtmp) {
+ if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
+ } else
+ jc.ival = DEFAULT_BLOCKSIZE;
+ if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
+ params[3] = (unsigned)jc.ival;
+
+ /* Get shuffle */
+ if(NCJdictget(jcodec,"shuffle",&jtmp)) {stat = NC_EFILTER; goto done;}
+ if(jtmp) {
+ if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
+ } else
+ jc.ival = BLOSC_NOSHUFFLE;
+ params[5] = (unsigned)jc.ival;
+
+ /* Get compname */
+ if(NCJdictget(jcodec,"cname",&jtmp)) {stat = NC_EFILTER; goto done;}
+ if(jtmp) {
+ if(NCJcvt(jtmp,NCJ_STRING,&jc)) {stat = NC_EFILTER; goto done;}
+ if(jc.sval == NULL || strlen(jc.sval) == 0) {stat = NC_EFILTER; goto done;}
+ if((compcode = blosc_compname_to_compcode(jc.sval)) < 0) {stat = NC_EFILTER; goto done;}
+ } else
+ compcode = DEFAULT_COMPCODE;
+ params[6] = (unsigned)compcode;
+
+ if(nparamsp) *nparamsp = 7;
+ if(paramsp) {*paramsp = params; params = NULL;}
+
+done:
+ if(jc.sval) {
+ free(jc.sval);
+ }
+ if(params) {
+ free(params);
+ }
+ NCJreclaim(jcodec);
+ return stat;
+}
+
+static int
+NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
+{
+ int stat = NC_NOERR;
+ char json[1024];
+ const char* compname = NULL;
+
+ if(nparams == 0 || params == NULL)
+ {stat = NC_EINVAL; goto done;}
+
+ /* Get the sub-compressor name */
+ if(blosc_compcode_to_compname((int)params[6],&compname) < 0) {stat = NC_EFILTER; goto done;}
+
+ snprintf(json,sizeof(json),
+ "{\"id\": \"blosc\",\"clevel\": %u,\"blocksize\": %u,\"cname\": \"%s\",\"shuffle\": %d}",
+ params[4],params[3],compname,params[5]);
+
+ if(codecp) {
+ if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
+ }
+
+done:
+ return stat;
+}
+#endif
+
+/**************************************************/
+
+NCZ_codec_t* NCZ_stdfilters_codecs[] = {
+#ifdef HAVE_BZ2
+&NCZ_bzip2_codec,
+#endif
+#ifdef HAVE_ZSTD
+&NCZ_zstd_codec,
+#endif
+#ifdef HAVE_BLOSC
+&NCZ_blosc_codec,
+#endif
+NULL
+};
+
+/* External Export API */
+DLLEXPORT
+const void*
+NCZ_codec_info_defaults(void)
+{
+ return (void*)&NCZ_stdfilters_codecs;
+}
+
diff --git a/plugins/stdinstall.in b/plugins/stdinstall.in
new file mode 100755
index 000000000..74a73765a
--- /dev/null
+++ b/plugins/stdinstall.in
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+set -x
+
+INSTALLDIR=@PLUGIN_INSTALL_DIR@
+
+# Load the findplugins function
+. ${builddir}/findplugin.sh
+echo "findplugin.sh loaded"
+
+installplugin() {
+PLUG="$1"
+# Locate the plugin path and the library name; argument order is critical
+findplugin ${PLUG}
+if ! test -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" ; then
+ echo "Not exists: ${HDF5_PLUGIN_DIR}/$HDF5_PLUGIN_LIB ; ignored"
+ return
+fi
+if ! test -d "${INSTALLDIR}" ; then
+ echo "Not exists: ${INSTALLDIR} ; creating"
+ mkdir ${INSTALLDIR}
+fi
+echo "Installing: $HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB into $INSTALLDIR"
+cp -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" $INSTALLDIR
+}
+
+if test "x$USEPLUGINS" != x ; then
+ if test "x$INSTALLDIR" != x ; then
+ installplugin h5bzip2
+ installplugin h5zstd
+ installplugin h5blosc
+ if test "x$FEATURE_NCZARR" ; then
+ installplugin h5fletcher32
+ installplugin h5shuffle
+ installplugin h5deflate
+ installplugin h5szip
+ installplugin nczdefaults
+ installplugin nczszip
+ fi
+ fi
+fi
diff --git a/test_common.in b/test_common.in
index e010870b4..eff6baf8e 100644
--- a/test_common.in
+++ b/test_common.in
@@ -28,9 +28,11 @@ FEATURE_PARALLEL=@HAS_PARALLEL@
# Define selected features of the build
FEATURE_HDF5=@HAS_HDF5@
+FEATURE_NCZARR=@HAS_NCZARR@
FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@
FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@
FEATURE_FILTERTESTS=@DO_FILTER_TESTS@
+FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@
set -e
diff --git a/unit_test/test_pathcvt.c b/unit_test/test_pathcvt.c
index 6c12b107f..b0230f53b 100644
--- a/unit_test/test_pathcvt.c
+++ b/unit_test/test_pathcvt.c
@@ -108,12 +108,12 @@ static Test PATHTESTS[] = {
char* macros[128];
/*Forward */
-#ifdef DEBUG
-static const char* kind2string(int kind);
-#endif
static char* expand(const char* s);
static void setmacros(void);
static void reclaimmacros(void);
+#ifdef DEBUG
+static const char* kind2string(int kind);
+#endif
int
main(int argc, char** argv)