mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-31 17:50:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into ejh_doc_4

Browse Source
This commit is contained in:
Edward Hartnett 2022-05-05 08:16:40 -06:00 committed by GitHub
commit c3d201a8b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
54 changed files with 2943 additions and 1111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/run_tests_ubuntu.yml vendored
View File

@ -4,7 +4,7 @@
name: Run Ubuntu/Linux netCDF Tests
on: [ pull_request ]
on: [pull_request]
jobs:

79
CMakeLists.txt
View File

@ -623,26 +623,6 @@ ENDIF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
# 3. is nczarr enabled?
# We need separate flags for cases 1 and 2
# We need to determine if libsz is available both for HDF5 and NCZarr
# If user has specified the `SZIP_LIBRARY`, use it; otherwise try to find...
IF(NOT SZIP_LIBRARY)
FIND_LIBRARY(SZIP PATH NAMES szip sz sz2)
IF(SZIP)
SET(SZIP_LIBRARY ${SZIP})
ELSE()
UNSET(SZIP_LIBRARY)
UNSET(SZIP)
ENDIF()
ENDIF()
IF(SZIP_LIBRARY)
SET(SZIP_FOUND yes)
SET(HAVE_SZ yes)
ELSE()
SET(SZIP_FOUND no)
SET(HAVE_SZ no)
ENDIF()
##
# Option to Enable HDF5
#
@ -905,15 +885,6 @@ IF(USE_HDF5)
int x = 1;}" USE_HDF5_SZIP)
IF(USE_HDF5_SZIP)
SET(HAVE_H5Z_SZIP yes)
# If user has specified the `SZIP_LIBRARY`, use it; otherwise try to find...
IF(SZIP_FOUND)
SET(CMAKE_REQUIRED_LIBRARIES ${SZIP_LIBRARY} ${CMAKE_REQUIRED_LIBRARIES})
MESSAGE(STATUS "HDF5 has szip.")
ELSE()
MESSAGE(FATAL_ERROR "HDF5 Requires SZIP, but cannot find libszip or libsz.")
ENDIF()
ELSE()
SET(HAVE_H5Z_SZIP no)
ENDIF()
####
@ -1114,31 +1085,53 @@ string(TOLOWER "${filter}" downfilter)
IF(${filter}_FOUND)
INCLUDE_DIRECTORIES(${filter}_INCLUDE_DIRS})
SET(ENABLE_${upfilter} TRUE)
SET(STD_FILTERS "${STD_FILTERS},${downfilter}")
SET(HAVE_${upfilter} ON)
SET(STD_FILTERS "${STD_FILTERS} ${downfilter}")
MESSAGE(">>> Standard Filter: ${downfilter}")
ELSE()
SET(ENABLE_${upfilter} FALSE)
SET(HAVE_${upfilter} OFF)
ENDIF()
endmacro(set_std_filter)
# Locate some compressors
FIND_PACKAGE(Szip)
FIND_PACKAGE(Bz2)
FIND_PACKAGE(Blosc)
FIND_PACKAGE(Zstd)
IF(Zstd_FOUND)
SET(HAVE_ZSTD yes)
ELSE()
SET(HAVE_ZSTD no)
ENDIF()
# Accumulate standard filters
set(STD_FILTERS "deflate") # Always have deflate */
set_std_filter(SZIP)
set(STD_FILTERS "deflate") # Always have deflate*/
set_std_filter(Szip)
SET(HAVE_SZ ${Szip_FOUND})
set_std_filter(Blosc)
set_std_filter(Zstd)
set_std_filter(Bz2)
IF(NOT Bz2_FOUND)
set(STD_FILTERS "${STD_FILTERS},bzip2") # Always have bzip2 */
IF(Bz2_FOUND)
set_std_filter(Bz2)
ELSE()
# The reason we use a local version is to support a more comples test case
MESSAGE(WARNING "libbz2 not found using built-in version")
SET(HAVE_LOCAL_BZ2 ON)
SET(HAVE_BZ2 ON)
set(STD_FILTERS "${STD_FILTERS} bz2")
ENDIF()
# If user wants, then install selected plugins
SET(PLUGIN_INSTALL_DIR "" CACHE STRING "Whether and where we should install plugins")
SET(ENABLE_PLUGIN_INSTALL OFF)
if(DEFINED PLUGIN_INSTALL_DIR OR DEFINED CACHE{PLUGIN_INSTALL_DIR})
IF(PLUGIN_INSTALL_DIR STREQUAL "")
MESSAGE(WARNING "No plugin directory value specified; option ignored.")
UNSET(PLUGIN_INSTALL_DIR)
UNSET(PLUGIN_INSTALL_DIR CACHE)
SET(PLUGIN_INSTALL_DIR_SETTING "N.A.")
ELSE()
SET(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}")
SET(ENABLE_PLUGIN_INSTALL ON)
ENDIF()
ELSE()
SET(PLUGIN_INSTALL_DIR_SETTING "N.A.")
ENDIF()
# See if we have libzip
@ -2528,6 +2521,10 @@ IF(ENABLE_NCZARR)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nczarr_test/findplugin.sh @ONLY NEWLINE_STYLE LF)
ENDIF()
IF(ENABLE_PLUGINS)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/plugins/findplugin.sh @ONLY NEWLINE_STYLE LF)
ENDIF()
IF(ENABLE_EXAMPLES)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF)
ENDIF()

2
RELEASE_NOTES.md
View File

@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.2 - TBD
* [Enhancement] Update the documentation to match the current filter capabilities See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249).
* [Enhancement] Support installation of pre-built standard filters into user-specified location. See [Github #2318](https://github.com/Unidata/netcdf-c/pull/2318).
* [Enhancement] Improve filter support. More specifically (1) add nc_inq_filter_avail to check if a filter is available, (2) add the notion of standard filters, (3) cleanup szip support to fix interaction with NCZarr. See [Github #2245](https://github.com/Unidata/netcdf-c/pull/2245).
* [Enhancement] Switch to tinyxml2 as the default xml parser implementation. See [Github #2170](https://github.com/Unidata/netcdf-c/pull/2170).
* [Bug Fix] Require that the type of the variable in nc_def_var_filter is not variable length. See [Github #/2231](https://github.com/Unidata/netcdf-c/pull/2231).

64
cmake/modules/FindBzip2.cmake Normal file
View File

@ -0,0 +1,64 @@
# Searches for an installation of the bzip2 library. On success, it sets the following variables:
#
# Bzip2_FOUND Set to true to indicate the bzip2 library was found
# Bzip2_INCLUDE_DIRS The directory containing the header file bzip2/bzip2.h
# Bzip2_LIBRARIES The libraries needed to use the bzip2 library
#
# To specify an additional directory to search, set Bzip2_ROOT.
#
# Author: Siddhartha Chaudhuri, 2009
#
# Look for the header, first in the user-specified location and then in the system locations
SET(Bzip2_INCLUDE_DOC "The directory containing the header file bzip2.h")
FIND_PATH(Bzip2_INCLUDE_DIRS NAMES bzip2.h bzip2/bzip2.h PATHS ${Bzip2_ROOT} ${Bzip2_ROOT}/include DOC ${Bzip2_INCLUDE_DOC} NO_DEFAULT_PATH)
IF(NOT Bzip2_INCLUDE_DIRS) # now look in system locations
FIND_PATH(Bzip2_INCLUDE_DIRS NAMES bzlib.h DOC ${Bzip2_INCLUDE_DOC})
ENDIF(NOT Bzip2_INCLUDE_DIRS)
SET(Bzip2_FOUND FALSE)
IF(Bzip2_INCLUDE_DIRS)
SET(Bzip2_LIBRARY_DIRS ${Bzip2_INCLUDE_DIRS})
IF("${Bzip2_LIBRARY_DIRS}" MATCHES "/include$")
# Strip off the trailing "/include" in the path.
GET_FILENAME_COMPONENT(Bzip2_LIBRARY_DIRS ${Bzip2_LIBRARY_DIRS} PATH)
ENDIF("${Bzip2_LIBRARY_DIRS}" MATCHES "/include$")
IF(EXISTS "${Bzip2_LIBRARY_DIRS}/lib")
SET(Bzip2_LIBRARY_DIRS ${Bzip2_LIBRARY_DIRS}/lib)
ENDIF(EXISTS "${Bzip2_LIBRARY_DIRS}/lib")
# Find Bzip2 libraries
FIND_LIBRARY(Bzip2_DEBUG_LIBRARY NAMES bzip2d bzip2_d libbzip2d libbzip2_d libbzip2
PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug
PATHS ${Bzip2_LIBRARY_DIRS} NO_DEFAULT_PATH)
FIND_LIBRARY(Bzip2_RELEASE_LIBRARY NAMES bzip2 libbzip2
PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release
PATHS ${Bzip2_LIBRARY_DIRS} NO_DEFAULT_PATH)
SET(Bzip2_LIBRARIES )
IF(Bzip2_DEBUG_LIBRARY AND Bzip2_RELEASE_LIBRARY)
SET(Bzip2_LIBRARIES debug ${Bzip2_DEBUG_LIBRARY} optimized ${Bzip2_RELEASE_LIBRARY})
ELSEIF(Bzip2_DEBUG_LIBRARY)
SET(Bzip2_LIBRARIES ${Bzip2_DEBUG_LIBRARY})
ELSEIF(Bzip2_RELEASE_LIBRARY)
SET(Bzip2_LIBRARIES ${Bzip2_RELEASE_LIBRARY})
ENDIF(Bzip2_DEBUG_LIBRARY AND Bzip2_RELEASE_LIBRARY)
IF(Bzip2_LIBRARIES)
SET(Bzip2_FOUND TRUE)
ENDIF(Bzip2_LIBRARIES)
ENDIF(Bzip2_INCLUDE_DIRS)
IF(Bzip2_FOUND)
# IF(NOT Bzip2_FIND_QUIETLY)
MESSAGE(STATUS "Found Bzip2: headers at ${Bzip2_INCLUDE_DIRS}, libraries at ${Bzip2_LIBRARY_DIRS}")
MESSAGE(STATUS " library is ${Bzip2_LIBRARIES}")
# ENDIF(NOT Bzip2_FIND_QUIETLY)
ELSE(Bzip2_FOUND)
IF(Bzip2_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Bzip2 library not found")
ENDIF(Bzip2_FIND_REQUIRED)
ENDIF(Bzip2_FOUND)

179
cmake/modules/FindSZIP.cmake
View File

@ -1,179 +0,0 @@
# - Find SZIP library
# - Derived from the FindTiff.cmake that is included with cmake
# Find the native SZIP includes and library
# This module defines
# SZIP_INCLUDE_DIRS, where to find tiff.h, etc.
# SZIP_LIBRARIES, libraries to link against to use SZIP.
# SZIP_FOUND, If false, do not try to use SZIP.
# also defined, but not for general use are
# SZIP_LIBRARY, where to find the SZIP library.
# SZIP_LIBRARY_DEBUG - Debug version of SZIP library
# SZIP_LIBRARY_RELEASE - Release Version of SZIP library
# MESSAGE (STATUS "Finding SZIP library and headers..." )
############################################
#
# Check the existence of the libraries.
#
############################################
# This macro was taken directly from the FindQt4.cmake file that is included
# with the CMake distribution. This is NOT my work. All work was done by the
# original authors of the FindQt4.cmake file. Only minor modifications were
# made to remove references to Qt and make this file more generally applicable
#########################################################################
MACRO (SZIP_ADJUST_LIB_VARS basename)
IF (${basename}_INCLUDE_DIR)
# if only the release version was found, set the debug variable also to the release version
IF (${basename}_LIBRARY_RELEASE AND NOT ${basename}_LIBRARY_DEBUG)
SET (${basename}_LIBRARY_DEBUG ${${basename}_LIBRARY_RELEASE})
SET (${basename}_LIBRARY ${${basename}_LIBRARY_RELEASE})
SET (${basename}_LIBRARIES ${${basename}_LIBRARY_RELEASE})
ENDIF (${basename}_LIBRARY_RELEASE AND NOT ${basename}_LIBRARY_DEBUG)
# if only the debug version was found, set the release variable also to the debug version
IF (${basename}_LIBRARY_DEBUG AND NOT ${basename}_LIBRARY_RELEASE)
SET (${basename}_LIBRARY_RELEASE ${${basename}_LIBRARY_DEBUG})
SET (${basename}_LIBRARY ${${basename}_LIBRARY_DEBUG})
SET (${basename}_LIBRARIES ${${basename}_LIBRARY_DEBUG})
ENDIF (${basename}_LIBRARY_DEBUG AND NOT ${basename}_LIBRARY_RELEASE)
IF (${basename}_LIBRARY_DEBUG AND ${basename}_LIBRARY_RELEASE)
# if the generator supports configuration types then set
# optimized and debug libraries, or if the CMAKE_BUILD_TYPE has a value
IF (CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE)
SET (${basename}_LIBRARY optimized ${${basename}_LIBRARY_RELEASE} debug ${${basename}_LIBRARY_DEBUG})
ELSE(CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE)
# if there are no configuration types and CMAKE_BUILD_TYPE has no value
# then just use the release libraries
SET (${basename}_LIBRARY ${${basename}_LIBRARY_RELEASE} )
ENDIF (CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE)
SET (${basename}_LIBRARIES optimized ${${basename}_LIBRARY_RELEASE} debug ${${basename}_LIBRARY_DEBUG})
ENDIF (${basename}_LIBRARY_DEBUG AND ${basename}_LIBRARY_RELEASE)
SET (${basename}_LIBRARY ${${basename}_LIBRARY} CACHE FILEPATH "The ${basename} library")
IF (${basename}_LIBRARY)
SET (${basename}_FOUND 1)
ENDIF (${basename}_LIBRARY)
ENDIF (${basename}_INCLUDE_DIR )
# Make variables changeble to the advanced user
MARK_AS_ADVANCED (${basename}_LIBRARY ${basename}_LIBRARY_RELEASE ${basename}_LIBRARY_DEBUG ${basename}_INCLUDE_DIR )
ENDMACRO (SZIP_ADJUST_LIB_VARS)
# Look for the header file.
SET (SZIP_INCLUDE_SEARCH_DIRS
$ENV{SZIP_INSTALL}/include
$ENV{SZIP_INSTALL}/include/szip
/usr/include
/usr/include/szip
)
SET (SZIP_LIB_SEARCH_DIRS
$ENV{SZIP_INSTALL}/lib
/usr/lib
)
SET (SZIP_BIN_SEARCH_DIRS
$ENV{SZIP_INSTALL}/bin
/usr/bin
)
FIND_PATH (SZIP_INCLUDE_DIR
NAMES szlib.h
PATHS ${SZIP_INCLUDE_SEARCH_DIRS}
NO_DEFAULT_PATH
)
IF (WIN32 AND NOT MINGW)
SET (SZIP_SEARCH_DEBUG_NAMES "sz_d;libsz_d")
SET (SZIP_SEARCH_RELEASE_NAMES "sz;libsz;szip")
ELSE (WIN32 AND NOT MINGW)
SET (SZIP_SEARCH_DEBUG_NAMES "sz_d")
SET (SZIP_SEARCH_RELEASE_NAMES "sz")
ENDIF (WIN32 AND NOT MINGW)
# Look for the library.
FIND_LIBRARY (SZIP_LIBRARY_DEBUG
NAMES ${SZIP_SEARCH_DEBUG_NAMES}
PATHS ${SZIP_LIB_SEARCH_DIRS}
NO_DEFAULT_PATH
)
FIND_LIBRARY (SZIP_LIBRARY_RELEASE
NAMES ${SZIP_SEARCH_RELEASE_NAMES}
PATHS ${SZIP_LIB_SEARCH_DIRS}
NO_DEFAULT_PATH
)
SZIP_ADJUST_LIB_VARS (SZIP)
IF (SZIP_INCLUDE_DIR AND SZIP_LIBRARY)
SET (SZIP_FOUND 1)
SET (SZIP_LIBRARIES ${SZIP_LIBRARY})
SET (SZIP_INCLUDE_DIRS ${SZIP_INCLUDE_DIR})
IF (SZIP_LIBRARY_DEBUG)
GET_FILENAME_COMPONENT (SZIP_LIBRARY_PATH ${SZIP_LIBRARY_DEBUG} PATH)
SET (SZIP_LIB_DIR ${SZIP_LIBRARY_PATH})
ELSEIF (SZIP_LIBRARY_RELEASE)
GET_FILENAME_COMPONENT (SZIP_LIBRARY_PATH ${SZIP_LIBRARY_RELEASE} PATH)
SET (SZIP_LIB_DIR ${SZIP_LIBRARY_PATH})
ENDIF (SZIP_LIBRARY_DEBUG)
ELSE (SZIP_INCLUDE_DIR AND SZIP_LIBRARY)
SET (SZIP_FOUND 0)
SET (SZIP_LIBRARIES)
SET (SZIP_INCLUDE_DIRS)
ENDIF (SZIP_INCLUDE_DIR AND SZIP_LIBRARY)
# Report the results.
IF (NOT SZIP_FOUND)
SET (SZIP_DIR_MESSAGE
"SZip was not found. Make sure SZIP_LIBRARY and SZIP_INCLUDE_DIR are set or set the SZIP_INSTALL environment variable."
)
IF (NOT SZIP_FIND_QUIETLY)
MESSAGE (STATUS "${SZIP_DIR_MESSAGE}")
ELSE (NOT SZIP_FIND_QUIETLY)
IF (SZIP_FIND_REQUIRED)
MESSAGE (FATAL_ERROR "SZip was NOT found and is Required by this project")
ENDIF (SZIP_FIND_REQUIRED)
ENDIF (NOT SZIP_FIND_QUIETLY)
ENDIF (NOT SZIP_FOUND)
IF (SZIP_FOUND)
INCLUDE (CheckSymbolExists)
#############################################
# Find out if SZIP was build using dll's
#############################################
# Save required variable
SET (CMAKE_REQUIRED_INCLUDES_SAVE ${CMAKE_REQUIRED_INCLUDES})
SET (CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})
# Add SZIP_INCLUDE_DIR to CMAKE_REQUIRED_INCLUDES
SET (CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES};${SZIP_INCLUDE_DIRS}")
CHECK_SYMBOL_EXISTS (SZIP_BUILT_AS_DYNAMIC_LIB "SZconfig.h" HAVE_SZIP_DLL)
IF (HAVE_SZIP_DLL STREQUAL "TRUE")
SET (HAVE_SZIP_DLL "1")
ENDIF (HAVE_SZIP_DLL STREQUAL "TRUE")
# Restore CMAKE_REQUIRED_INCLUDES and CMAKE_REQUIRED_FLAGS variables
SET (CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_SAVE})
SET (CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})
#
#############################################
ENDIF (SZIP_FOUND)
IF (FIND_SZIP_DEBUG)
MESSAGE (STATUS "SZIP_INCLUDE_DIR: ${SZIP_INCLUDE_DIR}")
MESSAGE (STATUS "SZIP_INCLUDE_DIRS: ${SZIP_INCLUDE_DIRS}")
MESSAGE (STATUS "SZIP_LIBRARY_DEBUG: ${SZIP_LIBRARY_DEBUG}")
MESSAGE (STATUS "SZIP_LIBRARY_RELEASE: ${SZIP_LIBRARY_RELEASE}")
MESSAGE (STATUS "HAVE_SZIP_DLL: ${HAVE_SZIP_DLL}")
MESSAGE (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
ENDIF (FIND_SZIP_DEBUG)

64
cmake/modules/FindSzip.cmake Normal file
View File

@ -0,0 +1,64 @@
# Searches for an installation of the szip library. On success, it sets the following variables:
#
# Szip_FOUND Set to true to indicate the szip library was found
# Szip_INCLUDE_DIRS The directory containing the header file szip/szip.h
# Szip_LIBRARIES The libraries needed to use the szip library
#
# To specify an additional directory to search, set Szip_ROOT.
#
# Author: Siddhartha Chaudhuri, 2009
#
# Look for the header, first in the user-specified location and then in the system locations
SET(Szip_INCLUDE_DOC "The directory containing the header file szip.h")
FIND_PATH(Szip_INCLUDE_DIRS NAMES szlib.h szip.h szip/szip.h PATHS ${Szip_ROOT} ${Szip_ROOT}/include DOC ${Szip_INCLUDE_DOC} NO_DEFAULT_PATH)
IF(NOT Szip_INCLUDE_DIRS) # now look in system locations
FIND_PATH(Szip_INCLUDE_DIRS NAMES szlib.h szip.h szip/szip.h DOC ${Szip_INCLUDE_DOC})
ENDIF(NOT Szip_INCLUDE_DIRS)
SET(Szip_FOUND FALSE)
IF(Szip_INCLUDE_DIRS)
SET(Szip_LIBRARY_DIRS ${Szip_INCLUDE_DIRS})
IF("${Szip_LIBRARY_DIRS}" MATCHES "/include$")
# Strip off the trailing "/include" in the path.
GET_FILENAME_COMPONENT(Szip_LIBRARY_DIRS ${Szip_LIBRARY_DIRS} PATH)
ENDIF("${Szip_LIBRARY_DIRS}" MATCHES "/include$")
IF(EXISTS "${Szip_LIBRARY_DIRS}/lib")
SET(Szip_LIBRARY_DIRS ${Szip_LIBRARY_DIRS}/lib)
ENDIF(EXISTS "${Szip_LIBRARY_DIRS}/lib")
# Find Szip libraries
FIND_LIBRARY(Szip_DEBUG_LIBRARY NAMES szipd szip_d libszipd libszip_d szip libszip sz2 libsz2
PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug
PATHS ${Szip_LIBRARY_DIRS} NO_DEFAULT_PATH)
FIND_LIBRARY(Szip_RELEASE_LIBRARY NAMES szip libszip sz libsz sz2 libsz2
PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release
PATHS ${Szip_LIBRARY_DIRS} NO_DEFAULT_PATH)
SET(Szip_LIBRARIES )
IF(Szip_DEBUG_LIBRARY AND Szip_RELEASE_LIBRARY)
SET(Szip_LIBRARIES debug ${Szip_DEBUG_LIBRARY} optimized ${Szip_RELEASE_LIBRARY})
ELSEIF(Szip_DEBUG_LIBRARY)
SET(Szip_LIBRARIES ${Szip_DEBUG_LIBRARY})
ELSEIF(Szip_RELEASE_LIBRARY)
SET(Szip_LIBRARIES ${Szip_RELEASE_LIBRARY})
ENDIF(Szip_DEBUG_LIBRARY AND Szip_RELEASE_LIBRARY)
IF(Szip_LIBRARIES)
SET(Szip_FOUND TRUE)
ENDIF(Szip_LIBRARIES)
ENDIF(Szip_INCLUDE_DIRS)
IF(Szip_FOUND)
# IF(NOT Szip_FIND_QUIETLY)
MESSAGE(STATUS "Found Szip: headers at ${Szip_INCLUDE_DIRS}, libraries at ${Szip_LIBRARY_DIRS}")
MESSAGE(STATUS " library is ${Szip_LIBRARIES}")
# ENDIF(NOT Szip_FIND_QUIETLY)
ELSE(Szip_FOUND)
IF(Szip_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Szip library not found")
ENDIF(Szip_FIND_REQUIRED)
ENDIF(Szip_FOUND)

2
config.h.cmake.in
View File

@ -173,7 +173,7 @@ are set when opening a binary file on Windows. */
#cmakedefine HAVE_ATEXIT 1
/* Define to 1 if bzip2 library available. */
#cmakedefine HAVE_BZIP2 1
#cmakedefine HAVE_BZ2 1
/* Define to 1 if zstd library available. */
#cmakedefine HAVE_ZSTD 1

58
configure.ac
View File

@ -61,6 +61,7 @@ AC_CONFIG_HEADERS([config.h])
##
SAVE_CFLAGS="${CFLAGS}"
AC_LANG_PUSH([C])
AC_LANG_COMPILER_REQUIRE
CFLAGS="${CFLAGS} -fno-strict-aliasing"
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
@ -667,24 +668,21 @@ fi
AC_MSG_CHECKING([whether libzstd library is available])
AC_MSG_RESULT([${have_zstd}])
# See if we have libbzip2 or libbz2
AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no])
if test "x$have_bzip2" = "xyes" ; then
# See if we have libbz2
AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bz2=yes],[have_bz2=no])
if test "x$have_bz2" = "xyes" ; then
AC_SEARCH_LIBS([BZ2_bzCompress],[bz2 bz2.dll cygbz2.dll], [], [])
AC_DEFINE([HAVE_BZIP2], [1], [if true, bz2 library is installed])
AC_DEFINE([HAVE_BZ2], [1], [if true, bz2 library is installed])
fi
AC_MSG_CHECKING([whether libbz2 library is available])
AC_MSG_RESULT([${have_bzip2}])
AC_MSG_RESULT([${have_bz2}])
if test "x$have_bzip2" = "xno" ; then
AC_CHECK_LIB([bzip2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no])
if test "x$have_bzip2" = "xyes" ; then
AC_SEARCH_LIBS([BZ2_bzCompress],[bzip2 bzip2.dll cygbzip2.dll], [], [])
AC_DEFINE([HAVE_BZIP2], [1], [if true, bzip2 library is installed])
fi
AC_MSG_CHECKING([whether libbzip2 library is available])
AC_MSG_RESULT([${have_bzip2}])
if test "x$have_bz2" = "xno" ; then
have_local_bz2=yes
else
have_local_bz2=no
fi
AM_CONDITIONAL(HAVE_LOCAL_BZ2, [test "x$have_local_bz2" = xyes])
# Note that szip management is tricky.
# This is because we have three things to consider:
@ -1753,7 +1751,7 @@ AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes])
AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes])
AM_CONDITIONAL(HAVE_BLOSC, [test "x$have_blosc" = xyes])
AM_CONDITIONAL(HAVE_BZIP2, [test "x$have_bzip2" = xyes])
AM_CONDITIONAL(HAVE_BZ2, [test "x$have_bz2" = xyes])
AM_CONDITIONAL(HAVE_ZSTD, [test "x$have_zstd" = xyes])
# If the machine doesn't have a long long, and we want netCDF-4, then
@ -1864,26 +1862,45 @@ AC_SUBST(HAS_SZLIB_WRITE, [$have_sz])
AC_SUBST(HAS_ZSTD,[$have_zstd])
# Always available
std_filters="deflate,bzip2"
std_filters="deflate bz2"
if test "x$enable_szlib" = xyes ; then
std_filters="${std_filters},szip"
std_filters="${std_filters} szip"
fi
# We need to include szip iff HDF5 && HDF5_HAS_SZIP || !HDF5 && NCZARR && libsz
if test "x$enable_hdf5" = xyes && test "x$enable_hdf5_szip" = xyes ; then
std_filters="${std_filters},szip"
std_filters="${std_filters} szip"
fi
if test "x$enable_hdf5" = xno && test "x$have_sz" = xyes ; then
std_filters="${std_filters},szip"
std_filters="${std_filters} szip"
fi
if test "x$have_blosc" = xyes ; then
std_filters="${std_filters},blosc"
std_filters="${std_filters} blosc"
fi
if test "x$have_zstd" = xyes ; then
std_filters="${std_filters},zst"
std_filters="${std_filters} zstd"
fi
AC_SUBST(STD_FILTERS,[$std_filters])
# If user wants, then install selected standard filters
AC_MSG_CHECKING([whether and where we should install plugins])
AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=<absolute directory>],
[Install selected standard filters in specified directory])])
AC_MSG_RESULT([$with_plugin_dir])
if test "x$with_plugin_dir" = x ; then
AC_MSG_WARN([No plugin directory value specified; option ignored.])
with_plugin_dir=
with_plugin_dir_setting="N.A."
enable_plugin_dir=no
else
with_plugin_dir_setting="$with_plugin_dir"
enable_plugin_dir=yes
fi
AM_CONDITIONAL([ENABLE_PLUGIN_DIR], [test "x$enable_plugin_dir" = xyes])
AC_SUBST([PLUGIN_INSTALL_DIR], [$with_plugin_dir])
# Better value for libnetcdf.settings
AC_SUBST([PLUGIN_INSTALL_DIR_SETTING], [$with_plugin_dir_setting])
# Access netcdf specific version of config.h
AH_BOTTOM([#include "ncconfigure.h"])
@ -1973,6 +1990,7 @@ AC_MSG_NOTICE([generating header files and makefiles])
AC_CONFIG_FILES(test_common.sh:test_common.in)
AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(plugins/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in)
AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)

3
docs/Doxyfile.in
View File

@ -753,7 +753,8 @@ INPUT = \
@abs_top_srcdir@/docs/COPYRIGHT.md \
@abs_top_srcdir@/docs/credits.md \
@abs_top_srcdir@/docs/tutorial.dox \
@abs_top_srcdir@/docs/internal.dox \
@abs_top_srcdir@/docs/internal.md \
@abs_top_srcdir@/docs/dispatch.md \
@abs_top_srcdir@/docs/inmeminternal.dox \
@abs_top_srcdir@/docs/indexing.dox \
@abs_top_srcdir@/docs/testserver.dox \

15
docs/FAQ.md
View File

@ -1079,9 +1079,22 @@ and writable by programs that used older versions of the libraries.
However, programs linked to older library versions will not be able to
create new data objects with the new less-restrictive names.
How difficult is it to convert my application to handle arbitrary netCDF-4 files? {#How-difficult-is-it-to-convert-my-application-to-handle-arbitrary-netCDF-4-files}
Can I use UTF-8 File Names with Windows? {#Can-I-use-UTF-8-File-Names-with-Windows}
-----------------
Starting with Windows 10 build 17134, Windows can support use of
the UTF-8 character set. We strongly encourage Windows users to
enable this feature. This requires the following steps.
1. In the "run" toolbar, execute the command "intl.cpl".
2. Move to the Administrative tab.
3. Move to "Change system locale"
4. Check the box at the bottom labeled something like
"Beta: Use Unicode UTF-8 for worldwide language support"
How difficult is it to convert my application to handle arbitrary netCDF-4 files? {#How-difficult-is-it-to-convert-my-application-to-handle-arbitrary-netCDF-4-files}
-----------------
Modifying an application to fully support the new enhanced data model
may be relatively easy or arbitrarily difficult :-), depending on what

2
docs/Makefile.am
View File

@ -9,7 +9,7 @@
# These files will be included with the dist.
EXTRA_DIST = netcdf.m4 DoxygenLayout.xml Doxyfile.in footer.html \
mainpage.dox tutorial.dox architecture.dox internal.dox \
windows-binaries.md building-with-cmake.md CMakeLists.txt groups.dox \
windows-binaries.md dispatch.md building-with-cmake.md CMakeLists.txt groups.dox \
notes.md install-fortran.md credits.md auth.md filters.md \
obsolete/fan_utils.html indexing.dox inmemory.md FAQ.md \
known_problems.md COPYRIGHT.md inmeminternal.dox testserver.dox \

507
docs/dispatch.md Normal file
View File

@ -0,0 +1,507 @@
Internal Dispatch Table Architecture
============================
<!-- double header is needed to workaround doxygen bug -->
# Internal Dispatch Table Architecture
\tableofcontents
# Introduction {#dispatch_intro}
The netcdf-c library uses an internal dispatch mechanism
as the means for wrapping the netcdf-c API around a wide variety
of underlying storage and stream data formats.
As of last check, the following formats are supported and each
has its own dispatch table.
Warning: some of the listed function signatures may be out of date
and the specific code should be consulted to see the actual parameters.
<table>
<tr><th>Format<td>Directory<th>NC_FORMATX Name
<tr><td>NetCDF-classic<td>libsrc<td>NC_FORMATX_NC3
<tr><td>NetCDF-enhanced<td>libhdf5<td>NC_FORMATX_NC_HDF5
<tr><td>HDF4<td>libhdf4<td>NC_FORMATX_NC_HDF4
<tr><td>PNetCDF<td>libsrcp<td>NC_FORMATX_PNETCDF
<tr><td>DAP2<td>libdap2<td>NC_FORMATX_DAP2
<tr><td>DAP4<td>libdap4<td>NC_FORMATX_DAP4
<tr><td>UDF0<td>N.A.<td>NC_FORMATX_UDF0
<tr><td>UDF1<td>N.A.<td>NC_FORMATX_UDF1
<tr><td>NCZarr<td>libnczarr<td>NC_FORMATX_NCZARR
</table>
Note that UDF0 and UDF1 allow for user-defined dispatch tables to
be implemented.
The idea is that when a user opens or creates a netcdf file, a
specific dispatch table is chosen. A dispatch table is a struct
containing an entry for (almost) every function in the netcdf-c API.
During execution, netcdf API calls are channeled through that
dispatch table to the appropriate function for implementing that
API call. The functions in the dispatch table are not quite the
same as those defined in *netcdf.h*. For simplicity and
compactness, some netcdf.h API calls are mapped to the same
dispatch table function. In addition to the functions, the first
entry in the table defines the model that this dispatch table
implements. It will be one of the NC_FORMATX_XXX values.
The second entry in the table is the version of the dispatch table.
The rule is that previous entries may not be removed, but new entries
may be added, and adding new entries increases the version number.
The dispatch table represents a distillation of the netcdf API down to
a minimal set of internal operations. The format of the dispatch table
is defined in the file *libdispatch/ncdispatch.h*. Every new dispatch
table must define this minimal set of operations.
# Adding a New Dispatch Table
In order to make this process concrete, let us assume we plan to add
an in-memory implementation of netcdf-3.
## Defining configure.ac flags
Define a *–-enable* flag option for *configure.ac*. For our
example, we assume the option "--enable-ncm" and the
internal corresponding flag "enable_ncm". If you examine the existing
*configure.ac* and see how, for example, *--enable_dap2* is
defined, then it should be clear how to do it for your code.
## Defining a "name space"
Choose some prefix of characters to identify the new dispatch
system. In effect we are defining a name-space. For our in-memory
system, we will choose "NCM" and "ncm". NCM is used for non-static
procedures to be entered into the dispatch table and ncm for all other
non-static procedures. Note that the chosen prefix should probably start
with "nc" or "NC" in order to avoid name conflicts outside the netcdf-c library.
## Extend include/netcdf.h
Modify the file *include/netcdf.h* to add an NC_FORMATX_XXX flag
by adding a flag for this dispatch format at the appropriate places.
````
#define NC_FORMATX_NCM 7
````
Add any format specific new error codes.
````
#define NC_ENCM (?)
````
## Extend include/ncdispatch.h
Modify the file *include/ncdispatch.h* to
add format specific data and initialization functions;
note the use of our NCM namespace.
````
#ifdef ENABLE_NCM
extern NC_Dispatch* NCM_dispatch_table;
extern int NCM_initialize(void);
#endif
````
## Define the dispatch table functions
Define the functions necessary to fill in the dispatch table. As a
rule, we assume that a new directory is defined, *libsrcm*, say. Within
this directory, we need to define *Makefile.am* and *CMakeLists.txt*.
We also need to define the source files
containing the dispatch table and the functions to be placed in the
dispatch table -– call them *ncmdispatch.c* and *ncmdispatch.h*. Look at
*libsrc/nc3dispatch.[ch]* or *libnczarr/zdispatch.[ch]* for examples.
Similarly, it is best to take existing *Makefile.am* and *CMakeLists.txt*
files (from *libsrcp* for example) and modify them.
## Adding the dispatch code to libnetcdf
Provide for the inclusion of this library in the final libnetcdf
library. This is accomplished by modifying *liblib/Makefile.am* by
adding something like the following.
````
if ENABLE_NCM
libnetcdf_la_LIBADD += $(top_builddir)/libsrcm/libnetcdfm.la
endif
````
## Extend library initialization
Modify the *NC_initialize* function in *liblib/nc_initialize.c* by adding
appropriate references to the NCM dispatch function.
````
#ifdef ENABLE_NCM
extern int NCM_initialize(void);
#endif
...
int NC_initialize(void)
{
...
#ifdef ENABLE_NCM
if((stat = NCM_initialize())) return stat;
#endif
...
}
````
Finalization is handled in an analogous fashion.
## Testing the new dispatch table
Add a directory of tests: *ncm_test*, say. The file *ncm_test/Makefile.am*
will look something like this.
````
# These files are created by the tests.
CLEANFILES = ...
# These are the tests which are always run.
TESTPROGRAMS = test1 test2 ...
test1_SOURCES = test1.c ...
...
# Set up the tests.
check_PROGRAMS = $(TESTPROGRAMS)
TESTS = $(TESTPROGRAMS)
# Any extra files required by the tests
EXTRA_DIST = ...
````
# Top-Level build of the dispatch code
Provide for *libnetcdfm* to be constructed by adding the following to
the top-level *Makefile.am*.
````
if ENABLE_NCM
NCM=libsrcm
NCMTESTDIR=ncm_test
endif
...
SUBDIRS = ... $(DISPATCHDIR) $(NCM) ... $(NCMTESTDIR)
````
# Choosing a Dispatch Table
The dispatch table is ultimately chosen by the function
NC_infermodel() in libdispatch/dinfermodel.c. This function is
invoked by the NC_create and the NC_open procedures. This can
be, unfortunately, a complex process. The detailed operation of
NC_infermodel() is defined in the companion document in docs/dinternal.md.
In any case, the choice of dispatch table is currently based on the following
pieces of information.
1. The mode argument – this can be used to detect, for example, what kind
of file to create: netcdf-3, netcdf-4, 64-bit netcdf-3, etc.
Using a mode flag is the most common mechanism, in which case
*netcdf.h* needs to be modified to define the relevant mode flag.
2. The file path – this can be used to detect, for example, a DAP url
versus a normal file system file. If the path looks like a URL, then
the fragment part of the URL is examined to determine the specific
dispatch function.
3. The file contents - when the contents of a real file are available,
the contents of the file can be used to determine the dispatch table.
As a rule, this is likely to be useful only for *nc_open*.
4. If the file is being opened vs being created.
5. Is parallel IO available?
The *NC_infermodel* function returns two values.
1. model - this is used by nc_open and nc_create to choose the dispatch table.
2. newpath - in some case, usually URLS, the path may be rewritten to include extra information for use by the dispatch functions.
# Special Dispatch Table Signatures.
The entries in the dispatch table do not necessarily correspond
to the external API. In many cases, multiple related API functions
are merged into a single dispatch table entry.
## Create/Open
The create table entry and the open table entry in the dispatch table
have the following signatures respectively.
````
int (*create)(const char *path, int cmode,
size_t initialsz, int basepe, size_t *chunksizehintp,
int useparallel, void* parameters,
struct NC_Dispatch* table, NC* ncp);
int (*open)(const char *path, int mode,
int basepe, size_t *chunksizehintp,
int use_parallel, void* parameters,
struct NC_Dispatch* table, NC* ncp);
````
The key difference is that these are the union of all the possible
create/open signatures from the include/netcdfXXX.h files. Note especially the last
three parameters. The parameters argument is a pointer to arbitrary data
to provide extra info to the dispatcher.
The table argument is included in case the create
function (e.g. *NCM_create_) needs to invoke other dispatch
functions. The very last argument, ncp, is a pointer to an NC
instance. The raw NC instance will have been created by *libdispatch/dfile.c*
and is passed to e.g. open with the expectation that it will be filled in
by the dispatch open function.
## Accessing Data with put_vara() and get_vara()
````
int (*put_vara)(int ncid, int varid, const size_t *start, const size_t *count,
const void *value, nc_type memtype);
````
````
int (*get_vara)(int ncid, int varid, const size_t *start, const size_t *count,
void *value, nc_type memtype);
````
Most of the parameters are similar to the netcdf API parameters. The
last parameter, however, is the type of the data in
memory. Additionally, instead of using an "int islong" parameter, the
memtype will be either ::NC_INT or ::NC_INT64, depending on the value
of sizeof(long). This means that even netcdf-3 code must be prepared
to encounter the ::NC_INT64 type.
## Accessing Attributes with put_attr() and get_attr()
````
int (*get_att)(int ncid, int varid, const char *name,
void *value, nc_type memtype);
````
````
int (*put_att)(int ncid, int varid, const char *name, nc_type datatype, size_t len,
const void *value, nc_type memtype);
````
Again, the key difference is the memtype parameter. As with
put/get_vara, it used ::NC_INT64 to encode the long case.
## Pre-defined Dispatch Functions
It is sometimes not necessary to implement all the functions in the
dispatch table. Some pre-defined functions are available which may be
used in many cases.
## Inquiry Functions
Many of The netCDF inquiry functions operate from an in-memory model of
metadata. Once a file is opened, or a file is created, this
in-memory metadata model is kept up to date. Consequenty the inquiry
functions do not depend on the dispatch layer code. These functions
can be used by all dispatch layers which use the internal netCDF
enhanced data model.
- NC4_inq
- NC4_inq_type
- NC4_inq_dimid
- NC4_inq_dim
- NC4_inq_unlimdim
- NC4_inq_att
- NC4_inq_attid
- NC4_inq_attname
- NC4_get_att
- NC4_inq_varid
- NC4_inq_var_all
- NC4_show_metadata
- NC4_inq_unlimdims
- NC4_inq_ncid
- NC4_inq_grps
- NC4_inq_grpname
- NC4_inq_grpname_full
- NC4_inq_grp_parent
- NC4_inq_grp_full_ncid
- NC4_inq_varids
- NC4_inq_dimids
- NC4_inq_typeids
- NC4_inq_type_equal
- NC4_inq_user_type
- NC4_inq_typeid
## NCDEFAULT get/put Functions
The mapped (varm) get/put functions have been
implemented in terms of the array (vara) functions. So dispatch layers
need only implement the vara functions, and can use the following
functions to get the and varm functions:
- NCDEFAULT_get_varm
- NCDEFAULT_put_varm
For the netcdf-3 format, the strided functions (nc_get/put_vars)
are similarly implemented in terms of the vara functions. So the following
convenience functions are available.
- NCDEFAULT_get_vars
- NCDEFAULT_put_vars
For the netcdf-4 format, the vars functions actually exist, so
the default vars functions are not used.
## Read-Only Functions
Some dispatch layers are read-only (ex. HDF4). Any function which
writes to a file, including nc_create(), needs to return error code
::NC_EPERM. The following read-only functions are available so that
these don't have to be re-implemented in each read-only dispatch layer:
- NC_RO_create
- NC_RO_redef
- NC_RO__enddef
- NC_RO_sync
- NC_RO_set_fill
- NC_RO_def_dim
- NC_RO_rename_dim
- NC_RO_rename_att
- NC_RO_del_att
- NC_RO_put_att
- NC_RO_def_var
- NC_RO_rename_var
- NC_RO_put_vara
- NC_RO_def_var_fill
## Classic NetCDF Only Functions
There are two functions that are only used in the classic code. All
other dispatch layers (except PnetCDF) return error ::NC_ENOTNC3 for
these functions. The following functions are provided for this
purpose:
- NOTNC3_inq_base_pe
- NOTNC3_set_base_pe
# HDF4 Dispatch Layer as a Simple Example
The HDF4 dispatch layer is about the simplest possible dispatch
layer. It is read-only, classic model. It will serve as a nice, simple
example of a dispatch layer.
Note that the HDF4 layer is optional in the netCDF build. Not all
users will have HDF4 installed, and those users will not build with
the HDF4 dispatch layer enabled. For this reason HDF4 code is guarded
as follows.
````
#ifdef USE_HDF4
...
#endif /*USE_HDF4*/
````
Code in libhdf4 is only compiled if HDF4 is
turned on in the build.
### The netcdf.h File
In the main netcdf.h file, we have the following:
````
#define NC_FORMATX_NC_HDF4 (3)
````
### The ncdispatch.h File
In ncdispatch.h we have the following:
````
#ifdef USE_HDF4
extern NC_Dispatch* HDF4_dispatch_table;
extern int HDF4_initialize(void);
extern int HDF4_finalize(void);
#endif
````
### The netcdf_meta.h File
The netcdf_meta.h file allows for easy determination of what features
are in use. For HDF4, It contains the following, set by configure:
````
...
#define NC_HAS_HDF4 0 /*!< HDF4 support. */
...
````
### The hdf4dispatch.h File
The file *hdf4dispatch.h* contains prototypes and
macro definitions used within the HDF4 code in libhdf4. This include
file should not be used anywhere except in libhdf4.
### Initialization Code Changes in liblib Directory
The file *nc_initialize.c* is modified to include the following:
````
#ifdef USE_HDF4
extern int HDF4_initialize(void);
extern int HDF4_finalize(void);
#endif
````
### Changes to libdispatch/dfile.c
In order for a dispatch layer to be used, it must be correctly
determined in functions *NC_open()* or *NC_create()* in *libdispatch/dfile.c*.
HDF4 has a magic number that is detected in
*NC_interpret_magic_number()*, which allows *NC_open* to automatically
detect an HDF4 file.
Once HDF4 is detected, the *model* variable is set to *NC_FORMATX_NC_HDF4*,
and later this is used in a case statement:
````
case NC_FORMATX_NC_HDF4:
dispatcher = HDF4_dispatch_table;
break;
````
This sets the dispatcher to the HDF4 dispatcher, which is defined in
the libhdf4 directory.
### Dispatch Table in libhdf4/hdf4dispatch.c
The file *hdf4dispatch.c* contains the definition of the HDF4 dispatch
table. It looks like this:
````
/* This is the dispatch object that holds pointers to all the
* functions that make up the HDF4 dispatch interface. */
static NC_Dispatch HDF4_dispatcher = {
NC_FORMATX_NC_HDF4,
NC_DISPATCH_VERSION,
NC_RO_create,
NC_HDF4_open,
NC_RO_redef,
NC_RO__enddef,
NC_RO_sync,
...
NC_NOTNC4_set_var_chunk_cache,
NC_NOTNC4_get_var_chunk_cache,
...
};
````
Note that most functions use some of the predefined dispatch
functions. Functions that start with NC_RO* are read-only, they return
::NC_EPERM. Functions that start with NOTNC4* return ::NC_ENOTNC4.
Only the functions that start with NC_HDF4* need to be implemented for
the HDF4 dispatch layer. There are 6 such functions:
- NC_HDF4_open
- NC_HDF4_abort
- NC_HDF4_close
- NC_HDF4_inq_format
- NC_HDF4_inq_format_extended
- NC_HDF4_get_vara
### HDF4 Reading Code
The code in *hdf4file.c* opens the HDF4 SD dataset, and reads the
metadata. This metadata is stored in the netCDF internal metadata
model, allowing the inq functions to work.
The code in *hdf4var.c* does an *nc_get_vara()* on the HDF4 SD
dataset. This is all that is needed for all the nc_get_* functions to
work.
# Point of Contact {#filters_poc}
*Author*: Dennis Heimbigner<br>
*Email*: dmh at ucar dot edu<br>
*Initial Version*: 12/22/2021<br>
*Last Revised*: 12/22/2021

720
docs/filters.md
View File

File diff suppressed because it is too large Load Diff

639
docs/internal.md Normal file
View File

@ -0,0 +1,639 @@
Notes On the Internals of the NetCDF-C Library
============================
<!-- double header is needed to workaround doxygen bug -->
# Notes On the Internals of the NetCDF-C Library {#intern_head}
\tableofcontents
This document attempts to record important information about
the internal architecture and operation of the netcdf-c library.
# 1. Including C++ Code in the netcdf-c Library {#intern_c++}
The state of C compiler technology has reached the point where
it is possible to include C++ code into the netcdf-c library
code base. Two examples are:
1. The AWS S3 SDK wrapper *libdispatch/ncs3sdk.cpp* file.
2. The TinyXML wrapper *ncxml\_tinyxml2.cpp* file.
However there are some consequences that must be handled for this to work.
Specifically, the compiler must be told that the C++ runtime is needed
in the following ways.
## Modifications to *lib\_flags.am*
Suppose we have a flag *ENABLE\_XXX* where that XXX
feature entails using C++ code. Then the following must be added
to *lib\_flags.am*
````
if ENABLE_XXX
AM_LDFLAGS += -lstdc++
endif
````
## Modifications to *libxxx/Makefile.am*
The Makefile in which the C++ code is included and compiled
(assumed here to be the *libxxx* directory) must have this set.
````
AM_CXXFLAGS = -std=c++11
````
It is possible that other values (e.g. *-std=c++14*) may also work.
# 2. Managing instances of complex data types
For a long time, there have been known problems with the
management of complex types containing VLENs. This also
involves the string type because it is stored as a VLEN of
chars.
The term complex type refers to any type that directly or
recursively references a VLEN type. So an array of VLENS, a
compound with a VLEN field, and so on.
In order to properly handle instances of these complex types, it
is necessary to have function that can recursively walk
instances of such types to perform various actions on them. The
term "deep" is also used to mean recursive.
Two deep walking operations are provided by the netcdf-c library
to aid in managing instances of complex structures.
* free'ing an instance of the complex type
* copying an instance of the complex type.
Previously The netcdf-c library only did shallow free and shallow copy of
complex types. This meant that only the top level was properly
free'd or copied, but deep internal blocks in the instance were
not touched. This led to a host of memory leaks and failures
when the deep data was effectively shared between the netcdf-c library
internally and the user's data.
Note that the term "vector" is used to mean a contiguous (in
memory) sequence of instances of some type. Given an array with,
say, dimensions 2 X 3 X 4, this will be stored in memory as a
vector of length 2*3*4=24 instances.
The use cases are primarily these.
## nc\_get\_vars
Suppose one is reading a vector of instances using nc\_get\_vars
(or nc\_get\_vara or nc\_get\_var, etc.). These functions will
return the vector in the top-level memory provided. All
interior blocks (form nested VLEN or strings) will have been
dynamically allocated. Note that computing the size of the vector
may be tricky because the strides must be taken into account.
After using this vector of instances, it is necessary to free
(aka reclaim) the dynamically allocated memory, otherwise a
memory leak occurs. So, the recursive reclaim function is used
to walk the returned instance vector and do a deep reclaim of
the data.
Currently functions are defined in netcdf.h that are supposed to
handle this: nc\_free\_vlen(), nc\_free\_vlens(), and
nc\_free\_string(). Unfortunately, these functions only do a
shallow free, so deeply nested instances are not properly
handled by them. They are marked in the description as
deprecated in favor of the newer recursive function.
## nc\_put\_vars
Suppose one is writing a vector of instances using nc\_put\_vars
(or nc\_put\_vara or nc\_put\_var, etc.). These functions will
write the contents of the vector to the specified variable.
Note that internally, the data passed to the nc\_put\_xxx function is
immediately written so there is no need to copy it internally. But the
caller may need to reclaim the vector of data that was created and passed
in to the nc\_put\_xxx function.
After writing this vector of instances, and assuming it was dynamically
created, at some point it will be necessary to reclaim that data.
So again, the recursive reclaim function can be used
to walk the returned instance vector and do a deep reclaim of
the data.
## nc\_put\_att
Suppose one is writing a vector of instances as the data of an attribute
using, say, nc\_put\_att.
Internally, the incoming attribute data must be copied and stored
so that changes/reclamation of the input data will not affect
the attribute. Note that this copying behavior is different from
writing to a variable, where the data is written immediately.
Again, the code inside the netcdf library used to use only shallow copying
rather than deep copy. As a result, one saw effects such as described
in Github Issue https://github.com/Unidata/netcdf-c/issues/2143.
Also, after defining the attribute, it may be necessary for the user
to free the data that was provided as input to nc\_put\_att() as in the
nc\_put\_xxx functions (previously described).
## nc\_get\_att
Suppose one is reading a vector of instances as the data of an attribute
using, say, nc\_get\_att.
Internally, the existing attribute data must be copied and returned
to the caller, and the caller is responsible for reclaiming
the returned data.
Again, the code inside the netcdf library used to only do shallow copying
rather than deep copy. So this could lead to memory leaks and errors
because the deep data was shared between the library and the user.
## New Instance Walking API
Proper recursive functions were added to the netcdf-c library to
provide reclaim and copy functions and use those as needed.
These functions are defined in libdispatch/dinstance.c and their
signatures are defined in include/netcdf.h. For back
compatibility, corresponding "ncaux\_XXX" functions are defined
in include/netcdf\_aux.h.
````
int nc_reclaim_data(int ncid, nc_type xtypeid, void* memory, size_t count);
int nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t count);
int nc_copy_data(int ncid, nc_type xtypeid, const void* memory, size_t count, void* copy);
int nc_copy_data_all(int ncid, nc_type xtypeid, const void* memory, size_t count, void** copyp);
````
There are two variants. The first two, nc\_reclaim\_data() and
nc\_copy\_data(), assume the top-level vector is managed by the
caller. For reclaim, this is so the user can use, for example, a
statically allocated vector. For copy, it assumes the user
provides the space into which the copy is stored.
The second two, nc\_reclaim\_data\_all() and
nc\_copy\_data\_all(), allows the functions to manage the
top-level. So for nc\_reclaim\_data\_all, the top level is
assumed to be dynamically allocated and will be free'd by
nc\_reclaim\_data\_all(). The nc\_copy\_data\_all() function
will allocate the top level and return a pointer to it to the
user. The user can later pass that pointer to
nc\_reclaim\_data\_all() to reclaim the instance(s).
# Internal Changes
The netcdf-c library internals are changed to use the proper reclaim
and copy functions. This also allows some simplification of the code
since the stdata and vldata fields of NC\_ATT\_INFO are no longer needed.
Currently this is commented out using the SEPDATA \#define macro.
When the bugs are found and fixed, all this code will be removed.
## Optimizations
In order to make these functions as efficient as possible, it is
desirable to classify all types as to whether or not they contain
variable-size data. If a type is fixed sized (i.e. does not contain
variable-size data) then it can be freed or copied as a single chunk.
This significantly increases the performance for such types.
For variable-size types, it is necessary to walk each instance of the type
and recursively reclaim or copy it. As another optimization,
if the type is a vector of strings, then the per-instance walk can be
sped up by doing the reclaim or copy inline.
The rules for classifying types as fixed or variable size are as follows.
1. All atomic types, except string, are fixed size.
2. All enum type and opaque types are fixed size.
3. All string types and VLEN types are variable size.
4. A compound type is fixed size if all of the types of its
fields are fixed size. Otherwise it has variable size.
The classification of types can be made at the time the type is defined
or is read in from some existing file. The reclaim and copy functions
use this information to speed up the handling of fixed size types.
# Warnings
1. The new API functions require that the type information be
accessible. This means that you cannot use these functions
after the file has been closed. After the file is closed, you
are on your own.
2. There is still one known failure that has not been solved; it is
possibly an HDF5 memory leak. All the failures revolve around
some variant of this .cdl file. The proximate cause of failure is
the use of a VLEN FillValue.
````
netcdf x {
types:
float(*) row_of_floats ;
dimensions:
m = 5 ;
variables:
row_of_floats ragged_array(m) ;
row_of_floats ragged_array:_FillValue = {-999} ;
data:
ragged_array = {10, 11, 12, 13, 14}, {20, 21, 22, 23}, {30, 31, 32},
{40, 41}, _ ;
}
````
# 3. Inferring File Types
As described in the companion document -- docs/dispatch.md --
when nc\_create() or nc\_open() is called, it must figure out what
kind of file is being created or opened. Once it has figured out
the file kind, the appropriate "dispatch table" can be used
to process that file.
## The Role of URLs
Figuring out the kind of file is referred to as model inference
and is, unfortunately, a complicated process. The complication
is mostly a result of allowing a path argument to be a URL.
Inferring the file kind from a URL requires deep processing of
the URL structure: the protocol, the host, the path, and the fragment
parts in particular. The query part is currently not used because
it usually contains information to be processed by the server
receiving the URL.
The "fragment" part of the URL may be unfamiliar.
The last part of a URL may optionally contain a fragment, which
is syntactically of this form in this pseudo URL specification.
````
<protocol>://<host>/<path>?<query>#<fragment>
````
The form of the fragment is similar to a query and takes this general form.
````
'#'<key>=<value>&<key>=<value>&...
````
The key is a simple name, the value is any sequence of characters,
although URL special characters such as '&' must be URL encoded in
the '%XX' form where each X is a hexadecimal digit.
An example might look like this non-sensical example:
````
https://host.com/path#mode=nczarr,s3&bytes
````
It is important to note that the fragment part is not intended to be
passed to the server, but rather is processed by the client program.
It is this property that allows the netcdf-c library to use it to
pass information deep into the dispatch table code that is processing the
URL.
## Model Inference Inputs
The inference algorithm is given the following information
from which it must determine the kind of file being accessed.
### Mode
The mode is a set of flags that are passed as the second
argument to nc\_create and nc\_open. The set of flags is define in
the netcdf.h header file. Generally it specifies the general
format of the file: netcdf-3 (classic) or netcdf-4 (enhanced).
Variants of these can also be specified, e.g. 64-bit netcdf-3 or
classic netcdf-4.
In the case where the path argument is a simple file path,
using a mode flag is the most common mechanism for specifying
the model.
### Path
The file path, the first argument to nc\_create and nc\_open,
Can be either a simple file path or a URL.
If it is a URL, then it will be deeply inspected to determine
the model.
### File Contents
When the contents of a real file are available,
the contents of the file can be used to determine the dispatch table.
As a rule, this is likely to be useful only for *nc\_open*.
It also requires access to functions that can open and read at least
the initial part of the file.
As a rule, the initial small prefix of the file is read
and examined to see if it matches any of the so-called
"magic numbers" that indicate the kind of file being read.
### Open vs Create
Is the file being opened or is it being created?
### Parallelism
Is parallel IO available?
## Model Inference Outputs
The inferencing algorithm outputs two pieces of information.
1. model - this is used by nc\_open and nc\_create to choose the dispatch table.
2. newpath - in some case, usually URLS, the path may be rewritten to include extra information for use by the dispatch functions.
The model output is actually a struct containing two fields:
1. implementation - this is a value from the NC\_FORMATX\_xxx
values in netcdf.h. It generally determines the dispatch
table to use.
2. format -- this is an NC\_FORMAT\_xxx value defining, in effect,
the netcdf-format to which the underlying format is to be
translated. Thus it can tell the netcdf-3 dispatcher that it
should actually implement CDF5 rather than standard netcdf classic.
## The Inference Algorithm
The construction of the model is primarily carried out by the function
*NC\_infermodel()* (in *libdispatch/dinfermodel.c).
It is given the following parameters:
1. path -- (IN) absolute file path or URL
2. modep -- (IN/OUT) the set of mode flags given to *NC\_open* or *NC\_create*.
3. iscreate -- (IN) distinguish open from create.
4. useparallel -- (IN) indicate if parallel IO can be used.
5. params -- (IN/OUT) arbitrary data dependent on the mode and path.
6. model -- (IN/OUT) place to store inferred model.
7. newpathp -- (OUT) the canonical rewrite of the path argument.
As a rule, these values are used in the this order of preference
to infer the model.
1. file contents -- highest precedence
2. url (if it is one) -- using the "mode=" key in the fragment (see below).
3. mode flags
4. default format -- lowest precedence
The sequence of steps is as follows.
### URL processing -- processuri()
If the path appears to be a URL, then it is parsed
and processed by the processuri function as follows.
1. Protocol --
The protocol is extracted and tested against the list of
legal protocols. If not found, then it is an error.
If found, then it is replaced by a substitute -- if specified.
So, for example, the protocol "dods" is replaced the protocol "http"
(note that at some point "http" will be replaced with "https").
Additionally, one or more "key=value" strings is appended
to the existing fragment of the url. So, again for "dods",
the fragment is extended by the string "mode=dap2".
Thus replacing "dods" does not lose information, but rather transfers
it to the fragment for later use.
2. Fragment --
After the protocol is processed, the initial fragment processing occurs
by converting it to a list data structure of the form
````
{<key>,<value>,<key>,<value>,<key>,<value>....}
````
### Macro Processing -- processmacros()
If the fragment list produced by processuri() is non-empty, then
it is processed for "macros". Notice that if the original path
was not a URL, then the fragment list is empty and this
processing will be bypassed. In any case, It is convenient to
allow some singleton fragment keys to be expanded into larger
fragment components. In effect, those singletons act as
macros. They can help to simplify the user's URL. The term
singleton means a fragment key with no associated value:
"#bytes", for example.
The list of fragments is searched looking for keys whose
value part is NULL or the empty string. Then the table
of macros is searched for that key and if found, then
a key and values is appended to the fragment list and the singleton
is removed.
### Mode Inference -- processinferences()
This function just processes the list of values associated
with the "mode" key. It is similar to a macro in that
certain mode values are added or removed based on tables
of "inferences" and "negations".
Again, the purpose is to allow users to provide simplified URL fragments.
The list of mode values is repeatedly searched and whenever a value
is found that is in the "modeinferences" table, then the associated inference value
is appended to the list of mode values. This process stops when no changes
occur. This form of inference allows the user to specify "mode=zarr"
and have it converted to "mode=nczarr,zarr". This avoids the need for the
dispatch table code to do the same inference.
After the inferences are made, The list of mode values is again
repeatedly searched and whenever a value
is found that is in the "modenegations" table, then the associated negation value
is removed from the list of mode values, assuming it is there. This process stops when no changes
occur. This form of inference allows the user to make sure that "mode=bytes,nczarr"
has the bytes mode take precedence by removing the "nczarr" value. Such illegal
combinations can occur because of previous processing steps.
### Fragment List Normalization
As the fragment list is processed, duplicates appear with the same key.
A function -- cleanfragments() -- is applied to clean up the fragment list
by coalesing the values of duplicate keys and removing duplicate key values.
### S3 Rebuild
If the URL is determined to be a reference to a resource on the Amazon S3 cloud,
then the URL needs to be converted to what is called "path format".
There are four S3 URL formats:
1. Virtual -- ````https://<bucket>.s3.<region>.amazonaws.com/<path>````
2. Path -- ````https://s3.<region>.amazonaws.com/<bucket>/<path>````
3. S3 -- ````s3://<bucket>/<path>````
4. Other -- ````https://<host>/<bucket>/<path>````
The S3 processing converts all of these to the Path format. In the "S3" format case
it is necessary to find or default the region from examining the ".aws" directory files.
### File Rebuild
If the URL protocol is "file" and its path is a relative file path,
then it is made absolute by prepending the path of the current working directory.
In any case, after S3 or File rebuilds, the URL is completely
rebuilt using any modified protocol, host, path, and
fragments. The query is left unchanged in the current algorithm.
The resulting rebuilt URL is passed back to the caller.
### Mode Key Processing
The set of values of the fragment's "mode" key are processed one by one
to see if it is possible to determine the model.
There is a table for format interpretations that maps a mode value
to the model's implementation and format. So for example,
if the mode value "dap2" is encountered, then the model
implementation is set to NC\_FORMATX\_DAP2 and the format
is set to NC\_FORMAT\_CLASSIC.
### Non-Mode Key Processing
If processing the mode does not tell us the implementation, then
all other fragment keys are processed to see if the implementaton
(and format) can be deduced. Currently this does nothing.
### URL Defaults
If the model is still not determined and the path is a URL, then
the implementation is defaulted to DAP2. This is for back
compatibility when all URLS implied DAP2.
### Mode Flags
In the event that the path is not a URL, then it is necessary
to use the mode flags and the isparallel arguments to choose a model.
This is just a straight forward flag checking exercise.
### Content Inference -- check\_file\_type()
If the path is being opened (as opposed to created), then
it may be possible to actually read the first few bytes of the
resource specified by the path and use that to determine the
model. If this succeeds, then it takes precedence over
all other model inferences.
### Flag Consistency
Once the model is known, then the set of mode flags
is modified to be consistent with that information.
So for example, if DAP2 is the model, then all netcdf-4 mode flags
and some netcdf-3 flags are removed from the set of mode flags
because DAP2 provides only a standard netcdf-classic format.
# 4. Adding a Standard Filter
The standard filter system extends the netcdf-c library API to
support a fixed set of "standard" filters. This is similar to the
way that deflate and szip are currently supported.
For background, the file filter.md should be consulted.
In general, the API for a standard filter has the following prototypes.
The case of zstandard (libzstd) is used as an example.
````
int nc_def_var_zstandard(int ncid, int varid, int level);
int nc_inq_var_zstandard(int ncid, int varid, int* has_filterp, int* levelp);
````
So generally the API has the ncid and the varid as fixed, and then
a list of parameters specific to the filter -- level in this case.
For the inquire function, there is an additional argument -- has_filterp --
that is set to 1 if the filter is defined for the given variable
and is 0 if not.
The remainder of the inquiry parameters are pointers to memory
into which the parameters are stored -- levelp in this case.
It is important to note that including a standard filter still
requires three supporting objects:
1. The implementing library for the filter. For example,
libzstd must be installed in order to use the zstandard
API.
2. A HDF5 wrapper for the filter must be installed in the
directory pointed to by the HDF5_PLUGIN_PATH environment
variable.
3. (Optional) An NCZarr Codec implementation must be installed
in the the HDF5_PLUGIN_PATH directory.
## Adding a New Standard Filter
The implementation of a standard filter must be loaded from one
of several locations.
1. It can be part of libnetcdf.so (preferred),
2. it can be loaded as part of the client code,
3. or it can be loaded as part of an external library such as libccr.
However, the three objects listed above need to be
stored in the HDF5_PLUGIN_DIR directory, so adding a standard
filter still requires modification to the netcdf build system.
This limitation may be lifted in the future.
### Build Changes
In order to detect a standard library, the following changes
must be made for Automake (configure.ac/Makefile.am)
and CMake (CMakeLists.txt)
#### Configure.ac
Configure.ac must have a block that similar to this that locates
the implementing library.
````
# See if we have libzstd
AC_CHECK_LIB([zstd],[ZSTD_compress],[have_zstd=yes],[have_zstd=no])
if test "x$have_zstd" = "xyes" ; then
AC_SEARCH_LIBS([ZSTD_compress],[zstd zstd.dll cygzstd.dll], [], [])
AC_DEFINE([HAVE_ZSTD], [1], [if true, zstd library is available])
fi
AC_MSG_CHECKING([whether libzstd library is available])
AC_MSG_RESULT([${have_zstd}])
````
Note the the entry point (*ZSTD_compress*) is library dependent
and is used to see if the library is available.
#### Makefile.am
It is assumed you have an HDF5 wrapper for zstd. If you want it
to be built as part of the netcdf-c library then you need to
add the following to *netcdf-c/plugins/Makefile.am*.
````
if HAVE_ZSTD
noinst_LTLIBRARIES += libh5zstd.la
libh5szip_la_SOURCES = H5Zzstd.c H5Zzstd.h
endif
````
# Need our version of szip if libsz available and we are not using HDF5
if HAVE_SZ
noinst_LTLIBRARIES += libh5szip.la
libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h
endif
#### CMakeLists.txt
In an analog to *configure.ac*, a block like
this needs to be in *netcdf-c/CMakeLists.txt*.
````
FIND_PACKAGE(Zstd)
set_std_filter(Zstd)
````
The FIND_PACKAGE requires a CMake module for the filter
in the cmake/modules directory.
The *set_std_filter* function is a macro.
An entry in the file config.h.cmake.in will also be needed.
````
/* Define to 1 if zstd library available. */
#cmakedefine HAVE_ZSTD 1
````
### Implementation Template
As a template, here is the implementation for zstandard.
It can be used as the template for adding other standard filters.
It is currently located in *netcdf-d/libdispatch/dfilter.c*, but
could be anywhere as indicated above.
````
#ifdef HAVE_ZSTD
int
nc_def_var_zstandard(int ncid, int varid, int level)
{
int stat = NC_NOERR;
unsigned ulevel;
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_ZSTD))) goto done;
/* Filter is available */
/* Level must be between -131072 and 22 on Zstandard v. 1.4.5 (~202009)
Earlier versions have fewer levels (especially fewer negative levels) */
if (level < -131072 || level > 22)
return NC_EINVAL;
ulevel = (unsigned) level; /* Keep bit pattern */
if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_ZSTD,1,&ulevel))) goto done;
done:
return stat;
}
int
nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp)
{
int stat = NC_NOERR;
size_t nparams;
unsigned params = 0;
int hasfilter = 0;
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_ZSTD))) goto done;
/* Filter is available */
/* Get filter info */
stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_ZSTD,&nparams,NULL);
if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;}
if(stat != NC_NOERR) goto done;
hasfilter = 1;
if(nparams != 1) {stat = NC_EFILTER; goto done;}
if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_ZSTD,&nparams,&params))) goto done;
done:
if(levelp) *levelp = (int)params;
if(hasfilterp) *hasfilterp = hasfilter;
return stat;
}
#endif /*HAVE_ZSTD*/
````
# Point of Contact {#intern_poc}
*Author*: Dennis Heimbigner<br>
*Email*: dmh at ucar dot edu<br>
*Initial Version*: 12/22/2021<br>
*Last Revised*: 01/25/2022

37
docs/nczarr.md
View File

@ -481,9 +481,9 @@ collections — High-performance dataset datatypes](https://docs.python.org/2/li
<a name="ref_xarray">[7]</a> [XArray Zarr Encoding Specification](http://xarray.pydata.org/en/latest/internals.html#zarr-encoding-specification)<br>
<a name="ref_xarray">[8]</a> [Dynamic Filter Loading](https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf)<br>
<a name="ref_xarray">[9]</a> [Officially Registered Custom HDF5 Filters](https://portal.hdfgroup.org/display/support/Registered+Filter+Plugins)<br>
<a name="ref_xarray">[10]</a> [C-Blosc Compressor Implementation](https://github.com/Blosc/c-blosc)
<a name="ref_awssdk_conda">[11]</a> [Conda-forge / packages / aws-sdk-cpp]
(https://anaconda.org/conda-forge/aws-sdk-cpp)<br>
<a name="ref_xarray">[10]</a> [C-Blosc Compressor Implementation](https://github.com/Blosc/c-blosc)<br>
<a name="ref_awssdk_conda">[11]</a> [Conda-forge / packages / aws-sdk-cpp](https://anaconda.org/conda-forge/aws-sdk-cpp)<br>
<a name="ref_gdal">[12]</a> [GDAL Zarr](https://gdal.org/drivers/raster/zarr.html)<br>
# Appendix A. Building NCZarr Support {#nczarr_build}
@ -524,8 +524,7 @@ Note also that if S3 support is enabled, then you need to have a C++ compiler in
The necessary CMake flags are as follows (with defaults)
1.
-DENABLE_NCZARR=off -- equivalent to the Automake _--disable-nczarr_ option.
1. -DENABLE_NCZARR=off -- equivalent to the Automake _--disable-nczarr_ option.
2. -DENABLE_NCZARR_S3=off -- equivalent to the Automake _--enable-nczarr-s3_ option.
3. -DENABLE_NCZARR_S3_TESTS=off -- equivalent to the Automake _--enable-nczarr-s3-tests_ option.
@ -562,7 +561,7 @@ Building this package from scratch has proven to be a formidable task.
This appears to be due to dependencies on very specific versions of,
for example, openssl.
## **nix** Build
## *\*nix\** Build
For linux, the following context works. Of course your mileage may vary.
* OS: ubuntu 21
@ -682,7 +681,7 @@ Some of the relevant limits are as follows:
Note that the limit is defined in terms of bytes and not (Unicode) characters.
This affects the depth to which groups can be nested because the key encodes the full path name of a group.
# Appendix D. Alternative Mechanisms for Accessing Remote Datasets
# Appendix D. Alternative Mechanisms for Accessing Remote Datasets {#nczarr_altremote}
The NetCDF-C library contains an alternate mechanism for accessing traditional netcdf-4 files stored in Amazon S3: The byte-range mechanism.
The idea is to treat the remote data as if it was a big file.
@ -706,7 +705,7 @@ Specifically, Thredds servers support such access using the HttpServer access me
https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes
````
# Appendix E. AWS Selection Algorithms.
# Appendix E. AWS Selection Algorithms. {#nczarr_awsselect}
If byterange support is enabled, the netcdf-c library will parse the files
````
@ -764,7 +763,7 @@ Picking an access-key/secret-key pair is always determined
by the current active profile. To choose to not use keys
requires that the active profile must be "none".
# Appendix F. NCZarr Version 1 Meta-Data Representation
# Appendix F. NCZarr Version 1 Meta-Data Representation. {#nczarr_version1}
In NCZarr Version 1, the NCZarr specific metadata was represented using new objects rather than as keys in existing Zarr objects.
Due to conflicts with the Zarr specification, that format is deprecated in favor of the one described above.
@ -779,6 +778,26 @@ The content of these objects is the same as the contents of the corresponding ke
* ''.nczarray <=> ''_NCZARR_ARRAY_''
* ''.nczattr <=> ''_NCZARR_ATTR_''
# Appendix G. JSON Attribute Convention. {#nczarr_version1}
An attribute may be encountered on read whose value when parsed
by JSON is a dictionary. As a special conventions, the value
converted to a string and stored as the value of the attribute
and the type of the attribute is treated as char.
When writing a character valued attribute, it's value is examined
to see if it looks like a JSON dictionary (i.e. "{...}")
and is parseable as JSON.
If so, then the attribute value is treated as one long string,
parsed as JSON, and stored in the .zattr file in JSON form.
These conventions are intended to help support various
attributes created by other packages where the attribute is a
complex JSON dictionary. An example is the GDAL Driver
convention <a href="#ref_gdal">[12]</a>. The value is a complex
JSON dictionary and it is desirable to both read and write that kind of
information through the netcdf API.
# Point of Contact {#nczarr_poc}
__Author__: Dennis Heimbigner<br>

2
examples/C/Makefile.am
View File

@ -7,7 +7,9 @@
# Un comment to use a more verbose test driver
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#TESTS_ENVIRONMENT = export SETX=1;
# Put together AM_CPPFLAGS and AM_LDFLAGS.
include $(top_srcdir)/lib_flags.am

13
examples/C/run_filter.sh
View File

@ -10,26 +10,25 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../../test_common.sh
echo "*** Running filter example for netCDF-4."
set -e
if test -f ${builddir}/findplugin.sh ; then
echo "*** running test_filter example..."
. ${builddir}/findplugin.sh
# Locate the plugin path and the library names; argument order is critical
# Find bzip2 and capture
findplugin h5bzip2
BZIP2PATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
BZIP2PATH="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
# Verify
if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
export HDF5_PLUGIN_PATH
echo "*** running filter_example..."
rm -f ./bzip2.nc
export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
echo "*** running filter_example..."
${execdir}/filter_example
#rm -f ./bzip2.nc
fi # Filter enabled
rm -f ./bzip2.nc
echo "*** Filter example successful!"
exit 0

44
include/netcdf_filter_build.h
View File

@ -34,17 +34,46 @@
/* Defined flags for filter invocation (not stored); powers of two */
#define NCZ_FILTER_DECODE 0x00000001
/* External Discovery Function */
/* External Discovery Functions */
/*
Obtain a pointer to an instance of NCZ_codec_class_t.
NCZ_get_codec_info(void) -- returns pointer to instance of NCZ_codec_class_t.
Instance an be recast based on version+sort to the plugin type specific info.
So the void* return value is typically actually of type NCZ_codec_class_t*.
Signature: typedef const void* (*NCZ_get_codec_info_proto)(void);
The current object returned by NCZ_get_codec_info is a
pointer to an instance of NCZ_codec_t.
The key to this struct are the several function pointers that do
initialize/finalize and conversion between codec JSON and HDF5
parameters. The function pointers defined in NCZ_codec_t
manipulate HDF5 parameters and NumCodec JSON.
Obtain a pointer to an instance of NCZ_codec_class_t.
NCZ_get_codec_info(void) -- returns pointer to instance of NCZ_codec_class_t.
Instance an be recast based on version+sort to the plugin type specific info.
So the void* return value is typically actually of type NCZ_codec_class_t*.
*/
typedef const void* (*NCZ_get_codec_info_proto)(void);
/*
Obtain a pointer to a NULL terminated vector of NCZ_codec_class_t*.
NCZ_codec_info_defaults(void) -- returns pointer to a vector of pointers to instances of NCZ_codec_class_t. The vector is NULL terminated.
So the void* return value is typically actually of type NCZ_codec_class_t**.
Signature: typedef const void* (*NCZ_codec_info_defaults_proto)(void);
This entry point is used to return the codec information for
multiple filters that otherwise do not have codec information defined.
*/
typedef const void* (*NCZ_codec_info_defaults_proto)(void);
/* The current object returned by NCZ_get_plugin_info is a
pointer to an instance of NCZ_codec_t.
@ -86,7 +115,7 @@ int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp);
* Convert a set of visible parameters to a set of working parameters using extra environmental information.
Also allows for changes to the visible parameters. Invoked before filter is actually used.
int (*NCZ_build_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
@param ncid -- (in) ncid of the variable's group
@param varid -- (in) varid of the variable
@ -96,8 +125,15 @@ int (*NCZ_build_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** v
@params wparamsp -- (out) vector of working parameters
@return -- a netcdf-c error code.
* Convert a set of working parameters to a set of visible parameters using extra environmental information, if needed.
Invoked before filter metadata is written.
* Convert an HDF5 vector of visible parameters to a JSON representation.
int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp);
@param nparams -- (in) the length of the HDF5 unsigned vector
@param params -- (in) pointer to the HDF5 unsigned vector.
@param codecp -- (out) store the string representation of the codec; caller must free.
@return -- a netcdf-c error code.
*/
/*

8
libdispatch/dutil.c
View File

@ -211,11 +211,11 @@ NC_mktmp(const char* base)
int fd = -1;
char* tmp = NULL;
size_t len;
#define MAXTRIES 4
#ifdef HAVE_MKSTEMP
mode_t mask;
#else
#ifndef HAVE_MKSTEMP
int tries;
#define MAXTRIES 4
#else
mode_t mask;
#endif
len = strlen(base)+6+1;

32
libnczarr/zfilter.c
View File

@ -52,9 +52,11 @@
#include "netcdf_filter_build.h"
#include "netcdf_aux.h"
#undef DEBUG
#undef DEBUGF
#undef DEBUGL
#if 0
#define DEBUG
#define DEBUGF
#define DEBUGL
#endif
/* If set, then triage potential shared libraries based on extension */
#define NAMEOPT
@ -177,7 +179,7 @@ NCJtrace(const NCjson* j)
const char*
printplugin(const NCZ_Plugin* plugin)
{
char* plbuf = malloc(4096);
static char plbuf[4096];
char plbuf2[2000];
char plbuf1[2000];
@ -194,8 +196,7 @@ printplugin(const NCZ_Plugin* plugin)
static char*
printparams(size_t nparams, const unsigned* params)
{
char* ppbuf = malloc(4096);
static char ppbuf[4096];
if(nparams == 0)
snprintf(ppbuf,4096,"{0,%p}",params);
else
@ -1378,20 +1379,27 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
/* See what we have */
{
H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type");
H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info");
NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info");
NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults");
const H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type");
const H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info");
const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info");
const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults");
if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL)
{stat = NC_ENOFILTER; goto done;}
if(cpd != NULL) {
/* Deal with defaults first */
NCZ_codec_t** cp = NULL;
const NCZ_codec_t** cp = NULL;
nclistpush(default_libs,lib);
for(cp=cpd();*cp;cp++) {
cp = (const NCZ_codec_t**)cpd();
#ifdef DEBUGL
fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp);
#endif
for(;*cp;cp++) {
struct CodecAPI* c0;
#ifdef DEBUGL
fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id);
#endif
c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI));
if(c0 == NULL) {stat = NC_ENOMEM; goto done1;}
c0->codec = *cp;

10
libnczarr/zfilter.h
View File

@ -25,16 +25,6 @@
#define plugin_dir_win "%s/hdf5/lib/plugin"
#define win32_root_env "ALLUSERSPROFILE"
/*
Return a NULL terminated vector of pointers to instances of ''NCZ_codec_t''.
The value returned is actually of type ''NCZ_codec_t**'',
but is of type ''void*'' to allow for extensions.
The list of returned items are used to try to provide defaults
for any HDF5 filters that have no corresponding Codec.
This is for internal use only.
*/
typedef void* (*NCZ_codec_info_defaults_proto)(void);
/* Opaque */
struct NCZ_Filter;

3
libnetcdf.settings.in
View File

@ -9,6 +9,7 @@ Configured On: @CONFIG_DATE@
Host System: @host_cpu@-@host_vendor@-@host_os@
Build Directory: @abs_top_builddir@
Install Prefix: @prefix@
Plugin Install Prefix: @PLUGIN_INSTALL_DIR_SETTING@
# Compiling Options
-----------------
@ -49,4 +50,4 @@ Logging: @HAS_LOGGING@
SZIP Write Support: @HAS_SZLIB_WRITE@
Standard Filters: @STD_FILTERS@
ZSTD Support: @HAS_ZSTD@
Benchmarks: @HAS_BENCHMARKS@
Benchmarks: @HAS_BENCHMARKS@

4
nc_test/CMakeLists.txt
View File

@ -91,7 +91,9 @@ IF(BUILD_UTILITIES)
add_sh_test(nc_test run_diskless5)
add_sh_test(nc_test run_inmemory)
IF(LARGE_FILE_TESTS)
add_sh_test(nc_test run_diskless2)
IF(NOT USE_PARALLEL)
add_sh_test(nc_test run_diskless2)
ENDIF()
ENDIF()
IF(ENABLE_BYTERANGE)

2
nc_test/Makefile.am
View File

@ -86,8 +86,10 @@ endif
if BUILD_UTILITIES
TESTS += run_diskless.sh run_diskless5.sh run_inmemory.sh
if LARGE_FILE_TESTS
if ! ENABLE_PARALLEL
TESTS += run_diskless2.sh
endif
endif
if BUILD_MMAP
TESTS += run_mmap.sh
run_mmap.log: run_diskless.log

3
nc_test/run_diskless2.sh
View File

@ -5,9 +5,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
set -e
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
# Get the target OS and CPU
CPU=`uname -p`
OS=`uname`

8
nc_test4/Makefile.am
View File

@ -109,9 +109,15 @@ ref_filteredvv.cdl ref_multi.cdl \
ref_ncgenF.cdl ref_nccopyF.cdl \
ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
ref_filter_order_create.txt ref_filter_order_read.txt \
ref_any.cdl tst_specific_filters.sh \
ref_any.cdl tst_specific_filters.sh \
tst_virtual_datasets.c
# The tst_filterinstall test can only be run after an install
# occurred with --with-plugin-dir enabled. So there is no point
#in running it via make check. It is kept here so it can be
# manually invoked if desired
EXTRA_DIST += tst_filterinstall.sh
CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt \
tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl tst_elena_*.cdl \

32
nc_test4/findplugin.in
View File

@ -9,12 +9,10 @@
# $2 is 1 if we are running under cmake
# $3 is 1 if we are running using Visual Studio, blank otherwise
# $4 is the build type; only used if $3 is 1
# Optional Input:
# HDF5_PLUGIN_PATH environment variable
# Outputs:
# return code is 0 is success, 1 if failed
# Variable HDF5_PLUGIN_LIB is set to the library file name
# Variable HDF5_PLUGIN_PATH is setthe absolute path to the
# Variable HDF5_PLUGIN_DIR is set to the absolute path to the
# directory containing the plugin library file
# Local variables are prefixed with FP_
#
@ -27,7 +25,7 @@ findplugin() {
FP_NAME="$1"
FP_PLUGIN_LIB=
FP_PLUGIN_PATH=
FP_PLUGIN_DIR=
# Figure out the plugin file name
# Test for visual studio before cygwin since both might be true
@ -43,14 +41,6 @@ else # Presumably some form on *nix"
FP_PLUGIN_LIB="lib${FP_NAME}.so"
fi
# If HDF5_PLUGIN_PATH is defined, then it overrides everything else.
if test "x$HDF5_PLUGIN_PATH" != x ; then
HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB"
# HDF5_PLUGIN_PATH already set
HDF5_PLUGIN_PATH=`${NCPATHCVT} $HDF5_PLUGIN_PATH`
return 0;
fi
# Figure out the path to where the lib is stored
# This can probably be simplified
@ -59,40 +49,38 @@ cd ${TOPBUILDDIR}/plugins
FP_PLUGINS=`pwd`
cd ${CURWD}
# Case 1: Cmake with Visual Studio
# Do not know where to look for a dylib
# Case 1: Cmake with Visual Studio
if test "x$FP_ISCMAKE" != x -a "x${FP_ISMSVC}" != x ; then
# Case 1a: ignore the build type directory
if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
FP_PLUGIN_PATH="${FP_PLUGINS}"
FP_PLUGIN_DIR="${FP_PLUGINS}"
fi
else # Case 2: automake
# Case 2a: look in .libs
if test -f "${FP_PLUGINS}/.libs/${FP_PLUGIN_LIB}" ; then
FP_PLUGIN_PATH="${FP_PLUGINS}/.libs"
FP_PLUGIN_DIR="${FP_PLUGINS}/.libs"
else # Case 2: look in FP_PLUGINS directly
if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
FP_PLUGIN_PATH="${FP_PLUGINS}"
FP_PLUGIN_DIR="${FP_PLUGINS}"
fi
fi
fi
# Verify
if test "x$FP_PLUGIN_PATH" = x ; then
echo "***Fail: Could not locate a usable HDF5_PLUGIN_PATH"
if test "x$FP_PLUGIN_DIR" = x ; then
echo "***Fail: Could not locate a usable HDF5_PLUGIN_DIR"
return 1
fi
if ! test -f "$FP_PLUGIN_PATH/$FP_PLUGIN_LIB" ; then
if ! test -f "$FP_PLUGIN_DIR/$FP_PLUGIN_LIB" ; then
echo "***Fail: Could not locate a usable HDF5_PLUGIN_LIB"
return 1
fi
FP_PLUGIN_PATH=`${NCPATHCVT} $FP_PLUGIN_PATH`
FP_PLUGIN_DIR=`${NCPATHCVT} $FP_PLUGIN_DIR`
# Set the final output variables
HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB"
HDF5_PLUGIN_PATH="$FP_PLUGIN_PATH"
HDF5_PLUGIN_DIR="$FP_PLUGIN_DIR"
return 0
}

1
nc_test4/run_par_test.sh.in
View File

@ -52,7 +52,6 @@ echo "Parallel I/O test for Collective I/O, contributed by HDF Group."
if test "@HAS_PAR_FILTERS@" = "yes"; then
echo
echo "Parallel I/O test with zlib."
@MPIEXEC@ -n 1 ./tst_parallel_zlib
@MPIEXEC@ -n 4 ./tst_parallel_zlib
echo

29
nc_test4/tst_filter.sh
View File

@ -43,36 +43,37 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
# Hide/unhide the noop filter
hidenoop() {
rm -fr ${HDF5_PLUGIN_PATH}/save
mkdir ${HDF5_PLUGIN_PATH}/save
mv ${NOOPPATH} ${HDF5_PLUGIN_PATH}/save
rm -fr ${HDF5_PLUGIN_DIR}/save
mkdir ${HDF5_PLUGIN_DIR}/save
mv ${NOOPDIR} ${HDF5_PLUGIN_DIR}/save
}
unhidenoop() {
mv ${HDF5_PLUGIN_PATH}/save/${NOOPLIB} ${HDF5_PLUGIN_PATH}
rm -fr ${HDF5_PLUGIN_PATH}/save
mv ${HDF5_PLUGIN_DIR}/save/${NOOPLIB} ${HDF5_PLUGIN_DIR}
rm -fr ${HDF5_PLUGIN_DIR}/save
}
# Locate the plugin path and the library names; argument order is critical
# Locate the plugin dir and the library names; argument order is critical
# Find bzip2 and capture
findplugin h5bzip2
BZIP2LIB="${HDF5_PLUGIN_LIB}"
BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}"
BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}"
# Find misc and capture
findplugin h5misc
MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
# Find noop and capture
findplugin h5noop
NOOPLIB="${HDF5_PLUGIN_LIB}"
NOOPPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
NOOPDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
export HDF5_PLUGIN_PATH
echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}"
export HDF5_PLUGIN_DIR
export HDF5_PLUGIN_PATH="$HDF5_PLUGIN_DIR"
# Verify
if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi
if ! test -f ${NOOPPATH} ; then echo "Unable to locate ${NOOPPATH}"; exit 1; fi
if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi
if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi
if ! test -f ${NOOPDIR} ; then echo "Unable to locate ${NOOPDIR}"; exit 1; fi
# See if we have szip
if avail szip; then HAVE_SZIP=1; else HAVE_SZIP=0; fi

111
nc_test4/tst_filterinstall.sh Executable file
View File

@ -0,0 +1,111 @@
#!/bin/bash
# Test the filter install
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
if test "x$TESTNCZARR" = x1 ; then
. ./test_nczarr.sh
fi
set -e
# Use this plugin path
export HDF5_PLUGIN_PATH="${FEATURE_PLUGIN_INSTALL_DIR}"
# Function to remove selected -s attributes from file;
# These attributes might be platform dependent
sclean() {
cat $1 \
| sed -e '/:_IsNetcdf4/d' \
| sed -e '/:_Endianness/d' \
| sed -e '/_NCProperties/d' \
| sed -e '/_SuperblockVersion/d' \
| sed -e '/_Format/d' \
| sed -e '/global attributes:/d' \
| cat > $2
}
# Function to extract _Filter attribute from a file
# These attributes might be platform dependent
getfilterattr() {
V="$1"
sed -e '/${V}.*:_Filter/p' -ed <$2 >$3
}
# Function to extract _Codecs attribute from a file
# These attributes might be platform dependent
getcodecsattr() {
V="$1"
sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3
}
trimleft() {
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
}
setfilter() {
FF="$1"
FSRC="$2"
FDST="$3"
FIH5="$4"
FICX="$5"
FFH5="$6"
FFCX="$7"
if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi
if test "x$FFCX" = x ; then FFCX="$FICX" ; fi
rm -f $FDST
cat ${srcdir}/$FSRC \
| sed -e "s/ref_any/${FF}/" \
| sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \
| sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \
| sed -e 's/"/\\"/g' -e 's/@/"/g' \
| cat > $FDST
}
# Execute the specified tests
runfilter() {
zext=$1
zfilt="$2"
zparams="$3"
zcodec="$4"
echo "*** Testing processing of filter $zfilt for map $zext"
if test "x$TESTNCZARR" = x1 ; then
fileargs "tmp_filt_${zfilt}"
deletemap $zext $file
else
file="tmp_filt_${zfilt}.nc"
rm -f $file
fi
setfilter $zfilt ref_any.cdl "tmp_filt_${zfilt}.cdl" "$zparams" "$zcodec"
if test "x$TESTNCZARR" = x1 ; then
${NCGEN} -4 -lb -o $fileurl "tmp_filt_${zfilt}.cdl"
${NCDUMP} -n $zfilt -sF $fileurl > "tmp_filt_${zfilt}.tmp"
else
${NCGEN} -4 -lb -o $file "tmp_filt_${zfilt}.cdl"
${NCDUMP} -n $zfilt -sF $file > "tmp_filt_${zfilt}.tmp"
fi
sclean "tmp_filt_${zfilt}.tmp" "tmp_filt_${zfilt}.dump"
}
testbzip2() {
zext=$1
if ! avail bzip2; then return 0; fi
runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]'
diff -b -w "tmp_filt_bzip2.cdl" "tmp_filt_bzip2.dump"
}
testset() {
# Which test cases to exercise
testbzip2 $1
}
if test "x$TESTNCZARR" = x1 ; then
testset file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi
else
testset nc
fi

7
nc_test4/tst_parallel_zlib.c
View File

@ -60,6 +60,13 @@ main(int argc, char **argv)
/*printf("mpi_name: %s size: %d rank: %d\n", mpi_name,
mpi_size, mpi_rank);*/
/* This program must have exactly 4 processors. */
if (mpi_size != 4) {
printf("Error: mpi_size must be 4 with this test. mpi_size: %d used.\n",
mpi_size);
ERR;
}
#ifdef USE_MPE
MPE_Init_log();
s_init = MPE_Log_get_event_number();

15
nc_test4/tst_specific_filters.sh
View File

@ -28,9 +28,6 @@ echo "findplugin.sh loaded"
# Assume all test filters are in same plugin dir
findplugin h5bzip2
echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
export HDF5_PLUGIN_PATH
# Function to remove selected -s attributes from file;
# These attributes might be platform dependent
sclean() {
@ -167,14 +164,14 @@ testzstd() {
testset() {
# Which test cases to exercise
if test "x$TESTNCZARR" = x1 ; then
# testfletcher32 $1
testfletcher32 $1
testshuffle $1
fi
# testdeflate $1
# testszip $1
# testbzip2 $1
# testblosc $1
# testzstd $1
testdeflate $1
testszip $1
testbzip2 $1
testblosc $1
testzstd $1
}
if test "x$TESTNCZARR" = x1 ; then

2
ncdap_test/Makefile.am
View File

@ -10,7 +10,7 @@ include $(top_srcdir)/lib_flags.am
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#TEST_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#TESTS_ENVIRONMENT += export SETX=1;
#TESTS_ENVIRONMENT = export SETX=1;
# Note which tests depend on other tests. Necessary for make -j check.
TEST_EXTENSIONS = .sh

4
ncdump/ncfilteravail.c
View File

@ -15,6 +15,10 @@
#include <unistd.h>
#endif
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#if defined(_WIN32) && !defined(__MINGW32__)
#include "XGetopt.h"
#else

4
ncdump/ncpathcvt.c
View File

@ -15,6 +15,10 @@
#include <unistd.h>
#endif
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#if defined(_WIN32) && !defined(__MINGW32__)
#include "XGetopt.h"
#else

4
ncdump/ncvalidator.c
View File

@ -73,6 +73,10 @@ THIS SOFTWARE.
#include <unistd.h> /* read() getopt() */
#endif
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#if defined(_WIN32) && !defined(__MINGW32__)
#include <io.h>
#include "XGetopt.h"

9
ncdump/printfqn.c
View File

@ -1,12 +1,17 @@
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#if defined(_WIN32) && ! defined(__MINGW32__)
#include "XGetopt.h"
#else
#include <getopt.h>
#endif
#include <netcdf.h>

2
ncdump/test_scope.sh
View File

@ -31,6 +31,8 @@ diff -wB ${srcdir}/$1.cdl ${execdir}/copy_$1.cdl
}
typescope() {
echo ">>>"
ls -l ${execdir}/printfqn* ${execdir}/$1.nc ${execdir}/$1_copy.nc
REFT=`${execdir}/printfqn -f ${execdir}/$1.nc -v test_variable -t`
COPYT=`${execdir}/printfqn -f ${execdir}/$1_copy.nc -v test_variable -t`
if test "x$REFT" != "x$COPYT" ; then

2
ncdump/tst_nccopy3.sh
View File

@ -8,7 +8,9 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -x
set -e
echo ""
# get some config.h parameters

1
ncdump/tst_output.sh
View File

@ -4,6 +4,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
# This shell script tests the output from several previous tests.
set -x
set -e
echo ""

11
nczarr_test/Makefile.am
View File

@ -154,7 +154,7 @@ EXTRA_DIST += ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes
CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl
BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh
BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh run_filterinstall.sh
test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c
rm -f $@
echo "#define TESTNCZARR" > $@
@ -174,6 +174,15 @@ run_specific_filters.sh: $(top_srcdir)/nc_test4/tst_specific_filters.sh
chmod a+x $@
rm -f run_specific_filters.tmp
run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh
rm -f $@ run_filterinstall.tmp
echo "#!/bin/bash" > run_filterinstall.tmp
echo "TESTNCZARR=1" >> run_filterinstall.tmp
cat $(top_srcdir)/nc_test4/tst_filterinstall.sh >> run_filterinstall.tmp
tr -d '\r' < run_filterinstall.tmp > $@
chmod a+x $@
rm -f run_filterinstall.tmp
# Remove directories
clean-local:
rm -fr tmp*.file results.file results.s3 results.zip

14
nczarr_test/run_filter.sh
View File

@ -51,21 +51,21 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
# Find misc and capture
findplugin h5misc
MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
# Find noop and capture
findplugin h5noop
NOOPLIB="${HDF5_PLUGIN_LIB}"
NOOPPATH="${HDF5_PLUGIN_PATH}/${NOOPLIB}"
NOOPDIR="${HDF5_PLUGIN_DIR}/${NOOPLIB}"
# Find bzip2 and capture
findplugin h5bzip2
BZIP2LIB="${HDF5_PLUGIN_LIB}"
BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}"
BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}"
# Verify
if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi
if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi
if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi
# Execute the specified tests
@ -144,11 +144,11 @@ ${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt
# Remove irrelevant -s output
sclean ./tmp_known_$zext.txt tmp_known_$zext.dump
# Now hide the filter code
mv ${NOOPPATH} ./${NOOPLIB}.save
mv ${NOOPDIR} ./${NOOPLIB}.save
# dump and clean noop.nc header when filter is not avail
${NCDUMP} -hs $fileurl > ./tmp_unk_$zext.txt
# Restore the filter code
mv ./${NOOPLIB}.save ${NOOPPATH}
mv ./${NOOPLIB}.save ${NOOPDIR}
# Verify that the filter is no longer defined
UNK=`sed -e '/var:_Filter/p' -e d ./tmp_unk_$zext.txt`
test "x$UNK" = x

6
nczarr_test/test_nczarr.sh
View File

@ -133,12 +133,12 @@ if test "x$FP_USEPLUGINS" = xyes; then
echo "findplugin.sh loaded"
# Locate the plugin path and the library names; argument order is critical
# Find bzip2 and capture
# Find misc in order to determine HDF5_PLUGIN+PATH.
# Assume all test filters are in same plugin dir
findplugin h5misc
echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
export HDF5_PLUGIN_PATH
echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}"
export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
fi # USEPLUGINS
resetrc() {

67
plugins/CMakeLists.txt
View File

@ -10,12 +10,6 @@ FILE(READ H5Znoop.c NOOP_SOURCE)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "#define NOOP_INSTANCE 1\n")
FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "${NOOP_SOURCE}")
SET(h5bzip2_SOURCES H5Zbzip2.c)
IF(NOT HAVE_BZIP2)
SET(h5bzip2_SOURCES ${h5bzip2_SOURCES} blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c)
ENDIF()
SET(h5misc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h)
SET(h5noop_SOURCES H5Znoop.c H5Zutil.c h5noop.h)
@ -29,11 +23,10 @@ SET(h5fletcher32_SOURCES H5Zfletcher32.c H5checksum.c)
SET(h5deflate_SOURCES H5Zdeflate.c)
SET(nczmisc_SOURCES NCZmisc.c)
SET(nczdefaults_SOURCES NCZdefaults.c)
IF(ENABLE_FILTER_TESTING)
IF(BUILD_UTILITIES)
SET(nczhdf5filters_SOURCES NCZhdf5filters.c)
SET(nczstdfilters_SOURCES NCZstdfilters.c)
IF(ENABLE_PLUGINS)
# LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined
SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
@ -64,7 +57,8 @@ buildplugin(h5fletcher32 "h5fletcher32")
buildplugin(h5deflate "h5deflate")
buildplugin(nczmisc "nczmisc")
buildplugin(nczdefaults "nczdefaults" netcdf)
buildplugin(nczhdf5filters "nczhdf5filters" netcdf)
buildplugin(nczstdfilters "nczstdfilters" netcdf)
IF(ENABLE_BLOSC)
SET(h5blosc_SOURCES H5Zblosc.c)
@ -80,17 +74,54 @@ ENDIF()
IF(HAVE_SZ)
SET(h5szip_SOURCES H5Zszip.c H5Zszip.h)
buildplugin(h5szip "h5szip" ${Szip_LIBRARIES})
SET(nczszip_SOURCES NCZszip.c)
buildplugin(nczszip "nczszip" netcdf)
ENDIF()
IF(HAVE_LOCAL_BZ2)
SET(h5bzip2_SOURCES H5Zbzip2.c blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h)
buildplugin(h5bzip2 "h5bzip2")
ELSE()
SET(h5bzip2_SOURCES H5Zbzip2.c)
buildplugin(h5bzip2 "h5bzip2" ${Bzip2_LIBRARIES})
# Note we use name h5bzip2 instead of bzip2 to avoid logical
# target name clash with examples/C/hdf5plugins
SET_TARGET_PROPERTIES(h5bzip2 PROPERTIES OUTPUT_NAME "bzip2")
ENDIF()
ENDIF(BUILD_UTILITIES)
ENDIF(ENABLE_FILTER_TESTING)
# Installation
IF(ENABLE_PLUGIN_INSTALL)
MACRO(installplugin PLUG)
IF(MSVC)
SET(BUILD_PLUGIN_LIB "${PLUG}.dll")
ELSE()
SET(BUILD_PLUGIN_LIB "lib${PLUG}.so")
ENDIF()
MESSAGE(STATUS "Installing: ${BUILD_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}")
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${BUILD_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR})
ENDMACRO()
install(DIRECTORY DESTINATION ${PLUGIN_INSTALL_DIR})
IF(Bzip2_FOUND)
installplugin(h5bzip2)
ENDIF()
IF(Zstd_FOUND)
installplugin(h5zstd)
ENDIF()
IF(Blosc_FOUND)
installplugin(h5blosc)
ENDIF()
IF(ENABLE_NCZARR)
installplugin(h5fletcher32)
installplugin(h5shuffle)
installplugin(h5deflate)
installplugin(nczhdf5filters)
installplugin(nczstdfilters)
IF(Szip_FOUND)
installplugin(h5szip)
ENDIF()
ENDIF()
ENDIF(ENABLE_PLUGIN_INSTALL)
ENDIF(ENABLE_PLUGINS)
# Copy some test files from current source dir to out-of-tree build dir.
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)

240
plugins/H5Zblosc.c
View File

@ -125,8 +125,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define PUSH_ERR(f,m,s,...) fprintf(stderr,"%s\n",s)
#endif /*USE_HDF5*/
static int h5z_blosc_initialized = 0;
static size_t blosc_filter(unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t* buf_size, void** buf);
@ -162,7 +160,7 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space)
hid_t super_type;
H5T_class_t classt;
r = GET_FILTER(dcpl, FILTER_BLOSC, &flags, &nelements, values, 0, NULL);
r = GET_FILTER(dcpl, H5Z_FILTER_BLOSC, &flags, &nelements, values, 0, NULL);
if (r < 0) return -1;
if (nelements < 4) nelements = 4; /* First 4 slots reserved. */
@ -209,7 +207,7 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space)
fprintf(stderr, "Blosc: Computed buffer size %d\n", bufsize);
#endif
r = H5Pmodify_filter(dcpl, FILTER_BLOSC, flags, nelements, values);
r = H5Pmodify_filter(dcpl, H5Z_FILTER_BLOSC, flags, nelements, values);
if (r < 0) return -1;
return 1;
@ -365,7 +363,7 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts,
const H5Z_class2_t blosc_H5Filter[1] = {
{
H5Z_CLASS_T_VERS,
(H5Z_filter_t)(FILTER_BLOSC),
(H5Z_filter_t)(H5Z_FILTER_BLOSC),
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"blosc",
@ -379,235 +377,3 @@ const H5Z_class2_t blosc_H5Filter[1] = {
H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_FILTER; }
const void* H5PLget_plugin_info(void) { return blosc_H5Filter; }
/* Provide the codec support for the HDF5 blosc library */
/* NCZarr Filter Objects */
#define DEFAULT_LEVEL 9
#define DEFAULT_BLOCKSIZE 1
#define DEFAULT_TYPESIZE 1
#define DEFAULT_COMPCODE BLOSC_LZ4
/* Forward */
static void NCZ_blosc_codec_finalize(void);
static int NCZ_blosc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
static int NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
/* Structure for NCZ_PLUGIN_CODEC */
static NCZ_codec_t NCZ_blosc_codec = {/* NCZ_codec_t codec fields */
NCZ_CODEC_CLASS_VER, /* Struct version number */
NCZ_CODEC_HDF5, /* Struct sort */
"blosc", /* Standard name/id of the codec */
FILTER_BLOSC, /* HDF5 alias for blosc */
NULL, /*NCZ_blosc_codec_initialize*/
NCZ_blosc_codec_finalize,
NCZ_blosc_codec_to_hdf5,
NCZ_blosc_hdf5_to_codec,
NCZ_blosc_modify_parameters,
};
/* External Export API */
const void*
NCZ_get_codec_info(void)
{
if(!h5z_blosc_initialized) {
h5z_blosc_initialized = 1;
blosc_init();
}
return (void*)&NCZ_blosc_codec;
}
/* NCZarr Interface Functions */
/* Create the true parameter set:
Visible parameters:
param[0] -- reserved
param[1] -- reserved
param[2] -- reserved
param[3] -- variable chunksize in bytes | 0 (=>default)
param[4] -- compression level
param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
param[6] -- compressor index
Working parameters:
param[0] -- filter revision
param[1] -- blosc version
param[2] -- variable type size in bytes
param[3] -- variable chunksize in bytes
param[4] -- compression level
param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
param[6] -- compressor index
*/
static void
NCZ_blosc_codec_finalize(void)
{
if(h5z_blosc_initialized) {
blosc_destroy();
h5z_blosc_initialized = 0;
}
}
static int
NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
{
int i,stat = NC_NOERR;
nc_type vtype;
int storage, ndims;
size_t* chunklens = NULL;
size_t typesize, chunksize;
char vname[NC_MAX_NAME+1];
unsigned* params = NULL;
size_t nparams;
size_t vnparams = *vnparamsp;
unsigned* vparams = *vparamsp;
if(vnparams < 7)
{stat = NC_EFILTER; goto done;}
nparams = 7;
if(vparams == NULL)
{stat = NC_EFILTER; goto done;}
if(wnparamsp == NULL || wparamsp == NULL)
{stat = NC_EFILTER; goto done;}
vnparams = *vnparamsp;
vparams = *vparamsp;
/* Get variable info */
if((stat = nc_inq_var(ncid,varid,vname,&vtype,&ndims,NULL,NULL))) goto done;
if(ndims == 0) {stat = NC_EFILTER; goto done;}
/* Get the typesize */
if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
/* Compute chunksize */
if((chunklens = (size_t*)calloc(ndims,sizeof(size_t)))==NULL) goto done;
if((stat = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
if(storage != NC_CHUNKED) {stat = NC_EFILTER; goto done;}
chunksize = typesize;
for(i=0;i<ndims;i++) chunksize *= chunklens[i];
if((params = (unsigned*)malloc(vnparams*sizeof(unsigned)))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(params,vparams,vnparams*sizeof(unsigned));
params[0] = FILTER_BLOSC_VERSION;
params[1] = BLOSC_VERSION_FORMAT;
params[2] = (unsigned)typesize;
params[3] = chunksize;
params[4] = params[4];
params[5] = params[5];
params[6] = params[6];
*wnparamsp = nparams;
nullfree(*wparamsp);
*wparamsp = params; params = NULL;
done:
nullfree(chunklens);
nullfree(params);
FUNC_LEAVE_NOAPI(stat)
}
static int
NCZ_blosc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
NCjson* jcodec = NULL;
NCjson* jtmp = NULL;
unsigned* params = NULL;
struct NCJconst jc = {0,0,0,NULL};
int compcode;
/* parse the JSON */
if(NCJparse(codec_json,0,&jcodec)) {stat = NC_EFILTER; goto done;}
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
/* Verify the codec ID */
if(NCJdictget(jcodec,"id",&jtmp))
{stat = NC_EFILTER; goto done;}
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;}
if(strcmp(NCJstring(jtmp),NCZ_blosc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
if((params = (unsigned*)calloc(7,sizeof(unsigned)))==NULL) {stat = NC_ENOMEM; goto done;}
/* Get compression level*/
if(NCJdictget(jcodec,"clevel",&jtmp)) {stat = NC_EFILTER; goto done;}
if(jtmp) {
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
} else
jc.ival = DEFAULT_LEVEL;
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
params[4] = (unsigned)jc.ival;
/* Get blocksize */
if(NCJdictget(jcodec,"blocksize",&jtmp)) {stat = NC_EFILTER; goto done;}
if(jtmp) {
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
} else
jc.ival = DEFAULT_BLOCKSIZE;
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
params[3] = (unsigned)jc.ival;
/* Get shuffle */
if(NCJdictget(jcodec,"shuffle",&jtmp)) {stat = NC_EFILTER; goto done;}
if(jtmp) {
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
} else
jc.ival = BLOSC_NOSHUFFLE;
params[5] = (unsigned)jc.ival;
/* Get compname */
if(NCJdictget(jcodec,"cname",&jtmp)) {stat = NC_EFILTER; goto done;}
if(jtmp) {
if(NCJcvt(jtmp,NCJ_STRING,&jc)) {stat = NC_EFILTER; goto done;}
if(jc.sval == NULL || strlen(jc.sval) == 0) {stat = NC_EFILTER; goto done;}
if((compcode = blosc_compname_to_compcode(jc.sval)) < 0) {stat = NC_EFILTER; goto done;}
} else
compcode = DEFAULT_COMPCODE;
params[6] = (unsigned)compcode;
if(nparamsp) *nparamsp = 7;
if(paramsp) {*paramsp = params; params = NULL;}
done:
if(jc.sval) {
free(jc.sval);
}
if(params) {
free(params);
}
NCJreclaim(jcodec);
return stat;
}
static int
NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
{
int stat = NC_NOERR;
char json[1024];
const char* compname = NULL;
if(nparams == 0 || params == NULL)
{stat = NC_EINVAL; goto done;}
/* Get the sub-compressor name */
if(blosc_compcode_to_compname((int)params[6],&compname) < 0) {stat = NC_EFILTER; goto done;}
snprintf(json,sizeof(json),
"{\"id\": \"blosc\",\"clevel\": %u,\"blocksize\": %u,\"cname\": \"%s\",\"shuffle\": %d}",
params[4],params[3],compname,params[5]);
if(codecp) {
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}

16
plugins/H5Zblosc.h
View File

@ -29,13 +29,6 @@ extern "C" {
#include "netcdf_filter_build.h"
#include "blosc.h"
/* Filter revision number, starting at 1 */
/* #define FILTER_BLOSC_VERSION 1 */
#define FILTER_BLOSC_VERSION 2 /* multiple compressors since Blosc 1.3 */
/* Filter ID registered with the HDF Group */
#define FILTER_BLOSC 32001
#ifdef _MSC_VER
#ifdef DLL_EXPORT /* define when building the library */
#define DECLSPEC __declspec(dllexport)
@ -46,6 +39,15 @@ extern "C" {
#define DECLSPEC extern
#endif
/* Filter revision number, starting at 1 */
/* #define FILTER_BLOSC_VERSION 1 */
#define FILTER_BLOSC_VERSION 2 /* multiple compressors since Blosc 1.3 */
#define DEFAULT_LEVEL 9
#define DEFAULT_BLOCKSIZE 1
#define DEFAULT_TYPESIZE 1
#define DEFAULT_COMPCODE BLOSC_LZ4
/* HDF5 Plugin API */
DECLSPEC H5PL_type_t H5PLget_plugin_type(void);
DECLSPEC const void* H5PLget_plugin_info(void);

89
plugins/H5Zbzip2.c
View File

@ -214,92 +214,3 @@ H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts,
return 0;
}
/**************************************************/
/* NCZarr Filter Objects */
/* Provide the codec support for the HDF5 bzip library */
static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */
NCZ_CODEC_CLASS_VER, /* Struct version number */
NCZ_CODEC_HDF5, /* Struct sort */
"bz2", /* Standard name/id of the codec */
H5Z_FILTER_BZIP2, /* HDF5 alias for bzip2 */
NULL, /*NCZ_bzip2_codec_initialize*/
NULL, /*NCZ_bzip2_codec_finalize*/
NCZ_bzip2_codec_to_hdf5,
NCZ_bzip2_hdf5_to_codec,
NULL, /*NCZ_bzip2_modify_parameters*/
};
/* External Export API */
DLLEXPORT
const void*
NCZ_get_codec_info(void)
{
return (void*)&NCZ_bzip2_codec;
}
static int
NCZ_bzip2_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
NCjson* jcodec = NULL;
NCjson* jtmp = NULL;
unsigned* params = NULL;
struct NCJconst jc;
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EINTERNAL; goto done;}
if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
{stat = NC_ENOMEM; goto done;}
/* parse the JSON */
if(NCJparse(codec_json,0,&jcodec))
{stat = NC_EFILTER; goto done;}
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
/* Verify the codec ID */
if(NCJdictget(jcodec,"id",&jtmp))
{stat = NC_EFILTER; goto done;}
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
if(strcmp(NCJstring(jtmp),NCZ_bzip2_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
/* Get Level */
if(NCJdictget(jcodec,"level",&jtmp))
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
params[0] = (unsigned)jc.ival;
*nparamsp = 1;
*paramsp = params; params = NULL;
done:
if(params) free(params);
NCJreclaim(jcodec);
return stat;
}
static int
NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
{
int stat = NC_NOERR;
unsigned level = 0;
char json[1024];
if(nparams == 0 || params == NULL)
{stat = NC_EFILTER; goto done;}
level = params[0];
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level);
if(codecp) {
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}

89
plugins/H5Zzstd.c
View File

@ -166,92 +166,3 @@ cleanupAndFail:
return 0;
}
/**************************************************/
/* NCZarr Filter Objects */
/* Provide the codec support for the HDF5 zstandard library */
static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */
NCZ_CODEC_CLASS_VER, /* Struct version number */
NCZ_CODEC_HDF5, /* Struct sort */
"zstd", /* Standard name/id of the codec */
H5Z_FILTER_ZSTD, /* HDF5 alias for zstd */
NULL, /*NCZ_zstd_codec_initialize*/
NULL, /*NCZ_zstd_codec_finalize*/
NCZ_zstd_codec_to_hdf5,
NCZ_zstd_hdf5_to_codec,
NULL, /*NCZ_zstd_modify_parameters*/
};
/* External Export API */
DLLEXPORT
const void*
NCZ_get_codec_info(void)
{
return (void*)&NCZ_zstd_codec;
}
static int
NCZ_zstd_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
NCjson* jcodec = NULL;
NCjson* jtmp = NULL;
unsigned* params = NULL;
struct NCJconst jc;
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EINTERNAL; goto done;}
if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
{stat = NC_ENOMEM; goto done;}
/* parse the JSON */
if(NCJparse(codec_json,0,&jcodec))
{stat = NC_EFILTER; goto done;}
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
/* Verify the codec ID */
if(NCJdictget(jcodec,"id",&jtmp))
{stat = NC_EFILTER; goto done;}
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
/* Get Level */
if(NCJdictget(jcodec,"level",&jtmp))
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
params[0] = (unsigned)jc.ival;
*nparamsp = 1;
*paramsp = params; params = NULL;
done:
if(params) free(params);
NCJreclaim(jcodec);
return stat;
}
static int
NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
{
int stat = NC_NOERR;
unsigned level = 0;
char json[1024];
if(nparams == 0 || params == NULL)
{stat = NC_EFILTER; goto done;}
level = params[0];
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level);
if(codecp) {
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}

59
plugins/Makefile.am
View File

@ -14,34 +14,22 @@ if ISMINGW
LDADD = ${top_builddir}/liblib/libnetcdf.la
endif
# BZIP2 version 1.0.8 (https://sourceware.org/bzip2/)
BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h
EXTRA_DIST = CMakeLists.txt
# The HDF5 filter wrappers
EXTRA_DIST += \
EXTRA_DIST += stdinstall.sh \
H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \
H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \
H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h H5Zzstd.c H5Zzstd.h
# The Codec filter wrappers
EXTRA_DIST += NCZdefaults.c NCZszip.c
EXTRA_DIST += NCZhdf5filters.c NCZstdfilters.c
# The Filter implementations
EXTRA_DIST += H5checksum.c
EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE
if ENABLE_FILTER_TESTING
lib_LTLIBRARIES += libh5bzip2.la
libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h
if ! HAVE_BZIP2
libh5bzip2_la_SOURCES += ${BZIP2SRC}
endif
noinst_LTLIBRARIES += libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la
noinst_LTLIBRARIES += libnczdefaults.la
if ENABLE_NCZARR_FILTERS
noinst_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la
@ -51,27 +39,39 @@ libh5deflate_la_SOURCES = H5Zdeflate.c
# Need our version of szip if libsz available and we are not using HDF5
if HAVE_SZ
noinst_LTLIBRARIES += libh5szip.la libnczszip.la
noinst_LTLIBRARIES += libh5szip.la
libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h
libnczszip_la_SOURCES = NCZszip.c
endif
libnczdefaults_la_SOURCES = NCZdefaults.c
endif # ENABLE_NCZARR_FILTERS
if ENABLE_PLUGINS
libnczstdfilters_la_SOURCES = NCZstdfilters.c
libnczhdf5filters_la_SOURCES = NCZhdf5filters.c
if HAVE_BLOSC
noinst_LTLIBRARIES += libh5blosc.la
libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h
endif
if HAVE_ZSTD
noinst_LTLIBRARIES += libh5zstd.la
libh5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h
endif
libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h
noinst_LTLIBRARIES += libnczhdf5filters.la
noinst_LTLIBRARIES += libnczstdfilters.la
if HAVE_BLOSC
noinst_LTLIBRARIES += libh5blosc.la
endif
if HAVE_ZSTD
noinst_LTLIBRARIES += libh5zstd.la
endif
endif #ENABLE_PLUGINS
libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h
libnczmisc_la_SOURCES = NCZmisc.c
# The noop filter is to allow testing of multifilters and filter order
@ -79,6 +79,14 @@ libnczmisc_la_SOURCES = NCZmisc.c
libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h
libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h
# Bzip2 is used to test more complex filters
libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h
BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h
EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE
if HAVE_LOCAL_BZ2
libh5bzip2_la_SOURCES += ${BZIP2SRC}
endif
endif #ENABLE_FILTER_TESTING
BUILT_SOURCES = H5Znoop1.c
@ -87,6 +95,9 @@ H5Znoop1.c: Makefile H5Znoop.c
echo '#define NOOP_INSTANCE 1' > $@
cat ${srcdir}/H5Znoop.c >> $@
noinst_LTLIBRARIES += libh5bzip2.la
# Record where bzip2 came from; may be out of date
BZIP2VER = 1.0.8
BZIP2DIR = bzip2-${BZIP2VER}
BZIP2URL = https://sourceware.org/pub/bzip2/${BZIP2DIR}.tar.gz
@ -96,3 +107,7 @@ bzip2::
tar -zxf ${BZIP2DIR}.tar.gz
cd ${BZIP2DIR}; cp ${BZIP2SRC} ..; cp LICENSE ../BZIP2_LICENSE ; cd ..
rm -fr ./${BZIP2DIR}
# Custom install
install-exec-hook:
sh ./stdinstall.sh

225
plugins/NCZdefaults.c → plugins/NCZhdf5filters.c
View File

@ -21,6 +21,11 @@ Author: Dennis Heimbigner
#include "netcdf_filter_build.h"
#include "netcdf_json.h"
#ifdef HAVE_SZ
#include <szlib.h>
#include "H5Zszip.h"
#endif
#define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */
#define H5Z_FILTER_SHUFFLE 2 /*shuffle the data */
#define H5Z_FILTER_FLETCHER32 3 /*fletcher32 checksum of EDC */
@ -40,6 +45,12 @@ static int NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparam
static int NCZ_deflate_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_deflate_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
#ifdef HAVE_SZ
static int NCZ_szip_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
static int NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
#endif
/**************************************************/
static NCZ_codec_t NCZ_shuffle_codec = {
@ -302,10 +313,224 @@ done:
/**************************************************/
#ifdef HAVE_SZ
static NCZ_codec_t NCZ_szip_codec = {
NCZ_CODEC_CLASS_VER, /* Struct version number */
NCZ_CODEC_HDF5, /* Struct sort */
"szip", /* Standard name/id of the codec */
H5Z_FILTER_SZIP, /* HDF5 alias for szip */
NULL, /*NCZ_szip_codec_initialize*/
NULL, /*NCZ_szip_codec_finalize*/
NCZ_szip_codec_to_hdf5,
NCZ_szip_hdf5_to_codec,
NCZ_szip_modify_parameters,
};
static int
NCZ_szip_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
unsigned* params = NULL;
size_t nparams = 2; /* No. of visible parameters */
NCjson* json = NULL;
NCjson* jtmp = NULL;
struct NCJconst jc = {0,0,0,NULL};
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EINTERNAL; goto done;}
if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL)
{stat = NC_ENOMEM; goto done;}
if(NCJparse(codec_json,0,&json))
{stat = NC_EFILTER; goto done;}
if(NCJdictget(json,"mask",&jtmp) || jtmp == NULL)
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
params[H5Z_SZIP_PARM_MASK] = (unsigned)jc.ival;
jtmp = NULL;
if(NCJdictget(json,"pixels-per-block",&jtmp) || jtmp == NULL)
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
params[H5Z_SZIP_PARM_PPB] = (unsigned)jc.ival;
*nparamsp = nparams;
*paramsp = params; params = NULL;
done:
NCJreclaim(json);
nullfree(params);
return stat;
}
static int
NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
{
int stat = NC_NOERR;
char json[2048];
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": %u, \"pixels-per-block\": %u}",
NCZ_szip_codec.codecid,
params[H5Z_SZIP_PARM_MASK],
params[H5Z_SZIP_PARM_PPB]);
if(codecp) {
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
static int
NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
{
int i,ret_value = NC_NOERR;
nc_type vtype;
size_t typesize, scanline, dtype_precision, npoints;
int ndims, storage, dtype_order;
int dimids[NC_MAX_VAR_DIMS];
char vname[NC_MAX_NAME+1];
size_t chunklens[NC_MAX_VAR_DIMS];
unsigned* params = NULL;
unsigned* vparams = NULL;
size_t wnparams = 4;
if(wnparamsp == NULL || wparamsp == NULL)
{ret_value = NC_EFILTER; goto done;}
if(vnparamsp == NULL || vparamsp == NULL)
{ret_value = NC_EFILTER; goto done;}
if(*vnparamsp > 0 && *vparamsp == NULL)
{ret_value = NC_EFILTER; goto done;}
vparams = *vparamsp;
/* Get variable info */
if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done;
/* Get the typesize */
if((ret_value = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
/* Get datatype's precision, in case is less than full bits */
dtype_precision = typesize*8;
if(dtype_precision > 24) {
if(dtype_precision <= 32)
dtype_precision = 32;
else if(dtype_precision <= 64)
dtype_precision = 64;
} /* end if */
if(ndims == 0) {ret_value = NC_EFILTER; goto done;}
/* Set "local" parameter for this dataset's "pixels-per-scanline" */
if((ret_value = nc_inq_dimlen(ncid,dimids[ndims-1],&scanline))) goto done;
/* Get number of elements for the dataspace; use
total number of elements in the chunk to define the new 'scanline' size */
/* Compute chunksize */
if((ret_value = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
if(storage != NC_CHUNKED) {ret_value = NC_EFILTER; goto done;}
npoints = 1;
for(i=0;i<ndims;i++) npoints *= chunklens[i];
/* Get datatype's endianness order */
if((ret_value = nc_inq_var_endian(ncid,varid,&dtype_order))) goto done;
if((params = (unsigned*)malloc(wnparams*sizeof(unsigned)))==NULL)
{ret_value = NC_ENOMEM; goto done;}
params[H5Z_SZIP_PARM_MASK] = vparams[H5Z_SZIP_PARM_MASK];
params[H5Z_SZIP_PARM_PPB] = vparams[H5Z_SZIP_PARM_PPB];
/* Set "local" parameter for this dataset's "bits-per-pixel" */
params[H5Z_SZIP_PARM_BPP] = dtype_precision;
/* Adjust scanline if it is smaller than number of pixels per block or
if it is bigger than maximum pixels per scanline, or there are more than
SZ_MAX_BLOCKS_PER_SCANLINE blocks per scanline */
if(scanline < vparams[H5Z_SZIP_PARM_PPB]) {
if(npoints < vparams[H5Z_SZIP_PARM_PPB])
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "pixels per block greater than total number of elements in the chunk")
scanline = MIN((vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), npoints);
} else {
if(scanline <= SZ_MAX_PIXELS_PER_SCANLINE)
scanline = MIN((vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), scanline);
else
scanline = vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE;
} /* end else */
/* Assign the final value to the scanline */
params[H5Z_SZIP_PARM_PPS] = (unsigned)scanline;
/* Set the correct mask flags */
/* From H5Pdcpl.c#H5Pset_szip */
params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~H5_SZIP_CHIP_OPTION_MASK);
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_ALLOW_K13_OPTION_MASK;
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_RAW_OPTION_MASK;
params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~(H5_SZIP_LSB_OPTION_MASK | H5_SZIP_MSB_OPTION_MASK));
/* From H5Zszip.c#H5Z__set_local_szip */
params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~(H5_SZIP_LSB_OPTION_MASK | H5_SZIP_MSB_OPTION_MASK));
switch(dtype_order) {
case NC_ENDIAN_LITTLE: /* Little-endian byte order */
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_LSB_OPTION_MASK;
break;
case NC_ENDIAN_BIG: /* Big-endian byte order */
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_MSB_OPTION_MASK;
break;
default:
HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
} /* end switch */
*wnparamsp = wnparams;
nullfree(*wparamsp);
*wparamsp = params; params = NULL;
done:
nullfree(params);
FUNC_LEAVE_NOAPI(ret_value)
}
#if 0
static int
NCZ_szip_visible_parameters(int ncid, int varid, size_t nparamsin, const unsigned int* paramsin, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
unsigned* params = NULL;
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EFILTER; goto done;}
if((params = (unsigned*)malloc(2*sizeof(unsigned)))==NULL)
{stat = NC_ENOMEM; goto done;}
params[H5Z_SZIP_PARM_MASK] = paramsin[H5Z_SZIP_PARM_MASK];
params[H5Z_SZIP_PARM_PPB] = paramsin[H5Z_SZIP_PARM_PPB];
nullfree(*paramsp);
*paramsp = params; params = NULL;
done:
nullfree(params);
return stat;
}
#endif
#endif /*HAVE_SZ*/
/**************************************************/
NCZ_codec_t* NCZ_default_codecs[] = {
&NCZ_shuffle_codec,
&NCZ_fletcher32_codec,
&NCZ_zlib_codec,
#ifdef HAVE_SZ
&NCZ_szip_codec,
#endif
NULL
};

460
plugins/NCZstdfilters.c Normal file
View File

@ -0,0 +1,460 @@
/* Copyright 2003-2018, University Corporation for Atmospheric
* Research. See the COPYRIGHT file for copying and redistribution
* conditions.
*/
/*
Author: Dennis Heimbigner
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include "netcdf.h"
#include "netcdf_filter.h"
#include "netcdf_filter_build.h"
#include "netcdf_json.h"
/* Provide Codec information for the standard filters */
#ifndef H5Z_FILTER_BZIP2
#define H5Z_FILTER_BZIP2 1
#define H5Z_FILTER_ZSTD 2
#define H5Z_FILTER_BLOSC 3
#endif
#ifdef HAVE_BLOSC
#include "H5Zblosc.h"
#endif
/**************************************************/
/* NCZarr Filter Objects */
/* Forward */
#ifdef HAVE_BZ2
static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
#endif
#ifdef HAVE_ZSTD
static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
#endif
#ifdef HAVE_BLOSC
static int NCZ_blosc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
static int NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
static void NCZ_blosc_codec_finalize(void);
#endif
/**************************************************/
/* Provide the codec support for bzip2 filter */
#ifdef HAVE_BZ2
static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */
NCZ_CODEC_CLASS_VER, /* Struct version number */
NCZ_CODEC_HDF5, /* Struct sort */
"bz2", /* Standard name/id of the codec */
H5Z_FILTER_BZIP2, /* HDF5 alias for bzip2 */
NULL, /*NCZ_bzip2_codec_initialize*/
NULL, /*NCZ_bzip2_codec_finalize*/
NCZ_bzip2_codec_to_hdf5,
NCZ_bzip2_hdf5_to_codec,
NULL, /*NCZ_bzip2_modify_parameters*/
};
/* External Export API */
DLLEXPORT
const void*
NCZ_get_codec_info(void)
{
return (void*)&NCZ_bzip2_codec;
}
static int
NCZ_bzip2_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
NCjson* jcodec = NULL;
NCjson* jtmp = NULL;
unsigned* params = NULL;
struct NCJconst jc;
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EINTERNAL; goto done;}
if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
{stat = NC_ENOMEM; goto done;}
/* parse the JSON */
if(NCJparse(codec_json,0,&jcodec))
{stat = NC_EFILTER; goto done;}
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
/* Verify the codec ID */
if(NCJdictget(jcodec,"id",&jtmp))
{stat = NC_EFILTER; goto done;}
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
if(strcmp(NCJstring(jtmp),NCZ_bzip2_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
/* Get Level */
if(NCJdictget(jcodec,"level",&jtmp))
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
params[0] = (unsigned)jc.ival;
*nparamsp = 1;
*paramsp = params; params = NULL;
done:
if(params) free(params);
NCJreclaim(jcodec);
return stat;
}
static int
NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
{
int stat = NC_NOERR;
unsigned level = 0;
char json[1024];
if(nparams == 0 || params == NULL)
{stat = NC_EFILTER; goto done;}
level = params[0];
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level);
if(codecp) {
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
#endif
/**************************************************/
/* Provide the codec support for zstandard filter */
#ifdef HAVE_ZSTD
static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */
NCZ_CODEC_CLASS_VER, /* Struct version number */
NCZ_CODEC_HDF5, /* Struct sort */
"zstd", /* Standard name/id of the codec */
H5Z_FILTER_ZSTD, /* HDF5 alias for zstd */
NULL, /*NCZ_zstd_codec_initialize*/
NULL, /*NCZ_zstd_codec_finalize*/
NCZ_zstd_codec_to_hdf5,
NCZ_zstd_hdf5_to_codec,
NULL, /*NCZ_zstd_modify_parameters*/
};
static int
NCZ_zstd_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
NCjson* jcodec = NULL;
NCjson* jtmp = NULL;
unsigned* params = NULL;
struct NCJconst jc;
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EINTERNAL; goto done;}
if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
{stat = NC_ENOMEM; goto done;}
/* parse the JSON */
if(NCJparse(codec_json,0,&jcodec))
{stat = NC_EFILTER; goto done;}
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
/* Verify the codec ID */
if(NCJdictget(jcodec,"id",&jtmp))
{stat = NC_EFILTER; goto done;}
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
/* Get Level */
if(NCJdictget(jcodec,"level",&jtmp))
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
params[0] = (unsigned)jc.ival;
*nparamsp = 1;
*paramsp = params; params = NULL;
done:
if(params) free(params);
NCJreclaim(jcodec);
return stat;
}
static int
NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
{
int stat = NC_NOERR;
unsigned level = 0;
char json[1024];
if(nparams == 0 || params == NULL)
{stat = NC_EFILTER; goto done;}
level = params[0];
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level);
if(codecp) {
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
#endif
/**************************************************/
/* Provide the codec support for blosc filter */
#ifdef HAVE_BLOSC
/* Structure for NCZ_PLUGIN_CODEC */
static NCZ_codec_t NCZ_blosc_codec = {/* NCZ_codec_t codec fields */
NCZ_CODEC_CLASS_VER, /* Struct version number */
NCZ_CODEC_HDF5, /* Struct sort */
"blosc", /* Standard name/id of the codec */
H5Z_FILTER_BLOSC, /* HDF5 alias for blosc */
NULL, /*NCZ_blosc_codec_initialize*/
NCZ_blosc_codec_finalize,
NCZ_blosc_codec_to_hdf5,
NCZ_blosc_hdf5_to_codec,
NCZ_blosc_modify_parameters,
};
/* NCZarr Interface Functions */
/* Create the true parameter set:
Visible parameters:
param[0] -- reserved
param[1] -- reserved
param[2] -- reserved
param[3] -- variable chunksize in bytes | 0 (=>default)
param[4] -- compression level
param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
param[6] -- compressor index
Working parameters:
param[0] -- filter revision
param[1] -- blosc version
param[2] -- variable type size in bytes
param[3] -- variable chunksize in bytes
param[4] -- compression level
param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE
param[6] -- compressor index
*/
void blosc_destroy(void);
static int ncz_blosc_initialized = 0;
static void
NCZ_blosc_codec_finalize(void)
{
if(ncz_blosc_initialized) {
blosc_destroy();
ncz_blosc_initialized = 0;
}
}
static int
NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
{
int i,stat = NC_NOERR;
nc_type vtype;
int storage, ndims;
size_t* chunklens = NULL;
size_t typesize, chunksize;
char vname[NC_MAX_NAME+1];
unsigned* params = NULL;
size_t nparams;
size_t vnparams = *vnparamsp;
unsigned* vparams = *vparamsp;
if(vnparams < 7)
{stat = NC_EFILTER; goto done;}
nparams = 7;
if(vparams == NULL)
{stat = NC_EFILTER; goto done;}
if(wnparamsp == NULL || wparamsp == NULL)
{stat = NC_EFILTER; goto done;}
vnparams = *vnparamsp;
vparams = *vparamsp;
/* Get variable info */
if((stat = nc_inq_var(ncid,varid,vname,&vtype,&ndims,NULL,NULL))) goto done;
if(ndims == 0) {stat = NC_EFILTER; goto done;}
/* Get the typesize */
if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
/* Compute chunksize */
if((chunklens = (size_t*)calloc(ndims,sizeof(size_t)))==NULL) goto done;
if((stat = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
if(storage != NC_CHUNKED) {stat = NC_EFILTER; goto done;}
chunksize = typesize;
for(i=0;i<ndims;i++) chunksize *= chunklens[i];
if((params = (unsigned*)malloc(vnparams*sizeof(unsigned)))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(params,vparams,vnparams*sizeof(unsigned));
params[0] = FILTER_BLOSC_VERSION;
params[1] = BLOSC_VERSION_FORMAT;
params[2] = (unsigned)typesize;
params[3] = chunksize;
params[4] = params[4];
params[5] = params[5];
params[6] = params[6];
*wnparamsp = nparams;
nullfree(*wparamsp);
*wparamsp = params; params = NULL;
done:
nullfree(chunklens);
nullfree(params);
FUNC_LEAVE_NOAPI(stat)
}
static int
NCZ_blosc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
NCjson* jcodec = NULL;
NCjson* jtmp = NULL;
unsigned* params = NULL;
struct NCJconst jc = {0,0,0,NULL};
int compcode;
/* parse the JSON */
if(NCJparse(codec_json,0,&jcodec)) {stat = NC_EFILTER; goto done;}
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
/* Verify the codec ID */
if(NCJdictget(jcodec,"id",&jtmp))
{stat = NC_EFILTER; goto done;}
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;}
if(strcmp(NCJstring(jtmp),NCZ_blosc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
if((params = (unsigned*)calloc(7,sizeof(unsigned)))==NULL) {stat = NC_ENOMEM; goto done;}
/* Get compression level*/
if(NCJdictget(jcodec,"clevel",&jtmp)) {stat = NC_EFILTER; goto done;}
if(jtmp) {
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
} else
jc.ival = DEFAULT_LEVEL;
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
params[4] = (unsigned)jc.ival;
/* Get blocksize */
if(NCJdictget(jcodec,"blocksize",&jtmp)) {stat = NC_EFILTER; goto done;}
if(jtmp) {
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
} else
jc.ival = DEFAULT_BLOCKSIZE;
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;}
params[3] = (unsigned)jc.ival;
/* Get shuffle */
if(NCJdictget(jcodec,"shuffle",&jtmp)) {stat = NC_EFILTER; goto done;}
if(jtmp) {
if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;}
} else
jc.ival = BLOSC_NOSHUFFLE;
params[5] = (unsigned)jc.ival;
/* Get compname */
if(NCJdictget(jcodec,"cname",&jtmp)) {stat = NC_EFILTER; goto done;}
if(jtmp) {
if(NCJcvt(jtmp,NCJ_STRING,&jc)) {stat = NC_EFILTER; goto done;}
if(jc.sval == NULL || strlen(jc.sval) == 0) {stat = NC_EFILTER; goto done;}
if((compcode = blosc_compname_to_compcode(jc.sval)) < 0) {stat = NC_EFILTER; goto done;}
} else
compcode = DEFAULT_COMPCODE;
params[6] = (unsigned)compcode;
if(nparamsp) *nparamsp = 7;
if(paramsp) {*paramsp = params; params = NULL;}
done:
if(jc.sval) {
free(jc.sval);
}
if(params) {
free(params);
}
NCJreclaim(jcodec);
return stat;
}
static int
NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
{
int stat = NC_NOERR;
char json[1024];
const char* compname = NULL;
if(nparams == 0 || params == NULL)
{stat = NC_EINVAL; goto done;}
/* Get the sub-compressor name */
if(blosc_compcode_to_compname((int)params[6],&compname) < 0) {stat = NC_EFILTER; goto done;}
snprintf(json,sizeof(json),
"{\"id\": \"blosc\",\"clevel\": %u,\"blocksize\": %u,\"cname\": \"%s\",\"shuffle\": %d}",
params[4],params[3],compname,params[5]);
if(codecp) {
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
#endif
/**************************************************/
NCZ_codec_t* NCZ_stdfilters_codecs[] = {
#ifdef HAVE_BZ2
&NCZ_bzip2_codec,
#endif
#ifdef HAVE_ZSTD
&NCZ_zstd_codec,
#endif
#ifdef HAVE_BLOSC
&NCZ_blosc_codec,
#endif
NULL
};
/* External Export API */
DLLEXPORT
const void*
NCZ_codec_info_defaults(void)
{
return (void*)&NCZ_stdfilters_codecs;
}

44
plugins/stdinstall.in Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -x
INSTALLDIR=@PLUGIN_INSTALL_DIR@
# Load the findplugins function
. ${builddir}/findplugin.sh
echo "findplugin.sh loaded"
installplugin() {
PLUG="$1"
# Locate the plugin path and the library name; argument order is critical
findplugin ${PLUG}
if ! test -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" ; then
echo "Not exists: ${HDF5_PLUGIN_DIR}/$HDF5_PLUGIN_LIB ; ignored"
return
fi
if ! test -d "${INSTALLDIR}" ; then
echo "Not exists: ${INSTALLDIR} ; creating"
mkdir ${INSTALLDIR}
fi
echo "Installing: $HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB into $INSTALLDIR"
cp -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" $INSTALLDIR
}
if test "x$USEPLUGINS" != x ; then
if test "x$INSTALLDIR" != x ; then
installplugin h5bzip2
installplugin h5zstd
installplugin h5blosc
if test "x$FEATURE_NCZARR" ; then
installplugin h5fletcher32
installplugin h5shuffle
installplugin h5deflate
installplugin h5szip
installplugin nczdefaults
installplugin nczszip
fi
fi
fi

3
test_common.in
View File

@ -24,12 +24,15 @@ FP_ISREGEDIT=@ISREGEDIT@
# Feature flags
FEATURE_HDF5=@HAS_HDF5@
FEATURE_PARALLEL=@HAS_PARALLEL@
# Define selected features of the build
FEATURE_HDF5=@HAS_HDF5@
FEATURE_NCZARR=@HAS_NCZARR@
FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@
FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@
FEATURE_FILTERTESTS=@DO_FILTER_TESTS@
FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@
set -e

6
unit_test/test_pathcvt.c
View File

@ -108,12 +108,12 @@ static Test PATHTESTS[] = {
char* macros[128];
/*Forward */
#ifdef DEBUG
static const char* kind2string(int kind);
#endif
static char* expand(const char* s);
static void setmacros(void);
static void reclaimmacros(void);
#ifdef DEBUG
static const char* kind2string(int kind);
#endif
int
main(int argc, char** argv)