mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-25 17:40:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2245 from DennisHeimbigner/filterenhance.dmh

Browse Source
This commit is contained in:
Ward Fisher 2022-03-21 16:45:27 -06:00 committed by GitHub
commit cd0f1690e8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
98 changed files with 2427 additions and 1327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
.github/workflows/mingw.yml vendored
View File

@ -1,29 +0,0 @@
name: NetCDF-Build MinGW
on: [workflow_dispatch,push]
jobs:
build:
runs-on: windows-latest
defaults:
run:
shell: msys2 {0}
steps:
- uses: actions/checkout@v2
- uses: msys2/setup-msys2@v2
with:
msystem: MINGW64
update: true
install: git mingw-w64-x86_64-toolchain automake libtool autoconf make mingw-w64-x86_64-hdf5 unzip
- name: CI-Build
shell: msys2 {0}
run: |
echo 'Running in MSYS2!'
set -e
pwd
git clone --single-branch --branch moreosfixes.tmp https://github.com/DennisHeimbigner/netcdf-c.git
cd netcdf-c
autoreconf -i --force
./configure --prefix=/builddir -enable-shared --disable-static --disable-plugins --disable-logging --disable-dap-remote-tests --disable-byterange
make -j LDFLAGS="-no-undefined -Wl,--export-all-symbols"
make check

2
.github/workflows/run_tests_ubuntu.yml vendored
View File

@ -4,7 +4,7 @@
name: Run Ubuntu/Linux netCDF Tests
on: [pull_request]
on: [ pull_request ]
jobs:

2
.github/workflows/run_tests_win_mingw.yml vendored
View File

@ -7,7 +7,7 @@
name: Run MSYS2, MinGW64-based Tests
on: [ pull_request ]
on: [pull_request]
jobs:

138
CMakeLists.txt
View File

@ -42,7 +42,7 @@ SET(PACKAGE_VERSION ${VERSION})
# Version of the dispatch table. This must match the value in
# configure.ac.
SET(NC_DISPATCH_VERSION 4)
SET(NC_DISPATCH_VERSION 5)
# Get system configuration, Use it to determine osname, os release, cpu. These
# will be used when committing to CDash.
@ -601,6 +601,33 @@ ENDIF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
# SET(BUILD_RPC ON CACHE BOOL "")
#ENDIF()
# Note that szip management is tricky.
# This is because we have three things to consider:
# 1. is libsz available?
# 2. is szip enabled in HDF5?
# 3. is nczarr enabled?
# We need separate flags for cases 1 and 2
# We need to determine if libsz is available both for HDF5 and NCZarr
# If user has specified the `SZIP_LIBRARY`, use it; otherwise try to find...
IF(NOT SZIP_LIBRARY)
FIND_LIBRARY(SZIP PATH NAMES szip sz sz2)
IF(SZIP)
SET(SZIP_LIBRARY ${SZIP})
ELSE()
UNSET(SZIP_LIBRARY)
UNSET(SZIP)
ENDIF()
ENDIF()
IF(SZIP_LIBRARY)
SET(SZIP_FOUND yes)
SET(HAVE_SZ yes)
ELSE()
SET(SZIP_FOUND no)
SET(HAVE_SZ no)
ENDIF()
##
# Option to Enable HDF5
#
@ -860,19 +887,18 @@ IF(USE_HDF5)
#error
#endif
int main() {
int x = 1;}" USE_SZIP)
IF(USE_SZIP)
int x = 1;}" USE_HDF5_SZIP)
IF(USE_HDF5_SZIP)
SET(HAVE_H5Z_SZIP yes)
# If user has specified the `SZIP_LIBRARY`, use it; otherwise try to find...
IF(NOT SZIP_LIBRARY)
FIND_LIBRARY(SZIP PATH NAMES szip sz)
SET(SZIP_LIBRARY ${SZIP})
IF(NOT SZIP)
IF(SZIP_FOUND)
SET(CMAKE_REQUIRED_LIBRARIES ${SZIP_LIBRARY} ${CMAKE_REQUIRED_LIBRARIES})
MESSAGE(STATUS "HDF5 has szip.")
ELSE()
MESSAGE(FATAL_ERROR "HDF5 Requires SZIP, but cannot find libszip or libsz.")
ENDIF()
ENDIF()
SET(HAVE_H5Z_SZIP 1)
SET(CMAKE_REQUIRED_LIBRARIES ${SZIP_LIBRARY} ${CMAKE_REQUIRED_LIBRARIES})
MESSAGE(STATUS "HDF5 has szip.")
ELSE()
SET(HAVE_H5Z_SZIP no)
ENDIF()
####
@ -904,9 +930,6 @@ IF(USE_HDF5)
#Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0)
CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5Pset_all_coll_metadata_ops "" HDF5_HAS_COLL_METADATA_OPS)
CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5free_memory "" HAVE_H5FREE_MEMORY)
CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5allocate_memory "" HAVE_H5ALLOCATE_MEMORY)
CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5resize_memory "" HAVE_H5RESIZE_MEMORY)
IF(HDF5_PARALLEL)
SET(HDF5_CC h5pcc)
@ -1068,30 +1091,33 @@ ELSE()
SET(ENABLE_ZLIB FALSE)
ENDIF()
# See if we have libblosc
IF(!MSVC)
FIND_PACKAGE(Blosc)
ENDIF()
# Define a test flag for have blosc library
IF(Blosc_FOUND)
INCLUDE_DIRECTORIES(${Blosc_INCLUDE_DIRS})
SET(ENABLE_BLOSC TRUE)
macro(set_std_filter filter)
# Upper case the filter name
string(TOUPPER "${filter}" upfilter)
string(TOLOWER "${filter}" downfilter)
# Define a test flag for filter
IF(${filter}_FOUND)
INCLUDE_DIRECTORIES(${filter}_INCLUDE_DIRS})
SET(ENABLE_${upfilter} TRUE)
SET(STD_FILTERS "${STD_FILTERS},${downfilter}")
ELSE()
SET(ENABLE_BLOSC FALSE)
SET(ENABLE_${upfilter} FALSE)
ENDIF()
endmacro(set_std_filter)
# See if we have libszip
IF(!MSVC)
#FIND_PACKAGE(SZIP)
#FIND_LIBRARY(SZIP PATH NAMES szip sz)
SET(SZIP_LIBRARY ${SZIP})
ENDIF()
# Define a test flag for have szip library
IF(SZIP_FOUND)
INCLUDE_DIRECTORIES(${SZIP_INCLUDE_DIRS})
SET(ENABLE_SZIP TRUE)
ELSE()
SET(ENABLE_SZIP FALSE)
# Locate some compressors
FIND_PACKAGE(Bz2)
FIND_PACKAGE(Blosc)
FIND_PACKAGE(Zstd)
# Accumulate standard filters
set(STD_FILTERS "deflate") # Always have deflate */
set_std_filter(SZIP)
set_std_filter(Blosc)
set_std_filter(Zstd)
set_std_filter(Bz2)
IF(NOT Bz2_FOUND)
set(STD_FILTERS "${STD_FILTERS},bzip2") # Always have bzip2 */
ENDIF()
# See if we have libzip
@ -1173,21 +1199,22 @@ IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_NCZARR_S3)
SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCARR S3 TESTS" FORCE)
ENDIF()
# See if aws-s3-sdk is available
# But only if enabled
# Note we check for the library after checking for enable_nczarr_s3
# because for some reason this screws up if we unconditionally test for sdk
# and it is not available. Fix someday
IF(ENABLE_NCZARR_S3)
find_package(AWSSDK REQUIRED COMPONENTS s3;core)
IF(AWSSDK_FOUND)
SET(service s3;core)
AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES)
SET(ENABLE_S3_SDK ON CACHE BOOL "S3 SDK" FORCE)
# See if aws-s3-sdk is available
find_package(AWSSDK REQUIRED COMPONENTS s3;core)
IF(AWSSDK_FOUND)
SET(service s3;core)
AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES)
SET(ENABLE_S3_SDK ON CACHE BOOL "S3 SDK" FORCE)
ELSE()
SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
ENDIF()
ELSE()
SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
ENDIF()
ELSE(ENABLE_NCZARR_S3)
# Unconditionally disable
SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
ENDIF(ENABLE_NCZARR_S3)
IF(NOT ENABLE_S3_SDK)
IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS)
@ -1506,11 +1533,19 @@ IF(NOT BUILD_SHARED_LIBS)
ENDIF()
OPTION(ENABLE_NCZARR_FILTERS "Enable NCZarr filters" yes)
OPTION(ENABLE_NCZARR_FILTERS_TESTING "Enable NCZarr filter testing." yes)
# Constraints
IF (NOT ENABLE_PLUGINS)
SET(ENABLE_NCZARR_FILTERS OFF CACHE BOOL "Enable NCZarr Filters." FORCE)
MESSAGE(WARNING "ENABLE_FILTER_TESTING requires shared libraries. Disabling.")
SET(ENABLE_NCZARR_FILTERS OFF CACHE BOOL "Enable NCZarr Filters." FORCE)
ENDIF()
IF (NOT ENABLE_NCZARR)
MESSAGE(WARNING "ENABLE_NCZARR==NO => ENABLE_NCZARR_FILTERS==NO AND ENABLE_NCZARR_FILTER_TESTING==NO")
SET(ENABLE_NCZARR_FILTERS OFF CACHE BOOL "Disable NCZARR_FILTERS" FORCE)
ENDIF()
OPTION(ENABLE_NCZARR_FILTERS_TESTING "Enable NCZarr filter testing." yes)
IF (NOT ENABLE_NCZARR_FILTERS)
SET(ENABLE_NCZARR_FILTER_TESTING OFF CACHE BOOL "Enable NCZarr Filter Testing" FORCE)
ENDIF()
@ -2370,9 +2405,6 @@ is_enabled(ENABLE_V2_API HAS_NC2)
is_enabled(ENABLE_NETCDF_4 HAS_NC4)
is_enabled(ENABLE_HDF4 HAS_HDF4)
is_enabled(USE_HDF5 HAS_HDF5)
is_enabled(USE_SZIP HAS_SZIP)
is_enabled(USE_SZIP HAS_SZIP_WRITE)
is_enabled(USE_SZIP HAS_SZLIB_WRITE)
is_enabled(STATUS_PNETCDF HAS_PNETCDF)
is_enabled(STATUS_PARALLEL HAS_PARALLEL)
is_enabled(ENABLE_PARALLEL4 HAS_PARALLEL4)
@ -2386,7 +2418,6 @@ is_enabled(JNA HAS_JNA)
is_enabled(ENABLE_ZERO_LENGTH_COORD_BOUND RELAX_COORD_BOUND)
is_enabled(USE_CDF5 HAS_CDF5)
is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
is_enabled(HAVE_H5Z_SZIP HAS_SZLIB)
is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
is_enabled(ENABLE_NCZARR HAS_NCZARR)
is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS)
@ -2395,7 +2426,8 @@ is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
is_enabled(ENABLE_QUANTIZE HAS_QUANTIZE)
is_enabled(ENABLE_LOGGING HAS_LOGGING)
is_enabled(ENABLE_FILTER_TESTING DO_FILTER_TESTS)
is_enabled(ENABLE_BLOSC HAS_BLOSC)
is_enabled(HAVE_SZ HAS_SZIP)
is_enabled(HAVE_SZ HAS_SZLIB_WRITE)
# Generate file from template.
CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"

1
RELEASE_NOTES.md
View File

@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.2 - TBD
* [Enhancement] Improve filter support. More specifically (1) add nc_inq_filter_avail to check if a filter is available, (2) add the notion of standard filters, (3) cleanup szip support to fix interaction with NCZarr. See [Github #2245](https://github.com/Unidata/netcdf-c/pull/2245).
* [Bug Fix] Require that the type of the variable in nc_def_var_filter is not variable length. See [Github #/2231](https://github.com/Unidata/netcdf-c/pull/2231).
* [File Change] Apply HDF5 v1.8 format compatibility when writing to previous files, as well as when creating new files. The superblock version remains at 2 for newly created files. Full backward read/write compatibility for netCDF-4 is maintained in all cases. See [Github #2176](https://github.com/Unidata/netcdf-c/issues/2176).
* [Enhancement] Add ability to set dataset alignment for netcdf-4/HDF5 files. See [Github #2206](https://github.com/Unidata/netcdf-c/pull/2206).

64
cmake/modules/FindBz2.cmake Normal file
View File

@ -0,0 +1,64 @@
# Searches for an installation of the bz2 library. On success, it sets the following variables:
#
# Bz2_FOUND Set to true to indicate the bz2 library was found
# Bz2_INCLUDE_DIRS The directory containing the header file bz2/bz2.h
# Bz2_LIBRARIES The libraries needed to use the bz2 library
#
# To specify an additional directory to search, set Bz2_ROOT.
#
# Author: Siddhartha Chaudhuri, 2009
#
# Look for the header, first in the user-specified location and then in the system locations
SET(Bz2_INCLUDE_DOC "The directory containing the header file bz2.h")
FIND_PATH(Bz2_INCLUDE_DIRS NAMES bz2.h bz2/bz2.h PATHS ${Bz2_ROOT} ${Bz2_ROOT}/include DOC ${Bz2_INCLUDE_DOC} NO_DEFAULT_PATH)
IF(NOT Bz2_INCLUDE_DIRS) # now look in system locations
FIND_PATH(Bz2_INCLUDE_DIRS NAMES bzlib.h DOC ${Bz2_INCLUDE_DOC})
ENDIF(NOT Bz2_INCLUDE_DIRS)
SET(Bz2_FOUND FALSE)
IF(Bz2_INCLUDE_DIRS)
SET(Bz2_LIBRARY_DIRS ${Bz2_INCLUDE_DIRS})
IF("${Bz2_LIBRARY_DIRS}" MATCHES "/include$")
# Strip off the trailing "/include" in the path.
GET_FILENAME_COMPONENT(Bz2_LIBRARY_DIRS ${Bz2_LIBRARY_DIRS} PATH)
ENDIF("${Bz2_LIBRARY_DIRS}" MATCHES "/include$")
IF(EXISTS "${Bz2_LIBRARY_DIRS}/lib")
SET(Bz2_LIBRARY_DIRS ${Bz2_LIBRARY_DIRS}/lib)
ENDIF(EXISTS "${Bz2_LIBRARY_DIRS}/lib")
# Find Bz2 libraries
FIND_LIBRARY(Bz2_DEBUG_LIBRARY NAMES bz2d bz2_d libbz2d libbz2_d libbz2
PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug
PATHS ${Bz2_LIBRARY_DIRS} NO_DEFAULT_PATH)
FIND_LIBRARY(Bz2_RELEASE_LIBRARY NAMES bz2 libbz2
PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release
PATHS ${Bz2_LIBRARY_DIRS} NO_DEFAULT_PATH)
SET(Bz2_LIBRARIES )
IF(Bz2_DEBUG_LIBRARY AND Bz2_RELEASE_LIBRARY)
SET(Bz2_LIBRARIES debug ${Bz2_DEBUG_LIBRARY} optimized ${Bz2_RELEASE_LIBRARY})
ELSEIF(Bz2_DEBUG_LIBRARY)
SET(Bz2_LIBRARIES ${Bz2_DEBUG_LIBRARY})
ELSEIF(Bz2_RELEASE_LIBRARY)
SET(Bz2_LIBRARIES ${Bz2_RELEASE_LIBRARY})
ENDIF(Bz2_DEBUG_LIBRARY AND Bz2_RELEASE_LIBRARY)
IF(Bz2_LIBRARIES)
SET(Bz2_FOUND TRUE)
ENDIF(Bz2_LIBRARIES)
ENDIF(Bz2_INCLUDE_DIRS)
IF(Bz2_FOUND)
# IF(NOT Bz2_FIND_QUIETLY)
MESSAGE(STATUS "Found Bz2: headers at ${Bz2_INCLUDE_DIRS}, libraries at ${Bz2_LIBRARY_DIRS}")
MESSAGE(STATUS " library is ${Bz2_LIBRARIES}")
# ENDIF(NOT Bz2_FIND_QUIETLY)
ELSE(Bz2_FOUND)
IF(Bz2_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Bz2 library not found")
ENDIF(Bz2_FIND_REQUIRED)
ENDIF(Bz2_FOUND)

64
cmake/modules/FindZstd.cmake Normal file
View File

@ -0,0 +1,64 @@
# Searches for an installation of the zstd library. On success, it sets the following variables:
#
# Zstd_FOUND Set to true to indicate the zstd library was found
# Zstd_INCLUDE_DIRS The directory containing the header file zstd/zstd.h
# Zstd_LIBRARIES The libraries needed to use the zstd library
#
# To specify an additional directory to search, set Zstd_ROOT.
#
# Author: Siddhartha Chaudhuri, 2009
#
# Look for the header, first in the user-specified location and then in the system locations
SET(Zstd_INCLUDE_DOC "The directory containing the header file zstd.h")
FIND_PATH(Zstd_INCLUDE_DIRS NAMES zstd.h zstd/zstd.h PATHS ${Zstd_ROOT} ${Zstd_ROOT}/include DOC ${Zstd_INCLUDE_DOC} NO_DEFAULT_PATH)
IF(NOT Zstd_INCLUDE_DIRS) # now look in system locations
FIND_PATH(Zstd_INCLUDE_DIRS NAMES zstd.h zstd/zstd.h DOC ${Zstd_INCLUDE_DOC})
ENDIF(NOT Zstd_INCLUDE_DIRS)
SET(Zstd_FOUND FALSE)
IF(Zstd_INCLUDE_DIRS)
SET(Zstd_LIBRARY_DIRS ${Zstd_INCLUDE_DIRS})
IF("${Zstd_LIBRARY_DIRS}" MATCHES "/include$")
# Strip off the trailing "/include" in the path.
GET_FILENAME_COMPONENT(Zstd_LIBRARY_DIRS ${Zstd_LIBRARY_DIRS} PATH)
ENDIF("${Zstd_LIBRARY_DIRS}" MATCHES "/include$")
IF(EXISTS "${Zstd_LIBRARY_DIRS}/lib")
SET(Zstd_LIBRARY_DIRS ${Zstd_LIBRARY_DIRS}/lib)
ENDIF(EXISTS "${Zstd_LIBRARY_DIRS}/lib")
# Find Zstd libraries
FIND_LIBRARY(Zstd_DEBUG_LIBRARY NAMES zstdd zstd_d libzstdd libzstd_d libzstd
PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug
PATHS ${Zstd_LIBRARY_DIRS} NO_DEFAULT_PATH)
FIND_LIBRARY(Zstd_RELEASE_LIBRARY NAMES zstd libzstd
PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release
PATHS ${Zstd_LIBRARY_DIRS} NO_DEFAULT_PATH)
SET(Zstd_LIBRARIES )
IF(Zstd_DEBUG_LIBRARY AND Zstd_RELEASE_LIBRARY)
SET(Zstd_LIBRARIES debug ${Zstd_DEBUG_LIBRARY} optimized ${Zstd_RELEASE_LIBRARY})
ELSEIF(Zstd_DEBUG_LIBRARY)
SET(Zstd_LIBRARIES ${Zstd_DEBUG_LIBRARY})
ELSEIF(Zstd_RELEASE_LIBRARY)
SET(Zstd_LIBRARIES ${Zstd_RELEASE_LIBRARY})
ENDIF(Zstd_DEBUG_LIBRARY AND Zstd_RELEASE_LIBRARY)
IF(Zstd_LIBRARIES)
SET(Zstd_FOUND TRUE)
ENDIF(Zstd_LIBRARIES)
ENDIF(Zstd_INCLUDE_DIRS)
IF(Zstd_FOUND)
# IF(NOT Zstd_FIND_QUIETLY)
MESSAGE(STATUS "Found Zstd: headers at ${Zstd_INCLUDE_DIRS}, libraries at ${Zstd_LIBRARY_DIRS}")
MESSAGE(STATUS " library is ${Zstd_LIBRARIES}")
# ENDIF(NOT Zstd_FIND_QUIETLY)
ELSE(Zstd_FOUND)
IF(Zstd_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Zstd library not found")
ENDIF(Zstd_FIND_REQUIRED)
ENDIF(Zstd_FOUND)

25
config.h.cmake.in
View File

@ -178,6 +178,15 @@ are set when opening a binary file on Windows. */
/* Define to 1 if you have the `atexit function. */
#cmakedefine HAVE_ATEXIT 1
/* Define to 1 if bzip2 library available. */
#cmakedefine HAVE_BZIP2 1
/* Define to 1 if zstd library available. */
#cmakedefine HAVE_ZSTD 1
/* Define to 1 if blosc library available. */
#cmakedefine HAVE_BLOSC 1
/* Define to 1 if you have hdf5_coll_metadata_ops */
#cmakedefine HDF5_HAS_COLL_METADATA_OPS 1
@ -259,6 +268,9 @@ are set when opening a binary file on Windows. */
/* Define to 1 if you have the `H5Z_SZIP' function. */
#cmakedefine HAVE_H5Z_SZIP 1
/* Define to 1 if you have libsz */
#cmakedefine HAVE_SZ 1
/* Define to 1 if you have the <hdf5.h> header file. */
#cmakedefine HAVE_HDF5_H 1
@ -422,16 +434,6 @@ are set when opening a binary file on Windows. */
/* Define to 1 if the system has the type `ushort'. */
#cmakedefine HAVE_USHORT 1
/* if true, H5free_memory() will be used to free hdf5-allocated memory in
nc4file. */
#cmakedefine HAVE_H5FREE_MEMORY 1
/* if true, H5allocate_memory() will be used. */
#cmakedefine HAVE_H5ALLOCATE_MEMORY 1
/* if true, H5resize_memory() will be used. */
#cmakedefine HAVE_H5RESIZE_MEMORY 1
/* if true, hdf5 has parallelism enabled */
#cmakedefine HDF5_PARALLEL 1
@ -620,9 +622,6 @@ with zip */
/* if true, enable nczarr blosc support */
#cmakedefine ENABLE_BLOSC 1
/* if true, enable nczarr szip support */
#cmakedefine USE_SZIP 1
/* Version number of package */
#cmakedefine VERSION "${netCDF_VERSION}"

130
configure.ac
View File

@ -599,16 +599,64 @@ if test "x$enable_nczarr" = xyes; then
fi
AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
# See if we have libblosc
AC_CHECK_LIB([blosc],[blosc_init],[enable_blosc=yes],[enable_blosc=no])
if test "x$enable_blosc" = "xyes" ; then
AC_SEARCH_LIBS([blosc_init],[blosc blosc.dll cygblosc.dll], [], [])
AC_DEFINE([ENABLE_BLOSC], [1], [if true, blosc library is available])
fi
AC_MSG_CHECKING([whether libblosc library is available])
AC_MSG_RESULT([${enable_blosc}])
##########
# Look for Standardized libraries
##########
# See if we have libzip
# See if we have libblosc
AC_CHECK_LIB([blosc],[blosc_init],[have_blosc=yes],[have_blosc=no])
if test "x$have_blosc" = "xyes" ; then
AC_SEARCH_LIBS([blosc_init],[blosc blosc.dll cygblosc.dll], [], [])
AC_DEFINE([HAVE_BLOSC], [1], [if true, blosc library is available])
fi
# See if we have libzstd
AC_CHECK_LIB([zstd],[ZSTD_compress],[have_zstd=yes],[have_zstd=no])
if test "x$have_zstd" = "xyes" ; then
AC_SEARCH_LIBS([ZSTD_compress],[zstd zstd.dll cygzstd.dll], [], [])
AC_DEFINE([HAVE_ZSTD], [1], [if true, zstd library is available])
fi
AC_MSG_CHECKING([whether libzstd library is available])
AC_MSG_RESULT([${have_zstd}])
# See if we have libbzip2 or libbz2
AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no])
if test "x$have_bzip2" = "xyes" ; then
AC_SEARCH_LIBS([BZ2_bzCompress],[bz2 bz2.dll cygbz2.dll], [], [])
AC_DEFINE([HAVE_BZIP2], [1], [if true, bz2 library is installed])
fi
AC_MSG_CHECKING([whether libbz2 library is available])
AC_MSG_RESULT([${have_bzip2}])
if test "x$have_bzip2" = "xno" ; then
AC_CHECK_LIB([bzip2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no])
if test "x$have_bzip2" = "xyes" ; then
AC_SEARCH_LIBS([BZ2_bzCompress],[bzip2 bzip2.dll cygbzip2.dll], [], [])
AC_DEFINE([HAVE_BZIP2], [1], [if true, bzip2 library is installed])
fi
AC_MSG_CHECKING([whether libbzip2 library is available])
AC_MSG_RESULT([${have_bzip2}])
fi
# Note that szip management is tricky.
# This is because we have three things to consider:
# 1. is libsz available?
# 2. is szip enabled in HDF5?
# 3. is nczarr enabled?
# We need separate flags for cases 1 and 2
# See if we have libsz (usually via libaec)
AC_CHECK_LIB([sz],[SZ_BufftoBuffCompress],[have_sz=yes],[have_sz=no])
if test "x$have_sz" = "xyes" ; then
AC_SEARCH_LIBS([SZ_BufftoBuffCompress],[sz sz.dll cygsz.dll], [], [])
AC_DEFINE([HAVE_SZ], [1], [if true, libsz (==szip) is available])
fi
AC_MSG_CHECKING([whether libsz library is available])
AC_MSG_RESULT([${have_sz}])
##########
# See if we have libzip for NCZarr
AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
if test "x$have_zip" = "xyes" ; then
AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [], [])
@ -1333,7 +1381,7 @@ fi
# Set defaults
hdf5_parallel=no
hdf5_supports_par_filters=no
enable_szlib=no
enable_hdf5_szip=no
has_ross3=no
if test "x$enable_hdf5" = xyes; then
@ -1360,7 +1408,7 @@ if test "x$enable_hdf5" = xyes; then
# H5Pset_fapl_mpiposix and H5Pget_fapl_mpiposix have been removed since HDF5 1.8.12.
# Use H5Pset_fapl_mpio and H5Pget_fapl_mpio, instead.
AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5free_memory H5resize_memory H5allocate_memory H5Pset_all_coll_metadata_ops H5Literate])
AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate])
# Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0)
if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then
@ -1404,10 +1452,10 @@ if test "x$enable_hdf5" = xyes; then
# must be able to link to szip library.
AC_MSG_CHECKING([whether szlib was used when building HDF5])
if test "x$ac_cv_func_H5Z_SZIP" = xyes; then
enable_szlib=yes
AC_DEFINE([USE_SZIP], [1], [if true, compile in szip compression in netCDF-4 variables])
enable_hdf5_szip=yes
AC_DEFINE([HAVE_H5Z_SZIP], [1], [if true, compile in szip compression in netCDF-4 variables])
fi
AC_MSG_RESULT([$enable_szlib])
AC_MSG_RESULT([$enable_hdf5_szip])
# Check to see if HDF5 library is 1.10.6 or greater.
# Used to control path name conversion
@ -1422,15 +1470,6 @@ if test "x$enable_hdf5" = xyes; then
AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8])
fi
else # !enable_hdf5
# See if we have libsz
AC_CHECK_LIB([sz],[SZ_BufftoBuffCompress],[enable_szlib=yes],[enable_szlib=no])
if test "x$enable_szlib" = "xyes" ; then
AC_SEARCH_LIBS([SZ_BufftoBuffCompress],[sz sz.dll cygsz.dll], [], [])
AC_DEFINE([USE_SZIP], [1], [if true, szip library is available])
fi
AC_MSG_CHECKING([whether szip library is available])
AC_MSG_RESULT([${enable_szlib}])
fi
AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ])
@ -1635,6 +1674,12 @@ AC_MSG_WARN([--disable-filter-testing => --disable-nczarr-filter-testing])
enable_nczarr_filter_testing=no
fi
if test "x$enable_nczarr" = xno; then
AC_MSG_WARN([--disable-nczarr => --disable-nczarr-filters])
enable_nczarr_filters=no
enable_nczarr_filter_testing=no
fi
if test "x$enable_nczarr_filters" = xyes; then
AC_DEFINE([ENABLE_NCZARR_FILTERS], [1], [if true, enable NCZarr filters])
fi
@ -1660,7 +1705,6 @@ AM_CONDITIONAL(ENABLE_CDF5, [test "x$enable_cdf5" = xyes])
AM_CONDITIONAL(ENABLE_DAP_REMOTE_TESTS, [test "x$enable_dap_remote_tests" = xyes])
AM_CONDITIONAL(ENABLE_DAP_AUTH_TESTS, [test "x$enable_dap_auth_tests" = xyes])
AM_CONDITIONAL(ENABLE_DAP_LONG_TESTS, [test "x$enable_dap_long_tests" = xyes])
AM_CONDITIONAL(USE_SZIP, [test "x$ac_cv_func_H5Z_SZIP" = xyes])
AM_CONDITIONAL(USE_PNETCDF_DIR, [test ! "x$PNETCDFDIR" = x])
AM_CONDITIONAL(USE_LOGGING, [test "x$enable_logging" = xyes])
AM_CONDITIONAL(CROSS_COMPILING, [test "x$cross_compiling" = xyes])
@ -1682,8 +1726,11 @@ AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
AM_CONDITIONAL(ENABLE_BLOSC, [test "x$enable_blosc" = xyes])
AM_CONDITIONAL(ENABLE_SZIP, [test "x$enable_szip" = xyes])
AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes])
AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes])
AM_CONDITIONAL(HAVE_BLOSC, [test "x$have_blosc" = xyes])
AM_CONDITIONAL(HAVE_BZIP2, [test "x$have_bzip2" = xyes])
AM_CONDITIONAL(HAVE_ZSTD, [test "x$have_zstd" = xyes])
# If the machine doesn't have a long long, and we want netCDF-4, then
# we've got problems!
@ -1771,8 +1818,6 @@ AC_SUBST(HAS_HDF4,[$enable_hdf4])
AC_SUBST(HAS_HDF5,[$enable_hdf5])
AC_SUBST(HAS_PNETCDF,[$enable_pnetcdf])
AC_SUBST(HAS_LOGGING, [$enable_logging])
AC_SUBST(HAS_SZLIB,[$enable_szlib])
AC_SUBST(HAS_SZLIB_WRITE, [$enable_szlib])
AC_SUBST(HAS_PARALLEL,[$enable_parallel])
AC_SUBST(HAS_PARALLEL4,[$enable_parallel4])
AC_SUBST(HAS_DISKLESS,[yes])
@ -1789,8 +1834,29 @@ AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
AC_SUBST([HAS_QUANTIZE],[yes])
AC_SUBST(HAS_LOGGING,[$enable_logging])
AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing])
AC_SUBST(HAVE_BLOSC,[$enable_blosc])
AC_SUBST(HAVE_SZIP,[$enable_szip])
AC_SUBST(HAS_SZLIB,[$have_sz])
AC_SUBST(HAS_SZLIB_WRITE, [$have_sz])
# Always available
std_filters="deflate,bzip2"
if test "x$enable_szlib" = xyes ; then
std_filters="${std_filters},szip"
fi
# We need to include szip iff HDF5 && HDF5_HAS_SZIP || !HDF5 && NCZARR && libsz
if test "x$enable_hdf5" = xyes && test "x$enable_hdf5_szip" = xyes ; then
std_filters="${std_filters},szip"
fi
if test "x$enable_hdf5" = xno && test "x$have_sz" = xyes ; then
std_filters="${std_filters},szip"
fi
if test "x$have_blosc" = xyes ; then
std_filters="${std_filters},blosc"
fi
if test "x$have_zstd" = xyes ; then
std_filters="${std_filters},zst"
fi
AC_SUBST(STD_FILTERS,[$std_filters])
# Access netcdf specific version of config.h
AH_BOTTOM([#include "ncconfigure.h"])
@ -1838,7 +1904,6 @@ AX_SET_META([NC_HAS_NC2],[$nc_build_v2],[yes])
AX_SET_META([NC_HAS_NC4],[$enable_netcdf_4],[yes])
AX_SET_META([NC_HAS_HDF4],[$enable_hdf4],[yes])
AX_SET_META([NC_HAS_HDF5],[$enable_hdf5],[yes])
AX_SET_META([NC_HAS_SZIP],[$ac_cv_func_H5Z_SZIP],[yes])
AX_SET_META([NC_HAS_DAP2],[$enable_dap],[yes])
AX_SET_META([NC_HAS_DAP4],[$enable_dap4],[yes])
AX_SET_META([NC_HAS_DISKLESS],[yes],[yes])
@ -1854,6 +1919,7 @@ AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes])
AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes])
AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes])
AX_SET_META([NC_HAS_SZIP],[$enable_hdf5_szip],[yes])
# This is the version of the dispatch table. If the dispatch table is
# changed, this should be incremented, so that user-defined format
@ -1861,7 +1927,7 @@ AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes])
# dispatch table to submit. If this is changed, make sure the value in
# CMakeLists.txt also changes to match.
AC_SUBST([NC_DISPATCH_VERSION], [4])
AC_SUBST([NC_DISPATCH_VERSION], [5])
AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.])
#####

8
h5_test/tst_h_enums.c
View File

@ -121,11 +121,7 @@ main()
if (the_value != val[i]) ERR;
member_name = H5Tget_member_name(typeid, i);
if (strcmp(member_name, love_how[i])) ERR;
#ifdef HAVE_H5FREE_MEMORY
H5free_memory(member_name);
#else
free(member_name);
#endif
}
/* Now read the data in the attribute and make sure it's what we
@ -239,11 +235,7 @@ main()
if (the_value != val[i]) ERR;
member_name = H5Tget_member_name(typeid, i);
if (strcmp(member_name, lang[i])) ERR;
#ifdef HAVE_H5FREE_MEMORY
H5free_memory(member_name);
#else
free(member_name);
#endif
}
/* Now read the data in the dataset and make sure it's what we

4
h5_test/tst_h_par_compress.c
View File

@ -25,11 +25,11 @@
/* The following code, when uncommented, adds szip testing for
* parallel I/O. However, this currently fails. I have a support
* request in to HDF5 about this. Ed 7/8/20 */
/* #ifdef USE_SZIP */
/* #ifdef HAVE_H5Z_SZIP */
/* #define NUM_COMPRESS_FILTERS 2 */
/* #else */
/* #define NUM_COMPRESS_FILTERS 1 */
/* #endif /\* USE_SZIP *\/ */
/* #endif /\* HAVE_H5Z_SZIP *\/ */
#define NUM_COMPRESS_FILTERS 1
int

4
h5_test/tst_h_vars.c
View File

@ -409,7 +409,7 @@ main()
}
SUMMARIZE_ERR;
#ifdef USE_SZIP
#ifdef HAVE_H5Z_SZIP
printf("*** Checking szip functionality...");
#define SZIP_VAR_NAME "szip_var"
#define SZIP_DIM1_LEN 32
@ -596,6 +596,6 @@ main()
} /* next file */
}
SUMMARIZE_ERR;
#endif /* USE_SZIP */
#endif /* HAVE_H5Z_SZIP */
FINAL_RESULTS;
}

2
include/Makefile.am
View File

@ -20,7 +20,7 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \
ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \
ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \
hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h \
ncfilter.h ncjson.h ncxml.h ncs3sdk.h
ncjson.h ncxml.h ncs3sdk.h
if USE_DAP
noinst_HEADERS += ncdap.h

1
include/hdf5internal.h
View File

@ -182,6 +182,7 @@ int nc4_hdf5_get_chunk_cache(int ncid, size_t *sizep, size_t *nelemsp,
int NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int filterid, size_t nparams, const unsigned int *params);
int NC4_hdf5_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int *filterids);
int NC4_hdf5_inq_var_filter_info(int ncid, int varid, unsigned int filterid, size_t* nparamsp, unsigned int *params);
int NC4_hdf5_inq_filter_avail(int ncid, unsigned id);
/* Filterlist management */

2
include/nc4internal.h
View File

@ -204,8 +204,6 @@ typedef struct NC_VAR_INFO
int storage; /**< Storage of this var, compact, contiguous, or chunked. */
int endianness; /**< What endianness for the var? */
int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */
nc_bool_t shuffle; /**< True if var has shuffle filter applied. */
nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */
struct ChunkCache {
size_t size; /**< Size in bytes of the var chunk cache. */
size_t nelems; /**< Number of slots in var chunk cache. */

9
include/ncconfigure.h
View File

@ -152,13 +152,4 @@ typedef long long fileoffset_t;
#define NC_UNUSED(var) (void)var
#endif
/* Protect old HDF5 code (pre 1.8.12) */
#ifndef HAVE_H5ALLOCATE_MEMORY
#ifndef H5allocate_memory
#define H5allocate_memory(size,clear) ((clear)?calloc(1,(size)):malloc(size))
#define H5free_memory(buf) free(buf)
#define H5resize_memory(buf,size) realloc(buf,size)
#endif
#endif
#endif /* NCCONFIGURE_H */

20
include/ncfilter.h
View File

@ -1,20 +0,0 @@
/* Copyright 2018-2018 University Corporation for Atmospheric
Research/Unidata. */
/**
* @file
* @internal This header file contains macros, types and prototypes
* used to build and manipulate filters. It contains definitions
* for multiple file formats.
*
* @author Dennis Heimbigner
*/
#ifndef NCFILTER_H
#define NCFILTER_H
/**************************************************/
/* Internal filter related structures */
/* Obsolete */
#endif /*NCFILTER_H*/

2
include/nchttp.h
View File

@ -34,7 +34,7 @@ typedef struct NC_HTTP_STATE {
extern int nc_http_init(NC_HTTP_STATE** state);
extern int nc_http_init_verbose(NC_HTTP_STATE** state, int verbose);
extern int nc_http_size(NC_HTTP_STATE* state, const char* url, long long unsigned* sizep);
extern int nc_http_size(NC_HTTP_STATE* state, const char* url, long long* sizep);
extern int nc_http_read(NC_HTTP_STATE* state, const char* url, size64_t start, size64_t count, NCbytes* buf);
extern int nc_http_write(NC_HTTP_STATE* state, const char* url, NCbytes* payload);
extern int nc_http_close(NC_HTTP_STATE* state);

3
include/netcdf_dispatch.h.in
View File

@ -150,6 +150,8 @@ struct NC_Dispatch
/* Version 4 Add quantization. */
int (*def_var_quantize)(int ncid, int varid, int quantize_mode, int nsd);
int (*inq_var_quantize)(int ncid, int varid, int *quantize_modep, int *nsdp);
/* Version 5 adds filter availability */
int (*inq_filter_avail)(int ncid, unsigned id);
};
#if defined(__cplusplus)
@ -189,6 +191,7 @@ extern "C" {
EXTERNL int NC_NOTNC4_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params);
EXTERNL int NC_NOOP_inq_var_filter_ids(int ncid, int varid, size_t* nfilters, unsigned int* filterids);
EXTERNL int NC_NOOP_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params);
EXTERNL int NC_NOOP_inq_filter_avail(int ncid, unsigned id);
EXTERNL int NC_NOTNC4_def_grp(int, const char *, int *);
EXTERNL int NC_NOTNC4_rename_grp(int, const char *);

49
include/netcdf_filter.h
View File

@ -17,14 +17,20 @@
/* Must match values in <H5Zpublic.h> */
#ifndef H5Z_FILTER_DEFLATE
#define H5Z_FILTER_DEFLATE 1
#define H5Z_FILTER_SHUFFLE 2
#define H5Z_FILTER_FLETCHER32 3
#endif
#ifndef H5Z_FILTER_SZIP
#define H5Z_FILTER_SZIP 4
#define H5_SZIP_ALLOW_K13_OPTION_MASK 1
#define H5_SZIP_CHIP_OPTION_MASK 2
#define H5_SZIP_EC_OPTION_MASK 4
#define H5_SZIP_LSB_OPTION_MASK 8
#define H5_SZIP_MSB_OPTION_MASK 16
#define H5_SZIP_NN_OPTION_MASK 32
#define H5_SZIP_MAX_PIXELS_PER_BLOCK 32
#define H5_SZIP_RAW_OPTION_MASK 128
#define NC_SZIP_EC 4 /**< Selects entropy coding method for szip. */
#define NC_SZIP_NN 32 /**< Selects nearest neighbor coding method for szip. */
@ -35,6 +41,29 @@
/** The maximum allowed setting for pixels_per_block when calling nc_def_var_szip(). */
#define NC_MAX_PIXELS_PER_BLOCK 32
/* Other Standard Filters */
#ifndef H5Z_FILTER_ZSTD
#define H5Z_FILTER_ZSTD 32015
#endif
#ifndef H5Z_FILTER_BZIP2
#define H5Z_FILTER_BZIP2 307
#endif
#ifndef H5Z_FILTER_BLOSC
#define H5Z_FILTER_BLOSC 32001
#endif
#ifndef BLOSC_SHUFFLE
enum BLOSC_SHUFFLE {
BLOSC_NOSHUFFLE=0, /* no shuffle */
BLOSC_SHUFFLE=1, /* byte-wise shuffle */
BLOSC_BITSHUFFLE=2 /* bit-wise shuffle */
};
#endif
/*Assumes use of c-blosc library */
#ifndef BLOSC_BLOSCLZ
enum BLOSC_SUBCOMPRESSORS {BLOSC_LZ=0, BLOSC_LZ4=1, BLOSC_LZ4HC=2, BLOSC_SNAPPY=3, BLOSC_ZLIB=4, BLOSC_ZSTD=5};
#endif
#if defined(__cplusplus)
extern "C" {
#endif
@ -60,6 +89,26 @@ EXTERNL int nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t*
/* End HDF5 Format Declarations */
/**************************************************/
/* Allow user to interrogate if a filter is available*/
/* Would prefer to get list of all known filters, but HDF5 does not support that */
/* See if filter is available */
EXTERNL int nc_inq_filter_avail(int ncid, unsigned id);
/**************************************************/
/* Functions for accessing standardized filters */
EXTERNL int nc_def_var_bzip2(int ncid, int varid, int level);
EXTERNL int nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp);
/* Note use of 'zstandard' instead of 'zstd' */
EXTERNL int nc_def_var_zstandard(int ncid, int varid, int level);
EXTERNL int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp);
EXTERNL int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle);
EXTERNL int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep);
#if defined(__cplusplus)
}
#endif

13
include/netcdf_filter_hdf5_build.h
View File

@ -204,4 +204,17 @@ typedef const void* (*H5PL_get_plugin_info_proto)(void);
}
#endif
/* Protect old HDF5 code (pre 1.8.12) */
#ifdef USE_HDF5
# if H5_VERSION_LE(1,8,11)
# define H5allocate_memory(size,clear) ((clear)?calloc(1,(size)):malloc(size))
# define H5free_memory(buf) free(buf)
# define H5resize_memory(buf,size) realloc(buf,size)
# endif
#else
# define H5allocate_memory(size,clear) ((clear)?calloc(1,(size)):malloc(size))
# define H5free_memory(buf) free(buf)
# define H5resize_memory(buf,size) realloc(buf,size)
#endif
#endif /*NETCDF_FILTER_HDF5_BUILD_H*/

4
lib_flags.am
View File

@ -13,6 +13,10 @@ if USE_DAP
AM_CPPFLAGS += -I${top_srcdir}/oc2
endif
if ENABLE_NCZARR
AM_CPPFLAGS += -I${top_srcdir}/libnczarr
endif
if ENABLE_S3_SDK
AM_LDFLAGS += -lstdc++
endif

2
libdap2/ncd2dispatch.c
View File

@ -7,6 +7,7 @@
#include "ncd2dispatch.h"
#include "ncrc.h"
#include "ncoffsets.h"
#include "netcdf_dispatch.h"
#ifdef DEBUG2
#include "dapdump.h"
#endif
@ -181,6 +182,7 @@ NC_NOOP_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
NC_NOOP_inq_filter_avail,
};
const NC_Dispatch* NCD2_dispatch_table = NULL; /* moved here from ddispatch.c */

14
libdap4/ncd4dispatch.c
View File

@ -799,6 +799,18 @@ NCD4_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams,
return (ret);
}
static int
NCD4_inq_filter_avail(int ncid, unsigned id)
{
NC* ncp;
int ret;
int substrateid;
if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) return (ret);
substrateid = makenc4id(ncp,ncid);
ret = nc_inq_filter_avail(substrateid, id);
return (ret);
}
/**************************************************/
/*
Following functions are overridden to handle
@ -1011,4 +1023,6 @@ NCD4_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NCD4_inq_var_quantize,
NCD4_inq_filter_avail,
};

6
libdispatch/CMakeLists.txt
View File

@ -21,7 +21,7 @@ ENDIF(ENABLE_BYTERANGE)
IF(ENABLE_S3_SDK)
SET(libdispatch_SOURCES ${libdispatch_SOURCES} ncs3sdk.cpp awsincludes.h)
ENDIF(ENABLE_S3_SDK)
ENDIF()
IF(REGEDIT)
SET(libdispatch_SOURCES ${libdispatch_SOURCES} dreg.c)
@ -36,6 +36,10 @@ IF(MPI_C_LIBRARIES)
target_link_libraries(dispatch PUBLIC ${MPI_C_LIBRARIES})
ENDIF(MPI_C_LIBRARIES)
IF(ENABLE_NCZARR)
target_include_directories(dispatch PUBLIC ../libnczarr)
ENDIF(ENABLE_NCZARR)
IF(ENABLE_S3_SDK)
target_include_directories(dispatch PUBLIC ${AWSSDK_INCLUDE_DIRS})
IF(NOT MSVC)

2
libdispatch/derror.c
View File

@ -261,7 +261,7 @@ const char *nc_strerror(int ncerr1)
case NC_EFILTER:
return "NetCDF: Filter error: bad id or parameters or duplicate filter";
case NC_ENOFILTER:
return "NetCDF: Filter error: unimplemented filter encountered";
return "NetCDF: Filter error: undefined filter encountered";
case NC_ECANTEXTEND:
return "NetCDF: Attempt to extend dataset during NC_INDEPENDENT I/O operation. Use nc_var_par_access to set mode NC_COLLECTIVE before extending variable.";
case NC_EMPI: return "NetCDF: MPI operation failed.";

180
libdispatch/dfilter.c
View File

@ -20,6 +20,10 @@
#include "hdf5internal.h"
#endif
#ifdef ENABLE_NCZARR
#include "zdispatch.h"
#endif
/*
Unified filter related code
*/
@ -189,6 +193,30 @@ nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsi
return stat;
}
/**************************************************/
/* Test if filter is available. Would prefer
returning a list of all available filters, but HDF5
does not support that capability.
@param file for which filter list is desired
@param id filter id of interest
@return NC_NOERR if the filter is available
@return NC_EBADID if ncid is invalid
@return NC_ENOFILTER if filter is not available.
*/
EXTERNL int
nc_inq_filter_avail(int ncid, unsigned id)
{
int stat = NC_NOERR;
NC* ncp;
stat = NC_check_id(ncid,&ncp);
if(stat != NC_NOERR) return stat;
if((stat = ncp->dispatch->inq_filter_avail(ncid,id))) goto done;
done:
return stat;
}
/**************************************************/
/* Support direct user defined filters */
@ -255,3 +283,155 @@ int stat = NC_NOERR;
}
#endif /*ENABLE_CLIENTSIDE_FILTERS*/
/**************************************************/
/* Functions for accessing standardized filters */
int
nc_def_var_bzip2(int ncid, int varid, int level)
{
int stat = NC_NOERR;
unsigned ulevel;
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2))) goto done;
/* Filter is available */
/* 1 <= Level <= 9 */
if (level < 1 || level > 9)
return NC_EINVAL;
ulevel = (unsigned) level; /* Keep bit pattern */
if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_BZIP2,1,&ulevel))) goto done;
done:
return stat;
}
int
nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp)
{
int stat = NC_NOERR;
size_t nparams;
unsigned params = 0;
int hasfilter = 0;
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2))) goto done;
/* Filter is available */
/* Get filter info */
stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BZIP2,&nparams,NULL);
if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;}
if(stat != NC_NOERR) goto done;
hasfilter = 1;
if(nparams != 1) {stat = NC_EFILTER; goto done;}
if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BZIP2,&nparams,&params))) goto done;
done:
if(levelp) *levelp = (int)params;
if(hasfilterp) *hasfilterp = hasfilter;
return stat;
}
int
nc_def_var_zstandard(int ncid, int varid, int level)
{
#ifdef HAVE_ZSTD
int stat = NC_NOERR;
unsigned ulevel;
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_ZSTD))) goto done;
/* Filter is available */
/* Level must be between -131072 and 22 on Zstandard v. 1.4.5 (~202009)
Earlier versions have fewer levels (especially fewer negative levels) */
if (level < -131072 || level > 22)
return NC_EINVAL;
ulevel = (unsigned) level; /* Keep bit pattern */
if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_ZSTD,1,&ulevel))) goto done;
done:
return stat;
#else
return NC_NOERR;
#endif /*HAVE_ZSTD*/
}
int
nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp)
{
#ifdef HAVE_ZSTD
int stat = NC_NOERR;
size_t nparams;
unsigned params = 0;
int hasfilter = 0;
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_ZSTD))) goto done;
/* Filter is available */
/* Get filter info */
stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_ZSTD,&nparams,NULL);
if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;}
if(stat != NC_NOERR) goto done;
hasfilter = 1;
if(nparams != 1) {stat = NC_EFILTER; goto done;}
if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_ZSTD,&nparams,&params))) goto done;
done:
if(levelp) *levelp = (int)params;
if(hasfilterp) *hasfilterp = hasfilter;
return stat;
#else
return NC_NOERR;
#endif /*HAVE_ZSTD*/
}
int
nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle)
{
#ifdef HAVE_BLOSC
int stat = NC_NOERR;
unsigned params[7];;
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BLOSC))) goto done;
/* Filter is available */
/* Verify parameters */
if(addshuffle > (unsigned)BLOSC_BITSHUFFLE) {stat = NC_EINVAL; goto done;}
if(subcompressor > (unsigned)BLOSC_ZSTD) {stat = NC_EINVAL; goto done;}
/* Set the parameters */
params[0] = 0;
params[1] = 0;
params[2] = 0;
params[3] = blocksize;
params[4] = level;
params[5] = addshuffle;
params[6] = subcompressor;
if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_BLOSC,7,params))) goto done;
done:
return stat;
#else
return NC_NOERR;
#endif
}
int
nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep)
{
#ifdef HAVE_BLOSC
int stat = NC_NOERR;
size_t nparams;
unsigned params[7];
int hasfilter = 0;
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BLOSC))) goto done;
/* Filter is available */
/* Get filter info */
stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BLOSC,&nparams,NULL);
if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;}
if(stat != NC_NOERR) goto done;
hasfilter = 1;
if(nparams != 7) {stat = NC_EFILTER; goto done;}
if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BLOSC,&nparams,params))) goto done;
if(blocksizep) *blocksizep = params[3];
if(levelp) *levelp = params[4];
if(addshufflep) *addshufflep = params[5];
if(subcompressorp) *subcompressorp = params[6];
done:
if(hasfilterp) *hasfilterp = hasfilter;
return stat;
#else
return NC_NOERR;
#endif
}

2
libdispatch/dhttp.c
View File

@ -276,7 +276,7 @@ Assume URL etc has already been set.
*/
int
nc_http_size(NC_HTTP_STATE* state, const char* objecturl, long long unsigned* sizep)
nc_http_size(NC_HTTP_STATE* state, const char* objecturl, long long* sizep)
{
int stat = NC_NOERR;
const char* hdr = NULL;

2
libdispatch/dinfermodel.c
View File

@ -52,7 +52,7 @@ struct MagicFile {
struct NCURI* uri;
int omode;
NCmodel* model;
long long unsigned filelen;
long long filelen;
int use_parallel;
int iss3;
void* parameters; /* !NULL if inmemory && !diskless */

7
libdispatch/dnotnc4.c
View File

@ -97,6 +97,13 @@ NC_NOOP_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparam
return NC_ENOFILTER;
}
int
NC_NOOP_inq_filter_avail(int ncid, unsigned id)
{
NC_UNUSED(ncid);
return NC_ENOFILTER;
}
/**
* @internal Not allowed for classic model.
*

6
libhdf4/hdf4dispatch.c
View File

@ -13,6 +13,7 @@
#include "hdf4dispatch.h"
#include "nc4dispatch.h"
#include "hdf5dispatch.h"
#include "netcdf_filter.h"
/* This is the dispatch object that holds pointers to all the
* functions that make up the HDF4 dispatch interface. */
@ -106,6 +107,11 @@ static const NC_Dispatch HDF4_dispatcher = {
NC_NOOP_inq_var_filter_ids,
NC_NOOP_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
NC_NOOP_inq_filter_avail,
};
const NC_Dispatch *HDF4_dispatch_table = NULL;

1
libhdf5/H5FDhttp.c
View File

@ -49,6 +49,7 @@
/*
Define a simple #ifdef test for the version of H5FD_class_t we are using
*/
#if H5_VERS_MAJOR == 1
#if H5_VERS_MINOR < 10
#define H5FDCLASS1 1

1
libhdf5/hdf5dispatch.c
View File

@ -109,6 +109,7 @@ static const NC_Dispatch HDF5_dispatcher = {
NC4_def_var_quantize,
NC4_inq_var_quantize,
NC4_hdf5_inq_filter_avail,
};
const NC_Dispatch* HDF5_dispatch_table = NULL; /* moved here from ddispatch.c */

57
libhdf5/hdf5filter.c
View File

@ -287,7 +287,18 @@ NC4_hdf5_addfilter(NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const un
}
fi->flags = flags;
if(!olddef) {
nclistpush(flist,fi);
size_t pos = nclistlength(flist);
/* Need to be careful about where we insert fletcher32 and shuffle */
if(nclistlength(flist) > 0) {
if(id == H5Z_FILTER_FLETCHER32)
pos = 0; /* alway first filter */
else if(id == H5Z_FILTER_SHUFFLE) {
/* See if first filter is fletcher32 */
struct NC_HDF5_Filter* f0 = (struct NC_HDF5_Filter*)nclistget(flist,0);
if(f0->filterid == H5Z_FILTER_FLETCHER32) pos = 1; else pos = 0;
}
}
nclistinsert(flist,pos,fi);
PRINTFILTERLIST(var,"add");
}
fi = NULL; /* either way,its in the var->filters list */
@ -354,10 +365,8 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
struct NC_HDF5_Filter* oldspec = NULL;
int flags = 0;
htri_t avail = -1;
#ifdef HAVE_H5Z_SZIP
int havedeflate = 0;
int haveszip = 0;
#endif
LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid));
@ -386,13 +395,20 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
#endif /* HDF5_SUPPORTS_PAR_FILTERS */
#endif /* USE_PARALLEL */
/* Lookup incoming id to see if already defined */
/* See if this filter is missing or not */
if((avail = H5Zfilter_avail(id)) < 0)
{stat = NC_EHDFERR; goto done;} /* Something in HDF5 went wrong */
if(avail == 0)
{stat = NC_ENOFILTER; goto done;} /* filter not available */
/* Lookup incoming id to see if already defined for this variable*/
switch((stat=NC4_hdf5_filter_lookup(var,id,&oldspec))) {
case NC_NOERR: break; /* already defined */
case NC_ENOFILTER: break; /*not defined*/
default: goto done;
}
#ifdef HAVE_H5Z_SZIP
stat = NC_NOERR; /* reset */
/* See if deflate &/or szip is defined */
switch ((stat = NC4_hdf5_filter_lookup(var,H5Z_FILTER_DEFLATE,NULL))) {
case NC_NOERR: havedeflate = 1; break;
@ -404,11 +420,8 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
case NC_ENOFILTER: haveszip = 0; break;
default: goto done;
}
#endif /* HAVE_H5Z_SZIP */
stat = NC_NOERR; /* reset */
/* See if this filter is missing or not */
if((avail = H5Zfilter_avail(id)) < 0)
{stat = NC_EHDFERR; goto done;} /* Something in HDF5 went wrong */
if(!avail) {
NC_HDF5_VAR_INFO_T* hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
flags |= NC_HDF5_FILTER_MISSING;
@ -425,25 +438,18 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
level = (int)params[0];
if (level < NC_MIN_DEFLATE_LEVEL || level > NC_MAX_DEFLATE_LEVEL)
{stat = THROW(NC_EINVAL); goto done;}
#ifdef HAVE_H5Z_SZIP
/* If szip compression is already applied, return error. */
if(haveszip) {stat = THROW(NC_EINVAL); goto done;}
#endif
}
#ifdef HAVE_H5Z_SZIP
if(id == H5Z_FILTER_SZIP) { /* Do error checking */
if(nparams != 2)
{stat = THROW(NC_EFILTER); goto done;}/* incorrect no. of parameters */
/* Pixels per block must be an even number, < 32. */
/* Pixels per block must be an even number, <= 32. */
if (params[1] % 2 || params[1] > NC_MAX_PIXELS_PER_BLOCK)
{stat = THROW(NC_EINVAL); goto done;}
/* If zlib compression is already applied, return error. */
if(havedeflate) {stat = THROW(NC_EINVAL); goto done;}
}
#else /*!HAVE_H5Z_SZIP*/
if(id == H5Z_FILTER_SZIP)
{stat = THROW(NC_EFILTER); goto done;} /* Not allowed */
#endif
/* Filter => chunking */
var->storage = NC_CHUNKED;
/* Determine default chunksizes for this variable unless already specified */
@ -456,7 +462,6 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
goto done;
}
}
#ifdef HAVE_H5Z_SZIP
/* More error checking */
if(id == H5Z_FILTER_SZIP) { /* szip X chunking error checking */
/* For szip, the pixels_per_block parameter must not be greater
@ -470,7 +475,6 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
if (params[1] > num_elem)
{stat = THROW(NC_EINVAL); goto done;}
}
#endif
/* addfilter can handle case where filter is already defined, and will just replace parameters */
if((stat = NC4_hdf5_addfilter(var,id,nparams,params,flags)))
goto done;
@ -588,3 +592,18 @@ NC4_hdf5_filter_finalize(void)
{
return NC_NOERR;
}
/* Test if filter available */
int
NC4_hdf5_inq_filter_avail(int ncid, unsigned id)
{
int stat = NC_NOERR;
htri_t avail = -1;
NC_UNUSED(ncid);
/* See if this filter is available or not */
if((avail = H5Zfilter_avail(id)) < 0)
{stat = NC_EHDFERR; goto done;} /* Something in HDF5 went wrong */
if(avail == 0) stat = NC_ENOFILTER;
done:
return stat;
}

15
libhdf5/hdf5open.c
View File

@ -16,7 +16,6 @@
#include "ncrc.h"
#include "ncauth.h"
#include "ncmodel.h"
#include "ncfilter.h"
#include "ncpathmgr.h"
#ifdef ENABLE_BYTERANGE
@ -1087,14 +1086,6 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var)
{stat = NC_EHDFERR; goto done;} /* Something in HDF5 went wrong */
switch (filter)
{
case H5Z_FILTER_SHUFFLE:
var->shuffle = NC_TRUE;
break;
case H5Z_FILTER_FLETCHER32:
var->fletcher32 = NC_TRUE;
break;
case H5Z_FILTER_DEFLATE:
if (cd_nelems != CD_NELEMS_ZLIB ||
cd_values[0] > NC_MAX_DEFLATE_LEVEL)
@ -2015,13 +2006,7 @@ hdf5free(void* memory)
#ifndef JNA
/* On Windows using the microsoft runtime, it is an error
for one library to free memory allocated by a different library.*/
#ifdef HAVE_H5FREE_MEMORY
if(memory != NULL) H5free_memory(memory);
#else
#ifndef _MSC_VER
if(memory != NULL) free(memory);
#endif
#endif
#endif
}

29
libhdf5/hdf5var.c
View File

@ -533,31 +533,30 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
}
/* Shuffle filter? */
if (shuffle)
{
if(*shuffle) var->shuffle = *shuffle; /* Once set, cannot be unset */
if(var->shuffle)
var->storage = NC_CHUNKED;
if (shuffle && *shuffle) {
retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_SHUFFLE,NULL,NULL);
if(!retval || retval == NC_ENOFILTER) {
if((retval = nc_def_var_filter(ncid,varid,H5Z_FILTER_SHUFFLE,0,NULL))) return retval;
var->storage = NC_CHUNKED;
}
}
/* Fletcher32 checksum error protection? */
if (fletcher32)
{
if(*fletcher32) var->fletcher32 = *fletcher32; /* cannot be unset */
if(var->fletcher32)
if (fletcher32 && fletcher32) {
retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_FLETCHER32,NULL,NULL);
if(!retval || retval == NC_ENOFILTER) {
if((retval = nc_def_var_filter(ncid,varid,H5Z_FILTER_FLETCHER32,0,NULL))) return retval;
var->storage = NC_CHUNKED;
}
}
#ifdef USE_PARALLEL
/* If deflate, shuffle, or fletcher32 was turned on with
/* If filter is being applied with
* parallel I/O writes, then switch to collective access. HDF5
* requires collevtive access for filter use with parallel
* I/O. */
if (shuffle || fletcher32)
{
if (h5->parallel && (nclistlength((NClist*)var->filters) > 0 || var->shuffle || var->fletcher32))
if (h5->parallel && (nclistlength((NClist*)var->filters) > 0))
var->parallel_access = NC_COLLECTIVE;
}
#endif /* USE_PARALLEL */
/* Handle storage settings. */
@ -568,7 +567,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
* no filters in use for this data. */
if (*storage != NC_CHUNKED)
{
if (nclistlength(((NClist*)var->filters)) > 0 || var->fletcher32 || var->shuffle)
if (nclistlength(((NClist*)var->filters)) > 0)
return NC_EINVAL;
for (d = 0; d < var->ndims; d++)
if (var->dim[d]->unlimited)

72
libhdf5/nc4hdf.c
View File

@ -824,18 +824,6 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
}
}
/* If the user wants to fletcher error correction, set that up now. */
/* Since it is a checksum of sorts, flatcher is always applied first */
if (var->fletcher32)
if (H5Pset_fletcher32(plistid) < 0)
BAIL(NC_EHDFERR);
/* If the user wants to shuffle the data, set that up now. */
if (var->shuffle) {
if (H5Pset_shuffle(plistid) < 0)
BAIL(NC_EHDFERR);
}
/* If the user wants to compress the data, using either zlib
* (a.k.a deflate) or szip, or another filter, set that up now.
* Szip and zip can be turned on
@ -847,35 +835,39 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
NClist* filters = (NClist*)var->filters;
for(j=0;j<nclistlength(filters);j++) {
struct NC_HDF5_Filter* fi = (struct NC_HDF5_Filter*)nclistget(filters,j);
{
if(fi->filterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */
unsigned level;
if(fi->nparams != 1)
BAIL(NC_EFILTER);
level = (int)fi->params[0];
if(H5Pset_deflate(plistid, level) < 0)
BAIL(NC_EFILTER);
} else if(fi->filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */
int options_mask;
int bits_per_pixel;
if(fi->nparams != 2)
BAIL(NC_EFILTER);
options_mask = (int)fi->params[0];
bits_per_pixel = (int)fi->params[1];
if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
BAIL(NC_EFILTER);
} else {
herr_t code = H5Pset_filter(plistid, fi->filterid,
#if 0
H5Z_FLAG_MANDATORY,
if(fi->filterid == H5Z_FILTER_FLETCHER32) {
if(H5Pset_fletcher32(plistid) < 0)
BAIL(NC_EHDFERR);
} else if(fi->filterid == H5Z_FILTER_SHUFFLE) {
if(H5Pset_shuffle(plistid) < 0)
BAIL(NC_EHDFERR);
} else if(fi->filterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */
unsigned level;
if(fi->nparams != 1)
BAIL(NC_EFILTER);
level = (int)fi->params[0];
if(H5Pset_deflate(plistid, level) < 0)
BAIL(NC_EFILTER);
} else if(fi->filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */
int options_mask;
int bits_per_pixel;
if(fi->nparams != 2)
BAIL(NC_EFILTER);
options_mask = (int)fi->params[0];
bits_per_pixel = (int)fi->params[1];
if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
BAIL(NC_EFILTER);
} else {
herr_t code = H5Pset_filter(plistid, fi->filterid,
#if 1
H5Z_FLAG_MANDATORY,
#else
H5Z_FLAG_OPTIONAL,
H5Z_FLAG_OPTIONAL,
#endif
fi->nparams, fi->params);
if(code < 0)
BAIL(NC_EFILTER);
}
}
fi->nparams, fi->params);
if(code < 0)
BAIL(NC_EFILTER);
}
}
}
@ -898,7 +890,7 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
/* If there are no unlimited dims, and no filters, and the user
* has not specified chunksizes, use contiguous variable for
* better performance. */
if (!var->shuffle && !var->fletcher32 && nclistlength((NClist*)var->filters) == 0 &&
if (nclistlength((NClist*)var->filters) == 0 &&
(var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim)
var->storage = NC_CONTIGUOUS;

15
liblib/CMakeLists.txt
View File

@ -77,9 +77,22 @@ SET(TLL_LIBS ${TLL_LIBS} ${HAVE_LIBM} ${ZLIB_LIBRARY})
# Add extra dependencies specified via NC_EXTRA_DEPS
SET(TLL_LIBS ${TLL_LIBS} ${EXTRA_DEPS})
IF(ENABLE_BLOSC)
IF(ENABLE_NCZARR_ZIP)
SET(TLL_LIBS ${TLL_LIBS} ${Zip_LIBRARIES})
ENDIF()
IF(Blosc_FOUND)
SET(TLL_LIBS ${TLL_LIBS} ${Blosc_LIBRARIES})
ENDIF()
IF(Zstd_FOUND)
SET(TLL_LIBS ${TLL_LIBS} ${Zstd_LIBRARIES})
ENDIF()
IF(Bz2_FOUND)
SET(TLL_LIBS ${TLL_LIBS} ${Bz2_LIBRARIES})
ENDIF()
IF(SZIP_FOUND)
SET(TLL_LIBS ${TLL_LIBS} ${SZIP_LIBRARIES})
ENDIF()
IF(HAVE_LIBDL)
SET(TLL_LIBS ${LIBDL} ${TLL_LIBS})

35
libncpoco/cp_unix.c
View File

@ -19,6 +19,8 @@
#include "ncpoco.h"
#include "ncpathmgr.h"
#undef DEBUG
/* Note: cygwin is missing RTLD_LOCAL, set it to 0 */
#if !defined(RTLD_LOCAL)
#define RTLD_LOCAL 0
@ -29,13 +31,32 @@
#endif
#if !defined(RTLD_LAZY)
#define RTLD_LAZY 0
#define RTLD_LAZY 1
#endif
#if !defined(RTLD_NOW)
#define RTLD_NOW 2
#endif
#ifdef USE_MUTEX
static pthread_mutex_t mutex;
#endif
static void
ncperr(const char* fcn, NCPSharedLib* lib)
{
const char* msg = dlerror();
lib->err.msg[0] = '\0';
if(msg != NULL) {
strlcat(lib->err.msg,fcn,sizeof(lib->err.msg));
strlcat(lib->err.msg,": ",sizeof(lib->err.msg));
strlcat(lib->err.msg,msg,sizeof(lib->err.msg));
#ifdef DEBUG
fprintf(stderr,">>> %s\n",lib->err.msg);
#endif
}
}
int
ncp_unix_initialize(void)
{
@ -96,7 +117,7 @@ static int
load(NCPSharedLib* lib , const char* path0, int flags)
{
int ret = NC_NOERR;
int realflags = RTLD_LAZY;
int realflags = RTLD_LAZY; /* versus RTLD_NOW which does not appear to work */
char* path = NULL;
if((path = NCpathcvt(path0))==NULL) {ret = NC_ENOMEM; goto done;}
@ -113,9 +134,7 @@ load(NCPSharedLib* lib , const char* path0, int flags)
lib->state.flags = realflags;
lib->state.handle = dlopen(lib->path, lib->state.flags);
if(lib->state.handle == NULL) {
const char* msg = dlerror();
if(msg == NULL) msg = "";
strncpy(lib->err.msg,msg,sizeof(lib->err.msg));
ncperr("dlopen",lib);
ret = NC_ENOTFOUND;
goto ldone;
}
@ -150,8 +169,12 @@ getsymbol(NCPSharedLib* lib, const char* name)
{
void* result = NULL;
lock();
if(lib->state.handle != NULL)
if(lib->state.handle != NULL) {
result = dlsym(lib->state.handle, name);
if(result == NULL) {
ncperr("dlsym",lib);
}
}
unlock();
return result;
}

1
libnczarr/Makefile.am
View File

@ -7,6 +7,7 @@
# Get AM_CPPFLAGS and AM_LDFLAGS
include $(top_srcdir)/lib_flags.am
AM_CPPFLAGS += -D_LARGEFILE_SOURCE
AM_CPPFLAGS += -I$(top_srcdir)/libsrc4 -I$(top_srcdir)/libncpoco
libnczarr_la_CPPFLAGS = ${AM_CPPFLAGS}

2
libnczarr/zcache.h
View File

@ -53,7 +53,7 @@ typedef struct NCZChunkCache {
/**************************************************/
#define FILTERED(cache) (nclistlength((NClist*)(cache)->var->filters) || (cache)->var->shuffle || (cache)->var->fletcher32);
#define FILTERED(cache) (nclistlength((NClist*)(cache)->var->filters))
extern int NCZ_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption);
extern int NCZ_adjust_var_cache(NC_VAR_INFO_T *var);

2
libnczarr/zdebug.h
View File

@ -9,7 +9,7 @@
#undef ZDEBUG1 /* detailed debug */
#undef ZCATCH /* Warning: significant performance impact */
#define ZTRACING /* Warning: significant performance impact */
#undef ZTRACING /* Warning: significant performance impact */
#include "ncexternl.h"
#include "nclog.h"

13
libnczarr/zdispatch.c
View File

@ -106,6 +106,7 @@ static const NC_Dispatch NCZ_dispatcher = {
NCZ_inq_var_filter_info,
NCZ_def_var_quantize,
NCZ_inq_var_quantize,
NCZ_inq_filter_avail,
};
const NC_Dispatch* NCZ_dispatch_table = NULL; /* moved here from ddispatch.c */
@ -157,7 +158,7 @@ NCZ_show_metadata(int ncid)
}
#ifndef ENABLE_NCZARR_FILTERS
int
int
NCZ_def_var_filter(int ncid, int varid, unsigned int id , size_t n , const unsigned int *params)
{
NC_UNUSED(ncid);
@ -188,6 +189,16 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, u
NC_UNUSED(params);
return REPORT(NC_ENOFILTER,"inq_var_filter_info");
}
int
NCZ_inq_filter_avail(int ncid, unsigned id)
{
NC_UNUSED(ncid);
NC_UNUSED(id);
return REPORT(NC_ENOFILTER,"inq_filter_avail");
}
#endif /*ENABLE_NCZARR_FILTERS*/
/**************************************************/

1
libnczarr/zdispatch.h
View File

@ -172,6 +172,7 @@ EXTERNL int NCZ_inq_unlimdims(int, int *, int *);
EXTERNL int NCZ_def_var_filter(int ncid, int varid, unsigned int filterid, size_t nparams, const unsigned int *params);
EXTERNL int NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int *filterids);
EXTERNL int NCZ_inq_var_filter_info(int ncid, int varid, unsigned int filterid, size_t* nparamsp, unsigned int *params);
EXTERNL int NCZ_inq_filter_avail(int ncid, unsigned id);
EXTERNL int NCZ_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd);
EXTERNL int NCZ_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp);

280
libnczarr/zfilter.c
View File

@ -56,6 +56,9 @@
#undef DEBUGF
#undef DEBUGL
/* If set, then triage potential shared libraries based on extension */
#define NAMEOPT
#define NULLIFY(x) ((x)?(x):"NULL")
/* Hold the loaded filter plugin information */
@ -65,6 +68,7 @@ typedef struct NCZ_Plugin {
NCPSharedLib* hdf5lib; /* source of the filter */
} hdf5;
struct CodecAPI {
int defaulted; /* codeclib was a defaulting library */
const NCZ_codec_t* codec;
NCPSharedLib* codeclib; /* of the source codec; null if same as hdf5 */
} codec;
@ -134,13 +138,15 @@ typedef struct NCZ_Filter {
} NCZ_Filter;
/* WARNING: GLOBAL DATA */
/* TODO: move to common global state */
/* All possible HDF5 filter plugins */
/* Convert to linked list or hash table or equivalent since very sparse */
/* Consider onverting to linked list or hash table or equivalent since very sparse */
NCZ_Plugin* loaded_plugins[H5Z_FILTER_MAX];
int loaded_plugins_max = -1;
static NCZ_codec_t** codec_defaults; /* NULL terminated */
static NCPSharedLib* default_lib; /* source of the defaults */
static NClist* codec_defaults = NULL; /* NClist<struct CodecAPI*> */
static NClist* default_libs = NULL; /* NClist<NCPSharedLib>; sources of the defaults */
static int NCZ_filter_initialized = 0;
@ -166,12 +172,12 @@ NCJtrace(const NCjson* j)
#endif
#ifdef DEBUGF
#if defined(DEBUGF) || defined(DEBUGL)
const char*
printplugin(const NCZ_Plugin* plugin)
{
static char plbuf[4096];
char* plbuf = malloc(4096);
char plbuf2[2000];
char plbuf1[2000];
@ -181,23 +187,23 @@ printplugin(const NCZ_Plugin* plugin)
snprintf(plbuf1,sizeof(plbuf1),"hdf5={id=%u name=%s}",plugin->hdf5.filter->id,plugin->hdf5.filter->name);
if(plugin->codec.codec)
snprintf(plbuf2,sizeof(plbuf2),"codec={codecid=%s hdf5id=%u}",plugin->codec.codec->codecid,plugin->codec.codec->hdf5id);
snprintf(plbuf,sizeof(plbuf),"plugin={%s %s}",plbuf1,plbuf2);
snprintf(plbuf,4096,"plugin={%s %s}",plbuf1,plbuf2);
return plbuf;
}
static const char*
static char*
printparams(size_t nparams, const unsigned* params)
{
static char ppbuf[4096];
char* ppbuf = malloc(4096);
if(nparams == 0)
snprintf(ppbuf,sizeof(ppbuf),"{0,%p}",params);
snprintf(ppbuf,4096,"{0,%p}",params);
else
snprintf(ppbuf,sizeof(ppbuf),"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params));
snprintf(ppbuf,4096,"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params));
return ppbuf;
}
static const char*
static char*
printnczparams(const NCZ_Params p)
{
return printparams(p.nparams,p.params);
@ -255,6 +261,10 @@ static int ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter);
static int paramnczclone(NCZ_Params* dst, const NCZ_Params* src);
static int paramclone(size_t nparams, unsigned** dstp, const unsigned* src);
#ifdef NAMEOPT
static int pluginnamecheck(const char* name);
#endif
/**************************************************/
/**
* @file
@ -660,7 +670,7 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp,
spec->hdf5.working.nparams,spec->hdf5.working.params,
&spec->hdf5.visible.nparams,&spec->hdf5.visible.params);
#ifdef DEBUGF
fprintf(stderr,"DEBUGF: NCZ_visible_parameters: ncid=%d varid=%d working=%s visible=%s\n",ncid,varid,
fprintf(stderr,">>> DEBUGF: NCZ_visible_parameters: ncid=%d varid=%d working=%s visible=%s\n",ncid,varid,
printnczparams(spec->hdf5.visible),printnczparams(spec->hdf5.working));
#endif
if(stat) goto done;
@ -679,6 +689,32 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp,
done:
return ZUNTRACEX(stat,"nparams=%u",(unsigned)(nparamsp?*nparamsp:0));
}
/* Test if a specific filter is available.
@param file for which use of a filter is desired
@param id the filter id of interest
@return NC_NOERR if the filter is available
@return NC_ENOFILTER if the filter is not available
@return NC_EBADID if ncid is invalid
@return NC_EFILTER if ncid format does not support filters
*/
int
NCZ_inq_filter_avail(int ncid, unsigned id)
{
int stat = NC_NOERR;
struct NCZ_Plugin* plug = NULL;
NC_UNUSED(ncid);
ZTRACE(1,"ncid=%d id=%u",ncid,id);
if((stat = NCZ_filter_initialize())) goto done;
/* Check the available filters list */
if((stat = NCZ_plugin_loaded((int)id, &plug))) goto done;
if(plug == NULL)
stat = NC_ENOFILTER;
done:
return ZUNTRACE(stat);
}
#endif /*ENABLE_NCZARR_FILTERS*/
/**************************************************/
@ -689,14 +725,17 @@ NCZ_filter_initialize(void)
{
int stat = NC_NOERR;
ZTRACE(6,"");
if(NCZ_filter_initialized) goto done;
{
NCZ_filter_initialized = 1;
memset(loaded_plugins,0,sizeof(loaded_plugins));
default_libs = nclistnew();
codec_defaults = nclistnew();
NCZ_filter_initialized = 1;
memset(loaded_plugins,0,sizeof(loaded_plugins));
#ifdef ENABLE_NCZARR_FILTERS
if((stat = NCZ_load_all_plugins())) goto done;
if((stat = NCZ_load_all_plugins())) goto done;
#endif
}
done:
return ZUNTRACE(stat);
}
@ -710,15 +749,34 @@ NCZ_filter_finalize(void)
if(!NCZ_filter_initialized) goto done;
#ifdef ENABLE_NCZARR_FILTERS
/* Reclaim all loaded filters */
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: finalize reclaim:\n");
#endif
for(i=0;i<=loaded_plugins_max;i++) {
NCZ_unload_plugin(loaded_plugins[i]);
loaded_plugins[i] = NULL;
if(loaded_plugins[i]) {
NCZ_unload_plugin(loaded_plugins[i]);
loaded_plugins[i] = NULL;
}
}
/* Reclaim the codec defaults */
if(nclistlength(codec_defaults) > 0) {
for(i=0;i<nclistlength(codec_defaults);i++) {
struct CodecAPI* ca = (struct CodecAPI*)nclistget(codec_defaults,i);
nullfree(ca);
}
}
/* Reclaim the defaults library contents; Must occur as last act */
if(nclistlength(default_libs) > 0) {
for(i=0;i<nclistlength(default_libs);i++) {
NCPSharedLib* l = (NCPSharedLib*)nclistget(default_libs,i);
if(l != NULL) (void)ncpsharedlibfree(l);
}
}
/* Reclaim the defaults library; Must occur as last act */
if(default_lib != NULL) {(void)ncpsharedlibfree(default_lib); default_lib = NULL; codec_defaults = NULL;}
#else
memset(loaded_plugins,0,sizeof(loaded_plugins));
#endif
nclistfree(default_libs); default_libs = NULL;
nclistfree(codec_defaults); codec_defaults = NULL;
done:
NCZ_filter_initialized = 0;
return ZUNTRACE(stat);
@ -780,7 +838,7 @@ NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* cha
size_t next_used = 0;
#ifdef DEBUG
fprintf(stderr,"current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(unsigned)current_used,current_buf);
fprintf(stderr,">>> current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(unsigned)current_used,current_buf);
#endif
/* Apply in proper order */
if(encode) {
@ -793,7 +851,7 @@ fprintf(stderr,"current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(uns
next_used = 0;
next_used = ff->filter(0,f->hdf5.working.nparams,f->hdf5.working.params,current_used,&next_alloc,&next_buf);
#ifdef DEBUG
fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf);
fprintf(stderr,">>> next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf);
#endif
if(next_used == 0) {stat = NC_EFILTER; lastbuffer = next_buf; goto done; }
/* If the filter did not need to create a new buffer, then next == current else current was reclaimed */
@ -812,7 +870,7 @@ fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)
next_used = 0;
next_used = ff->filter(H5Z_FLAG_REVERSE,f->hdf5.working.nparams,f->hdf5.working.params,current_used,&next_alloc,&next_buf);
#ifdef DEBUG
fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf);
fprintf(stderr,">>> next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf);
#endif
if(next_used == 0) {stat = NC_EFILTER; lastbuffer = next_buf; goto done;}
/* If the filter did not need to create a new buffer, then next == current else current was reclaimed */
@ -822,7 +880,7 @@ fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)
}
}
#ifdef DEBUG
fprintf(stderr,"current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(unsigned)current_used,current_buf);
fprintf(stderr,">>> current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(unsigned)current_used,current_buf);
#endif
/* return results */
if(outlenp) {*outlenp = current_used;} /* or should it be current_alloc? */
@ -864,7 +922,7 @@ NCZ_filter_jsonize(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Fil
if(filter->plugin->codec.codec->NCZ_hdf5_to_codec) {
stat = filter->plugin->codec.codec->NCZ_hdf5_to_codec(filter->hdf5.visible.nparams,filter->hdf5.visible.params,&filter->codec.codec);
#ifdef DEBUGF
fprintf(stderr,"DEBUGF: NCZ_hdf5_to_codec: visible=%s codec=%s\n",printnczparams(filter->hdf5.visible),filter->codec.codec);
fprintf(stderr,">>> DEBUGF: NCZ_hdf5_to_codec: visible=%s codec=%s\n",printnczparams(filter->hdf5.visible),filter->codec.codec);
#endif
if(stat) goto done;
} else
@ -921,7 +979,7 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j
if(plugin->codec.codec->NCZ_codec_to_hdf5) {
stat = plugin->codec.codec->NCZ_codec_to_hdf5(codec.codec,&hdf5.visible.nparams,&hdf5.visible.params);
#ifdef DEBUGF
fprintf(stderr,"DEBUGF: NCZ_codec_to_hdf5: codec=%s, hdf5=%s\n",printcodec(codec),printhdf5(hdf5));
fprintf(stderr,">>> DEBUGF: NCZ_codec_to_hdf5: codec=%s, hdf5=%s\n",printcodec(codec),printhdf5(hdf5));
#endif
if(stat) goto done;
}
@ -1052,7 +1110,7 @@ done:
static int
NCZ_load_all_plugins(void)
{
int i,ret = NC_NOERR;
int i,j,ret = NC_NOERR;
const char* pluginroot = NULL;
struct stat buf;
NClist* dirs = nclistnew();
@ -1063,7 +1121,7 @@ NCZ_load_all_plugins(void)
ZTRACE(6,"");
#ifdef DEBUGL
fprintf(stderr,"DEBUGL: NCZ_load_all_plugins\n");
fprintf(stderr,">>> DEBUGL: NCZ_load_all_plugins\n");
#endif
/* Find the plugin directory root(s) */
@ -1107,20 +1165,35 @@ NCZ_load_all_plugins(void)
/* Try to load plugins from this directory */
if((ret = NCZ_load_plugin_dir(dir))) goto done;
}
if(codec_defaults != NULL) { /* Try to provide default for any HDF5 filters without matching Codec. */
NCZ_codec_t** dfalts = NULL;
/* Search the defaults */
for(dfalts = codec_defaults;*dfalts;dfalts++) {
NCZ_codec_t* codec = *dfalts;
int hdf5id = codec->hdf5id;
NCZ_Plugin* p = loaded_plugins[hdf5id]; /* get candidate */
if(p != NULL && p->hdf5.filter != NULL && p->codec.codec == NULL) {
#ifdef DEBUGL
fprintf(stderr,"DEBUGL: plugin defaulted: id=%u, codec=%s\n",hdf5id,codec->codecid);
{ int i;
fprintf(stderr,"codec_defaults:");
for(i=0;i<nclistlength(codec_defaults);i++) {
struct CodecAPI* codec = (struct CodecAPI*)nclistget(codec_defaults,i);
fprintf(stderr," %d",codec->codec->hdf5id);
}
fprintf(stderr,"\n");
}
#endif
p->codec.codec = codec; codec = NULL;
p->codec.codeclib = NULL;
if(nclistlength(codec_defaults)) { /* Try to provide default for any HDF5 filters without matching Codec. */
/* Search the defaults */
for(j=0;j<nclistlength(codec_defaults);j++) {
struct CodecAPI* dfalt = (struct CodecAPI*)nclistget(codec_defaults,j);
if(dfalt->codec != NULL) {
const NCZ_codec_t* codec = dfalt->codec;
int hdf5id = codec->hdf5id;
NCZ_Plugin* p = NULL;
if(hdf5id < 0 || hdf5id > loaded_plugins_max) {ret = NC_EFILTER; goto done;}
p = loaded_plugins[hdf5id]; /* get candidate */
if(p != NULL && p->hdf5.filter != NULL
&& p->codec.codec == NULL) {
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: plugin defaulted: id=%u, codec=%s src=%s\n",hdf5id,codec->codecid,dfalt->codeclib->path);
#endif
p->codec.codec = codec;
p->codec.codeclib = dfalt->codeclib;
p->codec.defaulted = 1;
}
}
}
}
@ -1133,12 +1206,17 @@ NCZ_load_all_plugins(void)
if((p = loaded_plugins[i]) != NULL) {
if(p->hdf5.filter == NULL || p->codec.codec == NULL) {
#ifdef DEBUGL
fprintf(stderr,"DEBUGL: plugin expunged: id=%u\n",p->hdf5.filter->id);
fprintf(stderr,">>> DEBUGL: Incomplete plugin: expunged: id=%u; reasons: %s %s\n",i,
(p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":""));
#endif
/* expunge this entry */
(void)NCZ_unload_plugin(p);
loaded_plugins[i] = NULL;
}
#ifdef DEBUGL
else
fprintf(stderr,">>> DEBUGL: plugin accepted: id=%u\n",i);
#endif
}
}
}
@ -1152,7 +1230,7 @@ NCZ_load_all_plugins(void)
if(p->codec.codec && p->codec.codec->NCZ_codec_initialize)
p->codec.codec->NCZ_codec_initialize();
#ifdef DEBUGL
fprintf(stderr,"DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id);
fprintf(stderr,">>> DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id);
#endif
}
}
@ -1213,7 +1291,7 @@ NCZ_load_plugin_dir(const char* path)
ZTRACE(7,"path=%s",path);
#ifdef DEBUGL
fprintf(stderr,"DEBUGL: NCZ_load_plugin_dir: path=%s\n",path);
fprintf(stderr,">>> DEBUGL: NCZ_load_plugin_dir: path=%s\n",path);
#endif
if(path == NULL) {stat = NC_EINVAL; goto done;}
@ -1236,9 +1314,12 @@ NCZ_load_plugin_dir(const char* path)
strlcat(file,"/",flen);
strlcat(file,name,flen);
/* See if can load the file */
switch ((stat = NCZ_load_plugin(file,&plugin))) {
stat = NCZ_load_plugin(file,&plugin);
switch (stat) {
case NC_NOERR: break;
case NC_ENOFILTER: case NC_ENOTFOUND: stat = NC_NOERR; break; /* will cause it to be ignored */
case NC_ENOFILTER: case NC_ENOTFOUND:
stat = NC_NOERR;
break; /* will cause it to be ignored */
default: goto done;
}
if(plugin != NULL) {
@ -1247,11 +1328,11 @@ NCZ_load_plugin_dir(const char* path)
loaded_plugins[id] = plugin;
if(id > loaded_plugins_max) loaded_plugins_max = id;
#ifdef DEBUGL
fprintf(stderr,"DEBUGL: plugin loaded: %s\n",printplugin(plugin));
fprintf(stderr,">>> DEBUGL: plugin loaded: %s\n",printplugin(plugin));
#endif
} else {
#ifdef DEBUGL
fprintf(stderr,"DEBUGL: plugin duplicate: %s\n",printplugin(plugin));
fprintf(stderr,">>> DEBUGL: plugin duplicate: %s\n",printplugin(plugin));
#endif
NCZ_unload_plugin(plugin); /* its a duplicate */
}
@ -1280,23 +1361,21 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
ZTRACE(8,"path=%s",path);
#ifdef DEBUGL
fprintf(stderr,"DEBUGL: NCZ_load_plugin: path=%s\n",path);
#endif
if(plugp) *plugp = NULL;
#ifdef _WIN32
#if defined NAMEOPT || defined _WIN32
/*triage because visual studio does a popup if the file will not load*/
if(memcmp(path+(strlen(path)-4),".dll",4) != 0) {
stat = NC_ENOFILTER; goto done;
}
if(!pluginnamecheck(path)) {stat = NC_ENOFILTER; goto done;}
#endif
/* load the shared library */
if((stat = ncpsharedlibnew(&lib))) goto done;
if((stat = ncpload(lib,path,flags))) goto done;
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s\n",path);
#endif
/* See what we have */
{
H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type");
@ -1304,14 +1383,25 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info");
NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults");
/* Deal with defaults first */
if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL)
{stat = NC_ENOFILTER; goto done;}
if(cpd != NULL) {
if(codec_defaults == NULL) {
codec_defaults = (NCZ_codec_t**)cpd();
default_lib = lib; lib = NULL;
/* Deal with defaults first */
NCZ_codec_t** cp = NULL;
nclistpush(default_libs,lib);
for(cp=cpd();*cp;cp++) {
struct CodecAPI* c0;
c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI));
if(c0 == NULL) {stat = NC_ENOMEM; goto done1;}
c0->codec = *cp;
c0->codeclib = lib;
nclistpush(codec_defaults,c0); c0 = NULL;
}
done1:
lib = NULL;
goto done;
}
}
if(gpt != NULL && gpi != NULL) {
/* get HDF5 info */
@ -1331,14 +1421,11 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
}
}
/* Ignore this library if neither h5class nor codec are defined */
if(h5class == NULL && codec == NULL) {stat = NC_ENOFILTER; goto done;}
#ifdef DEBUGL
fprintf(stderr,"DEBUGL: load: %s:",path);
if(h5class) fprintf(stderr," %u",(unsigned)h5class->id);
if(codec) fprintf(stderr," %u/%s",codec->hdf5id,codec->codecid);
fprintf(stderr,"\n");
fprintf(stderr,">>> DEBUGL: load: %s:",path);
if(h5class) fprintf(stderr,">>> %u",(unsigned)h5class->id);
if(codec) fprintf(stderr,">>> %u/%s",codec->hdf5id,codec->codecid);
fprintf(stderr,">>> \n");
#endif
if(h5class != NULL && codec != NULL) {
@ -1379,7 +1466,7 @@ fprintf(stderr,"\n");
#ifdef DEBUGL
if(plugin)
fprintf(stderr,"DEBUGL: load_plugin: %s\n",printplugin(plugin));
fprintf(stderr,">>> DEBUGL: load_plugin: %s\n",printplugin(plugin));
#endif
@ -1404,19 +1491,49 @@ NCZ_unload_plugin(NCZ_Plugin* plugin)
if(plugin) {
#ifdef DEBUGL
fprintf(stderr,"DEBUGL: unload: %s\n",printplugin(plugin));
fprintf(stderr,">>> DEBUGL: unload: %s\n",printplugin(plugin));
#endif
if(plugin->codec.codec && plugin->codec.codec->NCZ_codec_finalize)
plugin->codec.codec->NCZ_codec_finalize();
if(plugin->hdf5.filter != NULL) loaded_plugins[plugin->hdf5.filter->id] = NULL;
if(plugin->hdf5.hdf5lib != NULL) (void)ncpsharedlibfree(plugin->hdf5.hdf5lib);
if(plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib);
memset(plugin,0,sizeof(NCZ_Plugin));
if(!plugin->codec.defaulted && plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib);
memset(plugin,0,sizeof(NCZ_Plugin));
free(plugin);
}
return ZUNTRACE(NC_NOERR);
}
#ifdef NAMEOPT
static int
pluginnamecheck(const char* name)
{
size_t count,len;
long i;
const char* p;
if(name == NULL) return 0;
/* get basename */
p = strrchr(name,'/');
if(p != NULL) name = (p+1);
len = strlen(name);
if(len == 0) return 0;
i = (long)(len-1);
count = 1;
p = name+i;
for(;i>=0;i--,count++,p--) {
char c = *p;
if(c == '/') break;
if(c == '.') {
if(count >= 3 && memcmp(p,".so",3)==0) return 1;
if(count >= 4 && memcmp(p,".dll",4)==0) return 1;
if(count >= 6 && memcmp(p,".dylib",6)==0) return 1;
}
}
return 0;
}
#endif
/**************************************************/
/* _Codecs attribute */
@ -1458,12 +1575,17 @@ ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter)
assert(filter->flags & FLAG_VISIBLE);
/* Convert the visible parameters to working parameters; may also modify the visible params */
#ifdef DEBUGF
fprintf(stderr,">>> DEBUGF: NCZ_modify_parameters: before: visible=%s working=%s\n",
printparams(filter->hdf5.visible.nparams,filter->hdf5.visible.params),
printparams(filter->hdf5.working.nparams,filter->hdf5.working.params));
#endif
if(filter->plugin && filter->plugin->codec.codec->NCZ_modify_parameters) {
stat = filter->plugin->codec.codec->NCZ_modify_parameters(ncidfor(var),var->hdr.id,
&filter->hdf5.visible.nparams, &filter->hdf5.visible.params,
&filter->hdf5.working.nparams, &filter->hdf5.working.params);
#ifdef DEBUGF
fprintf(stderr,"DEBUGF: NCZ_modify_parameters: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id,
fprintf(stderr,">>> DEBUGF: NCZ_modify_parameters: stat=%d ncid=%d varid=%d filter=%s\n",stat, (int)ncidfor(var),(int)var->hdr.id,
printfilter(filter));
#endif
if(stat) goto done;
@ -1477,10 +1599,15 @@ ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter)
nullfree(filter->hdf5.working.params);
if((stat = paramnczclone(&filter->hdf5.working,&filter->hdf5.visible))) goto done;
}
#ifdef DEBUGF
fprintf(stderr,">>> DEBUGF: NCZ_modify_parameters: after: visible=%s working=%s\n",
printparams(filter->hdf5.visible.nparams,filter->hdf5.visible.params),
printparams(filter->hdf5.working.nparams,filter->hdf5.working.params));
#endif
filter->flags |= FLAG_WORKING;
}
#ifdef DEBUGF
fprintf(stderr,"DEBUGF: ensure_working_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter));
fprintf(stderr,">>> DEBUGF: ensure_working_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter));
#endif
done:
return THROW(stat);
@ -1528,7 +1655,7 @@ rebuild_visible(const NC_VAR_INFO_T* var, NCZ_Filter* filter)
}
filter->flags |= FLAG_VISIBLE;
#ifdef DEBUGF
fprintf(stderr,"DEBUGF: rebuild_visible_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter));
fprintf(stderr,">>> DEBUGF: rebuild_visible_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter));
#endif
done:
nullfree(visible0);
@ -1555,7 +1682,7 @@ NCZ_filter_setup(NC_VAR_INFO_T* var)
/* Initialize the working parameters */
if((stat = ensure_working(var,filter))) goto done;
#ifdef DEBUGF
fprintf(stderr,"DEBUGF: NCZ_filter_setup: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id,
fprintf(stderr,">>> DEBUGF: NCZ_filter_setup: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id,
printfilter(filter));
#endif
}
@ -1564,6 +1691,7 @@ done:
return ZUNTRACE(stat);
}
/**************************************************/
/* Clone an hdf5 parameter set */

2
libnczarr/zfilter.h
View File

@ -48,5 +48,5 @@ int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, s
int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**);
int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter);
int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data);
#endif /*ZFILTER_H*/

28
libnczarr/zvar.c
View File

@ -600,23 +600,21 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
#endif
/* Shuffle filter? */
if (shuffle)
{
if(*shuffle) var->shuffle = *shuffle;
var->storage = NC_CHUNKED;
if(var->shuffle) {
if((retval = NCZ_def_var_filter(ncid,varid,2,0,NULL))) goto done;
}
if (shuffle && *shuffle) {
retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_SHUFFLE,NULL,NULL);
if(!retval || retval == NC_ENOFILTER) {
if((retval = NCZ_def_var_filter(ncid,varid,H5Z_FILTER_SHUFFLE,0,NULL))) return retval;
var->storage = NC_CHUNKED;
}
}
/* Fletcher32 checksum error protection? */
if (fletcher32)
{
if(*fletcher32) var->fletcher32 = *fletcher32;
var->storage = NC_CHUNKED;
if(var->fletcher32) {
if((retval = NCZ_def_var_filter(ncid,varid,3,0,NULL))) goto done;
}
if (fletcher32 && fletcher32) {
retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_FLETCHER32,NULL,NULL);
if(!retval || retval == NC_ENOFILTER) {
if((retval = NCZ_def_var_filter(ncid,varid,H5Z_FILTER_FLETCHER32,0,NULL))) return retval;
var->storage = NC_CHUNKED;
}
}
/* Handle storage settings. */
@ -628,7 +626,7 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
* no filters in use for this data. */
if (storage != NC_CHUNKED)
{
if (nclistlength(((NClist*)var->filters)) > 0 || var->fletcher32 || var->shuffle)
if (nclistlength(((NClist*)var->filters)) > 0)
{retval = NC_EINVAL; goto done;}
for (d = 0; d < var->ndims; d++)
if (var->dim[d]->unlimited)

4
libnetcdf.settings.in
View File

@ -41,10 +41,10 @@ JNA Support: @HAS_JNA@
CDF5 Support: @HAS_CDF5@
ERANGE Fill Support: @HAS_ERANGE_FILL@
Relaxed Boundary Check: @RELAX_COORD_BOUND@
SZIP Support: @HAS_SZLIB@
SZIP Write Support: @HAS_SZLIB_WRITE@
Parallel Filters: @HAS_PAR_FILTERS@
NCZarr Support: @HAS_NCZARR@
Multi-Filter Support: @HAS_MULTIFILTERS@
Quantization: @HAS_QUANTIZE@
Logging: @HAS_LOGGING@
SZIP Write Support: @HAS_SZLIB_WRITE@
Standard Filters: @STD_FILTERS@

2
libsrc/nc3dispatch.c
View File

@ -169,6 +169,8 @@ NC_NOOP_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
NC_NOOP_inq_filter_avail,
};
const NC_Dispatch* NC3_dispatch_table = NULL; /*!< NC3 Dispatch table, moved here from ddispatch.c */

17
libsrc4/nc4var.c
View File

@ -242,11 +242,16 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
*storagep = var->storage;
/* Filter stuff. */
if (shufflep)
*shufflep = (int)var->shuffle;
if (fletcher32p)
*fletcher32p = (int)var->fletcher32;
if (shufflep) {
retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_SHUFFLE,0,NULL);
if(retval && retval != NC_ENOFILTER) return retval;
*shufflep = (retval == NC_NOERR?1:0);
}
if (fletcher32p) {
retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_FLETCHER32,0,NULL);
if(retval && retval != NC_ENOFILTER) return retval;
*fletcher32p = (retval == NC_NOERR?1:0);
}
if (deflatep)
return NC_EFILTER;
@ -476,7 +481,7 @@ NC4_var_par_access(int ncid, int varid, int par_access)
/* If zlib, shuffle, or fletcher32 filters are in use, then access
* must be collective. Fail an attempt to set such a variable to
* independent access. */
if ((nclistlength((NClist*)var->filters) > 0 || var->shuffle || var->fletcher32) &&
if (nclistlength((NClist*)var->filters) > 0)
par_access == NC_INDEPENDENT)
return NC_EINVAL;

2
libsrcp/ncpdispatch.c
View File

@ -1466,6 +1466,8 @@ NC_NOOP_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
NC_NOOP_inq_filter_avail,
};
const NC_Dispatch *NCP_dispatch_table = NULL; /* moved here from ddispatch.c */

9
nc_test4/CMakeLists.txt
View File

@ -40,22 +40,21 @@ IF(BUILD_UTILITIES)
ADD_SH_TEST(nc_test4 tst_misc)
build_bin_test(tst_fillonly)
ADD_SH_TEST(nc_test4 test_fillonly)
IF(ENABLE_FILTER_TESTING)
IF(USE_HDF5 AND ENABLE_FILTER_TESTING)
build_bin_test(tst_filterparser)
build_bin_test(test_filter)
build_bin_test(test_filter_misc)
build_bin_test(tst_multifilter)
build_bin_test(test_filter_order)
build_bin_test(test_filter_repeat)
build_bin_test(tst_filter_avail)
build_bin_test(test_filter_vlen)
ADD_SH_TEST(nc_test4 tst_filter)
IF(ENABLE_BLOSC)
ADD_SH_TEST(nc_test4 tst_specific_filters)
ENDIF()
IF(ENABLE_CLIENTSIDE_FILTERS)
add_bin_test(nc_test4 test_filter_reg)
ENDIF(ENABLE_CLIENTSIDE_FILTERS)
ENDIF(ENABLE_FILTER_TESTING)
ENDIF(USE_HDF5 AND ENABLE_FILTER_TESTING)
ENDIF(BUILD_UTILITIES)
@ -84,7 +83,7 @@ IF(LARGE_FILE_TESTS)
ENDIF()
IF(USE_SZIP)
IF(HAVE_H5Z_SZIP)
BUILD_BIN_TEST(test_szip)
BUILD_BIN_TEST(h5testszip)
IF(BUILD_UTILITIES)

12
nc_test4/Makefile.am
View File

@ -71,21 +71,21 @@ check_PROGRAMS += tst_fillonly
TESTS += test_fillonly.sh
# Szip Tests (requires ncdump)
if USE_SZIP
if HAVE_H5Z_SZIP
check_PROGRAMS += test_szip h5testszip
TESTS += tst_szip.sh
endif
# Filter Tests (requires ncdump and ncgen)
# Filter Tests (requires ncdump and ncgen and HDF5)
if USE_HDF5
if ENABLE_FILTER_TESTING
extradir =
check_PROGRAMS += test_filter test_filter_misc test_filter_order test_filter_repeat test_filter_vlen
check_PROGRAMS += tst_multifilter
check_PROGRAMS += tst_multifilter tst_filter_avail
TESTS += tst_filter.sh
if ENABLE_BLOSC
TESTS += tst_specific_filters.sh
endif
endif
endif # USE_HDF5
endif # BUILD_UTILITIES
# This are extra tests that will only be run if netcdf-4 is configured
@ -102,7 +102,7 @@ ref_tst_xplatform2_1.nc ref_tst_xplatform2_2.nc ref_tst_dims.nc \
ref_tst_interops4.nc CMakeLists.txt run_grp_rename.sh \
run_empty_vlen_test.sh ref_hdf5_compat1.nc ref_hdf5_compat2.nc \
ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5 \
ref_szip.cdl tst_filter.sh bzip2.cdl ref_filtered.cdl \
ref_szip.cdl tst_filter.sh bzip2.cdl noop.cdl ref_filtered.cdl \
ref_unfiltered.cdl ref_bzip2.c findplugin.in ref_unfilteredvv.cdl \
ref_filteredvv.cdl ref_multi.cdl \
ref_ncgenF.cdl ref_nccopyF.cdl \

83
nc_test4/noop.cdl Normal file
View File

@ -0,0 +1,83 @@
netcdf bzip2 {
dimensions:
dim0 = 4 ;
dim1 = 4 ;
dim2 = 4 ;
dim3 = 4 ;
variables:
float var(dim0, dim1, dim2, dim3) ;
var:_Storage = "chunked" ;
var:_ChunkSizes = 4, 4, 4, 4 ;
var:_Filter = "40000,17" ;
var:_NoFill = "true" ;
// global attributes:
:_Format = "netCDF-4" ;
data:
var =
0, 1, 2, 3,
4, 5, 6, 7,
8, 9, 10, 11,
12, 13, 14, 15,
16, 17, 18, 19,
20, 21, 22, 23,
24, 25, 26, 27,
28, 29, 30, 31,
32, 33, 34, 35,
36, 37, 38, 39,
40, 41, 42, 43,
44, 45, 46, 47,
48, 49, 50, 51,
52, 53, 54, 55,
56, 57, 58, 59,
60, 61, 62, 63,
64, 65, 66, 67,
68, 69, 70, 71,
72, 73, 74, 75,
76, 77, 78, 79,
80, 81, 82, 83,
84, 85, 86, 87,
88, 89, 90, 91,
92, 93, 94, 95,
96, 97, 98, 99,
100, 101, 102, 103,
104, 105, 106, 107,
108, 109, 110, 111,
112, 113, 114, 115,
116, 117, 118, 119,
120, 121, 122, 123,
124, 125, 126, 127,
128, 129, 130, 131,
132, 133, 134, 135,
136, 137, 138, 139,
140, 141, 142, 143,
144, 145, 146, 147,
148, 149, 150, 151,
152, 153, 154, 155,
156, 157, 158, 159,
160, 161, 162, 163,
164, 165, 166, 167,
168, 169, 170, 171,
172, 173, 174, 175,
176, 177, 178, 179,
180, 181, 182, 183,
184, 185, 186, 187,
188, 189, 190, 191,
192, 193, 194, 195,
196, 197, 198, 199,
200, 201, 202, 203,
204, 205, 206, 207,
208, 209, 210, 211,
212, 213, 214, 215,
216, 217, 218, 219,
220, 221, 222, 223,
224, 225, 226, 227,
228, 229, 230, 231,
232, 233, 234, 235,
236, 237, 238, 239,
240, 241, 242, 243,
244, 245, 246, 247,
248, 249, 250, 251,
252, 253, 254, 255 ;
}

218
nc_test4/test_filter_vlen.c
View File

@ -66,24 +66,13 @@ static int nerrs = 0;
static int ncid, varid;
static int dimids[MAXDIMS];
static size_t odom[MAXDIMS];
static float* array = NULL;
static float* expected = NULL;
static unsigned int filterid = 0;
static size_t nparams = 0;
static unsigned int params[MAXPARAMS];
/* Forward */
static int test_test1(void);
static void init(int argc, char** argv);
static void reset(void);
static void odom_reset(void);
static int odom_more(void);
static int odom_next(void);
static int odom_offset(void);
static float expectedvalue(void);
static void verifyparams(void);
#define ERRR do { \
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
@ -100,16 +89,7 @@ check(int err,int line)
}
return NC_NOERR;
}
static void
report(const char* msg, int lineno)
{
fprintf(stderr,"fail: line=%d %s\n",lineno,msg);
exit(1);
}
#define CHECK(x) check(x,__LINE__)
#define REPORT(x) report(x,__LINE__)
static int
verifychunks(void)
@ -156,69 +136,6 @@ defvar(nc_type xtype)
return NC_NOERR;
}
static void
setvarfilter(void)
{
CHECK(nc_def_var_filter(ncid,varid,TEST_ID,NPARAMS,baseline));
verifyparams();
}
static void
verifyparams(void)
{
int i;
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params));
if(filterid != TEST_ID) REPORT("id mismatch");
if(nparams != NPARAMS) REPORT("nparams mismatch");
for(i=0;i<nparams;i++) {
if(params[i] != baseline[i])
REPORT("param mismatch");
}
}
static int
openfile(void)
{
unsigned int* params = NULL;
/* Open the file and check it. */
CHECK(nc_open(testfile, NC_NOWRITE, &ncid));
CHECK(nc_inq_varid(ncid, "var", &varid));
/* Check the compression algorithm */
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,NULL));
if(nparams > 0) {
params = (unsigned int*)malloc(sizeof(unsigned int)*nparams);
if(params == NULL)
return NC_ENOMEM;
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params));
}
if(filterid != TEST_ID) {
fprintf(stderr,"open: test id mismatch: %d\n",filterid);
free(params);
return NC_EFILTER;
}
if(nparams != NPARAMS) {
size_t i;
fprintf(stderr,"nparams mismatch\n");
for(nerrs=0,i=0;i<nparams;i++) {
if(params[i] != baseline[i]) {
fprintf(stderr,"open: testparam mismatch: %ld\n",(unsigned long)i);
nerrs++;
}
}
}
free(params);
if(nerrs > 0) return NC_EFILTER;
/* Verify chunking */
if(!verifychunks())
return 0;
fflush(stderr);
return 1;
}
static int
setchunking(void)
{
@ -231,91 +148,6 @@ setchunking(void)
return NC_NOERR;
}
static void
fill(void)
{
odom_reset();
if(1) {
int i;
if(actualproduct <= 1) abort();
for(i=0;i<actualproduct;i++)
expected[i] = (float)i;
} else {
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
expected[offset] = expect;
odom_next();
}
}
}
static int
compare(void)
{
int errs = 0;
fprintf(stderr,"data comparison: |array|=%ld\n",(unsigned long)actualproduct);
if(1)
{
int i;
for(i=0;i<actualproduct;i++) {
if(expected[i] != array[i]) {
fprintf(stderr,"data mismatch: array[%d]=%f expected[%d]=%f\n",
i,array[i],i,expected[i]);
errs++;
if(errs >= MAXERRS)
break;
}
}
} else
{
odom_reset();
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
if(array[offset] != expect) {
fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n",
offset,array[offset],expect);
errs++;
if(errs >= MAXERRS)
break;
}
odom_next();
}
}
if(errs == 0)
fprintf(stderr,"no data errors\n");
return (errs == 0);
}
static void
showparameters(void)
{
int i;
fprintf(stderr,"test: nparams=%ld: params=",(unsigned long)nparams);
for(i=0;i<nparams;i++) {
fprintf(stderr," %u",params[i]);
}
fprintf(stderr,"\n");
for(i=0;i<ndims;i++) {
if(i==0)
fprintf(stderr,"dimsizes=%ld",(unsigned long)dimsize[i]);
else
fprintf(stderr,",%ld",(unsigned long)dimsize[i]);
}
fprintf(stderr,"\n");
for(i=0;i<ndims;i++) {
if(i==0)
fprintf(stderr,"chunksizes=%ld",(unsigned long)chunksize[i]);
else
fprintf(stderr,",%ld",(unsigned long)chunksize[i]);
}
fprintf(stderr,"\n");
fflush(stderr);
}
static void
insert(int index, void* src, size_t size)
{
@ -401,56 +233,6 @@ reset()
memset(array,0,sizeof(float)*actualproduct);
}
static void
odom_reset(void)
{
memset(odom,0,sizeof(odom));
}
static int
odom_more(void)
{
return (odom[0] < dimsize[0]);
}
static int
odom_next(void)
{
int i; /* do not make unsigned */
for(i=ndims-1;i>=0;i--) {
odom[i] += 1;
if(odom[i] < dimsize[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows*/
odom[i] = 0; /* reset this position*/
}
return 1;
}
static int
odom_offset(void)
{
int i;
int offset = 0;
for(i=0;i<ndims;i++) {
offset *= dimsize[i];
offset += odom[i];
}
return offset;
}
static float
expectedvalue(void)
{
int i;
float offset = 0;
for(i=0;i<ndims;i++) {
offset *= dimsize[i];
offset += odom[i];
}
return offset;
}
static void
init(int argc, char** argv)
{

52
nc_test4/tst_filter.sh
View File

@ -41,15 +41,15 @@ trimleft() {
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
}
# Hide/unhide the bzip2 filter
hidebzip2() {
# Hide/unhide the noop filter
hidenoop() {
rm -fr ${HDF5_PLUGIN_PATH}/save
mkdir ${HDF5_PLUGIN_PATH}/save
mv ${BZIP2PATH} ${HDF5_PLUGIN_PATH}/save
mv ${NOOPPATH} ${HDF5_PLUGIN_PATH}/save
}
unhidebzip2() {
mv ${HDF5_PLUGIN_PATH}/save/${BZIP2LIB} ${HDF5_PLUGIN_PATH}
unhidenoop() {
mv ${HDF5_PLUGIN_PATH}/save/${NOOPLIB} ${HDF5_PLUGIN_PATH}
rm -fr ${HDF5_PLUGIN_PATH}/save
}
@ -61,6 +61,10 @@ BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}"
# Find misc and capture
findplugin h5misc
MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
# Find noop and capture
findplugin h5noop
NOOPLIB="${HDF5_PLUGIN_LIB}"
NOOPPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
export HDF5_PLUGIN_PATH
@ -68,14 +72,10 @@ export HDF5_PLUGIN_PATH
# Verify
if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi
if ! test -f ${NOOPPATH} ; then echo "Unable to locate ${NOOPPATH}"; exit 1; fi
# See if we have szip
HAVE_SZIP=0
if test -f ${TOPBUILDDIR}/libnetcdf.settings ; then
if grep "SZIP Support:[ ]*yes" <${TOPBUILDDIR}/libnetcdf.settings ; then
HAVE_SZIP=1
fi
fi
if avail szip; then HAVE_SZIP=1; else HAVE_SZIP=0; fi
# Execute the specified tests
@ -181,31 +181,31 @@ fi
if test "x$UNK" = x1 ; then
echo "*** Testing access to filter info when filter dll is not available"
rm -f bzip2.nc ./tmp_filter.txt
# xfail build bzip2.nc
hidebzip2
if ${NCGEN} -lb -4 -o bzip2.nc ${srcdir}/bzip2.cdl ; then
rm -f noop.nc ./tmp_filter.txt
# xfail build noop.nc
hidenoop
if ${NCGEN} -lb -4 -o noop.nc ${srcdir}/noop.cdl ; then
echo "*** FAIL: ncgen"
else
echo "*** XFAIL: ncgen"
fi
unhidebzip2
# build bzip2.nc
${NCGEN} -lb -4 -o bzip2.nc ${srcdir}/bzip2.cdl
unhidenoop
# build noop.nc
${NCGEN} -lb -4 -o noop.nc ${srcdir}/noop.cdl
# Now hide the filter code
hidebzip2
hidenoop
rm -f ./tmp_filter.txt
# This will xfail
if ${NCDUMP} -s bzip2.nc > ./tmp_filter.txt ; then
echo "*** FAIL: ncdump -hs bzip2.nc"
if ${NCDUMP} -s noop.nc > ./tmp_filter.txt ; then
echo "*** FAIL: ncdump -hs noop.nc"
else
echo "*** XFAIL: ncdump -hs bzip2.nc"
echo "*** XFAIL: ncdump -hs noop.nc"
fi
# Restore the filter code
unhidebzip2
unhidenoop
# Verify we can see filter when using -h
rm -f ./tmp_filter.txt
${NCDUMP} -hs bzip2.nc > ./tmp_filter.txt
${NCDUMP} -hs noop.nc > ./tmp_filter.txt
echo "*** Pass: unknown filter"
fi
@ -222,7 +222,7 @@ echo "*** Testing multiple filters"
rm -f ./tmp_multifilter.nc ./tmp_multi.txt ./tmp_smulti.cdl
rm -f tmp_nccopyF.cdl tmp_nccopyF.nc tmp_ncgenF.cdl tmp_ncgenF.nc
${execdir}/tst_multifilter
${NCDUMP} -hs -n multifilter tmp_multifilter.nc >./tmp_multi.cdl
${NCDUMP} -hsF -n multifilter tmp_multifilter.nc >./tmp_multi.cdl
# Remove irrelevant -s output
sclean ./tmp_multi.cdl ./tmp_smulti.cdl
diff -b -w ${srcdir}/ref_multi.cdl ./tmp_smulti.cdl
@ -237,7 +237,7 @@ diff -b -w ${srcdir}/ref_nccopyF.cdl ./tmp_nccopyFs.cdl
echo "*** ncgen with multiple filters"
${NCGEN} -4 -lb -o tmp_ncgenF.nc ${srcdir}/ref_nccopyF.cdl
# Need to fix name using -n
${NCDUMP} -hs -n nccopyF tmp_ncgenF.nc > ./tmp_ncgenF.cdl
${NCDUMP} -hsF -n nccopyF tmp_ncgenF.nc > ./tmp_ncgenF.cdl
sclean tmp_ncgenF.cdl tmp_ncgenFs.cdl
diff -b -w ${srcdir}/ref_nccopyF.cdl ./tmp_ncgenFs.cdl
echo "*** Pass: multiple filters"

106
nc_test4/tst_filter_avail.c Normal file
View File

@ -0,0 +1,106 @@
/*
Copyright 2018, UCAR/Unidata
See COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#ifdef USE_HDF5
#include <hdf5.h>
#endif
#include "netcdf.h"
#include "netcdf_aux.h"
#include "netcdf_filter.h"
#undef DEBUG
#define MAXPARAMS 32
#ifdef TESTNCZARR
#define DFALT_TESTFILE "file://tmp_filter_avail.file#mode=nczarr,file"
#else
#define DFALT_TESTFILE "tmp_filter_avail.nc"
#endif
static const char* testfile = NULL;
static int nerrs = 0;
static int ncid;
/* Forward */
static int test_test1(void);
static void init(int argc, char** argv);
#define ERRR do { \
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \
__FILE__, __LINE__); \
nerrs++;\
} while (0)
static int
check(int err,int line)
{
if(err != NC_NOERR) {
fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err));
}
return NC_NOERR;
}
#define CHECK(x) check(x,__LINE__)
static int
test_test1(void)
{
int stat = NC_NOERR;
printf("test1: bzip2 availability\n");
CHECK(nc_create(testfile,NC_NETCDF4|NC_CLOBBER,&ncid));
CHECK(nc_enddef(ncid));
switch (stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2)) {
case NC_NOERR: break;
case NC_ENOFILTER: break;
default: CHECK(stat); goto done;
}
if(stat == NC_ENOFILTER) {
printf("*** FAIL: filter %d not available\n",H5Z_FILTER_BZIP2);
} else {
printf("*** PASS: filter %d available\n",H5Z_FILTER_BZIP2);
}
CHECK(nc_abort(ncid));
done:
return stat;
}
/**************************************************/
/* Utilities */
static void
init(int argc, char** argv)
{
/* get the testfile path */
if(argc > 1)
testfile = argv[1];
else
testfile = DFALT_TESTFILE;
}
/**************************************************/
int
main(int argc, char **argv)
{
#ifdef USE_HDF5
#ifdef DEBUG
H5Eprint1(stderr);
nc_set_log_level(1);
#endif
#endif
init(argc,argv);
if(test_test1() != NC_NOERR) ERRR;
exit(nerrs > 0?1:0);
}

4
nc_test4/tst_interops5.c
View File

@ -180,7 +180,7 @@ main(int argc, char **argv)
}
}
SUMMARIZE_ERR;
#ifdef USE_SZIP
#ifdef HAVE_H5Z_SZIP
printf("*** testing HDF5 compatibility with szip...");
{
@ -254,7 +254,7 @@ main(int argc, char **argv)
}
SUMMARIZE_ERR;
#endif /* USE_SZIP */
#endif /* HAVE_H5Z_SZIP */
/* This test suggested by user brentd42 to find a memory problem in
* function rec_read_metadata(). This test demonstrates the bug on
* address sanitizer runs. See

8
nc_test4/tst_parallel5.c
View File

@ -370,7 +370,7 @@ main(int argc, char **argv)
}
if (!mpi_rank)
SUMMARIZE_ERR;
#ifdef USE_SZIP
#ifdef HAVE_H5Z_SZIP
#ifdef HDF5_SUPPORTS_PAR_FILTERS
#define SZIP_DIM_LEN 256
#define SZIP_DIM_NAME "Barrels"
@ -419,9 +419,9 @@ main(int argc, char **argv)
if (!mpi_rank)
SUMMARIZE_ERR;
#endif /* HDF5_SUPPORTS_PAR_FILTERS */
#endif /* USE_SZIP */
#endif /* HAVE_H5Z_SZIP */
#ifdef USE_SZIP
#ifdef HAVE_H5Z_SZIP
#ifdef HDF5_SUPPORTS_PAR_FILTERS
#define SZIP_DIM_LEN 256
#define SZIP_DIM_NAME "Barrels"
@ -469,7 +469,7 @@ main(int argc, char **argv)
if (!mpi_rank)
SUMMARIZE_ERR;
#endif /* HDF5_SUPPORTS_PAR_FILTERS */
#endif /* USE_SZIP */
#endif /* HAVE_H5Z_SZIP */
/* Shut down MPI. */
MPI_Finalize();

5
nc_test4/tst_quantize.c
View File

@ -210,7 +210,6 @@ main(int argc, char **argv)
char url[4096];
snprintf(url,sizeof(url),template,file_name);
strcpy(file_name,url);
fprintf(stderr,"\n>>> type url = |%s|\n",file_name);
}
#endif
/* Create file. */
@ -228,7 +227,8 @@ fprintf(stderr,"\n>>> type url = |%s|\n",file_name);
{
if (nc_open(file_name, NC_NETCDF4, &ncid)) ERR;
if (nc_inq_varid(ncid,VAR_NAME,&varid)) ERR;
if (nc_inq_var_quantize(ncid, varid, &quantize_mode, &nsd_in)) ERR;
if (nc_inq_var_quantize(ncid, varid, &quantize_mode, &nsd_in))
ERR;
if (quantize_mode) ERR;
if (nc_close(ncid)) ERR;
}
@ -931,7 +931,6 @@ fprintf(stderr,"\n>>> type url = |%s|\n",file_name);
* set compression level to 1, which is usually the best
* choice. */
if (nc_def_var_deflate(ncid, varid1, 0, 1, 1)) ERR;
if (nc_def_var_deflate(ncid, varid2, 0, 1, 1)) ERR;
#endif
#endif
/* Write the data. */

119
nc_test4/tst_specific_filters.sh
View File

@ -1,10 +1,24 @@
#!/bin/bash
# Test the implementations of specific filters
# Also test nc_inq_filter_avail
# WARNING: This file is also used to build nczarr_test/run_specific_filters.sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
if test "x$TESTNCZARR" = x1 ; then
. "$srcdir/test_nczarr.sh"
BLOSCARGS="32001,0,0,0,256,5,1,1"
BLOSCCODEC='[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 256,\"cname\": \"lz4\",\"shuffle\": 1}]'
else
BLOSCARGS="32001,2,2,4,256,5,1,1"
BLOSCCODEC='[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 256,\"cname\": \"lz4\",\"shuffle\": 1}]'
fi
# Load the findplugins function
. ${builddir}/findplugin.sh
echo "findplugin.sh loaded"
@ -17,8 +31,6 @@ findplugin h5bzip2
echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
export HDF5_PLUGIN_PATH
set -e
# Function to remove selected -s attributes from file;
# These attributes might be platform dependent
sclean() {
@ -50,6 +62,7 @@ trimleft() {
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
}
setfilter() {
FF="$1"
FSRC="$2"
@ -76,56 +89,98 @@ zext=$1
zfilt="$2"
zparams="$3"
zcodec="$4"
echo "*** Testing processing of filter $zfilt"
file="tmp_${zfilt}.nc"
rm -f "tmp_${zfilt},nc"
setfilter $zfilt ref_any.cdl "tmp_${zfilt}.cdl" "$zparams" "$zcodec"
if ${NCGEN} -4 -lb -o $file "tmp_${zfilt}.cdl" ; then
${NCDUMP} -n $zfilt -s $file > "tmp_${zfilt}.tmp"
sclean "tmp_${zfilt}.tmp" "tmp_${zfilt}.dump"
echo "*** Testing processing of filter $zfilt for map $zext"
if test "x$TESTNCZARR" = x1 ; then
fileargs "tmp_filt_${zfilt}"
deletemap $zext $file
else
file="tmp_filt_${zfilt}.nc"
rm -f $file
fi
unset NCTRACING
setfilter $zfilt ref_any.cdl "tmp_filt_${zfilt}.cdl" "$zparams" "$zcodec"
if test "x$TESTNCZARR" = x1 ; then
${NCGEN} -4 -lb -o $fileurl "tmp_filt_${zfilt}.cdl"
${NCDUMP} -n $zfilt -sF $fileurl > "tmp_filt_${zfilt}.tmp"
else
${NCGEN} -4 -lb -o $file "tmp_filt_${zfilt}.cdl"
${NCDUMP} -n $zfilt -sF $file > "tmp_filt_${zfilt}.tmp"
fi
sclean "tmp_filt_${zfilt}.tmp" "tmp_filt_${zfilt}.dump"
}
testfletcher32() {
zext=$1
runfilter $zext fletcher32 '3' '[{\"id\": \"fletcher32\"}]'
# need to do fixup
sed -e '/_Fletcher32 = "true"/d' -e '/_Filter = 3'/d -e '/_Codecs = \"[{\"id\": \"fletcher32\"}]\"/d' \
< tmp_filt_fletcher32.cdl > tmp_filt_fletcher32x.dump
diff -b -w "tmp_filt_fletcher32.cdl" "tmp_filt_fletcher32x.dump"
}
testshuffle() {
zext=$1
runfilter $zext shuffle '2' '[{\"id\": \"shuffle\",\"elementsize\": \"0\"}]'
# need to replace _Filter
sed -e '/_Shuffle = "true"/d' -e '/_Filter = 2'/d -e '/_Codecs = \"[{\"id\": \"shuffle\"}]\"/d' \
< tmp_filt_shuffle.cdl > tmp_filt_shufflex.dump
diff -b -w "tmp_filt_shuffle.cdl" "tmp_filt_shufflex.dump"
}
testdeflate() {
zext=$1
if ! avail deflate; then return 0; fi
runfilter $zext deflate '1,9' '[{\"id\": \"zlib\",\"level\": \"9\"}]'
if test -f "tmp_deflate.dump" ; then
# need to replace _DeflateLevel
sed -e 's/_DeflateLevel = 9/_Filter = "1,9"/' < tmp_deflate.dump > tmp_deflatex.dump
diff -b -w "tmp_deflate.cdl" "tmp_deflatex.dump"
else
echo "XFAIL: filter=deflate"
fi
# need to replace _DeflateLevel
# sed -e 's/_DeflateLevel = 9/_Filter = "1,9"/' < tmp_filt_deflate.dump > tmp_filt_deflatex.dump
diff -b -w "tmp_filt_deflate.cdl" "tmp_filt_deflate.dump"
}
testbzip2() {
zext=$1
if ! avail bzip2; then return 0; fi
runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]'
if test -f "tmp_bzip2.dump" ; then
diff -b -w "tmp_bzip2.cdl" "tmp_bzip2.dump"
else
echo "XFAIL: filter=bzip2"
fi
diff -b -w "tmp_filt_bzip2.cdl" "tmp_filt_bzip2.dump"
}
testszip() {
zext=$1
if ! avail szip; then return 0; fi
# H5_SZIP_NN_OPTION_MASK=32; H5_SZIP_MAX_PIXELS_PER_BLOCK_IN=32
runfilter $zext szip '4,32,32' '[{\"id\": \"szip\",\"mask\": 32,\"pixels-per-block\": 32}]'
diff -b -w "tmp_filt_szip.cdl" "tmp_filt_szip.dump"
}
testblosc() {
zext=$1
runfilter $zext blosc '32001,2,2,4,256,5,1,1' '[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 0,\"cname\": \"lz4\",\"shuffle\": 1}]'
if test -f "tmp_blosc.dump" ; then
diff -b -w "tmp_blosc.cdl" "tmp_blosc.dump"
else
echo "XFAIL: filter=blosc"
fi
if ! avail blosc; then return 0; fi
runfilter $zext blosc $BLOSCARGS "$BLOSCCODEC"
diff -b -w "tmp_filt_blosc.cdl" "tmp_filt_blosc.dump"
}
testzstd() {
zext=$1
if ! avail zstd; then return 0; fi
runfilter $zext zstd '32015,1' '[{\"id\": \"zstd\",\"level\": \"1\"}]'
diff -b -w "tmp_filt_zstd.cdl" "tmp_filt_zstd.dump"
}
testset() {
# Which test cases to exercise
if test "x$TESTNCZARR" = x1 ; then
# testfletcher32 $1
testshuffle $1
fi
# testdeflate $1
testbzip2 $1
# testszip $1
# testbzip2 $1
# testblosc $1
# testzstd $1
}
testset nc
exit 0
if test "x$TESTNCZARR" = x1 ; then
testset file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi
else
testset nc
fi

16
nc_test4/tst_udf.c
View File

@ -158,6 +158,13 @@ static NC_Dispatch tst_dispatcher = {
NC_NOOP_inq_var_filter_ids,
NC_NOOP_inq_var_filter_info,
#endif
#if NC_DISPATCH_VERSION >= 4
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
#endif
#if NC_DISPATCH_VERSION >= 5
NC_NOOP_inq_filter_avail,
#endif
};
/* This is the dispatch object that holds pointers to all the
@ -250,10 +257,17 @@ static NC_Dispatch tst_dispatcher_bad_version = {
NC_NOTNC4_def_var_filter,
NC_NOTNC4_set_var_chunk_cache,
NC_NOTNC4_get_var_chunk_cache,
#if NC_DISPATCH_VERSION >= 2
#if NC_DISPATCH_VERSION >= 3
NC_NOOP_inq_var_filter_ids,
NC_NOOP_inq_var_filter_info,
#endif
#if NC_DISPATCH_VERSION >= 4
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
#endif
#if NC_DISPATCH_VERSION >= 5
NC_NOOP_inq_filter_avail,
#endif
};
#define NUM_UDFS 2

13
nc_test4/tst_vars3.c
View File

@ -420,7 +420,7 @@ main(int argc, char **argv)
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
#ifdef USE_SZIP
#ifdef HAVE_H5Z_SZIP
printf("**** testing simple szip filter setup...");
{
int ncid;
@ -731,7 +731,7 @@ main(int argc, char **argv)
SUMMARIZE_ERR;
#else
/* This code is run if szip is not present in HDF5. It checks that
* nc_def_var_szip() returns NC_EFILTER in that case. */
* nc_def_var_szip() returns NC_ENOFILTER in that case. */
printf("**** testing szip handling when szip not built...");
{
int ncid;
@ -749,15 +749,16 @@ main(int argc, char **argv)
params[0] = NC_SZIP_NN; /* options_mask */
params[1] = NC_SZIP_EC_BPP_IN; /* pixels_per_block */
if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
{ int stat; if ((stat = nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN,
params)) != NC_EFILTER)
{ int stat;
if ((stat = nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN,
params)) != NC_ENOFILTER)
ERR;
}
if (nc_def_var_szip(ncid, varid, NC_SZIP_NN,
NC_SZIP_EC_BPP_IN) != NC_EFILTER) ERR;
NC_SZIP_EC_BPP_IN) != NC_ENOFILTER) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
#endif /* USE_SZIP */
#endif /* HAVE_H5Z_SZIP */
FINAL_RESULTS;
}

2
nc_test4/tst_vars4.c
View File

@ -34,7 +34,7 @@ main(int argc, char **argv)
nc_type type_in;
size_t len_in;
nc_set_log_level(3);
nc_set_log_level(-1);
if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;

65
ncdump/CMakeLists.txt
View File

@ -17,6 +17,7 @@ SET(ocprint_FILES ocprint.c)
SET(ncvalidator_FILES ncvalidator.c)
SET(printfqn_FILES printfqn.c)
SET(ncpathcvt_FILES ncpathcvt.c)
SET(ncfilteravail_FILES ncfilteravail.c)
SET(nchdf5version_FILES nchdf5version.c)
IF(USE_X_GETOPT)
@ -26,12 +27,14 @@ IF(USE_X_GETOPT)
SET(ncvalidator_FILES ${ncvalidator_FILES} XGetopt.c)
SET(printfqn_FILES ${printfqn_FILES} XGetopt.c)
SET(ncpathcvt_FILES ${ncpathcvt_FILES} XGetopt.c)
SET(ncfilteravail_FILES ${ncfilteravail_FILES} XGetopt.c)
ENDIF(USE_X_GETOPT)
ADD_EXECUTABLE(ncdump ${ncdump_FILES})
ADD_EXECUTABLE(nccopy ${nccopy_FILES})
ADD_EXECUTABLE(ncvalidator ${ncvalidator_FILES})
ADD_EXECUTABLE(ncpathcvt ${ncpathcvt_FILES})
ADD_EXECUTABLE(ncfilteravail ${ncfilteravail_FILES})
IF(USE_HDF5)
ADD_EXECUTABLE(nc4print nc4print.c nc4printer.c)
@ -47,6 +50,7 @@ TARGET_LINK_LIBRARIES(ncdump netcdf ${ALL_TLL_LIBS})
TARGET_LINK_LIBRARIES(nccopy netcdf ${ALL_TLL_LIBS})
TARGET_LINK_LIBRARIES(ncvalidator netcdf ${ALL_TLL_LIBS})
TARGET_LINK_LIBRARIES(ncpathcvt netcdf ${ALL_TLL_LIBS})
TARGET_LINK_LIBRARIES(ncfilteravail netcdf ${ALL_TLL_LIBS})
IF(USE_HDF5)
TARGET_LINK_LIBRARIES(nc4print netcdf ${ALL_TLL_LIBS})
@ -64,57 +68,30 @@ ENDIF(ENABLE_DAP)
# in MSVC builds. This is required to get
# test scripts to work.
####
IF(MSVC)
SET_TARGET_PROPERTIES(ncdump PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncdump PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncdump PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
macro(setbinprops name)
SET_TARGET_PROPERTIES(${name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(${name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(${name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
endmacro()
SET_TARGET_PROPERTIES(nccopy PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(nccopy PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(nccopy PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncpathcvt PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncpathcvt PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncpathcvt PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
setbinprops(ncdump)
setbinprops(nccopy)
setbinprops(ncvalidator)
setbinprops(ncpathcvt)
setbinprops(ncfilteravail)
IF(USE_HDF5)
SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(nchdf5version PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(nchdf5version PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(nchdf5version PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
setbinprops(printfqn)
setbinprops(nchdf5version)
ENDIF(USE_HDF5)
IF(ENABLE_DAP)
SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
setbinprops(ocprint)
ENDIF(ENABLE_DAP)
ENDIF()

4
ncdump/Makefile.am
View File

@ -41,6 +41,10 @@ ncvalidator_SOURCES = ncvalidator.c
noinst_PROGRAMS += ncpathcvt
ncpathcvt_SOURCES = ncpathcvt.c
# A non-installed utility program to test if a filter is available
noinst_PROGRAMS += ncfilteravail
ncfilteravail_SOURCES = ncfilteravail.c
# A simple netcdf-4 metadata -> xml printer. Do not install.
if USE_HDF5
bin_PROGRAMS += nc4print

1
ncdump/nccopy.c
View File

@ -31,7 +31,6 @@
#include "nccomps.h"
#include "list.h"
#include "ncpathmgr.h"
#include "ncfilter.h"
#undef DEBUGFILTER
#undef DEBUGCHUNK

4
ncdump/ncdump.1
View File

@ -7,7 +7,7 @@ ncdump \- Convert netCDF file to text form (CDL)
.HP
ncdump
.nh
\%[\-chistxw]
\%[\-chistxwF]
\%[\-v \fIvar1,...\fP]
\%[\-b \fIlang\fP]
\%[\-f \fIlang\fP]
@ -230,6 +230,8 @@ with client-side caching of entire variables.
.IP "\fB-x\fP"
Output XML (NcML) instead of CDL. The NcML does not include data values.
The NcML output option currently only works for netCDF classic model data.
.IP "\fB-F\fP"
Use _Filter and _Codecs attributes in place of _Fletcher32, _Shuffle, and _Deflate.
.SH EXAMPLES
.LP
Look at the structure of the data in the netCDF file `\fBfoo.nc\fP':

93
ncdump/ncdump.c
View File

@ -95,6 +95,7 @@ fspec_t formatting_specs = /* defaults, overridden by command-line options */
false, /* human-readable output for date-time values? */
false, /* use 'T' separator between date and time values as strings? */
false, /* output special attributes, eg chunking? */
false, /* if -F specified */
LANG_C, /* language conventions for indices */
false, /* for DAP URLs, client-side cache used */
0, /* if -v specified, number of variables in list */
@ -124,12 +125,13 @@ usage(void)
[-g grp1[,...]] Data and metadata for group(s) <grp1>,... only\n\
[-w] With client-side caching of variables for DAP URLs\n\
[-x] Output XML (NcML) instead of CDL\n\
[-F] Output _Filter and _Codecs instead of _Fletcher32, _Shuffle, and _Deflate\n\
[-Xp] Unconditionally suppress output of the properties attribute\n\
[-Ln] Set log level to n (>= 0); ignore if logging not enabled.\n\
file Name of netCDF file (or URL if DAP access enabled)\n"
(void) fprintf(stderr,
"%s [-c|-h] [-v ...] [[-b|-f] [c|f]] [-l len] [-n name] [-p n[,n]] [-k] [-x] [-s] [-t|-i] [-g ...] [-w] [-Ln] file\n%s",
"%s [-c|-h] [-v ...] [[-b|-f] [c|f]] [-l len] [-n name] [-p n[,n]] [-k] [-x] [-s] [-t|-i] [-g ...] [-w] [-F] [-Ln] file\n%s",
progname,
USAGE);
@ -1005,48 +1007,73 @@ pr_att_specials(
printf(" = \"unknown\" ;\n");
}
/* _Filter (including deflate and shuffle) */
/* _Checksum (fletcher32) */
if(!formatting_specs.filter_atts) {
int fletcher32 = 0;
NC_CHECK( nc_inq_var_fletcher32(ncid, varid, &fletcher32) );
if(fletcher32 != 0) {
pr_att_name(ncid, varp->name, NC_ATT_CHECKSUM);
printf(" = \"true\" ;\n");
}
}
/* _Shuffle */
if(!formatting_specs.filter_atts) {
int haveshuffle = 0;
NC_CHECK( nc_inq_var_deflate(ncid, varid, &haveshuffle, NULL, NULL));
if(haveshuffle) {
pr_att_name(ncid, varp->name, NC_ATT_SHUFFLE);
printf(" = \"true\" ;\n");
}
}
/* _Deflate*/
if(!formatting_specs.filter_atts) {
int havedeflate = 0;
int level = -1;
NC_CHECK( nc_inq_var_deflate(ncid, varid, NULL, &havedeflate, &level));
if(havedeflate) {
pr_att_name(ncid, varp->name, NC_ATT_DEFLATE);
printf(" = %d ;\n",level);
}
}
/* _Filter */
{
size_t nparams, nfilters, nbytes;
int shuffle=NC_NOSHUFFLE;
unsigned int* filterids = NULL;
unsigned int* params = NULL;
int usedeflateatt = 0;
/* Get applicable filter ids */
NC_CHECK(nc_inq_var_filter_ids(ncid, varid, &nfilters, NULL));
/* Get set of filters for this variable */
filterids = NULL;
if(nfilters > 0) {
filterids = (unsigned int*)malloc(sizeof(unsigned int)*nfilters);
if(filterids == NULL) NC_CHECK(NC_ENOMEM);
} else
filterids = NULL;
NC_CHECK(nc_inq_var_filter_ids(ncid, varid, &nfilters, filterids));
NC_CHECK(nc_inq_var_filter_ids(ncid, varid, &nfilters, filterids));
}
if(nfilters > 0) {
int k;
int pratt = 0;
int first = 1;
int _filter = 0;
for(k=0;k<nfilters;k++) {
NC_CHECK(nc_inq_var_filter_info(ncid, varid, filterids[k], &nparams, NULL));
if(!formatting_specs.filter_atts && (
filterids[k] == H5Z_FILTER_FLETCHER32
|| filterids[k] == H5Z_FILTER_SHUFFLE
|| filterids[k] == H5Z_FILTER_DEFLATE))
continue; /* Ignore fletcher32 and shuffle and deflate*/
_filter = 1;
if(nparams > 0) {
params = (unsigned int*)calloc(1,sizeof(unsigned int)*nparams);
NC_CHECK(nc_inq_var_filter_info(ncid, varid, filterids[k], &nbytes, params));
} else
params = NULL;
/* Use _Deflate if the first filter is zip */
if(k == 0 && filterids[k] == H5Z_FILTER_DEFLATE) {
pr_att_name(ncid, varp->name, NC_ATT_DEFLATE);
printf(" = %d", (int)params[0]);
pratt = 1;
usedeflateatt = 1;
nullfree(params); params = NULL;
continue;
}
if(pratt || k == 0) {
if(first) {
pr_att_name(ncid,varp->name,NC_ATT_FILTER);
printf(" = \"");
pratt = 0;
}
if(k > 0) printf("|");
} else
printf("|");
printf("%u",filterids[k]);
if(nparams > 0) {
int i;
@ -1054,17 +1081,11 @@ pr_att_specials(
printf(",%u",params[i]);
}
nullfree(params); params = NULL;
first = 0;
}
if(!usedeflateatt) printf("\"");
printf(" ;\n");
if(_filter) printf("\" ;\n");
}
if(filterids) free(filterids);
/* Finally, do Shuffle */
NC_CHECK( nc_inq_var_deflate(ncid, varid, &shuffle, NULL, NULL));
if(shuffle != NC_NOSHUFFLE) {
pr_att_name(ncid, varp->name, NC_ATT_SHUFFLE);
printf(" = \"true\" ;\n");
}
}
/* _Codecs*/
{
@ -1088,15 +1109,6 @@ pr_att_specials(
}
}
}
/* _Checksum */
{
int fletcher32 = 0;
NC_CHECK( nc_inq_var_fletcher32(ncid, varid, &fletcher32) );
if(fletcher32 != 0) {
pr_att_name(ncid, varp->name, NC_ATT_CHECKSUM);
printf(" = \"true\" ;\n");
}
}
/* _Endianness */
if(varp->tinfo->size > 1) /* Endianness is meaningless for 1-byte types */
{
@ -2261,7 +2273,7 @@ main(int argc, char *argv[])
}
opterr = 1;
while ((c = getopt(argc, argv, "b:cd:f:g:hikl:n:p:stv:xwKL:X:")) != EOF)
while ((c = getopt(argc, argv, "b:cd:f:g:hikl:n:p:stv:xwFKL:X:")) != EOF)
switch(c) {
case 'h': /* dump header only, no data */
formatting_specs.header_only = true;
@ -2375,6 +2387,9 @@ main(int argc, char *argv[])
#endif
ncsetlogging(1);
break;
case 'F':
formatting_specs.filter_atts = true;
break;
case '?':
usage();
exit(EXIT_FAILURE);

3
ncdump/ncdump.h
View File

@ -45,6 +45,9 @@ typedef struct { /* specification for how to format dump */
* _Endianness, _Format, _Checksum,
* _NoFill, _NetCDF4 */
bool_t filter_atts; /* if true, output filters using _Filter in place of
* _Fletcher32, _Shuffle, and _Deflate. */
Nclang data_lang; /* Specifies index conventions used in data
* comments, either LANG_C (C, 0-based,
* column major) or LANG_F (Fortran,

178
ncdump/ncfilteravail.c Normal file
View File

@ -0,0 +1,178 @@
/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#if defined(_WIN32) && !defined(__MINGW32__)
#include "XGetopt.h"
#else
#include <getopt.h>
#endif
#include "netcdf.h"
#include "ncpathmgr.h"
#include "netcdf_filter.h"
static const char* USAGE =
"ncfilteravail [-t <format>] <filtername>|<filterid> n"
"Options\n"
" -h help"
" -t the formatx name: nc4|nczarr (optional)\n"
"\n"
" <filtername> the filter name \n"
" <filterid> the filter id number \n"
"\n"
"Default format is nc4 if built, then nczarr if built else error.\n"
;
#undef DEBUG
struct Options {
char* format;
char* filtername;
unsigned filterid;
char* url;
} faoptions;
static const struct Filter {
char* name;
unsigned id;
} known_filters[] = {
{"deflate",1},
{"szip",4},
{"bzip2",307},
{"zstd",32015},
{"zstandard",32015},
{"blosc",32001},
{NULL,0}
};
static void usage(const char* msg);
static void
usage(const char* msg)
{
if(msg != NULL) fprintf(stderr,"%s\n",msg);
fprintf(stderr,"%s",USAGE);
if(msg == NULL) exit(0); else exit(1);
}
static int
getformat(const char* sformat)
{
if(sformat != NULL && strlen(sformat) > 0) {
if(strcmp(sformat,"nc4")==0) return NC_FORMATX_NC4;
if(strcmp(sformat,"nczarr")==0) return NC_FORMATX_NCZARR;
}
#ifdef USE_HDF5
return NC_FORMATX_NC4;
#else
# ifdef ENABLE_NCZARR
return NC_FORMATX_NCZARR;
# endif
#endif
return NC_FORMATX_UNDEFINED;
}
static int
makeurl(struct Options* faopt)
{
int stat = NC_NOERR;
int format;
char url[4096];
int pid;
#if defined(_WIN32) || defined(__MINGW32__)
pid = (int)_getpid();
#else
pid = (int)getpid();
#endif
switch (format = getformat(faopt->format)) {
case NC_FORMATX_NC4:
snprintf(url,sizeof(url),"tmp_%d.nc",pid);
break;
case NC_FORMATX_NCZARR:
snprintf(url,sizeof(url),"file://tmp_%d.file#mode=zarr,file",pid);
break;
default: stat = NC_EINVAL; break;
}
if(stat == NC_NOERR) faopt->url = strdup(url);
return stat;
}
static int
makefilterid(struct Options* faopt)
{
int stat = NC_NOERR;
const struct Filter* f = NULL;
for(f=known_filters;f->name;f++) {
if(strcmp(f->name,faopt->filtername)==0) {
faopt->filterid = f->id;
goto done;
}
}
/* See if it is a number */
if(1!=sscanf(faopt->filtername,"%u",&faopt->filterid))
{stat = NC_EINVAL; goto done;}
done:
return stat;
}
int
main(int argc, char** argv)
{
int c, ncid, stat = NC_NOERR;
int yesno = 0;
memset((void*)&faoptions,0,sizeof(faoptions));
while ((c = getopt(argc, argv, "t:")) != EOF) {
switch(c) {
case 'h': usage(""); break;
case 'x': faoptions.format = strdup(optarg); break;
case '?':
usage("unknown option");
break;
}
}
argc -= optind;
argv += optind;
/* If no filter arguments left or more than one, print usage message. */
if (argc == 0)
usage("no filter specified");
if (argc > 1)
usage("more than one filter specified");
faoptions.filtername = strdup(argv[0]);
if(makeurl(&faoptions)) {yesno = 0; goto done;}
if(makefilterid(&faoptions)) {yesno = 0; goto done;}
if((stat = nc_create(faoptions.url,NC_NETCDF4,&ncid))) goto done;
switch(stat = nc_inq_filter_avail(ncid,faoptions.filterid)) {
case NC_NOERR: yesno = 1; break;
case NC_ENOFILTER: yesno = 0; stat = NC_NOERR; break;
default: break;
}
if((stat = nc_abort(ncid))) goto done;
done:
printf("%s",(yesno?"yes":"no"));
if(stat) fprintf(stderr,"error: (%d) %s\n",stat,nc_strerror(stat));
return (stat?1:0);
}

10
ncdump/ref_tst_ncf213.cdl
View File

@ -22,25 +22,23 @@ variables:
int var2(dim1, dim2) ;
var2:_Storage = "chunked" ;
var2:_ChunkSizes = 6, 7 ;
var2:_Fletcher32 = "true" ;
var2:_Filter = "3" ;
var2:_Endianness = "big" ;
int var3(dim1, dim2, dim3) ;
var3:_Storage = "chunked" ;
var3:_ChunkSizes = 6, 7, 8 ;
var3:_DeflateLevel = 2 ;
var3:_Filter = "1,2" ;
var3:_Endianness = "little" ;
int var4(dim1, dim2, dim3) ;
var4:_Storage = "chunked" ;
var4:_ChunkSizes = 6, 7, 8 ;
var4:_DeflateLevel = 2 ;
var4:_Filter = "1,2" ;
var4:_Endianness = "little" ;
var4:_NoFill = "true" ;
obs_t var5(dim1) ;
var5:_Storage = "chunked" ;
var5:_ChunkSizes = 6 ;
var5:_DeflateLevel = 2 ;
var5:_Shuffle = "true" ;
var5:_Fletcher32 = "true" ;
var5:_Filter = "3|2,40|1,2" ;
var5:_NoFill = "true" ;
// global attributes:

6
ncdump/ref_tst_special_atts.cdl
View File

@ -32,16 +32,16 @@ variables:
int var4(dim1, dim2, dim3) ;
var4:_Storage = "chunked" ;
var4:_ChunkSizes = 6, 7, 8 ;
var4:_DeflateLevel = 2 ;
var4:_Shuffle = "true" ;
var4:_DeflateLevel = 2 ;
var4:_Endianness = "little" ;
var4:_NoFill = "true" ;
obs_t var5(dim1) ;
var5:_Storage = "chunked" ;
var5:_ChunkSizes = 6 ;
var5:_DeflateLevel = 2 ;
var5:_Shuffle = "true" ;
var5:_Fletcher32 = "true" ;
var5:_Shuffle = "true" ;
var5:_DeflateLevel = 2 ;
int var6(dim1) ;
var6:_Storage = "compact" ;
var6:_Endianness = "little" ;

3
ncdump/tst_nccopy3.sh
View File

@ -8,9 +8,6 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
export SETX=1
set -e
echo ""

1
ncdump/tst_nccopy4.sh
View File

@ -7,7 +7,6 @@ set -e
# For a netCDF-4 build, test nccopy on netCDF files in this directory
echo "@@@@@@"
if test -f tst_group_data${ext} ; then ${execdir}/tst_group_data ; fi
if test -f tst_enum_data${ext} ; then ${execdir}/tst_enum_data ; fi
if test -f tst_comp${ext} ; then ${execdir}/tst_comp ; fi

2
ncdump/tst_netcdf4_4.sh
View File

@ -68,7 +68,7 @@ diff -b tst_compounds4.cdl ${top_srcdir}/ncdump/ref_tst_compounds4.cdl
# rm -f tst_ncf213.cdl tst_ncf213.nc
# Remove specific _NCProperties values
${NCGEN} -b -o tst_ncf213.nc $srcdir/ref_tst_ncf213.cdl
${NCDUMP} -s -h tst_ncf213.nc > tst_ncf213.cdl
${NCDUMP} -shF tst_ncf213.nc > tst_ncf213.cdl
cleanncprops tst_ncf213.cdl tst_ncf213.tmp
cleanncprops ${srcdir}/ref_tst_ncf213.cdl ref_tst_ncf213.tmp
# Now compare

6
ncdump/utils.h
View File

@ -18,7 +18,11 @@
#define NC_GRP_DELIM '/'
typedef int bool_t;
enum {false=0, true=1};
#ifndef false
#define false 0
#define true 1
//enum {false=0, true=1};
#endif
struct safebuf_t;
/* Buffer structure for implementing growable strings, used in

2
ncgen/Makefile.am
View File

@ -61,7 +61,7 @@ CLEANFILES = c0.nc c0_64.nc c0_4.nc c0_4c.nc ref_camrun.c \
makeparser::
rm -f ncgenl.c lex.ncg.c
flex -Pncg -8 ncgen.l
flex -L -Pncg -8 ncgen.l
sed -e s/lex.ncg.c/ncgenl.c/g <lex.ncg.c >ncgenl.c
bison -pncg -t -d ncgen.y
rm -f ncgeny.c ncgeny.h

10
ncgen/genbin.c
View File

@ -7,6 +7,7 @@
#include "includes.h"
#include <ctype.h> /* for isprint() */
#include "netcdf_aux.h"
#include "netcdf_filter.h"
#ifdef ENABLE_BINARY
@ -248,6 +249,15 @@ genbin_definespecialattributes(Symbol* var)
int k;
for(k=0;k<special->nfilters;k++) {
NC_H5_Filterspec* nfs = special->_Filters[k];
/* See if the filter is available */
stat = nc_inq_filter_avail(var->container->nc_id, nfs->filterid);
switch (stat) {
case NC_NOERR: break;
case NC_ENOFILTER:
derror("Filter id=%u; filter not available",nfs->filterid);
/* fall thru */
default: CHECK_ERR(stat); break;
}
stat = nc_def_var_filter(var->container->nc_id,
var->nc_id,
nfs->filterid,

69
ncgen/ncgenl.c
View File

@ -1,6 +1,4 @@
#line 3 "ncgenl.c"
#define YY_INT_ALIGNED short int
/* A lexical scanner generated by flex */
@ -1533,8 +1531,6 @@ int yy_flex_debug = 0;
#define YY_MORE_ADJ 0
#define YY_RESTORE_YY_MORE_OFFSET
char *yytext;
#line 1 "ncgen.l"
#line 2 "ncgen.l"
/*********************************************************************
* Copyright 1993, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
@ -1679,8 +1675,6 @@ struct Specialtoken specials[] = {
{NULL,0} /* null terminate */
};
#line 1683 "ncgenl.c"
/* The most correct (validating) version of UTF8 character set
(Taken from: http://www.w3.org/2005/03/23-lex-U)
@ -1722,7 +1716,6 @@ ID ([A-Za-z_]|{UTF8})([A-Z.@#\[\]a-z_0-9+-]|{UTF8})*
/* Note: this definition of string will work for utf8 as well,
although it is a very relaxed definition
*/
#line 1726 "ncgenl.c"
#define INITIAL 0
#define ST_C_COMMENT 1
@ -1941,9 +1934,6 @@ YY_DECL
}
{
#line 225 "ncgen.l"
#line 1947 "ncgenl.c"
while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */
{
@ -2002,14 +1992,12 @@ do_action: /* This label is used only to access EOF actions. */
case 1:
YY_RULE_SETUP
#line 226 "ncgen.l"
{ /* whitespace */
break;
}
YY_BREAK
case 2:
YY_RULE_SETUP
#line 230 "ncgen.l"
{ /* comment */
break;
}
@ -2017,7 +2005,6 @@ YY_RULE_SETUP
case 3:
/* rule 3 can match eol */
YY_RULE_SETUP
#line 234 "ncgen.l"
{int len; char* s = NULL;
/* In netcdf4, this will be used in a variety
of places, so only remove escapes */
@ -2041,7 +2028,6 @@ yytext[MAXTRST-1] = '\0';
YY_BREAK
case 4:
YY_RULE_SETUP
#line 255 "ncgen.l"
{ /* drop leading 0x; pad to even number of chars */
char* p = yytext+2;
int len = yyleng - 2;
@ -2056,143 +2042,115 @@ YY_RULE_SETUP
YY_BREAK
case 5:
YY_RULE_SETUP
#line 267 "ncgen.l"
{return lexdebug(COMPOUND);}
YY_BREAK
case 6:
YY_RULE_SETUP
#line 268 "ncgen.l"
{return lexdebug(ENUM);}
YY_BREAK
case 7:
YY_RULE_SETUP
#line 269 "ncgen.l"
{return lexdebug(OPAQUE_);}
YY_BREAK
case 8:
YY_RULE_SETUP
#line 271 "ncgen.l"
{return lexdebug(FLOAT_K);}
YY_BREAK
case 9:
YY_RULE_SETUP
#line 272 "ncgen.l"
{return lexdebug(DOUBLE_K);}
YY_BREAK
case 10:
YY_RULE_SETUP
#line 273 "ncgen.l"
{return lexdebug(CHAR_K);}
YY_BREAK
case 11:
YY_RULE_SETUP
#line 274 "ncgen.l"
{return lexdebug(BYTE_K);}
YY_BREAK
case 12:
YY_RULE_SETUP
#line 275 "ncgen.l"
{return lexdebug(SHORT_K);}
YY_BREAK
case 13:
YY_RULE_SETUP
#line 276 "ncgen.l"
{return lexdebug(INT_K);}
YY_BREAK
case 14:
YY_RULE_SETUP
#line 277 "ncgen.l"
{return lexdebug(identcheck(UBYTE_K));}
YY_BREAK
case 15:
YY_RULE_SETUP
#line 278 "ncgen.l"
{return lexdebug(identcheck(USHORT_K));}
YY_BREAK
case 16:
YY_RULE_SETUP
#line 279 "ncgen.l"
{return lexdebug(identcheck(UINT_K));}
YY_BREAK
case 17:
YY_RULE_SETUP
#line 280 "ncgen.l"
{return lexdebug(identcheck(INT64_K));}
YY_BREAK
case 18:
YY_RULE_SETUP
#line 281 "ncgen.l"
{return lexdebug(identcheck(UINT64_K));}
YY_BREAK
case 19:
YY_RULE_SETUP
#line 282 "ncgen.l"
{return lexdebug(identcheck(STRING_K));}
YY_BREAK
case 20:
YY_RULE_SETUP
#line 284 "ncgen.l"
{return lexdebug(FLOAT_K);}
YY_BREAK
case 21:
YY_RULE_SETUP
#line 285 "ncgen.l"
{return lexdebug(INT_K);}
YY_BREAK
case 22:
YY_RULE_SETUP
#line 286 "ncgen.l"
{return lexdebug(INT_K);}
YY_BREAK
case 23:
YY_RULE_SETUP
#line 287 "ncgen.l"
{return lexdebug(identcheck(UINT_K));}
YY_BREAK
case 24:
YY_RULE_SETUP
#line 288 "ncgen.l"
{return lexdebug(identcheck(UINT_K));}
YY_BREAK
case 25:
YY_RULE_SETUP
#line 291 "ncgen.l"
{int32_val = -1;
return lexdebug(NC_UNLIMITED_K);}
YY_BREAK
case 26:
YY_RULE_SETUP
#line 294 "ncgen.l"
{return lexdebug(TYPES);}
YY_BREAK
case 27:
YY_RULE_SETUP
#line 295 "ncgen.l"
{return lexdebug(DIMENSIONS);}
YY_BREAK
case 28:
YY_RULE_SETUP
#line 296 "ncgen.l"
{return lexdebug(VARIABLES);}
YY_BREAK
case 29:
YY_RULE_SETUP
#line 297 "ncgen.l"
{return lexdebug(DATA);}
YY_BREAK
case 30:
YY_RULE_SETUP
#line 298 "ncgen.l"
{return lexdebug(GROUP);}
YY_BREAK
case 31:
YY_RULE_SETUP
#line 300 "ncgen.l"
{BEGIN(TEXT);return lexdebug(NETCDF);}
YY_BREAK
case 32:
YY_RULE_SETUP
#line 302 "ncgen.l"
{ /* missing value (pre-2.4 backward compatibility) */
if (yytext[0] == '-') {
double_val = -INFINITY;
@ -2205,7 +2163,6 @@ YY_RULE_SETUP
YY_BREAK
case 33:
YY_RULE_SETUP
#line 311 "ncgen.l"
{ /* missing value (pre-2.4 backward compatibility) */
double_val = NAN;
specialconstants = 1;
@ -2214,7 +2171,6 @@ YY_RULE_SETUP
YY_BREAK
case 34:
YY_RULE_SETUP
#line 317 "ncgen.l"
{/* missing value (pre-2.4 backward compatibility)*/
if (yytext[0] == '-') {
float_val = -INFINITYF;
@ -2227,7 +2183,6 @@ YY_RULE_SETUP
YY_BREAK
case 35:
YY_RULE_SETUP
#line 326 "ncgen.l"
{ /* missing value (pre-2.4 backward compatibility) */
float_val = NANF;
specialconstants = 1;
@ -2236,7 +2191,6 @@ YY_RULE_SETUP
YY_BREAK
case 36:
YY_RULE_SETUP
#line 332 "ncgen.l"
{
#ifdef USE_NETCDF4
if(l_flag == L_C || l_flag == L_BINARY)
@ -2249,7 +2203,6 @@ YY_RULE_SETUP
YY_BREAK
case 37:
YY_RULE_SETUP
#line 342 "ncgen.l"
{
bbClear(lextext);
bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */
@ -2260,7 +2213,6 @@ YY_RULE_SETUP
YY_BREAK
case 38:
YY_RULE_SETUP
#line 351 "ncgen.l"
{struct Specialtoken* st;
bbClear(lextext);
bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */
@ -2274,7 +2226,6 @@ YY_RULE_SETUP
case 39:
/* rule 39 can match eol */
YY_RULE_SETUP
#line 361 "ncgen.l"
{
int c;
char* p; char* q;
@ -2294,7 +2245,6 @@ YY_RULE_SETUP
YY_BREAK
case 40:
YY_RULE_SETUP
#line 378 "ncgen.l"
{ char* id = NULL; int len;
len = strlen(yytext);
len = unescape(yytext,len,ISIDENT,&id);
@ -2309,7 +2259,6 @@ YY_RULE_SETUP
YY_BREAK
case 41:
YY_RULE_SETUP
#line 390 "ncgen.l"
{
/*
We need to try to see what size of integer ((u)int).
@ -2390,7 +2339,6 @@ done: return 0;
YY_BREAK
case 42:
YY_RULE_SETUP
#line 468 "ncgen.l"
{
int c;
int token = 0;
@ -2441,7 +2389,6 @@ YY_RULE_SETUP
YY_BREAK
case 43:
YY_RULE_SETUP
#line 515 "ncgen.l"
{
if (sscanf((char*)yytext, "%le", &double_val) != 1) {
sprintf(errstr,"bad long or double constant: %s",(char*)yytext);
@ -2452,7 +2399,6 @@ YY_RULE_SETUP
YY_BREAK
case 44:
YY_RULE_SETUP
#line 522 "ncgen.l"
{
if (sscanf((char*)yytext, "%e", &float_val) != 1) {
sprintf(errstr,"bad float constant: %s",(char*)yytext);
@ -2464,7 +2410,6 @@ YY_RULE_SETUP
case 45:
/* rule 45 can match eol */
YY_RULE_SETUP
#line 529 "ncgen.l"
{
(void) sscanf((char*)&yytext[1],"%c",&byte_val);
return lexdebug(BYTE_CONST);
@ -2472,7 +2417,6 @@ YY_RULE_SETUP
YY_BREAK
case 46:
YY_RULE_SETUP
#line 533 "ncgen.l"
{
int oct = unescapeoct(&yytext[2]);
if(oct < 0) {
@ -2485,7 +2429,6 @@ YY_RULE_SETUP
YY_BREAK
case 47:
YY_RULE_SETUP
#line 542 "ncgen.l"
{
int hex = unescapehex(&yytext[3]);
if(byte_val < 0) {
@ -2498,7 +2441,6 @@ YY_RULE_SETUP
YY_BREAK
case 48:
YY_RULE_SETUP
#line 551 "ncgen.l"
{
switch ((char)yytext[2]) {
case 'a': byte_val = '\007'; break; /* not everyone under-
@ -2520,7 +2462,6 @@ YY_RULE_SETUP
case 49:
/* rule 49 can match eol */
YY_RULE_SETUP
#line 569 "ncgen.l"
{
lineno++ ;
break;
@ -2528,7 +2469,6 @@ YY_RULE_SETUP
YY_BREAK
case 50:
YY_RULE_SETUP
#line 574 "ncgen.l"
{/*initial*/
BEGIN(ST_C_COMMENT);
break;
@ -2537,21 +2477,18 @@ YY_RULE_SETUP
case 51:
/* rule 51 can match eol */
YY_RULE_SETUP
#line 579 "ncgen.l"
{/* continuation */
break;
}
YY_BREAK
case 52:
YY_RULE_SETUP
#line 583 "ncgen.l"
{/* final */
BEGIN(INITIAL);
break;
}
YY_BREAK
case YY_STATE_EOF(ST_C_COMMENT):
#line 588 "ncgen.l"
{/* final, error */
fprintf(stderr,"unterminated /**/ comment");
BEGIN(INITIAL);
@ -2560,17 +2497,14 @@ case YY_STATE_EOF(ST_C_COMMENT):
YY_BREAK
case 53:
YY_RULE_SETUP
#line 594 "ncgen.l"
{/* Note: this next rule will not work for UTF8 characters */
return lexdebug(yytext[0]) ;
}
YY_BREAK
case 54:
YY_RULE_SETUP
#line 597 "ncgen.l"
ECHO;
YY_BREAK
#line 2574 "ncgenl.c"
case YY_STATE_EOF(INITIAL):
case YY_STATE_EOF(TEXT):
yyterminate();
@ -3576,8 +3510,6 @@ void yyfree (void * ptr )
#define YYTABLES_NAME "yytables"
#line 597 "ncgen.l"
static int
lexdebug(int token)
{
@ -3676,7 +3608,6 @@ parseULL(int radix, char* text, int* failp)
return uint64;
}
/**
Given the raw bits, the sign char, the tag, and hasU
fill in the appropriate *_val field

19
nczarr_test/CMakeLists.txt
View File

@ -13,6 +13,19 @@ FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_quantize.c QSOURCE)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test_quantize.c "#define TESTNCZARR\n")
FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/test_quantize.c "${QSOURCE}")
FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_filter_avail.c ASOURCE)
STRING(PREPEND ASOURCE "#define TESTNCZARR\n")
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test_filter_avail.c "${ASOURCE}")
FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_specific_filters.sh SPSOURCE)
STRING(PREPEND SPSOURCE "#!/bin/bash\n")
STRING(PREPEND SPSOURCE "TESTNCZARR=1\n")
FILE(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.1 "${SPSOURCE}")
CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.1 ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh @ONLY NEWLINE_STYLE LF)
FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR} FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE)
FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.1)
FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh)
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh
${CMAKE_CURRENT_SOURCE_DIR}/ref*.cdl
${CMAKE_CURRENT_SOURCE_DIR}/ref*.txt)
@ -104,16 +117,16 @@ IF(ENABLE_TESTS)
ENDIF()
IF(ENABLE_FILTER_TESTING)
add_bin_test(nczarr_test tst_nczfilter)
build_bin_test(tst_nczfilter)
build_bin_test(testfilter)
build_bin_test(testfilter_misc)
build_bin_test(testfilter_multi)
build_bin_test(testfilter_order)
build_bin_test(testfilter_repeat)
build_bin_test(test_filter_avail)
ADD_SH_TEST(nczarr_test run_nczfilter)
ADD_SH_TEST(nczarr_test run_filter)
IF(ENABLE_BLOSC)
ADD_SH_TEST(nczarr_test run_specific_filters)
ENDIF()
ENDIF(ENABLE_FILTER_TESTING)
if(ENABLE_NCZARR_ZIP)
add_sh_test(nczarr_test run_newformat)

28
nczarr_test/Makefile.am
View File

@ -35,7 +35,7 @@ ut_projections_SOURCES = ut_projections.c ${commonsrc}
ut_chunking_SOURCES = ut_chunking.c ${commonsrc}
tst_fillonlyz_SOURCES = tst_fillonlyz.c ${tstcommonsrc}
check_PROGRAMS += tst_zchunks tst_zchunks2 tst_zchunks3 tst_fillonlyz test_quantize
check_PROGRAMS += tst_zchunks tst_zchunks2 tst_zchunks3 tst_fillonlyz test_quantize
TESTS += run_ut_map.sh
TESTS += run_ut_mapapi.sh
@ -86,16 +86,15 @@ endif
endif # BUILD_BENCHMARKS
if ENABLE_FILTER_TESTING
# Do simple filter functionality tests for nczarr
check_PROGRAMS += tst_nczfilter
TESTS += tst_nczfilter
TESTS += run_nczfilter.sh
# Echo filter tests from nc_test4
check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi
check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi test_filter_avail
TESTS += run_filter.sh
if ENABLE_BLOSC
TESTS += run_specific_filters.sh
endif
endif #ENABLE_FILTER_TESTING
@ -155,19 +154,32 @@ EXTRA_DIST += ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes
CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl
BUILT_SOURCES = test_quantize.c
BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh
test_quantize.c: ../nc_test4/tst_quantize.c
rm -f test_quantize.c
echo "#define TESTNCZARR" > test_quantize.c
cat ../nc_test4/tst_quantize.c >> test_quantize.c
test_filter_avail.c: ../nc_test4/tst_filter_avail.c
rm -f test_filter_avail.c
echo "#define TESTNCZARR" > test_filter_avail.c
cat ../nc_test4/tst_filter_avail.c >> test_filter_avail.c
run_specific_filters.sh: ../nc_test4/tst_specific_filters.sh
rm -f run_specific_filters.sh run_specific_filters.tmp
echo "#!/bin/bash" > run_specific_filters.tmp
echo "TESTNCZARR=1" >> run_specific_filters.tmp
cat ../nc_test4/tst_specific_filters.sh >> run_specific_filters.tmp
tr -d '\r' < run_specific_filters.tmp > run_specific_filters.sh
chmod a+x run_specific_filters.sh
rm -f run_specific_filters.tmp
# Remove directories
clean-local:
rm -fr tmp*.file results.file results.s3 results.zip
rm -fr rcmiscdir
DISTCLEANFILES = findplugin.sh test_quantize.c
DISTCLEANFILES = findplugin.sh test_quantize.c run_specific_filters.sh
# If valgrind is present, add valgrind targets.
@VALGRIND_CHECK_RULES@

25
nczarr_test/ncdumpchunks.c
View File

@ -229,12 +229,14 @@ static void
printchunk(Format* format, int* chunkdata, size_t indent)
{
int k[3];
int rank = format->rank;
unsigned cols[3], pos;
size_t* chl = format->chunklens;
memset(cols,0,sizeof(cols));
switch (format->rank) {
if(format->xtype == NC_UBYTE) rank = 0;
switch (rank) {
case 1:
cols[0] = 1;
cols[1] = 1;
@ -265,7 +267,15 @@ printchunk(Format* format, int* chunkdata, size_t indent)
k[2] = 0;
if(k[1] > 0) printf(" |");
for(k[2]=0;k[2]<cols[2];k[2]++) {
printf(" %02d", chunkdata[pos]);
if(format->xtype == NC_UBYTE) {
int l;
unsigned char* bchunkdata = (unsigned char*)(&chunkdata[pos]);
for(l=0;l<sizeof(int);l++) {
printf(" %02u", bchunkdata[l]);
}
} else {
printf(" %02d", chunkdata[pos]);
}
pos++;
}
}
@ -285,7 +295,7 @@ printchunk(Format* format, int* chunkdata, size_t indent)
int
dump(Format* format)
{
int* chunkdata = NULL; /*[CHUNKPROD];*/
void* chunkdata = NULL; /*[CHUNKPROD];*/
Odometer* odom = NULL;
int r;
size_t offset[NC_MAX_VAR_DIMS];
@ -458,8 +468,14 @@ main(int argc, char** argv)
memset(&format,0,sizeof(format));
while ((c = getopt(argc, argv, "v:DT:")) != EOF) {
/* Init some format fields */
format.xtype = NC_INT;
while ((c = getopt(argc, argv, "bv:DT:")) != EOF) {
switch(c) {
case 'b':
format.xtype = NC_UBYTE;
break;
case 'v':
strcpy(format.var_name,optarg);
break;
@ -512,7 +528,6 @@ main(int argc, char** argv)
if((stat=nc_inq_var_chunking(ncid,varid,&storage,format.chunklens))) usage(stat);
if(storage != NC_CHUNKED) usage(NC_EBADCHUNK);
if((stat=nc_get_att(ncid,varid,"_FillValue",&format.fillvalue))) usage(stat);
format.xtype = NC_INT;
for(i=0;i<format.rank;i++) {
if((stat=nc_inq_dimlen(ncid,dimids[i],&format.dimlens[i]))) usage(stat);

32
nczarr_test/run_filter.sh
View File

@ -48,21 +48,25 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
}
# Locate the plugin path and the library names; argument order is critical
# Find bzip2 and capture
findplugin h5bzip2
BZIP2LIB="${HDF5_PLUGIN_LIB}"
BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}"
# Find misc and capture
findplugin h5misc
MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
# Find noop and capture
findplugin h5noop
NOOPLIB="${HDF5_PLUGIN_LIB}"
NOOPPATH="${HDF5_PLUGIN_PATH}/${NOOPLIB}"
# Find bzip2 and capture
findplugin h5bzip2
BZIP2LIB="${HDF5_PLUGIN_LIB}"
BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}"
# Verify
if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi
echo "@@@@@@@@@@@"
find ${HDF5_PLUGIN_PATH}
# Execute the specified tests
testapi() {
@ -133,18 +137,18 @@ zext=$1
echo "*** Testing access to filter info when filter implementation is not available for map $zext"
fileargs tmp_known
deletemap $zext $file
# build bzip2.nc
${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/bzip2.cdl
# dump and clean bzip2.nc header when filter is avail
# build noop.nc
${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/noop.cdl
# dump and clean noop.nc header when filter is avail
${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt
# Remove irrelevant -s output
sclean ./tmp_known_$zext.txt tmp_known_$zext.dump
# Now hide the filter code
mv ${BZIP2PATH} ./${BZIP2LIB}.save
# dump and clean bzip2.nc header when filter is not avail
mv ${NOOPPATH} ./${NOOPLIB}.save
# dump and clean noop.nc header when filter is not avail
${NCDUMP} -hs $fileurl > ./tmp_unk_$zext.txt
# Restore the filter code
mv ./${BZIP2LIB}.save ${BZIP2PATH}
mv ./${NOOPLIB}.save ${NOOPPATH}
# Verify that the filter is no longer defined
UNK=`sed -e '/var:_Filter/p' -e d ./tmp_unk_$zext.txt`
test "x$UNK" = x
@ -167,7 +171,7 @@ echo "*** Testing multiple filters for map $zext"
fileargs tmp_multi
deletemap $zext $file
${execdir}/testfilter_multi $fileurl
${NCDUMP} -hs -n multifilter $fileurl >./tmp_multi_$zext.cdl
${NCDUMP} -hsF -n multifilter $fileurl >./tmp_multi_$zext.cdl
# Remove irrelevant -s output
sclean ./tmp_multi_$zext.cdl ./tmp_smulti_$zext.cdl
diff -b -w ${srcdir}/ref_multi.cdl ./tmp_smulti_$zext.cdl

14
nczarr_test/run_nczfilter.sh Executable file
View File

@ -0,0 +1,14 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. "$srcdir/test_nczarr.sh"
# This shell script runs tst_nczfilter.c
set -e
pwd
${execdir}/tst_nczfilter

4
nczarr_test/run_quantize.sh
View File

@ -22,5 +22,5 @@ testcase() {
}
testcase file
#if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi
#if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi

161
nczarr_test/run_specific_filters.sh
View File

@ -1,161 +0,0 @@
#!/bin/bash
# Test the implementations of specific filters
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. "$srcdir/test_nczarr.sh"
set -e
# Function to remove selected -s attributes from file;
# These attributes might be platform dependent
sclean() {
cat $1 \
| sed -e '/:_IsNetcdf4/d' \
| sed -e '/:_Endianness/d' \
| sed -e '/_NCProperties/d' \
| sed -e '/_SuperblockVersion/d' \
| sed -e '/_Format/d' \
| sed -e '/global attributes:/d' \
| cat > $2
}
# Function to extract _Filter attribute from a file
# These attributes might be platform dependent
getfilterattr() {
V="$1"
sed -e '/${V}.*:_Filter/p' -ed <$2 >$3
}
# Function to extract _Codecs attribute from a file
# These attributes might be platform dependent
getcodecsattr() {
V="$1"
sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3
}
trimleft() {
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
}
setfilter() {
FF="$1"
FSRC="$2"
FDST="$3"
FIH5="$4"
FICX="$5"
FFH5="$6"
FFCX="$7"
if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi
if test "x$FFCX" = x ; then FFCX="$FICX" ; fi
rm -f $FDST
cat ${srcdir}/$FSRC \
| sed -e "s/ref_any/${FF}/" \
| sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \
| sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \
| sed -e 's/"/\\"/g' -e 's/@/"/g' \
| cat > $FDST
}
# Execute the specified tests
runfilter() {
zext="$1"
zfilt="$2"
zparams="$3"
zcodec="$4"
echo "*** Testing processing of filter $zfilt for map $zext"
deletemap $zext "tmp_${zfilt}"
fileargs "tmp_${zfilt}"
setfilter $zfilt ref_any.cdl "tmp_${zfilt}.cdl" "$zparams" "$zcodec"
if ${NCGEN} -4 -lb -o $fileurl "tmp_${zfilt}.cdl" ; then
${NCDUMP} -n $zfilt -s $fileurl > "tmp_${zfilt}.tmp"
sclean "tmp_${zfilt}.tmp" "tmp_${zfilt}.dump"
fi
}
testfletcher32() {
zext=$1
runfilter $zext fletcher32 '3' '[{\"id\": \"fletcher32\"}]'
if test -f "tmp_fletcher32.dump" ; then
# need to remove _Filter
sed -e '/_Fletcher32 = "true"/d' < tmp_fletcher32.dump > tmp_fletcher32x.dump
diff -b -w "tmp_fletcher32.cdl" "tmp_fletcher32x.dump"
else
echo "XFAIL: filter=fletcher32 zext=$zext"
fi
}
testshuffle() {
zext=$1
runfilter $zext shuffle '2' '[{\"id\": \"shuffle\",\"elementsize\": \"0\"}]'
if test -f "tmp_shuffle.dump" ; then
# need to replace _Filter
sed -e 's/_Filter = "2,4"/_Filter = "2"/' -e '/_Shuffle = "true"/d' < tmp_shuffle.dump > tmp_shufflex.dump
diff -b -w "tmp_shuffle.cdl" "tmp_shufflex.dump"
else
echo "XFAIL: filter=shuffle zext=$zext"
fi
}
testdeflate() {
zext=$1
runfilter $zext deflate '1,9' '[{\"id\": \"zlib\",\"level\": \"9\"}]'
if test -f "tmp_deflate.dump" ; then
# need to replace _DeflateLevel
sed -e 's/_DeflateLevel = 9/_Filter = "1,9"/' < tmp_deflate.dump > tmp_deflatex.dump
diff -b -w "tmp_deflate.cdl" "tmp_deflatex.dump"
else
echo "XFAIL: filter=deflate zext=$zext"
fi
}
testbzip2() {
zext=$1
runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]'
if test -f "tmp_bzip2.dump" ; then
diff -b -w "tmp_bzip2.cdl" "tmp_bzip2.dump"
else
echo "XFAIL: filter=bzip2 zext=$zext"
fi
}
testszip() {
zext=$1
# H5_SZIP_NN_OPTION_MASK=32; H5_SZIP_MAX_PIXELS_PER_BLOCK_IN=32
runfilter $zext szip '4,32,32' '[{\"id\": \"szip\",\"mask\": 32,\"pixels-per-block\": 32}]'
if test -f "tmp_szip.dump" ; then
diff -b -w "tmp_szip.cdl" "tmp_szip.dump"
else
echo "XFAIL: filter=szip zext=$zext"
fi
}
testblosc() {
zext=$1
runfilter $zext blosc '32001,0,0,0,0,5,1,1' '[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 0,\"cname\": \"lz4\",\"shuffle\": 1}]'
if test -f "tmp_blosc.dump" ; then
diff -b -w "tmp_blosc.cdl" "tmp_blosc.dump"
else
echo "XFAIL: filter=blosc zext=$zext"
fi
}
testset() {
# Which test cases to exercise
testfletcher32 $1
testshuffle $1
testdeflate $1
testszip $1
testbzip2 $1
testblosc $1
}
testset file
#if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi
#if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi
exit 0

7
nczarr_test/tst_nczfilter.c
View File

@ -15,6 +15,8 @@
#define ERR(r) {fprintf(stderr,"fail: line %d: (%d) %s\n",__LINE__,(r),nc_strerror((r)));}
#define FILENAME "file://tmp_nczfilter.nc#mode=nczarr,file"
#define FILTERID 1
int
@ -33,7 +35,10 @@ main(int argc, char **argv)
size_t nparams;
unsigned inqparams[8];
furl = argv[1];
if(argc == 1)
furl = FILENAME;
else
furl = argv[1];
if ((ret=nc_create(furl, NC_NETCDF4, &ncid))) ERR(ret);
if ((ret=nc_def_dim(ncid, "d", 4, &dimid))) ERR(ret);

42
plugins/BZIP2_LICENSE Normal file
View File

@ -0,0 +1,42 @@
--------------------------------------------------------------------------
This program, "bzip2", the associated library "libbzip2", and all
documentation, are copyright (C) 1996-2019 Julian R Seward. All
rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. The origin of this software must not be misrepresented; you must
not claim that you wrote the original software. If you use this
software in a product, an acknowledgment in the product
documentation would be appreciated but is not required.
3. Altered source versions must be plainly marked as such, and must
not be misrepresented as being the original software.
4. The name of the author may not be used to endorse or promote
products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Julian Seward, jseward@acm.org
bzip2/libbzip2 version 1.0.8 of 13 July 2019
--------------------------------------------------------------------------

30
plugins/CMakeLists.txt
View File

@ -10,7 +10,11 @@ FILE(READ H5Znoop.c NOOP_SOURCE)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "#define NOOP_INSTANCE 1\n")
FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "${NOOP_SOURCE}")
SET(h5bzip2_SOURCES blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c H5Zbzip2.c ${NCJ})
SET(h5bzip2_SOURCES H5Zbzip2.c)
IF(NOT HAVE_BZIP2)
SET(h5bzip2_SOURCES ${h5bzip2_SOURCES} blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c)
ENDIF()
SET(h5misc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h)
@ -51,11 +55,6 @@ MACRO(buildplugin TARGET TARGETLIB)
ENDIF()
ENDMACRO()
# Note we use name h5bzip2 instead of bzip2 to avoid logical
# target name clash with examples/C/hdf5plugins
buildplugin(h5bzip2 "h5bzip2")
SET_TARGET_PROPERTIES(h5bzip2 PROPERTIES OUTPUT_NAME "bzip2")
buildplugin(h5misc "h5misc")
buildplugin(h5noop "h5noop")
buildplugin(h5noop1 "h5noop1")
@ -72,11 +71,24 @@ SET(h5blosc_SOURCES H5Zblosc.c)
buildplugin(h5blosc "h5blosc" netcdf;${Blosc_LIBRARIES})
ENDIF()
IF(ENABLE_SZIP)
SET(h5szip_SOURCES H5Zszip.c H5Zszip.h)
buildplugin(h5szip "h5szip" ${Szip_LIBRARIES})
IF(ENABLE_ZSTD)
SET(h5zstd_SOURCES H5Zzstd.c H5Zzstd.h)
buildplugin(h5zstd "h5zstd" netcdf;${Zstd_LIBRARIES})
ENDIF()
# Need our version of szip if libsz available and we are not using HDF5
IF(HAVE_SZ)
SET(h5szip_SOURCES H5Zszip.c H5Zszip.h)
buildplugin(h5szip "h5szip" ${Szip_LIBRARIES})
SET(nczszip_SOURCES NCZszip.c)
buildplugin(nczszip "nczszip" netcdf)
ENDIF()
buildplugin(h5bzip2 "h5bzip2" ${Bzip2_LIBRARIES})
# Note we use name h5bzip2 instead of bzip2 to avoid logical
# target name clash with examples/C/hdf5plugins
SET_TARGET_PROPERTIES(h5bzip2 PROPERTIES OUTPUT_NAME "bzip2")
ENDIF(BUILD_UTILITIES)
ENDIF(ENABLE_FILTER_TESTING)

2
plugins/H5Zblosc.c
View File

@ -469,7 +469,7 @@ NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** v
{stat = NC_EFILTER; goto done;}
nparams = 7;
if(vnparams > 0 && vparams == NULL)
if(vparams == NULL)
{stat = NC_EFILTER; goto done;}
if(wnparamsp == NULL || wparamsp == NULL)

72
plugins/H5Zszip.c
View File

@ -20,15 +20,15 @@
#include <math.h>
#include "netcdf_filter_build.h"
#ifdef USE_SZIP
#ifdef HAVE_SZ
#include <szlib.h>
#include "H5Zszip.h"
/* Local function prototypes */
static htri_t H5Z__can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id);
static herr_t H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id);
static size_t H5Z__filter_szip(unsigned flags, size_t cd_nelmts,
static htri_t H5Z___can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id);
static herr_t H5Z___set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id);
static size_t H5Z___filter_szip(unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf);
/* This message derives from H5Z */
@ -38,15 +38,15 @@ H5Z_class2_t H5Z_SZIP[1] = {{
1, /* Assume encoder present: check before registering */
1, /* decoder_present flag (set to true) */
"szip", /* Filter name for debugging */
H5Z__can_apply_szip, /* The "can apply" callback */
H5Z__set_local_szip, /* The "set local" callback */
H5Z__filter_szip, /* The actual filter function */
H5Z___can_apply_szip, /* The "can apply" callback */
H5Z___set_local_szip, /* The "set local" callback */
H5Z___filter_szip, /* The actual filter function */
}};
/*-------------------------------------------------------------------------
* Function: H5Z__can_apply_szip
* Function: H5Z___can_apply_szip
*
* Purpose: Check the parameters for szip compression for validity and
* whether they fit a particular dataset.
@ -67,12 +67,12 @@ H5Z_class2_t H5Z_SZIP[1] = {{
*-------------------------------------------------------------------------
*/
static htri_t
H5Z__can_apply_szip(hid_t H5_ATTR_UNUSED dcpl_id, hid_t type_id, hid_t H5_ATTR_UNUSED space_id)
H5Z___can_apply_szip(hid_t H5_ATTR_UNUSED dcpl_id, hid_t type_id, hid_t H5_ATTR_UNUSED space_id)
{
htri_t ret_value = TRUE; /* Return value */
#ifdef USE_HDF5
unsigned dtype_size; /* Datatype's size (in bits) */
H5T_order_t dtype_order; /* Datatype's endianness order */
htri_t ret_value = TRUE; /* Return value */
FUNC_ENTER_STATIC
@ -95,14 +95,14 @@ H5Z__can_apply_szip(hid_t H5_ATTR_UNUSED dcpl_id, hid_t type_id, hid_t H5_ATTR_U
done:
FUNC_LEAVE_NOAPI(ret_value)
#else
return SUCCEED;
#endif
} /* end H5Z__can_apply_szip() */
#else /*!USE_HDF5*/
return ret_value;
#endif /*USE_HDF5*/
} /* end H5Z___can_apply_szip() */
/*-------------------------------------------------------------------------
* Function: H5Z__set_local_szip
* Function: H5Z___set_local_szip
*
* Purpose: Set the "local" dataset parameters for szip compression.
*
@ -115,11 +115,11 @@ done:
*-------------------------------------------------------------------------
*/
#ifdef USE_HDF5
static herr_t
H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
H5Z___set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
{
herr_t ret_value = SUCCEED; /* Return value */
#ifdef USE_HDF5
unsigned flags; /* Filter flags */
size_t cd_nelmts = H5Z_SZIP_USER_NPARMS; /* Number of filter parameters */
@ -131,7 +131,6 @@ H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
size_t dtype_precision; /* Datatype's precision (in bits) */
size_t dtype_offset; /* Datatype's offset (in bits) */
hsize_t scanline; /* Size of dataspace's fastest changing dimension */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
@ -203,14 +202,14 @@ H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
/* Set the correct endianness flag for szip */
/* (Note: this may not handle non-atomic datatypes well) */
cd_values[H5Z_SZIP_PARM_MASK] &= ~(SZ_LSB_OPTION_MASK|SZ_MSB_OPTION_MASK);
cd_values[H5Z_SZIP_PARM_MASK] &= ~(H5_SZIP_LSB_OPTION_MASK|H5_SZIP_MSB_OPTION_MASK);
switch(dtype_order) {
case H5T_ORDER_LE: /* Little-endian byte order */
cd_values[H5Z_SZIP_PARM_MASK] |= SZ_LSB_OPTION_MASK;
cd_values[H5Z_SZIP_PARM_MASK] |= H5_SZIP_LSB_OPTION_MASK;
break;
case H5T_ORDER_BE: /* Big-endian byte order */
cd_values[H5Z_SZIP_PARM_MASK] |= SZ_MSB_OPTION_MASK;
cd_values[H5Z_SZIP_PARM_MASK] |= H5_SZIP_MSB_OPTION_MASK;
break;
case H5T_ORDER_ERROR:
@ -227,16 +226,14 @@ H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
done:
FUNC_LEAVE_NOAPI(ret_value)
#else
return SUCCEED;
#endif
} /* end H5Z__set_local_szip() */
#else /*!USE_HDF5*/
return ret_value;
#endif /*USE_HDF5*/
} /* end H5Z___set_local_szip() */
/*-------------------------------------------------------------------------
* Function: H5Z__filter_szip
* Function: H5Z___filter_szip
*
* Purpose: Implement an I/O filter around the 'rice' algorithm in
* libsz
@ -250,7 +247,7 @@ done:
*-------------------------------------------------------------------------
*/
static size_t
H5Z__filter_szip(unsigned flags, size_t cd_nelmts, const unsigned cd_values[],
H5Z___filter_szip(unsigned flags, size_t cd_nelmts, const unsigned cd_values[],
size_t nbytes, size_t *buf_size, void **buf)
{
size_t ret_value = 0; /* Return value */
@ -344,4 +341,19 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
}
#endif /* USE_SZIP */
/* External Discovery Functions */
DLLEXPORT
H5PL_type_t
H5PLget_plugin_type(void)
{
return H5PL_TYPE_FILTER;
}
DLLEXPORT
const void*
H5PLget_plugin_info(void)
{
return H5Z_SZIP;
}
#endif /* HAVE_SZ */

257
plugins/H5Zzstd.c Normal file
View File

@ -0,0 +1,257 @@
#include "config.h"
#include <sys/types.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <stdio.h>
#include "netcdf_filter_build.h"
/* WARNING:
Starting with HDF5 version 1.10.x, the plugin code MUST be
careful when using the standard *malloc()*, *realloc()*, and
*free()* function.
In the event that the code is allocating, reallocating, or
free'ing memory that either came from or will be exported to the
calling HDF5 library, then one MUST use the corresponding HDF5
functions *H5allocate_memory()*, *H5resize_memory()*,
*H5free_memory()* [5] to avoid memory failures.
Additionally, if your filter code leaks memory, then the HDF5 library
will generate an error.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include "netcdf_filter_build.h"
#include <netcdf_json.h>
#include "H5Zzstd.h"
/* Forward */
static htri_t H5Z_zstd_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id);
static size_t H5Z_filter_zstd(unsigned flags,size_t cd_nelmts,const unsigned cd_values[],
size_t nbytes,size_t *buf_size,void**buf);
const H5Z_class2_t H5Z_ZSTD[1] = {{
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)H5Z_FILTER_ZSTD, /* Filter id number */
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"zstd", /* Filter name for debugging */
(H5Z_can_apply_func_t)H5Z_zstd_can_apply, /* The "can apply" callback */
NULL, /* The "set local" callback */
(H5Z_func_t)H5Z_filter_zstd, /* The actual filter function */
}};
/* External Discovery Functions */
DLLEXPORT
H5PL_type_t
H5PLget_plugin_type(void)
{
return H5PL_TYPE_FILTER;
}
DLLEXPORT
const void*
H5PLget_plugin_info(void)
{
return H5Z_ZSTD;
}
/* Make this explicit */
/*
* The "can_apply" callback returns positive a valid combination, zero for an
* invalid combination and negative for an error.
*/
static htri_t
H5Z_zstd_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id)
{
return 1; /* Assume it can always apply */
}
static size_t
H5Z_filter_zstd(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
char *outbuf = NULL;
size_t outbuflen;
char *inbuf = NULL;
size_t inbuflen;
size_t outdatalen;
size_t err;
unsigned long long dsize;
/* Get level parameter */
if(cd_nelmts != 1) {
fprintf(stderr, "zstd: no level specified\n");
goto cleanupAndFail;
}
/* Prepare the input buffer. */
inbuflen = *buf_size;
inbuf = (char*)*buf;
if (flags & H5Z_FLAG_REVERSE) { /** Decompress data. */
dsize = ZSTD_getFrameContentSize(inbuf, inbuflen);
err = (size_t)dsize;
if(ZSTD_isError(err)) {
fprintf(stderr,"zstd: %s\n", ZSTD_getErrorName(err));
goto cleanupAndFail;
}
/* Prepare the output buffer. */
outbuflen = (size_t)dsize;
outbuf = H5allocate_memory(outbuflen,0);
if (outbuf == NULL) {
fprintf(stderr, "memory allocation failed for zstd compression\n");
goto cleanupAndFail;
}
dsize = ZSTD_decompress(outbuf, outbuflen, inbuf, inbuflen);
err = (size_t)dsize;
if(ZSTD_isError(err)) {
fprintf(stderr,"zstd: %s\n", ZSTD_getErrorName(err));
goto cleanupAndFail;
}
} else { /* Compress */
int level;
/* Prepare the output buffer. */
dsize = ZSTD_compressBound(inbuflen);
err = (size_t)dsize;
if(ZSTD_isError(err)) {
fprintf(stderr,"zstd: %s\n", ZSTD_getErrorName(err));
goto cleanupAndFail;
}
/* Prepare the output buffer. */
outbuflen = (size_t)dsize;
outbuf = H5allocate_memory(outbuflen,0);
if (outbuf == NULL) {
fprintf(stderr, "memory allocation failed for zstd compression\n");
goto cleanupAndFail;
}
level = (int)cd_values[0];
dsize = ZSTD_compress(outbuf, outbuflen, inbuf, inbuflen, level);
err = (size_t)dsize;
if(ZSTD_isError(err)) {
fprintf(stderr,"zstd: %s\n", ZSTD_getErrorName(err));
goto cleanupAndFail;
}
}
outdatalen = (size_t)dsize;
/* Always replace the input buffer with the output buffer. */
H5free_memory(*buf);
*buf = outbuf;
*buf_size = outbuflen;
return outdatalen;
cleanupAndFail:
if (outbuf)
H5free_memory(outbuf);
return 0;
}
/**************************************************/
/* NCZarr Filter Objects */
/* Provide the codec support for the HDF5 zstandard library */
static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */
NCZ_CODEC_CLASS_VER, /* Struct version number */
NCZ_CODEC_HDF5, /* Struct sort */
"zstd", /* Standard name/id of the codec */
H5Z_FILTER_ZSTD, /* HDF5 alias for zstd */
NULL, /*NCZ_zstd_codec_initialize*/
NULL, /*NCZ_zstd_codec_finalize*/
NCZ_zstd_codec_to_hdf5,
NCZ_zstd_hdf5_to_codec,
NULL, /*NCZ_zstd_modify_parameters*/
};
/* External Export API */
DLLEXPORT
const void*
NCZ_get_codec_info(void)
{
return (void*)&NCZ_zstd_codec;
}
static int
NCZ_zstd_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
NCjson* jcodec = NULL;
NCjson* jtmp = NULL;
unsigned* params = NULL;
struct NCJconst jc;
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EINTERNAL; goto done;}
if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL)
{stat = NC_ENOMEM; goto done;}
/* parse the JSON */
if(NCJparse(codec_json,0,&jcodec))
{stat = NC_EFILTER; goto done;}
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
/* Verify the codec ID */
if(NCJdictget(jcodec,"id",&jtmp))
{stat = NC_EFILTER; goto done;}
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;}
if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
/* Get Level */
if(NCJdictget(jcodec,"level",&jtmp))
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
params[0] = (unsigned)jc.ival;
*nparamsp = 1;
*paramsp = params; params = NULL;
done:
if(params) free(params);
NCJreclaim(jcodec);
return stat;
}
static int
NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
{
int stat = NC_NOERR;
unsigned level = 0;
char json[1024];
if(nparams == 0 || params == NULL)
{stat = NC_EFILTER; goto done;}
level = params[0];
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level);
if(codecp) {
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}

22
plugins/H5Zzstd.h Normal file
View File

@ -0,0 +1,22 @@
#ifndef H5ZSTD_H
#define H5ZSTD_H
#include "zstd.h"
#ifdef _MSC_VER
#ifdef DLL_EXPORT /* define when building the library */
#define DECLSPEC __declspec(dllexport)
#else
#define DECLSPEC __declspec(dllimport)
#endif
#else
#define DECLSPEC extern
#endif
/* declare the hdf5 interface */
DECLSPEC H5PL_type_t H5PLget_plugin_type(void);
DECLSPEC const void* H5PLget_plugin_info(void);
DECLSPEC const H5Z_class2_t H5Z_ZSTD[1];
#endif /*H5ZSTD_H*/

72
plugins/Makefile.am
View File

@ -7,6 +7,9 @@ include $(top_srcdir)/lib_flags.am
AM_LDFLAGS += -module -avoid-version -shared -export-dynamic \
-rpath ${abs_builddir} ${NOUNDEFINED}
lib_LTLIBRARIES =
noinst_LTLIBRARIES =
if ISMINGW
LDADD = ${top_builddir}/liblib/libnetcdf.la
endif
@ -14,46 +17,68 @@ endif
# BZIP2 version 1.0.8 (https://sourceware.org/bzip2/)
BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h
EXTRA_DIST = CMakeLists.txt \
H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \
NCZdefaults.c \
H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h \
H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \
${BZIP2SRC} H5checksum.c
EXTRA_DIST = CMakeLists.txt
# The HDF5 filter wrappers
EXTRA_DIST += \
H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \
H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \
H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h H5Zzstd.c H5Zzstd.h
# The Codec filter wrappers
EXTRA_DIST += NCZdefaults.c NCZszip.c
# The Filter implementations
EXTRA_DIST += H5checksum.c
EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE
if ENABLE_FILTER_TESTING
lib_LTLIBRARIES = libh5bzip2.la
libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h ${BZIP2SRC}
noinst_LTLIBRARIES = libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la \
libh5shuffle.la libh5fletcher32.la libh5deflate.la
lib_LTLIBRARIES += libh5bzip2.la
libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h
if ! HAVE_BZIP2
libh5bzip2_la_SOURCES += ${BZIP2SRC}
endif
noinst_LTLIBRARIES += libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la
noinst_LTLIBRARIES += libnczdefaults.la
if ENABLE_NCZARR_FILTERS
noinst_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la
libh5shuffle_la_SOURCES = H5Zshuffle.c
libh5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c
libh5deflate_la_SOURCES = H5Zdeflate.c
if ENABLE_BLOSC
# Need our version of szip if libsz available and we are not using HDF5
if HAVE_SZ
noinst_LTLIBRARIES += libh5szip.la libnczszip.la
libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h
libnczszip_la_SOURCES = NCZszip.c
endif
libnczdefaults_la_SOURCES = NCZdefaults.c
endif # ENABLE_NCZARR_FILTERS
if HAVE_BLOSC
noinst_LTLIBRARIES += libh5blosc.la
libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h
endif
if ENABLE_SZIP
noinst_LTLIBRARIES += libh5szip.la
libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h
if HAVE_ZSTD
noinst_LTLIBRARIES += libh5zstd.la
libh5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h
endif
libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h
libnczmisc_la_SOURCES = NCZmisc.c
libnczdefaults_la_SOURCES = NCZdefaults.c
# The noop filter is to allow testing of multifilters and filter order
# Need two distinct instances
libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h
libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h
endif #ENABLE_FILTER_TESTING
BUILT_SOURCES = H5Znoop1.c
@ -62,7 +87,12 @@ H5Znoop1.c: Makefile H5Znoop.c
echo '#define NOOP_INSTANCE 1' > $@
cat ${srcdir}/H5Znoop.c >> $@
# For reference: updating bzip2
BZIP2DIR=/cygdrive/d/bzip2-1.0.8
updatebzip2:
for b in ${BZIP2HDRS} ${BZIP2SRC} ; do cp -f ${BZIP2DIR}/$${b} . ; done
BZIP2VER = 1.0.8
BZIP2DIR = bzip2-${BZIP2VER}
BZIP2URL = https://sourceware.org/pub/bzip2/${BZIP2DIR}.tar.gz
bzip2::
rm -fr ./${BZIP2DIR} ${BZIP2SRC} BZIP2_LICENSE
wget ${BZIP2URL}
tar -zxf ${BZIP2DIR}.tar.gz
cd ${BZIP2DIR}; cp ${BZIP2SRC} ..; cp LICENSE ../BZIP2_LICENSE ; cd ..
rm -fr ./${BZIP2DIR}

227
plugins/NCZdefaults.c
View File

@ -16,11 +16,6 @@ Author: Dennis Heimbigner
#include <string.h>
#include <assert.h>
#ifdef USE_SZIP
#include <szlib.h>
#include "H5Zszip.h"
#endif
#include "netcdf.h"
#include "netcdf_filter.h"
#include "netcdf_filter_build.h"
@ -29,7 +24,6 @@ Author: Dennis Heimbigner
#define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */
#define H5Z_FILTER_SHUFFLE 2 /*shuffle the data */
#define H5Z_FILTER_FLETCHER32 3 /*fletcher32 checksum of EDC */
#define H5Z_FILTER_SZIP 4 /*szip compression */
/**************************************************/
/* NCZarr Filter Objects */
@ -46,12 +40,6 @@ static int NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparam
static int NCZ_deflate_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_deflate_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
#ifdef USE_SZIP
static int NCZ_szip_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
static int NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
#endif
/**************************************************/
static NCZ_codec_t NCZ_shuffle_codec = {
@ -117,10 +105,13 @@ NCZ_shuffle_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned**
if((params=(unsigned*)malloc(sizeof(unsigned)))==NULL)
{stat = NC_ENOMEM; goto done;}
params[0] = (unsigned)typesize;
*wnparamsp = 1;
nullfree(*wparamsp);
*wparamsp = params; params = NULL;
if(wnparamsp) *wnparamsp = 1;
if(wparamsp) {
nullfree(*wparamsp);
*wparamsp = params; params = NULL;
}
done:
nullfree(params);
@ -311,216 +302,10 @@ done:
/**************************************************/
#ifdef USE_SZIP
static NCZ_codec_t NCZ_szip_codec = {
NCZ_CODEC_CLASS_VER, /* Struct version number */
NCZ_CODEC_HDF5, /* Struct sort */
"szip", /* Standard name/id of the codec */
H5Z_FILTER_SZIP, /* HDF5 alias for szip */
NULL, /*NCZ_szip_codec_initialize*/
NULL, /*NCZ_szip_codec_finalize*/
NCZ_szip_codec_to_hdf5,
NCZ_szip_hdf5_to_codec,
NCZ_szip_modify_parameters,
};
static int
NCZ_szip_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
unsigned* params = NULL;
size_t nparams = 2; /* No. of visible parameters */
NCjson* json = NULL;
NCjson* jtmp = NULL;
struct NCJconst jc = {0,0,0,NULL};
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EINTERNAL; goto done;}
if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL)
{stat = NC_ENOMEM; goto done;}
if(NCJparse(codec_json,0,&json))
{stat = NC_EFILTER; goto done;}
if(NCJdictget(json,"mask",&jtmp) || jtmp == NULL)
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
params[H5Z_SZIP_PARM_MASK] = (unsigned)jc.ival;
jtmp = NULL;
if(NCJdictget(json,"pixels-per-block",&jtmp) || jtmp == NULL)
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
params[H5Z_SZIP_PARM_PPB] = (unsigned)jc.ival;
*nparamsp = nparams;
*paramsp = params; params = NULL;
done:
NCJreclaim(json);
nullfree(params);
return stat;
}
static int
NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
{
int stat = NC_NOERR;
char json[2048];
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": \"%u\", \"pixels-per-block\": \"%u\"}",
NCZ_szip_codec.codecid,
params[H5Z_SZIP_PARM_MASK],
params[H5Z_SZIP_PARM_PPB]);
if(codecp) {
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
static int
NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
{
int i,ret_value = NC_NOERR;
nc_type vtype;
size_t typesize, scanline, dtype_precision, npoints;
int ndims, storage, dtype_order;
int dimids[NC_MAX_VAR_DIMS];
char vname[NC_MAX_NAME+1];
size_t chunklens[NC_MAX_VAR_DIMS];
unsigned* params = NULL;
unsigned* vparams = NULL;
size_t wnparams = 4;
if(wnparamsp == NULL || wparamsp == NULL)
{ret_value = NC_EFILTER; goto done;}
if(vnparamsp == NULL || vparamsp == NULL)
{ret_value = NC_EFILTER; goto done;}
if(*vnparamsp > 0 && *vparamsp == NULL)
{ret_value = NC_EFILTER; goto done;}
vparams = *vparamsp;
/* Get variable info */
if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done;
/* Get the typesize */
if((ret_value = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
/* Get datatype's precision, in case is less than full bits */
dtype_precision = typesize;
if(dtype_precision > 24) {
if(dtype_precision <= 32)
dtype_precision = 32;
else if(dtype_precision <= 64)
dtype_precision = 64;
} /* end if */
if(ndims == 0) {ret_value = NC_EFILTER; goto done;}
/* Set "local" parameter for this dataset's "pixels-per-scanline" */
if((ret_value = nc_inq_dimlen(ncid,dimids[ndims-1],&scanline))) goto done;
/* Get number of elements for the dataspace; use
total number of elements in the chunk to define the new 'scanline' size */
/* Compute chunksize */
if((ret_value = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
if(storage != NC_CHUNKED) {ret_value = NC_EFILTER; goto done;}
npoints = 1;
for(i=0;i<ndims;i++) npoints *= chunklens[i];
/* Get datatype's endianness order */
if((ret_value = nc_inq_var_endian(ncid,varid,&dtype_order))) goto done;
if((params = (unsigned*)malloc(wnparams*sizeof(unsigned)))==NULL)
{ret_value = NC_ENOMEM; goto done;}
params[H5Z_SZIP_PARM_MASK] = vparams[H5Z_SZIP_PARM_MASK];
params[H5Z_SZIP_PARM_PPB] = vparams[H5Z_SZIP_PARM_PPB];
/* Set "local" parameter for this dataset's "bits-per-pixel" */
params[H5Z_SZIP_PARM_BPP] = dtype_precision;
/* Adjust scanline if it is smaller than number of pixels per block or
if it is bigger than maximum pixels per scanline, or there are more than
SZ_MAX_BLOCKS_PER_SCANLINE blocks per scanline */
if(scanline < vparams[H5Z_SZIP_PARM_PPB]) {
if(npoints < vparams[H5Z_SZIP_PARM_PPB])
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "pixels per block greater than total number of elements in the chunk")
scanline = MIN((vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), npoints);
} else {
if(scanline <= SZ_MAX_PIXELS_PER_SCANLINE)
scanline = MIN((vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), scanline);
else
scanline = vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE;
} /* end else */
/* Assign the final value to the scanline */
params[H5Z_SZIP_PARM_PPS] = (unsigned)scanline;
/* Set the correct endianness flag for szip */
/* (Note: this may not handle non-atomic datatypes well) */
params[H5Z_SZIP_PARM_MASK] &= ~(SZ_LSB_OPTION_MASK|SZ_MSB_OPTION_MASK);
switch(dtype_order) {
case NC_ENDIAN_LITTLE: /* Little-endian byte order */
params[H5Z_SZIP_PARM_MASK] |= SZ_LSB_OPTION_MASK;
break;
case NC_ENDIAN_BIG: /* Big-endian byte order */
params[H5Z_SZIP_PARM_MASK] |= SZ_MSB_OPTION_MASK;
break;
default:
HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
} /* end switch */
*wnparamsp = wnparams;
nullfree(*wparamsp);
*wparamsp = params; params = NULL;
done:
nullfree(params);
FUNC_LEAVE_NOAPI(ret_value)
}
#if 0
static int
NCZ_szip_visible_parameters(int ncid, int varid, size_t nparamsin, const unsigned int* paramsin, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
unsigned* params = NULL;
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EFILTER; goto done;}
if((params = (unsigned*)malloc(2*sizeof(unsigned)))==NULL)
{stat = NC_ENOMEM; goto done;}
params[H5Z_SZIP_PARM_MASK] = paramsin[H5Z_SZIP_PARM_MASK];
params[H5Z_SZIP_PARM_PPB] = paramsin[H5Z_SZIP_PARM_PPB];
nullfree(*paramsp);
*paramsp = params; params = NULL;
done:
nullfree(params);
return stat;
}
#endif
#endif /*USE_SZIP*/
/**************************************************/
NCZ_codec_t* NCZ_default_codecs[] = {
&NCZ_shuffle_codec,
&NCZ_fletcher32_codec,
&NCZ_zlib_codec,
#ifdef USE_SZIP
&NCZ_szip_codec,
#endif
NULL
};

255
plugins/NCZszip.c Normal file
View File

@ -0,0 +1,255 @@
/* Copyright 2003-2018, University Corporation for Atmospheric
* Research. See the COPYRIGHT file for copying and redistribution
* conditions.
*/
/*
Author: Dennis Heimbigner
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <szlib.h>
#include "H5Zszip.h"
#include "netcdf.h"
#include "netcdf_filter.h"
#include "netcdf_filter_build.h"
#include "netcdf_json.h"
#define H5Z_FILTER_SZIP 4 /*szip compression */
/**************************************************/
/* NCZarr Filter Objects */
static int NCZ_szip_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
static int NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
static int NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
static NCZ_codec_t NCZ_szip_codec = {
NCZ_CODEC_CLASS_VER, /* Struct version number */
NCZ_CODEC_HDF5, /* Struct sort */
"szip", /* Standard name/id of the codec */
H5Z_FILTER_SZIP, /* HDF5 alias for szip */
NULL, /*NCZ_szip_codec_initialize*/
NULL, /*NCZ_szip_codec_finalize*/
NCZ_szip_codec_to_hdf5,
NCZ_szip_hdf5_to_codec,
NCZ_szip_modify_parameters,
};
static int
NCZ_szip_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
unsigned* params = NULL;
size_t nparams = 2; /* No. of visible parameters */
NCjson* json = NULL;
NCjson* jtmp = NULL;
struct NCJconst jc = {0,0,0,NULL};
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EINTERNAL; goto done;}
if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL)
{stat = NC_ENOMEM; goto done;}
if(NCJparse(codec_json,0,&json))
{stat = NC_EFILTER; goto done;}
if(NCJdictget(json,"mask",&jtmp) || jtmp == NULL)
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
params[H5Z_SZIP_PARM_MASK] = (unsigned)jc.ival;
jtmp = NULL;
if(NCJdictget(json,"pixels-per-block",&jtmp) || jtmp == NULL)
{stat = NC_EFILTER; goto done;}
if(NCJcvt(jtmp,NCJ_INT,&jc))
{stat = NC_EFILTER; goto done;}
params[H5Z_SZIP_PARM_PPB] = (unsigned)jc.ival;
*nparamsp = nparams;
*paramsp = params; params = NULL;
done:
NCJreclaim(json);
nullfree(params);
return stat;
}
static int
NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
{
int stat = NC_NOERR;
char json[2048];
snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": %u, \"pixels-per-block\": %u}",
NCZ_szip_codec.codecid,
params[H5Z_SZIP_PARM_MASK],
params[H5Z_SZIP_PARM_PPB]);
if(codecp) {
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
static int
NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp)
{
int i,ret_value = NC_NOERR;
nc_type vtype;
size_t typesize, scanline, dtype_precision, npoints;
int ndims, storage, dtype_order;
int dimids[NC_MAX_VAR_DIMS];
char vname[NC_MAX_NAME+1];
size_t chunklens[NC_MAX_VAR_DIMS];
unsigned* params = NULL;
unsigned* vparams = NULL;
size_t wnparams = 4;
if(wnparamsp == NULL || wparamsp == NULL)
{ret_value = NC_EFILTER; goto done;}
if(vnparamsp == NULL || vparamsp == NULL)
{ret_value = NC_EFILTER; goto done;}
if(*vnparamsp > 0 && *vparamsp == NULL)
{ret_value = NC_EFILTER; goto done;}
vparams = *vparamsp;
/* Get variable info */
if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done;
/* Get the typesize */
if((ret_value = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done;
/* Get datatype's precision, in case is less than full bits */
dtype_precision = typesize*8;
if(dtype_precision > 24) {
if(dtype_precision <= 32)
dtype_precision = 32;
else if(dtype_precision <= 64)
dtype_precision = 64;
} /* end if */
if(ndims == 0) {ret_value = NC_EFILTER; goto done;}
/* Set "local" parameter for this dataset's "pixels-per-scanline" */
if((ret_value = nc_inq_dimlen(ncid,dimids[ndims-1],&scanline))) goto done;
/* Get number of elements for the dataspace; use
total number of elements in the chunk to define the new 'scanline' size */
/* Compute chunksize */
if((ret_value = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done;
if(storage != NC_CHUNKED) {ret_value = NC_EFILTER; goto done;}
npoints = 1;
for(i=0;i<ndims;i++) npoints *= chunklens[i];
/* Get datatype's endianness order */
if((ret_value = nc_inq_var_endian(ncid,varid,&dtype_order))) goto done;
if((params = (unsigned*)malloc(wnparams*sizeof(unsigned)))==NULL)
{ret_value = NC_ENOMEM; goto done;}
params[H5Z_SZIP_PARM_MASK] = vparams[H5Z_SZIP_PARM_MASK];
params[H5Z_SZIP_PARM_PPB] = vparams[H5Z_SZIP_PARM_PPB];
/* Set "local" parameter for this dataset's "bits-per-pixel" */
params[H5Z_SZIP_PARM_BPP] = dtype_precision;
/* Adjust scanline if it is smaller than number of pixels per block or
if it is bigger than maximum pixels per scanline, or there are more than
SZ_MAX_BLOCKS_PER_SCANLINE blocks per scanline */
if(scanline < vparams[H5Z_SZIP_PARM_PPB]) {
if(npoints < vparams[H5Z_SZIP_PARM_PPB])
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "pixels per block greater than total number of elements in the chunk")
scanline = MIN((vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), npoints);
} else {
if(scanline <= SZ_MAX_PIXELS_PER_SCANLINE)
scanline = MIN((vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), scanline);
else
scanline = vparams[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE;
} /* end else */
/* Assign the final value to the scanline */
params[H5Z_SZIP_PARM_PPS] = (unsigned)scanline;
/* Set the correct mask flags */
/* From H5Pdcpl.c#H5Pset_szip */
params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~H5_SZIP_CHIP_OPTION_MASK);
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_ALLOW_K13_OPTION_MASK;
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_RAW_OPTION_MASK;
params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~(H5_SZIP_LSB_OPTION_MASK | H5_SZIP_MSB_OPTION_MASK));
/* From H5Zszip.c#H5Z__set_local_szip */
params[H5Z_SZIP_PARM_MASK] &= (unsigned)(~(H5_SZIP_LSB_OPTION_MASK | H5_SZIP_MSB_OPTION_MASK));
switch(dtype_order) {
case NC_ENDIAN_LITTLE: /* Little-endian byte order */
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_LSB_OPTION_MASK;
break;
case NC_ENDIAN_BIG: /* Big-endian byte order */
params[H5Z_SZIP_PARM_MASK] |= H5_SZIP_MSB_OPTION_MASK;
break;
default:
HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
} /* end switch */
*wnparamsp = wnparams;
nullfree(*wparamsp);
*wparamsp = params; params = NULL;
done:
nullfree(params);
FUNC_LEAVE_NOAPI(ret_value)
}
#if 0
static int
NCZ_szip_visible_parameters(int ncid, int varid, size_t nparamsin, const unsigned int* paramsin, size_t* nparamsp, unsigned** paramsp)
{
int stat = NC_NOERR;
unsigned* params = NULL;
if(nparamsp == NULL || paramsp == NULL)
{stat = NC_EFILTER; goto done;}
if((params = (unsigned*)malloc(2*sizeof(unsigned)))==NULL)
{stat = NC_ENOMEM; goto done;}
params[H5Z_SZIP_PARM_MASK] = paramsin[H5Z_SZIP_PARM_MASK];
params[H5Z_SZIP_PARM_PPB] = paramsin[H5Z_SZIP_PARM_PPB];
nullfree(*paramsp);
*paramsp = params; params = NULL;
done:
nullfree(params);
return stat;
}
#endif
/**************************************************/
NCZ_codec_t* NCZ_szip_codecs[] = {
&NCZ_szip_codec,
NULL
};
/* External Export API */
DLLEXPORT
const void*
NCZ_codec_info_defaults(void)
{
return (void*)&NCZ_szip_codecs;
}

5
test_common.in
View File

@ -150,5 +150,10 @@ ncgen3c0="${top_srcdir}/ncgen3/c0.cdl"
ncgenc0="${top_srcdir}/ncgen/c0.cdl"
ncgenc04="${top_srcdir}/ncgen/c0_4.cdl"
# Test for filter availability
avail() {
if test yes = `${execdir}/../ncdump/ncfilteravail $1` ; then return 0 ; else echo "filter $1 not available" ; return 1; fi
}
# Make sure we are in builddir (not execdir)
cd $builddir