mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-19 17:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add filter support to NCZarr

Browse Source 
Filter support has three goals:

1. Use the existing HDF5 filter implementations,
2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr,
3. Allow filters to be used even when HDF5 is disabled

Detailed usage directions are define in docs/filters.md.

For now, the existing filter API is left in place. So filters
are defined using ''nc_def_var_filter'' using the HDF5 style
where the id and parameters are unsigned integers.

This is a big change since filters affect many parts of the code.

In the following, the terms "compressor" and "filter" and "codec" are generally
used synonomously.

### Filter-Related Changes:
* In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms.
* Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h.
* Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out.
* Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h.
* Add a number of new test to test the new nczarr filters.
* Let ncgen parse _Codecs attribute, although it is ignored.

### Plugin directory changes:
* Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file
* Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip
* Add a Codec defaulter (see docs/filters.md) for the big four filters.
* Make plugins work with windows by properly adding __declspec declaration.

### Misc. Non-Filter Changes
* Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5.
* Improve support for caching
* More fixes for path conversion code
* Fix misc. memory leaks
* Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath.
* Add a number of new test to test the non-filter fixes.
* Update the parsers
* Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
This commit is contained in:
Dennis Heimbigner 2021-09-02 17:04:26 -06:00
parent 09e0e04227
commit 11fe00ea05
175 changed files with 17390 additions and 6516 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/run_tests.yml vendored
View File

@ -199,9 +199,9 @@ jobs:
if: matrix.use_dap == 'dap_off'
- run: echo "ENABLE_DAP=ON" >> $GITHUB_ENV
if: matrix.use_dap == 'dap_on'
- run: echo "ENABLE_NCZARR=--disable-nczarr" >> $GITHUB_ENV
- run: echo "ENABLE_NCZARR=OFF" >> $GITHUB_ENV
if: matrix.use_nczarr == 'nczarr_off'
- run: echo "ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV
- run: echo "ENABLE_NCZARR=ON" >> $GITHUB_ENV
if: matrix.use_nczarr == 'nczarr_on'
- run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV

119
CMakeLists.txt
View File

@ -26,7 +26,7 @@ set(PACKAGE "netCDF" CACHE STRING "")
SET(NC_VERSION_MAJOR 4)
SET(NC_VERSION_MINOR 8)
SET(NC_VERSION_PATCH 2)
SET(NC_VERSION_PATCH 1)
SET(NC_VERSION_NOTE "-development")
SET(netCDF_VERSION ${NC_VERSION_MAJOR}.${NC_VERSION_MINOR}.${NC_VERSION_PATCH}${NC_VERSION_NOTE})
SET(VERSION ${netCDF_VERSION})
@ -118,6 +118,11 @@ MACRO(CHECK_C_LINKER_FLAG M_FLAG M_RESULT)
SET(CMAKE_REQUIRED_FLAGS "${T_REQ_FLAG}")
ENDMACRO()
# Enable 'dist and distcheck'.
# File adapted from http://ensc.de/cmake/FindMakeDist.cmake
FIND_PACKAGE(MakeDist)
# End 'enable dist and distcheck'
# Set the build type.
IF(NOT CMAKE_BUILD_TYPE)
SET(CMAKE_BUILD_TYPE DEBUG CACHE STRING "Choose the type of build, options are: None, Debug, Release."
@ -194,15 +199,6 @@ ENDIF()
OPTION(NC_FIND_SHARED_LIBS "Find dynamically-built versions of dependent libraries" ${BUILD_SHARED_LIBS})
##
# Check to see if C compiler supports -fno-strict-aliasing, in support of
# https://github.com/Unidata/netcdf-c/issues/1983
##
CHECK_C_COMPILER_FLAG(-fno-strict-aliasing CC_SUPPORTS_NO_STRICT_ALIASING)
IF(CC_SUPPORTS_NO_STRICT_ALIASING)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing")
ENDIF(CC_SUPPORTS_NO_STRICT_ALIASING)
##
# We've had a request to allow for non-versioned shared libraries.
# This seems reasonable enough to accommodate. See
@ -773,10 +769,6 @@ IF(USE_HDF5)
# are a hot mess between versions.
####
IF(HDF5_hdf5_LIBRARY AND NOT HDF5_C_LIBRARY)
SET(HDF5_C_LIBRARY ${HDF5_hdf5_LIBRARY})
ENDIF()
# Depending on the install, either HDF5_hdf_library or
# HDF5_C_LIBRARIES may be defined. We must check for either.
IF(HDF5_C_LIBRARIES AND NOT HDF5_hdf5_LIBRARY)
@ -933,7 +925,7 @@ int main() {
}" HAVE_LIBCURL_766)
IF (HAVE_LIBCURL_766)
# If libcurl version is >= 7.66, then can skip tests
# If libcurl version is >= 7.66, then can skip tests
# for these symbols which were added in an earlier version
set(HAVE_CURLOPT_USERNAME TRUE)
set(HAVE_CURLOPT_PASSWORD TRUE)
@ -1016,10 +1008,48 @@ OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF)
OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON)
SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
# See if we have zlib
FIND_PACKAGE(ZLIB)
# Define a test flag for have zlib library
IF(ZLIB_FOUND)
INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS})
SET(ENABLE_ZLIB TRUE)
ELSE()
SET(ENABLE_ZLIB FALSE)
ENDIF()
# See if we have libblosc
IF(!MSVC)
FIND_PACKAGE(Blosc)
ENDIF()
# Define a test flag for have blosc library
IF(Blosc_FOUND)
INCLUDE_DIRECTORIES(${Blosc_INCLUDE_DIRS})
SET(ENABLE_BLOSC TRUE)
ELSE()
SET(ENABLE_BLOSC FALSE)
ENDIF()
# See if we have libszip
IF(!MSVC)
#FIND_PACKAGE(SZIP)
#FIND_LIBRARY(SZIP PATH NAMES szip sz)
SET(SZIP_LIBRARY ${SZIP})
ENDIF()
message("xxx: ${SZIP_FOUND} ; ${SZIP}")
# Define a test flag for have szip library
IF(SZIP_FOUND)
INCLUDE_DIRECTORIES(${SZIP_INCLUDE_DIRS})
SET(ENABLE_SZIP TRUE)
ELSE()
SET(ENABLE_SZIP FALSE)
ENDIF()
# See if we have libzip
FIND_PACKAGE(Zip)
# Define a test flag for have curl library
# Define a test flag for have zip library
IF(Zip_FOUND)
INCLUDE_DIRECTORIES(${Zip_INCLUDE_DIRS})
SET(ENABLE_NCZARR_ZIP TRUE)
@ -1027,6 +1057,14 @@ ELSE()
SET(ENABLE_NCZARR_ZIP FALSE)
ENDIF()
# libdl is always available; built-in in Windows and OSX
SET(ENABLE_PLUGINS TRUE)
IF(NOT WIN32)
IF(HAVE_DLFCN_H)
INCLUDE_DIRECTORIES("dlfcn.h")
ENDIF()
ENDIF()
# Enable some developer-only tests
OPTION(ENABLE_EXTRA_TESTS "Enable Extra tests. Some may not work because of known issues. Developers only." OFF)
IF(ENABLE_EXTRA_TESTS)
@ -1049,9 +1087,9 @@ IF(MSVC)
FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c
DESTINATION ${netCDF_BINARY_DIR}/ncdump/)
FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c
DESTINATION ${netCDF_BINARY_DIR}/ncdap_test/)
DESTINATION ${netCDF_BINARY_DIR}/ncdap_test/)
ENDIF()
ENDIF()
@ -1386,20 +1424,21 @@ IF (ENABLE_PARALLEL_TESTS AND NOT USE_PARALLEL)
ENDIF()
# Enable special filter test; experimental when using cmake.
OPTION(ENABLE_FILTER_TESTING "Enable filter testing. Ignored if shared libraries or netCDF4 are not enabled" ${ENABLE_HDF5})
IF(ENABLE_FILTER_TESTING AND NOT ENABLE_HDF5)
MESSAGE(WARNING "ENABLE_FILTER_TESTING requires HDF5. Disabling.")
SET(ENABLE_FILTER_TESTING OFF)
OPTION(ENABLE_FILTER_TESTING "Enable filter testing. Ignored if shared libraries or netCDF4 are not enabled" yes)
IF(ENABLE_FILTER_TESTING)
if(NOT ENABLE_HDF5 AND NOT ENABLE_NCZARR)
MESSAGE(WARNING "ENABLE_FILTER_TESTING requires HDF5 and/or NCZarr. Disabling.")
SET(ENABLE_FILTER_TESTING OFF CACHE BOOL "Enable Filter Testing" FORCE)
ENDIF()
ENDIF()
IF(NOT BUILD_SHARED_LIBS)
MESSAGE(WARNING "ENABLE_FILTER_TESTING requires shared libraries. Disabling.")
SET(ENABLE_FILTER_TESTING OFF)
ENDIF()
OPTION(ENABLE_CLIENTSIDE_FILTERS "Enable client-side filter registration." OFF)
IF(NOT ENABLE_FILTER_TESTING)
SET(ENABLE_CLIENTSIDE_FILTERS OFF)
ENDIF()
SET(ENABLE_CLIENTSIDE_FILTERS OFF)
# Determine whether or not to generate documentation.
OPTION(ENABLE_DOXYGEN "Enable generation of doxygen-based documentation." OFF)
@ -1533,6 +1572,7 @@ CHECK_INCLUDE_FILE("libgen.h" HAVE_LIBGEN_H)
CHECK_INCLUDE_FILE("execinfo.h" HAVE_EXECINFO_H)
CHECK_INCLUDE_FILE("dirent.h" HAVE_DIRENT_H)
CHECK_INCLUDE_FILE("time.h" HAVE_TIME_H)
CHECK_INCLUDE_FILE("dlfcn.h" HAVE_DLFCN_H)
# Symbol Exists
CHECK_SYMBOL_EXISTS(isfinite "math.h" HAVE_DECL_ISFINITE)
@ -1931,7 +1971,7 @@ ENDIF(ENABLE_BASH_SCRIPT_TESTING)
MACRO(add_sh_test prefix F)
IF(HAVE_BASH)
ADD_TEST(${prefix}_${F} bash "-c" "export srcdir=${CMAKE_CURRENT_SOURCE_DIR};export TOPSRCDIR=${CMAKE_SOURCE_DIR};${CMAKE_CURRENT_BINARY_DIR}/${F}.sh ${ARGN}")
ADD_TEST(${prefix}_${F} bash "-c" "export srcdir=${CMAKE_CURRENT_SOURCE_DIR};export TOPSRCDIR=${CMAKE_SOURCE_DIR};${CMAKE_CURRENT_BINARY_DIR}/${F}.sh ${F}.sh ${ARGN}")
ENDIF()
ENDMACRO()
@ -2018,17 +2058,31 @@ IF(ENABLE_DAP4)
ADD_SUBDIRECTORY(libdap4)
ENDIF()
IF(ENABLE_PLUGINS)
ADD_SUBDIRECTORY(libncpoco)
ENDIF()
IF(ENABLE_NCZARR)
ADD_SUBDIRECTORY(libnczarr)
FILE(COPY ${netCDF_SOURCE_DIR}/unit_test/timer_utils.h
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
FILE(COPY ${netCDF_SOURCE_DIR}/unit_test/timer_utils.c
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter.c
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter_misc.c
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter_repeat.c
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter_order.c
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/tst_multifilter.c
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
ENDIF()
add_subdirectory(liblib)
IF(ENABLE_FILTER_TESTING)
IF(ENABLE_PLUGINS)
add_subdirectory(plugins)
ENDIF()
@ -2194,6 +2248,9 @@ INSTALL(PROGRAMS ${netCDF_BINARY_DIR}/nc-config
##
print_conf_summary()
# Enable Makedist files.
ADD_MAKEDIST()
ENABLE_MAKEDIST(README.md COPYRIGHT RELEASE_NOTES.md INSTALL INSTALL.cmake test_prog.c lib_flags.am cmake CMakeLists.txt COMPILE.cmake.txt config.h.cmake.in cmake_uninstall.cmake.in netcdf-config-version.cmake.in netcdf-config.cmake.in FixBundle.cmake.in nc-config.cmake.in configure configure.ac install-sh config.h.in config.sub CTestConfig.cmake.in)
#####
# Configure and print the libnetcdf.settings file.
@ -2245,6 +2302,8 @@ is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS)
is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)
is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
is_enabled(ENABLE_LOGGING HAS_LOGGING)
is_enabled(ENABLE_FILTER_TESTING DO_FILTER_TESTS)
is_enabled(ENABLE_BLOSC HAS_BLOSC)
# Generate file from template.
CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
@ -2302,6 +2361,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY
#####
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF)
IF(ENABLE_NCZARR)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nczarr_test/findplugin.sh @ONLY NEWLINE_STYLE LF)
ENDIF()
IF(ENABLE_EXAMPLES)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF)
ENDIF()

11
Makefile.am
View File

@ -81,13 +81,18 @@ HDF4_TEST_DIR = hdf4_test
LIBHDF4 = libhdf4
endif
# Build Plugin support
if ENABLE_PLUGINS
NCPOCO = libncpoco
endif
# Build Cloud Storage if desired.
if ENABLE_NCZARR
ZARR_TEST_DIR = nczarr_test
ZARR = libnczarr
ZARR = libnczarr
endif
# Optionally build plugins
# Optionally build test plugins
if ENABLE_FILTER_TESTING
PLUGIN_DIR = plugins
endif
@ -108,7 +113,7 @@ endif
# and run. ncgen must come before ncdump, because their tests
# depend on it.
SUBDIRS = include $(H5_TEST_DIR) libdispatch libsrc $(LIBSRC4_DIR) \
$(LIBSRCP) $(LIBHDF4) $(LIBHDF5) $(OCLIB) $(DAP2) ${DAP4} ${ZARR} liblib \
$(LIBSRCP) $(LIBHDF4) $(LIBHDF5) $(OCLIB) $(DAP2) ${DAP4} ${NCPOCO} ${ZARR} liblib \
$(NCGEN3) $(NCGEN) $(NCDUMP) ${PLUGIN_DIR} $(TESTDIRS) docs \
$(EXAMPLES)

64
cmake/modules/FindBlosc.cmake Executable file
View File

@ -0,0 +1,64 @@
# Searches for an installation of the blosc library. On success, it sets the following variables:
#
# Blosc_FOUND Set to true to indicate the blosc library was found
# Blosc_INCLUDE_DIRS The directory containing the header file blosc/blosc.h
# Blosc_LIBRARIES The libraries needed to use the blosc library
#
# To specify an additional directory to search, set Blosc_ROOT.
#
# Author: Siddhartha Chaudhuri, 2009
#
# Look for the header, first in the user-specified location and then in the system locations
SET(Blosc_INCLUDE_DOC "The directory containing the header file blosc.h")
FIND_PATH(Blosc_INCLUDE_DIRS NAMES blosc.h blosc/blosc.h PATHS ${Blosc_ROOT} ${Blosc_ROOT}/include DOC ${Blosc_INCLUDE_DOC} NO_DEFAULT_PATH)
IF(NOT Blosc_INCLUDE_DIRS) # now look in system locations
FIND_PATH(Blosc_INCLUDE_DIRS NAMES blosc.h blosc/blosc.h DOC ${Blosc_INCLUDE_DOC})
ENDIF(NOT Blosc_INCLUDE_DIRS)
SET(Blosc_FOUND FALSE)
IF(Blosc_INCLUDE_DIRS)
SET(Blosc_LIBRARY_DIRS ${Blosc_INCLUDE_DIRS})
IF("${Blosc_LIBRARY_DIRS}" MATCHES "/include$")
# Strip off the trailing "/include" in the path.
GET_FILENAME_COMPONENT(Blosc_LIBRARY_DIRS ${Blosc_LIBRARY_DIRS} PATH)
ENDIF("${Blosc_LIBRARY_DIRS}" MATCHES "/include$")
IF(EXISTS "${Blosc_LIBRARY_DIRS}/lib")
SET(Blosc_LIBRARY_DIRS ${Blosc_LIBRARY_DIRS}/lib)
ENDIF(EXISTS "${Blosc_LIBRARY_DIRS}/lib")
# Find Blosc libraries
FIND_LIBRARY(Blosc_DEBUG_LIBRARY NAMES bloscd blosc_d libbloscd libblosc_d libblosc
PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug
PATHS ${Blosc_LIBRARY_DIRS} NO_DEFAULT_PATH)
FIND_LIBRARY(Blosc_RELEASE_LIBRARY NAMES blosc libblosc
PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release
PATHS ${Blosc_LIBRARY_DIRS} NO_DEFAULT_PATH)
SET(Blosc_LIBRARIES )
IF(Blosc_DEBUG_LIBRARY AND Blosc_RELEASE_LIBRARY)
SET(Blosc_LIBRARIES debug ${Blosc_DEBUG_LIBRARY} optimized ${Blosc_RELEASE_LIBRARY})
ELSEIF(Blosc_DEBUG_LIBRARY)
SET(Blosc_LIBRARIES ${Blosc_DEBUG_LIBRARY})
ELSEIF(Blosc_RELEASE_LIBRARY)
SET(Blosc_LIBRARIES ${Blosc_RELEASE_LIBRARY})
ENDIF(Blosc_DEBUG_LIBRARY AND Blosc_RELEASE_LIBRARY)
IF(Blosc_LIBRARIES)
SET(Blosc_FOUND TRUE)
ENDIF(Blosc_LIBRARIES)
ENDIF(Blosc_INCLUDE_DIRS)
IF(Blosc_FOUND)
# IF(NOT Blosc_FIND_QUIETLY)
MESSAGE(STATUS "Found Blosc: headers at ${Blosc_INCLUDE_DIRS}, libraries at ${Blosc_LIBRARY_DIRS}")
MESSAGE(STATUS " library is ${Blosc_LIBRARIES}")
# ENDIF(NOT Blosc_FIND_QUIETLY)
ELSE(Blosc_FOUND)
IF(Blosc_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Blosc library not found")
ENDIF(Blosc_FIND_REQUIRED)
ENDIF(Blosc_FOUND)

20
config.h.cmake.in
View File

@ -153,6 +153,9 @@ are set when opening a binary file on Windows. */
/* if true, enable NCZARR */
#cmakedefine ENABLE_NCZARR 1
/* if true, enable nczarr filter support */
#cmakedefine ENABLE_NCZARR_FILTERS 1
/* if true, enable S3 support */
#cmakedefine ENABLE_NCZARR_S3 1
@ -165,6 +168,9 @@ are set when opening a binary file on Windows. */
/* if true, S3 SDK is available */
#cmakedefine ENABLE_S3_SDK 1
/* if true, Allow dynamically loaded plugins */
#cmakedefine ENABLE_PLUGINS 1
/* define the possible sources for remote test servers */
#cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}"
@ -223,6 +229,9 @@ are set when opening a binary file on Windows. */
/* Define to 1 if you have the <dirent.h> header file. */
#cmakedefine HAVE_DIRENT_H 1
/* Define to 1 if you have the <dlfcn.h> header file. */
#cmakedefine HAVE_DLFCN_H 1
/* Define to 1 if you have the <fcntl.h> header file. */
#cmakedefine HAVE_FCNTL_H 1
@ -611,11 +620,14 @@ with zip */
/* if true, use stdio instead of posixio */
#cmakedefine USE_STDIO 1
/* if true, compile in szip compression in netCDF-4 variables */
#cmakedefine USE_SZIP 1
/* if true, multi-filters enabled*/
#cmakedefine HAVE_MULTIFILTERS 1
#cmakedefine ENABLE_MULTIFILTERS 1
/* if true, enable nczarr blosc support */
#cmakedefine ENABLE_BLOSC 1
/* if true, enable nczarr szip support */
#cmakedefine USE_SZIP 1
/* Version number of package */
#cmakedefine VERSION "${netCDF_VERSION}"

123
configure.ac
View File

@ -596,6 +596,15 @@ if test "x$enable_nczarr" = xyes; then
fi
AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
# See if we have libblosc
AC_CHECK_LIB([blosc],[blosc_init],[enable_blosc=yes],[enable_blosc=no])
if test "x$enable_blosc" = "xyes" ; then
AC_SEARCH_LIBS([blosc_init],[blosc blosc.dll cygblosc.dll], [], [])
AC_DEFINE([ENABLE_BLOSC], [1], [if true, blosc library is available])
fi
AC_MSG_CHECKING([whether libblosc library is available])
AC_MSG_RESULT([${enable_blosc}])
# See if we have libzip
AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
if test "x$have_zip" = "xyes" ; then
@ -610,6 +619,7 @@ if test "x$enable_nczarr" = xno ; then
enable_nczarr_zip=no
fi
AC_MSG_CHECKING([whether nczarr zip support is enabled])
AC_MSG_RESULT([${enable_nczarr_zip}])
@ -1101,6 +1111,25 @@ if test "x$enable_byterange" = xyes; then
AC_DEFINE([ENABLE_BYTERANGE], [1], [if true, support byte-range read of remote datasets.])
fi
# Need libdl(d) for plugins
AC_CHECK_LIB([dl],[dlopen],[have_libdld=yes],[have_libdld=no])
if test "x$have_libdld" = "xyes" ; then
AC_SEARCH_LIBS([dlopen],[dl dld], [],[])
fi
# Does the user want plugins?
AC_MSG_CHECKING([whether dynamically loaded plugins is enabled])
AC_ARG_ENABLE([plugins],
[AS_HELP_STRING([--disable-plugins],
[allow dynamically loaded plugins])])
if test "x$have_libdld" = xno ; then enable_plugins=no; fi
test "x$enable_plugins" = xno || enable_plugins=yes
AC_MSG_RESULT($enable_plugins)
if test "x$enable_plugins" = xyes; then
AC_DEFINE([ENABLE_PLUGINS], [1], [if true, support dynamically loaded plugins])
fi
AM_CONDITIONAL(ENABLE_PLUGINS, [test "x$enable_plugins" = xyes])
AC_FUNC_ALLOCA
AC_CHECK_DECLS([isnan, isinf, isfinite],,,[#include <math.h>])
AC_STRUCT_ST_BLKSIZE
@ -1209,11 +1238,10 @@ AC_CHECK_SIZEOF(ssize_t)
$SLEEPCMD
AC_CHECK_SIZEOF([void*])
if test "x$enable_hdf5" = xyes || test "x$enable_dap" = xyes; then
if test "x$enable_hdf5" = xyes || test "x$enable_dap" = xyes ; then
AC_SEARCH_LIBS([deflate], [zlibwapi zlibstat zlib zlib1 z], [], [
AC_MSG_ERROR([Can't find or link to the z library. Turn off netCDF-4 and \
DAP clients with --disable-hdf5 --disable-dap, or see config.log for errors.])])
AC_SEARCH_LIBS([dlopen], [dl dld], [], [])
fi
# We need the math library
@ -1315,6 +1343,16 @@ if test "x$enable_hdf5" = xyes; then
if test "x$hdf5_version_1106" = xyes; then
AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8])
fi
else # !enable_hdf5
# See if we have libsz
AC_CHECK_LIB([sz],[SZ_BufftoBuffCompress],[enable_szlib=yes],[enable_szlib=no])
if test "x$enable_szlib" = "xyes" ; then
AC_SEARCH_LIBS([SZ_BufftoBuffCompress],[sz sz.dll cygsz.dll], [], [])
AC_DEFINE([USE_SZIP], [1], [if true, szip library is available])
fi
AC_MSG_CHECKING([whether szip library is available])
AC_MSG_RESULT([${enable_szlib}])
fi
AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ])
@ -1487,6 +1525,47 @@ if test "x$enable_logging" = xyes; then
AC_DEFINE([LOGGING], 1, [If true, turn on logging.])
fi
# Control NCZarr filters
# Does the user want to use NCZarr filters?
AC_MSG_CHECKING([whether we should enable NCZarr filters])
AC_ARG_ENABLE([nczarr-filters], [AS_HELP_STRING([--disable-nczarr-filters],
[disable NCZarr filters])])
test "x$enable_nczarr_filters" = xno || enable_nczarr_filters=yes
AC_MSG_RESULT([$enable_nczarr_filters])
if test "x$enable_nczarr_filters" = xyes; then
AC_DEFINE([ENABLE_NCZARR_FILTERS], [1], [if true, enable NCZarr filters])
fi
# Control filter test/example
AC_MSG_CHECKING([whether filter testing should be run])
AC_ARG_ENABLE([filter-testing],
[AS_HELP_STRING([--disable-filter-testing],
[Do not run filter test and example; requires shared libraries and HDF5|NCZarr])])
test "x$enable_filter_testing" = xno || enable_filter_testing=yes
AC_MSG_RESULT($enable_filter_testing)
if test "x$enable_nczarr" = xno ; then
enable_nczarr_filters=no
fi
if test "x$enable_hdf5" = xno && test "x$enable_nczarr" = xno ; then
AC_MSG_WARN([HDF5 and NCZarr are disabled => --disable-filter-testing])
enable_filter_testing=no
fi
if test "x$enable_shared" = xno ; then
AC_MSG_WARN([Shared libraries are disabled => --disable-filter-testing])
enable_filter_testing=no
enable_nczarr_filters=no
fi
# Client side filter registration is permanently disabled
enable_clientside_filters=no
AM_CONDITIONAL(ENABLE_CLIENTSIDE_FILTERS, [test x$enable_clientside_filters = xyes])
AM_CONDITIONAL(ENABLE_FILTER_TESTING, [test x$enable_filter_testing = xyes])
AM_CONDITIONAL(ENABLE_NCZARR_FILTERS, [test x$enable_nczarr_filters = xyes])
# Automake conditionals need to be called, whether the answer is yes
# or no.
AM_CONDITIONAL(BUILD_PARALLEL, [test x$enable_parallel = xyes])
@ -1523,6 +1602,8 @@ AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
AM_CONDITIONAL(HAVE_AWS, [test "x$have_aws" = xyes])
AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
AM_CONDITIONAL(ENABLE_BLOSC, [test "x$enable_blosc" = xyes])
AM_CONDITIONAL(ENABLE_SZIP, [test "x$enable_szip" = xyes])
# If the machine doesn't have a long long, and we want netCDF-4, then
# we've got problems!
@ -1598,30 +1679,6 @@ esac
NC_FLIBS="-lnetcdff $NC_LIBS"
# Control filter test/example
AC_MSG_CHECKING([whether filter testing should be run])
AC_ARG_ENABLE([filter-testing],
[AS_HELP_STRING([--disable-filter-testing],
[Do not run filter test and example; requires shared libraries and netCDF-4])])
test "x$enable_filter_testing" = xno || enable_filter_testing=yes
AC_MSG_RESULT($enable_filter_testing)
# Client side filter registration is permanently disabled
enable_clientside_filters=no
if test "x$enable_hdf5" = xno ; then
AC_MSG_WARN([HDF5 disabled => --disable-filter-testing && --disable-client-filters])
enable_filter_testing=no
fi
if test "x$enable_shared" = xno ; then
AC_MSG_WARN([Shared libraries are disabled => --disable-filter-testing])
enable_filter_testing=no
fi
AM_CONDITIONAL(ENABLE_CLIENTSIDE_FILTERS, [test x$enable_clientside_filters = xyes])
AM_CONDITIONAL(ENABLE_FILTER_TESTING, [test x$enable_filter_testing = xyes])
AC_SUBST(NC_LIBS,[$NC_LIBS])
AC_SUBST(HAS_DAP,[$enable_dap])
@ -1650,6 +1707,9 @@ AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
AC_SUBST(HAS_LOGGING,[$enable_logging])
AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing])
AC_SUBST(HAVE_BLOSC,[$enable_blosc])
AC_SUBST(HAVE_SZIP,[$enable_szip])
# Include some specifics for netcdf on windows.
#AH_VERBATIM([_WIN32_STRICMP],
@ -1745,9 +1805,8 @@ AM_CONDITIONAL([AX_IGNORE], [test xno = xyes])
AC_MSG_NOTICE([generating header files and makefiles])
AC_CONFIG_FILES(test_common.sh:test_common.in)
AC_CONFIG_FILES(nczarr_test/timer_utils.h:unit_test/timer_utils.h)
AC_CONFIG_FILES(nczarr_test/timer_utils.c:unit_test/timer_utils.c)
AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in)
AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)
@ -1756,6 +1815,13 @@ AC_CONFIG_FILES([h5_test/run_par_tests.sh], [chmod ugo+x h5_test/run_par_tests.s
AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh])
AC_CONFIG_FILES([nc_perf/run_par_bm_test.sh], [chmod ugo+x nc_perf/run_par_bm_test.sh])
AC_CONFIG_FILES([nc_perf/run_gfs_test.sh], [chmod ugo+x nc_perf/run_gfs_test.sh])
AC_CONFIG_FILES(nczarr_test/timer_utils.h:unit_test/timer_utils.h)
AC_CONFIG_FILES(nczarr_test/timer_utils.c:unit_test/timer_utils.c)
AC_CONFIG_FILES(nczarr_test/test_filter.c:nc_test4/test_filter.c)
AC_CONFIG_FILES(nczarr_test/test_filter_misc.c:nc_test4/test_filter_misc.c)
AC_CONFIG_FILES(nczarr_test/tst_multifilter.c:nc_test4/tst_multifilter.c)
AC_CONFIG_FILES(nczarr_test/test_filter_repeat.c:nc_test4/test_filter_repeat.c)
AC_CONFIG_FILES(nczarr_test/test_filter_order.c:nc_test4/test_filter_order.c)
AC_CONFIG_FILES([examples/C/run_par_test.sh], [chmod ugo+x examples/C/run_par_test.sh])
AC_CONFIG_FILES([nc-config], [chmod 755 nc-config])
AC_CONFIG_FILES([Makefile
@ -1782,6 +1848,7 @@ AC_CONFIG_FILES([Makefile
libdap4/Makefile
libhdf4/Makefile
libnczarr/Makefile
libncpoco/Makefile
libdispatch/Makefile
liblib/Makefile
ncdump/cdl/Makefile

2
dap4_test/baselinethredds/CMakeLists.txt
View File

@ -13,7 +13,7 @@ FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSION
# Rename file in support of https://github.com/Unidata/netcdf-c/issues/2077
##
IF(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds)
FILE(RENAME ${CMAKE_CURRENT_BINARY_DIR}/GOES16_TEST1.nc4.thredds ${CMAKE_CURRENT_BINARY_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds)
FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/GOES16_TEST1.nc4.thredds DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds)
ENDIF()
FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*)

952
docs/filters.md Normal file
View File

@ -0,0 +1,952 @@
NetCDF-4 Filter Support
============================
<!-- double header is needed to workaround doxygen bug -->
# NetCDF-4 Filter Support {#filters}
\tableofcontents
[TOC]
The netCDF library supports a general filter mechanism to apply various
kinds of filters to datasets before reading or writing.
The netCDF enhanced (aka netCDF-4) library inherits this capability since it depends on the HDF5 library.
The HDF5 library (1.8.11 and later) supports filters, and netCDF is based closely on that underlying HDF5 mechanism.
Filters assume that a variable has chunking defined and each chunk is filtered before writing and "unfiltered" after reading and before passing the data to the user.
In the event that multiple filters are defined on a variable, they are applied in first-defined order on writing and on the reverse order when reading.
The most common kind of filter is a compression-decompression filter, and that is the focus of this document.
For now, this document is strongly influenced by the HDF5 mechanism.
When other implementations (e.g. Zarr) support filters, this document will have multiple sections: one for each mechanism.
# A Warning on Backward Compatibility {#filters_compatibility}
The API defined in this document should accurately reflect
the current state of filters in the netCDF-c library.
Be aware that there was a short period in which the filter code was undergoing some revision and extension.
Those extensions have largely been reverted.
Unfortunately, some users may experience some compilation problems for previously working code because of these reversions.
In that case, please revise your code to adhere to this document. Apologies are extended for any inconvenience.
A user may encounter an incompatibility if any of the following appears in user code.
* The function _nc_inq_var_filter_ was returning the error value _NC_ENOFILTER_ if a variable had no associated filters.
It has been reverted to the previous case where it returned _NC_NOERR_ and the returned filter id was set to zero if the variable had no filters.
* The function _nc_inq_var_filterids_ was renamed to _nc_inq_var_filter_ids_.
* Some auxilliary functions for parsing textual filter specifications have been moved to __netcdf_aux.h__.
See <a href="#filters_appendixa">Appendix A</a>.
* All of the "filterx" functions have been removed. This is unlikely to cause problems because they had limited visibility.
* The undocumented function "nc_filter_remove" no longer exists.
For additional information, see <a href="#filters_appendixb">Appendix B</a>.
# Enabling A HDF5 Compression Filter {#filters_enable}
HDF5 supports dynamic loading of compression filters using the following process for reading of compressed data.
1. Assume that we have a dataset with one or more variables that were compressed using some algorithm.
How the dataset was compressed will be discussed subsequently.
2. Shared libraries or DLLs exist that implement the compress/decompress algorithm.
These libraries have a specific API so that the HDF5 library can locate, load, and utilize the compressor.
These libraries are expected to installed in a specific directory.
In order to compress a variable with an HDF5 compliant filter, the netcdf-c library must be given three pieces of information:
1. some unique identifier for the filter to be used,
2. a vector of parameters for controlling the action of the compression filter, and
3. a shared library implementation of the filter.
The meaning of the parameters is, of course, completely filter dependent and the filter description [3] needs to be consulted.
For bzip2, for example, a single parameter is provided representing the compression level.
It is legal to provide a zero-length set of parameters.
Defaults are not provided, so this assumes that the filter can operate with zero parameters.
Filter ids are assigned by the HDF group.
See [4] for a current list of assigned filter ids.
Note that ids above 32767 can be used for testing without registration.
The first two pieces of information can be provided in one of three ways: using __ncgen__, via an API call, or via command line parameters to __nccopy__.
In any case, remember that filtering also requires setting chunking, so the variable must also be marked with chunking information.
If compression is set for a non-chunked variable, the variable will forcibly be
converted to chunked using a default chunking algorithm.
## Using The API {#filters_API}
The necessary API methods are included in _netcdf_filter.h_ by default.
These functions implicitly use the HDF5 mechanisms and may produce an error if applied to a file format that is not compatible with the HDF5 mechanism.
1. Add a filter to the set of filters to be used when writing a variable.
This must be invoked after the variable has been created and before __nc_enddef__ is invoked.
````
int nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params);
Arguments:
* ncid -- File and group ID.
* varid -- Variable ID.
* id -- Filter ID.
* nparams -- Number of filter parameters.
* params -- Filter parameters.
Return codes:
* NC_NOERR -- No error.
* NC_ENOTNC4 -- Not a netCDF-4 file.
* NC_EBADID -- Bad ncid or bad filter id
* NC_ENOTVAR -- Invalid variable ID.
* NC_EINDEFINE -- called when not in define mode
* NC_ELATEDEF -- called after variable was created
* NC_EINVAL -- Scalar variable, or parallel enabled and parallel filters not supported or nparams or params invalid.
````
2. Query a variable to obtain a list of all filters associated with that variable.
The number of filters associated with the variable is stored in __nfiltersp__ (it may be zero).
The set of filter ids will be returned in __filterids__.
As is usual with the netcdf API, one is expected to call this function twice.
The first time to set __nfiltersp__ and the second to get the filter ids in client-allocated memory.
Any of these arguments can be NULL, in which case no value is returned.
````
int nc_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* filterids);
Arguments:
* ncid -- File and group ID.
* varid -- Variable ID.
* nfiltersp -- Stores number of filters found; may be zero.
* filterids -- Stores set of filter ids.
Return codes:
* NC_NOERR -- No error.
* NC_ENOTNC4 -- Not a netCDF-4 file.
* NC_EBADID -- Bad ncid
* NC_ENOTVAR -- Invalid variable ID.
````
3. Query a variable to obtain information about a specific filter associated with the variable.
The __id__ indicates the filter of interest.
The actual parameters are stored in __params__.
The number of parameters is returned in __nparamsp__.
As is usual with the netcdf API, one is expected to call this function twice.
The first time to set __nparamsp__ and the second to get the parameters in client-allocated memory.
Any of these arguments can be NULL, in which case no value is returned.
If the specified id is not attached to the variable, then NC_ENOFILTER is returned.
````
int nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, unsigned int* params);
Arguments:
* ncid -- File and group ID.
* varid -- Variable ID.
* id -- The filter id of interest.
* nparamsp -- Stores number of parameters.
* params -- Stores set of filter parameters.
Return codes:
* NC_NOERR -- No error.
* NC_ENOTNC4 -- Not a netCDF-4 file.
* NC_EBADID -- Bad ncid
* NC_ENOTVAR -- Invalid variable ID.
* NC_ENOFILTER -- Filter not defined for the variable.
````
4. Query a variable to obtain information about the first filter associated with the variable.
When netcdf-c was modified to support multiple filters per variable, the utility of this function became redundant since it returns info only about the first defined filter for the variable.
Internally, it is implemented using the functions __nc_inq_var_filter_ids__ and __nc_inq_filter_info__.
In any case, the filter id will be returned in the __idp__ argument.
If there are not filters, then zero is stored in this argument.
Otherwise, the number of parameters is stored in __nparamsp__ and the actual parameters in __params__.
As is usual with the netcdf API, one is expected to call this function twice.
The first time to get __nparamsp__ and the second to get the parameters in client-allocated memory.
Any of these arguments can be NULL, in which case no value is returned.
````
int nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsigned int* params);
Arguments:
* ncid -- File and group ID.
* varid -- Variable ID.
* idp -- Stores the id of the first found filter, set to zero if variable has no filters.
* nparamsp -- Stores number of parameters.
* params -- Stores set of filter parameters.
Return codes:
* NC_NOERR -- No error.
* NC_ENOTNC4 -- Not a netCDF-4 file.
* NC_EBADID -- Bad ncid
* NC_ENOTVAR -- Invalid variable ID.
````
## Using ncgen {#filters_NCGEN}
In a CDL file, compression of a variable can be specified by annotating it with the following attribute:
* ''_Filter'' &mdash; a string containing a comma separated list of constants specifying (1) the filter id to apply, and (2) a vector of constants representing the parameters for controlling the operation of the specified filter.
See the section on the <a href="#filters_syntax">parameter encoding syntax</a> for the details on the allowable kinds of constants.
This is a "special" attribute, which means that it will normally be invisible when using __ncdump__ unless the -s flag is specified.
This attribute may be repeated to specify multiple filters.
For backward compatibility it is probably better to use the ''_Deflate'' attribute instead of ''_Filter''. But using ''_Filter'' to specify deflation will work.
Note that the lexical order of declaration is important when more than one filter is specified for a variable because it determines the order in which the filters are applied.
### Example CDL File (Data elided)
````
netcdf bzip2szip {
dimensions:
dim0 = 4 ; dim1 = 4 ; dim2 = 4 ; dim3 = 4 ;
variables:
float var(dim0, dim1, dim2, dim3) ;
var:_Filter = "307,9|4,32,32" ; // bzip2 then szip
var:_Storage = "chunked" ;
var:_ChunkSizes = 4, 4, 4, 4 ;
data:
...
}
````
Note that the assigned filter id for bzip2 is 307 and for szip it is 4.
## Using nccopy {#filters_NCCOPY}
When copying a netcdf file using __nccopy__ it is possible to specify filter information for any output variable by using the "-F" option on the command line; for example:
````
nccopy -F "var,307,9" unfiltered.nc filtered.nc
````
Assume that _unfiltered.nc_ has a chunked but not bzip2 compressed variable named "var".
This command will copy that variable to the _filtered.nc_ output file but using filter with id 307 (i.e. bzip2) and with parameter(s) 9 indicating the compression level.
See the section on the <a href="#filters_syntax">parameter encoding syntax</a> for the details on the allowable kinds of constants.
The "-F" option can be used repeatedly, as long as a different variable is specified for each occurrence.
It can be convenient to specify that the same compression is to be applied to more than one variable. To support this, two additional *-F* cases are defined.
1. ````-F *,...```` means apply the filter to all variables in the dataset.
2. ````-F v1&v2&..,...```` means apply the filter to multiple variables.
Multiple filters can be specified using the pipeline notions '|'.
For example
1. ````-F v1&v2,307,9|4,32,32```` means apply filter 307 (bzip2) then filter 4 (szip) to the multiple variables.
Note that the characters '*', '&', and '|' are shell reserved characters, so you will probably need to escape or quote the filter spec in that environment.
As a rule, any input filter on an input variable will be applied to the equivalent output variable &mdash; assuming the output file type is netcdf-4.
It is, however, sometimes convenient to suppress output compression either totally or on a per-variable basis.
Total suppression of output filters can be accomplished by specifying a special case of "-F", namely this.
````
nccopy -F none input.nc output.nc
````
The expression ````-F *,none```` is equivalent to ````-F none````.
Suppression of output filtering for a specific set of variables can be accomplished using these formats.
````
nccopy -F "var,none" input.nc output.nc
nccopy -F "v1&v2&...,none" input.nc output.nc
````
where "var" and the "vi" are the fully qualified name of a variable.
The rules for all possible cases of the "-F none" flag are defined by this table.
<table>
<tr><th>-F none<th>-Fvar,...<th>Input Filter<th>Applied Output Filter
<tr><td>true<td>undefined<td>NA<td>unfiltered
<tr><td>true<td>none<td>NA<td>unfiltered
<tr><td>true<td>defined<td>NA<td>use output filter(s)
<tr><td>false<td>undefined<td>defined<td>use input filter(s)
<tr><td>false<td>none<td>NA<td>unfiltered
<tr><td>false<td>defined<td>NA<td>use output filter(s)
<tr><td>false<td>undefined<td>undefined<td>unfiltered
<tr><td>false<td>defined<td>defined<td>use output filter(s)
</table>
# Filter Specification Syntax {#filters_syntax}
The utilities <a href="#NCGEN">ncgen</a> and <a href="#NCCOPY">nccopy</a>, and also the output of __ncdump__, support the specification of filter ids, formats, and parameters in text format.
The BNF specification is defined in <a href="#filters_appendixc">Appendix C</a>.
Basically, These specifications consist of a filter id, a comma, and then a sequence of
comma separated constants representing the parameters.
The constants are converted within the utility to a proper set of unsigned int constants (see the <a href="#ParamEncode">parameter encoding section</a>).
To simplify things, various kinds of constants can be specified rather than just simple unsigned integers.
The __ncgen__ and __nccopy__ programs will encode them properly using the rules specified in the section on <a href="#filters_paramcoding">parameter encode/decode</a>.
Since the original types are lost after encoding, __ncdump__ will always show a simple list of unsigned integer constants.
The currently supported constants are as follows.
<table>
<tr halign="center"><th>Example<th>Type<th>Format Tag<th>Notes
<tr><td>-17b<td>signed 8-bit byte<td>b|B<td>Truncated to 8 bits and zero extended to 32 bits
<tr><td>23ub<td>unsigned 8-bit byte<td>u|U b|B<td>Truncated to 8 bits and zero extended to 32 bits
<tr><td>-25S<td>signed 16-bit short<td>s|S<td>Truncated to 16 bits and zero extended to 32 bits
<tr><td>27US<td>unsigned 16-bit short<td>u|U s|S<td>Truncated to 16 bits and zero extended to 32 bits
<tr><td>-77<td>implicit signed 32-bit integer<td>Leading minus sign and no tag<td>
<tr><td>77<td>implicit unsigned 32-bit integer<td>No tag<td>
<tr><td>93U<td>explicit unsigned 32-bit integer<td>u|U<td>
<tr><td>789f<td>32-bit float<td>f|F<td>
<tr><td>12345678.12345678d<td>64-bit double<td>d|D<td>LE encoding
<tr><td>-9223372036854775807L<td>64-bit signed long long<td>l|L<td>LE encoding
<tr><td>18446744073709551615UL<td>64-bit unsigned long long<td>u|U l|L<td>LE encoding
</table>
Some things to note.
1. In all cases, except for an untagged positive integer, the format tag is required and determines how the constant is converted to one or two unsigned int values.
2. For an untagged positive integer, the constant is treated as of the smallest type into which it fits (i.e. 8,16,32, or 64 bit).
3. For signed byte and short, the value is sign extended to 32 bits and then treated as an unsigned int value, but maintaining the bit-pattern.
4. For double, and signed|unsigned long long, they are converted as specified in the section on <a href="#filters_paramcoding">parameter encode/decode</a>.
5. In order to support mutiple filters, the argument to ''_Filter'' may be a pipeline separated (using '|') to specify a list of filters specs.
# Dynamic Loading Process {#filters_Process}
Each filter is assumed to be compiled into a separate dynamically loaded library.
For HDF5 conformant filters, these filter libraries are assumed to be in some specific location.
The details for writing such a filter are defined in the HDF5 documentation[1,2].
## Plugin directory {#filters_Plugindir}
The HDF5 loader expects plugins to be in a specified plugin directory.
The default directory is:
* "/usr/local/hdf5/lib/plugin” for linux/unix operating systems (including Cygwin)
* “%ALLUSERSPROFILE%\\hdf5\\lib\\plugin” for Windows systems, although the code does not appear to explicitly use this path.
The default may be overridden using the environment variable __HDF5_PLUGIN_PATH__.
## Plugin Library Naming {#filters_Pluginlib}
Given a plugin directory, HDF5 examines every file in that directory that conforms to a specified name pattern as determined by the platform on which the library is being executed.
<table>
<tr halign="center"><th>Platform<th>Basename<th>Extension
<tr halign="left"><td>Linux<td>lib*<td>.so*
<tr halign="left"><td>OSX<td>lib*<td>.so*
<tr halign="left"><td>Cygwin<td>cyg*<td>.dll*
<tr halign="left"><td>Windows<td>*<td>.dll
</table>
## Plugin Verification {#filters_Pluginverify}
For each dynamic library located using the previous patterns, HDF5 attempts to load the library and attempts to obtain information from it.
Specifically, It looks for two functions with the following signatures.
1. __H5PL_type_t H5PLget_plugin_type(void)__ &mdash; This function is expected to return the constant value __H5PL_TYPE_FILTER__ to indicate that this is a filter library.
2. __const void* H5PLget_plugin_info(void)__ &mdash; This function returns a pointer to a table of type __H5Z_class2_t__.
This table contains the necessary information needed to utilize the filter both for reading and for writing.
In particular, it specifies the filter id implemented by the library and it must match that id specified for the variable in __nc_def_var_filter__ in order to be used.
If plugin verification fails, then that plugin is ignored and the search continues for another, matching plugin.
# NCZarr Filter Support {#filters_nczarr}
The inclusion of Zarr support in the netcdf-c library creates the need to provide a new representation consistent with the way that Zarr files store filter information.
For Zarr, filters are represented using the JSON notation.
Each filter is defined by a JSON dictionary, and each such filter dictionary
is guaranteed to have a key named "id" whose value is a unique string defining the filter algorithm: "lz4" or "bzip2", for example.
The parameters of the filter are defined by additional -- algorithm specific -- keys in the filter dictionary.
One commonly used filter is "blosc", which has a JSON dictionary of this form.
````
{
"id": "blosc",
"cname": "lz4",
"clevel": 5,
"shuffle": 1
}
````
So it has three parameters:
1. "cname" -- the sub-algorithm used by the blosc compressor, LZ4 in this case.
2. "clevel" -- the compression level, 5 in this case.
3. "shuffle" -- is the input shuffled before compression, yes (1) in this case.
NCZarr has three constraints that must be met.
1. It must store its filter information in its metadata in the above JSON dictionary format.
2. It is required to re-use the HDF5 filter implementations.
This is to avoid having to rewrite the filter implementations
This means that some mechanism is needed to translate between the HDF5 id+parameter model and the Zarr JSON dictionary model.
3. It must be possible to modify the set of visible parameters in response to environment information such as the type of the associated variable; this is required to mimic the corresponding HDF5 capability.
Note that the term "visible parameters" is used here to refer to the parameters provided by "nc_def_var_filter" or those stored in the dataset's metadata as provided by the JSON codec. The term "working parameters" refers to the parameters given to the compressor itself and derived from the visible parameters.
The standard authority for defining Zarr filters is the list supported by the NumCodecs project [7].
Comparing the set of standard filters (aka codecs) defined by NumCodecs to the set of standard filters defined by HDF5 [3], it can be seen that the two sets overlap, but each has filters not defined by the other.
Note also that it is undesirable that a specific set of filters/codecs be built into the NCZarr implementation.
Rather, it is preferable for there be some extensible way to associate the JSON with the code implementing the codec. This mirrors the plugin model used by HDF5.
The mechanism provided to address these issues is similar to that taken by HDF5.
A shared library must exist that has certain well-defined entry points that allow the NCZarr code to determine information about a Codec.
The shared library exports a well-known function name to access Codec information and relate it to a corresponding HDF5 implementation,
## Processing Overview
There are several paths by which the NCZarr filter API is invoked.
1. The nc_def_var_filter function is invoked on a variable or
(1a) the metadata for a variable is read when opening an existing variable that has associated Codecs.
2. The visible parameters are converted to a set of working parameters.
3. The filter is invoked with the working parameters.
4. The dataset is closed using the final set of visible parameters.
### Step 1: Invoking nc_def_var_filter
In this case, the filter plugin is located and the set of visible parameters (from nc_def_var_filter) are provided.
### Step 1a: Reading metadata
In this case, the codec is read from the metadata and must be converted to a visible set of HDF5 style parameters.
It is possible that this set of visible parameters differs from the set that was provided by nc_def_var_filter.
If this is important, then the filter implementation is responsible for marking this difference using, for example, different number of parameters or some differing value.
### Step 2: Convert visible parameters to working parameters
Given environmental information such as the associated variables base type, the visible parameters
are converted to a potentially larger set of working parameters; additionally provide the opportunity
to modify the visible parameters.
### Step 3: Invoking the filter
As chunks are read or written, the filter is repeatedly invoked using the working parameters.
### Step 4: Closing the dataset
The visible parameters from step 2 are stored in the dataset's metadata.
It is desirable to determine if the set of visible parameters changes.
If no change is detected, then re-writing the compressor metadata may be avoided.
## Client API
Currently, there is no way to specify use of a filter via Codec through
the netcdf-c API. Rather, one must know the HDF5 id and parameters of
the filter of interest and use the functions ''nc_def_var_filter'' and ''nc_inq_var_filter''.
Internally, the NCZarr code will use information about known Codecs to convert the HDF5 filter reference to the corresponding Codec.
This restriction also holds for the specification of filters in ''ncgen'' and ''nccopy''.
## Special Codecs Attribute
A new special attribute is defined called ''_Codecs'' in parallel to the current ''_Filters'' special attribute. Its value is a string containing the JSON representation of the Codecs associated with a given variable.
This can be especially useful when a file is unreadable because it uses a filter not available to the netcdf-c library.
That is, no implementation was found in the e.g. ''HDF5_PLUGIN_PATH'' directory.
In this case ''ncdump -hs'' will display the raw Codec information so that it may be possible to see what filter is missing.
## Pre-Processing Filter Libraries
The process for using filters for NCZarr is defined to operate in several steps.
First, as with HDF5, all shared libraries in a specified directory
(e.g. ''HDF5_PLUGIN_PATH'') are scanned.
They are interrogated to see what kind of library they implement, if any.
This interrogation operates by seeing if certain well-known (function) names are defined in this library.
There will be two library types:
1. HDF5 -- exports a specific API: "H5Z\_plugin\_type" and "H5Z\_get\_plugin\_info".
2. Codec -- exports a specific API: "NCZ\_get\_codec\_info"
Note that a given library can export either or both of these APIs.
This means that we can have three types of libraries:
1. HDF5 only
2. Codec only
3. HDF5 + Codec
Suppose that our ''HDF5_PLUGIN_PATH'' location has an HDF5-only library.
Then by adding a corresponding, separate, Codec-only library to that same location, it is possible to make an HDF5 library usable by NCZarr.
It is possible to do this without having to modify the HDF5-only library.
Over time, it is possible to merge an HDF5-only library with a Codec-only library to produce a single, combined library.
## Using Plugin Libraries
The netcdf-c library processes all of the shared libraries by interrogating each one for the well-known APIs and recording the result.
Any libraries that do not export one or both of the well-known APIs is ignored.
Internally, the netcdf-c library pairs up each HDF5 library API with a corresponding Codec API by invoking the relevant well-known functions
(See <a href="#appendixe">Appendix E</a>).
This results in this table for associated codec and hdf5 libraries.
<table>
<tr><th>HDF5 API<th>Codec API<th>Action
<tr><td>Not defined<td>Not defined<td>Ignore
<tr><td>Defined<td>Not defined<td>Ignore
<tr><td>Defined<td>Defined<td>NCZarr usable
</table>
## Using the Codec API
Given a set of filters for which the HDF5 API and the Codec API
are defined, it is then possible to use the APIs to invoke the
filters and to process the meta-data in Codec JSON format.
### Writing an NCZarr Container
When writing, the user program will invoke the NetCDF API function *nc_def_var_filter*.
This function is currently defined to operate using HDF5-style id and parameters (unsigned ints).
The netcdf-c library examines its list of known filters to find one matching the HDF5 id provided by *nc_def_var_filter*.
The set of parameters provided is stored internally.
Then during writing of data, the corresponding HDF5 filter is invoked to encode the data.
When it comes time to write out the meta-data, the stored HDF5-style parameters are passed to a specific Codec function to obtain the corresponding JSON representation. Again see <a href="#appendixe">Appendix E</a>.
This resulting JSON is then written in the NCZarr metadata.
### Reading an NCZarr Container
When reading, the netcdf-c library will read the metadata for a given variable and will see that some set of filters are applied to this variable.
The metadata is encoded as Codec-style JSON.
Given a JSON Codec, it is parsed to provide a JSON dictionary containing the string "id" and the set of parameters as various keys.
The netcdf-c library examines its list of known filters to find one matching the Codec "id" string.
The JSON is passed to a Codec function to obtain the corresponding HDF5-style *unsigned int* parameter vector.
These parameters are stored for later use.
## Supporting Filter Chains
HDF5 supports *filter chains*, which is a sequence of filters where the output of one filter is provided as input to the next filter in the sequence.
When encoding, the filters are executed in the "forward" direction,
while when decoding the filters are executed in the "reverse" direction.
In the Zarr meta-data, a filter chain is divided into two parts:
the "compressor" and the "filters". The former is a single JSON codec
as described above. The latter is an ordered JSON array of codecs.
So if compressor is something like
"compressor": {"id": "c"...}
and the filters array is like this:
"filters": [ {"id": "f1"...}, {"id": "f2"...}...{"id": "fn"...}]
then the filter chain is (f1,f2,...fn,c) with f1 being applied first and c being applied last when encoding. On decode, the filter chain is executed in the order (c,fn...f2,f1).
So, an HDF5 filter chain is divided into two parts, where the last filter in the chain is assigned as the "compressor" and the remaining
filters are assigned as the "filters".
But independent of this, each codec, whether a compressor or a filter,
is stored in the JSON dictionary form described earlier.
## Extensions
The Codec style, using JSON, has the ability to provide very complex parameters that may be hard to encode as a vector of unsigned integers.
It might be desirable to consider exporting a JSON-base API out of the netcdf-c API to support user access to this complexity.
This would mean providing some alternate version of "nc_def_var_filter" that takes a string-valued argument instead of a vector of unsigned ints.
This extension is unlikely to be implemented until a compelling use-case is encountered.
One bad side-effect of this is that we then may have two classes of plugins.
One class can be used by both HDF5 and NCZarr, and a second class that is usable only with NCZarr.
## Using The NetCDF-C Plugins
As part of its testing, the NetCDF build process creates a number of shared libraries in the ''netcdf-c/plugins'' (or sometimes ''netcdf-c/plugins/.libs'') directory.
If you need a filter from that set, you may be able to set ''HDF5_PLUGIN_PATH''
to point to that directory or you may be able to copy the shared libraries out of that directory to your own location.
# Debugging {#filters_debug}
Depending on the debugger one uses, debugging plugins can be very difficult.
It may be necessary to use the old printf approach for debugging the filter itself.
One case worth mentioning is when there is a dataset that is using an unknown filter.
For this situation, you need to identify what filter(s) are used in the dataset.
This can be accomplished using this command.
````
ncdump -s -h <dataset filename>
````
Since ncdump is not being asked to access the data (the -h flag), it can obtain the filter information without failures.
Then it can print out the filter id and the parameters as well as the Codecs (via the -s flag).
## Test Cases {#filters_TestCase}
Within the netcdf-c source tree, the directory two directories contain test cases for testing dynamic filter operation.
* __netcdf-c/nc_test4__ provides tests for testing HDF5 filters.
* __netcdf-c/nczarr_test__ provides tests for testing NCZarr filters.
These tests are disabled if __--disable-shared__ or if __--disable-filter-tests__ is specified.
## HDF5 Example {#filters_Example}
A slightly simplified version of one of the HDF5 filter test cases is also available as an example within the netcdf-c source tree directory __netcdf-c/examples/C__.
The test is called __filter_example.c__ and it is executed as part of the __run_examples4.sh__ shell script.
The test case demonstrates dynamic filter writing and reading.
The files __example/C/hdf5plugins/Makefile.am__ and __example/C/hdf5plugins/CMakeLists.txt__ demonstrate how to build the hdf5 plugin for bzip2.
# Notes
## Order of Invocation for Multiple Filters
When multiple filters are defined on a variable, the order of application, when writing data to the file, is same as the order in which _nc_def_var_filter_ is called.
When reading a file the order of application is of necessity the reverse.
There are some special cases.
1. The fletcher32 filter is always applied first, if enabled.
2. If _nc_def_var_filter_ or _nc_def_var_deflate_ or _nc_def_var_szip_ is called multiple times with the same filter id, but possibly with different sets of parameters, then the position of that filter in the sequence of applictions does not change.
However the last set of parameters specified is used when actually writing the dataset.
3. Deflate and shuffle &mdash; these two are inextricably linked in the current API, but have quite different semantics.
If you call _nc_def_var_deflate_ multiple times, then the previous rule applies with respect to deflate.
However, the shuffle filter, if enabled, is ''always'' applied before applying any other filters, except fletcher32.
4. Once a filter is defined for a variable, it cannot be removed nor can its position in the filter order be changed.
## Memory Allocation Issues
Starting with HDF5 version 1.10.*, the plugin code MUST be careful when using the standard *malloc()*, *realloc()*, and *free()* function.
In the event that the code is allocating, reallocating, for
free'ing memory that either came from or will be exported to the
calling HDF5 library, then one MUST use the corresponding HDF5
functions *H5allocate_memory()*, *H5resize_memory()*,
*H5free_memory()* [5] to avoid memory failures.
Additionally, if your filter code leaks memory, then the HDF5 library generates a failure something like this.
````
H5MM.c:232: H5MM_final_sanity_check: Assertion `0 == H5MM_curr_alloc_bytes_s' failed.
````
One can look at the the code in plugins/H5Zbzip2.c and H5Zmisc.c as illustrations.
## SZIP Issues
The current szip plugin code in the HDF5 library has some behaviors that can catch the unwary.
These are handled internally to (mostly) hide them so that they should not affect users.
Specifically, this filter may do two things.
1. Add extra parameters to the filter parameters: going from the two parameters provided by the user to four parameters for internal use.
It turns out that the two parameters provided when calling nc_def_var_filter correspond to the first two parameters of the four parameters returned by nc_inq_var_filter.
2. Change the values of some parameters: the value of the __options_mask__ argument is known to add additional flag bits, and the __pixels_per_block__ parameter may be modified.
The reason for these changes is has to do with the fact that the szip API provided by the underlying H5Pset_szip function is actually a subset of the capabilities of the real szip implementation.
Presumably this is for historical reasons.
In any case, if the caller uses the __nc_inq_var_szip__ or the __nc_inq_var_filter__ functions, then the parameter values returned may differ from those originally specified.
## Supported Systems
The current matrix of OS X build systems known to work is as follows.
<table>
<tr><th>Build System<th>Supported OS
<tr><td>Automake<td>Linux, Cygwin, OSX
<tr><td>Cmake<td>Linux, Cygwin, OSX, Visual Studio
</table>
## Generic Plugin Build
If you do not want to use Automake or Cmake, the following has been known to work.
````
gcc -g -O0 -shared -o libbzip2.so <plugin source files> -L${HDF5LIBDIR} -lhdf5_hl -lhdf5 -L${ZLIBDIR} -lz
````
# References {#filters_References}
1. https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf
2. https://support.hdfgroup.org/HDF5/doc/TechNotes/TechNote-HDF5-CompressionTroubleshooting.pdf
3. https://portal.hdfgroup.org/display/support/Registered+Filter+Plugins
4. https://support.hdfgroup.org/services/contributions.html#filters
5. https://support.hdfgroup.org/HDF5/doc/RM/RM_H5.html
6. https://confluence.hdfgroup.org/display/HDF5/Filters
7. https://numcodecs.readthedocs.io/en/stable/
# Appendix A. HDF5 Parameter Encode/Decode {#filters_appendixa}
The filter id for an HDF5 format filter is an unsigned integer.
Further, the parameters passed to an HDF5 format filter are encoded internally as a vector of 32-bit unsigned integers.
It may be that the parameters required by a filter can naturally be encoded as unsigned integers.
The bzip2 compression filter, for example, expects a single integer value from zero thru nine.
This encodes naturally as a single unsigned integer.
Note that signed integers and single-precision (32-bit) float values also can easily be represented as 32 bit unsigned integers by proper casting to an unsigned integer so that the bit pattern is preserved.
Simple integer values of type short or char (or the unsigned versions) can also be mapped to an unsigned integer by truncating to 16 or 8 bits respectively and then zero extending.
Machine byte order (aka endian-ness) is an issue for passing some kinds of parameters.
You might define the parameters when compressing on a little endian machine, but later do the decompression on a big endian machine.
When using HDF5 format filters, byte order is not an issue for 32-bit values because HDF5 takes care of converting them between the local machine byte order and network byte order.
Parameters whose size is larger than 32-bits present a byte order problem.
This specifically includes double precision floats and (signed or unsigned) 64-bit integers.
For these cases, the machine byte order issue must be handled, in part, by the compression code.
This is because HDF5 will treat, for example, an unsigned long long as two 32-bit unsigned integers and will convert each to network order separately.
This means that on a machine whose byte order is different than the machine in which the parameters were initially created, the two integers will be separately
endian converted.
But this will be incorrect for 64-bit values.
So, we have this situation (for HDF5 only):
1. the 8 bytes come in as native machine order for the machine doing the call to *nc_def_var_filter*.
2. HDF5 divides the 8 bytes into 2 four byte pieces and ensures that each piece is in network (big) endian order.
3. When the filter is called, the two pieces are returned in the same order but with the bytes in each piece consistent with the native machine order for the machine executing the filter.
## Encoding Algorithms for HDF5
In order to properly extract the correct 8-byte value, we need to ensure that the values stored in the HDF5 file have a known format independent of the native format of the creating machine.
The idea is to do sufficient manipulation so that HDF5 will store the 8-byte value as a little endian value divided into two 4-byte integers.
Note that little-endian is used as the standard because it is the most common machine format.
When read, the filter code needs to be aware of this convention and do the appropriate conversions.
This leads to the following set of rules.
### Encoding
1. Encode on little endian (LE) machine: no special action is required.
The 8-byte value is passed to HDF5 as two 4-byte integers.
HDF5 byte swaps each integer and stores it in the file.
2. Encode on a big endian (BE) machine: several steps are required:
1. Do an 8-byte byte swap to convert the original value to little-endian format.
2. Since the encoding machine is BE, HDF5 will just store the value.
So it is necessary to simulate little endian encoding by byte-swapping each 4-byte integer separately.
3. This doubly swapped pair of integers is then passed to HDF5 and is stored unchanged.
### Decoding
1. Decode on LE machine: no special action is required.
HDF5 will get the two 4-bytes values from the file and byte-swap each separately.
The concatenation of those two integers will be the expected LE value.
2. Decode on a big endian (BE) machine: the inverse of the encode case must be implemented.
1. HDF5 sends the two 4-byte values to the filter.
2. The filter must then byte-swap each 4-byte value independently.
3. The filter then must concatenate the two 4-byte values into a single 8-byte value.
Because of the encoding rules, this 8-byte value will be in LE format.
4. The filter must finally do an 8-byte byte-swap on that 8-byte value to convert it to desired BE format.
To support these rules, some utility programs exist and are discussed in <a href="#filters_appendixb">Appendix B</a>.
# Appendix B. Support Utilities {#filters_appendixb}
Several functions are exported from the netcdf-c library for use by client programs and by filter implementations.
They are defined in the header file __netcdf_aux.h__.
The h5 tag indicates that they assume that the result of the parse is a set of unsigned integers &mdash; the format used by HDF5.
1. ````int ncaux_h5filterspec_parse(const char* txt, unsigned int* idp. size_t* nparamsp, unsigned int** paramsp);````
* txt contains the text of a sequence of comma separated constants
* idp will contain the first constant &mdash; the filter id
* nparamsp will contain the number of params
* paramsp will contain a vector of params &mdash; the caller must free
This function can parse single filter spec strings as defined in the section on <a href="#filters_syntax">Filter Specification Syntax</a>.
2. ````int ncaux_h5filterspec_parselist(const char* txt, int* formatp, size_t* nspecsp, struct NC_H5_Filterspec*** vectorp);````
* txt contains the text of a sequence '|' separated filter specs.
* formatp currently always returns 0.
* nspecsp will return the number of filter specifications.
* vectorp will return a pointer to a vector of pointers to filter specification instances &mdash; the caller must free.
This function parses a sequence of filter specifications each separated by a '|' character.
The text between '|' separators must be parsable by __ncaux_h5filterspec_parse__.
3. ````void ncaux_h5filterspec_free(struct NC_H5_Filterspec* f);````
* f is a pointer to an instance of ````struct NC_H5_Filterspec````
Typically this was returned as an element of the vector returned
by __ncaux_h5filterspec_parselist__.
This reclaims the parameters of the filter spec object as well as the object itself.
4. ````int ncaux_h5filterspec_fix8(unsigned char* mem8, int decode);````
* mem8 is a pointer to the 8-byte value either to fix.
* decode is 1 if the function should apply the 8-byte decoding algorithm
else apply the encoding algorithm.
This function implements the 8-byte conversion algorithms for HDF5.
Before calling *nc_def_var_filter* (unless *NC_parsefilterspec* was used), the client must call this function with the decode argument set to 0.
Inside the filter code, this function should be called with the decode argument set to 1.
Examples of the use of these functions can be seen in the test program ''nc_test4/tst_filterparser.c''.
Some of the above functions use a C struct defined in _netcdf_filter.h_.
The definition of that struct is as follows.
````
typedef struct NC_H5_Filterspec {
unsigned int filterid; /* ID for arbitrary filter. */
size_t nparams; /* nparams for arbitrary filter. */
unsigned int* params; /* Params for arbitrary filter. */
} NC_H5_Filterspec;
````
This struct in effect encapsulates all of the information about and HDF5 formatted filter &mdash; the id, the number of parameters, and the parameters themselves.
# Appendix C. Build Flags for Detecting the Filter Mechanism {#filters_appendixc}
The include file _netcdf_meta.h contains the following definition.
````
#define NC_HAS_MULTIFILTERS 1
````
This, in conjunction with the error code _NC_ENOFILTER_ in _netcdf.h_ can be used to see what filter mechanism is in place as described in the section on <a href="#filters_compatibility">incompatibities</a>.
1. !defined(NC_ENOFILTER) && !defined(NC_HAS_MULTIFILTERS) &mdash; indicates that the old pre-4.7.4 mechanism is in place.
It does not support multiple filters.
2. defined(NC_ENOFILTER) && !defined(NC_HAS_MULTIFILTERS) &mdash; indicates that the 4.7.4 mechanism is in place.
It does support multiple filters, but the error return codes for _nc_inq_var_filter_ are different and the filter spec parser functions are in a different location with different names.
3. defined(NC_ENOFILTER) && defined(NC_HAS_MULTIFILTERS) &mdash; indicates that the multiple filters are supported, and that _nc_inq_var_filter_ returns a filterid of zero to indicate that a variable has no filters.
Also, the filter spec parsers have the names and signatures described in this document and are define in _netcdf_aux.h_.
# Appendix D. BNF for Specifying Filters in Utilities {#filters_appendixd}
````
speclist: spec
| speclist '|' spec
;
spec: filterid
| filterid ',' parameterlist
;
filterid: unsigned32
;
parameterlist: parameter
| parameterlist ',' parameter
;
parameter: unsigned32
where
unsigned32: <32 bit unsigned integer>
````
# Appendix E. Codec API {#filters_appendixe}
The Codec API mirrors the HDF5 API closely. It has one well-known function that can be invoked to obtain information about the Codec as well as pointers to special functions to perform conversions.
## The Codec Plugin API
### NCZ_get_codec_info
This function returns a pointer to a C struct that provides detailed information about the codec plugin.
#### Signature
````
void* NCZ_get_codec_info(void);
````
The value returned is actually of type ''struct NCZ_codec_t'',
but is of type ''void*'' to allow for extensions.
### NCZ_codec_t
````
typedef struct NCZ_codec_t {
int version; /* Version number of the struct */
int sort; /* Format of remainder of the struct;
Currently always NCZ_CODEC_HDF5 */
const char* codecid; /* The name/id of the codec */
unsigned int hdf5id; /* corresponding hdf5 id */
void (*NCZ_codec_initialize)(void);
void (*NCZ_codec_finalize)(void);
int (*NCZ_codec_to_hdf5)(const char* codec, int* nparamsp, unsigned** paramsp);
int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp);
int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* nparamsp, unsigned** paramsp);
} NCZ_codec_t;
````
The semantics of the non-function fields is as follows:
1. ''version'' -- Version number of the struct.
2. ''sort'' -- Format of remainder of the struct; currently always NCZ_CODEC_HDF5.
3. ''codecid'' -- The name/id of the codec.
4. ''hdf5id'' -- The corresponding hdf5 id.
### NCZ_codec_to_hdf5
Given a JSON Codec representation, it will return a corresponding vector of unsigned integers representing the
visible parameters.
#### Signature
int (*NCZ_codec_to_hdf)(const char* codec, int* nparamsp, unsigned** paramsp);
#### Arguments
1. codec -- (in) ptr to JSON string representing the codec.
2. nparamsp -- (out) store the length of the converted HDF5 unsigned vector
3. paramsp -- (out) store a pointer to the converted HDF5 unsigned vector; caller must free the returned vector. Note the double indirection.
Return Value: a netcdf-c error code.
### NCZ_hdf5_to_codec
Given an HDF5 visible parameters vector of unsigned integers and its length,
return a corresponding JSON codec representation of those visible parameters.
#### Signature
int (*NCZ_hdf5_to_codec)(int ncid, int varid, size_t nparams, const unsigned* params, char** codecp);
#### Arguments
1. ncid -- the variables' containing group
2. varid -- the containing variable
3. nparams -- (in) the length of the HDF5 visible parameters vector
4. params -- (in) pointer to the HDF5 visible parameters vector.
5. codecp -- (out) store the string representation of the codec; caller must free.
Return Value: a netcdf-c error code.
### NCZ_modify_parameters
Extract environment information from the (ncid,varid) and use it to convert a set of visible parameters
to a set of working parameters; also provide option to modify visible parameters.
#### Signature
int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
#### Arguments
1. ncid -- (in) group id containing the variable.
2. varid -- (in) the id of the variable to which this filter is being attached.
3. vnparamsp -- (in/out) the count of visible parameters
4. vparamsp -- (in/out) the set of visible parameters
5. wnparamsp -- (out) the count of working parameters
4. wparamsp -- (out) the set of working parameters
Return Value: a netcdf-c error code.
### NCZ_codec_initialize
Some compressors may require library initialization.
This function is called as soon as a shared library is loaded and matched with an HDF5 filter.
#### Signature
int (*NCZ_codec_initialize)(void);
Return Value: a netcdf-c error code.
### NCZ_codec_finalize
Some compressors (like blosc) require invoking a finalize function in order to avoid memory loss.
This function is called during a call to ''nc_finalize'' to do any finalization.
If the client code does not invoke ''nc_finalize'' then memory checkers may complain about lost memory.
#### Signature
int (*NCZ_codec_finalize)(void);
Return Value: a netcdf-c error code.
## Multi-Codec API
As an aid to clients, it is convenient if a single shared library can provide multiple ''NCZ_code_t'' instances at one time.
This API is not intended to be used by plugin developers.
A shared library must only export this function.
### NCZ_codec_info_defaults
Return a NULL terminated vector of pointers to instances of ''NCZ_codec_t''.
#### Signature
void* NCZ_codec_info_defaults(void);
The value returned is actually of type ''NCZ_codec_t**'',
but is of type ''void*'' to allow for extensions.
The list of returned items are used to try to provide defaults
for any HDF5 filters that have no corresponding Codec.
This is for internal use only.
# Appendix F. Pre-built Filters
As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory.
They may be in that directory or the "plugins/.libs" subdirectory.
It may be possible for users to utilize some of those libraries to provide filter support for general use.
One simple way to reuse these libraries is to make the environment variable "HDF5_PLUGIN_PATH" refer to the plugin directory or the subdirectory, whichever contains the shared libraries.
This may not be possible if the user already has other filter shared libraries in use.
The second way is to copy the necessary shared libraries from the plugins directory to the user's HDF5_PLUGIN_PATH directory.
If the user is using HDF5, then the following filters are probably usable:
* ''libh5bzip2.so'' -- an HDF5 filter for bzip2 compression
* ''libh5blosc.so'' -- an HDF5 filter for blosc compression
If the user is using NCZarr filters, then you can install the following additional shared libraries:
* libh5shuffle.so -- shuffle filter
* libh5fletcher32.so -- fletcher32 checksum
* libh5deflate.so -- deflate compression
* libnczdefaults.so -- provide NCZarr support for shuffle, fletcher32, and deflate.
The shuffle, fletcher32, and deflate filters in this case will be ignored by HDF5 and only used by the NCZarr code.
But in order to use them, it needs additional Codec capabilities provided by the libnczdefauts.so shared library.
Note also that if you disable HDF5 support, but leave NCZarr support enabled, then all of the above filters
should continue to work.
# Point of Contact {#filters_poc}
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu<br>
__Initial Version__: 1/10/2018<br>
__Last Revised__: 7/17/2021

27
docs/nczarr.md
View File

@ -17,6 +17,10 @@ A note on terminology in this document.
1. The term "dataset" is used to refer to all of the Zarr objects constituting
the meta-data and data.
There are some important "caveats" of which to be aware when using this software.
1. NCZarr currently is not thread-safe. So any attempt to use it with parallelism, including MPIO, is likely to fail.
# The NCZarr Data Model {#nczarr_data_model}
NCZarr uses a data model <a href="#ref_nczarr">[4]</a> that, by design, extends the Zarr Version 2 Specification <a href="#ref_zarrv2">[6]</a> to add support for the NetCDF-4 data model.
@ -456,6 +460,8 @@ collections — High-performance dataset datatypes](https://docs.python.org/2/li
<a name="ref_xarray">[8]</a> [Dynamic Filter Loading](https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf)<br>
<a name="ref_xarray">[9]</a> [Officially Registered Custom HDF5 Filters](https://portal.hdfgroup.org/display/support/Registered+Filter+Plugins)<br>
<a name="ref_xarray">[10]</a> [C-Blosc Compressor Implementation](https://github.com/Blosc/c-blosc)
<a name="ref_awssdk_conda">[11]</a> [Conda-forge / packages / aws-sdk-cpp]
(https://anaconda.org/conda-forge/aws-sdk-cpp)<br>
# Appendix A. Building NCZarr Support {#nczarr_build}
@ -548,17 +554,18 @@ it any difficulties were reported to Unidata as a Github issue.
In order to use the S3 storage driver, it is necessary to install the Amazon [aws-sdk-cpp library](https://github.com/aws/aws-sdk-cpp.git).
As a starting point, here are the CMake options used by Unidata to build that library.
It assumes that it is being executed in a build directory, `build` say, and that `build/../CMakeLists.txt exists`.
```
cmake -DBUILD_ONLY=s3
```
The expected set of installed libraries are as follows:
* aws-cpp-sdk-s3
* aws-cpp-sdk-core
Building this package from scratch has proven to be a formidable task.
This appears to be due to dependencies on very specific versions of,
for example, openssl.
This library depends on libcurl, so you may need to install that
before building the sdk library.
It is recommended that you use conda to install this package on linux.
See [11] for the relevant conda package information.
For Windows we do not yet have solution. If you successfully install
on Windows, please let us know how you did it.
This library depends on libcurl and openssl, so you may need to install those
before installing the sdk library.
# Appendix C. Amazon S3 Imposed Limits {#nczarr_s3limits}

8
include/Makefile.am
View File

@ -7,7 +7,7 @@
# Ed Hartnett, Dennis Heimbigner, Ward Fisher
include_HEADERS = netcdf.h netcdf_meta.h netcdf_mem.h netcdf_aux.h \
netcdf_filter.h netcdf_filter_build.h netcdf_dispatch.h
netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h netcdf_dispatch.h netcdf_json.h
if BUILD_PARALLEL
include_HEADERS += netcdf_par.h
@ -31,3 +31,9 @@ noinst_HEADERS += nchttp.h
endif
EXTRA_DIST = CMakeLists.txt XGetopt.h netcdf_meta.h.in netcdf_dispatch.h.in
BUILT_SOURCES = netcdf_json.h
netcdf_json.h: Makefile ncjson.h ${srcdir}/../libdispatch/ncjson.c
sed -e 's/NCJSON_H/NETCDF_JSON_H/' -e '/ncjson.h/d' <ncjson.h > $@
sed -e '/ncjson.h/d' < ${srcdir}/../libdispatch/ncjson.c >> $@

2
include/hdf5internal.h
View File

@ -194,6 +194,8 @@ struct NC_HDF5_Filter {
unsigned int* params; /**< Params for arbitrary filter. */
};
int NC4_hdf5_filter_initialize(void);
int NC4_hdf5_filter_finalize(void);
int NC4_hdf5_filter_remove(NC_VAR_INFO_T* var, unsigned int id);
int NC4_hdf5_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NC_HDF5_Filter** fi);
int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params, int flags);

11
include/nc4internal.h
View File

@ -100,8 +100,7 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP, NCFIL} NC_SORT;
/** Hidden attributes; immutable and unreadable thru API. */
#define HIDDENATTRFLAG 1
/** Readonly global attributes; readable, but immutable thru the
* API. */
/** Readonly attributes; readable, but immutable thru the API. */
#define READONLYFLAG 2
/** Subset of readonly flags; readable by name only thru the API. */
@ -110,6 +109,9 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP, NCFIL} NC_SORT;
/** Subset of readonly flags; Value is actually in file. */
#define MATERIALIZEDFLAG 8
/** Per-variable attribute, as opposed to global */
#define VARFLAG 16
/** Boolean type, to make the code easier to read. */
typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t;
@ -201,9 +203,9 @@ typedef struct NC_VAR_INFO
int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */
nc_bool_t shuffle; /**< True if var has shuffle filter applied. */
nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */
size_t chunk_cache_size; /**< Size in bytes of the var chunk chache. */
size_t chunk_cache_size; /**< Size in bytes of the var chunk cache. */
size_t chunk_cache_nelems; /**< Number of slots in var chunk cache. */
float chunk_cache_preemption; /**< Chunk cache preemtion policy. */
float chunk_cache_preemption; /**< Chunk cache preemption policy. */
void *format_var_info; /**< Pointer to any binary format info. */
void* filters; /**< Record of the list of filters to be applied to var data; format dependent */
} NC_VAR_INFO_T;
@ -461,6 +463,7 @@ extern const NC_reservedatt* NC_findreserved(const char* name);
#define NC_ATT_DIMID_NAME "_Netcdf4Dimid"
#define NC_ATT_NC3_STRICT_NAME "_nc3_strict"
#define NC_XARRAY_DIMS "_ARRAY_DIMENSIONS"
#define NC_ATT_CODECS "_Codecs"
#define NC_NCZARR_ATTR "_NCZARR_ATTR"
#endif /* _NC4INTERNAL_ */

11
include/ncconfigure.h
View File

@ -71,6 +71,9 @@ extern int fileno(FILE*);
#ifndef HAVE_STRLCAT
#define strlcat(d,s,n) strcat_s((d),(n),(s))
#endif
#ifndef HAVE_STRCASECMP
#define strcasecmp _stricmp
#endif
#endif
/* handle null arguments */
@ -136,5 +139,13 @@ typedef long long fileoffset_t;
#define NC_UNUSED(var) (void)var
#endif
/* Protect old HDF5 code (pre 1.8.12) */
#ifndef HAVE_H5ALLOCATE_MEMORY
#ifndef H5allocate_memory
#define H5allocate_memory(size,clear) ((clear)?calloc(1,(size)):malloc(size))
#define H5free_memory(buf) free(buf)
#define H5resize_memory(buf,size) realloc(buf,size)
#endif
#endif
#endif /* NCCONFIGURE_H */

122
include/ncjson.h
View File

@ -1,20 +1,19 @@
/* Copyright 2018, UCAR/Unidata.
Copyright 2018 Unidata
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
See the COPYRIGHT file for more information.
*/
#ifndef NCJSON_H
#define NCJSON_H 1
#ifndef DLLEXPORT
#ifdef _WIN32
#define DLLEXPORT __declspec(dllexport)
#else
#define DLLEXPORT
#endif
#endif
/**************************************************/
/* Json object sorts (note use of term sort rather than e.g. type or discriminant) */
#define NCJ_UNDEF 0
#define NCJ_STRING 1
@ -29,12 +28,9 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
/* No flags are currently defined, but the argument is a placeholder */
/* Define a struct to store primitive values
as unquoted strings. The sort will
provide more info.
Do not bother with a union since
the amount of saved space is minimal.
/* Define a struct to store primitive values as unquoted
strings. The sort will provide more info. Do not bother with
a union since the amount of saved space is minimal.
*/
typedef struct NCjson {
@ -46,62 +42,65 @@ typedef struct NCjson {
} list; /* sort == DICT|ARRAY */
} NCjson;
/* Support Windows declspec */
#ifndef EXTERNL
# ifdef _WIN32
# ifdef NCJSON_INTERNAL /* define when compiling code */
# define EXTERNL __declspec(dllexport) extern
# else
# define EXTERNL __declspec(dllimport) extern
# endif
# else /* !_WIN32 */
# define EXTERNL extern
# endif
#endif /* !defined EXTERNL */
/* Structure to hold result of convertinf one json sort to value of another type;
don't use union so we can know when to reclaim sval
*/
struct NCJconst {int bval; long long ival; double dval; char* sval;};
/**************************************************/
/* Extended API */
/* Return 0 if ok else -1 */
#if defined(__cplusplus)
extern "C" {
#endif
/* int return value is either 1 (ok) or 0 (failure) */
/* Parse a JSON string */
DLLEXPORT int NCJparse(const char* text, unsigned flags, NCjson** jsonp);
/* Parse */
EXTERNL int NCJparse(const char* text, unsigned flags, NCjson** jsonp);
/* Reclaim a JSON tree */
DLLEXPORT extern void NCJreclaim(NCjson* json);
/* Build */
EXTERNL int NCJnew(int sort, NCjson** object);
/* Create a new JSON node of a given sort */
DLLEXPORT extern int NCJnew(int sort, NCjson** objectp);
/* Recursively free NCjson instance */
EXTERNL void NCJreclaim(NCjson*);
/* Create new json object with given string content */
DLLEXPORT extern int NCJnewstring(int sort, const char* value, NCjson** jsonp);
/* Assign a nul terminated string value to an NCjson object as its contents */
EXTERNL int NCJnewstring(int sort, const char* value, NCjson** jsonp);
/* Create new json object with given counted string content */
DLLEXPORT extern int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);
/* Assign a counted string value to an NCjson object as its contents */
EXTERNL int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);
/* Append value to an array or dict object. */
EXTERNL int NCJappend(NCjson* object, NCjson* value);
/* Insert key-value pair into a dict object. key will be copied */
EXTERNL int NCJinsert(NCjson* object, char* key, NCjson* value);
/* Unparser to convert NCjson object to text in buffer */
EXTERNL int NCJunparse(const NCjson* json, unsigned flags, char** textp);
/* Utilities */
EXTERNL int NCJaddstring(NCjson*, int sort, const char* s);
EXTERNL int NCJdictget(const NCjson* dict, const char* key, NCjson** valuep);
/* dump NCjson* object to output file */
EXTERNL void NCJdump(const NCjson* json, unsigned flags, FILE*);
/* Get dict key value by name */
DLLEXPORT extern int NCJdictget(const NCjson* dict, const char* key, NCjson** valuep);
/* Convert one json sort to value of another type; don't use union so we can know when to reclaim sval */
struct NCJconst {int bval; long long ival; double dval; char* sval;};
EXTERNL int NCJcvt(const NCjson* value, int outsort, struct NCJconst* output);
DLLEXPORT extern int NCJcvt(const NCjson* value, int outsort, struct NCJconst* output);
#ifndef NETCDF_JSON_H
/* Insert an atomic value to an array or dict object. */
DLLEXPORT int NCJaddstring(NCjson* json, int sort, const char* s);
/* Append value to an array or dict object. */
DLLEXPORT extern int NCJappend(NCjson* object, NCjson* value);
/* Insert key-value pair into a dict object. key will be copied */
DLLEXPORT extern int NCJinsert(NCjson* object, char* key, NCjson* value);
/* Unparser to convert NCjson object to text in buffer */
DLLEXPORT extern int NCJunparse(const NCjson* json, unsigned flags, char** textp);
/* Deep clone a json object */
EXTERNL int NCJclone(const NCjson* json, NCjson** clonep);
DLLEXPORT extern int NCJclone(const NCjson* json, NCjson** clonep);
/* dump NCjson* object to output file */
DLLEXPORT extern void NCJdump(const NCjson* json, unsigned flags, FILE*);
#endif
#if defined(__cplusplus)
}
#endif
/* Getters */
#define NCJsort(x) ((x)->sort)
@ -119,8 +118,7 @@ EXTERNL int NCJclone(const NCjson* json, NCjson** clonep);
/* Misc */
#define NCJisatomic(j) ((j)->sort != NCJ_ARRAY && (j)->sort != NCJ_DICT && (j)->sort != NCJ_NULL && (j)->sort != NCJ_UNDEF)
#if defined(__cplusplus)
}
#endif
/**************************************************/
#endif /*NCJSON_H*/

1
include/nclist.h
View File

@ -22,6 +22,7 @@ typedef struct NClist {
EXTERNL NClist* nclistnew(void);
EXTERNL int nclistfree(NClist*);
EXTERNL int nclistfreeall(NClist*);
EXTERNL int nclistclearall(NClist*);
EXTERNL int nclistsetalloc(NClist*,size_t);
EXTERNL int nclistsetlength(NClist*,size_t);

1
include/netcdf_aux.h
View File

@ -66,6 +66,7 @@ typedef struct NC_H5_Filterspec {
EXTERNL int ncaux_h5filterspec_parse(const char* txt, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp);
EXTERNL int ncaux_h5filterspec_parselist(const char* txt0, int* formatp, size_t* nspecsp, struct NC_H5_Filterspec*** vectorp);
EXTERNL int ncaux_h5filterspec_parse_parameter(const char* txt, size_t* nuiparamsp, unsigned int* uiparams);
EXTERNL void ncaux_h5filterspec_free(struct NC_H5_Filterspec* f);
EXTERNL void ncaux_h5filterspec_fix8(unsigned char* mem, int decode);

3
include/netcdf_filter.h
View File

@ -58,12 +58,11 @@ EXTERNL int nc_inq_var_filter_ids(int ncid, int varid, size_t* nfilters, unsigne
/* Learn about the filter with specified id wrt a variable */
EXTERNL int nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params);
/* End HDF5 Format Declarations */
/**************************************************/
#if defined(__cplusplus)
}
#endif
/**************************************************/
#endif /* NETCDF_FILTER_H */

167
include/netcdf_filter_build.h
View File

@ -20,107 +20,7 @@
#ifndef NETCDF_FILTER_BUILD_H
#define NETCDF_FILTER_BUILD_H 1
/**************************************************/
/* Build To the HDF5 C-API for Filters */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
/* Support headers */
#include <netcdf.h>
#include <netcdf_filter.h>
#ifdef USE_HDF5
#include <hdf5.h>
/* Older versions of the hdf library may define H5PL_type_t here */
#include <H5PLextern.h>
#else /*!USE_HDF5*/ /* Provide replacement definitions */
/* WARNING: In order make NCZARR independent of HDF5,
while still using HDF5-style filters, some HDF5
declarations need to be duplicated here with
different names. Watch out for changes in
the underlying HDF5 declarations.
See the file H5Zpublic.h for more detailed descriptions.
Note that these declarations are always enabled because
HDF5-style filters may have been created with these definitions
but for use by HDF5.
Note also that certain filters in the plugins directory will not build if HDF5 is not installed:
notably blosc.
*/
/* H5Z_FILTER_RESERVED => H5Z_FILTER_RESERVED */
#define H5Z_FILTER_RESERVED 256 /*filter ids below this value are reserved for library use */
/* H5Z_FILTER_MAX => H5Z_FILTER_MAX */
#define H5Z_FILTER_MAX 65535 /*maximum filter id */
/* Only a limited set of definition and invocation flags are allowed */
#define H5Z_FLAG_MANDATORY 0x0000 /*filter is mandatory */
#define H5Z_FLAG_OPTIONAL 0x0001 /*filter is optional */
#define H5Z_FLAG_REVERSE 0x0100 /*reverse direction; read */
typedef int htri_t;
typedef int herr_t;
typedef size_t hsize_t;
typedef long long hid_t;
#define H5allocate_memory(size,n) malloc(size)
#define H5free_memory(buf) free(buf)
/* htri_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id) => currently not supported; must be NULL. */
typedef htri_t (*H5Z_can_apply_func_t)(long long, long long, long long);
/* herr_t (*H5Z_set_local_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id); => currently not supported; must be NULL. */
typedef herr_t (*H5Z_set_local_func_t)(long long, long long, long long);
/* H5Z_funct_t => H5Z_filter_func_t */
typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
typedef int H5Z_filter_t;
#define H5Z_CLASS_T_VERS 1
/*
* The filter table maps filter identification numbers to structs that
* contain a pointers to the filter function and timing statistics.
*/
typedef struct H5Z_class2_t {
int version; /* Version number of the struct; should be H5Z_FILTER_CLASS_VER */
H5Z_filter_t id; /* Filter ID number */
unsigned encoder_present; /* Does this filter have an encoder? */
unsigned decoder_present; /* Does this filter have a decoder? */
const char *name; /* Comment for debugging */
H5Z_can_apply_func_t can_apply; /* The "can apply" callback for a filter */
H5Z_set_local_func_t set_local; /* The "set local" callback for a filter */
H5Z_func_t filter; /* The actual filter function */
} H5Z_class2_t;
/* The HDF5/H5Zarr dynamic loader looks for the following:*/
/* Plugin type used by the plugin library */
typedef enum H5PL_type_t {
H5PL_TYPE_ERROR = -1, /* Error */
H5PL_TYPE_FILTER = 0, /* Filter */
H5PL_TYPE_NONE = 1 /* This must be last! */
} H5PL_type_t;
#endif /*HAVE_HDF5_H*/
/* Following External Discovery Functions should be present for the dynamic loading of filters */
/* returns specific constant H5ZP_TYPE_FILTER */
typedef H5PL_type_t (*H5PL_get_plugin_type_proto)(void);
/* return <pointer to instance of H5Z_filter_class> */
typedef const void* (*H5PL_get_plugin_info_proto)(void);
#include "netcdf_filter_hdf5_build.h"
/**************************************************/
/* Build To a NumCodecs-style C-API for Filters */
@ -139,62 +39,65 @@ typedef const void* (*H5PL_get_plugin_info_proto)(void);
/*
Obtain a pointer to an instance of NCZ_codec_class_t.
NCZ_get_plugin_info(void) -- returns pointer to instance of NCZ_codec_class_t.
NCZ_get_codec_info(void) -- returns pointer to instance of NCZ_codec_class_t.
Instance an be recast based on version+sort to the plugin type specific info.
So the void* return value is typically actually of type NCZ_codec_class_t*.
*/
typedef const void* (*NCZ_get_plugin_info_proto)(void);
typedef const void* (*NCZ_get_codec_info_proto)(void);
/* The current object returned by NCZ_get_plugin_info is a
pointer to an instance of NCZ_codec_t.
The key to this struct are the four function pointers that do setup/reset/finalize
The key to this struct are the several function pointers that do initialize/finalize
and conversion between codec JSON and HDF5 parameters.
Setup context state for the codec converter
int (*NCZ_codec_setup)(int ncid, int varid, void** contextp);
The function pointers defined in NCZ_codec_t manipulate HDF5 parameters and NumCodec JSON.
@param ncid -- (in) ncid of the variable's group
@param varid -- (in) varid of the variable
@params contextp -- (out) context for this (var,codec) combination.
@return -- a netcdf-c error code.
* Initialize use of the filter. This is invoked when a filter is loaded.
Reclaim any codec resources from setup. Not same as finalize.
int (*NCZ_codec_reset)(void* context);
void (*NCZ_codec_initialize)(void);
@param context -- (in) context state
* Finalize use of the filter. Since HDF5 does not provide this functionality, the codec may need to do it.
See H5Zblosc.c for an example. This function is invoked when a filter is unloaded.
Finalize use of the plugin. Since HDF5 does not provide this functionality,
the codec may need to do it. See H5Zblosc.c for an example.
void (*NCZ_codec_finalize)(void);
@param context -- (in) context state
* Convert a JSON representation to an HDF5 representation. Invoked when a NumCodec JSON Codec is extracted
from Zarr metadata.
Convert a JSON representation to an HDF5 representation:
int (*NCZ_codec_to_hdf5)(void* context, const char* codec, int* nparamsp, unsigned** paramsp);
int (*NCZ_codec_to_hdf5)(const char* codec, int* nparamsp, unsigned** paramsp);
@param context -- (in) context state from setup.
@param codec -- (in) ptr to JSON string representing the codec.
@param nparamsp -- (out) store the length of the converted HDF5 unsigned vector
@param paramsp -- (out) store a pointer to the converted HDF5 unsigned vector;
caller frees. Note the double indirection.
@return -- a netcdf-c error code.
Convert an HDF5 representation to a JSON representation
int (*NCZ_hdf5_to_codec)(void* context, int nparams, const unsigned* params, char** codecp);
@param context -- (in) context state from setup.
* Convert an HDF5 vector of visible parameters to a JSON representation.
int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp);
@param nparams -- (in) the length of the HDF5 unsigned vector
@param params -- (in) pointer to the HDF5 unsigned vector.
@param codecp -- (out) store the string representation of the codec; caller must free.
@return -- a netcdf-c error code.
*/
/* QUESTION? do we want to provide a netcdf-specific
alternative to H5Z_set_local since NCZarr may not have HDF5 access?
HDF5: herr_t set_local(hid_t dcpl, hid_t type, hid_t space);
Proposed netcdf equivalent: int NCZ_set_local(int ncid, int varid, int* nparamsp, unsigned** paramsp);
where ncid+varid is equivalent to the space.
* Convert a set of visible parameters to a set of working parameters using extra environmental information.
Also allows for changes to the visible parameters. Invoked before filter is actually used.
int (*NCZ_build_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
@param ncid -- (in) ncid of the variable's group
@param varid -- (in) varid of the variable
@params vnparamsp -- (in/out) number of visible parameters
@params vparamsp -- (in/out) vector of visible parameters
@params wnparamsp -- (out) number of working parameters
@params wparamsp -- (out) vector of working parameters
@return -- a netcdf-c error code.
* Convert a set of working parameters to a set of visible parameters using extra environmental information, if needed.
Invoked before filter metadata is written.
*/
/*
@ -208,11 +111,11 @@ typedef struct NCZ_codec_t {
Currently always NCZ_CODEC_HDF5 */
const char* codecid; /* The name/id of the codec */
unsigned int hdf5id; /* corresponding hdf5 id */
int (*NCZ_codec_to_hdf5)(void* context, const char* codec, int* nparamsp, unsigned** paramsp);
int (*NCZ_hdf5_to_codec)(void* context, int nparams, const unsigned* params, char** codecp);
int (*NCZ_codec_setup)(int ncid, int varid, void** contextp);
int (*NCZ_codec_reset)(void* context);
void (*NCZ_codec_initialize)(void);
void (*NCZ_codec_finalize)(void);
int (*NCZ_codec_to_hdf5)(const char* codec, size_t* nparamsp, unsigned** paramsp);
int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp);
int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
} NCZ_codec_t;
#ifndef NC_UNUSED

207
include/netcdf_filter_hdf5_build.h Normal file
View File

@ -0,0 +1,207 @@
/* Copyright 2018, UCAR/Unidata and OPeNDAP, Inc.
See the COPYRIGHT file for more information. */
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/hdf5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* This include file is used if one wished to build a filter plugin
independent of HDF5. See examples in the plugins directory
*/
#ifndef NETCDF_FILTER_HDF5_BUILD_H
#define NETCDF_FILTER_HDF5_BUILD_H 1
/**************************************************/
/* Build To the HDF5 C-API for Filters */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
/* Support headers */
#include <netcdf.h>
#include <netcdf_filter.h>
#ifdef USE_HDF5
#include <hdf5.h>
/* Older versions of the hdf library may define H5PL_type_t here */
#include <H5PLextern.h>
#else /*!USE_HDF5*/ /* Provide replacement definitions */
/* WARNING: In order make NCZARR independent of HDF5,
while still using HDF5-style filters, some HDF5
declarations need to be duplicated here with
different names. Watch out for changes in
the underlying HDF5 declarations.
See the file H5Zpublic.h for more detailed descriptions.
Note that these declarations are always enabled because
HDF5-style filters may have been created with these definitions
but for use by HDF5.
Note also that certain filters in the plugins directory will not build if HDF5 is not installed:
notably blosc.
*/
/* H5Z_FILTER_RESERVED => H5Z_FILTER_RESERVED */
#define H5Z_FILTER_RESERVED 256 /*filter ids below this value are reserved for library use */
/* H5Z_FILTER_MAX => H5Z_FILTER_MAX */
#define H5Z_FILTER_MAX 65535 /*maximum filter id */
/* Only a limited set of definition and invocation flags are allowed */
#define H5Z_FLAG_MANDATORY 0x0000 /*filter is mandatory */
#define H5Z_FLAG_OPTIONAL 0x0001 /*filter is optional */
#define H5Z_FLAG_REVERSE 0x0100 /*reverse direction; read */
#define H5Z_FLAG_SKIP_EDC 0x0200 /*skip EDC filters for read */
typedef int htri_t;
typedef int herr_t;
typedef int hbool_t;
typedef size_t hsize_t;
typedef long long hid_t;
/* htri_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id) => currently not supported; must be NULL. */
typedef htri_t (*H5Z_can_apply_func_t)(long long, long long, long long);
/* herr_t (*H5Z_set_local_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id); => currently not supported; must be NULL. */
typedef herr_t (*H5Z_set_local_func_t)(long long, long long, long long);
/* H5Z_funct_t => H5Z_filter_func_t */
typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
typedef int H5Z_filter_t;
#define H5Z_CLASS_T_VERS 1
/*
* The filter table maps filter identification numbers to structs that
* contain a pointers to the filter function and timing statistics.
*/
typedef struct H5Z_class2_t {
int version; /* Version number of the struct; should be H5Z_FILTER_CLASS_VER */
H5Z_filter_t id; /* Filter ID number */
unsigned encoder_present; /* Does this filter have an encoder? */
unsigned decoder_present; /* Does this filter have a decoder? */
const char *name; /* Comment for debugging */
H5Z_can_apply_func_t can_apply; /* The "can apply" callback for a filter */
H5Z_set_local_func_t set_local; /* The "set local" callback for a filter */
H5Z_func_t filter; /* The actual filter function */
} H5Z_class2_t;
/* The HDF5/H5Zarr dynamic loader looks for the following:*/
/* Plugin type used by the plugin library */
typedef enum H5PL_type_t {
H5PL_TYPE_ERROR = -1, /* Error */
H5PL_TYPE_FILTER = 0, /* Filter */
H5PL_TYPE_NONE = 1 /* This must be last! */
} H5PL_type_t;
#endif /*USE_HDF5*/
/* Following External Discovery Functions should be present for the dynamic loading of filters */
/* returns specific constant H5ZP_TYPE_FILTER */
typedef H5PL_type_t (*H5PL_get_plugin_type_proto)(void);
/* return <pointer to instance of H5Z_filter_class> */
typedef const void* (*H5PL_get_plugin_info_proto)(void);
/*************************/
/* Following are always defined */
/* Misc Macros */
#ifndef HGOTO_ERROR
#define HGOTO_ERROR(pline, err, action, msg) {fprintf(stderr,"%s\n",msg); ret_value = -1; goto done;}
#endif
#ifndef H5_ATTR_FALLTHROUGH
#define H5_ATTR_FALLTHROUGH /*FALLTHROUGH*/
#endif
#ifndef H5MM_memcpy
#define H5MM_memcpy memcpy
#endif
#ifndef H5MM_malloc
#define H5MM_malloc malloc
#endif
#ifndef H5MM_realloc
#define H5MM_realloc realloc
#endif
#ifndef H5MM_xfree
#define H5MM_xfree nullfree
#endif
#ifndef H5_ATTR_UNUSED
#define H5_ATTR_UNUSED
#endif
#ifndef FUNC_ENTER_STATIC
#define FUNC_ENTER_STATIC
#endif
#ifndef FUNC_LEAVE_NOAPI
#define FUNC_LEAVE_NOAPI(ret_value) return ret_value;
#endif
#ifndef FALSE
#define FALSE 0
#define TRUE 1
#endif
#ifndef SUCCEED
#define SUCCEED 0
#define FAIL -1
#endif
#ifndef FUNC_ENTER_NOAPI_NOINIT_NOERR
#define FUNC_ENTER_NOAPI_NOINIT_NOERR
#endif
#ifndef FUNC_ENTER_STATIC_NOERR
#define FUNC_ENTER_STATIC_NOERR
#endif
#ifndef FUNC_LEAVE_NOAPI
#define FUNC_LEAVE_NOAPI(x) return x;
#endif
#ifndef FUNC_LEAVE_NOAPI_VOID
#define FUNC_LEAVE_NOAPI_VOID return;
#endif
#ifndef H5_CHECKED_ASSIGN
#define H5_CHECKED_ASSIGN(dst,dtype,src,stype) (dst) = (dtype)(src)
#endif
#ifndef H5_CHECK_OVERFLOW
#define H5_CHECK_OVERFLOW(dst,dtype,stype)
#endif
#ifndef HDceil
#define HDceil(x) ceil(x)
#endif
#ifndef HDassert
#define HDassert(x)
#endif
#ifndef HDmemset
#define HDmemset memset
#endif
#ifndef UINT32ENCODE
# define UINT32ENCODE(p, i) { \
*(p) = (uint8_t)( (i) & 0xff); (p)++; \
*(p) = (uint8_t)(((i) >> 8) & 0xff); (p)++; \
*(p) = (uint8_t)(((i) >> 16) & 0xff); (p)++; \
*(p) = (uint8_t)(((i) >> 24) & 0xff); (p)++; \
}
# define UINT32DECODE(p, i) { \
(i) = (uint32_t)(*(p) & 0xff); (p)++; \
(i) |= ((uint32_t)(*(p) & 0xff) << 8); (p)++; \
(i) |= ((uint32_t)(*(p) & 0xff) << 16); (p)++; \
(i) |= ((uint32_t)(*(p) & 0xff) << 24); (p)++; \
}
#endif
#endif /*NETCDF_FILTER_HDF5_BUILD_H*/

1143
include/netcdf_json.h Normal file
View File

File diff suppressed because it is too large Load Diff

2
libdap2/dce.y
View File

@ -5,7 +5,7 @@
They will be commennted out when building a java parser.
*/
%pure-parser
%define api.pure
%lex-param {DCEparsestate* parsestate}
%parse-param {DCEparsestate* parsestate}
%{

1391
libdap2/dcetab.c
View File

File diff suppressed because it is too large Load Diff

24
libdap2/dcetab.h
View File

@ -1,8 +1,9 @@
/* A Bison parser, made by GNU Bison 3.0.4. */
/* A Bison parser, made by GNU Bison 3.7.6. */
/* Bison interface for Yacc-like parsers in C
Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc.
Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation,
Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -15,7 +16,7 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. */
along with this program. If not, see <https://www.gnu.org/licenses/>. */
/* As a special exception, you may create a larger work that contains
part or all of the Bison parser skeleton and distribute that work
@ -30,6 +31,10 @@
This special exception was added by the Free Software Foundation in
version 2.2 of Bison. */
/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual,
especially those whose name start with YY_ or yy_. They are
private implementation details that can be changed or removed. */
#ifndef YY_DCE_DCE_TAB_H_INCLUDED
# define YY_DCE_DCE_TAB_H_INCLUDED
/* Debug traces. */
@ -40,15 +45,20 @@
extern int dcedebug;
#endif
/* Token type. */
/* Token kinds. */
#ifndef YYTOKENTYPE
# define YYTOKENTYPE
enum yytokentype
{
SCAN_WORD = 258,
SCAN_STRINGCONST = 259,
SCAN_NUMBERCONST = 260
YYEMPTY = -2,
YYEOF = 0, /* "end of file" */
YYerror = 256, /* error */
YYUNDEF = 257, /* "invalid token" */
SCAN_WORD = 258, /* SCAN_WORD */
SCAN_STRINGCONST = 259, /* SCAN_STRINGCONST */
SCAN_NUMBERCONST = 260 /* SCAN_NUMBERCONST */
};
typedef enum yytokentype yytoken_kind_t;
#endif
/* Value type. */

47
libdispatch/daux.c
View File

@ -735,7 +735,52 @@ ncaux_h5filterspec_parse(const char* txt, unsigned int* idp, size_t* nparamsp, u
if(paramsp) {*paramsp = params; params = NULL;}
done:
nullfree(params);
nullfree(sdata);
nullfree(sdata0);
return stat;
}
/*
Parse a filter parameter string into a sequence of unsigned ints.
@param txt - a string containing the parameter string.
@param nuiparamsp - store the number of unsigned ints here
@param uiparamsp - store the vector of unsigned ints here; caller frees.
@return NC_NOERR if parse succeeded
@return NC_EINVAL otherwise
*/
EXTERNL int
ncaux_h5filterspec_parse_parameter(const char* txt, size_t* nuiparamsp, unsigned int* uiparams)
{
int stat = NC_NOERR;
char* p;
char* sdata0 = NULL; /* what to free */
char* sdata = NULL; /* sdata0 with leading prefix skipped */
size_t nuiparams = 0;
size_t len;
if(txt == NULL)
{stat = NC_EINVAL; goto done;}
len = strlen(txt);
if(len == 0)
{stat = NC_EINVAL; goto done;}
if((sdata0 = (char*)calloc(1,len+1+1))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(sdata0,txt,len);
sdata = sdata0;
p = sdata;
nuiparams = 0;
len = strlen(p);
/* skip leading white space */
while(strchr(" ",*p) != NULL) {p++; len--;}
if((stat = filterspec_cvt(p,&nuiparams,uiparams))) goto done;
/* Now return results */
if(nuiparamsp) *nuiparamsp = nuiparams;
done:
nullfree(sdata0);
return stat;
}

3
libdispatch/dfilter.c
View File

@ -26,9 +26,6 @@ Unified filter related code
/**************************************************/
/* Per-variable filters */
/* The original HDF5-based functions are left
but are now wrappers around the filterx functions.
*/
/**
Find the set of filters (if any) associated with a variable.

408
libdispatch/dfilterx.c
View File

@ -15,124 +15,50 @@
#include "netcdf_filter.h"
#include "ncdispatch.h"
#include "nc4internal.h"
#include "ncfilter.h"
#ifdef USE_HDF5
#include "hdf5internal.h"
#endif
#include "ncjson.h"
/*
Unified filter related code
NCZarr filter API
*/
#ifndef H5Z_FILTER_SZIP
/** ID of HDF SZIP filter. */
#define H5Z_FILTER_SZIP 4
#endif
/*Mnemonic*/
#define USENAME 1
#define LPAREN '('
#define RPAREN ')'
#define LBRACK '['
#define RBRACK ']'
#define NUMCHAR "0123456789"
#define NAMECHAR1 "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
#define NAMECHARN (NAMECHAR1 NUMCHAR "_-")
const struct FilterName {
const char* name; /* name or alias as assigned by HDF group*/
unsigned int id; /* id as assigned by HDF group*/
} known_filters[] = {
{"zip", 2}, /* Standard zlib compression */
{"zlib", 2}, /* alias */
{"deflate", 2}, /* alias */
{"szip", 4}, /* Standard szip compression */
{"bzip2", 307}, /* BZIP2 lossless compression used by PyTables */
{"lzf", 32000}, /* LZF lossless compression used by H5Py project */
{"blosc", 32001}, /* Blosc lossless compression used by PyTables */
{"mafisc", 32002}, /* Modified LZMA compression filter, MAFISC (Multidimensional Adaptive Filtering Improved Scientific data Compression) */
{"snappy", 32003}, /* Snappy lossless compression. */
{"lz4", 32004}, /* LZ4 fast lossless compression algorithm */
{"apax", 32005}, /* Samplify's APAX Numerical Encoding Technology */
{"cbf", 32006}, /* All imgCIF/CBF compressions and decompressions, including Canonical, Packed, Packed Vesrsion 2, Byte Offset and Nibble Offset. */
{"jpeg-xr", 32007}, /* Enables images to be compressed/decompressed with JPEG-XR compression */
{"bitshuffle", 32008}, /* Extreme version of shuffle filter that shuffles data at bit level instead of byte level. */
{"spdp", 32009}, /* SPDP fast lossless compression algorithm for single- and double-precision floating-point data. */
{"lpc-rice", 32010}, /* LPC-Rice multi-threaded lossless compression */
{"ccsds-123", 32011}, /* ESA CCSDS-123 multi-threaded compression filter */
{"jpeg-ls", 32012}, /* CharLS JPEG-LS multi-threaded compression filter */
{"zfp", 32013}, /* Rate, accuracy or precision bounded compression for floating-point arrays */
{"fpzip", 32014}, /* Fast and Efficient Lossy or Lossless Compressor for Floating-Point Data */
{"zstandard", 32015}, /* Real-time compression algorithm with wide range of compression / speed trade-off and fast decoder */
{"b3d", 32016}, /* GPU based image compression method developed for light-microscopy applications */
{"sz", 32017}, /* An error-bounded lossy compressor for scientific floating-point data */
{"fcidecomp", 32018}, /* EUMETSAT CharLS compression filter for use with netCDF */
{"user-defined", 32768}, /* First user-defined filter */
{NULL,0}
};
/**************************************************/
/*Forward*/
static unsigned int NC_filterx_lookup(const char* filtername);
static const char* NC_filterx_toname(unsigned int id);
static int NC_filterx_transferstringvec(size_t n, char** vec, char** copy);
/**************************************************/
/* Per-variable filters extended string-based */
/* Per-variable filters */
/**
Find the set of filters (if any) associated with a variable.
Assumes NCZarr format using json
\param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
\param varid Variable ID
\param nfilters return no. of filters
\param ids return the filter ids (caller allocates)
\param jsonp a JSON formatted string is returned in this argument
\returns ::NC_NOERR No error.
\returns ::NC_ENOTNC4 Not a netCDF-4 file.
\returns ::NC_EBADID Bad ncid.
\returns ::NC_EBADID Bad ncid
\returns ::NC_ENOTVAR Invalid variable ID.
\returns ::NC_EINVAL Invalid arguments
\ingroup variables
\author Dennis Heimbigner
*/
EXTERNL int
nc_inq_var_filterx_ids(int ncid, int varid, size_t* nfiltersp, char** ids)
nc_inq_var_filterx_ids(int ncid, int varid, char** textp)
{
NC* ncp;
int stat = NC_NOERR;
NC_FILTERX_OBJ ncids;
NC* ncp;
int stat = NC_NOERR;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_inq_var_filterids);
memset(&ncids,0,sizeof(ncids));
ncids.usort = NC_FILTER_UNION_IDS;
ncids.u.ids.nfilters = 0;
ncids.u.ids.filterids = NULL;
if((stat = ncp->dispatch->filter_actions(ncid,varid, NCFILTER_FILTERIDS, &ncids))) goto done;
if(nfiltersp) *nfiltersp = ncids.u.ids.nfilters;
if(ids) {
if((stat = NC_filterx_transferstringvec(ncids.u.ids.nfilters, ncids.u.ids.filterids, ids))) goto done;
}
TRACE(nc_inq_var_filterx_ids);
if((stat = NC_check_id(ncid,&ncp))) return stat;
if((stat = ncp->dispatch->inq_var_filterx_ids(ncid,varid,textp))) goto done;
done:
NC_filterx_freestringvec(ncids.u.ids.nfilters,ncids.u.ids.filterids);
return stat;
}
return stat;
}
/**
Find the the param info about filter (if any)
associated with a variable and with specified id.
This is a wrapper for nc_inq_var_all().
Assumes HDF5 format using unsigned ints.
\param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
@ -140,7 +66,6 @@ nc_inq_ncid().
\param varid Variable ID
\param id The filter id of interest
\param formatp (Out) Storage for the filter format
\param nparamsp (Out) Storage which will get the number of parameters to the filter
\param params (Out) Storage which will get associated parameters.
Note: the caller must allocate and free.
@ -149,38 +74,27 @@ Note: the caller must allocate and free.
\returns ::NC_ENOTNC4 Not a netCDF-4 file.
\returns ::NC_EBADID Bad ncid.
\returns ::NC_ENOTVAR Invalid variable ID.
\returns ::NC_ENOFILTER No filter defined.
\returns ::NC_ENOFILTER Specified filter not defined for this variable.
\ingroup variables
\author Dennis Heimbigner
*/
EXTERNL int
nc_inq_var_filterx_info(int ncid, int varid, const char* id, size_t* nparamsp, char** params)
nc_inq_var_filterx_info(int ncid, int varid, const char* id, char** textp)
{
NC* ncp;
int stat = NC_NOERR;
NC_FILTERX_OBJ spec;
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_inq_var_filter_info);
TRACE(nc_inq_var_filterx_info);
if(stat != NC_NOERR) return stat;
if((stat = ncp->dispatch->inq_var_filterx_info(ncid,varid,id,textp))) goto done;
memset(&spec,0,sizeof(spec));
spec.usort = NC_FILTER_UNION_SPEC;
spec.u.spec.filterid = (char*)id;
spec.u.spec.params = NULL;
if((stat = ncp->dispatch->filter_actions(ncid,varid,NCFILTER_INFO,&spec))) goto done;
if(nparamsp) *nparamsp = spec.u.spec.nparams;
if(params) {
if((stat = NC_filterx_transferstringvec(spec.u.spec.nparams, spec.u.spec.params, params))) goto done;
}
done:
NC_filterx_freestringvec(spec.u.spec.nparams,spec.u.spec.params);
return stat;
return stat;
}
/**
Define a new variable filter.
Define a new variable filter
Assumes HDF5 format using unsigned ints.
Only variables with chunked storage can use filters.
@param ncid File and group ID.
@ -196,210 +110,134 @@ done:
*/
EXTERNL int
nc_def_var_filterx(int ncid, int varid, const char* id, size_t nparams, const char** params)
nc_def_var_filterx(int ncid, int varid, const char* json)
{
NC* ncp;
NC_FILTERX_OBJ spec;
int stat = NC_NOERR;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_def_var_filter);
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
TRACE(nc_def_var_filterx);
if(stat != NC_NOERR) return stat;
if((stat = ncp->dispatch->def_var_filterx(ncid,varid,json))) goto done;
memset(&spec,0,sizeof(spec));
spec.usort = NC_FILTER_UNION_SPEC;
spec.u.spec.filterid = (char*)id;
spec.u.spec.nparams = nparams;
spec.u.spec.params = (char**)params; /* discard const */
if((stat=ncp->dispatch->filter_actions(ncid,varid,NCFILTER_DEF,&spec))) goto done;
done:
return stat;
}
/**
Remove all filters with specified id from a variable
Find the first filter (if any) associated with a variable.
\param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
@param ncid File and group ID.
@param varid Variable ID.
@param id filter to remove
\param varid Variable ID
@return ::NC_NOERR No error.
@return ::NC_EBADID Bad ID.
@author Dennis Heimbigner
\param textp Storage which will get the filter info (id + parameters) in json format
This is redundant over the multi-filter API, so
it can be implemented in terms of those functions.
\returns ::NC_NOERR No error.
\returns ::NC_ENOTNC4 Not a netCDF-4 file.
\returns ::NC_EBADID Bad ncid.
\returns ::NC_ENOTVAR Invalid variable ID.
\ingroup variables
\author Dennis Heimbigner
*/
EXTERNL int
nc_var_filterx_remove(int ncid, int varid, const char* id)
nc_inq_var_filterx(int ncid, int varid, char** textp)
{
NC* ncp;
int stat = NC_NOERR;
NC_FILTERX_OBJ spec;
char* text = NULL;
NCjson* json = NULL;
NCjson* jid = NULL;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_var_filter_remove);
TRACE(nc_inq_var_filterx);
if((stat = NC_check_id(ncid,&ncp))) goto done;
memset(&spec,0,sizeof(spec));
spec.usort = NC_FILTER_UNION_SPEC;
spec.u.spec.filterid = (char*)id;
if((stat=ncp->dispatch->filter_actions(ncid,varid,NCFILTER_REMOVE,&spec))) goto done;
done:
/* Get the filters on this variable */
if((stat = nc_inq_var_filterx_ids(ncid,varid,&text))) goto done;
/* Parse it */
if((stat = NCJparse(text,0,&json))) goto done;
if(json->sort != NCJ_ARRAY)
{stat = NC_EFILTER; goto done;}
if(NCJlength(json) == 0 || NCJcontents(json) == NULL)
{stat = NC_ENOFILTER; goto done;}
jid = NCJith(json,0);
if(jid->sort == NCJ_DICT || jid->sort == NCJ_ARRAY)
{stat = NC_EFILTER; goto done;}
/* Get info about the first filter */
if((stat = nc_inq_var_filterx_info(ncid,varid,NCJstring(jid),textp)))
{stat = NC_ENOFILTER; goto done;}
done:
NCJreclaim(json);
return stat;
}
/**************************************************/
/* Utilities */
/* Support direct user defined filters */
int
NC_cvtX2I_idlist(size_t n, const char** xidlist, unsigned int* ids)
#ifdef ENABLE_CLIENTSIDE_FILTERS
/* Use void* to avoid having to include hdf.h*/
EXTERNL int
nc_filterx_client_register(unsigned int id, void* info)
{
int i,stat = NC_NOERR;
for(i=0;i<n;i++) {
if((stat = NC_cvtX2I_id(xidlist[i],&ids[i]))) break;
}
int stat = NC_NOERR;
#ifdef USE_HDF5
NC_FILTERX_OBJ_HDF5 client;
if(id == 0 ||info == NULL)
return NC_EINVAL;
memset(&client,0,sizeof(client));
client.hdr.format = NC_FILTERX_FORMAT_HDF5;
client.sort = NC_FILTERX_SORT_CLIENT;
client.u.client.id = id;
client.u.client.info = info;
/* Note use of a global function, not part of the dispatch table */
stat = nc4_global_filterx_action(NCFILTER_CLIENT_REG, id, &client);
#else
stat = NC_ENOTBUILT;
#endif
return stat;
}
int
NC_cvtX2I_params(size_t nparams, const char** xparamslist, unsigned int* params)
EXTERNL int
nc_filterx_client_unregister(unsigned int id)
{
int i,stat = NC_NOERR;
unsigned int id;
for(i=0;i<nparams;i++) {
/* See if this param looks like an unsigned int */
if(sscanf(xparamslist[i],"%u",&id) == 1) {
params[i] = id;
}
}
int stat = NC_NOERR;
#ifdef USE_HDF5
stat = nc4_global_filterx_action(NCFILTER_CLIENT_UNREG, id, NULL);
#else
stat = NC_ENOTBUILT;
#endif
return stat;
}
int
NC_cvtI2X_idlist(int n, const unsigned int* ids, char** xid)
/* Use void* to avoid having to include hdf.h*/
EXTERNL int
nc_filterx_client_inq(unsigned int id, void* infop)
{
int i,stat = NC_NOERR;
char sid[1024];
/* For now, do not attempt a name conversion */
for(i=0;i<n;i++) {
snprintf(sid,sizeof(sid),"%u",ids[i]);
if((xid[i] = strdup(sid)) == NULL)
{stat = NC_ENOMEM; goto done;}
int stat = NC_NOERR;
#ifdef USE_HDF5
H5Z_class2_t* hct = (H5Z_class2_t*)infop;
NC_FILTERX_OBJ_HDF5 client;
if(id == 0 ||infop == NULL)
return NC_EINVAL;
memset(&client,0,sizeof(client));
client.hdr.format = NC_FILTERX_FORMAT_HDF5;
client.sort = NC_FILTERX_SORT_CLIENT;
client.u.client.id = id;
client.u.client.info = hct;
/* Note use of a global function, not part of the dispatch table */
stat = nc4_global_filterx_action(NCFILTER_CLIENT_INQ, id, &client);
if(stat == NC_NOERR) {
*hct = *(H5Z_class2_t*)client.u.client.info;
}
done:
#else
stat = NC_ENOTBUILT;
#endif
return stat;
}
int
NC_cvtI2X_params(int n, const unsigned int* ids, char** params)
{
int i,stat = NC_NOERR;
char sid[1024];
for(i=0;i<n;i++) {
snprintf(sid,sizeof(sid),"%u",ids[i]);
if((params[i] = strdup(sid))==NULL)
{stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
/* Convert an xid; allow a name */
int
NC_cvtX2I_id(const char* xid, unsigned int* idp)
{
unsigned int id;
/* See if this id looks like an unsigned int */
if(sscanf(xid,"%u",&id) != 1) {
id = NC_filterx_lookup(xid);
}
if(idp) *idp = id;
return (id > 0 ? NC_NOERR : NC_EINVAL);
}
/* Convert an int id to a string; optional name conversion */
int
NC_cvtI2X_id(unsigned int id, char** xidp, int usename)
{
char xid[NC_MAX_NAME];
snprintf(xid,sizeof(xid),"%u",id);
if(usename) {/* See if this id has a name */
const char* name = NC_filterx_toname(id);
if(name != NULL) {xid[0] = '\0'; strlcat(xid,name,sizeof(xid));}
}
if(xidp) {
if((*xidp = strdup(xid))==NULL) return NC_ENOMEM;
}
return NC_NOERR;
}
static unsigned int
NC_filterx_lookup(const char* filtername)
{
const struct FilterName* p = known_filters;
/* Try a name lookup */
for(;p->name;p++) {
if(strcasecmp(p->name,filtername)==0)
return p->id;
}
return 0; /* no match */
}
static const char*
NC_filterx_toname(unsigned int id)
{
const struct FilterName* p = known_filters;
for(;p->name;p++) {
if(p->id == id)
return p->name;
}
return NULL; /* no match */
}
void
NC_filterx_freestringvec(size_t n, char** vec)
{
int i;
if(vec != NULL) {
for(i=0;i<n;i++) {
if(vec[i]) free(vec[i]);
}
free(vec);
}
}
static int
NC_filterx_transferstringvec(size_t n, char** vec, char** copy)
{
int i, stat = NC_NOERR;
for(i=0;i<n;i++) {
copy[i] = vec[i];
vec[i] = NULL;
}
return stat;
}
int
NC_filterx_copy(size_t n, const char** vec, char*** copyp)
{
char** copy = NULL;
int i, stat = NC_NOERR;
if((copy = calloc(sizeof(char*),n+1))==NULL)
{stat = NC_ENOMEM; goto done;}
for(i=0;i<n;i++) {
char* s = nulldup(vec[i]);
if(s == NULL) {stat = NC_ENOMEM; goto done;}
copy[i] = s;
}
if(copyp) {*copyp = copy; copy = NULL;}
done:
if(copy) NC_filterx_freestringvec(n,copy);
return stat;
}
#endif /*ENABLE_CLIENTSIDE_FILTERS*/

15
libdispatch/dpathmgr.c
View File

@ -167,21 +167,23 @@ NCpathcanonical(const char* srcpath, char** canonp)
/* parse the src path */
if((stat = parsepath(srcpath,&path))) {goto done;}
switch (path.kind) {
case NCPD_NIX:
case NCPD_CYGWIN:
case NCPD_REL:
/* use as is */
canon = path.path; path.path = NULL;
break;
case NCPD_NIX:
case NCPD_CYGWIN:
case NCPD_MSYS:
case NCPD_WIN: /* convert to cywin form */
len = strlen(path.path) + strlen("/cygdrive/X") + 1;
len = nulllen(path.path) + strlen("/cygdrive/X") + 1;
canon = (char*)malloc(len);
if(canon != NULL) {
canon[0] = '\0';
strlcat(canon,"/cygdrive/X",len);
canon[10] = path.drive;
strlcat(canon,path.path,len);
if(path.drive != 0) {
strlcat(canon,"/cygdrive/X",len);
canon[10] = path.drive;
}
if(path.path != NULL) strlcat(canon,path.path,len);
}
break;
default: goto done; /* return NULL */
@ -194,6 +196,7 @@ done:
return stat;
}
EXTERNL
char* /* caller frees */
NCpathabsolute(const char* relpath)

1
libdispatch/drc.c
View File

@ -495,6 +495,7 @@ done:
nullfree(path);
path = NULL;
}
errno = 0; /* silently ignore errors */
return (ret);
}

4
libdispatch/dvarinq.c
View File

@ -713,8 +713,10 @@ nc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp)
stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_SZIP,&nparams,params);
switch (stat) {
case NC_NOERR:
if(nparams != 2)
if(nparams < 2)
return NC_EFILTER; /* bad # params */
if(nparams > 2)
nparams = 2; /* for compatibility, only return 2 params */
break;
case NC_ENOFILTER:
case NC_ENOTNC4:

710
libdispatch/ncjson.c
View File

File diff suppressed because it is too large Load Diff

18
libdispatch/nclist.c
View File

@ -56,18 +56,26 @@ Free a list and its contents
*/
int
nclistfreeall(NClist* l)
{
nclistclearall(l);
return nclistfree(l);
}
/*
Free the contents of a list
*/
int
nclistclearall(NClist* l)
{
size_t i,len;
void** content = NULL;
if(l == NULL) return TRUE;
len = l->length;
content = nclistextract(l);
for(i=0;i<len;i++) {
void* value = content[i];
void* value = l->content[i];
if(value != NULL) free(value);
}
if(content != NULL) free(content);
return nclistfree(l);
nclistsetlength(l,0);
return TRUE;
}
int

2
libhdf5/hdf5attr.c
View File

@ -460,7 +460,7 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
&& (ra->flags & READONLYFLAG))
return NC_ENAMEINUSE;
/* case 2: grp=NA, varid!=NC_GLOBAL, flags & HIDDENATTRFLAG */
if (varid != NC_GLOBAL && (ra->flags & HIDDENATTRFLAG))
if (varid != NC_GLOBAL && (ra->flags & (HIDDENATTRFLAG|READONLYFLAG)))
return NC_ENAMEINUSE;
}

12
libhdf5/hdf5debug.c
View File

@ -15,14 +15,17 @@
#define STSIZE 1000
#if !defined _WIN32 && !defined __CYGWIN__
#ifdef H5BACKTRACE
# if !defined _WIN32 && !defined __CYGWIN__
static void* stacktrace[STSIZE];
# endif
#endif
int
nch5breakpoint(int err)
{
#if !defined _WIN32 && !defined __CYGWIN__
#ifdef H5BACKTRACE
# if !defined _WIN32 && !defined __CYGWIN__
int count = 0;
char** trace = NULL;
int i;
@ -32,9 +35,10 @@ nch5breakpoint(int err)
fprintf(stderr,"backtrace:\n");
for(i=0;i<count;i++)
fprintf(stderr,"[%03d] %s\n",i,trace[i]);
#if 0
# if 0
if(trace != NULL) free(trace);
#endif
# endif
# endif
#endif
return err;
}

2
libhdf5/hdf5debug.h
View File

@ -8,6 +8,8 @@
/* Warning: significant performance impact */
#undef H5CATCH
#undef H5BACKTRACE
#ifdef H5CATCH
/* Place breakpoint to catch errors close to where they occur*/
#define THROW(e) nch5throw(e)

17
libhdf5/hdf5filter.c
View File

@ -19,8 +19,13 @@
#include "netcdf.h"
#include "netcdf_filter.h"
#ifdef ENABLE_BLOSC
#include <blosc.h>
#endif
#undef TFILTERS
/* Forward */
static int NC4_hdf5_filter_free(struct NC_HDF5_Filter* spec);
/**************************************************/
@ -571,3 +576,15 @@ NC4_hdf5_find_missing_filter(NC_VAR_INFO_T* var, unsigned int* idp)
if(idp) *idp = id;
return stat;
}
int
NC4_hdf5_filter_initialize(void)
{
return NC_NOERR;
}
int
NC4_hdf5_filter_finalize(void)
{
return NC_NOERR;
}

2
libhdf5/hdf5internal.c
View File

@ -82,6 +82,7 @@ nc4_hdf5_initialize(void)
if (set_auto(NULL, NULL) < 0)
LOG((0, "Couldn't turn off HDF5 error messages!"));
LOG((1, "HDF5 error messages have been turned off."));
NC4_hdf5_filter_initialize();
nc4_hdf5_initialized = 1;
}
@ -94,6 +95,7 @@ nc4_hdf5_finalize(void)
{
/* Reclaim global resources */
NC4_provenance_finalize();
NC4_hdf5_filter_finalize();
nc4_hdf5_initialized = 0;
}

16
liblib/CMakeLists.txt
View File

@ -37,6 +37,10 @@ IF(ENABLE_NCZARR)
SET(liblib_LIBS ${liblib_LIBS} nczarr)
ENDIF()
IF(ENABLE_PLUGINS)
SET(liblib_LIBS ${liblib_LIBS} ncpoco)
ENDIF()
FOREACH(LIBS ${liblib_LIBS})
SET(LARGS ${LARGS} $<TARGET_OBJECTS:${LIBS}>)
ENDFOREACH()
@ -72,6 +76,10 @@ SET(TLL_LIBS ${TLL_LIBS} ${HAVE_LIBM} ${ZLIB_LIBRARY})
# Add extra dependencies specified via NC_EXTRA_DEPS
SET(TLL_LIBS ${TLL_LIBS} ${EXTRA_DEPS})
IF(ENABLE_BLOSC)
SET(TLL_LIBS ${TLL_LIBS} ${Blosc_LIBRARIES})
ENDIF()
IF(HAVE_LIBDL)
SET(TLL_LIBS ${LIBDL} ${TLL_LIBS})
ENDIF()
@ -117,6 +125,14 @@ IF(ENABLE_S3_SDK)
TARGET_LINK_LIBRARIES(netcdf ${AWSSDK_LINK_LIBRARIES})
ENDIF()
IF(NOT WIN32)
IF(NOT APPLE)
IF(CMAKE_DL_LIBS)
TARGET_LINK_LIBRARIES(netcdf ${CMAKE_DL_LIBS})
ENDIF()
ENDIF()
ENDIF()
IF(TLL_LIBS)
LIST(REMOVE_DUPLICATES TLL_LIBS)
ENDIF()

5
liblib/Makefile.am
View File

@ -78,6 +78,11 @@ libnetcdf_la_LIBADD += ${aws_cpp_sdk_core_LIBS} ${aws_cpp_sdk_s3_LIBS}
endif
endif #ENABLE_NCZARR
if ENABLE_PLUGINS
AM_CPPFLAGS += -I${top_srcdir}/libncpoco
libnetcdf_la_LIBADD += ${top_builddir}/libncpoco/libncpoco.la
endif #ENABLE_PLUGINS
if ISCYGWIN
# Force binary mode for file read/write
AM_LDFLAGS += -lbinmode

8
libncpoco/CMakeLists.txt Executable file
View File

@ -0,0 +1,8 @@
SET(libncpoco_SOURCES ncpoco.c ncpoco.h)
IF(MSVC)
SET(libncpoco_SOURCES ${libncpoco_SOURCES} cp_win32.c)
ELSE()
SET(libncpoco_SOURCES ${libncpoco_SOURCES} cp_unix.c)
ENDIF()
add_library(ncpoco OBJECT ${libncpoco_SOURCES})
ADD_EXTRA_DIST(${libncpoco_SOURCES} CMakeLists.txt)

39
libncpoco/COPYRIGHT Executable file
View File

@ -0,0 +1,39 @@
The NetCDF Copyright.
Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
2015, 2016
University Corporation for Atmospheric Research/Unidata.
Portions of this software were developed by the Unidata Program at the
University Corporation for Atmospheric Research.
Access and use of this software shall impose the following obligations
and understandings on the user. The user is granted the right, without
any fee or cost, to use, copy, modify, alter, enhance and distribute
this software, and any derivative works thereof, and its supporting
documentation for any purpose whatsoever, provided that this entire
notice appears in all copies of the software, derivative works and
supporting documentation. Further, UCAR requests that the user credit
UCAR/Unidata in any publications that result from the use of this
software or in any product that includes this software, although this
is not an obligation. The names UCAR and/or Unidata, however, may not
be used in any advertising or publicity to endorse or promote any
products or commercial entity unless specific written permission is
obtained from UCAR/Unidata. The user also understands that
UCAR/Unidata is not obligated to provide the user with any support,
consulting, training or assistance of any kind with regard to the use,
operation and performance of this software nor to provide the user
with any updates, revisions, new versions or "bug fixes."
THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.

40
libncpoco/Makefile.am Executable file
View File

@ -0,0 +1,40 @@
# Copyright 2009, UCAR/Unidata
# See the COPYRIGHT file for more information.
# Use automake or CMake for building under nix
# Use CMake for building under windows
# Get AM_CPPFLAGS and AM_LDFLAGS
include $(top_srcdir)/lib_flags.am
#AM_CPPFLAGS += -D_LARGEFILE_SOURCE
#AM_CPPFLAGS += -I$(top_srcdir)/libsrc4 -I$(top_srcdir)/libdap4
#libnczarr_la_CPPFLAGS = ${AM_CPPFLAGS}
AM_CXXFLAGS =
# This is our output. The ZARR convenience library.
noinst_LTLIBRARIES = libncpoco.la
libncpoco_la_LIBADD =
libncpoco_la_LDFLAGS =
libncpoco_la_SOURCES = ncpoco.c ncpoco.h
if ISMSVC
libncpoco_la_SOURCES += cp_win32.c
else
libncpoco_la_SOURCES += cp_unix.c
endif
EXTRA_DIST = CMakeLists.txt README.md COPYRIGHT
##################################################
# Testing
# check_LTLIBRARIES = libcpt.la
# libcpt_la_SOURCES = cptestlib.c
#
# # Normally check libraries are created only as archives,
# # but we need a shared lib. This appears to do the trick
# libcpt_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -rpath ${abs_builddir}
#
# AM_LDFLAGS += libcpoco.la
# check_PROGRAMS = cp_test
# TESTS = cp_test
#

128
libncpoco/README.md Normal file
View File

@ -0,0 +1,128 @@
# The cpoco Multi-Platform Dynamic-Loading Library
## Description
The primary goal of the cpoco project is to provide a C language
version of the poco multi-platform dynamic loading library. The
poco libraries are written in C++.
The secondary goal of cpoco is to support dynamic loading of
HDF5 filters by the [netCDF C library](http://www.unidata.ucar.edu/netcdf/).
# Mutual Exclusion Support
Internally, cpoco (like poco) supports serialized access to the dynamic
loading functions using mutual exclusion locks. For *nix* systems, this usually requires pthreads support.
In any case, it is possible to disable the use of mutual exclusion if you know
you are operating in a single threaded environment: see the [installation](#installation) section below.
# Implementataion Restrictions
Currently support is provided for the following systems.
* libdl supporting operating systems: e.g. linux, os-x, cygwin.
* Windows-32 api
# Installation
## Automake
Automake-based configuration is provided using ./configure is provided for systems supporting autoconf/automake.
Use this command to see the available options.
```bash
./configure --help
```
The most important options are these.
* --disable-mutex -- disable using mutual exclusion (default is enabled)
* --disable-pthread -- disable using pthreads (default is enabled if operating system provides it)
* --enable-shared -- build a shared library (default is enabled)
* --enable-static -- build a static library (default is enabled)
* --prefix=<installation directory> -- defaults to /usr/local
Note that is --enable-shared is disabled, then the test program
will not run because the test shared library (libcpt) cannot be built.
Use these commands to build, test, and install using autoconf.
```bash
# Invoke configure
./configure <options>
make all
# If testing is desired
make check
# Optional
make install
```
The <options> are those shown by `./configure --help`.
## CMake
Cmake-based configuration is provided provided for systems supporting it. If building for windows, then this is the only option provided.
Use these commands to build using cmake.
```bash
# Create a build directory
rm -fr build
mkdir build
cd build
# Invoke cmake
cmake <flags> ..
cmake --build .
# If testing is desired
CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --target test
# Optional
cmake --build . --target install
```
The <flags> are these:
* -DENABLE_MUTEX -- use mutual exclusion (default is true)
* -DENABLE_PTHREAD -- use pthreads (if available) (default is true)
* -DCMAKE_INSTALL_PREFIX=<path> -- installation directory (defaults to /usr/local or c:/Program Files)
# Support
__Author__: Dennis Heimbigner
__Organization__: University Corporation for Atmospheric Research
__Initial Release__: 2015-9-28
__Last Modified__: 2018-03-27
__Copyright__: Copyright 2018, UCAR/Unidata; see COPYRIGHT file for copying and redistribution conditions. This code is Derived from the poco library (See [Poco Information](#poco-information) below).
# Change Log
1. (2016-09-28) Initial release.
2. (2018-03-27) Updated to make fix some automake problems.
# Poco Information
* Base poco version: 1.7.5 (2016-08-29)
* Poco web page: http://pocoproject.org
## Poco License
> Boost Software License - Version 1.0 - August 17th, 2003
>
> Permission is hereby granted, free of charge, to any person or organization
> obtaining a copy of the software and accompanying documentation covered by
> this license (the "Software") to use, reproduce, display, distribute,
> execute, and transmit the Software, and to prepare derivative works of the
> Software, and to permit third-parties to whom the Software is furnished to
> do so, all subject to the following:
>
> The copyright notices in the Software and this entire statement, including
> the above license grant, this restriction and the following disclaimer,
> must be included in all copies of the Software, in whole or in part, and
> all derivative works of the Software, unless such copies or derivative
> works are solely in the form of machine-executable object code generated by
> a source language processor.
>
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
> SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
> FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
> ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
> DEALINGS IN THE SOFTWARE.

25
libncpoco/SourceLicence Normal file
View File

@ -0,0 +1,25 @@
## Poco License
> Boost Software License - Version 1.0 - August 17th, 2003
>
> Permission is hereby granted, free of charge, to any person or organization
> obtaining a copy of the software and accompanying documentation covered by
> this license (the "Software") to use, reproduce, display, distribute,
> execute, and transmit the Software, and to prepare derivative works of the
> Software, and to permit third-parties to whom the Software is furnished to
> do so, all subject to the following:
>
> The copyright notices in the Software and this entire statement, including
> the above license grant, this restriction and the following disclaimer,
> must be included in all copies of the Software, in whole or in part, and
> all derivative works of the Software, unless such copies or derivative
> works are solely in the form of machine-executable object code generated by
> a source language processor.
>
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
> SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
> FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
> ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
> DEALINGS IN THE SOFTWARE.

247
libncpoco/cp_test.c Executable file
View File

@ -0,0 +1,247 @@
/*
Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH.
and Contributors.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <stdarg.h>
#include <assert.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "cpoco.h"
/* Possible names of the test shared library */
static const char* SHAREDLIBS[] = {
"./libcpt.so",
"./cygcpt.dll",
"./cpt.dll",
"./.libs/libcpt.so",
"./.libs/cygcpt.dll",
"./.libs/cpt.dll",
"./.libs/cygcpt-0.dll",
NULL};
typedef int (*GimmeFiveFunc)();
#define CHECK(err) check(CE_ERR,0,__LINE__,__FILE__,(err))
#define CHECKNO(err) check(CE_ERR,1,__LINE__,__FILE__,(err))
#define CHECKBOOL(b) check(CE_BOOL,0,__LINE__,__FILE__,(b))
#define CHECKNOBOOL(b) check(CE_BOOL,1,__LINE__,__FILE__,(b))
#define CHECKSTR(s1,s2) check(CE_STR,0,__LINE__,__FILE__,(s1),(s2))
#define CHECKSYM(sym) check(CE_SYM,0,__LINE__,__FILE__,(sym))
#define CHECKNOSYM(sym) check(CE_SYM,1,__LINE__,__FILE__,(sym))
enum CE {
CE_ERR,
CE_BOOL,
CE_STR,
CE_SYM,
};
static void
check(enum CE ce, int invert, int lineno, const char* file, ...)
{
va_list args;
va_start (args,file);
int err;
char* s1;
char* s2;
void* sym;
switch (ce) {
case CE_ERR:
err = va_arg(args,int);
if((!invert && err != CP_OK) || (invert && err == CP_OK)) {
fprintf(stderr,"(%s:%d): %s\n",file,lineno,cperrstr(err));
goto fail;
}
break;
case CE_BOOL:
err = va_arg(args,int);
if((err && invert) || (!err && !invert)) {
fprintf(stderr,"(%s:%d): boolean fail\n",file,lineno);
goto fail;
}
break;
case CE_STR:
s1 = va_arg(args,char*);
s2 = va_arg (args,char*);
if(s1 == NULL) s1 = "null";
if(s2 == NULL) s2 = "null";
if((!invert && strcmp(s1,s2) != 0) || (invert && strcmp(s1,s2) == 0)) {
fprintf(stderr,"(%s:%d): mismatch: %s::%s \n",file,lineno,s1,s2);
goto fail;
}
break;
case CE_SYM:
sym = va_arg(args,void*);
if((!invert && sym == NULL) || (invert && sym != NULL)) {
fprintf(stderr,"(%s:%d): get symbol failure\n",file,lineno);
goto fail;
}
break;
default: abort();
}
va_end (args); // clean up the system stack
return;
fail:
exit(1);
}
static void
test1(const char* path)
{
SharedLib* sl = NULL;
void* p1;
CHECK(cpsharedlibnew(&sl));
CHECK(cpload(sl,path,0));
CHECKBOOL(cpisloaded(sl));
CHECKNO(cpload(sl,path,0));
CHECKSTR(cpgetpath(sl),path);
CHECKBOOL(cpisloaded(sl));
CHECKSYM(cpgetsymbol(sl,"gimmeFive"));
CHECKNOSYM(cpgetsymbol(sl,"fooBar123"));
p1 = cpgetsymbol(sl,"pocoBuildManifest");
CHECK((p1 != NULL));
p1 = cpgetsymbol(sl,"fooBar123");
cpunload(sl);
CHECKNOBOOL(cpisloaded(sl));
}
static void
test2(const char* path)
{
SharedLib* sl = NULL;
GimmeFiveFunc gimmeFive;
CHECK(cpsharedlibnew(&sl));
CHECK(cpload(sl,path,0));
CHECKSTR(cpgetpath(sl),path);
CHECKBOOL(cpisloaded(sl));
gimmeFive = (GimmeFiveFunc) cpgetsymbol(sl,"gimmeFive");
CHECKSYM(gimmeFive);
CHECKBOOL((gimmeFive() == 5));
CHECK(cpunload(sl));
CHECKNOBOOL(cpisloaded(sl));
}
static void
test3(const char* path)
{
SharedLib* sl = NULL;
CHECK(cpsharedlibnew(&sl));
CHECK(cpload(sl,path,0));
CHECKSTR(cpgetpath(sl),path);
CHECKBOOL(cpisloaded(sl));
CHECKNO(cpload(sl,"NoSuchLibrary",0));
CHECKBOOL(cpisloaded(sl));
CHECKNO(cpload(sl,path,0));
CHECKBOOL(cpisloaded(sl));
CHECKNO(cpload(sl,path,0));
CHECKBOOL(cpisloaded(sl));
CHECK(cpunload(sl));
CHECKNOBOOL(cpisloaded(sl));
}
static char*
abspath(const char* prefixpath, const char* relpath)
{
char* full = NULL;
if(prefixpath == NULL) {
#ifdef _MSC_VER
full = _fullpath(NULL,relpath,8192);
#else
full = realpath(relpath, NULL);
#endif
} else { /* concatenate */
full = malloc(strlen(prefixpath)+strlen(relpath)+2);
strcpy(full,prefixpath);
#ifdef _MSC_VER
strcat(full,"\\");
#else
strcat(full,"/");
#endif
strcat(full,relpath);
}
fprintf(stderr,"abspath=%s\n",full);
return full;
}
static const char*
findlib(const char* prefixpath)
{
char* path;
const char** p;
for(p=SHAREDLIBS;*p;p++) {
FILE* f;
path = abspath(prefixpath,*p);
if(path == NULL) continue;
f = fopen(path,"r");
fprintf(stderr,"findlib: trying %s\n",path);
if(f != NULL) {
fclose(f);
return *p;
}
}
fprintf(stderr,"findlib: failed\n");
return NULL;
}
int
main(int argc, char** argv)
{
const char* prefixpath = NULL;
const char* relpath = NULL;
const char* path = NULL;
switch (argc) {
default:
case 3:
relpath = argv[2];
/* fall thru */
case 2:
prefixpath = argv[1];
break;
case 1:
case 0:
break;
}
if(prefixpath == NULL) {
fprintf(stderr,"No prefix path specified\n");
return 1;
}
if(relpath == NULL)
relpath = findlib(prefixpath);
if(relpath == NULL) {
fprintf(stderr,"No dynamic test library specified\n");
return 1;
}
path = abspath(prefixpath,relpath);
if(path == NULL) {
fprintf(stderr,"Cannot locate dynamic test library: %s\n",relpath);
exit(1);
}
printf("Using shared test library: %s\n",path);
test1(path);
test2(path);
test3(path);
return 0;
}

173
libncpoco/cp_unix.c Executable file
View File

@ -0,0 +1,173 @@
/*********************************************************************
* Copyright 1993, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* Derived from poco library from Boost Software: see nccpoco/LICENSE file.
*********************************************************************/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef HAVE_DLFCN_H
#include <dlfcn.h>
#endif
#ifdef USE_MUTEX
#include <pthread.h>
#endif
#include "netcdf.h"
#include "ncpoco.h"
#include "ncpathmgr.h"
/* Note: cygwin is missing RTLD_LOCAL, set it to 0 */
#if !defined(RTLD_LOCAL)
#define RTLD_LOCAL 0
#endif
#if !defined(RTLD_GLOBAL)
#define RTLD_GLOBAL 0
#endif
#if !defined(RTLD_LAZY)
#define RTLD_LAZY 0
#endif
#ifdef USE_MUTEX
static pthread_mutex_t mutex;
#endif
int
ncp_unix_initialize()
{
int ret = 1;
#ifdef USE_MUTEX
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL);
if(pthread_mutex_init(&mutex, &attr)) {
fprintf(stderr,"cp_unix: Cannot create mutext\n");
ret = 1;
}
pthread_mutexattr_destroy(&attr);
#endif
return ret;
}
int
ncp_unix_finalize()
{
#ifdef USE_MUTEX
pthread_mutex_destroy(&mutex);
#endif
return 1;
}
/**************************************************/
#ifdef USE_MUTEX
static void lock(void) {pthread_mutex_lock(&mutex);}
#else
#define lock()
#endif
#ifdef USE_MUTEX
static void unlock(void) {pthread_mutex_unlock(&mutex);}
#else
#define unlock()
#endif
/**************************************************/
static int
init(NCPSharedLib* lib)
{
int ret = NC_NOERR;
return ret;
}
static int
reclaim(NCPSharedLib* lib)
{
int ret = NC_NOERR;
return ret;
}
static int
load(NCPSharedLib* lib , const char* path0, int flags)
{
int ret = NC_NOERR;
int realflags = RTLD_LAZY;
char* path = NULL;
if((path = NCpathcvt(path0))==NULL) {ret = NC_ENOMEM; goto done;}
lock();
if(lib->state.handle != NULL)
{ret = NC_EEXIST; goto ldone;}
lib->path = nulldup(path);
lib->flags = flags;
if(flags & NCP_LOCAL)
realflags |= RTLD_LOCAL;
else
realflags |= RTLD_GLOBAL;
lib->state.flags = realflags;
lib->state.handle = dlopen(lib->path, lib->state.flags);
if(lib->state.handle == NULL) {
const char* msg = dlerror();
if(msg == NULL) msg = "";
strncpy(lib->err.msg,msg,sizeof(lib->err.msg));
ret = NC_ENOTFOUND;
goto ldone;
}
ldone:
unlock();
done:
nullfree(path);
return ret;
}
static int
unload(NCPSharedLib* lib)
{
int ret = NC_NOERR;
lock();
if(lib->state.handle != NULL) {
dlclose(lib->state.handle);
lib->state.handle = NULL;
}
unlock();
return ret;
}
static int
isLoaded(NCPSharedLib* lib)
{
return lib->state.handle != NULL;
}
static void*
getsymbol(NCPSharedLib* lib, const char* name)
{
void* result = NULL;
lock();
if(lib->state.handle != NULL)
result = dlsym(lib->state.handle, name);
unlock();
return result;
}
static const char*
getpath(NCPSharedLib* lib)
{
return lib->path;
}
struct NCPAPI ncp_unix_api = {
init,
reclaim,
load,
unload,
isLoaded,
getsymbol,
getpath,
};

186
libncpoco/cp_win32.c Executable file
View File

@ -0,0 +1,186 @@
/*********************************************************************
* Copyright 1993, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* Derived from poco library from Boost Software: see nccpoco/LICENSE file.
*********************************************************************/
#ifdef _WIN32
#include "config.h"
#include <windows.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <windows.h>
#include <errhandlingapi.h>
#include "netcdf.h"
#include "ncpoco.h"
#include "ncpathmgr.h"
#ifdef USE_MUTEX
static CRITICAL_SECTION mutex;
#endif
static const char* driveletters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
/* Forward */
static int isAbsolutePath(const char* path);
/**************************************************/
int
ncp_win32_initialize()
{
int ret = 1;
#ifdef USE_MUTEX
InitializeCriticalSectionAndSpinCount(&mutex, 4000);
#endif
(void)SetErrorMode(SEM_FAILCRITICALERRORS|SEM_NOGPFAULTERRORBOX|SEM_NOOPENFILEERRORBOX);
return ret;
}
int
ncp_win32_finalize()
{
#ifdef USE_MUTEX
DeleteCriticalSection(&mutex);
#endif
return 1;
}
/**************************************************/
#ifdef USE_MUTEX
static void lock(void) {EnterCriticalSection(&mutex);}
#else
#define lock()
#endif
#ifdef USE_MUTEX
static void unlock(void) {LeaveCriticalSection(&mutex);}
#else
#define unlock()
#endif
/**************************************************/
static int
init(NCPSharedLib* lib)
{
int ret = NC_NOERR;
return ret;
}
static int
reclaim(NCPSharedLib* lib)
{
int ret = NC_NOERR;
return ret;
}
static int
load(NCPSharedLib* lib , const char* path0, int flags)
{
int ret = NC_NOERR;
DWORD realflags = 0;
char* path = NULL;
path = NCpathcvt(path0);
if(path == NULL) {ret = NC_EINVAL; goto done;}
lock();
if(lib->state.handle != NULL)
{ret = NC_EEXIST; goto ldone;}
lib->path = nulldup(path);
lib->flags = flags;
if(isAbsolutePath(path)) realflags |= LOAD_WITH_ALTERED_SEARCH_PATH;
lib->state.flags = realflags;
lib->state.handle = LoadLibraryExA(path, 0, realflags);
if(lib->state.handle == NULL) {
int errcode = GetLastError();
char* msg = NULL;
FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, errcode, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &msg, 0, NULL);
if(msg) {
strncpy(lib->err.msg,msg,sizeof(lib->err.msg));
} else
lib->err.msg[0] = '\0';
ret = NC_ENOTFOUND;
goto ldone;
}
ldone:
unlock();
done:
nullfree(path);
return ret;
}
static int
unload(NCPSharedLib* lib)
{
int ret = NC_NOERR;
lock();
if(lib->state.handle != NULL) {
FreeLibrary((HMODULE)lib->state.handle);
lib->state.handle = NULL;
goto done;
}
done:
unlock();
return ret;
}
static int
isLoaded(NCPSharedLib* lib)
{
return lib->state.handle != NULL;
}
static void*
getsymbol(NCPSharedLib* lib, const char* name)
{
void* result = NULL;
lock();
if(lib->state.handle != NULL) {
result = (void*)GetProcAddress((HMODULE)lib->state.handle,name);
}
unlock();
return result;
}
static const char*
getpath(NCPSharedLib* lib)
{
return lib->path;
}
static int
isAbsolutePath(const char* path)
{
if(path == NULL || path[0] == '\0')
return 0;
if(strchr(driveletters,path[0]) != NULL
&& path[1] == ':'
&& (path[2] == '/' || path[2] == '\\'))
return 1;
if(path[0] == '/' || path[2] == '\\')
return 1;
return 0;
}
struct NCPAPI ncp_win32_api = {
init,
reclaim,
load,
unload,
isLoaded,
getsymbol,
getpath,
};
#endif /*_WIN32*/

10
libncpoco/cptestlib.c Executable file
View File

@ -0,0 +1,10 @@
#include "assert.h"
#define DLL_EXPORT
#include "cpoco.h"
EXTERNL int gimmeFive()
{
return 5;
}

115
libncpoco/ncpoco.c Executable file
View File

@ -0,0 +1,115 @@
/*********************************************************************
* Copyright 1993, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* Derived from poco library from Boost Software: see nccpoco/SourceLicense file.
*********************************************************************/
#include "config.h"
#include <stdlib.h>
#include <string.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "netcdf.h"
#include "ncpoco.h"
/**************************************************/
#ifdef _WIN32
extern struct NCPAPI ncp_win32_api;
#else
extern struct NCPAPI ncp_unix_api;
#endif
/**************************************************/
/* Creates a SharedLib object. */
EXTERNL int
ncpsharedlibnew(NCPSharedLib** libp)
{
int ret = NC_NOERR;
NCPSharedLib* lib;
lib = (NCPSharedLib*)calloc(1,sizeof(NCPSharedLib));
if(lib == 0)
{ret = NC_ENOMEM; goto done;}
/* fill in the api */
#ifdef _WIN32
lib->api = ncp_win32_api;
#else
lib->api = ncp_unix_api;
#endif
ret = lib->api.init(lib);
if(ret == NC_NOERR && libp)
*libp = lib;
done:
return ret;
}
/* free this shared library */
EXTERNL int
ncpsharedlibfree(NCPSharedLib* lib)
{
int ret = NC_NOERR;
if(lib == NULL) return NC_NOERR;
ret = lib->api.unload(lib);
ret = lib->api.reclaim(lib);
/* reclaim common stuff */
nullfree(lib->path);
free(lib);
return ret;
}
/*
Loads a shared library from the given path using specified flags.
Returns error if a library has already been loaded or cannot be loaded.
*/
EXTERNL int
ncpload(NCPSharedLib* lib, const char* path, int flags)
{
if(lib == NULL || path == NULL) return NC_EINVAL;
return lib->api.load(lib,path,flags);
}
EXTERNL int
ncpunload(NCPSharedLib* lib) /* Unloads a shared library. */
{
if(lib == NULL) return NC_EINVAL;
return lib->api.unload(lib);
}
EXTERNL int
ncpisloaded(NCPSharedLib* lib) /* Returns 1 iff a library has been loaded. */
{
if(lib == NULL) return 0;
return lib->api.isloaded(lib);
}
/* Returns the address of the symbol with the given name.
For functions, this is the entry point of the function.
Return error if the symbol does not exist
*/
EXTERNL void*
ncpgetsymbol(NCPSharedLib* lib,const char* name)
{
if(lib == NULL) return NULL;
return lib->api.getsymbol(lib,name);
}
/* Returns the path of the library, as specified in
a call to load()
*/
EXTERNL const char*
ncpgetpath(NCPSharedLib* lib)
{
if(lib == NULL) return NULL;
return lib->api.getpath(lib);
}
/* Returns the last err msg. */
EXTERNL const char*
ncpgeterrmsg(NCPSharedLib* lib)
{
if(lib == NULL) return NULL;
return (lib->err.msg[0] == '\0' ? NULL : lib->err.msg);
}

76
libncpoco/ncpoco.h Executable file
View File

@ -0,0 +1,76 @@
/*********************************************************************
* Copyright 1993, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* Derived from poco library from Boost Software: see nccpoco/SourceLicence file.
*********************************************************************/
#ifndef NCPOCO_H
#define NCPOCO_H
/* Use NCP or nccp or ncpoco as our "namespace */
/* Forward */
typedef struct NCPSharedLib NCPSharedLib;
typedef enum NCP_Flags NCP_Flags;
enum NCP_Flags {
NCP_GLOBAL = 1, /* (default) use RTLD_GLOBAL; ignored on platforms that do not use dlopen(). */
NCP_LOCAL = 2 /* use RTLD_LOCAL; RTTI will not work with this flag. */
};
struct NCPSharedLib {
char* path;
int flags;
struct NCPstate {
void* handle;
int flags;
} state;
struct {
char msg[4096];
} err;
struct NCPAPI {
int (*init)(NCPSharedLib*);
int (*reclaim)(NCPSharedLib*); /* reclaim resources; do not free lib */
int (*load)(NCPSharedLib*,const char*,int);
int (*unload)(NCPSharedLib*);
int (*isloaded)(NCPSharedLib*);
void* (*getsymbol)(NCPSharedLib*,const char*);
const char* (*getpath)(NCPSharedLib*);
} api;
};
/**************************************************/
/* SharedLib API */
EXTERNL int ncpsharedlibnew(NCPSharedLib**); /* Creates a NCPSharedLib object. */
EXTERNL int ncpsharedlibfree(NCPSharedLib*); /* free this shared library */
/*
Loads a shared library from the given path using specified flags.
Returns error + message if a library has already been loaded or cannot be loaded.
*/
EXTERNL int ncpload(NCPSharedLib*,const char* path, int flags);
EXTERNL int ncpunload(NCPSharedLib*); /* Unloads a shared library. */
EXTERNL int ncpisloaded(NCPSharedLib*); /* Returns 1 iff a library has been loaded.*/
/* Returns the address of the symbol with the given name.
For functions, this is the entry point of the function.
Return NULL if the symbol does not exist
*/
EXTERNL void* ncpgetsymbol(NCPSharedLib*,const char* name);
/* Returns the path of the library, as specified in
a call to load(NCPSharedLib*)
*/
EXTERNL const char* ncpgetpath(NCPSharedLib*);
/* Return last err msg */
EXTERNL const char* ncpgeterrmsg(NCPSharedLib* lib);
EXTERNL const char* intstr(int err1);
#endif /*NCPOCO_H*/

3
libnczarr/CMakeLists.txt
View File

@ -42,6 +42,7 @@ zinternal.h
zmap.h
zodom.h
zprovenance.h
zfilter.h
zdebug.h
)
@ -58,6 +59,8 @@ ENDIF()
add_library(nczarr OBJECT ${libnczarr_SOURCES})
TARGET_INCLUDE_DIRECTORIES(nczarr PUBLIC ../libncpoco)
IF(MPI_C_INCLUDE_PATH)
target_include_directories(nczarr PUBLIC ${MPI_C_INCLUDE_PATH})
ENDIF(MPI_C_INCLUDE_PATH)

5
libnczarr/Makefile.am
View File

@ -8,7 +8,7 @@
# Get AM_CPPFLAGS and AM_LDFLAGS
include $(top_srcdir)/lib_flags.am
AM_CPPFLAGS += -D_LARGEFILE_SOURCE
AM_CPPFLAGS += -I$(top_srcdir)/libsrc4 -I$(top_srcdir)/libdap4
AM_CPPFLAGS += -I$(top_srcdir)/libsrc4 -I$(top_srcdir)/libncpoco
libnczarr_la_CPPFLAGS = ${AM_CPPFLAGS}
AM_CXXFLAGS =
libnczarr_la_LIBADD =
@ -38,7 +38,6 @@ zcvt.c \
zdim.c \
zdispatch.c \
zfile.c \
zfilter.c \
zgrp.c \
zinternal.c \
zmap.c \
@ -51,6 +50,7 @@ ztype.c \
zutil.c \
zvar.c \
zwalk.c \
zfilter.c \
zdebug.c \
zarr.h \
zcache.h \
@ -62,6 +62,7 @@ zinternal.h \
zmap.h \
zodom.h \
zprovenance.h \
zfilter.h \
zdebug.h
if ENABLE_NCZARR_ZIP

1
libnczarr/awsincludes.h
View File

@ -20,6 +20,7 @@
#include <aws/core/utils/memory/stl/AWSString.h>
#include <aws/core/utils/logging/DefaultLogSystem.h>
#include <aws/core/utils/logging/AWSLogging.h>
#include <aws/core/Version.h>
#include <ios>
#include <iostream>
#include <fstream>

865
libnczarr/obsolete/zjson.c
View File

@ -1,865 +0,0 @@
/* Copyright 2018, UCAR/Unidata.
See the COPYRIGHT file for more information.
*/
#include "zincludes.h"
#include "zjson.h"
#undef DEBUG
#define NCJ_LBRACKET '['
#define NCJ_RBRACKET ']'
#define NCJ_LBRACE '{'
#define NCJ_RBRACE '}'
#define NCJ_COLON ':'
#define NCJ_COMMA ','
#define NCJ_QUOTE '"'
#define NCJ_ESCAPE '\\'
#define NCJ_TAG_TRUE "true"
#define NCJ_TAG_FALSE "false"
#define NCJ_TAG_NULL "null"
#define NCJ_EOF -1
#define NCJ_ERR -2
/* WORD Subsumes Number also */
#define WORD "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$+-."
/*//////////////////////////////////////////////////*/
typedef struct NCJparser {
char* text;
char* pos;
size_t yylen; /* |yytext| */
char* yytext; /* string or word */
long long num;
int tf;
int err;
} NCJparser;
/**************************************************/
/* Forward */
static int NCJparseR(NCJparser* parser, NCjson**);
static int NCJparseArray(NCJparser* parser, NClist* array);
static int NCJparseDict(NCJparser* parser, NClist* dict);
static int testbool(const char* word);
static int testint(const char* word);
static int testdouble(const char* word);
static int testnull(const char* word);
static int NCJlex(NCJparser* parser);
static int NCJyytext(NCJparser*, char* start, ptrdiff_t pdlen);
static void NCJreclaimArray(NClist*);
static void NCJreclaimDict(NClist*);
static int NCJunparseR(const NCjson* json, NCbytes* buf, int flags);
static int NCJunescape(NCJparser* parser);
static int NCJescape(const char* text, NCbytes* buf);
static int NCJappendquoted(const char* value, NCbytes* buf);
static int NCJcloneArray(NClist* array, NCjson** clonep);
static int NCJcloneDict(NClist* dict, NCjson** clonep);
#ifdef DEBUG
static char* tokenname(int token);
#endif
/**************************************************/
int
NCJparse(const char* text, unsigned flags, NCjson** jsonp)
{
int stat = NC_NOERR;
size_t len;
NCJparser* parser = NULL;
NCjson* json = NULL;
/* Need at least 1 character of input */
if(text == NULL || text[0] == '\0')
{stat = NC_EINVAL; goto done;}
if(jsonp == NULL) goto done;
parser = calloc(1,sizeof(NCJparser));
if(parser == NULL)
{stat = NC_ENOMEM; goto done;}
len = strlen(text);
parser->text = (char*)malloc(len+1+1);
if(parser->text == NULL)
{stat = NC_ENOMEM; goto done;}
parser->text[0] = '\0';
strlcat(parser->text,text,len+1);
parser->text[len] = '\0';
parser->text[len+1] = '\0';
parser->pos = &parser->text[0];
#ifdef DEBUG
fprintf(stderr,"json: |%s|\n",parser->text);
#endif
if((stat=NCJparseR(parser,&json))) goto done;
*jsonp = json;
json = NULL;
done:
if(parser != NULL) {
nullfree(parser->text);
nullfree(parser->yytext);
free(parser);
}
(void)NCJreclaim(json);
return THROW(stat);
}
/*
Simple recursive descent
intertwined with dict and list parsers.
Invariants:
1. The json argument is provided by caller and filled in by NCJparseR.
2. Each call pushed back last unconsumed token
*/
static int
NCJparseR(NCJparser* parser, NCjson** jsonp)
{
int stat = NC_NOERR;
int token = NCJ_ERR;
NCjson* json = NULL;
if(jsonp == NULL)
{stat = NC_EINTERNAL; goto done;}
if((token = NCJlex(parser)) == NCJ_ERR)
{stat = NC_EINVAL; goto done;}
switch (token) {
case NCJ_EOF:
break;
case NCJ_NULL:
if((stat = NCJnew(NCJ_NULL,&json))) goto done;
break;
case NCJ_BOOLEAN:
if((stat = NCJnew(NCJ_BOOLEAN,&json))) goto done;
json->value = strdup(parser->yytext);
break;
case NCJ_INT:
if((stat = NCJnew(NCJ_INT,&json))) goto done;
json->value = strdup(parser->yytext);
break;
case NCJ_DOUBLE:
if((stat = NCJnew(NCJ_DOUBLE,&json))) goto done;
json->value = strdup(parser->yytext);
break;
case NCJ_STRING:
if((stat = NCJnew(NCJ_STRING,&json))) goto done;
json->value = strdup(parser->yytext);
break;
case NCJ_LBRACE:
if((stat = NCJnew(NCJ_DICT,&json))) goto done;
if((stat = NCJparseDict(parser, json->contents))) goto done;
break;
case NCJ_LBRACKET:
if((stat = NCJnew(NCJ_ARRAY,&json))) goto done;
if((stat = NCJparseArray(parser, json->contents))) goto done;
break;
case NCJ_RBRACE: /* We hit end of the dict we are parsing */
parser->pos--; /* pushback so NCJparseArray will catch */
json = NULL;
break;
case NCJ_RBRACKET:
parser->pos--; /* pushback so NCJparseDict will catch */
json = NULL;
break;
default:
stat = NC_EINVAL;
break;
}
if(jsonp && json) {*jsonp = json; json = NULL;}
done:
NCJreclaim(json);
return THROW(stat);
}
static int
NCJparseArray(NCJparser* parser, NClist* array)
{
int stat = NC_NOERR;
int token = NCJ_ERR;
NCjson* element = NULL;
int stop = 0;
/* [ ^e1,e2, ...en] */
while(!stop) {
/* Recurse to get the value ei (might be null) */
if((stat = NCJparseR(parser,&element))) goto done;
token = NCJlex(parser); /* Get next token */
/* Next token should be comma or rbracket */
switch(token) {
case NCJ_RBRACKET:
if(element != NULL) nclistpush(array,element);
element = NULL;
stop = 1;
break;
case NCJ_COMMA:
/* Append the ei to the list */
if(element == NULL) {stat = NC_EINVAL; goto done;} /* error */
nclistpush(array,element);
element = NULL;
break;
case NCJ_EOF:
case NCJ_ERR:
default:
stat = NC_EINVAL;
goto done;
}
}
done:
if(element != NULL)
NCJreclaim(element);
return THROW(stat);
}
static int
NCJparseDict(NCJparser* parser, NClist* dict)
{
int stat = NC_NOERR;
int token = NCJ_ERR;
NCjson* value = NULL;
NCjson* key = NULL;
int stop = 0;
/* { ^k1:v1,k2:v2, ...kn:vn] */
while(!stop) {
/* Get the key, which must be a word of some sort */
token = NCJlex(parser);
switch(token) {
case NCJ_STRING:
case NCJ_BOOLEAN:
case NCJ_INT: case NCJ_DOUBLE: {
if((stat=NCJnewstring(token,parser->yytext,&key))) goto done;
} break;
case NCJ_RBRACE: /* End of containing Dict */
stop = 1;
continue; /* leave loop */
case NCJ_EOF: case NCJ_ERR:
default:
stat = NC_EINVAL;
goto done;
}
/* Next token must be colon*/
switch((token = NCJlex(parser))) {
case NCJ_COLON: break;
case NCJ_ERR: case NCJ_EOF:
default: stat = NC_EINVAL; goto done;
}
/* Get the value */
if((stat = NCJparseR(parser,&value))) goto done;
/* Next token must be comma or RBRACE */
switch((token = NCJlex(parser))) {
case NCJ_RBRACE:
stop = 1;
/* fall thru */
case NCJ_COMMA:
/* Insert key value into dict: key first, then value */
nclistpush(dict,key);
key = NULL;
nclistpush(dict,value);
value = NULL;
break;
case NCJ_EOF:
case NCJ_ERR:
default:
stat = NC_EINVAL;
goto done;
}
}
done:
if(key != NULL)
NCJreclaim(key);
if(value != NULL)
NCJreclaim(value);
return THROW(stat);
}
static int
NCJlex(NCJparser* parser)
{
int c;
int token = 0;
char* start;
ptrdiff_t count;
while(token == 0) { /* avoid need to goto when retrying */
c = *parser->pos;
if(c == '\0') {
token = NCJ_EOF;
} else if(c <= ' ' || c == '\177') {
parser->pos++;
continue; /* ignore whitespace */
} else if(strchr(WORD, c) != NULL) {
start = parser->pos;
for(;;) {
c = *parser->pos++;
if(c == '\0' || strchr(WORD,c) == NULL) break; /* end of word */
}
/* Pushback c if not whitespace */
parser->pos--;
count = ((parser->pos) - start);
if(NCJyytext(parser,start,count)) goto done;
/* Discriminate the word string to get the proper sort */
if(testbool(parser->yytext) == NC_NOERR)
token = NCJ_BOOLEAN;
/* do int test first since double subsumes int */
else if(testint(parser->yytext) == NC_NOERR)
token = NCJ_INT;
else if(testdouble(parser->yytext) == NC_NOERR)
token = NCJ_DOUBLE;
else if(testnull(parser->yytext) == NC_NOERR)
token = NCJ_NULL;
else
token = NCJ_STRING;
} else if(c == NCJ_QUOTE) {
parser->pos++;
start = parser->pos;
for(;;) {
c = *parser->pos++;
if(c == NCJ_ESCAPE) parser->pos++;
else if(c == NCJ_QUOTE || c == '\0') break;
}
if(c == '\0') {
parser->err = NC_EINVAL;
token = NCJ_ERR;
goto done;
}
count = ((parser->pos) - start) - 1; /* -1 for trailing quote */
if(NCJyytext(parser,start,count)) goto done;
if(NCJunescape(parser)) goto done;
token = NCJ_STRING;
} else { /* single char token */
if(NCJyytext(parser,parser->pos,1)) goto done;
token = *parser->pos++;
}
#ifdef DEBUG
fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,parser->yytext);
#endif
} /*for(;;)*/
done:
if(parser->err) token = NCJ_ERR;
return token;
}
static int
testnull(const char* word)
{
if(strcasecmp(word,NCJ_TAG_NULL)==0)
return NC_NOERR;
return NC_EINVAL;
}
static int
testbool(const char* word)
{
if(strcasecmp(word,NCJ_TAG_TRUE)==0
|| strcasecmp(word,NCJ_TAG_FALSE)==0)
return NC_NOERR;
return NC_EINVAL;
}
static int
testint(const char* word)
{
int ncvt;
long long i;
int count = 0;
/* Try to convert to number */
ncvt = sscanf(word,"%lld%n",&i,&count);
return (ncvt == 1 && strlen(word)==count ? NC_NOERR : NC_EINVAL);
}
static int
testdouble(const char* word)
{
int ncvt;
double d;
int count = 0;
/* Check for Nan and Infinity */
if(strcasecmp("nan",word)==0) return NC_NOERR;
if(strcasecmp("infinity",word)==0) return NC_NOERR;
if(strcasecmp("-infinity",word)==0) return NC_NOERR;
/* Allow the XXXf versions as well */
if(strcasecmp("nanf",word)==0) return NC_NOERR;
if(strcasecmp("infinityf",word)==0) return NC_NOERR;
if(strcasecmp("-infinityf",word)==0) return NC_NOERR;
/* Try to convert to number */
ncvt = sscanf(word,"%lg%n",&d,&count);
return (ncvt == 1 && strlen(word)==count ? NC_NOERR : NC_EINVAL);
}
static int
NCJyytext(NCJparser* parser, char* start, ptrdiff_t pdlen)
{
size_t len = (size_t)pdlen;
if(parser->yytext == NULL) {
parser->yytext = (char*)malloc(len+1);
parser->yylen = len;
} else if(parser->yylen <= len) {
parser->yytext = (char*) realloc(parser->yytext,len+1);
parser->yylen = len;
}
if(parser->yytext == NULL) return NC_ENOMEM;
memcpy(parser->yytext,start,len);
parser->yytext[len] = '\0';
return NC_NOERR;
}
/**************************************************/
void
NCJreclaim(NCjson* json)
{
if(json == NULL) return;
switch(json->sort) {
case NCJ_INT:
case NCJ_DOUBLE:
case NCJ_BOOLEAN:
case NCJ_STRING:
nullfree(json->value);
break;
case NCJ_DICT:
NCJreclaimDict(json->contents);
nclistfree(json->contents);
break;
case NCJ_ARRAY:
NCJreclaimArray(json->contents);
nclistfree(json->contents);
break;
default: break; /* nothing to reclaim */
}
free(json);
}
static void
NCJreclaimArray(NClist* array)
{
int i;
for(i=0;i<nclistlength(array);i++) {
NCjson* j = nclistget(array,i);
NCJreclaim(j);
}
}
static void
NCJreclaimDict(NClist* dict)
{
return NCJreclaimArray(dict);
}
int
NCJclone(NCjson* json, NCjson** clonep)
{
int stat = NC_NOERR;
NCjson* clone = NULL;
if(json == NULL) goto done;
switch(json->sort) {
case NCJ_INT:
case NCJ_DOUBLE:
case NCJ_BOOLEAN:
case NCJ_STRING:
if((stat=NCJnew(json->sort,&clone))) goto done;
if((clone->value = strdup(json->value)) == NULL)
{stat = NC_ENOMEM; goto done;}
break;
case NCJ_NULL:
if((stat=NCJnew(json->sort,&clone))) goto done;
break;
case NCJ_DICT:
if((stat=NCJcloneDict(json->contents,&clone))) goto done;
break;
case NCJ_ARRAY:
if((stat=NCJcloneArray(json->contents,&clone))) goto done;
break;
default: break; /* nothing to clone */
}
done:
if(stat == NC_NOERR && clonep) {*clonep = clone; clone = NULL;}
NCJreclaim(clone);
return stat;
}
static int
NCJcloneArray(NClist* array, NCjson** clonep)
{
int i, stat=NC_NOERR;
NCjson* clone = NULL;
if((stat=NCJnew(NCJ_ARRAY,&clone))) goto done;
for(i=0;i<nclistlength(array);i++) {
NCjson* elem = nclistget(array,i);
NCjson* elemclone = NULL;
if((stat=NCJclone(elem,&elemclone))) goto done;
nclistpush(clone->contents,elemclone);
}
done:
if(stat == NC_NOERR && clonep) {*clonep = clone; clone = NULL;}
NCJreclaim(clone);
return stat;
}
static int
NCJcloneDict(NClist* dict, NCjson** clonep)
{
int stat = NC_NOERR;
NCjson* clone = NULL;
if((stat=NCJcloneArray(dict,&clone))) goto done;
/* Convert from array to dict */
clone->sort = NCJ_DICT;
done:
if(stat == NC_NOERR && clonep) {*clonep = clone; clone = NULL;}
NCJreclaim(clone);
return stat;
}
/**************************************************/
/* Build Functions */
int
NCJnew(int sort, NCjson** objectp)
{
int stat = NC_NOERR;
NCjson* object = NULL;
if((object = calloc(1,sizeof(NCjson))) == NULL)
{stat = NC_ENOMEM; goto done;}
object->sort = sort;
switch (sort) {
case NCJ_INT:
case NCJ_DOUBLE:
case NCJ_BOOLEAN:
case NCJ_STRING:
case NCJ_NULL:
break;
case NCJ_DICT:
object->contents = nclistnew();
break;
case NCJ_ARRAY:
object->contents = nclistnew();
break;
default:
stat = NC_EINVAL;
goto done;
}
if(objectp) {*objectp = object; object = NULL;}
done:
if(stat) NCJreclaim(object);
return THROW(stat);
}
int
NCJnewstring(int sort, const char* value, NCjson** jsonp)
{
return NCJnewstringn(sort,strlen(value),value,jsonp);
}
int
NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp)
{
int stat = NC_NOERR;
NCjson* json = NULL;
if(jsonp) *jsonp = NULL;
if((stat = NCJnew(sort,&json)))
goto done;
if((json->value = malloc(len+1))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(json->value,value,len);
json->value[len] = '\0';
if(jsonp) *jsonp = json;
json = NULL; /* avoid memory errors */
done:
if(stat) NCJreclaim(json);
return THROW(stat);
}
/* Insert key-value pair into a dict object.
key will be strdup'd
*/
int
NCJinsert(NCjson* object, char* key, NCjson* value)
{
if(object == NULL || object->sort != NCJ_DICT)
return NC_EINTERNAL;
NCJaddstring(object,NCJ_STRING,key);
nclistpush(object->contents,value);
return NC_NOERR;
}
/* Remove a key-value pair from a dict object.
*/
int
NCJremove(NCjson* dict, char* key, NCjson** keyp, NCjson** valuep)
{
int i;
if(dict == NULL || dict->sort != NCJ_DICT)
return NC_ENCZARR;
for(i=0;i<nclistlength(dict->contents);i+=2) {
NCjson* tmp = (NCjson*)nclistget(dict->contents,i);
if(strcmp(tmp->value,key)==0) {
if(keyp) *keyp = tmp;
if(valuep) *valuep = (NCjson*)nclistget(dict->contents,i+1);
nclistremove(dict->contents,i+1);
nclistremove(dict->contents,i);
break;
}
}
return NC_NOERR;
}
int
NCJaddstring(NCjson* dictarray, int sort, const char* value)
{
int stat = NC_NOERR;
NCjson* jvalue = NULL;
switch (dictarray->sort) {
case NCJ_DICT:
if((stat = NCJnewstring(sort,value,&jvalue))) goto done;
nclistpush(dictarray->contents,jvalue);
break;
case NCJ_ARRAY:
if((stat = NCJnewstring(sort,value,&jvalue))) goto done;
nclistpush(dictarray->contents,jvalue);
break;
default: stat = NC_EINVAL; goto done;
}
done:
return THROW(stat);
}
int
NCJdictith(NCjson* object, size_t i, NCjson** keyp, NCjson** valuep)
{
if(object == NULL || object->sort != NCJ_DICT)
return NC_EINTERNAL;
if(i >= (nclistlength(object->contents)/2))
return NC_EINVAL;
if(keyp) *keyp = nclistget(object->contents,2*i);
if(valuep) *valuep = nclistget(object->contents,(2*i)+1);
return NC_NOERR;
}
int
NCJdictget(NCjson* object, const char* key, NCjson** valuep)
{
int i;
if(object == NULL || object->sort != NCJ_DICT)
return NC_EINTERNAL;
if(valuep) *valuep = NULL;
for(i=0;i<nclistlength(object->contents);i+=2) {
const NCjson* k = nclistget(object->contents,i);
assert(k != NULL && k->sort == NCJ_STRING);
if(strcmp(k->value,key)==0) {
if(valuep) *valuep = nclistget(object->contents,i+1);
break;
}
}
return NC_NOERR;
}
/* Append value to an array or dict object.
*/
int
NCJappend(NCjson* object, NCjson* value)
{
if(object == NULL)
return NC_EINTERNAL;
switch (object->sort) {
case NCJ_ARRAY:
nclistpush(object->contents,value);
break;
case NCJ_DICT:
nclistpush(object->contents,value);
break;
default:
return NC_EINTERNAL;
}
return NC_NOERR;
}
int
NCJarrayith(NCjson* object, size_t i, NCjson** valuep)
{
if(object == NULL || object->sort != NCJ_ARRAY)
return NC_EINTERNAL;
if(valuep) *valuep = nclistget(object->contents,i);
return NC_NOERR;
}
/* Unescape the text in parser->yytext; can
do in place because unescaped string will
always be shorter */
static int
NCJunescape(NCJparser* parser)
{
char* p = parser->yytext;
char* q = p;
int c;
for(;(c=*p++);) {
if(c == NCJ_ESCAPE) {
c = *p++;
switch (c) {
case 'b': c = '\b'; break;
case 'f': c = '\f'; break;
case 'n': c = '\n'; break;
case 'r': c = '\r'; break;
case 't': c = '\t'; break;
case NCJ_QUOTE: c = c; break;
case NCJ_ESCAPE: c = c; break;
default: c = c; break;/* technically not Json conformant */
}
}
*q++ = c;
}
*q = '\0';
return NC_NOERR;
}
/**************************************************/
/* Unparser to convert NCjson object to text in buffer */
int
NCJunparse(const NCjson* json, int flags, char** textp)
{
int stat = NC_NOERR;
NCbytes* buf = ncbytesnew();
if((stat = NCJunparseR(json,buf,flags)))
goto done;
if(textp) {
ncbytesnull(buf);
*textp = ncbytesextract(buf);
}
done:
ncbytesfree(buf);
return THROW(stat);
}
static int
NCJunparseR(const NCjson* json, NCbytes* buf, int flags)
{
int stat = NC_NOERR;
int i;
switch (json->sort) {
case NCJ_STRING:
NCJappendquoted(json->value,buf);
break;
case NCJ_INT:
case NCJ_DOUBLE:
case NCJ_BOOLEAN:
ncbytescat(buf,json->value);
break;
case NCJ_DICT:
ncbytesappend(buf,NCJ_LBRACE);
for(i=0;i<nclistlength(json->contents);) {
const NCjson* key = NULL;
const NCjson* value = NULL;
if(i > 0) ncbytesappend(buf,NCJ_COMMA);
key = nclistget(json->contents,i);
NCJunparseR(key,buf,flags);
ncbytesappend(buf,NCJ_COLON);
ncbytesappend(buf,' ');
/* Allow for the possibility of a short dict entry */
i++;
if(i >= nclistlength(json->contents)) { /*short*/
ncbytescat(buf,"?");
} else {
value = nclistget(json->contents,i);
NCJunparseR(value,buf,flags);
i++;
}
}
ncbytesappend(buf,NCJ_RBRACE);
break;
case NCJ_ARRAY:
ncbytesappend(buf,NCJ_LBRACKET);
for(i=0;i<nclistlength(json->contents);i++) {
const NCjson* value = nclistget(json->contents,i);
if(i > 0) ncbytesappend(buf,NCJ_COMMA);
NCJunparseR(value,buf,flags);
}
ncbytesappend(buf,NCJ_RBRACKET);
break;
case NCJ_NULL:
ncbytescat(buf,"null");
break;
default:
stat = NC_EINVAL; goto done;
}
done:
return THROW(stat);
}
/* Escape a string and append to buf */
static int
NCJescape(const char* text, NCbytes* buf)
{
const char* p = text;
int c;
for(;(c=*p++);) {
char replace = 0;
switch (c) {
case '\b': replace = 'b'; break;
case '\f': replace = 'f'; break;
case '\n': replace = 'n'; break;
case '\r': replace = 'r'; break;
case '\t': replace = 't'; break;
case NCJ_QUOTE: replace = '\''; break;
case NCJ_ESCAPE: replace = '\\'; break;
default: break;
}
if(replace) {
ncbytesappend(buf,NCJ_ESCAPE);
ncbytesappend(buf,replace);
} else
ncbytesappend(buf,c);
}
return NC_NOERR;
}
static int
NCJappendquoted(const char* value, NCbytes* buf)
{
ncbytesappend(buf,'"');
NCJescape(value,buf);
ncbytesappend(buf,'"');
return NC_NOERR;
}
void
NCJdump(const NCjson* json, int flags)
{
char* text = NULL;
(void)NCJunparse(json,0,&text);
fprintf(stderr,"%s\n",text);
fflush(stderr);
}
#ifdef DEBUG
static char*
tokenname(int token)
{
switch (token) {
case NCJ_STRING: return "NCJ_STRING";
case NCJ_INT: return "NCJ_INT";
case NCJ_DOUBLE: return "NCJ_DOUBLE";
case NCJ_BOOLEAN: return "NCJ_BOOLEAN";
case NCJ_DICT: return "NCJ_DICT";
case NCJ_ARRAY: return "NCJ_ARRAY";
case NCJ_NULL: return "NCJ_NULL";
default:
if(token > ' ' && token <= 127) {
static char s[4];
s[0] = '\'';
s[1] = (char)token;
s[2] = '\'';
s[3] = '\0';
return s;
} else
break;
}
return "NCJ_UNDEF";
}
#endif

93
libnczarr/obsolete/zjson.h
View File

@ -1,93 +0,0 @@
/* Copyright 2018, UCAR/Unidata.
See the COPYRIGHT file for more information.
*/
#ifndef NCJSON_H
#define NCJSON_H 1
#include "ncexternl.h"
/* Json object sorts */
#define NCJ_UNDEF 0
#define NCJ_STRING 1
#define NCJ_INT 2
#define NCJ_DOUBLE 3
#define NCJ_BOOLEAN 4
#define NCJ_DICT 5
#define NCJ_ARRAY 6
#define NCJ_NULL 7
/* Don't bother with unions: define
a struct to store primitive values
as unquoted strings. Sort will
provide more info.
Also, this does not use a true hashmap
but rather an envv style list where name
and value alternate. This works under
the assumption that we are generally
iterating over the Dict rather than
probing it.
*/
typedef struct NCjson {
int sort;
char* value;
NClist* contents; /* For array|dict */
} NCjson;
#define NCJF_MULTILINE 1
/* Parse */
EXTERNL int NCJparse(const char* text, unsigned flags, NCjson** jsonp);
/* Build */
EXTERNL int NCJnew(int sort, NCjson** object);
/* Convert a nul terminated string value to an NCjson object */
EXTERNL int NCJnewstring(int sort, const char* value, NCjson** jsonp);
/* Convert a counted string value to an NCjson object (+ nul term)*/
EXTERNL int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);
/* Insert key-value pair into a dict object.
key will be strdup'd.
*/
EXTERNL int NCJinsert(NCjson* object, char* key, NCjson* value);
/* Remove a key-value pair from a dict object.
*/
EXTERNL int NCJremove(NCjson* object, char* key, NCjson** keyp, NCjson** valuep);
/* Insert a string value into a json Dict|Array */
EXTERNL int NCJaddstring(NCjson* dictarray, int sort, const char* value);
/* Get ith pair from dict */
EXTERNL int NCJdictith(NCjson* object, size_t i, NCjson** keyp, NCjson** valuep);
/* Get value for key from dict */
EXTERNL int NCJdictget(NCjson* object, const char* key, NCjson** valuep);
/* Append value to an array or dict object. */
EXTERNL int NCJappend(NCjson* object, NCjson* value);
/* Get ith element from array */
EXTERNL int NCJarrayith(NCjson* object, size_t i, NCjson** valuep);
/* Unparser to convert NCjson object to text in buffer */
EXTERNL int NCJunparse(const NCjson* json, int flags, char** textp);
/* Utilities */
EXTERNL void NCJreclaim(NCjson*);
EXTERNL int NCJclone(NCjson* json, NCjson** clonep); /* deep clone */
/* dump NCjson* object */
EXTERNL void NCJdump(const NCjson* json, int flags);
/* Macro defined functions */
#define NCJlength(json) \
((json)->sort == NCJ_DICT ? (nclistlength((json)->contents)/2) \
: ((json)->sort == NCJ_ARRAY ? (nclistlength((json)->contents)) \
: 1))
#endif /*NCJSON_H*/

6
libnczarr/zarr.c
View File

@ -307,8 +307,8 @@ ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjs
char* tkey = NULL;
NCZMAP* map = zinfo->map;
assert((jattrs->sort = NCJ_DICT));
assert((jtypes->sort = NCJ_DICT));
assert((NCJsort(jattrs) == NCJ_DICT));
assert((NCJsort(jtypes) == NCJ_DICT));
if(container->sort == NCGRP) {
NC_GRP_INFO_T* grp = (NC_GRP_INFO_T*)container;
@ -332,7 +332,7 @@ ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjs
NCjson* k = NULL;
NCjson* v = NULL;
/* remove any previous version */
if(NCJremove(jattrs,NCZ_V2_ATTRS,&k,&v) == NC_NOERR) {
if(!NCJremove(jattrs,NCZ_V2_ATTRS,&k,&v)) {
NCJreclaim(k); NCJreclaim(v);
}
}

10
libnczarr/zarr.h
View File

@ -31,13 +31,13 @@ extern int ncz_close_file(NC_FILE_INFO_T* file, int abort);
/* zcvt.c */
extern int NCZ_convert1(NCjson* jsrc, nc_type, unsigned char* memory0);
extern int NCZ_stringconvert1(nc_type typid, unsigned char* src, char** strp);
extern int NCZ_stringconvert1(nc_type typid, size_t len, char* src, NCjson* jvalue);
extern int NCZ_stringconvert(nc_type typid, size_t len, void* data0, NCjson** jdatap);
/* zsync.c */
extern int ncz_sync_file(NC_FILE_INFO_T* file, int isclose);
extern int ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
extern int ncz_sync_atts(NC_FILE_INFO_T*, NC_OBJ* container, NCindex* attlist);
extern int ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose);
extern int ncz_sync_atts(NC_FILE_INFO_T*, NC_OBJ* container, NCindex* attlist, int isclose);
extern int ncz_read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
extern int ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container);
extern int ncz_read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
@ -67,10 +67,10 @@ extern int NCZ_comma_parse(const char* s, NClist* list);
extern int NCZ_swapatomicdata(size_t datalen, void* data, int typesize);
extern char** NCZ_clonestringvec(size_t len, const char** vec);
extern void NCZ_freestringvec(size_t len, char** vec);
extern int NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fillchunkp);
extern int NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, const void* fill, void** fillchunkp);
extern int NCZ_s3clear(ZS3INFO* s3);
extern int NCZ_ischunkname(const char* name,char dimsep);
extern char* NCZ_chunkpath(struct ChunkKey key,char dimsep);
extern char* NCZ_chunkpath(struct ChunkKey key);
/* Export */
EXTERNL int NCZ_s3urlprocess(NCURI* url, ZS3INFO* s3);

62
libnczarr/zattr.c
View File

@ -10,10 +10,10 @@
*/
#include "zincludes.h"
#include "zfilter.h"
#undef ADEBUG
/**
* @internal Get the attribute list for either a varid or NC_GLOBAL
*
@ -73,6 +73,7 @@ ncz_getattlist(NC_GRP_INFO_T *grp, int varid, NC_VAR_INFO_T **varp, NCindex **at
* the file, they are constructed on the fly.
*
* @param h5 Pointer to ZARR file info struct.
* @param var Pointer to var info struct; NULL signals global.
* @param name Name of attribute.
* @param filetypep Pointer that gets type of the attribute data in
* file.
@ -87,21 +88,40 @@ ncz_getattlist(NC_GRP_INFO_T *grp, int varid, NC_VAR_INFO_T **varp, NCindex **at
* @author Dennis Heimbigner
*/
int
ncz_get_att_special(NC_FILE_INFO_T* h5, const char* name,
ncz_get_att_special(NC_FILE_INFO_T* h5, NC_VAR_INFO_T* var, const char* name,
nc_type* filetypep, nc_type mem_type, size_t* lenp,
int* attnump, void* data)
{
int stat = NC_NOERR;
/* Fail if asking for att id */
if(attnump)
return NC_EATTMETA;
{stat = NC_EATTMETA; goto done;}
/* Handle the per-var case(s) first */
if(var != NULL) {
if(strcmp(name,NC_ATT_CODECS)==0) {
NClist* filters = (NClist*)var->filters;
if(mem_type == NC_NAT) mem_type = NC_CHAR;
if(mem_type != NC_CHAR)
{stat = NC_ECHAR; goto done;}
if(filetypep) *filetypep = NC_CHAR;
if(lenp) *lenp = 0;
if(filters == NULL) goto done;
if((stat = NCZ_codec_attr(var,lenp,data))) goto done;
}
goto done;
}
/* The global reserved attributes */
if(strcmp(name,NCPROPS)==0) {
int len;
if(h5->provenance.ncproperties == NULL)
return NC_ENOTATT;
{stat = NC_ENOTATT; goto done;}
if(mem_type == NC_NAT) mem_type = NC_CHAR;
if(mem_type != NC_CHAR)
return NC_ECHAR;
{stat = NC_ECHAR; goto done;}
if(filetypep) *filetypep = NC_CHAR;
len = strlen(h5->provenance.ncproperties);
if(lenp) *lenp = len;
@ -127,10 +147,12 @@ ncz_get_att_special(NC_FILE_INFO_T* h5, const char* name,
case NC_INT64: *((long long*)data) = (long long)iv; break;
case NC_UINT64: *((unsigned long long*)data) = (unsigned long long)iv; break;
default:
return NC_ERANGE;
{stat = NC_ERANGE; goto done;}
}
}
return NC_NOERR;
done:
return stat;
}
/**
@ -571,7 +593,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
if (*(char **)var->fill_value)
free(*(char **)var->fill_value);
}
free(var->fill_value);
free(var->fill_value); var->fill_value = NULL;
}
#ifdef LOOK
@ -805,11 +827,10 @@ NCZ_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
return retval;
/* If this is one of the reserved atts, use nc_get_att_special. */
if (!var)
{
const NC_reservedatt *ra = NC_findreserved(norm_name);
if (ra && ra->flags & NAMEONLYFLAG)
return ncz_get_att_special(h5, norm_name, xtypep, NC_NAT, lenp, NULL,
return ncz_get_att_special(h5, var, norm_name, xtypep, NC_NAT, lenp, NULL,
NULL);
}
@ -846,11 +867,10 @@ NCZ_inq_attid(int ncid, int varid, const char *name, int *attnump)
return retval;
/* If this is one of the reserved atts, use nc_get_att_special. */
if (!var)
{
const NC_reservedatt *ra = NC_findreserved(norm_name);
if (ra && ra->flags & NAMEONLYFLAG)
return ncz_get_att_special(h5, norm_name, NULL, NC_NAT, NULL, attnump,
return ncz_get_att_special(h5, var, norm_name, NULL, NC_NAT, NULL, attnump,
NULL);
}
@ -924,12 +944,11 @@ NCZ_get_att(int ncid, int varid, const char *name, void *value,
&h5, &grp, &var, NULL)))
return retval;
/* If this is one of the reserved atts, use nc_get_att_special. */
if (!var)
/* If this is one of the reserved global atts, use nc_get_att_special. */
{
const NC_reservedatt *ra = NC_findreserved(norm_name);
if (ra && ra->flags & NAMEONLYFLAG)
return ncz_get_att_special(h5, norm_name, NULL, NC_NAT, NULL, NULL,
return ncz_get_att_special(h5, var, norm_name, NULL, NC_NAT, NULL, NULL,
value);
}
@ -1014,6 +1033,16 @@ ncz_makeattr(NC_OBJ* container, NCindex* attlist, const char* name, nc_type type
int stat = NC_NOERR;
NC_ATT_INFO_T* att = NULL;
NCZ_ATT_INFO_T* zatt = NULL;
void* clone = NULL;
size_t typesize, clonesize;
NC_GRP_INFO_T* grp = (container->sort == NCGRP ? (NC_GRP_INFO_T*)container
: ((NC_VAR_INFO_T*)container)->container);
/* Duplicate the values */
if ((stat = nc4_get_typelen_mem(grp->nc4_info, typeid, &typesize))) goto done;
clonesize = len*typesize;
if((clone = malloc(clonesize))==NULL) {stat = NC_ENOMEM; goto done;}
memcpy(clone,values,clonesize);
if((stat=nc4_att_list_add(attlist,name,&att)))
goto done;
@ -1030,11 +1059,12 @@ ncz_makeattr(NC_OBJ* container, NCindex* attlist, const char* name, nc_type type
/* Fill in the attribute's type and value */
att->nc_typeid = typeid;
att->len = len;
att->data = values;
att->data = clone; clone = NULL;
att->dirty = NC_TRUE;
if(attp) {*attp = att; att = NULL;}
done:
nullfree(clone);
if(stat) {
if(att) nc4_att_list_del(attlist,att);
nullfree(zatt);

30
libnczarr/zcache.h
View File

@ -12,6 +12,16 @@
struct NCxcache;
/* Note in the following: the term "real"
refers to the unfiltered/uncompressed data
The term filtered refers to the result of running
the real data through the filter chain. Note that the
sizeof the filtered data might be larger than the size of
the real data.
The term "raw" is used to refer to the data on disk and it may either
be real or filtered.
*/
typedef struct NCZCacheEntry {
struct List {void* next; void* prev; void* unused;} list;
int modified;
@ -21,24 +31,30 @@ typedef struct NCZCacheEntry {
char* chunkkey; /* name of the chunk */
} key;
size64_t hashkey;
void* data;
int isfiltered; /* 1=>data contains filtered data else real data */
size64_t size; /* |data| */
void* data; /* contains either filtered or real data */
} NCZCacheEntry;
typedef struct NCZChunkCache {
const NC_VAR_INFO_T* var; /* backlink */
NC_VAR_INFO_T* var; /* backlink */
size64_t ndims; /* true ndims == var->ndims + scalar */
size64_t chunksize;
void* fillchunk; /* enough fillvalues to fill a chunk */
size_t maxentries; /* Max number of entries allowed */
NClist* mru; /* all cache entries in mru order */
size64_t chunksize; /* for real data */
void* fillchunk; /* enough fillvalues to fill a real chunk */
size_t maxentries; /* Max number of entries allowed; maxsize can override */
size_t maxsize; /* Maximum space used by cache; 0 => nolimit */
size_t used; /* How much total space is being used */
NClist* mru; /* NClist<NCZCacheEntry> all cache entries in mru order */
struct NCxcache* xcache;
char dimension_separator;
} NCZChunkCache;
/**************************************************/
#define FILTERED(cache) (nclistlength((NClist*)(cache)->var->filters) || (cache)->var->shuffle || (cache)->var->fletcher32);
extern int NCZ_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption);
extern int NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
extern int NCZ_adjust_var_cache(NC_VAR_INFO_T *var);
extern int NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t, char dimsep, NCZChunkCache** cachep);
extern void NCZ_free_chunk_cache(NCZChunkCache* cache);
extern int NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap);

1
libnczarr/zchunking.h
View File

@ -67,7 +67,6 @@ struct Common {
size64_t* memshape;
void* memory;
size_t typesize;
void* fillvalue;
size64_t chunkcount; /* computed product of chunklens; warning indices, not bytes */
int swap; /* var->format_info_file->native_endianness == var->endianness */
size64_t shape[NC_MAX_VAR_DIMS]; /* shape of the output hyperslab */

1
libnczarr/zclose.c
View File

@ -4,6 +4,7 @@
*********************************************************************/
#include "zincludes.h"
#include "zfilter.h"
/* Forward */
static int zclose_group(NC_GRP_INFO_T*);

6
libnczarr/zcreate.c
View File

@ -66,10 +66,8 @@ ncz_create_file(const char *path, int cmode, size_t initialsz, const char** cont
return NC_NOERR;
exit: /*failure exit*/
if(!h5) return retval;
#ifdef LOOK
ncz_close_ncz_file(h5, 1, NULL); /* treat like abort */
#endif
if(retval && h5)
ncz_closeorabort(h5, NULL, 1); /* treat like abort */
return retval;
}

123
libnczarr/zcvt.c
View File

@ -22,6 +22,10 @@ struct ZCVT {
double float64v;
};
/* Forward */
static int typeid2jtype(nc_type typeid);
/* Convert an NCJ_STRING to a memory equivalent value of specified dsttype */
int
NCZ_convert1(NCjson* jsrc, nc_type dsttype, unsigned char* memory)
{
@ -31,7 +35,7 @@ NCZ_convert1(NCjson* jsrc, nc_type dsttype, unsigned char* memory)
int outofrange = 0;
/* Convert the incoming jsrc string to a restricted set of values */
switch (jsrc->sort) {
switch (NCJsort(jsrc)) {
case NCJ_INT: /* convert to (u)int64 */
if(NCJstring(jsrc)[0] == '-') {
if(sscanf(NCJstring(jsrc),"%lld",&zcvt.int64v) != 1)
@ -70,10 +74,10 @@ NCZ_convert1(NCjson* jsrc, nc_type dsttype, unsigned char* memory)
default: stat = NC_EINTERNAL; goto done;
}
/* Now, do the down conversion into memory */
/* Now, do the down conversion */
switch (dsttype) {
case NC_BYTE: {
signed char* p = (signed char*)memory;
signed char* p = (signed char*)memory;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
@ -234,8 +238,9 @@ done:
return stat;
}
/* Convert a memory value to a JSON string value */
int
NCZ_stringconvert1(nc_type srctype, unsigned char* src, char** strp)
NCZ_stringconvert1(nc_type srctype, size_t len, char* src, NCjson* jvalue)
{
int stat = NC_NOERR;
struct ZCVT zcvt;
@ -282,8 +287,8 @@ NCZ_stringconvert1(nc_type srctype, unsigned char* src, char** strp)
dsttype = NC_DOUBLE;
} break;
case NC_DOUBLE: {
zcvt.float64v= (double)(*((double*)src));
dsttype = NC_DOUBLE;
zcvt.float64v= (double)(*((double*)src));
} break;
default: stat = NC_EINTERNAL; goto done;
}
@ -297,11 +302,33 @@ NCZ_stringconvert1(nc_type srctype, unsigned char* src, char** strp)
snprintf(s,sizeof(s),"%llu",zcvt.uint64v);
} break;
case NC_DOUBLE: {
snprintf(s,sizeof(s),"%lg",zcvt.float64v); /* handles NAN? */
#ifdef _WIN32
switch (_fpclass(zcvt.float64v)) {
case _FPCLASS_SNAN: case _FPCLASS_QNAN:
strcpy(s,"Nan"); break;
case _FPCLASS_NINF:
strcpy(s,"-Infinity"); break;
case _FPCLASS_PINF:
strcpy(s,"Infinity"); break;
default:
snprintf(s,sizeof(s),"%lg",zcvt.float64v); /* handles NAN? */
break;
}
#else
if(isnan(zcvt.float64v))
strcpy(s,"NaN");
else if(isinf(zcvt.float64v) && zcvt.float64v < 0)
strcpy(s,"-Infinity");
else if(isinf(zcvt.float64v) && zcvt.float64v > 0)
strcpy(s,"Infinity");
else {
snprintf(s,sizeof(s),"%lg",zcvt.float64v); /* handles NAN? */
}
#endif
} break;
default: stat = NC_EINTERNAL; goto done;
}
if(strp) *strp = strdup(s);
NCJsetstring(jvalue,strdup(s));
done:
return stat;
}
@ -316,6 +343,9 @@ NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap)
char* str = NULL;
NCjson* jvalue = NULL;
NCjson* jdata = NULL;
int jtype = NCJ_UNDEF;
jtype = typeid2jtype(typeid);
if((stat = NC4_inq_atomic_type(typeid, NULL, &typelen)))
goto done;
@ -325,61 +355,15 @@ NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap)
/* Create a string valued json object */
if((stat = NCJnewstringn(NCJ_STRING,len,src,&jdata)))
goto done;
} else { /* all other cases */
if(len == 0) {stat = NC_EINVAL; goto done;}
if(len > 1) {
if((stat = NCJnew(NCJ_ARRAY,&jdata))) goto done;
} else /* return a singletone */
jdata = NULL;
} else if(len == 1) { /* create singleton */
if((stat = NCJnew(jtype,&jdata))) goto done;
if((stat = NCZ_stringconvert1(typeid, len, src, jdata))) goto done;
} else { /* len > 1 create array of values */
if((stat = NCJnew(NCJ_ARRAY,&jdata))) goto done;
for(i=0;i<len;i++) {
char* special = NULL;
double d;
if((stat = NCZ_stringconvert1(typeid, src, &str)))
goto done;
switch (typeid) {
case NC_BYTE: case NC_SHORT: case NC_INT: case NC_INT64:
case NC_UBYTE: case NC_USHORT: case NC_UINT: case NC_UINT64:
if((stat=NCJnew(NCJ_INT,&jvalue))) goto done;
break;
case NC_FLOAT:
case NC_DOUBLE: {
if(typeid == NC_FLOAT)
d = (double)(*((float*)src));
else
d = *((double*)src);
#ifdef _WIN32
switch (_fpclass(d)) {
case _FPCLASS_SNAN: case _FPCLASS_QNAN:
special = "Nan"; break;
case _FPCLASS_NINF:
special = "-Infinity"; break;
case _FPCLASS_PINF:
special = "Infinity"; break;
default: break;
}
#else
if(isnan(d))
special = "NaN";
else if(isinf(d) && d < 0)
special = "-Infinity";
else if(isinf(d) && d > 0)
special = "Infinity";
else {}
#endif
if((stat=NCJnew(NCJ_DOUBLE,&jvalue))) goto done;
} break;
case NC_CHAR:
if((stat=NCJnew(NCJ_STRING,&jvalue))) goto done;
break;
default: stat = NC_EINTERNAL; goto done;
}
if(special) {nullfree(str); str = strdup(special);}
NCJstring(jvalue) = str;
str = NULL;
if(len == 1)
jdata = jvalue;
else
NCJappend(jdata,jvalue);
if((stat = NCJnew(jtype,&jvalue))) goto done;
if((stat = NCZ_stringconvert1(typeid, len, src, jvalue))) goto done;
NCJappend(jdata,jvalue);
jvalue = NULL;
src += typelen;
}
@ -392,3 +376,20 @@ done:
NCJreclaim(jdata);
return stat;
}
static int
typeid2jtype(nc_type typeid)
{
switch (typeid) {
case NC_BYTE: case NC_SHORT: case NC_INT: case NC_INT64:
case NC_UBYTE: case NC_USHORT: case NC_UINT: case NC_UINT64:
return NCJ_INT;
case NC_FLOAT:
case NC_DOUBLE:
return NCJ_DOUBLE;
case NC_CHAR:
return NCJ_STRING;
default: break;
}
return NCJ_UNDEF;
}

9
libnczarr/zdebug.c
View File

@ -288,6 +288,15 @@ nczprint_idvector(size_t len, const int* ids)
return nczprint_vector(len,v);
}
char*
nczprint_paramvector(size_t len, const unsigned* params)
{
size64_t v[4096];
size_t i;
for(i=0;i<len;i++) v[i] = params[i];
return nczprint_vector(len,v);
}
char*
nczprint_sizevector(size_t len, const size_t* sizes)
{

7
libnczarr/zdebug.h
View File

@ -14,6 +14,12 @@
#include "ncexternl.h"
#include "nclog.h"
#ifdef LOGGING
#define ZLOG(tag,...) nclog(tag,__VA_ARGS__)
#else
#define ZLOG(tag,...)
#endif
#ifdef ZCATCH
/* Place breakpoint on zbreakpoint to catch errors close to where they occur*/
#define THROW(e) zthrow((e),__FILE__, __func__, __LINE__)
@ -53,6 +59,7 @@ EXTERNL char* nczprint_projectionx(const NCZProjection proj, int raw);
EXTERNL char* nczprint_sliceprojectionsx(const NCZSliceProjections slp, int raw);
EXTERNL char* nczprint_vector(size_t,const size64_t*);
EXTERNL char* nczprint_idvector(size_t,const int*);
EXTERNL char* nczprint_paramvector(size_t,const unsigned*);
EXTERNL char* nczprint_sizevector(size_t,const size_t*);
EXTERNL char* nczprint_envv(const char** envv);

7
libnczarr/zdispatch.c
View File

@ -15,7 +15,6 @@
static int NCZ_var_par_access(int ncid, int varid, int par_access);
static int NCZ_show_metadata(int ncid);
static const NC_Dispatch NCZ_dispatcher = {
NC_FORMATX_NCZARR,
@ -96,11 +95,11 @@ static const NC_Dispatch NCZ_dispatcher = {
NC_NOTNC4_inq_enum_member,
NC_NOTNC4_inq_enum_ident,
NC_NOTNC4_def_opaque,
NC_NOTNC4_def_var_deflate,
NC_NOTNC4_def_var_fletcher32,
NCZ_def_var_deflate,
NCZ_def_var_fletcher32,
NCZ_def_var_chunking,
NCZ_def_var_endian,
NC_NOTNC4_def_var_filter,
NCZ_def_var_filter,
NCZ_set_var_chunk_cache,
NC4_get_var_chunk_cache,
NCZ_inq_var_filter_ids,

4
libnczarr/zdispatch.h
View File

@ -173,6 +173,10 @@ EXTERNL int NCZ_def_var_filter(int ncid, int varid, unsigned int filterid, size_
EXTERNL int NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int *filterids);
EXTERNL int NCZ_inq_var_filter_info(int ncid, int varid, unsigned int filterid, size_t* nparamsp, unsigned int *params);
EXTERNL int NCZ_def_var_filterx(int ncid, int varid, const char* text);
EXTERNL int NCZ_inq_var_filterx_ids(int ncid, int varid, char** textp);
EXTERNL int NCZ_inq_var_filterx_info(int ncid, int varid, const char* id, char** textp);
#if defined(__cplusplus)
}
#endif

18
libnczarr/zfile.c
View File

@ -13,6 +13,7 @@
*/
#include "zincludes.h"
#include "zfilter.h"
/* Forward */
static int NCZ_enddef(int ncid);
@ -103,19 +104,24 @@ NCZ_enddef(int ncid)
ZTRACE(1,"ncid=%d",ncid);
LOG((1, "%s: ncid 0x%x", __func__, ncid));
/* Find pointer to group and zinfo. */
if ((stat = nc4_find_grp_h5(ncid, &grp, &h5)))
goto done;
/* When exiting define mode, mark all variable written. */
/* When exiting define mode, process all variables */
for (i = 0; i < nclistlength(h5->allgroups); i++) {
NC_GRP_INFO_T* g = nclistget(h5->allgroups,i);
for (j = 0; j < ncindexsize(g->vars); j++) {
var = (NC_VAR_INFO_T *)ncindexith(g->vars, j);
assert(var);
var->written_to = NC_TRUE;
/* set the fill value and _FillValue attribute */
if((stat = ncz_get_fill_value(h5,var,NULL))) goto done; /* ensure var->fill_value is set */
assert(var->fill_value != NULL);
var->written_to = NC_TRUE; /* mark it written */
/* rebuild the fill chunk */
if((stat = NCZ_adjust_var_cache(var))) goto done;
/* Build the filter working parameters for any filters */
if((stat = NCZ_filter_setup(var))) goto done;
}
}
stat = ncz_enddef_netcdf4_file(h5);
@ -236,13 +242,13 @@ ncz_closeorabort(NC_FILE_INFO_T* h5, void* params, int abort)
NC_UNUSED(params);
ZTRACE(1,"file=%s abort=%d",h5->hdr.name,abort);
ZTRACE(3,"file=%s abort=%d",h5->hdr.name,abort);
LOG((2, "%s: file: %p", __func__, h5));
/* If we're in define mode, but not redefing the file, delete it. */
if(!abort) {
/* Invoke enddef if needed, which mean sync */
/* Invoke enddef if needed, which mean sync first */
if(h5->flags & NC_INDEF) h5->flags ^= NC_INDEF;
/* Sync the file unless this is a read-only file. */
if(!h5->no_write) {

1522
libnczarr/zfilter.c
View File

File diff suppressed because it is too large Load Diff

52
libnczarr/zfilter.h Normal file
View File

@ -0,0 +1,52 @@
/* Copyright 2018-2018 University Corporation for Atmospheric
Research/Unidata. */
/**
* @file This header file containsfilter related macros, types, and prototypes for
* the filter code in libnczarr. This header should not be included in
* code outside libnczarr.
*
* @author Dennis Heimbigner
*/
#ifndef ZFILTER_H
#define ZFILTER_H
/* zfilter.c */
/* Dispatch functions are also in zfilter.c */
/* Filterlist management */
/*Mnemonic*/
#define ENCODING 1
/* list of environment variables to check for plugin roots */
#define plugin_env "HDF5_PLUGIN_PATH"
#define plugin_dir_unix "/usr/local/hdf5/plugin"
#define plugin_dir_win "%s/hdf5/lib/plugin"
#define win32_root_env "ALLUSERSPROFILE"
/*
Return a NULL terminated vector of pointers to instances of ''NCZ_codec_t''.
The value returned is actually of type ''NCZ_codec_t**'',
but is of type ''void*'' to allow for extensions.
The list of returned items are used to try to provide defaults
for any HDF5 filters that have no corresponding Codec.
This is for internal use only.
*/
typedef void* (*NCZ_codec_info_defaults_proto)(void);
/* Opaque */
struct NCZ_Filter;
int NCZ_filter_initialize(void);
int NCZ_filter_finalize(void);
int NCZ_addfilter(NC_FILE_INFO_T*, NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params);
int NCZ_filter_setup(NC_VAR_INFO_T* var);
int NCZ_filter_freelist(NC_VAR_INFO_T* var);
int NCZ_codec_freelist(NCZ_VAR_INFO_T* zvar);
int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, size_t insize, void* indata, size_t* outlen, void** outdata, int encode);
int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**);
int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter);
int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data);
#endif /*ZFILTER_H*/

66
libnczarr/zinternal.c
View File

@ -16,6 +16,7 @@
*/
#include "zincludes.h"
#include "zfilter.h"
/* These are the default chunk cache sizes for ZARR files created or
* opened with netCDF-4. */
@ -76,6 +77,7 @@ NCZ_initialize_internal(void)
ngs->zarr.dimension_separator = dimsep[0];
}
}
done:
return stat;
}
@ -89,6 +91,7 @@ NCZ_finalize_internal(void)
{
/* Reclaim global resources */
ncz_initialized = 0;
NCZ_filter_finalize();
return NC_NOERR;
}
@ -631,6 +634,7 @@ ncz_find_grp_var_att(int ncid, int varid, const char *name, int attnum,
/**
* @internal What fill value should be used for a variable?
* Side effects: set as default if necessary and build _FillValue attribute.
*
* @param h5 Pointer to file info struct.
* @param var Pointer to variable info struct.
@ -644,32 +648,45 @@ int
ncz_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp)
{
size_t size;
int retval;
int retval = NC_NOERR;
#if 0 /*LOOK*/
/* Find out how much space we need for this type's fill value. */
if (var->type_info->nc_type_class == NC_VLEN)
size = sizeof(nc_vlen_t);
else if (var->type_info->nc_type_class == NC_STRING)
size = sizeof(char *);
else
#endif
{
if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size)))
return retval;
if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size))) goto done;
}
assert(size);
/* Allocate the space. */
if (!((*fillp) = calloc(1, size)))
return NC_ENOMEM;
/* If the user has set a fill_value for this var, use, otherwise find the default fill value. */
/* If the user has set a fill_value for this var, use, otherwise
* find the default fill value. */
if (var->fill_value)
{
LOG((4, "Found a fill value for var %s", var->hdr.name));
if (var->fill_value == NULL) {
/* initialize the fill_value to the default */
/* Allocate the fill_value space. */
if((var->fill_value = calloc(1, size))==NULL)
{retval = NC_ENOMEM; goto done;}
if((retval = nc4_get_default_fill_value(var->type_info->hdr.id, var->fill_value))) {
/* Note: release memory, but don't return error on failure */
free(var->fill_value);
var->fill_value = NULL;
retval = NC_NOERR;
goto done;
}
}
assert(var->fill_value != NULL);
LOG((4, "Found a fill value for var %s", var->hdr.name));
#if 0 /*LOOK*/
/* Need to copy both vlen and a single basetype */
if (var->type_info->nc_type_class == NC_VLEN)
{
nc_vlen_t *in_vlen = (nc_vlen_t *)(var->fill_value), *fv_vlen = (nc_vlen_t *)(*fillp);
nc_vlen_t *in_vlen = (nc_vlen_t *)(var->fill_value);
nc_vlen-t *fv_vlen = (nc_vlen_t *)fill;
size_t basetypesize = 0;
if((retval=nc4_get_typelen_mem(h5, var->type_info->u.v.base_nc_typeid, &basetypesize)))
@ -694,20 +711,21 @@ ncz_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp)
return NC_ENOMEM;
}
}
else
memcpy((*fillp), var->fill_value, size);
}
else
{
if (nc4_get_default_fill_value(var->type_info->hdr.id, *fillp))
{
/* Note: release memory, but don't return error on failure */
free(*fillp);
*fillp = NULL;
}
#endif /*0*/
/* Create _FillValue Attribute */
if((retval = ncz_create_fillvalue(var))) goto done;
if(fillp) {
void* fill = NULL;
/* Allocate the return space. */
if((fill = calloc(1, size))==NULL)
{retval = NC_ENOMEM; goto done;}
memcpy(fill, var->fill_value, size);
*fillp = fill;
fill = NULL;
}
return NC_NOERR;
done:
return retval;
}
#ifdef LOGGING

25
libnczarr/zinternal.h
View File

@ -56,6 +56,9 @@
#define ZATTRS ".zattrs"
#define ZARRAY ".zarray"
/* Pure Zarr pseudo names */
#define ZDIMANON "_zdim"
/* V2 Reserved Attributes */
/*
Inserted into /.zgroup
@ -91,13 +94,16 @@ Inserted into any .zattrs ? or should it go into the container?
#define islegaldimsep(c) ((c) != '\0' && strchr(LEGAL_DIM_SEPARATORS,(c)) != NULL)
/* Mnemonics */
#define ZCLOSE 1 /* this is closeorabort as opposed to enddef */
/* Mnemonics */
#define ZCLOSE 1 /* this is closeorabort as opposed to enddef */
/* Useful macro */
#define ncidforx(file,grpid) ((file)->controller->ext_ncid | (grpid))
#define ncidfor(var) ncidforx((var)->container->nc4_info,(var)->container->hdr.id)
/**************************************************/
/* Forward */
@ -246,23 +252,6 @@ int ncz_makeattr(NC_OBJ*, NCindex* attlist, const char* name, nc_type typid, siz
int ncz_gettype(NC_FILE_INFO_T*, NC_GRP_INFO_T*, int xtype, NC_TYPE_INFO_T** typep);
int ncz_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
/* zfilter.c */
/* Dispatch functions are also in zfilter.c */
/* Filterlist management */
/* The NC_VAR_INFO_T->filters field is an NClist of this struct */
struct NCZ_Filter {
int flags; /**< Flags describing state of this filter. */
#define NCZ_FILTER_MISSING 1 /* Signal filter implementation is not available */
unsigned int filterid; /**< ID for arbitrary filter. */
size_t nparams; /**< nparams for arbitrary filter. */
unsigned int* params; /**< Params for arbitrary filter. */
};
int NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp);
int NCZ_addfilter(NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params);
int NCZ_filter_freelist(NC_VAR_INFO_T* var);
/* Undefined */
/* Find var, doing lazy var metadata read if needed. */
int ncz_find_grp_file_var(int ncid, int varid, NC_FILE_INFO_T** file,

10
libnczarr/zmap.c
View File

@ -409,20 +409,20 @@ nczm_canonicalpath(const char* path, char** cpathp)
{
int ret = NC_NOERR;
char* cpath = NULL;
char* tmp = NULL;
char* tmp1 = NULL;
if(path == NULL)
{cpath = NULL; goto done;}
/* Process path to make it be windows compatible */
if((tmp = NCpathcvt(path))==NULL) {ret = NC_ENOMEM; goto done;}
/* Process path to make it be absolute*/
if((tmp1 = NCpathabsolute(path))==NULL) {ret = NC_ENOMEM; goto done;}
/* Fix slashes to be forward for now */
if((ret = nczm_localize(tmp,&cpath,!LOCALIZE))) goto done;
if((ret = nczm_localize(tmp1,&cpath,!LOCALIZE))) goto done;
if(cpathp) {*cpathp = cpath; cpath = NULL;}
done:
nullfree(tmp);
nullfree(tmp1);
nullfree(cpath);
return THROW(ret);
}

5
libnczarr/zmap.h
View File

@ -110,9 +110,10 @@ model, only content-bearing objects actually exist. Note that
this different than, say, a direvtory tree where a key will
always lead to something: a directory or a file.
In any case, the zmap API returns two distinguished error code:
In any case, the zmap API returns three distinguished error code:
1. NC_NOERR if a operation succeeded
2. NC_EEMPTY is returned when accessing a key that has no content.
2. NC_EEMPTY is returned when accessing a key that has no content or does not exist.
This does not preclude other errors being returned such NC_EACCESS or NC_EPERM or NC_EINVAL
if there are permission errors or illegal function arguments, for example.
It also does not preclude the use of other error codes internal to the zmap

4
libnczarr/zmap_file.c
View File

@ -740,9 +740,7 @@ platformopenfile(ZFMAP* zfmap, const char* canonpath, FD* fd)
/* Try to open file (will localize) */
fd->fd = NCopen3(canonpath, ioflags, permissions);
if(fd->fd < 0)
{
fprintf(stderr,"xxx: canonpath=%s\n",canonpath);
stat = platformerr(errno); goto done;} /* could not open */
{stat = platformerr(errno); goto done;} /* could not open */
done:
errno = 0;
return ZUNTRACEX(stat,"fd=%d",(fd?fd->fd:-1));

31
libnczarr/zmap_zip.c
View File

@ -101,8 +101,6 @@ static int
zipcreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp)
{
int stat = NC_NOERR;
char* truepath = NULL;
char* dataset = NULL;
ZZMAP* zzmap = NULL;
NCURI* url = NULL;
zip_flags_t zipflags = 0;
@ -124,12 +122,6 @@ zipcreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
if(strcasecmp(url->protocol,"file") != 0)
{stat = NC_EURL; goto done;}
/* Canonicalize the root path */
if((stat = NCpathcanonical(url->path,&truepath))) goto done;
/* Extract the dataset name */
if((stat = nczm_basename(truepath,&dataset))) goto done;
/* Build the zz state */
if((zzmap = calloc(1,sizeof(ZZMAP))) == NULL)
{stat = NC_ENOMEM; goto done;}
@ -140,10 +132,11 @@ zipcreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
/* create => NC_WRITE */
zzmap->map.mode = mode;
zzmap->map.api = &zapi;
zzmap->root = truepath;
truepath = NULL;
zzmap->dataset = dataset;
dataset = NULL;
/* Since root is in canonical form, we need to convert to local form */
if((zzmap->root = NCpathcvt(url->path))==NULL)
{stat = NC_ENOMEM; goto done;}
/* Extract the dataset name */
if((stat = nczm_basename(url->path,&zzmap->dataset))) goto done;
/* Set zip openflags */
zipflags |= ZIP_CREATE;
@ -168,8 +161,6 @@ zipcreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
done:
ncurifree(url);
nullfree(truepath);
nullfree(dataset);
if(zzmap) zipclose((NCZMAP*)zzmap,1);
return ZUNTRACE(stat);
}
@ -187,8 +178,6 @@ static int
zipopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp)
{
int stat = NC_NOERR;
char* truepath = NULL;
char* dataset = NULL;
ZZMAP* zzmap = NULL;
NCURI*url = NULL;
zip_flags_t zipflags = 0;
@ -209,9 +198,6 @@ zipopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m
if(strcasecmp(url->protocol,"file") != 0)
{stat = NC_EURL; goto done;}
/* Canonicalize the root path */
if((stat = NCpathcanonical(url->path,&truepath))) goto done;
/* Build the zz state */
if((zzmap = calloc(1,sizeof(ZZMAP))) == NULL)
{stat = NC_ENOMEM; goto done;}
@ -221,8 +207,9 @@ zipopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m
zzmap->map.flags = flags;
zzmap->map.mode = mode;
zzmap->map.api = (NCZMAP_API*)&zapi;
zzmap->root = truepath;
truepath = NULL;
/* Since root is in canonical form, we need to convert to local form */
if((zzmap->root = NCpathcvt(url->path))==NULL)
{stat = NC_ENOMEM; goto done;}
/* Set zip open flags */
zipflags |= ZIP_CHECKCONS;
@ -258,8 +245,6 @@ zipopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m
done:
ncurifree(url);
nullfree(truepath);
nullfree(dataset);
if(zzmap) zipclose((NCZMAP*)zzmap,0);
return ZUNTRACE(stat);

2
libnczarr/zopen.c
View File

@ -124,7 +124,7 @@ ncz_open_file(const char *path, int mode, const char** controls, int ncid)
exit:
if (stat && h5)
ncz_close_file(h5, 1); /* treat like abort*/
ncz_closeorabort(h5, NULL, 1); /* treat like abort*/
return ZUNTRACE(stat);
}

273
libnczarr/zsync.c
View File

@ -4,12 +4,13 @@
*********************************************************************/
#include "zincludes.h"
#include "zfilter.h"
#undef FILLONCLOSE
/* Forward */
static int ncz_collect_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NCjson** jdimsp);
static int ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var);
static int ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose);
static int ncz_jsonize_atts(NCindex* attlist, NCjson** jattrsp);
static int load_jatts(NCZMAP* map, NC_OBJ* container, int nczarrv1, NCjson** jattrsp, NClist** atypes);
@ -63,7 +64,7 @@ ncz_sync_file(NC_FILE_INFO_T* file, int isclose)
ZTRACE(3,"file=%s isclose=%d",file->controller->path,isclose);
/* Write out root group recursively */
if((stat = ncz_sync_grp(file, file->root_grp)))
if((stat = ncz_sync_grp(file, file->root_grp, isclose)))
goto done;
done:
@ -111,7 +112,7 @@ done:
* @author Dennis Heimbigner
*/
int
ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp)
ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
{
int i,stat = NC_NOERR;
NCZ_FILE_INFO_T* zinfo = NULL;
@ -205,19 +206,19 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp)
/* Build the .zattrs object */
assert(grp->att);
if((stat = ncz_sync_atts(file,(NC_OBJ*)grp, grp->att)))
if((stat = ncz_sync_atts(file,(NC_OBJ*)grp, grp->att, isclose)))
goto done;
/* Now synchronize all the variables */
for(i=0; i<ncindexsize(grp->vars); i++) {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
if((stat = ncz_sync_var(file,var))) goto done;
if((stat = ncz_sync_var(file,var,isclose))) goto done;
}
/* Now recurse to synchronize all the subgrps */
for(i=0; i<ncindexsize(grp->children); i++) {
NC_GRP_INFO_T* g = (NC_GRP_INFO_T*)ncindexith(grp->children,i);
if((stat = ncz_sync_grp(file,g))) goto done;
if((stat = ncz_sync_grp(file,g,isclose))) goto done;
}
done:
@ -234,15 +235,17 @@ done:
}
/**
* @internal Synchronize variable data from memory to map.
* @internal Synchronize variable meta data from memory to map.
*
* @param file Pointer to file struct
* @param var Pointer to var struct
* @param isclose If this called as part of nc_close() as opposed to nc_enddef().
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose)
{
int i,stat = NC_NOERR;
NCZ_FILE_INFO_T* zinfo = NULL;
@ -259,9 +262,9 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
NCjson* jfill = NULL;
size64_t shape[NC_MAX_VAR_DIMS];
NCZ_VAR_INFO_T* zvar = var->format_var_info;
NClist* filterchain = NULL;
NCjson* jfilter = NULL;
LOG((3, "%s: dims: %s", __func__, key));
zinfo = file->format_file_info;
map = zinfo->map;
@ -338,30 +341,11 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
if(var->no_fill) {
if((stat=NCJnew(NCJ_NULL,&jfill))) goto done;
} else {/*!var->no_fill*/
int fillsort;
int atomictype = var->type_info->hdr.id;
/* A scalar value providing the default value to use for uninitialized
portions of the array, or ``null`` if no fill_value is to be used. */
/* Use the defaults defined in netdf.h */
assert(atomictype > 0 && atomictype <= NC_MAX_ATOMIC_TYPE && atomictype != NC_STRING);
if((stat = ncz_fill_value_sort(atomictype,&fillsort))) goto done;
if(var->fill_value == NULL) { /* use default */
size_t typelen;
if((stat = NC4_inq_atomic_type(atomictype, NULL, &typelen))) goto done;
var->fill_value = (atomictype == NC_CHAR ? malloc(typelen+1) : malloc(typelen));
if(var->fill_value == NULL) {stat = NC_ENOMEM; goto done;}
if((stat = nc4_get_default_fill_value(atomictype,var->fill_value))) goto done;
}
assert(var->fill_value != NULL);
/* Convert var->fill_value to a string */
if((stat = NCZ_stringconvert(atomictype,1,var->fill_value,&jfill))) goto done;
if(NCJsort(jfill) == NCJ_ARRAY) { /* stringconvert should prevent this from happening */
assert(NCJlength(jfill) > 0);
jtmp = NCJith(jfill,0);
if((stat = NCJclone(jtmp,&jtmp))) goto done; /* clone so we can free it later */
NCJreclaim(jfill);
jfill = jtmp;
jtmp = NULL;
}
assert(jfill->sort != NCJ_ARRAY);
}
if((stat = NCJinsert(jvar,"fill_value",jfill))) goto done;
jfill = NULL;
@ -373,20 +357,44 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
/* Default to C for now */
if((stat = NCJaddstring(jvar,NCJ_STRING,"C"))) goto done;
/* Compressor and Filters */
filterchain = (NClist*)var->filters;
/* compressor key */
/* A JSON object identifying the primary compression codec and providing
/* From V2 Spec: A JSON object identifying the primary compression codec and providing
configuration parameters, or ``null`` if no compressor is to be used. */
if((stat = NCJaddstring(jvar,NCJ_STRING,"compressor"))) goto done;
/* Default to null for now */
if((stat = NCJnew(NCJ_NULL,&jtmp))) goto done;
if((stat = NCJappend(jvar,jtmp))) goto done;
if(nclistlength(filterchain) > 0) {
struct NCZ_Filter* filter = (struct NCZ_Filter*)nclistget(filterchain,nclistlength(filterchain)-1);
/* encode up the compressor */
if((stat = NCZ_filter_jsonize(file,var,filter,&jtmp))) goto done;
} else { /* no filters at all */
/* Default to null */
if((stat = NCJnew(NCJ_NULL,&jtmp))) goto done;
}
if(jtmp && (stat = NCJappend(jvar,jtmp))) goto done;
jtmp = NULL;
/* filters key */
if((stat = NCJaddstring(jvar,NCJ_STRING,"filters"))) goto done;
/* From V2 Spec: A list of JSON objects providing codec configurations,
or null if no filters are to be applied. Each codec configuration
object MUST contain a "id" key identifying the codec to be used. */
/* A list of JSON objects providing codec configurations, or ``null``
if no filters are to be applied. */
if((stat = NCJnew(NCJ_NULL,&jtmp))) goto done;
if((stat = NCJaddstring(jvar,NCJ_STRING,"filters"))) goto done;
if(nclistlength(filterchain) > 1) {
int k;
/* jtmp holds the array of filters */
if((stat = NCJnew(NCJ_ARRAY,&jtmp))) goto done;
for(k=0;k<nclistlength(filterchain)-1;k++) {
struct NCZ_Filter* filter = (struct NCZ_Filter*)nclistget(filterchain,k);
/* encode up the filter as a string */
if((stat = NCZ_filter_jsonize(file,var,filter,&jfilter))) goto done;
if((stat = NCJappend(jtmp,jfilter))) goto done;
}
} else { /* no filters at all */
if((stat = NCJnew(NCJ_NULL,&jtmp))) goto done;
}
if((stat = NCJappend(jvar,jtmp))) goto done;
jtmp = NULL;
@ -460,15 +468,9 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
/* Build .zattrs object */
assert(var->att);
if((stat = ncz_sync_atts(file,(NC_OBJ*)var, var->att)))
if((stat = ncz_sync_atts(file,(NC_OBJ*)var, var->att, isclose)))
goto done;
/* flush only chunks that have been written */
if(zvar->cache) {
if((stat = NCZ_flush_chunk_cache(zvar->cache)))
goto done;
}
done:
nclistfreeall(dimrefs);
nullfree(fullpath);
@ -481,6 +483,37 @@ done:
return THROW(stat);
}
/**
* @internal Synchronize variable meta data and data from memory to map.
*
* @param file Pointer to file struct
* @param var Pointer to var struct
* @param isclose If this called as part of nc_close() as opposed to nc_enddef().
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose)
{
int stat = NC_NOERR;
NCZ_VAR_INFO_T* zvar = var->format_var_info;
if(!isclose) {
if((stat = ncz_sync_var_meta(file,var,isclose))) goto done;
}
/* flush only chunks that have been written */
if(zvar->cache) {
if((stat = NCZ_flush_chunk_cache(zvar->cache)))
goto done;
}
done:
return THROW(stat);
}
/*
Flush all chunks to disk. Create any that are missing
and fill as needed.
@ -535,17 +568,7 @@ ncz_write_var(NC_VAR_INFO_T* var)
default: goto done; /* some other error */
}
/* If we reach here, then chunk does not exist, create it with fill */
/* ensure fillchunk exists */
if(zvar->cache->fillchunk == NULL) {
nc_type typecode;
size_t typesize;
void* fillvalue = NULL;
typecode = var->type_info->hdr.id;
if((stat = NC4_inq_atomic_type(typecode, NULL, &typesize))) goto done;
if((stat = ncz_get_fill_value(file, var, &fillvalue))) goto done;
if((stat = NCZ_create_fill_chunk(zvar->cache->chunksize, typesize, fillvalue, &zvar->cache->fillchunk)))
goto done;
}
assert(zvar->cache->fillchunk != NULL);
if((stat=nczmap_write(map,key,0,zvar->cache->chunksize,zvar->cache->fillchunk))) goto done;
next:
nullfree(key);
@ -570,7 +593,7 @@ done:
* @author Dennis Heimbigner
*/
int
ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist)
ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isclose)
{
int i,stat = NC_NOERR;
NCZ_FILE_INFO_T* zinfo = NULL;
@ -883,7 +906,7 @@ computeattrinfo(const char* name, NClist* atypes, NCjson* values,
int stat = NC_NOERR;
int i;
size_t len, typelen;
void* data;
void* data = NULL;
nc_type typeid;
/* Get type info for the given att */
@ -927,7 +950,7 @@ computeattrdata(nc_type* typeidp, NCjson* values, size_t* typelenp, size_t* lenp
/* Get assumed type */
if(typeidp) typeid = *typeidp;
if(typeid == NC_NAT) inferattrtype(values,&typeid);
if(typeid == NC_NAT) if((stat = inferattrtype(values,&typeid))) goto done;
if(typeid == NC_NAT) {stat = NC_EBADTYPE; goto done;}
/* Collect the length of the attribute; might be a singleton */
@ -936,10 +959,11 @@ computeattrdata(nc_type* typeidp, NCjson* values, size_t* typelenp, size_t* lenp
case NCJ_ARRAY:
count = NCJlength(values);
break;
case NCJ_STRING: /* requires special handling as an array of characters */
if(typeid == NC_CHAR)
case NCJ_STRING: /* requires special handling as an array of characters; also look out for empty string */
if(typeid == NC_CHAR) {
count = strlen(NCJstring(values));
else
if(count == 0) count = 1; /* Actually a single nul char, probably default fill value ugh!*/
} else
count = 1;
break;
default:
@ -973,20 +997,27 @@ done:
}
static int
inferattrtype(NCjson* values, nc_type* typeidp)
inferattrtype(NCjson* value, nc_type* typeidp)
{
int stat = NC_NOERR;
nc_type typeid;
NCjson* j = NULL;
unsigned long long u64;
long long i64;
int negative = 0;
if(NCJsort(value) == NCJ_ARRAY && NCJlength(value) == 0)
{typeid = NC_NAT; goto done;}
switch (NCJsort(values)) {
case NCJ_ARRAY:
if(NCJlength(values) == 0) return NC_EINVAL;
j = NCJith(values,0);
if(NCJsort(value) == NCJ_NULL)
{typeid = NC_NAT; goto done;}
if(value->sort == NCJ_ARRAY) {
j=NCJith(value,0);
return inferattrtype(j,typeidp);
}
switch (NCJsort(value)) {
case NCJ_NULL:
typeid = NC_CHAR;
return NC_NOERR;
@ -996,15 +1027,15 @@ inferattrtype(NCjson* values, nc_type* typeidp)
default: /* atomic */
break;
}
if(NCJstring(values))
negative = (NCJstring(values)[0] == '-');
switch (NCJsort(values)) {
if(NCJstring(value) != NULL)
negative = (NCJstring(value)[0] == '-');
switch (value->sort) {
case NCJ_INT:
if(negative) {
sscanf(NCJstring(values),"%lld",&i64);
sscanf(NCJstring(value),"%lld",&i64);
u64 = (unsigned long long)i64;
} else
sscanf(NCJstring(values),"%llu",&u64);
sscanf(NCJstring(value),"%llu",&u64);
typeid = mininttype(u64,negative);
break;
case NCJ_DOUBLE:
@ -1017,10 +1048,11 @@ inferattrtype(NCjson* values, nc_type* typeidp)
typeid = NC_CHAR;
break;
default:
return NC_ENCZARR;
stat = NC_ENCZARR;
}
done:
if(typeidp) *typeidp = typeid;
return NC_NOERR;
return stat;
}
static int
@ -1192,7 +1224,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container)
NCjson* jattrs = NULL;
NClist* atypes = NULL;
nc_type typeid;
size_t len;
size_t len, typelen;
void* data = NULL;
NC_ATT_INFO_T* fillvalueatt = NULL;
@ -1255,10 +1287,11 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container)
/* Create the attribute */
/* Collect the attribute's type and value */
if((stat = computeattrinfo(NCJstring(key),atypes,value,
&typeid,NULL,&len,&data)))
&typeid,&typelen,&len,&data)))
goto done;
if((stat = ncz_makeattr(container,attlist,NCJstring(key),typeid,len,data,&att)))
goto done;
nullfree(data); data = NULL; /* passed to the attribute */
/* Is this _FillValue ? */
if(strcmp(att->hdr.name,_FillValue)==0) fillvalueatt = att;
}
@ -1279,6 +1312,7 @@ done:
NCJreclaim(jattrs);
nclistfreeall(atypes);
nullfree(fullpath);
nullfree(data);
nullfree(key);
return THROW(stat);
}
@ -1345,6 +1379,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
NCjson* jncvar = NULL;
NCjson* jdimrefs = NULL;
NCjson* jvalue = NULL;
NCjson* jfilter = NULL;
int purezarr = 0;
int xarray = 0;
int formatv1 = 0;
@ -1376,6 +1411,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
zvar->common.file = file;
/* Set filter list */
assert(var->filters == NULL);
var->filters = (void*)nclistnew();
/* Construct var path */
@ -1389,7 +1425,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
if((stat=NCZ_readdict(map,key,&jvar)))
goto done;
nullfree(key); key = NULL;
assert((NCJsort(jvar) == NCJ_DICT));
assert(NCJsort(jvar) == NCJ_DICT);
/* Extract the .zarray info from jvar */
@ -1450,6 +1486,23 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
zvar->dimension_separator = ngs->zarr.dimension_separator; /* use global value */
assert(islegaldimsep(zvar->dimension_separator)); /* we are hosed */
}
/* fill_value; must precede calls to adjust cache */
{
if((stat = NCJdictget(jvar,"fill_value",&jvalue))) goto done;
if(jvalue == NULL || NCJsort(jvalue) == NCJ_NULL)
var->no_fill = 1;
else {
size_t fvlen;
typeid = var->type_info->hdr.id;
var->no_fill = 0;
if((stat = computeattrdata(&typeid, jvalue, NULL, &fvlen, &var->fill_value)))
goto done;
assert(typeid == var->type_info->hdr.id);
/* Note that we do not create the _FillValue
attribute here to avoid having to read all
the attributes and thus foiling lazy read.*/
}
}
/* chunks */
{
int rank;
@ -1478,25 +1531,9 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
/* Create the cache */
if((stat = NCZ_create_chunk_cache(var,var->type_info->size*zvar->chunkproduct,zvar->dimension_separator,&zvar->cache)))
goto done;
if((stat = NCZ_adjust_var_cache(var))) goto done;
}
}
/* fill_value */
{
if((stat = NCJdictget(jvar,"fill_value",&jvalue))) goto done;
if(jvalue == NULL || NCJsort(jvalue) == NCJ_NULL)
var->no_fill = 1;
else {
size_t fvlen;
typeid = var->type_info->hdr.id;
var->no_fill = 0;
if((stat = computeattrdata(&typeid, jvalue, NULL, &fvlen, &var->fill_value)))
goto done;
assert(typeid == var->type_info->hdr.id);
/* Note that we do not create the _FillValue
attribute here to avoid having to read all
the attributes and thus foiling lazy read.*/
}
}
/* Capture row vs column major; currently, column major not used*/
{
if((stat = NCJdictget(jvar,"order",&jvalue))) goto done;
@ -1504,15 +1541,39 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
((NCZ_VAR_INFO_T*)var->format_var_info)->order = 1;
else ((NCZ_VAR_INFO_T*)var->format_var_info)->order = 0;
}
/* compressor ignored */
{
if((stat = NCJdictget(jvar,"compressor",&jvalue))) goto done;
/* ignore */
}
/* filters ignored */
/* filters key */
/* From V2 Spec: A list of JSON objects providing codec configurations,
or null if no filters are to be applied. Each codec configuration
object MUST contain a "id" key identifying the codec to be used. */
/* Do filters key before compressor key so final filter chain is in correct order */
{
int k;
if(var->filters == NULL) var->filters = (void*)nclistnew();
if((stat = NCZ_filter_initialize())) goto done;
if((stat = NCJdictget(jvar,"filters",&jvalue))) goto done;
/* ignore */
if(jvalue != NULL && NCJsort(jvalue) != NCJ_NULL) {
if(NCJsort(jvalue) != NCJ_ARRAY) {stat = NC_EFILTER; goto done;}
for(k=0;;k++) {
jfilter = NULL;
jfilter = NCJith(jvalue,k);
if(jfilter == NULL) break; /* done */
if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;}
if((stat = NCZ_filter_build(file,var,jfilter))) goto done;
}
}
}
/* compressor key */
/* From V2 Spec: A JSON object identifying the primary compression codec and providing
configuration parameters, or ``null`` if no compressor is to be used. */
{
if(var->filters == NULL) var->filters = (void*)nclistnew();
if((stat = NCZ_filter_initialize())) goto done;
if((stat = NCJdictget(jvar,"compressor",&jfilter))) goto done;
if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) {
if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;}
if((stat = NCZ_filter_build(file,var,jfilter))) goto done;
}
}
if(!purezarr) {
@ -1572,11 +1633,15 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
for(j=0;j<rank;j++)
var->dimids[j] = var->dim[j]->hdr.id;
/* At this point, we can finalize the filters */
if((stat = NCZ_filter_setup(var))) goto done;
/* Clean up from last cycle */
nclistfreeall(dimnames); dimnames = nclistnew();
nullfree(varpath); varpath = NULL;
nullfree(shapes); shapes = NULL;
NCJreclaim(jvar); jvar = NULL;
if(formatv1) {NCJreclaim(jncvar); jncvar = NULL;}
NCJreclaim(jvar); jvar = NULL;
}
done:
@ -1882,7 +1947,7 @@ decodeints(NCjson* jshape, size64_t* shapes)
for(i=0;i<NCJlength(jshape);i++) {
long long v;
NCjson* jv = NCJith(jshape,i);
if((stat = NCZ_convert1(jv,NC_INT64,(char*)&v))) goto done;
if((stat = NCZ_convert1(jv,NC_INT64,(unsigned char*)&v))) goto done;
if(v < 0) {stat = THROW(NC_ENCZARR); goto done;}
shapes[i] = (size64_t)v;
}
@ -2040,7 +2105,7 @@ ncz_get_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
/* Is this a deflated variable with a chunksize greater than the
* current cache size? */
if ((retval = nc4_adjust_var_cache(var->container, var)))
if ((retval = nc4_adjust_var_cache(var)))
BAIL(retval);
if (var->coords_read && !var->dimscale)
@ -2134,7 +2199,7 @@ computedimrefs(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int purezarr, int xarra
for(i=0;i<ndims;i++) {
/* Compute the set of absolute paths to dimrefs */
char zdimname[4096];
snprintf(zdimname,sizeof(zdimname),"/.zdim_%llu",shapes[i]);
snprintf(zdimname,sizeof(zdimname),"/%s_%llu",ZDIMANON,shapes[i]);
nclistpush(dimnames,strdup(zdimname));
}
}

5
libnczarr/ztracedispatch.h
View File

@ -471,9 +471,8 @@ static int NCZTR_def_var_fletcher32(int ncid, int varid, int fletcher32)
static int NCZTR_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int *params)
{
int stat = NC_NOERR;
ZTRACE(0,"NCZ_def_var_filter(ncid,varid,id,nparams,params)");
stat = NCZ_def_var_filter(ncid,varid,id,nparams,params);
return ZUNTRACE(stat);
return (stat);
}
#endif
@ -562,7 +561,7 @@ static const NC_Dispatch NCZ_dispatcher_trace = {
NC_NOTNC4_def_var_fletcher32,
NCZTR_def_var_chunking,
NCZTR_def_var_endian,
NC_NOTNC4_def_var_filter,
NCZ_def_var_filter,
NCZTR_set_var_chunk_cache,
NCZTR_get_var_chunk_cache,
NCZTR_inq_var_filter_ids,

18
libnczarr/zutil.c
View File

@ -340,7 +340,7 @@ NCZ_readdict(NCZMAP* zmap, const char* key, NCjson** jsonp)
if((stat = NCZ_downloadjson(zmap,key,&json)))
goto done;
if(json->sort != NCJ_DICT) {stat = NC_ENCZARR; goto done;}
if(NCJsort(json) != NCJ_DICT) {stat = NC_ENCZARR; goto done;}
if(jsonp) {*jsonp = json; json = NULL;}
done:
NCJreclaim(json);
@ -364,7 +364,7 @@ NCZ_readarray(NCZMAP* zmap, const char* key, NCjson** jsonp)
if((stat = NCZ_downloadjson(zmap,key,&json)))
goto done;
if(json->sort != NCJ_ARRAY) {stat = NC_ENCZARR; goto done;}
if(NCJsort(json) != NCJ_ARRAY) {stat = NC_ENCZARR; goto done;}
if(jsonp) {*jsonp = json; json = NULL;}
done:
NCJreclaim(json);
@ -414,9 +414,9 @@ ncz_default_fill_value(nc_type nctype, const char** dfaltp)
/**
@internal Given an nc_type, produce the corresponding
fill value sort
fill value JSON type
@param nctype - [in] nc_type
@param sortp - [out] pointer to hold pointer to the sort
@param sortp - [out] pointer to hold pointer to the JSON type
@return NC_NOERR
@author Dennis Heimbigner
*/
@ -732,12 +732,17 @@ NCZ_freestringvec(size_t len, char** vec)
/* create a fill chunk */
int
NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fillchunkp)
NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, const void* fill, void** fillchunkp)
{
int i;
void* fillchunk = NULL;
if((fillchunk = malloc(chunksize))==NULL)
return NC_ENOMEM;
if(fill == NULL) {
/* use zeros */
memset(fillchunk,0,chunksize);
goto done;
}
switch (typesize) {
case 1: {
unsigned char c = *((unsigned char*)fill);
@ -764,6 +769,7 @@ NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fi
memcpy(p,fill,typesize);
} break;
}
done:
if(fillchunkp) {*fillchunkp = fillchunk; fillchunk = NULL;}
nullfree(fillchunk);
return NC_NOERR;
@ -901,7 +907,7 @@ NCZ_ischunkname(const char* name,char dimsep)
}
char*
NCZ_chunkpath(struct ChunkKey key,char dimsep)
NCZ_chunkpath(struct ChunkKey key)
{
size_t plen = nulllen(key.varkey)+1+nulllen(key.chunkkey);
char* path = (char*)malloc(plen+1);

28
libnczarr/zvar.c
View File

@ -390,6 +390,7 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims,
var->atts_read = NC_TRUE;
/* Set the filter list */
assert(var->filters == NULL);
var->filters = (void*)nclistnew();
/* Point to the type, and increment its ref. count */
@ -461,10 +462,6 @@ var->type_info->rc++;
if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,zvar->dimension_separator,&zvar->cache)))
BAIL(retval);
/* Is this a variable with a chunksize greater than the current cache size? */
if ((retval = NCZ_adjust_var_cache(grp, var)))
BAIL(retval);
/* Return the varid. */
if (varidp)
*varidp = var->hdr.id;
@ -599,6 +596,9 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
{
if(*shuffle) var->shuffle = *shuffle;
var->storage = NC_CHUNKED;
if(var->shuffle) {
if((retval = NCZ_def_var_filter(ncid,varid,2,0,NULL))) goto done;
}
}
/* Fletcher32 checksum error protection? */
@ -606,6 +606,9 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
{
if(*fletcher32) var->fletcher32 = *fletcher32;
var->storage = NC_CHUNKED;
if(var->fletcher32) {
if((retval = NCZ_def_var_filter(ncid,varid,3,0,NULL))) goto done;
}
}
/* Handle storage settings. */
@ -675,9 +678,6 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
for (d = 0; d < var->ndims; d++)
zvar->chunkproduct *= var->chunksizes[d];
zvar->chunksize = zvar->chunkproduct * var->type_info->size;
/* Adjust the cache. */
if ((retval = NCZ_adjust_var_cache(grp, var)))
goto done;
}
#ifdef LOGGING
@ -2002,10 +2002,22 @@ NCZ_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
goto done;
assert(grp && h5);
/* Short-circuit the filter-related inquiries */
if(shufflep) {
*shufflep = 0;
if((retval = NCZ_inq_var_filter_info(ncid,varid,2,NULL,NULL))==NC_NOERR)
*shufflep = 1;
}
if(fletcher32p) {
*fletcher32p = 0;
if((retval = NCZ_inq_var_filter_info(ncid,varid,3,NULL,NULL))==NC_NOERR)
*fletcher32p = 1;
}
/* Now that lazy atts have been read, use the libsrc4 function to
* get the answers. */
retval = NC4_inq_var_all(ncid, varid, name, xtypep, ndimsp, dimidsp, nattsp,
shufflep, unused4, unused5, fletcher32p,
NULL, unused4, unused5, NULL,
storagep, chunksizesp, no_fill, fill_valuep,
endiannessp, unused1, unused2, unused3);
done:

12
libnczarr/zwalk.c
View File

@ -25,7 +25,6 @@ static unsigned int wdebug = 0;
static int NCZ_walk(NCZProjection** projv, NCZOdometer* chunkodom, NCZOdometer* slpodom, NCZOdometer* memodom, const struct Common* common, void* chunkdata);
static int rangecount(NCZChunkRange range);
static int readfromcache(void* source, size64_t* chunkindices, void** chunkdata);
static int NCZ_fillchunk(void* chunkdata, struct Common* common);
static int iswholechunk(struct Common* common,NCZSlice*);
static int wholechunk_indices(struct Common* common, NCZSlice* slices, size64_t* chunkindices);
@ -123,8 +122,6 @@ NCZ_transferslice(NC_VAR_INFO_T* var, int reading,
common.typesize = typesize;
common.cache = zvar->cache;
if((stat = ncz_get_fill_value(common.file, common.var, &common.fillvalue))) goto done;
/* We need to talk scalar into account */
common.rank = var->ndims + zvar->scalar;
common.scalar = zvar->scalar;
@ -160,6 +157,9 @@ NCZ_transferslice(NC_VAR_INFO_T* var, int reading,
common.reader.source = ((NCZ_VAR_INFO_T*)(var->format_var_info))->cache;
common.reader.read = readfromcache;
/* verify */
assert(var->no_fill || var->fill_value != NULL);
if(common.scalar) {
if((stat = NCZ_transferscalar(&common))) goto done;
}
@ -227,7 +227,6 @@ NCZ_transfer(struct Common* common, NCZSlice* slices)
/* Read the chunk */
switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) {
case NC_EEMPTY: /* cache created the chunk */
if((stat = NCZ_fillchunk(chunkdata,common))) goto done;
break;
case NC_NOERR: break;
default: goto done;
@ -299,7 +298,6 @@ NCZ_transfer(struct Common* common, NCZSlice* slices)
stat = common->reader.read(common->reader.source, chunkindices, &chunkdata);
switch (stat) {
case NC_EEMPTY: /* cache created the chunk */
if((stat = NCZ_fillchunk(chunkdata,common))) goto done;
break;
case NC_NOERR: break;
default: goto done;
@ -503,6 +501,7 @@ unsigned srcidx = srcoff / sizeof(unsigned); (void)srcidx;
}
#endif
#if 0
/* This function may not be necessary if code in zvar does it instead */
static int
NCZ_fillchunk(void* chunkdata, struct Common* common)
@ -523,6 +522,7 @@ NCZ_fillchunk(void* chunkdata, struct Common* common)
done:
return stat;
}
#endif
/* Break out this piece so we can use it for unit testing */
int
@ -681,7 +681,6 @@ NCZ_clearcommon(struct Common* common)
{
NCZ_clearsliceprojections(common->rank,common->allprojections);
nullfree(common->allprojections);
nullfree(common->fillvalue);
}
/* Does the User want all of one and only chunk? */
@ -730,7 +729,6 @@ NCZ_transferscalar(struct Common* common)
chunkindices[0] = 0;
switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) {
case NC_EEMPTY: /* cache created the chunk */
if((stat = NCZ_fillchunk(chunkdata,common))) goto done;
break;
case NC_NOERR: break;
default: goto done;

273
libnczarr/zxcache.c
View File

@ -13,20 +13,20 @@
#include "zincludes.h"
#include "zcache.h"
#include "ncxcache.h"
#include "zfilter.h"
#undef DEBUG
#undef FILLONREAD
#undef FLUSH
#define LEAFLEN 32
/* Forward */
static int get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry);
static int put_chunk(NCZChunkCache* cache, const NCZCacheEntry*);
static int put_chunk(NCZChunkCache* cache, NCZCacheEntry*);
static int makeroom(NCZChunkCache* cache);
static int flushcache(NCZChunkCache* cache);
static int constraincache(NCZChunkCache* cache);
/**************************************************/
/* Dispatch table per-var cache functions */
@ -57,20 +57,20 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl
NC_FILE_INFO_T *h5;
NC_VAR_INFO_T *var;
NCZ_VAR_INFO_T *zvar;
int retval;
int retval = NC_NOERR;
/* Check input for validity. */
if (preemption < 0 || preemption > 1)
return NC_EINVAL;
{retval = NC_EINVAL; goto done;}
/* Find info for this file and group, and set pointer to each. */
if ((retval = nc4_find_nc_grp_h5(ncid, NULL, &grp, &h5)))
return retval;
goto done;
assert(grp && h5);
/* Find the var. */
if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
return NC_ENOTVAR;
{retval = NC_ENOTVAR; goto done;}
assert(var && var->hdr.id == varid);
zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
@ -81,12 +81,10 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl
var->chunk_cache_nelems = nelems;
var->chunk_cache_preemption = preemption;
#ifdef LOOK
/* Reopen the dataset to bring new settings into effect. */
if ((retval = nc4_reopen_dataset(grp, var)))
return retval;
#endif
return NC_NOERR;
/* Fix up cache */
if((retval = NCZ_adjust_var_cache(var))) goto done;
done:
return retval;
}
/**
@ -103,31 +101,36 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl
* @author Ed Hartnett
*/
int
NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
NCZ_adjust_var_cache(NC_VAR_INFO_T *var)
{
size64_t cachesize,nelems;
int stat = NC_NOERR;
NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
/* empty the cache */
zvar->cache->maxentries = 0;
makeroom(zvar->cache);
/* completely empty the cache */
flushcache(zvar->cache);
#ifdef DEBUG
fprintf(stderr,"xxx: adjusting cache for: %s\n",var->hdr.name);
#endif
/* Reset the parameters */
/* The total cache size is considered fixed here, so modify nelems */
cachesize = var->chunk_cache_size;
nelems = floordiv(cachesize , zvar->chunksize);
if(nelems == 0) nelems = 1;
zvar->cache->maxentries = nelems;
zvar->cache->maxsize = var->chunk_cache_size;
zvar->cache->maxentries = var->chunk_cache_nelems;
#ifdef DEBUG
fprintf(stderr,"%s.cache.adjust: size=%ld nelems=%ld\n",
var->hdr.name,(unsigned long)cachesize,(unsigned long)zvar->cache->maxentries);
var->hdr.name,(unsigned long)zvar->cache->maxsize,(unsigned long)zvar->cache->maxentries);
#endif
/* One more thing, adjust the chunksize */
zvar->cache->chunksize = zvar->chunksize;
/* and also free the fillchunk */
/* and also rebuild the fillchunk */
nullfree(zvar->cache->fillchunk);
zvar->cache->fillchunk = NULL;
return NC_NOERR;
if(var->no_fill)
stat = NCZ_create_fill_chunk(zvar->cache->chunksize,var->type_info->size,NULL,&zvar->cache->fillchunk);
else {
assert(var->fill_value != NULL);
stat = NCZ_create_fill_chunk(zvar->cache->chunksize,var->type_info->size,var->fill_value,&zvar->cache->fillchunk);
}
return stat;
}
/**************************************************/
@ -149,7 +152,6 @@ NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, char dimsep, NCZC
int stat = NC_NOERR;
NCZChunkCache* cache = NULL;
void* fill = NULL;
size_t nelems, cachesize;
NCZ_VAR_INFO_T* zvar = NULL;
if(chunksize == 0) return NC_EINVAL;
@ -164,21 +166,15 @@ NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, char dimsep, NCZC
cache->fillchunk = NULL;
cache->chunksize = chunksize;
cache->dimension_separator = dimsep;
zvar->cache = cache;
/* Figure out the actual cache size */
cachesize = var->chunk_cache_size;
nelems = (cachesize / chunksize);
if(nelems == 0) nelems = 1;
/* Make consistent */
cachesize = nelems * chunksize;
cache->maxentries = nelems;
#ifdef FLUSH
cache->maxentries = 1;
#endif
#ifdef DEBUG
fprintf(stderr,"%s.cache: nelems=%ld size=%ld\n",
var->hdr.name,(unsigned long)cache->maxentries,(unsigned long)(cache->maxentries*cache->chunksize));
var->hdr.name,(unsigned long)cache->maxentries,(unsigned long)cache->maxsize);
#endif
if((stat = ncxcachenew(LEAFLEN,&cache->xcache))) goto done;
if((cache->mru = nclistnew()) == NULL)
@ -191,6 +187,17 @@ done:
return THROW(stat);
}
static void
free_cache_entry(NCZCacheEntry* entry)
{
if(entry) {
nullfree(entry->data);
nullfree(entry->key.varkey);
nullfree(entry->key.chunkkey);
nullfree(entry);
}
}
void
NCZ_free_chunk_cache(NCZChunkCache* cache)
{
@ -204,7 +211,7 @@ NCZ_free_chunk_cache(NCZChunkCache* cache)
NCZCacheEntry* entry = nclistremove(cache->mru,0);
(void)ncxcacheremove(cache->xcache,entry->hashkey,&ptr);
assert(ptr == entry);
nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey); nullfree(entry);
free_cache_entry(entry);
}
#ifdef DEBUG
fprintf(stderr,"|cache.free|=%ld\n",nclistlength(cache->mru));
@ -237,11 +244,10 @@ NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap
{
int stat = NC_NOERR;
int rank = cache->ndims;
NC_FILE_INFO_T* file = cache->var->container->nc4_info;
NCZCacheEntry* entry = NULL;
ncexhashkey_t hkey = 0;
int created = 0;
/* the hash key */
hkey = ncxcachekey(indices,sizeof(size64_t)*cache->ndims);
/* See if already in cache */
@ -258,38 +264,22 @@ NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap
}
if(entry == NULL) { /*!found*/
/* Make room in the cache */
if((stat=makeroom(cache))) goto done;
/* Create a new entry */
if((entry = calloc(1,sizeof(NCZCacheEntry)))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(entry->indices,indices,rank*sizeof(size64_t));
/* Create the local copy space */
if((entry->data = calloc(1,cache->chunksize)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* Create the key for this cache */
if((stat = NCZ_buildchunkpath(cache,indices,&entry->key))) goto done;
entry->hashkey = hkey;
/* Try to read the object in toto */
stat=get_chunk(cache,entry);
switch (stat) {
case NC_NOERR: break;
case NC_EEMPTY:
/* If the file is read-only, then fake the chunk */
entry->modified = (file->no_write?0:1);
#ifdef FILLONREAD
/* apply fill value */
memcpy(entry->data,cache->fillchunk,cache->chunksize);
#else
memset(entry->data,0,cache->chunksize);
#endif
created = 1;
break;
default: goto done;
}
/* Try to read the object from "disk" */
if((stat=get_chunk(cache,entry))) goto done;
nclistpush(cache->mru,entry);
cache->used += entry->size;
if((stat = ncxcacheinsert(cache->xcache,entry->hashkey,entry))) goto done;
/* Ensure cache constraints not violated */
if((stat=makeroom(cache))) goto done;
}
#ifdef DEBUG
fprintf(stderr,"|cache.read.lru|=%ld\n",nclistlength(cache->mru));
#endif
@ -298,33 +288,34 @@ fprintf(stderr,"|cache.read.lru|=%ld\n",nclistlength(cache->mru));
done:
if(created && stat == NC_NOERR) stat = NC_EEMPTY; /* tell upper layers */
if(entry) {nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey);}
nullfree(entry);
if(entry) free_cache_entry(entry);
return THROW(stat);
}
#if 0
int
NCZ_write_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap)
NCZ_write_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void* content)
{
int stat = NC_NOERR;
int rank = cache->ndims;
NCZCacheEntry* entry = NULL;
ncexhashkey_t hkey;
/* and the hash key */
/* create the hash key */
hkey = ncxcachekey(indices,sizeof(size64_t)*cache->ndims);
if(entry == NULL) { /*!found*/
if((stat=makeroom(cache))) goto done;
/* Create a new entry */
if((entry = calloc(1,sizeof(NCZCacheEntry)))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(entry->indices,indices,rank*sizeof(size64_t));
/* Create the local copy space */
if((entry->data = calloc(1,cache->chunksize)) == NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = NCZ_buildchunkpath(cache,indices,&entry->key))) goto done;
entry->hashkey = hkey;
/* Create the local copy space */
entry->size = cache->chunksize;
if((entry->data = calloc(1,cache->chunksize)) == NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(entry->data,content,cache->chunksize);
}
entry->modified = 1;
nclistpush(cache->mru,entry); /* MRU order */
@ -333,18 +324,51 @@ fprintf(stderr,"|cache.write|=%ld\n",nclistlength(cache->mru));
#endif
entry = NULL;
/* Ensure cache constraints not violated */
if((stat=makeroom(cache))) goto done;
done:
if(entry) {nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey);}
nullfree(entry);
if(entry) free_cache_entry(entry);
return THROW(stat);
}
#endif
/* Constrain cache, but allow at least one entry */
static int
makeroom(NCZChunkCache* cache)
{
int stat = NC_NOERR;
/* Sanity check; make sure at least one entry is always allowed */
if(nclistlength(cache->mru) == 1)
goto done;
stat = constraincache(cache);
done:
return stat;
}
/* Completely flush cache */
static int
flushcache(NCZChunkCache* cache)
{
cache->maxentries = 0;
return constraincache(cache);
}
/* Remove entries to ensure cache is not
violating any of its constraints.
On entry, constraints might be violated.
*/
static int
constraincache(NCZChunkCache* cache)
{
int stat = NC_NOERR;
/* Flush from LRU end if we are at capacity */
while(nclistlength(cache->mru) > cache->maxentries) {
while(nclistlength(cache->mru) > cache->maxentries || cache->used > cache->maxsize) {
int i;
void* ptr;
NCZCacheEntry* e = ncxcachelast(cache->xcache); /* last entry is the least recently used */
@ -359,6 +383,9 @@ makeroom(NCZChunkCache* cache)
nclistremove(cache->mru,i);
if(e->modified) /* flush to file */
stat=put_chunk(cache,e);
/* Decrement space used */
assert(cache->used >= e->size);
cache->used -= e->size;
/* reclaim */
nullfree(e->data); nullfree(e->key.varkey); nullfree(e->key.chunkkey); nullfree(e);
}
@ -478,23 +505,43 @@ NCZ_buildchunkkey(size_t R, const size64_t* chunkindices, char dimsep, char** ke
* @author Dennis Heimbigner
*/
static int
put_chunk(NCZChunkCache* cache, const NCZCacheEntry* entry)
put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
{
int stat = NC_NOERR;
NC_FILE_INFO_T* file = NULL;
NCZ_FILE_INFO_T* zfile = NULL;
NCZMAP* map = NULL;
char* path = NULL;
ZTRACE(5,"cache.var=%s entry.key=%s",cache->var->hdr.name,entry->key);
LOG((3, "%s: var: %p", __func__, cache->var));
zfile = ((cache->var->container)->nc4_info)->format_file_info;
file = (cache->var->container)->nc4_info;
zfile = file->format_file_info;
map = zfile->map;
{
char* path = NCZ_chunkpath(entry->key,cache->dimension_separator);
stat = nczmap_write(map,path,0,cache->chunksize,entry->data);
nullfree(path);
/* Make sure the entry is in filtered state */
if(!entry->isfiltered) {
NC_VAR_INFO_T* var = cache->var;
void* filtered = NULL; /* pointer to the filtered data */
size_t flen; /* length of filtered data */
/* Get the filter chain to apply */
NClist* filterchain = (NClist*)var->filters;
if(nclistlength(filterchain) > 0) {
/* Apply the filter chain to get the filtered data */
if((stat = NCZ_applyfilterchain(file,var,filterchain,entry->size,entry->data,&flen,&filtered,ENCODING))) goto done;
/* Fix up the cache entry */
/* Note that if filtered is different from entry->data, then entry->data will have been freed */
entry->data = filtered;
entry->size = flen;
entry->isfiltered = 1;
}
}
path = NCZ_chunkpath(entry->key);
stat = nczmap_write(map,path,0,entry->size,entry->data);
nullfree(path); path = NULL;
switch(stat) {
case NC_NOERR:
break;
@ -502,6 +549,7 @@ put_chunk(NCZChunkCache* cache, const NCZCacheEntry* entry)
default: goto done;
}
done:
nullfree(path);
return ZUNTRACE(stat);
}
@ -522,6 +570,9 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
NCZMAP* map = NULL;
NC_FILE_INFO_T* file = NULL;
NCZ_FILE_INFO_T* zfile = NULL;
size64_t size;
int empty = 0;
char* path = NULL;
ZTRACE(5,"cache.var=%s entry.key=%s sep=%d",cache->var->hdr.name,entry->key,cache->dimension_separator);
@ -530,14 +581,67 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
file = (cache->var->container)->nc4_info;
zfile = file->format_file_info;
map = zfile->map;
assert(map && entry->data);
assert(map);
{
char* path = NCZ_chunkpath(entry->key,cache->dimension_separator);
stat = nczmap_read(map,path,0,cache->chunksize,(char*)entry->data);
nullfree(path);
/* get size of the "raw" data on "disk" */
path = NCZ_chunkpath(entry->key);
stat = nczmap_len(map,path,&size);
nullfree(path); path = NULL;
switch(stat) {
case NC_NOERR: break;
case NC_EEMPTY: empty = 1; stat = NC_NOERR; break;
default: goto done;
}
if(!empty) {
/* Make sure we have a place to read it */
entry->size = size;
entry->isfiltered = FILTERED(cache); /* Is the data being read filtered? */
if((entry->data = (void*)malloc(entry->size)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* Read the raw data */
path = NCZ_chunkpath(entry->key);
stat = nczmap_read(map,path,0,entry->size,(char*)entry->data);
nullfree(path); path = NULL;
switch (stat) {
case NC_NOERR: break;
case NC_EEMPTY: empty = 1; stat = NC_NOERR;break;
default: goto done;
}
}
if(empty) {
/* fake the chunk */
entry->modified = (file->no_write?0:1);
entry->size = cache->chunksize;
if((entry->data = (void*)malloc(entry->size)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* apply fill value */
assert(cache->fillchunk);
memcpy(entry->data,cache->fillchunk,entry->size);
entry->isfiltered = 0;
stat = NC_NOERR;
}
/* Make sure the entry is in unfiltered state */
if(entry->isfiltered) {
NC_VAR_INFO_T* var = cache->var;
void* unfiltered = NULL; /* pointer to the unfiltered data */
void* filtered = NULL; /* pointer to the filtered data */
size_t unflen; /* length of unfiltered data */
/* Get the filter chain to apply */
NClist* filterchain = (NClist*)var->filters;
if(nclistlength(filterchain) == 0) {stat = NC_EFILTER; goto done;}
/* Apply the filter chain to get the unfiltered data */
filtered = entry->data;
entry->data = NULL;
if((stat = NCZ_applyfilterchain(file,var,filterchain,entry->size,filtered,&unflen,&unfiltered,!ENCODING))) goto done;
/* Fix up the cache entry */
entry->data = unfiltered;
entry->size = unflen;
entry->isfiltered = 0;
}
done:
nullfree(path);
return ZUNTRACE(stat);
}
@ -561,4 +665,3 @@ done:
nullfree(varkey);
return THROW(stat);
}

10
libsrc4/nc4internal.c
View File

@ -40,6 +40,7 @@ static const NC_reservedatt NC_reserved[] = {
{NC_ATT_NAME, READONLYFLAG|HIDDENATTRFLAG}, /*NAME*/
{NC_ATT_REFERENCE_LIST, READONLYFLAG|HIDDENATTRFLAG}, /*REFERENCE_LIST*/
{NC_XARRAY_DIMS, READONLYFLAG|HIDDENATTRFLAG}, /*_ARRAY_DIMENSIONS*/
{NC_ATT_CODECS, VARFLAG|READONLYFLAG|NAMEONLYFLAG|HIDDENATTRFLAG}, /*_Codecs*/
{NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/
{ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/
{NCPROPS, READONLYFLAG|NAMEONLYFLAG|MATERIALIZEDFLAG}, /*_NCProperties*/
@ -352,6 +353,7 @@ nc4_find_nc_grp_h5(int ncid, NC **nc, NC_GRP_INFO_T **grp, NC_FILE_INFO_T **h5)
NC_FILE_INFO_T *my_h5 = NULL;
NC *my_nc;
int retval;
size_t index;
/* Look up file metadata. */
if ((retval = NC_check_id(ncid, &my_nc)))
@ -360,7 +362,8 @@ nc4_find_nc_grp_h5(int ncid, NC **nc, NC_GRP_INFO_T **grp, NC_FILE_INFO_T **h5)
assert(my_h5 && my_h5->root_grp);
/* If we can't find it, the grp id part of ncid is bad. */
if (!(my_grp = nclistget(my_h5->allgroups, (ncid & GRP_ID_MASK))))
index = (ncid & GRP_ID_MASK);
if (!(my_grp = nclistget(my_h5->allgroups,index)))
return NC_EBADID;
/* Return pointers to caller, if desired. */
@ -1336,7 +1339,7 @@ var_free(NC_VAR_INFO_T *var)
/* Delete any fill value allocation. */
if (var->fill_value)
free(var->fill_value);
{free(var->fill_value); var->fill_value = NULL;}
/* Release type information */
if (var->type_info)
@ -1837,12 +1840,13 @@ NC_findreserved(const char* name)
int n = NRESERVED;
int L = 0;
int R = (n - 1);
for(;;) {
if(L > R) break;
int m = (L + R) / 2;
const NC_reservedatt* p = &NC_reserved[m];
int cmp = strcmp(p->name,name);
if(cmp == 0) return p;
if(cmp == 0) return p;
if(cmp < 0)
L = (m + 1);
else /*cmp > 0*/

7
nc_test4/CMakeLists.txt
View File

@ -8,6 +8,8 @@
# This is the cmake build file for the nc_test4 directory.
# Ward Fisher, Ed Hartnett
SET(srcdir ${CMAKE_CURRENT_SOURCE_DIR})
# Some extra tests
SET(NC4_TESTS tst_dims tst_dims2 tst_dims3 tst_files tst_files4
tst_vars tst_varms tst_unlim_vars tst_converts tst_converts2
@ -40,6 +42,9 @@ IF(ENABLE_FILTER_TESTING)
build_bin_test(test_filter_order)
build_bin_test(test_filter_repeat)
ADD_SH_TEST(nc_test4 tst_filter)
IF(ENABLE_BLOSC)
ADD_SH_TEST(nc_test4 tst_specific_filters)
ENDIF()
IF(ENABLE_CLIENTSIDE_FILTERS)
add_bin_test(nc_test4 test_filter_reg)
ENDIF(ENABLE_CLIENTSIDE_FILTERS)
@ -80,6 +85,7 @@ IF(USE_SZIP)
ENDIF()
ENDIF()
# Copy some test files from current source dir to out-of-tree build dir.
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/ref_bzip2.c ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl)
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
@ -89,6 +95,7 @@ ENDIF()
FOREACH(CTEST ${NC4_TESTS})
add_bin_test(nc_test4 ${CTEST})
SET_TESTS_PROPERTIES(nc_test4_${CTEST} PROPERTIES ENVIRONMENT srcdir=${CMAKE_CURRENT_SOURCE_DIR})
ENDFOREACH()
IF(TEST_PARALLEL4)

11
nc_test4/Makefile.am
View File

@ -78,6 +78,9 @@ extradir =
check_PROGRAMS += test_filter test_filter_misc test_filter_order test_filter_repeat
check_PROGRAMS += tst_multifilter
TESTS += tst_filter.sh
if ENABLE_BLOSC
TESTS += tst_specific_filters.sh
endif
endif
endif # BUILD_UTILITIES
@ -100,10 +103,11 @@ ref_unfiltered.cdl ref_bzip2.c findplugin.in ref_unfilteredvv.cdl \
ref_filteredvv.cdl ref_multi.cdl \
ref_ncgenF.cdl ref_nccopyF.cdl \
ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
ref_filter_order_create.txt ref_filter_order_read.txt
ref_filter_order_create.txt ref_filter_order_read.txt \
ref_any.cdl tst_specific_filters.sh
CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc \
tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt \
tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl tst_elena_*.cdl \
tst_simple*.cdl tst_chunks.cdl pr_A1.* tauu_A1.* usi_01.* thetau_01.* \
tst_*.h5 tst_grp_rename.cdl tst_grp_rename.dmp ref_grp_rename.cdl \
@ -111,7 +115,8 @@ foo1.nc tst_*.h4 test.nc testszip.nc test.h5 szip_dump.cdl \
perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \
floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \
testfilter_reg.nc filterrepeat.txt tmp_fillonly.nc \
testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1
testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1 \
tmp_*.txt tmp_*.nc
DISTCLEANFILES = findplugin.sh run_par_test.sh

34
nc_test4/findplugin.in
View File

@ -3,12 +3,14 @@
# Define a function that attempts to locate a
# plugin with a given canonical name.
# Assumptions:
# 1. plugins is a top-level directory
# 1. plugins is a top-level directory, possibly in "build"
# Inputs:
# $1 is the canonical name
# $2 is 1 if we are running under cmake
# $3 is 1 if we are running using Visual Studio, blank otherwise
# $4 is the build type; only used if $3 is 1
# Optional Input:
# HDF5_PLUGIN_PATH environment variable
# Outputs:
# return code is 0 is success, 1 if failed
# Variable HDF5_PLUGIN_LIB is set to the library file name
@ -30,8 +32,6 @@ FP_ISMSVC=@ISMSVC@
FP_ISCYGWIN=@ISCYGWIN@
FP_ISOSX=@ISOSX@
FP_PLUGINS="$TOPBUILDDIR/plugins"
FP_PLUGIN_LIB=
FP_PLUGIN_PATH=
@ -47,16 +47,30 @@ else # Presumably some form on *nix"
FP_PLUGIN_LIB="lib${FP_NAME}.so"
fi
# If HDF5_PLUGIN_PATH is defined, then it overrides everything else.
if test "x$HDF5_PLUGIN_PATH" != x ; then
HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB"
# HDF5_PLUGIN_PATH already set
HDF5_PLUGIN_PATH=`${NCPATHCVT} $HDF5_PLUGIN_PATH`
return 0;
fi
# Figure out the path to where the lib is stored
# This can probably be simplified
CURWD=`pwd`
cd ${TOPBUILDDIR}/plugins
FP_PLUGINS=`pwd`
cd ${CURWD}
# Case 1: Cmake with Visual Studio
# Do not know where to look for a dylib
# Case 1: Cmake with Visual Studio
if test "x$FP_ISCMAKE" != x -a "x${FP_ISMSVC}" != x ; then
# Case 1a: ignore the build type directory
if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
FP_PLUGIN_PATH="${FP_PLUGINS}"
fi
# Case 1a: ignore the build type directory
if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
FP_PLUGIN_PATH="${FP_PLUGINS}"
fi
else # Case 2: automake
# Case 2a: look in .libs
if test -f "${FP_PLUGINS}/.libs/${FP_PLUGIN_LIB}" ; then
@ -78,11 +92,7 @@ if ! test -f "$FP_PLUGIN_PATH/$FP_PLUGIN_LIB" ; then
return 1
fi
# If we are operating using both Visual Studio and Cygwin,
# then we need to convert the FP_PLUGIN_PATH to windows format
if test "x$FP_ISMSVC" != x -a "x$FP_ISCYGWIN" != x ; then
FP_PLUGIN_PATH=`cygpath -wl $FP_PLUGIN_PATH`
fi
FP_PLUGIN_PATH=`${NCPATHCVT} $FP_PLUGIN_PATH`
# Set the final output variables
HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB"

53
nc_test4/ref_any.cdl Normal file
View File

@ -0,0 +1,53 @@
netcdf ref_any {
dimensions:
dim0 = 4 ;
dim1 = 4 ;
dim2 = 4 ;
variables:
int ivar(dim0, dim1, dim2) ;
ivar:_Storage = @chunked@ ;
ivar:_ChunkSizes = 4, 4, 4 ;
ivar:_Filter = @IH5@ ;
float fvar(dim0, dim1, dim2) ;
fvar:_Storage = @chunked@ ;
fvar:_ChunkSizes = 4, 4, 4 ;
fvar:_Filter = @FH5@ ;
data:
ivar =
0, 1, 2, 3,
4, 5, 6, 7,
8, 9, 10, 11,
12, 13, 14, 15,
16, 17, 18, 19,
20, 21, 22, 23,
24, 25, 26, 27,
28, 29, 30, 31,
32, 33, 34, 35,
36, 37, 38, 39,
40, 41, 42, 43,
44, 45, 46, 47,
48, 49, 50, 51,
52, 53, 54, 55,
56, 57, 58, 59,
60, 61, 62, 63 ;
fvar =
0.5, 1.5, 2.5, 3.5,
4.5, 5.5, 6.5, 7.5,
8.5, 9.5, 10.5, 11.5,
12.5, 13.5, 14.5, 15.5,
16.5, 17.5, 18.5, 19.5,
20.5, 21.5, 22.5, 23.5,
24.5, 25.5, 26.5, 27.5,
28.5, 29.5, 30.5, 31.5,
32.5, 33.5, 34.5, 35.5,
36.5, 37.5, 38.5, 39.5,
40.5, 41.5, 42.5, 43.5,
44.5, 45.5, 46.5, 47.5,
48.5, 49.5, 50.5, 51.5,
52.5, 53.5, 54.5, 55.5,
56.5, 57.5, 58.5, 59.5,
60.5, 61.5, 62.5, 63.5 ;
}

2
nc_test4/ref_filtered.cdl
View File

@ -13,7 +13,7 @@ group: g {
float var(dim0, dim1, dim2, dim3) ;
var:_Storage = "chunked" ;
var:_ChunkSizes = 4, 4, 4, 4 ;
var:_Filter = "307,9,4";
var:_Filter = "307,9";
var:_NoFill = "true" ;
// group attributes:

4
nc_test4/ref_filteredvv.cdl
View File

@ -6,7 +6,7 @@ variables:
float var1(dim0, dim1) ;
var1:_Storage = "chunked" ;
var1:_ChunkSizes = 2, 2 ;
var1:_Filter = "307,9,4" ;
var1:_Filter = "307,9" ;
var1:_NoFill = "true" ;
// global attributes:
@ -24,7 +24,7 @@ group: g {
float var2(dim0, dim1) ;
var2:_Storage = "chunked" ;
var2:_ChunkSizes = 2, 2 ;
var2:_Filter = "307,9,4" ;
var2:_Filter = "307,9" ;
var2:_NoFill = "true" ;
// group attributes:

8
nc_test4/ref_filterx_hdf5.txt Normal file
View File

@ -0,0 +1,8 @@
test1: def filter repeat .
test1: compression.
direction=compress id=40000 cd_nelmts=2 cd_values= 0 18
test1: decompression.
filter(40000): params=[0,18]
direction=decompress id=40000 cd_nelmts=2 cd_values= 0 18
data comparison: |array|=256
no data errors

16
nc_test4/test_filter.c
View File

@ -51,7 +51,7 @@ data:
/* The compression level used in this example */
#define BZIP2_LEVEL 9
#define TESTFILE "bzip2.nc"
#define DFALT_TESTFILE "tmp_bzip2.nc"
/* Point at which we give up */
#define MAXERRS 8
@ -72,6 +72,8 @@ static size_t chunks[NDIMS];
static int nerrs = 0;
static const char* testfile = NULL;
static int ncid, varid;
static int dimids[NDIMS];
static float* array = NULL;
@ -175,7 +177,7 @@ test_bzip2(void)
memset(array,0,sizeof(float)*actualproduct);
/* Create a file */
CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid));
/* Do not use fill for this file */
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
@ -234,7 +236,7 @@ test_bzip2(void)
memset(array,0,sizeof(float)*actualproduct);
/* Open the file */
CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid));
CHECK(nc_open(testfile, NC_NOWRITE, &ncid));
/* Get the variable id */
CHECK(nc_inq_varid(ncid, "var", &varid));
@ -278,6 +280,13 @@ static void
init(int argc, char** argv)
{
int i;
/* get the testfile path */
if(argc > 1)
testfile = argv[1];
else
testfile = DFALT_TESTFILE;
/* Setup various variables */
actualproduct = 1;
chunkproduct = 1;
@ -298,7 +307,6 @@ init(int argc, char** argv)
int
main(int argc, char **argv)
{
H5Eprint1(stderr);
init(argc,argv);
if(test_bzip2() != NC_NOERR) ERRR;
exit(nerrs > 0?1:0);

15
nc_test4/test_filter_misc.c
View File

@ -32,9 +32,11 @@
static unsigned int baseline[NPARAMS];
static const char* testfile = NULL;
#define MAXDIMS 8
#define TESTFILE "testmisc.nc"
#define DFALT_TESTFILE "tmp_misc.nc"
#define spec "32768, -17b, 23ub, -25S, 27US, 77, 93U, 789f, 12345678.12345678d, -9223372036854775807L, 18446744073709551615UL"
@ -132,7 +134,7 @@ create(void)
int i;
/* Create a file with one big variable, but whose dimensions arte not a multiple of chunksize (to see what happens) */
CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
for(i=0;i<ndims;i++) {
char dimname[1024];
@ -169,7 +171,7 @@ openfile(void)
unsigned int* params = NULL;
/* Open the file and check it. */
CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid));
CHECK(nc_open(testfile, NC_NOWRITE, &ncid));
CHECK(nc_inq_varid(ncid, "var", &varid));
/* Check the compression algorithm */
@ -517,6 +519,13 @@ static void
init(int argc, char** argv)
{
int i;
/* get the testfile path */
if(argc > 1)
testfile = argv[1];
else
testfile = DFALT_TESTFILE;
/* Setup various variables */
totalproduct = 1;
actualproduct = 1;

30
nc_test4/test_filter_order.c
View File

@ -25,7 +25,7 @@
#define MAXDIMS 8
#define TESTFILE "testfilter_order.nc"
#define DFALT_TESTFILE "tmp_filter_order.nc"
#define NPARAMS 1
#define PARAMVAL 17
@ -43,6 +43,8 @@ static size_t chunkproduct = 1; /* x-product over actual chunks */
static int nerrs = 0;
static const char* testfile = NULL;
static int ncid, varid;
static int dimids[MAXDIMS];
static size_t odom[MAXDIMS];
@ -112,7 +114,7 @@ create(void)
int i;
/* Create a file with one big variable */
CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
for(i=0;i<ndims;i++) {
char dimname[1024];
@ -170,7 +172,7 @@ openfile(void)
int k;
/* Open the file and check it. */
CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid));
CHECK(nc_open(testfile, NC_NOWRITE, &ncid));
CHECK(nc_inq_varid(ncid, "var", &varid));
/* Check the compression algorithms */
@ -380,14 +382,30 @@ expectedvalue(void)
return offset;
}
static void
usage(void)
{
fprintf(stderr,"usage: test_filter_order read|create [path]\n");
exit(1);
}
static void
init(int argc, char** argv)
{
int i;
creating = 1; /* default is test1 */
if(argc > 1 && strcmp(argv[1],"read")==0)
creating = 0;
if(argc == 1)
usage();
if(strcmp(argv[1],"read")==0) creating = 0;
else if(strcmp(argv[1],"create")==0) creating = 1;
else usage();
/* get the testfile path */
if(argc > 2)
testfile = argv[2];
else
testfile = DFALT_TESTFILE;
/* Setup various variables */
totalproduct = 1;

18
nc_test4/test_filter_reg.c
View File

@ -15,7 +15,7 @@
#include <H5PLextern.h>
#include "netcdf.h"
#include "netcdf_filter.h"
#include "netcdf_filter_hdf5_build_.h"
#undef TESTODDSIZE
@ -553,40 +553,24 @@ H5Z_filter_reg(unsigned int flags, size_t cd_nelmts,
if (flags & H5Z_FLAG_REVERSE) {
/* Replace buffer */
#ifdef HAVE_H5ALLOCATE_MEMORY
newbuf = H5allocate_memory(*buf_size,0);
#else
newbuf = malloc(*buf_size);
#endif
if(newbuf == NULL) abort();
if(*buf != NULL) {
memcpy(newbuf,*buf,*buf_size);
/* reclaim old buffer */
#ifdef HAVE_H5FREE_MEMORY
H5free_memory(*buf);
#else
free(*buf);
#endif
}
*buf = newbuf;
} else {
/* Replace buffer */
#ifdef HAVE_H5ALLOCATE_MEMORY
newbuf = H5allocate_memory(*buf_size,0);
#else
newbuf = malloc(*buf_size);
#endif
if(newbuf == NULL) abort();
if(*buf != NULL) {
memcpy(newbuf,*buf,*buf_size);
/* reclaim old buffer */
#ifdef HAVE_H5FREE_MEMORY
H5free_memory(*buf);
#else
free(*buf);
#endif
}
*buf = newbuf;

16
nc_test4/test_filter_repeat.c
View File

@ -25,7 +25,7 @@
#define MAXDIMS 8
#define TESTFILE "testfilter_reg.nc"
#define DFALT_TESTFILE "testfilter_reg.nc"
#define NPARAMS 1
#define PARAMVAL 17
@ -42,6 +42,9 @@ static size_t chunkproduct = 1; /* x-product over actual chunks */
static int nerrs = 0;
static char* testfile = NULL;
static int ncid, varid;
static int dimids[MAXDIMS];
static size_t odom[MAXDIMS];
@ -111,7 +114,7 @@ create(void)
int i;
/* Create a file with one big variable */
CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
for(i=0;i<ndims;i++) {
char dimname[1024];
@ -151,7 +154,7 @@ openfile(void)
int k;
/* Open the file and check it. */
CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid));
CHECK(nc_open(testfile, NC_NOWRITE, &ncid));
CHECK(nc_inq_varid(ncid, "var", &varid));
/* Verify chunking */
@ -338,6 +341,13 @@ static void
init(int argc, char** argv)
{
int i;
/* get the testfile path */
if(argc > 1)
testfile = argv[1];
else
testfile = DFALT_TESTFILE;
/* Setup various variables */
totalproduct = 1;
actualproduct = 1;

457
nc_test4/test_filterx_hdf5.c Normal file
View File

@ -0,0 +1,457 @@
/*
Copyright 2018, UCAR/Unidata
See COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <sys/types.h>
/* Test using the filterx interface
for HDF5 filters
*/
#include "netcdf.h"
#include "netcdf_filter.h"
#include "ncbytes.h"
#include "ncjson.h"
#undef TESTODDSIZE
#undef DEBUG
#define FILTER_ID 40000
#define MAXERRS 8
#define MAXPARAMS 32
#define MAXDIMS 8
#define DFALT_TESTFILE "tmp_filterx_hdf5.nc"
#define NPARAMS 1
#define PARAMVAL 17
#define NDIMS 4
static size_t dimsize[NDIMS] = {4,4,4,4};
static size_t chunksize[NDIMS] = {4,4,4,4};
static size_t ndims = NDIMS;
static size_t totalproduct = 1; /* x-product over max dims */
static size_t actualproduct = 1; /* x-product over actualdims */
static size_t chunkproduct = 1; /* x-product over actual chunks */
static int nerrs = 0;
static char* testfile = NULL;
static int ncid, varid;
static int dimids[MAXDIMS];
static size_t odom[MAXDIMS];
static float* array = NULL;
static float* expected = NULL;
/* Forward */
static int filter_test1(void);
static void init(int argc, char** argv);
static void reset(void);
static void odom_reset(void);
static int odom_more(void);
static int odom_next(void);
static int odom_offset(void);
static float expectedvalue(void);
static int vector2json(size_t n, unsigned* values, char** textp);
static int json2vector(const NCjson* jarray, size_t* np, unsigned** valuesp);
#define ERRR do { \
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \
__FILE__, __LINE__); \
nerrs++;\
} while (0)
static int
check(int err,int line)
{
if(err != NC_NOERR) {
fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err));
}
return NC_NOERR;
}
#if 0
static void
report(const char* msg, int lineno)
{
fprintf(stderr,"fail: line=%d %s\n",lineno,msg);
exit(1);
}
#endif
#define CHECK(x) check(x,__LINE__)
#define REPORT(x) report(x,__LINE__)
static int
verifychunks(void)
{
int i;
int store = -1;
size_t localchunks[MAXDIMS];
memset(localchunks,0,sizeof(localchunks));
CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks));
if(store != NC_CHUNKED) {
fprintf(stderr,"bad chunk store\n");
return 0;
}
for(i=0;i<ndims;i++) {
if(chunksize[i] != localchunks[i]) {
fprintf(stderr,"bad chunk size: %d\n",i);
return 0;
}
}
return 1;
}
static int
create(void)
{
int i;
/* Create a file with one big variable */
CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
for(i=0;i<ndims;i++) {
char dimname[1024];
snprintf(dimname,sizeof(dimname),"dim%d",i);
CHECK(nc_def_dim(ncid, dimname, dimsize[i], &dimids[i]));
}
CHECK(nc_def_var(ncid, "var", NC_FLOAT, ndims, dimids, &varid));
return NC_NOERR;
}
static void
deffilter(unsigned int id, size_t nparams, unsigned int* params)
{
char template[8192];
char* tmp = NULL;
CHECK(vector2json(nparams,params,&tmp));
snprintf(template,sizeof(template),"{\"id\":%u, \"parameters\":%s}",id,tmp);
/* Register filter */
CHECK(nc_def_var_filterx(ncid,varid,template));
nullfree(tmp);
}
static void
printfilter(unsigned int id)
{
char* buf = NULL;
char xid[64];
snprintf(xid,sizeof(xid),"%u",id);
CHECK(nc_inq_var_filterx_info(ncid,varid,xid,&buf));
printf("filter(%s): params=%s\n",xid,buf);
nullfree(buf);
}
static int
openfile(void)
{
unsigned int* filterids = NULL;
size_t nfilters = 0;
int k;
char* buf = NULL;
NCjson* json = NULL;
/* Open the file and check it. */
CHECK(nc_open(testfile, NC_NOWRITE, &ncid));
CHECK(nc_inq_varid(ncid, "var", &varid));
/* Verify chunking */
if(!verifychunks())
return 0;
/* Check the compression algorithms */
CHECK(nc_inq_var_filterx_ids(ncid,varid,&buf));
CHECK(NCJparse(buf,0,&json));
CHECK(json2vector(json,&nfilters,&filterids));
for(k=0;k<nfilters;k++)
printfilter(filterids[k]);
fflush(stderr);
nullfree(buf);
nullfree(filterids);
NCJreclaim(json);
return 1;
}
static int
setchunking(void)
{
int store;
store = NC_CHUNKED;
CHECK(nc_def_var_chunking(ncid,varid,store,chunksize));
if(!verifychunks())
return NC_EINVAL;
return NC_NOERR;
}
static void
fill(void)
{
odom_reset();
if(1) {
int i;
if(actualproduct <= 1) abort();
for(i=0;i<actualproduct;i++)
expected[i] = (float)i;
} else {
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
expected[offset] = expect;
odom_next();
}
}
}
static int
compare(void)
{
int errs = 0;
printf("data comparison: |array|=%ld\n",(unsigned long)actualproduct);
if(1)
{
int i;
for(i=0;i<actualproduct;i++) {
if(expected[i] != array[i]) {
fprintf(stderr,"data mismatch: array[%d]=%f expected[%d]=%f\n",
i,array[i],i,expected[i]);
errs++;
if(errs >= MAXERRS)
break;
}
}
} else
{
odom_reset();
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
if(array[offset] != expect) {
fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n",
offset,array[offset],expect);
errs++;
if(errs >= MAXERRS)
break;
}
odom_next();
}
}
if(errs == 0)
printf("no data errors\n");
return (errs == 0);
}
static int
filter_test1(void)
{
int ok = 1;
unsigned int params[MAXPARAMS];
reset();
printf("test1: def filter repeat .\n");
create();
setchunking();
params[0] = 1;
params[1] = 17;
deffilter(FILTER_ID,2,params);
params[0] = 0;
params[1] = 18;
deffilter(FILTER_ID,2,params);
CHECK(nc_enddef(ncid));
/* Fill in the array */
fill();
printf("test1: compression.\n");
/* write array */
CHECK(nc_put_var(ncid,varid,expected));
CHECK(nc_close(ncid));
printf("test1: decompression.\n");
reset();
openfile();
CHECK(nc_get_var_float(ncid, varid, array));
ok = compare();
CHECK(nc_close(ncid));
return ok;
}
/**************************************************/
/* Utilities */
static void
reset()
{
memset(array,0,sizeof(float)*actualproduct);
}
static void
odom_reset(void)
{
memset(odom,0,sizeof(odom));
}
static int
odom_more(void)
{
return (odom[0] < dimsize[0]);
}
static int
odom_next(void)
{
int i; /* do not make unsigned */
for(i=ndims-1;i>=0;i--) {
odom[i] += 1;
if(odom[i] < dimsize[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows*/
odom[i] = 0; /* reset this position*/
}
return 1;
}
static int
odom_offset(void)
{
int i;
int offset = 0;
for(i=0;i<ndims;i++) {
offset *= dimsize[i];
offset += odom[i];
}
return offset;
}
static float
expectedvalue(void)
{
int i;
float offset = 0;
for(i=0;i<ndims;i++) {
offset *= dimsize[i];
offset += odom[i];
}
return offset;
}
static void
init(int argc, char** argv)
{
int i;
/* get the testfile path */
if(argc > 1)
testfile = argv[1];
else
testfile = DFALT_TESTFILE;
/* Setup various variables */
totalproduct = 1;
actualproduct = 1;
chunkproduct = 1;
for(i=0;i<NDIMS;i++) {
totalproduct *= dimsize[i];
if(i < ndims) {
actualproduct *= dimsize[i];
chunkproduct *= chunksize[i];
}
}
/* Allocate max size */
array = (float*)calloc(1,sizeof(float)*actualproduct);
expected = (float*)calloc(1,sizeof(float)*actualproduct);
}
/**************************************************/
/* Utilities */
static int
vector2json(size_t n, unsigned* values, char** textp)
{
int stat = NC_NOERR;
size_t i,jlen;
char tmp[16];
char* json = NULL;
jlen = n*12
+2 /* [] */
+n /* commas */
+1 /* nul term */
;
if((json = malloc(jlen))==NULL)
{stat = NC_ENOMEM; goto done;}
json[0] = '\0';
strlcat(json,"[",jlen);
for(i=0;i<n;i++) {
snprintf(tmp,sizeof(tmp),"%u%s",values[i],(i == n-1?"":","));
strlcat(json,tmp,jlen);
}
strlcat(json,"]",jlen);
if(textp) {*textp = json; json = NULL;}
done:
nullfree(json);
return stat;
}
static int
json2vector(const NCjson* jarray, size_t* np, unsigned** valuesp)
{
int i,stat = NC_NOERR;
unsigned* values = NULL;
struct NCJconst con;
if(NCJsort(jarray) != NCJ_ARRAY)
{stat = NC_EINVAL; goto done;}
if(NCJlength(jarray) > 0 && NCJcontents(jarray) != NULL) {
if((values = (unsigned*)malloc(sizeof(unsigned)*NCJlength(jarray)))==NULL)
{stat = NC_ENOMEM; goto done;}
for(i=0;i<NCJlength(jarray);i++) {
if((stat=NCJcvt(NCJith(jarray,i),NCJ_INT,&con))) goto done;
values[i] = (unsigned)con.ival;
}
}
if(np) *np = NCJlength(jarray);
if(valuesp) {*valuesp = values; values = NULL;}
done:
nullfree(values);
return stat;
}
/**************************************************/
int
main(int argc, char **argv)
{
#ifdef DEBUG
H5Eprint(stderr);
nc_set_log_level(1);
#endif
init(argc,argv);
if(!filter_test1()) ERRR;
exit(nerrs > 0?1:0);
}

3
nc_test4/tst_dims.c
View File

@ -1332,7 +1332,8 @@ main(int argc, char **argv)
strcat(file_in, REF_FILE_NAME);
/* Reopen and check it out again. */
if (nc_open(file_in, NC_NOWRITE, &ncid)) ERR;
if ((ret = nc_open(file_in, NC_NOWRITE, &ncid)))
ERR;
/* Check it out. */
ret = nc_inq_dim(ncid, dimid, name_in, &len_in);

165
nc_test4/tst_filter.sh
View File

@ -59,7 +59,7 @@ findplugin h5bzip2
BZIP2LIB="${HDF5_PLUGIN_LIB}"
BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}"
# Find misc and capture
findplugin misc
findplugin h5misc
MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}"
echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
@ -81,99 +81,99 @@ fi
if test "x$API" = x1 ; then
echo "*** Testing dynamic filters using API"
rm -f ./bzip2.nc ./bzip2.dump ./tst_filter.txt
rm -f ./tmp_bzip2.nc ./tmp_bzip2.dump ./tmp_filter.txt
${execdir}/test_filter
${NCDUMP} -s bzip2.nc > ./tst_filter.txt
${NCDUMP} -s -n bzip2 tmp_bzip2.nc > ./tmp_filter.txt
# Remove irrelevant -s output
sclean ./tst_filter.txt ./bzip2.dump
diff -b -w ${srcdir}/bzip2.cdl ./bzip2.dump
sclean ./tmp_filter.txt ./tmp_bzip2.dump
diff -b -w ${srcdir}/bzip2.cdl ./tmp_bzip2.dump
echo "*** Pass: API dynamic filter"
fi
if test "x$MISC" = x1 ; then
echo
echo "*** Testing dynamic filters parameter passing"
rm -f ./testmisc.nc tst_filter.txt tst_filter2.txt
rm -f ./tmp_misc.nc tmp_filter.txt tmp_filter2.txt
${execdir}/test_filter_misc
# Verify the parameters via ncdump
${NCDUMP} -s testmisc.nc > ./tst_filter.txt
${NCDUMP} -s tmp_misc.nc > ./tmp_misc.txt
# Extract the parameters
getfilterattr ./tst_filter.txt ./tst_filter2.txt
rm -f ./tst_filter.txt
trimleft ./tst_filter2.txt ./tst_filter.txt
rm -f ./tst_filter2.txt
cat >./tst_filter2.txt <<EOF
getfilterattr ./tmp_misc.txt ./tmp_filter2.txt
rm -f ./tmp_misc.txt
trimleft ./tmp_filter2.txt ./tmp_filter.txt
rm -f ./tmp_filter2.txt
cat >./tmp_filter2.txt <<EOF
var:_Filter = "32768,3,239,23,65511,27,77,93,1145389056,3287505826,1097305129,1,2147483648,4294967295,4294967295" ;
EOF
diff -b -w ./tst_filter.txt ./tst_filter2.txt
diff -b -w ./tmp_filter.txt ./tmp_filter2.txt
echo "*** Pass: parameter passing"
fi
if test "x$NG" = x1 ; then
echo "*** Testing dynamic filters using ncgen"
rm -f ./bzip2.nc ./bzip2.dump ./tst_filter.txt
${NCGEN} -lb -4 -o bzip2.nc ${srcdir}/bzip2.cdl
${NCDUMP} -s bzip2.nc > ./tst_filter.txt
rm -f ./tmp_bzip2.nc ./tmp_bzip2.dump ./tmp_filter.txt
${NCGEN} -lb -4 -o tmp_bzip2.nc ${srcdir}/bzip2.cdl
${NCDUMP} -s -n bzip2 tmp_bzip2.nc > ./tmp_filter.txt
# Remove irrelevant -s output
sclean ./tst_filter.txt ./bzip2.dump
diff -b -w ${srcdir}/bzip2.cdl ./bzip2.dump
sclean ./tmp_filter.txt ./tmp_bzip2.dump
diff -b -w ${srcdir}/bzip2.cdl ./tmp_bzip2.dump
echo "*** Pass: ncgen dynamic filter"
fi
if test "x$NCP" = x1 ; then
echo "*** Testing dynamic filters using nccopy"
rm -f ./unfiltered.nc ./filtered.nc ./tmp.nc ./filtered.dump ./tst_filter.txt
rm -f ./tmp_unfiltered.nc ./tmp_filtered.nc ./tmp.nc ./tmp_filtered.dump ./tmp_filter.txt
# Create our input test files
${NCGEN} -4 -lb -o unfiltered.nc ${srcdir}/ref_unfiltered.cdl
${NCGEN} -4 -lb -o unfilteredvv.nc ${srcdir}/ref_unfilteredvv.cdl
${NCGEN} -4 -lb -o tmp_unfiltered.nc ${srcdir}/ref_unfiltered.cdl
${NCGEN} -4 -lb -o tmp_unfilteredvv.nc ${srcdir}/ref_unfilteredvv.cdl
echo " *** Testing simple filter application"
${NCCOPY} -M0 -F "/g/var,307,9,4" unfiltered.nc filtered.nc
${NCDUMP} -s filtered.nc > ./tst_filter.txt
${NCCOPY} -M0 -F "/g/var,307,9" tmp_unfiltered.nc tmp_filtered.nc
${NCDUMP} -s -n filtered tmp_filtered.nc > ./tmp_filter.txt
# Remove irrelevant -s output
sclean ./tst_filter.txt ./filtered.dump
diff -b -w ${srcdir}/ref_filtered.cdl ./filtered.dump
sclean ./tmp_filter.txt ./tmp_filtered.dump
diff -b -w ${srcdir}/ref_filtered.cdl ./tmp_filtered.dump
echo " *** Pass: nccopy simple filter"
echo " *** Testing '*' filter application"
${NCCOPY} -M0 -F "*,307,9,4" unfilteredvv.nc filteredvv.nc
${NCDUMP} -s filteredvv.nc > ./tst_filtervv.txt
${NCCOPY} -M0 -F "*,307,9" tmp_unfilteredvv.nc tmp_filteredvv.nc
${NCDUMP} -s -n filteredvv tmp_filteredvv.nc > ./tmp_filtervv.txt
# Remove irrelevant -s output
sclean ./tst_filtervv.txt ./filteredvv.dump
diff -b -w ${srcdir}/ref_filteredvv.cdl ./filteredvv.dump
sclean ./tmp_filtervv.txt ./tmp_filteredvv.dump
diff -b -w ${srcdir}/ref_filteredvv.cdl ./tmp_filteredvv.dump
echo " *** Pass: nccopy '*' filter"
echo " *** Testing 'v&v' filter application"
${NCCOPY} -M0 -F "var1&/g/var2,307,9,4" unfilteredvv.nc filteredvbar.nc
${NCDUMP} -n filteredvv -s filteredvbar.nc > ./tst_filtervbar.txt
${NCCOPY} -M0 -F "var1&/g/var2,307,9" tmp_unfilteredvv.nc tmp_filteredvbar.nc
${NCDUMP} -n filteredvv -s tmp_filteredvbar.nc > ./tmp_filtervbar.txt
# Remove irrelevant -s output
sclean ./tst_filtervbar.txt ./filteredvbar.dump
diff -b -w ${srcdir}/ref_filteredvv.cdl ./filteredvbar.dump
sclean ./tmp_filtervbar.txt ./tmp_filteredvbar.dump
diff -b -w ${srcdir}/ref_filteredvv.cdl ./tmp_filteredvbar.dump
echo " *** Pass: nccopy 'v|v' filter"
echo " *** Testing pass-thru of filters"
rm -f ./tst_filter.txt tst_filter2.txt ./tst_filter2.nc
rm -f ./tmp_filter.txt tmp_filter2.txt ./tmp_filter2.nc
# Prevent failure by allowing any chunk size
${NCCOPY} -M0 ./filtered.nc ./tst_filter2.nc
${NCDUMP} -s tst_filter2.nc > ./tst_filter.txt
sed -e '/_Filter/p' -e d < ./tst_filter.txt >tst_filter2.txt
test -s tst_filter2.txt
${NCCOPY} -M0 ./tmp_filtered.nc ./tmp_filter2.nc
${NCDUMP} -s tmp_filter2.nc > ./tmp_filter.txt
sed -e '/_Filter/p' -e d < ./tmp_filter.txt >tmp_filter2.txt
test -s tmp_filter2.txt
echo " *** Pass: pass-thru of filters"
echo " *** Testing -F none"
rm -f ./tst_none.txt ./tst_none2.txt ./tst_none.nc
${NCCOPY} -M0 -F none ./filtered.nc ./tst_none.nc
${NCDUMP} -hs tst_none.nc > ./tst_none.txt
sed -e '/_Filter/p' -e d < ./tst_none.txt >./tst_none2.txt
test ! -s tst_none2.txt
rm -f ./tmp_none.txt ./tmp_none2.txt ./tmp_none.nc
${NCCOPY} -M0 -F none ./tmp_filtered.nc ./tmp_none.nc
${NCDUMP} -hs tmp_none.nc > ./tmp_none.txt
sed -e '/_Filter/p' -e d < ./tmp_none.txt >./tmp_none2.txt
test ! -s tmp_none2.txt
echo " *** Pass: -F none"
echo " *** Testing -F var,none "
rm -f ./tst_vnone.txt ./tst_vnone.nc tst_vnone2.txt
${NCCOPY} -M0 -F "/g/var,none" ./filtered.nc ./tst_vnone.nc
${NCDUMP} -s tst_vnone.nc > ./tst_vnone.txt
sed -e '/_Filter/p' -e d < ./tst_vnone.txt >tst_vnone2.txt
test ! -s tst_vnone2.txt
rm -f ./tmp_vnone.txt ./tmp_vnone.nc tmp_vnone2.txt
${NCCOPY} -M0 -F "/g/var,none" ./tmp_filtered.nc ./tmp_vnone.nc
${NCDUMP} -s tmp_vnone.nc > ./tmp_vnone.txt
sed -e '/_Filter/p' -e d < ./tmp_vnone.txt >tmp_vnone2.txt
test ! -s tmp_vnone2.txt
echo " *** Pass: -F var,none"
echo "*** Pass: all nccopy filter tests"
@ -210,75 +210,60 @@ echo "*** Pass: unknown filter"
fi
if test "x$NGC" = x1 ; then
rm -f ./test_bzip2.c
rm -f ./tmp_bzip2.c
echo "*** Testing dynamic filters using ncgen with -lc"
${NCGEN} -lc -4 ${srcdir}/bzip2.cdl > test_bzip2.c
diff -b -w ${srcdir}/ref_bzip2.c ./test_bzip2.c
${NCGEN} -lc -4 ${srcdir}/bzip2.cdl > tmp_bzip2.c
diff -b -w ${srcdir}/ref_bzip2.c ./tmp_bzip2.c
echo "*** Pass: ncgen dynamic filter"
fi
if test "x$MULTI" = x1 ; then
echo "*** Testing multiple filters"
rm -f ./multifilter.nc ./multi.txt ./smulti.cdl
rm -f nccopyF.cdl nccopyF.nc ncgenF.cdl ncgenF.nc
rm -f ./tmp_multifilter.nc ./tmp_multi.txt ./tmp_smulti.cdl
rm -f tmp_nccopyF.cdl tmp_nccopyF.nc tmp_ncgenF.cdl tmp_ncgenF.nc
${execdir}/tst_multifilter
${NCDUMP} -hs multifilter.nc >./multi.cdl
${NCDUMP} -hs -n multifilter tmp_multifilter.nc >./tmp_multi.cdl
# Remove irrelevant -s output
sclean ./multi.cdl ./smulti.cdl
diff -b -w ${srcdir}/ref_multi.cdl ./smulti.cdl
sclean ./tmp_multi.cdl ./tmp_smulti.cdl
diff -b -w ${srcdir}/ref_multi.cdl ./tmp_smulti.cdl
echo "*** nccopy -F with multiple filters"
if ! test -f unfiltered.nc ; then
${NCGEN} -4 -lb -o unfiltered.nc ${srcdir}/ref_unfiltered.cdl
if ! test -f tmp_unfiltered.nc ; then
${NCGEN} -4 -lb -o tmp_unfiltered.nc ${srcdir}/ref_unfiltered.cdl
fi
${NCCOPY} "-F/g/var,307,4|40000" unfiltered.nc nccopyF.nc
${NCDUMP} -hs nccopyF.nc > ./nccopyF.cdl
sclean nccopyF.cdl nccopyFs.cdl
diff -b -w ${srcdir}/ref_nccopyF.cdl ./nccopyFs.cdl
${NCCOPY} "-F/g/var,307,4|40000" tmp_unfiltered.nc tmp_nccopyF.nc
${NCDUMP} -hs -n nccopyF tmp_nccopyF.nc > ./tmp_nccopyF.cdl
sclean tmp_nccopyF.cdl tmp_nccopyFs.cdl
diff -b -w ${srcdir}/ref_nccopyF.cdl ./tmp_nccopyFs.cdl
echo "*** ncgen with multiple filters"
${NCGEN} -4 -lb -o ncgenF.nc ${srcdir}/ref_nccopyF.cdl
${NCGEN} -4 -lb -o tmp_ncgenF.nc ${srcdir}/ref_nccopyF.cdl
# Need to fix name using -n
${NCDUMP} -hs -n nccopyF ncgenF.nc > ./ncgenF.cdl
sclean ncgenF.cdl ncgenFs.cdl
diff -b -w ${srcdir}/ref_nccopyF.cdl ./ncgenFs.cdl
${NCDUMP} -hs -n nccopyF tmp_ncgenF.nc > ./tmp_ncgenF.cdl
sclean tmp_ncgenF.cdl tmp_ncgenFs.cdl
diff -b -w ${srcdir}/ref_nccopyF.cdl ./tmp_ncgenFs.cdl
echo "*** Pass: multiple filters"
fi
if test "x$REP" = x1 ; then
echo "*** Testing filter re-definition invocation"
rm -f filterrepeat.txt
${execdir}/test_filter_repeat >filterrepeat.txt
diff -b -w ${srcdir}/ref_filter_repeat.txt filterrepeat.txt
rm -f tmp_filterrepeat.txt
${execdir}/test_filter_repeat >tmp_filterrepeat.txt
diff -b -w ${srcdir}/ref_filter_repeat.txt tmp_filterrepeat.txt
fi
if test "x$ORDER" = x1 ; then
echo "*** Testing multiple filter order of invocation on create"
rm -f crfilterorder.txt
${execdir}/test_filter_order create >crfilterorder.txt
diff -b -w ${srcdir}/ref_filter_order_create.txt crfilterorder.txt
rm -f tmp_crfilterorder.txt
${execdir}/test_filter_order create >tmp_crfilterorder.txt
diff -b -w ${srcdir}/ref_filter_order_create.txt tmp_crfilterorder.txt
echo "*** Testing multiple filter order of invocation on read"
rm -f rdfilterorder.txt
${execdir}/test_filter_order read >rdfilterorder.txt
diff -b -w ${srcdir}/ref_filter_order_read.txt rdfilterorder.txt
rm -f tmp_rdfilterorder.txt
${execdir}/test_filter_order read >tmp_rdfilterorder.txt
diff -b -w ${srcdir}/ref_filter_order_read.txt tmp_rdfilterorder.txt
fi
echo "*** Pass: all selected tests passed"
#cleanup
rm -f testmisc.nc
rm -f unfiltered.nc unfilteredvv.nc filtered.nc filtered.dump
rm -f filteredvv.nc tst_filtervv.txt filteredvv.dump
rm -f filteredvbar.nc tst_filtervbar.txt filteredvbar.dump
rm -f tst_filter2.nc tst_filter2.txt
rm -f tst_none.nc tst_none.txt tst_none2.txt testfilter_reg.nc
rm -f tst_vnone.nc tst_vnone.txt tst_vnone2.txt
rm -f bzip2.nc bzip2.dump tst_filter.txt bzip2x.dump
rm -f test_bzip2.c
rm -f multifilter.nc multi.cdl smulti.cdl
rm -f nccopyF.nc nccopyF.cdl ncgenF.nc ncgenF.cdl
rm -f ncgenFs.cdl nccopyFs.cdl
rm -f crfilterorder.txt rdfilterorder.txt
exit 0

4
nc_test4/tst_interops.c
View File

@ -36,7 +36,9 @@ printRes(const char *msg, int ires)
if (ires < 0)
{
printf("bad ires: %d\n", ires);
/*H5Eprint2(ires, stdout);*/
#ifdef DEBUG
H5Eprint2(ires, stdout);
#endif
exit(1);
}
}

19
nc_test4/tst_multifilter.c
View File

@ -27,7 +27,7 @@ Test support for multiple filters per variable
#define NFILTERS 3
#define TESTFILE "multifilter.nc"
#define DFALT_TESTFILE "tmp_multifilter.nc"
/* Point at which we give up */
#define MAXERRS 8
@ -36,6 +36,8 @@ Test support for multiple filters per variable
#define DIMSIZE 4
#define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/
static const char* testfile = NULL;
static size_t dimsize = DIMSIZE;
static size_t chunksize = CHUNKSIZE;
static size_t actualdims = NDIMS;
@ -199,7 +201,7 @@ test_multi(void)
memset(array,0,sizeof(float)*actualproduct);
/* Create a file */
CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid));
/* Do not use fill for this file */
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
@ -262,7 +264,7 @@ test_multi(void)
memset(array,0,sizeof(float)*actualproduct);
/* Open the file */
CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid));
CHECK(nc_open(testfile, NC_NOWRITE, &ncid));
/* Get the variable id */
CHECK(nc_inq_varid(ncid, "var", &varid));
@ -289,6 +291,13 @@ static void
init(int argc, char** argv)
{
int i;
/* get the testfile path */
if(argc > 1)
testfile = argv[1];
else
testfile = DFALT_TESTFILE;
/* Setup various variables */
actualproduct = 1;
chunkproduct = 1;
@ -309,7 +318,9 @@ init(int argc, char** argv)
int
main(int argc, char **argv)
{
H5Eprint1(stderr);
#ifdef DEBUG
H5Eprint(stderr);
#endif
init(argc,argv);
if(test_multi() != NC_NOERR) ERRR;
exit(nerrs > 0?1:0);

Some files were not shown because too many files have changed in this diff Show More