mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-19 17:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3034 from DennisHeimbigner/minpluginx.dmh

Browse Source 
Extend the netcdf API to support programmatic changes to the plugin search path
This commit is contained in:
Ward Fisher 2024-11-13 01:11:54 +01:00 committed by GitHub
commit fb75ad6588
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
96 changed files with 4112 additions and 2661 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/main-cmake.yml vendored
View File

@ -1,6 +1,6 @@
name: NetCDF-C CMake CI - Windows
on: [pull_request, workflow_dispatch]
on: [ pull_request, workflow_dispatch]
env:
REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }}

131
CMakeLists.txt
View File

@ -54,7 +54,6 @@ if (NOT DEFINED NETCDF_IS_TOP_LEVEL)
endif ()
endif ()
################################
# The target
################################
@ -72,8 +71,6 @@ endif()
add_library(netcdf)
add_library(netCDF::netcdf ALIAS netcdf)
# Version of the dispatch table. This must match the value in
# configure.ac.
set(NC_DISPATCH_VERSION 5)
@ -363,12 +360,12 @@ endif()
set(DEFAULT_CHUNK_SIZE 16777216 CACHE STRING "Default Chunk Cache Size.")
set(DEFAULT_CHUNK_CACHE_SIZE 16777216U CACHE STRING "Default Chunk Cache Size.")
set(DEFAULT_CHUNKS_IN_CACHE 1000 CACHE STRING "Default number of chunks in cache.")
set(DEFAULT_CHUNK_CACHE_PREEMPTION 0.75 CACHE STRING "Default file chunk cache preemption policy (a number between 0 and 1, inclusive.")
set(DEFAULT_CHUNK_CACHE_PREEMPTION 0.75 CACHE STRING "Default file chunk cache preemption policy (a number between 0 and 1, inclusive).")
# HDF5 default cache size values
set(CHUNK_CACHE_SIZE ${DEFAULT_CHUNK_CACHE_SIZE} CACHE STRING "Default HDF5 Chunk Cache Size.")
set(CHUNK_CACHE_NELEMS ${DEFAULT_CHUNKS_IN_CACHE} CACHE STRING "Default maximum number of elements in cache.")
set(CHUNK_CACHE_PREEMPTION ${DEFAULT_CHUNK_CACHE_PREEMPTION} CACHE STRING "Default file chunk cache preemption policy for HDf5 files(a number between 0 and 1, inclusive.")
set(CHUNK_CACHE_PREEMPTION ${DEFAULT_CHUNK_CACHE_PREEMPTION} CACHE STRING "Default file chunk cache preemption policy for HDf5 files(a number between 0 and 1, inclusive.)")
set(NETCDF_LIB_NAME "" CACHE STRING "Default name of the netcdf library.")
set(TEMP_LARGE "." CACHE STRING "Where to put large temp files if large file tests are run.")
@ -629,76 +626,66 @@ option(NETCDF_ENABLE_FILTER_BZ2 "Enable use of Bz2 compression library if it i
option(NETCDF_ENABLE_FILTER_BLOSC "Enable use of blosc compression library if it is available." ON)
option(NETCDF_ENABLE_FILTER_ZSTD "Enable use of Zstd compression library if it is available." ON)
# If user wants, then install selected plugins (default on)
set(NETCDF_PLUGIN_INSTALL_DIR "YES" CACHE STRING "Whether and where we should install plugins; defaults to yes")
if(NOT NETCDF_ENABLE_PLUGINS)
unset(NETCDF_PLUGIN_INSTALL_DIR CACHE)
endif()
# If user wants, then install selected plugins (default off)
option(NETCDF_PLUGIN_INSTALL "Enable plugin installation" NO)
# This is ugly, but seems necessary because of CMake's boolean structure
set(boolval FALSE)
if(DEFINED NETCDF_PLUGIN_INSTALL_DIR)
booleanize(${NETCDF_PLUGIN_INSTALL_DIR} boolval)
if(boolval)
set(ENABLE_PLUGIN_INSTALL YES)
# No actual value was specified
unset(NETCDF_PLUGIN_INSTALL_DIR CACHE)
else()
if(boolval STREQUAL "NOTFOUND")
# Must be an actual value
set(ENABLE_PLUGIN_INSTALL YES)
else()
set(ENABLE_PLUGIN_INSTALL NO)
endif()
endif()
# Note: the term PLATFORMDEFAULT stands for:
# -- /usr/loca/hdf5/lib/plugin If on a *nix* machine
# -- %ALLUSERSPROFILE%/hdf5/lib/plugins If on a windows or Mingw platform
if(ISMSVC OR ISMINGW)
set(PLATFORMDEFAULT "$ENV{ALLUSERSPROFILE}\\hdf5\\lib\\plugin")
set(PLATFORMSEP ";")
else()
set(ENABLE_PLUGIN_INSTALL NO)
set(PLATFORMDEFAULT "/usr/local/hdf5/lib/plugin")
set(PLATFORMSEP ":")
endif()
# Ensure no defined plugin dir if not enabled
if(NOT ENABLE_PLUGIN_INSTALL)
unset(NETCDF_PLUGIN_INSTALL_DIR CACHE)
# Internally, the variable DEFAULT_PLUGIN_INSTALL_DIR is the default
# directory into which plugins are installed; this may be undefined if
# plugin installation is disabled (the usual case). It is exported as
# NETCDF_DEFAULT_PLUGIN_INSTALL_DIR.
#
# Similarly the variable DEFAULT_PLUGIN_SEARCH_PATH is the default list
# of directories to search to locate plugins.
#
# See configure.ac to see a table defining the rules for computing these two variables.
# Inferences about plugins
if(NETCDF_WITH_PLUGIN_DIR)
set(NETCDF_ENABLE_PLUGINS yes)
endif()
# canonical form is all forward slashes
string(REPLACE "\\" "/" DEFAULT_PLUGIN_INSTALL_DIR "${DEFAULT_PLUGIN_INSTALL_DIR}")
string(REPLACE "\\" "/" DEFAULT_PLUGIN_SEARCH_PATH "${DEFAULT_PLUGIN_SEARCH_PATH}")
if(NOT NETCDF_ENABLE_PLUGINS)
unset(NETCDF_PLUGIN_INSTALL)
unset(NETCDF_WITH_PLUGIN_DIR)
endif()
if(ENABLE_PLUGIN_INSTALL)
if(NOT DEFINED NETCDF_PLUGIN_INSTALL_DIR)
# Default to HDF5_PLUGIN_PATH or its default directories
if(DEFINED ENV{HDF5_PLUGIN_PATH})
set(NETCDF_PLUGIN_INSTALL_DIR "$ENV{HDF5_PLUGIN_PATH}")
else()
if(ISMSVC OR ISMINGW)
set(NETCDF_PLUGIN_INSTALL_DIR "$ENV{ALLUSERSPROFILE}\\hdf5\\lib\\plugin")
else()
if(NOT DEFINED CMAKE_INSTALL_PREFIX)
set(NETCDF_PLUGIN_INSTALL_DIR "/usr/local/hdf5/lib/plugin")
else()
set(NETCDF_PLUGIN_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/hdf5/lib/plugin")
endif(NOT DEFINED CMAKE_INSTALL_PREFIX)
endif(ISMSVC OR ISMINGW)
endif(DEFINED ENV{HDF5_PLUGIN_PATH})
message(STATUS "Defaulting to -DPLUGIN_INSTALL_DIR=${NETCDF_PLUGIN_INSTALL_DIR}")
endif()
endif(ENABLE_PLUGIN_INSTALL)
if(ENABLE_PLUGIN_INSTALL)
# Use the lowest priority dir in the path
if(NOT ISMSVC AND NOT ISMINGW)
string(REPLACE ":" ";" PATH_LIST ${NETCDF_PLUGIN_INSTALL_DIR})
else()
set(PATH_LIST ${NETCDF_PLUGIN_INSTALL_DIR})
endif()
# Get last element
list(GET PATH_LIST -1 NETCDF_PLUGIN_INSTALL_DIR)
set(PLUGIN_INSTALL_DIR_SETTING "${NETCDF_PLUGIN_INSTALL_DIR}")
message(STATUS "Final value of-DPLUGIN_INSTALL_DIR=${NETCDF_PLUGIN_INSTALL_DIR}")
else() # No option specified
unset(NETCDF_PLUGIN_INSTALL_DIR)
unset(NETCDF_PLUGIN_INSTALL_DIR CACHE)
set(PLUGIN_INSTALL_DIR_SETTING "N.A.")
if (DEFINED NETCDF_WITH_PLUGIN_DIR) # Table row 3
set(DEFAULT_PLUGIN_INSTALL_DIR "${NETCDF_WITH_PLUGIN_DIR}")
set(DEFAULT_PLUGIN_SEARCH_PATH "${NETCDF_WITH_PLUGIN_DIR}${PLATFORMSEP}${PLATFORMDEFAULT}")
elseif (DEFINED CMAKE_INSTALL_PREFIX) # Table row 2
set(DEFAULT_PLUGIN_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/hdf5/lib/plugin")
set(DEFAULT_PLUGIN_SEARCH_PATH "${DEFAULT_PLUGIN_INSTALL_DIR}${PLATFORMSEP}${PLATFORMDEFAULT}")
else() # Table row 1
unset(DEFAULT_PLUGIN_INSTALL_DIR)
set(DEFAULT_PLUGIN_SEARCH_PATH "${PLATFORMDEFAULT}")
endif()
# canonical form is all forward slashes
string(REPLACE "\\" "/" DEFAULT_PLUGIN_INSTALL_DIR "${DEFAULT_PLUGIN_INSTALL_DIR}")
string(REPLACE "\\" "/" DEFAULT_PLUGIN_SEARCH_PATH "${DEFAULT_PLUGIN_SEARCH_PATH}")
message(STATUS "ENABLE_PLUGIN_INSTALL=${ENABLE_PLUGIN_INSTALL} PLUGIN_INSTALL_DIR=${NETCDF_PLUGIN_INSTALL_DIR}")
# Inferences
if (DEFINED DEFAULT_PLUGIN_INSTALL_DIR)
set(ENABLE_PLUGIN_DIR yes)
else()
set(ENABLE_PLUGIN_DIR no)
endif()
set(NETCDF_PLUGIN_INSTALL_DIR "${DEFAULT_PLUGIN_INSTALL_DIR}")
set(NETCDF_PLUGIN_SEARCH_PATH "${DEFAULT_PLUGIN_SEARCH_PATH}")
# Try to enable NCZarr zip support
option(NETCDF_ENABLE_NCZARR_ZIP "Enable NCZarr ZIP support." ${NETCDF_ENABLE_NCZARR})
@ -753,10 +740,10 @@ if(NOT WIN32)
endif()
# Options for S3 Support
#option(NETCDF_ENABLE_S3 "Enable S3 support." OFF)
option(NETCDF_ENABLE_S3_AWS "Enable S3 support via AWS-CPP-SDK" OFF)
option(NETCDF_ENABLE_S3_INTERNAL "Enable S3 Internal support." OFF)
#option(NETCDF_ENABLE_S3 "Enable S3 support." OFF)
cmake_dependent_option(NETCDF_ENABLE_S3 "Enable S3 Support" ON "NETCDF_ENABLE_S3_AWS OR NETCDF_ENABLE_S3_INTERNAL" OFF)
option(NETCDF_ENABLE_NCZARR_S3 "Enable NCZarr S3 support; Deprecated in favor of NETCDF_ENABLE_S3" ${NETCDF_ENABLE_S3})
@ -877,7 +864,7 @@ if(NETCDF_ENABLE_TESTS)
# See https://github.com/Unidata/netcdf-c/issues/2627 for more information.
###
option(NETCDF_ENABLE_BENCHMARKS "Run benchmark Tests." OFF)
set(BUILD_BENCHMARKS ${NETCDF_ENABLE_BENCHMARKS} CACHE BOOL "alias for NETCDF_ENABLE_BENCHMARKS")
###
# End known-failures.
@ -1380,8 +1367,8 @@ endif(NETCDF_ENABLE_MMAP)
# Used in the `configure_file` calls below
set(ISCMAKE "yes")
if(MSVC)
set(ISMSVC ON CACHE BOOL "" FORCE)
set(REGEDIT ON CACHE BOOL "" FORCE)
set(ISMSVC yes CACHE BOOL "" FORCE)
set(REGEDIT yes CACHE BOOL "" FORCE)
# Get windows major version and build number
execute_process(COMMAND "systeminfo" OUTPUT_VARIABLE WININFO)
if(WININFO STREQUAL "")
@ -1564,8 +1551,6 @@ endif()
# STATIC_DEFINE netcdf_BUILT_AS_STATIC
#)
##
# Brute force, grab all of the dlls from the dependency directory,
# install them in the binary dir. Grab all of the .libs, put them

2
RELEASE_NOTES.md
View File

@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release
## 4.9.3 - TBD
* Extend the netcdf API to support programmatic changes to the plugin search path. See [Github #3034](https://github.com/Unidata/netcdf-c/pull/3034) for more information.
## Known Issue
> Parallel operation using `mpich 4.2.0` (the default on `Ubuntu 24.04`) results in 'unexpected results' when running `nc_test4/run_par_test.sh`. This can be fixed by removing `mpich` and associated libraries and development packages and installing `mpich 4.2.2` by hand, or by using `openmpi` provided via `apt`.

8
cmake/dependencies.cmake
View File

@ -226,13 +226,13 @@ if(USE_HDF5)
endif(USE_HDF5)
################################
# Curl Libraryies
# Curl Libraries
# Only needed for DAP (DAP2 or DAP4)
# and NCZARR with S3 Support
# and NCZARR S3 support
# and byterange support
################################
if( (NETCDF_ENABLE_DAP AND (NETCDF_ENABLE_DAP2 OR NETCDF_ENABLE_DAP4 OR NETCDF_ENABLE_BYTERANGE_SUPPORT)) OR (NETCDF_ENABLE_NCZARR AND NETCDF_ENABLENCZARR_S3))
if( NETCDF_ENABLE_DAP2 OR NETCDF_ENABLE_DAP4 OR NETCDF_ENABLE_BYTERANGE_SUPPORT OR NETCDF_ENABLE_NCZARR_S3)
# See if we have libcurl
find_package(CURL)
#target_compile_options(netcdf

14
cmake/netcdf_functions_macros.cmake
View File

@ -324,4 +324,16 @@ function(is_disabled feature ret_val)
set(${ret_val} "yes" PARENT_SCOPE)
set("NC_${ret_val}" 1 PARENT_SCOPE)
endif(${feature})
endfunction()
endfunction()
# Extract the last element from a path string
function(getlastdir s ret_val)
if(NOT ISMSVC AND NOT ISMINGW)
string(REPLACE ":" ";" list "${s}")
else()
set(list ${s})
endif()
list(GET list -1 last)
set(${ret_val} "${last}" PARENT_SCOPE)
endfunction()

9
config.h.cmake.in
View File

@ -124,6 +124,9 @@ are set when opening a binary file on Windows. */
#define ERANGE_FILL 1
#endif
/* if true, use hdf5 S3 virtual file reader */
#cmakedefine NETCDF_ENABLE_HDF5 1
/* if true, use hdf5 S3 virtual file reader */
#cmakedefine NETCDF_ENABLE_HDF5_ROS3 1
@ -160,6 +163,12 @@ are set when opening a binary file on Windows. */
/* if true, Allow dynamically loaded plugins */
#cmakedefine NETCDF_ENABLE_PLUGINS 1
/* Define the plugin install dir */
#cmakedefine NETCDF_PLUGIN_INSTALL_DIR "${NETCDF_PLUGIN_INSTALL_DIR}"
/* Define the plugin search path */
#cmakedefine NETCDF_PLUGIN_SEARCH_PATH "${NETCDF_PLUGIN_SEARCH_PATH}"
/* if true, enable S3 support */
#cmakedefine NETCDF_ENABLE_S3 1

138
configure.ac
View File

@ -1,4 +1,4 @@
o# -*- Autoconf -*-
# -*- Autoconf -*-
## Process this file with autoconf to produce a configure script.
# This is part of Unidata's netCDF package. Copyright 2005-2018, see
@ -945,8 +945,6 @@ else
AC_MSG_RESULT([${have_zip}])
enable_nczarr_zip=${have_zip} # alias
AC_MSG_CHECKING([whether nczarr zip support is enabled])
AC_MSG_RESULT([${enable_nczarr_zip}])
@ -958,15 +956,10 @@ else
AC_MSG_CHECKING([whether netcdf S3 support should be enabled])
AC_ARG_ENABLE([s3],
[AS_HELP_STRING([--enable-s3],
[enable netcdf S3 support])])
[enable netcdf S3 support (default off)])])
test "x$enable_s3" = xyes || enable_s3=no
AC_MSG_RESULT($enable_s3)
if test "x$enable_remote_functionality" = xno ; then
AC_MSG_WARN([--disable-remote-functionality => --disable-s3])
enable_s3=no
fi
# --enable-nczarr-s3 is synonym for --enable-s3 (but...)
AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled])
AC_ARG_ENABLE([nczarr-s3],
@ -975,11 +968,16 @@ else
AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)])
# Set enable_s3 instead of enable_nczarr_s3
if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_remote_functionality" = xyes; then
if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes ; then
enable_s3=yes # back compatibility
fi
unset enable_nczarr_s3
if test "x$enable_remote_functionality" = xno ; then
AC_MSG_WARN([--disable-remote-functionality => --disable-s3])
enable_s3=no
fi
# Note we check for the library after checking for enable_s3
# because for some reason this fails if we unconditionally test for sdk
# and it is not available. Fix someday
@ -1175,7 +1173,7 @@ AC_ARG_ENABLE([benchmarks],
are timed. We use these tests to check netCDF performance.])])
test "x$enable_benchmarks" = xyes || enable_benchmarks=no
AC_MSG_RESULT($enable_benchmarks)
if test "x$enable_HDF5" = xno -a "x$enable_benchmarks" = xyes; then
if test "x$enable_hdf5" = xno -a "x$enable_benchmarks" = xyes; then
AC_MSG_ERROR([Can't use benchmarks if HDF5 is disabled.])
fi
AM_CONDITIONAL(BUILD_BENCHMARKS, [test x$enable_benchmarks = xyes])
@ -1653,7 +1651,7 @@ enable_hdf5_szip=no
has_hdf5_ros3=no
if test "x$enable_hdf5" = xyes; then
AC_DEFINE([NETCDF_ENABLE_HDF5], [1], [if true, use HDF5])
AC_DEFINE([USE_HDF5], [1], [if true, use HDF5])
AC_DEFINE([H5_USE_16_API], [1], [use HDF5 1.6 API])
@ -2153,62 +2151,82 @@ fi
AC_SUBST(STD_FILTERS,[$std_filters])
# The --with-plugin-dir gives the user control of the plugin
# directory. If set to 'yes' (the default), then first directory in
# the HDF5_PLUGIN_PATH, or /usr/local/hdf5/lib/plugin (the default
# HDF5 plugin dir) will be used. If another directory is provided, it
# will be used. If 'no', then plugins will not be installed.
# The --with-plugin-dir option gives the user control of the plugin install
# directory. This also used as one of the default plugin directory when
# searching for plugins.
# If a specific directory is provided, it will be used.
# If not defined or defined with the value no, or --without-plugin-dir, then
# plugin installation will be suppressed (but see below).
AC_MSG_CHECKING([whether and where we should install plugins])
AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=<absolute directory>|yes|no|--without-plugin-dir],
AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=<absolute directory>|no|--without-plugin-dir],
[Install selected standard filters in specified or default directory])],
[],[with_plugin_dir=no])
if test "x$have_zstd" = xyes; then
if test "x$enable_plugins" = xyes && test "x$with_plugin_dir" = xno; then
with_plugin_dir=yes
fi
fi
AC_MSG_RESULT([$with_plugin_dir])
if test "x$with_plugin_dir" = xno ; then # option missing|disabled
with_plugin_dir=no
with_plugin_dir_setting="N.A."
enable_plugin_dir=no
elif test "x$with_plugin_dir" = xyes ; then # --with-plugin-dir, no argument
# Default to last dir (lowest search priority) in HDF5_PLUGIN_PATH
PLUGIN_PATH="$HDF5_PLUGIN_PATH"
if test "x${PLUGIN_PATH}" = x ; then
if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then
PLUGIN_PATH="${ALLUSERSPROFILE}\\hdfd5\\lib\\plugin"
else
if test "x${prefix}" = xNONE ; then
PLUGIN_PATH="/usr/local/hdf5/lib/plugin"
else
PLUGIN_PATH="${prefix}/hdf5/lib/plugin"
fi
fi
fi
# Use the lowest priority dir in the path
if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then
PLUGIN_DIR=`echo "$PLUGIN_PATH" | tr ';' ' '`
else
PLUGIN_DIR=`echo "$PLUGIN_PATH" | tr ':' ' '`
fi
for pp in ${PLUGIN_DIR} ; do last="$pp"; done
PLUGIN_DIR="$last"
with_plugin_dir_setting="$PLUGIN_DIR"
# canonical form is all forward slashes
with_plugin_dir=`echo "$PLUGIN_DIR" | tr '\\\\' '/'`
enable_plugin_dir=yes
AC_MSG_NOTICE([Defaulting to --with-plugin-dir=$with_plugin_dir])
else # --with-plugin-dir=<dir|path>
with_plugin_dir_setting="$with_plugin_dir"
# Using selected filter options forces installation
if test "x$have_zstd" = xyes; then
AC_MSG_WARN([--enable_filter-zstd => --enable-plugins.])
enable_plugins=yes
fi
# Note: the term PLATFORMDEFAULT stands for:
# -- /usr/loca/hdf5/lib/plugin If on a *nix* machine
# -- %ALLUSERSPROFILE%/hdf5/lib/plugins If on a windows or Mingw platform
if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then
PLATFORMDEFAULT="${ALLUSERSPROFILE}\\hdf5\\lib\\plugin"
PLATFORMSEP=";"
else
PLATFORMDEFAULT="/usr/local/hdf5/lib/plugin"
PLATFORMSEP=":"
fi
# Internally, the variable DEFAULT_PLUGIN_INSTALL_DIR is the default
# directory into which plugins are installed; this may be undefined if
# plugin installation is disabled (the usual case). It is exported as
# NETCDF_PLUGIN_INSTALL_DIR.
#
# Similarly the variable DEFAULT_PLUGIN_SEARCH_PATH is the default list
# of directories to search to locate plugins. It is exported as
# NETCDF_PLUGIN_SEARCH_PATH
#
# See the document docs/pluginpath.md for a more detailed discussion
# of how NETCDF_PLUGIN_INSTALL_DIR and NETCDF_PLUGIN_SEARCH_PATH are
# computed.
# Canonicalize --with-prefix-dir and --prefix
if test "x$with_plugin_dir" = xno ; then unset with_plugin_dir ; fi
if test "xprefix" = x ; then prefix=NONE ; fi
if test "x$with_plugin_dir" != x ; then # Table row 3
DEFAULT_PLUGIN_INSTALL_DIR="${with_plugin_dir}"
DEFAULT_PLUGIN_SEARCH_PATH="${with_plugin_dir}${PLATFORMSEP}${PLATFORMDEFAULT}"
elif test "x$prefix" != xNONE ; then # Table row 2
DEFAULT_PLUGIN_INSTALL_DIR="${prefix}/hdf5/lib/plugin"
DEFAULT_PLUGIN_SEARCH_PATH="${prefix}/hdf5/lib/plugin${PLATFORMSEP}${PLATFORMDEFAULT}"
else # Table row 1
unset DEFAULT_PLUGIN_INSTALL_DIR
DEFAULT_PLUGIN_SEARCH_PATH="${PLATFORMDEFAULT}"
fi
# canonical form is all forward slashes
DEFAULT_PLUGIN_INSTALL_DIR=`echo "$DEFAULT_PLUGIN_INSTALL_DIR" | tr '\\\\' '/'`
DEFAULT_PLUGIN_SEARCH_PATH=`echo "$DEFAULT_PLUGIN_SEARCH_PATH" | tr '\\\\' '/'`
# Inferences
if test "x$DEFAULT_PLUGIN_INSTALL_DIR" = x ; then # option missing|disabled
enable_plugin_dir=no
else
enable_plugin_dir=yes
fi
if test "x$enable_plugin_dir" = xyes ; then
AC_DEFINE([ENABLE_PLUGIN_DIR], [1], [If true, enable filter installation])
fi
AM_CONDITIONAL([ENABLE_PLUGIN_DIR], [test "x$enable_plugin_dir" = xyes])
AC_SUBST([PLUGIN_INSTALL_DIR], [$with_plugin_dir])
# Better value for libnetcdf.settings
AC_SUBST([PLUGIN_INSTALL_DIR_SETTING], [$with_plugin_dir_setting])
AC_SUBST(NETCDF_PLUGIN_INSTALL_DIR, [$PLUGIN_INSTALL_DIR])
AC_DEFINE_UNQUOTED([NETCDF_PLUGIN_INSTALL_DIR], ["${DEFAULT_PLUGIN_INSTALL_DIR}"], [Final Install Dir])
AC_DEFINE_UNQUOTED([NETCDF_PLUGIN_SEARCH_PATH], ["${DEFAULT_PLUGIN_SEARCH_PATH}"], [Final Search Path])
AC_SUBST([NETCDF_PLUGIN_INSTALL_DIR], [${DEFAULT_PLUGIN_INSTALL_DIR}])
AC_SUBST([NETCDF_PLUGIN_SEARCH_PATH], [${DEFAULT_PLUGIN_SEARCH_PATH}])
# Access netcdf specific version of config.h
AH_BOTTOM([#include "ncconfigure.h"])

1
docs/Doxyfile.in
View File

@ -965,6 +965,7 @@ INPUT = @abs_top_srcdir@/docs/mainpage.dox \
@abs_top_srcdir@/docs/all-error-codes.md \
@abs_top_srcdir@/docs/filters.md \
@abs_top_srcdir@/docs/quickstart_filters.md \
@abs_top_srcdir@/docs/pluginpath.md \
@abs_top_srcdir@/docs/quickstart_paths.md \
@abs_top_srcdir@/docs/quickstart_env.md \
@abs_top_srcdir@/include/netcdf.h \

2
docs/Makefile.am
View File

@ -16,7 +16,7 @@ known_problems.md COPYRIGHT.md inmeminternal.dox testserver.dox \
byterange.md nczarr.md quantize.md all-error-codes.md \
quickstart_paths.md cloud.md header.html attribute_conventions.md \
file_format_specifications.md quickstart_filters.md quickstart_env.md \
doxygen-awesome-css netcdf-50x50.png
doxygen-awesome-css netcdf-50x50.png pluginpath.md
# Turn off parallel builds in this directory.
.NOTPARALLEL:

99
docs/filters.md
View File

@ -13,9 +13,8 @@ should be aware.
* ***Auto Install of filters***<br>
An option is now provided to automatically install
HDF5 filters into a default location, or optionally
into a user-specified location. This is described in
[Appendix H](#filters_appendixh)
(with supporting information in [Appendix G](#filters_appendixg)).
into a user-specified location.
This is described in the *pluginpath.md* document.
* ***NCZarr Filter Support***<br>
[NCZarr filters](#filters_nczarr) are now supported.
@ -367,32 +366,8 @@ The details for writing such a filter are defined in the HDF5 documentation[1,2]
### Plugin directory {#filters_plugindir}
The HDF5 loader searches for plugins in a number of directories.
This search is contingent on the presence or absence of the environment
variable named ***HDF5\_PLUGIN\_PATH***.
As with all other "...PATH" variables, it is a sequence of absolute
directories separated by a separator character. For *nix* operating systems,
this separator is the colon (':') character. For Windows and Mingw, the
separator is the semi-colon (';') character. So for example:
* Linux: `export HDF5_PLUGIN_PATH=/usr/lib:/usr/local/lib`
* Windows: `export HDF5_PLUGIN_PATH=c:\\ProgramData\\hdf5\\plugin;c:\\tools\\lib`
If HDF5\_PLUGIN\_PATH is defined, then the loader will search each directory
in the path from left to right looking for shared libraries with specific
exported symbols representing the entry points into the library.
If HDF5\_PLUGIN\_PATH is not defined, the loader defaults to using
these default directories:
* Linux: `/usr/local/hdf5/lib/plugin`
* Windows: `%ALLUSERSPROFILE%\\hdf5\\lib\\plugin`
It should be noted that there is a difference between the search order
for HDF5 versus NCZarr. The HDF5 loader will search only the directories
specificed in HDF5\_PLUGIN\_PATH. In NCZarr, the loader
searches HDF5\_PLUGIN\_PATH and as a last resort,
it also searches the default directory.
The netcdf-c process for installing and locating plugins is described
in detail in the *pluginpath.md* document.
### Plugin Library Naming {#filters_Pluginlib}
@ -1164,7 +1139,7 @@ When installing the netcdf library, the following other libraries must be instal
2. The HDF5 wrapper for *libzstd.so* -- There are several options for obtaining this (see [Appendix G](#filters_appendixg).)
3. (Optional) The Zarr wrapper for *libzstd.so* -- you need this if you intend to read/write Zarr datasets that were compressed using zstandard; again see [Appendix G](#filters_appendixg).
## Appendix G. Finding Filters {#filters_appendixg}
## Appendix G. Finding Filter Implementations {#filters_appendixg}
A major problem for filter users is finding an implementation of an HDF5 filter wrapper and (optionally)
its corresponding NCZarr wrapper. There are several ways to do this.
@ -1190,40 +1165,7 @@ so they are only usable with netcdf-4. This will change in the future.
As part of the overall build process, a number of filter wrappers are built as shared libraries in the "plugins" directory.
These wrappers can be installed as part of the overall netcdf-c installation process.
WARNING: the installer still needs to make sure that the actual filter/compression libraries are installed: e.g. libzstd and/or libblosc.
The target location into which libraries in the "plugins" directory are installed is specified
using a special *./configure* option
````
--with-plugin-dir=<directorypath>
or
--with-plugin-dir
````
or its corresponding *cmake* option.
````
-DPLUGIN_INSTALL_DIR=<directorypath>
or
-DPLUGIN_INSTALL_DIR=YES
````
This option defaults to the value "yes", which means that filters are
installed by default. This can be disabled by one of the following options.
````
--without-plugin-dir (automake)
or
--with-plugin-dir=no (automake)
or
-DPLUGIN_INSTALL_DIR=NO (CMake)
````
If the option is specified with no argument (automake) or with the value "YES" (CMake),
then it defaults (in order) to the following directories:
1. If the HDF5\_PLUGIN\_PATH environment variable is defined, then last directory in the list of directories in the path is used.
2. (a) `/usr/local/hdf5/lib/plugin` for linux/unix operating systems (including Cygwin)<br>
(b) `%ALLUSERSPROFILE%\\hdf5\\lib\\plugin` for Windows and MinGW
See the document *pluginpaths.md* for details on the installation process.
If NCZarr is enabled, then in addition to wrappers for the standard filters,
additional libraries will be installed to support NCZarr access to filters.
Currently, this list includes the following:
@ -1243,31 +1185,6 @@ provided by the *lib__nczh5filters.so* shared library. Note also that
if you disable HDF5 support, but leave NCZarr support enabled,
then all of the above filters should continue to work.
### HDF5\_PLUGIN\_PATH
At the moment, NetCDF uses the existing HDF5 environment variable
*HDF5\_PLUGIN\_PATH* to locate the directories in which filter wrapper
shared libraries are located. This is used both for the HDF5 filter
wrappers but also the NCZarr codec wrappers.
*HDF5\_PLUGIN\_PATH* is a typical Windows or Unix style
path-list. That is it is a sequence of absolute directory paths
separated by a specific separator character. For Windows, the
separator character is a semicolon (';') and for Unix, it is a a
colon (':').
So, if HDF5\_PLUGIN\_PATH is defined at build time, and
*--with-plugin-dir* is specified with no argument then the last
directory in the path will be the one into which filter wrappers are
installed. Otherwise the default directories are used.
The important thing to note is that at run-time, there are several cases to consider:
1. HDF5\_PLUGIN\_PATH is defined and is the same value as it was at build time -- no action needed
2. HDF5\_PLUGIN\_PATH is defined and is has a different value from build time -- the user is responsible for ensuring that the run-time path includes the same directory used at build time, otherwise this case will fail.
3. HDF5\_PLUGIN\_PATH is not defined at either run-time or build-time -- no action needed
4. HDF5\_PLUGIN\_PATH is not defined at run-time but was defined at build-time -- this will probably fail
## Appendix I. A Warning on Backward Compatibility {#filters_appendixi}
The API defined in this document should accurately reflect the
@ -1290,9 +1207,9 @@ A user may encounter an incompatibility if any of the following appears in user
For additional information, see [Appendix B](#filters_appendixb).
## Point of Contact {#filters_poc}
## History {#filters_history}
*Author*: Dennis Heimbigner<br>
*Email*: dmh at ucar dot edu<br>
*Email*: dennis.heimbigner@gmail.com<br>
*Initial Version*: 1/10/2018<br>
*Last Revised*: 5/18/2022

2
docs/footer.html
View File

@ -1,4 +1,4 @@
<!-- HTML footer for doxygen 1.9.1-->
<!-- HTML footer for doxygen 1.10.0-->
<!-- start footer part -->
<!--BEGIN GENERATE_TREEVIEW-->
<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->

44
docs/header.html
View File

@ -1,54 +1,78 @@
<!-- HTML header for doxygen 1.9.1-->
<!-- HTML header for doxygen 1.10.0-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<html xmlns="http://www.w3.org/1999/xhtml" lang="$langISO">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta http-equiv="X-UA-Compatible" content="IE=11"/>
<meta name="generator" content="Doxygen $doxygenversion"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
<!--BEGIN PROJECT_ICON-->
<link rel="icon" href="$relpath^$projecticon" type="image/x-icon" />
<!--END PROJECT_ICON-->
<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
<!--BEGIN DISABLE_INDEX-->
<!--BEGIN FULL_SIDEBAR-->
<script type="text/javascript">var page_layout=1;</script>
<!--END FULL_SIDEBAR-->
<!--END DISABLE_INDEX-->
<script type="text/javascript" src="$relpath^jquery.js"></script>
<script type="text/javascript" src="$relpath^dynsections.js"></script>
<!--BEGIN COPY_CLIPBOARD-->
<script type="text/javascript" src="$relpath^clipboard.js"></script>
<!--END COPY_CLIPBOARD-->
$treeview
$search
$mathjax
$darkmode
<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
$extrastylesheet
</head>
<body>
<!--BEGIN DISABLE_INDEX-->
<!--BEGIN FULL_SIDEBAR-->
<div id="side-nav" class="ui-resizable side-nav-resizable"><!-- do not remove this div, it is closed by doxygen! -->
<!--END FULL_SIDEBAR-->
<!--END DISABLE_INDEX-->
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
<!--BEGIN TITLEAREA-->
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
<tbody>
<tr style="height: 56px;">
<tr id="projectrow">
<!--BEGIN PROJECT_LOGO-->
<td id="projectlogo"><img alt="Logo" src="$relpath^$projectlogo"/></td>
<td id="projectlogo"><img alt="Logo" src="$relpath^$projectlogo"$logosize/></td>
<!--END PROJECT_LOGO-->
<!--BEGIN PROJECT_NAME-->
<td id="projectalign" style="padding-left: 0.5em;">
<div id="projectname">$projectname
<!--BEGIN PROJECT_NUMBER-->&#160;<span id="projectnumber">$projectnumber</span><!--END PROJECT_NUMBER-->
<td id="projectalign">
<div id="projectname">$projectname<!--BEGIN PROJECT_NUMBER--><span id="projectnumber">&#160;$projectnumber</span><!--END PROJECT_NUMBER-->
</div>
<!--BEGIN PROJECT_BRIEF--><div id="projectbrief">$projectbrief</div><!--END PROJECT_BRIEF-->
</td>
<!--END PROJECT_NAME-->
<!--BEGIN !PROJECT_NAME-->
<!--BEGIN PROJECT_BRIEF-->
<td style="padding-left: 0.5em;">
<td>
<div id="projectbrief">$projectbrief</div>
</td>
<!--END PROJECT_BRIEF-->
<!--END !PROJECT_NAME-->
<!--BEGIN DISABLE_INDEX-->
<!--BEGIN SEARCHENGINE-->
<td>$searchbox</td>
<!--BEGIN !FULL_SIDEBAR-->
<td>$searchbox</td>
<!--END !FULL_SIDEBAR-->
<!--END SEARCHENGINE-->
<!--END DISABLE_INDEX-->
</tr>
<!--BEGIN SEARCHENGINE-->
<!--BEGIN FULL_SIDEBAR-->
<tr><td colspan="2">$searchbox</td></tr>
<!--END FULL_SIDEBAR-->
<!--END SEARCHENGINE-->
</tbody>
</table>
</div>

2
docs/indexing.dox
View File

@ -2,7 +2,7 @@
\internal
\page Hashed + Indexed Access to Metadata Objects
\page Indexed Access to Metadata Objects
\tableofcontents

208
docs/pluginpath.md Normal file
View File

@ -0,0 +1,208 @@
Appendix E. NetCDF-4 Plugin Path Support {#pluginpaths}
==================================
[TOC]
## Plugin Path Overview {#pluginpaths_overview}
The processes by which plugins are installed into some directory
and the process by which plugins are located are unfortunately
complicated. This is in part due to the historical requirements
to support existing HDF5 and Zarr mechanisms.
This document describes the following major processes:
1. *Discovery* -- at run-time, any reference to a plugin must do a search
to locate a dynamic library that implements the plugin.
2. *Plugin Path Management* -- at run-time, the client program may wish to programmatically
set the sequence of directories to use in locating plugins.
3. *Installation* -- during the build of the netcdf-c library,
any compiled plugins may optionally be installed into some directory.
## Discovering a Specific Plugin at Run-Time
The netcdf-c library maintains an internal sequence of directory paths
-- collectively called the *plugin path* -- that controls the search
for plugin libraries. Basically, when looking for a specific plugin,
each directory in the plugin path is examined in order. For each such
directory, the files in that directory are checked to see if it
contains the specified plugin. The details of how a file is processed
is described in the document *filters.md*.
The netcdf-c search algorithm is closely tied to the HDF5
discovery process. The HDF5 process searches its own internal
plugin path (sequence of directories) in order to discover
a specific plugin library.
The addition of NCZarr support to the netcdf-c library
requires yet another plugin path (sequence of directories)
for its search process.
It is important to know that the plugin path is completely controlled
by a global plugin path. If it changes, then this global plugin path
is propagated to HDF5 and NCZarr to ensure that all such plugin paths
use the same sequence of directories for discovery.
## Programmatic Management of the Plugin Path
As of netcdf-c version 4.9.3, it is possible for a client program
to set the global plugin path to control plugin discovery.
Since the global path and the HDF5 and NCZarr paths are kept in
sync, this means that both HDF5 and NCZarr will look in the same
directories in order to locate specified plugins.
[Appendix E.1](#pluginpath_appendixe1) defines the current API for
managing the global plugin path.
Note that it is best practice for a client program to use the API
to set the plugin path before any calls to *nc_open* or *nc_create*.
Modifying the plugin paths later may fail because it cannot be guaranteed
that the underlying implementations (i.e. HDF5 or NCZarr) will
take notice of the change.
When the netcdf-c library initializes itself, it chooses an initial
global plugin path using the following rules, which are those used
by the HDF5 library:
1. If *HDF5\_PLUGIN\_PATH* environment variable is defined,
then its value is used as the initial plugin path.
2. If *HDF5\_PLUGIN\_PATH* is not defined, then the
initial plugin path is either:
* /usr/local/hdf5/plugin -- for Linux/Unix/Cygwin,
* %ALLUSERSPROFILE%/hdf5/lib/plugin -- for Windows/Mingw.
This initial global plugin path will be propagated to HDF5 and NCZarr.
## Installing Plugins at Build-Time
At build-time, the target location directory into which libraries implementing plugins are installed is specified using a special *./configure* option
````
--with-plugin-dir=<directorypath>
````
or its corresponding *cmake* option.
````
-DNETCDF_WITH_PLUGIN_DIR=<directorypath>
````
## Build-Time Operations
At build time, certain plugin-related constants are constructed.
1. NETCDF_PLUGIN_INSTALL_DIR -- the directory into which compiled plugins should be installed
2. NETCDF_PLUGIN_SEARCH_PATH -- the default search path to be used at run-time if not over-ridden by the *HDF5_PLUGIN_PATH* environment variable.
<table style="border:2px solid black;border-collapse:collapse">
<tr style="outline: thin solid;" align="center"><td colspan="4">Table showing the build-time computation of DEFAULT_PLUGIN_INSTALL_DIR and DEFAULT_PLUGIN_SEARCH_PATH.</td>
<tr style="outline: thin solid" ><th>--with-plugin-dir<th>--prefix<th>DEFAULT_PLUGIN_INSTALL_DIR<th>DEFAULT_PLUGIN_SEARCH_PATH
<tr style="outline: thin solid" ><td>undefined<td>undefined<td>undefined<td>PLATFORMDEFALT
<tr style="outline: thin solid" ><td>undefined<td>&lt;abspath-prefix&gt;<td>&lt;abspath-prefix&gt;/hdf5/lib/plugin<td>&lt;abspath-prefix&gt;/hdf5/lib/plugin&lt;SEP&gt;PLATFORMDEFALT
<tr style="outline: thin solid" ><td>&lt;abspath-plugins&gt;<td>N.A.<td>&lt;abspath-plugins&gt;<td>&lt;abspath-plugins&gt;&lt;SEP&gt;PLATFORMDEFALT
</table>
Notes:
1. HDF5_PLUGIN_PATH is ignored at build time.
2. ';' is used as a placeholder for PLATFORMSEP.
3. The term PLATFORMDEFAULT stands for:
- /usr/local/hdf5/lib/plugin If on a *nix* machine
- %ALLUSERSPROFILE%/hdf5/lib/plugins If on a windows or Mingw platform
4. The term SEP stands for:
- ':' If on a *nix* machine
- ';' If on a windows or Mingw platform
## Run-Time Operations
When the netcdf-c library initializes itself (at runtime), it chooses an
initial global plugin path for the config.h value.
This value defaults to *NETCDF_PLUGIN_SEARCH_PATH*.
If, however, HDF5_PLUGIN_PATH is defined, then it is used to override
*NETCDF_PLUGIN_SEARCH_PATH*.
<table style="border:2px solid black;border-collapse:collapse">
<tr style="outline: thin solid" align="center"><td colspan="2">Table showing the computation of the initial global plugin path</td>
<tr style="outline: thin solid"><th>HDF5_PLUGIN_PATH<th>Initial global plugin path
<tr style="outline: thin solid"><td>undefined<td>NETCDF_PLUGIN_SEARCH_PATH
<tr style="outline: thin solid"><td>&lt;path1;...pathn&gt;<td>&lt;path1;...pathn&gt;
</table>
## Multi-Threaded Access to the Plugin Path.
Specifically, note that modifying the plugin path must be done "atomically".
That is, in a multi-threaded environment, it is important that the sequence of actions involved in setting up the plugin path must be done by a single processor or in some other way as to guarantee that two or more processors are not simultaneously accessing the plugin path get/set operations.
As an example, assume there exists a mutex lock called PLUGINLOCK.
Then any processor accessing the plugin paths should operate as follows:
````
lock(PLUGINLOCK);
nc_plugin_path_get(...);
<rebuild plugin path>
nc_plugin_path_set(...);
unlock(PLUGINLOCK);
````
## Internal Architecture
It is assumed here that there only needs to be a single set of
plugin path directories that is shared by all filter code and is
independent of any file descriptor; it is global in other words.
This means, for example, that the path list for NCZarr and for
HDF5 will always be the same.
However, and internally, processing the set of plugin paths
depends on the particular NC_FORMATX value (NC_FORMATX_NC_HDF5
and NC_FORMATX_NCZARR, currently). So the *nc_plugin_path_set*
function, will take the paths it is given and propagate them to
each of the NC_FORMATX dispatchers to store in a way that is
appropriate to the given dispatcher.
There is a complication with respect to the *nc_plugin_path_get* function.
It is possible for users to bypass the netcdf API and modify the HDF5 plugin paths directly. This can result in an inconsistent plugin path between the value
used by HDF5 and the global value used by netcdf-c. Since there is no obvious fix for this, we warn the user of this possibility and otherwise ignore it.
## Appendix E.1. Programmatic Plugin Path API{#pluginpath_appendixe1}
The API makes use of a counted vector of strings representing the sequence of directories in the path. The relevant type definition is as follows.
````
typedef struct NCPluginList {size_t ndirs; char** dirs;} NCPluginList;
````
The API proposed in this PR looks like this (from netcdf-c/include/netcdf_filter.h).
* ````int nc_plugin_path_ndirs(size_t* ndirsp);````
This function returns the number of directories in the sequence if internal directories of the internal plugin path list.
The argument is as follows:
- *ndirsp* store the number of directories in this memory.
* ````int nc_plugin_path_get(NCPluginList* dirs);````
This function returns the current sequence of directories from the internal plugin path list. Since this function does not modify the plugin path, it does not need to be locked; it is only when used to get the path to be modified that locking is required.
The argument is as follows:
- *dirs* counted vector for storing the sequence of directies in the internal path list.
If the value of *dirs.dirs is NULL (the normal case), then memory is allocated to hold the vector of directories. Otherwise, use the memory of *dirs.dirs* to hold the vector of directories.
* ````int nc_plugin_path_set(const NCPluginList* dirs);````
This function empties the current internal path sequence and replaces it with the sequence of directories argument. Using an *ndirs* argument of 0 will clear the set of plugin paths.
The argument are as follows:
- *dirs* counted vector for storing the sequence of directies in the internal path list.
*HDF5\_PLUGIN\_PATH* is a typical Windows or Unix style
path-list. That is it is a sequence of absolute directory paths
separated by a specific separator character. For Windows, the
separator character is a semicolon (';') and for Unix, it is a
colon (':').
At the moment, NetCDF optionally (i.e. not overridden) uses the
existing HDF5 environment variable *HDF5\_PLUGIN\_PATH* to
locate the directories in which plugin libraries are located. It
also optionally uses the last directory in the path as the
installation directory. This is used both for the HDF5 filter
wrappers but also the NCZarr codec wrappers.
## History {#pluginpath_history}
*Author*: Dennis Heimbigner<br>
*Email*: dennis.heimbigner@gmail.com<br>
*Initial Version*: 9/28/2024<br>
*Last Revised*: 9/28/2024

6
docs/quickstart_env.md
View File

@ -1,6 +1,8 @@
Appendix D.3. Environment Variables and .RC files {#nc_env_quickstart}
Appendix G. Environment Variables and .RC files QuickStart {#nc_env_quickstart}
==============================
[TOC]
The netCDF-c library provides several parameterization mechanisms to
control its behavior at run-time. The term _run-time_ means that the
library's behavior can be changed every time the library is initialized
@ -147,7 +149,7 @@ Other keys are as follows:
* oc2/occurlfunctions.c
- HTTP.NETRC -- alternate way to specify the path of the .netrc file
## Point of Contact {#nc_env_poc}
## History {#nc_env_history}
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu<br>

12
docs/quickstart_paths.md
View File

@ -1,6 +1,8 @@
Appendix D.2. Specifying Paths for NetCDF-C {#nc_paths_quickstart}
Appendix F. Specifying Paths for NetCDF-C QuickStart {#nc_paths_quickstart}
==============================
[TOC]
A key concept in netcdf-c is the notion of a "path".
A path specifies some dataset that is of interest to a user.
It is the primary argument to the *nc_open* and *nc_create*
@ -13,10 +15,10 @@ for using the NetCDF-C library.
## Classification of Paths {#nc_paths_kinds}
Basically, there are two kinds of paths:
1. File system paths, and
2. Uniform Resource Locator (URL) paths.
1. <a href="#qpaths_filesystem">File system paths</a>, and
2. <a href="#qpaths_url">Uniform Resource Locator (URL) paths</a>.
### File System Paths
### File System Paths {#qpaths_filesystem}
The most common form of path accepted by the NetCDF-C library is a file system path.
Every user of some computer operating system is familiar with the idea of a file system path.
@ -37,7 +39,7 @@ Windows has a notion of a drive ("d:") and each drive serves as the root
of its own file system. Windows uses "\\" as its file separator, although
many programs also accept "/".
## Uniform Resource Locator (URL) Paths
## Uniform Resource Locator (URL) Paths {#qpaths_url}
The NetCDF-C library can access datasets that reside on remote computers,
Hence NetCDF-C now also accepts URLs to specify those remote datasets.

3
h5_test/Makefile.am
View File

@ -62,4 +62,5 @@ h5_err_macros.h CMakeLists.txt run_h5_zstd_tests.sh
# Clean up test results.
CLEANFILES = tst_h_*.h5
DISTCLEANFILES = run_par_test.sh
DISTCLEANFILES = run_par_test.sh findplugin.sh

29
include/CMakeLists.txt
View File

@ -41,10 +41,37 @@ IF(NETCDF_ENABLE_PNETCDF OR NETCDF_ENABLE_PARALLEL4)
COMPONENT headers)
ENDIF()
INSTALL(FILES ${netCDF_SOURCE_DIR}/include/netcdf_json.h
INSTALL(FILES ${netCDF_BINARY_DIR}/include/netcdf_json.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
COMPONENT headers)
INSTALL(FILES ${netCDF_BINARY_DIR}/include/netcdf_proplist.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
COMPONENT headers)
FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.h)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} Makefile.am CMakeLists.txt)
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")
# One time read ncextern.h
file(READ ncexternl.h NCEXTH0)
# Built source: netcdf_json.h
file(READ ncjson.h JSONH0)
STRING(REPLACE "NCJSON_H" "NETCDF_JSON_H" JSONH1 "${JSONH0}")
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/netcdf_json.h "${JSONH1}")
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_json.h "#ifdef NETCDF_JSON_H\n")
file(READ ${CMAKE_SOURCE_DIR}/libdispatch/ncjson.c JSONC0)
STRING(REGEX REPLACE "#include[ ]+\"ncjson.h\"" "" JSONC1 "${JSONC0}")
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_json.h "${JSONC1}")
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_json.h "#endif /*NETCDF_JSON_H*/\n")
# Built source: netcdf_proplist.h
file(READ ncproplist.h PLISTH0)
STRING(REPLACE "NCPROPLIST_H" "NETCDF_PROPLIST_H" PLISTH1 "${PLISTH0}")
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/netcdf_proplist.h "${PLISTH1}")
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_proplist.h "#ifdef NETCDF_PROPLIST_H\n")
file(READ ${CMAKE_SOURCE_DIR}/libdispatch/ncproplist.c PLISTC0)
STRING(REGEX REPLACE "#include[ ]+\"ncproplist.h\"" "" PLISTC1 "${PLISTC0}")
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_proplist.h "${PLISTC1}")
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_proplist.h "#endif /*NETCDF_PROPLIST_H*/\n")

37
include/Makefile.am
View File

@ -6,11 +6,13 @@
# Ed Hartnett, Dennis Heimbigner, Ward Fisher
include_HEADERS = netcdf.h netcdf_meta.h netcdf_mem.h netcdf_aux.h \
netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h netcdf_dispatch.h
# Installed headers
# Built source
include_HEADERS += netcdf_json.h
include_HEADERS = netcdf.h netcdf_meta.h netcdf_mem.h netcdf_aux.h \
netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h \
netcdf_dispatch.h
include_HEADERS += netcdf_json.h netcdf_proplist.h
if BUILD_PARALLEL
include_HEADERS += netcdf_par.h
@ -23,7 +25,7 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \
ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \
ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \
hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h \
ncjson.h ncxml.h ncs3sdk.h
ncjson.h ncxml.h ncs3sdk.h ncproplist.h ncplugins.h
if USE_DAP
noinst_HEADERS += ncdap.h
@ -35,19 +37,32 @@ endif
EXTRA_DIST = CMakeLists.txt XGetopt.h netcdf_meta.h.in netcdf_dispatch.h.in
BUILT_SOURCES = netcdf_json.h
MAINTAINERCLEANFILES = netcdf_json.h netcdf_proplist.h
BUILT_SOURCES = netcdf_json.h netcdf_proplist.h
# netcdf_json.h is constructed as a header-only file for use by
# nczarr code wrappers in the plugin directory. It is
# constructed by joining libdispatch/ncjson.c with
# include/ncjson.h. Additionally all the entry points are
# marked with a macro (OPTSTATIC) that marks the entry point as
# static inside netcdf_json.h. This is an ugly hack to avoid
# static inside netcdf_json.h. This is an ugly hack to avoid
# having to reference libnetcdf in the nczarr code wrappers.
# Note also that we incorporate the core of ncexternl.h into the netcdf_json.h file.
# Give the recipe for building netcdf_json.h
netcdf_json.h: ${top_srcdir}/libdispatch/ncjson.c ${top_srcdir}/include/ncjson.h
sed -e 's/NCJSON_H/NETCDF_JSON_H/' -e '/ncjson.h/d' -e '/#endif[^!]*!NETCDF_JSON_H!/d' <${srcdir}/ncjson.h > netcdf_json.h
netcdf_json.h: ${top_srcdir}/libdispatch/ncjson.c ${top_srcdir}/include/ncjson.h ${top_srcdir}/include/ncexternl.h
rm -fr ${srcdir}/netcdf_json.h
cat ${srcdir}/ncjson.h | sed -e 's/NCJSON_H/NETCDF_JSON_H/' >> ${srcdir}/netcdf_json.h
echo '#ifdef NETCDF_JSON_H' >> netcdf_json.h
sed -e '/ncjson.h/d' < ${srcdir}/../libdispatch/ncjson.c >> netcdf_json.h
echo '#endif /*NETCDF_JSON_H*/' >> netcdf_json.h
sed -e '/ncjson.h/d' < ${srcdir}/../libdispatch/ncjson.c >> ${srcdir}/netcdf_json.h
echo '#endif /*NETCDF_JSON_H*/' >> ${srcdir}/netcdf_json.h
# netcdf_proplist is analogous to netcdf_json but, of course, using libdispatch/ncproplist.c and include/ncproplist.h
# Give the recipe for building netcdf_proplist.h. Basically same as for netcdf_json.h
netcdf_proplist.h: ${top_srcdir}/libdispatch/ncproplist.c ${top_srcdir}/include/ncproplist.h ${top_srcdir}/include/ncexternl.h
rm -fr ${srcdir}/netcdf_proplist.h
cat ${srcdir}/ncproplist.h | sed -e 's/NCPROPLIST_H/NETCDF_PROPLIST_H/' >> ${srcdir}/netcdf_proplist.h
echo '#ifdef NETCDF_PROPLIST_H' >> netcdf_proplist.h
sed -e '/ncproplist.h/d' < ${srcdir}/../libdispatch/ncproplist.c >> ${srcdir}/netcdf_proplist.h
echo '#endif /*NETCDF_PROPLIST_H*/' >> ${srcdir}/netcdf_proplist.h

9
include/nc4internal.h
View File

@ -466,9 +466,18 @@ typedef struct NCglobalstate {
char* home; /* track $HOME */
char* cwd; /* track getcwd */
struct NCRCinfo* rcinfo; /* Currently only one rc file per session */
NClist* pluginpaths; /* Global Plugin State */
struct GlobalZarr { /* Zarr specific parameters */
char dimension_separator;
int default_zarrformat;
NClist* pluginpaths; /* NCZarr mirror of plugin paths */
NClist* codec_defaults;
NClist* default_libs;
/* All possible HDF5 filter plugins */
/* Consider onverting to linked list or hash table or
equivalent since very sparse */
struct NCZ_Plugin** loaded_plugins; //[H5Z_FILTER_MAX+1];
size_t loaded_plugins_max; /* plugin filter id index. 0<loaded_plugins_max<=H5Z_FILTER_MAX */
} zarr;
struct GlobalAWS { /* AWS S3 specific parameters/defaults */
char* default_region;

2
include/ncdispatch.h
View File

@ -35,7 +35,7 @@
/* Given a filename, check its magic number */
/* Change magic number size from 4 to 8 to be more precise for HDF5 */
#define MAGIC_NUMBER_LEN ((size_t)8)
#define MAGIC_NUMBER_LEN ((unsigned long long)8)
#define MAGIC_HDF5_FILE 1
#define MAGIC_HDF4_FILE 2
#define MAGIC_CDF1_FILE 1 /* std classic format */

37
include/ncjson.h
View File

@ -2,38 +2,20 @@
See the COPYRIGHT file for more information.
*/
#ifndef NCJSON_H
#define NCJSON_H 1
#define NCJSON_H
/*
WARNING:
If you modify this file,
then you need to got to
the include/ directory
and do the command:
make makepluginjson
*/
#if defined(DLL_NETCDF) /* define when library is a DLL */
# if defined(DLL_EXPORT) /* define when building the library */
# define MSC_EXTRA __declspec(dllexport)
# else
# define MSC_EXTRA __declspec(dllimport)
# endif
#else
# define MSC_EXTRA
#endif /* defined(DLL_NETCDF) */
#ifndef EXTERNL
# define EXTERNL MSC_EXTRA extern
#endif
/* Override for plugins */
#ifndef OPTEXPORT
#ifdef NETCDF_JSON_H
#define OPTEXPORT static
#else /*NETCDF_JSON_H*/
#define OPTEXPORT MSC_EXTRA
#else /*!NETCDF_JSON_H*/
#ifdef _WIN32
#define OPTEXPORT __declspec(dllexport)
#else
#define OPTEXPORT extern
#endif
#endif /*NETCDF_JSON_H*/
#endif /*OPTEXPORT*/
/**************************************************/
/* Json object sorts (note use of term sort rather than e.g. type or discriminant) */
@ -157,3 +139,4 @@ OPTEXPORT const char* NCJtotext(const NCjson* json);
#endif /*NCJSON_H*/

16
include/ncpathmgr.h
View File

@ -33,9 +33,9 @@ Assumptions about Input path:
2. It conforms to the format expected by one of the following:
Linux (/x/y/...),
Cygwin (/cygdrive/D/...),
Windows|MINGW (D:\...),
Windows|MINGW|MSYS (D:\...),
Windows network path (\\mathworks\...)
MSYS (/D/...),
MSYS (/D/...) but only if local platform is Windows or MSYS.
4. It is encoded in the local platform character set. Note that
for most systems, this is utf-8. But for Windows, the
encoding is most likely some form of ANSI code page, probably
@ -50,11 +50,11 @@ Parsing Rules:
2. A leading '/cygdrive/D' will be converted to
drive letter D if D is alpha-char.
3. A leading D:/... is treated as a windows drive letter
4. A leading /d/... is treated as a windows drive letter
if the platform is MSYS2.
5. A leading // is a windows network path and is converted
4. A leading // is a windows network path and is converted
to a drive letter using the fake drive letter "/".
So '//svc/x/y' translates to '/:/svc/x/y'.
5. If the platform is Windows or MSYS, then a leading /D/ is treated
as a drive letter.
6. All other cases are assumed to be Unix variants with no drive letter.
After parsing, the following pieces of information are kept in a struct.
@ -74,12 +74,13 @@ The re-write rules (unparsing) are given the above three pieces
of info + the current platform + the root mount point (if any).
The conversion rules are as follows.
Platform | No Input Driv | Input Drive
Platform | No Input Drive | Input Drive
----------------------------------------------------
NCPD_NIX | <path> | /<drive>/path
NCPD_CYGWIN | /<path> | /cygdrive/<drive>/<path>
NCPD_WIN | <mountpoint>/<path> | <drive>:<path>
NCPD_MSYS | <mountpoint>/<path> | <drive>:<path>
NCPD_MINGW | <mountpoint>/<path> | <drive>:<path>
Notes:
1. MINGW without MSYS is treated like WIN.
@ -169,6 +170,9 @@ EXTERNL int NChasdriveletter(const char* path);
because it first converts to wide character and then to utf8. */
EXTERNL int NCpath2utf8(const char* path, char** u8p);
/* Convert stdin, stdout, stderr to use binary mode (\r\n -> \n) */
EXTERNL int NCstdbinary(void);
/* Wrap various stdio and unistd IO functions.
It is especially important to use for windows so that
NCpathcvt (above) is invoked on the path.

39
include/ncplugins.h Normal file
View File

@ -0,0 +1,39 @@
/*
Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT for license information.
*/
/*
Common functionality for plugin paths/
For internal use only.
*/
#ifndef NCPLUGINS_H
#define NCPLUGINS_H
/* Opaque */
struct NCPluginList;
#if defined(__cplusplus)
extern "C" {
#endif
EXTERNL int NCZ_plugin_path_initialize(void);
EXTERNL int NCZ_plugin_path_finalize(void);
EXTERNL int NCZ_plugin_path_ndirs(size_t* ndirsp);
EXTERNL int NCZ_plugin_path_get(struct NCPluginList* dirs);
EXTERNL int NCZ_plugin_path_set(struct NCPluginList* dirs);
EXTERNL int NC4_hdf5_plugin_path_initialize(void);
EXTERNL int NC4_hdf5_plugin_path_finalize(void);
EXTERNL int NC4_hdf5_plugin_path_ndirs(size_t* ndirsp);
EXTERNL int NC4_hdf5_plugin_path_get(struct NCPluginList* dirs);
EXTERNL int NC4_hdf5_plugin_path_set(struct NCPluginList* dirs);
#if defined(__cplusplus)
}
#endif
#endif /*NCPLUGINS_H*/

108
include/ncproplist.h Normal file
View File

@ -0,0 +1,108 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* $Header$
*********************************************************************/
#ifndef NCPROPLIST_H
#define NCPROPLIST_H
#ifndef OPTEXPORT
#ifdef NETCDF_PROPLIST_H
#define OPTEXPORT static
#else /*!NETCDF_PROPLIST_H*/
#ifdef _WIN32
#define OPTEXPORT __declspec(dllexport)
#else
#define OPTEXPORT extern
#endif
#endif /*NETCDF_PROPLIST_H*/
#endif /*OPTEXPORT*/
/**************************************************/
/*
This is used to store a property list mapping a small number of
fixed-sized key strings to an arbitrary uintptr_t value. The
uintptr_t type is used to ensure that the value can be a pointer or a
small string upto sizeof(uintptr_t) - 1 (for trailing nul). The big
problem is reclaiming the value if it a pointer. The fact that the
number of keys is small makes it feasible to use linear search.
This is currently only used for plugins, but may be extended to other uses.
*/
/*! Proplist-related structs.
NOTES:
1. 'value' is the an arbitrary uintptr_t integer or void* pointer.
WARNINGS:
1. It is critical that |uintptr_t| == |void*|
*/
#define NCPROPSMAXKEY 31 /* characters assert (NCPROPSMAXKEY+1)/8 == 0*/
/* Returns 0 => error; 1 => success */
typedef int (*NCPreclaimfcn)(uintptr_t userdata, const char* key, void* value, uintptr_t size);
/* The property list proper is a sequence of these objects */
typedef struct NCProperty {
char key[NCPROPSMAXKEY+1]; /* copy of the key string; +1 for trailing nul */
uintptr_t flags;
# define NCPF_SIMPLE (1<<0) /* non-reclaimable */
# define NCPF_BYTES (1<<1) /* reclaimable bytes */
# define NCPF_COMPLEX (1<<2) /* extended case */
uintptr_t value;
uintptr_t size; /* size = |value| as ptr to memory, if string, then include trailing nul */
uintptr_t userdata; /* extra data for following functions */
NCPreclaimfcn reclaim;
} NCProperty;
/*
The property list object.
*/
typedef struct NCproplist {
size_t alloc; /* allocated space to hold properties */
size_t count; /* # of defined properties */
NCProperty* properties;
} NCproplist;
/**************************************************/
/* Extended API */
#if defined(_cplusplus_) || defined(__cplusplus__)
extern "C" {
#endif
/* Create, free, etc. */
OPTEXPORT NCproplist* ncproplistnew(void);
OPTEXPORT int ncproplistfree(NCproplist*);
/* Locate a proplist entry */
OPTEXPORT int ncproplistadd(NCproplist* plist,const char* key, uintptr_t value); /* use when reclaim not needed */
/* Insert properties */
OPTEXPORT int ncproplistadd(NCproplist* plist,const char* key, uintptr_t value); /* use when reclaim not needed */
OPTEXPORT int ncproplistaddstring(NCproplist* plist, const char* key, const char* str); /* use when value is simple string (char*) */
OPTEXPORT int ncproplistaddbytes(NCproplist* plist, const char* key, void* value, uintptr_t size); /* use when value is simple ptr and reclaim is simple free function */
OPTEXPORT int ncproplistaddx(NCproplist* plist, const char* key, void* value, uintptr_t size, uintptr_t userdata, NCPreclaimfcn); /* fully extended case */
OPTEXPORT int ncproplistclone(const NCproplist* src, NCproplist* clone);
/*
Lookup key and return value.
@return ::NC_NOERR if found ::NC_EINVAL otherwise; returns the data in datap if !null
*/
OPTEXPORT int ncproplistget(const NCproplist*, const char* key, uintptr_t* datap, uintptr_t* sizep);
/* Iteration support */
/* Return the number of properties in the property list */
#define ncproplistlen(plist) (((NCproplist)(plist))->count)
/* get the ith key+value */
OPTEXPORT int ncproplistith(const NCproplist*, size_t i, char* const * keyp, uintptr_t const * valuep, uintptr_t* sizep);
#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__)
}
#endif
#endif /*NCPROPLIST_H*/

97
include/netcdf_aux.h
View File

@ -96,9 +96,104 @@ EXTERNL int ncaux_abort_compound(void* tag);
EXTERNL int ncaux_add_field(void* tag, const char *name, nc_type field_type,
int ndims, const int* dimsizes);
/**************************************************/
/* Path-list Utilities */
/* Opaque */
struct NCPluginList;
/**
Parse a string into a sequence of path directories.
The pathlist argument has the following syntax:
paths := <empty> | dirlist
dirlist := dir | dirlist separator dir
separator := ';' | ':'
dir := <OS specific directory path>
@param pathlist a string encoding a list of directories
@param sep one of ';' | ':' | '\0' where '\0' means use the platform's default separator.
@param dirs a pointer to an NCPluginPath object for returning the number and vector of directories from the parse.
@return ::NC_NOERR | NC_EXXX
Note: If dirs->dirs is not NULL, then this function
will allocate the space for the vector of directory path.
The user is then responsible for free'ing that vector
(or call ncaux_plugin_path_reclaim).
Author: Dennis Heimbigner
*/
EXTERNL int ncaux_plugin_path_parse(const char* pathlist, char sep, struct NCPluginList* dirs);
/**
Concatenate a vector of directories with the separator between.
This is more-or-less the inverse of the ncaux_plugin_path_parse function
The resulting string has following syntax:
paths := <empty> | dirlist
dirlist := dir | dirlist separator dir
separator := ';' | ':'
dir := <OS specific directory path>
@param dirs a pointer to an NCPluginList object giving the number and vector of directories to concatenate.
@param sep one of ';', ':', or '\0'
@param catlen length of the cat arg including a nul terminator
@param cat user provided space for holding the concatenation; nul termination guaranteed if catlen > 0.
@return ::NC_NOERR
@return ::NC_EINVAL for illegal arguments
Note: If dirs->dirs is not NULL, then this function
will allocate the space for the vector of directory path.
The user is then responsible for free'ing that vector
(or call ncaux_plugin_path_reclaim).
Author: Dennis Heimbigner
*/
EXTERNL int ncaux_plugin_path_tostring(const struct NCPluginList* dirs, char sep, char** catp);
/*
Clear the contents of a NCPluginList object.
@param dirs a pointer to an NCPluginList object giving the number and vector of directories to reclaim
@return ::NC_NOERR
@return ::NC_EINVAL for illegal arguments
Author: Dennis Heimbigner
*/
EXTERNL int ncaux_plugin_path_clear(struct NCPluginList* dirs);
/*
Reclaim a NCPluginList object possibly produced by ncaux_plugin_parse function.
WARNING: do not call with a static or stack allocated object.
@param dirs a pointer to an NCPluginList object giving the number and vector of directories to reclaim
@return ::NC_NOERR
@return ::NC_EINVAL for illegal arguments
Author: Dennis Heimbigner
*/
EXTERNL int ncaux_plugin_path_reclaim(struct NCPluginList* dirs);
/*
Modify a plugin path set to append a new directory to the end.
@param dirs a pointer to an NCPluginList object giving the number and vector of directories to which 'dir' argument is appended.
@return ::NC_NOERR
@return ::NC_EINVAL for illegal arguments
Author: Dennis Heimbigner
*/
EXTERNL int ncaux_plugin_path_append(struct NCPluginList* dirs, const char* dir);
/*
Modify a plugin path set to prepend a new directory to the front.
@param dirs a pointer to an NCPluginList object giving the number and vector of directories to which 'dir' argument is appended.
@return ::NC_NOERR
@return ::NC_EINVAL for illegal arguments
Author: Dennis Heimbigner
*/
EXTERNL int ncaux_plugin_path_prepend(struct NCPluginList* dirs, const char* dir);
#if defined(__cplusplus)
}
#endif
#endif /*NCAUX_H*/

63
include/netcdf_filter.h
View File

@ -12,8 +12,8 @@
#ifndef NETCDF_FILTER_H
#define NETCDF_FILTER_H 1
/* API for libdispatch/dfilter.c
*/
/**************************************************/
/* API for libdispatch/dfilter.c */
/* Must match values in <H5Zpublic.h> */
#ifndef H5Z_FILTER_DEFLATE
@ -110,9 +110,68 @@ EXTERNL int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *leve
EXTERNL int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle);
EXTERNL int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep);
/* Filter path query/set */
EXTERNL int nc_filter_path_query(int id);
#if defined(__cplusplus)
}
#endif
/**************************************************/
/* API for libdispatch/dplugin.c */
/* Combine the vector of directory path plus it's length in a single struct. */
typedef struct NCPluginList {
size_t ndirs; /* |dirs| */
char** dirs;
} NCPluginList;
/* Externally visible plugin path functions */
#if defined(__cplusplus)
extern "C" {
#endif
/**
* Return the length of the current sequence of directories
* in the internal global plugin path list.
* @param ndirsp length is returned here
* @return NC_NOERR | NC_EXXX
*
* @author Dennis Heimbigner
*/
EXTERNL int nc_plugin_path_ndirs(size_t* ndirsp);
/**
* Return the current sequence of directories in the internal global
* plugin path list. Since this function does not modify the plugin path,
* it can be called at any time.
* @param dirs pointer to an NCPluginList object
* @return NC_NOERR | NC_EXXX
* @author Dennis Heimbigner
*
* WARNING: if dirs->dirs is NULL, then space for the directory
* vector will be allocated. If not NULL, then the specified space will
* be overwritten with the vector.
*
* @author: Dennis Heimbigner
*/
EXTERNL int nc_plugin_path_get(NCPluginList* dirs);
/**
* Empty the current internal path sequence
* and replace with the sequence of directories argument.
*
* Using a paths argument of NULL or npaths argument of 0 will clear the set of plugin paths.
* @param dirs pointer to an NCPluginList object containing
* the number and vector of directory paths
* @return NC_NOERR | NC_EXXX
*
* @author Dennis Heimbigner
*/
EXTERNL int nc_plugin_path_set(NCPluginList* dirs);
#if defined(__cplusplus)
}
#endif
#endif /* NETCDF_FILTER_H */

1310
include/netcdf_json.h
View File

File diff suppressed because it is too large Load Diff

3
libdispatch/CMakeLists.txt
View File

@ -11,6 +11,7 @@ target_sources(dispatch
dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dpathmgr.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c
daux.c dinstance.c dinstance_intern.c
dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c ncjson.c ds3util.c dparallel.c dmissing.c
ncproplist.c
)
if (NETCDF_ENABLE_DLL)
@ -33,7 +34,7 @@ set_property(SOURCE dinstance_intern.c dinstance.c dvarput.c
# Netcdf-4 only functions. Must be defined even if not used
target_sources(dispatch
PRIVATE
dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c
dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c dplugins.c
)
if(BUILD_V2)

5
libdispatch/Makefile.am
View File

@ -21,7 +21,8 @@ dinternal.c ddispatch.c dutf8.c nclog.c dstring.c ncuri.c nclist.c \
ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c dauth.c doffsets.c \
dpathmgr.c dutil.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c \
daux.c dinstance.c dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c \
ncjson.c ds3util.c dparallel.c dmissing.c dinstance_intern.c
ncjson.c ds3util.c dparallel.c dmissing.c dinstance_intern.c \
ncproplist.c
# Add the utf8 codebase
libdispatch_la_SOURCES += utf8proc.c utf8proc.h
@ -34,7 +35,7 @@ libdispatch_la_SOURCES += drc.c
# Add functions only found in netCDF-4.
# They are always defined, even if they just return an error
libdispatch_la_SOURCES += dgroup.c dvlen.c dcompound.c dtype.c denum.c \
dopaque.c dfilter.c
dopaque.c dfilter.c dplugins.c
# Add V2 API convenience library if needed.
if BUILD_V2

215
libdispatch/daux.c
View File

@ -30,6 +30,7 @@ See COPYRIGHT for license information.
#include "nclog.h"
#include "ncrc.h"
#include "netcdf_filter.h"
#include "ncpathmgr.h"
struct NCAUX_FIELD {
char* name;
@ -57,6 +58,7 @@ static int computefieldinfo(struct NCAUX_CMPD* cmpd);
static int filterspec_cvt(const char* txt, size_t* nparamsp, unsigned int* params);
EXTERNL int nc_dump_data(int ncid, nc_type xtype, void* memory, size_t count, char** bufp);
EXTERNL int nc_parse_plugin_pathlist(const char* path0, NClist* dirlist);
/**************************************************/
/*
@ -615,7 +617,7 @@ ncaux_h5filterspec_parselist(const char* txt0, int* formatp, size_t* nspecsp, NC
done:
nullfree(spec);
if(vector) {
int i;
size_t i;
for(i=0;i<nspecs;i++)
ncaux_h5filterspec_free(vector[i]);
nullfree(vector);
@ -955,3 +957,214 @@ ncaux_dump_data(int ncid, int xtype, void* memory, size_t count, char** bufp)
return nc_dump_data(ncid, xtype, memory, count, bufp);
}
/**************************************************/
/* Path List Utilities */
/* Path-list Parser:
@param pathlist0 the string to parse
@param dirs return the parsed directories -- see note below.
@param sep the separator parsing: one of ';' | ':' | '\0', where zero means use platform default.
@return NC_NOERR || NC_EXXX
Note: If dirs->dirs is not NULL, then this function
will allocate the space for the vector of directory path.
The user is then responsible for free'ing that vector
(or call ncaux_plugin_path_reclaim).
*/
EXTERNL int
ncaux_plugin_path_parse(const char* pathlist0, char sep, NCPluginList* dirs)
{
int stat = NC_NOERR;
size_t i;
char* path = NULL;
char* p;
size_t count;
size_t plen;
char seps[2] = "\0\0"; /* will contain all allowable separators */
if(dirs == NULL) {stat = NC_EINVAL; goto done;}
if(pathlist0 == NULL || pathlist0[0] == '\0') {dirs->ndirs = 0; goto done;}
/* If a separator is specified, use it, otherwise search for ';' or ':' */
seps[0] = sep;
if(sep == 0) {
if(NCgetlocalpathkind() == NCPD_WIN
|| NCgetlocalpathkind() == NCPD_MSYS)
seps[0] = ';';
else
seps[0] = ':';
}
plen = strlen(pathlist0); /* assert plen > 0 */
if((path = malloc(plen+1+1))==NULL) {stat = NC_ENOMEM; goto done;}
memcpy(path,pathlist0,plen);
path[plen] = '\0'; path[plen+1] = '\0'; /* double null term */
for(count=0,p=path;*p;p++) {
if(strchr(seps,*p) == NULL)
continue; /* non-separator */
else {
*p = '\0';
count++;
}
}
count++; /* count last piece */
/* Save and allocate */
dirs->ndirs = count;
if(dirs->dirs == NULL) {
if((dirs->dirs = (char**)calloc(count,sizeof(char*)))==NULL)
{stat = NC_ENOMEM; goto done;}
}
/* capture the parsed pieces */
for(p=path,i=0;i<count;i++) {
size_t len = strlen(p);
dirs->dirs[i] = strdup(p);
p = p+len+1; /* point to next piece */
}
done:
nullfree(path);
return stat;
}
/*
Path-list concatenator where given a vector of directories,
concatenate all dirs with specified separator.
If the separator is 0, then use the default platform separator.
@param dirs the counted directory vector to concatenate
@param sep one of ';', ':', or '\0'
@param catp return the concatenation; WARNING: caller frees.
@return ::NC_NOERR
@return ::NC_EINVAL for illegal arguments
*/
EXTERNL int
ncaux_plugin_path_tostring(const NCPluginList* dirs, char sep, char** catp)
{
int stat = NC_NOERR;
NCbytes* buf = ncbytesnew();
size_t i;
if(dirs == NULL) {stat = NC_EINVAL; goto done;}
if(dirs->ndirs > 0 && dirs->dirs == NULL) {stat = NC_EINVAL; goto done;}
if(sep == '\0')
#ifdef _WIN32
sep = ';';
#else
sep = ':';
#endif
if(dirs->ndirs > 0) {
for(i=0;i<dirs->ndirs;i++) {
if(i>0) ncbytesappend(buf,sep);
if(dirs->dirs[i] != NULL) ncbytescat(buf,dirs->dirs[i]);
}
}
ncbytesnull(buf);
if(catp) *catp = ncbytesextract(buf);
done:
ncbytesfree(buf);
return stat;
}
/*
Clear an NCPluginList object possibly produced by ncaux_plugin_parse function.
@param dirs the object to clear
@return ::NC_NOERR
@return ::NC_EINVAL for illegal arguments
*/
EXTERNL int
ncaux_plugin_path_clear(NCPluginList* dirs)
{
int stat = NC_NOERR;
size_t i;
if(dirs == NULL || dirs->ndirs == 0 || dirs->dirs == NULL) goto done;
for(i=0;i<dirs->ndirs;i++) {
if(dirs->dirs[i] != NULL) free(dirs->dirs[i]);
dirs->dirs[i] = NULL;
}
free(dirs->dirs);
dirs->dirs = NULL;
dirs->ndirs = 0;
done:
return stat;
}
/*
Reclaim an NCPluginList object.
@param dir the object to reclaim
@return ::NC_NOERR
@return ::NC_EINVAL for illegal arguments
*/
EXTERNL int
ncaux_plugin_path_reclaim(NCPluginList* dirs)
{
int stat = NC_NOERR;
if((stat = ncaux_plugin_path_clear(dirs))) goto done;
nullfree(dirs);
done:
return stat;
}
/*
Modify a plugin path set to append a new directory to the end.
@param dirs a pointer to an NCPluginPath object giving the number and vector of directories to which 'dir' argument is appended.
@return ::NC_NOERR
@return ::NC_EINVAL for illegal arguments
WARNING: dirs->dirs may be reallocated.
Author: Dennis Heimbigner
*/
EXTERNL int
ncaux_plugin_path_append(NCPluginList* dirs, const char* dir)
{
int stat = NC_NOERR;
char** newdirs = NULL;
char** olddirs = NULL;
if(dirs == NULL || dir == NULL) {stat = NC_EINVAL; goto done;}
olddirs = dirs->dirs; dirs->dirs = NULL;
if((newdirs = (char**)calloc(dirs->ndirs+1,sizeof(char*)))==NULL)
{stat = NC_ENOMEM; goto done;}
if(dirs->ndirs > 0)
memcpy(newdirs,olddirs,sizeof(char*)*dirs->ndirs);
nullfree(olddirs);
dirs->dirs = newdirs; newdirs = NULL;
dirs->dirs[dirs->ndirs] = nulldup(dir);
dirs->ndirs++;
done:
return stat;
}
/*
Modify a plugin path set to prepend a new directory to the front.
@param dirs a pointer to an NCPluginList object giving the number and vector of directories to which 'dir' argument is appended.
@return ::NC_NOERR
@return ::NC_EINVAL for illegal arguments
WARNING: dirs->dirs may be reallocated.
Author: Dennis Heimbigner
*/
EXTERNL int
ncaux_plugin_path_prepend(struct NCPluginList* dirs, const char* dir)
{
int stat = NC_NOERR;
char** newdirs = NULL;
char** olddirs = NULL;
if(dirs == NULL || dir == NULL) {stat = NC_EINVAL; goto done;}
olddirs = dirs->dirs; dirs->dirs = NULL;
if((newdirs = (char**)calloc(dirs->ndirs+1,sizeof(char*)))==NULL)
{stat = NC_ENOMEM; goto done;}
if(dirs->ndirs > 0)
memcpy(&newdirs[1],olddirs,sizeof(char*)*dirs->ndirs);
nullfree(olddirs);
dirs->dirs = newdirs; newdirs = NULL;
dirs->dirs[0] = nulldup(dir);
dirs->ndirs++;
done:
return stat;
}

11
libdispatch/dcopy.c
View File

@ -59,7 +59,8 @@ NC_compare_nc_types(int ncid1, int typeid1, int ncid2, int typeid2, int *equalp)
}
else
{
int i, ret, equal1;
size_t i;
int ret, equal1;
char name1[NC_MAX_NAME];
char name2[NC_MAX_NAME];
size_t size1, size2;
@ -283,7 +284,9 @@ nc_copy_var(int ncid_in, int varid_in, int ncid_out)
char name[NC_MAX_NAME + 1];
char att_name[NC_MAX_NAME + 1];
nc_type xtype;
int ndims, dimids_in[NC_MAX_VAR_DIMS], dimids_out[NC_MAX_VAR_DIMS], natts, real_ndims;
int ndims;
int dimids_in[NC_MAX_VAR_DIMS], dimids_out[NC_MAX_VAR_DIMS];
int natts, real_ndims;
int varid_out;
int a, d;
void *data = NULL;
@ -295,14 +298,14 @@ nc_copy_var(int ncid_in, int varid_in, int ncid_out)
int src_format, dest_format;
char type_name[NC_MAX_NAME+1];
char dimname_in[NC_MAX_NAME + 1];
int i;
size_t i;
/* Learn about this var. */
if ((retval = nc_inq_var(ncid_in, varid_in, name, &xtype,
&ndims, dimids_in, &natts)))
return retval;
/* find corresponding dimids in the output file */
for(i = 0; i < ndims; i++) {
for(i = 0; i < (size_t)ndims; i++) {
dimids_out[i] = dimids_in[i];
if ((retval = nc_inq_dimname(ncid_in, dimids_in[i], dimname_in)))
return retval;

3
libdispatch/ddispatch.c
View File

@ -154,7 +154,7 @@ static int NC_createglobalstate(void);
/** \defgroup global_state Global state functions. */
/** \{
\ingroup global state
\ingroup global_state
*/
/* NCglobal state management */
@ -216,6 +216,7 @@ NC_freeglobalstate(void)
NC_rcclear(nc_globalstate->rcinfo);
free(nc_globalstate->rcinfo);
}
nclistfree(nc_globalstate->pluginpaths);
free(nc_globalstate);
nc_globalstate = NULL;
}

8
libdispatch/dfile.c
View File

@ -1748,6 +1748,13 @@ nc_inq_type(int ncid, nc_type xtype, char *name, size_t *size)
return ncp->dispatch->inq_type(ncid,xtype,name,size);
}
/** \defgroup dispatch dispatch functions. */
/** \{
\ingroup dispatch
*/
/**
Check the create mode parameter for sanity.
@ -2214,3 +2221,4 @@ nc__pseudofd(void)
}
return pseudofd++;
}
/** \} */

5
libdispatch/dinfermodel.c
View File

@ -1284,7 +1284,7 @@ check_file_type(const char *path, int omode, int use_parallel,
if((status = openmagic(&magicinfo))) goto done;
/* Verify we have a large enough file */
if(magicinfo.filelen < (unsigned long long)MAGIC_NUMBER_LEN)
if(MAGIC_NUMBER_LEN >= (unsigned long long)magicinfo.filelen)
{status = NC_ENOTNC; goto done;}
if((status = readmagic(&magicinfo,0L,magic)) != NC_NOERR) {
status = NC_ENOTNC;
@ -1306,7 +1306,7 @@ check_file_type(const char *path, int omode, int use_parallel,
{
size_t pos = 512L;
for(;;) {
if((pos+MAGIC_NUMBER_LEN) > magicinfo.filelen)
if((pos+MAGIC_NUMBER_LEN) > (unsigned long long)magicinfo.filelen)
{status = NC_ENOTNC; goto done;}
if((status = readmagic(&magicinfo,pos,magic)) != NC_NOERR)
{status = NC_ENOTNC; goto done; }
@ -1642,7 +1642,6 @@ isdaoscontainer(const char* path)
#else
(void)getxattr(path, p, xvalue, (size_t)xlen);
#endif
fprintf(stderr,"@@@ %s=|%s|\n",p,xvalue);
/* Look for '.daos' in the value */
if(strstr(xvalue,".daos") != NULL) {rc = 1; break;} /* success */
}

5
libdispatch/dinstance.c
View File

@ -484,17 +484,18 @@ static int
dump_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* offset, NCbytes* buf)
{
int stat = NC_NOERR;
size_t i;
int i;
ptrdiff_t saveoffset;
int ndims;
int dimsizes[NC_MAX_VAR_DIMS];
int fid;
saveoffset = offset->offset;
ncbytescat(buf,"<");
/* Get info about each field in turn and dump it */
for(int fid=0;fid<nfields;fid++) {
for(fid=0;fid<(int)nfields;fid++) {
size_t fieldalignment;
nc_type fieldtype;
char name[NC_MAX_NAME];

2
libdispatch/dnotnc4.c
View File

@ -94,6 +94,7 @@ NC_NOOP_inq_var_filter_ids(int ncid, int varid, size_t* nfilters, unsigned int*
int
NC_NOOP_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params)
{
NC_UNUSED(ncid);
return NC_ENOFILTER;
}
@ -101,6 +102,7 @@ int
NC_NOOP_inq_filter_avail(int ncid, unsigned id)
{
NC_UNUSED(ncid);
NC_UNUSED(id);
return NC_ENOFILTER;
}

69
libdispatch/dpathmgr.c
View File

@ -104,11 +104,11 @@ static struct MountPoint {
char drive;
} mountpoint;
/* Pick the target kind for testing */
static int testkind = 0;
/* Pick the platform kind for testing */
static int platform = 0;
static int parsepath(const char* inpath, struct Path* path);
static int unparsepath(struct Path* p, char** pathp, int target);
static int unparsepath(struct Path* p, char** pathp, int platform);
static int getwdpath(void);
static void clearPath(struct Path* path);
static void pathinit(void);
@ -136,7 +136,7 @@ NCpathcvt(const char* inpath)
char* tmp1 = NULL;
char* result = NULL;
struct Path inparsed = empty;
int target = NCgetlocalpathkind();
int platform= NCgetlocalpathkind();
if(inpath == NULL) goto done; /* defensive driving */
@ -152,7 +152,7 @@ NCpathcvt(const char* inpath)
if(pathdebug > 0)
fprintf(stderr,">>> NCpathcvt: inparsed=%s\n",printPATH(&inparsed));
if((stat = unparsepath(&inparsed,&result,target)))
if((stat = unparsepath(&inparsed,&result,platform)))
{REPORT(stat,"NCpathcvt: unparsepath"); goto done;}
done:
@ -268,7 +268,7 @@ NCpathcvt_test(const char* inpath, int ukind, int udrive)
mountpoint.drive = (char)udrive;
mountpoint.defined = (mountpoint.drive || nulllen(mountpoint.prefix) > 0);
testkind = ukind;
platform = ukind;
result = NCpathcvt(inpath);
mountpoint = old;
return result;
@ -721,6 +721,19 @@ NCpath2utf8(const char* s, char** u8p)
return ansi2utf8(s,u8p);
}
int
NCstdbinary(void)
{
int fd;
fd = _fileno(stdin);
if(_setmode(fd,_O_BINARY)<0) return NC_EINVAL;
fd = _fileno(stdout);
if(_setmode(fd,_O_BINARY)<0) return NC_EINVAL;
fd = _fileno(stderr);
if(_setmode(fd,_O_BINARY)<0) return NC_EINVAL;
return NC_NOERR;
}
#else /*!WINPATH*/
int
@ -736,6 +749,13 @@ NCpath2utf8(const char* path, char** u8p)
done:
return stat;
}
int
NCstdbinary(void)
{
return NC_NOERR;
}
#endif /*!WINPATH*/
EXTERNL int
@ -781,6 +801,7 @@ parsepath(const char* inpath, struct Path* path)
char* tmp1 = NULL;
size_t len;
char* p;
int platform = NCgetlocalpathkind();
assert(path);
memset(path,0,sizeof(struct Path));
@ -851,13 +872,17 @@ parsepath(const char* inpath, struct Path* path)
{stat = NC_ENOMEM; goto done;}
path->kind = NCPD_WIN; /* Might be MINGW */
}
#if 0
/* X. look for MSYS2 path /D/... */
else if(len >= 2
/* The /D/x/y/z MSYS paths cause much parsing confusion.
So only use it if the current platform is windows of mingw.
Otherwise use windows paths
*/
/* X. look for MSYS path /D/... */
else if((platform== NCPD_WIN || platform == NCPD_MSYS)
&& len >= 2
&& (tmp1[0] == '/')
&& strchr(windrive,tmp1[1]) != NULL
&& (tmp1[2] == '/' || tmp1[2] == '\0')) {
/* Assume this is an MSYS2 path */
/* Assume this is an MSYS path */
path->drive = tmp1[1];
/* Remainder */
if(tmp1[2] == '\0')
@ -868,7 +893,6 @@ parsepath(const char* inpath, struct Path* path)
{stat = NC_ENOMEM; goto done;}
path->kind = NCPD_MSYS;
}
#endif
/* 5. look for *nix* path; note this includes MSYS2 paths as well */
else if(len >= 1 && tmp1[0] == '/') {
/* Assume this is a *nix path */
@ -888,7 +912,7 @@ done:
}
static int
unparsepath(struct Path* xp, char** pathp, int target)
unparsepath(struct Path* xp, char** pathp, int platform)
{
int stat = NC_NOERR;
size_t len;
@ -902,7 +926,7 @@ unparsepath(struct Path* xp, char** pathp, int target)
if(xp->kind == NCPD_REL) {
/* Pass thru relative paths, but with proper slashes */
if((path = strdup(xp->path))==NULL) stat = NC_ENOMEM;
if(target == NCPD_WIN || target == NCPD_MSYS) {
if(platform == NCPD_WIN || platform == NCPD_MSYS) {
char* p;
for(p=path;*p;p++) {if(*p == '/') *p = '\\';} /* back slash*/
}
@ -910,12 +934,12 @@ unparsepath(struct Path* xp, char** pathp, int target)
}
/* We need a two level switch with an arm
for every pair of (xp->kind,target)
for every pair of (xp->kind,platform)
*/
#define CASE(k,t) case ((k)*10+(t))
switch (xp->kind*10 + target) {
switch (xp->kind*10 + platform) {
CASE(NCPD_NIX,NCPD_NIX):
assert(xp->drive == 0);
len = nulllen(xp->path)+1;
@ -973,13 +997,20 @@ unparsepath(struct Path* xp, char** pathp, int target)
CASE(NCPD_CYGWIN,NCPD_NIX):
len = nulllen(xp->path);
if(xp->drive != 0)
len += (cdlen + 2); /* /cygdrive/D */
len++;
len += (cdlen + 1); /* strlen("/cygdrive/D") */
len++; /* nul term */
if((path = (char*)malloc(len))==NULL)
{stat = NCTHROW(NC_ENOMEM); goto done;}
path[0] = '\0';
if(xp->drive != 0) {
/* There is no good/standard way to map a windows
drive letter to a *nix* path.*/
#if 0
strlcat(path,"/cygdrive/",len);
#else
/* so, just use "/D" where D is the drive letter */
strlcat(path,"/",len);
#endif
sdrive[0] = xp->drive; sdrive[1] = '\0';
strlcat(path,sdrive,len);
}
@ -1090,7 +1121,7 @@ unparsepath(struct Path* xp, char** pathp, int target)
}
if(pathdebug > 0)
fprintf(stderr,">>> unparse: target=%s xp=%s path=|%s|\n",NCgetkindname(target),printPATH(xp),path);
fprintf(stderr,">>> unparse: platform=%s xp=%s path=|%s|\n",NCgetkindname(platform),printPATH(xp),path);
exit:
if(pathp) {*pathp = path; path = NULL;}
@ -1149,7 +1180,7 @@ int
NCgetlocalpathkind(void)
{
int kind = NCPD_UNKNOWN;
if(testkind) return testkind;
if(platform) return platform;
#ifdef __CYGWIN__
kind = NCPD_CYGWIN;
#elif defined _MSC_VER /* not _WIN32 */

338
libdispatch/dplugins.c Normal file
View File

@ -0,0 +1,338 @@
/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
/**************************************************/
/* Global state plugin path implementation */
/**
* @file
* Functions for working with plugins.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef _MSC_VER
#include <io.h>
#endif
#include "netcdf.h"
#include "netcdf_filter.h"
#include "ncdispatch.h"
#include "nc4internal.h"
#include "nclog.h"
#include "ncbytes.h"
#include "ncplugins.h"
#include "netcdf_aux.h"
/*
Unified plugin related code
*/
/**************************************************/
/* Plugin-path API */
/* list of environment variables to check for plugin roots */
#define PLUGIN_ENV "HDF5_PLUGIN_PATH"
/* Control path verification */
#define PLUGINPATHVERIFY "NC_PLUGIN_PATH_VERIFY"
/*Forward*/
static int buildinitialpluginpath(NCPluginList* dirs);
static int NC_plugin_path_initialized = 0;
static int NC_plugin_path_verify = 1;
/**
* This function is called as part of nc_initialize.
* Its purpose is to initialize the plugin paths state.
*
* @return NC_NOERR
*
* @author Dennis Heimbigner
*/
EXTERNL int
nc_plugin_path_initialize(void)
{
int stat = NC_NOERR;
struct NCglobalstate* gs = NULL;
NCPluginList dirs = {0,NULL};
int hdf5found = 0; /* 1 => we got a legit plugin path set from HDF5 */
if(!NC_initialized) nc_initialize();
if(NC_plugin_path_initialized != 0) goto done;
NC_plugin_path_initialized = 1;
if(getenv(PLUGINPATHVERIFY) != NULL) NC_plugin_path_verify = 1;
gs = NC_getglobalstate();
/**
* When the netcdf-c library initializes itself (at runtime), it chooses an
* initial global plugin path using the following rules, which are those used
* by the HDF5 library, except as modified for plugin install (which HDF5 does not support).
*/
/* Initialize the implementations */
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
if((stat = NCZ_plugin_path_initialize())) goto done;
#endif
#ifdef USE_HDF5
if((stat = NC4_hdf5_plugin_path_initialize())) goto done;
#endif
/* Compute the initial global plugin path */
assert(dirs.ndirs == 0 && dirs.dirs == NULL);
if((stat = buildinitialpluginpath(&dirs))) goto done; /* Construct a default */
/* Sync to the actual implementations */
#ifdef USE_HDF5
if(!hdf5found)
{if((stat = NC4_hdf5_plugin_path_set(&dirs))) goto done;}
#endif
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
if((stat = NCZ_plugin_path_set(&dirs))) goto done;
#endif
/* Set the global plugin dirs sequence */
assert(gs->pluginpaths == NULL);
gs->pluginpaths = nclistnew();
if(dirs.ndirs > 0) {
size_t i;
char** dst;
nclistsetlength(gs->pluginpaths,dirs.ndirs);
dst = (char**)nclistcontents(gs->pluginpaths);
assert(dst != NULL);
for(i=0;i<dirs.ndirs;i++)
dst[i] = strdup(dirs.dirs[i]);
}
done:
ncaux_plugin_path_clear(&dirs);
return NCTHROW(stat);
}
/**
* This function is called as part of nc_finalize()
* Its purpose is to clean-up plugin path state.
*
* @return NC_NOERR
*
* @author Dennis Heimbigner
*/
int
nc_plugin_path_finalize(void)
{
int stat = NC_NOERR;
struct NCglobalstate* gs = NC_getglobalstate();
if(NC_plugin_path_initialized == 0) goto done;
NC_plugin_path_initialized = 0;
NC_plugin_path_verify = 0;
/* Finalize the actual implementatios */
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
if((stat = NCZ_plugin_path_finalize())) goto done;
#endif
#ifdef USE_HDF5
if((stat = NC4_hdf5_plugin_path_finalize())) goto done;
#endif
nclistfreeall(gs->pluginpaths); gs->pluginpaths = NULL;
done:
return NCTHROW(stat);
}
/**
* Return the length of the current sequence of directories
* in the internal global plugin path list.
* @param ndirsp length is returned here
* @return NC_NOERR | NC_EXXX
*
* @author Dennis Heimbigner
*/
int
nc_plugin_path_ndirs(size_t* ndirsp)
{
int stat = NC_NOERR;
size_t ndirs = 0;
struct NCglobalstate* gs = NC_getglobalstate();
if(gs->pluginpaths == NULL) gs->pluginpaths = nclistnew(); /* suspenders and belt */
ndirs = nclistlength(gs->pluginpaths);
/* Verify that the implementation plugin paths are consistent in length*/
if(NC_plugin_path_verify) {
#ifdef NETCDF_ENABLE_HDF5
{
size_t ndirs5 = 0;
if((stat=NC4_hdf5_plugin_path_ndirs(&ndirs5))) goto done;
assert(ndirs5 == ndirs);
}
#endif /*NETCDF_ENABLE_HDF5*/
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
{
size_t ndirsz = 0;
if((stat=NCZ_plugin_path_ndirs(&ndirsz))) goto done;
assert(ndirsz == ndirs);
}
#endif /*NETCDF_ENABLE_NCZARR_FILTERS*/
}
if(ndirsp) *ndirsp = ndirs;
done:
return NCTHROW(stat);
}
/**
* Return the current sequence of directories in the internal global
* plugin path list. Since this function does not modify the plugin path,
* it can be called at any time.
* @param dirs pointer to an NCPluginList object
* @return NC_NOERR | NC_EXXX
* @author Dennis Heimbigner
*
* WARNING: if dirs->dirs is NULL, then space for the directory
* vector will be allocated. If not NULL, then the specified space will
* be overwritten with the vector.
*
* @author: Dennis Heimbigner
*/
int
nc_plugin_path_get(NCPluginList* dirs)
{
int stat = NC_NOERR;
struct NCglobalstate* gs = NC_getglobalstate();
size_t i;
if(gs->pluginpaths == NULL) gs->pluginpaths = nclistnew(); /* suspenders and belt */
if(dirs == NULL) goto done;
dirs->ndirs = nclistlength(gs->pluginpaths);
if(dirs->ndirs > 0 && dirs->dirs == NULL) {
if((dirs->dirs = (char**)calloc(dirs->ndirs,sizeof(char*)))==NULL)
{stat = NC_ENOMEM; goto done;}
}
for(i=0;i<dirs->ndirs;i++) {
const char* dir = nclistget(gs->pluginpaths,i);
dirs->dirs[i] = nulldup(dir);
}
/* Verify that the implementation plugin paths are consistent */
if(NC_plugin_path_verify) {
#ifdef NETCDF_ENABLE_HDF5
{
size_t i;
NCPluginList l5 = {0,NULL};
if((stat=NC4_hdf5_plugin_path_get(&l5))) goto done;
assert(l5.ndirs == nclistlength(gs->pluginpaths));
for(i=0;i<l5.ndirs;i++) {
assert(strcmp(dirs->dirs[i],l5.dirs[i])==0);
nullfree(l5.dirs[i]);
}
nullfree(l5.dirs);
}
#endif /*NETCDF_ENABLE_HDF5*/
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
{
size_t i;
NCPluginList lz = {0,NULL};
if((stat=NCZ_plugin_path_get(&lz))) goto done;
assert(lz.ndirs == nclistlength(gs->pluginpaths));
for(i=0;i<lz.ndirs;i++) {
assert(strcmp(dirs->dirs[i],lz.dirs[i])==0);
nullfree(lz.dirs[i]);
}
nullfree(lz.dirs);
}
#endif /*NETCDF_ENABLE_NCZARR_FILTERS*/
}
done:
return NCTHROW(stat);
}
/**
* Empty the current internal path sequence
* and replace with the sequence of directories argument.
* Using a dirs->ndirs argument of 0 will clear the set of plugin dirs.
*
* @param dirs to overwrite the current internal dir list
* @return NC_NOERR | NC_EXXX
*
* @author Dennis Heimbigner
*/
int
nc_plugin_path_set(NCPluginList* dirs)
{
int stat = NC_NOERR;
struct NCglobalstate* gs = NC_getglobalstate();
if(dirs == NULL) {stat = NC_EINVAL; goto done;}
/* Clear the current dir list */
nclistfreeall(gs->pluginpaths);
gs->pluginpaths = nclistnew();
if(dirs->ndirs > 0) {
size_t i;
assert(gs->pluginpaths != NULL);
for(i=0;i<dirs->ndirs;i++) {
nclistpush(gs->pluginpaths,nulldup(dirs->dirs[i]));
}
}
/* Sync the global plugin path set to the individual implementations */
#ifdef NETCDF_ENABLE_HDF5
if((stat = NC4_hdf5_plugin_path_set(dirs))) goto done;
#endif
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
if((stat = NCZ_plugin_path_set(dirs))) goto done;
#endif
done:
return NCTHROW(stat);
}
/**
* When the netcdf-c library initializes itself (at runtime), it chooses an
* initial global plugin path using the following rules, which are those used
* by the HDF5 library, except as modified for plugin install (which HDF5 does not support).
*
* Note: In the following, PLATFORMPATH is:
* -- /usr/local/lhdf5/lib/plugin if platform is *nix*
* -- %ALLUSERSPROFILE%/hdf5/lib/plugin if platform is Windows or Mingw
* and in the following, PLATFORMSEP is:
* -- ":" if platform is *nix*
* -- ";" if platform is Windows or Mingw
* and
* NETCDF_PLUGIN_SEARCH_PATH is the value constructed at build-time (see configure.ac)
*
* Table showing the computation of the initial global plugin path
* =================================================
* | HDF5_PLUGIN_PATH | Initial global plugin path |
* =================================================
* | undefined | NETCDF_PLUGIN_SEARCH_PATH |
* -------------------------------------------------
* | <path1;...pathn> | <path1;...pathn> |
* -------------------------------------------------
*/
static int
buildinitialpluginpath(NCPluginList* dirs)
{
int stat = NC_NOERR;
const char* hdf5path = NULL;
/* Find the plugin directory root(s) */
hdf5path = getenv(PLUGIN_ENV); /* HDF5_PLUGIN_PATH */
if(hdf5path == NULL) {
/* Use NETCDF_PLUGIN_SEARCH_PATH */
hdf5path = NETCDF_PLUGIN_SEARCH_PATH;
}
if((stat = ncaux_plugin_path_parse(hdf5path,'\0',dirs))) goto done;
done:
return stat;
}

67
libdispatch/ds3util.c
View File

@ -50,6 +50,8 @@ static int endswith(const char* s, const char* suffix);
static void freeentry(struct AWSentry* e);
static int awsparse(const char* text, NClist* profiles);
extern void awsprofiles(void);
/**************************************************/
/* Capture environmental Info */
@ -255,7 +257,7 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp)
ncurirebuild(newurl);
/* return various items */
#ifdef AWSDEBUG
fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=|%s| region=|%s|\n",uri->uri,bucket,region);
fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=|%s| region=|%s|\n",newurl->uri,bucket,region);
#endif
if(newurlp) {*newurlp = newurl; newurl = NULL;}
if(s3 != NULL) {
@ -507,22 +509,7 @@ NC_aws_load_credentials(NCglobalstate* gstate)
gstate->rcinfo->s3profiles = profiles; profiles = NULL;
#ifdef AWSDEBUG
{int i,j;
fprintf(stderr,">>> profiles:\n");
for(i=0;i<nclistlength(gstate->rcinfo->s3profiles);i++) {
struct AWSprofile* p = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i);
fprintf(stderr," [%s]",p->name);
for(j=0;j<nclistlength(p->entries);j++) {
struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j);
if(strcmp(e->key,"aws_access_key_id")
fprintf(stderr," %s=%d",e->key,(int)strlen(e->value));
else if(strcmp(e->key,"aws_secret_access_key")
fprintf(stderr," %s=%d",e->key,(int)strlen(e->value));
else fprintf(stderr," %s=%s",e->key,e->value);
}
fprintf(stderr,"\n");
}
}
awsprofiles();
#endif
done:
@ -666,7 +653,7 @@ NC_getdefaults3region(NCURI* uri, const char** regionp)
if(region == NULL)
region = NC_getglobalstate()->aws.default_region; /* Force use of the Amazon default */
#ifdef AWSDEBUG
fprintf(stderr,">>> activeregion = |%s|\n",region));
fprintf(stderr,">>> activeregion = |%s|\n",region);
#endif
if(regionp) *regionp = region;
return stat;
@ -936,10 +923,52 @@ freeentry(struct AWSentry* e)
{
if(e) {
#ifdef AWSDEBUG
fprintf(stderr,">>> freeentry: key=%p value=%p\n",e->key,e->value);
fprintf(stderr,">>> freeentry: key=%s value=%s\n",e->key,e->value);
#endif
nullfree(e->key);
nullfree(e->value);
nullfree(e);
}
}
/* Provide profile-related dumper(s) */
void
awsdumpprofile(struct AWSprofile* p)
{
size_t j;
if(p == NULL) {
fprintf(stderr," <NULL>");
goto done;
}
fprintf(stderr," [%s]",p->name);
if(p->entries == NULL) {
fprintf(stderr,"<NULL>");
goto done;
}
for(j=0;j<nclistlength(p->entries);j++) {
struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j);
fprintf(stderr," %s=%s",e->key,e->value);
}
done:
fprintf(stderr,"\n");
}
void
awsdumpprofiles(NClist* profiles)
{
size_t i;
NCglobalstate* gs = NC_getglobalstate();
for(i=0;i<nclistlength(gs->rcinfo->s3profiles);i++) {
struct AWSprofile* p = (struct AWSprofile*)nclistget(profiles,i);
awsdumpprofile(p);
}
}
void
awsprofiles(void)
{
NCglobalstate* gs = NC_getglobalstate();
fprintf(stderr,">>> profiles from global->rcinfo->s3profiles:\n");
awsdumpprofiles(gs->rcinfo->s3profiles);
}

2
libdispatch/ncbytes.c
View File

@ -23,7 +23,7 @@ static int
ncbytesfail(void)
{
fflush(stdout);
fprintf(stderr,"bytebuffer failure\n");
fprintf(stderr,"NCbytes failure\n");
fflush(stderr);
#ifdef NCBYTESDEBUG
abort();

32
libdispatch/ncjson.c
View File

@ -6,15 +6,6 @@
TODO: make utf8 safe
*/
/*
WARNING:
If you modify this file,
then you need to got to
the include/ directory
and do the command:
make makenetcdfjson
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
@ -119,7 +110,7 @@ static int bytesappendquoted(NCJbuf* buf, const char* s);
static int bytesappend(NCJbuf* buf, const char* s);
static int bytesappendc(NCJbuf* bufp, char c);
/* Hide everything for plugins */
/* Static'ize everything for plugins */
#ifdef NETCDF_JSON_H
#define OPTSTATIC static
static int NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp);
@ -474,7 +465,7 @@ testint(const char* word)
int count = 0;
/* Try to convert to number */
ncvt = sscanf(word,"%lld%n",&i,&count);
return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR));
return NCJTHROW((ncvt == 1 && strlen(word)==(size_t)count ? NCJ_OK : NCJ_ERR));
}
static int
@ -493,7 +484,7 @@ testdouble(const char* word)
if(0==(int)strcasecmp("-infinityf",word)) return NCJTHROW(NCJ_OK);
/* Try to convert to number */
ncvt = sscanf(word,"%lg%n",&d,&count);
return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR));
return NCJTHROW((ncvt == 1 && strlen(word)==(size_t)count ? NCJ_OK : NCJ_ERR));
}
static int
@ -540,7 +531,7 @@ NCJreclaim(NCjson* json)
static void
NCJreclaimArray(struct NCjlist* array)
{
int i;
size_t i;
for(i=0;i<array->len;i++) {
NCJreclaim(array->contents[i]);
}
@ -618,7 +609,8 @@ done:
OPTSTATIC int
NCJdictget(const NCjson* dict, const char* key, const NCjson** valuep)
{
int i,stat = NCJ_OK;
int stat = NCJ_OK;
size_t i;
if(dict == NULL || dict->sort != NCJ_DICT)
{stat = NCJTHROW(NCJ_ERR); goto done;}
@ -850,7 +842,8 @@ done:
static int
NCJcloneArray(const NCjson* array, NCjson** clonep)
{
int i, stat=NCJ_OK;
int stat=NCJ_OK;
size_t i;
NCjson* clone = NULL;
if((stat=NCJnew(NCJ_ARRAY,&clone))==NCJ_ERR) goto done;
for(i=0;i<NCJlength(array);i++) {
@ -868,7 +861,8 @@ done:
static int
NCJcloneDict(const NCjson* dict, NCjson** clonep)
{
int i, stat=NCJ_OK;
int stat=NCJ_OK;
size_t i;
NCjson* clone = NULL;
if((stat=NCJnew(NCJ_DICT,&clone))==NCJ_ERR) goto done;
for(i=0;i<NCJlength(dict);i++) {
@ -927,7 +921,6 @@ NCJinsertstring(NCjson* object, const char* key, const char* value)
else
NCJnewstring(NCJ_STRING,value,&jvalue);
NCJinsert(object,key,jvalue);
done:
return NCJTHROW(stat);
}
@ -941,7 +934,6 @@ NCJinsertint(NCjson* object, const char* key, long long ivalue)
snprintf(digits,sizeof(digits),"%lld",ivalue);
NCJnewstring(NCJ_STRING,digits,&jvalue);
NCJinsert(object,key,jvalue);
done:
return NCJTHROW(stat);
}
@ -982,7 +974,7 @@ static int
NCJunparseR(const NCjson* json, NCJbuf* buf, unsigned flags)
{
int stat = NCJ_OK;
int i;
size_t i;
switch (NCJsort(json)) {
case NCJ_STRING:
@ -1146,5 +1138,7 @@ netcdf_supresswarnings(void)
ignore = (void*)NCJparse;
ignore = (void*)NCJdump;
ignore = (void*)NCJtotext;
ignore = (void*)NCJinsertstring;
ignore = (void*)NCJinsertint;
ignore = ignore;
}

61
libdispatch/nclist.c
View File

@ -13,6 +13,8 @@
int nclistisnull(void* e) {return e == NULL;}
#define NCLISTDEBUG 1
#ifndef TRUE
#define TRUE 1
#endif
@ -23,6 +25,18 @@ int nclistisnull(void* e) {return e == NULL;}
#define DEFAULTALLOC 16
#define ALLOCINCR 16
static int
nclistfail(void)
{
fflush(stdout);
fprintf(stderr,"NClist failure\n");
fflush(stderr);
#ifdef NCLISTDEBUG
abort();
#endif
return FALSE;
}
NClist* nclistnew(void)
{
NClist* l;
@ -83,7 +97,7 @@ int
nclistsetalloc(NClist* l, size_t sz)
{
void** newcontent = NULL;
if(l == NULL) return FALSE;
if(l == NULL) return nclistfail();
if(sz <= 0) {sz = (l->length?2*l->length:DEFAULTALLOC);}
if(l->alloc >= sz) {return TRUE;}
newcontent=(void**)calloc(sz,sizeof(void*));
@ -99,8 +113,8 @@ nclistsetalloc(NClist* l, size_t sz)
int
nclistsetlength(NClist* l, size_t newlen)
{
if(l == NULL) return FALSE;
if(newlen > l->alloc && !nclistsetalloc(l,newlen)) return FALSE;
if(l == NULL) return nclistfail();
if(newlen > l->alloc && !nclistsetalloc(l,newlen)) return nclistfail();
if(newlen > l->length) {
/* clear any extension */
memset(&l->content[l->length],0,(newlen - l->length)*sizeof(void*));
@ -112,7 +126,8 @@ nclistsetlength(NClist* l, size_t newlen)
void*
nclistget(const NClist* l, size_t index)
{
if(l == NULL || l->length == 0) return NULL;
if(l == NULL) return (nclistfail(),NULL);
if(l->length == 0) return NULL;
if(index >= l->length) return NULL;
return l->content[index];
}
@ -123,10 +138,10 @@ nclistget(const NClist* l, size_t index)
int
nclistset(NClist* l, size_t index, void* elem)
{
if(l == NULL) return FALSE;
if(!nclistsetalloc(l,index+1)) return FALSE;
if(l == NULL) return nclistfail();
if(!nclistsetalloc(l,index+1)) return nclistfail();
if(index >= l->length) {
if(!nclistsetlength(l,index+1)) return FALSE;
if(!nclistsetlength(l,index+1)) return nclistfail();
}
l->content[index] = elem;
return TRUE;
@ -136,11 +151,13 @@ nclistset(NClist* l, size_t index, void* elem)
int
nclistinsert(NClist* l, size_t index, void* elem)
{
long i; /* do not make unsigned */
if(l == NULL) return FALSE;
if(index > l->length) return FALSE;
size_t i;
if(l == NULL) return nclistfail();
if(index > l->length) return nclistfail();
nclistsetalloc(l,0);
for(i=(long)l->length;i>index;i--) l->content[i] = l->content[i-1];
if(l->length > 0) {
for(i=l->length;i>index;i--) l->content[i] = l->content[i-1];
}
l->content[index] = elem;
l->length++;
return TRUE;
@ -149,8 +166,10 @@ nclistinsert(NClist* l, size_t index, void* elem)
int
nclistpush(NClist* l, const void* elem)
{
if(l == NULL) return FALSE;
if(l == NULL) return nclistfail();
if(l->length >= l->alloc) nclistsetalloc(l,0);
if(l->content == NULL)
nclistsetalloc(l,0);
l->content[l->length] = (void*)elem;
l->length++;
return TRUE;
@ -159,7 +178,8 @@ nclistpush(NClist* l, const void* elem)
void*
nclistpop(NClist* l)
{
if(l == NULL || l->length == 0) return NULL;
if(l == NULL) return (nclistfail(),NULL);
if(l->length == 0) return NULL;
l->length--;
return l->content[l->length];
}
@ -167,7 +187,8 @@ nclistpop(NClist* l)
void*
nclisttop(NClist* l)
{
if(l == NULL || l->length == 0) return NULL;
if(l == NULL) return (nclistfail(),NULL);
if(l->length == 0) return NULL;
return l->content[l->length - 1];
}
@ -176,7 +197,8 @@ nclistremove(NClist* l, size_t i)
{
size_t len;
void* elem;
if(l == NULL || (len=l->length) == 0) return NULL;
if(l == NULL) return (nclistfail(),NULL);
if((len=l->length) == 0) return NULL;
if(i >= len) return NULL;
elem = l->content[i];
for(i+=1;i<len;i++) l->content[i-1] = l->content[i];
@ -219,7 +241,8 @@ nclistelemremove(NClist* l, void* elem)
size_t len;
size_t i;
int found = 0;
if(l == NULL || (len=l->length) == 0) return 0;
if(l == NULL) return nclistfail();
if((len=l->length) == 0) return 0;
for(i=0;i<nclistlength(l);i++) {
void* candidate = l->content[i];
if(elem == candidate) {
@ -242,7 +265,8 @@ nclistunique(NClist* l)
{
size_t i,j,k,len;
void** content;
if(l == NULL || l->length == 0) return 1;
if(l == NULL) return nclistfail();
if(l->length == 0) return 1;
len = l->length;
content = l->content;
for(i=0;i<len;i++) {
@ -304,7 +328,8 @@ nclistextract(NClist* l)
int
nclistnull(NClist* l)
{
if(l == NULL || l->length == 0) return 1;
if(l == NULL) return nclistfail();
if(l->length == 0) return 1;
nclistpush(l,NULL);
nclistsetlength(l,l->length-1);
return 1;

350
libdispatch/ncproplist.c Normal file
View File

@ -0,0 +1,350 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* $Header$
*********************************************************************/
#include "config.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include "ncdispatch.h"
#include "nccrc.h"
#include "ncproplist.h"
#undef DEBUG
#define ASSERTIONS
#ifdef ASSERTIONS
#define ASSERT(x) assert(x)
#else
#define ASSERT(x)
#endif
/**************************************************/
#define MINPROPS 2
#define EXPANDFACTOR 1
#define hasspace(plist,nelems) ((plist)->alloc >= ((plist)->count + (nelems)))
#define emptyprop {" ",0,0,0,NULL}
/**************************************************/
/* Forward */
static int ncproplistinit(NCproplist* plist);
static int extendplist(NCproplist* plist, size_t nprops);
/* Static'ize everything for plugins */
#ifdef NETCDF_PROPLIST_H
#define OPTSTATIC static
static NCproplist* ncproplistnew(void);
static int ncproplistfree(NCproplist* plist);
static int ncproplistadd(NCproplist* plist, const char* key, uintptr_t value);
static int ncproplistaddbytes(NCproplist* plist, const char* key, void* value, uintptr_t size);
static int ncproplistaddstring(NCproplist* plist, const char* key, const char* str);
static int ncproplistaddx(NCproplist* plist, const char* key, void* value, uintptr_t size, uintptr_t userdata, NCPreclaimfcn fcn);
static int ncproplistclone(const NCproplist* src, NCproplist* clone);
static int ncproplistget(const NCproplist* plist, const char* key, uintptr_t* valuep, uintptr_t* sizep);
static int ncproplistith(const NCproplist* plist, size_t i, char* const * keyp, uintptr_t const * valuep, uintptr_t* sizep);
#else /*!NETCDF_PROPLIST_H*/
#define OPTSTATIC
#endif /*NETCDF_PROPLIST_H*/
/**
* Create new property list
* @return pointer to the created property list.
*/
OPTSTATIC NCproplist*
ncproplistnew(void)
{
NCproplist* plist = NULL;
plist = calloc(1,sizeof(NCproplist));
if(ncproplistinit(plist) != NC_NOERR)
{ncproplistfree(plist); plist = NULL;}
return plist;
}
/**
* Reclaim memory used by a property list
* @param plist to reclaim
* @return NC_NOERR if succeed, NC_EXXX otherwise.
*/
OPTSTATIC int
ncproplistfree(NCproplist* plist)
{
int stat = NC_NOERR;
size_t i;
if(plist == NULL) goto done;
if(plist->properties != NULL) {
for(i=0;i<plist->count;i++) {
NCProperty* prop = &plist->properties[i];
void* ptr = (void*)prop->value; /* convert to ptr */
assert(prop->flags & (NCPF_SIMPLE|NCPF_BYTES|NCPF_COMPLEX));
if(prop->flags & NCPF_SIMPLE) continue; /* no reclaim needed */
if(prop->flags & NCPF_BYTES) {
if(ptr != NULL) free(ptr);
} else { /* (prop->flags & NCPF_COMPLEX) */
int ok;
assert(prop->reclaim != NULL);
ok = prop->reclaim(prop->userdata, prop->key, ptr, prop->size);
if(!ok && stat == NC_NOERR) stat = NC_EINVAL;
}
}
free(plist->properties);
}
free(plist);
done:
return stat;
}
/**
* Add a non-reclaimable entry to the property list
* @param plist into which the value is be inserted.
* @param key
* @param value
* @return NC_NOERR if succeed, NC_EXXX otherwise.
*/
OPTSTATIC int
ncproplistadd(NCproplist* plist, const char* key, uintptr_t value)
{
int stat = NC_NOERR;
NCProperty* prop = NULL;
size_t keylen;
if(plist == NULL) goto done;
if(!hasspace(plist,1)) {if((stat = extendplist(plist,(plist->count+1)*EXPANDFACTOR))) goto done;} /* extra space */
prop = &plist->properties[plist->count];
keylen = strlen(key);
if(keylen > NCPROPSMAXKEY) keylen = NCPROPSMAXKEY; /* truncate */
memcpy(prop->key,key,keylen);
prop->key[keylen] = '\0';
prop->value = value;
prop->flags = NCPF_SIMPLE;
plist->count++;
done:
return stat;
}
/**
* Add a reclaimable entry to the property list, where the value
* can be reclaimed using a simple free();
* @param plist into which the value is be inserted.
* @param key
* @param value ptr to memory chunk
* @param size |*value|
* @return NC_NOERR if succeed, NC_EXXX otherwise.
*/
OPTSTATIC int
ncproplistaddbytes(NCproplist* plist, const char* key, void* value, uintptr_t size)
{
int stat = NC_NOERR;
NCProperty* prop = NULL;
size_t keylen;
if(plist == NULL) goto done;
if(!hasspace(plist,1)) {if((stat = extendplist(plist,(plist->count+1)*EXPANDFACTOR))) goto done;} /* extra space */
prop = &plist->properties[plist->count];
keylen = strlen(key);
if(keylen > NCPROPSMAXKEY) keylen = NCPROPSMAXKEY; /* truncate */
memcpy(prop->key,key,keylen);
prop->key[keylen] = '\0';
prop->value = (uintptr_t)value;
prop->flags = NCPF_BYTES;
plist->count++;
done:
return stat;
}
/**
* Add a reclaimable entry to the property list, where the value
* can be reclaimed using a simple free();
* @param plist into which the value is be inserted.
* @param key
* @param value ptr to memory chunk
* @param size |*value|
* @return NC_NOERR if succeed, NC_EXXX otherwise.
*/
OPTSTATIC int
ncproplistaddstring(NCproplist* plist, const char* key, const char* str)
{
uintptr_t size = 0;
if(str) size = (uintptr_t)strlen(str);
return ncproplistaddbytes(plist,key,(void*)str,size);
}
/**
* Most general case for adding a property.
* @param plist into which the value is be inserted.
* @param key
* @param value
* @param size
* @param userdata extra environment data for the reclaim function.
* @param fcn the reclaim function
* @return NC_NOERR if succeed, NC_EXXX otherwise.
*/
OPTSTATIC int
ncproplistaddx(NCproplist* plist, const char* key, void* value, uintptr_t size, uintptr_t userdata, NCPreclaimfcn fcn)
{
int stat = NC_NOERR;
NCProperty* prop = NULL;
size_t keylen;
if(plist == NULL) goto done;
if(!hasspace(plist,1)) {if((stat = extendplist(plist,(plist->count+1)*EXPANDFACTOR))) goto done;} /* extra space */
prop = &plist->properties[plist->count];
keylen = strlen(key);
if(keylen > NCPROPSMAXKEY) keylen = NCPROPSMAXKEY; /* truncate */
memcpy(prop->key,key,keylen);
prop->key[keylen] = '\0';
prop->value = (uintptr_t)value;
prop->size = size;
prop->reclaim = fcn;
prop->userdata = userdata;
prop->flags = NCPF_COMPLEX;
plist->count++;
done:
return stat;
}
OPTSTATIC int
ncproplistclone(const NCproplist* src, NCproplist* clone)
{
int stat = NC_NOERR;
size_t i;
NCProperty* srcprops;
NCProperty* cloneprops;
if(src == NULL || clone == NULL) {stat = NC_EINVAL; goto done;}
if((stat=ncproplistinit(clone))) goto done;
if((stat=extendplist(clone,src->count))) goto done;
srcprops = src->properties;
cloneprops = clone->properties;
for(i=0;i<src->count;i++) {
cloneprops[i] = srcprops[i];
strncpy(cloneprops[i].key,srcprops[i].key,sizeof(cloneprops[i].key));
#if 0
cloneprops[i]->flags = srcprops[i]->flags;
cloneprops[i]->value = srcprops[i]->value;
cloneprops[i]->size = srcprops[i]->size;
cloneprops[i]->userdata = srcprops[i]->userdata;
cloneprops[i]->reclaim = srcprops->reclaim;
#endif
}
done:
return stat;
}
/* Increase size of a plist to be at lease nprops properties */
static int
extendplist(NCproplist* plist, size_t nprops)
{
int stat = NC_NOERR;
size_t newsize = plist->count + nprops;
NCProperty* newlist = NULL;
if((plist->alloc >= newsize) || (nprops == 0))
goto done; /* Already enough space */
newlist = realloc(plist->properties,newsize*sizeof(NCProperty));
if(newlist == NULL) {stat = NC_ENOMEM; goto done;}
plist->properties = newlist; newlist = NULL;
plist->alloc = newsize;
done:
return stat;
}
/**
* Lookup key and return value and size
* @param plist to search
* @param key for which to search
* @param valuep returned value
* @param sizep returned size
* @return NC_NOERR if key found, NC_ENOOBJECT if key not found; NC_EXXX otherwise
*/
OPTSTATIC int
ncproplistget(const NCproplist* plist, const char* key, uintptr_t* valuep, uintptr_t* sizep)
{
int stat = NC_ENOOBJECT; /* assume not found til proven otherwise */
size_t i;
NCProperty* props;
uintptr_t value = 0;
uintptr_t size = 0;
if(plist == NULL || key == NULL) goto done;
for(i=0,props=plist->properties;i<plist->count;i++,props++) {
if(strcmp(props->key,key)==0) {
value = props->value;
size = props->size;
stat = NC_NOERR; /* found */
break;
}
}
if(valuep) *valuep = value;
if(sizep) *sizep = size;
done:
return stat;
}
/* Iteration support */
/**
* Get the ith key+value.a
* @param plist to search
* @param i which property to get.
* @param keyp return i'th key
* @param valuep return i'th value
* @param valuep return i'th size
* @return NC_NOERR if success, NC_EINVAL otherwise
*/
OPTSTATIC int
ncproplistith(const NCproplist* plist, size_t i, char* const * keyp, uintptr_t const * valuep, uintptr_t* sizep)
{
int stat = NC_NOERR;
NCProperty* prop = NULL;
if(plist == NULL) goto done;
if(i >= plist->count) {stat = NC_EINVAL; goto done;}
prop = &plist->properties[i];
if(keyp) *((char**)keyp) = (char*)prop->key;
if(valuep) *((uintptr_t*)valuep) = (uintptr_t)prop->value;
if(sizep) *sizep = prop->size;
done:
return stat;
}
/**************************************************/
/* Support Functions */
/**
* Initialize a new property list
*/
static int
ncproplistinit(NCproplist* plist)
{
/* Assume property list will hold at lease MINPROPS properties */
plist->alloc = MINPROPS;
plist->count = 0;
plist->properties = (NCProperty*)calloc(MINPROPS,sizeof(NCProperty));
return (plist->properties?NC_NOERR:NC_ENOMEM);
}
/* Suppress unused statics warning */
static void
ncproplist_unused(void)
{
void* unused = ncproplist_unused;
unused = ncproplistnew;
unused = ncproplistfree;
unused = ncproplistadd;
unused = ncproplistaddbytes;
unused = ncproplistaddstring;
unused = ncproplistaddx;
unused = ncproplistclone;
unused = ncproplistget;
unused = ncproplistith;
unused = ncproplistinit;
unused = (void*)ncproplistith;
unused = unused;
}

9
libdispatch/ncs3sdk_h5.c
View File

@ -959,11 +959,12 @@ s3objectsinfo(NClist* contents, NClist* keys, NClist* lengths)
for(i=0;i<nclistlength(contents);i++) {
struct Object* s3_object = (struct Object*)nclistget(contents,i);
if((stat = s3objectinfo1(s3_object,&key,&length))) goto done;
nclistpush(keys,key); key = NULL;
nclistpush(lengths,(void*)length);
if(keys != NULL) {nclistpush(keys,key);} else {nullfree(key);}
key = NULL;
if(lengths != NULL) nclistpush(lengths,(void*)length);
}
nclistnull(keys);
nclistnull(lengths);
if(keys != NULL) nclistnull(keys);
if(lengths != NULL) nclistnull(lengths);
done:
nullfree(key); /* avoid mem leak */
return NCTHROW(stat);

2
libdispatch/ncuri.c
View File

@ -832,7 +832,7 @@ ncuriremoveparam(NCURI* uri, const char* key)
static int
ncfind(NClist* params, const char* key)
{
int i;
size_t i;
if(key == NULL) return -1;
if(params == NULL) return -1;
for(i=0;i<nclistlength(params);i+=2) {

2
libhdf5/CMakeLists.txt
View File

@ -13,7 +13,7 @@ add_library(netcdfhdf5 OBJECT)
target_sources(netcdfhdf5 PRIVATE
nc4hdf.c nc4info.c hdf5file.c hdf5attr.c
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c
hdf5var.c nc4mem.c nc4memcb.c hdf5dispatch.c hdf5filter.c
hdf5var.c nc4mem.c nc4memcb.c hdf5dispatch.c hdf5filter.c hdf5plugins.c
hdf5set_format_compatibility.c hdf5debug.c
)

2
libhdf5/Makefile.am
View File

@ -15,7 +15,7 @@ noinst_LTLIBRARIES = libnchdf5.la
# The source files.
libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c \
hdf5var.c nc4mem.c nc4memcb.c hdf5dispatch.c hdf5filter.c \
hdf5var.c nc4mem.c nc4memcb.c hdf5dispatch.c hdf5filter.c hdf5plugins.c \
hdf5set_format_compatibility.c hdf5debug.c hdf5debug.h hdf5err.h
if NETCDF_ENABLE_BYTERANGE

1
libhdf5/hdf5dispatch.c
View File

@ -10,6 +10,7 @@
#include "config.h"
#include "hdf5internal.h"
#include "hdf5dispatch.h"
#ifdef NETCDF_ENABLE_BYTERANGE
#include "H5FDhttp.h"

4
libhdf5/hdf5internal.c
View File

@ -15,9 +15,11 @@
*/
#include "config.h"
#include <stddef.h>
#include "netcdf.h"
#include "ncplugins.h"
#include "hdf5internal.h"
#include "hdf5err.h" /* For BAIL2 */
#include <stddef.h>
#ifdef _WIN32
#include <windows.h>
#endif

245
libhdf5/hdf5plugins.c Normal file
View File

@ -0,0 +1,245 @@
/* Copyright 2003-2018, University Corporation for Atmospheric
* Research. See the COPYRIGHT file for copying and redistribution
* conditions.
*/
/**
* @file @internal netcdf-4 functions for the plugin list.
*
* @author Dennis Heimbigner
*/
#include "config.h"
#include <stddef.h>
#include <stdlib.h>
#include "netcdf.h"
#include "ncbytes.h"
#include "hdf5internal.h"
#include "hdf5debug.h"
#include "ncplugins.h"
#undef TPLUGINS
/**************************************************/
/**
* @file
* @internal
* Internal netcdf hdf5 plugin path functions.
*
* @author Dennis Heimbigner
*/
/**************************************************/
/**
* Return the length of the current sequence of directories
* in the internal global plugin path list.
* @param ndirsp length is returned here
* @return NC_NOERR | NC_EXXX
*
* @author Dennis Heimbigner
*/
int
NC4_hdf5_plugin_path_ndirs(size_t* ndirsp)
{
int stat = NC_NOERR;
size_t ndirs = 0;
unsigned undirs = 0;
herr_t hstat = 0;
/* Get the length of the HDF5 plugin path set */
if((hstat = H5PLsize(&undirs))<0) goto done;
ndirs = (size_t)undirs;
if(ndirsp) *ndirsp = ndirs;
done:
return THROW(stat);
}
/**
* Return the current sequence of directories in the internal global
* plugin path list. Since this function does not modify the plugin path,
* it can be called at any time.
* @param dirs pointer to an NCPluginList object
* @return NC_NOERR | NC_EXXX
* @author Dennis Heimbigner
*
* WARNING: if dirs->dirs is NULL, then space for the directory
* vector will be allocated. If not NULL, then the specified space will
* be overwritten with the vector.
*
* @author: Dennis Heimbigner
*/
int
NC4_hdf5_plugin_path_get(NCPluginList* dirs)
{
int stat = NC_NOERR;
unsigned i;
herr_t hstat = 0;
unsigned undirs = 0;
ssize_t dirlen;
char* dirbuf = NULL;
if(dirs == NULL) {stat = NC_EINVAL; goto done;}
/* Get the length of the HDF5 plugin path set */
if((hstat = H5PLsize(&undirs))<0) goto done;
dirs->ndirs = (size_t)undirs;
/* Copy out the paths from the HDF5 library */
/* Watch out for nul term handling WRT dir string length */
if(dirs->dirs == NULL) {
if((dirs->dirs=(char**)calloc(dirs->ndirs,sizeof(char*)))==NULL)
{stat = NC_ENOMEM; goto done;}
}
for(i=0;i<undirs;i++) {
if((dirlen = H5PLget(i, NULL, 0))<0) {stat = NC_EHDFERR; goto done;}
nullfree(dirbuf); dirbuf = NULL; /* suspenders and belt */
if((dirbuf = (char*)malloc((size_t)dirlen+1))==NULL) {stat = NC_ENOMEM; goto done;} /* dirlen does not include nul term */
if((dirlen = H5PLget(i, dirbuf, ((size_t)dirlen)+1))<0) {stat = NC_EHDFERR; goto done;}
dirs->dirs[i] = dirbuf; dirbuf = NULL;
}
done:
if(hstat < 0 && stat != NC_NOERR) stat = NC_EHDFERR;
return THROW(stat);
}
/**
* Empty the current internal path sequence
* and replace with the sequence of directories argument.
* Using a dirs->ndirs argument of 0 will clear the set of plugin dirs.
*
* @param dirs to overwrite the current internal dir list
* @return NC_NOERR | NC_EXXX
*
* @author Dennis Heimbigner
*/
int
NC4_hdf5_plugin_path_set(NCPluginList* dirs)
{
int stat = NC_NOERR;
size_t i;
herr_t hstat = 0;
unsigned undirs = 0;
/* validate */
if(dirs == NULL || (dirs->ndirs > 0 && dirs->dirs == NULL))
{stat = NC_EINVAL; goto done;}
/* Clear the current path list */
if((hstat = H5PLsize(&undirs))<0) goto done;
if(undirs > 0) {
for(i=0;i<undirs;i++) {
/* Always remove the first element to avoid index confusion */
if((hstat = H5PLremove(0))<0) {stat = NC_EINVAL; goto done;}
}
}
/* Insert the new path list */
for(i=0;i<dirs->ndirs;i++) {
/* Always append */
if((hstat = H5PLappend(dirs->dirs[i]))<0)
{stat = NC_EINVAL; goto done;}
}
done:
if(hstat < 0 && stat != NC_NOERR) stat = NC_EHDFERR;
return stat;
}
int
NC4_hdf5_plugin_path_initialize(void)
{
return NC_NOERR;
}
int
NC4_hdf5_plugin_path_finalize(void)
{
return NC_NOERR;
}
/**************************************************/
/* Debug printer for HDF5 plugin paths */
static NCbytes* ncppbuf;
const char*
NC4_hdf5_plugin_path_tostring(void)
{
int stat = NC_NOERR;
herr_t hstat = 0;
unsigned npaths = 0;
char* dir = NULL;
if(ncppbuf == NULL) ncppbuf = ncbytesnew();
ncbytesclear(ncppbuf);
if((hstat = H5PLsize(&npaths))<0) {stat = NC_EINVAL; goto done;}
if(npaths > 0) {
ssize_t dirlen = 0;
unsigned i;
for(i=0;i<npaths;i++) {
dirlen = H5PLget(i,NULL,0);
if(dirlen < 0) {stat = NC_EINVAL; goto done;}
if((dir = (char*)malloc(1+(size_t)dirlen))==NULL)
{stat = NC_ENOMEM; goto done;}
/* returned dirlen does not include the nul terminator, but the length argument must include it */
dirlen = H5PLget(i,dir,(size_t)(dirlen+1));
dir[dirlen] = '\0';
if(i > 0) ncbytescat(ncppbuf,";");
ncbytescat(ncppbuf,dir);
nullfree(dir); dir = NULL;
}
}
done:
nullfree(dir);
if(stat != NC_NOERR) ncbytesclear(ncppbuf);
ncbytesnull(ncppbuf);
return ncbytescontents(ncppbuf);
}
/**************************************************/
#ifdef TPLUGINS
static void
printplugin1(struct NC_HDF5_Plugin* nfs)
{
int i;
if(nfs == NULL) {
fprintf(stderr,"{null}");
return;
}
fprintf(stderr,"{%u,(%u)",nfs->pluginid,(int)nfs->nparams);
for(i=0;i<nfs->nparams;i++) {
fprintf(stderr," %s",nfs->params[i]);
}
fprintf(stderr,"}");
}
static void
printplugin(struct NC_HDF5_Plugin* nfs, const char* tag, int line)
{
fprintf(stderr,"%s: line=%d: ",tag,line);
printplugin1(nfs);
fprintf(stderr,"\n");
}
static void
printpluginlist(NC_VAR_INFO_T* var, const char* tag, int line)
{
int i;
const char* name;
if(var == NULL) name = "null";
else if(var->hdr.name == NULL) name = "?";
else name = var->hdr.name;
fprintf(stderr,"%s: line=%d: var=%s plugins=",tag,line,name);
if(var != NULL) {
NClist* plugins = (NClist*)var->plugins;
for(i=0;i<nclistlength(plugins);i++) {
struct NC_HDF5_Plugin* nfs = (struct NC_HDF5_Plugin*)nclistget(plugins,i);
fprintf(stderr,"[%d]",i);
printplugin1(nfs);
}
}
fprintf(stderr,"\n");
}
#endif /*TPLUGINS*/

6
libnczarr/CMakeLists.txt
View File

@ -19,7 +19,6 @@ zcvt.c
zdim.c
zdispatch.c
zfile.c
zfilter.c
zgrp.c
zinternal.c
zmap.c
@ -42,6 +41,7 @@ zinternal.h
zmap.h
zodom.h
zprovenance.h
zplugins.h
zfilter.h
zdebug.h
)
@ -50,6 +50,10 @@ if(NETCDF_ENABLE_NCZARR_ZIP)
set(libnczarr_SOURCES ${libnczarr_SOURCES} zmap_zip.c)
endif()
if(NETCDF_ENABLE_NCZARR_FILTERS)
set(libnczarr_SOURCES ${libnczarr_SOURCES} zfilter.c zplugins.c)
endif()
if(NETCDF_ENABLE_S3)
set(libnczarr_SOURCES ${libnczarr_SOURCES} zmap_s3sdk.c)
endif()

3
libnczarr/Makefile.am
View File

@ -61,6 +61,7 @@ zinternal.h \
zmap.h \
zodom.h \
zprovenance.h \
zplugins.h \
zfilter.h \
zdebug.h
@ -69,7 +70,7 @@ libnczarr_la_SOURCES += zmap_zip.c
endif
if NETCDF_ENABLE_NCZARR_FILTERS
libnczarr_la_SOURCES += zfilter.c
libnczarr_la_SOURCES += zfilter.c zplugins.c
endif
if NETCDF_ENABLE_S3

2
libnczarr/zdispatch.c
View File

@ -10,6 +10,7 @@
*/
#include "zincludes.h"
#include "zplugins.h"
/* Forward */
static int NCZ_var_par_access(int ncid, int varid, int par_access);
@ -198,7 +199,6 @@ NCZ_inq_filter_avail(int ncid, unsigned id)
return REPORT(NC_ENOFILTER,"inq_filter_avail");
}
#endif /*NETCDF_ENABLE_NCZARR_FILTERS*/
/**************************************************/

2
libnczarr/zfile.c
View File

@ -294,7 +294,7 @@ NCZ_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp)
NC_FILE_INFO_T* file;
NC_GRP_INFO_T *grp;
int stat = NC_NOERR;
int i;
size_t i;
LOG((2, "%s: ncid 0x%x", __func__, ncid));

731
libnczarr/zfilter.c
View File

@ -50,7 +50,8 @@
#include "ncpoco.h"
#include "netcdf_filter.h"
#include "netcdf_filter_build.h"
#include "netcdf_aux.h"
#include "zfilter.h"
#include "zplugins.h"
#if 0
#define DEBUG
@ -63,20 +64,6 @@
#define NULLIFY(x) ((x)?(x):"NULL")
/* Hold the loaded filter plugin information */
typedef struct NCZ_Plugin {
int incomplete;
struct HDF5API {
const H5Z_class2_t* filter;
NCPSharedLib* hdf5lib; /* source of the filter */
} hdf5;
struct CodecAPI {
int defaulted; /* codeclib was a defaulting library */
const NCZ_codec_t* codec;
NCPSharedLib* codeclib; /* of the source codec; null if same as hdf5 */
} codec;
} NCZ_Plugin;
/* The NC_VAR_INFO_T->filters field is an NClist of this struct */
/*
Each filter can have two parts: HDF5 and Codec.
@ -148,14 +135,6 @@ typedef struct NCZ_Filter {
/* WARNING: GLOBAL DATA */
/* TODO: move to common global state */
/* All possible HDF5 filter plugins */
/* Consider onverting to linked list or hash table or equivalent since very sparse */
NCZ_Plugin* loaded_plugins[H5Z_FILTER_MAX];
int loaded_plugins_max = -1;
static NClist* codec_defaults = NULL; /* NClist<struct CodecAPI*> */
static NClist* default_libs = NULL; /* NClist<NCPSharedLib>; sources of the defaults */
static int NCZ_filter_initialized = 0;
/**************************************************/
@ -179,61 +158,7 @@ NCJtrace(const NCjson* j)
#define SEXISTS(x,p) (((x) && *(x)? (*(x))-> p : "null"))
#endif
#if defined(DEBUGF) || defined(DEBUGL)
const char*
printplugin(const NCZ_Plugin* plugin)
{
static char plbuf[4096];
char plbuf2[2000];
char plbuf1[2000];
if(plugin == NULL) return "plugin=NULL";
plbuf2[0] = '\0'; plbuf1[0] = '\0';
if(plugin->hdf5.filter)
snprintf(plbuf1,sizeof(plbuf1),"hdf5={id=%u name=%s}",plugin->hdf5.filter->id,plugin->hdf5.filter->name);
if(plugin->codec.codec)
snprintf(plbuf2,sizeof(plbuf2),"codec={codecid=%s hdf5id=%u}",plugin->codec.codec->codecid,plugin->codec.codec->hdf5id);
snprintf(plbuf,4096,"plugin={%s %s}",plbuf1,plbuf2);
return plbuf;
}
static char*
printparams(size_t nparams, const unsigned* params)
{
static char ppbuf[4096];
if(nparams == 0)
snprintf(ppbuf,4096,"{0,%p}",params);
else
snprintf(ppbuf,4096,"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params));
return ppbuf;
}
static char*
printnczparams(const NCZ_Params p)
{
return printparams(p.nparams,p.params);
}
static const char*
printcodec(const NCZ_Codec c)
{
static char pcbuf[4096];
snprintf(pcbuf,sizeof(pcbuf),"{id=%s codec=%s}",
c.id,NULLIFY(c.codec));
return pcbuf;
}
static const char*
printhdf5(const NCZ_HDF5 h)
{
static char phbuf[4096];
snprintf(phbuf,sizeof(phbuf),"{id=%u visible=%s working=%s}",
h.id, printnczparams(h.visible), printnczparams(h.working));
return phbuf;
}
static const char*
printfilter(const NCZ_Filter* f)
{
@ -249,29 +174,16 @@ printfilter(const NCZ_Filter* f)
/* Forward */
static int NCZ_load_all_plugins(void);
static int NCZ_load_plugin_dir(const char* path);
static int NCZ_load_plugin(const char* path, NCZ_Plugin** plugp);
static int NCZ_unload_plugin(NCZ_Plugin* plugin);
static int NCZ_plugin_loaded(int filterid, NCZ_Plugin** pp);
static int NCZ_plugin_save(int filterid, NCZ_Plugin* p);
static int NCZ_filter_free(NCZ_Filter* spec);
static int NCZ_filter_hdf5_clear(NCZ_HDF5* spec);
static int NCZ_filter_codec_clear(NCZ_Codec* spec);
static int NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp);
static int getentries(const char* path, NClist* contents);
static int NCZ_split_plugin_path(const char* path0, NClist* list);
static int ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter);
static int paramnczclone(NCZ_Params* dst, const NCZ_Params* src);
static int paramclone(size_t nparams, unsigned** dstp, const unsigned* src);
#ifdef NAMEOPT
static int pluginnamecheck(const char* name);
#endif
/**************************************************/
/**
* @file
@ -367,7 +279,7 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t
if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew();
/* Before anything else, find the matching plugin */
if((stat = NCZ_plugin_loaded(id,&plugin))) goto done;
if((stat = NCZ_plugin_loaded((size_t)id,&plugin))) goto done;
if(plugin == NULL) {
stat = THROW(NC_ENOFILTER);
goto done;
@ -607,7 +519,7 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
/* For szip, the pixels_per_block parameter must not be greater
* than the number of elements in a chunk of data. */
size_t num_elem = 1;
int d;
size_t d;
for (d = 0; d < var->ndims; d++)
if (var->dim[d]->len)
num_elem *= var->dim[d]->len;
@ -736,7 +648,7 @@ NCZ_inq_filter_avail(int ncid, unsigned id)
ZTRACE(1,"ncid=%d id=%u",ncid,id);
if((stat = NCZ_filter_initialize())) goto done;
/* Check the available filters list */
if((stat = NCZ_plugin_loaded((int)id, &plug))) goto done;
if((stat = NCZ_plugin_loaded((size_t)id, &plug))) goto done;
if(plug == NULL || plug->incomplete)
stat = THROW(NC_ENOFILTER);
done:
@ -756,14 +668,11 @@ NCZ_filter_initialize(void)
if(NCZ_filter_initialized) goto done;
default_libs = nclistnew();
codec_defaults = nclistnew();
NCZ_filter_initialized = 1;
memset(loaded_plugins,0,sizeof(loaded_plugins));
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
if((stat = NCZ_load_all_plugins())) goto done;
#endif
done:
return ZUNTRACE(stat);
}
@ -772,75 +681,12 @@ int
NCZ_filter_finalize(void)
{
int stat = NC_NOERR;
int i;
ZTRACE(6,"");
if(!NCZ_filter_initialized) goto done;
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
/* Reclaim all loaded filters */
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: finalize reclaim:\n");
#endif
for(i=0;i<=loaded_plugins_max;i++) {
if(loaded_plugins[i]) {
NCZ_unload_plugin(loaded_plugins[i]);
loaded_plugins[i] = NULL;
}
}
/* Reclaim the codec defaults */
if(nclistlength(codec_defaults) > 0) {
for(i=0;i<nclistlength(codec_defaults);i++) {
struct CodecAPI* ca = (struct CodecAPI*)nclistget(codec_defaults,i);
nullfree(ca);
}
}
/* Reclaim the defaults library contents; Must occur as last act */
if(nclistlength(default_libs) > 0) {
for(i=0;i<nclistlength(default_libs);i++) {
NCPSharedLib* l = (NCPSharedLib*)nclistget(default_libs,i);
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: NCZ_filter_finalize: reclaim default_lib[i]=%p\n",l);
#endif
if(l != NULL) (void)ncpsharedlibfree(l);
}
}
#else
memset(loaded_plugins,0,sizeof(loaded_plugins));
#endif
nclistfree(default_libs); default_libs = NULL;
nclistfree(codec_defaults); codec_defaults = NULL;
done:
NCZ_filter_initialized = 0;
return ZUNTRACE(stat);
}
static int
NCZ_plugin_save(int filterid, NCZ_Plugin* p)
{
int stat = NC_NOERR;
ZTRACE(6,"filterid=%d p=%p",filterid,p);
if(filterid <= 0 || filterid >= H5Z_FILTER_MAX)
{stat = NC_EINVAL; goto done;}
if(filterid > loaded_plugins_max) loaded_plugins_max = filterid;
loaded_plugins[filterid] = p;
done:
return ZUNTRACE(stat);
}
static int
NCZ_plugin_loaded(int filterid, NCZ_Plugin** pp)
{
int stat = NC_NOERR;
struct NCZ_Plugin* plug = NULL;
ZTRACE(6,"filterid=%d",filterid);
if(filterid <= 0 || filterid >= H5Z_FILTER_MAX)
{stat = NC_EINVAL; goto done;}
if(filterid <= loaded_plugins_max)
plug = loaded_plugins[filterid];
if(pp) *pp = plug;
done:
return ZUNTRACEX(stat,"plugin=%p",*pp);
}
int
NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* chain, size_t inlen, void* indata, size_t* outlenp, void** outdatap, int encode)
{
@ -977,7 +823,7 @@ done:
int
NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex)
{
int i,stat = NC_NOERR;
int stat = NC_NOERR;
NCZ_Filter* filter = NULL;
const NCjson* jvalue = NULL;
NCZ_Plugin* plugin = NULL;
@ -1002,13 +848,7 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j
if(NCJunparse(jfilter,0,&codec.codec)<0) {stat = NC_EFILTER; goto done;}
/* Find the plugin for this filter */
for(i=0;i<=loaded_plugins_max;i++) {
if (!loaded_plugins[i]) continue;
if(!loaded_plugins[i] || !loaded_plugins[i]->codec.codec) continue; /* no plugin or no codec */
if(strcmp(NCJstring(jvalue), loaded_plugins[i]->codec.codec->codecid) == 0)
{plugin = loaded_plugins[i]; break;}
}
if((stat = NCZ_plugin_loaded_byname(NCJstring(jvalue),&plugin))) goto done;
/* Will always have a filter; possibly unknown */
if((filter = calloc(1,sizeof(NCZ_Filter)))==NULL) {stat = NC_ENOMEM; goto done;}
filter->chainindex = chainindex;
@ -1049,561 +889,6 @@ done:
return ZUNTRACE(stat);
}
/**************************************************/
/* Filter loading */
/*
Get entries in a path that is assumed to be a directory.
*/
#ifdef _WIN32
static int
getentries(const char* path, NClist* contents)
{
/* Iterate over the entries in the directory */
int ret = NC_NOERR;
errno = 0;
WIN32_FIND_DATA FindFileData;
HANDLE dir = NULL;
char* ffpath = NULL;
char* lpath = NULL;
size_t len;
char* d = NULL;
ZTRACE(6,"path=%s",path);
/* We need to process the path to make it work with FindFirstFile */
len = strlen(path);
/* Need to terminate path with '/''*' */
ffpath = (char*)malloc(len+2+1);
memcpy(ffpath,path,len);
if(path[len-1] != '/') {
ffpath[len] = '/';
len++;
}
ffpath[len] = '*'; len++;
ffpath[len] = '\0';
/* localize it */
if((ret = nczm_localize(ffpath,&lpath,LOCALIZE))) goto done;
dir = FindFirstFile(lpath, &FindFileData);
if(dir == INVALID_HANDLE_VALUE) {
/* Distinquish not-a-directory from no-matching-file */
switch (GetLastError()) {
case ERROR_FILE_NOT_FOUND: /* No matching files */ /* fall thru */
ret = NC_NOERR;
goto done;
case ERROR_DIRECTORY: /* not a directory */
default:
ret = NC_EEMPTY;
goto done;
}
}
do {
char* p = NULL;
const char* name = NULL;
name = FindFileData.cFileName;
if(strcmp(name,".")==0 || strcmp(name,"..")==0)
continue;
nclistpush(contents,strdup(name));
} while(FindNextFile(dir, &FindFileData));
done:
if(dir) FindClose(dir);
nullfree(lpath);
nullfree(ffpath);
nullfree(d);
errno = 0;
return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents));
}
#else /* !_WIN32 */
int
getentries(const char* path, NClist* contents)
{
int ret = NC_NOERR;
errno = 0;
DIR* dir = NULL;
ZTRACE(6,"path=%s",path);
dir = NCopendir(path);
if(dir == NULL)
{ret = (errno); goto done;}
for(;;) {
const char* name = NULL;
struct dirent* de = NULL;
errno = 0;
de = readdir(dir);
if(de == NULL)
{ret = (errno); goto done;}
if(strcmp(de->d_name,".")==0 || strcmp(de->d_name,"..")==0)
continue;
name = de->d_name;
nclistpush(contents,strdup(name));
}
done:
if(dir) NCclosedir(dir);
errno = 0;
return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents));
}
#endif /*_WIN32*/
static int
NCZ_load_all_plugins(void)
{
size_t i,j;
int ret = NC_NOERR;
char* pluginroots = NULL;
struct stat buf;
NClist* dirs = nclistnew();
char* defaultpluginpath = NULL;
ZTRACE(6,"");
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: NCZ_load_all_plugins\n");
#endif
/* Setup the plugin path default */
{
#ifdef _WIN32
const char* win32_root;
char dfalt[4096];
win32_root = getenv(WIN32_ROOT_ENV);
if(win32_root != NULL && strlen(win32_root) > 0) {
snprintf(dfalt,sizeof(dfalt),PLUGIN_DIR_WIN,win32_root);
defaultpluginpath = strdup(dfalt);
}
#else /*!_WIN32*/
defaultpluginpath = strdup(PLUGIN_DIR_UNIX);
#endif
}
/* Find the plugin directory root(s) */
pluginroots = getenv(PLUGIN_ENV); /* Usually HDF5_PLUGIN_PATH */
if(pluginroots != NULL && strlen(pluginroots) == 0) pluginroots = NULL;
if(pluginroots == NULL) {
pluginroots = defaultpluginpath;
}
assert(pluginroots != NULL);
ZTRACEMORE(6,"pluginroots=%s",(pluginroots?pluginroots:"null"));
if((ret = NCZ_split_plugin_path(pluginroots,dirs))) goto done;
/* Add the default to end of the dirs list if not already there */
if(!nclistmatch(dirs,defaultpluginpath,0)) {
nclistpush(dirs,defaultpluginpath);
defaultpluginpath = NULL;
}
for(i=0;i<nclistlength(dirs);i++) {
const char* dir = (const char*)nclistget(dirs,i);
/* Make sure the root is actually a directory */
errno = 0;
ret = NCstat(dir, &buf);
#if 1
ZTRACEMORE(6,"stat: ret=%d, errno=%d st_mode=%d",ret,errno,buf.st_mode);
#endif
if(ret < 0) {errno = 0; ret = NC_NOERR; continue;} /* ignore unreadable directories */
if(! S_ISDIR(buf.st_mode))
ret = NC_EINVAL;
if(ret) goto done;
/* Try to load plugins from this directory */
if((ret = NCZ_load_plugin_dir(dir))) goto done;
}
#ifdef DEBUGL
{ int i;
fprintf(stderr,"codec_defaults:");
for(i=0;i<nclistlength(codec_defaults);i++) {
struct CodecAPI* codec = (struct CodecAPI*)nclistget(codec_defaults,i);
fprintf(stderr," %d",codec->codec->hdf5id);
}
fprintf(stderr,"\n");
}
#endif
if(nclistlength(codec_defaults)) { /* Try to provide default for any HDF5 filters without matching Codec. */
/* Search the defaults */
for(j=0;j<nclistlength(codec_defaults);j++) {
struct CodecAPI* dfalt = (struct CodecAPI*)nclistget(codec_defaults,j);
if(dfalt->codec != NULL) {
const NCZ_codec_t* codec = dfalt->codec;
int hdf5id = codec->hdf5id;
NCZ_Plugin* p = NULL;
if(hdf5id < 0 || hdf5id > loaded_plugins_max) {ret = NC_EFILTER; goto done;}
p = loaded_plugins[hdf5id]; /* get candidate */
if(p != NULL && p->hdf5.filter != NULL
&& p->codec.codec == NULL) {
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: plugin defaulted: id=%u, codec=%s src=%s\n",hdf5id,codec->codecid,dfalt->codeclib->path);
#endif
p->codec.codec = codec;
p->codec.codeclib = dfalt->codeclib;
p->codec.defaulted = 1;
}
}
}
}
/* Mark all plugins for which we do not have both HDF5 and codec */
{
int i;
NCZ_Plugin* p;
for(i=0;i<loaded_plugins_max;i++) {
if((p = loaded_plugins[i]) != NULL) {
if(p->hdf5.filter == NULL || p->codec.codec == NULL) {
/* mark this entry as incomplete */
p->incomplete = 1;
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: Incomplete plugin: id=%u; reasons: %s %s\n",i,
(p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":""));
#endif
}
#ifdef DEBUGL
else
fprintf(stderr,">>> DEBUGL: plugin accepted: id=%u\n",i);
#endif
}
}
}
/* Iniitalize all remaining plugins */
{
int i;
NCZ_Plugin* p;
for(i=0;i<loaded_plugins_max;i++) {
if((p = loaded_plugins[i]) != NULL) {
if(p->incomplete) continue;
if(p->hdf5.filter != NULL && p->codec.codec != NULL) {
if(p->codec.codec && p->codec.codec->NCZ_codec_initialize)
p->codec.codec->NCZ_codec_initialize();
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id);
#endif
}
}
}
}
done:
nullfree(defaultpluginpath);
nclistfreeall(dirs);
errno = 0;
return ZUNTRACE(ret);
}
static int
NCZ_split_plugin_path(const char* path0, NClist* list)
{
int i,stat = NC_NOERR;
char* path = NULL;
char* p;
int count;
size_t plen;
#ifdef _WIN32
const char* seps = ";";
#else
const char* seps = ";:";
#endif
if(path0 == NULL || path0[0] == '\0') goto done;
plen = strlen(path0);
if((path = malloc(plen+1+1))==NULL) {stat = NC_ENOMEM; goto done;}
memcpy(path,path0,plen);
path[plen] = '\0'; path[plen+1] = '\0'; /* double null term */
for(count=0,p=path;*p;p++) {
if(strchr(seps,*p) != NULL) {*p = '\0'; count++;}
}
count++; /* for last piece */
for(p=path,i=0;i<count;i++) {
size_t len = strlen(p);
if(len > 0)
nclistpush(list,strdup(p));
p = p+len+1; /* point to next piece */
}
done:
nullfree(path);
return stat;
}
/* Load all the filters within a specified directory */
static int
NCZ_load_plugin_dir(const char* path)
{
size_t i;
int stat = NC_NOERR;
size_t pathlen;
NClist* contents = nclistnew();
char* file = NULL;
ZTRACE(7,"path=%s",path);
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: NCZ_load_plugin_dir: path=%s\n",path);
#endif
if(path == NULL) {stat = NC_EINVAL; goto done;}
pathlen = strlen(path);
if(pathlen == 0) {stat = NC_EINVAL; goto done;}
if((stat = getentries(path,contents))) goto done;
for(i=0;i<nclistlength(contents);i++) {
const char* name = (const char*)nclistget(contents,i);
size_t nmlen = strlen(name);
size_t flen = pathlen+1+nmlen+1;
int id;
NCZ_Plugin* plugin = NULL;
assert(nmlen > 0);
nullfree(file); file = NULL;
if((file = (char*)malloc(flen))==NULL) {stat = NC_ENOMEM; goto done;}
file[0] = '\0';
strlcat(file,path,flen);
strlcat(file,"/",flen);
strlcat(file,name,flen);
/* See if can load the file */
stat = NCZ_load_plugin(file,&plugin);
switch (stat) {
case NC_NOERR: break;
case NC_ENOFILTER: case NC_ENOTFOUND:
stat = NC_NOERR;
break; /* will cause it to be ignored */
default: goto done;
}
if(plugin != NULL) {
id = plugin->hdf5.filter->id;
if(loaded_plugins[id] == NULL) {
loaded_plugins[id] = plugin;
if(id > loaded_plugins_max) loaded_plugins_max = id;
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: plugin loaded: %s\n",printplugin(plugin));
#endif
} else {
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: plugin duplicate: %s\n",printplugin(plugin));
#endif
NCZ_unload_plugin(plugin); /* its a duplicate */
}
} else
stat = NC_NOERR; /*ignore failure */
}
done:
nullfree(file);
nclistfreeall(contents);
return ZUNTRACE(stat);
}
int
loadcodecdefaults(const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp)
{
int stat = NC_NOERR;
int lib_used = 0;
nclistpush(default_libs,lib);
for(;*cp;cp++) {
struct CodecAPI* c0;
#ifdef DEBUGL
fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id);
#endif
c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI));
if(c0 == NULL) {stat = NC_ENOMEM; goto done;}
c0->codec = *cp;
c0->codeclib = lib;
lib_used = 1; /* remember */
nclistpush(codec_defaults,c0); c0 = NULL;
}
done:
if(lib_usedp) *lib_usedp = lib_used;
return stat;
}
static int
NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
{
int stat = NC_NOERR;
NCZ_Plugin* plugin = NULL;
const H5Z_class2_t* h5class = NULL;
H5PL_type_t h5type = 0;
const NCZ_codec_t** cp = NULL;
const NCZ_codec_t* codec = NULL;
NCPSharedLib* lib = NULL;
int flags = NCP_GLOBAL;
int h5id = -1;
assert(path != NULL && strlen(path) > 0 && plugp != NULL);
ZTRACE(8,"path=%s",path);
if(plugp) *plugp = NULL;
#if defined NAMEOPT || defined _WIN32
/*triage because visual studio does a popup if the file will not load*/
if(!pluginnamecheck(path)) {stat = NC_ENOFILTER; goto done;}
#endif
/* load the shared library */
if((stat = ncpsharedlibnew(&lib))) goto done;
if((stat = ncpload(lib,path,flags))) goto done;
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s lib=%p\n",path,lib);
#endif
/* See what we have */
{
const H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type");
const H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info");
const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info");
const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults");
if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL)
{stat = THROW(NC_ENOFILTER); goto done;}
/* We can have cpd or we can have (gpt && gpi && npi) but not both sets */
if(cpd != NULL) {
cp = (const NCZ_codec_t**)cpd();
} else {/* cpd => !gpt && !gpi && !npi */
if(gpt != NULL && gpi != NULL) { /* get HDF5 info */
h5type = gpt();
h5class = gpi();
/* Verify */
if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;}
if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;}
}
if(npi != NULL) {/* get Codec info */
codec = npi();
/* Verify */
if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;}
if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;}
}
}
}
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: load: %s:",path);
if(h5class) fprintf(stderr,">>> %u",(unsigned)h5class->id);
if(codec) fprintf(stderr,">>> %u/%s",codec->hdf5id,codec->codecid);
fprintf(stderr,">>> \n");
#endif
/* Handle defaults separately */
if(cp != NULL) {
int used = 0;
#ifdef DEBUGL
fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp);
#endif
if((stat = loadcodecdefaults(path,cp,lib,&used))) goto done;
if(used) lib = NULL;
goto done;
}
if(h5class != NULL && codec != NULL) {
/* Verify consistency of the HDF5 and the Codec */
if(h5class->id != codec->hdf5id) goto done; /* ignore */
}
/* There are several cases to consider:
1. This library has both HDF5 API and Codec API => merge
2. This library has HDF5 API only and Codec API was already found in another library => merge
3. This library has Codec API only and HDF5 API was already found in another library => merge
*/
/* Get any previous plugin entry for this id; may be NULL */
if(h5class != NULL) {
h5id = h5class->id;
if((stat = NCZ_plugin_loaded(h5class->id,&plugin))) goto done;
} else if(codec != NULL) {
h5id = codec->hdf5id;
if((stat = NCZ_plugin_loaded(codec->hdf5id,&plugin))) goto done;
}
if(plugin == NULL) {
/* create new entry */
if((plugin = (NCZ_Plugin*)calloc(1,sizeof(NCZ_Plugin)))==NULL) {stat = NC_ENOMEM; goto done;}
}
/* Fill in the plugin */
if(h5class != NULL && plugin->hdf5.filter == NULL) {
plugin->hdf5.filter = h5class;
plugin->hdf5.hdf5lib = lib;
lib = NULL;
}
if(codec != NULL && plugin->codec.codec == NULL) {
plugin->codec.codec = codec;
plugin->codec.codeclib = lib;
lib = NULL;
}
#ifdef DEBUGL
if(plugin)
fprintf(stderr,">>> DEBUGL: load_plugin: %s\n",printplugin(plugin));
#endif
/* Cleanup */
if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) /* Works for NULL case also */
plugin->codec.codeclib = NULL;
if((stat=NCZ_plugin_save(h5id,plugin))) goto done;
plugin = NULL;
done:
if(lib)
(void)ncpsharedlibfree(lib);
if(plugin) NCZ_unload_plugin(plugin);
return ZUNTRACEX(stat,"plug=%p",*plugp);
}
static int
NCZ_unload_plugin(NCZ_Plugin* plugin)
{
ZTRACE(9,"plugin=%p",plugin);
if(plugin) {
#ifdef DEBUGL
fprintf(stderr,">>> DEBUGL: unload: %s\n",printplugin(plugin));
#endif
if(plugin->codec.codec && plugin->codec.codec->NCZ_codec_finalize)
plugin->codec.codec->NCZ_codec_finalize();
if(plugin->hdf5.filter != NULL) loaded_plugins[plugin->hdf5.filter->id] = NULL;
if(plugin->hdf5.hdf5lib != NULL) (void)ncpsharedlibfree(plugin->hdf5.hdf5lib);
if(!plugin->codec.defaulted && plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib);
memset(plugin,0,sizeof(NCZ_Plugin));
free(plugin);
}
return ZUNTRACE(NC_NOERR);
}
#ifdef NAMEOPT
static int
pluginnamecheck(const char* name)
{
size_t count,len;
long i;
const char* p;
if(name == NULL) return 0;
/* get basename */
p = strrchr(name,'/');
if(p != NULL) name = (p+1);
len = strlen(name);
if(len == 0) return 0;
i = (long)(len-1);
count = 1;
p = name+i;
for(;i>=0;i--,count++,p--) {
char c = *p;
if(c == '/') break;
if(c == '.') {
if(count >= 3 && memcmp(p,".so",3)==0) return 1;
if(count >= 4 && memcmp(p,".dll",4)==0) return 1;
if(count >= 6 && memcmp(p,".dylib",6)==0) return 1;
}
}
return 0;
}
#endif
/**************************************************/
/* _Codecs attribute */

6
libnczarr/zfilter.h
View File

@ -19,12 +19,6 @@
/*Mnemonic*/
#define ENCODING 1
/* list of environment variables to check for plugin roots */
#define PLUGIN_ENV "HDF5_PLUGIN_PATH"
#define PLUGIN_DIR_UNIX "/usr/local/hdf5/plugin"
#define PLUGIN_DIR_WIN "%s/hdf5/lib/plugin"
#define WIN32_ROOT_ENV "ALLUSERSPROFILE"
/* Opaque */
struct NCZ_Filter;

12
libnczarr/zinternal.c
View File

@ -113,7 +113,7 @@ find_var_dim_max_length(NC_GRP_INFO_T *grp, int varid, int dimid,
*maxlen = 0;
/* Find this var. */
var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid);
var = (NC_VAR_INFO_T*)ncindexith(grp->vars,(size_t)varid);
if (!var) return NC_ENOTVAR;
assert(var->hdr.id == varid);
@ -230,7 +230,7 @@ ncz_find_dim_len(NC_GRP_INFO_T *grp, int dimid, size_t **len)
{
NC_VAR_INFO_T *var;
int retval;
int i;
size_t i;
assert(grp && len);
LOG((3, "%s: grp->name %s dimid %d", __func__, grp->hdr.name, dimid));
@ -382,7 +382,7 @@ static int
close_dims(NC_GRP_INFO_T *grp)
{
NC_DIM_INFO_T *dim;
int i;
size_t i;
for (i = 0; i < ncindexsize(grp->dim); i++)
{
@ -417,7 +417,7 @@ close_dims(NC_GRP_INFO_T *grp)
static int
close_types(NC_GRP_INFO_T *grp)
{
int i;
size_t i;
for (i = 0; i < ncindexsize(grp->type); i++)
{
@ -456,7 +456,7 @@ close_types(NC_GRP_INFO_T *grp)
static int
ncz_rec_grp_NCZ_del(NC_GRP_INFO_T *grp)
{
int i;
size_t i;
int retval;
assert(grp && grp->format_grp_info);
@ -608,7 +608,7 @@ ncz_find_grp_var_att(int ncid, int varid, const char *name, int attnum,
if (att)
{
my_att = use_name ? (NC_ATT_INFO_T *)ncindexlookup(attlist, my_norm_name) :
(NC_ATT_INFO_T *)ncindexith(attlist, attnum);
(NC_ATT_INFO_T *)ncindexith(attlist, (size_t)attnum);
if (!my_att)
return NC_ENOTATT;
}

6
libnczarr/zmap.c
View File

@ -292,8 +292,8 @@ nczm_divide_at(const char* key, int nsegs, char** prefixp, char** suffixp)
size_t len, i;
ptrdiff_t delta;
const char* p;
int abssegs = (nsegs >= 0 ?nsegs: -nsegs);
int presegs = 0;
size_t abssegs = (size_t)(nsegs >= 0 ?nsegs: -nsegs);
size_t presegs = 0;
/* Special case */
if(key == NULL || strlen(key) == 0) goto done;
@ -439,7 +439,7 @@ nczm_segment1(const char* path, char** seg1p)
delta = (q-p);
if((seg1 = (char*)malloc((size_t)delta+1))==NULL)
{ret = NC_ENOMEM; goto done;}
memcpy(seg1,p,delta);
memcpy(seg1,p,(size_t)delta);
seg1[delta] = '\0';
if(seg1p) {*seg1p = seg1; seg1 = NULL;}

772
libnczarr/zplugins.c Normal file
View File

@ -0,0 +1,772 @@
/* Copyright 2003-2018, University Corporation for Atmospheric
* Research. See the COPYRIGHT file for copying and redistribution
* conditions.
*/
/**
* @file @internal netcdf-4 functions for the plugin list.
*
* @author Dennis Heimbigner
*/
#include "config.h"
#include <stddef.h>
#include <stdlib.h>
#include "zincludes.h"
#include "ncpathmgr.h"
#include "ncpoco.h"
#include "netcdf_filter.h"
#include "netcdf_filter_build.h"
#include "zfilter.h"
#include "ncplugins.h"
#include "zplugins.h"
#ifdef _WIN32
#include <windows.h>
#endif
/**************************************************/
/* Forward */
static int NCZ_load_plugin(const char* path, NCZ_Plugin** plugp);
static int NCZ_unload_plugin(NCZ_Plugin* plugin);
static int NCZ_load_plugin_dir(const char* path);
static int NCZ_plugin_save(size_t filterid, NCZ_Plugin* p);
static int getentries(const char* path, NClist* contents);
static int loadcodecdefaults(const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp);
#if defined(NAMEOPT) || defined(_WIN32)
static int pluginnamecheck(const char* name);
#endif
/**************************************************/
/**
* @file
* @internal
* Internal netcdf zarr plugin path functions.
*
* @author Dennis Heimbigner
*/
/**
* This function is called as part of nc_plugin_path_initialize.
* Its purpose is to initialize the plugin state.
*
* @return NC_NOERR
*
* @author Dennis Heimbigner
*/
int
NCZ_plugin_path_initialize(void)
{
int stat = NC_NOERR;
struct NCglobalstate* gs = NC_getglobalstate();
gs->zarr.pluginpaths = nclistnew();
gs->zarr.default_libs = nclistnew();
gs->zarr.codec_defaults = nclistnew();
gs->zarr.loaded_plugins = (struct NCZ_Plugin**)calloc(H5Z_FILTER_MAX+1,sizeof(struct NCZ_Plugin*));
if(gs->zarr.loaded_plugins == NULL) {stat = NC_ENOMEM; goto done;}
done:
return stat;
}
/**
* This function is called as part of nc_plugin_path_finalize.
* Its purpose is to clean-up the plugin state.
*
* @return NC_NOERR
*
* @author Dennis Heimbigner
*/
int
NCZ_plugin_path_finalize(void)
{
int stat = NC_NOERR;
size_t i;
struct NCglobalstate* gs = NC_getglobalstate();
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
/* Reclaim all loaded filters */
for(i=1;i<=gs->zarr.loaded_plugins_max;i++) {
if(gs->zarr.loaded_plugins[i]) {
NCZ_unload_plugin(gs->zarr.loaded_plugins[i]);
gs->zarr.loaded_plugins[i] = NULL;
}
}
/* Reclaim the codec defaults */
if(nclistlength(gs->zarr.codec_defaults) > 0) {
for(i=0;i<nclistlength(gs->zarr.codec_defaults);i++) {
struct CodecAPI* ca = (struct CodecAPI*)nclistget(gs->zarr.codec_defaults,i);
nullfree(ca);
}
}
/* Reclaim the defaults library contents; Must occur as last act */
if(nclistlength(gs->zarr.default_libs) > 0) {
for(i=0;i<nclistlength(gs->zarr.default_libs);i++) {
NCPSharedLib* l = (NCPSharedLib*)nclistget(gs->zarr.default_libs,i);
if(l != NULL) (void)ncpsharedlibfree(l);
}
}
#endif
gs->zarr.loaded_plugins_max = 0;
nullfree(gs->zarr.loaded_plugins); gs->zarr.loaded_plugins = NULL;
nclistfree(gs->zarr.default_libs); gs->zarr.default_libs = NULL;
nclistfree(gs->zarr.codec_defaults); gs->zarr.codec_defaults = NULL;
nclistfreeall(gs->zarr.pluginpaths); gs->zarr.pluginpaths = NULL;
return THROW(stat);
}
/**
* Return the length of the current sequence of directories
* in the internal global plugin path list.
* @param ndirsp length is returned here
* @return NC_NOERR | NC_EXXX
*
* @author Dennis Heimbigner
*/
int
NCZ_plugin_path_ndirs(size_t* ndirsp)
{
int stat = NC_NOERR;
size_t ndirs = 0;
struct NCglobalstate* gs = NC_getglobalstate();
if(gs->zarr.pluginpaths == NULL) gs->zarr.pluginpaths = nclistnew(); /* suspenders and belt */
ndirs = nclistlength(gs->zarr.pluginpaths);
if(ndirsp) *ndirsp = ndirs;
done:
return THROW(stat);
}
/**
* Return the current sequence of directories in the internal global
* plugin path list. Since this function does not modify the plugin path,
* it can be called at any time.
* @param dirs pointer to an NCPluginList object
* @return NC_NOERR | NC_EXXX
* @author Dennis Heimbigner
*
* WARNING: if dirs->dirs is NULL, then space for the directory
* vector will be allocated. If not NULL, then the specified space will
* be overwritten with the vector.
*
* @author: Dennis Heimbigner
*/
int
NCZ_plugin_path_get(NCPluginList* dirs)
{
int stat = NC_NOERR;
struct NCglobalstate* gs = NC_getglobalstate();
if(dirs == NULL) {stat = NC_EINVAL; goto done;}
if(gs->zarr.pluginpaths == NULL) gs->zarr.pluginpaths = nclistnew(); /* suspenders and belt */
dirs->ndirs = nclistlength(gs->zarr.pluginpaths);
if(dirs->dirs == NULL && dirs->ndirs > 0) {
if((dirs->dirs = (char**)calloc(dirs->ndirs,sizeof(char*)))==NULL)
{stat = NC_ENOMEM; goto done;}
}
if(dirs->ndirs > 0) {
size_t i;
for(i=0;i<dirs->ndirs;i++) {
const char* dir = (const char*)nclistget(gs->zarr.pluginpaths,i);
dirs->dirs[i] = nulldup(dir);
}
}
done:
return THROW(stat);
}
/**
* Empty the current internal path sequence
* and replace with the sequence of directories argument.
* Using a dirs->ndirs argument of 0 will clear the set of plugin dirs.
*
* @param dirs to overwrite the current internal dir list
* @return NC_NOERR | NC_EXXX
*
* @author Dennis Heimbigner
*/
int
NCZ_plugin_path_set(NCPluginList* dirs)
{
int stat = NC_NOERR;
struct NCglobalstate* gs = NC_getglobalstate();
if(dirs == NULL) {stat = NC_EINVAL; goto done;}
if(dirs->ndirs > 0 && dirs->dirs == NULL) {stat = NC_EINVAL; goto done;}
/* Clear the current dir list */
nclistfreeall(gs->zarr.pluginpaths);
gs->zarr.pluginpaths = nclistnew();
if(dirs->ndirs > 0) {
size_t i;
for(i=0;i<dirs->ndirs;i++) {
nclistpush(gs->zarr.pluginpaths,nulldup(dirs->dirs[i]));
}
}
done:
return THROW(stat);
}
/**************************************************/
/* Filter<->Plugin interface */
int
NCZ_load_all_plugins(void)
{
int ret = NC_NOERR;
size_t i,j;
struct NCglobalstate* gs = NC_getglobalstate();
struct stat buf;
NClist* dirs = nclistnew();
char* defaultpluginpath = NULL;
ZTRACE(6,"");
for(i=0;i<nclistlength(gs->pluginpaths);i++) {
const char* dir = (const char*)nclistget(gs->pluginpaths,i);
/* Make sure the root is actually a directory */
errno = 0;
ret = NCstat(dir, &buf);
#if 1
ZTRACEMORE(6,"stat: ret=%d, errno=%d st_mode=%d",ret,errno,buf.st_mode);
#endif
if(ret < 0) {errno = 0; ret = NC_NOERR; continue;} /* ignore unreadable directories */
if(! S_ISDIR(buf.st_mode))
ret = NC_EINVAL;
if(ret) goto done;
/* Try to load plugins from this directory */
if((ret = NCZ_load_plugin_dir(dir))) goto done;
}
if(nclistlength(gs->zarr.codec_defaults)) { /* Try to provide default for any HDF5 filters without matching Codec. */
/* Search the defaults */
for(j=0;j<nclistlength(gs->zarr.codec_defaults);j++) {
struct CodecAPI* dfalt = (struct CodecAPI*)nclistget(gs->zarr.codec_defaults,j);
if(dfalt->codec != NULL) {
const NCZ_codec_t* codec = dfalt->codec;
size_t hdf5id = codec->hdf5id;
NCZ_Plugin* p = NULL;
if(hdf5id <= 0 || hdf5id > gs->zarr.loaded_plugins_max) {ret = NC_EFILTER; goto done;}
p = gs->zarr.loaded_plugins[hdf5id]; /* get candidate */
if(p != NULL && p->hdf5.filter != NULL
&& p->codec.codec == NULL) {
p->codec.codec = codec;
p->codec.codeclib = dfalt->codeclib;
p->codec.defaulted = 1;
}
}
}
}
/* Mark all plugins for which we do not have both HDF5 and codec */
{
size_t i;
NCZ_Plugin* p;
for(i=1;i<gs->zarr.loaded_plugins_max;i++) {
if((p = gs->zarr.loaded_plugins[i]) != NULL) {
if(p->hdf5.filter == NULL || p->codec.codec == NULL) {
/* mark this entry as incomplete */
p->incomplete = 1;
}
}
}
}
/* Iniitalize all remaining plugins */
{
size_t i;
NCZ_Plugin* p;
for(i=1;i<gs->zarr.loaded_plugins_max;i++) {
if((p = gs->zarr.loaded_plugins[i]) != NULL) {
if(p->incomplete) continue;
if(p->hdf5.filter != NULL && p->codec.codec != NULL) {
if(p->codec.codec && p->codec.codec->NCZ_codec_initialize)
p->codec.codec->NCZ_codec_initialize();
}
}
}
}
done:
nullfree(defaultpluginpath);
nclistfreeall(dirs);
errno = 0;
return ZUNTRACE(ret);
}
/* Load all the filters within a specified directory */
static int
NCZ_load_plugin_dir(const char* path)
{
size_t i;
int stat = NC_NOERR;
size_t pathlen;
NClist* contents = nclistnew();
char* file = NULL;
struct NCglobalstate* gs = NC_getglobalstate();
ZTRACE(7,"path=%s",path);
if(path == NULL) {stat = NC_EINVAL; goto done;}
pathlen = strlen(path);
if(pathlen == 0) {stat = NC_EINVAL; goto done;}
if((stat = getentries(path,contents))) goto done;
for(i=0;i<nclistlength(contents);i++) {
const char* name = (const char*)nclistget(contents,i);
size_t nmlen = strlen(name);
size_t flen = pathlen+1+nmlen+1;
size_t id;
NCZ_Plugin* plugin = NULL;
assert(nmlen > 0);
nullfree(file); file = NULL;
if((file = (char*)malloc(flen))==NULL) {stat = NC_ENOMEM; goto done;}
file[0] = '\0';
strlcat(file,path,flen);
strlcat(file,"/",flen);
strlcat(file,name,flen);
/* See if can load the file */
stat = NCZ_load_plugin(file,&plugin);
switch (stat) {
case NC_NOERR: break;
case NC_ENOFILTER: case NC_ENOTFOUND:
stat = NC_NOERR;
break; /* will cause it to be ignored */
default: goto done;
}
if(plugin != NULL) {
id = (size_t)plugin->hdf5.filter->id;
if(gs->zarr.loaded_plugins[id] == NULL) {
gs->zarr.loaded_plugins[id] = plugin;
if(id > gs->zarr.loaded_plugins_max) gs->zarr.loaded_plugins_max = id;
} else {
NCZ_unload_plugin(plugin); /* its a duplicate */
}
} else
stat = NC_NOERR; /*ignore failure */
}
done:
nullfree(file);
nclistfreeall(contents);
return ZUNTRACE(stat);
}
int
NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
{
int stat = NC_NOERR;
NCZ_Plugin* plugin = NULL;
const H5Z_class2_t* h5class = NULL;
H5PL_type_t h5type = 0;
const NCZ_codec_t** cp = NULL;
const NCZ_codec_t* codec = NULL;
NCPSharedLib* lib = NULL;
int flags = NCP_GLOBAL;
size_t h5id = 0;
assert(path != NULL && strlen(path) > 0 && plugp != NULL);
ZTRACE(8,"path=%s",path);
if(plugp) *plugp = NULL;
#if defined NAMEOPT || defined _WIN32
/*triage because visual studio does a popup if the file will not load*/
if(!pluginnamecheck(path)) {stat = NC_ENOFILTER; goto done;}
#endif
/* load the shared library */
if((stat = ncpsharedlibnew(&lib))) goto done;
if((stat = ncpload(lib,path,flags))) goto done;
/* See what we have */
{
const H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type");
const H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info");
const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info");
const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults");
if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL)
{stat = THROW(NC_ENOFILTER); goto done;}
/* We can have cpd or we can have (gpt && gpi && npi) but not both sets */
if(cpd != NULL) {
cp = (const NCZ_codec_t**)cpd();
} else {/* cpd => !gpt && !gpi && !npi */
if(gpt != NULL && gpi != NULL) { /* get HDF5 info */
h5type = gpt();
h5class = gpi();
/* Verify */
if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;}
if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;}
}
if(npi != NULL) {/* get Codec info */
codec = npi();
/* Verify */
if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;}
if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;}
}
}
}
/* Handle defaults separately */
if(cp != NULL) {
int used = 0;
if((stat = loadcodecdefaults(path,cp,lib,&used))) goto done;
if(used) lib = NULL;
goto done;
}
if(h5class != NULL && codec != NULL) {
/* Verify consistency of the HDF5 and the Codec */
if(((size_t)h5class->id) != codec->hdf5id) goto done; /* ignore */
}
/* There are several cases to consider:
1. This library has both HDF5 API and Codec API => merge
2. This library has HDF5 API only and Codec API was already found in another library => merge
3. This library has Codec API only and HDF5 API was already found in another library => merge
*/
/* Get any previous plugin entry for this id; may be NULL */
if(h5class != NULL) {
h5id = (size_t)h5class->id;
if((stat = NCZ_plugin_loaded(h5id,&plugin))) goto done;
} else if(codec != NULL) {
h5id = (size_t)codec->hdf5id;
if((stat = NCZ_plugin_loaded(h5id,&plugin))) goto done;
}
if(plugin == NULL) {
/* create new entry */
if((plugin = (NCZ_Plugin*)calloc(1,sizeof(NCZ_Plugin)))==NULL) {stat = NC_ENOMEM; goto done;}
}
/* Fill in the plugin */
if(h5class != NULL && plugin->hdf5.filter == NULL) {
plugin->hdf5.filter = h5class;
plugin->hdf5.hdf5lib = lib;
lib = NULL;
}
if(codec != NULL && plugin->codec.codec == NULL) {
plugin->codec.codec = codec;
plugin->codec.codeclib = lib;
lib = NULL;
}
/* Cleanup */
if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) /* Works for NULL case also */
plugin->codec.codeclib = NULL;
if((stat=NCZ_plugin_save(h5id,plugin))) goto done;
plugin = NULL;
done:
if(lib)
(void)ncpsharedlibfree(lib);
if(plugin) NCZ_unload_plugin(plugin);
return ZUNTRACEX(stat,"plug=%p",*plugp);
}
int
NCZ_unload_plugin(NCZ_Plugin* plugin)
{
struct NCglobalstate* gs = NC_getglobalstate();
ZTRACE(9,"plugin=%p",plugin);
if(plugin) {
if(plugin->codec.codec && plugin->codec.codec->NCZ_codec_finalize)
plugin->codec.codec->NCZ_codec_finalize();
if(plugin->hdf5.filter != NULL) gs->zarr.loaded_plugins[plugin->hdf5.filter->id] = NULL;
if(plugin->hdf5.hdf5lib != NULL) (void)ncpsharedlibfree(plugin->hdf5.hdf5lib);
if(!plugin->codec.defaulted && plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib);
memset(plugin,0,sizeof(NCZ_Plugin));
free(plugin);
}
return ZUNTRACE(NC_NOERR);
}
int
NCZ_plugin_loaded(size_t filterid, NCZ_Plugin** pp)
{
int stat = NC_NOERR;
struct NCglobalstate* gs = NC_getglobalstate();
struct NCZ_Plugin* plug = NULL;
ZTRACE(6,"filterid=%d",filterid);
if(filterid <= 0 || filterid >= H5Z_FILTER_MAX)
{stat = NC_EINVAL; goto done;}
if(filterid <= gs->zarr.loaded_plugins_max)
plug = gs->zarr.loaded_plugins[filterid];
if(pp) *pp = plug;
done:
return ZUNTRACEX(stat,"plugin=%p",*pp);
}
int
NCZ_plugin_loaded_byname(const char* name, NCZ_Plugin** pp)
{
int stat = NC_NOERR;
size_t i;
struct NCZ_Plugin* plug = NULL;
struct NCglobalstate* gs = NC_getglobalstate();
ZTRACE(6,"pluginname=%s",name);
if(name == NULL) {stat = NC_EINVAL; goto done;}
for(i=1;i<=gs->zarr.loaded_plugins_max;i++) {
if (!gs->zarr.loaded_plugins[i]) continue;
if(!gs->zarr.loaded_plugins[i] || !gs->zarr.loaded_plugins[i]->codec.codec) continue; /* no plugin or no codec */
if(strcmp(name, gs->zarr.loaded_plugins[i]->codec.codec->codecid) == 0)
{plug = gs->zarr.loaded_plugins[i]; break;}
}
if(pp) *pp = plug;
done:
return ZUNTRACEX(stat,"plugin=%p",*pp);
}
static int
NCZ_plugin_save(size_t filterid, NCZ_Plugin* p)
{
int stat = NC_NOERR;
struct NCglobalstate* gs = NC_getglobalstate();
ZTRACE(6,"filterid=%d p=%p",filterid,p);
if(filterid <= 0 || filterid >= H5Z_FILTER_MAX)
{stat = NC_EINVAL; goto done;}
if(filterid > gs->zarr.loaded_plugins_max) gs->zarr.loaded_plugins_max = filterid;
gs->zarr.loaded_plugins[filterid] = p;
done:
return ZUNTRACE(stat);
}
static int
loadcodecdefaults(const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp)
{
int stat = NC_NOERR;
int lib_used = 0;
struct NCglobalstate* gs = NC_getglobalstate();
nclistpush(gs->zarr.default_libs,lib);
for(;*cp;cp++) {
struct CodecAPI* c0;
c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI));
if(c0 == NULL) {stat = NC_ENOMEM; goto done;}
c0->codec = *cp;
c0->codeclib = lib;
lib_used = 1; /* remember */
nclistpush(gs->zarr.codec_defaults,c0); c0 = NULL;
}
done:
if(lib_usedp) *lib_usedp = lib_used;
return stat;
}
#if defined(NAMEOPT) || defined(_WIN32)
static int
pluginnamecheck(const char* name)
{
size_t count,len;
long i;
const char* p;
if(name == NULL) return 0;
/* get basename */
p = strrchr(name,'/');
if(p != NULL) name = (p+1);
len = strlen(name);
if(len == 0) return 0;
i = (long)(len-1);
count = 1;
p = name+i;
for(;i>=0;i--,count++,p--) {
char c = *p;
if(c == '/') break;
if(c == '.') {
if(count >= 3 && memcmp(p,".so",3)==0) return 1;
if(count >= 4 && memcmp(p,".dll",4)==0) return 1;
if(count >= 6 && memcmp(p,".dylib",6)==0) return 1;
}
}
return 0;
}
#endif
/**************************************************/
/*
Get entries in a path that is assumed to be a directory.
*/
#ifdef _WIN32
static int
getentries(const char* path, NClist* contents)
{
/* Iterate over the entries in the directory */
int ret = NC_NOERR;
errno = 0;
WIN32_FIND_DATA FindFileData;
HANDLE dir = NULL;
char* ffpath = NULL;
char* lpath = NULL;
size_t len;
char* d = NULL;
ZTRACE(6,"path=%s",path);
/* We need to process the path to make it work with FindFirstFile */
len = strlen(path);
/* Need to terminate path with '/''*' */
ffpath = (char*)malloc(len+2+1);
memcpy(ffpath,path,len);
if(path[len-1] != '/') {
ffpath[len] = '/';
len++;
}
ffpath[len] = '*'; len++;
ffpath[len] = '\0';
/* localize it */
if((ret = nczm_localize(ffpath,&lpath,LOCALIZE))) goto done;
dir = FindFirstFile(lpath, &FindFileData);
if(dir == INVALID_HANDLE_VALUE) {
/* Distinquish not-a-directory from no-matching-file */
switch (GetLastError()) {
case ERROR_FILE_NOT_FOUND: /* No matching files */ /* fall thru */
ret = NC_NOERR;
goto done;
case ERROR_DIRECTORY: /* not a directory */
default:
ret = NC_EEMPTY;
goto done;
}
}
do {
char* p = NULL;
const char* name = NULL;
name = FindFileData.cFileName;
if(strcmp(name,".")==0 || strcmp(name,"..")==0)
continue;
nclistpush(contents,strdup(name));
} while(FindNextFile(dir, &FindFileData));
done:
if(dir) FindClose(dir);
nullfree(lpath);
nullfree(ffpath);
nullfree(d);
errno = 0;
return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents));
}
#else /* !_WIN32 */
int
getentries(const char* path, NClist* contents)
{
int ret = NC_NOERR;
errno = 0;
DIR* dir = NULL;
ZTRACE(6,"path=%s",path);
dir = NCopendir(path);
if(dir == NULL)
{ret = (errno); goto done;}
for(;;) {
const char* name = NULL;
struct dirent* de = NULL;
errno = 0;
de = readdir(dir);
if(de == NULL)
{ret = (errno); goto done;}
if(strcmp(de->d_name,".")==0 || strcmp(de->d_name,"..")==0)
continue;
name = de->d_name;
nclistpush(contents,strdup(name));
}
done:
if(dir) NCclosedir(dir);
errno = 0;
return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents));
}
#endif /*_WIN32*/
/**************************************************/
#if defined(DEBUGF) || defined(DEBUGL)
const char*
printplugin(const NCZ_Plugin* plugin)
{
static char plbuf[4096];
char plbuf2[2000];
char plbuf1[2000];
if(plugin == NULL) return "plugin=NULL";
plbuf2[0] = '\0'; plbuf1[0] = '\0';
if(plugin->hdf5.filter)
snprintf(plbuf1,sizeof(plbuf1),"hdf5={id=%u name=%s}",plugin->hdf5.filter->id,plugin->hdf5.filter->name);
if(plugin->codec.codec)
snprintf(plbuf2,sizeof(plbuf2),"codec={codecid=%s hdf5id=%u}",plugin->codec.codec->codecid,plugin->codec.codec->hdf5id);
snprintf(plbuf,4096,"plugin={%s %s}",plbuf1,plbuf2);
return plbuf;
}
static char*
printparams(size_t nparams, const unsigned* params)
{
static char ppbuf[4096];
if(nparams == 0)
snprintf(ppbuf,4096,"{0,%p}",params);
else
snprintf(ppbuf,4096,"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params));
return ppbuf;
}
static char*
printnczparams(const NCZ_Params p)
{
return printparams(p.nparams,p.params);
}
static const char*
printcodec(const NCZ_Codec c)
{
static char pcbuf[4096];
snprintf(pcbuf,sizeof(pcbuf),"{id=%s codec=%s}",
c.id,NULLIFY(c.codec));
return pcbuf;
}
static const char*
printhdf5(const NCZ_HDF5 h)
{
static char phbuf[4096];
snprintf(phbuf,sizeof(phbuf),"{id=%u visible=%s working=%s}",
h.id, printnczparams(h.visible), printnczparams(h.working));
return phbuf;
}
#endif /* defined(DEBUGF) || defined(DEBUGL) */
/* Suppress selected unused static functions */
static void static_unused(void)
{
void* p = NULL;
p = p;
p = static_unused;
#if defined(DEBUGF) || defined(DEBUGL)
(void)printplugin(NULL);
(void)printparams(0, NULL);
(void)printnczparams(const NCZ_Params p);
(void)printcodec(const NCZ_Codec c);
(void)printhdf5(const NCZ_HDF5 h);
#endif
}

45
libnczarr/zplugins.h Normal file
View File

@ -0,0 +1,45 @@
/* Copyright 2018-2018 University Corporation for Atmospheric
Research/Unidata. */
/**
* @file This header file containsplugin related macros, types, and prototypes for
* the plugin code in libnczarr. This header should not be included in
* code outside libnczarr.
*
* @author Dennis Heimbigner
*/
#ifndef ZPLUGIN_H
#define ZPLUGIN_H
/* zplugin.c */
/* Pluginlist management */
/* Opaque Handles */
struct H5Z_class2_t;
struct NCZ_codec_t;
struct NCPSharedLib;
/* Hold the loaded filter plugin information */
typedef struct NCZ_Plugin {
int incomplete;
struct HDF5API {
const struct H5Z_class2_t* filter;
struct NCPSharedLib* hdf5lib; /* source of the filter */
} hdf5;
struct CodecAPI {
int defaulted; /* codeclib was a defaulting library */
int ishdf5raw; /* The codec is the hdf5raw codec */
const struct NCZ_codec_t* codec;
struct NCPSharedLib* codeclib; /* of the codec; null if same as hdf5 */
} codec;
} NCZ_Plugin;
int NCZ_load_all_plugins(void);
int NCZ_plugin_loaded(size_t filterid, NCZ_Plugin** pp);
int NCZ_plugin_loaded_byname(const char* name, NCZ_Plugin** pp);
#endif /*ZPLUGIN_H*/

2
libnetcdf.settings.in
View File

@ -9,7 +9,6 @@ Configured On: @CONFIG_DATE@
Host System: @host_cpu@-@host_vendor@-@host_os@
Build Directory: @abs_top_builddir@
Install Prefix: @prefix@
Plugin Install Prefix: @PLUGIN_INSTALL_DIR_SETTING@
# Compiling Options
-----------------
@ -53,6 +52,7 @@ Relaxed Boundary Check: @RELAX_COORD_BOUND@
Plugins Enabled: @HAS_PLUGINS@
Plugin Install Dir: @NETCDF_PLUGIN_INSTALL_DIR@
Plugin Search Path: @NETCDF_PLUGIN_SEARCH_PATH@
Quantization: @HAS_QUANTIZE@
Logging: @HAS_LOGGING@

12
libsrc4/nc4dispatch.c
View File

@ -25,7 +25,9 @@ extern NC_Dispatch UDF0_DISPATCH;
extern NC_Dispatch UDF1_DISPATCH;
#endif /* USE_UDF1 */
extern int nc_plugin_path_initialize(void);
extern int nc_plugin_path_finalize(void);
/**
* @internal Initialize netCDF-4. If user-defined format(s) have been
* specified in configure, load their dispatch table(s).
@ -62,6 +64,11 @@ NC4_initialize(void)
}
#endif
#endif
#if defined(USE_HDF5) || defined(NETCDF_ENABLE_NCZARR)
nc_plugin_path_initialize();
#endif
NC_initialize_reserved();
return ret;
}
@ -75,5 +82,8 @@ NC4_initialize(void)
int
NC4_finalize(void)
{
#if defined(USE_HDF5) || defined(NETCDF_ENABLE_NCZARR)
nc_plugin_path_finalize();
#endif
return NC_NOERR;
}

2
nc_test/tst_atts3.c
View File

@ -189,7 +189,7 @@ main(int argc, char **argv)
size_t att_len;
int i;
char *speech_in;
char *speech_in = NULL;
/* This won't work, because classic files can't create these types. */
#ifdef TEST_PNETCDF

2
nc_test4/CMakeLists.txt
View File

@ -40,6 +40,8 @@ IF(NETCDF_BUILD_UTILITIES)
ADD_SH_TEST(nc_test4 tst_misc)
build_bin_test(tst_fillonly)
ADD_SH_TEST(nc_test4 test_fillonly)
# H5 and nczarr Fixed string support
build_bin_test(build_fixedstring)
ADD_SH_TEST(nc_test4 tst_fixedstring)
IF(USE_HDF5 AND NETCDF_ENABLE_FILTER_TESTING)
build_bin_test(tst_filterparser)

17
nc_test4/Makefile.am
View File

@ -68,7 +68,8 @@ TESTS += run_grp_rename.sh tst_misc.sh
check_PROGRAMS += tst_fillonly
TESTS += test_fillonly.sh
# H5 and nczarr Fixed string support
# H5 Fixed string support
check_PROGRAMS += build_fixedstring
TESTS += tst_fixedstring.sh
# Szip Tests (requires ncdump)
@ -143,7 +144,7 @@ ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
ref_filter_order_create.txt ref_filter_order_read.txt ref_any.cdl \
tst_specific_filters.sh tst_unknown.sh tst_virtual_datasets.c \
noop1.cdl unknown.cdl tst_broken_files.c ref_bloscx.cdl \
tst_bloscfail.sh tst_fixedstring.sh ref_fixedstring.h5 \
tst_bloscfail.sh tst_fixedstring.sh \
ref_fixedstring.cdl tst_filterinstall.sh tst_filter_vlen.sh \
tst_filter_misc.sh run_zstd_test.sh run_par_warn_test.sh.in \
ref_tmp_tst_warn_out.txt
@ -159,7 +160,7 @@ floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \
testfilter_reg.nc filterrepeat.txt tmp_fillonly.nc \
testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1 \
tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp \
tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp tmp_*.cdl
tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp tmp_*.cdl ref_fixedstring.h5
DISTCLEANFILES = findplugin.sh run_par_test.sh run_par_warn_test.sh
@ -168,13 +169,3 @@ clean-local:
# If valgrind is present, add valgrind targets.
@VALGRIND_CHECK_RULES@
# The (otherwise unused) program build_fixedstring.c
# is used to generate the test file ref_fixedstring.h5.
# That test file is build and included as part of the distribution,
# so the build_fixedstring.c program generally does not need to
# be executed unless the test file needs to be modified..
check_PROGRAMS += build_fixedstring
ref_fixedstring.h5: build_fixedstring.c
${srcdir}/buildfixedstring

13
nc_test4/ref_fixedstring.cdl
View File

@ -1,9 +1,16 @@
netcdf ref_fixedstring {
dimensions:
phony_dim_0 = 3 ;
phony_dim_0 = 4 ;
variables:
string test(phony_dim_0) ;
string v1 ;
string vn(phony_dim_0) ;
// global attributes:
:att1 = "abcd" ;
string :attn = "abcd", "efgh", "ijkl", "mnop" ;
data:
test = "foo", "bar", "baz" ;
v1 = "abcd" ;
vn = "abcd", "efgh", "ijkl", "mnop" ;
}

BIN
nc_test4/ref_fixedstring.h5
View File

Binary file not shown.

7
nc_test4/tst_fixedstring.sh
View File

@ -6,10 +6,11 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
set -e
# Note, the test file for this is ref_fixedstring.h5
# But is is generated by the (otherwise unused) program
# ../h5_test/tst_h_fixedstrings.c.
# and is generated on the fly by build_fixedstring.
${execdir}/build_fixedstring
echo "*** Test reading a file with HDF5 fixed length strings"
rm -f ./tmp_fixedstring.cdl
$NCDUMP ${srcdir}/ref_fixedstring.h5 > ./tmp_fixedstring.cdl
$NCDUMP ${execdir}/ref_fixedstring.h5 > ./tmp_fixedstring.cdl
diff -b -w ${srcdir}/ref_fixedstring.cdl ./tmp_fixedstring.cdl
rm -f ${execdir}/ref_fixedstring.h5

6
ncdump/CMakeLists.txt
View File

@ -23,7 +23,8 @@ set(printfqn_FILES printfqn.c ${XGETOPTSRC})
set(ncpathcvt_FILES ncpathcvt.c ${XGETOPTSRC})
set(ncfilteravail_FILES ncfilteravail.c ${XGETOPTSRC})
set(nchdf5version_FILES nchdf5version.c)
set(echon_FILES echon.c ${XGETOPTSRC})
##
# Turn off inclusion of particular files when using the cmake-native
# option to turn on Unity Builds.
@ -42,6 +43,7 @@ add_executable(nccopy ${nccopy_FILES})
add_executable(ncvalidator ${ncvalidator_FILES})
add_executable(ncpathcvt ${ncpathcvt_FILES})
add_executable(ncfilteravail ${ncfilteravail_FILES})
add_executable(echon ${echon_FILES})
if(USE_HDF5)
add_executable(nc4print nc4print.c nc4printer.c)
@ -63,6 +65,7 @@ target_link_libraries(nccopy netcdf ${ALL_TLL_LIBS})
target_link_libraries(ncvalidator netcdf ${ALL_TLL_LIBS})
target_link_libraries(ncpathcvt netcdf ${ALL_TLL_LIBS})
target_link_libraries(ncfilteravail netcdf ${ALL_TLL_LIBS})
target_link_libraries(echon netcdf ${ALL_TLL_LIBS})
if(NETCDF_ENABLE_DAP)
target_link_libraries(ocprint netcdf ${ALL_TLL_LIBS})
@ -90,6 +93,7 @@ if(WIN32)
setbinprops(ncvalidator)
setbinprops(ncpathcvt)
setbinprops(ncfilteravail)
setbinprops(echon)
if(USE_HDF5)
setbinprops(printfqn)

4
ncdump/Makefile.am
View File

@ -37,6 +37,10 @@ utils.h utils.c dimmap.h dimmap.c list.c list.h
noinst_PROGRAMS += ncvalidator
ncvalidator_SOURCES = ncvalidator.c
# An equivalent to echo that always implements -n flag
noinst_PROGRAMS += echon
echon_SOURCES = echon.c
# A non-installed utility program to convert paths; similar to cygpath
noinst_PROGRAMS += ncpathcvt
ncpathcvt_SOURCES = ncpathcvt.c

106
ncdump/echon.c Normal file
View File

@ -0,0 +1,106 @@
/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#ifdef _WIN32
#include <windows.h>
#include <io.h>
#include <fcntl.h>
#endif
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#if defined(_WIN32) && ! defined(__MINGW32__)
#include "XGetopt.h"
#endif
extern int NCstdbinary(void);
int
main(int argc, char** argv)
{
int i;
int len;
char* input = NULL;
char* output = NULL;
int noeol = 1; /* Default to -n vs -N */
int escape = 1; /* Ditto */
char* p = NULL;
char* q = NULL;
char c;
NCstdbinary(); /* avoid \r\n for windows */
opterr = 1;
while ((c = getopt(argc, argv,"enxNX")) != EOF) {
switch(c) {
case 'e': goto nomoreargs;
case 'x': escape = 1; break;
case 'n': noeol = 1; break;
case 'X': escape = 0; break;
case 'N': noeol = 0; break;
case '?': default: break; /* ignore */
}
}
nomoreargs:
/* leave non-flag args */
argc -= optind;
argv += optind;
/* Compute the max possible length of output */
for(len=0,i=0;i<argc;i++) len += (int)strlen(argv[i]);
len += (argc - 1); /* inter-arg spacing */
input = (char*)calloc(1,(size_t)len+1+1); /* +1 for nul term +1 for '\n' */
assert(input != NULL);
input[0] = '\0';
/* Concat all the arguments with ' ' between them */
for(i=0;i<argc;i++) {
if(i > 0) strcat(input," ");
strcat(input,argv[i]);
}
output = (char*)calloc(1,(size_t)len+1+1); /* +1 for nul term +1 for '\n' */
assert(output != NULL);
output[0] = '\0';
/* Optionally de-escape the input */
for(p=input,q=output;*p;) {
if(escape) {
switch (*p) {
default: *q++ = *p; break;
case '\\':
p++;
switch(*p) {
case 'a': *q++ = '\a'; break;
case 'b': *q++ = '\b'; break;
case 'f': *q++ = '\f'; break;
case 'r': *q++ = '\r'; break;
case 'n': *q++ = '\n'; break;
case 't': *q++ = '\t'; break;
case 'v': *q++ = '\v'; break;
default: *q++ = *p; break;
}
break;
}
p++;
} else
*q++ = *p++;
}
if(!noeol) *p++ = '\n';
*p = '\0';
fputs(output,stdout);
fflush(stdout);
free(input);
free(output);
return 0;
}

104
ncdump/ncpathcvt.c
View File

@ -26,20 +26,24 @@
#endif
#include "netcdf.h"
#include "netcdf_filter.h"
#include "netcdf_aux.h"
#include "ncpathmgr.h"
#include "ncbytes.h"
static const char* USAGE =
"ncpathcvt [-c|-C|-m|-u|-w] [-h] [-e] [-F] [-d <driveletter>] [-B<char>] [-k] [-p] PATH\n"
"ncpathcvt [-c|-m|-u|-w] [-e] [-h] [-k] [-p] [-x] [-F] [-D <driveletter>] [-B<char>] [-S<char>] PATH\n"
"Options\n"
" -h help"
" -e add backslash escapes to '\' and ' '\n"
" -d <driveletter> use driveletter when needed; defaults to 'c'\n"
" -B <char> convert occurrences of <char> to blank\n"
" -D <driveletter> use driveletter when needed; defaults to 'c'\n"
" -F convert occurrences of '\\' to '/'"
" -S <char> use <char> as path separator when parsing;\n"
" currently limited to ';' or ':' but defaults to ';'\n"
"Output type options:\n"
" -c convert to Cygwin form of path\n"
" -C return canonical form of path\n"
" -m convert to MSYS form of path\n"
" -m convert to MSYS form of path: currently an alias for -w\n"
" -u convert to Unix form of path\n"
" -w convert to Windows form of path\n"
"Other options:\n"
@ -56,10 +60,10 @@ struct Options {
int escapes;
int drive;
int debug;
int canon;
int blank;
int slash;
int pathkind;
int sep;
} cvtoptions;
static char* escape(const char* path);
@ -186,37 +190,66 @@ printpathkind(const char* path)
exit(0);
}
int
processdir(const char* indir, char** cvtdirp)
{
char* cvtdir = NULL;
if(cvtoptions.target == NCPD_UNKNOWN) {
cvtdir = NCpathcvt(indir);
} else {
cvtdir = NCpathcvt_test(indir,cvtoptions.target,(char)cvtoptions.drive);
}
if(cvtdir && cvtoptions.escapes) {
char* dir = cvtdir; cvtdir = NULL;
cvtdir = escape(dir);
free(dir);
}
if(cvtdir && cvtoptions.slash) {
char* dir = cvtdir; cvtdir = NULL;
cvtdir = slash(dir);
free(dir);
}
if(cvtdirp) {*cvtdirp = cvtdir; cvtdir = NULL;}
if(cvtdir) free(cvtdir);
return 0;
}
int
main(int argc, char** argv)
{
int c;
char* cvtpath = NULL;
char* inpath, *canon = NULL;
char* inpath = NULL;
NCPluginList indirs = {0,NULL};
NCbytes* outpath = ncbytesnew();
int stat = NC_NOERR;
size_t i;
memset((void*)&cvtoptions,0,sizeof(cvtoptions));
cvtoptions.drive = 'c';
cvtoptions.sep = ';';
while ((c = getopt(argc, argv, "B:CFcD:d:ehkmpuwX")) != EOF) {
while ((c = getopt(argc, argv, "B:D:FS:Xchkmpuwx")) != EOF) {
switch(c) {
case 'c': cvtoptions.target = NCPD_CYGWIN; break;
case 'd': cvtoptions.drive = optarg[0]; break;
case 'e': cvtoptions.escapes = 1; break;
case 'h': usage(NULL); break;
case 'k': printlocalkind(); break;
case 'm': cvtoptions.target = NCPD_MSYS; break;
case 'm': cvtoptions.target = NCPD_WIN; break; /* Aliased */
case 'p': cvtoptions.pathkind = 1; break;
case 'u': cvtoptions.target = NCPD_NIX; break;
case 'w': cvtoptions.target = NCPD_WIN; break;
case 'x': cvtoptions.escapes = 1; break;
case 'B':
cvtoptions.blank = optarg[0];
if(cvtoptions.blank < ' ' || cvtoptions.blank == '\177')
usage("Bad -B argument");
break;
case 'C': cvtoptions.canon = 1; break;
case 'D': cvtoptions.drive = optarg[0]; break;
case 'F': cvtoptions.slash = 1; break;
case 'D':
sscanf(optarg,"%d",&cvtoptions.debug);
break;
case 'S': cvtoptions.sep = optarg[0]; break;
case 'X': printenv(); break;
case '?':
usage("unknown option");
@ -252,33 +285,22 @@ main(int argc, char** argv)
goto done;
}
/* Canonicalize */
if(NCpathcanonical(inpath,&canon))
usage("Could not convert to canonical form");
/* Break using the path separator */
if((stat = ncaux_plugin_path_parse(inpath,cvtoptions.sep,&indirs)))
{usage(nc_strerror(stat));}
for(i=0;i<indirs.ndirs;i++) {
char* outdir = NULL;
if((stat = processdir(indirs.dirs[i],&outdir)))
{usage(nc_strerror(stat));}
if(i > 0) ncbytesappend(outpath,cvtoptions.sep);
ncbytescat(outpath,outdir);
nullfree(outdir);
}
printf("%s",ncbytescontents(outpath));
if(cvtoptions.canon) {
cvtpath = canon; canon = NULL;
} else if(cvtoptions.target == NCPD_UNKNOWN) {
cvtpath = NCpathcvt(canon);
} else {
cvtpath = NCpathcvt_test(canon,cvtoptions.target,(char)cvtoptions.drive);
}
if(cvtpath && cvtoptions.escapes) {
char* path = cvtpath; cvtpath = NULL;
cvtpath = escape(path);
free(path);
}
if(cvtpath && cvtoptions.slash) {
char* path = cvtpath; cvtpath = NULL;
cvtpath = slash(path);
free(path);
}
printf("%s",cvtpath);
done:
if(canon) free(canon);
if(inpath) free(inpath);
if(cvtpath) free(cvtpath);
ncaux_plugin_path_clear(&indirs);
ncbytesfree(outpath);
return 0;
}

40
ncdump/ref_pathcvt.txt
View File

@ -1,45 +1,39 @@
path: -u: |/xxx/x/y| => |/xxx/x/y|
path: -c: |/xxx/x/y| => |/cygdrive/c/xxx/x/y|
path: -m: |/xxx/x/y| => |/xxx/x/y|
path: -w: |/xxx/x/y| => |c:\\xxx\\x\\y|
path: -C: |/xxx/x/y| => |/xxx/x/y|
path: -u: |d:/x/y| => |/d/x/y|
path: -c: |d:/x/y| => |/cygdrive/d/x/y|
path: -m: |d:/x/y| => |/d/x/y|
path: -w: |d:/x/y| => |d:\\x\\y|
path: -C: |d:/x/y| => |/cygdrive/d/x/y|
path: -u: |/cygdrive/d/x/y| => |/d/x/y|
path: -c: |/cygdrive/d/x/y| => |/cygdrive/d/x/y|
path: -m: |/cygdrive/d/x/y| => |/d/x/y|
path: -w: |/cygdrive/d/x/y| => |d:\\x\\y|
path: -C: |/cygdrive/d/x/y| => |/cygdrive/d/x/y|
path: -u: |/d/x/y| => |/d/x/y|
path: -c: |/d/x/y| => |/cygdrive/d/x/y|
path: -m: |/d/x/y| => |/d/x/y|
path: -c: |/d/x/y| => |/cygdrive/c/d/x/y|
path: -w: |/d/x/y| => |d:\\x\\y|
path: -C: |/d/x/y| => |/cygdrive/d/x/y|
path: -u: |/cygdrive/d| => |/d|
path: -c: |/cygdrive/d| => |/cygdrive/d|
path: -m: |/cygdrive/d| => |/d|
path: -w: |/cygdrive/d| => |d:|
path: -C: |/cygdrive/d| => |/cygdrive/d|
path: -u: |/d| => |/d|
path: -c: |/d| => |/cygdrive/d|
path: -m: |/d| => |/d|
path: -c: |/d| => |/cygdrive/c/d|
path: -w: |/d| => |d:|
path: -C: |/d| => |/cygdrive/d|
path: -u: |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn| => |/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn|
path: -c: |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn| => |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn|
path: -m: |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn| => |/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn|
path: -w: |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn| => |d:\\git\\netcdf-c\\dap4_test\\test_anon_dim.2.syn|
path: -C: |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn| => |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn|
path: -u: |d:\x\y| => |/d/x/y|
path: -c: |d:\x\y| => |/cygdrive/d/x/y|
path: -m: |d:\x\y| => |/d/x/y|
path: -w: |d:\x\y| => |d:\\x\\y|
path: -C: |d:\x\y| => |/cygdrive/d/x/y|
path: -u: |d:\x\y w\z| => |/d/x/y\ w/z|
path: -c: |d:\x\y w\z| => |/cygdrive/d/x/y\ w/z|
path: -m: |d:\x\y w\z| => |/d/x/y\ w/z|
path: -w: |d:\x\y w\z| => |d:\\x\\y\ w\\z|
path: -C: |d:\x\y w\z| => |/cygdrive/d/x/y\ w/z|
path: -u: |d:\x\y@w\z| => |/d/x/y\ w/z|
path: -c: |d:\x\y@w\z| => |/cygdrive/d/x/y\ w/z|
path: -w: |d:\x\y@w\z| => |d:\\x\\y\ w\\z|
path: -u: |/xxx/x/y;/cygdrive/d/x/y| => |/xxx/x/y;/d/x/y|
path: -c: |/xxx/x/y;/cygdrive/d/x/y| => |/cygdrive/c/xxx/x/y;/cygdrive/d/x/y|
path: -w: |/xxx/x/y;/cygdrive/d/x/y| => |c:\\xxx\\x\\y;d:\\x\\y|
path: -u: |/d/x/y;/cygdrive/d| => |/d/x/y;/d|
path: -c: |/d/x/y;/cygdrive/d| => |/cygdrive/c/d/x/y;/cygdrive/d|
path: -w: |/d/x/y;/cygdrive/d| => |d:\\x\\y;d:|
path: -u: |cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn;d:\x\y| => |cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn;/d/x/y|
path: -c: |cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn;d:\x\y| => |cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn;/cygdrive/d/x/y|
path: -w: |cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn;d:\x\y| => |cygdrive\\d\\git\\netcdf-c\\dap4_test\\test_anon_dim.2.syn;d:\\x\\y|
path: -u: |d:\x\y@w\z| => |/d/x/y\ w/z|
path: -c: |d:\x\y@w\z| => |/cygdrive/d/x/y\ w/z|
path: -w: |d:\x\y@w\z| => |d:\\x\\y\ w\\z|

55
ncdump/testpathcvt.sh
View File

@ -9,39 +9,58 @@ set -e
export MSYS2_ARG_CONV_EXCL='*'
# We need to find the drive letter, if any
DL=`${NCPATHCVT} -c -e / | sed -e 's|/cygdrive/\([a-zA-Z]\)/.*|\1|'`
DL=`${NCPATHCVT} -c -x / | sed -e 's|/cygdrive/\([a-zA-Z]\)/.*|\1|'`
if test "x$DL" != x ; then
# Lower case drive letter
DLL=`echon "$DL" | tr '[:upper:]' '[:lower:]'`
DL="-d $DLL"
DLL=`echon -e "$DL" | tr '[:upper:]' '[:lower:]'`
DL="-D $DLL"
fi
testcase1() {
testcaseD() {
T="$1"
P="$2"
echon "path: $T: |$P| => |" >>tmp_pathcvt.txt
${NCPATHCVT} -B"@" ${DL} "$T" -e "$P" >>tmp_pathcvt.txt
# Fixup for shell handling of '\'
PX=`echon -n "$P" | sed -e 's/\\\\\\\\/\\\\/g'`
echon -e "path: $T: |$P| => |" >>tmp_pathcvt.txt
${NCPATHCVT} -B"@" ${DL} "$T" -x "$PX" >>tmp_pathcvt.txt
echo "|" >> tmp_pathcvt.txt
}
testcase() {
testcase1 "-u" "$1"
testcase1 "-c" "$1"
testcase1 "-m" "$1"
testcase1 "-w" "$1"
testcase1 "-C" "$1"
testcaseP() {
T="$1"
P="$2"
# Fixup for shell handling of '\'
PX=`echon -n "$P" | sed -e 's/\\\\\\\\/\\\\/g'`
echon -e "path: $T: |$P| => |" >>tmp_pathcvt.txt
${NCPATHCVT} -S';' -B"@" ${DL} "$T" -x "$PX" >>tmp_pathcvt.txt
echo "|" >> tmp_pathcvt.txt
}
# Note that -m is not tested as it is currently an alias for -w
testcase1() {
testcaseD "-u" "$1"
testcaseD "-c" "$1"
testcaseD "-w" "$1"
}
testcase2() {
testcaseP "-u" "$1"
testcaseP "-c" "$1"
testcaseP "-w" "$1"
}
rm -f tmp_pathcvt.txt
# '@' will get translated to embedded blank
PATHS="/xxx/x/y d:/x/y /cygdrive/d/x/y /d/x/y /cygdrive/d /d /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn d:\\x\\y d:\\x\\y@w\\z"
for p in $PATHS ; do
testcase $p
TESTPATHS1="/xxx/x/y d:/x/y /cygdrive/d/x/y /d/x/y /cygdrive/d /d /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn d:\\\\x\\\\y d:\\\\x\\\\y@w\\\\z"
for p in $TESTPATHS1 ; do
testcase1 "$p"
done
TESTPATHS2="/xxx/x/y;/cygdrive/d/x/y /d/x/y;/cygdrive/d cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn;d:\\\\x\\\\y d:\\\\x\\\\y@w\\\\z"
for p in $TESTPATHS2 ; do
testcase2 "$p"
done
exit
diff -w ${srcdir}/ref_pathcvt.txt ./tmp_pathcvt.txt

2
nczarr_test/CMakeLists.txt
View File

@ -130,7 +130,7 @@ IF(NETCDF_ENABLE_TESTS)
ENDIF()
# Helper programs for testing
BUILD_BIN_TEST(zhex)
build_bin_test(zhex)
build_bin_test_with_util_lib(zisjson ut_util)
TARGET_INCLUDE_DIRECTORIES(zisjson PUBLIC ../libnczarr)
build_bin_test_with_util_lib(zs3parse ut_util)

46
oc2/dap.y
View File

@ -245,29 +245,29 @@ errorprog : /*empty*/ {$$=null;} | SCAN_PROG '=' WORD_WORD ';' {$$=$3;}
*/
name:
WORD_WORD {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_ALIAS {$$=strdup($1);}
| SCAN_ARRAY {$$=strdup($1);}
| SCAN_ATTR {$$=strdup($1);}
| SCAN_BYTE {$$=strdup($1);}
| SCAN_DATASET {$$=strdup($1);}
| SCAN_DATA {$$=strdup($1);}
| SCAN_ERROR {$$=strdup($1);}
| SCAN_FLOAT32 {$$=strdup($1);}
| SCAN_FLOAT64 {$$=strdup($1);}
| SCAN_GRID {$$=strdup($1);}
| SCAN_INT16 {$$=strdup($1);}
| SCAN_INT32 {$$=strdup($1);}
| SCAN_MAPS {$$=strdup($1);}
| SCAN_SEQUENCE {$$=strdup($1);}
| SCAN_STRING {$$=strdup($1);}
| SCAN_STRUCTURE {$$=strdup($1);}
| SCAN_UINT16 {$$=strdup($1);}
| SCAN_UINT32 {$$=strdup($1);}
| SCAN_URL {$$=strdup($1);}
| SCAN_CODE {$$=strdup($1);}
| SCAN_MESSAGE {$$=strdup($1);}
| SCAN_PROG {$$=strdup($1);}
| SCAN_PTYPE {$$=strdup($1);}
| SCAN_ALIAS {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_ARRAY {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_ATTR {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_BYTE {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_DATASET {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_DATA {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_ERROR {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_FLOAT32 {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_FLOAT64 {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_GRID {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_INT16 {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_INT32 {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_MAPS {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_SEQUENCE {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_STRING {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_STRUCTURE {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_UINT16 {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_UINT32 {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_URL {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_CODE {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_MESSAGE {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_PROG {$$=dapdecode(parsestate->lexstate,$1);}
| SCAN_PTYPE {$$=dapdecode(parsestate->lexstate,$1);}
;
%%

46
oc2/dapy.c
View File

@ -2048,139 +2048,139 @@ yyreduce:
case 84: /* name: SCAN_ALIAS */
#line 248 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2053 "dapy.c"
break;
case 85: /* name: SCAN_ARRAY */
#line 249 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2059 "dapy.c"
break;
case 86: /* name: SCAN_ATTR */
#line 250 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2065 "dapy.c"
break;
case 87: /* name: SCAN_BYTE */
#line 251 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2071 "dapy.c"
break;
case 88: /* name: SCAN_DATASET */
#line 252 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2077 "dapy.c"
break;
case 89: /* name: SCAN_DATA */
#line 253 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2083 "dapy.c"
break;
case 90: /* name: SCAN_ERROR */
#line 254 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2089 "dapy.c"
break;
case 91: /* name: SCAN_FLOAT32 */
#line 255 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2095 "dapy.c"
break;
case 92: /* name: SCAN_FLOAT64 */
#line 256 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2101 "dapy.c"
break;
case 93: /* name: SCAN_GRID */
#line 257 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2107 "dapy.c"
break;
case 94: /* name: SCAN_INT16 */
#line 258 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2113 "dapy.c"
break;
case 95: /* name: SCAN_INT32 */
#line 259 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2119 "dapy.c"
break;
case 96: /* name: SCAN_MAPS */
#line 260 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2125 "dapy.c"
break;
case 97: /* name: SCAN_SEQUENCE */
#line 261 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2131 "dapy.c"
break;
case 98: /* name: SCAN_STRING */
#line 262 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2137 "dapy.c"
break;
case 99: /* name: SCAN_STRUCTURE */
#line 263 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2143 "dapy.c"
break;
case 100: /* name: SCAN_UINT16 */
#line 264 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2149 "dapy.c"
break;
case 101: /* name: SCAN_UINT32 */
#line 265 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2155 "dapy.c"
break;
case 102: /* name: SCAN_URL */
#line 266 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2161 "dapy.c"
break;
case 103: /* name: SCAN_CODE */
#line 267 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2167 "dapy.c"
break;
case 104: /* name: SCAN_MESSAGE */
#line 268 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2173 "dapy.c"
break;
case 105: /* name: SCAN_PROG */
#line 269 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2179 "dapy.c"
break;
case 106: /* name: SCAN_PTYPE */
#line 270 "dap.y"
{yyval=strdup(yyvsp[0]);}
{yyval=dapdecode(parsestate->lexstate,yyvsp[0]);}
#line 2185 "dapy.c"
break;

3
oc2/ocinternal.c
View File

@ -86,6 +86,9 @@ ocinternalinitialize(void)
/* Compute some xdr related flags */
xxdr_init();
if(getenv("OCDEBUG") != NULL)
ocdebug = atoi(getenv("OCDEBUG"));
return OCTHROW(stat);
}

20
plugins/CMakeLists.txt
View File

@ -118,7 +118,6 @@ else()
buildplugin(h5bzip2 "h5bzip2" ${Bzip2_LIBRARIES})
endif()
# Installation
if(ENABLE_PLUGIN_INSTALL)
@ -128,7 +127,7 @@ macro(installplugin PLUG)
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${INSTALLED_PLUGIN_LIB} DESTINATION ${NETCDF_PLUGIN_INSTALL_DIR})
endmacro()
install(DIRECTORY DESTINATION ${PLUGIN_INSTALL_DIR})
install(DIRECTORY DESTINATION ${NETCDF_PLUGIN_INSTALL_DIR})
if(Bzip2_FOUND OR Bz2_FOUND)
installplugin(h5bzip2)
endif()
@ -137,20 +136,6 @@ if(Zstd_FOUND)
endif()
if(Blosc_FOUND)
installplugin(h5blosc)
endif()
if(NETCDF_ENABLE_NCZARR)
installplugin(h5fletcher32)
installplugin(h5shuffle)
installplugin(h5deflate)
installplugin(zhdf5filters)
installplugin(zstdfilters)
if(Szip_FOUND)
installplugin(h5szip)
endif()
endif()
endif(ENABLE_PLUGIN_INSTALL)
endif(NETCDF_ENABLE_PLUGINS)
# Copy some test files from current source dir to out-of-tree build dir.
@ -158,3 +143,6 @@ file(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
if(MSVC)
file(COPY ${COPY_FILES} DESTINATION ${RUNTIME_OUTPUT_DIRECTORY}/)
endif()
endif(ENABLE_PLUGIN_INSTALL)
endif(NETCDF_ENABLE_PLUGINS)

2
plugins/H5Zblosc.c
View File

@ -107,6 +107,7 @@ param[6] -- compressor to use
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include "netcdf_filter_build.h"
#include <netcdf_json.h>
@ -375,6 +376,5 @@ const H5Z_class2_t blosc_H5Filter[1] = {
}
};
H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_FILTER; }
const void* H5PLget_plugin_info(void) { return blosc_H5Filter; }

2
plugins/H5Zblosc.h
View File

@ -36,7 +36,7 @@ extern "C" {
#ifdef DLL_EXPORT /* define when building the library */
#define DECLSPEC __declspec(dllexport)
#else
#define DECLSPEC __declspec(dllimport)
#define DECLSPEC __declspec(dllimport)
#endif
#else
#define DECLSPEC extern

3
plugins/H5Znoop.c
View File

@ -5,8 +5,7 @@
#include <stdio.h>
#include <sys/types.h>
#include <netcdf_json.h>
#include "netcdf_json.h"
#include "netcdf_filter_build.h"
#ifndef NOOP_INSTANCE

2
plugins/Makefile.am
View File

@ -18,7 +18,7 @@ RPATH = -rpath $(abs_top_builddir)/.libs
# This is where the plugins are to be installed
if ENABLE_PLUGIN_DIR
plugindir = @PLUGIN_INSTALL_DIR@
plugindir = @NETCDF_PLUGIN_INSTALL_DIR@
else
plugindir = ${ALTPLUGINDIR}
AM_LDFLAGS += ${RPATH}

16
test_common.in
View File

@ -11,11 +11,12 @@ export TEST_COMMON_SH=1
# Define various global constants
# Define location of execution
# Define various build and install paths
abs_top_srcdir=@abs_top_srcdir@
abs_top_builddir=@abs_top_builddir@
TOPSRCDIR="${abs_top_srcdir}"
TOPBUILDDIR="${abs_top_builddir}"
FP_ISCMAKE=@ISCMAKE@
FP_ISMSVC=@ISMSVC@
FP_WINVERMAJOR=@WINVERMAJOR@
@ -35,7 +36,6 @@ FEATURE_PARALLEL=@HAS_PARALLEL@
# Define selected features of the build
FEATURE_HDF5=@HAS_HDF5@
FEATURE_FILTERTESTS=@DO_FILTER_TESTS@
FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@
FEATURE_BYTERANGE=@HAS_BYTERANGE@
FEATURE_ROS3=@HAS_HDF5_ROS3@
FEATURE_S3_AWS=@HAS_S3_AWS@
@ -46,6 +46,9 @@ FEATURE_S3TESTS=@NETCDF_ENABLE_S3_TESTING@
FEATURE_NCZARR_ZIP=@HAS_NCZARR_ZIP@
FEATURE_LARGE_TESTS=@DO_LARGE_TESTS@
FEATURE_PLUGIN_INSTALL_DIR="@NETCDF_PLUGIN_INSTALL_DIR@"
FEATURE_PLUGIN_SEARCH_PATH="@NETCDF_PLUGIN_SEARCH_PATH@"
# Thredds-test server is currently disabled
#FEATURE_THREDDSTEST=1
@ -120,7 +123,6 @@ if test "x$SETX" = x1 ; then set -x ; fi
system=`uname`
if test "x${system##MINGW*}" = x; then
alias pwd='pwd -W'
fi
# We assume that TOPSRCDIR and TOPBUILDDIR are defined
@ -197,19 +199,19 @@ export HOME
fi
fi
# OS/X apparently has no echo -n option, so fake it
echon() { echo "$@" | tr -d '\r\n' ; }
# OS/X and windows apparently have no echo -n option, so fake it
echon() {
${execdir}/../ncdump/echon -n $@
}
# Test for filter availability
avail() {
if test yes = `${execdir}/../ncdump/ncfilteravail $1` ; then return 0 ; else echo "filter $1 not available" ; return 1; fi
}
# Make sure we are in builddir (not execdir)
cd $builddir
# Parallel make can cause inter-test interference (mostly because of historical naming issues).
# As a protection against this, the isolate() function supports the creation of an
# isolation directory in which created products are stored.

17
unit_test/CMakeLists.txt
View File

@ -8,6 +8,10 @@
# This is the cmake build file for unit_test/ directory.
# @author Ward Fisher
IF(USE_X_GETOPT)
SET(XGETOPTSRC "${CMAKE_CURRENT_SOURCE_DIR}/../libdispatch/XGetopt.c")
ENDIF()
# Currently (August 21, 2019): Some tests will not work with
# Visual Studio. The unit tests are for internal netCDF functions,
# so we don't want to make them external, which would be required to
@ -16,10 +20,6 @@
SET(UNIT_TESTS test_ncuri)
add_bin_test(unit_test test_ncuri)
IF(USE_X_GETOPT)
SET(XGETOPTSRC "${CMAKE_CURRENT_SOURCE_DIR}/../libdispatch/XGetopt.c")
ENDIF()
IF(NETCDF_ENABLE_HDF5)
IF(NOT WIN32)
add_bin_test(unit_test tst_nclist)
@ -27,6 +27,13 @@ IF(NETCDF_ENABLE_HDF5)
ENDIF(NOT WIN32)
build_bin_test(tst_reclaim ${XGETOPTSRC})
add_sh_test(unit_test run_reclaim_tests)
if(NETCDF_ENABLE_NCZARR_FILTERS)
build_bin_test(tst_pluginpaths ${XGETOPTSRC})
add_sh_test(unit_test run_pluginpaths)
add_sh_test(unit_test run_dfaltpluginpath)
build_bin_test(ncpluginpath ${XGETOPTSRC})
add_sh_test(unit_test run_dfaltpluginpath)
endif()
ENDIF(NETCDF_ENABLE_HDF5)
# Path convert test(s)
@ -44,8 +51,10 @@ IF(NETCDF_BUILD_UTILITIES)
ENDIF()
# Performance tests
if(BUILD_BENCHMARKS)
add_bin_test(unit_test tst_exhash timer_utils.c)
add_bin_test(unit_test tst_xcache timer_utils.c)
endif()
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh)
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE)

24
unit_test/Makefile.am
View File

@ -6,7 +6,7 @@
# functions, to allow Windows to work. Since we have not extern'd
# these functions, they will only be run under the autotools build.
# Ed Hartnett 8/9/19
# Dennis Heimbigner 8/27/2024
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
@ -22,15 +22,21 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la
check_PROGRAMS =
TESTS =
noinst_PROGRAMS =
noinst_PROGRAMS += ncpluginpath
ncpluginpath_SOURCES = ncpluginpath.c
check_PROGRAMS += tst_nclist test_ncuri test_pathcvt
TESTS += tst_nclist test_ncuri test_pathcvt
# Performance tests
if BUILD_BENCHMARKS
check_PROGRAMS += tst_exhash tst_xcache
tst_exhash_SOURCES = tst_exhash.c timer_utils.c timer_utils.h
tst_xcache_SOURCES = tst_xcache.c timer_utils.c timer_utils.h
TESTS += tst_nclist test_ncuri test_pathcvt tst_exhash tst_xcache
TESTS += tst_exhash tst_xcache
endif
if USE_HDF5
check_PROGRAMS += tst_nc4internal tst_reclaim
@ -38,6 +44,13 @@ TESTS += tst_nc4internal
TESTS += run_reclaim_tests.sh
endif # USE_HDF5
if USE_HDF5
if NETCDF_ENABLE_NCZARR_FILTERS
check_PROGRAMS += tst_pluginpaths
TESTS += run_pluginpaths.sh run_dfaltpluginpath.sh
endif
endif
if NETCDF_ENABLE_S3
if NETCDF_ENABLE_S3_TESTALL
check_PROGRAMS += test_s3sdk
@ -47,10 +60,11 @@ check_PROGRAMS += aws_config
TESTS += run_aws_config.sh
endif
EXTRA_DIST = CMakeLists.txt run_s3sdk.sh run_reclaim_tests.sh
EXTRA_DIST = CMakeLists.txt run_s3sdk.sh run_reclaim_tests.sh run_aws_config.sh run_pluginpaths.sh run_dfaltpluginpath.sh
EXTRA_DIST += nctest_netcdf4_classic.nc reclaim_tests.cdl
EXTRA_DIST += ref_get.txt ref_set.txt
CLEANFILES = reclaim_tests*.txt reclaim_tests.nc
CLEANFILES = reclaim_tests*.txt reclaim_tests.nc tmp_*.txt
# If valgrind is present, add valgrind targets.
@VALGRIND_CHECK_RULES@

3
unit_test/aws_config.c
View File

@ -46,8 +46,7 @@ main(int argc, char** argv)
int c = 0,stat = NC_NOERR;
/* Load RC and .aws/config */
CHECK(nc_initialize());
CHECK(NC_s3sdkinitialize());
CHECK(nc_initialize()); /* Will invoke NC_s3sdkinitialize()); */
NCglobalstate* gs = NC_getglobalstate();
//Not checking, aborts if ~/.aws/config doesn't exist
CHECK(NC_aws_load_credentials(gs));

165
unit_test/ncpluginpath.c Normal file
View File

@ -0,0 +1,165 @@
/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
#include <stddef.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef _WIN32
#include <windows.h>
#include <fcntl.h>
#endif
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#if defined(_WIN32) && !defined(__MINGW32__)
#include "XGetopt.h"
#else
#include <getopt.h>
#endif
#include "netcdf.h"
#include "netcdf_filter.h"
#include "netcdf_aux.h"
#include "ncplugins.h"
extern int NCstdbinary(void);
static const char* USAGE =
"ncpluginpath [-f global|hdf5|nczarr][-h][-n]"
"Options\n"
" -f which plugin path to print.\n"
" global - print the global plugin path\n"
" hdf5 - print the hdf5 plugin path\n"
" nczarr - print the nczarr plugin path\n"
" If -f is not specified, then it defaults to 'global'\n"
" -h print the usage message\n"
" -n print the length of the current internal plugin path list\n"
"\n"
;
#undef DEBUG
static void usage(const char* msg);
static void
usage(const char* msg)
{
if(msg != NULL) fprintf(stderr,"%s\n",msg);
fprintf(stderr,"%s",USAGE);
if(msg == NULL) exit(0); else exit(1);
}
static int
getformatx(const char* sformat)
{
if(sformat != NULL && strlen(sformat) > 0) {
if(strcmp(sformat,"global")==0) return NC_FORMATX_UNDEFINED;
if(strcmp(sformat,"hdf5")==0) return NC_FORMATX_NC_HDF5;
if(strcmp(sformat,"nczarr")==0) return NC_FORMATX_NCZARR;
}
return NC_FORMATX_UNDEFINED;
}
#if 0
static const char*
getsource(int formatx)
{
switch (formatx) {
case NC_FORMATX_NC_HDF5: return "hdf5";
case NC_FORMATX_NCZARR: return "nczarr";
default: break;
}
return "global";
}
#endif
static int
getfrom(int formatx, char** textp)
{
int stat = NC_NOERR;
NCPluginList dirs = {0,NULL};
char* text = NULL;
/* Get a plugin path */
switch (formatx) {
case 0: /* Global */
if((stat=nc_plugin_path_get(&dirs))) goto done;
break;
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
case NC_FORMATX_NCZARR:
if((stat=NCZ_plugin_path_get(&dirs))) goto done;
break;
#endif
#ifdef USE_HDF5
case NC_FORMATX_NC_HDF5:
if((stat=NC4_hdf5_plugin_path_get(&dirs))) goto done;
break;
#endif
default: abort();
}
if((stat = ncaux_plugin_path_tostring(&dirs,';',&text))) goto done;
*textp = text; text = NULL;
done:
if(text != NULL) free(text);
ncaux_plugin_path_clear(&dirs);
return stat;
}
int
main(int argc, char** argv)
{
int stat = NC_NOERR;
int c;
int formatx = NC_FORMATX_UNDEFINED;
char* text = NULL;
int ndirsflag = 0;
NCstdbinary(); /* avoid \r\n on windows */
nc_initialize();
while ((c = getopt(argc, argv, "f:hn")) != EOF) {
switch(c) {
case 'f':
formatx = getformatx(optarg);
break;
case 'n':
ndirsflag = 1;
break;
case '?':
usage("unknown option");
break;
}
}
if(ndirsflag) {
size_t ndirs = 0;
char sndirs[64];
if((stat = nc_plugin_path_ndirs(&ndirs))) goto done;
snprintf(sndirs,sizeof(sndirs),"%zu",ndirs);
text = strdup(sndirs);
} else {
if((stat = getfrom(formatx,&text))) goto done;
}
printf("%s",text); /* suppress trailing eol */
done:
if(text != NULL) free(text);
nc_finalize();
return (stat?1:0);
}

3
unit_test/ref_get.txt Normal file
View File

@ -0,0 +1,3 @@
testget(global): /zero;/one;/two;/three;/four
testget(hdf5): /zero;/one;/two;/three;/four
testget(nczarr): /zero;/one;/two;/three;/four

6
unit_test/ref_set.txt Normal file
View File

@ -0,0 +1,6 @@
testset(global): before: /zero;/one;/two;/three;/four
testset(global): after: /zero;/one;/mod;/two;/three;/four
testset(hdf5): before: /zero;/one;/two;/three;/four
testset(hdf5): after: /zero;/one;/mod;/two;/three;/four
testset(nczarr): before: /zero;/one;/two;/three;/four
testset(nczarr): after: /zero;/one;/mod;/two;/three;/four

34
unit_test/run_aws_config.sh
View File

@ -7,7 +7,7 @@ set -e
#CMD="valgrind --leak-check=full"
isolate "testdir_ut_aws_config"
isolate "testdir_aws_config"
THISDIR=`pwd`
cd $ISOPATH
@ -19,6 +19,10 @@ test_cleanup() {
}
trap test_cleanup EXIT
test_cleanup
mkdir -p $THISDIR/.aws
cat << 'EOF' > $THISDIR/.aws/config
[uni]
region = somewhere-1
@ -37,31 +41,37 @@ endpoint_url = https://endpoint.example.com/
EOF
cat << 'EOF' > $THISDIR/.aws/credentials
[play]
aws_access_key_id = DummyKeys
aws_secret_access_key = DummySecret
[uni]
region = somewhere-2
endpoint_url = https://example.com/bucket/prefix/2
key = value-overwritten
[play]
aws_access_key_id = DummyKeys
aws_secret_access_key = DummySecret
EOF
echo -e "Testing loading AWS configuration in ${THISDIR}/.aws/config"
NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=unidata ${CMD} ${execdir}/aws_config endpoint_url region dummy_key
export NC_TEST_AWS_DIR=${THISDIR}
export AWS_PROFILE=unidata
${CMD} ${execdir}/aws_config endpoint_url region dummy_key
echo "Status: $?"
exit
export AWS_PROFILE=play
${CMD} ${execdir}/aws_config endpoint_url region
echo "Status: $?"
NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=play ${CMD} ${execdir}/aws_config endpoint_url region
export AWS_PROFILE=uni
${CMD} ${execdir}/aws_config endpoint_url region key
echo "Status: $?"
NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=uni ${CMD} ${execdir}/aws_config endpoint_url region key
echo "Status: $?"
NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=uni ${CMD} ${execdir}/aws_config key=value-overwritten region=somewhere-2 endpoint_url=https://example.com/bucket/prefix/2 extrakey=willbepropagated
export AWS_PROFILE=uni
${CMD} ${execdir}/aws_config key=value-overwritten region=somewhere-2 endpoint_url=https://example.com/bucket/prefix/2 extrakey=willbepropagated
echo "Status: $?"
# Will use profile=no
NC_TEST_AWS_DIR=${THISDIR} ${CMD} ${execdir}/aws_config 2>&1 | grep -q 'Active profile:no'
unset AWS_PROFILE
${CMD} ${execdir}/aws_config 2>&1 | grep -q 'Active profile:no'
echo "Status: $?"
echo -e "Finished"

73
unit_test/run_dfaltpluginpath.sh Executable file
View File

@ -0,0 +1,73 @@
#!/bin/sh
# Test the various plugin path defaults that can occur
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
#CMD="valgrind --leak-check=full"
if test "x$FP_MSVC" = x && test "x$FP_ISMINGW" = x ; then
FEATURE_PLUGIN_SEARCH_PATH=`echon "${FEATURE_PLUGIN_SEARCH_PATH}" | tr ':' ';'`
fi
TESTHPPWIN=`${execdir}/../ncdump/ncpathcvt -S';' -w "/tmp;${HOME}"`
TESTHPPNIX=`${execdir}/../ncdump/ncpathcvt -S';' -u "/tmp;${HOME}"`
#DFALTWIN=`${execdir}/../ncdump/ncpathcvt -S';' -w "C:\\ProgramData\hdf5\\lib\\plugin"`
#DFALTNIX=`${execdir}/../ncdump/ncpathcvt -S';' -u "/usr/local/hdf5/lib/plugin"`
DFALTWIN=`${execdir}/../ncdump/ncpathcvt -S';' -w "${FEATURE_PLUGIN_SEARCH_PATH}"`
DFALTNIX=`${execdir}/../ncdump/ncpathcvt -S';' -u "${FEATURE_PLUGIN_SEARCH_PATH}"`
FAIL=
# Test with no HDF5_PLUGIN_PATH
testnohpp() {
unset HDF5_PLUGIN_PATH
NOHPP1=`${execdir}/ncpluginpath -f global`
if test "x$ISMSVC" = x && test "x$ISMINGW" = x ; then
NOHPP1=`${execdir}/../ncdump/ncpathcvt -S';' -u "$NOHPP1"`
BASELINE="$DFALTNIX"
else
NOHPP1=`${execdir}/../ncdump/ncpathcvt -S';' -w "$NOHPP1"`
BASELINE="$DFALTWIN"
fi
if test "x$NOHPP1" = "x$BASELINE"; then TF=yes ; else TF=no; fi
echo "** compare: $NOHPP1 :: $BASELINE == $TF"
if test "$TF" = yes ; then
echo "***PASS: default plugin path = |$NOHPP1|"
else
FAIL=1
echo "***FAIL: default plugin path = |$NOHPP1|"
fi
}
# Test with given HDF5_PLUGIN_PATH
testhpp() {
unset HDF5_PLUGIN_PATH
if test "x$ISMSVC" = x && test "x$ISMINGW" = x ; then
export HDF5_PLUGIN_PATH="$TESTHPPNIX"
HPP1=`${execdir}/ncpluginpath -f global`
HPP1=`${execdir}/../ncdump/ncpathcvt -S';' -u "$HPP1"`
BASELINE="$TESTHPPNIX"
else
export HDF5_PLUGIN_PATH="$TESTHPPWIN"
HPP1=`${execdir}/ncpluginpath -f global`
HPP1=`${execdir}/../ncdump/ncpathcvt -S';' -w "$HPP1"`
BASELINE="$TESTHPPWIN"
fi
unset TF
if test "x$HPP1" = "x$BASELINE"; then TF=yes ; else TF=no; fi
echo "** compare: $HPP1 :: $BASELINE == $TF"
if test "$TF" = yes ; then
echo "***PASS: default plugin path: |$HPP1| HDF5_PLUGIN_PATH=|$HDF5_PLUGIN_PATH|"
else
FAIL=1
echo "***FAIL: default plugin path: |$HPP1| HDF5_PLUGIN_PATH=|$HDF5_PLUGIN_PATH|"
fi
}
testnohpp
testhpp
if test "x$FAIL" != x ; then exit 1; fi
exit 0

115
unit_test/run_pluginpaths.sh Executable file
View File

@ -0,0 +1,115 @@
#!/bin/bash
# Test the programmatic API for manipulating the plugin paths.
# This script is still full of cruft that needs to be removed
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
IMPLS=
if test "x$FEATURE_HDF5" = xyes ; then IMPLS="$IMPLS hdf5"; fi
if test "x$FEATURE_NCZARR" = xyes ; then IMPLS="$IMPLS nczarr"; fi
# Remove leading blank
IMPLS=`echo "$IMPLS" | cut -d' ' -f2,3`
echo "IMPLS=|$IMPLS|"
#VERBOSE=1
# Bash under windows/mingw has bad habit of translating '/' to '\\'
# for command line arguments
export MSYS2_ARG_CONV_EXCL="*"
DFALT="/zero;/one;/two;/three;/four"
DFALTSET="/zero;/one;/mod;/two;/three;/four"
DFALTHDF5="/zero;/one;/two;/hdf5;/three;/four"
DFALTNCZARR="/zero;/one;/two;/nczarr;three;/four;/five"
if test "x$TESTNCZARR" = x1 ; then
. "$srcdir/test_nczarr.sh"
s3isolate "testdir_pluginpath"
THISDIR=`pwd`
cd $ISOPATH
fi
TP="${execdir}/tst_pluginpaths"
filenamefor() {
# tmp|ref_action
eval "filename=${1}_$2"
}
dfaltfor() {
case "$1" in
hdf5) eval "dfalt=\"$DFALTHDF5\"" ;;
nczarr) eval "dfalt=\"$DFALTNCZARR\"" ;;
all) eval "dfalt=\"$DFALT\"" ;;
esac
}
modfor() {
local formatx="$1"
local dfalt="$2"
case "$formatx" in
hdf5) mod="${dfalt};/modhdf5" ;;
nczarr) mod="/modnczarr;${dfalt}" ;;
all) mode="${dfalt}" ;;
esac
}
#####
# Test that global state is same as that for HDF5 and NCZarr.
# It is difficult to test for outside interference, so not attempted.
testget() {
filenamefor tmp get
# print out the global state
printf "testget(global): %s\n" `${TP} -x "set:${DFALT},get:global"` >> ${filename}.txt
# print out the HDF5 state
printf "testget(hdf5): %s\n" `${TP} -x "set:${DFALT},get:hdf5"` >> ${filename}.txt ;
# print out the NCZarr state
printf "testget(nczarr): %s\n" `${TP} -x "set:${DFALT},get:nczarr"` >> ${filename}.txt ;
}
# Set the global state to some value and verify that it was sync'd to hdf5 and nczarr
testset() {
filenamefor tmp set
# print out the global state, modify it and print again
printf "testset(global): before: %s\n" `${TP} -x "set:${DFALT},get:global"` >> ${filename}.txt
printf "testset(global): after: %s\n" `${TP} -x "set:${DFALT},set:${DFALTSET},get:global"` >> ${filename}.txt
# print out the HDF5 state
printf "testset(hdf5): before: %s\n" `${TP} -x "set:${DFALT},get:hdf5"` >> ${filename}.txt
printf "testset(hdf5): after: %s\n" `${TP} -x "set:${DFALT},set:${DFALTSET},get:hdf5"` >> ${filename}.txt
# print out the NCZarr state
printf "testset(nczarr): before: %s\n" `${TP} -x "set:${DFALT},get:nczarr"` >> ${filename}.txt
printf "testset(nczarr): after: %s\n" `${TP} -x "set:${DFALT},set:${DFALTSET},get:nczarr"` >> ${filename}.txt
}
#########################
cleanup() {
rm -f tmp_*.txt
}
init() {
cleanup
}
# Verify output for a specific action
verify() {
for action in get set ; do
if diff -wBb ${srcdir}/ref_${action}.txt tmp_${action}.txt ; then
echo "***PASS: $action"
else
echo "***FAIL: $action"
exit 1
fi
done
}
init
testget
testset
verify
cleanup

16
unit_test/test_pathcvt.c
View File

@ -13,7 +13,7 @@ Test the NCpathcvt
#include "netcdf.h"
#include "ncpathmgr.h"
#undef DEBUG
#define DEBUG
#define NKINDS 4
static const int kinds[NKINDS] = {NCPD_NIX,NCPD_MSYS,NCPD_CYGWIN,NCPD_WIN};
@ -44,31 +44,31 @@ static Test PATHTESTS[] = {
"d:\\x\\y" /*NCPD_WIN*/
}},
{"/cygdrive/d/x/y",{
"/cygdrive/d/x/y", /*NCPD_LINUX*/
"/d/x/y", /*NCPD_LINUX*/
"d:\\x\\y", /*NCPD_MSYS*/
"/cygdrive/d/x/y", /*NCPD_CYGWIN*/
"d:\\x\\y" /*NCPD_WIN*/
}},
{"/d/x/y",{
"/d/x/y", /*NCPD_LINUX*/
"c:\\d\\x\\y", /*NCPD_MSYS*/
"d:\\x\\y", /*NCPD_MSYS*/
"/cygdrive/c/d/x/y", /*NCPD_CYGWIN*/
"c:\\d\\x\\y" /*NCPD_WIN*/
"d:\\x\\y" /*NCPD_WIN*/
}},
{"/cygdrive/d",{
"/cygdrive/d", /*NCPD_LINUX*/
"/d", /*NCPD_LINUX*/
"d:", /*NCPD_MSYS*/
"/cygdrive/d", /*NCPD_CYGWIN*/
"d:" /*NCPD_WIN*/
}},
{"/d", {
"/d", /*NCPD_LINUX*/
"c:\\d", /*NCPD_MSYS*/
"d:", /*NCPD_MSYS*/
"/cygdrive/c/d", /*NCPD_CYGWIN*/
"c:\\d" /*NCPD_WIN*/
"d:" /*NCPD_WIN*/
}},
{"/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn",{
"/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn", /*NCPD_LINUX*/
"/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn", /*NCPD_LINUX*/
"d:\\git\\netcdf-c\\dap4_test\\test_anon_dim.2.syn", /*NCPD_MSYS*/
"/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn", /*NCPD_CYGWIN*/
"d:\\git\\netcdf-c\\dap4_test\\test_anon_dim.2.syn" /*NCPD_WIN*/

349
unit_test/tst_pluginpaths.c Normal file
View File

@ -0,0 +1,349 @@
/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdlib.h>
#include <stddef.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#if defined(_WIN32) && !defined(__MINGW32__)
#include "XGetopt.h"
#endif
#include "netcdf.h"
#include "netcdf_filter.h"
#include "netcdf_aux.h"
#include "ncplugins.h"
#include "ncpathmgr.h"
#undef DEBUG
/* Always use ';' as path separator */
#define SEP ';'
/* Define max number of -x actions */
#define NACTIONS 64
/* Define max length of -x action string */
#define NACTIONLEN 4096
/* Define max no. of dirs in path list */
#define NDIRSMAX 64
typedef enum Action {
ACT_NONE=0,
ACT_GET=1,
ACT_SET=2,
/* Synthetic Actions */
ACT_CLEAR=3,
} Action;
static struct ActionTable {
Action op;
const char* opname;
} actiontable[] = {
{ACT_NONE,"none"},
{ACT_GET,"get"},
{ACT_SET,"set"},
{ACT_CLEAR,"clear"},
{ACT_NONE,NULL}
};
static struct FormatXTable {
const char* name;
int formatx;
} formatxtable[] = {
{"global",0},
{"hdf5",NC_FORMATX_NC_HDF5},
{"nczarr",NC_FORMATX_NCZARR},
{"zarr",NC_FORMATX_NCZARR},
{NULL,0}
};
/* command line options */
struct Dumpptions {
int debug;
size_t nactions;
struct Execute {
Action action;
char* name;
char* arg;
} actions[NACTIONS];
} dumpoptions;
/* Forward */
#define NCCHECK(expr) nccheck((expr),__LINE__)
static void ncbreakpoint(int stat) {stat=stat;}
static int nccheck(int stat, int line)
{
if(stat) {
fprintf(stderr,"%d: %s\n",line,nc_strerror(stat));
fflush(stderr);
ncbreakpoint(stat);
exit(1);
}
return stat;
}
static void
pluginusage(void)
{
fprintf(stderr,"usage: tst_pluginpath [-d] -x <command>[:<arg>],<command>[:<arg>]...\n");
fprintf(stderr,"\twhere <command> is one of: read | write | clear| or formatx.\n");
fprintf(stderr,"\twhere <arg> is arbitrary string (with '\\,' to escape commas); arg can be missing or empty.\n");
exit(1);
}
static int
decodeformatx(const char* name)
{
struct FormatXTable* p = formatxtable;
for(;p->name != NULL;p++) {
if(strcasecmp(p->name,name)==0) return p->formatx;
}
return -1;
}
static Action
decodeop(const char* name)
{
struct ActionTable* p = actiontable;
for(;p->opname != NULL;p++) {
if(strcasecmp(p->opname,name)==0) return p->op;
}
return ACT_NONE;
}
/* Unescape all escaped characters in s */
static void
descape(char* s)
{
char* p;
char* q;
if(s == NULL) goto done;
for(p=s,q=s;*p;) {
if(*p == '\\' && p[1] != '\0') p++;
*q++ = *p++;
}
*q = *p; /* nul terminate */
done:
return;
}
/* A version of strchr that takes escapes into account */
static char*
xstrchr(char* s, char c)
{
int leave;
char* p;
for(leave=0,p=s;!leave;p++) {
switch (*p) {
case '\\': p++; break;
case '\0': return NULL; break;
default: if(*p == c) return p; break;
}
}
return NULL;
}
static void
parseactionlist(const char* cmds0)
{
size_t i,cmdlen;
char cmds[NACTIONLEN+2];
char* p;
char* q;
size_t ncmds;
int leave;
memset(cmds,0,sizeof(cmds));
if(cmds0 == NULL) cmdlen = 0; else cmdlen = strlen(cmds0);
if(cmdlen == 0) {fprintf(stderr,"error: -x must have non-empty argument.\n"); pluginusage();}
if(cmdlen > NACTIONLEN) {fprintf(stderr,"error: -x argument too lone; max is %zu\n",(size_t)NACTIONLEN); pluginusage();}
strncpy(cmds,cmds0,cmdlen);
/* split into command + formatx + arg strings and count */
ncmds = 0;
fprintf(stderr,"$$$ cmds=|%s|\n",cmds);
for(leave=0,p=cmds;!leave;p=q) {
q = xstrchr(p,',');
fprintf(stderr,"$$$ p=|%s| q=|%s|\n",p,q);
if(q == NULL) {
q = cmds+cmdlen; /* point to trailing nul */
leave = 1;
} else {
*q++ = '\0'; /* overwrite ',' and skip to start of the next command*/
}
ncmds++;
}
fprintf(stderr,"$$$ ncmds=%d\n",(int)ncmds);
if(ncmds > NACTIONS) {fprintf(stderr,"error: -x must have not more than %zu commands.\n",(size_t)NACTIONS); pluginusage();}
dumpoptions.nactions = ncmds;
/* Now process each command+formatx+arg triple */
for(p=cmds,i=0;i<ncmds;i++) {
size_t clen,alen;
clen = strlen(p);
if(clen > NACTIONLEN) {fprintf(stderr,"error: -x cmd '%s' too long; max is %zu\n",p,(size_t)NACTIONLEN); pluginusage();}
/* search for ':' taking escapes into account */
q = xstrchr(p,':');
if(q == NULL)
q = p+clen; /* point to trailing nul */
else
*q++ = '\0'; /* overwrite ':' and skip to start of the arg*/
dumpoptions.actions[i].name = nulldup(p);
/* Get the argument, if any */
alen = strlen(q);
if(alen > 0) {
dumpoptions.actions[i].arg = strdup(q);
}
p += (clen+1); /* move to next cmd+arg pair */
}
/* De-escape names and args and compute action enum */
for(i=0;i<dumpoptions.nactions;i++) {
descape(dumpoptions.actions[i].name);
descape(dumpoptions.actions[i].arg);
dumpoptions.actions[i].action = decodeop(dumpoptions.actions[i].name);
fprintf(stderr,"$$$ [%d] name=|%s| arg=|%s|\n",(int)i,dumpoptions.actions[i].name,dumpoptions.actions[i].arg);
}
return;
}
static int
getfrom(int formatx, char** textp)
{
int stat = NC_NOERR;
NCPluginList dirs = {0,NULL};
char* text = NULL;
/* Get a plugin path */
switch (formatx) {
case 0: /* Global */
if((stat=nc_plugin_path_get(&dirs))) goto done;
break;
case NC_FORMATX_NCZARR:
if((stat=NCZ_plugin_path_get(&dirs))) goto done;
break;
case NC_FORMATX_NC_HDF5:
if((stat=NC4_hdf5_plugin_path_get(&dirs))) goto done;
break;
default: abort();
}
if((stat = ncaux_plugin_path_tostring(&dirs,SEP,&text))) goto done;
if(textp) {*textp = text; text = NULL;}
done:
nullfree(text);
ncaux_plugin_path_clear(&dirs);
return NCCHECK(stat);
}
/**************************************************/
static int
actionclear(const struct Execute* action)
{
int stat = NC_NOERR;
NCPluginList dirs = {0,NULL};
if((stat=nc_plugin_path_set(&dirs))) goto done;
done:
return NCCHECK(stat);
}
static int
actionget(const struct Execute* action)
{
int stat = NC_NOERR;
char* text = NULL;
int formatx = decodeformatx(action->arg);
/* Get global plugin path */
if((stat = getfrom(formatx,&text))) goto done;
printf("%s\n",text);
done:
nullfree(text);
return NCCHECK(stat);
}
static int
actionset(const struct Execute* action)
{
int stat = NC_NOERR;
const char* text = action->arg;
NCPluginList dirs = {0,NULL};
if(text == NULL) text = "";
if((stat=ncaux_plugin_path_parse(text,0,&dirs))) goto done;
if((stat=nc_plugin_path_set(&dirs))) goto done;
done:
ncaux_plugin_path_clear(&dirs);
return NCCHECK(stat);
}
int
main(int argc, char** argv)
{
int stat = NC_NOERR;
int c;
size_t i;
if((stat = NCstdbinary())) {
fprintf(stderr,"@@@@ NCstdbinary failed\n"); fflush(stderr);
}
/* Init options */
memset((void*)&dumpoptions,0,sizeof(dumpoptions));
while ((c = getopt(argc, argv, "dvx:X:")) != EOF) {
switch(c) {
case 'd':
dumpoptions.debug = 1;
break;
case 'v':
pluginusage();
goto done;
case 'x':
parseactionlist(optarg);
break;
case '?':
fprintf(stderr,"unknown option\n");
{stat = NC_EINVAL; goto done;}
}
}
for(i=0;i<dumpoptions.nactions;i++) {
#ifdef DEBUG
fprintf(stderr,">>>> [%zu] %s(%d) : %s\n",i,
dumpoptions.actions[i].name,
dumpoptions.actions[i].action,
dumpoptions.actions[i].arg);
#endif
switch (dumpoptions.actions[i].action) {
default:
fprintf(stderr,"Illegal action: %s\n",dumpoptions.actions[i].name);
pluginusage();
break;
case ACT_CLEAR: if((stat=actionclear(&dumpoptions.actions[i]))) goto done; break;
case ACT_GET: if((stat=actionget(&dumpoptions.actions[i]))) goto done; break;
case ACT_SET: if((stat=actionset(&dumpoptions.actions[i]))) goto done; break;
}
}
done:
fflush(stdout);
if(stat)
fprintf(stderr,"fail: %s\n",nc_strerror(stat));
return (stat ? 1 : 0);
}