2016-09-01 05:38:59 +08:00
# -*- Autoconf -*-
2010-06-03 21:25:25 +08:00
## Process this file with autoconf to produce a configure script.
2018-08-17 01:43:01 +08:00
# This is part of Unidata's netCDF package. Copyright 2005-2018, see
2010-06-03 21:25:25 +08:00
# the COPYRIGHT file for more information.
2017-11-18 06:35:14 +08:00
# Ed Hartnett, Ward Fisher, Dennis Heimbigner
2010-06-03 21:25:25 +08:00
2014-09-05 05:42:11 +08:00
# Running autoconf on this file will trigger a warning if
2010-06-03 21:25:25 +08:00
# autoconf is not at least the specified version.
AC_PREREQ([2.59])
2014-09-05 05:42:11 +08:00
# Initialize with name, version, and support email address.
2022-06-04 00:52:00 +08:00
AC_INIT([netCDF],[4.9.1-development],[support-netcdf@unidata.ucar.edu],[netcdf-c])
2010-06-03 21:25:25 +08:00
2016-05-03 06:07:14 +08:00
##
# Prefer an empty CFLAGS variable instead of the default -g -O2.
# See:
# * http://www.gnu.org/savannah-checkouts/gnu/autoconf/manual/autoconf-2.69/html_node/C-Compiler.html#C-Compiler
##
: ${CFLAGS=""}
2014-09-11 06:50:45 +08:00
AC_SUBST([NC_VERSION_MAJOR]) NC_VERSION_MAJOR=4
2022-06-04 00:52:00 +08:00
AC_SUBST([NC_VERSION_MINOR]) NC_VERSION_MINOR=9
AC_SUBST([NC_VERSION_PATCH]) NC_VERSION_PATCH=1
2015-06-20 02:29:15 +08:00
AC_SUBST([NC_VERSION_NOTE]) NC_VERSION_NOTE="-development"
2014-05-31 03:36:35 +08:00
2022-05-20 12:00:40 +08:00
##
# These linker flags specify libtool version info.
# See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning
# for information regarding incrementing `-version-info`.
# These values should match those in CMakeLists.txt
2022-06-04 00:52:00 +08:00
AC_SUBST([netCDF_SO_VERSION]) netCDF_SO_VERSION=20:0:1
2022-05-20 12:00:40 +08:00
2014-05-31 03:36:35 +08:00
#####
# Set some variables used to generate a libnetcdf.settings file,
# pattered after the files generated by libhdf4, libhdf5.
#####
2010-06-03 21:25:25 +08:00
# Create the VERSION file, which contains the package version from
# AC_INIT.
2014-05-31 03:36:35 +08:00
echo AC_PACKAGE_VERSION>VERSION
2010-06-03 21:25:25 +08:00
AC_SUBST(PACKAGE_VERSION)
AC_MSG_NOTICE([netCDF AC_PACKAGE_VERSION])
# Keep libtool macros in an m4 directory.
AC_CONFIG_MACRO_DIR([m4])
2014-05-31 03:36:35 +08:00
# Configuration Date
2016-05-24 05:31:40 +08:00
if test "x$SOURCE_DATE_EPOCH" != "x" ; then
2016-05-24 05:24:32 +08:00
AC_SUBST([CONFIG_DATE]) CONFIG_DATE="`date -u -d "${SOURCE_DATE_EPOCH}"`"
2016-05-15 23:20:18 +08:00
else
AC_SUBST([CONFIG_DATE]) CONFIG_DATE="`date`"
fi
2014-05-31 03:36:35 +08:00
2010-06-03 21:25:25 +08:00
# Find out about the host we're building on.
AC_CANONICAL_HOST
# Find out about the target we're building for.
AC_CANONICAL_TARGET
AC_CONFIG_HEADERS([config.h])
2021-07-28 01:16:58 +08:00
##
# Check to see if the compiler supports -fno-strict-aliasing and, if so,
# add that to the C compiler flags. This is in support of https://github.com/Unidata/netcdf-c/issues/1983.
##
SAVE_CFLAGS="${CFLAGS}"
AC_LANG_PUSH([C])
2022-04-19 16:47:33 +08:00
AC_LANG_COMPILER_REQUIRE
2021-07-28 01:16:58 +08:00
CFLAGS="${CFLAGS} -fno-strict-aliasing"
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
[int i = 0;]])],
[have_no_strict_aliasing=yes],
[have_no_strict_aliasing=no])
AC_MSG_CHECKING([whether compiler supports -fno-strict-aliasing])
AC_MSG_RESULT([$have_no_strict_aliasing])
AC_LANG_POP([C])
if test $have_no_strict_aliasing = no; then
CFLAGS=$SAVE_CFLAGS
fi
2016-04-12 03:42:14 +08:00
##
# Some files need to exist in build directories
# that do not correspond to their source directory, or
# the test program makes an assumption about where files
# live. AC_CONFIG_LINKS provides a mechanism to link/copy files
# if an out-of-source build is happening.
##
AC_CONFIG_LINKS([nc_test4/ref_hdf5_compat1.nc:nc_test4/ref_hdf5_compat1.nc])
AC_CONFIG_LINKS([nc_test4/ref_hdf5_compat2.nc:nc_test4/ref_hdf5_compat2.nc])
AC_CONFIG_LINKS([nc_test4/ref_hdf5_compat3.nc:nc_test4/ref_hdf5_compat3.nc])
2018-03-05 18:45:18 +08:00
AC_CONFIG_LINKS([hdf4_test/ref_chunked.hdf4:hdf4_test/ref_chunked.hdf4])
AC_CONFIG_LINKS([hdf4_test/ref_contiguous.hdf4:hdf4_test/ref_contiguous.hdf4])
2021-08-19 00:24:42 +08:00
AC_CONFIG_LINKS([dap4_test/baselinethredds/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds:dap4_test/baselinethredds/GOES16_TEST1.nc4.thredds])
2016-06-02 05:20:36 +08:00
AM_INIT_AUTOMAKE([foreign dist-zip subdir-objects])
2013-09-25 05:23:27 +08:00
2012-03-17 04:20:15 +08:00
# Check for the existence of this file before proceeding.
2012-09-07 03:44:03 +08:00
AC_CONFIG_SRCDIR([include/netcdf.h])
2010-06-03 21:25:25 +08:00
2021-12-24 13:18:56 +08:00
# Figure out platforms of special interest
case "`uname`" in
CYGWIN*) ISCYGWIN=yes;;
Darwin*) ISOSX=yes;;
WIN*) ISMSVC=yes;;
MINGW*) ISMINGW=yes;;
esac
if test "x$MSYSTEM" != x ; then
ISMINGW=yes
ISMSYS=yes
fi
2022-02-09 11:53:30 +08:00
# Get windows version info
WINVER=`systeminfo | sed -e '/^OS Version:/p' -ed | sed -e 's|[^0-9]*\([0-9.]*\).*|\1|'`
WINVERMAJOR=`echo $WVER | sed -e 's|\([^.]*\)[.]\([^.]*\)[.]\(.*\)|\1|'`
WINVERBUILD=`echo $WVER | sed -e 's|\([^.]*\)[.]\([^.]*\)[.]\(.*\)|\3|'`
if test "x$WINVERMAJOR" = x ; then WINVERMAJOR=0; fi
if test "x$WINVERBUILD" = x ; then WINVERBUILD=0; fi
AC_DEFINE_UNQUOTED([WINVERMAJOR], [$WINVERMAJOR], [windows version major])
AC_DEFINE_UNQUOTED([WINVERBUILD], [$WINVERBUILD], [windows version build])
2020-08-18 09:15:47 +08:00
AC_MSG_NOTICE([checking supported formats])
# An explicit disable of netcdf-4 | netcdf4 is treated as if it was disable-hdf5
AC_MSG_CHECKING([whether we should build with netcdf4 (alias for HDF5)])
AC_ARG_ENABLE([netcdf4], [AS_HELP_STRING([--disable-netcdf4],
Codify cross-platform file paths
The netcdf-c code has to deal with a variety of platforms:
Windows, OSX, Linux, Cygwin, MSYS, etc. These platforms differ
significantly in the kind of file paths that they accept. So in
order to handle this, I have created a set of replacements for
the most common file system operations such as _open_ or _fopen_
or _access_ to manage the file path differences correctly.
A more limited version of this idea was already implemented via
the ncwinpath.h and dwinpath.c code. So this can be viewed as a
replacement for that code. And in path in many cases, the only
change that was required was to replace '#include <ncwinpath.h>'
with '#include <ncpathmgt.h>' and then replace file operation
calls with the NCxxx equivalent from ncpathmgr.h Note that
recently, the ncwinpath.h was renamed ncpathmgmt.h, so this pull
request should not require dealing with winpath.
The heart of the change is include/ncpathmgmt.h, which provides
alternate operations such as NCfopen or NCaccess and which properly
parse and rebuild path arguments to work for the platform on which
the code is executing. This mostly matters for Windows because of the
way that it uses backslash and drive letters, as compared to *nix*.
One important feature is that the user can do string manipulations
on a file path without having to worry too much about the platform
because the path management code will properly handle most mixed cases.
So one can for example concatenate a path suffix that uses forward
slashes to a Windows path and have it work correctly.
The conversion code is in libdispatch/dpathmgr.c, and the
important function there is NCpathcvt which does the proper
conversions to the local path format.
As a rule, most code should just replace their file operations with
the corresponding NCxxx ones defined in include/ncpathmgmt.h. These
NCxxx functions all call NCpathcvt on their path arguments before
executing the actual file operation.
In some rare cases, the client may need to directly use NCpathcvt,
but this should be avoided as much as possible. If there is a need
for supporting a new file operation not already in ncpathmgmt.h, then
use the code in dpathmgr.c as a template. Also please notify Unidata
so we can include it as a formal part or our supported operations.
Also, if you see an operation in the library that is not using the
NCxxx form, then please submit an issue so we can fix it.
Misc. Changes:
* Clean up the utf8 testing code; it is impossible to get some
tests to work under windows using shell scripts; the args do
not pass as utf8 but as some other encoding.
* Added an extra utf8 test case: test_unicode_path.sh
* Add a true test for HDF5 1.10.6 or later because as noted in
PR https://github.com/Unidata/netcdf-c/pull/1794,
HDF5 changed its Windows file path handling.
2021-03-05 04:41:31 +08:00
[(deprecated synonym for --enable-hdf5)])])
2020-08-18 09:15:47 +08:00
test "x$enable_netcdf4" = xno || enable_netcdf4=yes
Codify cross-platform file paths
The netcdf-c code has to deal with a variety of platforms:
Windows, OSX, Linux, Cygwin, MSYS, etc. These platforms differ
significantly in the kind of file paths that they accept. So in
order to handle this, I have created a set of replacements for
the most common file system operations such as _open_ or _fopen_
or _access_ to manage the file path differences correctly.
A more limited version of this idea was already implemented via
the ncwinpath.h and dwinpath.c code. So this can be viewed as a
replacement for that code. And in path in many cases, the only
change that was required was to replace '#include <ncwinpath.h>'
with '#include <ncpathmgt.h>' and then replace file operation
calls with the NCxxx equivalent from ncpathmgr.h Note that
recently, the ncwinpath.h was renamed ncpathmgmt.h, so this pull
request should not require dealing with winpath.
The heart of the change is include/ncpathmgmt.h, which provides
alternate operations such as NCfopen or NCaccess and which properly
parse and rebuild path arguments to work for the platform on which
the code is executing. This mostly matters for Windows because of the
way that it uses backslash and drive letters, as compared to *nix*.
One important feature is that the user can do string manipulations
on a file path without having to worry too much about the platform
because the path management code will properly handle most mixed cases.
So one can for example concatenate a path suffix that uses forward
slashes to a Windows path and have it work correctly.
The conversion code is in libdispatch/dpathmgr.c, and the
important function there is NCpathcvt which does the proper
conversions to the local path format.
As a rule, most code should just replace their file operations with
the corresponding NCxxx ones defined in include/ncpathmgmt.h. These
NCxxx functions all call NCpathcvt on their path arguments before
executing the actual file operation.
In some rare cases, the client may need to directly use NCpathcvt,
but this should be avoided as much as possible. If there is a need
for supporting a new file operation not already in ncpathmgmt.h, then
use the code in dpathmgr.c as a template. Also please notify Unidata
so we can include it as a formal part or our supported operations.
Also, if you see an operation in the library that is not using the
NCxxx form, then please submit an issue so we can fix it.
Misc. Changes:
* Clean up the utf8 testing code; it is impossible to get some
tests to work under windows using shell scripts; the args do
not pass as utf8 but as some other encoding.
* Added an extra utf8 test case: test_unicode_path.sh
* Add a true test for HDF5 1.10.6 or later because as noted in
PR https://github.com/Unidata/netcdf-c/pull/1794,
HDF5 changed its Windows file path handling.
2021-03-05 04:41:31 +08:00
AC_MSG_RESULT([$enable_netcdf4 (deprecated; Please use with --disable-hdf5)])
2020-08-18 09:15:47 +08:00
AC_MSG_CHECKING([whether we should build with netcdf-4 (alias for HDF5)])
AC_ARG_ENABLE([netcdf-4], [AS_HELP_STRING([--disable-netcdf-4],
[(synonym for --disable-netcdf4)])])
test "x$enable_netcdf_4" = xno || enable_netcdf_4=yes
AC_MSG_RESULT([$enable_netcdf_4])
# Propagate the alias
if test "x$enable_netcdf_4" = xno ; then enable_netcdf4=no; fi
if test "x$enable_netcdf4" = xno ; then enable_netcdf_4=no; fi
# Does the user want to use HDF5?
AC_MSG_CHECKING([whether we should build with HDF5])
AC_ARG_ENABLE([hdf5], [AS_HELP_STRING([--disable-hdf5],
[do not build with HDF5])])
test "x$enable_hdf5" = xno || enable_hdf5=yes
if test "x$enable_netcdf4" = xno ; then enable_hdf5=no ; fi
# disable-netcdf4 is synonym for disable-hdf5
AC_MSG_RESULT([$enable_hdf5])
# Check whether we want to enable CDF5 support.
AC_MSG_CHECKING([whether CDF5 support should be disabled])
AC_ARG_ENABLE([cdf5],
[AS_HELP_STRING([--disable-cdf5],
[build without CDF5 support. @<:@default: auto@:>@])],
[enable_cdf5=${enableval}], [enable_cdf5=auto]
)
AC_MSG_RESULT($enable_cdf5)
# Does the user want to turn on HDF4 read ability?
AC_MSG_CHECKING([whether reading of HDF4 SD files is to be enabled])
AC_ARG_ENABLE([hdf4], [AS_HELP_STRING([--enable-hdf4],
[build netcdf with HDF4 read capability (HDF4, HDF5 and zlib required)])])
test "x$enable_hdf4" = xyes || enable_hdf4=no
AC_MSG_RESULT($enable_hdf4)
AC_MSG_CHECKING([whether parallel I/O for classic files is to be enabled])
AC_ARG_ENABLE([pnetcdf], [AS_HELP_STRING([--enable-pnetcdf],
[build with parallel I/O for classic files. @<:@default: disabled@:>@])])
test "x$enable_pnetcdf" = xyes || enable_pnetcdf=no
AC_MSG_RESULT($enable_pnetcdf)
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
# We need curl for remote operations
AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
if test "x$found_curl" = "xyes" ; then
AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll cygcurl.dll], [],[])
fi
2020-08-18 09:15:47 +08:00
## Capture the state of the --enable-dap flag => enable dap2+dap4
AC_MSG_CHECKING([whether DAP client(s) are to be built])
AC_ARG_ENABLE([dap],
[AS_HELP_STRING([--disable-dap],
[build without DAP client support.])])
test "x$enable_dap" = xno || enable_dap=yes
AC_MSG_RESULT($enable_dap)
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
if test "x$enable_dap" = xyes & test "x$found_curl" = xno ; then
AC_MSG_WARN([curl required for dap access. DAP support disabled.])
enable_dap=no
fi
AC_MSG_CHECKING([whether netcdf zarr storage format should be disabled])
2020-08-18 09:15:47 +08:00
AC_ARG_ENABLE([nczarr],
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
[AS_HELP_STRING([--disable-nczarr],
[disable netcdf zarr storage support])])
test "x$enable_nczarr" = xno || enable_nczarr=yes
2020-08-18 09:15:47 +08:00
AC_MSG_RESULT($enable_nczarr)
# HDF5 | HDF4 | NCZarr => netcdf-4
if test "x$enable_hdf5" = xyes || test "x$enable_hdf4" = xyes || test "x$enable_nczarr" = xyes ; then
enable_netcdf_4=yes
fi
AC_MSG_CHECKING([whether netcdf-4 should be forcibly enabled])
AC_MSG_RESULT([$enable_netcdf_4])
# Synonym
enable_netcdf4=${enable_netcdf_4}
2010-06-03 21:25:25 +08:00
AC_MSG_NOTICE([checking user options])
# Did the user specify a default minimum blocksize (NCIO_MINBLOCKSIZE) for posixio?
AC_MSG_CHECKING([whether a NCIO_MINBLOCKSIZE was specified])
AC_ARG_WITH([minblocksize],
[AS_HELP_STRING([--with-minblocksize=<integer>],
[Specify minimum I/O blocksize for netCDF classic and 64-bit offset format files.])],
[NCIO_MINBLOCKSIZE=$with_minblocksize], [NCIO_MINBLOCKSIZE=256])
AC_MSG_RESULT([$NCIO_MINBLOCKSIZE])
AC_DEFINE_UNQUOTED([NCIO_MINBLOCKSIZE], [$NCIO_MINBLOCKSIZE], [min blocksize for posixio.])
2018-03-29 03:54:05 +08:00
# Find valgrind, if available, and add targets for it.
AX_VALGRIND_DFLT([sgcheck], [off])
AX_VALGRIND_CHECK
2018-10-31 10:48:12 +08:00
AM_CONDITIONAL(ENABLE_VALGRIND, [test "x$VALGRIND_ENABLED" = xyes])
2018-03-29 03:54:05 +08:00
2015-05-27 04:29:28 +08:00
###
# Doxygen and doxygen-related options.
###
2011-08-22 21:53:37 +08:00
AC_ARG_ENABLE([doxygen],
2012-05-19 04:31:50 +08:00
[AS_HELP_STRING([--enable-doxygen],
[Enable generation of documentation.])])
test "x$enable_doxygen" = xyes || enable_doxygen=no
2014-09-05 05:42:11 +08:00
AM_CONDITIONAL([BUILD_DOCS], [test "x$enable_doxygen" = xyes])
2011-08-22 21:53:37 +08:00
2015-05-27 04:29:28 +08:00
AC_ARG_ENABLE([doxygen-tasks],
[AS_HELP_STRING([--enable-doxygen-tasks],
[Enable Doxygen-generated test, todo and bug list documentation. Developers only.])])
test "x$enable_doxygen_tasks" = xyes || enable_doxygen_tasks=no
AM_CONDITIONAL([SHOW_DOXYGEN_TAG_LIST], [test "x$enable_doxygen_tasks" = xyes])
AC_SUBST([SHOW_DOXYGEN_TAG_LIST], [$enable_doxygen_tasks])
2015-08-18 02:19:51 +08:00
###
# Determine if we should build documentation
# configured for releases on the Unidata web server.
###
AC_ARG_ENABLE([doxygen-build-release-docs],
[AS_HELP_STRING([--enable-doxygen-build-release-docs],
[Build release documentation. This is of interest only to developers.])])
test "x$enable_doxygen_build_release_docs" = xyes || enable_doxygen_build_release_docs=no
AM_CONDITIONAL([DOXYGEN_BUILD_RELEASE_DOCS], [test "x$enable_doxygen_build_release_docs" = xyes])
if test $enable_doxygen_build_release_docs = yes; then
AC_SUBST([DOXYGEN_CSS_FILE], ["release.css"])
AC_SUBST([DOXYGEN_HEADER_FILE], ["release_header.html"])
AC_SUBST([DOXYGEN_SEARCHENGINE], ["NO"])
else
AC_SUBST([DOXYGEN_CSS_FILE], [])
AC_SUBST([DOXYGEN_HEADER_FILE], [])
AC_SUBST([DOXYGEN_SEARCHENGINE], ["YES"])
fi
2017-11-13 21:37:07 +08:00
AC_SUBST([DOXYGEN_SERVER_BASED_SEARCH], ["NO"])
2015-05-27 04:29:28 +08:00
AC_ARG_ENABLE([doxygen-pdf-output],
[AS_HELP_STRING([--enable-doxygen-pdf-output],
[Build netCDF library documentation in PDF format. Experimental.])])
AM_CONDITIONAL([NC_ENABLE_DOXYGEN_PDF_OUTPUT], [test "x$enable_doxygen_pdf_output" = xyes])
AC_SUBST([NC_ENABLE_DOXYGEN_PDF_OUTPUT], [$enable_doxygen_pdf_output])
2014-02-05 07:13:18 +08:00
AC_ARG_ENABLE([dot],
[AS_HELP_STRING([--enable-dot],
[Use dot (provided by graphviz) to generate charts and graphs in the doxygen-based documentation.])])
test "x$enable_dot" = xyes || enable_dot=no
2011-08-22 21:53:37 +08:00
AC_ARG_ENABLE([internal-docs],
[AS_HELP_STRING([--enable-internal-docs],
[Include documentation of library internals. This is of interest only to those developing the netCDF library.])])
2011-09-16 00:57:16 +08:00
test "x$enable_internal_docs" = xyes || enable_internal_docs=no
2014-09-05 05:42:11 +08:00
AC_SUBST([BUILD_INTERNAL_DOCS], [$enable_internal_docs])
2011-08-22 21:53:37 +08:00
2018-07-02 11:56:59 +08:00
# Doxygen is apparently buggy when trying to combine a markdown
# file with @internal. The equivalent can be faked using
# the Doxygen ENABLED_SECTIONS mechanism. See docs/testserver.dox
# to see how this is done.
sections=
if test "x$enable_internal_docs" = xyes ; then
sections="$sections INTERNAL"
fi
AC_SUBST([ENABLED_DOC_SECTIONS], [$sections])
2010-06-03 21:25:25 +08:00
AC_MSG_CHECKING([if fsync support is enabled])
AC_ARG_ENABLE([fsync],
2013-03-06 04:29:49 +08:00
[AS_HELP_STRING([--enable-fsync],
[enable fsync support])],
[],
[enable_fsync=no])
2010-06-03 21:25:25 +08:00
test "x$enable_fsync" = xno || enable_fsync=yes
AC_MSG_RESULT($enable_fsync)
if test "x$enable_fsync" = xyes ; then
AC_DEFINE([USE_FSYNC], [1], [if true, include experimental fsync code])
fi
2014-04-16 11:25:44 +08:00
# Temporary until JNA bug is fixed (which is probably never).
2018-09-05 01:27:47 +08:00
# The problem being solved is this:
# > On Windows using the microsoft runtime, it is an error
# > for one library to free memory allocated by a different library.
# This is probably only an issue when using the netcdf-c library
# via JNA under Java.
AC_MSG_CHECKING([if jna bug workaround is enabled])
2014-04-10 09:55:20 +08:00
AC_ARG_ENABLE([jna],
[AS_HELP_STRING([--enable-jna],
[enable jna bug workaround])],
[],
[enable_jna=no])
test "x$enable_jna" = xno || enable_jna=yes
AC_MSG_RESULT($enable_jna)
if test "x$enable_jna" = xyes ; then
AC_DEFINE([JNA], [1], [if true, include jna bug workaround code])
fi
2019-08-09 23:31:24 +08:00
# Does the user want to turn off unit tests (useful for test coverage
# analysis).
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
AC_MSG_CHECKING([if unit tests should be disabled])
2019-08-09 23:31:24 +08:00
AC_ARG_ENABLE([unit-tests],
[AS_HELP_STRING([--disable-unit-tests],
2019-08-14 00:55:44 +08:00
[Disable tests in unit_test directory. Other tests still run.])])
2019-08-09 23:31:24 +08:00
test "x$enable_unit_tests" = xno || enable_unit_tests=yes
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
AC_MSG_RESULT($enable_unit_tests)
2019-08-09 23:31:24 +08:00
AM_CONDITIONAL([BUILD_UNIT_TESTS], [test "x$enable_unit_tests" = xyes])
2013-06-11 05:48:11 +08:00
# Does the user require dynamic loading?
# This is only for those hdf5 installs that support it.
AC_MSG_CHECKING([do we require hdf5 dynamic-loading support])
AC_ARG_ENABLE([dynamic-loading], [AS_HELP_STRING([--enable-dynamic-loading],
[enable dynamic loading for use with supported hdf5 installs (libdl, HDF5 required)])])
2014-05-06 05:37:47 +08:00
test "x$enable_dynamic_loading" = xno || enable_dynamic_loading=yes
2013-06-11 05:48:11 +08:00
AC_MSG_RESULT([$enable_dynamic_loading])
2010-06-03 21:25:25 +08:00
# Does the user want to turn on extra HDF4 file tests?
AC_MSG_CHECKING([whether to fetch some sample HDF4 files from Unidata ftp site to test HDF4 reading (requires wget)])
AC_ARG_ENABLE([hdf4-file-tests], [AS_HELP_STRING([--enable-hdf4-file-tests],
[get some HDF4 files from Unidata ftp site and test that they can be read])])
test "x$enable_hdf4" = xyes -a "x$enable_hdf4_file_tests" = xyes || enable_hdf4_file_tests=no
if test "x$enable_hdf4_file_tests" = xyes; then
AC_DEFINE([USE_HDF4_FILE_TESTS], 1, [If true, use use wget to fetch some sample HDF4 data, and then test against it.])
fi
AC_MSG_RESULT($enable_hdf4_file_tests)
2014-11-19 06:33:16 +08:00
# Does the user want to try to install netcdf-fortran
# automatically?
AC_MSG_CHECKING([whether we should attempt to install netcdf-fortran (EXPERIMENTAL)])
2018-09-15 06:08:11 +08:00
AC_ARG_ENABLE([remote-fortran-bootstrap], [AS_HELP_STRING([--enable-remote-fortran-bootstrap],
2014-11-19 06:33:16 +08:00
[Download and install netcdf-fortran (EXPERIMENTAL)])])
2014-11-21 06:56:11 +08:00
test "x$enable_remote_fortran_bootstrap" = xyes || enable_remote_fortran_bootstrap=no
AC_MSG_RESULT([$enable_remote_fortran_bootstrap])
2014-11-19 06:33:16 +08:00
2010-06-03 21:25:25 +08:00
# Does the user want to run extra parallel tests when parallel netCDF-4 is built?
AC_MSG_CHECKING([whether parallel IO tests should be run])
AC_ARG_ENABLE([parallel-tests],
[AS_HELP_STRING([--enable-parallel-tests],
2018-02-27 08:08:26 +08:00
[Run extra parallel IO tests. Requires netCDF-4
with parallel I/O support.])])
2010-06-03 21:25:25 +08:00
test "x$enable_parallel_tests" = xyes || enable_parallel_tests=no
AC_MSG_RESULT($enable_parallel_tests)
2018-08-18 18:17:24 +08:00
# Did the user specify an MPI launcher other than mpiexec?
2018-08-16 21:23:41 +08:00
AC_MSG_CHECKING([whether a user specified program to run mpi programs])
AC_ARG_WITH([mpiexec],
[AS_HELP_STRING([--with-mpiexec=<command>],
[Specify command to launch MPI parallel tests.])],
[MPIEXEC=$with_mpiexec], [MPIEXEC=mpiexec])
AC_MSG_RESULT([$MPIEXEC])
AC_SUBST([MPIEXEC], [$MPIEXEC])
2010-06-03 21:25:25 +08:00
# Did the user specify a default chunk size?
AC_MSG_CHECKING([whether a default chunk size in bytes was specified])
AC_ARG_WITH([default-chunk-size],
[AS_HELP_STRING([--with-default-chunk-size=<integer>],
[Specify default size of chunks in bytes.])],
2010-06-23 06:41:04 +08:00
[DEFAULT_CHUNK_SIZE=$with_default_chunk_size], [DEFAULT_CHUNK_SIZE=4194304])
2010-06-03 21:25:25 +08:00
AC_MSG_RESULT([$DEFAULT_CHUNK_SIZE])
AC_DEFINE_UNQUOTED([DEFAULT_CHUNK_SIZE], [$DEFAULT_CHUNK_SIZE], [default chunk size in bytes])
# Did the user specify a max per-var cache size?
AC_MSG_CHECKING([whether a maximum per-variable cache size for HDF5 was specified])
AC_ARG_WITH([max-default-cache-size],
[AS_HELP_STRING([--with-max-default-cache-size=<integer>],
[Specify maximum size (in bytes) for the default per-var chunk cache.])],
[MAX_DEFAULT_CACHE_SIZE=$with_max_default_cache_size], [MAX_DEFAULT_CACHE_SIZE=67108864])
AC_MSG_RESULT([$MAX_DEFAULT_CACHE_SIZE])
AC_DEFINE_UNQUOTED([MAX_DEFAULT_CACHE_SIZE], [$MAX_DEFAULT_CACHE_SIZE], [max size of the default per-var chunk cache.])
# Did the user specify a number of chunks in default per-var cache size?
AC_MSG_CHECKING([whether a number of chunks for the default per-variable cache was specified])
AC_ARG_WITH([default-chunks-in-cache],
[AS_HELP_STRING([--with-default-chunks-in-cache=<integer>],
[Specify the number of chunks to store in default per-variable cache.])],
[DEFAULT_CHUNKS_IN_CACHE=$with_default_chunks_in_cache], [DEFAULT_CHUNKS_IN_CACHE=10])
AC_MSG_RESULT([$DEFAULT_CHUNKS_IN_CACHE])
AC_DEFINE_UNQUOTED([DEFAULT_CHUNKS_IN_CACHE], [$DEFAULT_CHUNKS_IN_CACHE], [num chunks in default per-var chunk cache.])
2020-11-20 08:01:04 +08:00
# Did the user specify a default cache size for HDF5?
2010-06-03 21:25:25 +08:00
AC_MSG_CHECKING([whether a default file cache size for HDF5 was specified])
AC_ARG_WITH([chunk-cache-size],
[AS_HELP_STRING([--with-chunk-cache-size=<integer>],
[Specify default file cache chunk size for HDF5 files in bytes.])],
2019-11-26 07:33:04 +08:00
[CHUNK_CACHE_SIZE=$with_chunk_cache_size], [CHUNK_CACHE_SIZE=16777216])
2010-06-03 21:25:25 +08:00
AC_MSG_RESULT([$CHUNK_CACHE_SIZE])
AC_DEFINE_UNQUOTED([CHUNK_CACHE_SIZE], [$CHUNK_CACHE_SIZE], [default file chunk cache size in bytes.])
# Did the user specify a default cache nelems?
AC_MSG_CHECKING([whether a default file cache maximum number of elements for HDF5 was specified])
AC_ARG_WITH([chunk-cache-nelems],
[AS_HELP_STRING([--with-chunk-cache-nelems=<integer>],
[Specify default maximum number of elements in the file chunk cache chunk for HDF5 files (should be prime number).])],
2019-11-26 07:33:04 +08:00
[CHUNK_CACHE_NELEMS=$with_chunk_cache_nelems], [CHUNK_CACHE_NELEMS=4133])
2010-06-03 21:25:25 +08:00
AC_MSG_RESULT([$CHUNK_CACHE_NELEMS])
AC_DEFINE_UNQUOTED([CHUNK_CACHE_NELEMS], [$CHUNK_CACHE_NELEMS], [default file chunk cache nelems.])
# Did the user specify a default cache preemption?
AC_MSG_CHECKING([whether a default cache preemption for HDF5 was specified])
AC_ARG_WITH([chunk-cache-preemption],
[AS_HELP_STRING([--with-chunk-cache-preemption=<float between 0 and 1 inclusive>],
[Specify default file chunk cache preemption policy for HDF5 files (a number between 0 and 1, inclusive).])],
[CHUNK_CACHE_PREEMPTION=$with_chunk_cache_preemption], [CHUNK_CACHE_PREEMPTION=0.75])
AC_MSG_RESULT([$CHUNK_CACHE_PREEMPTION])
AC_DEFINE_UNQUOTED([CHUNK_CACHE_PREEMPTION], [$CHUNK_CACHE_PREEMPTION], [default file chunk cache preemption policy.])
# Does the user want to enable netcdf-4 logging?
AC_MSG_CHECKING([whether netCDF-4 logging is enabled])
AC_ARG_ENABLE([logging],
[AS_HELP_STRING([--enable-logging],
[enable logging capability (only applies when netCDF-4 is built). \
This debugging features is only of interest to netCDF developers. \
2014-09-05 05:42:11 +08:00
Ignored if netCDF-4 is not enabled.])])
2010-06-03 21:25:25 +08:00
test "x$enable_logging" = xyes || enable_logging=no
AC_MSG_RESULT([$enable_logging])
2018-05-13 01:27:11 +08:00
# Does the user want to turn off nc_set_log_level() function? (It will
# always be defined if --enable-logging is used.)
AC_MSG_CHECKING([whether nc_set_log_level() function is included (will do nothing unless enable-logging is also used)])
AC_ARG_ENABLE([set_log_level_func], [AS_HELP_STRING([--disable-set-log-level-func],
[disable the nc_set_log_level function])])
test "x$enable_set_log_level_func" = xno -a "x$enable_logging" = xno || enable_set_log_level_func=yes
2018-05-13 20:35:19 +08:00
if test "x$enable_set_log_level_func" = xyes -a "x$enable_netcdf_4" = xyes; then
2018-05-13 01:46:04 +08:00
AC_DEFINE([ENABLE_SET_LOG_LEVEL], 1, [If true, define nc_set_log_level.])
2018-05-13 01:27:11 +08:00
fi
AC_MSG_RESULT($enable_set_log_level_func)
2011-04-29 01:11:21 +08:00
2020-09-02 03:44:24 +08:00
# CURLOPT_USERNAME is not defined until curl version 7.19.1
# CURLOPT_PASSWORD is not defined until curl version 7.19.1
# CURLOPT_KEYPASSWD is not defined until curl version 7.16.4
# CURLINFO_RESPONSE_CODE is not defined until curl version 7.10.7
# CURLOPT_CHUNK_BGN_FUNCTION is not defined until curl version 7.21.0
# CURL_MAX_READ_SIZE is not defined until 7.59
# Save/restore CFLAGS
SAVECFLAGS="$CFLAGS"
CFLAGS="${curl_cflags}"
AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[#include "curl/curl.h"],
[[int x = CURLOPT_USERNAME;]])],
[haveusername=yes],
[haveusername=no])
AC_MSG_CHECKING([whether CURLOPT_USERNAME is defined])
AC_MSG_RESULT([${haveusername}])
if test $haveusername = yes; then
AC_DEFINE([HAVE_CURLOPT_USERNAME],[1],[Is CURLOPT_USERNAME defined])
fi
AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[#include "curl/curl.h"],
[[int x = CURLOPT_PASSWORD;]])],
[havepassword=yes],
[havepassword=no])
AC_MSG_CHECKING([whether CURLOPT_PASSWORD is defined])
AC_MSG_RESULT([${havepassword}])
if test $havepassword = yes; then
AC_DEFINE([HAVE_CURLOPT_PASSWORD],[1],[Is CURLOPT_PASSWORD defined])
fi
AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[#include "curl/curl.h"],
[[int x = CURLOPT_KEYPASSWD;]])],
[havekeypassword=yes],
[havekeypassword=no])
AC_MSG_CHECKING([whether CURLOPT_KEYPASSWD is defined])
AC_MSG_RESULT([${havekeypassword}])
if test $havekeypassword = yes; then
AC_DEFINE([HAVE_CURLOPT_KEYPASSWD],[1],[Is CURLOPT_KEYPASSWD defined])
fi
AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[#include "curl/curl.h"],
[[int x = CURLINFO_RESPONSE_CODE;]])],
[haveresponsecode=yes],
[haveresponsecode=no])
AC_MSG_CHECKING([whether CURLINFO_RESPONSE_CODE is defined])
AC_MSG_RESULT([${haveresponsecode}])
if test $haveresponsecode = yes; then
AC_DEFINE([HAVE_CURLINFO_RESPONSE_CODE],[1],[Is CURLINFO_RESPONSE_CODE defined])
fi
AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[#include "curl/curl.h"],
[[int x = CURLOPT_BUFFERSIZE;]])],
[havecurloption=yes],
[havecurloption=no])
AC_MSG_CHECKING([whether CURLOPT_BUFFERSIZE is defined])
AC_MSG_RESULT([${havecurloption}])
if test $havecurloption = yes; then
AC_DEFINE([HAVE_CURLOPT_BUFFERSIZE],[1],[Is CURLOPT_BUFFERSIZE defined])
fi
AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[#include "curl/curl.h"],
[[int x = CURLOPT_TCP_KEEPALIVE;]])],
[havecurloption=yes],
[havecurloption=no])
AC_MSG_CHECKING([whether CURLOPT_TCP_KEEPALIVE is defined])
AC_MSG_RESULT([${havecurloption}])
if test $havecurloption = yes; then
AC_DEFINE([HAVE_CURLOPT_KEEPALIVE],[1],[Is CURLOPT_TCP_KEEPALIVE defined])
fi
# CURLOPT_VERIFYHOST semantics differ depending on version
AC_MSG_CHECKING([whether libcurl is version 7.66 or later?])
AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[#include "curl/curl.h"],
[[
#if LIBCURL_VERSION_NUM < 0x074200
error "<7.66";
#endif
]])], [libcurl766=yes], [libcurl766=no])
AC_MSG_RESULT([$libcurl766])
if test x$libcurl66 = xno; then
AC_DEFINE([HAVE_LIBCURL_766],[1],[Is libcurl version 7.66 or later])
fi
CFLAGS="$SAVECFLAGS"
2022-03-16 05:33:13 +08:00
###
# Libxml2 control block.
###
AC_MSG_CHECKING([whether to search for and use external libxml2])
AC_ARG_ENABLE([libxml2],
[AS_HELP_STRING([--disable-libxml2],
[disable detection and use of libxml2 in favor of the bundled ezxml interpreter])])
test "x$enable_libxml2" = xno || enable_libxml2=yes
AC_MSG_RESULT([$enable_libxml2])
have_libxml2=no
if test "x$enable_libxml2" = xyes; then
# We can optionally use libxml2 for DAP4, if available
AC_CHECK_LIB([xml2],[xmlReadMemory],[have_libxml2=yes],[have_libxml2=no])
if test "x$have_libxml2" = "xyes" ; then
AC_SEARCH_LIBS([xmlReadMemory],[xml2 xml2.dll cygxml2.dll], [],[])
fi
if test "x$have_libxml2" = xyes; then
XML2FLAGS=`xml2-config --cflags`
AC_SUBST([XML2FLAGS],${XML2FLAGS})
AC_DEFINE([HAVE_LIBXML2], [1], [if true, use libxml2])
fi
fi
if test "x$enable_libxml2" = xyes; then
XMLPARSER="libxml2"
else
XMLPARSER="tinyxml2 (bundled)"
fi
# Need a condition and subst for this
AM_CONDITIONAL(ENABLE_LIBXML2, [test "x$enable_libxml2" = xyes])
AC_SUBST([XMLPARSER],[${XMLPARSER}])
###
# End Libxml2 block
###
2018-11-09 22:39:12 +08:00
# --enable-dap => enable-dap4
enable_dap4=$enable_dap
2021-08-13 17:06:42 +08:00
AC_MSG_CHECKING([whether dap remote testing should be enabled])
2010-06-03 21:25:25 +08:00
AC_ARG_ENABLE([dap-remote-tests],
2021-12-11 09:53:00 +08:00
[AS_HELP_STRING([--enable-dap-remote-tests],
[enable dap remote tests])])
2021-12-17 06:43:22 +08:00
test "x$enable_dap_remote_tests" = xno || enable_dap_remote_tests=yes
2010-06-03 21:25:25 +08:00
if test "x$enable_dap" = "xno" ; then
enable_dap_remote_tests=no
fi
AC_MSG_RESULT($enable_dap_remote_tests)
2019-03-12 22:39:06 +08:00
# Default is not to do the remote authorization tests.
2014-03-08 03:04:38 +08:00
AC_MSG_CHECKING([whether dap remote authorization testing should be enabled (default off)])
2014-03-11 06:40:52 +08:00
AC_ARG_ENABLE([dap-auth-tests],
2014-03-08 03:04:38 +08:00
[AS_HELP_STRING([--enable-dap-auth-tests],
[enable dap remote authorization tests])])
test "x$enable_dap_auth_tests" = xyes || enable_dap_auth_tests=no
# dap must be enabled
if test "x$enable_dap" = "xno" ; then
enable_dap_auth_tests=no
fi
# if remote tests are disabled, then so is this
if test "x$enable_dap_remote_tests" = "xno" ; then
enable_dap_remote_tests=no
fi
AC_MSG_RESULT($enable_dap_auth_tests)
2017-03-09 08:01:10 +08:00
# Did the user specify a list of test servers to try for remote tests?
AC_MSG_CHECKING([which remote test server(s) to use])
AC_ARG_WITH([testservers],
2018-06-27 03:58:45 +08:00
[AS_HELP_STRING([--with-testservers=<host:port>,<host:port>...],
2017-03-09 08:01:10 +08:00
[Specify the testserver(s) to try for remote tests.])],
[REMOTETESTSERVERS=$with_testservers], [REMOTETESTSERVERS=no])
msg="$REMOTETESTSERVERS"
if test "x$REMOTETESTSERVERS" = xno ; then
2020-01-09 06:18:31 +08:00
svclist="remotetest.unidata.ucar.edu"
2018-06-27 03:58:45 +08:00
REMOTETESTSERVERS="$svclist"
2017-03-09 08:01:10 +08:00
fi
2018-06-27 03:58:45 +08:00
AC_MSG_RESULT([$svclist])
2017-03-09 08:01:10 +08:00
AC_DEFINE_UNQUOTED([REMOTETESTSERVERS], ["$REMOTETESTSERVERS"], [the testservers for remote tests.])
2010-12-16 05:45:05 +08:00
# Set the config.h flags
if test "x$enable_dap" = xyes; then
AC_DEFINE([USE_DAP], [1], [if true, build DAP Client])
AC_DEFINE([ENABLE_DAP], [1], [if true, build DAP Client])
fi
if test "x$enable_dap_remote_tests" = xyes; then
AC_DEFINE([ENABLE_DAP_REMOTE_TESTS], [1], [if true, do remote tests])
fi
2010-06-03 21:25:25 +08:00
AC_MSG_CHECKING([whether the time-consuming dap tests should be enabled (default off)])
AC_ARG_ENABLE([dap-long-tests],
[AS_HELP_STRING([--enable-dap-long-tests],
[enable dap long tests])])
test "x$enable_dap_long_tests" = xyes || enable_dap_long_tests=no
if test "x$enable_dap_remote_tests" = "xno" ; then
enable_dap_long_tests=no
fi
AC_MSG_RESULT([$enable_dap_long_tests])
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
# Control zarr storage
if test "x$enable_nczarr" = xyes ; then
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
if test "x$enable_netcdf_4" = xno ; then
AC_MSG_WARN([netCDF-4 disabled, so you must not enable nczarr])
enable_nczarr=no
fi
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
fi
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
if test "x$enable_nczarr" = xyes; then
AC_DEFINE([ENABLE_NCZARR], [1], [if true, build NCZarr Client])
AC_SUBST(ENABLE_NCZARR)
fi
AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
##########
# Look for Standardized libraries
##########
2021-09-03 07:04:26 +08:00
# See if we have libblosc
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AC_CHECK_LIB([blosc],[blosc_init],[have_blosc=yes],[have_blosc=no])
if test "x$have_blosc" = "xyes" ; then
2021-09-03 07:04:26 +08:00
AC_SEARCH_LIBS([blosc_init],[blosc blosc.dll cygblosc.dll], [], [])
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AC_DEFINE([HAVE_BLOSC], [1], [if true, blosc library is available])
fi
# See if we have libzstd
AC_CHECK_LIB([zstd],[ZSTD_compress],[have_zstd=yes],[have_zstd=no])
if test "x$have_zstd" = "xyes" ; then
AC_SEARCH_LIBS([ZSTD_compress],[zstd zstd.dll cygzstd.dll], [], [])
AC_DEFINE([HAVE_ZSTD], [1], [if true, zstd library is available])
fi
AC_MSG_CHECKING([whether libzstd library is available])
AC_MSG_RESULT([${have_zstd}])
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
# See if we have libbz2
AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bz2=yes],[have_bz2=no])
if test "x$have_bz2" = "xyes" ; then
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AC_SEARCH_LIBS([BZ2_bzCompress],[bz2 bz2.dll cygbz2.dll], [], [])
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
AC_DEFINE([HAVE_BZ2], [1], [if true, bz2 library is installed])
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
fi
AC_MSG_CHECKING([whether libbz2 library is available])
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
AC_MSG_RESULT([${have_bz2}])
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
if test "x$have_bz2" = "xno" ; then
have_local_bz2=yes
2022-05-28 05:45:34 +08:00
AC_MSG_NOTICE([Defaulting to internal libbz2])
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
else
have_local_bz2=no
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
fi
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
AM_CONDITIONAL(HAVE_LOCAL_BZ2, [test "x$have_local_bz2" = xyes])
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
# Note that szip management is tricky.
# This is because we have three things to consider:
# 1. is libsz available?
# 2. is szip enabled in HDF5?
# 3. is nczarr enabled?
# We need separate flags for cases 1 and 2
# See if we have libsz (usually via libaec)
AC_CHECK_LIB([sz],[SZ_BufftoBuffCompress],[have_sz=yes],[have_sz=no])
if test "x$have_sz" = "xyes" ; then
AC_SEARCH_LIBS([SZ_BufftoBuffCompress],[sz sz.dll cygsz.dll], [], [])
AC_DEFINE([HAVE_SZ], [1], [if true, libsz (==szip) is available])
2021-09-03 07:04:26 +08:00
fi
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AC_MSG_CHECKING([whether libsz library is available])
AC_MSG_RESULT([${have_sz}])
##########
2021-09-03 07:04:26 +08:00
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
# See if we have libzip for NCZarr
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
if test "x$have_zip" = "xyes" ; then
AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [], [])
fi
AC_MSG_CHECKING([whether libzip library is available])
AC_MSG_RESULT([${have_zip}])
enable_nczarr_zip=${have_zip} # alias
if test "x$enable_nczarr" = xno ; then
enable_nczarr_zip=no
fi
2021-09-03 07:04:26 +08:00
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
AC_MSG_CHECKING([whether nczarr zip support is enabled])
AC_MSG_RESULT([${enable_nczarr_zip}])
if test "x$enable_nczarr_zip" = xyes ; then
AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
fi
AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
2020-10-17 05:04:51 +08:00
# Check for enabling of S3 support
2020-06-29 09:53:40 +08:00
AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled])
2020-10-17 05:04:51 +08:00
AC_ARG_ENABLE([nczarr-s3],
[AS_HELP_STRING([--enable-nczarr-s3],
2020-08-18 09:15:47 +08:00
[enable netcdf zarr S3 support; make sure to set LDFLAGS])])
2020-10-17 05:04:51 +08:00
test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
if test "x$enable_nczarr" = xno ; then
enable_nczarr_s3=no
2020-10-17 05:04:51 +08:00
fi
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
AC_MSG_RESULT($enable_nczarr_s3)
2020-10-17 05:04:51 +08:00
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
# Note we check for the library after checking for enable_nczarr_s3
# because for some reason this screws up if we unconditionally test for sdk
# and it is not available. Fix someday
2021-09-28 08:36:33 +08:00
enable_s3_sdk=no
2020-12-17 11:48:02 +08:00
if test "x$enable_nczarr_s3" = xyes ; then
2020-10-17 05:04:51 +08:00
# See if we have the s3 aws library
# Check for the AWS S3 SDK library
AC_LANG_PUSH([C++])
2021-09-28 08:36:33 +08:00
AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [enable_s3_sdk=yes],[enable_s3_sdk=no])
2020-10-17 05:04:51 +08:00
AC_LANG_POP
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
fi
2020-10-17 05:04:51 +08:00
AC_MSG_CHECKING([whether AWS S3 SDK library is available])
2021-09-28 08:36:33 +08:00
AC_MSG_RESULT([$enable_s3_sdk])
2020-10-17 05:04:51 +08:00
2021-09-28 08:36:33 +08:00
if test "x$enable_s3_sdk" = xno ; then
AC_MSG_WARN([No S3 library available; disabling S3 support])
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
enable_nczarr_s3=no
2021-09-28 08:36:33 +08:00
fi
if test "x$enable_s3_sdk" = xyes ; then
2020-10-17 05:04:51 +08:00
AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
fi
2021-09-28 08:36:33 +08:00
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716
The Zarr version 2 spec has been extended to include the ability
to choose the dimension separator in chunk name keys. The legal
separators has been extended from {'.'} to {'.' '/'}. So now it
is possible to use a key like "0/1/2/0" for chunk names.
This PR implements this for NCZarr. The V2 spec now says that
this separator can be set on a per-variable basis. For now, I
have chosen to allow this be set only globally by adding a key
named "ZARR.DIMENSION_SEPARATOR=<char>" in the
.daprc/.dodsrc/ncrc file. Currently, the only legal separator
characters are '.' (the default) and '/'. On writing, this key
will only be written if its value is different than the default.
This change caused problems because supporting a separator of '/'
is difficult to parse when keys/paths use '/' as the path separator.
A test case was added for this.
Additionally, make nczarr be enabled default by default. This required
some additional changes so that if zip and/or AWS S3 sdk are unavailable,
then they are disabled for NCZarr.
In addition the following unrelated changes were made.
1. Tested that pure-zarr mode could read an nczarr formatted store.
1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added.
1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl
is not found then the other options that depend upon it properly
are disabled.
1. I decided that xarray support should be enabled by default for pure
zarr. In order to allow disabling, I added a new mode flag "noxarray".
1. Certain test in nczarr_test depend on use of .dodsrc. In order for these
to work when testing in parallel, some inter-test dependencies needed to
be added.
1. Improved authorization testing to use changes in thredds.ucar.edu
2021-04-25 09:48:15 +08:00
# Check for enabling S3 testing
AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
AC_ARG_ENABLE([nczarr-s3-tests],
[AS_HELP_STRING([--enable-nczarr-s3-tests],
[enable netcdf zarr S3 testing])])
test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no
AC_MSG_RESULT($enable_nczarr_s3_tests)
# Disable S3 tests if S3 support is disabled
if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then
AC_MSG_ERROR([NCZarr S3 support is disabled; please remove option --enable-nczarr-s3-tests])
2020-10-17 05:04:51 +08:00
enable_nczarr_s3_tests=no
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
fi
2020-06-29 09:53:40 +08:00
2020-10-17 05:04:51 +08:00
if test "x$enable_nczarr_s3" = xyes ; then
AC_DEFINE([ENABLE_NCZARR_S3], [1], [if true, build libnczarr with S3 support enabled])
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
fi
2020-10-17 05:04:51 +08:00
AM_CONDITIONAL(ENABLE_NCZARR_S3, [test "x$enable_nczarr_s3" = xyes])
if test "x$enable_nczarr_s3_tests" = xyes ; then
AC_DEFINE([ENABLE_NCZARR_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled])
fi
AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes])
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
2021-01-29 11:11:01 +08:00
if test "x$enable_nczarr_s3_tests" = xyes ; then
AC_MSG_WARN([*** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***])
fi
# Set default
2020-11-20 08:01:04 +08:00
# Did the user specify a default cache size for NCZarr?
AC_MSG_CHECKING([whether a default file cache size for NCZarr was specified])
AC_ARG_WITH([chunk-cache-size-nczarr],
[AS_HELP_STRING([--with-chunk-cache-size-nczarr=<integer>],
[Specify default maximum space used by the chunk cache NCZarr.])],
[CHUNK_CACHE_SIZE_NCZARR=$with_chunk_cache_size_nczarr], [CHUNK_CACHE_SIZE_NCZARR=4194304])
AC_MSG_RESULT([$CHUNK_CACHE_SIZE_NCZARR])
AC_DEFINE_UNQUOTED([CHUNK_CACHE_SIZE_NCZARR], [$CHUNK_CACHE_SIZE_NCZARR], [default nczarr chunk cache size.])
2020-09-28 02:43:46 +08:00
# This has multiversion capability
AC_MSG_CHECKING([whether multi-filter support is enabled])
2020-10-17 05:04:51 +08:00
has_multifilters=yes
AC_MSG_RESULT($has_multifilters)
AC_DEFINE([HAS_MULTIFILTERS], [1], [if true, multi-filter support enabled])
2020-09-28 02:43:46 +08:00
2017-11-21 06:50:47 +08:00
# Check whether we want to enable strict null byte header padding.
2017-11-22 03:49:14 +08:00
# See https://github.com/Unidata/netcdf-c/issues/657 for more information.
2017-11-21 06:50:47 +08:00
AC_MSG_CHECKING([whether to enable strict null-byte header padding when reading (default off)])
AC_ARG_ENABLE([strict-null-byte-header-padding],
[AS_HELP_STRING([--enable-strict-null-byte-header-padding],
[enable strict null-byte header padding when reading netCDF3 files.])])
test "x$enable_strict_null_byte_header_padding" = xyes || enable_strict_null_byte_header_padding=no
AC_MSG_RESULT($enable_strict_null_byte_header_padding)
if test "x$enable_strict_null_byte_header_padding" = xyes; then
AC_DEFINE([USE_STRICT_NULL_BYTE_HEADER_PADDING], [1], [if true, enable strict null byte header padding])
fi
AM_CONDITIONAL(USE_STRICT_NULL_BYTE_HEADER_PADDING, [test x$enable_strict_null_byte_header_padding = xyes ])
2010-06-03 21:25:25 +08:00
# Does the user want to use the ffio module?
AC_MSG_CHECKING([whether FFIO will be used])
AC_ARG_ENABLE([ffio],
[AS_HELP_STRING([--enable-ffio],
[use ffio instead of posixio (ex. on the Cray)])])
test "x$enable_ffio" = xyes || enable_ffio=no
AC_MSG_RESULT($enable_ffio)
2012-03-26 09:34:32 +08:00
if test "x$enable_ffio" = xyes; then
AC_DEFINE([USE_FFIO], [1], [if true, use ffio instead of posixio])
fi
2010-06-03 21:25:25 +08:00
AM_CONDITIONAL(USE_FFIO, [test x$enable_ffio = xyes])
2015-08-20 07:14:13 +08:00
# Does the user want to use the stdio module?
AC_MSG_CHECKING([whether STDIO will be used])
AC_ARG_ENABLE([stdio],
[AS_HELP_STRING([--enable-stdio],
[use stdio instead of posixio (ex. on the Cray)])])
test "x$enable_stdio" = xyes || enable_stdio=no
AC_MSG_RESULT($enable_stdio)
if test "x$enable_stdio" = xyes; then
AC_DEFINE([USE_STDIO], [1], [if true, use stdio instead of posixio])
fi
AM_CONDITIONAL(USE_STDIO, [test x$enable_stdio = xyes])
2011-04-27 04:57:24 +08:00
nc_build_c=yes
2010-06-03 21:25:25 +08:00
nc_build_v2=yes
nc_build_utilities=yes
2012-04-13 10:41:00 +08:00
nc_build_tests=yes
2010-06-03 21:25:25 +08:00
nc_build_examples=yes
# Does the user want to build examples?
AC_MSG_CHECKING([whether examples should be built])
AC_ARG_ENABLE([examples],
[AS_HELP_STRING([--disable-examples],
[don't build the netCDF examples during make check \
(examples are treated as extra tests by netCDF)])])
test "x$enable_examples" = xno && nc_build_examples=no
AC_MSG_RESULT($nc_build_examples)
AM_CONDITIONAL(BUILD_EXAMPLES, [test x$nc_build_examples = xyes])
# Does the user want to disable the V2 API?
AC_MSG_CHECKING([whether v2 netCDF API should be built])
AC_ARG_ENABLE([v2],
[AS_HELP_STRING([--disable-v2],
[turn off the netCDF version 2 API])])
test "x$enable_v2" = xno && nc_build_v2=no
AC_MSG_RESULT($nc_build_v2)
AM_CONDITIONAL(BUILD_V2, [test x$nc_build_v2 = xyes])
if test "x$nc_build_v2" = xno; then
AC_DEFINE_UNQUOTED(NO_NETCDF_2, 1, [do not build the netCDF version 2 API])
2014-09-05 06:17:13 +08:00
else
AC_DEFINE_UNQUOTED(USE_NETCDF_2, 1, [build the netCDF version 2 API])
2010-06-03 21:25:25 +08:00
fi
2020-05-15 02:18:42 +08:00
# Does the user want to disable ncgen/ncdump/nccopy/...?
2010-09-15 23:25:20 +08:00
AC_MSG_CHECKING([whether the ncgen/ncdump/nccopy should be built])
2010-06-03 21:25:25 +08:00
AC_ARG_ENABLE([utilities],
[AS_HELP_STRING([--disable-utilities],
2010-09-15 23:25:20 +08:00
[don't build netCDF utilities ncgen, ncdump, and nccopy])])
2010-06-03 21:25:25 +08:00
test "x$nc_build_c" = xno && enable_utilities=no
test "x$enable_utilities" = xno && nc_build_utilities=no
AC_MSG_RESULT($nc_build_utilities)
AM_CONDITIONAL(BUILD_UTILITIES, [test x$nc_build_utilities = xyes])
2012-04-13 10:41:00 +08:00
# Does the user want to disable all tests?
AC_MSG_CHECKING([whether test should be built and run])
AC_ARG_ENABLE([testsets],
[AS_HELP_STRING([--disable-testsets],
[don't build or run netCDF tests])])
test "x$enable_testsets" = xno || enable_testsets=yes
nc_build_tests=$enable_testsets
AC_MSG_RESULT($nc_build_tests)
AM_CONDITIONAL(BUILD_TESTSETS, [test x$nc_build_tests = xyes])
2010-06-03 21:25:25 +08:00
# Does the user want to run tests for large files (> 2GiB)?
AC_MSG_CHECKING([whether large file (> 2GB) tests should be run])
AC_ARG_ENABLE([large-file-tests],
[AS_HELP_STRING([--enable-large-file-tests],
[Run tests which create very large data files (~13 GB disk space
required, but it will be recovered when tests are complete). See
option --with-temp-large to specify temporary directory])])
test "x$enable_large_file_tests" = xyes || enable_large_file_tests=no
AC_MSG_RESULT($enable_large_file_tests)
AM_CONDITIONAL(LARGE_FILE_TESTS, [test x$enable_large_file_tests = xyes])
if test "x$enable_large_file_tests" = xyes; then
AC_DEFINE([LARGE_FILE_TESTS], [1], [do large file tests])
fi
# Does the user want to run benchmarks?
2018-12-17 22:59:33 +08:00
AC_MSG_CHECKING([whether benchmarks should be run])
2010-06-03 21:25:25 +08:00
AC_ARG_ENABLE([benchmarks],
[AS_HELP_STRING([--enable-benchmarks],
2018-12-17 22:59:33 +08:00
[Run benchmarks. This will cause sample data files from the Unidata ftp
site to be fetched. The benchmarks are a bunch of extra tests, which
are timed. We use these tests to check netCDF performance.])])
2010-06-03 21:25:25 +08:00
test "x$enable_benchmarks" = xyes || enable_benchmarks=no
AC_MSG_RESULT($enable_benchmarks)
2020-08-18 09:15:47 +08:00
if test "x$enable_HDF5" = xno -a "x$enable_benchmarks" = xyes; then
AC_MSG_ERROR([Can't use benchmarks if HDF5 is disabled.])
2019-03-18 22:40:18 +08:00
fi
2010-06-03 21:25:25 +08:00
AM_CONDITIONAL(BUILD_BENCHMARKS, [test x$enable_benchmarks = xyes])
# Does the user want to use extreme numbers in testing.
AC_MSG_CHECKING([whether extreme numbers should be used in tests])
AC_ARG_ENABLE([extreme-numbers],
[AS_HELP_STRING([--disable-extreme-numbers],
[don't use extreme numbers during testing, such as MAX_INT - 1])])
case "$host_cpu $host_os" in
*386*solaris*)
test "x$enable_extreme_numbers" = xyes || enable_extreme_numbers=no
;;
*)
test "x$enable_extreme_numbers" = xno || enable_extreme_numbers=yes
;;
2014-09-05 05:42:11 +08:00
esac
2010-06-03 21:25:25 +08:00
AC_MSG_RESULT($enable_extreme_numbers)
if test "x$enable_extreme_numbers" = xyes; then
AC_DEFINE(USE_EXTREME_NUMBERS, 1, [set this to use extreme numbers in tests])
fi
# If the env. variable TEMP_LARGE is set, or if
# --with-temp-large=<directory>, use it as a place for the large
# (i.e. > 2 GiB) files created during the large file testing.
AC_MSG_CHECKING([where to put large temp files if large file tests are run])
AC_ARG_WITH([temp-large],
[AS_HELP_STRING([--with-temp-large=<directory>],
[specify directory where large files (i.e. >2 GB) \
will be written, if large files tests are run with
--enable-large-file-tests])],
[TEMP_LARGE=$with_temp_large])
TEMP_LARGE=${TEMP_LARGE-.}
AC_MSG_RESULT($TEMP_LARGE)
#AC_SUBST(TEMP_LARGE)
AC_DEFINE_UNQUOTED([TEMP_LARGE], ["$TEMP_LARGE"], [Place to put very large netCDF test files.])
2018-08-26 11:44:41 +08:00
# Specify extra values to add to _NCProperties attribute
# --with-ncproperties-extra="<name>=<value>|...".
# Note: need to figure out a way to do this programmatically also
AC_MSG_CHECKING([Extra values for _NCProperties])
AC_ARG_WITH([ncproperties-extra],
[AS_HELP_STRING([--with-ncproperties-extra="<name>=<value>,...],
[specify extra pairs for _NCProperties])],
[NCPROPERTIES_EXTRA=$with_ncproperties_extra],
[NCPROPERTIES_EXTRA=""])
AC_MSG_RESULT([$NCPROPERTIES_EXTRA])
AC_DEFINE_UNQUOTED([NCPROPERTIES_EXTRA], ["$NCPROPERTIES_EXTRA"], [Extra pairs for _NCProperties])
2018-06-02 21:13:08 +08:00
# Did the user specify a user-defined format 0?
AC_MSG_CHECKING([whether user-defined format 0 was specified])
AC_ARG_WITH([udf0],
[AS_HELP_STRING([--with-udf0=<dispatch_name>],
[Specify a dispatch table for user-defined format 0.])],
[UDF0_DISPATCH=$with_udf0])
AC_MSG_RESULT([$UDF0_DISPATCH])
if test -n "$UDF0_DISPATCH"; then
AC_DEFINE_UNQUOTED([UDF0_DISPATCH], [$UDF0_DISPATCH], [dispatch table for user-defined format 0.])
2018-06-03 20:28:55 +08:00
AC_DEFINE_UNQUOTED([UDF0_DISPATCH_FUNC], [get_$UDF0_DISPATCH()], [function to get dispatch table for user-defined format 0.])
2018-06-02 21:13:08 +08:00
AC_DEFINE([USE_UDF0], [1], [if true, use user-defined format 0 in utilities])
2018-06-03 20:14:23 +08:00
AC_CHECK_LIB([$UDF0_DISPATCH], [get_$UDF0_DISPATCH], [],
[AC_MSG_ERROR([Can't find or link to the user-defined format 0 library.])],
[])
2018-06-02 21:13:08 +08:00
fi
2018-06-07 02:54:03 +08:00
# Did the user specify a magic number for user-defined format 0?
AC_MSG_CHECKING([whether a magic number for user-defined format 0 was specified])
AC_ARG_WITH([udf0-magic-number],
[AS_HELP_STRING([--with-udf0-magic-number=<magic_number>],
[Specify a magic number for user-defined format 0 (ignored unless --with-udf0 is also used).])],
[UDF0_MAGIC_NUMBER=$with_udf0_magic_number])
AC_MSG_RESULT([$UDF0_MAGIC_NUMBER])
2018-06-02 21:13:08 +08:00
# Did the user specify a user-defined format 1?
AC_MSG_CHECKING([whether user-defined format 1 was specified])
AC_ARG_WITH([udf1],
[AS_HELP_STRING([--with-udf1=<dispatch_name>],
[Specify a dispatch table for user-defined format 1.])],
[UDF1_DISPATCH=$with_udf1])
AC_MSG_RESULT([$UDF1_DISPATCH])
if test -n "$UDF1_DISPATCH"; then
AC_DEFINE_UNQUOTED([UDF1_DISPATCH], [$UDF1_DISPATCH], [dispatch table for user-defined format 1.])
2018-06-03 20:28:55 +08:00
AC_DEFINE_UNQUOTED([UDF1_DISPATCH_FUNC], [get_$UDF1_DISPATCH()], [function to get dispatch table for user-defined format 1.])
2018-06-02 21:13:08 +08:00
AC_DEFINE([USE_UDF1], [1], [if true, use user-defined format 1 in utilities])
2018-06-03 20:28:55 +08:00
AC_CHECK_LIB([$UDF1_DISPATCH], [get_$UDF1_DISPATCH], [],
[AC_MSG_ERROR([Can't find or link to the user-defined format 1 library.])],
[])
2018-06-02 21:13:08 +08:00
fi
2018-06-07 02:54:03 +08:00
# Did the user specify a magic number for user-defined format 0?
AC_MSG_CHECKING([whether a magic number for user-defined format 1 was specified])
AC_ARG_WITH([udf1-magic-number],
[AS_HELP_STRING([--with-udf1-magic-number=<magic_number>],
[Specify a magic number for user-defined format 1 (ignored unless --with-udf1 is also used).])],
[UDF1_MAGIC_NUMBER=$with_udf1_magic_number])
AC_MSG_RESULT([$UDF1_MAGIC_NUMBER])
2010-06-03 21:25:25 +08:00
# According to the autoconf mailing list gurus, we must test for
# compilers unconditionally. That is, we can't skip looking for the
# fortran compilers, just because the user doesn't want fortran. This
# is due to a limitation in autoconf.
# Find the C compiler.
AC_MSG_NOTICE([finding C compiler])
2014-05-31 03:48:55 +08:00
## Compiler with version information. This consists of the full path
## name of the compiler and the reported version number.
AC_SUBST([CC_VERSION])
## Strip anything that looks like a flag off of $CC
CC_NOFLAGS=`echo $CC | sed 's/ -.*//'`
if `echo $CC_NOFLAGS | grep ^/ >/dev/null 2>&1`; then
CC_VERSION="$CC"
else
CC_VERSION="$CC";
for x in `echo $PATH | sed -e 's/:/ /g'`; do
if test -x $x/$CC_NOFLAGS; then
CC_VERSION="$x/$CC"
break
fi
done
fi
if test -n "$cc_version_info"; then
CC_VERSION="$CC_VERSION ( $cc_version_info)"
fi
2011-04-27 04:57:24 +08:00
AC_PROG_CC
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
AC_PROG_CXX
2010-06-03 21:25:25 +08:00
AM_PROG_CC_C_O
AC_C_CONST
# Set up libtool.
AC_MSG_NOTICE([setting up libtool])
2011-07-02 00:14:25 +08:00
LT_PREREQ([2.2])
2017-11-30 20:40:17 +08:00
LT_INIT()
2010-11-30 06:23:16 +08:00
2010-06-03 21:25:25 +08:00
AC_MSG_NOTICE([finding other utilities])
2015-01-13 01:36:11 +08:00
# Is m4 installed? If not, bail.
2015-01-14 00:56:37 +08:00
AC_CHECK_PROGS([NC_M4], [m4])
2015-01-13 01:36:11 +08:00
if test -z "$NC_M4"; then
AC_MSG_ERROR([Cannot find m4 utility. Install m4 and try again.])
fi
2011-07-21 05:39:14 +08:00
# Is doxygen installed? If so, have configure construct the Doxyfile.
AC_CHECK_PROGS([DOXYGEN], [doxygen])
2014-09-05 05:42:11 +08:00
if test -z "$DOXYGEN"; then
2011-08-22 21:53:37 +08:00
AC_MSG_WARN([Doxygen not found - documentation will not be built])
fi
2014-02-05 07:13:18 +08:00
2014-09-05 05:42:11 +08:00
# Is graphviz/dot installed? If so, we'll use dot to create
2014-02-05 07:13:18 +08:00
# graphs in the documentation.
AC_CHECK_PROGS([DOT], [dot])
if test -z "$DOT"; then
AC_MSG_WARN([dot not found - will use simple charts in documentation])
HAVE_DOT=NO
elif test "x$enable_dot" = xno; then
HAVE_DOT=NO
2014-09-05 05:42:11 +08:00
else
2014-02-05 07:13:18 +08:00
HAVE_DOT=YES
2014-09-05 05:42:11 +08:00
fi
2011-08-22 21:53:37 +08:00
# If we have doxygen, and it's enabled, then process the file.
if test "x$enable_doxygen" != xno; then
if test -n "$DOXYGEN"; then
2014-09-05 05:42:11 +08:00
AC_SUBST(HAVE_DOT)
2014-09-19 02:26:42 +08:00
AC_CONFIG_FILES([docs/Doxyfile])
2011-08-22 21:53:37 +08:00
fi
2012-03-15 07:26:48 +08:00
# Note: the list of files to input to doxygen
2014-05-22 04:40:39 +08:00
# has been moved to docs/Doxyfile.in so
2012-03-15 07:26:48 +08:00
# that make distcheck works correctly.
# Any new inputs should be inserted into
2014-05-22 04:40:39 +08:00
# docs/Doxyfile.in and possibley docs/Makefile.am
2011-07-21 05:39:14 +08:00
fi
# Find the install program.
2010-06-03 21:25:25 +08:00
AC_PROG_INSTALL
# Check to see if any macros must be set to enable large (>2GB) files.
AC_SYS_LARGEFILE
AC_MSG_NOTICE([displaying some results])
## This next macro just prints some results for debugging
## support issues.
UD_DISPLAY_RESULTS
# For nightly build testing, output CC, FC, etc.
2011-08-12 01:54:34 +08:00
echo "CPPFLAGS=$CPPFLAGS CC=$CC CFLAGS=$CFLAGS LDFLAGS=$LDFLAGS LIBS=$LIBS" >> comps.txt
2010-06-03 21:25:25 +08:00
AC_MSG_NOTICE([checking types, headers, and functions])
2011-07-13 20:36:22 +08:00
AC_CHECK_HEADERS([sys/param.h])
2018-02-03 10:57:55 +08:00
AC_CHECK_HEADERS([libgen.h])
2011-03-22 04:31:56 +08:00
#AC_CHECK_HEADERS([locale.h])
2010-06-03 21:25:25 +08:00
AC_HEADER_STDC
2020-11-20 08:01:04 +08:00
AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h])
2010-06-03 21:25:25 +08:00
2012-05-16 01:10:41 +08:00
# Do sys/resource.h separately
#AC_CHECK_HEADERS([sys/resource.h],[havesysresource=1],[havesysresource=0])
#if test "x$enable_dll" != xyes ; then
AC_CHECK_HEADERS([sys/resource.h])
#fi
2017-07-06 00:03:48 +08:00
# See if we have ftw.h to walk directory trees
AC_CHECK_HEADERS([ftw.h])
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
# See if we can do stack tracing programmatically
AC_CHECK_HEADERS([execinfo.h])
2010-06-03 21:25:25 +08:00
# Check for these functions...
2019-05-30 04:35:29 +08:00
AC_CHECK_FUNCS([strlcat snprintf strcasecmp fileno \
2018-02-05 23:17:25 +08:00
strdup strtoll strtoull \
mkstemp mktemp random \
2020-05-14 21:20:57 +08:00
getrlimit gettimeofday fsync MPI_Comm_f2c MPI_Info_f2c \
strncasecmp])
2012-04-09 06:47:38 +08:00
2020-12-07 09:19:53 +08:00
# See if clock_gettime is available and its arg types.
AC_CHECK_FUNCS([clock_gettime])
AC_CHECK_TYPES([struct timespec])
2022-03-16 05:33:13 +08:00
# disable dap4 if hdf5 is disabled
2018-12-01 23:24:56 +08:00
if test "x$enable_hdf5" = "xno" ; then
2017-03-09 08:01:10 +08:00
AC_MSG_WARN([netcdf-4 not enabled; disabling DAP4])
enable_dap4=no
fi
2022-03-16 05:33:13 +08:00
if test "x$ISOSX" = xyes && "x$have_libxml2" = xno ; then
AC_MSG_ERROR([Error: OSX requires libxml2 => --disable-dap4.])
enable_dap4=no
2012-08-03 05:04:47 +08:00
fi
2022-03-16 05:33:13 +08:00
if test "x$enable_dap4" = xyes; then
AC_DEFINE([ENABLE_DAP4], [1], [if true, build DAP4 Client])
2021-11-02 12:37:05 +08:00
fi
2021-11-04 03:40:47 +08:00
2012-05-16 01:10:41 +08:00
# check for useful, but not essential, memio support
AC_CHECK_FUNCS([memmove getpagesize sysconf])
2012-06-24 03:25:49 +08:00
# Does the user want to allow use of mmap for NC_DISKLESS?
AC_MSG_CHECKING([whether mmap is enabled for in-memory files])
2012-04-23 04:34:21 +08:00
AC_ARG_ENABLE([mmap],
[AS_HELP_STRING([--enable-mmap],
2012-06-24 03:25:49 +08:00
[allow mmap for in-memory files])])
2012-04-23 04:34:21 +08:00
test "x$enable_mmap" = xyes || enable_mmap=no
AC_MSG_RESULT($enable_mmap)
2019-04-20 10:32:26 +08:00
# check for mmap availability before committing to use mmap
2018-12-06 10:20:43 +08:00
have_mmap="$enable_mmap"
2019-04-20 10:32:26 +08:00
AC_CHECK_FUNCS([mmap],[havemmapfcn=yes],[havemmapfcn=no])
if test "x$havemmapfcn" = xno ; then
have_mmap=no
fi
# check for mremap availability; not strictly needed
AC_CHECK_FUNCS([mremap],[havemremapfcn=yes],[havemmapfcn=no])
2012-05-16 01:10:41 +08:00
2018-12-06 10:20:43 +08:00
# Check for MAP_ANONYMOUS
AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[#include <sys/mman.h>],
[[int x = MAP_ANONYMOUS;]])],
[havemapanon=yes],
[havemapanon=no])
AC_MSG_CHECKING([whether MAP_ANONYMOUS is defined])
AC_MSG_RESULT([${havemapanon}])
if test "x$havemapanon" != xyes ; then
have_mmap=no
2018-10-11 03:32:17 +08:00
fi
2018-12-06 10:20:43 +08:00
if test "x$have_mmap" != xyes ; then
2022-03-28 06:35:39 +08:00
if test "x$enable_mmap" = xyes ; then
AC_MSG_WARN([mmap functionality is not available: disabling mmap])
else
AC_MSG_NOTICE([mmap functionality is not available: disabling mmap])
fi
2012-04-23 04:34:21 +08:00
enable_mmap=no
fi
2012-04-13 10:41:00 +08:00
2018-05-12 05:30:19 +08:00
if test "x$enable_mmap" = xyes; then
2012-05-16 01:10:41 +08:00
AC_DEFINE([USE_MMAP], [1], [if true, use mmap for in-memory files])
2012-04-13 10:41:00 +08:00
fi
2012-04-09 06:47:38 +08:00
Provide byte-range reading of remote datasets
re: issue https://github.com/Unidata/netcdf-c/issues/1251
Assume that you have the URL to a remote dataset
which is a normal netcdf-3 or netcdf-4 file.
This PR allows the netcdf-c to read that dataset's
contents as a netcdf file using HTTP byte ranges
if the remote server supports byte-range access.
Originally, this PR was set up to access Amazon S3 objects,
but it can also access other remote datasets such as those
provided by a Thredds server via the HTTPServer access protocol.
It may also work for other kinds of servers.
Note that this is not intended as a true production
capability because, as is known, this kind of access to
can be quite slow. In addition, the byte-range IO drivers
do not currently do any sort of optimization or caching.
An additional goal here is to gain some experience with
the Amazon S3 REST protocol.
This architecture and its use documented in
the file docs/byterange.dox.
There are currently two test cases:
1. nc_test/tst_s3raw.c - this does a simple open, check format, close cycle
for a remote netcdf-3 file and a remote netcdf-4 file.
2. nc_test/test_s3raw.sh - this uses ncdump to investigate some remote
datasets.
This PR also incorporates significantly changed model inference code
(see the superceded PR https://github.com/Unidata/netcdf-c/pull/1259).
1. It centralizes the code that infers the dispatcher.
2. It adds support for byte-range URLs
Other changes:
1. NC_HDF5_finalize was not being properly called by nc_finalize().
2. Fix minor bug in ncgen3.l
3. fix memory leak in nc4info.c
4. add code to walk the .daprc triples and to replace protocol=
fragment tag with a more general mode= tag.
Final Note:
Th inference code is still way too complicated. We need to move
to the validfile() model used by netcdf Java, where each
dispatcher is asked if it can process the file. This decentralizes
the inference code. This will be done after all the major new
dispatchers (PIO, Zarr, etc) have been implemented.
2019-01-02 09:27:36 +08:00
# Does the user want to allow reading of remote data via range headers?
AC_MSG_CHECKING([whether byte range support is enabled])
AC_ARG_ENABLE([byterange],
[AS_HELP_STRING([--enable-byterange],
[allow byte-range I/O])])
test "x$enable_byterange" = xyes || enable_byterange=no
AC_MSG_RESULT($enable_byterange)
# Need curl for byte ranges
2019-05-05 02:29:42 +08:00
if test "x$found_curl" = xno && test "x$enable_byterange" = xyes ; then
Provide byte-range reading of remote datasets
re: issue https://github.com/Unidata/netcdf-c/issues/1251
Assume that you have the URL to a remote dataset
which is a normal netcdf-3 or netcdf-4 file.
This PR allows the netcdf-c to read that dataset's
contents as a netcdf file using HTTP byte ranges
if the remote server supports byte-range access.
Originally, this PR was set up to access Amazon S3 objects,
but it can also access other remote datasets such as those
provided by a Thredds server via the HTTPServer access protocol.
It may also work for other kinds of servers.
Note that this is not intended as a true production
capability because, as is known, this kind of access to
can be quite slow. In addition, the byte-range IO drivers
do not currently do any sort of optimization or caching.
An additional goal here is to gain some experience with
the Amazon S3 REST protocol.
This architecture and its use documented in
the file docs/byterange.dox.
There are currently two test cases:
1. nc_test/tst_s3raw.c - this does a simple open, check format, close cycle
for a remote netcdf-3 file and a remote netcdf-4 file.
2. nc_test/test_s3raw.sh - this uses ncdump to investigate some remote
datasets.
This PR also incorporates significantly changed model inference code
(see the superceded PR https://github.com/Unidata/netcdf-c/pull/1259).
1. It centralizes the code that infers the dispatcher.
2. It adds support for byte-range URLs
Other changes:
1. NC_HDF5_finalize was not being properly called by nc_finalize().
2. Fix minor bug in ncgen3.l
3. fix memory leak in nc4info.c
4. add code to walk the .daprc triples and to replace protocol=
fragment tag with a more general mode= tag.
Final Note:
Th inference code is still way too complicated. We need to move
to the validfile() model used by netcdf Java, where each
dispatcher is asked if it can process the file. This decentralizes
the inference code. This will be done after all the major new
dispatchers (PIO, Zarr, etc) have been implemented.
2019-01-02 09:27:36 +08:00
AC_MSG_ERROR([curl required for byte range support. Install curl or build without --enable-byterange.])
enable_byterange=no
fi
2019-02-25 07:54:13 +08:00
if test "x$enable_byterange" = xyes; then
AC_DEFINE([ENABLE_BYTERANGE], [1], [if true, support byte-range read of remote datasets.])
Provide byte-range reading of remote datasets
re: issue https://github.com/Unidata/netcdf-c/issues/1251
Assume that you have the URL to a remote dataset
which is a normal netcdf-3 or netcdf-4 file.
This PR allows the netcdf-c to read that dataset's
contents as a netcdf file using HTTP byte ranges
if the remote server supports byte-range access.
Originally, this PR was set up to access Amazon S3 objects,
but it can also access other remote datasets such as those
provided by a Thredds server via the HTTPServer access protocol.
It may also work for other kinds of servers.
Note that this is not intended as a true production
capability because, as is known, this kind of access to
can be quite slow. In addition, the byte-range IO drivers
do not currently do any sort of optimization or caching.
An additional goal here is to gain some experience with
the Amazon S3 REST protocol.
This architecture and its use documented in
the file docs/byterange.dox.
There are currently two test cases:
1. nc_test/tst_s3raw.c - this does a simple open, check format, close cycle
for a remote netcdf-3 file and a remote netcdf-4 file.
2. nc_test/test_s3raw.sh - this uses ncdump to investigate some remote
datasets.
This PR also incorporates significantly changed model inference code
(see the superceded PR https://github.com/Unidata/netcdf-c/pull/1259).
1. It centralizes the code that infers the dispatcher.
2. It adds support for byte-range URLs
Other changes:
1. NC_HDF5_finalize was not being properly called by nc_finalize().
2. Fix minor bug in ncgen3.l
3. fix memory leak in nc4info.c
4. add code to walk the .daprc triples and to replace protocol=
fragment tag with a more general mode= tag.
Final Note:
Th inference code is still way too complicated. We need to move
to the validfile() model used by netcdf Java, where each
dispatcher is asked if it can process the file. This decentralizes
the inference code. This will be done after all the major new
dispatchers (PIO, Zarr, etc) have been implemented.
2019-01-02 09:27:36 +08:00
fi
2021-10-30 10:06:37 +08:00
# Does the user want to disable atexit?
AC_MSG_CHECKING([whether nc_finalize should be invoked at exit])
AC_ARG_ENABLE([atexit-finalize],
[AS_HELP_STRING([--disable-atexit-finalize],
[disable invoking nc_finalize at exit])])
test "x$enable_atexit_finalize" = xno || enable_atexit_finalize=yes
AC_MSG_RESULT($enable_atexit_finalize)
2021-11-04 03:40:47 +08:00
2021-10-30 10:06:37 +08:00
# Check for atexit
AC_CHECK_FUNCS([atexit])
# If no atexit, then disable atexit finalize
if test "x$enable_atexit_finalize" = xyes ; then
if test "x$ac_cv_func_function" = xno ; then
enable_atexit_finalize=no
AC_MSG_ERROR([atexit() required for enable-atexit-finalize.])
fi
fi
if test "x$enable_atexit_finalize" = xyes ; then
AC_DEFINE([ENABLE_ATEXIT_FINALIZE], [1], [If true, enable nc_finalize via atexit()])
fi
2021-09-03 07:04:26 +08:00
# Need libdl(d) for plugins
AC_CHECK_LIB([dl],[dlopen],[have_libdld=yes],[have_libdld=no])
if test "x$have_libdld" = "xyes" ; then
AC_SEARCH_LIBS([dlopen],[dl dld], [],[])
fi
# Does the user want plugins?
AC_MSG_CHECKING([whether dynamically loaded plugins is enabled])
AC_ARG_ENABLE([plugins],
[AS_HELP_STRING([--disable-plugins],
2021-12-24 13:18:56 +08:00
[disallow dynamically loaded plugins])])
2021-09-03 07:04:26 +08:00
test "x$enable_plugins" = xno || enable_plugins=yes
2021-12-24 13:18:56 +08:00
AC_MSG_RESULT([$enable_plugins])
if test "x$have_libdld" = xno && test "x$ISMSVC" = xno; then
enable_plugins=no
2022-06-04 00:52:00 +08:00
AC_MSG_WARN([libdld required for enable-plugins.])
2021-12-24 13:18:56 +08:00
fi
if test "x$enable_plugins" = xyes && test "x$enable_shared" = xno; then
AC_MSG_WARN([--disable-shared => --disable-plugins])
enable_plugins=no
fi
2021-09-03 07:04:26 +08:00
if test "x$enable_plugins" = xyes; then
AC_DEFINE([ENABLE_PLUGINS], [1], [if true, support dynamically loaded plugins])
fi
AM_CONDITIONAL(ENABLE_PLUGINS, [test "x$enable_plugins" = xyes])
2021-12-24 13:18:56 +08:00
AC_SUBST(USEPLUGINS, [${enable_plugins}])
2021-09-03 07:04:26 +08:00
2010-06-03 21:25:25 +08:00
AC_FUNC_ALLOCA
2018-02-05 23:17:25 +08:00
AC_CHECK_DECLS([isnan, isinf, isfinite],,,[#include <math.h>])
2010-06-03 21:25:25 +08:00
AC_STRUCT_ST_BLKSIZE
UD_CHECK_IEEE
2022-02-01 08:16:23 +08:00
AC_CHECK_TYPES([size_t, ssize_t, schar, uchar, longlong, ushort, uint, int64, uint64, size64_t, ssize64_t, _off64_t, uint64_t, ptrdiff_t])
2010-06-03 21:25:25 +08:00
AC_TYPE_OFF_T
2018-03-17 01:46:18 +08:00
AC_TYPE_UINTPTR_T
AC_C_CHAR_UNSIGNED
2010-06-03 21:25:25 +08:00
AC_C_BIGENDIAN
2015-09-24 02:49:08 +08:00
2020-10-17 05:04:51 +08:00
AM_CONDITIONAL(ISCYGWIN, [test "x$ISCYGWIN" = xyes])
AM_CONDITIONAL(ISMSVC, [test "x$ISMSVC" = xyes])
2021-05-20 11:04:53 +08:00
AM_CONDITIONAL(ISOSX, [test "x$ISOSX" = xyes])
2021-11-04 02:49:54 +08:00
AM_CONDITIONAL(ISMINGW, [test "x$ISMINGW" = xyes])
2021-12-24 13:18:56 +08:00
AM_CONDITIONAL(ISMSYS, [test "x$ISMSYS" = xyes])
2021-05-20 11:04:53 +08:00
AC_SUBST([ISMSVC], [${ISMSVC}])
2022-02-09 11:53:30 +08:00
AC_SUBST([WINVERMAJOR], [${WINVERMAJOR}])
AC_SUBST([WINVERBUILD], [${WINVERBUILD}])
2021-05-20 11:04:53 +08:00
AC_SUBST([ISCYGWIN], [${ISCYGWIN}])
AC_SUBST([ISOSX], [${ISOSX}])
2021-11-04 02:49:54 +08:00
AC_SUBST([ISMINGW], [${ISMINGW}])
2021-12-24 13:18:56 +08:00
AC_SUBST([ISMSYS], [${ISMSYS}])
if test "x$ISMSVC" != x ; then REGEDIT=yes; fi
if test "x$ISMSYS" != x ; then REGEDIT=yes; fi
if test "x$ISCYGWIN" != x ; then REGEDIT=yes; fi
if test "x$REGEDIT" = xyes; then
AC_DEFINE([REGEDIT], 1, [dreg.c usable])
fi
AM_CONDITIONAL(REGEDIT, [test "x$REGEDIT" = xyes])
AC_SUBST([ISREGEDIT], [yes])
2020-10-17 05:04:51 +08:00
2015-09-24 02:49:08 +08:00
###
# Crude hack to work around an issue
# in Cygwin.
###
SLEEPCMD=""
2020-10-17 05:04:51 +08:00
if test "x$ISCYGWIN" = "xyes"; then
2015-09-24 02:49:08 +08:00
SLEEPCMD="sleep 5"
AC_MSG_NOTICE([Pausing between sizeof() checks to mitigate a Cygwin issue.])
fi
$SLEEPCMD
2010-06-03 21:25:25 +08:00
AC_CHECK_SIZEOF(short)
2015-09-24 02:49:08 +08:00
$SLEEPCMD
2010-06-03 21:25:25 +08:00
AC_CHECK_SIZEOF(int)
2015-09-24 02:49:08 +08:00
$SLEEPCMD
2010-06-03 21:25:25 +08:00
AC_CHECK_SIZEOF(long)
2015-09-24 02:49:08 +08:00
$SLEEPCMD
2012-05-31 09:28:42 +08:00
AC_CHECK_SIZEOF(long long)
2015-09-24 02:49:08 +08:00
$SLEEPCMD
2010-06-03 21:25:25 +08:00
AC_CHECK_SIZEOF(float)
2015-09-24 02:49:08 +08:00
$SLEEPCMD
2010-06-03 21:25:25 +08:00
AC_CHECK_SIZEOF(double)
2015-09-24 02:49:08 +08:00
$SLEEPCMD
2010-06-03 21:25:25 +08:00
AC_CHECK_SIZEOF(off_t)
2015-09-24 02:49:08 +08:00
$SLEEPCMD
2010-06-03 21:25:25 +08:00
AC_CHECK_SIZEOF(size_t)
2015-09-16 05:18:59 +08:00
$SLEEPCMD
2015-08-16 06:26:35 +08:00
AC_CHECK_SIZEOF(unsigned long long)
2017-09-17 06:35:52 +08:00
2020-08-18 09:15:47 +08:00
2018-02-03 14:05:14 +08:00
if test "$ac_cv_sizeof_size_t" -lt "8" ; then
2018-07-02 08:42:03 +08:00
if test "x${enable_cdf5}" = xyes ; then
dnl unable to support CDF5, but --enable-cdf5 is explicitly set
AC_MSG_ERROR([Unable to support CDF5 feature because size_t is less than 8 bytes])
fi
2018-02-03 14:05:14 +08:00
enable_cdf5=no
else
2018-07-02 08:42:03 +08:00
if test "x${enable_cdf5}" != xno ; then
enable_cdf5=yes
fi
2018-02-03 14:05:14 +08:00
fi
if test "x${enable_cdf5}" = xyes; then
AC_DEFINE([ENABLE_CDF5], [1], [if true, enable CDF5 Support])
fi
AM_CONDITIONAL(ENABLE_CDF5, [test x$enable_cdf5 = xyes ])
2015-08-16 06:26:35 +08:00
2016-10-29 07:16:49 +08:00
$SLEEPCMD
if test "$ac_cv_type_uchar" = yes ; then
AC_CHECK_SIZEOF(uchar)
else
AC_CHECK_SIZEOF(unsigned char)
fi
2015-11-20 04:44:07 +08:00
$SLEEPCMD
2015-08-16 06:26:35 +08:00
if test "$ac_cv_type_ushort" = yes ; then
AC_CHECK_SIZEOF(ushort)
else
AC_CHECK_SIZEOF(unsigned short int)
fi
2015-11-20 04:44:07 +08:00
$SLEEPCMD
2015-08-16 06:26:35 +08:00
if test "$ac_cv_type_uint" = yes ; then
AC_CHECK_SIZEOF(uint)
else
AC_CHECK_SIZEOF(unsigned int)
fi
2015-09-16 05:18:59 +08:00
$SLEEPCMD
2015-08-16 06:26:35 +08:00
if test "$ac_cv_type_ushort" = yes ; then
AC_CHECK_SIZEOF(ushort)
else
AC_CHECK_SIZEOF(unsigned short int)
fi
2015-09-16 05:18:59 +08:00
$SLEEPCMD
2015-08-16 06:26:35 +08:00
if test "$ac_cv_type_uint" = yes ; then
AC_CHECK_SIZEOF(uint)
else
AC_CHECK_SIZEOF(unsigned int)
fi
2015-11-06 04:40:35 +08:00
$SLEEPCMD
2017-03-09 08:01:10 +08:00
AC_CHECK_SIZEOF(ssize_t)
$SLEEPCMD
AC_CHECK_SIZEOF([void*])
2010-06-03 21:25:25 +08:00
2021-09-03 07:04:26 +08:00
if test "x$enable_hdf5" = xyes || test "x$enable_dap" = xyes ; then
2012-06-12 03:44:32 +08:00
AC_SEARCH_LIBS([deflate], [zlibwapi zlibstat zlib zlib1 z], [], [
2011-05-25 18:54:57 +08:00
AC_MSG_ERROR([Can't find or link to the z library. Turn off netCDF-4 and \
2020-08-18 09:15:47 +08:00
DAP clients with --disable-hdf5 --disable-dap, or see config.log for errors.])])
2011-02-09 20:38:07 +08:00
fi
2012-05-16 01:10:41 +08:00
# We need the math library
2014-09-05 05:42:11 +08:00
AC_CHECK_LIB([m], [floor], [],
2011-06-02 03:48:33 +08:00
[AC_MSG_ERROR([Can't find or link to the math library.])])
2010-06-03 21:25:25 +08:00
if test "x$enable_netcdf_4" = xyes; then
2018-11-29 22:02:31 +08:00
AC_DEFINE([USE_NETCDF4], [1], [if true, build netCDF-4])
fi
2020-02-17 03:59:33 +08:00
# Set defaults
2018-11-29 22:02:31 +08:00
hdf5_parallel=no
2020-01-17 00:19:24 +08:00
hdf5_supports_par_filters=no
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
enable_hdf5_szip=no
2020-09-25 04:33:58 +08:00
has_ross3=no
2020-02-17 03:59:33 +08:00
2018-11-29 22:02:31 +08:00
if test "x$enable_hdf5" = xyes; then
2011-12-13 12:16:52 +08:00
2018-05-09 01:58:01 +08:00
AC_DEFINE([USE_HDF5], [1], [if true, use HDF5])
2010-06-03 21:25:25 +08:00
AC_DEFINE([H5_USE_16_API], [1], [use HDF5 1.6 API])
2011-05-13 00:31:48 +08:00
# Check for the main hdf5 and hdf5_hl library.
2011-12-13 12:16:52 +08:00
2021-09-28 08:36:33 +08:00
AC_SEARCH_LIBS([H5Fflush], [hdf5 hdf5.dll], [],
[AC_MSG_ERROR([Can't find or link to the hdf5 library. Use --disable-hdf5, or see config.log for errors.])])
AC_SEARCH_LIBS([H5DSis_scale], [hdf5_hl hdf5_hl.dll], [],
2020-08-18 09:15:47 +08:00
[AC_MSG_ERROR([Can't find or link to the hdf5 library. Use --disable-hdf5, or see config.log for errors.])])
2014-09-05 05:42:11 +08:00
2018-11-09 23:45:27 +08:00
AC_CHECK_HEADERS([hdf5.h], [], [AC_MSG_ERROR([Compiling a test with HDF5 failed. Either hdf5.h cannot be found, or config.log should be checked for other reason.])])
2011-03-15 18:19:08 +08:00
2020-01-23 20:23:33 +08:00
# Was HDF5 built with zlib as netCDF requires?
2020-06-25 10:28:15 +08:00
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include "H5public.h"],
2020-01-23 20:23:33 +08:00
[[#if !H5_HAVE_ZLIB_H
# error
#endif]
])], [], [AC_MSG_ERROR([HDF5 was not built with zlib, which is required. Rebuild HDF5 with zlib.])])
2018-09-23 09:22:34 +08:00
# H5Pset_fapl_mpiposix and H5Pget_fapl_mpiposix have been removed since HDF5 1.8.12.
# Use H5Pset_fapl_mpio and H5Pget_fapl_mpio, instead.
2021-03-23 06:59:15 +08:00
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate])
2021-03-23 03:40:11 +08:00
2016-12-02 02:56:33 +08:00
# Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0)
if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then
AC_DEFINE([HDF5_HAS_COLL_METADATA_OPS], [1], [if true, use collective metadata ops in parallel netCDF-4])
fi
2015-08-16 06:26:35 +08:00
# If parallel is available in hdf5, enable it in the C code. Also add some stuff to netcdf.h.
2011-03-15 18:19:08 +08:00
if test "x$ac_cv_func_H5Pget_fapl_mpio" = xyes -o "x$ac_cv_func_H5Pget_fapl_mpiposix" = xyes; then
2015-08-16 06:26:35 +08:00
hdf5_parallel=yes
2010-06-03 21:25:25 +08:00
fi
2015-08-16 06:26:35 +08:00
AC_MSG_CHECKING([whether parallel io is enabled in hdf5])
AC_MSG_RESULT([$hdf5_parallel])
2014-09-05 05:42:11 +08:00
2020-05-15 02:59:00 +08:00
# See if H5Dread_chunk is available
AC_SEARCH_LIBS([H5Dread_chunk],[hdf5_hldll hdf5_hl], [has_readchunks=yes], [has_readdhunks=no])
2020-11-12 01:51:24 +08:00
# See if hdf5 library supports Read-Only S3 (byte-range) driver
2020-09-25 04:33:58 +08:00
AC_SEARCH_LIBS([H5Pset_fapl_ros3],[hdf5_hldll hdf5_hl], [has_ros3=yes], [has_ros3=no])
2020-11-29 04:00:06 +08:00
if test "x$has_ros3" = xyes && test "x$enable_byterange" = xyes; then
2020-09-25 04:33:58 +08:00
AC_DEFINE([ENABLE_HDF5_ROS3], [1], [if true, support byte-range using hdf5 virtual file driver.])
fi
2020-02-07 04:25:25 +08:00
# Check to see if HDF5 library is 1.10.3 or greater. If so, allows
# parallel I/O with filters. This allows zlib/szip compression to
# be used with parallel I/O, which is very helpful to HPC users.
2020-05-15 02:59:00 +08:00
if test "x$has_readchunks" = xyes; then
2020-01-21 22:42:14 +08:00
AC_DEFINE([HDF5_SUPPORTS_PAR_FILTERS], [1], [if true, HDF5 is at least version 1.10.3 and allows parallel I/O with zip])
2020-01-17 00:19:24 +08:00
hdf5_supports_par_filters=yes
2019-12-20 20:54:21 +08:00
fi
2020-02-07 04:25:25 +08:00
AC_MSG_CHECKING([whether HDF5 allows parallel filters])
2020-05-15 02:59:00 +08:00
AC_MSG_RESULT([$has_readchunks])
2019-12-20 20:54:21 +08:00
2020-02-06 22:47:15 +08:00
# Check to see if user asked for parallel build, but HDF5 does not support it.
2018-02-27 08:08:26 +08:00
if test "x$hdf5_parallel" = "xno"; then
if test "x$enable_parallel_tests" = "xyes"; then
AC_MSG_ERROR([Parallel tests requested, but no parallel HDF5 installation detected.])
fi
fi
2020-02-07 00:17:49 +08:00
# Check whether HDF5 was built with the SZLIB library. If so we
# must be able to link to szip library.
2018-11-15 01:36:27 +08:00
AC_MSG_CHECKING([whether szlib was used when building HDF5])
2011-04-30 04:39:32 +08:00
if test "x$ac_cv_func_H5Z_SZIP" = xyes; then
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
enable_hdf5_szip=yes
AC_DEFINE([HAVE_H5Z_SZIP], [1], [if true, compile in szip compression in netCDF-4 variables])
2010-06-03 21:25:25 +08:00
fi
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AC_MSG_RESULT([$enable_hdf5_szip])
2020-05-15 02:18:42 +08:00
2020-10-17 05:04:51 +08:00
# Check to see if HDF5 library is 1.10.6 or greater.
# Used to control path name conversion
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <H5public.h>]], [[
#if (H5_VERS_MAJOR*10000 + H5_VERS_MINOR*100 + H5_VERS_RELEASE < 11006)
choke me
#endif
]])], [hdf5_version_1106=yes], [hdf5_version_1106=no])
AC_MSG_CHECKING([whether HDF5 library is version 1.10.6 or later])
AC_MSG_RESULT([$hdf5_version_1106])
if test "x$hdf5_version_1106" = xyes; then
AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8])
fi
2021-09-03 07:04:26 +08:00
2018-11-29 22:02:31 +08:00
fi
2020-10-17 05:04:51 +08:00
2020-05-15 02:59:00 +08:00
AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ])
2014-09-05 05:42:11 +08:00
2018-11-29 22:02:31 +08:00
# If the user wants hdf4 built in, check it out.
if test "x$enable_hdf4" = xyes; then
AC_CHECK_LIB([jpeg], [jpeg_CreateCompress], [],
[AC_MSG_ERROR([Jpeg library required for --enable-hdf4 builds.])])
AC_CHECK_HEADERS([mfhdf.h], [], [nc_mfhdf_h_missing=yes])
if test "x$nc_mfhdf_h_missing" = xyes; then
AC_MSG_ERROR([Cannot find mfhdf.h, yet --enable-hdf4 was used.])
2010-06-03 21:25:25 +08:00
fi
2018-11-29 22:02:31 +08:00
AC_CHECK_LIB([df], [Hclose], [], [AC_MSG_ERROR([Can't find or link to the hdf4 df library. See config.log for errors.])])
AC_CHECK_LIB([mfhdf], [NC_arrayfill], [AC_MSG_ERROR([HDF4 library must be built with --disable-netcdf.])], [])
AC_CHECK_LIB([mfhdf], [SDcreate], [], [AC_MSG_ERROR([Can't find or link to the hdf4 mfhdf library. See config.log for errors.])])
AC_CHECK_LIB([jpeg], [jpeg_set_quality], [], [AC_MSG_ERROR([Can't find or link to the jpeg library (required by hdf4). See config.log for errors.])])
AC_DEFINE([USE_HDF4], [1], [if true, use HDF4 too])
2010-06-03 21:25:25 +08:00
fi
2015-08-16 06:26:35 +08:00
# There are several cases for parallelism:
2018-09-18 00:47:40 +08:00
# 1. PnetCDF enabled => we want to parallelism for CDF-1,CDF-2,and CDF-5
2015-08-16 06:26:35 +08:00
# 2. hdf5 has mpio enabled
# a. do not want to use it for netcdf4
# b. do want to use it for netcdf4
2017-10-27 04:05:04 +08:00
# Should we provide parallel io for netcdf-4?
2018-11-29 22:02:31 +08:00
if test "x$enable_hdf5" = xyes ; then
2017-10-27 04:05:04 +08:00
AC_ARG_ENABLE([parallel4],
[AS_HELP_STRING([--disable-parallel4],
[disable parallel I/O for netcdf-4, even if it's enabled in libhdf5])],
2019-01-03 02:04:28 +08:00
[user_set_parallel4=${enableval}]2)
2017-10-27 04:05:04 +08:00
test "x$enable_parallel4" = xno || enable_parallel4=yes
# If user wants parallel IO for netCDF-4, make sure HDF5 can provide it.
if test "x$enable_parallel4" = xyes; then
if test "x$hdf5_parallel" = xno; then
# If user specifically asked for parallel4, then error out.
if test "x$user_set_parallel4" = xyes; then
AC_MSG_ERROR([Paralllel IO in netCDF-4 requested, but HDF5 does not provide parallel IO.])
fi
# User didn't specify, so disable parallel4
enable_parallel4=no
AC_MSG_WARN([Parallel io disabled for netcdf-4 because hdf5 does not support])
fi
fi
2015-08-16 06:26:35 +08:00
else
2017-10-27 04:05:04 +08:00
enable_parallel4=no
2015-08-16 06:26:35 +08:00
fi
2018-11-15 01:36:27 +08:00
AC_MSG_CHECKING([whether parallel I/O is enabled for netcdf-4])
AC_MSG_RESULT($enable_parallel4)
2015-08-16 06:26:35 +08:00
# We have already tested for parallel io in netcdf4
2018-09-18 00:47:40 +08:00
# parallel I/O for CDF-1, 2, and 5 files can also be done through PnetCDF
# See if the PnetCDF lib is available and of the
2015-08-16 06:26:35 +08:00
# right version (1.6.0 or later)
if test "x$enable_pnetcdf" = xyes; then
pnetcdf_conflict=no
AC_CHECK_LIB([pnetcdf], [ncmpi_create], [],[pnetcdf_conflict=yes])
2017-10-26 19:38:31 +08:00
if test "x$pnetcdf_conflict" = xyes ; then
2018-09-18 00:47:40 +08:00
AC_MSG_ERROR([Cannot link to PnetCDF library.])
2017-11-21 06:50:47 +08:00
fi
2017-10-26 19:38:31 +08:00
2015-08-16 06:26:35 +08:00
# Pnetcdf did not support utf-8 until 1.6.0
AC_MSG_CHECKING([Is libpnetcdf version 1.6.0 or later?])
2020-06-04 01:13:35 +08:00
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <pnetcdf.h>]], [[
2018-07-28 01:18:37 +08:00
#if (PNETCDF_VERSION_MAJOR*1000 + PNETCDF_VERSION_MINOR < 1006)
choke me
#endif
]])], [pnetcdf16=yes], [pnetcdf16=no])
2015-08-16 06:26:35 +08:00
AC_MSG_RESULT([$pnetcdf16])
if test x$pnetcdf16 = xno; then
2017-10-26 19:38:31 +08:00
AC_MSG_ERROR([--enable-pnetcdf requires version 1.6.0 or later])
2015-08-16 06:26:35 +08:00
fi
fi
2018-09-18 00:47:40 +08:00
# Now, set enable_parallel if either enable_pnetcdf or enable_parallel4 is set
2015-08-16 06:26:35 +08:00
if test "x$enable_pnetcdf" = xyes -o "x$enable_parallel4" = xyes; then
2018-09-18 00:47:40 +08:00
enable_parallel=yes
2015-08-16 06:26:35 +08:00
else
2018-09-18 00:47:40 +08:00
enable_parallel=no
2015-08-16 06:26:35 +08:00
fi
2017-10-17 00:30:47 +08:00
AM_CONDITIONAL(ENABLE_PARALLEL, [test x$enable_parallel = xyes ])
2015-08-16 06:26:35 +08:00
if test "x$hdf5_parallel" = xyes; then
# Provide more precise parallel control
AC_DEFINE([HDF5_PARALLEL], [1], [if true, hdf5 has parallelism enabled])
fi
# Set config flags
if test "x$enable_parallel4" = xyes; then
# Provide more precise parallel control
AC_DEFINE([USE_PARALLEL4], [1], [if true, parallel netcdf-4 is in use])
fi
if test "x$enable_pnetcdf" = xyes; then
2018-09-18 00:47:40 +08:00
AC_DEFINE([USE_PNETCDF], [1], [if true, PnetCDF is used])
2015-08-16 06:26:35 +08:00
fi
# If enable_parallel is in use, enable it in the C code. Also add some stuff to netcdf.h.
if test "x$enable_parallel" = xyes; then
2018-09-18 00:47:40 +08:00
AC_DEFINE([USE_PARALLEL], [1], [if true, PnetCDF or parallel netcdf-4 is in use])
2015-08-16 06:26:35 +08:00
fi
2016-11-04 04:14:14 +08:00
AC_ARG_ENABLE([erange_fill],
2017-04-06 07:09:39 +08:00
[AS_HELP_STRING([--enable-erange-fill],
[Enable use of fill value when out-of-range type
conversion causes NC_ERANGE error. @<:@default: disabled@:>@])],
2018-07-17 03:34:22 +08:00
[enable_erange_fill=${enableval}], [enable_erange_fill=auto]
2016-11-04 04:14:14 +08:00
)
2016-11-15 00:41:10 +08:00
# check PnetCDF's settings on enable_erange_fill and relax_coord_bound
2016-11-13 13:58:09 +08:00
if test "x$enable_pnetcdf" = xyes; then
UD_CHECK_HEADER_PATH([pnetcdf.h])
2016-11-15 00:41:10 +08:00
AC_MSG_CHECKING([if erange-fill is enabled in PnetCDF])
2020-06-04 01:13:35 +08:00
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <pnetcdf.h>]], [[
2018-07-17 03:34:22 +08:00
#if !defined(PNETCDF_ERANGE_FILL) || PNETCDF_ERANGE_FILL == 0
choke me
#endif]])], [enable_erange_fill_pnetcdf=yes], [enable_erange_fill_pnetcdf=no])
2017-10-26 19:38:31 +08:00
AC_MSG_RESULT([$enable_erange_fill_pnetcdf])
2018-07-17 03:34:22 +08:00
if test "x$enable_erange_fill" = xauto ; then
enable_erange_fill=$enable_erange_fill_pnetcdf
elif test "$enable_erange_fill" != "$enable_erange_fill_pnetcdf"; then
if test "$enable_erange_fill" = yes; then
AC_MSG_ERROR([Enabling erange-fill conflicts with PnetCDF setting])
2016-11-15 00:41:10 +08:00
else
2018-07-17 03:34:22 +08:00
AC_MSG_ERROR([Disabling erange-fill conflicts with PnetCDF setting])
2016-11-15 00:41:10 +08:00
fi
fi
AC_MSG_CHECKING([if relax-coord-bound is enabled in PnetCDF])
2020-06-04 01:13:35 +08:00
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <pnetcdf.h>]], [[
2018-07-17 03:34:22 +08:00
#if !defined(PNETCDF_RELAX_COORD_BOUND) || PNETCDF_RELAX_COORD_BOUND == 0
choke me
#endif]])], [relax_coord_bound_pnetcdf=yes], [relax_coord_bound_pnetcdf=no])
2017-10-26 19:38:31 +08:00
AC_MSG_RESULT([$relax_coord_bound_pnetcdf])
2018-12-03 21:17:23 +08:00
if test x"$relax_coord_bound_pnetcdf" != xyes; then
AC_MSG_ERROR([PNetCDF must be built with relax-coord-bound])
2016-11-13 13:58:09 +08:00
fi
2018-07-17 03:34:22 +08:00
else
2019-07-14 05:07:50 +08:00
if test "x$enable_erange_fill" = xauto; then
# if --enable-erange-fill is not used, default setting is no
enable_erange_fill=no
fi
2018-07-17 03:34:22 +08:00
fi
2016-11-15 00:41:10 +08:00
if test "x$enable_erange_fill" = xyes ; then
if test "x$M4FLAGS" = x ; then
M4FLAGS="-DERANGE_FILL"
else
M4FLAGS="$M4FLAGS -DERANGE_FILL"
fi
2017-08-21 03:32:57 +08:00
AC_DEFINE([ERANGE_FILL], [1], [if true, use _FillValue for NC_ERANGE data elements])
2016-11-15 00:41:10 +08:00
fi
AC_SUBST(M4FLAGS)
2014-11-19 06:33:16 +08:00
# Check for downloading/building fortran via postinstall script.
2014-11-21 06:56:11 +08:00
if test "x$enable_remote_fortran_bootstrap" = xyes; then
2014-11-19 06:33:16 +08:00
AC_DEFINE([BUILD_FORTRAN], 1, [If true, will attempt to download and build netcdf-fortran.])
fi
2010-06-03 21:25:25 +08:00
# No logging for netcdf-3.
if test "x$enable_netcdf_4" = xno; then
enable_logging=no
fi
if test "x$enable_logging" = xyes; then
AC_DEFINE([LOGGING], 1, [If true, turn on logging.])
fi
2021-09-03 07:04:26 +08:00
# Control NCZarr filters
# Does the user want to use NCZarr filters?
AC_MSG_CHECKING([whether we should enable NCZarr filters])
AC_ARG_ENABLE([nczarr-filters], [AS_HELP_STRING([--disable-nczarr-filters],
[disable NCZarr filters])])
test "x$enable_nczarr_filters" = xno || enable_nczarr_filters=yes
AC_MSG_RESULT([$enable_nczarr_filters])
# Control filter test/example
AC_MSG_CHECKING([whether filter testing should be run])
AC_ARG_ENABLE([filter-testing],
[AS_HELP_STRING([--disable-filter-testing],
[Do not run filter test and example; requires shared libraries and HDF5|NCZarr])])
test "x$enable_filter_testing" = xno || enable_filter_testing=yes
AC_MSG_RESULT($enable_filter_testing)
2021-12-24 13:18:56 +08:00
# Apply constraints
if test "x$enable_plugins" = xno ; then
AC_MSG_WARN([--disable-plugins => --disable-nczarr-filters])
2021-09-03 07:04:26 +08:00
enable_nczarr_filters=no
fi
2021-12-24 13:18:56 +08:00
if test "x$enable_plugins" = xno ; then
AC_MSG_WARN([--disable-plugins => --disable-filter-testing])
2021-09-03 07:04:26 +08:00
enable_filter_testing=no
fi
2021-12-24 13:18:56 +08:00
if test "x$enable_filter_testing" = xno; then
AC_MSG_WARN([--disable-filter-testing => --disable-nczarr-filter-testing])
enable_nczarr_filter_testing=no
fi
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
if test "x$enable_nczarr" = xno; then
AC_MSG_WARN([--disable-nczarr => --disable-nczarr-filters])
enable_nczarr_filters=no
enable_nczarr_filter_testing=no
fi
2021-12-24 13:18:56 +08:00
if test "x$enable_nczarr_filters" = xyes; then
AC_DEFINE([ENABLE_NCZARR_FILTERS], [1], [if true, enable NCZarr filters])
2021-09-03 07:04:26 +08:00
fi
# Client side filter registration is permanently disabled
enable_clientside_filters=no
AM_CONDITIONAL(ENABLE_CLIENTSIDE_FILTERS, [test x$enable_clientside_filters = xyes])
AM_CONDITIONAL(ENABLE_FILTER_TESTING, [test x$enable_filter_testing = xyes])
AM_CONDITIONAL(ENABLE_NCZARR_FILTERS, [test x$enable_nczarr_filters = xyes])
2010-06-03 21:25:25 +08:00
# Automake conditionals need to be called, whether the answer is yes
# or no.
AM_CONDITIONAL(BUILD_PARALLEL, [test x$enable_parallel = xyes])
2015-08-16 06:26:35 +08:00
AM_CONDITIONAL(TEST_PARALLEL4, [test "x$enable_parallel4" = xyes -a "x$enable_parallel_tests" = xyes])
2010-06-03 21:25:25 +08:00
AM_CONDITIONAL(BUILD_DAP, [test "x$enable_dap" = xyes])
AM_CONDITIONAL(USE_DAP, [test "x$enable_dap" = xyes]) # Alias
2017-03-09 08:01:10 +08:00
# Provide protocol specific flags
AM_CONDITIONAL(ENABLE_DAP, [test "x$enable_dap" = xyes])
AM_CONDITIONAL(ENABLE_DAP4, [test "x$enable_dap4" = xyes])
2017-11-21 06:50:47 +08:00
AM_CONDITIONAL(USE_STRICT_NULL_BYTE_HEADER_PADDING, [test x$enable_strict_null_byte_header_padding = xyes])
2017-09-14 05:25:40 +08:00
AM_CONDITIONAL(ENABLE_CDF5, [test "x$enable_cdf5" = xyes])
2010-06-03 21:25:25 +08:00
AM_CONDITIONAL(ENABLE_DAP_REMOTE_TESTS, [test "x$enable_dap_remote_tests" = xyes])
2014-03-08 03:04:38 +08:00
AM_CONDITIONAL(ENABLE_DAP_AUTH_TESTS, [test "x$enable_dap_auth_tests" = xyes])
2010-06-03 21:25:25 +08:00
AM_CONDITIONAL(ENABLE_DAP_LONG_TESTS, [test "x$enable_dap_long_tests" = xyes])
AM_CONDITIONAL(USE_PNETCDF_DIR, [test ! "x$PNETCDFDIR" = x])
AM_CONDITIONAL(USE_LOGGING, [test "x$enable_logging" = xyes])
AM_CONDITIONAL(CROSS_COMPILING, [test "x$cross_compiling" = xyes])
AM_CONDITIONAL(USE_NETCDF4, [test x$enable_netcdf_4 = xyes])
2018-05-09 01:58:01 +08:00
AM_CONDITIONAL(USE_HDF5, [test x$enable_hdf5 = xyes])
2010-06-03 21:25:25 +08:00
AM_CONDITIONAL(USE_HDF4, [test x$enable_hdf4 = xyes])
AM_CONDITIONAL(USE_HDF4_FILE_TESTS, [test x$enable_hdf4_file_tests = xyes])
AM_CONDITIONAL(USE_RENAMEV3, [test x$enable_netcdf_4 = xyes -o x$enable_dap = xyes])
2014-11-21 06:56:11 +08:00
AM_CONDITIONAL(BUILD_FORTRAN, [test x$enable_remote_fortran_bootstrap = xyes])
2010-06-03 21:25:25 +08:00
AM_CONDITIONAL(USE_PNETCDF, [test x$enable_pnetcdf = xyes])
AM_CONDITIONAL(USE_DISPATCH, [test x$enable_dispatch = xyes])
2012-05-16 01:10:41 +08:00
AM_CONDITIONAL(BUILD_MMAP, [test x$enable_mmap = xyes])
2015-05-27 04:29:28 +08:00
AM_CONDITIONAL(BUILD_DOCS, [test x$enable_doxygen = xyes])
AM_CONDITIONAL(SHOW_DOXYGEN_TAG_LIST, [test x$enable_doxygen_tasks = xyes])
2018-03-18 06:25:13 +08:00
AM_CONDITIONAL(ENABLE_METADATA_PERF, [test x$enable_metadata_perf = xyes])
2019-02-25 07:54:13 +08:00
AM_CONDITIONAL(ENABLE_BYTERANGE, [test "x$enable_byterange" = xyes])
2020-01-25 08:15:29 +08:00
AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes])
2020-02-17 03:59:33 +08:00
AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
2021-09-28 08:36:33 +08:00
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
2020-10-17 05:04:51 +08:00
AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes])
AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes])
AM_CONDITIONAL(HAVE_BLOSC, [test "x$have_blosc" = xyes])
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
AM_CONDITIONAL(HAVE_BZ2, [test "x$have_bz2" = xyes])
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AM_CONDITIONAL(HAVE_ZSTD, [test "x$have_zstd" = xyes])
2010-06-03 21:25:25 +08:00
# If the machine doesn't have a long long, and we want netCDF-4, then
# we've got problems!
if test "x$enable_netcdf_4" = xyes; then
AC_TYPE_LONG_LONG_INT
AC_TYPE_UNSIGNED_LONG_LONG_INT
dnl if test ! "x$ac_cv_type_long_long_int" = xyes -o ! "x$ac_cv_type_unsigned_long_long_int" = xyes; then
dnl AC_MSG_ERROR([This platform does not support long long types. These are required for netCDF-4.])
dnl fi
fi
# Create the file name for a "make ftpbin" which is used to generate a
# binary distribution. For each release we generate binary releases on
# the thousands of machines in Unidata's vast underground complex at
# an undisclosed location in the Rocky Mountains. The binary
# distributions, along with the 25-foot thick cement slabs and the
# giant springs, will help distribute netCDF even after a catastrophic
# meteor strike.
AC_MSG_CHECKING([what to call the output of the ftpbin target])
BINFILE_NAME=binary-netcdf-$PACKAGE_VERSION
test "x$enable_netcdf_4" = xno && BINFILE_NAME=${BINFILE_NAME}_nc3
BINFILE_NAME=${BINFILE_NAME}.tar
AC_SUBST(BINFILE_NAME)
AC_MSG_RESULT([$BINFILE_NAME $FC $CXX])
2015-11-03 07:26:50 +08:00
##
# Bugfix for Cygwin.
##
AC_MSG_CHECKING([if libtool needs -no-undefined flag to build shared libraries])
2020-11-12 01:51:24 +08:00
NOUNDEFINED=""
2015-11-03 07:26:50 +08:00
case "`uname`" in
CYGWIN*|MINGW*|AIX*)
## Add in the -no-undefined flag to LDFLAGS for libtool.
2015-11-04 01:37:42 +08:00
AC_MSG_RESULT([yes])
2015-11-03 07:26:50 +08:00
NOUNDEFINED=" -no-undefined"
;;
*)
## Don't add anything
AC_MSG_RESULT([no])
;;
esac
2010-06-03 21:25:25 +08:00
2011-05-18 03:14:35 +08:00
AC_MSG_CHECKING([value of LIBS])
AC_MSG_RESULT([$LIBS])
2010-06-03 21:25:25 +08:00
# Flags for nc-config script; by design $prefix, $includir, $libdir,
# etc. are left as shell variables in the script so as to facilitate
# relocation
2014-09-05 05:42:11 +08:00
if test "x$with_netcdf_c_lib" = x ; then
2010-06-03 21:25:25 +08:00
NC_LIBS="-lnetcdf"
else
NC_LIBS="$with_netcdf_c_lib"
fi
if test "x$enable_shared" != xyes; then
2011-06-10 21:34:12 +08:00
NC_LIBS="$LDFLAGS $NC_LIBS $LIBS"
2014-09-05 05:42:11 +08:00
fi
2010-06-03 21:25:25 +08:00
case "x$target_os" in
2014-09-05 05:42:11 +08:00
xsolaris*)
2010-06-03 21:25:25 +08:00
NEWNCLIBS=""
for x in $NC_LIBS ; do
case "$x" in
-L*) r=`echo "$x" | sed -e 's|^-L|-R|'`
NEWNCLIBS="$NEWNCLIBS $x $r"
;;
*) NEWNCLIBS="$NEWNCLIBS $x" ;;
esac
done
NC_LIBS="$NEWNCLIBS"
;;
*);;
esac
2011-04-27 04:57:24 +08:00
NC_FLIBS="-lnetcdff $NC_LIBS"
2010-06-03 21:25:25 +08:00
2020-09-16 06:56:17 +08:00
2014-05-31 05:30:58 +08:00
AC_SUBST(NC_LIBS,[$NC_LIBS])
AC_SUBST(HAS_DAP,[$enable_dap])
2017-09-27 01:33:33 +08:00
AC_SUBST(HAS_DAP2,[$enable_dap])
2017-03-09 08:01:10 +08:00
AC_SUBST(HAS_DAP4,[$enable_dap4])
2014-05-31 05:30:58 +08:00
AC_SUBST(HAS_NC2,[$nc_build_v2])
AC_SUBST(HAS_NC4,[$enable_netcdf_4])
2017-09-14 05:25:40 +08:00
AC_SUBST(HAS_CDF5,[$enable_cdf5])
2014-05-31 05:30:58 +08:00
AC_SUBST(HAS_HDF4,[$enable_hdf4])
2022-04-26 20:18:52 +08:00
AC_SUBST(HAS_BENCHMARKS,[$enable_benchmarks])
2018-05-09 01:58:01 +08:00
AC_SUBST(HAS_HDF5,[$enable_hdf5])
2014-05-31 05:30:58 +08:00
AC_SUBST(HAS_PNETCDF,[$enable_pnetcdf])
2016-03-01 02:22:10 +08:00
AC_SUBST(HAS_LOGGING, [$enable_logging])
2014-05-31 05:30:58 +08:00
AC_SUBST(HAS_PARALLEL,[$enable_parallel])
2015-08-16 06:26:35 +08:00
AC_SUBST(HAS_PARALLEL4,[$enable_parallel4])
2018-05-12 05:30:19 +08:00
AC_SUBST(HAS_DISKLESS,[yes])
2014-05-31 05:30:58 +08:00
AC_SUBST(HAS_MMAP,[$enable_mmap])
AC_SUBST(HAS_JNA,[$enable_jna])
2018-02-03 11:31:45 +08:00
AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill])
2019-02-25 07:54:13 +08:00
AC_SUBST(HAS_BYTERANGE,[$enable_byterange])
2020-01-25 08:15:29 +08:00
AC_SUBST(RELAX_COORD_BOUND,[yes])
2020-02-17 03:59:33 +08:00
AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters])
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
AC_SUBST(HAS_NCZARR,[$enable_nczarr])
2020-10-17 05:04:51 +08:00
AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
2021-01-29 11:11:01 +08:00
AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
2021-08-24 14:45:38 +08:00
AC_SUBST([HAS_QUANTIZE],[yes])
2021-08-24 14:17:03 +08:00
AC_SUBST(HAS_LOGGING,[$enable_logging])
2021-09-03 07:04:26 +08:00
AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing])
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AC_SUBST(HAS_SZLIB,[$have_sz])
AC_SUBST(HAS_SZLIB_WRITE, [$have_sz])
2022-04-11 22:03:24 +08:00
AC_SUBST(HAS_ZSTD,[$have_zstd])
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
# Always available
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
std_filters="deflate bz2"
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
if test "x$enable_szlib" = xyes ; then
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
std_filters="${std_filters} szip"
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
fi
# We need to include szip iff HDF5 && HDF5_HAS_SZIP || !HDF5 && NCZARR && libsz
if test "x$enable_hdf5" = xyes && test "x$enable_hdf5_szip" = xyes ; then
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
std_filters="${std_filters} szip"
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
fi
if test "x$enable_hdf5" = xno && test "x$have_sz" = xyes ; then
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
std_filters="${std_filters} szip"
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
fi
if test "x$have_blosc" = xyes ; then
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
std_filters="${std_filters} blosc"
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
fi
if test "x$have_zstd" = xyes ; then
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
std_filters="${std_filters} zstd"
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
fi
AC_SUBST(STD_FILTERS,[$std_filters])
2018-07-14 05:32:43 +08:00
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
# If user wants, then install selected standard filters
AC_MSG_CHECKING([whether and where we should install plugins])
2022-05-25 10:05:19 +08:00
AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=<absolute directory>|no|--without-plugin-dir],
[Install selected standard filters in specified or default directory])],
2022-06-11 05:03:36 +08:00
[],[with_plugin_dir=no])
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
AC_MSG_RESULT([$with_plugin_dir])
2022-05-17 03:58:42 +08:00
if test "x$with_plugin_dir" = xno ; then # option missing|disabled
2022-06-04 00:52:00 +08:00
with_plugin_dir=no
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
with_plugin_dir_setting="N.A."
enable_plugin_dir=no
2022-05-17 03:58:42 +08:00
elif test "x$with_plugin_dir" = xyes ; then # --with-plugin-dir, no argument
# Default to last dir (lowest search priority) in HDF5_PLUGIN_PATH
PLUGIN_PATH="$HDF5_PLUGIN_PATH"
if test "x${PLUGIN_PATH}" = x ; then
if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then
PLUGIN_PATH="${ALLUSERSPROFILE}\\hdfd5\\lib\\plugin"
else
PLUGIN_PATH="/usr/local/hdf5/lib/plugin"
fi
fi
# Use the lowest priority dir in the path
if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then
2022-05-25 10:05:19 +08:00
PLUGIN_DIR=`echo "$PLUGIN_PATH" | tr ';' ' '`
2022-05-17 03:58:42 +08:00
else
2022-05-25 10:05:19 +08:00
PLUGIN_DIR=`echo "$PLUGIN_PATH" | tr ':' ' '`
2022-05-17 03:58:42 +08:00
fi
2022-05-25 10:05:19 +08:00
for pp in ${PLUGIN_DIR} ; do last="$pp"; done
PLUGIN_DIR="$last"
2022-05-17 03:58:42 +08:00
with_plugin_dir_setting="$PLUGIN_DIR"
# canonical form is all forward slashes
with_plugin_dir=`echo "$PLUGIN_DIR" | tr '\\\\' '/'`
enable_plugin_dir=yes
2022-05-25 10:05:19 +08:00
AC_MSG_NOTICE([Defaulting to --with-plugin-dir=$with_plugin_dir])
2022-05-17 03:58:42 +08:00
else # --with-plugin-dir=<dir|path>
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
with_plugin_dir_setting="$with_plugin_dir"
enable_plugin_dir=yes
fi
AM_CONDITIONAL([ENABLE_PLUGIN_DIR], [test "x$enable_plugin_dir" = xyes])
AC_SUBST([PLUGIN_INSTALL_DIR], [$with_plugin_dir])
# Better value for libnetcdf.settings
AC_SUBST([PLUGIN_INSTALL_DIR_SETTING], [$with_plugin_dir_setting])
2011-07-15 06:24:02 +08:00
# Access netcdf specific version of config.h
AH_BOTTOM([#include "ncconfigure.h"])
2010-06-03 21:25:25 +08:00
##################################################
# Uncomment this to keep a copy of autoconf defines at this point, for
# debugging purposes.
# cp confdefs.h my_config.h
2014-09-05 07:20:15 +08:00
#####
2014-09-05 05:42:11 +08:00
# Create output variables from various
2014-05-31 05:30:58 +08:00
# shell variables, for use in generating
# libnetcdf.settings.
2014-09-05 07:20:15 +08:00
#####
2014-05-31 05:30:58 +08:00
AC_SUBST([enable_shared])
AC_SUBST([enable_static])
AC_SUBST([CFLAGS])
AC_SUBST([CPPFLAGS])
AC_SUBST([LDFLAGS])
AC_SUBST([AM_CFLAGS])
AC_SUBST([AM_CPPFLAGS])
AC_SUBST([AM_LDFLAGS])
2015-11-03 07:26:50 +08:00
AC_SUBST([NOUNDEFINED])
2014-05-31 05:30:58 +08:00
2014-09-11 06:50:45 +08:00
# Args:
# 1. netcdf_meta.h variable
# 2. conditional variable that is yes or no.
# 3. default condition
#
# example: AX_SET_META([NC_HAS_NC2],[$nc_build_v2],[]) # Because it checks for no.
# AX_SET_META([NC_HAS_HDF4],[$enable_hdf4],[yes])
AC_DEFUN([AX_SET_META],[
if [ test "x$2" = x$3 ]; then
AC_SUBST([$1]) $1=1
2014-09-06 01:09:46 +08:00
else
2014-09-11 06:50:45 +08:00
AC_SUBST([$1]) $1=0
2014-09-06 01:09:46 +08:00
fi
])
2014-09-05 07:20:15 +08:00
#####
# Define values used in include/netcdf_meta.h
#####
2014-09-11 06:50:45 +08:00
AC_SUBST([NC_VERSION]) NC_VERSION=$VERSION
AX_SET_META([NC_HAS_NC2],[$nc_build_v2],[yes])
AX_SET_META([NC_HAS_NC4],[$enable_netcdf_4],[yes])
AX_SET_META([NC_HAS_HDF4],[$enable_hdf4],[yes])
2022-04-26 20:18:52 +08:00
AX_SET_META([NC_HAS_BENCHMARKS],[$enable_benchmarks],[yes])
2020-08-18 09:15:47 +08:00
AX_SET_META([NC_HAS_HDF5],[$enable_hdf5],[yes])
2017-09-27 01:33:33 +08:00
AX_SET_META([NC_HAS_DAP2],[$enable_dap],[yes])
2017-03-09 08:01:10 +08:00
AX_SET_META([NC_HAS_DAP4],[$enable_dap4],[yes])
2018-05-12 05:30:19 +08:00
AX_SET_META([NC_HAS_DISKLESS],[yes],[yes])
2014-09-11 06:50:45 +08:00
AX_SET_META([NC_HAS_MMAP],[$enable_mmap],[yes])
AX_SET_META([NC_HAS_JNA],[$enable_jna],[yes])
AX_SET_META([NC_HAS_PNETCDF],[$enable_pnetcdf],[yes])
AX_SET_META([NC_HAS_PARALLEL],[$enable_parallel],[yes])
2015-08-16 06:26:35 +08:00
AX_SET_META([NC_HAS_PARALLEL4],[$enable_parallel4],[yes])
2017-09-27 01:33:33 +08:00
AX_SET_META([NC_HAS_CDF5],[$enable_cdf5],[yes])
2018-01-26 03:46:46 +08:00
AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes])
2020-01-17 00:19:24 +08:00
AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes])
2019-02-25 07:54:13 +08:00
AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes])
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
2020-10-17 05:04:51 +08:00
AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes])
2021-08-24 14:17:03 +08:00
AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes])
2022-04-11 22:03:24 +08:00
AX_SET_META([NC_HAS_QUANTIZE],[yes],[yes])
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AX_SET_META([NC_HAS_SZIP],[$enable_hdf5_szip],[yes])
2022-04-11 22:03:24 +08:00
AX_SET_META([NC_HAS_ZSTD],[$have_zstd],[yes])
Fix nczarr-experimental to better support CMake and find AWS libraries
The primary fix is to improve CMake build support.
Specific changes include:
* CMake: Provide a better soln to locating the AWS SDK
libraries; the new way is the preferred method as described in
the aws-cpp-sdk documentation.
* CMake (and Automake): allow -DENABLE_S3_SDK (default off) to suppress
looking for AWS libraries.
* CMake: add the complete set of nczarr tests
* CMake: add EXTERNL as needed to various .h files.
* Improve support for windows drive letters in paths.
* Add nczarr and s3 flags to nc-config
* For VisualStudio X nczarr, cleanup the NAN+INFINITY handling
* Convert _MSC_VER -> _WIN32 and vice versa as needed
* NCZarr - support multiple platform paths including windows, cygwin.
mingw, etc.
* NCZarr - sort the test outputs because different platforms
produce directory contents in different orders.
One big change concerns netcdf-c/CMakeLists.txt and netcdf-c/configure.ac.
In the current versions, it was the case that --disable-hdf5
disabled netcdf-4 (libsrc4). With nczarr, this can no longer
be the case because nczarr requires libsrc4 even if libhdf5
is disabled. So, I modified the above files to move the
format options (HDF5, NCZarr, HDF4, etc) to a single place
near the front of the files. Now it is the case that:
* Enabling any of the formats that require libsrc4
also does an implicit --enable-netcdf4.
* --disable-netcdf4 | --disable-netcdf-4 now becomes
and alias for --disable-hdf5.
There are probably some bugs in this change in terms of
dependencies between format options.
Problems:
* CMake S3 support is still not working for Visual Studio
* A recent issue points out that there is work to do on handling
UTF8 filenames, but that will be addressed in a separate fix.
Notes:
* Consider converting all of our includes/.h files to use EXTERNL
2020-07-13 02:21:56 +08:00
2020-07-15 20:58:16 +08:00
# This is the version of the dispatch table. If the dispatch table is
# changed, this should be incremented, so that user-defined format
# applications like PIO can determine whether they have an appropriate
# dispatch table to submit. If this is changed, make sure the value in
# CMakeLists.txt also changes to match.
2021-02-01 12:40:08 +08:00
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
AC_SUBST([NC_DISPATCH_VERSION], [5])
2021-02-01 12:40:08 +08:00
AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.])
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
2019-03-12 22:55:30 +08:00
#####
# End netcdf_meta.h definitions.
#####
2012-07-18 06:23:37 +08:00
2019-03-12 22:55:30 +08:00
# This would be true for a cmake build.
2018-01-17 02:00:09 +08:00
AC_SUBST([ISCMAKE], [])
2019-03-12 22:55:30 +08:00
2020-10-17 05:04:51 +08:00
# Provide conditional to temporarily suppress tests and such
AM_CONDITIONAL([AX_IGNORE], [test xno = xyes])
2019-03-12 22:55:30 +08:00
AC_MSG_NOTICE([generating header files and makefiles])
AC_CONFIG_FILES(test_common.sh:test_common.in)
2018-01-17 02:00:09 +08:00
AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in)
2021-09-03 07:04:26 +08:00
AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in)
Support installation of filters into user-specified location
re: https://github.com/Unidata/netcdf-c/issues/2294
Ed Hartnett suggested that the netcdf library installation process
be extended to install the standard filters into a user specified
location. The user can then set HDF5_PLUGIN_PATH to that location.
This PR provides that capability using:
````
configure option: --with-plugin-dir=<absolute directory path>
cmake option: -DPLUGIN_INSTALL_DIR=<absolute directory path>
````
Currently, the following plugins are always installed, if
available: bzip2, zstd, blosc.
If NCZarr is enabled, then additional plugins are installed:
fletcher32, shuffle, deflate, szip.
Additionally, the necessary codec support is installed
for each of the above filters that is installed.
## Changes:
1. Cleanup handling of built-in bzip2.
2. Add documentation to docs/filters.md
3. Re-factor the NCZarr codec libraries
4. Add a test, although it can only be exercised after
the library is installed, so it cannot be used during
normal testing.
5. Cleanup use of HDF5_PLUGIN_PATH in the filter test cases.
2022-04-30 04:31:55 +08:00
AC_CONFIG_FILES(plugins/findplugin.sh:nc_test4/findplugin.in)
2018-03-17 01:46:18 +08:00
AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in)
2018-09-05 01:27:47 +08:00
AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in)
2022-05-15 06:05:48 +08:00
AC_CONFIG_FILES([nc_test/run_pnetcdf_tests.sh:nc_test/run_pnetcdf_tests.sh.in],[chmod ugo+x nc_test/run_pnetcdf_tests.sh])
2018-09-05 01:27:47 +08:00
AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)
2020-01-01 06:42:58 +08:00
AC_CONFIG_FILES(dap4_test/pingurl4.c:ncdap_test/pingurl.c)
2020-02-28 05:06:45 +08:00
AC_CONFIG_FILES([h5_test/run_par_tests.sh], [chmod ugo+x h5_test/run_par_tests.sh])
2018-08-17 01:43:01 +08:00
AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh])
2019-03-17 22:14:27 +08:00
AC_CONFIG_FILES([nc_perf/run_par_bm_test.sh], [chmod ugo+x nc_perf/run_par_bm_test.sh])
2020-07-01 00:40:05 +08:00
AC_CONFIG_FILES([nc_perf/run_gfs_test.sh], [chmod ugo+x nc_perf/run_gfs_test.sh])
2021-09-03 07:04:26 +08:00
AC_CONFIG_FILES(nczarr_test/timer_utils.h:unit_test/timer_utils.h)
AC_CONFIG_FILES(nczarr_test/timer_utils.c:unit_test/timer_utils.c)
AC_CONFIG_FILES(nczarr_test/test_filter.c:nc_test4/test_filter.c)
AC_CONFIG_FILES(nczarr_test/test_filter_misc.c:nc_test4/test_filter_misc.c)
AC_CONFIG_FILES(nczarr_test/tst_multifilter.c:nc_test4/tst_multifilter.c)
AC_CONFIG_FILES(nczarr_test/test_filter_repeat.c:nc_test4/test_filter_repeat.c)
AC_CONFIG_FILES(nczarr_test/test_filter_order.c:nc_test4/test_filter_order.c)
2019-03-12 23:00:31 +08:00
AC_CONFIG_FILES([examples/C/run_par_test.sh], [chmod ugo+x examples/C/run_par_test.sh])
2018-08-17 01:43:01 +08:00
AC_CONFIG_FILES([nc-config], [chmod 755 nc-config])
2010-06-03 21:25:25 +08:00
AC_CONFIG_FILES([Makefile
netcdf.pc
2018-05-13 22:38:21 +08:00
libnetcdf.settings
postinstall.sh
include/netcdf_meta.h
2021-02-01 12:40:08 +08:00
include/netcdf_dispatch.h
2010-06-04 04:22:55 +08:00
include/Makefile
2014-09-06 01:09:46 +08:00
h5_test/Makefile
2018-03-05 18:45:18 +08:00
hdf4_test/Makefile
2010-06-03 21:25:25 +08:00
libsrc/Makefile
libsrc4/Makefile
2018-05-09 01:58:01 +08:00
libhdf5/Makefile
2015-08-15 10:38:30 +08:00
libsrcp/Makefile
2010-06-03 21:25:25 +08:00
ncdump/Makefile
ncgen3/Makefile
ncgen/Makefile
examples/Makefile
examples/C/Makefile
examples/CDL/Makefile
2012-08-01 04:34:13 +08:00
oc2/Makefile
2012-06-13 05:50:02 +08:00
libdap2/Makefile
2017-03-09 08:01:10 +08:00
libdap4/Makefile
2018-02-08 21:20:58 +08:00
libhdf4/Makefile
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
libnczarr/Makefile
2021-09-03 07:04:26 +08:00
libncpoco/Makefile
2021-11-02 12:37:05 +08:00
libncxml/Makefile
2012-06-13 05:50:02 +08:00
libdispatch/Makefile
liblib/Makefile
2014-09-19 08:26:06 +08:00
ncdump/cdl/Makefile
ncdump/expected/Makefile
2017-03-09 08:01:10 +08:00
docs/Makefile
docs/images/Makefile
2019-08-09 23:31:24 +08:00
unit_test/Makefile
2017-03-09 08:01:10 +08:00
nctest/Makefile
nc_test4/Makefile
2019-03-17 22:03:27 +08:00
nc_perf/Makefile
2017-03-09 08:01:10 +08:00
nc_test/Makefile
2012-06-13 05:50:02 +08:00
ncdap_test/Makefile
ncdap_test/testdata3/Makefile
ncdap_test/expected3/Makefile
ncdap_test/expectremote3/Makefile
2017-03-09 08:01:10 +08:00
dap4_test/Makefile
2018-04-22 10:10:47 +08:00
plugins/Makefile
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
nczarr_test/Makefile
2018-08-17 01:43:01 +08:00
])
2010-06-03 21:25:25 +08:00
AC_OUTPUT()
2014-05-31 05:30:58 +08:00
cat libnetcdf.settings
