2018-12-07 05:29:57 +08:00
|
|
|
# Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
|
|
|
|
|
# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
|
|
|
|
|
# 2015, 2016, 2017, 2018
|
|
|
|
|
# University Corporation for Atmospheric Research/Unidata.
|
|
|
|
|
|
|
|
|
|
# See netcdf-c/COPYRIGHT file for more info.
|
2024-02-07 23:53:45 +08:00
|
|
|
add_library(dispatch OBJECT)
|
|
|
|
|
|
2024-02-08 00:06:46 +08:00
|
|
|
target_sources(dispatch
|
|
|
|
|
PRIVATE
|
|
|
|
|
dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dpathmgr.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c
|
|
|
|
|
daux.c dinstance.c dinstance_intern.c
|
|
|
|
|
dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c ncjson.c ds3util.c dparallel.c dmissing.c
|
2024-02-07 23:53:45 +08:00
|
|
|
)
|
2022-04-02 12:11:28 +08:00
|
|
|
|
2024-03-01 01:36:47 +08:00
|
|
|
if (ENABLE_DLL)
|
|
|
|
|
target_compile_definitions(dispatch PRIVATE DLL_NETCDF DLL_EXPORT)
|
|
|
|
|
endif()
|
2022-04-02 12:11:28 +08:00
|
|
|
|
2024-01-23 04:37:35 +08:00
|
|
|
##
|
|
|
|
|
# Turn off inclusion of particular files when using the cmake-native
|
|
|
|
|
# option to turn on Unity Builds.
|
|
|
|
|
#
|
|
|
|
|
# For more information, see:
|
|
|
|
|
# * https://github.com/Unidata/netcdf-c/pull/2839/
|
|
|
|
|
# * https://cmake.org/cmake/help/latest/prop_tgt/UNITY_BUILD.html
|
|
|
|
|
# * https://cmake.org/cmake/help/latest/prop_tgt/UNITY_BUILD_MODE.html#prop_tgt:UNITY_BUILD_MODE
|
|
|
|
|
##
|
2024-01-12 17:18:29 +08:00
|
|
|
set_property(SOURCE dinstance_intern.c dinstance.c dvarput.c
|
|
|
|
|
PROPERTY
|
|
|
|
|
SKIP_UNITY_BUILD_INCLUSION ON)
|
|
|
|
|
|
2019-07-21 03:59:40 +08:00
|
|
|
# Netcdf-4 only functions. Must be defined even if not used
|
2024-02-07 23:53:45 +08:00
|
|
|
target_sources(dispatch
|
2024-02-08 00:06:46 +08:00
|
|
|
PRIVATE
|
|
|
|
|
dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c
|
2024-02-07 23:53:45 +08:00
|
|
|
)
|
2012-07-18 04:50:43 +08:00
|
|
|
|
2024-01-27 06:04:15 +08:00
|
|
|
if(BUILD_V2)
|
2024-02-07 23:53:45 +08:00
|
|
|
target_sources(dispatch
|
2024-02-08 00:06:46 +08:00
|
|
|
PRIVATE
|
|
|
|
|
dv2i.c
|
2024-02-07 23:53:45 +08:00
|
|
|
)
|
2024-01-27 06:04:15 +08:00
|
|
|
endif(BUILD_V2)
|
2012-07-18 04:50:43 +08:00
|
|
|
|
2024-01-27 06:04:15 +08:00
|
|
|
if(ENABLE_BYTERANGE)
|
2024-02-07 23:53:45 +08:00
|
|
|
target_sources(dispatch
|
2024-02-08 00:06:46 +08:00
|
|
|
PRIVATE
|
|
|
|
|
dhttp.c
|
2024-02-07 23:53:45 +08:00
|
|
|
)
|
2019-02-25 07:54:13 +08:00
|
|
|
ENDIF(ENABLE_BYTERANGE)
|
Provide byte-range reading of remote datasets
re: issue https://github.com/Unidata/netcdf-c/issues/1251
Assume that you have the URL to a remote dataset
which is a normal netcdf-3 or netcdf-4 file.
This PR allows the netcdf-c to read that dataset's
contents as a netcdf file using HTTP byte ranges
if the remote server supports byte-range access.
Originally, this PR was set up to access Amazon S3 objects,
but it can also access other remote datasets such as those
provided by a Thredds server via the HTTPServer access protocol.
It may also work for other kinds of servers.
Note that this is not intended as a true production
capability because, as is known, this kind of access to
can be quite slow. In addition, the byte-range IO drivers
do not currently do any sort of optimization or caching.
An additional goal here is to gain some experience with
the Amazon S3 REST protocol.
This architecture and its use documented in
the file docs/byterange.dox.
There are currently two test cases:
1. nc_test/tst_s3raw.c - this does a simple open, check format, close cycle
for a remote netcdf-3 file and a remote netcdf-4 file.
2. nc_test/test_s3raw.sh - this uses ncdump to investigate some remote
datasets.
This PR also incorporates significantly changed model inference code
(see the superceded PR https://github.com/Unidata/netcdf-c/pull/1259).
1. It centralizes the code that infers the dispatcher.
2. It adds support for byte-range URLs
Other changes:
1. NC_HDF5_finalize was not being properly called by nc_finalize().
2. Fix minor bug in ncgen3.l
3. fix memory leak in nc4info.c
4. add code to walk the .daprc triples and to replace protocol=
fragment tag with a more general mode= tag.
Final Note:
Th inference code is still way too complicated. We need to move
to the validfile() model used by netcdf Java, where each
dispatcher is asked if it can process the file. This decentralizes
the inference code. This will be done after all the major new
dispatchers (PIO, Zarr, etc) have been implemented.
2019-01-02 09:27:36 +08:00
|
|
|
|
2023-04-26 07:15:06 +08:00
|
|
|
IF(ENABLE_S3)
|
2024-01-27 06:04:15 +08:00
|
|
|
if(ENABLE_S3_INTERNAL)
|
2024-02-07 23:53:45 +08:00
|
|
|
target_sources(dispatch
|
2024-02-08 00:06:46 +08:00
|
|
|
PRIVATE
|
|
|
|
|
ncs3sdk_h5.c nch5s3comms.c nch5s3comms.h nccurl_sha256.c nccurl_sha256.h nccurl_hmac.c nccurl_hmac.h nccurl_setup.h
|
2024-02-07 23:53:45 +08:00
|
|
|
)
|
2024-01-27 06:04:15 +08:00
|
|
|
else()
|
2024-02-07 23:53:45 +08:00
|
|
|
target_sources(dispatch
|
2024-02-08 00:06:46 +08:00
|
|
|
PRIVATE
|
|
|
|
|
ncs3sdk_aws.cpp awsincludes.h
|
2024-02-07 23:53:45 +08:00
|
|
|
)
|
2024-01-27 06:04:15 +08:00
|
|
|
endif()
|
|
|
|
|
endif()
|
2021-10-30 10:06:37 +08:00
|
|
|
|
2024-01-27 06:04:15 +08:00
|
|
|
if(REGEDIT)
|
2024-02-07 23:53:45 +08:00
|
|
|
target_sources(dispatch
|
2024-02-08 00:06:46 +08:00
|
|
|
PRIVATE
|
|
|
|
|
dreg.c
|
2024-02-07 23:53:45 +08:00
|
|
|
)
|
2024-01-27 06:04:15 +08:00
|
|
|
endif(REGEDIT)
|
2021-12-24 13:18:56 +08:00
|
|
|
|
2024-01-27 06:04:15 +08:00
|
|
|
|
|
|
|
|
if (USE_HDF5)
|
2024-02-16 18:51:20 +08:00
|
|
|
target_link_libraries(dispatch PRIVATE HDF5::HDF5)
|
2024-01-27 06:04:15 +08:00
|
|
|
endif(USE_HDF5)
|
|
|
|
|
|
|
|
|
|
if(STATUS_PARALLEL)
|
2023-09-29 22:42:19 +08:00
|
|
|
target_link_libraries(dispatch PUBLIC MPI::MPI_C)
|
2024-01-27 06:04:15 +08:00
|
|
|
endif(STATUS_PARALLEL)
|
2012-07-18 04:50:43 +08:00
|
|
|
|
2024-01-18 01:17:59 +08:00
|
|
|
if (ENABLE_PNETCDF)
|
2024-01-24 18:07:46 +08:00
|
|
|
target_link_libraries(dispatch PUBLIC PNETCDF::PNETCDF)
|
2024-01-18 01:17:59 +08:00
|
|
|
endif()
|
|
|
|
|
|
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
|
|
|
IF(ENABLE_NCZARR)
|
|
|
|
|
target_include_directories(dispatch PUBLIC ../libnczarr)
|
2024-01-27 06:04:15 +08:00
|
|
|
endif(ENABLE_NCZARR)
|
Enhance/Fix filter support
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214
The primary change is to support so-called "standard filters".
A standard filter is one that is defined by the following
netcdf-c API:
````
int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params);
int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params);
````
So for example, zstandard would be a standard filter by defining
the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*.
In order to define these functions, we need a new dispatch function:
````
int nc_inq_filter_avail(int ncid, unsigned filterid);
````
This function, combined with the existing filter API can be used
to implement arbitrary standard filters using a simple code pattern.
Note that I would have preferred that this function return a list
of all available filters, but HDF5 does not support that functionality.
So this PR implements the dispatch function and implements
the following standard functions:
+ bzip2
+ zstandard
+ blosc
Specific test cases are also provided for HDF5 and NCZarr.
Over time, other specific standard filters will be defined.
## Primary Changes
* Add nc_inq_filter_avail() to netcdf-c API.
* Add standard filter implementations to test use of *nc_inq_filter_avail*.
* Bump the dispatch table version number and add to all the relevant
dispatch tables (libsrc, libsrcp, etc).
* Create a program to invoke nc_inq_filter_avail so that it is accessible
to shell scripts.
* Cleanup szip support to properly support szip
when HDF5 is disabled. This involves detecting
libsz separately from testing if HDF5 supports szip.
* Integrate shuffle and fletcher32 into the existing
filter API. This means that, for example, nc_def_var_fletcher32
is now a wrapper around nc_def_var_filter.
* Extend the Codec defaulting to allow multiple default shared libraries.
## Misc. Changes
* Modify configure.ac/CMakeLists.txt to look for the relevant
libraries implementing standard filters.
* Modify libnetcdf.settings to list available standard filters
(including deflate and szip).
* Add CMake test modules to locate libbz2 and libzstd.
* Cleanup the HDF5 memory manager function use in the plugins.
* remove unused file include//ncfilter.h
* remove tests for the HDF5 memory operations e.g. H5allocate_memory.
* Add flag to ncdump to force use of _Filter instead of _Deflate
or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
|
|
|
|
2024-01-27 06:04:15 +08:00
|
|
|
if(ENABLE_S3)
|
|
|
|
|
if(ENABLE_S3_AWS)
|
2023-04-26 07:15:06 +08:00
|
|
|
target_include_directories(dispatch PUBLIC ${AWSSDK_INCLUDE_DIRS})
|
2024-01-27 06:04:15 +08:00
|
|
|
if(NOT MSVC)
|
2023-04-26 07:15:06 +08:00
|
|
|
target_compile_features(dispatch PUBLIC cxx_std_11)
|
2024-01-27 06:04:15 +08:00
|
|
|
endif()
|
|
|
|
|
else()
|
2023-04-26 07:15:06 +08:00
|
|
|
target_include_directories(dispatch PUBLIC ../libncxml)
|
2024-01-27 06:04:15 +08:00
|
|
|
endif()
|
|
|
|
|
endif()
|
2021-10-30 10:06:37 +08:00
|
|
|
|
2024-01-27 06:04:15 +08:00
|
|
|
if(ENABLE_TESTS)
|
2023-10-06 19:48:00 +08:00
|
|
|
BUILD_BIN_TEST(ncrandom)
|
2024-01-27 06:04:15 +08:00
|
|
|
endif()
|
2023-04-26 07:15:06 +08:00
|
|
|
|
2024-01-27 06:04:15 +08:00
|
|
|
file(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.c)
|
|
|
|
|
set(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am)
|
2016-04-07 10:38:51 +08:00
|
|
|
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")
|
