netcdf-c/CMakeLists.txt

1844 lines
61 KiB
CMake
Raw Normal View History

## This is a CMake file, part of Unidata's netCDF package.
2018-03-27 01:53:31 +08:00
# Copyright 2012-2018, see the COPYRIGHT file for more information.
2015-10-15 01:08:54 +08:00
#
##################################
# Set Project Properties
##################################
2024-02-23 23:02:15 +08:00
cmake_minimum_required(VERSION 3.20.0)
#Project Name
project(netCDF
LANGUAGES C CXX
HOMEPAGE_URL "https://www.unidata.ucar.edu/software/netcdf/"
DESCRIPTION "NetCDF is a set of software libraries and machine-independent data formats that support the creation, access, and sharing of array-oriented scientific data."
2024-01-11 06:16:49 +08:00
VERSION 4.9.3
)
#Add custom CMake Module
set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules/;${PROJECT_SOURCE_DIR}/cmake"
CACHE INTERNAL "Location of our custom CMake modules.")
set(PACKAGE "netCDF" CACHE STRING "")
2024-01-18 05:17:50 +08:00
include(netcdf_functions_macros)
# Backport of built-in `PROJECT_IS_TOP_LEVEL` from CMake 3.21
if (NOT DEFINED NETCDF_IS_TOP_LEVEL)
set(NETCDF_IS_TOP_LEVEL OFF)
if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
set(NETCDF_IS_TOP_LEVEL ON)
endif ()
endif ()
################################
# The target
################################
##
# Default building shared libraries.
# BUILD_SHARED_LIBS is provided by/used by
# CMake directly.
##
option(BUILD_SHARED_LIBS "Configure netCDF as a shared library." ON)
if(BUILD_SHARED_LIBS)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
endif()
add_library(netcdf)
add_library(netCDF::netcdf ALIAS netcdf)
#####
# Version Info:
#
# Release Version
# Library Version
# SO Version
#
# SO Version is computed from library version. See:
# http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning
#####
2024-01-18 06:07:22 +08:00
set(NC_VERSION_NOTE "-development")
set(netCDF_VERSION ${PROJECT_VERSION}${NC_VERSION_NOTE})
set(VERSION ${netCDF_VERSION})
set(NC_VERSION ${netCDF_VERSION})
set(PACKAGE_VERSION ${VERSION})
# These values should match those in configure.ac
2024-01-18 06:07:22 +08:00
set(netCDF_LIB_VERSION 19)
set(netCDF_SO_VERSION 19)
# Version of the dispatch table. This must match the value in
# configure.ac.
2024-01-18 06:07:22 +08:00
set(NC_DISPATCH_VERSION 5)
# Get system configuration, Use it to determine osname, os release, cpu. These
# will be used when committing to CDash.
find_program(UNAME NAMES uname)
2024-01-18 06:07:22 +08:00
if(UNAME)
getuname(osname -s)
getuname(osrel -r)
getuname(cpu -m)
set(TMP_BUILDNAME "${osname}-${osrel}-${cpu}")
2024-01-18 06:07:22 +08:00
endif()
Improve UTF8 Support On Windows re: Issue https://github.com/Unidata/netcdf-c/issues/2190 The primary purpose of this PR is to improve the utf8 support for windows. This is persuant to a change in Windows that supports utf8 natively (almost). The almost means that it is still utf16 internally and the set of characters representable by utf8 is larger than those representable by utf16. This leaves open the question in the Issue about handling the Windows 1252 character set. This required the following changes: 1. Test the Windows build and major version in order to see if native utf8 is supported. 2. If native utf8 is supported, Modify dpathmgr.c to call the 8-bit version of the windows fopen() and open() functions. 3. In support of this, programs that use XGetOpt (Windows versions) need to get the command line as utf8 and then parse to arc+argv as utf8. This requires using a homegrown command line parser named XCommandLineToArgvA. 4. Add a utility program called "acpget" that prints out the current Windows code page and locale. Additionally, some technical debt was cleaned up as follows: 1. Unify all the places which attempt to read all or a part of a file into the dutil.c#NC_readfile code. 2. Similary unify all the code that creates temp files into dutil.c#NC_mktmp code. 3. Convert almost all remaining calls to fopen() and open() to NCfopen() and NCopen3(). This is to ensure that path management is used consistently. This touches a number of files. 4. extern->EXTERNL as needed to get it to work under Windows.
2022-02-09 11:53:30 +08:00
# Define some Platforms
if(osname MATCHES "CYGWIN.*")
2024-01-18 06:07:22 +08:00
set(ISCYGWIN yes)
Improve UTF8 Support On Windows re: Issue https://github.com/Unidata/netcdf-c/issues/2190 The primary purpose of this PR is to improve the utf8 support for windows. This is persuant to a change in Windows that supports utf8 natively (almost). The almost means that it is still utf16 internally and the set of characters representable by utf8 is larger than those representable by utf16. This leaves open the question in the Issue about handling the Windows 1252 character set. This required the following changes: 1. Test the Windows build and major version in order to see if native utf8 is supported. 2. If native utf8 is supported, Modify dpathmgr.c to call the 8-bit version of the windows fopen() and open() functions. 3. In support of this, programs that use XGetOpt (Windows versions) need to get the command line as utf8 and then parse to arc+argv as utf8. This requires using a homegrown command line parser named XCommandLineToArgvA. 4. Add a utility program called "acpget" that prints out the current Windows code page and locale. Additionally, some technical debt was cleaned up as follows: 1. Unify all the places which attempt to read all or a part of a file into the dutil.c#NC_readfile code. 2. Similary unify all the code that creates temp files into dutil.c#NC_mktmp code. 3. Convert almost all remaining calls to fopen() and open() to NCfopen() and NCopen3(). This is to ensure that path management is used consistently. This touches a number of files. 4. extern->EXTERNL as needed to get it to work under Windows.
2022-02-09 11:53:30 +08:00
endif()
2023-08-31 00:17:00 +08:00
if(osname MATCHES "Darwin.*")
2024-01-18 06:07:22 +08:00
set(ISOSX yes)
Improve UTF8 Support On Windows re: Issue https://github.com/Unidata/netcdf-c/issues/2190 The primary purpose of this PR is to improve the utf8 support for windows. This is persuant to a change in Windows that supports utf8 natively (almost). The almost means that it is still utf16 internally and the set of characters representable by utf8 is larger than those representable by utf16. This leaves open the question in the Issue about handling the Windows 1252 character set. This required the following changes: 1. Test the Windows build and major version in order to see if native utf8 is supported. 2. If native utf8 is supported, Modify dpathmgr.c to call the 8-bit version of the windows fopen() and open() functions. 3. In support of this, programs that use XGetOpt (Windows versions) need to get the command line as utf8 and then parse to arc+argv as utf8. This requires using a homegrown command line parser named XCommandLineToArgvA. 4. Add a utility program called "acpget" that prints out the current Windows code page and locale. Additionally, some technical debt was cleaned up as follows: 1. Unify all the places which attempt to read all or a part of a file into the dutil.c#NC_readfile code. 2. Similary unify all the code that creates temp files into dutil.c#NC_mktmp code. 3. Convert almost all remaining calls to fopen() and open() to NCfopen() and NCopen3(). This is to ensure that path management is used consistently. This touches a number of files. 4. extern->EXTERNL as needed to get it to work under Windows.
2022-02-09 11:53:30 +08:00
endif()
if(MSVC)
2024-01-18 06:07:22 +08:00
set(ISMSVC yes)
Improve UTF8 Support On Windows re: Issue https://github.com/Unidata/netcdf-c/issues/2190 The primary purpose of this PR is to improve the utf8 support for windows. This is persuant to a change in Windows that supports utf8 natively (almost). The almost means that it is still utf16 internally and the set of characters representable by utf8 is larger than those representable by utf16. This leaves open the question in the Issue about handling the Windows 1252 character set. This required the following changes: 1. Test the Windows build and major version in order to see if native utf8 is supported. 2. If native utf8 is supported, Modify dpathmgr.c to call the 8-bit version of the windows fopen() and open() functions. 3. In support of this, programs that use XGetOpt (Windows versions) need to get the command line as utf8 and then parse to arc+argv as utf8. This requires using a homegrown command line parser named XCommandLineToArgvA. 4. Add a utility program called "acpget" that prints out the current Windows code page and locale. Additionally, some technical debt was cleaned up as follows: 1. Unify all the places which attempt to read all or a part of a file into the dutil.c#NC_readfile code. 2. Similary unify all the code that creates temp files into dutil.c#NC_mktmp code. 3. Convert almost all remaining calls to fopen() and open() to NCfopen() and NCopen3(). This is to ensure that path management is used consistently. This touches a number of files. 4. extern->EXTERNL as needed to get it to work under Windows.
2022-02-09 11:53:30 +08:00
endif()
if(osname MATCHES "MINGW.*" OR osname MATCHES "MSYS.*")
2024-01-18 06:07:22 +08:00
set(ISMINGW yes)
set(MINGW yes)
Improve UTF8 Support On Windows re: Issue https://github.com/Unidata/netcdf-c/issues/2190 The primary purpose of this PR is to improve the utf8 support for windows. This is persuant to a change in Windows that supports utf8 natively (almost). The almost means that it is still utf16 internally and the set of characters representable by utf8 is larger than those representable by utf16. This leaves open the question in the Issue about handling the Windows 1252 character set. This required the following changes: 1. Test the Windows build and major version in order to see if native utf8 is supported. 2. If native utf8 is supported, Modify dpathmgr.c to call the 8-bit version of the windows fopen() and open() functions. 3. In support of this, programs that use XGetOpt (Windows versions) need to get the command line as utf8 and then parse to arc+argv as utf8. This requires using a homegrown command line parser named XCommandLineToArgvA. 4. Add a utility program called "acpget" that prints out the current Windows code page and locale. Additionally, some technical debt was cleaned up as follows: 1. Unify all the places which attempt to read all or a part of a file into the dutil.c#NC_readfile code. 2. Similary unify all the code that creates temp files into dutil.c#NC_mktmp code. 3. Convert almost all remaining calls to fopen() and open() to NCfopen() and NCopen3(). This is to ensure that path management is used consistently. This touches a number of files. 4. extern->EXTERNL as needed to get it to work under Windows.
2022-02-09 11:53:30 +08:00
endif()
###
# Allow for some customization of the buildname.
# This will make it easier to identify different builds,
# based on values passed from command line/shell scripts.
#
# For ctest scripts, we can use CTEST_BUILD_NAME.
###
2024-01-18 06:07:22 +08:00
set(BUILDNAME_PREFIX "" CACHE STRING "")
set(BUILDNAME_SUFFIX "" CACHE STRING "")
2024-01-18 06:07:22 +08:00
if(BUILDNAME_PREFIX)
set(TMP_BUILDNAME "${BUILDNAME_PREFIX}-${TMP_BUILDNAME}")
endif()
2024-01-18 06:07:22 +08:00
if(BUILDNAME_SUFFIX)
set(TMP_BUILDNAME "${TMP_BUILDNAME}-${BUILDNAME_SUFFIX}")
endif()
2024-01-18 06:07:22 +08:00
if(NOT BUILDNAME)
set(BUILDNAME "${TMP_BUILDNAME}" CACHE STRING "Build name variable for CDash")
endif()
###
# End BUILDNAME customization.
###
# For CMAKE_INSTALL_LIBDIR
2024-01-18 06:07:22 +08:00
include(GNUInstallDirs)
2012-10-03 04:56:46 +08:00
2024-01-18 06:07:22 +08:00
if(MSVC)
set(GLOBAL PROPERTY USE_FOLDERS ON)
target_compile_options(netcdf PUBLIC "/utf-8")
2024-01-18 06:07:22 +08:00
endif()
2012-10-03 04:56:46 +08:00
# auto-configure style checks, other CMake modules.
2024-01-18 06:07:22 +08:00
include(CheckLibraryExists)
include(CheckIncludeFile)
include(CheckIncludeFiles)
include(CheckTypeSize)
include(CheckFunctionExists)
include(CheckCXXSourceCompiles)
include(CheckCSourceCompiles)
include(TestBigEndian)
include(CheckSymbolExists)
include(GetPrerequisites)
include(CheckCCompilerFlag)
# A check to see if the system is big endian
2017-02-09 05:51:41 +08:00
TEST_BIG_ENDIAN(BIGENDIAN)
2024-01-18 06:07:22 +08:00
if(${BIGENDIAN})
set(WORDS_BIGENDIAN "1")
endif(${BIGENDIAN})
2017-02-09 05:51:41 +08:00
# Define a function to convert various true or false values
# to either TRUE|FALSE (uppercase).
# If value is not a recognized boolean, then return NOTFOUND
#1, ON, YES, TRUE, Y,
#0, OFF, NO, FALSE, N, IGNORE, NOTFOUND -NOTFOUND ""
2024-01-18 06:07:22 +08:00
set(TRUELIST "on;yes;y;true")
set(FALSELIST "off;no;n;false;0;ignore;notfound")
# Set the build type.
2024-01-18 06:07:22 +08:00
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE DEBUG CACHE STRING "Choose the type of build, options are: None, Debug, Release."
FORCE)
2024-01-18 06:07:22 +08:00
endif()
# Set build type uppercase
2024-01-18 06:07:22 +08:00
string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
# Determine the configure date.
2014-12-16 01:56:14 +08:00
2024-01-18 06:07:22 +08:00
if(DEFINED ENV{SOURCE_DATE_EPOCH})
execute_process(
COMMAND "date" "-u" "-d" "@$ENV{SOURCE_DATE_EPOCH}"
OUTPUT_VARIABLE CONFIG_DATE
)
2024-01-18 06:07:22 +08:00
else()
execute_process(
COMMAND date
OUTPUT_VARIABLE CONFIG_DATE
)
2024-01-18 06:07:22 +08:00
endif()
if(CONFIG_DATE)
2014-12-16 01:56:14 +08:00
string(STRIP ${CONFIG_DATE} CONFIG_DATE)
2024-01-18 06:07:22 +08:00
endif()
2014-09-30 02:04:35 +08:00
##
# Allow for extra dependencies.
##
2024-01-18 06:07:22 +08:00
set(EXTRA_DEPS "")
2014-09-30 02:04:35 +08:00
################################
# End Project Properties
################################
################################
# Set CTest Properties
################################
2024-01-18 06:07:22 +08:00
enable_testing()
include(CTest)
# Set Memory test program for non-MSVC based builds.
# Assume valgrind for now.
2024-01-18 06:07:22 +08:00
if((NOT MSVC) AND (NOT MINGW) AND (NOT ISCYGWIN))
set(CTEST_MEMORYCHECK_COMMAND valgrind CACHE STRING "")
endif()
# Set variable to define the build type.
2024-01-18 06:07:22 +08:00
include(GenerateExportHeader)
################################
# End CTest Properties
################################
################################
# Compiler and Linker Configuration
################################
2024-01-18 06:07:22 +08:00
option(NC_FIND_SHARED_LIBS "Find dynamically-built versions of dependent libraries" ${BUILD_SHARED_LIBS})
##
# We've had a request to allow for non-versioned shared libraries.
# This seems reasonable enough to accommodate. See
# https://github.com/Unidata/netcdf-c/issues/228 for more info.
##
2024-01-18 06:07:22 +08:00
option(ENABLE_SHARED_LIBRARY_VERSION "Encode the library SO version in the file name of the generated library file." ON)
# Set some default linux gcc & apple compiler options for
# debug builds.
2024-01-18 06:07:22 +08:00
if(CMAKE_COMPILER_IS_GNUCC OR APPLE)
option(NETCDF_ENABLE_COVERAGE_TESTS "Enable compiler flags needed to perform coverage tests." OFF)
option(NETCDF_ENABLE_CONVERSION_WARNINGS "Enable warnings for implicit conversion from 64 to 32-bit datatypes." ON)
2024-01-18 06:07:22 +08:00
option(ENABLE_LARGE_FILE_TESTS "Enable large file tests." OFF)
# Debugging flags
2024-01-18 06:07:22 +08:00
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Wall")
# Check to see if -Wl,--no-undefined is supported.
CHECK_C_LINKER_FLAG("-Wl,--no-undefined" LIBTOOL_HAS_NO_UNDEFINED)
2024-01-18 06:07:22 +08:00
if(LIBTOOL_HAS_NO_UNDEFINED)
set(CMAKE_SHARED_LINKER_FLAGS_DEBUG "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -Wl,--no-undefined")
endif()
set(CMAKE_REQUIRED_FLAGS "${TMP_CMAKE_REQUIRED_FLAGS}")
# Coverage tests need to have optimization turned off.
if(NETCDF_ENABLE_COVERAGE_TESTS)
2024-01-18 06:07:22 +08:00
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -coverage -fprofile-arcs -ftest-coverage")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -coverage -fprofile-arcs -ftest-coverage")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
message(STATUS "Coverage Tests: On.")
endif()
This PR adds EXPERIMENTAL support for accessing data in the cloud using a variant of the Zarr protocol and storage format. This enhancement is generically referred to as "NCZarr". The data model supported by NCZarr is netcdf-4 minus the user-defined types and the String type. In this sense it is similar to the CDF-5 data model. More detailed information about enabling and using NCZarr is described in the document NUG/nczarr.md and in a [Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in). WARNING: this code has had limited testing, so do use this version for production work. Also, performance improvements are ongoing. Note especially the following platform matrix of successful tests: Platform | Build System | S3 support ------------------------------------ Linux+gcc | Automake | yes Linux+gcc | CMake | yes Visual Studio | CMake | no Additionally, and as a consequence of the addition of NCZarr, major changes have been made to the Filter API. NOTE: NCZarr does not yet support filters, but these changes are enablers for that support in the future. Note that it is possible (probable?) that there will be some accidental reversions if the changes here did not correctly mimic the existing filter testing. In any case, previously filter ids and parameters were of type unsigned int. In order to support the more general zarr filter model, this was all converted to char*. The old HDF5-specific, unsigned int operations are still supported but they are wrappers around the new, char* based nc_filterx_XXX functions. This entailed at least the following changes: 1. Added the files libdispatch/dfilterx.c and include/ncfilter.h 2. Some filterx utilities have been moved to libdispatch/daux.c 3. A new entry, "filter_actions" was added to the NCDispatch table and the version bumped. 4. An overly complex set of structs was created to support funnelling all of the filterx operations thru a single dispatch "filter_actions" entry. 5. Move common code to from libhdf5 to libsrc4 so that it is accessible to nczarr. Changes directly related to Zarr: 1. Modified CMakeList.txt and configure.ac to support both C and C++ -- this is in support of S3 support via the awd-sdk libraries. 2. Define a size64_t type to support nczarr. 3. More reworking of libdispatch/dinfermodel.c to support zarr and to regularize the structure of the fragments section of a URL. Changes not directly related to Zarr: 1. Make client-side filter registration be conditional, with default off. 2. Hack include/nc4internal.h to make some flags added by Ed be unique: e.g. NC_CREAT, NC_INDEF, etc. 3. cleanup include/nchttp.h and libdispatch/dhttp.c. 4. Misc. changes to support compiling under Visual Studio including: * Better testing under windows for dirent.h and opendir and closedir. 5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags and to centralize error reporting. 6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them. 7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible. Changes Left TO-DO: 1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
# Warnings for 64-to-32 bit conversions.
if(NETCDF_ENABLE_CONVERSION_WARNINGS)
CHECK_C_COMPILER_FLAG(-Wconversion CC_HAS_WCONVERSION)
CHECK_C_COMPILER_FLAG(-Wshorten-64-to-32 CC_HAS_SHORTEN_64_32)
2024-01-18 06:07:22 +08:00
if(CC_HAS_SHORTEN_64_32)
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Wshorten-64-to-32")
endif()
if(CC_HAS_WCONVERSION)
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Wconversion")
endif()
endif(NETCDF_ENABLE_CONVERSION_WARNINGS)
2024-01-18 06:07:22 +08:00
endif(CMAKE_COMPILER_IS_GNUCC OR APPLE)
# End default linux gcc & apple compiler options.
# Use relative pathnames in __FILE__ macros on MINGW:
2024-01-18 06:07:22 +08:00
if(MINGW)
CHECK_C_COMPILER_FLAG("-fmacro-prefix-map='${CMAKE_SOURCE_DIR}'=." CC_HAS_MACRO_PREFIX_MAP)
2024-01-18 06:07:22 +08:00
if(CC_HAS_MACRO_PREFIX_MAP)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fmacro-prefix-map='${CMAKE_SOURCE_DIR}'=.")
endif()
endif()
# Suppress CRT Warnings.
2012-11-08 01:34:05 +08:00
# Only necessary for Windows
2024-01-18 06:07:22 +08:00
if(MSVC)
target_compile_definitions(netcdf PRIVATE _CRT_SECURE_NO_WARNINGS)
2024-01-18 06:07:22 +08:00
endif()
# Support ANSI format specifiers for *printf on MINGW:
2024-01-18 06:07:22 +08:00
if(MINGW)
target_compile_definitions(netcdf PRIVATE __USE_MINGW_ANSI_STDIO=1)
2024-01-18 06:07:22 +08:00
endif()
#####
# System inspection checks
#####
2024-01-18 06:07:22 +08:00
include_directories(${CMAKE_CURRENT_BINARY_DIR}/include)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/oc2)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/libsrc)
set(CMAKE_REQUIRED_INCLUDES ${CMAKE_CURRENT_SOURCE_DIR}/libsrc)
################################
# End Compiler Configuration
################################
##
# Configuration for post-install RPath
# Adapted from http://www.cmake.org/Wiki/CMake_RPATH_handling
##
2024-01-18 06:07:22 +08:00
if(NOT WIN32 AND BUILD_SHARED_LIBS)
2014-03-07 23:46:26 +08:00
# use, i.e. don't skip the full RPATH for the build tree
2024-01-18 06:07:22 +08:00
set(CMAKE_SKIP_BUILD_RPATH FALSE)
2014-03-07 23:46:26 +08:00
# when building, don't use the install RPATH already
# (but later on when installing)
2024-01-18 06:07:22 +08:00
set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
2014-03-07 23:46:26 +08:00
if(APPLE)
set(CMAKE_MACOSX_RPATH ON)
endif(APPLE)
2014-03-07 23:46:26 +08:00
# add the automatically determined parts of the RPATH
# which point to directories outside the build tree to the install RPATH
2024-01-18 06:07:22 +08:00
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
2014-03-07 23:46:26 +08:00
# the RPATH to be used when installing,
# but only if it's not a system directory
2024-01-18 06:07:22 +08:00
list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}" isSystemDir)
if("${isSystemDir}" STREQUAL "-1")
set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
endif("${isSystemDir}" STREQUAL "-1")
2024-01-18 06:07:22 +08:00
endif()
##
# End configuration for post-install RPath
##
################################
2012-10-03 04:56:46 +08:00
# Option checks
################################
# Default Cache variables.
2024-01-18 06:07:22 +08:00
set(DEFAULT_CHUNK_SIZE 16777216 CACHE STRING "Default Chunk Cache Size.")
set(DEFAULT_CHUNK_CACHE_SIZE 16777216U CACHE STRING "Default Chunk Cache Size.")
set(DEFAULT_CHUNKS_IN_CACHE 1000 CACHE STRING "Default number of chunks in cache.")
set(DEFAULT_CHUNK_CACHE_PREEMPTION 0.75 CACHE STRING "Default file chunk cache preemption policy (a number between 0 and 1, inclusive.")
# HDF5 default cache size values
2024-01-18 06:07:22 +08:00
set(CHUNK_CACHE_SIZE ${DEFAULT_CHUNK_CACHE_SIZE} CACHE STRING "Default HDF5 Chunk Cache Size.")
set(CHUNK_CACHE_NELEMS ${DEFAULT_CHUNKS_IN_CACHE} CACHE STRING "Default maximum number of elements in cache.")
set(CHUNK_CACHE_PREEMPTION ${DEFAULT_CHUNK_CACHE_PREEMPTION} CACHE STRING "Default file chunk cache preemption policy for HDf5 files(a number between 0 and 1, inclusive.")
2024-01-18 06:07:22 +08:00
set(NETCDF_LIB_NAME "" CACHE STRING "Default name of the netcdf library.")
set(TEMP_LARGE "." CACHE STRING "Where to put large temp files if large file tests are run.")
set(NCPROPERTIES_EXTRA "" CACHE STRING "Specify extra pairs for _NCProperties.")
2024-01-18 06:07:22 +08:00
if(NOT NETCDF_LIB_NAME STREQUAL "")
set(MOD_NETCDF_NAME ON)
endif()
2012-10-03 04:56:46 +08:00
# Set the appropriate compiler/architecture for universal OSX binaries.
2024-01-18 06:07:22 +08:00
if(${CMAKE_SYSTEM_NAME} EQUAL "Darwin")
set(CMAKE_OSX_ARCHITECTURES i386;x86_64)
endif(${CMAKE_SYSTEM_NAME} EQUAL "Darwin")
2012-10-03 04:56:46 +08:00
# Option to use Static Runtimes in MSVC
2024-01-18 06:07:22 +08:00
if(MSVC)
option(NC_USE_STATIC_CRT "Use static CRT Libraries ('\\MT')." OFF)
if(NC_USE_STATIC_CRT)
set(USE_STATIC_CRT ON)
specify_static_crt_flag()
2024-01-18 06:07:22 +08:00
endif()
endif()
# Option to build netCDF Version 2
OPTION (ENABLE_V2_API "Build netCDF Version 2." ON)
2024-01-18 06:07:22 +08:00
set(BUILD_V2 ${ENABLE_V2_API})
if(NOT ENABLE_V2_API)
set(NO_NETCDF_2 ON)
else(NOT ENABLE_V2_API)
set(USE_NETCDF_2 TRUE)
endif(NOT ENABLE_V2_API)
2012-10-03 04:56:46 +08:00
# Option to build utilities
2024-03-19 04:29:24 +08:00
option(NETCDF_BUILD_UTILITIES "Build ncgen, ncgen3, ncdump." ON)
2012-10-03 04:56:46 +08:00
# Option to use MMAP
2024-01-18 06:07:22 +08:00
option(ENABLE_MMAP "Use MMAP." ON)
2013-02-15 04:24:02 +08:00
2012-10-03 04:56:46 +08:00
# Option to use examples.
option(NETCDF_ENABLE_EXAMPLES "Build Examples" ON)
2012-10-03 04:56:46 +08:00
###
# Allow the user to specify libraries
# to link against, similar to automakes 'LIBS' variable.
###
2024-01-18 06:07:22 +08:00
set(NC_EXTRA_DEPS "" CACHE STRING "Additional libraries to link against.")
if(NC_EXTRA_DEPS)
string(REPLACE " " ";" DEPS_LIST ${NC_EXTRA_DEPS})
foreach(_DEP ${DEPS_LIST})
string(REGEX REPLACE "^-l" "" _LIB ${_DEP})
FIND_LIBRARY("${_LIB}_DEP" NAMES "${_LIB}" "lib${_LIB}")
2024-01-18 06:07:22 +08:00
message(${${_LIB}_DEP})
if("${${_LIB}_DEP}" STREQUAL "${_LIB}_DEP-NOTFOUND")
message(FATAL_ERROR "Error finding ${_LIB}.")
else()
message(STATUS "Found ${_LIB}: ${${_LIB}_DEP}")
endif()
set(EXTRA_DEPS ${EXTRA_DEPS} "${${_LIB}_DEP}")
endforeach()
2024-01-18 06:07:22 +08:00
message("Extra deps: ${EXTRA_DEPS}")
list(REMOVE_DUPLICATES EXTRA_DEPS)
set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${EXTRA_DEPS})
endif()
###
# End user-specified dependent libraries.
###
################################
# Format Option checks
################################
# We need to now treat enable-netcdf4 and enable-hdf5 as separate,
# but for back compatability, we need to treat enable-netcdf4
# as equivalent to enable-hdf5.
# We detect this using these rules:
# 1. if NETCDF_ENABLE_HDF5 is off then disable hdf5
# 2. if ENABLE_NETCDF4 is off then disable hdf5
# 3. else enable hdf5
2024-01-18 06:07:22 +08:00
option(ENABLE_NETCDF_4 "Use HDF5." ON)
option(ENABLE_NETCDF4 "Use HDF5." ON)
option(NETCDF_ENABLE_HDF5 "Use HDF5." ON)
if(NOT NETCDF_ENABLE_HDF5 OR NOT ENABLE_NETCDF4 OR NOT ENABLE_NETCDF_4)
set(NETCDF_ENABLE_HDF5 OFF CACHE BOOL "Use HDF5" FORCE)
2024-01-18 06:07:22 +08:00
endif()
option(NETCDF_ENABLE_HDF4 "Build netCDF-4 with HDF4 read capability(HDF4, HDF5 and Zlib required)." OFF)
option(NETCDF_ENABLE_DAP "Enable DAP2 and DAP4 Client." ON)
2024-01-18 06:07:22 +08:00
option(ENABLE_NCZARR "Enable NCZarr Client." ON)
option(ENABLE_PNETCDF "Build with parallel I/O for CDF-1, 2, and 5 files using PnetCDF." OFF)
set(NETCDF_ENABLE_CDF5 AUTO CACHE STRING "AUTO")
option(NETCDF_ENABLE_CDF5 "Enable CDF5 support" ON)
# Netcdf-4 support (i.e. libsrc4) is required by more than just HDF5 (e.g. NCZarr)
# So depending on what above formats are enabled, enable netcdf-4
if(NETCDF_ENABLE_HDF5 OR NETCDF_ENABLE_HDF4 OR ENABLE_NCZARR)
2024-01-18 06:07:22 +08:00
set(ENABLE_NETCDF_4 ON CACHE BOOL "Enable netCDF-4 API" FORCE)
set(ENABLE_NETCDF4 ON CACHE BOOL "Enable netCDF4 Alias" FORCE)
endif()
# enable|disable all forms of network access
2024-01-18 06:07:22 +08:00
option(ENABLE_REMOTE_FUNCTIONALITY "Enable|disable all forms remote data access (DAP, S3, etc)" ON)
message(">>> ENABLE_REMOTE_FUNCTIONALITY=${ENABLE_REMOTE_FUNCTIONALITY}")
if(NOT ENABLE_REMOTE_FUNCTIONALITY)
message(WARNING "ENABLE_REMOTE_FUNCTIONALITY=NO => NETCDF_ENABLE_DAP[4]=NO")
set(NETCDF_ENABLE_DAP OFF CACHE BOOL "ENABLE_REMOTE_FUNCTIONALITY=NO => NETCDF_ENABLE_DAP=NO" FORCE)
set(NETCDF_ENABLE_DAP4 OFF CACHE BOOL "ENABLE_REMOTE_FUNCTIONALITY=NO => NETCDF_ENABLE_DAP4=NO" FORCE)
2024-01-18 06:07:22 +08:00
endif()
2012-10-03 04:56:46 +08:00
# Option to Build DLL
2024-01-18 06:07:22 +08:00
if(WIN32)
option(NETCDF_ENABLE_DLL "Build a Windows DLL." ${BUILD_SHARED_LIBS})
if(NETCDF_ENABLE_DLL)
2024-01-18 06:07:22 +08:00
set(BUILD_DLL ON CACHE BOOL "")
target_compile_definitions(netcdf PRIVATE DLL_EXPORT)
target_compile_definitions(netcdf PUBLIC DLL_NETCDF)
2024-01-18 06:07:22 +08:00
endif()
endif()
2012-10-03 04:56:46 +08:00
# Did the user specify a default minimum blocksize for posixio?
2024-01-18 06:07:22 +08:00
set(NCIO_MINBLOCKSIZE 256 CACHE STRING "Minimum I/O Blocksize for netCDF classic and 64-bit offset format files.")
if(ENABLE_NETCDF_4)
set(USE_NETCDF4 ON CACHE BOOL "")
set(ENABLE_NETCDF_4 ON CACHE BOOL "")
set(ENABLE_NETCDF4 ON CACHE BOOL "")
else()
set(USE_HDF4_FILE_TESTS OFF)
set(USE_HDF4 OFF)
set(NETCDF_ENABLE_HDF4_FILE_TESTS OFF)
set(NETCDF_ENABLE_HDF4 OFF)
2024-01-18 06:07:22 +08:00
endif()
2012-10-03 04:56:46 +08:00
# Option Logging, only valid for netcdf4.
2024-01-18 06:07:22 +08:00
option(ENABLE_LOGGING "Enable Logging." OFF)
if(NOT ENABLE_NETCDF_4)
set(ENABLE_LOGGING OFF)
endif()
if(ENABLE_LOGGING)
target_compile_definitions(netcdf PRIVATE LOGGING ENABLE_SET_LOG_LEVEL)
2024-01-18 06:07:22 +08:00
set(LOGGING ON)
set(ENABLE_SET_LOG_LEVEL ON)
endif()
option(ENABLE_SET_LOG_LEVEL_FUNC "Enable definition of nc_set_log_level()." ON)
if(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC)
target_compile_definitions(netcdf PRIVATE -DENABLE_SET_LOG_LEVEL)
2024-01-18 06:07:22 +08:00
set(ENABLE_SET_LOG_LEVEL ON)
endif()
2017-11-21 04:52:06 +08:00
# Option to allow for strict null file padding.
# See https://github.com/Unidata/netcdf-c/issues/657 for more information
2024-01-18 06:07:22 +08:00
option(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING "Enable strict null byte header padding." OFF)
2024-01-18 06:07:22 +08:00
if(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
set(USE_STRICT_NULL_BYTE_HEADER_PADDING ON CACHE BOOL "")
endif(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
2017-11-21 04:52:06 +08:00
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
# Note that szip management is tricky.
# This is because we have three things to consider:
# 1. is libsz available?
# 2. is szip enabled in HDF5?
# 3. is nczarr enabled?
# We need separate flags for cases 1 and 2
set(USE_HDF5 ${NETCDF_ENABLE_HDF5})
if(NETCDF_ENABLE_DAP)
2024-01-18 06:07:22 +08:00
set(USE_DAP ON CACHE BOOL "")
set(NETCDF_ENABLE_DAP2 ON CACHE BOOL "")
if(NETCDF_ENABLE_HDF5)
2024-01-18 06:07:22 +08:00
message(STATUS "Enabling DAP4")
set(NETCDF_ENABLE_DAP4 ON CACHE BOOL "")
2024-01-18 06:07:22 +08:00
else()
message(STATUS "Disabling DAP4")
set(NETCDF_ENABLE_DAP4 OFF CACHE BOOL "")
endif(NETCDF_ENABLE_HDF5)
2017-09-27 05:26:34 +08:00
2024-01-18 06:07:22 +08:00
else()
set(NETCDF_ENABLE_DAP2 OFF CACHE BOOL "")
set(NETCDF_ENABLE_DAP4 OFF CACHE BOOL "")
2024-01-18 06:07:22 +08:00
endif()
2012-10-03 04:56:46 +08:00
Provide byte-range reading of remote datasets re: issue https://github.com/Unidata/netcdf-c/issues/1251 Assume that you have the URL to a remote dataset which is a normal netcdf-3 or netcdf-4 file. This PR allows the netcdf-c to read that dataset's contents as a netcdf file using HTTP byte ranges if the remote server supports byte-range access. Originally, this PR was set up to access Amazon S3 objects, but it can also access other remote datasets such as those provided by a Thredds server via the HTTPServer access protocol. It may also work for other kinds of servers. Note that this is not intended as a true production capability because, as is known, this kind of access to can be quite slow. In addition, the byte-range IO drivers do not currently do any sort of optimization or caching. An additional goal here is to gain some experience with the Amazon S3 REST protocol. This architecture and its use documented in the file docs/byterange.dox. There are currently two test cases: 1. nc_test/tst_s3raw.c - this does a simple open, check format, close cycle for a remote netcdf-3 file and a remote netcdf-4 file. 2. nc_test/test_s3raw.sh - this uses ncdump to investigate some remote datasets. This PR also incorporates significantly changed model inference code (see the superceded PR https://github.com/Unidata/netcdf-c/pull/1259). 1. It centralizes the code that infers the dispatcher. 2. It adds support for byte-range URLs Other changes: 1. NC_HDF5_finalize was not being properly called by nc_finalize(). 2. Fix minor bug in ncgen3.l 3. fix memory leak in nc4info.c 4. add code to walk the .daprc triples and to replace protocol= fragment tag with a more general mode= tag. Final Note: Th inference code is still way too complicated. We need to move to the validfile() model used by netcdf Java, where each dispatcher is asked if it can process the file. This decentralizes the inference code. This will be done after all the major new dispatchers (PIO, Zarr, etc) have been implemented.
2019-01-02 09:27:36 +08:00
# Option to support byte-range reading of remote datasets
option(NETCDF_ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." ON)
if(NOT ENABLE_REMOTE_FUNCTIONALITY)
message(WARNING "ENABLE_REMOTE_FUNCTIONALITY=NO => NETCDF_ENABLE_BYTERANGE=NO")
set(NETCDF_ENABLE_BYTERANGE OFF CACHE BOOL "ENABLE_REMOTE_FUNCTIONALITY=NO => NETCDF_ENABLE_BYTERANGE=NO" FORCE)
2024-01-18 06:07:22 +08:00
endif()
2012-10-03 04:56:46 +08:00
# Option to Enable DAP long tests, remote tests.
option(NETCDF_ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON)
option(NETCDF_ENABLE_EXTERNAL_SERVER_TESTS "Enable external Server remote tests." OFF)
option(NETCDF_ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF)
if(NOT NETCDF_ENABLE_DAP)
set(NETCDF_ENABLE_DAP_REMOTE_TESTS OFF CACHE BOOL "" FORCE)
set(NETCDF_ENABLE_EXTERNAL_SERVER_TESTS OFF CACHE BOOL "" FORCE)
set(NETCDF_ENABLE_DAP_LONG_TESTS OFF CACHE BOOL "" FORCE)
2024-01-18 06:07:22 +08:00
endif()
2024-01-18 06:07:22 +08:00
set(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
set(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
Add support for Zarr string type to NCZarr * re: https://github.com/Unidata/netcdf-c/pull/2278 * re: https://github.com/Unidata/netcdf-c/issues/2485 * re: https://github.com/Unidata/netcdf-c/issues/2474 This PR subsumes PR https://github.com/Unidata/netcdf-c/pull/2278. Actually is a bit an omnibus covering several issues. ## PR https://github.com/Unidata/netcdf-c/pull/2278 Add support for the Zarr string type. Zarr strings are restricted currently to be of fixed size. The primary issue to be addressed is to provide a way for user to specify the size of the fixed length strings. This is handled by providing the following new attributes special: 1. **_nczarr_default_maxstrlen** — This is an attribute of the root group. It specifies the default maximum string length for string types. If not specified, then it has the value of 64 characters. 2. **_nczarr_maxstrlen** — This is a per-variable attribute. It specifies the maximum string length for the string type associated with the variable. If not specified, then it is assigned the value of **_nczarr_default_maxstrlen**. This PR also requires some hacking to handle the existing netcdf-c NC_CHAR type, which does not exist in zarr. The goal was to choose numpy types for both the netcdf-c NC_STRING type and the netcdf-c NC_CHAR type such that if a pure zarr implementation read them, it would still work and an NC_CHAR type would be handled by zarr as a string of length 1. For writing variables and NCZarr attributes, the type mapping is as follows: * "|S1" for NC_CHAR. * ">S1" for NC_STRING && MAXSTRLEN==1 * ">Sn" for NC_STRING && MAXSTRLEN==n Note that it is a bit of a hack to use endianness, but it should be ok since for string/char, the endianness has no meaning. For reading attributes with pure zarr (i.e. with no nczarr atribute types defined), they will always be interpreted as of type NC_CHAR. ## Issue: https://github.com/Unidata/netcdf-c/issues/2474 This PR partly fixes this issue because it provided more comprehensive support for Zarr attributes that are JSON valued expressions. This PR still does not address the problem in that issue where the _ARRAY_DIMENSION attribute is incorrectly set. Than can only be fixed by the creator of the datasets. ## Issue: https://github.com/Unidata/netcdf-c/issues/2485 This PR also fixes the scalar failure shown in this issue. It generally cleans up scalar handling. It also adds a note to the documentation describing that NCZarr supports scalars while Zarr does not and also how scalar interoperability is achieved. ## Misc. Other Changes 1. Convert the nczarr special attributes and keys to be all lower case. So "_NCZARR_ATTR" now used "_nczarr_attr. Support back compatibility for the upper case names. 2. Cleanup my too-clever-by-half handling of scalars in libnczarr.
2022-08-28 10:21:13 +08:00
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
# Locate some compressors
option(NETCDF_ENABLE_FILTER_SZIP "Enable use of Szip compression library if it is available. Required if ENABLE_NCZARR is true." ON)
option(NETCDF_ENABLE_FILTER_BZ2 "Enable use of Bz2 compression library if it is available." ON)
option(NETCDF_ENABLE_FILTER_BLOSC "Enable use of blosc compression library if it is available." ON)
option(NETCDF_ENABLE_FILTER_ZSTD "Enable use of Zstd compression library if it is available." ON)
# If user wants, then install selected plugins (default on)
2024-01-18 06:07:22 +08:00
set(PLUGIN_INSTALL_DIR "NO" CACHE STRING "Whether and where we should install plugins; defaults to yes")
# This is ugly, but seems necessary because of CMake's boolean structure
2024-01-18 06:07:22 +08:00
set(boolval FALSE)
if(DEFINED PLUGIN_INSTALL_DIR)
booleanize(${PLUGIN_INSTALL_DIR} boolval)
2024-01-18 06:07:22 +08:00
if(boolval)
set(ENABLE_PLUGIN_INSTALL YES)
# No actual value was specified
unset(PLUGIN_INSTALL_DIR CACHE)
2024-01-18 06:07:22 +08:00
else()
if(boolval STREQUAL "NOTFOUND")
# Must be an actual value
2024-01-18 06:07:22 +08:00
set(ENABLE_PLUGIN_INSTALL YES)
else()
set(ENABLE_PLUGIN_INSTALL NO)
endif()
endif()
else()
set(ENABLE_PLUGIN_INSTALL NO)
endif()
# Ensure no defined plugin dir if not enabled
2024-01-18 06:07:22 +08:00
if(NOT ENABLE_PLUGIN_INSTALL)
unset(PLUGIN_INSTALL_DIR CACHE)
2024-01-18 06:07:22 +08:00
endif()
2024-01-18 06:07:22 +08:00
if(ENABLE_PLUGIN_INSTALL)
if(NOT DEFINED PLUGIN_INSTALL_DIR)
# Default to HDF5_PLUGIN_PATH or its default directories
2024-01-18 06:07:22 +08:00
if(DEFINED ENV{HDF5_PLUGIN_PATH})
set(PLUGIN_INSTALL_DIR "$ENV{HDF5_PLUGIN_PATH}")
else()
if(ISMSVC OR ISMINGW)
set(PLUGIN_INSTALL_DIR "$ENV{ALLUSERSPROFILE}\\hdf5\\lib\\plugin")
else()
set(PLUGIN_INSTALL_DIR "/usr/local/hdf5/lib/plugin")
endif()
endif()
message("Defaulting to -DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}")
endif()
endif()
if(ENABLE_PLUGIN_INSTALL)
# Use the lowest priority dir in the path
2024-01-18 06:07:22 +08:00
if(NOT ISMSVC AND NOT ISMINGW)
string(REPLACE ":" ";" PATH_LIST ${PLUGIN_INSTALL_DIR})
else()
set(PATH_LIST ${PLUGIN_INSTALL_DIR})
endif()
# Get last element
2024-01-18 06:07:22 +08:00
list(GET PATH_LIST -1 PLUGIN_INSTALL_DIR)
set(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}")
message("Final value of-DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}")
else() # No option specified
unset(PLUGIN_INSTALL_DIR)
unset(PLUGIN_INSTALL_DIR CACHE)
2024-01-18 06:07:22 +08:00
set(PLUGIN_INSTALL_DIR_SETTING "N.A.")
endif()
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
# Try to enable NCZarr zip support
2024-01-18 06:07:22 +08:00
option(ENABLE_NCZARR_ZIP "Enable NCZarr ZIP support." OFF)
include(CMakeDependentOption)
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
# libdl is always available; built-in in Windows and OSX
cmake_dependent_option(ENABLE_PLUGINS "Enable dynamically loaded plugins (default on)."
ON "BUILD_SHARED_LIBS" OFF)
2024-03-01 02:53:13 +08:00
MESSAGE(STATUS "ENABLE_PLUGINS: ${ENABLE_PLUGINS}")
2024-01-18 06:07:22 +08:00
if(MINGW)
2024-03-01 02:53:13 +08:00
message(STATUS "MINGW Detected, disabling plugins.")
2024-01-18 06:07:22 +08:00
set(ENABLE_PLUGINS OFF CACHE BOOL "Disable plugins" FORCE)
else()
if(NOT WIN32)
if(HAVE_DLFCN_H)
include_directories("dlfcn.h")
endif()
endif()
endif()
2024-03-01 02:53:13 +08:00
2024-01-18 06:07:22 +08:00
if(ENABLE_PLUGINS)
set(USEPLUGINS yes)
endif()
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
# Enable some developer-only tests
option(NETCDF_ENABLE_EXTRA_TESTS "Enable Extra tests. Some may not work because of known issues. Developers only." OFF)
if(NETCDF_ENABLE_EXTRA_TESTS)
2024-01-18 06:07:22 +08:00
set(EXTRA_TESTS ON)
endif()
2012-10-03 04:56:46 +08:00
# Option to use bundled XGetopt in place of getopt(). This is mostly useful
# for MSVC builds. If not building utilities or some tests,
# getopt() isn't required at all.
2024-01-18 06:07:22 +08:00
if(MSVC)
option(ENABLE_XGETOPT "Enable bundled XGetOpt instead of external getopt()." ON)
if(ENABLE_XGETOPT)
set(USE_X_GETOPT ON CACHE BOOL "")
endif()
endif()
2012-10-03 04:56:46 +08:00
2024-01-18 06:07:22 +08:00
set(MATH "")
if(NOT WIN32)
# STDIO instead of posixio.
2024-01-18 06:07:22 +08:00
option(ENABLE_STDIO "If true, use stdio instead of posixio (ex. on the Cray)" OFF)
if(ENABLE_STDIO)
set(USE_STDIO ON CACHE BOOL "")
endif()
2014-03-07 23:46:26 +08:00
# FFIO insteaad of PosixIO
option(NETCDF_ENABLE_FFIO "If true, use ffio instead of posixio" OFF)
if(NETCDF_ENABLE_FFIO)
2024-01-18 06:07:22 +08:00
set(USE_FFIO ON CACHE BOOL "")
endif()
endif()
2012-10-03 04:56:46 +08:00
Fix byterange handling of some URLS re: Issue The byterange handling of the following URLS fails. ### Problem 1: "https://crudata.uea.ac.uk/cru/data/temperature/HadCRUT.4.6.0.0.median.nc#mode=bytes" It turns out that byterange in hdf5 has two possible targets: S3 and not-S3 (e.g. a thredds server or the crudata URL above). Each uses a different HDF5 Virtual File Driver (VFD). I incorrectly set up the byterange code in libhdf5 so that it would choose one or the other of the two VFD's for any netcdf-c library build. The fix is to allow it to choose either one at run-time. ### Problem 2: "https://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" When given what appears to be an S3-related URL, the netcdf-c library code converts it into a canonical, so-called "path" format. In casing out the possible input URL formats, I missed the case where the host contains the bucket ("noaa-goes16"), but not the region. So the fix was to check for this case. ## Misc. Related Changes 1. Since S3 is used in more than just NCZarr, I changed the automake/cmake options to replace "--enable-nczarr-s3" with "--enable-s3", but keeping the former option as a synonym for the latter. This also entailed cleaning up libnetcdf.settings WRT S3 support 2. Added the above URLS as additional test cases ## Misc. Un-Related Changes 1. CURLOPT_PUT is deprecated in favor to CURLOPT_UPLOAD 2. Fix some minor warnings ## Open Problems * Under Ubuntu, either libcrypto or aws-sdk-cpp has a memory leak.
2023-03-03 10:51:02 +08:00
# Options for S3 Support
2024-01-18 06:07:22 +08:00
option(ENABLE_S3 "Enable S3 support." OFF)
option(ENABLE_S3_INTERNAL "Enable S3 Internal support." OFF)
option(ENABLE_NCZARR_S3 "Enable NCZarr S3 support; Deprecated in favor of ENABLE_S3" OFF)
2024-01-18 06:07:22 +08:00
if(NOT ENABLE_REMOTE_FUNCTIONALITY)
set(ENABLE_S3 OFF CACHE BOOL "" FORCE)
set(ENABLE_S3_INTERNAL OFF CACHE BOOL "" FORCE)
set(ENABLE_NCZARR_S3 OFF CACHE BOOL "" FORCE)
endif()
# Control S3 Testing: Multi-valued option
2024-01-18 06:07:22 +08:00
set(WITH_S3_TESTING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none) PUBLIC")
SET_PROPERTY(CACHE WITH_S3_TESTING PROPERTY STRINGS ON OFF PUBLIC) #
2024-01-18 06:07:22 +08:00
if(WITH_S3_TESTING STREQUAL "")
set(WITH_S3_TESTING OFF CACHE STRING "") # Default
endif()
2024-01-18 06:07:22 +08:00
if(WITH_S3_TESTING)
message(WARNING "**** DO NOT USE WITH_S3_TESTING=ON UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***")
2024-01-18 06:07:22 +08:00
endif()
This PR adds EXPERIMENTAL support for accessing data in the cloud using a variant of the Zarr protocol and storage format. This enhancement is generically referred to as "NCZarr". The data model supported by NCZarr is netcdf-4 minus the user-defined types and the String type. In this sense it is similar to the CDF-5 data model. More detailed information about enabling and using NCZarr is described in the document NUG/nczarr.md and in a [Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in). WARNING: this code has had limited testing, so do use this version for production work. Also, performance improvements are ongoing. Note especially the following platform matrix of successful tests: Platform | Build System | S3 support ------------------------------------ Linux+gcc | Automake | yes Linux+gcc | CMake | yes Visual Studio | CMake | no Additionally, and as a consequence of the addition of NCZarr, major changes have been made to the Filter API. NOTE: NCZarr does not yet support filters, but these changes are enablers for that support in the future. Note that it is possible (probable?) that there will be some accidental reversions if the changes here did not correctly mimic the existing filter testing. In any case, previously filter ids and parameters were of type unsigned int. In order to support the more general zarr filter model, this was all converted to char*. The old HDF5-specific, unsigned int operations are still supported but they are wrappers around the new, char* based nc_filterx_XXX functions. This entailed at least the following changes: 1. Added the files libdispatch/dfilterx.c and include/ncfilter.h 2. Some filterx utilities have been moved to libdispatch/daux.c 3. A new entry, "filter_actions" was added to the NCDispatch table and the version bumped. 4. An overly complex set of structs was created to support funnelling all of the filterx operations thru a single dispatch "filter_actions" entry. 5. Move common code to from libhdf5 to libsrc4 so that it is accessible to nczarr. Changes directly related to Zarr: 1. Modified CMakeList.txt and configure.ac to support both C and C++ -- this is in support of S3 support via the awd-sdk libraries. 2. Define a size64_t type to support nczarr. 3. More reworking of libdispatch/dinfermodel.c to support zarr and to regularize the structure of the fragments section of a URL. Changes not directly related to Zarr: 1. Make client-side filter registration be conditional, with default off. 2. Hack include/nc4internal.h to make some flags added by Ed be unique: e.g. NC_CREAT, NC_INDEF, etc. 3. cleanup include/nchttp.h and libdispatch/dhttp.c. 4. Misc. changes to support compiling under Visual Studio including: * Better testing under windows for dirent.h and opendir and closedir. 5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags and to centralize error reporting. 6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them. 7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible. Changes Left TO-DO: 1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
Fix byterange handling of some URLS re: Issue The byterange handling of the following URLS fails. ### Problem 1: "https://crudata.uea.ac.uk/cru/data/temperature/HadCRUT.4.6.0.0.median.nc#mode=bytes" It turns out that byterange in hdf5 has two possible targets: S3 and not-S3 (e.g. a thredds server or the crudata URL above). Each uses a different HDF5 Virtual File Driver (VFD). I incorrectly set up the byterange code in libhdf5 so that it would choose one or the other of the two VFD's for any netcdf-c library build. The fix is to allow it to choose either one at run-time. ### Problem 2: "https://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" When given what appears to be an S3-related URL, the netcdf-c library code converts it into a canonical, so-called "path" format. In casing out the possible input URL formats, I missed the case where the host contains the bucket ("noaa-goes16"), but not the region. So the fix was to check for this case. ## Misc. Related Changes 1. Since S3 is used in more than just NCZarr, I changed the automake/cmake options to replace "--enable-nczarr-s3" with "--enable-s3", but keeping the former option as a synonym for the latter. This also entailed cleaning up libnetcdf.settings WRT S3 support 2. Added the above URLS as additional test cases ## Misc. Un-Related Changes 1. CURLOPT_PUT is deprecated in favor to CURLOPT_UPLOAD 2. Fix some minor warnings ## Open Problems * Under Ubuntu, either libcrypto or aws-sdk-cpp has a memory leak.
2023-03-03 10:51:02 +08:00
# ENABLE_NCZARR_S3 is now an alias for ENABLE_S3 (but...)
if (NOT ENABLE_S3 AND ENABLE_NCZARR_S3)
2024-01-18 06:07:22 +08:00
set(ENABLE_S3 ON CACHE BOOL "NCARR S3" FORCE) # For back compatibility
endif()
unset(ENABLE_NCZARR_S3)
2024-01-18 06:07:22 +08:00
if(NOT ENABLE_REMOTE_FUNCTIONALITY)
message(WARNING "ENABLE_REMOTE_FUNCTIONALITY=NO => disable all s3 functionality")
set(ENABLE_S3 OFF CACHE BOOL "" FORCE)
set(ENABLE_S3_INTERNAL OFF CACHE BOOL "" FORCE)
set(ENABLE_NCZARR_S3 OFF CACHE BOOL "" FORCE)
set(NETCDF_ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE)
2024-01-18 06:07:22 +08:00
set(WITH_S3_TESTING OFF CACHE STRING "" FORCE)
endif()
if(ENABLE_S3)
if(NOT ENABLE_S3_AWS AND NOT ENABLE_S3_INTERNAL)
Fix byterange handling of some URLS re: Issue The byterange handling of the following URLS fails. ### Problem 1: "https://crudata.uea.ac.uk/cru/data/temperature/HadCRUT.4.6.0.0.median.nc#mode=bytes" It turns out that byterange in hdf5 has two possible targets: S3 and not-S3 (e.g. a thredds server or the crudata URL above). Each uses a different HDF5 Virtual File Driver (VFD). I incorrectly set up the byterange code in libhdf5 so that it would choose one or the other of the two VFD's for any netcdf-c library build. The fix is to allow it to choose either one at run-time. ### Problem 2: "https://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" When given what appears to be an S3-related URL, the netcdf-c library code converts it into a canonical, so-called "path" format. In casing out the possible input URL formats, I missed the case where the host contains the bucket ("noaa-goes16"), but not the region. So the fix was to check for this case. ## Misc. Related Changes 1. Since S3 is used in more than just NCZarr, I changed the automake/cmake options to replace "--enable-nczarr-s3" with "--enable-s3", but keeping the former option as a synonym for the latter. This also entailed cleaning up libnetcdf.settings WRT S3 support 2. Added the above URLS as additional test cases ## Misc. Un-Related Changes 1. CURLOPT_PUT is deprecated in favor to CURLOPT_UPLOAD 2. Fix some minor warnings ## Open Problems * Under Ubuntu, either libcrypto or aws-sdk-cpp has a memory leak.
2023-03-03 10:51:02 +08:00
message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_S3=NO")
2024-01-18 06:07:22 +08:00
set(ENABLE_S3 OFF CACHE BOOL "S3 support" FORCE)
endif()
if(ENABLE_S3_AWS AND ENABLE_S3_INTERNAL)
Improve S3 Documentation and Support ## Improvements to S3 Documentation * Create a new document *quickstart_paths.md* that give a summary of the legal path formats used by netcdf-c. This includes both file paths and URL paths. * Modify *nczarr.md* to remove most of the S3 related text. * Move the S3 text from *nczarr.md* to a new document *cloud.md*. * Add some S3-related text to the *byterange.md* document. Hopefully, this will make it easier for users to find the information they want. ## Rebuild NCZarr Testing In order to avoid problems with running make check in parallel, two changes were made: 1. The *nczarr_test* test system was rebuilt. Now, for each test. any generated files are kept in a test-specific directory, isolated from all other test executions. 2. Similarly, since the S3 test bucket is shared, any generated S3 objects are isolated using a test-specific key path. ## Other S3 Related Changes * Add code to ensure that files created on S3 are reclaimed at end of testing. * Used the bash "trap" command to ensure S3 cleanup even if the test fails. * Cleanup the S3 related configure.ac flag set since S3 is used in several places. So now one should use the option *--enable-s3* instead of *--enable-nczarr-s3*, although the latter is still kept as a deprecated alias for the former. * Get some of the github actions yml to work with S3; required fixing various test scripts adding a secret to access the Unidata S3 bucket. * Cleanup S3 portion of libnetcdf.settings.in and netcdf_meta.h.in and test_common.in. * Merge partial S3 support into dhttp.c. * Create an experimental s3 access library especially for use with Windows. It is enabled by using the options *--enable-s3-internal* (automake) or *-DENABLE_S3_INTERNAL=ON* (CMake). Also add a unit-test for it. * Move some definitions from ncrc.h to ncs3sdk.h ## Other Changes * Provide a default implementation of strlcpy and move this and similar defaults into *dmissing.c*.
2023-04-26 07:15:06 +08:00
message(WARNING "Both aws-sdk-cpp and s3-internal enabled => use s3-internal")
2024-01-18 06:07:22 +08:00
set(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE)
endif()
endif()
2024-01-18 06:07:22 +08:00
if(NOT ENABLE_S3)
if(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING)
message(WARNING "S3 support is disabled => WITH_S3_TESTING=OFF")
2024-01-18 06:07:22 +08:00
set(WITH_S3_TESTING OFF CACHE STRING "" FORCE)
endif()
endif()
2024-01-18 06:07:22 +08:00
option(ENABLE_LIBXML2 "Link against libxml2 if it is available, use the packaged tinyxml2 parser otherwise." ON)
set(XMLPARSER "tinyxml2 (bundled)")
if(NOT NETCDF_ENABLE_BYTERANGE AND NETCDF_ENABLE_HDF5_ROS3)
message(WARNING "ROS3 support requires NETCDF_ENABLE_BYTERANGE=TRUE; disabling ROS3 support")
set(NETCDF_ENABLE_HDF5_ROS3 OFF CACHE BOOL "ROS3 support" FORCE)
2024-01-18 06:07:22 +08:00
endif()
##
2012-10-03 04:56:46 +08:00
# Enable Tests
##
2024-01-18 06:07:22 +08:00
option(ENABLE_TESTS "Enable basic tests, run with 'make test'." ON)
if(ENABLE_TESTS)
set(BUILD_TESTSETS ON CACHE BOOL "")
# Options for CTest-based tests, dashboards.
2024-01-18 06:07:22 +08:00
set(NC_CTEST_PROJECT_NAME "netcdf-c" CACHE STRING "Project Name for CTest-based testing purposes.")
set(NC_CTEST_DROP_SITE "cdash.unidata.ucar.edu:443" CACHE STRING "Dashboard location for CTest-based testing purposes.")
set(NC_CTEST_DROP_LOC_PREFIX "" CACHE STRING "Prefix for Dashboard location on remote server when using CTest-based testing.")
set(SUBMIT_URL "https://cdash.unidata.ucar.edu:443")
find_program(HOSTNAME_CMD NAMES hostname)
if(NOT WIN32)
set(HOSTNAME_ARG "-s")
endif()
if(HOSTNAME_CMD)
execute_process(COMMAND ${HOSTNAME_CMD} "${HOSTNAME_ARG}" OUTPUT_VARIABLE HOSTNAME OUTPUT_STRIP_TRAILING_WHITESPACE)
set(NC_CTEST_SITE "${HOSTNAME}" CACHE STRING "Hostname of test machine.")
endif()
if(NC_CTEST_SITE)
set(SITE "${NC_CTEST_SITE}" CACHE STRING "")
endif()
2015-02-03 06:14:22 +08:00
###
# This option dictates whether or not to turn on
# tests which are known to fail. This is not the
# same thing as an 'expected failure'. Rather, these
# are tests that will need to be fixed eventually.
#
2019-09-18 10:27:43 +08:00
# By placing them here, we can occasionally turn this
# flag on and see if any known failures have been
# fixed in the course of code improvement/other bug
# fixes.
#
# To use this, simply add as a fencepost around tests
# which are known to fail.
###
option(NETCDF_ENABLE_FAILING_TESTS "Run tests which are known to fail, check to see if any have been fixed." OFF)
###
2019-11-16 22:20:45 +08:00
# Option to turn on unit testing. See
# https://github.com/Unidata/netcdf-c/pull/1472 for more information.
###
2024-01-18 06:07:22 +08:00
option(ENABLE_UNIT_TESTS "Run Unit Tests." ON)
###
# Option to turn on performance testing.
# See https://github.com/Unidata/netcdf-c/issues/2627 for more information.
###
option(NETCDF_ENABLE_BENCHMARKS "Run benchmark Tests." OFF)
###
# End known-failures.
###
MARK_AS_ADVANCED(NETCDF_ENABLE_FAILING_TESTS)
2024-01-18 06:07:22 +08:00
endif()
2012-10-03 04:56:46 +08:00
###
# Option to enable extreme numbers during testing.
###
option(NETCDF_ENABLE_EXTREME_NUMBERS "Enable extreme numbers during testing, such as MAX_INT-1" ON)
if(NETCDF_ENABLE_EXTREME_NUMBERS)
2024-01-18 06:07:22 +08:00
set(USE_EXTREME_NUMBERS ON)
endif()
2012-10-03 04:56:46 +08:00
# Enable Large file tests
2024-01-18 06:07:22 +08:00
if(ENABLE_LARGE_FILE_TESTS)
set(LARGE_FILE_TESTS ON)
endif()
2012-10-03 04:56:46 +08:00
2024-01-18 06:07:22 +08:00
option(ENABLE_METADATA_PERF_TESTS "Enable test of metadata performance." OFF)
if(ENABLE_METADATA_PERF_TESTS)
set(ENABLE_METADATA_PERF ON)
endif()
# Location for large file tests.
2024-01-18 06:07:22 +08:00
set(TEMP_LARGE "." CACHE STRING "Location to store large file tests.")
option(NETCDF_ENABLE_FSYNC "Enable experimental fsync code." OFF)
if(NETCDF_ENABLE_FSYNC)
2024-01-18 06:07:22 +08:00
set(USE_FSYNC ON)
endif()
2012-10-03 04:56:46 +08:00
# Temporary
2014-04-10 05:20:16 +08:00
OPTION (ENABLE_JNA "Enable jna bug fix code." OFF)
2024-01-18 06:07:22 +08:00
if(ENABLE_JNA)
set(JNA ON)
endif()
2014-04-10 05:20:16 +08:00
# Linux specific large file support flags.
# Modelled after check in CMakeLists.txt for hdf5.
option(NETCDF_ENABLE_LARGE_FILE_SUPPORT "Enable large file support." ON)
if(NETCDF_ENABLE_LARGE_FILE_SUPPORT)
2024-01-18 06:07:22 +08:00
if(MSVC)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /LARGEADDRESSAWARE")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /LARGEADDRESSAWARE")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} /LARGEADDRESSAWARE")
else()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64")
endif()
endif()
option(NETCDF_ENABLE_EXAMPLE_TESTS "Run extra example tests. Requires GNU Sed. Ignored if HDF5 is not Enabled" OFF)
if(NOT NETCDF_ENABLE_HDF5 AND NETCDF_ENABLE_EXAMPLE_TESTS)
set(NETCDF_ENABLE_EXAMPLE_TESTS OFF)
2024-01-18 06:07:22 +08:00
endif()
2012-10-03 04:56:46 +08:00
2024-01-17 01:20:43 +08:00
##################################
# Dependencies
##################################
include(cmake/dependencies.cmake)
################################
# End Dependencies
################################
# Enable Parallel IO with netCDF-4/HDF5 files using HDF5 parallel I/O.
2024-01-18 06:07:22 +08:00
set(STATUS_PARALLEL "OFF")
set(IMPORT_MPI "")
2024-01-18 06:07:22 +08:00
option(ENABLE_PARALLEL4 "Build netCDF-4 with parallel IO" "${HDF5_PARALLEL}")
if(ENABLE_PARALLEL4 AND NETCDF_ENABLE_HDF5)
2024-01-18 06:07:22 +08:00
if(NOT HDF5_PARALLEL)
set(USE_PARALLEL OFF CACHE BOOL "")
message(STATUS "Cannot find HDF5 library built with parallel support. Disabling parallel build.")
else()
set(HDF5_PARALLEL ON CACHE BOOL "")
set(USE_PARALLEL ON CACHE BOOL "")
set(USE_PARALLEL4 ON CACHE BOOL "")
set(STATUS_PARALLEL "ON")
configure_file("${netCDF_SOURCE_DIR}/nc_test4/run_par_test.sh.in"
2018-08-17 22:52:34 +08:00
"${netCDF_BINARY_DIR}/tmp/run_par_test.sh" @ONLY NEWLINE_STYLE LF)
2024-01-18 06:07:22 +08:00
file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_test.sh"
2018-08-17 22:52:34 +08:00
DESTINATION ${netCDF_BINARY_DIR}/nc_test4
FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
2020-02-28 05:06:45 +08:00
configure_file("${netCDF_SOURCE_DIR}/h5_test/run_par_tests.sh.in"
"${netCDF_BINARY_DIR}/tmp/run_par_tests.sh" @ONLY NEWLINE_STYLE LF)
2024-01-18 06:07:22 +08:00
file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_tests.sh"
2020-02-28 05:06:45 +08:00
DESTINATION ${netCDF_BINARY_DIR}/h5_test
FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
set(IMPORT_MPI "include(CMakeFindDependencyMacro)\nfind_dependency(MPI COMPONENTS C)")
2024-01-18 06:07:22 +08:00
endif()
endif()
2013-02-07 07:09:19 +08:00
# Options to enable parallel IO for classic formats with PnetCDF library.
set(STATUS_PNETCDF ${ENABLE_PNETCDF})
set(USE_PNETCDF ${ENABLE_PNETCDF})
2024-01-18 06:07:22 +08:00
if(ENABLE_PNETCDF)
set(STATUS_PARALLEL ON)
set(USE_PARALLEL ON CACHE BOOL "")
###
# Generate pnetcdf test
###
configure_file("${netCDF_SOURCE_DIR}/nc_test/run_pnetcdf_tests.sh.in"
"${netCDF_BINARY_DIR}/nc_test/run_pnetcdf_tests.sh")
2024-01-18 06:07:22 +08:00
endif()
2013-02-07 07:09:19 +08:00
# Options to enable use of fill values for elements causing NC_ERANGE
set(NETCDF_ENABLE_ERANGE_FILL AUTO CACHE STRING "AUTO")
option(NETCDF_ENABLE_ERANGE_FILL "Enable use of fill value when out-of-range type conversion causes NC_ERANGE error." OFF)
if(NETCDF_ENABLE_ERANGE_FILL) # enable or auto
string(TOUPPER ${NETCDF_ENABLE_ERANGE_FILL} NETCDF_ENABLE_ERANGE_FILL)
if(NETCDF_ENABLE_ERANGE_FILL AND NOT NETCDF_ENABLE_ERANGE_FILL STREQUAL "AUTO")
# explicitly enabled
set(NETCDF_ENABLE_ERANGE_FILL ON)
2024-01-18 06:07:22 +08:00
else()
if(NOT NETCDF_ENABLE_ERANGE_FILL STREQUAL "AUTO")
set(NETCDF_ENABLE_ERANGE_FILL OFF)
2024-01-18 06:07:22 +08:00
endif()
endif()
endif(NETCDF_ENABLE_ERANGE_FILL)
# Now NETCDF_ENABLE_ERANGE_FILL is either AUTO, ON, or OFF
# More relaxed coordinate check is now mandatory for all builds.
2024-01-18 06:07:22 +08:00
set(ENABLE_ZERO_LENGTH_COORD_BOUND ON)
# check and conform with PnetCDF settings on ERANGE_FILL and RELAX_COORD_BOUND
if(USE_PNETCDF)
if(NETCDF_ENABLE_ERANGE_FILL STREQUAL "AUTO") # not set on command line
set(NETCDF_ENABLE_ERANGE_FILL "${PNETCDF_HAS_ERANGE_FILL}")
2024-01-18 06:07:22 +08:00
else()
# user explicitly set this option on command line
if(NOT NETCDF_ENABLE_ERANGE_FILL STREQUAL "${PNETCDF_HAS_ERANGE_FILL}")
if(NETCDF_ENABLE_ERANGE_FILL)
2024-01-18 06:07:22 +08:00
message(FATAL_ERROR "Enabling erange-fill conflicts with PnetCDF setting")
else()
message(FATAL_ERROR "Disabling erange-fill conflicts with PnetCDF setting")
endif()
endif()
endif()
endif()
if(NETCDF_ENABLE_ERANGE_FILL)
2024-01-18 06:07:22 +08:00
message(STATUS "Enabling use of fill value when NC_ERANGE")
set(M4FLAGS "-DERANGE_FILL" CACHE STRING "")
endif()
2024-01-18 06:07:22 +08:00
if(ENABLE_ZERO_LENGTH_COORD_BOUND)
message(STATUS "Enabling a more relaxed check for NC_EINVALCOORDS")
target_compile_definitions(netcdf PRIVATE RELAX_COORD_BOUND)
2024-01-18 06:07:22 +08:00
endif()
# Enable Parallel Tests.
2024-01-18 06:07:22 +08:00
option(ENABLE_PARALLEL_TESTS "Enable Parallel IO Tests. Requires HDF5/NetCDF4 with parallel I/O Support." "${HDF5_PARALLEL}")
if(ENABLE_PARALLEL_TESTS AND USE_PARALLEL)
set(TEST_PARALLEL ON CACHE BOOL "")
if(USE_NETCDF4)
set(TEST_PARALLEL4 ON CACHE BOOL "")
endif()
endif()
2018-02-27 08:16:15 +08:00
IF (ENABLE_PARALLEL_TESTS AND NOT USE_PARALLEL)
2024-01-18 06:07:22 +08:00
message(FATAL_ERROR "Parallel tests requested, but no parallel HDF5 installation detected.")
endif()
2012-10-03 04:56:46 +08:00
# Enable special filter test; experimental when using cmake.
option(NETCDF_ENABLE_FILTER_TESTING "Enable filter testing. Ignored if shared libraries or netCDF4 are not enabled" ${ENABLE_PLUGINS})
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
if(NETCDF_ENABLE_FILTER_TESTING)
if(NOT NETCDF_ENABLE_HDF5 AND NOT ENABLE_NCZARR)
message(WARNING "NETCDF_ENABLE_FILTER_TESTING requires HDF5 and/or NCZarr. Disabling.")
set(NETCDF_ENABLE_FILTER_TESTING OFF CACHE BOOL "Enable Filter Testing" FORCE)
2024-01-18 06:07:22 +08:00
endif()
endif()
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
2024-01-18 06:07:22 +08:00
if(NOT BUILD_SHARED_LIBS)
message(WARNING "NETCDF_ENABLE_FILTER_TESTING requires shared libraries. Disabling.")
set(NETCDF_ENABLE_FILTER_TESTING OFF)
2024-01-18 06:07:22 +08:00
endif()
option(ENABLE_NCZARR_FILTERS "Enable NCZarr filters" ${ENABLE_PLUGINS})
option(ENABLE_NCZARR_FILTERS_TESTING "Enable NCZarr filter testing." ${ENABLE_NCZARR_FILTERS})
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
# Constraints
2024-03-01 02:53:13 +08:00
if (NOT ENABLE_PLUGINS AND ENABLE_NCZARR_FILTERS)
message(WARNING "ENABLE_NCZARR_FILTERS requires ENABLE_PLUGINS. Disabling.")
2024-01-18 06:07:22 +08:00
set(ENABLE_NCZARR_FILTERS OFF CACHE BOOL "Enable NCZarr Filters." FORCE)
endif()
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
IF (NOT ENABLE_NCZARR)
2024-01-18 06:07:22 +08:00
message(WARNING "ENABLE_NCZARR==NO => ENABLE_NCZARR_FILTERS==NO AND ENABLE_NCZARR_FILTER_TESTING==NO")
set(ENABLE_NCZARR_FILTERS OFF CACHE BOOL "Disable NCZARR_FILTERS" FORCE)
endif()
Support MSYS2/Mingw platform re: The current netcdf-c release has some problems with the mingw platform on windows. Mostly they are path issues. Changes to support mingw+msys2: ------------------------------- * Enable option of looking into the windows registry to find the mingw root path. In aid of proper path handling. * Add mingw+msys as a specific platform in configure.ac and move testing of the platform to the front so it is available early. * Handle mingw X libncpoco (dynamic loader) properly even though mingw does not yet support it. * Handle mingw X plugins properly even though mingw does not yet support it. * Alias pwd='pwd -W' to better handle paths in shell scripts. * Plus a number of other minor compile irritations. * Disallow the use of multiple nc_open's on the same file for windows (and mingw) because windows does not seem to handle these properly. Not sure why we did not catch this earlier. * Add mountpoint info to dpathmgr.c to help support mingw. * Cleanup dpathmgr conversions. Known problems: --------------- * I have not been able to get shared libraries to work, so plugins/filters must be disabled. * There is some kind of problem with libcurl that I have not solved, so all uses of libcurl (currently DAP+Byterange) must be disabled. Misc. other fixes: ------------------ * Cleanup the relationship between ENABLE_PLUGINS and various other flags in CMakeLists.txt and configure.ac. * Re-arrange the TESTDIRS order in Makefile.am. * Add pseudo-breakpoint to nclog.[ch] for debugging. * Improve the documentation of the path manager code in ncpathmgr.h * Add better support for relative paths in dpathmgr.c * Default the mode args to NCfopen to include "b" (binary) for windows. * Add optional debugging output in various places. * Make sure that everything builds with plugins disabled. * Fix numerous (s)printf inconsistencies betweenb the format spec and the arguments.
2021-12-24 13:18:56 +08:00
IF (NOT ENABLE_NCZARR_FILTERS)
2024-01-18 06:07:22 +08:00
set(ENABLE_NCZARR_FILTER_TESTING OFF CACHE BOOL "Enable NCZarr Filter Testing" FORCE)
endif()
Support MSYS2/Mingw platform re: The current netcdf-c release has some problems with the mingw platform on windows. Mostly they are path issues. Changes to support mingw+msys2: ------------------------------- * Enable option of looking into the windows registry to find the mingw root path. In aid of proper path handling. * Add mingw+msys as a specific platform in configure.ac and move testing of the platform to the front so it is available early. * Handle mingw X libncpoco (dynamic loader) properly even though mingw does not yet support it. * Handle mingw X plugins properly even though mingw does not yet support it. * Alias pwd='pwd -W' to better handle paths in shell scripts. * Plus a number of other minor compile irritations. * Disallow the use of multiple nc_open's on the same file for windows (and mingw) because windows does not seem to handle these properly. Not sure why we did not catch this earlier. * Add mountpoint info to dpathmgr.c to help support mingw. * Cleanup dpathmgr conversions. Known problems: --------------- * I have not been able to get shared libraries to work, so plugins/filters must be disabled. * There is some kind of problem with libcurl that I have not solved, so all uses of libcurl (currently DAP+Byterange) must be disabled. Misc. other fixes: ------------------ * Cleanup the relationship between ENABLE_PLUGINS and various other flags in CMakeLists.txt and configure.ac. * Re-arrange the TESTDIRS order in Makefile.am. * Add pseudo-breakpoint to nclog.[ch] for debugging. * Improve the documentation of the path manager code in ncpathmgr.h * Add better support for relative paths in dpathmgr.c * Default the mode args to NCfopen to include "b" (binary) for windows. * Add optional debugging output in various places. * Make sure that everything builds with plugins disabled. * Fix numerous (s)printf inconsistencies betweenb the format spec and the arguments.
2021-12-24 13:18:56 +08:00
2024-01-18 06:07:22 +08:00
set(ENABLE_CLIENTSIDE_FILTERS OFF)
# Determine whether or not to generate documentation.
option(NETCDF_ENABLE_DOXYGEN "Enable generation of doxygen-based documentation." OFF)
if(NETCDF_ENABLE_DOXYGEN)
# Offer the option to build internal documentation.
option(NETCDF_ENABLE_INTERNAL_DOCS "Build internal documentation. This is of interest to developers only." OFF)
if(NETCDF_ENABLE_INTERNAL_DOCS)
2024-01-18 06:07:22 +08:00
set(BUILD_INTERNAL_DOCS yes CACHE STRING "")
else()
set(BUILD_INTERNAL_DOCS no CACHE STRING "")
endif()
###
#
# If we are building release documentation, we need to set some
# variables that will be used in the Doxygen.in template.
###
option(NETCDF_ENABLE_DOXYGEN_BUILD_RELEASE_DOCS "Build release documentation. This is of interest only to the netCDF developers." OFF)
if(NETCDF_ENABLE_DOXYGEN_BUILD_RELEASE_DOCS)
2024-01-18 06:07:22 +08:00
set(DOXYGEN_CSS_FILE "${CMAKE_SOURCE_DIR}/docs/release.css" CACHE STRING "")
set(DOXYGEN_HEADER_FILE "${CMAKE_SOURCE_DIR}/docs/release_header.html" CACHE STRING "")
set(DOXYGEN_SEARCHENGINE "NO" CACHE STRING "")
set(NETCDF_ENABLE_DOXYGEN_SERVER_BASED_SEARCH NO CACHE STRING "")
2024-01-18 06:07:22 +08:00
else()
set(DOXYGEN_CSS_FILE "" CACHE STRING "")
set(DOXYGEN_HEADER_FILE "" CACHE STRING "")
set(DOXYGEN_SEARCHENGINE "YES" CACHE STRING "")
2015-11-03 03:46:09 +08:00
# If not using release document configuration,
# provide an option for server-based search.
option(NETCDF_ENABLE_DOXYGEN_SERVER_SIDE_SEARCH "Configure Doxygen with server-based search." OFF)
if(NETCDF_ENABLE_DOXYGEN_SERVER_SIDE_SEARCH)
2024-01-18 06:07:22 +08:00
set(DOXYGEN_SERVER_BASED_SEARCH "YES" CACHE STRING "")
else()
set(DOXYGEN_SERVER_BASED_SEARCH "NO" CACHE STRING "")
endif(NETCDF_ENABLE_DOXYGEN_SERVER_SIDE_SEARCH)
2015-11-03 03:46:09 +08:00
endif(NETCDF_ENABLE_DOXYGEN_BUILD_RELEASE_DOCS)
# Option to turn on the TODO list in the doxygen-generated documentation.
2024-03-19 04:29:56 +08:00
option(NETCDF_DOXYGEN_ENABLE_TASKS "Turn on test, todo, bug lists in documentation. This is of interest to developers only." OFF)
if(NETCDF_DOXYGEN_ENABLE_TASKS)
2024-01-18 06:07:22 +08:00
set(SHOW_DOXYGEN_TAG_LIST YES CACHE STRING "")
2024-03-19 04:29:56 +08:00
else(NETCDF_DOXYGEN_ENABLE_TASKS)
2024-01-18 06:07:22 +08:00
set(SHOW_DOXYGEN_TODO_LIST NO CACHE STRING "")
2024-03-19 04:29:56 +08:00
endif(NETCDF_DOXYGEN_ENABLE_TASKS)
2024-01-18 06:07:22 +08:00
option(NETCDF_ENABLE_DOXYGEN_PDF_OUTPUT "[EXPERIMENTAL] Turn on PDF output for Doxygen-generated documentation." OFF)
2024-01-18 06:07:22 +08:00
if(NETCDF_ENABLE_DOXYGEN_PDF_OUTPUT)
set(NC_NETCDF_ENABLE_DOXYGEN_PDF_OUTPUT "YES" CACHE STRING "")
2024-01-18 06:07:22 +08:00
else()
set(NC_NETCDF_ENABLE_DOXYGEN_PDF_OUTPUT "NO" CACHE STRING "")
2024-01-18 06:07:22 +08:00
endif()
find_program(NC_DOT NAMES dot)
# Specify whether or not 'dot' was found on the system path.
2024-01-18 06:07:22 +08:00
if(NC_DOT)
set(HAVE_DOT YES CACHE STRING "")
else(NC_DOT)
set(HAVE_DOT NO CACHE STRING "")
endif(NC_DOT)
endif()
2012-10-03 04:56:46 +08:00
# Always enable DISKLESS
option(NETCDF_ENABLE_DISKLESS "Enable in-memory files" ON)
# Always enable quantization.
2024-01-18 06:07:22 +08:00
option(ENABLE_QUANTIZE "Enable variable quantization" ON)
# By default, MSVC has a stack size of 1000000.
# Allow a user to override this.
2024-01-18 06:07:22 +08:00
if(MSVC)
set(NC_MSVC_STACK_SIZE 40000000 CACHE STRING "Default stack size for MSVC-based projects.")
2013-10-10 06:05:50 +08:00
# By default, CMake sets the stack to 1000000.
# Remove this limitation.
# See here for more details:
# http://www.cmake.org/pipermail/cmake/2009-April/028710.html
2024-01-18 06:07:22 +08:00
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /STACK:${NC_MSVC_STACK_SIZE}")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /STACK:${NC_MSVC_STACK_SIZE}")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} /STACK:${NC_MSVC_STACK_SIZE}")
endif()
2012-10-03 04:56:46 +08:00
# Set some of the options as advanced.
MARK_AS_ADVANCED(NETCDF_ENABLE_INTERNAL_DOCS VALGRIND_TESTS NETCDF_ENABLE_COVERAGE_TESTS )
MARK_AS_ADVANCED(NETCDF_ENABLE_DAP_REMOTE_TESTS NETCDF_ENABLE_DAP_LONG_TESTS USE_REMOTE_CDASH NETCDF_ENABLE_EXTERNAL_SERVER_TESTS)
MARK_AS_ADVANCED(NETCDF_ENABLE_DOXYGEN_BUILD_RELEASE_DOCS NETCDF_DOXYGEN_ENABLE_TASKS NETCDF_ENABLE_DOXYGEN_SERVER_SIDE_SEARCH)
MARK_AS_ADVANCED(ENABLE_SHARED_LIBRARY_VERSION)
################################
# Option checks
################################
2012-10-03 04:56:46 +08:00
# Library include checks
2024-01-18 06:07:22 +08:00
CHECK_INCLUDE_file("math.h" HAVE_MATH_H)
CHECK_INCLUDE_file("unistd.h" HAVE_UNISTD_H)
# Solve a compatibility issue in ncgen/, which checks
# for NO_UNISTD_H
2024-01-18 06:07:22 +08:00
if(NOT HAVE_UNISTD_H)
set(YY_NO_UNISTD_H TRUE)
endif()
CHECK_INCLUDE_file("alloca.h" HAVE_ALLOCA_H)
CHECK_INCLUDE_file("malloc.h" HAVE_MALLOC_H)
CHECK_INCLUDE_file("fcntl.h" HAVE_FCNTL_H)
CHECK_INCLUDE_file("getopt.h" HAVE_GETOPT_H)
CHECK_INCLUDE_file("locale.h" HAVE_LOCALE_H)
CHECK_INCLUDE_file("stdint.h" HAVE_STDINT_H)
CHECK_INCLUDE_file("stdio.h" HAVE_STDIO_H)
if(MSVC)
CHECK_INCLUDE_file("io.h" HAVE_IO_H)
endif(MSVC)
CHECK_INCLUDE_file("stdlib.h" HAVE_STDLIB_H)
CHECK_INCLUDE_file("ctype.h" HAVE_CTYPE_H)
CHECK_INCLUDE_file("stdarg.h" HAVE_STDARG_H)
CHECK_INCLUDE_file("strings.h" HAVE_STRINGS_H)
CHECK_INCLUDE_file("signal.h" HAVE_SIGNAL_H)
CHECK_INCLUDE_file("sys/param.h" HAVE_SYS_PARAM_H)
CHECK_INCLUDE_file("sys/stat.h" HAVE_SYS_STAT_H)
CHECK_INCLUDE_file("sys/time.h" HAVE_SYS_TIME_H)
CHECK_INCLUDE_file("sys/types.h" HAVE_SYS_TYPES_H)
CHECK_INCLUDE_file("sys/mman.h" HAVE_SYS_MMAN_H)
CHECK_INCLUDE_file("sys/resource.h" HAVE_SYS_RESOURCE_H)
CHECK_INCLUDE_file("fcntl.h" HAVE_FCNTL_H)
CHECK_INCLUDE_file("inttypes.h" HAVE_INTTYPES_H)
CHECK_INCLUDE_file("pstdint.h" HAVE_PSTDINT_H)
CHECK_INCLUDE_file("endian.h" HAVE_ENDIAN_H)
CHECK_INCLUDE_file("BaseTsd.h" HAVE_BASETSD_H)
CHECK_INCLUDE_file("stddef.h" HAVE_STDDEF_H)
CHECK_INCLUDE_file("string.h" HAVE_STRING_H)
CHECK_INCLUDE_file("winsock2.h" HAVE_WINSOCK2_H)
CHECK_INCLUDE_file("ftw.h" HAVE_FTW_H)
CHECK_INCLUDE_file("libgen.h" HAVE_LIBGEN_H)
CHECK_INCLUDE_file("execinfo.h" HAVE_EXECINFO_H)
CHECK_INCLUDE_file("dirent.h" HAVE_DIRENT_H)
CHECK_INCLUDE_file("time.h" HAVE_TIME_H)
CHECK_INCLUDE_file("dlfcn.h" HAVE_DLFCN_H)
# Symbol Exists
CHECK_SYMBOL_EXISTS(isfinite "math.h" HAVE_DECL_ISFINITE)
CHECK_SYMBOL_EXISTS(isnan "math.h" HAVE_DECL_ISNAN)
CHECK_SYMBOL_EXISTS(isinf "math.h" HAVE_DECL_ISINF)
CHECK_SYMBOL_EXISTS(st_blksize "sys/stat.h" HAVE_STRUCT_STAT_ST_BLKSIZE)
CHECK_SYMBOL_EXISTS(alloca "alloca.h" HAVE_ALLOCA)
CHECK_SYMBOL_EXISTS(snprintf "stdio.h" HAVE_SNPRINTF)
2012-10-03 04:56:46 +08:00
# Type checks
# Aliases for automake consistency
2024-01-18 06:07:22 +08:00
set(SIZEOF_VOIDSTAR ${CMAKE_SIZEOF_VOID_P})
set(SIZEOF_VOIDP ${SIZEOF_VOIDSTAR})
CHECK_TYPE_SIZE("char" SIZEOF_CHAR)
CHECK_TYPE_SIZE("double" SIZEOF_DOUBLE)
CHECK_TYPE_SIZE("float" SIZEOF_FLOAT)
CHECK_TYPE_SIZE("int" SIZEOF_INT)
CHECK_TYPE_SIZE("uint" SIZEOF_UINT)
2024-01-18 06:07:22 +08:00
if(SIZEOF_UINT)
set(HAVE_UINT TRUE)
endif(SIZEOF_UINT)
2017-12-21 10:53:30 +08:00
CHECK_TYPE_SIZE("schar" SIZEOF_SCHAR)
2024-01-18 06:07:22 +08:00
if(SIZEOF_SCHAR)
set(HAVE_SCHAR TRUE)
endif(SIZEOF_SCHAR)
2017-12-21 10:53:30 +08:00
CHECK_TYPE_SIZE("long" SIZEOF_LONG)
CHECK_TYPE_SIZE("long long" SIZEOF_LONG_LONG)
2024-01-18 06:07:22 +08:00
if(SIZEOF_LONG_LONG)
set(HAVE_LONG_LONG_INT TRUE)
endif(SIZEOF_LONG_LONG)
CHECK_TYPE_SIZE("unsigned long long" SIZEOF_UNSIGNED_LONG_LONG)
CHECK_TYPE_SIZE("off_t" SIZEOF_OFF_T)
CHECK_TYPE_SIZE("off64_t" SIZEOF_OFF64_T)
CHECK_TYPE_SIZE("short" SIZEOF_SHORT)
CHECK_TYPE_SIZE("ushort" SIZEOF_USHORT)
2024-01-18 06:07:22 +08:00
if(SIZEOF_USHORT)
set(HAVE_USHORT TRUE)
endif(SIZEOF_USHORT)
CHECK_TYPE_SIZE("_Bool" SIZEOF__BOOL)
CHECK_TYPE_SIZE("size_t" SIZEOF_SIZE_T)
# Check whether to turn on or off CDF5 support.
2024-01-18 06:07:22 +08:00
if(SIZEOF_SIZE_T EQUAL 4)
if(NETCDF_ENABLE_CDF5) # enable or auto
string(TOUPPER ${NETCDF_ENABLE_CDF5} NETCDF_ENABLE_CDF5)
if(NETCDF_ENABLE_CDF5 AND NOT NETCDF_ENABLE_CDF5 STREQUAL "AUTO") # explicitly enabled
2024-01-18 06:07:22 +08:00
message(FATAL_ERROR "Unable to support CDF5 feature because size_t is less than 8 bytes")
endif(NETCDF_ENABLE_CDF5 AND NOT NETCDF_ENABLE_CDF5 STREQUAL "AUTO")
set(NETCDF_ENABLE_CDF5 OFF) # cannot support CDF5
2024-01-18 06:07:22 +08:00
set(USE_CDF5 OFF CACHE BOOL "") # cannot support CDF5
endif(NETCDF_ENABLE_CDF5)
2024-01-18 06:07:22 +08:00
else(SIZEOF_SIZE_T EQUAL 4)
if(NETCDF_ENABLE_CDF5) # explicitly set by user or not set
2024-01-18 06:07:22 +08:00
set(USE_CDF5 ON CACHE BOOL "")
else(NETCDF_ENABLE_CDF5) # explicitly disabled by user
2024-01-18 06:07:22 +08:00
set(USE_CDF5 OFF CACHE BOOL "")
endif(NETCDF_ENABLE_CDF5)
2024-01-18 06:07:22 +08:00
endif(SIZEOF_SIZE_T EQUAL 4)
CHECK_TYPE_SIZE("ssize_t" SIZEOF_SSIZE_T)
2024-01-18 06:07:22 +08:00
if(SIZEOF_SSIZE_T)
set(HAVE_SSIZE_T TRUE)
endif(SIZEOF_SSIZE_T)
CHECK_TYPE_SIZE("ptrdiff_t" SIZEOF_PTRDIFF_T)
2024-01-18 06:07:22 +08:00
if(SIZEOF_PTRDIFF_T)
set(HAVE_PTRDIFF_T TRUE)
endif(SIZEOF_PTRDIFF_T)
CHECK_TYPE_SIZE("uintptr_t" SIZEOF_UINTPTR_T)
2024-01-18 06:07:22 +08:00
if(SIZEOF_UINTPTR_T)
set(HAVE_UINTPTR_T TRUE)
endif(SIZEOF_UINTPTR_T)
Mitigate S3 test interference + Unlimited Dimensions in NCZarr This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
2023-09-27 06:56:48 +08:00
CHECK_TYPE_SIZE("mode_t" SIZEOF_MODE_T)
2024-01-18 06:07:22 +08:00
if(SIZEOF_MODE_T)
set(HAVE_MODE_T TRUE)
endif(SIZEOF_MODE_T)
2013-08-31 05:16:17 +08:00
# __int64 is used on Windows for large file support.
CHECK_TYPE_SIZE("__int64" SIZEOF___INT_64)
CHECK_TYPE_SIZE("int64_t" SIZEOF_INT64_T)
CHECK_TYPE_SIZE("uint64" SIZEOF_UINT64)
CHECK_TYPE_SIZE("unsigned char" SIZEOF_UCHAR)
CHECK_TYPE_SIZE("unsigned short int" SIZEOF_UNSIGNED_SHORT_INT)
CHECK_TYPE_SIZE("unsigned int" SIZEOF_UNSIGNED_INT)
CHECK_TYPE_SIZE("long long" SIZEOF_LONGLONG)
CHECK_TYPE_SIZE("unsigned long long" SIZEOF_ULONGLONG)
2012-10-03 04:56:46 +08:00
2020-11-20 13:24:13 +08:00
CHECK_TYPE_SIZE("uint64_t" SIZEOF_UINT64_T)
2024-01-18 06:07:22 +08:00
if(SIZEOF_UINT64_T)
set(HAVE_UINT64_T TRUE)
endif(SIZEOF_UINT64_T)
2020-11-20 13:24:13 +08:00
# On windows systems, we redefine off_t as __int64
# to enable LFS. This is true on 32 and 64 bit system.s
# We must redefine SIZEOF_OFF_T to match.
2024-01-18 06:07:22 +08:00
if(MSVC AND SIZEOF___INT_64)
set(SIZEOF_OFF_T ${SIZEOF___INT_64})
endif()
2013-08-31 05:16:17 +08:00
# Check for various functions.
2014-03-07 23:46:26 +08:00
CHECK_FUNCTION_EXISTS(fsync HAVE_FSYNC)
CHECK_FUNCTION_EXISTS(strlcat HAVE_STRLCAT)
Improve S3 Documentation and Support ## Improvements to S3 Documentation * Create a new document *quickstart_paths.md* that give a summary of the legal path formats used by netcdf-c. This includes both file paths and URL paths. * Modify *nczarr.md* to remove most of the S3 related text. * Move the S3 text from *nczarr.md* to a new document *cloud.md*. * Add some S3-related text to the *byterange.md* document. Hopefully, this will make it easier for users to find the information they want. ## Rebuild NCZarr Testing In order to avoid problems with running make check in parallel, two changes were made: 1. The *nczarr_test* test system was rebuilt. Now, for each test. any generated files are kept in a test-specific directory, isolated from all other test executions. 2. Similarly, since the S3 test bucket is shared, any generated S3 objects are isolated using a test-specific key path. ## Other S3 Related Changes * Add code to ensure that files created on S3 are reclaimed at end of testing. * Used the bash "trap" command to ensure S3 cleanup even if the test fails. * Cleanup the S3 related configure.ac flag set since S3 is used in several places. So now one should use the option *--enable-s3* instead of *--enable-nczarr-s3*, although the latter is still kept as a deprecated alias for the former. * Get some of the github actions yml to work with S3; required fixing various test scripts adding a secret to access the Unidata S3 bucket. * Cleanup S3 portion of libnetcdf.settings.in and netcdf_meta.h.in and test_common.in. * Merge partial S3 support into dhttp.c. * Create an experimental s3 access library especially for use with Windows. It is enabled by using the options *--enable-s3-internal* (automake) or *-DENABLE_S3_INTERNAL=ON* (CMake). Also add a unit-test for it. * Move some definitions from ncrc.h to ncs3sdk.h ## Other Changes * Provide a default implementation of strlcpy and move this and similar defaults into *dmissing.c*.
2023-04-26 07:15:06 +08:00
CHECK_FUNCTION_EXISTS(strlcpy HAVE_STRLCPY)
2014-03-07 23:46:26 +08:00
CHECK_FUNCTION_EXISTS(strdup HAVE_STRDUP)
CHECK_FUNCTION_EXISTS(strndup HAVE_STRNDUP)
2014-03-07 23:46:26 +08:00
CHECK_FUNCTION_EXISTS(strtoll HAVE_STRTOLL)
CHECK_FUNCTION_EXISTS(strcasecmp HAVE_STRCASECMP)
CHECK_FUNCTION_EXISTS(strtoull HAVE_STRTOULL)
2014-03-07 23:46:26 +08:00
CHECK_FUNCTION_EXISTS(mkstemp HAVE_MKSTEMP)
CHECK_FUNCTION_EXISTS(mktemp HAVE_MKTEMP)
CHECK_FUNCTION_EXISTS(random HAVE_RANDOM)
2014-03-07 23:46:26 +08:00
CHECK_FUNCTION_EXISTS(gettimeofday HAVE_GETTIMEOFDAY)
CHECK_FUNCTION_EXISTS(MPI_Comm_f2c HAVE_MPI_COMM_F2C)
2018-09-22 06:32:36 +08:00
CHECK_FUNCTION_EXISTS(MPI_Info_f2c HAVE_MPI_INFO_F2C)
2014-03-07 23:46:26 +08:00
CHECK_FUNCTION_EXISTS(memmove HAVE_MEMMOVE)
CHECK_FUNCTION_EXISTS(getpagesize HAVE_GETPAGESIZE)
CHECK_FUNCTION_EXISTS(sysconf HAVE_SYSCONF)
CHECK_FUNCTION_EXISTS(getrlimit HAVE_GETRLIMIT)
CHECK_FUNCTION_EXISTS(_filelengthi64 HAVE_FILE_LENGTH_I64)
CHECK_FUNCTION_EXISTS(mmap HAVE_MMAP)
CHECK_FUNCTION_EXISTS(mremap HAVE_MREMAP)
CHECK_FUNCTION_EXISTS(fileno HAVE_FILENO)
CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME)
CHECK_SYMBOL_EXISTS("struct timespec" "time.h" HAVE_STRUCT_TIMESPEC)
CHECK_FUNCTION_EXISTS(atexit HAVE_ATEXIT)
# Control invoking nc_finalize at exit
2024-03-19 04:30:20 +08:00
option(NETCDF_ENABLE_ATEXIT_FINALIZE "Invoke nc_finalize at exit." ON)
2024-01-18 06:07:22 +08:00
if(NOT HAVE_ATEXIT)
2024-03-19 04:30:20 +08:00
if(NETCDF_ENABLE_ATEXIT_FINALIZE AND NOT HAVE_ATEXIT)
set(NETCDF_ENABLE_ATEXIT_FINALIZE OFF CACHE BOOL "Enable ATEXIT" FORCE)
message(WARNING "NETCDF_ENABLE_ATEXIT_FINALIZE set but atexit() function not defined")
2024-01-18 06:07:22 +08:00
endif()
endif()
# Check to see if MAP_ANONYMOUS is defined.
2024-01-18 06:07:22 +08:00
if(MSVC)
message(WARNING "mmap not supported under visual studio: disabling MMAP support.")
set(ENABLE_MMAP OFF)
else()
2023-08-31 00:17:00 +08:00
CHECK_C_SOURCE_COMPILES("
#include <sys/mman.h>
int main() {int x = MAP_ANONYMOUS;}" HAVE_MAPANON)
2024-01-18 06:07:22 +08:00
if(NOT HAVE_MMAP OR NOT HAVE_MAPANON)
message(WARNING "mmap or MAP_ANONYMOUS not found: disabling MMAP support.")
set(ENABLE_MMAP OFF)
endif()
endif()
Revert/Improve nc_create + NC_DISKLESS behavior re: https://github.com/Unidata/netcdf-c/issues/1154 Inadvertently, the behavior of NC_DISKLESS with nc_create() was changed in release 4.6.1. Previously, the NC_WRITE flag needed to be explicitly used with NC_DISKLESS in order to cause the created file to be persisted to disk. Additional analyis indicated that the current NC_DISKLESS implementation was seriously flawed. This PR attempts to clean up and regularize the situation with respect to NC_DISKLESS control. One important aspect of diskless operation is that there are two different notions of write. 1. The file is read-write vs read-only when using the netcdf API. 2. The file is persisted or not to disk at nc_close(). Previously, these two were conflated. The rules now are as follows. 1. NC_DISKLESS + NC_WRITE means that the file is read/write using the netcdf API 2. NC_DISKLESS + NC_PERSIST means that the file is persisted to a disk file at nc_close. 3. NC_DISKLESS + NC_PERSIST + NC_WRITE means both 1 and 2. The NC_PERSIST flag is new and takes over the obsolete NC_MPIPOSIX flag. NC_MPIPOSIX is still defined, but is now an alias for the NC_MPIIO flag. It is also now the case that for netcdf-4, NC_DISKLESS is independent of NC_INMEMORY and in fact it is an error to specify both flags simultaneously. Finally, the MMAP code was fixed to use NC_PERSIST as well. Also marked MMAP as deprecated. Also added a test case to test various combinations of NC_DISKLESS, NC_PERSIST, and NC_WRITE. This PR affects a number of files and especially test cases that used NC_DISKLESS. Misc. Unrelated fixes 1. fixed some warnings in ncdump/dumplib.c
2018-10-11 03:32:17 +08:00
2024-01-18 06:07:22 +08:00
if(ENABLE_MMAP)
# Aliases
2024-01-18 06:07:22 +08:00
set(BUILD_MMAP ON)
set(USE_MMAP ON)
endif(ENABLE_MMAP)
#CHECK_FUNCTION_EXISTS(alloca HAVE_ALLOCA)
Improve UTF8 Support On Windows re: Issue https://github.com/Unidata/netcdf-c/issues/2190 The primary purpose of this PR is to improve the utf8 support for windows. This is persuant to a change in Windows that supports utf8 natively (almost). The almost means that it is still utf16 internally and the set of characters representable by utf8 is larger than those representable by utf16. This leaves open the question in the Issue about handling the Windows 1252 character set. This required the following changes: 1. Test the Windows build and major version in order to see if native utf8 is supported. 2. If native utf8 is supported, Modify dpathmgr.c to call the 8-bit version of the windows fopen() and open() functions. 3. In support of this, programs that use XGetOpt (Windows versions) need to get the command line as utf8 and then parse to arc+argv as utf8. This requires using a homegrown command line parser named XCommandLineToArgvA. 4. Add a utility program called "acpget" that prints out the current Windows code page and locale. Additionally, some technical debt was cleaned up as follows: 1. Unify all the places which attempt to read all or a part of a file into the dutil.c#NC_readfile code. 2. Similary unify all the code that creates temp files into dutil.c#NC_mktmp code. 3. Convert almost all remaining calls to fopen() and open() to NCfopen() and NCopen3(). This is to ensure that path management is used consistently. This touches a number of files. 4. extern->EXTERNL as needed to get it to work under Windows.
2022-02-09 11:53:30 +08:00
# Used in the `configure_file` calls below
2024-01-18 06:07:22 +08:00
set(ISCMAKE "yes")
if(MSVC)
set(ISMSVC ON CACHE BOOL "" FORCE)
set(REGEDIT ON CACHE BOOL "" FORCE)
# Get windows major version and build number
2024-01-18 06:07:22 +08:00
execute_process(COMMAND "systeminfo" OUTPUT_VARIABLE WININFO)
if(WININFO STREQUAL "")
set(WVM 0)
set(WVB 0)
else()
string(REGEX MATCH "\nOS Version:[ \t]+[0-9.]+" WINVERLINE "${WININFO}")
string(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\1" WVM "${WINVERLINE}")
string(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\3" WVB "${WINVERLINE}")
endif()
set(WINVERMAJOR ${WVM} CACHE STRING "" FORCE)
set(WINVERBUILD ${WVB} CACHE STRING "" FORCE)
endif()
Improve UTF8 Support On Windows re: Issue https://github.com/Unidata/netcdf-c/issues/2190 The primary purpose of this PR is to improve the utf8 support for windows. This is persuant to a change in Windows that supports utf8 natively (almost). The almost means that it is still utf16 internally and the set of characters representable by utf8 is larger than those representable by utf16. This leaves open the question in the Issue about handling the Windows 1252 character set. This required the following changes: 1. Test the Windows build and major version in order to see if native utf8 is supported. 2. If native utf8 is supported, Modify dpathmgr.c to call the 8-bit version of the windows fopen() and open() functions. 3. In support of this, programs that use XGetOpt (Windows versions) need to get the command line as utf8 and then parse to arc+argv as utf8. This requires using a homegrown command line parser named XCommandLineToArgvA. 4. Add a utility program called "acpget" that prints out the current Windows code page and locale. Additionally, some technical debt was cleaned up as follows: 1. Unify all the places which attempt to read all or a part of a file into the dutil.c#NC_readfile code. 2. Similary unify all the code that creates temp files into dutil.c#NC_mktmp code. 3. Convert almost all remaining calls to fopen() and open() to NCfopen() and NCopen3(). This is to ensure that path management is used consistently. This touches a number of files. 4. extern->EXTERNL as needed to get it to work under Windows.
2022-02-09 11:53:30 +08:00
2012-10-03 04:56:46 +08:00
#####
# End system inspection checks.
#####
# A basic script used to convert m4 files
2024-01-18 06:07:22 +08:00
find_program(NC_M4 NAMES m4 m4.exe)
if(NC_M4)
message(STATUS "Found m4: ${NC_M4}")
set(HAVE_M4 TRUE)
else()
message(STATUS "m4 not found.")
set(HAVE_M4 FALSE)
endif()
##specific
# Shell script Macro
##
# Determine if 'bash' is on the system.
##
option(NETCDF_ENABLE_BASH_SCRIPT_TESTING "Detection is typically automatic, but this option can be used to force enable/disable bash-script based tests." ON)
2024-01-18 06:07:22 +08:00
if(NETCDF_ENABLE_BASH_SCRIPT_TESTING)
2024-01-18 06:07:22 +08:00
find_program(HAVE_BASH bash)
if(HAVE_BASH)
string(COMPARE EQUAL "${HAVE_BASH}" "C:/Windows/System32/bash.exe" IS_BASH_EXE)
if(NOT IS_BASH_EXE)
message(STATUS "Found bash: ${HAVE_BASH}")
else()
message(STATUS "Ignoring ${HAVE_BASH}")
set(HAVE_BASH "")
endif()
else()
message(STATUS "Bash shell not found; disabling shell script tests.")
endif()
else(NETCDF_ENABLE_BASH_SCRIPT_TESTING)
2024-01-18 06:07:22 +08:00
set(HAVE_BASH "")
endif(NETCDF_ENABLE_BASH_SCRIPT_TESTING)
# Create config.h file.
configure_file("${netCDF_SOURCE_DIR}/config.h.cmake.in"
2014-03-07 23:46:26 +08:00
"${netCDF_BINARY_DIR}/config.h")
2024-01-18 06:07:22 +08:00
add_definitions(-DHAVE_CONFIG_H)
2024-01-18 06:07:22 +08:00
include_directories(${netCDF_BINARY_DIR})
2024-01-17 01:20:43 +08:00
# End autotools-style checks for config.h
2012-10-03 04:56:46 +08:00
#####
# Set the true names of all the libraries, if customized by external project
#####
# Recurse into other subdirectories.
add_subdirectory("include")
add_subdirectory(libdispatch)
add_subdirectory(libsrc)
2024-01-18 06:07:22 +08:00
if(USE_PNETCDF)
add_subdirectory(libsrcp)
2024-01-18 06:07:22 +08:00
endif(USE_PNETCDF)
2013-03-16 04:31:07 +08:00
2024-01-18 06:07:22 +08:00
if(USE_NETCDF4)
add_subdirectory(libsrc4)
2024-01-18 06:07:22 +08:00
endif()
2024-01-18 06:07:22 +08:00
if(USE_HDF5)
2018-05-09 01:58:01 +08:00
add_subdirectory(libhdf5)
2024-01-18 06:07:22 +08:00
endif(USE_HDF5)
2012-10-03 04:56:46 +08:00
2024-01-18 06:07:22 +08:00
if(USE_HDF4)
2018-02-08 21:20:58 +08:00
add_subdirectory(libhdf4)
2018-03-05 19:22:15 +08:00
add_subdirectory(hdf4_test)
2024-01-18 06:07:22 +08:00
endif(USE_HDF4)
if(NETCDF_ENABLE_DAP2)
2024-01-18 06:07:22 +08:00
add_subdirectory(oc2)
add_subdirectory(libdap2)
endif()
if(NETCDF_ENABLE_DAP4)
2024-01-18 06:07:22 +08:00
add_subdirectory(libdap4)
add_subdirectory(libncxml)
else()
if(ENABLE_S3_INTERNAL)
add_subdirectory(libncxml)
endif()
endif()
if(ENABLE_PLUGINS)
add_subdirectory(libncpoco)
endif()
if(ENABLE_NCZARR)
add_subdirectory(libnczarr)
file(COPY ${netCDF_SOURCE_DIR}/unit_test/timer_utils.h
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
2024-01-18 06:07:22 +08:00
file(COPY ${netCDF_SOURCE_DIR}/unit_test/timer_utils.c
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
2024-01-18 06:07:22 +08:00
file(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter.c
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
2024-01-18 06:07:22 +08:00
file(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter_misc.c
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
2024-01-18 06:07:22 +08:00
file(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter_repeat.c
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
2024-01-18 06:07:22 +08:00
file(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter_order.c
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
2024-01-18 06:07:22 +08:00
file(COPY ${netCDF_SOURCE_DIR}/nc_test4/tst_multifilter.c
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
2024-01-18 06:07:22 +08:00
endif()
This PR adds EXPERIMENTAL support for accessing data in the cloud using a variant of the Zarr protocol and storage format. This enhancement is generically referred to as "NCZarr". The data model supported by NCZarr is netcdf-4 minus the user-defined types and the String type. In this sense it is similar to the CDF-5 data model. More detailed information about enabling and using NCZarr is described in the document NUG/nczarr.md and in a [Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in). WARNING: this code has had limited testing, so do use this version for production work. Also, performance improvements are ongoing. Note especially the following platform matrix of successful tests: Platform | Build System | S3 support ------------------------------------ Linux+gcc | Automake | yes Linux+gcc | CMake | yes Visual Studio | CMake | no Additionally, and as a consequence of the addition of NCZarr, major changes have been made to the Filter API. NOTE: NCZarr does not yet support filters, but these changes are enablers for that support in the future. Note that it is possible (probable?) that there will be some accidental reversions if the changes here did not correctly mimic the existing filter testing. In any case, previously filter ids and parameters were of type unsigned int. In order to support the more general zarr filter model, this was all converted to char*. The old HDF5-specific, unsigned int operations are still supported but they are wrappers around the new, char* based nc_filterx_XXX functions. This entailed at least the following changes: 1. Added the files libdispatch/dfilterx.c and include/ncfilter.h 2. Some filterx utilities have been moved to libdispatch/daux.c 3. A new entry, "filter_actions" was added to the NCDispatch table and the version bumped. 4. An overly complex set of structs was created to support funnelling all of the filterx operations thru a single dispatch "filter_actions" entry. 5. Move common code to from libhdf5 to libsrc4 so that it is accessible to nczarr. Changes directly related to Zarr: 1. Modified CMakeList.txt and configure.ac to support both C and C++ -- this is in support of S3 support via the awd-sdk libraries. 2. Define a size64_t type to support nczarr. 3. More reworking of libdispatch/dinfermodel.c to support zarr and to regularize the structure of the fragments section of a URL. Changes not directly related to Zarr: 1. Make client-side filter registration be conditional, with default off. 2. Hack include/nc4internal.h to make some flags added by Ed be unique: e.g. NC_CREAT, NC_INDEF, etc. 3. cleanup include/nchttp.h and libdispatch/dhttp.c. 4. Misc. changes to support compiling under Visual Studio including: * Better testing under windows for dirent.h and opendir and closedir. 5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags and to centralize error reporting. 6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them. 7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible. Changes Left TO-DO: 1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
2012-10-03 04:56:46 +08:00
add_subdirectory(liblib)
2024-01-18 06:07:22 +08:00
if(ENABLE_PLUGINS)
add_subdirectory(plugins)
2024-01-18 06:07:22 +08:00
endif()
2012-10-10 04:56:27 +08:00
# Enable Utilities.
2024-03-19 04:29:24 +08:00
if(NETCDF_BUILD_UTILITIES)
2024-01-18 06:07:22 +08:00
include_directories(ncdump)
add_subdirectory(ncgen)
add_subdirectory(ncgen3)
add_subdirectory(ncdump)
endif()
2012-10-03 04:56:46 +08:00
# Enable tests
2024-01-18 06:07:22 +08:00
if(ENABLE_TESTS)
if(ENABLE_V2_API)
add_subdirectory(nctest)
endif()
add_subdirectory(nc_test)
if(USE_HDF5)
include_directories(h5_test)
add_subdirectory(nc_test4)
add_subdirectory(h5_test)
endif()
if(NETCDF_ENABLE_DAP2)
2024-01-18 06:07:22 +08:00
add_subdirectory(ncdap_test)
endif()
if(NETCDF_ENABLE_DAP4)
2024-01-18 06:07:22 +08:00
add_subdirectory(dap4_test)
endif()
if(NETCDF_ENABLE_EXAMPLES)
2024-01-18 06:07:22 +08:00
add_subdirectory(examples)
endif()
if(NETCDF_ENABLE_BENCHMARKS)
2024-01-18 06:07:22 +08:00
add_subdirectory(nc_perf)
endif(NETCDF_ENABLE_BENCHMARKS)
2024-01-18 06:07:22 +08:00
if(ENABLE_UNIT_TESTS)
add_subdirectory(unit_test)
endif(ENABLE_UNIT_TESTS)
if(ENABLE_NCZARR)
add_subdirectory(nczarr_test)
endif()
endif()
2012-10-03 04:56:46 +08:00
# Code to generate an export header
#GENERATE_EXPORT_HEADER(netcdf
2014-03-07 23:46:26 +08:00
# BASE_NAME netcdf
# EXPORT_MACRO_NAME netcdf_EXPORT
# EXPORT_FILE_NAME netcdf_Export.h
# STATIC_DEFINE netcdf_BUILT_AS_STATIC
#)
2012-10-03 04:56:46 +08:00
#####
# Build doxygen documentation, if need be.
#####
2024-01-18 06:07:22 +08:00
add_subdirectory(docs)
##
# Brute force, grab all of the dlls from the dependency directory,
# install them in the binary dir. Grab all of the .libs, put them
# in the libdir.
##
2024-01-18 06:07:22 +08:00
if(MSVC)
file(GLOB COPY_FILES ${CMAKE_PREFIX_PATH}/lib/*.lib)
install(FILES ${COPY_FILES}
DESTINATION ${CMAKE_INSTALL_LIBDIR}
COMPONENT dependencies)
2024-01-18 06:07:22 +08:00
file(GLOB COPY_FILES ${CMAKE_PREFIX_PATH}/bin/*.dll)
string(REGEX REPLACE "msv[.*].dll" "" COPY_FILES "${COPY_FILES}")
install(FILES ${COPY_FILES}
DESTINATION ${CMAKE_INSTALL_BINDIR}
COMPONENT dependencies)
2024-01-18 06:07:22 +08:00
endif()
2012-10-03 04:56:46 +08:00
# Subdirectory CMakeLists.txt files should specify their own
# 'install' files.
# Including 'CPack' kicks everything off.
2024-01-18 06:07:22 +08:00
include(InstallRequiredSystemLibraries)
configure_file(
2012-10-03 04:56:46 +08:00
${CMAKE_CURRENT_SOURCE_DIR}/FixBundle.cmake.in
${CMAKE_CURRENT_BINARY_DIR}/FixBundle.cmake
@ONLY
)
2012-10-03 04:56:46 +08:00
###
# Create pkgconfig files.
###
2024-01-18 06:07:22 +08:00
if(NOT DEFINED CMAKE_INSTALL_LIBDIR)
set(CMAKE_INSTALL_LIBDIR lib)
endif(NOT DEFINED CMAKE_INSTALL_LIBDIR)
# Set
2024-01-18 06:07:22 +08:00
set(prefix ${CMAKE_INSTALL_PREFIX})
set(exec_prefix ${CMAKE_INSTALL_PREFIX})
set(libdir ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR})
set(includedir ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR})
set(CC ${CMAKE_C_COMPILER})
# Process all dependency libraries and create a string
# used when parsing netcdf.pc.in
2024-01-18 06:07:22 +08:00
set(NC_LIBS "")
foreach(_LIB ${ALL_TLL_LIBS})
get_filename_component(_LIB_NAME ${_LIB} NAME_WE)
string(REGEX REPLACE "^lib" "" _NAME ${_LIB_NAME})
list(APPEND NC_LIBS "-l${_NAME}")
get_filename_component(_LIB_DIR ${_LIB} PATH)
list(APPEND LINKFLAGS "-L${_LIB_DIR}")
2024-01-19 05:07:13 +08:00
endforeach()
2024-01-18 06:07:22 +08:00
#set(NC_LIBS "-lnetcdf ${NC_LIBS}")
if(NC_LIBS)
string(REPLACE ";" " " NC_LIBS "${NC_LIBS}")
string(REPLACE "-lhdf5::hdf5-shared" "-lhdf5" NC_LIBS ${NC_LIBS})
string(REPLACE "-lhdf5::hdf5_hl-shared" "-lhdf5_hl" NC_LIBS ${NC_LIBS})
string(REPLACE "-lhdf5::hdf5-static" "-lhdf5" NC_LIBS ${NC_LIBS})
string(REPLACE "-lhdf5::hdf5_hl-static" "-lhdf5_hl" NC_LIBS ${NC_LIBS})
endif()
2024-01-18 06:07:22 +08:00
string(REPLACE ";" " " LINKFLAGS "${LINKFLAGS}")
2024-01-18 06:07:22 +08:00
list(REMOVE_DUPLICATES NC_LIBS)
list(REMOVE_DUPLICATES LINKFLAGS)
2024-01-18 06:07:22 +08:00
set(LIBS ${NC_LIBS})
set(NC_LIBS "-lnetcdf")
configure_file(
${netCDF_SOURCE_DIR}/netcdf.pc.in
${netCDF_BINARY_DIR}/netcdf.pc @ONLY)
2024-01-18 06:07:22 +08:00
if(NOT IS_DIRECTORY ${netCDF_BINARY_DIR}/tmp)
file(MAKE_DIRECTORY ${netCDF_BINARY_DIR}/tmp)
endif()
configure_file("${netCDF_SOURCE_DIR}/nc-config.cmake.in"
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
"${netCDF_BINARY_DIR}/tmp/nc-config" @ONLY
NEWLINE_STYLE LF)
2024-01-18 06:07:22 +08:00
file(COPY "${netCDF_BINARY_DIR}/tmp/nc-config"
DESTINATION ${netCDF_BINARY_DIR}/
FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
2024-01-18 06:07:22 +08:00
install(FILES ${netCDF_BINARY_DIR}/netcdf.pc
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
COMPONENT utilities)
2024-01-18 06:07:22 +08:00
install(PROGRAMS ${netCDF_BINARY_DIR}/nc-config
DESTINATION ${CMAKE_INSTALL_BINDIR}
COMPONENT utilities)
###
# End pkgconfig, nc-config file creation.
###
2012-10-03 04:56:46 +08:00
##
# Print the configuration summary
##
2012-10-03 04:56:46 +08:00
print_conf_summary()
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
# Enable Makedist files.
ADD_MAKEDIST()
ENABLE_MAKEDIST(README.md COPYRIGHT RELEASE_NOTES.md INSTALL INSTALL.cmake test_prog.c lib_flags.am cmake CMakeLists.txt COMPILE.cmake.txt config.h.cmake.in cmake_uninstall.cmake.in netcdf-config-version.cmake.in netcdf-config.cmake.in FixBundle.cmake.in nc-config.cmake.in configure configure.ac install-sh config.h.in config.sub CTestConfig.cmake.in)
#####
# Configure and print the libnetcdf.settings file.
#####
# Set variables to mirror those used by autoconf.
# This way we don't need to maintain two separate template
# files.
2024-01-18 06:07:22 +08:00
set(host_cpu "${cpu}")
set(host_vendor "${osname}")
set(host_os "${osrel}")
set(abs_top_builddir "${CMAKE_CURRENT_BINARY_DIR}")
set(abs_top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}")
string(RANDOM LENGTH 3 ALPHABET "0123456789" PLATFORMUID)
math(EXPR PLATFORMUID "${PLATFORMUID} + 1" OUTPUT_FORMAT DECIMAL)
Mitigate S3 test interference + Unlimited Dimensions in NCZarr This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
2023-09-27 06:56:48 +08:00
2024-01-18 06:07:22 +08:00
set(CC_VERSION "${CMAKE_C_COMPILER}")
# Build *FLAGS for libnetcdf.settings.
2024-01-18 06:07:22 +08:00
set(CFLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}}")
set(CPPFLAGS "${CMAKE_CPP_FLAGS} ${CMAKE_CPP_FLAGS_${CMAKE_BUILD_TYPE}}")
set(LDFLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CMAKE_SHARED_LINKER_FLAGS_${CMAKE_BUILD_TYPE}}")
is_disabled(BUILD_SHARED_LIBS enable_static)
is_enabled(BUILD_SHARED_LIBS enable_shared)
is_enabled(ENABLE_V2_API HAS_NC2)
is_enabled(ENABLE_NETCDF_4 HAS_NC4)
is_enabled(NETCDF_ENABLE_HDF4 HAS_HDF4)
is_enabled(USE_HDF5 HAS_HDF5)
2022-04-26 20:18:52 +08:00
is_enabled(OFF HAS_BENCHMARKS)
is_enabled(STATUS_PNETCDF HAS_PNETCDF)
is_enabled(STATUS_PARALLEL HAS_PARALLEL)
is_enabled(ENABLE_PARALLEL4 HAS_PARALLEL4)
is_enabled(NETCDF_ENABLE_DAP HAS_DAP)
is_enabled(NETCDF_ENABLE_DAP2 HAS_DAP2)
is_enabled(NETCDF_ENABLE_DAP4 HAS_DAP4)
is_enabled(NETCDF_ENABLE_BYTERANGE HAS_BYTERANGE)
is_enabled(NETCDF_ENABLE_DISKLESS HAS_DISKLESS)
is_enabled(USE_MMAP HAS_MMAP)
is_enabled(JNA HAS_JNA)
is_enabled(ENABLE_ZERO_LENGTH_COORD_BOUND RELAX_COORD_BOUND)
2017-09-27 04:01:21 +08:00
is_enabled(USE_CDF5 HAS_CDF5)
is_enabled(NETCDF_ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
Fix byterange handling of some URLS re: Issue The byterange handling of the following URLS fails. ### Problem 1: "https://crudata.uea.ac.uk/cru/data/temperature/HadCRUT.4.6.0.0.median.nc#mode=bytes" It turns out that byterange in hdf5 has two possible targets: S3 and not-S3 (e.g. a thredds server or the crudata URL above). Each uses a different HDF5 Virtual File Driver (VFD). I incorrectly set up the byterange code in libhdf5 so that it would choose one or the other of the two VFD's for any netcdf-c library build. The fix is to allow it to choose either one at run-time. ### Problem 2: "https://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" When given what appears to be an S3-related URL, the netcdf-c library code converts it into a canonical, so-called "path" format. In casing out the possible input URL formats, I missed the case where the host contains the bucket ("noaa-goes16"), but not the region. So the fix was to check for this case. ## Misc. Related Changes 1. Since S3 is used in more than just NCZarr, I changed the automake/cmake options to replace "--enable-nczarr-s3" with "--enable-s3", but keeping the former option as a synonym for the latter. This also entailed cleaning up libnetcdf.settings WRT S3 support 2. Added the above URLS as additional test cases ## Misc. Un-Related Changes 1. CURLOPT_PUT is deprecated in favor to CURLOPT_UPLOAD 2. Fix some minor warnings ## Open Problems * Under Ubuntu, either libcrypto or aws-sdk-cpp has a memory leak.
2023-03-03 10:51:02 +08:00
is_enabled(ENABLE_S3 HAS_S3)
Improve S3 Documentation and Support ## Improvements to S3 Documentation * Create a new document *quickstart_paths.md* that give a summary of the legal path formats used by netcdf-c. This includes both file paths and URL paths. * Modify *nczarr.md* to remove most of the S3 related text. * Move the S3 text from *nczarr.md* to a new document *cloud.md*. * Add some S3-related text to the *byterange.md* document. Hopefully, this will make it easier for users to find the information they want. ## Rebuild NCZarr Testing In order to avoid problems with running make check in parallel, two changes were made: 1. The *nczarr_test* test system was rebuilt. Now, for each test. any generated files are kept in a test-specific directory, isolated from all other test executions. 2. Similarly, since the S3 test bucket is shared, any generated S3 objects are isolated using a test-specific key path. ## Other S3 Related Changes * Add code to ensure that files created on S3 are reclaimed at end of testing. * Used the bash "trap" command to ensure S3 cleanup even if the test fails. * Cleanup the S3 related configure.ac flag set since S3 is used in several places. So now one should use the option *--enable-s3* instead of *--enable-nczarr-s3*, although the latter is still kept as a deprecated alias for the former. * Get some of the github actions yml to work with S3; required fixing various test scripts adding a secret to access the Unidata S3 bucket. * Cleanup S3 portion of libnetcdf.settings.in and netcdf_meta.h.in and test_common.in. * Merge partial S3 support into dhttp.c. * Create an experimental s3 access library especially for use with Windows. It is enabled by using the options *--enable-s3-internal* (automake) or *-DENABLE_S3_INTERNAL=ON* (CMake). Also add a unit-test for it. * Move some definitions from ncrc.h to ncs3sdk.h ## Other Changes * Provide a default implementation of strlcpy and move this and similar defaults into *dmissing.c*.
2023-04-26 07:15:06 +08:00
is_enabled(ENABLE_S3_AWS HAS_S3_AWS)
is_enabled(ENABLE_S3_INTERNAL HAS_S3_INTERNAL)
is_enabled(HAS_HDF5_ROS3 HAS_HDF5_ROS3)
This PR adds EXPERIMENTAL support for accessing data in the cloud using a variant of the Zarr protocol and storage format. This enhancement is generically referred to as "NCZarr". The data model supported by NCZarr is netcdf-4 minus the user-defined types and the String type. In this sense it is similar to the CDF-5 data model. More detailed information about enabling and using NCZarr is described in the document NUG/nczarr.md and in a [Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in). WARNING: this code has had limited testing, so do use this version for production work. Also, performance improvements are ongoing. Note especially the following platform matrix of successful tests: Platform | Build System | S3 support ------------------------------------ Linux+gcc | Automake | yes Linux+gcc | CMake | yes Visual Studio | CMake | no Additionally, and as a consequence of the addition of NCZarr, major changes have been made to the Filter API. NOTE: NCZarr does not yet support filters, but these changes are enablers for that support in the future. Note that it is possible (probable?) that there will be some accidental reversions if the changes here did not correctly mimic the existing filter testing. In any case, previously filter ids and parameters were of type unsigned int. In order to support the more general zarr filter model, this was all converted to char*. The old HDF5-specific, unsigned int operations are still supported but they are wrappers around the new, char* based nc_filterx_XXX functions. This entailed at least the following changes: 1. Added the files libdispatch/dfilterx.c and include/ncfilter.h 2. Some filterx utilities have been moved to libdispatch/daux.c 3. A new entry, "filter_actions" was added to the NCDispatch table and the version bumped. 4. An overly complex set of structs was created to support funnelling all of the filterx operations thru a single dispatch "filter_actions" entry. 5. Move common code to from libhdf5 to libsrc4 so that it is accessible to nczarr. Changes directly related to Zarr: 1. Modified CMakeList.txt and configure.ac to support both C and C++ -- this is in support of S3 support via the awd-sdk libraries. 2. Define a size64_t type to support nczarr. 3. More reworking of libdispatch/dinfermodel.c to support zarr and to regularize the structure of the fragments section of a URL. Changes not directly related to Zarr: 1. Make client-side filter registration be conditional, with default off. 2. Hack include/nc4internal.h to make some flags added by Ed be unique: e.g. NC_CREAT, NC_INDEF, etc. 3. cleanup include/nchttp.h and libdispatch/dhttp.c. 4. Misc. changes to support compiling under Visual Studio including: * Better testing under windows for dirent.h and opendir and closedir. 5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags and to centralize error reporting. 6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them. 7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible. Changes Left TO-DO: 1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
is_enabled(ENABLE_NCZARR HAS_NCZARR)
is_enabled(ENABLE_NCZARR_ZIP HAS_NCZARR_ZIP)
is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
is_enabled(ENABLE_QUANTIZE HAS_QUANTIZE)
is_enabled(ENABLE_LOGGING HAS_LOGGING)
is_enabled(NETCDF_ENABLE_FILTER_TESTING DO_FILTER_TESTS)
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
is_enabled(HAVE_SZ HAS_SZIP)
is_enabled(HAVE_SZ HAS_SZLIB_WRITE)
is_enabled(HAVE_ZSTD HAS_ZSTD)
is_enabled(HAVE_BLOSC HAS_BLOSC)
is_enabled(HAVE_BZ2 HAS_BZ2)
is_enabled(ENABLE_REMOTE_FUNCTIONALITY DO_REMOTE_FUNCTIONALITY)
if(ENABLE_S3_INTERNAL)
2024-01-18 06:07:22 +08:00
set(WHICH_S3_SDK "internal")
set(NC_WHICH_S3_SDK "internal")
elseif(ENABLE_S3_AWS)
2024-01-18 06:07:22 +08:00
set(WHICH_S3_SDK "aws-sdk-cpp")
set(NC_WHICH_S3_SDK "aws-sdk-cpp")
Improve S3 Documentation and Support ## Improvements to S3 Documentation * Create a new document *quickstart_paths.md* that give a summary of the legal path formats used by netcdf-c. This includes both file paths and URL paths. * Modify *nczarr.md* to remove most of the S3 related text. * Move the S3 text from *nczarr.md* to a new document *cloud.md*. * Add some S3-related text to the *byterange.md* document. Hopefully, this will make it easier for users to find the information they want. ## Rebuild NCZarr Testing In order to avoid problems with running make check in parallel, two changes were made: 1. The *nczarr_test* test system was rebuilt. Now, for each test. any generated files are kept in a test-specific directory, isolated from all other test executions. 2. Similarly, since the S3 test bucket is shared, any generated S3 objects are isolated using a test-specific key path. ## Other S3 Related Changes * Add code to ensure that files created on S3 are reclaimed at end of testing. * Used the bash "trap" command to ensure S3 cleanup even if the test fails. * Cleanup the S3 related configure.ac flag set since S3 is used in several places. So now one should use the option *--enable-s3* instead of *--enable-nczarr-s3*, although the latter is still kept as a deprecated alias for the former. * Get some of the github actions yml to work with S3; required fixing various test scripts adding a secret to access the Unidata S3 bucket. * Cleanup S3 portion of libnetcdf.settings.in and netcdf_meta.h.in and test_common.in. * Merge partial S3 support into dhttp.c. * Create an experimental s3 access library especially for use with Windows. It is enabled by using the options *--enable-s3-internal* (automake) or *-DENABLE_S3_INTERNAL=ON* (CMake). Also add a unit-test for it. * Move some definitions from ncrc.h to ncs3sdk.h ## Other Changes * Provide a default implementation of strlcpy and move this and similar defaults into *dmissing.c*.
2023-04-26 07:15:06 +08:00
else()
2024-01-18 06:07:22 +08:00
set(WHICH_S3_SDK "none")
set(NC_WHICH_S3_SDK "none")
Improve S3 Documentation and Support ## Improvements to S3 Documentation * Create a new document *quickstart_paths.md* that give a summary of the legal path formats used by netcdf-c. This includes both file paths and URL paths. * Modify *nczarr.md* to remove most of the S3 related text. * Move the S3 text from *nczarr.md* to a new document *cloud.md*. * Add some S3-related text to the *byterange.md* document. Hopefully, this will make it easier for users to find the information they want. ## Rebuild NCZarr Testing In order to avoid problems with running make check in parallel, two changes were made: 1. The *nczarr_test* test system was rebuilt. Now, for each test. any generated files are kept in a test-specific directory, isolated from all other test executions. 2. Similarly, since the S3 test bucket is shared, any generated S3 objects are isolated using a test-specific key path. ## Other S3 Related Changes * Add code to ensure that files created on S3 are reclaimed at end of testing. * Used the bash "trap" command to ensure S3 cleanup even if the test fails. * Cleanup the S3 related configure.ac flag set since S3 is used in several places. So now one should use the option *--enable-s3* instead of *--enable-nczarr-s3*, although the latter is still kept as a deprecated alias for the former. * Get some of the github actions yml to work with S3; required fixing various test scripts adding a secret to access the Unidata S3 bucket. * Cleanup S3 portion of libnetcdf.settings.in and netcdf_meta.h.in and test_common.in. * Merge partial S3 support into dhttp.c. * Create an experimental s3 access library especially for use with Windows. It is enabled by using the options *--enable-s3-internal* (automake) or *-DENABLE_S3_INTERNAL=ON* (CMake). Also add a unit-test for it. * Move some definitions from ncrc.h to ncs3sdk.h ## Other Changes * Provide a default implementation of strlcpy and move this and similar defaults into *dmissing.c*.
2023-04-26 07:15:06 +08:00
endif()
if(WITH_S3_TESTING STREQUAL PUBLIC)
set(ENABLE_S3_TESTING "public")
elseif(WITH_S3_TESTING)
set(ENABLE_S3_TESTING "yes")
set(ENABLE_S3_TESTALL "yes")
elseif(NOT WITH_S3_TESTING)
set(ENABLE_S3_TESTING "no")
else()
set(ENABLE_S3_TESTING "no")
Improve S3 Documentation and Support ## Improvements to S3 Documentation * Create a new document *quickstart_paths.md* that give a summary of the legal path formats used by netcdf-c. This includes both file paths and URL paths. * Modify *nczarr.md* to remove most of the S3 related text. * Move the S3 text from *nczarr.md* to a new document *cloud.md*. * Add some S3-related text to the *byterange.md* document. Hopefully, this will make it easier for users to find the information they want. ## Rebuild NCZarr Testing In order to avoid problems with running make check in parallel, two changes were made: 1. The *nczarr_test* test system was rebuilt. Now, for each test. any generated files are kept in a test-specific directory, isolated from all other test executions. 2. Similarly, since the S3 test bucket is shared, any generated S3 objects are isolated using a test-specific key path. ## Other S3 Related Changes * Add code to ensure that files created on S3 are reclaimed at end of testing. * Used the bash "trap" command to ensure S3 cleanup even if the test fails. * Cleanup the S3 related configure.ac flag set since S3 is used in several places. So now one should use the option *--enable-s3* instead of *--enable-nczarr-s3*, although the latter is still kept as a deprecated alias for the former. * Get some of the github actions yml to work with S3; required fixing various test scripts adding a secret to access the Unidata S3 bucket. * Cleanup S3 portion of libnetcdf.settings.in and netcdf_meta.h.in and test_common.in. * Merge partial S3 support into dhttp.c. * Create an experimental s3 access library especially for use with Windows. It is enabled by using the options *--enable-s3-internal* (automake) or *-DENABLE_S3_INTERNAL=ON* (CMake). Also add a unit-test for it. * Move some definitions from ncrc.h to ncs3sdk.h ## Other Changes * Provide a default implementation of strlcpy and move this and similar defaults into *dmissing.c*.
2023-04-26 07:15:06 +08:00
endif()
Mitigate S3 test interference + Unlimited Dimensions in NCZarr This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
2023-09-27 06:56:48 +08:00
# The Unidata testing S3 bucket
# WARNING: this must match the value in configure.ac
2024-01-18 06:07:22 +08:00
set(S3TESTBUCKET "unidata-zarr-test-data" CACHE STRING "S3 test bucket")
Mitigate S3 test interference + Unlimited Dimensions in NCZarr This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
2023-09-27 06:56:48 +08:00
# The working S3 path tree within the Unidata bucket.
# WARNING: this must match the value in configure.ac
2024-01-18 06:07:22 +08:00
set(S3TESTSUBTREE "netcdf-c" CACHE STRING "Working S3 path.")
Mitigate S3 test interference + Unlimited Dimensions in NCZarr This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
2023-09-27 06:56:48 +08:00
# Build a unique id based on the date
string(TIMESTAMP TESTUID "%s")
if(ENABLE_S3_TESTING)
2024-01-18 06:07:22 +08:00
file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/s3cleanup_${PLATFORMUID}.uids" "${TESTUID}\n")
Mitigate S3 test interference + Unlimited Dimensions in NCZarr This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
2023-09-27 06:56:48 +08:00
endif()
# Copy the CTest customization file into binary directory, as required.
2024-01-18 06:07:22 +08:00
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake")
Mitigate S3 test interference + Unlimited Dimensions in NCZarr This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
2023-09-27 06:56:48 +08:00
# Generate file from template.
2024-01-18 06:07:22 +08:00
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
"${CMAKE_CURRENT_BINARY_DIR}/libnetcdf.settings"
@ONLY)
# Read in settings file, print out.
# Avoid using system-specific calls so that this
# might also work on Windows.
2024-01-18 06:07:22 +08:00
file(READ "${CMAKE_CURRENT_BINARY_DIR}/libnetcdf.settings"
LIBNETCDF_SETTINGS)
2024-01-18 06:07:22 +08:00
message(STATUS ${LIBNETCDF_SETTINGS})
# Install libnetcdf.settings file into same location
# as the libraries.
2024-01-18 06:07:22 +08:00
install(FILES "${netCDF_BINARY_DIR}/libnetcdf.settings"
DESTINATION "${CMAKE_INSTALL_LIBDIR}"
COMPONENT libraries)
#####
# End libnetcdf.settings section.
#####
#####
# Create 'netcdf_meta.h' include file.
#####
configure_file(
${netCDF_SOURCE_DIR}/include/netcdf_meta.h.in
${netCDF_BINARY_DIR}/include/netcdf_meta.h @ONLY)
#####
# Create 'netcdf_dispatch.h' include file.
#####
configure_file(
${netCDF_SOURCE_DIR}/include/netcdf_dispatch.h.in
${netCDF_BINARY_DIR}/include/netcdf_dispatch.h @ONLY NEWLINE_STYLE LF)
####
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
# Build test_common.sh
#####
2024-01-18 06:07:22 +08:00
set(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_CURRENT_SOURCE_DIR}/test_common.in)
set(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}")
set(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY_DIR}/test_common.sh @ONLY NEWLINE_STYLE LF)
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
Mitigate S3 test interference + Unlimited Dimensions in NCZarr This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
2023-09-27 06:56:48 +08:00
####
# Build s3cleanup.sh and s3gc.sh
#####
2024-01-18 06:07:22 +08:00
set(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_CURRENT_SOURCE_DIR}/s3cleanup.in ${CMAKE_CURRENT_SOURCE_DIR}/s3gc.in)
set(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}")
set(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}")
Mitigate S3 test interference + Unlimited Dimensions in NCZarr This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
2023-09-27 06:56:48 +08:00
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/s3cleanup.in ${CMAKE_CURRENT_BINARY_DIR}/s3cleanup.sh @ONLY NEWLINE_STYLE LF)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/s3gc.in ${CMAKE_CURRENT_BINARY_DIR}/s3gc.sh @ONLY NEWLINE_STYLE LF)
#####
# Build and copy nc_test4/findplugin.sh to various places
#####
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF)
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nczarr_test/findplugin.sh @ONLY NEWLINE_STYLE LF)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/plugins/findplugin.sh @ONLY NEWLINE_STYLE LF)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF)
if(NETCDF_ENABLE_BENCHMARKS)
2024-01-18 06:07:22 +08:00
if(ENABLE_PARALLEL4)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_perf/run_par_bm_test.sh.in ${CMAKE_CURRENT_BINARY_DIR}/nc_perf/run_par_bm_test.sh @ONLY NEWLINE_STYLE LF)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_perf/run_gfs_test.sh.in ${CMAKE_CURRENT_BINARY_DIR}/nc_perf/run_gfs_test.sh @ONLY NEWLINE_STYLE LF)
endif(ENABLE_PARALLEL4)
endif(NETCDF_ENABLE_BENCHMARKS)
2023-02-17 06:56:25 +08:00
2024-01-18 06:07:22 +08:00
if(ENABLE_TESTS)
2020-08-12 22:38:00 +08:00
#####
# Build ncdap_test|dap4_test/findtestserver[4].c
#####
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_CURRENT_BINARY_DIR}/ncdap_test/findtestserver.c @ONLY NEWLINE_STYLE LF)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_CURRENT_BINARY_DIR}/dap4_test/findtestserver4.c @ONLY NEWLINE_STYLE LF)
2020-08-12 22:38:00 +08:00
#####
# Build dap4_test/pingurl4.c
#####
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ncdap_test/pingurl.c ${CMAKE_CURRENT_BINARY_DIR}/dap4_test/pingurl4.c @ONLY NEWLINE_STYLE LF)
Mitigate S3 test interference + Unlimited Dimensions in NCZarr This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
2023-09-27 06:56:48 +08:00
#####
# Build CTestCustom.cmake to cleanup S3 after tests are done.
#####
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake NEWLINE_STYLE LF)
2024-01-18 06:07:22 +08:00
endif()
2020-01-03 02:47:25 +08:00
if(DEFINED ENV{LIB_FUZZING_ENGINE})
add_subdirectory(fuzz)
endif(DEFINED ENV{LIB_FUZZING_ENGINE})
####
# Export files
####
# Create CMake package configuration files. With these, other packages using
# cmake should be able to find netcdf using find_package and find_library.
# The EXPORT call is paired with one in liblib.
set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/netCDF)
install(EXPORT netCDFTargets
DESTINATION ${ConfigPackageLocation}
COMPONENT headers
NAMESPACE netCDF::
)
export(EXPORT netCDFTargets FILE netCDFTargets.cmake NAMESPACE netCDF::)
include(CMakePackageConfigHelpers)
configure_package_config_file(
"${CMAKE_CURRENT_SOURCE_DIR}/netCDFConfig.cmake.in"
"${CMAKE_CURRENT_BINARY_DIR}/netCDFConfig.cmake"
INSTALL_DESTINATION "${ConfigPackageLocation}"
)
target_include_directories(netcdf
PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
)
# Create export configuration
write_basic_package_version_file(
netCDFConfigVersion.cmake
VERSION ${netCDF_VERSION}
COMPATIBILITY SameMajorVersion
)
install(
FILES
"${CMAKE_CURRENT_BINARY_DIR}/netCDFConfig.cmake"
"${CMAKE_CURRENT_BINARY_DIR}/netCDFConfigVersion.cmake"
DESTINATION ${ConfigPackageLocation}
COMPONENT headers
)
####
# End export files
####
2012-10-03 04:56:46 +08:00
# CPack inclusion must come last.
option(NETCDF_PACKAGE "Create netCDF-C package " ${NETCDF_IS_TOP_LEVEL})
if (NETCDF_PACKAGE)
include(CMakeInstallation.cmake)
endif()