mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-25 17:40:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Fix nczarr-experimental: improve build support, disengage hdf5 vs netcdf4 flags, and find AWS libraries"

Browse Source
This commit is contained in:
Ward Fisher 2020-08-17 15:51:01 -06:00 committed by GitHub
parent c025e85a4a
commit 16c27ca13f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
130 changed files with 1883 additions and 2593 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.travis.yml
View File

@ -12,13 +12,13 @@ services:
env:
matrix:
# Ubuntu
- DOCKIMG=unidata/nctests:serial USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc CFLAGS='-fsigned-char' AC_COPTS='--disable-hdf5 --disable-hdf4 --disable-dap-remote-tests --enable-cdf5 --enable-byterange' COPTS='-DENABLE_HDF5=OFF -DENABLE_HDF4=OFF -DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE -DENABLE_BYTERANGE=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-signed TESTFILTER=FALSE
- DOCKIMG=unidata/nctests:serial USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc CFLAGS='-fsigned-char' AC_COPTS='--disable-netcdf-4 --disable-hdf4 --disable-dap-remote-tests --enable-cdf5 --enable-byterange' COPTS='-DENABLE_NETCDF_4=OFF -DENABLE_HDF4=OFF -DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE -DENABLE_BYTERANGE=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-signed TESTFILTER=FALSE
- DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc CFLAGS='-fsigned-char' AC_COPTS='--disable-hdf5 --disable-hdf4 --disable-dap-remote-tests' COPTS='-DENABLE_HDF5=OFF -DENABLE_HDF4=OFF -DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-signed TESTFILTER=FALSE
- DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc CFLAGS='-fsigned-char' AC_COPTS='--disable-netcdf-4 --disable-hdf4 --disable-dap-remote-tests' COPTS='-DENABLE_NETCDF_4=OFF -DENABLE_HDF4=OFF -DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-signed TESTFILTER=FALSE
- DOCKIMG=unidata/nctests:serial USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc CFLAGS='-funsigned-char' AC_COPTS='--disable-hdf5 --disable-hdf4 --disable-dap-remote-tests --enable-cdf5' COPTS='-DENABLE_HDF5=OFF -DENABLE_HDF4=OFF -DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-unsigned TESTFILTER=FALSE
- DOCKIMG=unidata/nctests:serial USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc CFLAGS='-funsigned-char' AC_COPTS='--disable-netcdf-4 --disable-hdf4 --disable-dap-remote-tests --enable-cdf5' COPTS='-DENABLE_NETCDF_4=OFF -DENABLE_HDF4=OFF -DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-unsigned TESTFILTER=FALSE
- DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc CFLAGS='-funsigned-char' AC_COPTS='--disable-hdf5 --disable-hdf4 --disable-dap-remote-tests' COPTS='-DENABLE_HDF5=OFF -DENABLE_HDF4=OFF -DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-unsigned TESTFILTER=FALSE
- DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc CFLAGS='-funsigned-char' AC_COPTS='--disable-netcdf-4 --disable-hdf4 --disable-dap-remote-tests' COPTS='-DENABLE_NETCDF_4=OFF -DENABLE_HDF4=OFF -DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-unsigned TESTFILTER=FALSE
- DOCKIMG=unidata/nctests:serial USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc CFLAGS='-fsigned-char' AC_COPTS='--disable-dap-remote-tests --enable-cdf5' COPTS='-DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-signed TESTFILTER=TRUE

161
CMakeLists.txt
View File

@ -436,37 +436,8 @@ ENDIF()
# End user-specified dependent libraries.
###
################################
# Format Option checks
################################
# We need to now treat enable-netcdf4 and enable-hdf5 as separate,
# but for back compatability, we need to treat enable-netcdf4
# as equivalent to enable-hdf5.
# We detect this using these rules:
# 1. if ENABLE_HDF5 is off then disable hdf5
# 2. if ENABLE_NETCDF4 is off then disable hdf5
# 3. else enable hdf5
OPTION(ENABLE_NETCDF_4 "Use HDF5." ON)
OPTION(ENABLE_NETCDF4 "Use HDF5." ON)
OPTION(ENABLE_HDF5 "Use HDF5." ON)
IF(NOT ENABLE_HDF5 OR NOT ENABLE_NETCDF4 OR NOT ENABLE_NETCDF_4)
SET(ENABLE_HDF5 OFF CACHE BOOL "Use HDF5" FORCE)
ENDIF()
# Option to use HDF4
OPTION(ENABLE_HDF4 "Build netCDF-4 with HDF4 read capability(HDF4, HDF5 and Zlib required)." OFF)
OPTION(ENABLE_DAP "Enable DAP2 and DAP4 Client." ON)
OPTION(ENABLE_NCZARR "Enable NCZarr Client." ON)
OPTION(ENABLE_PNETCDF "Build with parallel I/O for CDF-1, 2, and 5 files using PnetCDF." OFF)
SET(ENABLE_CDF5 AUTO CACHE STRING "AUTO")
OPTION(ENABLE_CDF5 "Enable CDF5 support" ON)
# Netcdf-4 support (i.e. libsrc4) is required by more than just HDF5 (e.g. NCZarr)
# So depending on what above formats are enabled, enable netcdf-4
if(ENABLE_HDF5 OR ENABLE_HDF4 OR ENABLE_NCZARR)
SET(ENABLE_NETCDF_4 ON CACHE BOOL "Enable netCDF-4 API" FORCE)
SET(ENABLE_NETCDF4 ON CACHE BOOL "Enable netCDF4 Alias" FORCE)
endif()
IF(ENABLE_HDF4)
SET(USE_HDF4 ON)
# Check for include files, libraries.
@ -538,6 +509,8 @@ ENDIF()
# Did the user specify a default minimum blocksize for posixio?
SET(NCIO_MINBLOCKSIZE 256 CACHE STRING "Minimum I/O Blocksize for netCDF classic and 64-bit offset format files.")
# Build netCDF4
OPTION(ENABLE_NETCDF_4 "Enable netCDF-4" ON)
IF(ENABLE_NETCDF_4)
SET(USE_NETCDF4 ON CACHE BOOL "")
SET(ENABLE_NETCDF_4 ON CACHE BOOL "")
@ -575,10 +548,10 @@ IF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
ENDIF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
# Option for building RPC
#OPTION(ENABLE_RPC "Enable RPC Client and Server." OFF)
#IF(ENABLE_RPC)
# SET(BUILD_RPC ON CACHE BOOL "")
#ENDIF()
OPTION(ENABLE_RPC "Enable RPC Client and Server." OFF)
IF(ENABLE_RPC)
SET(BUILD_RPC ON CACHE BOOL "")
ENDIF()
##
# Option to Enable HDF5
@ -597,8 +570,10 @@ ENDIF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
# * HDF5_INCLUDE_DIR
# *
##
SET(USE_HDF5 ${ENABLE_HDF5})
IF(USE_HDF5)
OPTION(USE_HDF5 "Use HDF5." ${ENABLE_NETCDF_4})
IF(USE_HDF5 OR ENABLE_NETCDF_4)
SET(USE_HDF5 ON)
SET(USE_NETCDF4 ON)
##
# Accommodate developers who have hdf5 libraries and
# headers on their system, but do not have a the hdf
@ -806,7 +781,8 @@ IF(USE_HDF5)
#option to include HDF5 High Level header file (hdf5_hl.h) in case we are not doing a make install
INCLUDE_DIRECTORIES(${HDF5_HL_INCLUDE_DIR})
ENDIF(USE_HDF5)
ENDIF(USE_HDF5 OR ENABLE_NETCDF_4)
# See if we have libcurl
FIND_PACKAGE(CURL)
@ -864,13 +840,17 @@ int main() {
#endif
}" HAVE_LIBCURL_766)
# Option to Build DAP2+DAP4 Clients
OPTION(ENABLE_DAP "Enable DAP2 and DAP4 Client." ON)
IF(ENABLE_DAP)
SET(USE_DAP ON CACHE BOOL "")
SET(ENABLE_DAP2 ON CACHE BOOL "")
IF(ENABLE_HDF5)
IF(ENABLE_NETCDF_4)
SET(ENABLE_DAP4 ON CACHE BOOL "")
ENDIF(ENABLE_HDF5)
ELSE(ENABLE_NETCDF_4)
SET(ENABLE_DAP4 OFF CACHE BOOL "")
ENDIF(ENABLE_NETCDF_4)
ELSE()
SET(ENABLE_DAP2 OFF)
@ -946,26 +926,42 @@ IF(NOT WIN32)
ENDIF()
ENDIF()
# Options to Extend NCZarr support
OPTION(ENABLE_S3_SDK "Enable Search for AWS libraries." OFF)
OPTION(ENABLE_S3_TESTS "Enable NCZarr S3 tests." OFF)
# Option to Build NCZarr clients
OPTION(ENABLE_NCZARR "Enable NCZarr Client." OFF)
OPTION(ENABLE_S3_SDK "Enable NCZarr S3 support." ON)
IF(NOT ENABLE_S3_SDK)
IF(ENABLE_S3_TESTS)
message(WARNING "DISABLE_S3_SDK => DISABLE_S3_TESTS")
SET(ENABLE_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
# See if aws-s3-sdk is available
FIND_LIBRARY(AWS_CPP_SDK_CORE_LIB NAMES aws-cpp-sdk-core)
FIND_LIBRARY(AWS_CPP_SDK_S3_LIB NAMES aws-cpp-sdk-s3)
FIND_LIBRARY(AWS_C_COMMON_LIB NAMES aws-c-common)
IF(NOT AWS_CPP_SDK_CORE_LIB)
SET(AWS_CPP_SDK_CORE_LIB OFF)
ENDIF()
IF(NOT AWS_CPP_SDK_S3_LIB)
SET(AWS_CPP_SDK_S3_LIB OFF)
ENDIF()
IF(NOT AWS_C_COMMON_LIB)
SET(AWS_C_COMMON_LIB OFF)
ENDIF()
IF(ENABLE_S3_SDK)
IF(NOT AWS_CPP_SDK_CORE_LIB
OR NOT AWS_CPP_SDK_S3_LIB
OR NOT AWS_C_COMMON_LIB)
MESSAGE(WARNING "One of aws-c-common, aws-cpp-sdk-s3, aws-cpp-sdk-core libraries is not available: Turning off S3 support")
SET(ENABLE_S3_SDK OFF)
ELSE()
MESSAGE(STATUS "Found aws sdk libraries")
ENDIF()
ENDIF()
IF(ENABLE_S3_SDK)
# See if aws-s3-sdk is available
find_package(AWSSDK REQUIRED)
set(SERVICE s3)
AWSSDK_DETERMINE_LIBS_TO_LINK(SERVICE AWSSDK_LINK_LIBRARIES)
IF(NOT AWSSDK_FOUND)
SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
ENDIF()
ENDIF()
IF(CMAKE_COMPILER_IS_GNUCC OR APPLE)
# SET(LDFLAGS "${LDFLAGS} ${AWS_CPP_SDK_CORE_LIB} ${AWS_CPP_SDK_S3_LIB} ${AWS_C_COMMON_LIB}")
# SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lstdc++")
ENDIF(CMAKE_COMPILER_IS_GNUCC OR APPLE)
ENDIF(ENABLE_S3_SDK)
##
# Enable Tests
@ -1074,15 +1070,15 @@ IF(ENABLE_LARGE_FILE_SUPPORT)
ENDIF()
ENDIF()
OPTION(ENABLE_EXAMPLE_TESTS "Run extra example tests. Requires GNU Sed. Ignored if HDF5 is not Enabled" OFF)
IF(NOT ENABLE_HDF5 AND ENABLE_EXAMPLE_TESTS)
OPTION(ENABLE_EXAMPLE_TESTS "Run extra example tests. Requires GNU Sed. Ignored if netCDF-4 is not Enabled" OFF)
IF(NOT ENABLE_NETCDF_4 AND ENABLE_EXAMPLE_TESTS)
SET(ENABLE_EXAMPLE_TESTS OFF)
ENDIF()
# Enable Parallel IO with netCDF-4/HDF5 files using HDF5 parallel I/O.
SET(STATUS_PARALLEL "OFF")
OPTION(ENABLE_PARALLEL4 "Build netCDF-4 with parallel IO" "${HDF5_PARALLEL}")
IF(ENABLE_PARALLEL4 AND ENABLE_HDF5)
IF(ENABLE_PARALLEL4 AND ENABLE_NETCDF_4)
IF(NOT HDF5_PARALLEL)
SET(USE_PARALLEL OFF CACHE BOOL "")
MESSAGE(STATUS "Cannot find HDF5 library built with parallel support. Disabling parallel build.")
@ -1107,6 +1103,8 @@ ENDIF()
# Options to enable parallel IO for classic formats with PnetCDF library.
SET(STATUS_PNETCDF "OFF")
OPTION(ENABLE_PNETCDF "Build with parallel I/O for CDF-1, 2, and 5 files using PnetCDF." OFF)
IF(ENABLE_PNETCDF)
# Check for ncmpi_create in libpnetcdf, define USE_PNETCDF
# Does the user want to turn on PnetCDF read ability?
@ -1223,9 +1221,9 @@ IF (ENABLE_PARALLEL_TESTS AND NOT USE_PARALLEL)
ENDIF()
# Enable special filter test; experimental when using cmake.
OPTION(ENABLE_FILTER_TESTING "Enable filter testing. Ignored if shared libraries or netCDF4 are not enabled" ${ENABLE_HDF5})
IF(ENABLE_FILTER_TESTING AND NOT ENABLE_HDF5)
MESSAGE(WARNING "ENABLE_FILTER_TESTING requires HDF5. Disabling.")
OPTION(ENABLE_FILTER_TESTING "Enable filter testing. Ignored if shared libraries or netCDF4 are not enabled" ${ENABLE_NETCDF_4})
IF(ENABLE_FILTER_TESTING AND NOT ENABLE_NETCDF_4)
MESSAGE(WARNING "ENABLE_FILTER_TESTING requires netCDF-4. Disabling.")
SET(ENABLE_FILTER_TESTING OFF)
ENDIF()
IF(NOT BUILD_SHARED_LIBS)
@ -1305,6 +1303,7 @@ ENDIF()
# Always enable DISKLESS
OPTION(ENABLE_DISKLESS "Enable in-memory files" ON)
# By default, MSVC has a stack size of 1000000.
# Allow a user to override this.
IF(MSVC)
@ -1417,6 +1416,8 @@ CHECK_TYPE_SIZE("_Bool" SIZEOF__BOOL)
CHECK_TYPE_SIZE("size_t" SIZEOF_SIZE_T)
# Check whether to turn on or off CDF5 support.
SET(ENABLE_CDF5 AUTO CACHE STRING "AUTO")
OPTION(ENABLE_CDF5 "Enable CDF5 support" ON)
IF(SIZEOF_SIZE_T EQUAL 4)
IF(ENABLE_CDF5) # enable or auto
STRING(TOUPPER ${ENABLE_CDF5} ENABLE_CDF5)
@ -1597,10 +1598,10 @@ ENDMACRO()
# Build a binary used by a script, but don't make a test out of it.
MACRO(build_bin_test F)
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${F}.c")
ADD_EXECUTABLE(${F} "${CMAKE_CURRENT_SOURCE_DIR}/${F}.c" ${ARGN})
ADD_EXECUTABLE(${F} "${CMAKE_CURRENT_SOURCE_DIR}/${F}.c")
else()
# File should have been copied to the binary directory
ADD_EXECUTABLE(${F} "${CMAKE_CURRENT_BINARY_DIR}/${F}.c" ${ARGN})
ADD_EXECUTABLE(${F} "${CMAKE_CURRENT_BINARY_DIR}/${F}.c")
endif()
TARGET_LINK_LIBRARIES(${F} netcdf ${ALL_TLL_LIBS})
IF(MSVC)
@ -1670,6 +1671,23 @@ MACRO(add_bin_test prefix F)
ENDIF()
ENDMACRO()
# Binary tests which are used by a script looking for a specific name.
MACRO(build_bin_test_sourced F)
ADD_EXECUTABLE(${F} ${F}.c ${ARGN})
TARGET_LINK_LIBRARIES(${F} netcdf ${ALL_TLL_LIBS})
IF(MSVC)
SET_TARGET_PROPERTIES(${F}
PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT"
)
SET_TARGET_PROPERTIES(${F} PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(${F} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(${F} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
ENDIF()
ENDMACRO()
# A cmake script to print out information at the end of the configuration step.
MACRO(print_conf_summary)
MESSAGE("")
@ -1749,7 +1767,7 @@ ENDIF(ENABLE_BASH_SCRIPT_TESTING)
MACRO(add_sh_test prefix F)
IF(HAVE_BASH)
ADD_TEST(${prefix}_${F} bash "-c" "export srcdir=${CMAKE_CURRENT_SOURCE_DIR};export TOPSRCDIR=${CMAKE_SOURCE_DIR};${CMAKE_CURRENT_BINARY_DIR}/${F}.sh ${F}.sh ${ARGN}")
ADD_TEST(${prefix}_${F} bash "-c" "export srcdir=${CMAKE_CURRENT_SOURCE_DIR};export TOPSRCDIR=${CMAKE_SOURCE_DIR};${CMAKE_CURRENT_BINARY_DIR}/${F}.sh")
ENDIF()
ENDMACRO()
@ -1814,11 +1832,8 @@ IF(USE_PNETCDF)
add_subdirectory(libsrcp)
ENDIF(USE_PNETCDF)
IF(USE_NETCDF4)
add_subdirectory(libsrc4)
ENDIF()
IF(USE_HDF5)
add_subdirectory(libsrc4)
add_subdirectory(libhdf5)
ENDIF(USE_HDF5)
@ -1866,7 +1881,7 @@ IF(ENABLE_TESTS)
ADD_SUBDIRECTORY(nctest)
ENDIF()
ADD_SUBDIRECTORY(nc_test)
IF(USE_HDF5)
IF(USE_NETCDF4)
INCLUDE_DIRECTORIES(h5_test)
ADD_SUBDIRECTORY(nc_test4)
IF(BUILD_BENCHMARKS)
@ -1880,15 +1895,15 @@ IF(ENABLE_TESTS)
IF(ENABLE_DAP4)
ADD_SUBDIRECTORY(dap4_test)
ENDIF()
IF(ENABLE_NCZARR)
ADD_SUBDIRECTORY(nczarr_test)
ENDIF()
IF(ENABLE_EXAMPLES)
ADD_SUBDIRECTORY(examples)
ENDIF()
IF(ENABLE_UNIT_TESTS)
ADD_SUBDIRECTORY(unit_test)
ENDIF(ENABLE_UNIT_TESTS)
IF(ENABLE_NCZARR)
ADD_SUBDIRECTORY(nczarr_test)
ENDIF()
ENDIF()
# Code to generate an export header
@ -2041,7 +2056,7 @@ is_enabled(BUILD_SHARED_LIBS enable_shared)
is_enabled(ENABLE_V2_API HAS_NC2)
is_enabled(ENABLE_NETCDF_4 HAS_NC4)
is_enabled(ENABLE_HDF4 HAS_HDF4)
is_enabled(USE_HDF5 HAS_HDF5)
is_enabled(ENABLE_NETCDF_4 HAS_HDF5)
is_enabled(USE_SZIP HAS_SZIP)
is_enabled(USE_SZIP HAS_SZIP_WRITE)
is_enabled(STATUS_PNETCDF HAS_PNETCDF)
@ -2060,7 +2075,7 @@ is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
is_enabled(HAVE_H5Z_SZIP HAS_SZLIB)
is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
is_enabled(ENABLE_NCZARR HAS_NCZARR)
is_enabled(ENABLE_S3_TESTS DO_S3_TESTS)
is_enabled(ENABLE_S3_SDK HAS_S3_SDK)
# Generate file from template.
CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"

8
Makefile.am
View File

@ -53,7 +53,7 @@ endif
if USE_HDF5
H5_TEST_DIR = h5_test
LIBHDF5 = libhdf5
if USE_HDF5
if USE_NETCDF4
NC_TEST4 = nc_test4
endif
endif
@ -99,9 +99,9 @@ endif # BUILD_BENCHMARKS
# Define Test directories
if BUILD_TESTSETS
TESTDIRS = $(UNIT_TEST) $(V2_TEST) nc_test $(NC_TEST4)
TESTDIRS += $(BENCHMARKS_DIR) $(HDF4_TEST_DIR) $(NCDAP2TESTDIR) $(NCDAP4TESTDIR)
TESTDIRS += ${ZARR_TEST_DIR}
TESTDIRS = $(UNIT_TEST) $(V2_TEST) nc_test $(NC_TEST4) \
$(BENCHMARKS_DIR) $(HDF4_TEST_DIR) $(NCDAP2TESTDIR) $(NCDAP4TESTDIR) \
${ZARR_TEST_DIR}
endif
# This is the list of subdirs for which Makefiles will be constructed

99
NUG/nczarr.md
View File

@ -56,9 +56,7 @@ NCZarr support can be disabled using the _--disable-dap_.
In order to access a NCZarr data source through the netCDF API, the
file name normally used is replaced with a URL with a specific
format. Note specifically that there is no NC_NCZARR flag for
the mode argument of _nc_create_ or _nc_open_. In this case, it is
indicated by the URL path.
format.
## URL Format
The URL is the usual scheme:://host:port/path?query#fragment format.
@ -337,90 +335,51 @@ zarr format.
Currently only the following build cases are supported.
Operating System | Build System | NCZarr | S3 Support
-----------------------------------------------------------
Linux | Automake | yes | yes
Linux | CMake | yes | yes
Cygwin | Automake | yes | no
OSX | Automake | unknown | unknown
OSX | CMake | unknown | unknown
Visual Studio | CMake | yes | tests fail
Note: S3 support includes both compiling the S3 support code as well as running the S3 tests.
# Automake
Operating System | Supported Build Systems
------------------------------------------
Linux | Automake, CMake
OS-X | Automake, CMake
Visual Studio | N.A.
There are several options relevant to NCZarr support and to Amazon S3 support.
These are as follows.
1. _--enable-nczarr_ -- enable the NCZarr support. If disabled, then all of the following options are disabled or irrelevant.
2. _--enable-s3-sdk_ -- enable the use of the aws s3 sdk.
2. _--enable-s3-tests_ -- the s3 tests are currently only usable by Unidata personnel, so they are disabled by default.
2.   _aws-c-common aws-cpp-sdk-s3_ and _aws-cpp-sdk-core_ -- if these libraries are available, then Amazon S3 support is enabled for NCZarr.
3. _--disable-s3-sdk_ -- even if the aws libraries are available, this option will forcibly disable Amazon S3 support for using them.
<!--
3. '--enable-xarray-dimension' -- this enables the xarray support described in the section on <a href="#nczarr_compatibility">compatibility</a>.
4. '--enable-xarray-dimension' -- this enables the xarray support described in the section on <a href="#nczarr_compatibility">compatibility</a>.
-->
A note about using S3 with Automake. Automake does not handle C++ libraries, so
if S3 support is desired, and using Automake, then LDFLAGS
must be properly set, namely to this.
The CMake equivalents are as follows:
* _--enable-nczarr_ => ENABLE_NCZARR=ON
* _--disable-s3-sdk_ => ENABLE_S3_SDK=OFF
If S3 support is desired, then LDFLAGS should be properly set, namely this.
````
LDFLAGS="$LDFLAGS -L/usr/local/lib -laws-cpp-sdk-s3"
LDFLAGS="$LDFLAGS -L/usr/local/lib -laws-cpp-sdk-s3 aws-cpp-sdk-core"
````
The above assumes that these libraries were installed in
'/usr/local/lib', so the above requires modification if they
were installed elsewhere.
The above assumes that these libraries were installed in '/usr/local/lib', so the above
requires modification if they were installed elsewhere.
Note also that if S3 support is enabled, then you need to have a
C++ compiler installed because part of the S3 support code is
written in C++.
# CMake
The necessary CMake flags are as follows (with defaults)
1. -DENABLE_NCZARR=on -- equivalent to the Automake _--enable-nczarr_ option.
2. -DENABLE_S3_SDK=off -- quivalent to the Automake _--enable-s3-sdk_ option.
3. -DENABLE_S3_TESTS=off -- equivalent to the Automake _--enable-s3-tests_ option.
Note that unlike Automake, CMake can properly locate C++ libraries,
so it should not be necessary to specify _-laws-cpp-sdk-s3_ assuming
that the aws s3 libraries are installed in the default location.
For CMake with Visual Studio, the default location is here:
````
C:/Program Files (x86)/aws-cpp-sdk-all
````
Note also that if S3 support is enabled, then you need to have a C++ compiler installed
because part of the S3 support code is written in C++.
## Testing S3 Support
The relevant tests for S3 support are
_nczarr_test/run_ut_mapapi.sh_ and _nczarr_test/run_it_test2.sh_.
Currently, by default, testing of S3 with NCzarr is supported only for
Currently testing of S3 with NCzarr is supported only for
Unidata members of the NetCDF Development Group. This is because
it uses a specific bucket on a specific internal S3 appliance that
is inaccessible to the general user. This is controlled by
the _--enable_s3_tests_ option.
However, an untested mechanism exists by which others may be able
to run the tests. If someone else wants to attempt these tests, then
they need to define the environment variable name _NCS3PATH_.
The form of this variable is as follows:
an environment variable via this shell command:
````
NCS3PATH="https://<host>/<bucket>/<prefix>
export NETCDF_S3_TESTS=1
````
This assumes a Path Style address (see above) where
* host -- the complete host part of the url
* bucket -- a bucket in which testing can occur without fear of
damaging anything.
* prefix - prefix of the key; the actual root, typically _test_,
is appended to this to get the root key used by the test.
Example:
````
s3.us-west.amazonaws.com/testingbucket/segment1/segment2
````
If anyone tries to use this mechanism, it would be appreciated
it any difficulties were reported to Unidata.
If that environment variable is set and the aws libraries are
available, and --enable-s3-sdk is set, then the S3 tests will be
attempted. If someone else wants to attempt these tests, then
they will need to modify the tests
_nczarr_test/run_ut_mapapi.sh_ and _nczarr_test/run_it_test2.sh_.
# Appendix B. Building aws-sdk-cpp {#nczarr_s3sdk}
@ -431,8 +390,8 @@ As a starting point, here are the CMake options used by Unidata
to build that library. It assumes that it is being executed
in a build directory, `build` say, and that `build/../CMakeLists.txt exists`.
```
cmake -DBUILD_ONLY=s3
````
cmake -DFORCE_CURL=ON -DBUILD_ONLY=s3 -DMINIMIZE_SIZE=ON -DBUILD_DEPS=OFF -DCMAKE_CXX_STANDARD=14 ..
```
The expected set of installed libraries are as follows:
* aws-cpp-sdk-s3

6
config.h.cmake.in
View File

@ -145,12 +145,6 @@ are set when opening a binary file on Windows. */
/* if true, enable NCZARR */
#cmakedefine ENABLE_NCZARR 1
/* if true, enable S3 support */
#cmakedefine ENABLE_S3_SDK 1
/* if true, enable S3 testing*/
#cmakedefine ENABLE_S3_TESTS 1
/* define the possible sources for remote test servers */
#cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}"

261
configure.ac
View File

@ -74,78 +74,6 @@ AM_INIT_AUTOMAKE([foreign dist-zip subdir-objects])
# Check for the existence of this file before proceeding.
AC_CONFIG_SRCDIR([include/netcdf.h])
AC_MSG_NOTICE([checking supported formats])
# An explicit disable of netcdf-4 | netcdf4 is treated as if it was disable-hdf5
AC_MSG_CHECKING([whether we should build with netcdf4 (alias for HDF5)])
AC_ARG_ENABLE([netcdf4], [AS_HELP_STRING([--disable-netcdf4],
[(synonym for --enable-hdf5)])])
test "x$enable_netcdf4" = xno || enable_netcdf4=yes
AC_MSG_RESULT([$enable_netcdf4])
AC_MSG_CHECKING([whether we should build with netcdf-4 (alias for HDF5)])
AC_ARG_ENABLE([netcdf-4], [AS_HELP_STRING([--disable-netcdf-4],
[(synonym for --disable-netcdf4)])])
test "x$enable_netcdf_4" = xno || enable_netcdf_4=yes
AC_MSG_RESULT([$enable_netcdf_4])
# Propagate the alias
if test "x$enable_netcdf_4" = xno ; then enable_netcdf4=no; fi
if test "x$enable_netcdf4" = xno ; then enable_netcdf_4=no; fi
# Does the user want to use HDF5?
AC_MSG_CHECKING([whether we should build with HDF5])
AC_ARG_ENABLE([hdf5], [AS_HELP_STRING([--disable-hdf5],
[do not build with HDF5])])
test "x$enable_hdf5" = xno || enable_hdf5=yes
if test "x$enable_netcdf4" = xno ; then enable_hdf5=no ; fi
# disable-netcdf4 is synonym for disable-hdf5
AC_MSG_RESULT([$enable_hdf5])
# Check whether we want to enable CDF5 support.
AC_MSG_CHECKING([whether CDF5 support should be disabled])
AC_ARG_ENABLE([cdf5],
[AS_HELP_STRING([--disable-cdf5],
[build without CDF5 support. @<:@default: auto@:>@])],
[enable_cdf5=${enableval}], [enable_cdf5=auto]
)
AC_MSG_RESULT($enable_cdf5)
# Does the user want to turn on HDF4 read ability?
AC_MSG_CHECKING([whether reading of HDF4 SD files is to be enabled])
AC_ARG_ENABLE([hdf4], [AS_HELP_STRING([--enable-hdf4],
[build netcdf with HDF4 read capability (HDF4, HDF5 and zlib required)])])
test "x$enable_hdf4" = xyes || enable_hdf4=no
AC_MSG_RESULT($enable_hdf4)
AC_MSG_CHECKING([whether parallel I/O for classic files is to be enabled])
AC_ARG_ENABLE([pnetcdf], [AS_HELP_STRING([--enable-pnetcdf],
[build with parallel I/O for classic files. @<:@default: disabled@:>@])])
test "x$enable_pnetcdf" = xyes || enable_pnetcdf=no
AC_MSG_RESULT($enable_pnetcdf)
## Capture the state of the --enable-dap flag => enable dap2+dap4
AC_MSG_CHECKING([whether DAP client(s) are to be built])
AC_ARG_ENABLE([dap],
[AS_HELP_STRING([--disable-dap],
[build without DAP client support.])])
test "x$enable_dap" = xno || enable_dap=yes
AC_MSG_RESULT($enable_dap)
AC_MSG_CHECKING([whether netcdf zarr storage format should be enabled])
AC_ARG_ENABLE([nczarr],
[AS_HELP_STRING([--enable-nczarr],
[enable netcdf zarr storage support; requires netCDF-4 and libcurl])])
test "x$enable_nczarr" = xyes || enable_nczarr=no
AC_MSG_RESULT($enable_nczarr)
# HDF5 | HDF4 | NCZarr => netcdf-4
if test "x$enable_hdf5" = xyes || test "x$enable_hdf4" = xyes || test "x$enable_nczarr" = xyes ; then
enable_netcdf_4=yes
fi
AC_MSG_CHECKING([whether netcdf-4 should be forcibly enabled])
AC_MSG_RESULT([$enable_netcdf_4])
# Synonym
enable_netcdf4=${enable_netcdf_4}
AC_MSG_NOTICE([checking user options])
# Did the user specify a default minimum blocksize (NCIO_MINBLOCKSIZE) for posixio?
@ -275,6 +203,33 @@ test "x$enable_unit_tests" = xno || enable_unit_tests=yes
AC_MSG_RESULT($enable_unit_tests)
AM_CONDITIONAL([BUILD_UNIT_TESTS], [test "x$enable_unit_tests" = xyes])
# Does the user want to build netcdf-4?
AC_MSG_CHECKING([whether we should build netCDF-4])
AC_ARG_ENABLE([netcdf-4], [AS_HELP_STRING([--disable-netcdf-4],
[do not build with netcdf-4 (else HDF5 and zlib required)])])
test "x$enable_netcdf_4" = xno || enable_netcdf_4=yes
# Synonym
AC_ARG_ENABLE([netcdf4], [AS_HELP_STRING([--disable-netcdf4],
[(just a synonym for --disable-netcdf-4)])])
test "x$enable_netcdf4" = xno || enable_netcdf4=yesi
if test "x$enable_netcdf4" = xno ; then
enable_netcdf_4=no
fi
AC_MSG_RESULT([$enable_netcdf_4])
# Does the user want to use HDF5?
# WARNING: enable_hdf5 does not imply that use_netcdf_4 is enabled
AC_MSG_CHECKING([whether we should build with HDF5])
#enable_hdf5=$enable_netcdf_4
AC_ARG_ENABLE([hdf5], [AS_HELP_STRING([--disable-hdf5],
[do not build with HDF5])])
test "x$enable_hdf5" = xno || enable_hdf5=yes
if test "x$enable_netcdf_4" = xno ; then
enable_hdf5=no
fi
AC_MSG_RESULT([$enable_hdf5])
# Does the user require dynamic loading?
# This is only for those hdf5 installs that support it.
AC_MSG_CHECKING([do we require hdf5 dynamic-loading support])
@ -283,6 +238,15 @@ AC_ARG_ENABLE([dynamic-loading], [AS_HELP_STRING([--enable-dynamic-loading],
test "x$enable_dynamic_loading" = xno || enable_dynamic_loading=yes
AC_MSG_RESULT([$enable_dynamic_loading])
# Does the user want to turn on HDF4 read ability?
AC_MSG_CHECKING([whether reading of HDF4 SD files is to be enabled])
AC_ARG_ENABLE([hdf4], [AS_HELP_STRING([--enable-hdf4],
[build netcdf-4 with HDF4 read capability (HDF4, HDF5 and zlib required)])])
test "x$enable_hdf4" = xyes || enable_hdf4=no
if test "x$enable_hdf4" = xyes -a "x$enable_netcdf_4" = xno; then
AC_MSG_ERROR([NetCDF-4 is required for HDF4 features])
fi
AC_MSG_RESULT($enable_hdf4)
# Does the user want to turn on extra HDF4 file tests?
AC_MSG_CHECKING([whether to fetch some sample HDF4 files from Unidata ftp site to test HDF4 reading (requires wget)])
@ -395,6 +359,14 @@ if test "x$enable_set_log_level_func" = xyes -a "x$enable_netcdf_4" = xyes; then
fi
AC_MSG_RESULT($enable_set_log_level_func)
## Capture the state of the --enable-dap flag => enable dap2+dap4
AC_MSG_CHECKING([whether DAP client(s) are to be built])
AC_ARG_ENABLE([dap],
[AS_HELP_STRING([--disable-dap],
[build without DAP client support.])])
test "x$enable_dap" = xno || enable_dap=yes
AC_MSG_RESULT($enable_dap)
# We need curl for DAP and byterange
AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
if test "x$enable_dap" = "xyes" ; then
@ -469,6 +441,12 @@ AC_MSG_RESULT([$enable_dap_long_tests])
AM_CONDITIONAL(INTERNAL_OCLIB,[test "x" = "x"])
# Control zarr storage
AC_MSG_CHECKING([whether netcdf zarr storage format should be enabled])
AC_ARG_ENABLE([nczarr],
[AS_HELP_STRING([--enable-nczarr],
[enable netcdf zarr storage support; requires netCDF-4 and libcurl])])
test "x$enable_nczarr" = xyes || enable_nczarr=no
AC_MSG_RESULT($enable_nczarr)
if test "x$enable_nczarr" = xyes ; then
if test "x$enable_netcdf_4" = xno ; then
AC_MSG_ERROR([netCDF-4 disabled, so you must not enable nczarr])
@ -481,36 +459,41 @@ if test "x$enable_nczarr" = xyes; then
fi
AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
# Check for enabling S3 support
# Check for the AWS S3 SDK library
AC_LANG_PUSH([C++])
AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
AC_LANG_POP
AC_MSG_CHECKING([whether AWS S3 SDK library is available])
AC_MSG_RESULT([$have_aws])
# Check for forced disabling of S3 support
AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled])
AC_ARG_ENABLE([s3-sdk],
[AS_HELP_STRING([--enable-s3-sdk],
[enable netcdf zarr S3 support; make sure to set LDFLAGS])])
test "x$enable_s3_sdk" = xyes || enable_s3_sdk=no
have_aws=${enable_s3_sdk}
[AS_HELP_STRING([--disable-s3-sdk],
[forcibly disable usage of netcdf zarr S3 sdk])])
test "x$enable_s3_sdk" = xno || enable_s3_sdk=yes
AC_MSG_RESULT($enable_s3_sdk)
if test "x$have_aws" = xyes ; then
AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
fi
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
# Check for enabling S3 testing
AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
AC_ARG_ENABLE([s3-tests],
[AS_HELP_STRING([--enable-s3-tests],
[enable netcdf zarr S3 testing])])
test "x$enable_s3_tests" = xyes || enable_s3_tests=no
AC_MSG_RESULT($enable_s3_tests)
if test "x$have_aws" = xno ; then
AC_MSG_WARN([aws libraries not found, so default to --disable-s3-tests])
enable_s3_tests=no
if test "x$have_aws" = xno -a "x$enable_s3_sdk" = xno; then
have_aws=no
enable_s3_sdk=no
elif test "x$have_aws" = xno -a "x$enable_s3_sdk" = xyes; then
AC_MSG_WARN([aws libraries not found, so default to --disable-s3-sdk])
have_aws=no
enable_s3_sdk=no
elif test "x$have_aws" = xyes -a "x$enable_s3_sdk" = xno; then
AC_MSG_WARN([--disable-s3-sdk overrides presence of aws s3 libraries])
have_aws=no
enable_s3_sdk=no
elif test "x$have_aws" = xyes -a "x$enable_s3_sdk" = xyes; then
have_aws=yes
enable_s3_sdk=yes
fi
if test "x$enable_s3_tests" = xyes ; then
AC_DEFINE([ENABLE_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled])
if test "x$enable_s3_sdk" = xyes ; then
AC_DEFINE([ENABLE_S3_SDK], [1], [if true, build libnczarr with S3 support enabled])
fi
AM_CONDITIONAL(ENABLE_S3_TESTS, [test "x$enable_s3_tests" = xyes])
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
# Check whether we want to enable strict null byte header padding.
# See https://github.com/Unidata/netcdf-c/issues/657 for more information.
@ -624,8 +607,8 @@ AC_ARG_ENABLE([benchmarks],
are timed. We use these tests to check netCDF performance.])])
test "x$enable_benchmarks" = xyes || enable_benchmarks=no
AC_MSG_RESULT($enable_benchmarks)
if test "x$enable_HDF5" = xno -a "x$enable_benchmarks" = xyes; then
AC_MSG_ERROR([Can't use benchmarks if HDF5 is disabled.])
if test "x$enable_netcdf4" = xno -a "x$enable_benchmarks" = xyes; then
AC_MSG_ERROR([Can't use benchmarks if netCDF-4 is disabled.])
fi
AM_CONDITIONAL(BUILD_BENCHMARKS, [test x$enable_benchmarks = xyes])
@ -1048,7 +1031,13 @@ AC_CHECK_SIZEOF(size_t)
$SLEEPCMD
AC_CHECK_SIZEOF(unsigned long long)
# Check whether we want to enable CDF5 support.
AC_MSG_CHECKING([whether CDF5 support should be disabled])
AC_ARG_ENABLE([cdf5],
[AS_HELP_STRING([--disable-cdf5],
[build without CDF5 support. @<:@default: auto@:>@])],
[enable_cdf5=${enableval}], [enable_cdf5=auto]
)
if test "$ac_cv_sizeof_size_t" -lt "8" ; then
if test "x${enable_cdf5}" = xyes ; then
dnl unable to support CDF5, but --enable-cdf5 is explicitly set
@ -1060,6 +1049,7 @@ else
enable_cdf5=yes
fi
fi
AC_MSG_RESULT($enable_cdf5)
if test "x${enable_cdf5}" = xyes; then
AC_DEFINE([ENABLE_CDF5], [1], [if true, enable CDF5 Support])
@ -1103,10 +1093,10 @@ AC_CHECK_SIZEOF(ssize_t)
$SLEEPCMD
AC_CHECK_SIZEOF([void*])
if test "x$enable_hdf5" = xyes || test "x$enable_dap" = xyes; then
if test "x$enable_netcdf_4" = xyes || test "x$enable_dap" = xyes; then
AC_SEARCH_LIBS([deflate], [zlibwapi zlibstat zlib zlib1 z], [], [
AC_MSG_ERROR([Can't find or link to the z library. Turn off netCDF-4 and \
DAP clients with --disable-hdf5 --disable-dap, or see config.log for errors.])])
DAP clients with --disable-netcdf-4 --disable-dap, or see config.log for errors.])])
AC_SEARCH_LIBS([dlopen], [dl dld], [], [])
fi
@ -1131,9 +1121,9 @@ if test "x$enable_hdf5" = xyes; then
# Check for the main hdf5 and hdf5_hl library.
AC_SEARCH_LIBS([H5Fflush], [hdf5dll hdf5], [],
[AC_MSG_ERROR([Can't find or link to the hdf5 library. Use --disable-hdf5, or see config.log for errors.])])
[AC_MSG_ERROR([Can't find or link to the hdf5 library. Use --disable-netcdf-4, or see config.log for errors.])])
AC_SEARCH_LIBS([H5DSis_scale], [hdf5_hldll hdf5_hl], [],
[AC_MSG_ERROR([Can't find or link to the hdf5 high-level. Use --disable-hdf5, or see config.log for errors.])])
[AC_MSG_ERROR([Can't find or link to the hdf5 high-level. Use --disable-netcdf-4, or see config.log for errors.])])
AC_CHECK_HEADERS([hdf5.h], [], [AC_MSG_ERROR([Compiling a test with HDF5 failed. Either hdf5.h cannot be found, or config.log should be checked for other reason.])])
@ -1238,6 +1228,12 @@ AC_MSG_RESULT($enable_parallel4)
# We have already tested for parallel io in netcdf4
# parallel I/O for CDF-1, 2, and 5 files can also be done through PnetCDF
AC_MSG_CHECKING([whether parallel I/O for classic files is to be enabled])
AC_ARG_ENABLE([pnetcdf], [AS_HELP_STRING([--enable-pnetcdf],
[build with parallel I/O for classic files. @<:@default: disabled@:>@])])
test "x$enable_pnetcdf" = xyes || enable_pnetcdf=no
AC_MSG_RESULT($enable_pnetcdf)
# See if the PnetCDF lib is available and of the
# right version (1.6.0 or later)
if test "x$enable_pnetcdf" = xyes; then
@ -1390,7 +1386,6 @@ AM_CONDITIONAL(ENABLE_BYTERANGE, [test "x$enable_byterange" = xyes])
AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes])
AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
AM_CONDITIONAL(HAVE_AWS, [test "x$have_aws" = xyes])
# If the machine doesn't have a long long, and we want netCDF-4, then
# we've got problems!
@ -1473,6 +1468,48 @@ AC_ARG_ENABLE([filter-testing],
test "x$enable_filter_testing" = xno || enable_filter_testing=yes
AC_MSG_RESULT($enable_filter_testing)
# Control client filters
AC_MSG_CHECKING([whether client filters should be enabled])
AC_ARG_ENABLE([client-filters],
[AS_HELP_STRING([--enable-clientfilters],
[Enable client side filters; default off])])
test "x$enable_client_filters" = xyes || enable_client_filters=no
AC_MSG_RESULT($enable_client_filters)
if test "x$enable_netcdf_4" = xno ; then
AC_MSG_WARN([netCDF-4 disabled => --disable-filter-testing && --disable-client-filters])
enable_filter_testing=no
enable_client_filters=no
fi
if test "x$enable_hdf5" = xno ; then
AC_MSG_WARN([HDF5 disabled => --disable-filter-testing])
enable_filter_testing=no
fi
if test "x$enable_shared" = xno ; then
AC_MSG_WARN([Shared libraries are disabled => --disable-filter-testing])
enable_filter_testing=no
fi
AM_CONDITIONAL(ENABLE_CLIENT_FILTERS, [test x$enable_client_filters = xyes])
if test "x$enable_netcdf_4" = xno ; then
AC_MSG_WARN([netCDF-4 disabled => --disable-filter-testing && --disable-client-filters])
enable_filter_testing=no
enable_client_filters=no
fi
if test "x$enable_hdf5" = xno ; then
AC_MSG_WARN([HDF5 disabled => --disable-filter-testing])
enable_filter_testing=no
fi
if test "x$enable_shared" = xno ; then
AC_MSG_WARN([Shared libraries are disabled => --disable-filter-testing])
enable_filter_testing=no
fi
AM_CONDITIONAL(ENABLE_FILTER_TESTING, [test x$enable_filter_testing = xyes])
# Enable client side filter registration
AC_MSG_CHECKING([If client-side filters are enabled (default off)])
AC_ARG_ENABLE([clientside-filters],
@ -1485,20 +1522,7 @@ AC_MSG_RESULT($enable_clientside_filters)
if test "x$enable_clientside_filters" = xyes ; then
AC_DEFINE([ENABLE_CLIENTSIDE_FILTERS], [1], [if true, enable client-side filters])
fi
if test "x$enable_hdf5" = xno ; then
AC_MSG_WARN([HDF5 disabled => --disable-filter-testing && --disable-client-filters])
enable_filter_testing=no
enable_clientside_filters=no
fi
AM_CONDITIONAL(ENABLE_CLIENTSIDE_FILTERS, [test x$enable_clientside_filters = xyes])
AM_CONDITIONAL(ENABLE_FILTER_TESTING, [test x$enable_filter_testing = xyes])
if test "x$enable_shared" = xno ; then
AC_MSG_WARN([Shared libraries are disabled => --disable-filter-testing])
enable_filter_testing=no
fi
AC_SUBST(NC_LIBS,[$NC_LIBS])
AC_SUBST(HAS_DAP,[$enable_dap])
@ -1523,7 +1547,7 @@ AC_SUBST(HAS_BYTERANGE,[$enable_byterange])
AC_SUBST(RELAX_COORD_BOUND,[yes])
AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters])
AC_SUBST(HAS_NCZARR,[$enable_nczarr])
AC_SUBST(DO_S3_TESTS,[$enable_s3_tests])
AC_SUBST(HAS_S3_SDK,[$enable_s3_sdk])
# Include some specifics for netcdf on windows.
#AH_VERBATIM([_WIN32_STRICMP],
@ -1580,7 +1604,7 @@ AC_SUBST([NC_VERSION]) NC_VERSION=$VERSION
AX_SET_META([NC_HAS_NC2],[$nc_build_v2],[yes])
AX_SET_META([NC_HAS_NC4],[$enable_netcdf_4],[yes])
AX_SET_META([NC_HAS_HDF4],[$enable_hdf4],[yes])
AX_SET_META([NC_HAS_HDF5],[$enable_hdf5],[yes])
AX_SET_META([NC_HAS_HDF5],[$enable_netcdf_4],[yes])
AX_SET_META([NC_HAS_SZIP],[$ac_cv_func_H5Z_SZIP],[yes])
AX_SET_META([NC_HAS_DAP2],[$enable_dap],[yes])
AX_SET_META([NC_HAS_DAP4],[$enable_dap4],[yes])
@ -1595,6 +1619,7 @@ AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes])
AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes])
AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes])
AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
AX_SET_META([NC_HAS_NCZARR_S3],[$enable_s3_sdk],[yes])
# This is the version of the dispatch table. If the dispatch table is
# changed, this should be incremented, so that user-defined format

143
debug/cf
View File

@ -1,22 +1,20 @@
#!/bin/bash
#NB=1
DB=1
#DB=1
#X=-x
#ANSI=1
#MEM=1
MEM=1
#NOTUIL=1
FAST=1
#PROF=1
PROF=1
NCZARR=1
HDF5=1
DAP=1
S3=1
#S3TEST=1
#SZIP=1
#HDF4=1
#S3=1
SZIP=1
HDF4=1
#PNETCDF=1
#PAR4=1
@ -36,13 +34,14 @@ fi
CFLAGS=""
#CFLAGS="-Wall -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unused-parameter -Wconversion ${CFLAGS}"
#CFLAGS="-Wall -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unused-parameter -Wno-char-subscripts -Wno-pointer-sign -Wno-format ${CFLAGS}"
CFLAGS="-Wall -Wno-unused-parameter -Wno-char-subscripts -Wno-pointer-sign ${CFLAGS}"
#CFLAGS="-Wall ${CFLAGS}"
#CFLAGS="$CFLAGS -Wconversion"
#CFLAGS="-Wall -Wunused-parameter -Wno-char-subscripts -Wno-pointer-sign ${CFLAGS}"
CFLAGS="-Wall -Wno-unused-parameter -Wno-char-subscripts -Wno-pointer-sign ${CFLAGS}"
if test "x$MEM" = x1 ; then
CFLAGS="-fsanitize=address ${CFLAGS}"
export export NC_VLEN_NOTEST=1
fi
if test "x$ANSI" = x1 ; then
@ -77,26 +76,128 @@ CC=gcc
MALLOC_CHECK=""
CPPFLAGS=""
LDFLAGS="-L/usr/lib -L${stddir}/lib"
#LD_LIBRARY_PATH="/usr/lib:${stddir}/lib:$LD_LIBRARY_PATH"
LDFLAGS=""
CFLAGS="-g -O0 $CFLAGS -Wno-undefined"
#CFLAGS="-g -O0 $CFLAGS -Wno-undefined"
CFLAGS="-g -O0 $CFLAGS"
MAKE=make
IGNORE="test 0 = 1"
if test "x$HDF5" != x ; then
if test "x$HDF5" = "x1" ; then
CPPFLAGS="-I${stddir}/include $CPPFLAGS"
LDFLAGS="-lhdf5_hl -lhdf5 -lz $LDFLAGS"
else
FLAGS="$FLAGS --disable-netcdf-4"
#FLAGS="$FLAGS --disable-hdf5"
LDFLAGS="-L${stddir}/lib -lhdf5_hl -lhdf5 -lz $LDFLAGS"
LD_LIBRARY_PATH="${stddir}/lib:$LD_LIBRARY_PATH"
fi
if test "x$HDF4" = "x1" ; then
LDFLAGS="$LDFLAGS -lmfhdf -ldf -ljpeg"
fi
if test "x$SZIP" = "x1" ; then
LDFLAGS="$LDFLAGS -lsz -laec"
fi
FLAGS="--prefix ${PREFIX}"
if test "x$NCZARR" = "x1" ; then
FLAGS="$FLAGS --enable-nczarr"
if test "x$S3" = x1 ; then
CPPFLAGS="-I${stddir}/include $CPPFLAGS "
#CXXFLAGS="-std=c++11 $CXXFLAGS"
LDFLAGS="$LDFLAGS -L${stddir}/lib -laws-cpp-sdk-s3"
LDFLAGS="$LDFLAGS -laws-cpp-sdk-core"
#LDFLAGS="$LDFLAGS -laws-c-common"
#LDFLAGS="$LDFLAGS -laws-c-event-stream"
#LDFLAGS="$LDFLAGS -laws-checksums"
#LDFLAGS="$LDFLAGS -lstdc++"
else
FLAGS="$FLAGS --disable-s3-sdk"
fi
fi
export PKG_CONFIG_PATH=/usr/lib/pkgconfig
if curl-config --version >/dev/null ; then
TMP=`curl-config --cflags`
CPPFLAGS="$TMP $CPPFLAGS"
TMP=`curl-config --libs`
LDFLAGS="$LDFLAGS $TMP"
#LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$WH"
else
LDFLAGS="${LDFLAGS} -lcurl"
fi
#FLAGS="$FLAGS --disable-cxx"
#FLAGS="$FLAGS --disable-examples"
#FLAGS="$FLAGS --enable-cxx-4"
#FLAGS="$FLAGS --enable-dap-long-tests"
#FLAGS="$FLAGS --enable-ffio"
#FLAGS="$FLAGS --enable-benchmarks"
FLAGS="$FLAGS --enable-extreme-numbers"
#FLAGS="$FLAGS --enable-large-file-tests"
#FLAGS="$FLAGS --disable-testsets"
#FLAGS="$FLAGS --disable-dap-remote-tests"
#FLAGS="$FLAGS --enable-dap-auth-tests" -- requires a new remotetest server
#FLAGS="$FLAGS --enable-doxygen --enable-internal-docs"
FLAGS="$FLAGS --enable-logging"
#FLAGS="$FLAGS --disable-diskless"
FLAGS="$FLAGS --enable-mmap"
FLAGS="$FLAGS --enable-byterange"
#FLAGS="$FLAGS --enable-atexit-finalize"
#FLAGS="$FLAGS --with-udunits"
#FLAGS="$FLAGS --with-libcf"
#FLAGS="$FLAGS --enable-jna"
#FLAGS="$FLAGS --disable-properties-attribute"
#FLAGS="$FLAGS --disable-silent-rules"
#FLAGS="$FLAGS --disable-filter-testing"
#FLAGS="$FLAGS --enable-metadata-perf"
#FLAGS="$FLAGS --with-ncproperties-extra=key1=value1,key2=value2"
#FLAGS="$FLAGS --enable-valgrind"
#FLAGS="$FLAGS --enable-unfixed-memory-leaks"
if test "x$TESTSERVERS" != x ; then
FLAGS="$FLAGS --with-testservers=$TESTSERVERS"
fi
if test "x$PAR4" != x1 ; then
FLAGS="$FLAGS --disable-parallel4"
fi
if test "x$NOUTIL" = x1 ; then
FLAGS="$FLAGS --disable-utilities"
fi
if test "x${DB}" = x1 ; then
FLAGS="$FLAGS --disable-shared --enable-static"
#FLAGS="$FLAGS --enable-static"
else
FLAGS="$FLAGS --enable-shared"
fi
#TMP=
#rm -f ./test_mpi.tmp
#if test -f $stddir/lib/libhdf5.a ; then
# nm $stddir/lib/libhdf5.a | grep mpich_mpi_float$ >./test_mpi.tmp
#else
# if test -f $stddir/lib/libhdf5.so ; then
# nm $stddir/lib/libhdf5.so | grep mpich_mpi_float$ >./test_mpi.tmp
# fi
#fi
#if test -s ./test_mpi.tmp ; then
# PAR=1
#fi
#rm -f ./test_mpi.tmp
if test "x${PAR}" != x ; then
FLAGS="$FLAGS --enable-parallel"
fi
if test "x$HDF5" = "x" ; then
FLAGS="$FLAGS --disable-netcdf-4"
fi
if test "x$HDF4" = x1 ; then
FLAGS="$FLAGS --enable-hdf4 --enable-hdf4-file-tests"
fi
if test "x$DAP" = "x" ; then
FLAGS="$FLAGS --disable-dap"
fi
@ -116,15 +217,13 @@ if test "x$PNETCDF" = x1 ; then
FLAGS="$FLAGS --enable-pnetcdf"
fi
if test "x$NCZARR" != "x" ; then
FLAGS="$FLAGS --enable-nczarr"
fi
if test "x${PROF}" = x1 ; then
CFLAGS="${CFLAGS} -pg"
LDFLAGS="${LDFLAGS} -pg"
fi
CXXFLAGS="$CFLAGS $CXXFLAGS"
export PATH
export CC
export CPPFLAGS

33
debug/cf.cmake
View File

@ -4,16 +4,15 @@ NCC="c:/tools/hdf5"
# Is netcdf-4 and/or DAP enabled?
NCZARR=1
HDF5=1
NC4=1
DAP=1
S3=1
#S3TEST=1
#CDF5=1
#HDF4=1
HDF4=1
#S3=1
#TR=--trace
export SETX=1
#export SETX=1
for arg in "$@" ; do
case "$arg" in
@ -61,8 +60,8 @@ if test "x$DAP" = x ; then
FLAGS="$FLAGS -DENABLE_DAP=false"
fi
if test "x$HDF5" = x ; then
FLAGS="$FLAGS -DENABLE_HDF5=false"
if test "x$NC4" = x ; then
FLAGS="$FLAGS -DENABLE_NETCDF_4=false"
else
ignore=1
#FLAGS="$FLAGS -DDEFAULT_API_VERSION:STRING=v110"
@ -83,21 +82,16 @@ if test "x$TESTSERVERS" != x ; then
FLAGS="$FLAGS -DREMOTETESTSERVERS=${TESTSERVERS}"
fi
if test "x$S3" != x ; then
FLAGS="$FLAGS -DENABLE_S3_SDK=true"
else
FLAGS="$FLAGS -DENABLE_S3_SDK=false"
fi
if test "x$S3TEST" != x ; then
FLAGS="$FLAGS -DENABLE_S3_TESTS=true"
fi
# Enables
FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true"
FLAGS="$FLAGS -DENABLE_LOGGING=true"
#FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true"
#FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true"
#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
if test "x$S3" = x ; then
FLAGS="$FLAGS -DENABLE_S3_SDK=false"
fi
# Disables
FLAGS="$FLAGS -DENABLE_EXAMPLES=false"
@ -105,9 +99,8 @@ FLAGS="$FLAGS -DENABLE_CONVERSION_WARNINGS=false"
#FLAGS="$FLAGS -DENABLE_TESTS=false"
#FLAGS="$FLAGS -DENABLE_DISKLESS=false"
FLAGS="$FLAGS -DBUILD_UTILITIES=true"
FLAGS="$FLAGS -DENABLE_FILTER_TESTING=false"
FLAGS="$FLAGS -DCURL_NO_CURL_CMAKE=TRUE"
#FLAGS="$FLAGS -DCURL_NO_CURL_CMAKE=TRUE"
# Withs
FLAGS="$FLAGS -DNCPROPERTIES_EXTRA=\"key1=value1|key2=value2\""
@ -142,7 +135,7 @@ NCLIB="${NCLIB}/build/liblib"
#T="--trace-expand"
cmake "${G}" $FLAGS ..
if test "x$NOBUILD" = x ; then
make all
make VERBOSE=1 all
fi
if test "x$NOTEST" = x ; then
make test

2
include/Makefile.am
View File

@ -19,7 +19,7 @@ ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h \
nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \
ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \
ncwinpath.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \
hdf5dispatch.h ncmodel.h ncfilter.h isnan.h
hdf5dispatch.h ncmodel.h ncfilter.h
if USE_DAP
noinst_HEADERS += ncdap.h

9
include/hdf5dispatch.h
View File

@ -53,15 +53,6 @@ extern "C" {
NC4_HDF5_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
float preemption);
EXTERNL int
HDF5_def_dim(int ncid, const char *name, size_t len, int *idp);
EXTERNL int
HDF5_inq_dim(int ncid, int dimid, char *name, size_t *lenp);
EXTERNL int
HDF5_rename_dim(int ncid, int dimid, const char *name);
#if defined(__cplusplus)
}
#endif

9
include/nc4dispatch.h
View File

@ -63,12 +63,21 @@ extern "C" {
/* Begin _dim */
EXTERNL int
NC4_def_dim(int ncid, const char *name, size_t len, int *idp);
EXTERNL int
NC4_inq_dimid(int ncid, const char *name, int *idp);
EXTERNL int
NC4_inq_dim(int ncid, int dimid, char *name, size_t *lenp);
EXTERNL int
NC4_inq_unlimdim(int ncid, int *unlimdimidp);
EXTERNL int
NC4_rename_dim(int ncid, int dimid, const char *name);
/* End _dim */
/* Begin _att */

107
include/nc4internal.h
View File

@ -338,100 +338,100 @@ typedef struct
} nc_hvl_t;
/* Misc functions */
extern int NC4_inq_atomic_type(nc_type typeid1, char *name, size_t *size);
extern int NC4_lookup_atomic_type(const char *name, nc_type* idp, size_t *sizep);
int NC4_inq_atomic_type(nc_type typeid1, char *name, size_t *size);
int NC4_lookup_atomic_type(const char *name, nc_type* idp, size_t *sizep);
/* These functions convert between netcdf and HDF5 types. */
extern int nc4_get_typelen_mem(NC_FILE_INFO_T *h5, nc_type xtype, size_t *len);
extern int nc4_convert_type(const void *src, void *dest, const nc_type src_type,
int nc4_get_typelen_mem(NC_FILE_INFO_T *h5, nc_type xtype, size_t *len);
int nc4_convert_type(const void *src, void *dest, const nc_type src_type,
const nc_type dest_type, const size_t len, int *range_error,
const void *fill_value, int strict_nc3);
/* These functions do HDF5 things. */
extern int nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
extern int nc4_read_atts(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
int nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
int nc4_read_atts(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
/* Find items in the in-memory lists of metadata. */
extern int nc4_find_nc_grp_h5(int ncid, NC **nc, NC_GRP_INFO_T **grp,
int nc4_find_nc_grp_h5(int ncid, NC **nc, NC_GRP_INFO_T **grp,
NC_FILE_INFO_T **h5);
extern int nc4_find_grp_h5(int ncid, NC_GRP_INFO_T **grp, NC_FILE_INFO_T **h5);
extern int nc4_find_nc4_grp(int ncid, NC_GRP_INFO_T **grp);
extern int nc4_find_dim(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T **dim,
int nc4_find_grp_h5(int ncid, NC_GRP_INFO_T **grp, NC_FILE_INFO_T **h5);
int nc4_find_nc4_grp(int ncid, NC_GRP_INFO_T **grp);
int nc4_find_dim(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T **dim,
NC_GRP_INFO_T **dim_grp);
extern int nc4_find_var(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var);
extern int nc4_find_dim_len(NC_GRP_INFO_T *grp, int dimid, size_t **len);
extern int nc4_find_type(const NC_FILE_INFO_T *h5, int typeid1, NC_TYPE_INFO_T **type);
extern NC_TYPE_INFO_T *nc4_rec_find_named_type(NC_GRP_INFO_T *start_grp, char *name);
extern NC_TYPE_INFO_T *nc4_rec_find_equal_type(NC_GRP_INFO_T *start_grp, int ncid1,
int nc4_find_var(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var);
int nc4_find_dim_len(NC_GRP_INFO_T *grp, int dimid, size_t **len);
int nc4_find_type(const NC_FILE_INFO_T *h5, int typeid1, NC_TYPE_INFO_T **type);
NC_TYPE_INFO_T *nc4_rec_find_named_type(NC_GRP_INFO_T *start_grp, char *name);
NC_TYPE_INFO_T *nc4_rec_find_equal_type(NC_GRP_INFO_T *start_grp, int ncid1,
NC_TYPE_INFO_T *type);
extern int nc4_find_nc_att(int ncid, int varid, const char *name, int attnum,
int nc4_find_nc_att(int ncid, int varid, const char *name, int attnum,
NC_ATT_INFO_T **att);
extern int nc4_find_grp_h5_var(int ncid, int varid, NC_FILE_INFO_T **h5,
int nc4_find_grp_h5_var(int ncid, int varid, NC_FILE_INFO_T **h5,
NC_GRP_INFO_T **grp, NC_VAR_INFO_T **var);
extern int nc4_find_grp_att(NC_GRP_INFO_T *grp, int varid, const char *name,
int nc4_find_grp_att(NC_GRP_INFO_T *grp, int varid, const char *name,
int attnum, NC_ATT_INFO_T **att);
extern int nc4_get_typeclass(const NC_FILE_INFO_T *h5, nc_type xtype,
int nc4_get_typeclass(const NC_FILE_INFO_T *h5, nc_type xtype,
int *type_class);
/* Free various types */
extern int nc4_type_free(NC_TYPE_INFO_T *type);
int nc4_type_free(NC_TYPE_INFO_T *type);
/* These list functions add and delete vars, atts. */
extern int nc4_nc4f_list_add(NC *nc, const char *path, int mode);
extern int nc4_nc4f_list_del(NC_FILE_INFO_T *h5);
extern int nc4_file_list_add(int ncid, const char *path, int mode,
int nc4_nc4f_list_add(NC *nc, const char *path, int mode);
int nc4_nc4f_list_del(NC_FILE_INFO_T *h5);
int nc4_file_list_add(int ncid, const char *path, int mode,
void **dispatchdata);
extern int nc4_file_list_get(int ncid, char **path, int *mode,
int nc4_file_list_get(int ncid, char **path, int *mode,
void **dispatchdata);
extern int nc4_file_list_del(int ncid);
extern int nc4_file_change_ncid(int ncid, unsigned short new_ncid_index);
extern int nc4_var_list_add(NC_GRP_INFO_T* grp, const char* name, int ndims,
int nc4_file_list_del(int ncid);
int nc4_file_change_ncid(int ncid, unsigned short new_ncid_index);
int nc4_var_list_add(NC_GRP_INFO_T* grp, const char* name, int ndims,
NC_VAR_INFO_T **var);
extern int nc4_var_list_add2(NC_GRP_INFO_T* grp, const char* name,
int nc4_var_list_add2(NC_GRP_INFO_T* grp, const char* name,
NC_VAR_INFO_T **var);
extern int nc4_var_set_ndims(NC_VAR_INFO_T *var, int ndims);
extern int nc4_var_list_del(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
extern int nc4_dim_list_add(NC_GRP_INFO_T *grp, const char *name, size_t len,
int nc4_var_set_ndims(NC_VAR_INFO_T *var, int ndims);
int nc4_var_list_del(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
int nc4_dim_list_add(NC_GRP_INFO_T *grp, const char *name, size_t len,
int assignedid, NC_DIM_INFO_T **dim);
extern int nc4_dim_list_del(NC_GRP_INFO_T *grp, NC_DIM_INFO_T *dim);
extern int nc4_type_new(size_t size, const char *name, int assignedid,
int nc4_dim_list_del(NC_GRP_INFO_T *grp, NC_DIM_INFO_T *dim);
int nc4_type_new(size_t size, const char *name, int assignedid,
NC_TYPE_INFO_T **type);
extern int nc4_type_list_add(NC_GRP_INFO_T *grp, size_t size, const char *name,
int nc4_type_list_add(NC_GRP_INFO_T *grp, size_t size, const char *name,
NC_TYPE_INFO_T **type);
extern int nc4_type_list_del(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type);
extern int nc4_type_free(NC_TYPE_INFO_T *type);
extern int nc4_field_list_add(NC_TYPE_INFO_T* parent, const char *name,
int nc4_type_list_del(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type);
int nc4_type_free(NC_TYPE_INFO_T *type);
int nc4_field_list_add(NC_TYPE_INFO_T* parent, const char *name,
size_t offset, nc_type xtype, int ndims,
const int *dim_sizesp);
extern int nc4_att_list_add(NCindex *list, const char *name, NC_ATT_INFO_T **att);
extern int nc4_att_list_del(NCindex *list, NC_ATT_INFO_T *att);
extern int nc4_grp_list_add(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *parent, char *name,
int nc4_att_list_add(NCindex *list, const char *name, NC_ATT_INFO_T **att);
int nc4_att_list_del(NCindex *list, NC_ATT_INFO_T *att);
int nc4_grp_list_add(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *parent, char *name,
NC_GRP_INFO_T **grp);
extern int nc4_build_root_grp(NC_FILE_INFO_T *h5);
extern int nc4_rec_grp_del(NC_GRP_INFO_T *grp);
extern int nc4_enum_member_add(NC_TYPE_INFO_T *type, size_t size, const char *name,
int nc4_build_root_grp(NC_FILE_INFO_T *h5);
int nc4_rec_grp_del(NC_GRP_INFO_T *grp);
int nc4_enum_member_add(NC_TYPE_INFO_T *type, size_t size, const char *name,
const void *value);
extern int nc4_att_free(NC_ATT_INFO_T *att);
int nc4_att_free(NC_ATT_INFO_T *att);
/* Check and normalize names. */
extern int NC_check_name(const char *name);
extern int nc4_check_name(const char *name, char *norm_name);
extern int nc4_normalize_name(const char *name, char *norm_name);
extern int nc4_check_dup_name(NC_GRP_INFO_T *grp, char *norm_name);
int NC_check_name(const char *name);
int nc4_check_name(const char *name, char *norm_name);
int nc4_normalize_name(const char *name, char *norm_name);
int nc4_check_dup_name(NC_GRP_INFO_T *grp, char *norm_name);
/* Get the fill value for a var. */
extern int nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp);
int nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp);
/* Find default fill value. */
extern int nc4_get_default_fill_value(nc_type typecode, void *fill_value);
int nc4_get_default_fill_value(nc_type typecode, void *fill_value);
/* Get an att given pointers to file, group, and perhaps ver info. */
extern int nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var,
int nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var,
const char *name, nc_type *xtype, nc_type mem_type,
size_t *lenp, int *attnum, void *data);
/* Close the file. */
extern int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio);
int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio);
/* HDF5 initialization/finalization */
extern int nc4_hdf5_initialized;
@ -441,7 +441,7 @@ extern void nc4_hdf5_finalize(void);
/* This is only included if --enable-logging is used for configure; it
prints info about the metadata to stderr. */
#ifdef LOGGING
extern int log_metadata_nc(NC_FILE_INFO_T *h5);
int log_metadata_nc(NC_FILE_INFO_T *h5);
#endif
/** @internal Names of atomic types. */
@ -459,5 +459,4 @@ extern const NC_reservedatt* NC_findreserved(const char* name);
#define NC_ATT_FORMAT "_Format"
#define NC_ATT_DIMID_NAME "_Netcdf4Dimid"
#define NC_ATT_NC3_STRICT_NAME "_nc3_strict"
#endif /* _NC4INTERNAL_ */

2
include/nc_logging.h
View File

@ -51,7 +51,7 @@ void nc_log_hdf5(void);
#define BAIL_QUIET BAIL
#ifdef USE_NETCDF4
#ifdef USE_NETCDF_4
#ifndef ENABLE_SET_LOG_LEVEL
/* Define away any calls to nc_set_log_level(), if its not enabled. */
#define nc_set_log_level(e)

2
include/ncdispatch.h
View File

@ -55,7 +55,9 @@
#define T_uint NC_UINT
#define T_longlong NC_INT64
#define T_ulonglong NC_UINT64
#ifdef USE_NETCDF4
#define T_string NC_STRING
#endif
/* Synthetic type to handle special memtypes */
#define T_uchar NC_UBYTE

40
include/nclist.h
View File

@ -3,15 +3,13 @@
#ifndef NCLIST_H
#define NCLIST_H 1
#include "ncexternl.h"
/* Define the type of the elements in the list*/
#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__)
extern "C" {
#endif
EXTERNL int nclistnull(void*);
extern int nclistnull(void*);
typedef struct NClist {
size_t alloc;
@ -19,42 +17,42 @@ typedef struct NClist {
void** content;
} NClist;
EXTERNL NClist* nclistnew(void);
EXTERNL int nclistfree(NClist*);
EXTERNL int nclistfreeall(NClist*);
EXTERNL int nclistsetalloc(NClist*,size_t);
EXTERNL int nclistsetlength(NClist*,size_t);
extern NClist* nclistnew(void);
extern int nclistfree(NClist*);
extern int nclistfreeall(NClist*);
extern int nclistsetalloc(NClist*,size_t);
extern int nclistsetlength(NClist*,size_t);
/* Set the ith element; will overwrite previous contents; expand if needed */
EXTERNL int nclistset(NClist*,size_t,void*);
extern int nclistset(NClist*,size_t,void*);
/* Get value at position i */
EXTERNL void* nclistget(NClist*,size_t);/* Return the ith element of l */
extern void* nclistget(NClist*,size_t);/* Return the ith element of l */
/* Insert at position i; will push up elements i..|seq|. */
EXTERNL int nclistinsert(NClist*,size_t,void*);
extern int nclistinsert(NClist*,size_t,void*);
/* Remove element at position i; will move higher elements down */
EXTERNL void* nclistremove(NClist* l, size_t i);
extern void* nclistremove(NClist* l, size_t i);
/* Tail operations */
EXTERNL int nclistpush(NClist*,const void*); /* Add at Tail */
EXTERNL void* nclistpop(NClist*);
EXTERNL void* nclisttop(NClist*);
extern int nclistpush(NClist*,const void*); /* Add at Tail */
extern void* nclistpop(NClist*);
extern void* nclisttop(NClist*);
/* Look for pointer match */
EXTERNL int nclistcontains(NClist*, void*);
extern int nclistcontains(NClist*, void*);
/* Look for value match as string */
EXTERNL int nclistmatch(NClist*, const char*, int casesensistive);
extern int nclistmatch(NClist*, const char*, int casesensistive);
/* Remove element by value; only removes first encountered */
EXTERNL int nclistelemremove(NClist* l, void* elem);
extern int nclistelemremove(NClist* l, void* elem);
/* remove duplicates */
EXTERNL int nclistunique(NClist*);
extern int nclistunique(NClist*);
/* Create a clone of a list; if deep, then assume it is a list of strings */
EXTERNL NClist* nclistclone(NClist*, int deep);
extern NClist* nclistclone(NClist*, int deep);
EXTERNL void* nclistextract(NClist*);
extern void* nclistextract(NClist*);
/* Following are always "in-lined"*/
#define nclistclear(l) nclistsetlength((l),0)

4
include/ncwinpath.h
View File

@ -41,9 +41,6 @@
/* Path Converter */
EXTERNL char* NCpathcvt(const char* path);
/* path -> URL Path converter */
EXTERNL char* NCurlpath(const char* path);
/* Fix path in case it was escaped by shell */
EXTERNL char* NCdeescape(const char* name);
@ -81,6 +78,5 @@ EXTERNL int NCclosedir(DIR* ent);
/* Platform independent */
#define NCclose(fd) close(fd)
EXTERNL int NChasdriveletter(const char* path);
#endif /* _NCWINIO_H_ */

2
include/netcdf_meta.h.in
View File

@ -60,6 +60,6 @@
#define NC_RELAX_COORD_BOUND 1 /*!< RELAX_COORD_BOUND */
#define NC_DISPATCH_VERSION @NC_DISPATCH_VERSION@ /*!< Dispatch table version */
#define NC_HAS_PAR_FILTERS @NC_HAS_PAR_FILTERS@ /* Parallel I/O with filter support. */
#define NC_HAS_NCZARR @NC_HAS_NCZARR@
#define NC_HAS_ZARR @NC_HAS_ZARR@
#endif

2
lib_flags.am
View File

@ -13,7 +13,7 @@ if USE_DAP
AM_CPPFLAGS += -I${top_srcdir}/oc2
endif
if HAVE_AWS
if ENABLE_S3_SDK
AM_LDFLAGS += -lstdc++
endif

4
libdap2/CMakeLists.txt
View File

@ -20,9 +20,9 @@ set(man_MANS netcdf.3)
set(ARGS_MANPAGE -DAPI=C)
IF (USE_HDF5)
IF (USE_NETCDF4)
SET(ARGS_MANPAGE ${ARGS_MANPAGE} -DNETCDF4=TRUE)
ENDIF(USE_HDF5)
ENDIF(USE_NETCDF4)
IF (BUILD_DAP)
SET(ARGS_MANPAGE ${ARGS_MANPAGE} -DDAP=TRUE)

5
libdap2/Makefile.am
View File

@ -24,6 +24,11 @@ HDRS= nccommon.h constraints.h ncd2dispatch.h dapincludes.h \
dapodom.h getvara.h dapnc.h daputil.h dapdebug.h dapdump.h \
dceconstraints.h dcetab.h dceparselex.h
if USE_NETCDF4
#SRC += cdf4.c constraints4.c ncd4dispatch.c ncdap4.c getvara4.c
#HDRS +=constraints4.h ncd4dispatch.h ncdap4.h
endif
if ENABLE_DAP
if USE_NETCDF4

4
libdap4/CMakeLists.txt
View File

@ -20,9 +20,9 @@ set(man_MANS netcdf.3)
set(ARGS_MANPAGE -DAPI=C)
IF (USE_HDf5)
IF (USE_NETCDF4)
SET(ARGS_MANPAGE ${ARGS_MANPAGE} -DNETCDF4=TRUE)
ENDIF(USE_HDF5)
ENDIF(USE_NETCDF4)
IF (BUILD_DAP)
SET(ARGS_MANPAGE ${ARGS_MANPAGE} -DDAP=TRUE)

8
libdispatch/ddispatch.c
View File

@ -17,7 +17,7 @@ See LICENSE.txt for license information.
#endif
/* Required for getcwd, other functions. */
#ifdef _WIN32
#ifdef _MSC_VER
#include <direct.h>
#define getcwd _getcwd
#endif
@ -61,7 +61,7 @@ NCDISPATCH_initialize(void)
char* p;
char* q;
char cwd[4096];
#ifdef _WIN32
#ifdef _MSC_VER
tempdir = getenv("TEMP");
#else
tempdir = "/tmp";
@ -78,7 +78,7 @@ NCDISPATCH_initialize(void)
*q = *p;
}
*q = '\0';
#ifdef _WIN32
#ifdef _MSC_VER
#else
/* Canonicalize */
for(p=globalstate->tempdir;*p;p++) {
@ -105,7 +105,7 @@ NCDISPATCH_initialize(void)
*q = *p;
}
*q = '\0';
#ifdef _WIN32
#ifdef _MSC_VER
#else
/* Canonicalize */
for(p=home;*p;p++) {

2
libdispatch/dhttp.c
View File

@ -52,7 +52,7 @@ static void
Trace(const char* fcn)
{
fprintf(stdout,"xxx: %s\n",fcn);
dbgflush();
dbgflush();
}
#else
#define dbgflush()

19
libdispatch/dutil.c
View File

@ -351,22 +351,3 @@ done:
nclistfreeall(modelist);
return found;
}
#ifdef __APPLE__
int isinf(double x)
{
union { unsigned long long u; double f; } ieee754;
ieee754.f = x;
return ( (unsigned)(ieee754.u >> 32) & 0x7fffffff ) == 0x7ff00000 &&
( (unsigned)ieee754.u == 0 );
}
int isnan(double x)
{
union { unsigned long long u; double f; } ieee754;
ieee754.f = x;
return ( (unsigned)(ieee754.u >> 32) & 0x7fffffff ) +
( (unsigned)ieee754.u != 0 ) > 0x7ff00000;
}
#endif /*APPLE*/

52
libdispatch/dwinpath.c
View File

@ -45,6 +45,7 @@ Rules:
All other cases are passed thru unchanged
*/
/* Define legal windows drive letters */
static const char* windrive = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
@ -62,16 +63,9 @@ NCpathcvt(const char* path)
char* p;
char* q;
size_t pathlen;
int forwardslash;
if(path == NULL) goto done; /* defensive driving */
#ifdef _WIN32
forwardslash = 0;
#else
forwardslash = 1;
#endif
/* Check for path debug env vars */
if(pathdebug < 0) {
const char* s = getenv("NCPATHDEBUG");
@ -116,7 +110,6 @@ NCpathcvt(const char* path)
}
/* 3. Look for leading D: where D is a single-char drive letter */
/* This could be cygwin or Windows or mingw */
if(pathlen >= 2
&& strchr(windrive,path[0]) != NULL
&& path[1] == ':'
@ -136,24 +129,23 @@ NCpathcvt(const char* path)
goto done;
slashtrans:
/* In order to help debugging, and if not using MSC_VER or MINGW or CYGWIN,
/* In order to help debugging, and if not using MSC_VER or MINGW,
convert back slashes to forward, else convert forward to back
*/
if(!forwardslash) {
p = outpath;
/* In all #1 or #2 cases, translate '/' -> '\\' */
for(;*p;p++) {
if(*p == '/') {*p = '\\';}
}
p = outpath;
/* In all #1 or #2 cases, translate '/' -> '\\' */
for(;*p;p++) {
if(*p == '/') {*p = '\\';}
}
#ifdef PATHFORMAT
if(forwardslash) {
#ifndef _WIN32
p = outpath;
/* Convert '\' back to '/' */
for(;*p;p++) {
if(*p == '\\') {*p = '/';}
}
}
#endif /*!_WIN32*/
#endif /*PATHFORMAT*/
done:
@ -165,19 +157,6 @@ done:
return outpath;
}
/* Make path suitable for inclusion in url */
EXTERNL
char* /* caller frees */
NCurlpath(const char* path)
{
char* upath = NCpathcvt(path);
char* p = upath;
for(;*p;p++) {
if(*p == '\\') {*p = '/';}
}
return upath;
}
static char*
makeabsolute(const char* relpath)
{
@ -220,18 +199,6 @@ NCdeescape(const char* name)
return ename;
}
int
NChasdriveletter(const char* path)
{
/* Check for windows drive letter */
if(path == NULL || strlen(path) < 2)
return 0;
if(strchr(windrive,path[0]) != NULL && path[1] == ':')
return 1; /* windows path with drive letter */
return 0;
}
#ifdef WINPATH
/*
@ -335,7 +302,7 @@ NCmkdir(const char* path, int mode)
int status = 0;
char* cvtname = NCpathcvt(path);
if(cvtname == NULL) return -1;
status = mkdir(cvtname,mode);
status = _mkdir(cvtname,mode);
free(cvtname);
return status;
}
@ -360,4 +327,5 @@ done:
if(errno) return NULL;
return cwdbuf;
}
#endif /*WINPATH*/

3
libhdf4/hdf4dispatch.c
View File

@ -12,7 +12,6 @@
#include <stdlib.h>
#include "hdf4dispatch.h"
#include "nc4dispatch.h"
#include "hdf5dispatch.h"
/* This is the dispatch object that holds pointers to all the
* functions that make up the HDF4 dispatch interface. */
@ -38,7 +37,7 @@ static const NC_Dispatch HDF4_dispatcher = {
NC_RO_def_dim,
NC4_inq_dimid,
HDF5_inq_dim,
NC4_inq_dim,
NC4_inq_unlimdim,
NC_RO_rename_dim,

6
libhdf5/hdf5dim.c
View File

@ -38,7 +38,7 @@
* @author Ed Hartnett
*/
int
HDF5_def_dim(int ncid, const char *name, size_t len, int *idp)
NC4_def_dim(int ncid, const char *name, size_t len, int *idp)
{
NC *nc;
NC_GRP_INFO_T *grp;
@ -132,7 +132,7 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp)
* @author Ed Hartnett
*/
int
HDF5_inq_dim(int ncid, int dimid, char *name, size_t *lenp)
NC4_inq_dim(int ncid, int dimid, char *name, size_t *lenp)
{
NC *nc;
NC_FILE_INFO_T *h5;
@ -217,7 +217,7 @@ HDF5_inq_dim(int ncid, int dimid, char *name, size_t *lenp)
* @author Ed Hartnett
*/
int
HDF5_rename_dim(int ncid, int dimid, const char *name)
NC4_rename_dim(int ncid, int dimid, const char *name)
{
NC_GRP_INFO_T *grp;
NC_DIM_INFO_T *dim;

6
libhdf5/hdf5dispatch.c
View File

@ -35,11 +35,11 @@ static const NC_Dispatch HDF5_dispatcher = {
NC4_inq,
NC4_inq_type,
HDF5_def_dim,
NC4_def_dim,
NC4_inq_dimid,
HDF5_inq_dim,
NC4_inq_dim,
NC4_inq_unlimdim,
HDF5_rename_dim,
NC4_rename_dim,
NC4_HDF5_inq_att,
NC4_HDF5_inq_attid,

78
libhdf5/hdf5type.c
View File

@ -94,6 +94,84 @@ NC4_inq_type_equal(int ncid1, nc_type typeid1, int ncid2,
return NC_NOERR;
}
/**
* @internal Get the id of a type from the name.
*
* @param ncid File and group ID.
* @param name Name of type.
* @param typeidp Pointer that will get the type ID.
*
* @return ::NC_NOERR No error.
* @return ::NC_ENOMEM Out of memory.
* @return ::NC_EINVAL Bad size.
* @return ::NC_ENOTNC4 User types in netCDF-4 files only.
* @return ::NC_EBADTYPE Type not found.
* @author Ed Hartnett
*/
EXTERNL int
NC4_inq_typeid(int ncid, const char *name, nc_type *typeidp)
{
NC_GRP_INFO_T *grp;
NC_GRP_INFO_T *grptwo;
NC_FILE_INFO_T *h5;
NC_TYPE_INFO_T *type = NULL;
char *norm_name;
int i, retval;
/* Handle atomic types. */
for (i = 0; i < NUM_ATOMIC_TYPES; i++)
if (!strcmp(name, nc4_atomic_name[i]))
{
if (typeidp)
*typeidp = i;
return NC_NOERR;
}
/* Find info for this file and group, and set pointer to each. */
if ((retval = nc4_find_grp_h5(ncid, &grp, &h5)))
return retval;
assert(h5 && grp);
/* If the first char is a /, this is a fully-qualified
* name. Otherwise, this had better be a local name (i.e. no / in
* the middle). */
if (name[0] != '/' && strstr(name, "/"))
return NC_EINVAL;
/* Normalize name. */
if (!(norm_name = (char*)malloc(strlen(name) + 1)))
return NC_ENOMEM;
if ((retval = nc4_normalize_name(name, norm_name))) {
free(norm_name);
return retval;
}
/* Is the type in this group? If not, search parents. */
for (grptwo = grp; grptwo; grptwo = grptwo->parent) {
type = (NC_TYPE_INFO_T*)ncindexlookup(grptwo->type,norm_name);
if(type)
{
if (typeidp)
*typeidp = type->hdr.id;
break;
}
}
/* Still didn't find type? Search file recursively, starting at the
* root group. */
if (!type)
if ((type = nc4_rec_find_named_type(grp->nc4_info->root_grp, norm_name)))
if (typeidp)
*typeidp = type->hdr.id;
free(norm_name);
/* OK, I give up already! */
if (!type)
return NC_EBADTYPE;
return NC_NOERR;
}
/**
* @internal This internal function adds a new user defined type to
* the metadata of a group of an open file.

2
libhdf5/hdf5var.c
View File

@ -1910,7 +1910,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
/* We can't go beyond the largest current extent of
the unlimited dim. */
if ((retval = HDF5_inq_dim(ncid, dim->hdr.id, NULL, &ulen)))
if ((retval = NC4_inq_dim(ncid, dim->hdr.id, NULL, &ulen)))
BAIL(retval);
/* Check for out of bound requests. */

72
libhdf5/nc4hdf.c
View File

@ -2234,6 +2234,78 @@ nc4_rec_match_dimscales(NC_GRP_INFO_T *grp)
return retval;
}
/**
* @internal Get the class of a type
*
* @param h5 Pointer to the HDF5 file info struct.
* @param xtype NetCDF type ID.
* @param type_class Pointer that gets class of type, NC_INT,
* NC_FLOAT, NC_CHAR, or NC_STRING, NC_ENUM, NC_VLEN, NC_COMPOUND, or
* NC_OPAQUE.
*
* @return ::NC_NOERR No error.
* @author Ed Hartnett
*/
int
nc4_get_typeclass(const NC_FILE_INFO_T *h5, nc_type xtype, int *type_class)
{
int retval = NC_NOERR;
LOG((4, "%s xtype: %d", __func__, xtype));
assert(type_class);
/* If this is an atomic type, the answer is easy. */
if (xtype <= NC_STRING)
{
switch (xtype)
{
case NC_BYTE:
case NC_UBYTE:
case NC_SHORT:
case NC_USHORT:
case NC_INT:
case NC_UINT:
case NC_INT64:
case NC_UINT64:
/* NC_INT is class used for all integral types */
*type_class = NC_INT;
break;
case NC_FLOAT:
case NC_DOUBLE:
/* NC_FLOAT is class used for all floating-point types */
*type_class = NC_FLOAT;
break;
case NC_CHAR:
*type_class = NC_CHAR;
break;
case NC_STRING:
*type_class = NC_STRING;
break;
default:
BAIL(NC_EBADTYPE);
}
}
else
{
NC_TYPE_INFO_T *type;
/* See if it's a used-defined type */
if ((retval = nc4_find_type(h5, xtype, &type)))
BAIL(retval);
if (!type)
BAIL(NC_EBADTYPE);
*type_class = type->nc_type_class;
}
exit:
return retval;
}
/**
* @internal Report information about an open HDF5 object. This is
* called on any still-open objects when a HDF5 file close is

18
liblib/CMakeLists.txt
View File

@ -10,11 +10,10 @@ SET(liblib_LIBS dispatch netcdf3)
# Add target objects/modules based on options.
#####
IF(USE_NETCDF4)
IF(USE_HDF5 OR USE_NETCDF4)
SET(liblib_LIBS ${liblib_LIBS} netcdf4)
ENDIF()
IF(USE_HDF5)
SET(liblib_LIBS ${liblib_LIBS} netcdfhdf5)
ENDIF()
IF(USE_PNETCDF)
@ -37,6 +36,11 @@ IF(ENABLE_NCZARR)
SET(liblib_LIBS ${liblib_LIBS} nczarr)
ENDIF()
IF(ENABLE_S3_SDK)
SET(liblib_LIBS ${liblib_LIBS} ${AWS_CPP_SDK_CORE_LIB} ${AWS_CPP_SDK_S3_LIB} ${AWS_C_COMMON_LIB})
MESSAGE(STATUS "xxx: ${AWS_CPP_SDK_CORE_LIB} ${AWS_CPP_SDK_S3_LIB} ${AWS_C_COMMON_LIB}")
ENDIF(ENABLE_S3_SDK)
FOREACH(LIBS ${liblib_LIBS})
SET(LARGS ${LARGS} $<TARGET_OBJECTS:${LIBS}>)
ENDFOREACH()
@ -72,7 +76,7 @@ IF(HAVE_LIBDL)
SET(TLL_LIBS ${LIBDL} ${TLL_LIBS})
ENDIF()
IF(USE_HDF5)
IF(USE_HDF5 OR USE_NETCDF4)
IF(NOT MSVC)
# Some version of cmake define HDF5_hdf5_LIBRARY instead of
# HDF5_LIBRARY. Same with HDF5_HL_LIBRARIES
@ -104,17 +108,11 @@ IF(ENABLE_PNETCDF AND PNETCDF)
SET(TLL_LIBS ${TLL_LIBS} ${PNETCDF})
ENDIF()
IF(ENABLE_S3_SDK)
TARGET_LINK_DIRECTORIES(netcdf PUBLIC ${AWSSDK_LIB_DIR})
TARGET_LINK_LIBRARIES(netcdf ${AWSSDK_LINK_LIBRARIES})
ENDIF()
IF(TLL_LIBS)
LIST(REMOVE_DUPLICATES TLL_LIBS)
ENDIF()
TARGET_LINK_LIBRARIES(netcdf ${TLL_LIBS})
SET(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${TLL_LIBS})
IF(MSVC)
SET_TARGET_PROPERTIES(netcdf PROPERTIES

5
liblib/Makefile.am
View File

@ -44,7 +44,7 @@ endif # USE_PNETCDF
if USE_HDF5
AM_CPPFLAGS += -I${top_srcdir}/libhdf5
libnetcdf_la_LIBADD += ${top_builddir}/libhdf5/libnchdf5.la
endif # USE_HDF5
endif # USE_NETCDF4
# + hdf4
if USE_HDF4
@ -73,9 +73,6 @@ endif #USE_NETCDF4
if ENABLE_NCZARR
AM_CPPFLAGS += -I${top_srcdir}/libnczarr
libnetcdf_la_LIBADD += ${top_builddir}/libnczarr/libnczarr.la
if ENABLE_S3_SDK
libnetcdf_la_LIBADD += ${aws_cpp_sdk_core_LIBS} ${aws_cpp_sdk_s3_LIBS}
endif
endif #ENABLE_NCZARR
if ISCYGWIN

8
libnczarr/CMakeLists.txt
View File

@ -49,6 +49,7 @@ zdebug.h
)
IF(ENABLE_S3_SDK)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
SET(libnczarr_SOURCES ${libnczarr_SOURCES} zs3sdk.cpp zmap_s3sdk.c awsincludes.h zs3sdk.h)
ENDIF()
@ -56,13 +57,6 @@ ENDIF()
# the netCDF library.
add_library(nczarr OBJECT ${libnczarr_SOURCES})
IF(ENABLE_S3_SDK)
IF(NOT MSVC)
target_compile_features(nczarr PUBLIC cxx_std_11)
ENDIF()
target_include_directories(nczarr PRIVATE ${AWSSDK_INCLUDE_DIR})
ENDIF()
# Remember to package this file for CMake builds.
ADD_EXTRA_DIST(${libnczarr_SOURCES} CMakeLists.txt)

26
libnczarr/zchunking.h
View File

@ -6,8 +6,6 @@
#ifndef ZCHUNKING_H
#define ZCHUNKING_H
#include "ncexternl.h"
/* Callback functions so we can use with unit tests */
typedef int (*NCZ_reader)(void* source, size64_t* chunkindices, void** chunkdata);
@ -75,29 +73,29 @@ struct Common {
/**************************************************/
/* From zchunking.c */
EXTERNL int NCZ_compute_chunk_ranges(int rank, const NCZSlice*, const size64_t*, NCZChunkRange* ncr);
EXTERNL int NCZ_compute_projections(size64_t dimlen, size64_t chunklen, size64_t chunkindex, const NCZSlice* slice, size_t n, NCZProjection* projections);
EXTERNL int NCZ_compute_per_slice_projections(int rank, const NCZSlice*, const NCZChunkRange*, size64_t dimlen, size64_t chunklen, NCZSliceProjections* slp);
EXTERNL int NCZ_compute_all_slice_projections(int rank, const NCZSlice* slices, const size64_t* dimlen, const size64_t* chunklen, const NCZChunkRange*, NCZSliceProjections*);
extern int NCZ_compute_chunk_ranges(int rank, const NCZSlice*, const size64_t*, NCZChunkRange* ncr);
extern int NCZ_compute_projections(size64_t dimlen, size64_t chunklen, size64_t chunkindex, const NCZSlice* slice, size_t n, NCZProjection* projections);
extern int NCZ_compute_per_slice_projections(int rank, const NCZSlice*, const NCZChunkRange*, size64_t dimlen, size64_t chunklen, NCZSliceProjections* slp);
extern int NCZ_compute_all_slice_projections(int rank, const NCZSlice* slices, const size64_t* dimlen, const size64_t* chunklen, const NCZChunkRange*, NCZSliceProjections*);
/* From zwalk.c */
EXTERNL int ncz_chunking_init(void);
EXTERNL int NCZ_transferslice(NC_VAR_INFO_T* var, int reading,
extern int ncz_chunking_init(void);
extern int NCZ_transferslice(NC_VAR_INFO_T* var, int reading,
size64_t* start, size64_t* count, size64_t* stride,
void* memory, nc_type typecode);
EXTERNL int NCZ_transfer(struct Common* common, NCZSlice* slices);
EXTERNL size64_t NCZ_computelinearoffset(size_t, const size64_t*, const size64_t*);
extern int NCZ_transfer(struct Common* common, NCZSlice* slices);
extern size64_t NCZ_computelinearoffset(size_t, const size64_t*, const size64_t*);
/* Special entry points for unit testing */
struct Common;
struct NCZOdometer;
EXTERNL int NCZ_projectslices(size64_t* dimlens,
extern int NCZ_projectslices(size64_t* dimlens,
size64_t* chunklens,
NCZSlice* slices,
struct Common*, struct NCZOdometer**);
EXTERNL int NCZ_chunkindexodom(int rank, const NCZChunkRange* ranges, size64_t*, struct NCZOdometer** odom);
EXTERNL void NCZ_clearsliceprojections(int count, NCZSliceProjections* slpv);
EXTERNL void NCZ_clearcommon(struct Common* common);
extern int NCZ_chunkindexodom(int rank, const NCZChunkRange* ranges, size64_t*, struct NCZOdometer** odom);
extern void NCZ_clearsliceprojections(int count, NCZSliceProjections* slpv);
extern void NCZ_clearcommon(struct Common* common);
#define floordiv(x,y) ((x) / (y))

36
libnczarr/zcvt.c
View File

@ -5,12 +5,11 @@
#include "zincludes.h"
#include <math.h>
#ifdef _MSC_VER
#include <crtdbg.h>
#endif
#include "isnan.h"
/*
Code taken directly from libdap4/d4cvt.c
*/
@ -47,8 +46,6 @@ NCZ_convert1(NCjson* jsrc, nc_type dsttype, char* memory)
/* Capture nan and infinity values */
if(strcasecmp(jsrc->value,"nan")==0)
zcvt.float64v = NAN;
else if(strcasecmp(jsrc->value,"-nan")==0)
zcvt.float64v = - NAN;
else if(strcasecmp(jsrc->value,"infinity")==0)
zcvt.float64v = INFINITY;
else if(strcasecmp(jsrc->value,"-infinity")==0)
@ -327,8 +324,6 @@ NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap)
} else { /* for all other values, create an array of values */
if((stat = NCJnew(NCJ_ARRAY,&jdata))) goto done;
for(i=0;i<len;i++) {
char* special = NULL;
double d;
if((stat = NCZ_stringconvert1(typeid, src, &str)))
goto done;
switch (typeid) {
@ -336,39 +331,14 @@ NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap)
case NC_UBYTE: case NC_USHORT: case NC_UINT: case NC_UINT64:
if((stat=NCJnew(NCJ_INT,&jvalue))) goto done;
break;
case NC_FLOAT:
case NC_DOUBLE: {
if(typeid == NC_FLOAT)
d = (double)(*((float*)src));
else
d = *((double*)src);
#ifdef _WIN32
switch (_fpclass(d)) {
case _FPCLASS_SNAN: case _FPCLASS_QNAN:
special = "Nan"; break;
case _FPCLASS_NINF:
special = "-Infinity"; break;
case _FPCLASS_PINF:
special = "Infinity"; break;
default: break;
}
#else
if(isnan(d))
special = "NaN";
else if(isinf(d) < 0)
special = "-Infinity";
else if(isinf(d) > 0)
special = "Infinity";
else {}
#endif
case NC_FLOAT: case NC_DOUBLE:
if((stat=NCJnew(NCJ_DOUBLE,&jvalue))) goto done;
} break;
break;
case NC_CHAR:
if((stat=NCJnew(NCJ_STRING,&jvalue))) goto done;
break;
default: stat = NC_EINTERNAL; goto done;
}
if(special) {nullfree(str); str = strdup(special);}
jvalue->value = str;
str = NULL;
nclistpush(jdata->contents,jvalue);

42
libnczarr/zdebug.h
View File

@ -14,20 +14,18 @@
#undef ZCATCH /* Warning: significant performance impact */
#undef ZTRACING /* Warning: significant performance impact */
#include "ncexternl.h"
#include "nclog.h"
#ifdef ZCATCH
/* Place breakpoint on zbreakpoint to catch errors close to where they occur*/
#define THROW(e) zthrow((e),__FILE__,__LINE__)
#define ZCHECK(e) if((e)) {THROW(stat); goto done;} else {}
EXTERNL int zbreakpoint(int err);
EXTERNL int zthrow(int err, const char* fname, int line);
extern int zbreakpoint(int err);
extern int zthrow(int err, const char* fname, int line);
#else
#define ZCHECK(e) {if((e)) {goto done;}}
#define THROW(e) (e)
#endif
#include "nclog.h"
#ifdef ZTRACING
#define ZLOG(level,msg,...) nclog(level,msg,__VA_ARGS__)
#define ZTRACE(fmt,...) nclog(NCLOGDBG,"trace: %s:%s " fmt,__FILE__,__func__,__VA_ARGS__)
@ -37,27 +35,27 @@ EXTERNL int zthrow(int err, const char* fname, int line);
#endif
/* printers */
EXTERNL void nczprint_reclaim(void);
EXTERNL char* nczprint_slice(NCZSlice);
EXTERNL char* nczprint_slices(int rank, NCZSlice*);
EXTERNL char* nczprint_slab(int rank, NCZSlice*);
EXTERNL char* nczprint_odom(NCZOdometer*);
EXTERNL char* nczprint_chunkrange(NCZChunkRange);
EXTERNL char* nczprint_projection(NCZProjection);
EXTERNL char* nczprint_sliceprojections(NCZSliceProjections);
EXTERNL char* nczprint_allsliceprojections(int r, NCZSliceProjections* slp);
EXTERNL char* nczprint_vector(size_t,size64_t*);
EXTERNL char* nczprint_slicex(NCZSlice slice, int raw);
EXTERNL char* nczprint_slicesx(int rank, NCZSlice* slices, int raw);
EXTERNL char* nczprint_projectionx(NCZProjection proj, int raw);
EXTERNL char* nczprint_sliceprojectionsx(NCZSliceProjections slp, int raw);
extern void nczprint_reclaim(void);
extern char* nczprint_slice(NCZSlice);
extern char* nczprint_slices(int rank, NCZSlice*);
extern char* nczprint_slab(int rank, NCZSlice*);
extern char* nczprint_odom(NCZOdometer*);
extern char* nczprint_chunkrange(NCZChunkRange);
extern char* nczprint_projection(NCZProjection);
extern char* nczprint_sliceprojections(NCZSliceProjections);
extern char* nczprint_allsliceprojections(int r, NCZSliceProjections* slp);
extern char* nczprint_vector(size_t,size64_t*);
extern char* nczprint_slicex(NCZSlice slice, int raw);
extern char* nczprint_slicesx(int rank, NCZSlice* slices, int raw);
extern char* nczprint_projectionx(NCZProjection proj, int raw);
extern char* nczprint_sliceprojectionsx(NCZSliceProjections slp, int raw);
#ifdef ZDEBUG
EXTERNL void zdumpcommon(struct Common*);
extern void zdumpcommon(struct Common*);
#endif
#ifdef HAVE_EXECINFO_H
EXTERNL void NCZbacktrace(void);
extern void NCZbacktrace(void);
#endif
/* Define the possible unit tests (powers of 2) */
@ -66,7 +64,7 @@ EXTERNL void NCZbacktrace(void);
#define UTEST_TRANSFER 4
#ifdef ZUT
EXTERNL struct ZUTEST {
extern struct ZUTEST {
int tests;
void (*print)(int sort,...);
} zutest;

1
libnczarr/zdim.c
View File

@ -271,3 +271,4 @@ NCZ_inq_unlimdims(int ncid, int *ndimsp, int *unlimdimidsp)
if(ndimsp) *ndimsp = 0;
return NC_NOERR;
}

2
libnczarr/zdispatch.c
View File

@ -38,7 +38,7 @@ static const NC_Dispatch NCZ_dispatcher = {
NCZ_def_dim,
NC4_inq_dimid,
NCZ_inq_dim,
NC4_inq_dim,
NC4_inq_unlimdim,
NCZ_rename_dim,

2
libnczarr/zfilter.c
View File

@ -21,7 +21,7 @@
/* WARNING: GLOBAL VARIABLE */
#ifdef ENABLE_CLIENTSIDE_FILTERS
#ifdef ENABLE_CLIENT_FILTERS
/* Define list of registered filters */
static NClist* NCZ_registeredfilters = NULL; /** List<NC_FILTER_CLIENT_ZARR*> */

4
libnczarr/zincludes.h
View File

@ -20,15 +20,13 @@
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <malloc.h>
#include <stddef.h> /* size_t, ptrdiff_t */
#include <assert.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#endif
#ifdef _MSC_VER
#include <malloc.h>
#endif
#ifdef __cplusplus
extern "C" {

2
libnczarr/zinternal.h
View File

@ -204,7 +204,7 @@ int ncz_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
/* zfilter.c */
int NCZ_filter_actions(int ncid, int varid, int op, void* args);
#ifdef ENABLE_CLIENTSIDE_FILTERS
#ifdef ENABLE_CLIENT_FILTERS
int NCZ_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop);
#endif

45
libnczarr/zjson.c
View File

@ -5,8 +5,6 @@
#include "zincludes.h"
#include "zjson.h"
#undef DEBUG
#define NCJ_LBRACKET '['
#define NCJ_RBRACKET ']'
#define NCJ_LBRACE '{'
@ -57,9 +55,6 @@ static int NCJappendquoted(const char* value, NCbytes* buf);
static int NCJcloneArray(NClist* array, NCjson** clonep);
static int NCJcloneDict(NClist* dict, NCjson** clonep);
#ifdef DEBUG
static char* tokenname(int token);
#endif
/**************************************************/
int
NCJparse(const char* text, unsigned flags, NCjson** jsonp)
@ -69,8 +64,7 @@ NCJparse(const char* text, unsigned flags, NCjson** jsonp)
NCJparser* parser = NULL;
NCjson* json = NULL;
/* Need at least 1 character of input */
if(text == NULL || text[0] == '\0')
if(text == NULL)
{stat = NC_EINVAL; goto done;}
if(jsonp == NULL) goto done;
parser = calloc(1,sizeof(NCJparser));
@ -85,9 +79,6 @@ NCJparse(const char* text, unsigned flags, NCjson** jsonp)
parser->text[len] = '\0';
parser->text[len+1] = '\0';
parser->pos = &parser->text[0];
#ifdef DEBUG
fprintf(stderr,"json: |%s|\n",parser->text);
#endif
if((stat=NCJparseR(parser,&json))) goto done;
*jsonp = json;
json = NULL;
@ -334,9 +325,6 @@ NCJlex(NCJparser* parser)
if(NCJyytext(parser,parser->pos,1)) goto done;
token = *parser->pos++;
}
#ifdef DEBUG
fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,parser->yytext);
#endif
} /*for(;;)*/
done:
if(parser->err) token = NCJ_ERR;
@ -381,10 +369,6 @@ testdouble(const char* word)
if(strcasecmp("nan",word)==0) return NC_NOERR;
if(strcasecmp("infinity",word)==0) return NC_NOERR;
if(strcasecmp("-infinity",word)==0) return NC_NOERR;
/* Allow the XXXf versions as well */
if(strcasecmp("nanf",word)==0) return NC_NOERR;
if(strcasecmp("infinityf",word)==0) return NC_NOERR;
if(strcasecmp("-infinityf",word)==0) return NC_NOERR;
/* Try to convert to number */
ncvt = sscanf(word,"%lg%n",&d,&count);
return (ncvt == 1 && strlen(word)==count ? NC_NOERR : NC_EINVAL);
@ -815,30 +799,3 @@ NCJdump(const NCjson* json, int flags)
fprintf(stderr,"%s\n",text);
fflush(stderr);
}
#ifdef DEBUG
static char*
tokenname(int token)
{
switch (token) {
case NCJ_STRING: return "NCJ_STRING";
case NCJ_INT: return "NCJ_INT";
case NCJ_DOUBLE: return "NCJ_DOUBLE";
case NCJ_BOOLEAN: return "NCJ_BOOLEAN";
case NCJ_DICT: return "NCJ_DICT";
case NCJ_ARRAY: return "NCJ_ARRAY";
case NCJ_NULL: return "NCJ_NULL";
default:
if(token > ' ' && token <= 127) {
static char s[4];
s[0] = '\'';
s[1] = (char)token;
s[2] = '\'';
s[3] = '\0';
return s;
} else
break;
}
return "NCJ_UNDEF";
}
#endif

30
libnczarr/zjson.h
View File

@ -5,8 +5,6 @@
#ifndef NCJSON_H
#define NCJSON_H 1
#include "ncexternl.h"
/* Json object sorts */
#define NCJ_UNDEF 0
#define NCJ_STRING 1
@ -39,46 +37,46 @@ typedef struct NCjson {
#define NCJF_MULTILINE 1
/* Parse */
EXTERNL int NCJparse(const char* text, unsigned flags, NCjson** jsonp);
extern int NCJparse(const char* text, unsigned flags, NCjson** jsonp);
/* Build */
EXTERNL int NCJnew(int sort, NCjson** object);
extern int NCJnew(int sort, NCjson** object);
/* Convert a nul terminated string value to an NCjson object */
EXTERNL int NCJnewstring(int sort, const char* value, NCjson** jsonp);
extern int NCJnewstring(int sort, const char* value, NCjson** jsonp);
/* Convert a counted string value to an NCjson object (+ nul term)*/
EXTERNL int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);
extern int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);
/* Insert key-value pair into a dict object.
key will be strdup'd.
*/
EXTERNL int NCJinsert(NCjson* object, char* key, NCjson* value);
extern int NCJinsert(NCjson* object, char* key, NCjson* value);
/* Insert a string value into a json Dict|Array */
EXTERNL int NCJaddstring(NCjson* dictarray, int sort, const char* value);
extern int NCJaddstring(NCjson* dictarray, int sort, const char* value);
/* Get ith pair from dict */
EXTERNL int NCJdictith(NCjson* object, size_t i, const char** keyp, NCjson** valuep);
extern int NCJdictith(NCjson* object, size_t i, const char** keyp, NCjson** valuep);
/* Get value for key from dict */
EXTERNL int NCJdictget(NCjson* object, const char* key, NCjson** valuep);
extern int NCJdictget(NCjson* object, const char* key, NCjson** valuep);
/* Append value to an array or dict object. */
EXTERNL int NCJappend(NCjson* object, NCjson* value);
extern int NCJappend(NCjson* object, NCjson* value);
/* Get ith element from array */
EXTERNL int NCJarrayith(NCjson* object, size_t i, NCjson** valuep);
extern int NCJarrayith(NCjson* object, size_t i, NCjson** valuep);
/* Unparser to convert NCjson object to text in buffer */
EXTERNL int NCJunparse(const NCjson* json, int flags, char** textp);
extern int NCJunparse(const NCjson* json, int flags, char** textp);
/* Utilities */
EXTERNL void NCJreclaim(NCjson*);
EXTERNL int NCJclone(NCjson* json, NCjson** clonep); /* deep clone */
extern void NCJreclaim(NCjson*);
extern int NCJclone(NCjson* json, NCjson** clonep); /* deep clone */
/* dump NCjson* object */
EXTERNL void NCJdump(const NCjson* json, int flags);
extern void NCJdump(const NCjson* json, int flags);
/* Macro defined functions */
#define NCJlength(json) \

66
libnczarr/zmap.c
View File

@ -4,7 +4,6 @@
*/
#include "zincludes.h"
#include "ncwinpath.h"
/**************************************************/
/* Import the current implementations */
@ -199,8 +198,7 @@ nczm_join(NClist* segments, char** pathp)
ncbytescat(buf,"/");
else for(i=0;i<nclistlength(segments);i++) {
const char* seg = nclistget(segments,i);
if(seg[0] != '/')
ncbytescat(buf,"/");
ncbytescat(buf,"/");
ncbytescat(buf,seg);
}
@ -294,6 +292,8 @@ nczm_clear(NCZMAP* map)
return NC_NOERR;
}
static const char* driveletter = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
int
nczm_isabsolutepath(const char* path)
{
@ -304,65 +304,9 @@ nczm_isabsolutepath(const char* path)
case '\0': break;
default:
/* Check for windows drive letter */
if(NChasdriveletter(path)) return 1;
if(strchr(driveletter,path[0]) != NULL && path[1] == ':')
return 1; /* windows path with drive letter */
break;
}
return 0;
}
/* Convert forward slash to backslash ( !localize) or vice-versa (localize)*/
int
nczm_localize(const char* path, char** localpathp, int localize)
{
int stat = NC_NOERR;
char* localpath = NULL;
char* p;
int forward = 1;
int offset = 0;
#ifdef _MSC_VER
forward = (localize?0:1);
#endif
/* If path comes from a url, then it may start with: /x:/...
where x is a drive letter. If so, then remove leading / */
if(path[0] == '/' && NChasdriveletter(path+1))
offset = 1;
if((localpath = strdup(path+offset))==NULL) return NC_ENOMEM;
for(p=localpath;*p;p++) {
if(forward && *p == '\\') *p = '/';
else if(!forward && *p == '/') *p = '\\';
}
if(localpathp) {*localpathp = localpath; localpath = NULL;}
nullfree(localpath);
return stat;
}
/* Convert path0 to be:
1. absolute -- including drive letters
2. forward slashed -- we will convert back to back slash in
nczm_fixpath
*/
int
nczm_canonicalpath(const char* path, char** cpathp)
{
int ret = NC_NOERR;
char* cpath = NULL;
char* tmp = NULL;
if(path == NULL)
{cpath = NULL; goto done;}
/* Process path to make it be windows compatible */
if((tmp = NCpathcvt(path))==NULL) {ret = NC_ENOMEM; goto done;}
/* Fix slashes to be forward for now */
if((ret = nczm_localize(tmp,&cpath,!LOCALIZE))) goto done;
if(cpathp) {*cpathp = cpath; cpath = NULL;}
done:
nullfree(tmp);
nullfree(cpath);
return THROW(ret);
}

46
libnczarr/zmap.h
View File

@ -100,15 +100,10 @@ generic nczm_xxx functions below.
#ifndef ZMAP_H
#define ZMAP_H
#include "ncexternl.h"
#define NCZM_SEP "/"
#define NCZM_DOT '.'
/*Mnemonic*/
#define LOCALIZE 1
/* Forward */
typedef struct NCZMAP_API NCZMAP_API;
@ -184,7 +179,7 @@ Check if a specified content-bearing object exists or not.
@return NC_ENOTFOUND if the object does not exist
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_exists(NCZMAP* map, const char* key);
extern int nczmap_exists(NCZMAP* map, const char* key);
/**
Return the current size of a specified content-bearing object exists or not.
@ -195,7 +190,7 @@ Return the current size of a specified content-bearing object exists or not.
@return NC_ENOTFOUND if the object does not exist
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_len(NCZMAP* map, const char* key, size64_t* sizep);
extern int nczmap_len(NCZMAP* map, const char* key, size64_t* sizep);
/**
Create a specified content-bearing object.
@ -205,7 +200,7 @@ Create a specified content-bearing object.
@return NC_ENOTFOUND if the object does not exist
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_defineobj(NCZMAP* map, const char* key);
extern int nczmap_defineobj(NCZMAP* map, const char* key);
/**
Read the content of a specified content-bearing object.
@ -218,7 +213,7 @@ Read the content of a specified content-bearing object.
@return NC_ENOTFOUND if the object does not exist
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_read(NCZMAP* map, const char* key, size64_t start, size64_t count, void* content);
extern int nczmap_read(NCZMAP* map, const char* key, size64_t start, size64_t count, void* content);
/**
Write the content of a specified content-bearing object.
@ -231,7 +226,7 @@ Write the content of a specified content-bearing object.
@return NC_ENOTFOUND if the object does not exist
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_write(NCZMAP* map, const char* key, size64_t start, size64_t count, const void* content);
extern int nczmap_write(NCZMAP* map, const char* key, size64_t start, size64_t count, const void* content);
/**
Return a vector of keys representing the content-bearing
@ -242,7 +237,7 @@ objects that are immediately contained by the prefix key.
@return NC_NOERR if the operation succeeded
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_search(NCZMAP* map, const char* prefix, NClist* matches);
extern int nczmap_search(NCZMAP* map, const char* prefix, NClist* matches);
/**
Close a map
@ -252,45 +247,36 @@ Close a map
@return NC_ENOTFOUND if the object does not exist
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_close(NCZMAP* map, int deleteit);
extern int nczmap_close(NCZMAP* map, int deleteit);
/* Create/open and control a dataset using a specific implementation */
EXTERNL int nczmap_create(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp);
EXTERNL int nczmap_open(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp);
extern int nczmap_create(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp);
extern int nczmap_open(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp);
/* Utility functions */
/** Split a path into pieces along '/' character; elide any leading '/' */
EXTERNL int nczm_split(const char* path, NClist* segments);
extern int nczm_split(const char* path, NClist* segments);
/* Split a path into pieces along some character; elide any leading char */
EXTERNL int nczm_split_delim(const char* path, char delim, NClist* segments);
extern int nczm_split_delim(const char* path, char delim, NClist* segments);
/* Convenience: Join all segments into a path using '/' character */
EXTERNL int nczm_join(NClist* segments, char** pathp);
/* Convenience: Join all segments into a path using '/' character
but taking possible lead windows drive letter into account
*/
EXTERNL int nczm_joinpath(NClist* segments, char** pathp);
extern int nczm_join(NClist* segments, char** pathp);
/* Convenience: concat two strings; caller frees */
EXTERNL int nczm_concat(const char* prefix, const char* suffix, char** pathp);
extern int nczm_concat(const char* prefix, const char* suffix, char** pathp);
/* Break a key into prefix and suffix, where prefix is the first nsegs segments;
nsegs can be negative to specify that suffix is |nsegs| long
*/
EXTERNL int nczm_divide_at(const char* key, int nsegs, char** prefixp, char** suffixp);
extern int nczm_divide_at(const char* key, int nsegs, char** prefixp, char** suffixp);
/* Reclaim the content of a map but not the map itself */
EXTERNL int nczm_clear(NCZMAP* map);
extern int nczm_clear(NCZMAP* map);
/* Return 1 if path is absolute; takes Windows drive letters into account */
EXTERNL int nczm_isabsolutepath(const char* path);
/* Convert forward to back slash if needed */
EXTERNL int nczm_localize(const char* path, char** newpathp, int local);
EXTERNL int nczm_canonicalpath(const char* path, char** cpathp);
extern int nczm_isabsolutepath(const char* path);
#ifdef __cplusplus
}

72
libnczarr/zmap_nz4.c
View File

@ -35,7 +35,7 @@ For the object API, the mapping is as follows:
/* Define the "subclass" of NCZMAP */
typedef struct Z4MAP {
NCZMAP map;
char* root;
char* path;
int ncid;
} Z4MAP;
@ -51,6 +51,7 @@ static int zcreatedim(Z4MAP*, int, int* dimidp);
static int parseurl(const char* path0, NCURI** urip);
static void nc4ify(const char* zname, char* nc4name);
static void zify(const char* nc4name, char* zname);
static int z4getcwd(char** cwdp);
static int testcontentbearing(int grpid);
/* Define the Dataset level API */
@ -60,11 +61,14 @@ znc4create(const char *path, int mode, size64_t flags, void* parameters, NCZMAP*
{
int stat = NC_NOERR;
char* truepath = NULL;
char* local = NULL; /* localized truepath */
Z4MAP* z4map = NULL;
int ncid;
NCURI* url = NULL;
char* z4cwd = NULL;
if((stat=parseurl(path,&url)))
goto done;
/* Fix up mode */
mode = (NC_NETCDF4 | NC_WRITE | mode);
if(flags & FLAG_BYTERANGE)
@ -73,14 +77,19 @@ znc4create(const char *path, int mode, size64_t flags, void* parameters, NCZMAP*
if(!(mode & NC_WRITE))
{stat = NC_EPERM; goto done;}
if((stat=parseurl(path,&url)))
goto done;
/* Canonicalize the root path */
if((stat = nczm_canonicalpath(url->path,&truepath))) goto done;
/* Also get local path */
if((stat = nczm_localize(truepath,&local,LOCALIZE))) goto done;
/* Get cwd so we can use absolute paths */
if((stat = z4getcwd(&z4cwd))) goto done;
if(flags & FLAG_BYTERANGE)
truepath = ncuribuild(url,NULL,NULL,NCURIALL);
else {
/* Make the root path be absolute */
if(!nczm_isabsolutepath(url->path)) {
if((stat = nczm_concat(z4cwd,url->path,&truepath))) goto done;
} else
truepath = strdup(url->path);
}
/* Build the z4 state */
if((z4map = calloc(1,sizeof(Z4MAP))) == NULL)
{stat = NC_ENOMEM; goto done;}
@ -90,10 +99,10 @@ znc4create(const char *path, int mode, size64_t flags, void* parameters, NCZMAP*
z4map->map.mode = mode;
z4map->map.flags = flags;
z4map->map.api = &zapi;
z4map->root= truepath;
z4map->path = truepath;
truepath = NULL;
if((stat=nc_create(local,mode,&ncid)))
if((stat=nc_create(z4map->path,mode,&ncid)))
{stat = NC_EEMPTY; goto done;} /* could not open */
z4map->ncid = ncid;
@ -102,7 +111,7 @@ znc4create(const char *path, int mode, size64_t flags, void* parameters, NCZMAP*
done:
ncurifree(url);
nullfree(truepath);
nullfree(local);
nullfree(z4cwd);
if(stat) znc4close((NCZMAP*)z4map,1);
return (stat);
}
@ -112,10 +121,10 @@ znc4open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
{
int stat = NC_NOERR;
char* truepath = NULL;
char* local = NULL;
Z4MAP* z4map = NULL;
int ncid;
NCURI* url = NULL;
char* z4cwd = NULL;
/* Fixup mode */
mode = NC_NETCDF4 | mode;
@ -125,10 +134,18 @@ znc4open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
if((stat=parseurl(path,&url)))
goto done;
/* Canonicalize the root path */
if((stat = nczm_canonicalpath(url->path,&truepath))) goto done;
/* Also get local path */
if((stat = nczm_localize(truepath,&local,LOCALIZE))) goto done;
/* Get cwd so we can use absolute paths */
if((stat = z4getcwd(&z4cwd))) goto done;
if(flags & FLAG_BYTERANGE)
truepath = ncuribuild(url,NULL,NULL,NCURIALL);
else {
/* Make the root path be absolute */
if(!nczm_isabsolutepath(url->path)) {
if((stat = nczm_concat(z4cwd,url->path,&truepath))) goto done;
} else
truepath = strdup(url->path);
}
/* Build the z4 state */
if((z4map = calloc(1,sizeof(Z4MAP))) == NULL)
@ -139,10 +156,10 @@ znc4open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
z4map->map.mode = mode;
z4map->map.flags = flags;
z4map->map.api = (NCZMAP_API*)&zapi;
z4map->root = truepath;
z4map->path = truepath;
truepath = NULL;
if((stat=nc_open(local,mode,&ncid)))
if((stat=nc_open(z4map->path,mode,&ncid)))
{stat = NC_EEMPTY; goto done;} /* could not open */
z4map->ncid = ncid;
@ -150,7 +167,7 @@ znc4open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
done:
nullfree(truepath);
nullfree(local);
nullfree(z4cwd);
ncurifree(url);
if(stat) znc4close((NCZMAP*)z4map,0);
return (stat);
@ -168,7 +185,7 @@ znc4close(NCZMAP* map, int delete)
if(map == NULL) return NC_NOERR;
path = z4map->root;
path = z4map->path;
if((stat = nc_close(z4map->ncid)))
goto done;
@ -178,7 +195,7 @@ znc4close(NCZMAP* map, int delete)
}
done:
nullfree(z4map->root);
nullfree(z4map->path);
nczm_clear(map);
free(z4map);
return (stat);
@ -591,6 +608,17 @@ nc4ify(const char* zname, char* nc4name)
if(nc4name[0] == NCZM_DOT) nc4name[0] = ZDOTNC4;
}
static int
z4getcwd(char** cwdp)
{
char buf[4096];
char* cwd = NULL;
cwd = NCcwd(buf,sizeof(buf));
if(cwd == NULL) return errno;
if(cwdp) *cwdp = strdup(buf);
return NC_NOERR;
}
/**************************************************/
/* External API objects */

421
libnczarr/zmap_nzf.c
View File

@ -10,6 +10,15 @@
#include "zincludes.h"
#include <errno.h>
#if 0
#ifdef _WIN32
#ifndef __cplusplus
#include <windows.h>
#include <io.h>
#include <iostream>
#endif
#endif
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
@ -27,20 +36,6 @@
#include <dirent.h>
#endif
#ifdef _WIN32
#include <windows.h>
#ifndef S_ISDIR
#define S_ISDIR(mode) ((mode) & _S_IFDIR)
#define S_ISREG(mode) ((mode) & _S_IFREG)
#endif
#if 0
#ifndef __cplusplus
#include <io.h>
#include <iostream>
#endif
#endif
#endif
#include "fbits.h"
#include "ncwinpath.h"
@ -113,6 +108,7 @@ static FD FDNUL = {-1};
typedef struct ZFMAP {
NCZMAP map;
char* root;
char* cwd;
} ZFMAP;
/* Forward */
@ -131,13 +127,13 @@ static int platformcreatedir(ZFMAP* map, const char* truepath);
static int platformopenfile(ZFMAP* zfmap, const char* truepath, FD* fd);
static int platformopendir(ZFMAP* map, const char* truepath);
static int platformdircontent(ZFMAP* map, const char* path, NClist* contents);
static int platformdelete(ZFMAP* map, const char* path, int delroot);
static int platformdelete(ZFMAP* map, const char* path);
static int platformseek(ZFMAP* map, FD* fd, int pos, size64_t* offset);
static int platformread(ZFMAP* map, FD* fd, size64_t count, void* content);
static int platformwrite(ZFMAP* map, FD* fd, size64_t count, const void* content);
static int platformcwd(char** cwdp);
static void platformrelease(ZFMAP* zfmap, FD* fd);
static int platformtestcontentbearing(ZFMAP* zfmap, const char* truepath);
#ifdef VERIFY
static int verify(const char* path, int isdir);
#endif
@ -176,7 +172,8 @@ static int
zfilecreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp)
{
int stat = NC_NOERR;
char* truepath = NULL;
char* truepath = NULL; /* Might be a URL */
char* zfcwd = NULL;
ZFMAP* zfmap = NULL;
NCURI* url = NULL;
@ -199,8 +196,15 @@ zfilecreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP
if(strcasecmp(url->protocol,"file") != 0)
{stat = NC_EURL; goto done;}
/* Canonicalize the root path */
if((stat = nczm_canonicalpath(url->path,&truepath))) goto done;
/* Get cwd so we can use absolute paths
in case user gives us a relative path*/
if((stat = platformcwd(&zfcwd))) goto done;
/* Make the root path be absolute */
if(!nczm_isabsolutepath(url->path)) {
if((stat = nczm_concat(zfcwd,url->path,&truepath))) goto done;
} else
truepath = strdup(url->path);
#ifdef CHECKNESTEDDATASETS
if(isnesteddataset(truepath))
@ -219,10 +223,11 @@ zfilecreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP
zfmap->map.api = &zapi;
zfmap->root = truepath;
truepath = NULL;
zfmap->cwd = zfcwd; zfcwd = NULL;
/* If NC_CLOBBER, then delete below file tree */
/* If NC_CLOBBER, then delete file tree */
if(!fIsSet(mode,NC_NOCLOBBER))
platformdelete(zfmap,zfmap->root,0);
platformdelete(zfmap,zfmap->root);
/* make sure we can access the root directory; create if necessary */
if((stat = platformcreatedir(zfmap, zfmap->root)))
@ -235,6 +240,7 @@ zfilecreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP
done:
ncurifree(url);
nullfree(truepath);
nullfree(zfcwd);
if(stat)
zfileclose((NCZMAP*)zfmap,1);
return (stat);
@ -254,6 +260,7 @@ zfileopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
{
int stat = NC_NOERR;
char* truepath = NULL;
char* zfcwd = NULL;
ZFMAP* zfmap = NULL;
NCURI*url = NULL;
@ -267,14 +274,20 @@ zfileopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
if(flags & FLAG_BYTERANGE)
mode &= ~(NC_CLOBBER | NC_WRITE);
/* Get cwd so we can use absolute paths */
if((stat = platformcwd(&zfcwd))) goto done;
/* path must be a url with file: protocol*/
if((stat=zfparseurl(path,&url)))
goto done;
if(strcasecmp(url->protocol,"file") != 0)
{stat = NC_EURL; goto done;}
/* Canonicalize the root path */
if((stat = nczm_canonicalpath(url->path,&truepath))) goto done;
/* Make the root path be absolute */
if(!nczm_isabsolutepath(url->path)) {
if((stat = nczm_concat(zfcwd,url->path,&truepath))) goto done;
} else
truepath = strdup(url->path);
/* Build the z4 state */
if((zfmap = calloc(1,sizeof(ZFMAP))) == NULL)
@ -287,6 +300,7 @@ zfileopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
zfmap->map.api = (NCZMAP_API*)&zapi;
zfmap->root = truepath;
truepath = NULL;
zfmap->cwd = zfcwd; zfcwd = NULL;
/* Verify root dir exists */
switch (stat = platformopendir(zfmap,zfmap->root)) {
@ -302,6 +316,7 @@ zfileopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
done:
ncurifree(url);
nullfree(zfcwd);
nullfree(truepath);
if(stat) zfileclose((NCZMAP*)zfmap,0);
return (stat);
@ -457,14 +472,15 @@ zfileclose(NCZMAP* map, int delete)
ZTRACE("%d",delete);
if(zfmap == NULL) return NC_NOERR;
/* Delete the subtree below the root and the root */
/* Delete the subtree below the root */
if(delete) {
stat = platformdelete(zfmap,zfmap->root,1);
stat = platformdelete(zfmap,zfmap->root);
unlink(zfmap->root);
}
nczm_clear(map);
nullfree(zfmap->root);
zfmap->root = NULL;
nullfree(zfmap->cwd);
free(zfmap);
return (stat);
}
@ -482,7 +498,7 @@ zfilesearch(NCZMAP* map, const char* prefixkey, NClist* matches)
int i;
ZFMAP* zfmap = (ZFMAP*)map;
char* truepath = NULL;
NClist* nextlevel = nclistnew();
NClist* segments = nclistnew();
NCbytes* buf = ncbytesnew();
int trailing;
@ -494,19 +510,11 @@ zfilesearch(NCZMAP* map, const char* prefixkey, NClist* matches)
else if((stat = nczm_concat(zfmap->root,prefixkey,&truepath))) goto done;
trailing = (prefixkey[strlen(prefixkey)-1] == '/');
/* get names of the next level path entries */
switch (stat = platformdircontent(zfmap, truepath, nextlevel)) {
case NC_NOERR: /* ok */
break;
case NC_EEMPTY: /* not a dir */
stat = NC_NOERR;
/* get names of the next level path segments */
if((stat = platformdircontent(zfmap, truepath, segments)))
goto done;
case NC_ENOTFOUND: /* does not exist */
default:
goto done;
}
for(i=0;i<nclistlength(nextlevel);i++) {
const char* segment = nclistget(nextlevel,i);
for(i=0;i<nclistlength(segments);i++) {
const char* segment = nclistget(segments,i);
ncbytescat(buf,prefixkey);
if(!trailing) ncbytescat(buf,"/");
ncbytescat(buf,segment);
@ -514,8 +522,8 @@ zfilesearch(NCZMAP* map, const char* prefixkey, NClist* matches)
}
done:
nclistfreeall(nextlevel);
ncbytesfree(buf);
nclistfreeall(segments);
nullfree(truepath);
return THROW(stat);
}
@ -696,8 +704,8 @@ platformerr(int err)
}
/* Test type of the specified file.
@return NC_NOERR if found and is a content-bearing object (file)
@return NC_EEMPTY if exists but is not-content-bearing (a directory)
@return NC_NOERR if found and is a content-bearing object
@return NC_EEMPTY if exists but is not-content-bearing
@return NC_ENOTFOUND if not found
*/
static int
@ -705,20 +713,14 @@ platformtestcontentbearing(ZFMAP* zfmap, const char* truepath)
{
int ret = 0;
struct stat buf;
char* local = NULL;
/* Localize */
if((ret = nczm_localize(truepath,&local,LOCALIZE))) goto done;
errno = 0;
if((ret = stat(local, &buf)) < 0) {
if((ret = stat(truepath, &buf)) < 0) {
ret = platformerr(errno);
} else if(S_ISDIR(buf.st_mode)) {
ret = NC_EEMPTY;
} else
ret = NC_NOERR;
done:
nullfree(local);
errno = 0;
return ret;
}
@ -743,21 +745,24 @@ platformcreatefile(ZFMAP* zfmap, const char* truepath, FD* fd)
#ifdef O_BINARY
fSet(ioflags, O_BINARY);
#endif
if(fIsSet(mode, NC_NOCLOBBER))
fSet(createflags, O_EXCL);
else
fSet(createflags, O_TRUNC);
if(fIsSet(mode,NC_WRITE))
createflags = (ioflags|O_CREAT);
/* Try to create file (will also localize) */
/* Try to open file as if it exists */
fd->fd = NCopen3(truepath, createflags, permissions);
if(fd->fd < 0) { /* could not create */
stat = platformerr(errno);
goto done; /* could not open */
if(fd->fd < 0) {
if(errno == ENOENT) {
if(fIsSet(mode,NC_WRITE)) {
/* Try to create it */
createflags = (ioflags|O_CREAT);
fd->fd = NCopen3(truepath, createflags, permissions);
if(fd->fd < 0) goto done; /* could not create */
}
}
}
if(fd->fd < 0)
{stat = platformerr(errno); goto done;} /* could not open */
done:
errno = 0;
return THROW(stat);
@ -789,7 +794,7 @@ platformopenfile(ZFMAP* zfmap, const char* truepath, FD* fd)
assert(!"expected file, have dir");
#endif
/* Try to open file (will localize) */
/* Try to open file */
fd->fd = NCopen3(truepath, ioflags, permissions);
if(fd->fd < 0)
{stat = platformerr(errno); goto done;} /* could not open */
@ -807,7 +812,7 @@ platformcreatedir(ZFMAP* zfmap, const char* truepath)
errno = 0;
/* Try to access file as if it exists */
ret = NCaccess(truepath,ACCESS_MODE_EXISTS);
ret = NCaccess(truepath,F_OK);
if(ret < 0) { /* it does not exist, then it can be anything */
if(fIsSet(mode,NC_WRITE)) {
/* Try to create it */
@ -815,7 +820,7 @@ platformcreatedir(ZFMAP* zfmap, const char* truepath)
if(NCmkdir(truepath,NC_DEFAULT_DIR_PERMS) < 0)
{ret = platformerr(errno); goto done;}
/* try to access again */
ret = NCaccess(truepath,ACCESS_MODE_EXISTS);
ret = NCaccess(truepath,F_OK);
if(ret < 0)
{ret = platformerr(errno); goto done;}
} else
@ -827,6 +832,7 @@ done:
return THROW(ret);
}
/* Open a dir; fail if it does not exist */
static int
platformopendir(ZFMAP* zfmap, const char* truepath)
@ -835,7 +841,7 @@ platformopendir(ZFMAP* zfmap, const char* truepath)
errno = 0;
/* Try to access file as if it exists */
ret = NCaccess(truepath,ACCESS_MODE_EXISTS);
ret = NCaccess(truepath,F_OK);
if(ret < 0)
{ret = platformerr(errno); goto done;}
done:
@ -843,79 +849,76 @@ done:
return THROW(ret);
}
/**
Given a path, return the list of all files+dirs immediately below
the specified path: e.g. X s.t. path/X exists.
There are several possibilities:
1. path does not exist => return NC_ENOTFOUND
2. path is not a directory => return NC_EEMPTY and |contents| == 0
3. path is a directory => return NC_NOERR and |contents| >= 0
@return NC_NOERR if path is a directory
@return NC_EEMPTY if path is not a directory
@return NC_ENOTFOUND if path does not exist
*/
#ifdef _WIN32
static int
platformdircontent(ZFMAP* zfmap, const char* truepath, NClist* contents)
platformdircontent(ZFMAP* zfmap, const char* path, NClist* contents)
{
int ret = NC_NOERR;
int stat = NC_NOERR;
errno = 0;
WIN32_FIND_DATA FindFileData;
HANDLE dir;
char* ffpath = NULL;
char* lpath = NULL;
size_t len;
char* d = NULL;
switch (ret = platformtestcontentbearing(zfmap, truepath)) {
case NC_EEMPTY: ret = NC_NOERR; break; /* directory */
case NC_NOERR: ret = NC_EEMPTY; goto done;
default: goto done;
}
/* We need to process the path to make it work with FindFirstFile */
len = strlen(truepath);
/* Need to terminate path with '/''*' */
ffpath = (char*)malloc(len+2+1);
memcpy(ffpath,truepath,len);
if(truepath[len-1] != '/') {
ffpath[len] = '/';
len++;
}
ffpath[len] = '*'; len++;
ffpath[len] = '\0';
/* localize it */
if((ret = nczm_localize(ffpath,&lpath,LOCALIZE))) goto done;
dir = FindFirstFile(lpath, &FindFileData);
if(dir == INVALID_HANDLE_VALUE) {
/* Distinquish not-a-directory from no-matching-file */
switch (GetLastError()) {
case ERROR_FILE_NOT_FOUND: /* No matching files */ /* fall thru */
ret = NC_NOERR;
goto done;
case ERROR_DIRECTORY: /* not a directory */
default:
ret = NC_EEMPTY;
goto done;
}
}
dir = FindFirstFile(truepath, &FindFileData);
if(dir == INVALID_HANDLE_VALUE)
{stat = NC_ENOTFOUND; goto done;}
do {
char* p = NULL;
const char* name = NULL;
name = FindFileData.cFileName;
if(strcmp(name,".")==0 || strcmp(name,"..")==0)
continue;
nclistpush(contents,strdup(name));
} while(FindNextFile(dir, &FindFileData));
done:
if(dir) FindClose(dir);
nullfree(lpath);
nullfree(ffpath);
nullfree(d);
FindClose(dir);
errno = 0;
return THROW(stat);
}
static int
platformdeleter(ZFMAP* zfmap, NClist* segments)
{
int ret = NC_NOERR;
struct stat statbuf;
struct dirent* entry = NULL;
char* path = NULL;
HANDLE dir;
WIN32_FIND_DATA FindFileData;
if((ret = nczm_join(segments,&path))) goto done;
errno = 0;
ret = stat(path, &statbuf);
if(ret < 0) {
if(errno == ENOENT) {ret = NC_NOERR; goto done;}
else {ret = platformerr(errno); goto done;}
}
/* process this file */
if(S_ISDIR(statbuf.st_mode)) {
dir = FindFirstFile(path, &FindFileData);
if(dir == INVALID_HANDLE_VALUE)
{stat = NC_ENOTFOUND; goto done;}
do {
char* seg = NULL;
errno = 0;
char* name = NULL;
name = FindFileData.cFileName;
if(name == NULL) {ret = NC_EINTERNAL break;}
/* Ignore "." and ".." */
if(strcmp(name,".")==0) continue;
if(strcmp(name,"..")==0) continue;
/* append name to segments */
if((seg = strdup(name)) == NULL)
{ret = NC_ENOMEM; goto done;}
nclistpush(segments,seg);
/* recurse */
if((ret = platformdeleter(zfmap, segments))) goto done;
/* remove+reclaim last segment */
nclistpop(segments);
nullfree(seg);
} while(FindNextFile(dir, &FindFileData));
}
done:
FindClose(dir);
/* delete this file|dir */
remove(path);
nullfree(path);
errno = 0;
return THROW(ret);
}
@ -923,28 +926,22 @@ done:
#else /*!_WIN32*/
static int
platformdircontent(ZFMAP* zfmap, const char* truepath, NClist* contents)
platformdircontent(ZFMAP* zfmap, const char* path, NClist* contents)
{
int ret = NC_NOERR;
int stat = NC_NOERR;
errno = 0;
DIR* dir = NULL;
switch (ret = platformtestcontentbearing(zfmap, truepath)) {
case NC_EEMPTY: ret = NC_NOERR; break; /* directory */
case NC_NOERR: ret = NC_EEMPTY; goto done;
default: goto done;
}
dir = NCopendir(truepath);
if(dir == NULL)
{ret = platformerr(errno); goto done;}
dir = NCopendir(path);
if(dir == NULL && errno == ENOTDIR)
goto done;
else if(dir == NULL) {stat = platformerr(errno); goto done;}
for(;;) {
const char* name = NULL;
struct dirent* de = NULL;
errno = 0;
de = readdir(dir);
if(de == NULL)
{ret = platformerr(errno); goto done;}
if((de = readdir(dir)) == NULL)
{stat = platformerr(errno); goto done;}
if(strcmp(de->d_name,".")==0 || strcmp(de->d_name,"..")==0)
continue;
name = de->d_name;
@ -953,28 +950,19 @@ platformdircontent(ZFMAP* zfmap, const char* truepath, NClist* contents)
done:
if(dir) NCclosedir(dir);
errno = 0;
return THROW(ret);
return THROW(stat);
}
#endif /*_WIN32*/
#if 0
static int
platformdeleter(ZFMAP* zfmap, NClist* segments, int depth)
platformdeleter(ZFMAP* zfmap, NClist* segments)
{
int ret = NC_NOERR;
struct stat statbuf;
struct dirent* entry = NULL;
DIR* dir = NULL;
char* path = NULL;
char* tmp = NULL;
if((ret = nczm_join(segments,&path))) goto done;
/* When running on any platform that can accept drive letters */
if((ret = nczm_fixpath(path,&tmp))) goto done;
nullfree(path); path = NULL;
if((path = NCpathcvt(tmp))==NULL) {ret = NC_ENOMEM; goto done;}
errno = 0;
ret = stat(path, &statbuf);
if(ret < 0) {
@ -998,90 +986,34 @@ platformdeleter(ZFMAP* zfmap, NClist* segments, int depth)
{ret = NC_ENOMEM; goto done;}
nclistpush(segments,seg);
/* recurse */
if((ret = platformdeleter(zfmap, segments, depth+1))) goto done;
if((ret = platformdeleter(zfmap, segments))) goto done;
/* remove+reclaim last segment */
nclistpop(segments);
nullfree(seg);
}
/* Delete this file */
rmdir(path);
} else {
assert(S_ISREG(statbuf.st_mode));
unlink(path);
}
done:
if(dir) NCclosedir(dir);
/* delete this file|dir */
remove(path);
nullfree(path);
nullfree(tmp);
errno = 0;
return THROW(ret);
}
#endif /*0*/
#endif /*_WIN32*/
/* Deep file/dir deletion */
static int
platformdeleter(ZFMAP* zfmap, NCbytes* truepath, int delroot, int depth)
{
int ret = NC_NOERR;
int i;
NClist* contents = nclistnew();
size_t tpathlen = ncbyteslength(truepath);
char* local = NULL;
ret = platformdircontent(zfmap, ncbytescontents(truepath), contents);
switch (ret) {
case NC_NOERR: /* recurse to remove levels below */
for(i=0;i<nclistlength(contents);i++) {
const char* name = nclistget(contents,i);
/* append name to current path */
ncbytescat(truepath, "/");
ncbytescat(truepath, name);
/* recurse */
if ((ret = platformdeleter(zfmap, truepath,delroot,depth+1))) goto done;
ncbytessetlength(truepath,tpathlen); /* reset */
ncbytesnull(truepath);
}
if(depth > 0 || delroot) {
/* localize and delete */
if((ret = nczm_localize(ncbytescontents(truepath),&local,LOCALIZE))) goto done;
rmdir(local); /* kill this dir */
}
break;
case NC_EEMPTY: /* Not a directory */
ret = NC_NOERR;
/* localize and delete */
if(local) {nullfree(local); local = NULL;}
if((ret = nczm_localize(ncbytescontents(truepath),&local,LOCALIZE))) goto done;
unlink(local); /* kill this file */
break;
case NC_ENOTFOUND:
default:
goto done;
}
done:
nclistfreeall(contents);
nullfree(local);
ncbytessetlength(truepath,tpathlen);
ncbytesnull(truepath);
return THROW(ret);
}
/* Deep file/dir deletion; depth first */
static int
platformdelete(ZFMAP* zfmap, const char* rootpath, int delroot)
platformdelete(ZFMAP* zfmap, const char* path)
{
int stat = NC_NOERR;
NCbytes* truepath = ncbytesnew();
if(rootpath == NULL || strlen(rootpath) == 0) goto done;
ncbytescat(truepath,rootpath);
if(rootpath[strlen(rootpath)-1] == '/') /* elide trailing '/' */
ncbytessetlength(truepath,ncbyteslength(truepath)-1);
if((stat = platformdeleter(zfmap,truepath,delroot,0))) goto done;
NClist* segments = NULL;
if(path == NULL || strlen(path) == 0) goto done;
segments = nclistnew();
nclistpush(segments,strdup(path));
stat = platformdeleter(zfmap,segments);
done:
ncbytesfree(truepath);
nclistfreeall(segments);
errno = 0;
return THROW(stat);
}
@ -1138,7 +1070,7 @@ platformwrite(ZFMAP* zfmap, FD* fd, size64_t count, const void* content)
while(need > 0) {
ssize_t red = 0;
if((red = write(fd->fd,(void*)writepoint,need)) <= 0)
if((red = write(fd->fd,writepoint,need)) <= 0)
{ret = NC_EACCESS; goto done;}
need -= red;
writepoint += red;
@ -1147,7 +1079,6 @@ done:
return THROW(ret);
}
#if 0
static int
platformcwd(char** cwdp)
{
@ -1158,7 +1089,6 @@ platformcwd(char** cwdp)
if(cwdp) *cwdp = strdup(buf);
return NC_NOERR;
}
#endif
/* When we are finished accessing FD; essentially
equivalent to closing the file descriptor.
@ -1186,6 +1116,29 @@ platformclose(ZFMAP* zfmap, FD* fd)
}
#endif
#if 0
/* Test type of the specified file.
@return NC_NOERR if found and is a content-bearing object
@return NC_EEMPTY if exists but is not-content-bearing
@return NC_ENOTFOUND if not found
*/
static int
platformfiletype(ZFMAP* zfmap, const char* truepath)
{
int ret = 0;
struct stat buf;
errno = 0;
switch(ret = stat(truepath, &buf)) {
if(ret < 0) {ret = errno; goto done;}
if(!S_ISDIR(buf.st_mode)) {ret = ENOTDIR;}
done:
errno = 0;
return ret;
}
#endif
#ifdef VERIFY
static int
@ -1194,7 +1147,7 @@ verify(const char* path, int isdir)
int ret = 0;
struct stat buf;
ret = NCaccess(path,ACCESS_MODE_EXISTS);
ret = NCaccess(path,F_OK);
if(ret < 0)
return 1; /* If it does not exist, then it can be anything */
ret = stat(path,&buf);
@ -1204,39 +1157,3 @@ verify(const char* path, int isdir)
return 0;
}
#endif
#if 0
/* Return NC_EINVAL if path does not exist; els 1/0 in isdirp and local path in truepathp */
static int
testifdir(const char* path, int* isdirp, char** truepathp)
{
int ret = NC_NOERR;
char* tmp = NULL;
char* truepath = NULL;
struct stat statbuf;
/* Make path be windows compatible */
if((ret = nczm_fixpath(path,&tmp))) goto done;
if((truepath = NCpathcvt(tmp))==NULL) {ret = NC_ENOMEM; goto done;}
errno = 0;
ret = stat(truepath, &statbuf);
if(ret < 0) {
if(errno == ENOENT)
ret = NC_ENOTFOUND; /* path does not exist */
else
ret = platformerr(errno);
goto done;
}
/* Check for being a directory */
if(isdirp) {
if(S_ISDIR(statbuf.st_mode)) {*isdirp = 1;} else {*isdirp = 0;}
}
if(truepathp) {*truepathp = truepath; truepath = NULL;}
done:
errno = 0;
nullfree(tmp);
nullfree(truepath);
return THROW(ret);
}
#endif /* 0 */

4
libnczarr/zmap_s3sdk.c
View File

@ -435,11 +435,11 @@ zs3write(NCZMAP* map, const char* key, size64_t start, size64_t count, const voi
}
if(newsize > objsize) {
/* Zeroize the part of the object added */
memset(((char*)chunk)+objsize,0,(newsize-objsize));
memset(chunk+objsize,0,(newsize-objsize));
objsize = newsize;
}
/* overwrite with the contents */
memcpy(((char*)chunk)+start,content,count); /* remember there may be data above start+count */
memcpy(chunk+start,content,count); /* remember there may be data above start+count */
if((stat = NCZ_s3sdkwriteobject(z3map->s3client, z3map->bucket, truekey, objsize, chunk, &z3map->errmsg)))
goto done;

5
libnczarr/zs3sdk.cpp
View File

@ -5,6 +5,7 @@
#define NOOP
#include "awsincludes.h"
#include <stdlib.h>
#include <string.h>
@ -13,10 +14,6 @@
#include "netcdf.h"
#include "zs3sdk.h"
#ifdef __CYGWIN__
extern char* strdup(const char*);
#endif
#define size64_t unsigned long long
static Aws::SDKOptions zs3options;

150
libnczarr/zvar.c
View File

@ -586,6 +586,18 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
var->storage = NC_CHUNKED;
}
#ifdef USE_PARALLEL
/* If deflate, shuffle, or fletcher32 was turned on with
* parallel I/O writes, then switch to collective access. HDF5
* requires collevtive access for filter use with parallel
* I/O. */
if (shuffle || fletcher32)
{
if (h5->parallel && (nclistlength(var->filters) > 0 || var->shuffle || var->fletcher32))
var->parallel_access = NC_COLLECTIVE;
}
#endif /* USE_PARALLEL */
/* Handle storage settings. */
if (storagep)
{
@ -1268,6 +1280,44 @@ log_dim_info(NC_VAR_INFO_T *var, size64_t *fdims, size64_t *fmaxdims,
}
#endif /* LOGGING */
#ifdef USE_PARALLEL4
/**
* @internal Set the parallel access for a var (collective
* vs. independent).
*
* @param h5 Pointer to ZARR file info struct.
* @param var Pointer to var info struct.
* @param xfer_plistid H5FD_MPIO_COLLECTIVE or H5FD_MPIO_INDEPENDENT.
*
* @returns NC_NOERR No error.
* @author Dennis Heimbigner, Ed Hartnett
*/
static int
set_par_access(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, hid_t xfer_plistid)
{
/* If netcdf is built with parallel I/O, then parallel access can
* be used, and, if this file was opened or created for parallel
* access, we need to set the transfer mode. */
if (h5->parallel)
{
H5FD_mpio_xfer_t ncz_xfer_mode;
/* Decide on collective or independent. */
ncz_xfer_mode = (var->parallel_access != NC_INDEPENDENT) ?
H5FD_MPIO_COLLECTIVE : H5FD_MPIO_INDEPENDENT;
/* Set the mode in the transfer property list. */
if (H5Pset_dxpl_mpio(xfer_plistid, ncz_xfer_mode) < 0)
return NC_EPARINIT;
LOG((4, "%s: %d H5FD_MPIO_COLLECTIVE: %d H5FD_MPIO_INDEPENDENT: %d",
__func__, (int)ncz_xfer_mode, H5FD_MPIO_COLLECTIVE,
H5FD_MPIO_INDEPENDENT));
}
return NC_NOERR;
}
#endif /* USE_PARALLEL4 */
/**
* @internal Write a strided array of data to a variable. This is
* called by nc_put_vars() and other nc_put_vars_* functions, for
@ -1308,22 +1358,18 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
#ifdef LOOK
hid_t file_spaceid = 0, mem_spaceid = 0, xfer_plistid = 0;
#endif
size64_t fdims[NC_MAX_VAR_DIMS];
size64_t fdims[NC_MAX_VAR_DIMS], fmaxdims[NC_MAX_VAR_DIMS];
size64_t start[NC_MAX_VAR_DIMS], count[NC_MAX_VAR_DIMS];
size64_t stride[NC_MAX_VAR_DIMS];
#ifdef USE_PARALLEL4
int extend_possible = 0;
#endif
int retval, range_error = 0, i, d2;
void *bufr = NULL;
int need_to_convert = 0;
int zero_count = 0; /* true if a count is zero */
size_t len = 1;
size64_t fmaxdims[NC_MAX_VAR_DIMS];
NC_UNUSED(fmaxdims);
#ifndef LOOK
NC_UNUSED(fmaxdims);
#endif
/* Find info for this file, group, and var. */
if ((retval = nc4_find_grp_h5_var(ncid, varid, &h5, &grp, &var)))
return THROW(retval);
@ -1357,8 +1403,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
/* Check to see if any counts are zero. */
if (!count[i])
zero_count++;
fdims[i] = var->dim[i]->len;
fmaxdims[i] = (fdims[i] = var->dim[i]->len);
}
#ifdef LOOK
@ -1454,6 +1499,11 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
if ((xfer_plistid = H5Pcreate(H5P_DATASET_XFER)) < 0)
BAIL(NC_EHDFERR);
#ifdef USE_PARALLEL4
/* Set up parallel I/O, if needed. */
if ((retval = set_par_access(h5, var, xfer_plistid)))
BAIL(retval);
#endif
#endif /*LOOK*/
/* Read this hyperslab from memory. Does the dataset have to be
@ -1471,6 +1521,9 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
assert(dim && dim->hdr.id == var->dimids[d2]);
if (dim->unlimited)
{
#ifdef USE_PARALLEL4
extend_possible = 1;
#endif
if (!zero_count && endindex >= fdims[d2])
{
xtend_size[d2] = (long long unsigned)(endindex+1);
@ -1492,12 +1545,40 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
}
#endif
#ifdef LOOK
#ifdef USE_PARALLEL4
/* Check if anyone wants to extend. */
if (extend_possible && h5->parallel &&
NC_COLLECTIVE == var->parallel_access)
{
/* Form consensus opinion among all processes about whether
* to perform collective I/O. */
if (MPI_SUCCESS != MPI_Allreduce(MPI_IN_PLACE, &need_to_extend, 1,
MPI_INT, MPI_BOR, h5->comm))
BAIL(NC_EMPI);
}
#endif /* USE_PARALLEL4 */
#endif /*LOOK*/
#ifdef LOOK
/* If we need to extend it, we also need a new file_spaceid
to reflect the new size of the space. */
if (need_to_extend)
{
LOG((4, "extending dataset"));
#ifdef USE_PARALLEL4
if (h5->parallel)
{
if (NC_COLLECTIVE != var->parallel_access)
BAIL(NC_ECANTEXTEND);
/* Reach consensus about dimension sizes to extend to */
if (MPI_SUCCESS != MPI_Allreduce(MPI_IN_PLACE, xtend_size, var->ndims,
MPI_UNSIGNED_LONG_LONG, MPI_MAX,
h5->comm))
BAIL(NC_EMPI);
}
#endif /* USE_PARALLEL4 */
/* Convert xtend_size back to hsize_t for use with
* H5Dset_extent. */
for (d2 = 0; d2 < var->ndims; d2++)
@ -1609,8 +1690,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
NC_DIM_INFO_T *dim;
size_t file_type_size;
size64_t count[NC_MAX_VAR_DIMS];
size64_t fdims[NC_MAX_VAR_DIMS];
size64_t fmaxdims[NC_MAX_VAR_DIMS];
size64_t fdims[NC_MAX_VAR_DIMS], fmaxdims[NC_MAX_VAR_DIMS];
size64_t start[NC_MAX_VAR_DIMS];
size64_t stride[NC_MAX_VAR_DIMS];
void *fillvalue = NULL;
@ -1621,8 +1701,6 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
int need_to_convert = 0;
size_t len = 1;
NC_UNUSED(fmaxdims);
/* Find info for this file, group, and var. */
if ((retval = nc4_find_grp_h5_var(ncid, varid, &h5, &grp, &var)))
return THROW(retval);
@ -1816,6 +1894,12 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
if ((xfer_plistid = H5Pcreate(H5P_DATASET_XFER)) < 0)
BAIL(NC_EHDFERR);
#ifdef USE_PARALLEL4
/* Set up parallel I/O, if needed. */
if ((retval = set_par_access(h5, var, xfer_plistid)))
BAIL(retval);
#endif
/* Read this hyperslab into memory. */
LOG((5, "About to H5Dread some data..."));
if (H5Dread(ncz_var->hdf_datasetid,
@ -1844,6 +1928,44 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
range_error = 0;
}
} /* endif ! no_read */
else
{
#ifdef LOOK
#ifdef USE_PARALLEL4 /* Start block contributed by HDF group. */
/* For collective IO read, some processes may not have any element for reading.
Collective requires all processes to participate, so we use H5Sselect_none
for these processes. */
if (var->parallel_access == NC_COLLECTIVE)
{
/* Create the data transfer property list. */
if ((xfer_plistid = H5Pcreate(H5P_DATASET_XFER)) < 0)
BAIL(NC_EHDFERR);
if ((retval = set_par_access(h5, var, xfer_plistid)))
BAIL(retval);
if (H5Sselect_none(file_spaceid) < 0)
BAIL(NC_EHDFERR);
/* Since no element will be selected, we just get the memory
* space the same as the file space. */
if ((mem_spaceid = H5Dget_space(ncz_var->hdf_datasetid)) < 0)
BAIL(NC_EHDFERR);
if (H5Sselect_none(mem_spaceid) < 0)
BAIL(NC_EHDFERR);
/* Read this hyperslab into memory. */
LOG((5, "About to H5Dread some data..."));
#ifdef LOOK
if (H5Dread(ncz_var->hdf_datasetid,
((NCZ_TYPE_INFO_T *)var->type_info->format_type_info)->native_hdf_typeid,
mem_spaceid, file_spaceid, xfer_plistid, bufr) < 0)
BAIL(NC_EHDFERR);
#endif
}
#endif /* USE_PARALLEL4 */
#endif /*LOOK*/
}
/* Now we need to fake up any further data that was asked for,
using the fill values instead. First skip past the data we
just read, if any. */

1
libnetcdf.settings.in
View File

@ -44,3 +44,4 @@ SZIP Support: @HAS_SZLIB@
SZIP Write Support: @HAS_SZLIB_WRITE@
Parallel Filters: @HAS_PAR_FILTERS@
NCZarr Support: @HAS_NCZARR@
NCZarr S3: @HAS_S3_SDK@

4
libsrc/CMakeLists.txt
View File

@ -39,9 +39,9 @@ SET(MAN_FILES netcdf.3)
set(ARGS_MANPAGE -DAPI=C)
IF (USE_HDF5)
IF (USE_NETCDF4)
SET(ARGS_MANPAGE ${ARGS_MANPAGE} -DNETCDF4=TRUE)
ENDIF(USE_HDF5)
ENDIF(USE_NETCDF4)
IF (BUILD_DAP)
SET(ARGS_MANPAGE ${ARGS_MANPAGE} -DDAP=TRUE)

4
libsrc4/CMakeLists.txt
View File

@ -31,9 +31,9 @@ set(man_MANS netcdf.3)
set(ARGS_MANPAGE -DAPI=C)
IF (USE_HDF5)
IF (USE_NETCDF4)
SET(ARGS_MANPAGE ${ARGS_MANPAGE} -DNETCDF4=TRUE)
ENDIF(USE_HDF5)
ENDIF(USE_NETCDF4)
IF (BUILD_DAP)
SET(ARGS_MANPAGE ${ARGS_MANPAGE} -DDAP=TRUE)

150
libsrc4/nc4type.c
View File

@ -532,153 +532,3 @@ NC4_inq_enum_member(int ncid, nc_type typeid1, int idx, char *identifier,
return NC_NOERR;
}
/**
* @internal Get the id of a type from the name.
*
* @param ncid File and group ID.
* @param name Name of type.
* @param typeidp Pointer that will get the type ID.
*
* @return ::NC_NOERR No error.
* @return ::NC_ENOMEM Out of memory.
* @return ::NC_EINVAL Bad size.
* @return ::NC_ENOTNC4 User types in netCDF-4 files only.
* @return ::NC_EBADTYPE Type not found.
* @author Ed Hartnett
*/
EXTERNL int
NC4_inq_typeid(int ncid, const char *name, nc_type *typeidp)
{
NC_GRP_INFO_T *grp;
NC_GRP_INFO_T *grptwo;
NC_FILE_INFO_T *h5;
NC_TYPE_INFO_T *type = NULL;
char *norm_name;
int i, retval;
/* Handle atomic types. */
for (i = 0; i < NUM_ATOMIC_TYPES; i++)
if (!strcmp(name, nc4_atomic_name[i]))
{
if (typeidp)
*typeidp = i;
return NC_NOERR;
}
/* Find info for this file and group, and set pointer to each. */
if ((retval = nc4_find_grp_h5(ncid, &grp, &h5)))
return retval;
assert(h5 && grp);
/* If the first char is a /, this is a fully-qualified
* name. Otherwise, this had better be a local name (i.e. no / in
* the middle). */
if (name[0] != '/' && strstr(name, "/"))
return NC_EINVAL;
/* Normalize name. */
if (!(norm_name = (char*)malloc(strlen(name) + 1)))
return NC_ENOMEM;
if ((retval = nc4_normalize_name(name, norm_name))) {
free(norm_name);
return retval;
}
/* Is the type in this group? If not, search parents. */
for (grptwo = grp; grptwo; grptwo = grptwo->parent) {
type = (NC_TYPE_INFO_T*)ncindexlookup(grptwo->type,norm_name);
if(type)
{
if (typeidp)
*typeidp = type->hdr.id;
break;
}
}
/* Still didn't find type? Search file recursively, starting at the
* root group. */
if (!type)
if ((type = nc4_rec_find_named_type(grp->nc4_info->root_grp, norm_name)))
if (typeidp)
*typeidp = type->hdr.id;
free(norm_name);
/* OK, I give up already! */
if (!type)
return NC_EBADTYPE;
return NC_NOERR;
}
/**
* @internal Get the class of a type
*
* @param h5 Pointer to the HDF5 file info struct.
* @param xtype NetCDF type ID.
* @param type_class Pointer that gets class of type, NC_INT,
* NC_FLOAT, NC_CHAR, or NC_STRING, NC_ENUM, NC_VLEN, NC_COMPOUND, or
* NC_OPAQUE.
*
* @return ::NC_NOERR No error.
* @author Ed Hartnett
*/
int
nc4_get_typeclass(const NC_FILE_INFO_T *h5, nc_type xtype, int *type_class)
{
int retval = NC_NOERR;
LOG((4, "%s xtype: %d", __func__, xtype));
assert(type_class);
/* If this is an atomic type, the answer is easy. */
if (xtype <= NC_STRING)
{
switch (xtype)
{
case NC_BYTE:
case NC_UBYTE:
case NC_SHORT:
case NC_USHORT:
case NC_INT:
case NC_UINT:
case NC_INT64:
case NC_UINT64:
/* NC_INT is class used for all integral types */
*type_class = NC_INT;
break;
case NC_FLOAT:
case NC_DOUBLE:
/* NC_FLOAT is class used for all floating-point types */
*type_class = NC_FLOAT;
break;
case NC_CHAR:
*type_class = NC_CHAR;
break;
case NC_STRING:
*type_class = NC_STRING;
break;
default:
BAIL(NC_EBADTYPE);
}
}
else
{
NC_TYPE_INFO_T *type;
/* See if it's a used-defined type */
if ((retval = nc4_find_type(h5, xtype, &type)))
BAIL(retval);
if (!type)
BAIL(NC_EBADTYPE);
*type_class = type->nc_type_class;
}
exit:
return retval;
}

14
nc-config.in
View File

@ -27,8 +27,6 @@ has_cdf5="@HAS_CDF5@"
has_szlib="@HAS_SZLIB@"
has_parallel="@HAS_PARALLEL@"
has_parallel4="@HAS_PARALLEL4@"
has_nczarr="@HAS_NCZARR@"
has_s3_sdk="@HAS_S3_SDK@"
version="@PACKAGE_NAME@ @PACKAGE_VERSION@"
HAS_STATIC=""
@ -93,8 +91,6 @@ Available values for OPTION include:
--has-cdf5 whether cdf5 support is included in build
--has-parallel4 whether has parallel IO support via HDF5
--has-parallel whether has parallel IO support via HDF5 or PnetCDF
--has-nczarr whether nczarr enabled
--has-s3-sdk whether AWS S3 support enabled
--libs library linking information for netcdf
--static library linking information for statically-compiled netcdf
--prefix Install prefix
@ -176,8 +172,6 @@ fi
echo " --has-cdf5 -> $has_cdf5"
echo " --has-parallel4 -> $has_parallel4"
echo " --has-parallel -> $has_parallel"
echo " --has-nczarr -> $has_nczarr"
echo " --has-s3-sdk -> $has_s3_sdk"
echo
echo " --prefix -> $prefix"
echo " --includedir -> $includedir"
@ -280,14 +274,6 @@ while test $# -gt 0; do
echo $has_parallel
;;
--has-nczarr)
echo $has_nczarr
;;
--has-s3-sdk)
echo $has_s3_sdk
;;
--libs)
if [ "x$HAS_STATIC" = "xTRUE" ]; then
echo $libs $libsprivate

4
nc_test/CMakeLists.txt
View File

@ -59,8 +59,8 @@ IF(LARGE_FILE_TESTS)
ENDIF()
SET(TESTFILES ${TESTFILES} tst_diskless tst_diskless3 tst_diskless4 tst_diskless5 tst_inmemory tst_open_mem)
IF(USE_HDF5)
SET(TESTFILES ${TESTFILES} tst_diskless tst_diskless3 tst_diskless4 tst_diskless5 tst_inmemory tst_open_mem)
IF(USE_NETCDF4)
SET(TESTFILES ${TESTFILES} tst_diskless2)
SET(TESTS ${TESTS} tst_diskless6)
ENDIF()

14
nc_test/run_diskless.sh
View File

@ -17,16 +17,18 @@ FILE3=tst_diskless3.nc
echo ""
echo "*** Testing in-memory (diskless) files with and without persistence"
HASHDF5=`${top_builddir}/nc-config --has-hdf5`
echo "**** Test diskless netCDF classic file without persistence"
${execdir}/tst_diskless
echo "PASS: diskless netCDF classic file without persistence"
if test "x$FEATURE_HDF5" = "xyes" ; then
if test "x$HASHDF5" = "xyes" ; then
echo ""
echo "**** Test diskless netCDF enhanced file without persistence"
${execdir}/tst_diskless netcdf4
echo "PASS: diskless netCDF enhanced file without persistence"
fi #FEATURE_HDF5
fi #HASHDF5
echo ""
echo "**** Test diskless netCDF classic file with persistence"
@ -41,7 +43,7 @@ echo "#### $FILE1 not created"
echo "FAIL: diskless netCDF classic file with persistence"
fi
if test "x$FEATURE_HDF5" = "xyes" ; then
if test "x$HASHDF5" = "xyes" ; then
echo ""
echo "**** Test diskless netCDF enhanced file with persistence"
rm -f $FILE1
@ -55,10 +57,10 @@ echo "$FILE1 not created"
echo "FAIL: diskless netCDF enhanced file with persistence"
fi
fi #FEATURE_HDF5
fi #HASHDF5
# Do extended netcdf enhanced test
if test "x$FEATURE_HDF5" = "xyes" ; then
if test "x$HASHDF5" = "xyes" ; then
ok=""
echo ""
@ -89,7 +91,7 @@ else
echo "FAIL: extended enhanced diskless netCDF with persistence"
fi
fi #FEATURE_HDF5
fi #HASHDF5
echo ""
echo "**** Testing nc_open in-memory (diskless) files"

10
nc_test/run_inmemory.sh
View File

@ -1,5 +1,6 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
@ -21,19 +22,20 @@ OMEMFILE=f03tst_open_mem.nc
echo ""
echo "*** Testing in-memory operations"
HASHDF5=`${top_builddir}/nc-config --has-hdf5`
# Execute the core of the inmemory tests
${execdir}/tst_inmemory
if test "x$FEATURE_HDF5" = "xyes" ; then
if test "x$HASNC4" = xyes ; then
${execdir}/tst_open_mem ${srcdir}/${OMEMFILE}
fi
echo "**** Test ncdump of the resulting inmemory data: netcdf-3"
echo "**** Test ncdump of the resulting inmemory data"
${NCDUMP} -n "${FILE3}" ${FILE3}.nc > ${FILE3}.cdl
${NCDUMP} -n "${FILE3}" ${CREATE3}.nc > ${CREATE3}.cdl
diff -wb ${FILE3}.cdl ${CREATE3}.cdl
if test "x$FEATURE_HDF5" = "xyes" ; then
echo "**** Test ncdump of the resulting inmemory data: netcdf-4"
if test "x$HASHDF5" = "xyes" ; then
${NCDUMP} ${FILE4}.nc > ${FILE4}.cdl
${NCDUMP} -n ${FILE4} ${CREATE4}.nc > ${CREATE4}.cdl
diff -wb ${FILE4}.cdl ${CREATE4}.cdl

6
nc_test/test_byterange.sh
View File

@ -9,6 +9,10 @@ set -e
URL3="https://remotetest.unidata.ucar.edu/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes"
URL4="https://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
# See if netcdf-4 support is enabled
HAVENC4=`cat ${TOPBUILDDIR}/libnetcdf.settings | sed -e '/NetCDF-4[ ]*API:[ ]*yes/p' -e d`
if test "x$HAVENC4" = x ; then HAVENC4=no; else HAVENC4=yes; fi
rm -f tst_http_nc3.cdl tst_http_nc4.cdl
echo ""
@ -26,7 +30,7 @@ ${NCDUMP} -h "$URL3" >tst_http_nc3.cdl
# compare
diff tst_http_nc3.cdl ${srcdir}/ref_tst_http_nc3.cdl
if test "x$FEATURE_HDF5" = xyes ; then
if test "x$HAVENC4" = xyes ; then
echo "*** Testing reading NetCDF-4 file with http"
# Test using -k flag
K=`${NCDUMP} -k "$URL4"`

2
nc_test/tst_big_rvar.c
View File

@ -22,7 +22,7 @@
/* Test with both classic and 64-bit offset files. If netcdf-4 is
* included, test with both netCDF-4 format variants also. */
/* #ifdef USE_HDF5 */
/* #ifdef USE_NETCDF4 */
/* #define NUM_FORMATS (4) */
/* #else */
/* #define NUM_FORMATS (2) */

2
nc_test/tst_big_var.c
View File

@ -19,7 +19,7 @@
/* Test with both classic and 64-bit offset files. If netcdf-4 is
* included, test with both netCDF-4 format variants also. */
#ifdef USE_HDF5
#ifdef USE_NETCDF4
#define NUM_FORMATS (4)
#else
#define NUM_FORMATS (2)

2
nc_test/tst_big_var2.c
View File

@ -22,7 +22,7 @@
/* Test with both classic and 64-bit offset files. If netcdf-4 is
* included, test with both netCDF-4 format variants also. */
#ifdef USE_HDF5
#ifdef USE_NETCDF4
#define NUM_FORMATS (4)
#else
#define NUM_FORMATS (2)

2
nc_test/tst_big_var6.c
View File

@ -21,7 +21,7 @@
/* Test with both classic and 64-bit offset files. If netcdf-4 is
* included, test with both netCDF-4 format variants also. */
#ifdef USE_HDF5
#ifdef USE_NETCDF4
#define NUM_FORMATS (4)
#else
#define NUM_FORMATS (2)

2
nc_test/tst_byterange.c
View File

@ -34,7 +34,7 @@ struct TESTURLS {
const char* url;
} testurls[] = {
{NC_FORMAT_CLASSIC,"https://remotetest.unidata.ucar.edu/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes"},
#ifdef USE_HDF5
#ifdef USE_NETCDF4
{NC_FORMAT_NETCDF4,"https://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"},
#endif
{0,NULL}

2
nc_test/tst_def_var_fill.c
View File

@ -57,7 +57,7 @@ int main(int argc, char** argv) {
#ifndef ENABLE_CDF5
if (formats[k] == NC_FORMAT_CDF5) continue;
#endif
#ifndef USE_HDF5
#ifndef USE_NETCDF4
if (formats[k] == NC_FORMAT_NETCDF4 ||
formats[k] == NC_FORMAT_NETCDF4_CLASSIC)
continue;

2
nc_test/tst_default_format_pnetcdf.c
View File

@ -100,7 +100,7 @@ int main(int argc, char *argv[])
__FILE__, __LINE__, err);
nerrs++;
}
#ifdef USE_HDF5
#ifdef USE_NETCDF4
/* check illegal cmode */
cmode = NC_NETCDF4 | NC_64BIT_OFFSET;
err = nc_create_par(fname, cmode, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid);

2
nc_test/tst_diskless.c
View File

@ -105,7 +105,7 @@ main(int argc, char **argv)
/* ignore anything not recognized */
}
#ifndef USE_HDF5
#ifndef USE_NETCDF4
fprintf(stderr,"netcdf-4 not supported; ignored\n");
usenetcdf4 = 0;
#endif

2
nc_test/tst_large.c
View File

@ -17,7 +17,7 @@
/* Test with both classic and 64-bit offset files. If netcdf-4 is
* included, test with both netCDF-4 format variants also. */
#ifdef USE_HDF5
#ifdef USE_NETCDF4
#define NUM_FORMATS (4)
#else
#define NUM_FORMATS (2)

2
nc_test/tst_large_cdf5.c
View File

@ -20,7 +20,7 @@
/* Test with both classic and 64-bit offset files. If netcdf-4 is
* included, test with both netCDF-4 format variants also. */
#ifdef USE_HDF5
#ifdef USE_NETCDF4
#define NUM_FORMATS (4)
#else
#define NUM_FORMATS (2)

4
nc_test/tst_misc.c
View File

@ -61,7 +61,7 @@ main(int argc, char **argv)
}
SUMMARIZE_ERR;
#ifndef USE_HDF5
#ifndef USE_NETCDF4
printf("*** Trying to create netCDF-4 file without netCDF-4...");
{
int ncid;
@ -70,7 +70,7 @@ main(int argc, char **argv)
ERR;
}
SUMMARIZE_ERR;
#endif /* USE_HDF5 undefined */
#endif /* USE_NETCDF4 undefined */
#ifdef TEST_PNETCDF
MPI_Finalize();

2
nc_test/tst_names.c
View File

@ -234,7 +234,7 @@ main(int argc, char **argv)
NC_FORMAT_NETCDF4
,
NC_FORMAT_NETCDF4_CLASSIC
#endif /* USE_HDF5 */
#endif /* USE_NETCDF4 */
};
int num_formats = (sizeof formats) / (sizeof formats[0]);
char *format_names[] = {

2
nc_test/tst_open_mem.c
View File

@ -21,7 +21,7 @@ See \ref copyright file for more info.
#include "ncbytes.h"
#include "nc_tests.h"
#ifdef USE_HDF5
#ifdef USE_NETCDF4
#include <hdf5.h>
extern int H5Eprint1(FILE * stream);
#endif

11
nc_test4/findplugin.in
View File

@ -28,6 +28,9 @@ FP_NAME="$1"
FP_ISCMAKE=@ISCMAKE@
FP_ISMSVC=@MSVC@
# And topsrcdir
topbuilddir='@abs_top_builddir@'
# Are we operating under OS-X? (test using uname)
FP_OS=`uname | cut -d '_' -f 1`
if test "x$FP_OS" = xDarwin ; then FP_ISOSX=1; fi
@ -36,7 +39,7 @@ if test "x$FP_OS" = xDarwin ; then FP_ISOSX=1; fi
FP_OS=`uname | cut -d '_' -f 1`
if test "x$FP_OS" = xCYGWIN ; then FP_ISCYGWIN=1; fi
FP_PLUGINS="$TOPBUILDDIR/plugins"
FP_PLUGINS="$topbuilddir/plugins"
FP_PLUGIN_LIB=
FP_PLUGIN_PATH=
@ -96,3 +99,9 @@ HDF5_PLUGIN_PATH="$FP_PLUGIN_PATH"
return 0
}
# debug
if test "x$1" != x ; then
findplugin
echo "HDF5_PLUGIN_PATH=|$FP_PLUGIN_PATH|"
fi

4
nc_test4/tst_converts.c
View File

@ -44,11 +44,11 @@ determine_test_formats(int *num_formats, int *format)
format[ind++] = NC_FORMAT_64BIT_OFFSET;
/* Do we have netCDF-4 and netCDF-4 classic? */
#ifdef USE_HDF5
#ifdef USE_NETCDF4
num += 2;
format[ind++] = NC_FORMAT_NETCDF4_CLASSIC;
format[ind++] = NC_FORMAT_NETCDF4;
#endif /* USE_HDF5 */
#endif /* USE_NETCDF4 */
/* Do we have CDF5? */
#ifdef ENABLE_CDF5

5
nc_test4/tst_udf.c
View File

@ -12,7 +12,6 @@
#include "err_macros.h"
#include "netcdf.h"
#include "nc4dispatch.h"
#include "hdf5dispatch.h"
#include "netcdf_dispatch.h"
#define FILE_NAME "tst_udf.nc"
@ -87,7 +86,7 @@ static NC_Dispatch tst_dispatcher = {
NC_RO_def_dim,
NC4_inq_dimid,
HDF5_inq_dim,
NC4_inq_dim,
NC4_inq_unlimdim,
NC_RO_rename_dim,
@ -182,7 +181,7 @@ static NC_Dispatch tst_dispatcher_bad_version = {
NC_RO_def_dim,
NC4_inq_dimid,
HDF5_inq_dim,
NC4_inq_dim,
NC4_inq_unlimdim,
NC_RO_rename_dim,

10
ncdap_test/t_dap.c
View File

@ -39,6 +39,16 @@ See \ref copyright file for more info.
#define DIMSIZE 25
#define STRLEN 64
#ifndef USE_NETCDF4
#define NC_UBYTE 7 /* unsigned 1 byte int */
#define NC_USHORT 8 /* unsigned 2-byte int */
#define NC_UINT 9 /* unsigned 4-byte int */
#define NC_INT64 10 /* signed 8-byte int */
#define NC_UINT64 11 /* unsigned 8-byte int */
#define NC_STRING 12 /* string */
#endif
#define CHECK(expr) check(expr,__FILE__,__LINE__);
#define COMMA (i==0?"":",")

10
ncdap_test/t_dap3a.c
View File

@ -40,6 +40,16 @@ See \ref copyright file for more info.
#define DIMSIZE 25
#define STRLEN 64
#ifndef USE_NETCDF4
#define NC_UBYTE 7 /* unsigned 1 byte int */
#define NC_USHORT 8 /* unsigned 2-byte int */
#define NC_UINT 9 /* unsigned 4-byte int */
#define NC_INT64 10 /* signed 8-byte int */
#define NC_UINT64 11 /* unsigned 8-byte int */
#define NC_STRING 12 /* string */
#endif
#define CHECK(expr) check(expr,__FILE__,__LINE__);
#define COMMA (i==0?"":",")

10
ncdap_test/test_cvt.c
View File

@ -39,6 +39,16 @@ See \ref copyright file for more info.
#define DIMSIZE 25
#define STRLEN 64
#ifndef USE_NETCDF4
#define NC_UBYTE 7 /* unsigned 1 byte int */
#define NC_USHORT 8 /* unsigned 2-byte int */
#define NC_UINT 9 /* unsigned 4-byte int */
#define NC_INT64 10 /* signed 8-byte int */
#define NC_UINT64 11 /* unsigned 8-byte int */
#define NC_STRING 12 /* string */
#endif
#define CHECK(expr) check(expr,__FILE__,__LINE__);
#define COMMA (i==0?"":",")

28
ncdump/CMakeLists.txt
View File

@ -86,7 +86,7 @@ IF(ENABLE_TESTS)
TARGET_LINK_LIBRARIES(bom netcdf)
TARGET_LINK_LIBRARIES(tst_dimsizes netcdf)
IF(USE_HDF5)
IF(USE_NETCDF4)
ADD_EXECUTABLE(tst_fileinfo tst_fileinfo.c)
TARGET_LINK_LIBRARIES(tst_fileinfo netcdf)
ENDIF()
@ -120,7 +120,7 @@ IF(ENABLE_TESTS)
SET_TARGET_PROPERTIES(nctrunc PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
IF(USE_HDF5)
IF(USE_NETCDF4)
SET_TARGET_PROPERTIES(tst_fileinfo PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(tst_fileinfo PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
@ -128,7 +128,7 @@ IF(ENABLE_TESTS)
SET_TARGET_PROPERTIES(tst_fileinfo PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
ENDIF(USE_HDF5)
ENDIF(USE_NETCDF4)
ENDIF(MSVC)
# Base tests
@ -146,11 +146,11 @@ ENDIF(MSVC)
add_sh_test(ncdump tst_calendars)
build_bin_test_no_prefix(tst_utf8)
add_sh_test(ncdump run_utf8_tests)
IF(USE_HDF5)
IF(USE_NETCDF4)
add_sh_test(ncdump run_utf8_nc4_tests)
add_sh_test(ncdump tst_fileinfo)
add_sh_test(ncdump tst_hdf5_offset)
ENDIF(USE_HDF5)
ENDIF(USE_NETCDF4)
add_sh_test(ncdump test_unicode_directory)
add_sh_test(ncdump tst_null_byte_padding)
@ -180,12 +180,12 @@ ENDIF(MSVC)
add_sh_test(ncdump tst_inmemory_nc3)
add_sh_test(ncdump tst_nccopy_w3)
IF(USE_HDF5)
IF(USE_NETCDF4)
add_sh_test(ncdump tst_inmemory_nc4)
add_sh_test(ncdump tst_nccopy_w4)
ENDIF(USE_HDF5)
ENDIF(USE_NETCDF4)
IF(USE_HDF5)
IF(USE_NETCDF4)
build_bin_test_no_prefix(tst_create_files)
build_bin_test_no_prefix(tst_group_data)
build_bin_test_no_prefix(tst_enum_data)
@ -237,9 +237,9 @@ ENDIF(MSVC)
# only ran after ncdump was built.
###
add_sh_test(ncdump run_ncgen_tests)
IF(USE_HDF5)
IF(USE_NETCDF4)
add_sh_test(ncdump run_ncgen_nc4_tests)
ENDIF(USE_HDF5)
ENDIF(USE_NETCDF4)
add_sh_test(ncdump tst_grp_spec)
add_sh_test(ncdump tst_mud)
@ -255,14 +255,14 @@ ENDIF(MSVC)
SET_TESTS_PROPERTIES(ncdump_tst_nccopy5 PROPERTIES RUN_SERIAL TRUE)
ENDIF(HAVE_BASH)
IF(USE_HDF5)
IF(USE_NETCDF4)
add_sh_test(ncdump tst_ncgen4)
ENDIF(USE_HDF5)
ENDIF(USE_NETCDF4)
add_sh_test(ncdump tst_inttags)
IF(USE_HDF5)
IF(USE_NETCDF4)
add_sh_test(ncdump tst_inttags4)
ENDIF(USE_HDF5)
ENDIF(USE_NETCDF4)
add_sh_test(ncdump test_radix)

4
ncdump/Makefile.am
View File

@ -25,7 +25,7 @@ endif
# This is the program we're building, and it's sources.
bin_PROGRAMS = ncdump
ncdump_SOURCES = ncdump.c vardata.c dumplib.c indent.c nctime0.c \
ncdump.h vardata.h dumplib.h indent.h nctime0.h cdl.h utils.h \
ncdump.h vardata.h dumplib.h indent.h isnan.h nctime0.h cdl.h utils.h \
utils.c nciter.h nciter.c nccomps.h
# Another utility program that copies any netCDF file using only the
@ -84,7 +84,7 @@ TESTS += tst_iter.sh
endif
if USE_HDF5
# HDF5 has some extra C programs to build. These will be run by
# NetCDF-4 has some extra C programs to build. These will be run by
# the shell script tests.
check_PROGRAMS += tst_fileinfo tst_create_files tst_h_rdc0 \
tst_group_data tst_enum_data tst_opaque_data tst_string_data \

17
include/isnan.h → ncdump/isnan.h
View File

@ -13,22 +13,6 @@
#endif /* NO_FLOAT_H */
#include <math.h>
#ifndef NAN
#error "NAN undefined"
#endif
#ifndef INFINITY
#error "INFINITY undefined"
#endif
#ifndef NANF
#define NANF ((float)NAN)
#endif
#ifndef INFINITYF
#define INFINITYF ((float)INFINITY)
#endif
#ifdef __APPLE__
extern int isinf(double x);
extern int isnan(double x);
#else
#if ! (defined(isinf) || HAVE_DECL_ISINF)
#define isinf(x) (DBL_MAX/((double)(x))==0.0)
#endif /* !HAVE_DECL_ISINF */
@ -38,6 +22,5 @@ extern int isnan(double x);
#if ! (defined(isfinite) || HAVE_DECL_ISFINITE)
#define isfinite(x) (!(isinf(x)||isnan(x)))
#endif /* !HAVE_DECL_ISFINITE */
#endif /*!APPLE*/
#endif /* _ISNAN_H */

2
ncgen/CMakeLists.txt
View File

@ -34,7 +34,7 @@ TARGET_LINK_LIBRARIES(ncgen netcdf ${ALL_TLL_LIBS})
# Given a netcdf4 file, dump the actual chunk contents.
# Used to validate nczarr chunking code.
IF(USE_HDF5)
IF(USE_NETCDF4)
IF(NOT MSVC)
SET(ncdumpchunks_FILES ncdumpchunks.c)
ADD_EXECUTABLE(ncdumpchunks ${ncdumpchunks_FILES})

4
ncgen/Makefile.am
View File

@ -45,9 +45,9 @@ tst_gattenum.cdl tst_usuffix.cdl
# Moved to ncdump as these tests rely on ncdump having been built.
##
#TESTS = run_tests.sh
#if USE_HDF5
#if USE_NETCDF4
#TESTS += run_nc4_tests.sh
#endif # USE_HDF5
#endif # USE_NETCDF4
#endif # !BUILD_DLL
CLEANFILES = c0.nc c0_64.nc c0_4.nc c0_4c.nc ref_camrun.c \

2
ncgen/cvt.c
View File

@ -7,7 +7,7 @@
#include "includes.h"
#include "bytebuffer.h"
#include "isnan.h"
#include "../ncdump/isnan.h"
#include <math.h>
static char stmp[256];

7
ncgen/ncdumpchunks.c
View File

@ -49,14 +49,11 @@ usage(int err)
int
main(int argc, char** argv)
{
int i,stat = NC_NOERR;
int i,r,stat = NC_NOERR;
hid_t fileid, grpid, datasetid;
int* chunkdata = NULL; /*[CHUNKPROD];*/
#ifdef HDF5_SUPPORTS_PAR_FILTERS
int r;
hid_t dxpl_id = H5P_DEFAULT; /*data transfer property list */
int* chunkdata = NULL; /*[CHUNKPROD];*/
unsigned int filter_mask = 0;
#endif
const char* file_name = NULL;
const char* var_name = NULL;
int ncid, varid, dimids[NC_MAX_VAR_DIMS];

17
ncgen/ncgen.h
View File

@ -8,7 +8,7 @@
#ifdef _WIN32
#include <float.h>
#include "isnan.h"
#include "../ncdump/isnan.h"
#define strcasecmp _stricmp
#endif
@ -40,8 +40,23 @@
/* Extend nc types with NIL value*/
#define NC_NIL 32
/* Must be a better way to do this */
#ifndef INFINITE
#ifdef _MSC_VER
#define NC_INFINITE (DBL_MAX+DBL_MAX)
#define NC_INFINITEF NC_INFINITE
#define NAN (NC_INFINITE-NC_INFINITE)
#define NANF NAN
#else
#define NC_INFINITE (1.0/0.0)
#define NC_INFINITEF (1.0f/0.0f)
#define NAN (0.0/0.0)
#define NANF (0.0f/0.0f)
#endif
#define NEGNC_INFINITEF (-NC_INFINITEF)
#define NEGNC_INFINITE (-NC_INFINITEF)
#endif
/* nc_class is one of:
NC_GRP NC_DIM NC_VAR NC_ATT NC_TYPE

9
ncgen/ncgen.l
View File

@ -37,7 +37,6 @@
#include "ncgen.h"
#include "ncgeny.h"
#include "isnan.h"
EXTERNL int fileno(FILE*);
@ -291,9 +290,9 @@ group: {return lexdebug(GROUP);}
DoubleInf|-?Infinity { /* missing value (pre-2.4 backward compatibility) */
if (yytext[0] == '-') {
double_val = -INFINITY;
double_val = NEGNC_INFINITE;
} else {
double_val = INFINITY;
double_val = NC_INFINITE;
}
specialconstants = 1;
return lexdebug(DOUBLE_CONST);
@ -306,9 +305,9 @@ NaN|nan { /* missing value (pre-2.4 backward compatibility) */
FloatInf|-?Infinityf|-?Inff {/* missing value (pre-2.4 backward compatibility)*/
if (yytext[0] == '-') {
float_val = -INFINITYF;
float_val = NEGNC_INFINITEF;
} else {
float_val = INFINITYF;
float_val = NC_INFINITEF;
}
specialconstants = 1;
return lexdebug(FLOAT_CONST);

981
ncgen/ncgenl.c
View File

File diff suppressed because it is too large Load Diff

3
ncgen3/CMakeLists.txt
View File

@ -33,7 +33,6 @@ ENDIF()
ADD_EXECUTABLE(ncgen3 ${ncgen3_FILES})
TARGET_LINK_LIBRARIES(ncgen3 netcdf ${ALL_TLL_LIBS})
####
# We have to do a little tweaking
# to remove the Release/ and Debug/ directories
@ -53,7 +52,7 @@ ENDIF()
IF(ENABLE_TESTS)
SET(NCGEN3_TESTS run_tests)
IF(USE_HDF5)
IF(USE_NETCDF4)
SET(NCGEN3_TESTS ${NCGEN3_TESTS} run_nc4_tests)
ENDIF()

35
nczarr_test/CMakeLists.txt
View File

@ -18,54 +18,41 @@ FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSION
IF(ENABLE_TESTS)
SET(COMMONSRC ut_util.c ut_test.c)
IF(USE_X_GETOPT)
SET(COMMONSRC ${COMMONSRC} XGetopt.c)
ENDIF()
# Base tests
# The tests are set up as a combination of shell scripts and executables that
# must be run in a particular order. It is painful but will use macros to help
# keep it from being too bad.
BUILD_BIN_TEST(ut_map ${COMMONSRC})
BUILD_BIN_TEST(ut_mapapi ${COMMONSRC})
BUILD_BIN_TEST(ut_json ${COMMONSRC})
BUILD_BIN_TEST(ut_projections ${COMMONSRC})
BUILD_BIN_TEST(ut_chunking ${COMMONSRC})
BUILD_BIN_TEST(tst_chunks ${COMMONSRC})
BUILD_BIN_TEST(tst_chunks2 ${COMMONSRC})
BUILD_BIN_TEST_SOURCED(ut_map ${COMMONSRC})
BUILD_BIN_TEST_SOURCED(ut_mapapi ${COMMONSRC})
BUILD_BIN_TEST_SOURCED(ut_json ${COMMONSRC})
BUILD_BIN_TEST_SOURCED(ut_projections ${COMMONSRC})
BUILD_BIN_TEST_SOURCED(ut_chunking ${COMMONSRC})
TARGET_INCLUDE_DIRECTORIES(ut_map PUBLIC ../libnczarr)
TARGET_INCLUDE_DIRECTORIES(ut_mapapi PUBLIC ../libnczarr)
TARGET_INCLUDE_DIRECTORIES(ut_json PUBLIC ../libnczarr)
TARGET_INCLUDE_DIRECTORIES(ut_projections PUBLIC ../libnczarr)
TARGET_INCLUDE_DIRECTORIES(ut_chunking PUBLIC ../libnczarr)
TARGET_INCLUDE_DIRECTORIES(tst_chunks PUBLIC ../libnczarr)
TARGET_INCLUDE_DIRECTORIES(tst_chunks2 PUBLIC ../libnczarr)
# Helper programs for testing
BUILD_BIN_TEST(zmapio ${COMMONSRC})
BUILD_BIN_TEST_SOURCED(zmapio ${COMMONSRC})
TARGET_INCLUDE_DIRECTORIES(zmapio PUBLIC ../libnczarr)
BUILD_BIN_TEST(zhex)
BUILD_BIN_TEST(zisjson)
TARGET_INCLUDE_DIRECTORIES(zisjson PUBLIC ../libnczarr)
BUILD_BIN_TEST(zs3parse ${COMMONSRC})
TARGET_INCLUDE_DIRECTORIES(zs3parse PUBLIC ../libnczarr)
# Integration tests
# BUILD_BIN_TEST(it_meta)
# TARGET_INCLUDE_DIRECTORIES(it_meta PUBLIC ../libnczarr)
IF(BUILD_UTILITIES)
add_sh_test(nczarr_test run_ut_map)
add_sh_test(nczarr_test run_ut_mapapi)
add_sh_test(nczarr_test run_ut_misc)
add_sh_test(nczarr_test run_ut_chunk)
IF(USE_HDF5)
add_sh_test(nczarr_test run_it_test1)
ENDIF()
add_sh_test(nczarr_test run_it_test2)
ENDIF(BUILD_UTILITIES)
ENDIF(ENABLE_TESTS)
## Specify files to be distributed by 'make dist'
FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.c ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am zmapio.c zhex.c)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am zmapio.c)
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

32
nczarr_test/Makefile.am
View File

@ -6,8 +6,6 @@
# Put together AM_CPPFLAGS and AM_LDFLAGS.
include $(top_srcdir)/lib_flags.am
TESTS_ENVIRONMENT =
#TEST_EXTENSIONS = .sh
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
@ -43,29 +41,17 @@ TESTS += run_ut_map.sh
TESTS += run_ut_mapapi.sh
TESTS += run_ut_misc.sh
TESTS += run_ut_chunk.sh
TESTS += run_it_test2.sh
if USE_HDF5
TESTS += run_it_test1.sh
endif
TESTS += run_it_test1.sh run_it_test2.sh
ignorec = ut_allslices.c ut_transfer.c ut_vars.c ut_walk.c
ignoresh = run_meta_tests.sh run_unittests.sh
ignoreh = ut_varstest.h
# These programs are used by the test cases
noinst_PROGRAMS = zmapio
zmapio_SOURCES = zmapio.c
noinst_PROGRAMS += zhex
zhex_SOURCES = zhex.c
noinst_PROGRAMS += zisjson
zisjson_SOURCES = zisjson.c
noinst_PROGRAMS += zs3parse
zs3parse_SOURCES = zs3parse.c
EXTRA_DIST = \
run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_it_test1.sh run_it_test2.sh \
test_nczarr.sh \
ref_ut_map_readmeta_nzf.txt \
ref_ut_map_writedata_nzf.cdl \
ref_ut_map_writemeta_nzf.cdl \
@ -93,14 +79,8 @@ ref_ut_mapapi_data_nzf.cdl \
ref_ut_mapapi_data_s3.cdl \
ref_ut_mapapi_meta_nz4.cdl \
ref_ut_mapapi_meta_nzf.cdl \
ref_ut_mapapi_meta_s3.cdl \
ref_ut_mapapi_search_nz4.txt \
ref_ut_mapapi_search_nzf.txt \
ref_ut_mapapi_search_s3.txt
ref_ut_mapapi_meta_s3.cdl
CLEANFILES = test.nzf test.nz4 test.nz4 tst_chunks.nc.nzf tst_chunks.nc.nz4 \
testmap.nzf testmap.nz4 testmapapi.nzf testmapapi.nz4 \
tst_chunks.nz4 tst_chunks.nzf tst_chunks2.nc.nz4 \
ut_*.txt ut*.cdl
#tst_group_data.dmp tst_nans.dmp tst_solar_1.dmp tst_nul4.dmp dimscope.dmp
#CLEANFILES = \
# ut_map*.cdl ut_map*.txt \
# ut_mapapi*.cdl ut_mapapi*.txt

4
nczarr_test/ref_ut_map_create_nzf.cdl
View File

@ -1,2 +1,2 @@
testmap.nzf
testmap.nzf/.nczarr : ||
./testmap.nzf
./testmap.nzf/.nczarr : ||

12
nczarr_test/ref_ut_map_search_nz4.txt
View File

@ -1,8 +1,8 @@
[0] /
[1] /.nczarr
[2] /data1
[3] /data1/0
[4] /meta1
[5] /meta1/.zarray
[6] /meta2
[7] /meta2/.nczvar
[2] /meta1
[3] /meta1/.zarray
[4] /meta2
[5] /meta2/.nczvar
[6] /data1
[7] /data1/0

8
nczarr_test/ref_ut_map_search_nzf.txt
View File

@ -1,8 +1,8 @@
[0] /
[1] /.nczarr
[2] /data1
[3] /data1/0
[2] /meta2
[3] /meta2/.nczvar
[4] /meta1
[5] /meta1/.zarray
[6] /meta2
[7] /meta2/.nczvar
[6] /data1
[7] /data1/0

12
nczarr_test/ref_ut_map_write2meta_nzf.cdl
View File

@ -1,6 +1,6 @@
testmap.nzf
testmap.nzf/.nczarr : ||
testmap.nzf/meta1
testmap.nzf/meta1/.zarray : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4]}|
testmap.nzf/meta2
testmap.nzf/meta2/.nczvar : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4], "extra": 137}|
./testmap.nzf
./testmap.nzf/.nczarr : ||
./testmap.nzf/meta2
./testmap.nzf/meta2/.nczvar : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4], "extra": 137}|
./testmap.nzf/meta1
./testmap.nzf/meta1/.zarray : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4]}|

16
nczarr_test/ref_ut_map_writedata_nzf.cdl
View File

@ -1,8 +1,8 @@
testmap.nzf
testmap.nzf/.nczarr : ||
testmap.nzf/data1
testmap.nzf/data1/0 : |000000000100000002000000030000000400000005000000060000000700000008000000090000000a0000000b0000000c0000000d0000000e0000000f000000100000001100000012000000130000001400000015000000160000001700000018000000|
testmap.nzf/meta1
testmap.nzf/meta1/.zarray : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4]}|
testmap.nzf/meta2
testmap.nzf/meta2/.nczvar : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4], "extra": 137}|
./testmap.nzf
./testmap.nzf/.nczarr : ||
./testmap.nzf/meta2
./testmap.nzf/meta2/.nczvar : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4], "extra": 137}|
./testmap.nzf/meta1
./testmap.nzf/meta1/.zarray : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4]}|
./testmap.nzf/data1
./testmap.nzf/data1/0 : | 0 0 0 0 1 0 0 0 2 0 0 0 3 0 0 0 4 0 0 0 5 0 0 0 6 0 0 0 7 0 0 0 8 0 0 0 9 0 0 0 a 0 0 0 b 0 0 0 c 0 0 0 d 0 0 0 e 0 0 0 f 0 0 0 10 0 0 0 11 0 0 0 12 0 0 0 13 0 0 0 14 0 0 0 15 0 0 0 16 0 0 0 17 0 0 0 18 0 0 0|

8
nczarr_test/ref_ut_map_writemeta_nzf.cdl
View File

@ -1,4 +1,4 @@
testmap.nzf
testmap.nzf/.nczarr : ||
testmap.nzf/meta1
testmap.nzf/meta1/.zarray : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4]}|
./testmap.nzf
./testmap.nzf/.nczarr : ||
./testmap.nzf/meta1
./testmap.nzf/meta1/.zarray : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4]}|

4
nczarr_test/ref_ut_mapapi_data_nz4.cdl
View File

@ -2,7 +2,7 @@
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[2] /data1 : (100) |000000000100000002000000030000000400000005000000060000000700000008000000090000000a0000000b0000000c0000000d0000000e0000000f000000100000001100000012000000130000001400000015000000160000001700000018000000|
[4] /meta1/.zarray : (34) |{
[3] /meta1/.zarray : (34) |{
"shape": [1,2,3],
"dtype": "<1"}|
[4] /data1 : (100) |000000000100000002000000030000000400000005000000060000000700000008000000090000000a0000000b0000000c0000000d0000000e0000000f000000100000001100000012000000130000001400000015000000160000001700000018000000|

4
nczarr_test/ref_ut_mapapi_data_nzf.cdl
View File

@ -2,7 +2,7 @@
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[2] /data1 : (100) |000000000100000002000000030000000400000005000000060000000700000008000000090000000a0000000b0000000c0000000d0000000e0000000f000000100000001100000012000000130000001400000015000000160000001700000018000000|
[4] /meta1/.zarray : (34) |{
[3] /meta1/.zarray : (34) |{
"shape": [1,2,3],
"dtype": "<1"}|
[4] /data1 : (100) |000000000100000002000000030000000400000005000000060000000700000008000000090000000a0000000b0000000c0000000d0000000e0000000f000000100000001100000012000000130000001400000015000000160000001700000018000000|

Some files were not shown because too many files have changed in this diff Show More