mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-04-12 18:10:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

merged master

Browse Source
This commit is contained in:
Edward Hartnett 2020-07-03 13:57:47 -06:00
commit 4b78c0c4a3
276 changed files with 31291 additions and 4617 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
CMakeLists.txt
View File

@ -10,7 +10,7 @@
cmake_minimum_required(VERSION 3.6.1)
#Project Name
project(netCDF C)
project(netCDF LANGUAGES C CXX)
set(PACKAGE "netCDF" CACHE STRING "")
#####
@ -232,7 +232,7 @@ IF(CMAKE_COMPILER_IS_GNUCC OR APPLE)
MESSAGE(STATUS "Coverage Tests: On.")
ENDIF()
# Warnings for 64-to-32 bit conversions.
# Warnings for 64-to-32 bit conversions.
IF(ENABLE_CONVERSION_WARNINGS)
CHECK_C_COMPILER_FLAG(-Wconversion CC_HAS_WCONVERSION)
CHECK_C_COMPILER_FLAG(-Wshorten-64-to-32 CC_HAS_SHORTEN_64_32)
@ -270,8 +270,6 @@ SET(CMAKE_REQUIRED_INCLUDES ${CMAKE_CURRENT_SOURCE_DIR}/libsrc)
# End Compiler Configuration
################################
##
# Configuration for post-install RPath
# Adapted from http://www.cmake.org/Wiki/CMake_RPATH_handling
@ -906,6 +904,8 @@ IF(MSVC)
DESTINATION ${netCDF_BINARY_DIR}/ncgen/)
FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c
DESTINATION ${netCDF_BINARY_DIR}/ncdump/)
FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c
DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
ENDIF()
ENDIF()
@ -925,13 +925,49 @@ IF(NOT WIN32)
ENDIF()
ENDIF()
# Option to Build NCZarr clients
OPTION(ENABLE_NCZARR "Enable NCZarr Client." OFF)
OPTION(ENABLE_S3_SDK "Enable NCZarr S3 support." ON)
# See if aws-s3-sdk is available
FIND_LIBRARY(AWS_CPP_SDK_CORE_LIB NAMES aws-cpp-sdk-core)
FIND_LIBRARY(AWS_CPP_SDK_S3_LIB NAMES aws-cpp-sdk-s3)
FIND_LIBRARY(AWS_C_COMMON_LIB NAMES aws-c-common)
IF(NOT AWS_CPP_SDK_CORE_LIB)
SET(AWS_CPP_SDK_CORE_LIB OFF)
ENDIF()
IF(NOT AWS_CPP_SDK_S3_LIB)
SET(AWS_CPP_SDK_S3_LIB OFF)
ENDIF()
IF(NOT AWS_C_COMMON_LIB)
SET(AWS_C_COMMON_LIB OFF)
ENDIF()
IF(ENABLE_S3_SDK)
IF(NOT AWS_CPP_SDK_CORE_LIB
OR NOT AWS_CPP_SDK_S3_LIB
OR NOT AWS_C_COMMON_LIB)
MESSAGE(WARNING "One of aws-c-common, aws-cpp-sdk-s3, aws-cpp-sdk-core libraries is not available: Turning off S3 support")
SET(ENABLE_S3_SDK OFF)
ELSE()
MESSAGE(STATUS "Found aws sdk libraries")
ENDIF()
ENDIF()
IF(ENABLE_S3_SDK)
IF(CMAKE_COMPILER_IS_GNUCC OR APPLE)
# SET(LDFLAGS "${LDFLAGS} ${AWS_CPP_SDK_CORE_LIB} ${AWS_CPP_SDK_S3_LIB} ${AWS_C_COMMON_LIB}")
# SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lstdc++")
ENDIF(CMAKE_COMPILER_IS_GNUCC OR APPLE)
ENDIF(ENABLE_S3_SDK)
##
# Enable Tests
##
OPTION(ENABLE_TESTS "Enable basic tests, run with 'make test'." ON)
IF(ENABLE_TESTS)
SET(BUILD_TESTSETS ON CACHE BOOL "")
# Options for CTest-based tests, dashboards.
SET(NC_CTEST_PROJECT_NAME "netcdf-c" CACHE STRING "Project Name for CTest-based testing purposes.")
SET(NC_CTEST_DROP_SITE "cdash.unidata.ucar.edu:443" CACHE STRING "Dashboard location for CTest-based testing purposes.")
@ -967,6 +1003,10 @@ IF(ENABLE_TESTS)
OPTION(ENABLE_FAILING_TESTS "Run tests which are known to fail, check to see if any have been fixed." OFF)
# There is a special class of tests that currently fail because of memory leaks.
# As with failing tests, they can be enabled to work on them
OPTION(ENABLE_UNFIXED_MEMORY_LEAKS "Run tests with unfixed memory leaks" OFF)
###
# Option to turn on unit testing. See
# https://github.com/Unidata/netcdf-c/pull/1472 for more
@ -1325,6 +1365,8 @@ CHECK_INCLUDE_FILE("string.h" HAVE_STRING_H)
CHECK_INCLUDE_FILE("winsock2.h" HAVE_WINSOCK2_H)
CHECK_INCLUDE_FILE("ftw.h" HAVE_FTW_H)
CHECK_INCLUDE_FILE("libgen.h" HAVE_LIBGEN_H)
CHECK_INCLUDE_FILE("execinfo.h" HAVE_EXECINFO_H)
CHECK_INCLUDE_FILE("dirent.h" HAVE_DIRENT_H)
# Symbol Exists
CHECK_SYMBOL_EXISTS(isfinite "math.h" HAVE_DECL_ISFINITE)
@ -1622,6 +1664,23 @@ MACRO(add_bin_test prefix F)
ENDIF()
ENDMACRO()
# Binary tests which are used by a script looking for a specific name.
MACRO(build_bin_test_sourced F)
ADD_EXECUTABLE(${F} ${F}.c ${ARGN})
TARGET_LINK_LIBRARIES(${F} netcdf ${ALL_TLL_LIBS})
IF(MSVC)
SET_TARGET_PROPERTIES(${F}
PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT"
)
SET_TARGET_PROPERTIES(${F} PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(${F} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(${F} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
ENDIF()
ENDMACRO()
# A cmake script to print out information at the end of the configuration step.
MACRO(print_conf_summary)
MESSAGE("")
@ -1785,6 +1844,10 @@ IF(ENABLE_DAP4)
ADD_SUBDIRECTORY(libdap4)
ENDIF()
IF(ENABLE_NCZARR)
ADD_SUBDIRECTORY(libnczarr)
ENDIF()
add_subdirectory(liblib)
IF(ENABLE_FILTER_TESTING)
@ -1826,6 +1889,9 @@ IF(ENABLE_TESTS)
IF(ENABLE_DAP4)
ADD_SUBDIRECTORY(dap4_test)
ENDIF()
IF(ENABLE_NCZARR)
ADD_SUBDIRECTORY(nczarr_test)
ENDIF()
IF(ENABLE_EXAMPLES)
ADD_SUBDIRECTORY(examples)
ENDIF()
@ -2002,6 +2068,8 @@ is_enabled(USE_CDF5 HAS_CDF5)
is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
is_enabled(HAVE_H5Z_SZIP HAS_SZLIB)
is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
is_enabled(ENABLE_NCZARR HAS_NCZARR)
is_enabled(ENABLE_S3_SDK HAS_S3_SDK)
# Generate file from template.
CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
@ -2013,7 +2081,7 @@ CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
# might also work on Windows.
FILE(READ "${CMAKE_CURRENT_BINARY_DIR}/libnetcdf.settings"
LIBNETCDF_SETTINGS)
MESSAGE(${LIBNETCDF_SETTINGS})
MESSAGE(STATUS ${LIBNETCDF_SETTINGS})
# Install libnetcdf.settings file into same location
# as the libraries.

11
Makefile.am
View File

@ -81,6 +81,12 @@ HDF4_TEST_DIR = hdf4_test
LIBHDF4 = libhdf4
endif
# Build Cloud Storage if desired.
if ENABLE_NCZARR
ZARR_TEST_DIR = nczarr_test
ZARR = libnczarr
endif
# Optionally build plugins
if ENABLE_FILTER_TESTING
PLUGIN_DIR = plugins
@ -94,14 +100,15 @@ endif # BUILD_BENCHMARKS
# Define Test directories
if BUILD_TESTSETS
TESTDIRS = $(UNIT_TEST) $(V2_TEST) nc_test $(NC_TEST4) \
$(BENCHMARKS_DIR) $(HDF4_TEST_DIR) $(NCDAP2TESTDIR) $(NCDAP4TESTDIR)
$(BENCHMARKS_DIR) $(HDF4_TEST_DIR) $(NCDAP2TESTDIR) $(NCDAP4TESTDIR) \
${ZARR_TEST_DIR}
endif
# This is the list of subdirs for which Makefiles will be constructed
# and run. ncgen must come before ncdump, because their tests
# depend on it.
SUBDIRS = include $(H5_TEST_DIR) libdispatch libsrc $(LIBSRC4_DIR) \
$(LIBSRCP) $(LIBHDF4) $(LIBHDF5) $(OCLIB) $(DAP2) ${DAP4} liblib \
$(LIBSRCP) $(LIBHDF4) $(LIBHDF5) $(OCLIB) $(DAP2) ${DAP4} ${ZARR} liblib \
$(NCGEN3) $(NCGEN) $(NCDUMP) ${PLUGIN_DIR} $(TESTDIRS) docs \
$(EXAMPLES)

3
NUG/Doxyfile
View File

@ -794,7 +794,8 @@ WARN_LOGFILE =
INPUT = guide.dox \
bestpractices.md \
install.md \
DAP2.dox \
nczarr.md \
DAP2.dox \
DAP4.dox \
OPeNDAP.dox \
types.dox \

416
NUG/nczarr.md Executable file
View File

@ -0,0 +1,416 @@
# The NetCDF NCZarr Implementation {#nczarr_head}
[TOC]
# NCZarr Introduction {#nczarr_introduction}
Beginning with netCDF version 4.8.0, the Unidata NetCDF group
has extended the netcdf-c library to provide access to cloud
storage (e.g. Amazon S3
<a href="#ref_aws">[1]</a>
) by providing a mapping from a subset of the full netCDF Enhanced
(aka netCDF-4) data model to a variant of the Zarr
<a href="#ref_zarrv2">[4]</a>
data model that already has mappings to
key-value pair cloud storage systems.
The NetCDF version of this storage format is called NCZarr
<a href="#ref_nczarr">[2]</a>.
# The NCZarr Data Model {#nczarr_data_model}
NCZarr uses a data model <a href="#ref_nczarr">[2]</a> that is,
by design, similar to, but not identical with the Zarr Version 2
Specification <a href="#ref_zarrv2">[4]</a>. Briefly, the data
model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. As with netcdf-4 it supports
chunking. Eventually it will also support filters in a manner
similar to the way filters are supported in netcdf-4.
Specifically, the model supports the following.
- "Atomic" types: char, byte, ubyte, short, ushort, int, uint, int64, uint64.
- Shared (named) dimensions
- Attributes with specified types -- both global and per-variable
- Chunking
- Fill values
- Groups
- N-Dimensional variables
- Per-variable endianness (big or little)
With respect to full netCDF-4, the following concepts are
currently unsupported.
- String type
- User-defined types (enum, opaque, VLEN, and Compound)
- Unlimited dimensions
# Enabling NCZarr Support {#nczarr_enable}
NCZarr support is enabled if the _--enable-nczarr_ option
is used with './configure'. If NCZarr support is enabled, then
a usable version of _libcurl_ must be specified
using the _LDFLAGS_ environment variable (similar to the way
that the _HDF5_ libraries are referenced).
Refer to the installation manual for details.
NCZarr support can be disabled using the _--disable-dap_.
# Accessing Data Using the NCZarr Prototocol {#nczarr_accessing_data}
In order to access a NCZarr data source through the netCDF API, the
file name normally used is replaced with a URL with a specific
format.
## URL Format
The URL is the usual scheme:://host:port/path?query#fragment format.
There are some details that are important.
- Scheme: this should be _https_ or _s3_,or _file_.
The _s3_ scheme is equivalent
to "https" plus setting "mode=nczarr" (see below).
Specifying "file" is mostly used for testing.
- Host: Amazon S3 defines two forms: _Virtual_ and _Path_.
+ _Virtual_: the host includes the bucket name as in
__bucket.s3.&lt;region&gt;.amazonaws.com__
+ _Path_: the host does not include the bucket name, but
rather the bucket name is the first segment of the path.
For example __s3.&lt;region&gt;.amazonaws.com/bucket__
+ _Other_: It is possible to use other non-Amazon cloud storage, but
that is cloud library dependent.
- Query: currently not used.
- Fragment: the fragment is of the form _key=value&key=value&..._.
Depending on the key, the _=value_ part may be left out and some
default value will be used.
## Client Parameters
The fragment part of a URL is used to specify information
that is interpreted to specify what data format is to be used,
as well as additional controls for that data format.
For NCZarr support, the following _key=value_ pairs are allowd.
- mode=nczarr|zarr|s3|nz4|nzf... -- The mode key specifies
the particular format to be used by the netcdf-c library for
interpreting the dataset specified by the URL. Using _mode=nczarr_
causes the URL to be interpreted as a reference to a dataset
that is stored in NCZarr format. The modes _s3_, _nz4_, and _nzf_
tell the library what storage driver to use. The _s3_ is default]
and indicates using Amazon S3 or some equivalent. The other two,
_nz4_ and _nzf_ are again for testing. The _zarr_ mode tells the
library to use NCZarr, but to restrict its operation to operate on
pure Zarr Version 2 datasets.
- log=&lt;output-stream&gt;: this control turns on logging output,
which is useful for debugging and testing. If just _log_ is used
then it is equivalent to _log=stderr_.
# NCZarr Map Implementation {#nczarr_mapimpl}
Internally, the nczarr implementation has a map abstraction
that allows different storage formats to be used.
This is closely patterned on the same approach used in
the Python Zarr implementation, which relies on the Python
_MutableMap_
<a href="#ref_python">[3]</a> class.
In NCZarr, the corresponding type is called _zmap_.
The zmap model is a set of keys where each key maps to
an _object_ that can hold arbitrary data. The keys are assumed to
have following BNF grammar.
````
key: '/' segment
| key '/' segment
;
````
This key structure induces a tree structure where each segment
matches a node in the tree. This key/tree duality deliberately
matches that of a typical file system path in e.g. linux. The
key '/' is the root of the tree.
## Datasets
Within the key induced tree, each dataset (in the netCDF sense)
has a root which is specified by a specific key. All objects
making up the dataset (see the section on <a
href="#nczarr_purezarr">NCZarr vs Zarr </a>) reside in objects
(keys) below that dataset root key.
One restriction is that datasets cannot be nested in that
no dataset root key can be a prefix of another dataset root key.
## Zmap Implementatons
The primary zmap implementation is _s3_ (i.e. _mode=nczarr,s3_) and indicates
that the Amazon S3 cloud storage is to be used. Other storage formats
use a structured NetCDF-4 file format (_mode=nczarr,nz4_), or a
directory tree (_mode=nczarr,nzf_)
The latter two are used mostly for debugging and testing.
However, the _nzf_ format is important because it is intended
to match a corresponding storage format used by the Python
Zarr implementation. Hence it should serve to provide
interoperability between NCZarr and the Python Zarr.
# NCZarr versus Pure Zarr. {#nczarr_purezarr}
The NCZARR format extends the pure Zarr format by adding
extra objects such as _.nczarr_ and _.ncvar_. It is possible
to suppress the use of these extensions so that the netcdf
library can read and write a pure zarr formatted file.
This is controlled by using _mode=nczarr,zarr_ combination.
# Notes on Debugging NCZarr Access {#nczarr_debug}
The NCZarr support has a logging facility.
Turning on this logging can
sometimes give important information. Logging can be enabled by
using the client parameter "log" or "log=filename",or by
setting the environment variable NCLOGGING.
The first case will send log output to standard error and the
second will send log output to the specified file. The environment
variable is equivalent to _log_.
# Amazon S3 Storage {#nczarr_debug}
The Amazon AWS S3 storage driver currently uses the Amazon
AWS S3 Software Development Kit for C++ (aws-s3-sdk-cpp).
In order to use it, the client must provide some configuration
information. Specifically, the `~/.aws/config` file should
contain something like this.
```
[default]
output = json
aws_access_key_id=XXXX...
aws_secret_access_key=YYYY...
```
## Addressing Style
The notion of "addressing style" may need some expansion.
Amazon S3 accepts two forms for specifying the endpoint
for accessing the data.
1. Virtual -- the virtual addressing style places the bucket in
the host part of a URL. For example:
```
https://<bucketname>.s2.<region>.amazonaws.com/
```
1. Path -- the path addressing style places the bucket in
at the front of the path part of a URL. For example:
```
https://s2.<region>.amazonaws.com/<bucketname>/
```
The NCZarr code will accept either form, although internally,
it is standardized on path style. The reason for this
is that the bucket name forms the initial segment
in the keys.
# Zarr vs NCZarr {#nczarr_vs_zarr}
## Data Model
The NCZarr storage format is almost identical to that of
the the standard Zarr version 2 format.
The data model differs as follows.
1. Zarr supports filters -- NCZarr as yet does not
2. Zarr only supports anonymous dimensions -- NCZarr supports
only shared (named) dimensions.
3. Zarr attributes are untyped -- or perhaps more correctly
characterized as of type string.
## Storage Format
Consider both NCZarr and Zarr, and assume S3 notions of bucket
and object. In both systems, Groups and Variables (Array in Zarr)
map to S3 objects. Containment is modelled using the fact that
the container's key is a prefix of the variable's key.
So for example, if variable _v1_ is contained int top level group g1 -- _/g1 --
then the key for _v1_ is _/g1/v_.
Additional information is stored in special objects whose name
start with ".z".
In Zarr, the following special objects exist.
1. Information about a group is kept in a special object named
_.zgroup_; so for example the object _/g1/.zgroup_.
1. Information about an array is kept as a special object named _.zarray_;
so for example the object _/g1/v1/.zarray_.
1. Group-level attributes and variable-level attributes are stored
in a special object named _.zattr_;
so for example the objects _/g1/.zattr_ and _/g1/v1/.zattr.
The NCZarr format uses the same group and variable (array) objects
as Zarr. It also uses the Zarr special _.zXXX_ objects.
However, NCZarr adds some additional special objects.
1. _.nczarr_ -- this is in the top level group -- key _/.nczarr_.
It is in effect the "superblock" for the dataset and contains
any netcdf specific dataset level information. It is also used
to verify that a given key is the root of a dataset.
1. _.nczgroup_ -- this is a parallel object to _.zgroup_ and contains
any netcdf specific group information. Specifically it contains the following.
* dims -- the name and size of shared dimensions defined in this group.
* vars -- the name of variables defined in this group.
* groups -- the name of sub-groups defined in this group.
These lists allow walking the NCZarr dataset without having to use
the potentially costly S3 list operation.
1. _.nczvar_ -- this is a parallel object to _.zarray_ and contains
netcdf specific information. Specifically it contains the following.
* dimrefs -- the names of the shared dimensions referenced by the variable.
* storage -- indicates if the variable is chunked vs contiguous
in the netcdf sense.
1 _.nczattr_ -- this is parallel to the .zattr objects and stores
the attribute type information.
## Translation
With some constraints, it is possible for an nczarr library to read
Zarr and for a zarr library to read the nczarr format.
The latter case, zarr reading nczarr is possible if the zarr library
is willing to ignore objects whose name it does not recognized;
specifically anthing beginning with _.ncz_.
The former case, nczarr reading zarr is also
possible if the nczarr can simulate or infer the contents of
the missing _.nczXXX_ objects. As a rule this can be done as follows.
1. _.nczgroup_ -- The list of contained variables and sub-groups
can be computed using the S3 list operation to list the keys
"contained" in the key for a group. By looking for occurrences
of _.zgroup_, _.zattr_, _.zarray to infer the keys for the
contained groups, attribute sets, and arrays (variables).
Constructing the set of "shared dimensions" is carried out
by walking all the variables in the whole dataset and collecting
the set of unique integer shapes for the variables.
For each such dimension length, a top level dimension is created
named ".zdim<len>" where len is the integer length. The name
is subject to change.
1. _.nczvar_ -- The dimrefs are inferred by using the shape
in _.zarray_ and creating references to the simulated shared dimension.
netcdf specific information.
1. _.nczattr_ -- The type of each attribute is inferred by trying to parse the first attribute value string.
# Compatibility {#nczarr_compatibility}
In order to accomodate existing implementations, certain mode tags are
provided to tell the NCZarr code to look for information used
by specific implementations.
<!--
## XArray
The Xarray
<a href="#ref_xarray">[5]</a>
Zarr implementation uses its own mechanism for
specifying shared dimensions. It uses a special
attribute named ''_ARRAY_DIMENSIONS''.
The value of this attribute is a list of dimension names (strings), for example ````["time", "lon", "lat"]````.
If enabled and detected, then these dimension names are used
to define shared dimensions.
-->
# Examples {#nczarr_examples}
Here are a couple of examples using the _ncgen_ and _ncdump_ utilities.
1. Create an nczarr file using a local directory tree as storage.
```
ncgen -4 -lb -o "file:///home/user/dataset.nzf#mode=nczarr,nzf" dataset.cdl
```
1. Display the content of an nczarr file using a local directory tree as storage.
```
ncdump "file:///home/user/dataset.nzf#mode=nczarr,nzf"
```
1. Create an nczarr file using S3 as storage.
```
ncgen -4 -lb -o "s3://datasetbucket" dataset.cdl
```
1. Create an nczarr file using S3 as storage and keeping to the pure
zarr format.
```
ncgen -4 -lb -o "s3://datasetbucket#mode=zarr" dataset.cdl
```
# References {#nczarr_bib}
<a name="ref_aws">[1]</a> [Amazon Simple Storage Service Documentation](https://docs.aws.amazon.com/s3/index.html)<br>
<a name="ref_nczarr">[2]</a> [NetCDF ZARR Data Model Specification](https://www.unidata.ucar.edu/blogs/developer/en/entry/netcdf-zarr-data-model-specification)<br>
<a name="ref_python">[3]</a> [Python Documentation: 8.3. collections — High-performance container datatypes](https://docs.python.org/2/library/collections.html)<br>
<a name="ref_zarrv2">[4]</a> [Zarr Version 2 Specification](https://zarr.readthedocs.io/en/stable/spec/v2.html)<br>
<a name="ref_xarray">[5]</a> [XArray Zarr Encoding Specification](http://xarray.pydata.org/en/latest/internals.html#zarr-encoding-specification)<br>
# Appendix A. Building NCZarr Support {#nczarr_build}
Currently only the following build cases are supported.
Operating System | Supported Build Systems
------------------------------------------
Linux | Automake, CMake
OS-X | Automake, CMake
Visual Studio | N.A.
There are several options relevant to NCZarr support and to Amazon S3 support.
These are as follows.
1. _--enable-nczarr_ -- enable the NCZarr support. If disabled, then all of the following options are disabled or irrelevant.
2. &nbsp;&nbsp;_aws-c-common aws-cpp-sdk-s3_ and _aws-cpp-sdk-core_ -- if these libraries are available, then Amazon S3 support is enabled for NCZarr.
3. _--disable-s3-sdk_ -- even if the aws libraries are available, this option will forcibly disable Amazon S3 support for using them.
<!--
4. '--enable-xarray-dimension' -- this enables the xarray support described in the section on <a href="#nczarr_compatibility">compatibility</a>.
-->
The CMake equivalents are as follows:
* _--enable-nczarr_ => ENABLE_NCZARR=ON
* _--disable-s3-sdk_ => ENABLE_S3_SDK=OFF
If S3 support is desired, then LDFLAGS should be properly set, namely this.
````
LDFLAGS="$LDFLAGS -L/usr/local/lib -laws-cpp-sdk-s3 aws-cpp-sdk-core"
````
The above assumes that these libraries were installed in '/usr/local/lib', so the above
requires modification if they were installed elsewhere.
Note also that if S3 support is enabled, then you need to have a C++ compiler installed
because part of the S3 support code is written in C++.
## Testing S3 Support
Currently testing of S3 with NCzarr is supported only for
Unidata members of the NetCDF Development Group. This is because
it uses a specific bucket on a specific internal S3 appliance that
is inaccessible to the general user. This is controlled by
an environment variable via this shell command:
````
export NETCDF_S3_TESTS=1
````
If that environment variable is set and the aws libraries are
available, and --enable-s3-sdk is set, then the S3 tests will be
attempted. If someone else wants to attempt these tests, then
they will need to modify the tests
_nczarr_test/run_ut_mapapi.sh_ and _nczarr_test/run_it_test2.sh_.
# Appendix B. Building aws-sdk-cpp {#nczarr_s3sdk}
In order to use the S3 storage driver, it is necessary to
install the Amazon [aws-sdk-cpp library](https://github.com/aws/aws-sdk-cpp.git).
As a starting point, here are the CMake options used by Unidata
to build that library. It assumes that it is being executed
in a build directory, `build` say, and that `build/../CMakeLists.txt exists`.
```
cmake -DFORCE_CURL=ON -DBUILD_ONLY=s3 -DMINIMIZE_SIZE=ON -DBUILD_DEPS=OFF -DCMAKE_CXX_STANDARD=14 ..
```
The expected set of installed libraries are as follows:
* aws-cpp-sdk-s3
* aws-cpp-sdk-core
# Appendix C. Amazon S3 Imposed Limits {#nczarr_s3limits}
The Amazon S3 cloud storage imposes some significant limits
that are inherited by NCZarr (and Zarr also, for that matter).
Some of the relevant limits are as follows:
1. The maximum object size is 5 Gigabytes with a total
for all objects limited to 5 Terabytes.
2. S3 key names can be any UNICODE name with a maximum length of 1024
bytes. Note that the limit is defined in terms of bytes and not (Unicode) characters. This affects the depth to which groups can be nested because the key encodes the full path name of a group.
# __Point of Contact__ {#nczarr_poc}
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu<br>
__Initial Version__: 4/10/2020<br>
__Last Revised__: 6/8/2020

1
RELEASE_NOTES.md
View File

@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.0 - TBD
* [Enhancement] Add support for cloud storage using a variant of the Zarr storage format. Warning: this feature is highly experimental and is subject to rapid evolution [https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in].
* [Bug Fix] Fix nccopy to properly set default chunking parameters when not otherwise specified. This can significantly improve performance in selected cases. Note that if seeing slow performance with nccopy, then, as a work-around, specifically set the chunking parameters. [https://github.com/Unidata/netcdf-c/issues/1763].
* [Bug Fix] Fix some protocol bugs/differences between the netcdf-c library and the OPeNDAP Hyrax server. Also cleanup checksum handling [https://github.com/Unidata/netcdf-c/issues/1712].
* [Bug Fix] Add necessary __declspec declarations to allow compilation

12
config.h.cmake.in
View File

@ -127,6 +127,9 @@ are set when opening a binary file on Windows. */
/* if true, enable CDF5 Support */
#cmakedefine ENABLE_CDF5 1
/* if true, enable client side filters */
#cmakedefine ENABLE_CLIENT_FILTERS 1
/* if true, enable strict null byte header padding. */
#cmakedefine USE_STRICT_NULL_BYTE_HEADER_PADDING 1
@ -136,12 +139,12 @@ are set when opening a binary file on Windows. */
/* if true, build DAP4 Client */
#cmakedefine ENABLE_DAP4 1
/* if true, build DAP4 Client */
#cmakedefine ENABLE_DAP4 1
/* if true, do remote tests */
#cmakedefine ENABLE_DAP_REMOTE_TESTS 1
/* if true, enable NCZARR */
#cmakedefine ENABLE_NCZARR 1
/* define the possible sources for remote test servers */
#cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}"
@ -197,6 +200,9 @@ are set when opening a binary file on Windows. */
*/
#cmakedefine HAVE_DECL_ISNAN 1
/* Define to 1 if you have the <dirent.h> header file. */
#cmakedefine HAVE_DIRENT_H 1
/* Define to 1 if you have the <fcntl.h> header file. */
#cmakedefine HAVE_FCNTL_H 1

123
configure.ac
View File

@ -184,12 +184,23 @@ if test "x$enable_jna" = xyes ; then
AC_DEFINE([JNA], [1], [if true, include jna bug workaround code])
fi
# Should tests with unfixed memory leaks be run?
AC_MSG_CHECKING([if unfixed tests with memory leaks should be enabled])
AC_ARG_ENABLE([unfixed-memory-leaks],
[AS_HELP_STRING([--enable-unfixed-memory-leaks],
[Enable tests in the ncdump directory that have unfixed memory leaks])])
test "x$enable_unfixed_memory_leaks" = xyes || enable_unfixed_memory_leaks=no
AC_MSG_RESULT($enable_unfixed_memory_leaks)
AM_CONDITIONAL([ENABLE_UNFIXED_MEMORY_LEAKS], [test "x$enable_unfixed_memory_leaks" = xyes])
# Does the user want to turn off unit tests (useful for test coverage
# analysis).
AC_MSG_CHECKING([if unit tests should be disabled])
AC_ARG_ENABLE([unit-tests],
[AS_HELP_STRING([--disable-unit-tests],
[Disable tests in unit_test directory. Other tests still run.])])
test "x$enable_unit_tests" = xno || enable_unit_tests=yes
AC_MSG_RESULT($enable_unit_tests)
AM_CONDITIONAL([BUILD_UNIT_TESTS], [test "x$enable_unit_tests" = xyes])
# Does the user want to build netcdf-4?
@ -429,6 +440,61 @@ AC_MSG_RESULT([$enable_dap_long_tests])
AM_CONDITIONAL(INTERNAL_OCLIB,[test "x" = "x"])
# Control zarr storage
AC_MSG_CHECKING([whether netcdf zarr storage format should be enabled])
AC_ARG_ENABLE([nczarr],
[AS_HELP_STRING([--enable-nczarr],
[enable netcdf zarr storage support; requires netCDF-4 and libcurl])])
test "x$enable_nczarr" = xyes || enable_nczarr=no
AC_MSG_RESULT($enable_nczarr)
if test "x$enable_nczarr" = xyes ; then
if test "x$enable_netcdf_4" = xno ; then
AC_MSG_ERROR([netCDF-4 disabled, so you must not enable nczarr])
enable_nczarr=no
fi
fi
if test "x$enable_nczarr" = xyes; then
AC_DEFINE([ENABLE_NCZARR], [1], [if true, build NCZarr Client])
AC_SUBST(ENABLE_NCZARR)
fi
AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
# Check for the AWS S3 SDK library
AC_LANG_PUSH([C++])
AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
AC_LANG_POP
AC_MSG_CHECKING([whether AWS S3 SDK library is available])
AC_MSG_RESULT([$have_aws])
# Check for forced disabling of S3 support
AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled])
AC_ARG_ENABLE([s3-sdk],
[AS_HELP_STRING([--disable-s3-sdk],
[forcibly disable usage of netcdf zarr S3 sdk])])
test "x$enable_s3_sdk" = xno || enable_s3_sdk=yes
AC_MSG_RESULT($enable_s3_sdk)
if test "x$have_aws" = xno -a "x$enable_s3_sdk" = xno; then
have_aws=no
enable_s3_sdk=no
elif test "x$have_aws" = xno -a "x$enable_s3_sdk" = xyes; then
AC_MSG_WARN([aws libraries not found, so default to --disable-s3-sdk])
have_aws=no
enable_s3_sdk=no
elif test "x$have_aws" = xyes -a "x$enable_s3_sdk" = xno; then
AC_MSG_WARN([--disable-s3-sdk overrides presence of aws s3 libraries])
have_aws=no
enable_s3_sdk=no
elif test "x$have_aws" = xyes -a "x$enable_s3_sdk" = xyes; then
have_aws=yes
enable_s3_sdk=yes
fi
if test "x$enable_s3_sdk" = xyes ; then
AC_DEFINE([ENABLE_S3_SDK], [1], [if true, build libnczarr with S3 support enabled])
fi
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
# Check whether we want to enable strict null byte header padding.
# See https://github.com/Unidata/netcdf-c/issues/657 for more information.
AC_MSG_CHECKING([whether to enable strict null-byte header padding when reading (default off)])
@ -671,6 +737,7 @@ fi
AC_PROG_CC
AC_PROG_CXX
AM_PROG_CC_C_O
AC_C_CONST
@ -831,7 +898,7 @@ AC_CHECK_HEADERS([sys/param.h])
AC_CHECK_HEADERS([libgen.h])
#AC_CHECK_HEADERS([locale.h])
AC_HEADER_STDC
AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h])
AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h dirent.h])
# Do sys/resource.h separately
#AC_CHECK_HEADERS([sys/resource.h],[havesysresource=1],[havesysresource=0])
@ -842,6 +909,9 @@ AC_CHECK_HEADERS([sys/resource.h])
# See if we have ftw.h to walk directory trees
AC_CHECK_HEADERS([ftw.h])
# See if we can do stack tracing programmatically
AC_CHECK_HEADERS([execinfo.h])
# Check for these functions...
AC_CHECK_FUNCS([strlcat snprintf strcasecmp fileno \
strdup strtoll strtoull \
@ -923,7 +993,7 @@ AC_FUNC_ALLOCA
AC_CHECK_DECLS([isnan, isinf, isfinite],,,[#include <math.h>])
AC_STRUCT_ST_BLKSIZE
UD_CHECK_IEEE
AC_CHECK_TYPES([size_t, ssize_t, schar, uchar, longlong, ushort, uint, int64, uint64])
AC_CHECK_TYPES([size_t, ssize_t, schar, uchar, longlong, ushort, uint, int64, uint64, size64_t, ssize64_t, _off64_t])
AC_TYPE_OFF_T
AC_TYPE_UINTPTR_T
AC_C_CHAR_UNSIGNED
@ -1315,6 +1385,7 @@ AM_CONDITIONAL(ENABLE_METADATA_PERF, [test x$enable_metadata_perf = xyes])
AM_CONDITIONAL(ENABLE_BYTERANGE, [test "x$enable_byterange" = xyes])
AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes])
AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
# If the machine doesn't have a long long, and we want netCDF-4, then
# we've got problems!
@ -1389,19 +1460,6 @@ esac
NC_FLIBS="-lnetcdff $NC_LIBS"
# temporary to deal with a JNA problem
AC_MSG_CHECKING([If compilation is for use with JNA])
AC_ARG_ENABLE([jna],
[AS_HELP_STRING([--enable-jna],
[enable jna bug fix])],
[],
[enable_jna=no])
test "x$enable_jna" = xno || enable_jna=yes
AC_MSG_RESULT($enable_jna)
if test "x$enable_jna" = xyes ; then
AC_DEFINE([JNA], [1], [if true, include JNA bug fix])
fi
# Control filter test/example
AC_MSG_CHECKING([whether filter testing should be run])
AC_ARG_ENABLE([filter-testing],
@ -1410,9 +1468,35 @@ AC_ARG_ENABLE([filter-testing],
test "x$enable_filter_testing" = xno || enable_filter_testing=yes
AC_MSG_RESULT($enable_filter_testing)
# Control client filters
AC_MSG_CHECKING([whether client filters should be enabled])
AC_ARG_ENABLE([client-filters],
[AS_HELP_STRING([--enable-clientfilters],
[Enable client side filters; default off])])
test "x$enable_client_filters" = xyes || enable_client_filters=no
AC_MSG_RESULT($enable_client_filters)
if test "x$enable_netcdf_4" = xno ; then
AC_MSG_WARN([netCDF-4 disabled => --disable-filter-testing])
AC_MSG_WARN([netCDF-4 disabled => --disable-filter-testing && --disable-client-filters])
enable_filter_testing=no
enable_client_filters=no
fi
if test "x$enable_hdf5" = xno ; then
AC_MSG_WARN([HDF5 disabled => --disable-filter-testing])
enable_filter_testing=no
fi
if test "x$enable_shared" = xno ; then
AC_MSG_WARN([Shared libraries are disabled => --disable-filter-testing])
enable_filter_testing=no
fi
AM_CONDITIONAL(ENABLE_CLIENT_FILTERS, [test x$enable_client_filters = xyes])
if test "x$enable_netcdf_4" = xno ; then
AC_MSG_WARN([netCDF-4 disabled => --disable-filter-testing && --disable-client-filters])
enable_filter_testing=no
enable_client_filters=no
fi
if test "x$enable_hdf5" = xno ; then
@ -1462,6 +1546,8 @@ AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill])
AC_SUBST(HAS_BYTERANGE,[$enable_byterange])
AC_SUBST(RELAX_COORD_BOUND,[yes])
AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters])
AC_SUBST(HAS_NCZARR,[$enable_nczarr])
AC_SUBST(HAS_S3_SDK,[$enable_s3_sdk])
# Include some specifics for netcdf on windows.
#AH_VERBATIM([_WIN32_STRICMP],
@ -1532,7 +1618,10 @@ AX_SET_META([NC_HAS_CDF5],[$enable_cdf5],[yes])
AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes])
AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes])
AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes])
AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
AX_SET_META([NC_HAS_NCZARR_S3],[$enable_s3_sdk],[yes])
AC_SUBST([NC_DISPATCH_VERSION], [2])
#####
# End netcdf_meta.h definitions.
#####
@ -1578,6 +1667,7 @@ AC_CONFIG_FILES([Makefile
libdap2/Makefile
libdap4/Makefile
libhdf4/Makefile
libnczarr/Makefile
libdispatch/Makefile
liblib/Makefile
ncdump/cdl/Makefile
@ -1595,6 +1685,7 @@ AC_CONFIG_FILES([Makefile
ncdap_test/expectremote3/Makefile
dap4_test/Makefile
plugins/Makefile
nczarr_test/Makefile
])
AC_OUTPUT()

2
dap4_test/Makefile.am
View File

@ -14,7 +14,7 @@ include $(top_srcdir)/lib_flags.am
#TESTS_ENVIRONMENT = export SETX=1;
LDADD = ${top_builddir}/liblib/libnetcdf.la
AM_CPPFLAGS += -I$(top_srcdir)/libdap4 -I${topdir}/include
AM_CPPFLAGS += -I$(top_srcdir)/libdap4
# Set up the tests; do the .sh first, then .c
check_PROGRAMS =

2
dap4_test/test_raw.sh
View File

@ -1,7 +1,5 @@
#!/bin/sh
export SETX=1
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh

61
debug/cf
View File

@ -1,24 +1,27 @@
#!/bin/bash
#NB=1
#DB=1
#X=-x
#ANSI=1
#MEM=1 # export NC_VLEN_NOTEST=1
MEM=1
#NOTUIL=1
#FAST=1
#PROF=1
FAST=1
PROF=1
#NCZARR=1
NCZARR=1
HDF5=1
DAP=1
#SZIP=1
#HDF4=1
#S3=1
SZIP=1
HDF4=1
#PNETCDF=1
#PAR4=1
#TESTSERVERS="149.165.169.123:8080,remotetest.unidata.ucar.edu"
#TESTSERVERS="localhost:8080,149.165.169.123:8080"
#TESTSERVERS="149.165.169.123:8080"
#TESTSERVERS="remotetest.unidata.ucar.edu"
if test $# != 0 ; then
cmds=$@
@ -31,14 +34,14 @@ fi
CFLAGS=""
#CFLAGS="-Wall -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unused-parameter -Wconversion ${CFLAGS}"
#CFLAGS="-Wall -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unused-parameter -Wno-char-subscripts -Wno-pointer-sign -Wno-format ${CFLAGS}"
CFLAGS="-Wall -Wno-unused-parameter -Wno-char-subscripts -Wno-pointer-sign ${CFLAGS}"
#CFLAGS="-Wall ${CFLAGS}"
#CFLAGS="$CFLAGS -Wconversion"
#CFLAGS="-Wall -Wunused-parameter -Wno-char-subscripts -Wno-pointer-sign ${CFLAGS}"
CFLAGS="-Wall -Wno-unused-parameter -Wno-char-subscripts -Wno-pointer-sign ${CFLAGS}"
if test "x$MEM" = x1 ; then
export NC_VLEN_NOTEST=1
CFLAGS="-fsanitize=address ${CFLAGS}"
CFLAGS="-fsanitize=address ${CFLAGS}"
export export NC_VLEN_NOTEST=1
fi
if test "x$ANSI" = x1 ; then
@ -75,7 +78,8 @@ MALLOC_CHECK=""
CPPFLAGS=""
LDFLAGS=""
CFLAGS="-g -O0 $CFLAGS -Wno-undefined"
#CFLAGS="-g -O0 $CFLAGS -Wno-undefined"
CFLAGS="-g -O0 $CFLAGS"
MAKE=make
IGNORE="test 0 = 1"
@ -94,22 +98,35 @@ if test "x$SZIP" = "x1" ; then
LDFLAGS="$LDFLAGS -lsz -laec"
fi
FLAGS="--prefix ${PREFIX}"
if test "x$NCZARR" = "x1" ; then
FLAGS="$FLAGS --enable-nczarr"
if test "x$S3" = x1 ; then
CPPFLAGS="-I${stddir}/include $CPPFLAGS "
#CXXFLAGS="-std=c++11 $CXXFLAGS"
LDFLAGS="$LDFLAGS -L${stddir}/lib -laws-cpp-sdk-s3"
LDFLAGS="$LDFLAGS -laws-cpp-sdk-core"
#LDFLAGS="$LDFLAGS -laws-c-common"
#LDFLAGS="$LDFLAGS -laws-c-event-stream"
#LDFLAGS="$LDFLAGS -laws-checksums"
#LDFLAGS="$LDFLAGS -lstdc++"
else
FLAGS="$FLAGS --disable-s3-sdk"
fi
fi
export PKG_CONFIG_PATH=/usr/lib/pkgconfig
if curl-config --version >/dev/null ; then
TMP=`curl-config --cflags`
CPPFLAGS="$TMP $CPPFLAGS"
WH=`curl-config --prefix`
WH="${WH}/lib"
TMP=`curl-config --libs`
LDFLAGS="$LDFLAGS -L$WH $TMP"
LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$WH"
LDFLAGS="$LDFLAGS $TMP"
#LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$WH"
else
LDFLAGS="${LDFLAGS} -lcurl"
fi
CXXFLAGS="$CPPFLAGS $CXXFLAGS"
FLAGS="--prefix ${PREFIX}"
#FLAGS="$FLAGS --disable-cxx"
#FLAGS="$FLAGS --disable-examples"
#FLAGS="$FLAGS --enable-cxx-4"
@ -134,9 +151,9 @@ FLAGS="$FLAGS --enable-byterange"
#FLAGS="$FLAGS --disable-silent-rules"
#FLAGS="$FLAGS --disable-filter-testing"
#FLAGS="$FLAGS --enable-metadata-perf"
#FLAGS="$FLAGS --enable-extra-tests"
#FLAGS="$FLAGS --with-ncproperties-extra=key1=value1,key2=value2"
#FLAGS="$FLAGS --enable-valgrind"
#FLAGS="$FLAGS --enable-unfixed-memory-leaks"
if test "x$TESTSERVERS" != x ; then
FLAGS="$FLAGS --with-testservers=$TESTSERVERS"
@ -146,10 +163,6 @@ if test "x$PAR4" != x1 ; then
FLAGS="$FLAGS --disable-parallel4"
fi
if test "x$NCZARR" = x1 ; then
FLAGS="$FLAGS --enable-nczarr"
fi
if test "x$NOUTIL" = x1 ; then
FLAGS="$FLAGS --disable-utilities"
fi
@ -209,6 +222,8 @@ CFLAGS="${CFLAGS} -pg"
LDFLAGS="${LDFLAGS} -pg"
fi
CXXFLAGS="$CFLAGS $CXXFLAGS"
export PATH
export CC
export CPPFLAGS

45
debug/cf.cmake
View File

@ -1,16 +1,18 @@
# Visual Studio
NCC="c:/tools/hdf5"
# Is netcdf-4 and/or DAP enabled?
NCZARR=1
NC4=1
DAP=1
#CDF5=1
#HDF4=1
HDF4=1
#S3=1
#TR=--trace
NCC="c:/tools/nccmake"
export SETX=1
#export SETX=1
for arg in "$@" ; do
case "$arg" in
@ -42,23 +44,32 @@ fi
FLAGS=
FLAGS="$FLAGS -DNC_FIND_SHARED_LIBS=ON"
if test "x$VS" != x -a "x$INSTALL" != x ; then
<<<<<<< HEAD
FLAGS="-DCMAKE_PREFIX_PATH=${NCC}"
=======
FLAGS="$FLAGS -DCMAKE_PREFIX_PATH=${NCC}"
>>>>>>> master
fi
FLAGS="$FLAGS -DCMAKE_INSTALL_PREFIX=/tmp/netcdf"
mkdir -p /tmp/netcdf
if test "x$NCZARR" != x ; then
FLAGS="$FLAGS -DENABLE_NCZARR=true"
fi
if test "x$DAP" = x ; then
FLAGS="$FLAGS -DENABLE_DAP=false"
fi
if test "x$NC4" = x ; then
FLAGS="$FLAGS -DENABLE_NETCDF_4=false"
else
ignore=1
#FLAGS="-DHDF5_C_LIBRARY=${NCC}/lib/hdf5 -DHDF5_HL_LIBRARY=${NCC}/lib/hdf5_hl -DHDF5_INCLUDE_DIR=${NCC}/include"
#FLAGS="$FLAGS -DDEFAULT_API_VERSION:STRING=v110"
#FLAGS="$FLAGS -DHDF5_ROOT=c:/tools/hdf5"
#FLAGS="$FLAGS -DHDF5_ROOT_DIR_HINT=c:/tools/hdf5/cmake/hdf5/hdf5-config.cmake"
FLAGS="$FLAGS -DHDF5_DIR=c:/tools/hdf5/cmake/hdf5"
#hdf5-config.cmake
#FLAGS="-DHDF5_LIBRARIES=${NCC}/lib/hdf5 -DHDF5_HL_LIBRARY=${NCC}/lib/hdf5_hl -DHDF5_INCLUDE_DIR=${NCC}/include"
fi
if test "x$CDF5" != x ; then
FLAGS="$FLAGS -DENABLE_CDF5=true"
@ -76,7 +87,11 @@ FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true"
FLAGS="$FLAGS -DENABLE_LOGGING=true"
#FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true"
#FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true"
#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
if test "x$S3" = x ; then
FLAGS="$FLAGS -DENABLE_S3_SDK=false"
fi
# Disables
FLAGS="$FLAGS -DENABLE_EXAMPLES=false"
@ -85,6 +100,8 @@ FLAGS="$FLAGS -DENABLE_CONVERSION_WARNINGS=false"
#FLAGS="$FLAGS -DENABLE_DISKLESS=false"
FLAGS="$FLAGS -DBUILD_UTILITIES=true"
#FLAGS="$FLAGS -DCURL_NO_CURL_CMAKE=TRUE"
# Withs
FLAGS="$FLAGS -DNCPROPERTIES_EXTRA=\"key1=value1|key2=value2\""
@ -101,10 +118,12 @@ CFG="Release"
NCLIB="${NCLIB}/liblib"
export PATH="${NCLIB}:${PATH}"
#G=
#TR=--trace
cmake ${TR} "$G" -DCMAKE_BUILD_TYPE=${CFG} $FLAGS ..
if test "x$NOBUILD" = x ; then
cmake ${TR} --build . --config ${CFG}
#cmake ${TR} --build . --config ${CFG} --target ZERO_CHECK
cmake ${TR} --build . --config ${CFG} --target ALL_BUILD
#cmake ${TR} --build . --config ${CFG} --target ALL_BUILD
if test "x$NOTEST" = x ; then
cmake ${TR} --build . --config ${CFG} --target RUN_TESTS
fi
@ -116,7 +135,9 @@ NCLIB="${NCLIB}/build/liblib"
#T="--trace-expand"
cmake "${G}" $FLAGS ..
if test "x$NOBUILD" = x ; then
make all
make VERBOSE=1 all
fi
if test "x$NOTEST" = x ; then
make test
fi
fi

54
debug/libnczarr/Make0 Normal file
View File

@ -0,0 +1,54 @@
# Test c output
T=ut_projections
XSRC=ut_util.c ../nczarr_test/ztest_util.c
#ARGS=-r 1 -d 10 -c 5 -s '[0:15:2]'
#ARGS=-r 1 -d 13 -c 5 -s '[0:11:2]'
#ARGS=-r 2 -d 13,10 -c 5,3 -s '[0:11:2][1:9]'
ARGS=-r 1 -d 10 -c 3 -s '[1:9]'
#CMD=valgrind --leak-check=full
CMD=gdb --args
#PAR=1
#SZIP=1
#CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include -I../libnczarr
CFLAGS = -Wall -g -O0 -I.. -I../include -I../libnczarr
LDFLAGS = ../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm
ifdef PAR
CC=mpicc
LDFLAGS += -lmpich
else
CC=gcc
endif
ifdef SZIP
LDFLAGS += -lsz -laec
endif
LLP=/usr/local/lib:${LD_LIBRARY_PATH}
all:: cmp
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
${CMD} ./t ${ARGS}
cmp::
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
${CC} -o t ${CFLAGS} ${T}.c ${XSRC} ${FILTEROBJ} ${LDFLAGS}
filter::
${CC} ${CFLAGS} -c hdf5plugins/${FILTER}.c ${LDFLAGS}
cpp::
${CC} -E ${CFLAGS} ${T}.c > ${T}.txt
H5=h5testszip
EXT=testszip.nc
h5::
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
${CC} -o h5 ${CFLAGS} ${H5}.c ${SRC} ${LDFLAGS}; \
${CMD} ./h5 ${EXT}

2
docs/file_format_specifications.md
View File

@ -70,7 +70,7 @@ the intent of elements of the format.
// special2 chars are recently permitted in
// names (and require escaping in CDL).
// Note: '/' is not permitted.
special2 = ' ' | '!' | '"' | '#' | '$' | '%' | '&' | '\'' |
special2 = ' ' | '!' | '"' | '#' | '$' | '\%' | '&' | '\'' |
'(' | ')' | '*' | ',' | ':' | ';' | '<' | '=' |
'>' | '?' | '[' | '\\' | ']' | '^' | '`' | '{' |
'|' | '}' | '~'

8
examples/C/Makefile.am
View File

@ -9,10 +9,12 @@
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
LDADD = -lm
AM_CPPFLAGS = -I$(top_srcdir)/include
# Put together AM_CPPFLAGS and AM_LDFLAGS.
include $(top_srcdir)/lib_flags.am
AM_CPPFLAGS += -I$(top_srcdir)/include
AM_CPPFLAGS += -I$(top_builddir)/liblib
AM_LDFLAGS = ${top_builddir}/liblib/libnetcdf.la
AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la -lm
# These are the netCDF-3 examples.
check_PROGRAMS = simple_xy_wr simple_xy_rd sfc_pres_temp_wr \

2
include/Makefile.am
View File

@ -19,7 +19,7 @@ ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h \
nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \
ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \
ncwinpath.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \
hdf5dispatch.h ncmodel.h
hdf5dispatch.h ncmodel.h ncfilter.h
if USE_DAP
noinst_HEADERS += ncdap.h

26
include/hdf5internal.h
View File

@ -30,8 +30,10 @@
#define NC_EMPTY_SCALE "NC_EMPTY_SCALE"
/* This is an attribute I had to add to handle multidimensional
* coordinate variables. */
#define COORDINATES "_Netcdf4Coordinates"
* coordinate variables. See nc4internal.h:NC_ATT_COORDINATES.
* in nc4internal.h.
*/
#define COORDINATES NC_ATT_COORDINATES
#define COORDINATES_LEN (NC_MAX_NAME * 5)
/* This is used when the user defines a non-coordinate variable with
@ -40,17 +42,17 @@
/* An attribute in the HDF5 root group of this name means that the
* file must follow strict netCDF classic format rules. */
#define NC3_STRICT_ATT_NAME "_nc3_strict"
#define NC3_STRICT_ATT_NAME NC_ATT_NC3_STRICT_NAME
/* If this attribute is present on a dimscale variable, use the value
* as the netCDF dimid. */
#define NC_DIMID_ATT_NAME "_Netcdf4Dimid"
#define NC_DIMID_ATT_NAME NC_ATT_DIMID_NAME /*See nc4internal.h*/
/** This is the name of the class HDF5 dimension scale attribute. */
#define HDF5_DIMSCALE_CLASS_ATT_NAME "CLASS"
#define HDF5_DIMSCALE_CLASS_ATT_NAME NC_ATT_CLASS /*See nc4internal.h*/
/** This is the name of the name HDF5 dimension scale attribute. */
#define HDF5_DIMSCALE_NAME_ATT_NAME "NAME"
#define HDF5_DIMSCALE_NAME_ATT_NAME NC_ATT_NAME
/** Define Filter API Operations */
#define FILTER_REG 1
@ -159,9 +161,6 @@ extern int nc4_create_dim_wo_var(NC_DIM_INFO_T *dim);
* name, but the var is not a coord var of that dim. */
extern int nc4_give_var_secret_name(NC_VAR_INFO_T *var);
/* Get the fill value for a var. */
int nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp);
/* Find file, group, var, and att info, doing lazy reads if needed. */
int nc4_hdf5_find_grp_var_att(int ncid, int varid, const char *name, int attnum,
int use_name, char *norm_name, NC_FILE_INFO_T **h5,
@ -181,10 +180,15 @@ int nc4_get_var_meta(NC_VAR_INFO_T *var);
int nc4_hdf5_get_chunk_cache(int ncid, size_t *sizep, size_t *nelemsp,
float *preemptionp);
#ifdef ENABLE_CLIENTSIDE_FILTERS
/* Define Filter API Function */
int nc4_global_filter_action(int action, unsigned int id, struct NC_FILTER_OBJ_HDF5* infop);
int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* params, struct NC_FILTER_SPEC_HDF5**);
int NC4_hdf5_remove_filter(NC_VAR_INFO_T* var, unsigned int filterid);
#endif
/* Use this in dispatch table */
int NC4_hdf5_remove_filter(NC_VAR_INFO_T* var, const char* filterid);
/*Internal */
int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* params);
/* Support functions for provenance info (defined in nc4hdf.c) */
extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hdf.c*/

2
include/nc4dispatch.h
View File

@ -256,7 +256,7 @@ extern "C" {
NC4_show_metadata(int);
EXTERNL int
NC4_filter_actions(int, int, int, struct NC_Filterobject*);
NC4_filter_actions(int, int, int, void*);
#if defined(__cplusplus)
}

106
include/nc4internal.h
View File

@ -111,21 +111,13 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP, NCFIL} NC_SORT;
/** Subset of readonly flags; Value is actually in file. */
#define MATERIALIZEDFLAG 8
/* Generic reserved Attributes */
#define NC_ATT_REFERENCE_LIST "REFERENCE_LIST" /**< HDF5 reference list attribute name. */
#define NC_ATT_CLASS "CLASS" /**< HDF5 class atttribute name. */
#define NC_ATT_DIMENSION_LIST "DIMENSION_LIST" /**< HDF5 dimension list attribute name. */
#define NC_ATT_NAME "NAME" /**< HDF5 name atttribute name. */
#define NC_ATT_COORDINATES COORDINATES /**< Coordinates atttribute name. */
#define NC_ATT_FORMAT "_Format" /**< Format atttribute name. */
/** Boolean type, to make the code easier to read. */
typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t;
/* Forward declarations. */
struct NC_GRP_INFO;
struct NC_TYPE_INFO;
struct NC_FIlterobject;
struct NC_Filterobject;
/**
* This struct provides indexed Access to Meta-data objects. See the
@ -208,6 +200,7 @@ typedef struct NC_VAR_INFO
void *fill_value; /**< Pointer to fill value, or NULL. */
size_t *chunksizes; /**< For chunked storage, an array (size ndims) of chunksizes. */
int storage; /**< Storage of this var, compact, contiguous, or chunked. */
int endianness; /**< What endianness for the var? */
int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */
nc_bool_t shuffle; /**< True if var has shuffle filter applied. */
nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */
@ -215,7 +208,7 @@ typedef struct NC_VAR_INFO
size_t chunk_cache_nelems; /**< Number of slots in var chunk cache. */
float chunk_cache_preemption; /**< Chunk cache preemtion policy. */
void *format_var_info; /**< Pointer to any binary format info. */
NClist* filters; /**< List of filters to be applied to var data. */
NClist* filters; /**< List<NC_FILTERX_SPEC> of filters to be applied to var data; technically format dependent */
} NC_VAR_INFO_T;
/** This is a struct to handle the field metadata from a user-defined
@ -284,15 +277,17 @@ typedef struct NC_GRP_INFO
/* These constants apply to the cmode parameter in the
* HDF5_FILE_INFO_T defined below. */
#define NC_CREAT 2 /**< in create phase, cleared by ncendef */
#define NC_INDEF 8 /**< in define mode, cleared by ncendef */
#define NC_NSYNC 0x10 /**< synchronise numrecs on change */
#define NC_HSYNC 0x20 /**< synchronise whole header on change */
#define NC_NDIRTY 0x40 /**< numrecs has changed */
#define NC_HDIRTY 0x80 /**< header info has changed */
/* Make sure they do not conflict with defined flags in netcdf.h */
#define NC_CREAT 0x10002 /**< in create phase, cleared by ncendef */
#define NC_INDEF 0x10008 /**< in define mode, cleared by ncendef */
#define NC_NSYNC 0x10010 /**< synchronise numrecs on change */
#define NC_HSYNC 0x10020 /**< synchronise whole header on change */
#define NC_NDIRTY 0x10040 /**< numrecs has changed */
#define NC_HDIRTY 0x10080 /**< header info has changed */
/** This is the metadata we need to keep track of for each
* netcdf-4/HDF5 file. */
* netcdf-4/HDF5 file. */
typedef struct NC_FILE_INFO
{
NC_OBJ hdr;
@ -342,8 +337,9 @@ typedef struct
void *p; /**< Pointer to VL data */
} nc_hvl_t;
/** The names of the atomic data types. */
extern const char *nc4_atomic_name[NC_MAX_ATOMIC_TYPE + 1];
/* Misc functions */
int NC4_inq_atomic_type(nc_type typeid1, char *name, size_t *size);
int NC4_lookup_atomic_type(const char *name, nc_type* idp, size_t *sizep);
/* These functions convert between netcdf and HDF5 types. */
int nc4_get_typelen_mem(NC_FILE_INFO_T *h5, nc_type xtype, size_t *len);
@ -423,8 +419,11 @@ int nc4_check_name(const char *name, char *norm_name);
int nc4_normalize_name(const char *name, char *norm_name);
int nc4_check_dup_name(NC_GRP_INFO_T *grp, char *norm_name);
/* Get the fill value for a var. */
int nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp);
/* Find default fill value. */
int nc4_get_default_fill_value(const NC_TYPE_INFO_T *type_info, void *fill_value);
int nc4_get_default_fill_value(nc_type typecode, void *fill_value);
/* Get an att given pointers to file, group, and perhaps ver info. */
int nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var,
@ -445,60 +444,19 @@ extern void nc4_hdf5_finalize(void);
int log_metadata_nc(NC_FILE_INFO_T *h5);
#endif
/** @internal Names of atomic types. */
extern const char* nc4_atomic_name[NUM_ATOMIC_TYPES];
/* Binary searcher for reserved attributes */
extern const NC_reservedatt *NC_findreserved(const char *name);
/**************************************************/
/* Internal filter related structures */
/* Internal filter actions */
#define NCFILTER_DEF 1
#define NCFILTER_REMOVE 2
#define NCFILTER_INQ 3
#define NCFILTER_FILTERIDS 4
#define NCFILTER_INFO 5
#define NCFILTER_FREESPEC 6
#define NCFILTER_CLIENT_REG 10
#define NCFILTER_CLIENT_UNREG 11
#define NCFILTER_CLIENT_INQ 12
typedef enum NC_FILTER_SORT {
NC_FILTER_SORT_SPEC=((int)1),
NC_FILTER_SORT_IDS=((int)2),
NC_FILTER_SORT_CLIENT=((int)3),
} NC_FILTER_SORT;
/* Provide structs to pass args to filter_actions function for HDF5*/
typedef struct NC_FILTER_SPEC_HDF5 {
int active; /**< true iff HDF5 library was told to activate filter */
unsigned int filterid; /**< ID for arbitrary filter. */
size_t nparams; /**< nparams for arbitrary filter. */
unsigned int* params; /**< Params for arbitrary filter. */
} NC_FILTER_SPEC_HDF5;
typedef struct NC_FILTERIDS_HDF5 {
size_t nfilters; /**< number of filters */
unsigned int* filterids; /**< Filter ids. */
} NC_FILTERIDS_HDF5;
typedef struct NC_FILTER_CLIENT_HDF5 {
unsigned int id;
/* The filter info for hdf5 */
/* Avoid needing hdf.h by using void* */
void* info;
} NC_FILTER_CLIENT_HDF5;
typedef struct NC_FILTER_OBJ_HDF5 {
NC_Filterobject hdr; /* So we can cast it */
NC_FILTER_SORT sort; /* discriminate union */
union {
NC_FILTER_SPEC_HDF5 spec;
NC_FILTERIDS_HDF5 ids;
NC_FILTER_CLIENT_HDF5 client;
} u;
} NC_FILTER_OBJ_HDF5;
extern void NC4_freefilterspec(NC_FILTER_SPEC_HDF5* f);
extern const NC_reservedatt* NC_findreserved(const char* name);
/* Generic reserved Attributes */
#define NC_ATT_REFERENCE_LIST "REFERENCE_LIST"
#define NC_ATT_CLASS "CLASS"
#define NC_ATT_DIMENSION_LIST "DIMENSION_LIST"
#define NC_ATT_NAME "NAME"
#define NC_ATT_COORDINATES "_Netcdf4Coordinates" /*see hdf5internal.h:COORDINATES*/
#define NC_ATT_FORMAT "_Format"
#define NC_ATT_DIMID_NAME "_Netcdf4Dimid"
#define NC_ATT_NC3_STRICT_NAME "_nc3_strict"
#endif /* _NC4INTERNAL_ */

2
include/nc_provenance.h
View File

@ -20,6 +20,8 @@
* - Per file: _NCProperties attribute
*
* @author Dennis Heimbigner, Ward Fisher
*
* [This file is too hdf5 specific, need to clean so we can use with Zarr]
*/
#ifndef _NCPROVENANCE_

4
include/ncbytes.h
View File

@ -4,8 +4,6 @@
#ifndef NCBYTES_H
#define NCBYTES_H 1
#include "ncexternl.h"
typedef struct NCbytes {
int nonextendible; /* 1 => fail if an attempt is made to extend this buffer*/
unsigned long alloc;
@ -13,6 +11,8 @@ typedef struct NCbytes {
char* content;
} NCbytes;
#include "ncexternl.h"
#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__) || defined(__CPLUSPLUS)
extern "C" {
#endif

4
include/ncconfigure.h
View File

@ -114,6 +114,10 @@ typedef unsigned int uintptr_t;
#endif
#endif
#ifndef HAVE_SIZE64_T
typedef unsigned long long size64_t;
#endif
/* Provide a fixed size alternative to off_t or off64_t */
typedef long long fileoffset_t;

6
include/ncdispatch.h
View File

@ -143,6 +143,12 @@ extern int HDF4_initialize(void);
extern int HDF4_finalize(void);
#endif
#ifdef ENABLE_NCZARR
extern const NC_Dispatch* NCZ_dispatch_table;
extern int NCZ_initialize(void);
extern int NCZ_finalize(void);
#endif
/* User-defined formats.*/
extern NC_Dispatch* UDF0_dispatch_table;
extern char UDF0_magic_number[NC_MAX_MAGIC_NUMBER_LEN + 1];

86
include/ncfilter.h Normal file
View File

@ -0,0 +1,86 @@
/* Copyright 2018-2018 University Corporation for Atmospheric
Research/Unidata. */
/**
* @file
* @internal This header file contains macros, types and prototypes
* used to build and manipulate filters. It contains definitions
* for multiple file formats.
*
* @author Dennis Heimbigner
*/
#ifndef NCFILTER_H
#define NCFILTER_H
/**************************************************/
/* Internal filter related structures */
/* Internal filter actions */
#define NCFILTER_DEF 1
#define NCFILTER_REMOVE 2
#define NCFILTER_FILTERIDS 3
#define NCFILTER_INFO 4
#define NCFILTER_CLIENT_REG 10
#define NCFILTER_CLIENT_UNREG 11
#define NCFILTER_CLIENT_INQ 12
typedef enum NC_FILTER_UNION {
NC_FILTER_UNION_SPEC=((int)1),
NC_FILTER_UNION_IDS=((int)2),
NC_FILTER_UNION_CLIENT=((int)3),
} NC_FILTER_UNION;
/**************************************************/
/* Provide structs to pass args to filter_actions function using strings */
/* Originally, this used the HDF5 model where ids and params were unsigned ints.
To extend to zarr, we have converted so that ids and params are strings (char*).
Additionally we now allow a string id to be either an integer (as before but as a string)
or a name defined in the known_filters table in dfilter.c */
typedef struct NC_FILTERX_SPEC {
int active; /**< true iff underlying library was told to activate filter */
char* filterid; /**< ID for arbitrary filter. */
size_t nparams;
char** params; /**< Params for arbitrary filter. */
} NC_FILTERX_SPEC;
typedef struct NC_FILTERX_IDS {
size_t nfilters; /**< number of filters */
char** filterids; /**< Filter ids. */
} NC_FILTERX_IDS;
typedef struct NC_FILTERX_CLIENT {
char* id;
/* The filter info for x */
/* Avoid needing hdf.h by using void* */
void* info;
} NC_FILTERX_CLIENT;
typedef struct NC_FILTERX_OBJ {
NC_FILTER_UNION usort; /* discriminate union */
union {
NC_FILTERX_SPEC spec;
NC_FILTERX_IDS ids;
NC_FILTERX_CLIENT client;
} u;
} NC_FILTERX_OBJ;
struct NC_VAR_INFO;
EXTERNL int NC4_filterx_lookup(struct NC_VAR_INFO* var, const char* id, NC_FILTERX_SPEC** specp);
EXTERNL int NC4_filterx_add(struct NC_VAR_INFO* var, int active, const char* id, int nparams, const char** params);
EXTERNL int NC4_filterx_remove(struct NC_VAR_INFO* var, const char* xid);
EXTERNL int NC4_filterx_freelist(struct NC_VAR_INFO* var);
EXTERNL int NC4_filterx_free(NC_FILTERX_SPEC*);
EXTERNL int NC_cvtX2I_id(const char* xid, unsigned int* id);
EXTERNL int NC_cvtI2X_id(unsigned int id, char** xidp, int usename);
EXTERNL void NC_filterx_freestringvec(size_t n, char** vec);
EXTERNL int NC_filterx_copy(size_t n, const char** vec, char*** copyp);
/* As a rule, user provides space for output */
EXTERNL int NC_cvtI2X_idlist(int n, const unsigned int* ids, char** xid);
EXTERNL int NC_cvtI2X_params(int n, const unsigned int* iparams, char** xparams);
EXTERNL int NC_cvtX2I_params(size_t nparams, const char** xparamslist, unsigned int* params);
EXTERNL int NC_cvtX2I_idlist(size_t n, const char** xidlist, unsigned int* ids);
#endif /*NCFILTER_H*/

11
include/nchashmap.h
View File

@ -85,7 +85,7 @@ extern int NC_hashmapget(NC_hashmap*, const char* key, size_t keysize, uintptr_t
*/
extern int NC_hashmapsetdata(NC_hashmap*, const char* key, size_t keylen, uintptr_t newdata);
/** Returns the number of saved elements. */
/** Returns the number of active elements. */
extern size_t NC_hashmapcount(NC_hashmap*);
/** Reclaims the hashmap structure. */
@ -94,5 +94,14 @@ extern int NC_hashmapfree(NC_hashmap*);
/* Return the hash key for specified key; takes key+size*/
extern unsigned int NC_hashmapkey(const char* key, size_t size);
/* Return the ith entry info:
@param map
@param i
@param entryp contains 0 if not active, otherwise the data
@param keyp contains null if not active, otherwise the key
@return NC_EINVAL if no more entries
*/
extern int NC_hashmapith(NC_hashmap* map, size_t i, uintptr_t* datap, const char** keyp);
#endif /*NCHASHMAP_H*/

22
include/nchttp.h
View File

@ -8,8 +8,24 @@
#ifndef NCHTTP_H
#define NCHTTP_H
extern int nc_http_open(const char* objecturl, void** curlp, fileoffset_t* filelenp);
extern int nc_http_read(void* curl, const char* url, fileoffset_t start, fileoffset_t count, NCbytes* buf);
extern int nc_http_close(void* curl);
typedef enum HTTPMETHOD {
HTTPNONE=0, HTTPGET=1, HTTPPUT=2, HTTPPOST=3, HTTPHEAD=4
} HTTPMETHOD;
struct CURL; /* Forward */
typedef struct NC_HTTP_STATE {
struct CURL* curl;
long httpcode;
const char** headset; /* which headers to capture */
NClist* headers;
NCbytes* buf;
} NC_HTTP_STATE;
extern int nc_http_open(const char* objecturl, NC_HTTP_STATE** state, size64_t* lenp);
extern int nc_http_size(NC_HTTP_STATE* state, const char* url, size64_t* sizep);
extern int nc_http_read(NC_HTTP_STATE* state, const char* url, size64_t start, size64_t count, NCbytes* buf);
extern int nc_http_close(NC_HTTP_STATE* state);
extern int nc_http_headers(NC_HTTP_STATE* state, const NClist** headersp); /* only if headerson */
#endif /*NCHTTP_H*/

14
include/nclist.h
View File

@ -33,24 +33,24 @@ extern int nclistinsert(NClist*,size_t,void*);
extern void* nclistremove(NClist* l, size_t i);
/* Tail operations */
extern int nclistpush(NClist*,void*); /* Add at Tail */
extern int nclistpush(NClist*,const void*); /* Add at Tail */
extern void* nclistpop(NClist*);
extern void* nclisttop(NClist*);
/* Duplicate and return the content (null terminate) */
extern void** nclistdup(NClist*);
/* Look for value match */
/* Look for pointer match */
extern int nclistcontains(NClist*, void*);
/* Look for value match as string */
extern int nclistmatch(NClist*, const char*, int casesensistive);
/* Remove element by value; only removes first encountered */
extern int nclistelemremove(NClist* l, void* elem);
/* remove duplicates */
extern int nclistunique(NClist*);
/* Create a clone of a list */
extern NClist* nclistclone(NClist*);
/* Create a clone of a list; if deep, then assume it is a list of strings */
extern NClist* nclistclone(NClist*, int deep);
extern void* nclistextract(NClist*);

3
include/nclog.h
View File

@ -10,7 +10,8 @@
#include <stdarg.h>
#include "ncexternl.h"
#define NCENVFLAG "NCLOGFILE"
#define NCENVLOGFILE "NCLOGFILE"
#define NCENVLOGGING "NCLOGGING"
/* Suggested tag values */
#define NCLOGNOTE 0

28
include/ncuri.h
View File

@ -58,16 +58,22 @@ extern "C" {
#endif
EXTERNL int ncuriparse(const char* s, NCURI** ncuri);
extern void ncurifree(NCURI* ncuri);
EXTERNL void ncurifree(NCURI* ncuri);
/* Replace the protocol */
extern int ncurisetprotocol(NCURI*,const char* newprotocol);
EXTERNL int ncurisetprotocol(NCURI*,const char* newprotocol);
/* Replace the constraints */
EXTERNL int ncurisetquery(NCURI*,const char* query);
/* Replace the fragment list */
extern int ncurisetfragments(NCURI*, const char* fragments);
EXTERNL int ncurisetfragments(NCURI*, const char* fragments);
/* Replace a specific &key=...& in uri fragment */
EXTERNL int ncurisetfragmentkey(NCURI* duri,const char* key, const char* value);
/* append a specific &key=...& in uri fragment */
EXTERNL int ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value);
/* Construct a complete NC URI; caller frees returned string */
EXTERNL char* ncuribuild(NCURI*,const char* prefix, const char* suffix, int flags);
@ -76,28 +82,28 @@ EXTERNL char* ncuribuild(NCURI*,const char* prefix, const char* suffix, int flag
Null result => entry not found; !NULL=>found;
In any case, the result is imutable and should not be free'd.
*/
extern const char* ncurilookup(NCURI*, const char* param);
EXTERNL const char* ncurifragmentlookup(NCURI*, const char* param);
/*! Search the query for a given parameter
Null result => entry not found; !NULL=>found;
In any case, the result is imutable and should not be free'd.
*/
extern const char* ncuriquerylookup(NCURI*, const char* param);
EXTERNL const char* ncuriquerylookup(NCURI*, const char* param);
/* Obtain the complete list of fragment pairs in envv format */
extern const char** ncurifragmentparams(NCURI*);
EXTERNL const char** ncurifragmentparams(NCURI*);
/* Obtain the complete list of query pairs in envv format */
extern const char** ncuriqueryparams(NCURI*);
EXTERNL const char** ncuriqueryparams(NCURI*);
/* URL Encode/Decode */
extern char* ncuridecode(char* s);
EXTERNL char* ncuridecode(const char* s);
/* Partial decode */
extern char* ncuridecodepartial(char* s, const char* decodeset);
EXTERNL char* ncuridecodepartial(const char* s, const char* decodeset);
/* Encode using specified character set */
extern char* ncuriencodeonly(char* s, const char* allowable);
EXTERNL char* ncuriencodeonly(const char* s, const char* allowable);
/* Encode user or pwd */
extern char* ncuriencodeuserpwd(char* s);
EXTERNL char* ncuriencodeuserpwd(const char* s);
#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__) || defined(__CPLUSPLUS)
}

8
include/ncurlmodel.h
View File

@ -9,20 +9,12 @@
#ifndef NCURLMODEL_H
#define NCURLMODEL_H
/* Define the io handler to be used to do lowest level
access. This is above the libcurl level.
Note that cases (DAP2,DAP4) where the implementation is 1-1
with the iosp are not included.
*/
#define NC_IOSP_ZARR (1)
/* Track the information from a URL that will help us
infer how to access the data pointed to by that URL.
*/
typedef struct NCmode {
int format; /* NC_FORMAT_XXX value */
int implementation; /* NC_FORMATX_XXX value */
int iosp; /* NC_IOSP_XXX value (above) */
} NCmode;
/* return 1 if path looks like a url; 0 otherwise */

28
include/ncwinpath.h
View File

@ -11,10 +11,13 @@
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_DIRENT_H
#include <dirent.h>
#endif
#include "ncexternl.h"
#ifndef WINPATH
#ifdef _MSC_VER
#ifdef _WIN32
#define WINPATH 1
#endif
#ifdef __MINGW32__
@ -23,7 +26,7 @@
#endif
/* Define wrapper constants for use with NCaccess */
#ifdef _MSC_VER
#ifdef _WIN32
#define ACCESS_MODE_EXISTS 0
#define ACCESS_MODE_R 4
#define ACCESS_MODE_W 2
@ -38,6 +41,9 @@
/* Path Converter */
EXTERNL char* NCpathcvt(const char* path);
/* Fix path in case it was escaped by shell */
EXTERNL char* NCdeescape(const char* name);
#ifdef WINPATH
/* path converter wrappers*/
EXTERNL FILE* NCfopen(const char* path, const char* flags);
@ -45,16 +51,32 @@ EXTERNL int NCopen3(const char* path, int flags, int mode);
EXTERNL int NCopen2(const char* path, int flags);
EXTERNL int NCaccess(const char* path, int mode);
EXTERNL int NCremove(const char* path);
EXTERNL int NCmkdir(const char* path, int mode);
EXTERNL char* NCcwd(char* cwdbuf, size_t len);
#ifdef HAV_DIRENT_H
EXTERNL DIR* NCopendir(const char* path);
EXTERNL int NCclosedir(DIR* ent);
#endif
#else /*!WINPATH*/
#define NCfopen(path,flags) fopen((path),(flags))
#define NCopen3(path,flags,mode) open((path),(flags),(mode))
#define NCopen2(path,flags) open((path),(flags))
#define NCremove(path) remove(path)
#ifdef _MSC_VER
#ifdef _WIN32
#define NCaccess(path,mode) _access(path,mode)
#else
#define NCaccess(path,mode) access(path,mode)
#endif
#define NCmkdir(path, mode) mkdir(path,mode)
#define NCcwd(buf, len) getcwd(buf,len)
#ifdef HAVE_DIRENT_H
#define NCopendir(path) opendir(path)
#define NCclosedir(ent) closedir(ent)
#endif
#endif /*WINPATH*/
/* Platform independent */
#define NCclose(fd) close(fd)
#endif /* _NCWINIO_H_ */

10
include/netcdf.h
View File

@ -118,6 +118,8 @@ extern "C" {
currently unused:
0x0002
and the whole upper 16 bits
Note: nc4internal also defines flags in this space even tho it should not.
so check there around #define NC_CREAT.
*/
#define NC_NOWRITE 0x0000 /**< Set read-only access for nc_open(). */
@ -125,7 +127,6 @@ extern "C" {
#define NC_CLOBBER 0x0000 /**< Destroy existing file. Mode flag for nc_create(). */
#define NC_NOCLOBBER 0x0004 /**< Don't destroy existing file. Mode flag for nc_create(). */
#define NC_DISKLESS 0x0008 /**< Use diskless file. Mode flag for nc_open() or nc_create(). */
#define NC_MMAP 0x0010 /**< \deprecated Use diskless file with mmap. Mode flag for nc_open() or nc_create()*/
@ -216,7 +217,7 @@ Use this in mode flags for both nc_create() and nc_open(). */
#define NC_FORMATX_DAP4 (6)
#define NC_FORMATX_UDF0 (8)
#define NC_FORMATX_UDF1 (9)
#define NC_FORMATX_ZARR (10)
#define NC_FORMATX_NCZARR (10)
#define NC_FORMATX_UNDEFINED (0)
/* To avoid breaking compatibility (such as in the python library),
@ -476,8 +477,11 @@ by the desired type. */
#define NC_ENULLPAD (-134) /**< Header Bytes not Null-Byte padded */
#define NC_EINMEMORY (-135) /**< In-memory file error */
#define NC_ENOFILTER (-136) /**< Filter not defined on variable. */
#define NC_ENCZARR (-137) /**< Error at NCZarr layer. */
#define NC_ES3 (-138) /**< Generic S3 error */
#define NC_EEMPTY (-139) /**< Attempt to read empty NCZarr map key */
#define NC4_LAST_ERROR (-137) /**< @internal All netCDF errors > this. */
#define NC4_LAST_ERROR (-140) /**< @internal All netCDF errors > this. */
/* Errors for all remote access methods(e.g. DAP and CDMREMOTE)*/
#define NC_EURL (NC_EDAPURL) /**< Malformed URL */

26
include/netcdf_aux.h
View File

@ -54,6 +54,32 @@ EXTERNL size_t ncaux_type_alignment(int xtype, int ncid);
/* Takes type classes only */
EXTERNL size_t ncaux_class_alignment(int ncclass);
/**************************************************/
#define NCAUX_FILTERSPEC_VERSION 1
/* String specific filter info */
typedef struct NC_Filterspec {
int version;
char* filterid; /**< ID for arbitrary filter. */
size_t nparams;
char** params; /**< Params for arbitrary filter. */
} NC_Filterspec;
/* HDF5 specific filter info */
typedef struct NC_H5_Filterspec {
unsigned int filterid; /**< ID for arbitrary filter. */
size_t nparams; /**< nparams for arbitrary filter. */
unsigned int* params; /**< Params for arbitrary filter. */
} NC_H5_Filterspec;
EXTERNL void ncaux_filterfix8(unsigned char* mem, int decode);
EXTERNL int ncaux_filterspec_parselist(const char* listspec, char** formatp, size_t* nfilters, NC_Filterspec*** filtersp);
EXTERNL int ncaux_filterspec_parse(const char* txt, NC_Filterspec** specp);
EXTERNL void ncaux_filterspec_free(NC_Filterspec*);
EXTERNL int ncaux_filterspec_cvt(const NC_Filterspec* spec, NC_H5_Filterspec** spech5p);
#if defined(__cplusplus)
}
#endif

6
include/netcdf_dispatch.h
View File

@ -148,7 +148,7 @@ struct NC_Dispatch
/* Dispatch table Version 2 or later */
/* Handle all filter related actions. */
int (*filter_actions)(int ncid, int varid, int action, struct NC_Filterobject*);
int (*filter_actions)(int ncid, int varid, int action, void*);
};
#if defined(__cplusplus)
@ -220,7 +220,6 @@ extern "C" {
EXTERNL int NC_NOTNC4_inq_typeids(int, int *, int *);
EXTERNL int NC_NOTNC4_inq_user_type(int, nc_type, char *, size_t *,
nc_type *, size_t *, int *);
EXTERNL int NC_NOTNC4_filter_actions(int, int, int, struct NC_Filterobject*);
/* These functions are for dispatch layers that don't implement
* the enhanced model, but want to succeed anyway.
@ -228,7 +227,8 @@ extern "C" {
* In some cases (filter actions), some cases may succeed and some
* will fail.
*/
EXTERNL int NC_NOOP_filter_actions(int, int, int, struct NC_Filterobject*);
EXTERNL int NC_NOOP_filter_actions(int, int, int, void*);
EXTERNL int NC_NOTNC4_filter_actions(int, int, int, void*);
/* These functions are for dispatch layers that don't want to
* implement the deprecated varm functions. They return

83
include/netcdf_filter.h
View File

@ -11,7 +11,13 @@
#ifndef NETCDF_FILTER_H
#define NETCDF_FILTER_H 1
/* API for libdispatch/dfilter.c */
/* API for libdispatch/dfilter.c
Note that since this filter_actions() is visible
thru the dispatch table (via netcdf_dispatch.h)
and that can be seen by clients using user-defined
formats, then all argument types need
to be user visible as well.
*/
/* Must match values in <H5Zpublic.h> */
#ifndef H5Z_FILTER_DEFLATE
@ -24,6 +30,9 @@
#define H5_SZIP_EC_OPTION_MASK 4
#define H5_SZIP_NN_OPTION_MASK 32
#define H5_SZIP_MAX_PIXELS_PER_BLOCK 32
#define NC_SZIP_EC 4 /**< Selects entropy coding method for szip. */
#define NC_SZIP_NN 32 /**< Selects nearest neighbor coding method for szip. */
#endif
#define H5_SZIP_ALL_MASKS (H5_SZIP_CHIP_OPTION_MASK|H5_SZIP_EC_OPTION_MASK|H5_SZIP_NN_OPTION_MASK)
@ -35,28 +44,12 @@
extern "C" {
#endif
/* Define the formats for NC_FILTER classes */
/* Define the formats for NC_FILTER classes as aliases for NC_FORMATX_XXX*/
#define NC_FILTER_FORMAT_HDF5 (NC_FORMATX_NC_HDF5)
/* Define the sort for NC_FILTER classes */
//#define NC_FILTER_SORT_SPEC ((int)1) /* Use for NC_Filterspec; Must match nc4internal.h */
/* Define a Header Object for all filter-related objects */
/* provide a common generic struct field */
/*
format indicates e.g. HDF5
sort indicates the "subclass" of the superclass
*/
typedef struct NC_Filterobject {int format;} NC_Filterobject;
/* Generic version of Filterspec */
typedef struct NC_Filterspec {
NC_Filterobject hdr; /**< e.g. NC_FILTER_FORMAT_HDF5 */
} NC_Filterspec;
#define NCX_FILTER_FORMAT (NC_FORMATX_NCZARR)
/**************************************************/
/* HDF5 Specific filter functions */
/* HDF5 Specific filter functions (Deprecated) */
/*Define a filter for a variable */
EXTERNL int
@ -78,7 +71,7 @@ nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, un
/* Remove filter from variable*/
EXTERNL int nc_var_filter_remove(int ncid, int varid, unsigned int id);
/* Support direct user defined filters;
/* Support direct user defined filters if enabled during configure;
last arg is void*, but is actually H5Z_class2_t*.
It is void* to avoid having to reference hdf.h.
*/
@ -86,23 +79,43 @@ EXTERNL int nc_filter_client_register(unsigned int id, void*);
EXTERNL int nc_filter_client_unregister(unsigned int id);
EXTERNL int nc_filter_client_inq(unsigned int id, void*);
/* HDF5 specific filter info */
typedef struct NC4_Filterspec {
NC_Filterspec hdr;
/* HDF5 specific extensions */
unsigned int filterid; /**< ID for arbitrary filter. */
size_t nparams; /**< nparams for arbitrary filter. */
unsigned int* params; /**< Params for arbitrary filter. */
} NC4_Filterspec;
EXTERNL void NC4_filterfix8(unsigned char* mem, int decode);
EXTERNL int NC_parsefilterlist(const char* listspec, int* formatp, size_t* nfilters, NC_Filterspec*** filtersp);
EXTERNL int NC_parsefilterspec(const char* txt, int format, NC_Filterspec** specp);
/* End HDF5 Specific Declarations */
/**************************************************/
/* X (String-based extension) Declarations */
/*Define a filter for a variable */
EXTERNL int
nc_def_var_filterx(int ncid, int varid, const char* id, size_t nparams, const char** params);
/* Support inquiry about all the filters associated with a variable */
/* As is usual, it is expected that this will be called twice:
once to get the number of filters, and then a second time to read the ids */
EXTERNL int
nc_inq_var_filterx_ids(int ncid, int varid, size_t* nfilters, char** filteridsp);
/* Learn about the filter with specified id wrt a variable */
EXTERNL int
nc_inq_var_filterx_info(int ncid, int varid, const char* id, size_t* nparamsp, char** paramsp);
/* Remove filter from variable*/
EXTERNL int
nc_var_filterx_remove(int ncid, int varid, const char* id);
/* Support direct user defined filters if enabled during configure;
last arg is void*, but is actually H5Z_class2_t*.
It is void* to avoid having to reference hdf.h.
*/
EXTERNL int nc_filterx_client_register(const char* id, void*);
EXTERNL int nc_filterx_client_unregister(const char* id);
EXTERNL int nc_filterx_client_inq(const char* id, void*);
/* End X (String-based extension) Declarations */
/**************************************************/
/* Set szip compression for a variable. */
EXTERNL int nc_def_var_szip(int ncid, int varid, int options_mask, int pixels_per_block);
#if defined(__cplusplus)
}

2
include/netcdf_meta.h.in
View File

@ -60,4 +60,6 @@
#define NC_RELAX_COORD_BOUND 1 /*!< RELAX_COORD_BOUND */
#define NC_DISPATCH_VERSION @NC_DISPATCH_VERSION@ /*!< Dispatch table version */
#define NC_HAS_PAR_FILTERS @NC_HAS_PAR_FILTERS@ /* Parallel I/O with filter support. */
#define NC_HAS_ZARR @NC_HAS_ZARR@
#endif

4
lib_flags.am
View File

@ -13,4 +13,8 @@ if USE_DAP
AM_CPPFLAGS += -I${top_srcdir}/oc2
endif
if ENABLE_S3_SDK
AM_LDFLAGS += -lstdc++
endif
AM_TESTS_ENVIRONMENT = export TOPSRCDIR=${abs_top_srcdir}; export TOPBUILDDIR=${abs_top_builddir};

2
libdap2/cache.c
View File

@ -247,7 +247,7 @@ buildcachenode(NCDAPCOMMON* nccomm,
/* create the cache node */
cachenode = createnccachenode();
cachenode->isprefetch = isprefetch;
cachenode->vars = nclistclone(varlist);
cachenode->vars = nclistclone(varlist,0);
cachenode->datadds = dxdroot;
/* Give the constraint over to the cachenode */
cachenode->constraint = constraint;

2
libdap2/cdf.c
View File

@ -748,7 +748,7 @@ definedimsetplus(NCDAPCOMMON* nccomm/*notused*/, CDFnode* node)
if(node->array.dimset0 != NULL)
/* copy the dimset0 into dimset */
dimset = nclistclone(node->array.dimset0);
dimset = nclistclone(node->array.dimset0, 0);
/* Insert the sequence or string dims */
if(node->array.stringdim != NULL) {
if(dimset == NULL) dimset = nclistnew();

1
libdap2/dapcvt.c
View File

@ -28,7 +28,6 @@ dapconvert(nc_type srctype, nc_type dsttype, char* memory0, char* value0, size_t
size_t i;
char* memory = memory0;
char* value = value0;
/* In order to deal with the DAP upgrade problem,
try to preserve the bit patterns
*/

7
libdap2/dapdebug.h
View File

@ -5,8 +5,11 @@
#ifndef DEBUG_H
#define DEBUG_H
/* Warning: setting CATCHERROR has significant performance impact */
#define CATCHERROR
#if 0
#define DAPDEBUG 1
#define DAPDEBUG 2
#define OCDEBUG 1
#endif
@ -29,8 +32,6 @@
#undef DCEVERBOSE
#undef PARSEDEBUG
/* Warning: setting CATCHERROR has significant performance impact */
#undef CATCHERROR
#include <stdarg.h>
#include <assert.h>

6
libdap2/daputil.c
View File

@ -215,7 +215,7 @@ dapparamvalue(NCDAPCOMMON* nccomm, const char* key)
const char* value;
if(nccomm == NULL || key == NULL) return 0;
value=ncurilookup(nccomm->oc.url,key);
value=ncurifragmentlookup(nccomm->oc.url,key);
return value;
}
@ -231,7 +231,7 @@ dapparamcheck(NCDAPCOMMON* nccomm, const char* key, const char* subkey)
char* p;
if(nccomm == NULL || key == NULL) return 0;
if((value=ncurilookup(nccomm->oc.url,key)) == NULL)
if((value=ncurifragmentlookup(nccomm->oc.url,key)) == NULL)
return 0;
if(subkey == NULL) return 1;
p = strstr(value,subkey);
@ -711,6 +711,8 @@ oc_dumpnode(conn,*rootp);
} else if(httpcode >= 500) {
ncstat = NC_EDAPSVC;
} else if(httpcode == 401) {
ncstat = NC_EACCESS;
} else if(httpcode == 403) {
ncstat = NC_EAUTH;
} else if(httpcode == 404) {
ncstat = NC_ENOTFOUND;

2
libdap2/dceconstraints.c
View File

@ -63,7 +63,7 @@ slicedump(const char* prefix, DCEslice* s)
#else
size_t last = (s->first+s->length)-1;
fprintf(stderr,"%s: [%lu:%lu:%lu p=%lu l=%lu c=%lu]\n",
prefix,s->first,s->stride,last,s->stop,s->length,s->count);
prefix,s->first,s->stride,last,s->last,s->length,s->count);
#endif
}
#endif

6
libdap2/getvara.c
View File

@ -506,8 +506,8 @@ movetor(NCDAPCOMMON* nccomm,
if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;}
#ifdef DEBUG2
fprintf(stderr,"moveto: nctype=%d depth=%d dimindex=%d mode=%s",
xnode->nctype, depth,dimindex,oc_data_modestring(mode));
fprintf(stderr,"moveto: nctype=%d depth=%d dimindex=%d",
xnode->nctype, (int)depth, (int)dimindex);
fprintf(stderr," segment=%s hasstringdim=%d\n",
dcetostring((DCEnode*)segment),hasstringdim);
#endif
@ -903,7 +903,7 @@ slicestring(OClink conn, char* stringmem, DCEslice* slice, struct NCMEMORY* memo
fprintf(stderr,"moveto: slicestring: string/%lu=%s\n",stringlen,stringmem);
fprintf(stderr,"slicestring: %lu string=|%s|\n",stringlen,stringmem);
fprintf(stderr,"slicestring: slice=[%lu:%lu:%lu/%lu]\n",
slice->first,slice->stride,slice->stop,slice->declsize);
slice->first,slice->stride,slice->last,slice->declsize);
#endif
/* Stride across string; if we go past end of string, then pad*/

10
libdap2/ncd2dispatch.c
View File

@ -346,7 +346,7 @@ NCD2_open(const char* path, int mode, int basepe, size_t *chunksizehintp,
/* fail if we are unconstrainable but have constraints */
if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE)) {
if(dapcomm->oc.url->query != NULL) {
if(dapcomm->oc.url != NULL && dapcomm->oc.url->query != NULL) {
nclog(NCLOGWARN,"Attempt to constrain an unconstrainable data source: %s",
dapcomm->oc.url->query);
ncstat = THROW(NC_EDAPCONSTRAINT);
@ -393,7 +393,9 @@ NCD2_open(const char* path, int mode, int basepe, size_t *chunksizehintp,
/* Parse constraints to make sure they are syntactically correct */
ncstat = dapparsedapconstraints(dapcomm,dapcomm->oc.url->query,dapcomm->oc.dapconstraint);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;}
#ifdef DEBUG2
fprintf(stderr,"ce=%s\n",dumpconstraint(dapcomm->oc.dapconstraint));
#endif
/* Construct a url for oc minus any constraint and params*/
dapcomm->oc.urltext = ncuribuild(dapcomm->oc.url,NULL,NULL,NCURIBASE);
@ -822,7 +824,7 @@ fprintf(stderr,"\n");
nclistpush(unsignedatt->values,strdup("false"));
} else if(att->etype != var->etype) {/* other mismatches */
/* Log a message */
nclog(NCLOGERR,"_FillValue/Variable type mismatch: variable=%s",var->ncbasename);
nclog(NCLOGWARN,"_FillValue/Variable type mismatch: variable=%s",var->ncbasename);
/* See if mismatch is allowed */
if(FLAGSET(dapcomm->controls,NCF_FILLMISMATCH)) {
/* Forcibly change the attribute type to match */
@ -1227,7 +1229,7 @@ paramlookup(NCDAPCOMMON* state, const char* key)
{
const char* value = NULL;
if(state == NULL || key == NULL || state->oc.url == NULL) return NULL;
value = ncurilookup(state->oc.url,key);
value = ncurifragmentlookup(state->oc.url,key);
return value;
}

2
libdap2/ncdaperr.c
View File

@ -36,6 +36,8 @@ ocerrtoncerr(OCerror ocerr)
case OC_EDATADDS: return NC_EDATADDS;
case OC_ERCFILE: return NC_EDAP;
case OC_ENOFILE: return NC_ECANTREAD;
case OC_EAUTH: return NC_EAUTH;
case OC_EACCESS: return NC_EACCESS;
default: break;
}
return NC_EDAP; /* default;*/

600
libdap2/test_vara.c
View File

@ -1,600 +0,0 @@
/*! \file
Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
2015, 2016, 2017, 2018
University Corporation for Atmospheric Research/Unidata.
See \ref copyright file for more info.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "netcdf.h"
#include "nctestserver.h"
#define DTSTEST "/dts/test.06"
/* The DDS in netcdf classic form is as follows:
netcdf test {
dimensions:
t = 10 ;
x = 10 ;
y = 10 ;
z = 10 ;
variables:
double OneD(x) ;
double TwoD(x, y) ;
double ThreeD(x, y, z) ;
double FourD(x, y, z, t) ;
}
*/
/* Use the ThreeD variable */
#define NCVAR "ThreeD"
#define X 10
#define Y 10
#define Z 10
#define RANK 3
#define ERRCODE 2
#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);}
#undef DEBUG
#define DELTA 0.005
/* Setup an odometer */
typedef struct Odom {
int rank;
size_t* index;
size_t* stop;
size_t* start;
size_t* count;
} Odom;
static float target_data[X][Y][Z];
static float target[X][Y][Z];
static dims[RANK] = {X,Y,Z};
static Odom* odom_create(int rank);
static void odom_reclaim(Odom* odom);
static void odom_set(Odom* odom, size_t* start, size_t* count);
static int odom_more(Odom* odom);
static int odom_incr(Odom* odom);
static size_t odom_count(Odom* odom);
static int check(size_t* start, size_t* count);
/* Define whole variable start/count */
static size_t start0[RANK] = {0,0,0};
static size_t count0[RANK] = {X,Y,Z};
static int floateq(float f1, float f2)
{
float diff = f1 - f2;
if(diff >= 0 && diff < DELTA) return 1;
if(diff <= 0 && diff > -DELTA) return 1;
return 0;
}
int
main()
{
int ncid, varid;
int retval,i;
size_t start[RANK];
size_t count[RANK];
char URL[4096];
const char* svc = NULL;
const char* testservers = "";
#ifdef REMOTETESTSERVERS
testservers = REMOTETESTSERVERS;
#endif
/* Find Test Server */
svc = nc_findtestserver("dts",0,testservers);
if(svc == NULL) {
fprintf(stderr,"Cannot locate test server\n");
exit(1);
}
printf("Using test server: %s\n",svc);
strncpy(URL,svc,sizeof(URL));
strlcat(URL,DTSTEST,sizeof(URL));
memset((void*)target,0,sizeof(target));
if((retval = nc_open(URL, NC_NOWRITE, &ncid)))
ERR(retval);
if((retval = nc_inq_varid(ncid, NCVAR, &varid)))
ERR(retval);
printf("test 1: Read the whole variable\n");
memcpy(start,start0,sizeof(start0));
memcpy(count,count0,sizeof(count0));
if((retval = nc_get_vara_float(ncid,varid,start,count,(float*)target)))
ERR(retval);
if(!check(start,count)) goto fail;
printf("test 2: Read the top half\n");
memcpy(start,start0,sizeof(start0));
memcpy(count,count0,sizeof(count0));
start[0] += count0[0]/2;
count[0] = count0[0] - start[0];
if((retval = nc_get_vara_float(ncid,varid,start,count,(float*)target)))
ERR(retval);
if(!check(start,count)) goto fail;
printf("test 3: Read the slice with X=1\n");
memcpy(start,start0,sizeof(start0));
memcpy(count,count0,sizeof(count0));
start[0] = 1;
count[0] = 1;
if((retval = nc_get_vara_float(ncid,varid,start,count,(float*)target)))
ERR(retval);
if(!check(start,count)) goto fail;
if((retval = nc_close(ncid)))
ERR(retval);
printf("*** PASS\n");
return 0;
fail:
printf("*** FAIL\n");
return 1;
}
static int
check(size_t* start, size_t* count)
{
int ok = 1;
int index = 0;
Odom* odom = odom_create(RANK);
odom_set(odom,start,count);
float* result = (float*)target;
float* expected = (float*)target_data;
for(index=0;odom_more(odom);odom_incr(odom),index++) {
size_t offset = odom_count(odom);
if(!floateq(result[index],expected[offset])) {
fprintf(stderr,"fail: result[%lu] = %f ; expected[%lu] = %f\n",
index,result[index],offset,expected[offset]);
ok=0;
}
}
odom_reclaim(odom);
return ok;
}
static Odom* odom_create(int rank)
{
Odom* odom = (Odom*)malloc(sizeof(Odom));
/* Init the odometer */
odom->rank = rank;
odom->index = (size_t*)calloc(sizeof(size_t)*rank,1);
odom->stop = (size_t*)calloc(sizeof(size_t)*rank,1);
odom->start = (size_t*)calloc(sizeof(size_t)*rank,1);
odom->count = (size_t*)calloc(sizeof(size_t)*rank,1);
return odom;
}
static void odom_reclaim(Odom* odom)
{
free(odom->index);
free(odom->stop);
free(odom->start);
free(odom->count);
free(odom);
}
static void odom_set(Odom* odom, size_t* start, size_t* count)
{
int i;
/* Init the odometer */
for(i=0;i<odom->rank;i++) {
odom->start[i] = start[i];
odom->count[i] = count[i];
}
for(i=0;i<odom->rank;i++) {
odom->index[i] = odom->start[i];
odom->stop[i] = odom->start[i] + odom->count[i];
}
}
static int odom_more(Odom* odom)
{
return (odom->index[0] < odom->stop[0]?1:0);
}
static int odom_incr(Odom* odom)
{
int i; /* do not make unsigned */
if(odom->rank == 0) return 0;
for(i=odom->rank-1;i>=0;i--) {
odom->index[i]++;
if(odom->index[i] < odom->stop[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows*/
odom->index[i] = odom->start[i]; /* reset this position*/
}
return 1;
}
/* Convert current dapodometer settings to a single integer count*/
static size_t odom_count(Odom* odom)
{
int i;
size_t offset = 0;
for(i=0;i<odom->rank;i++) {
offset *= odom->count[i];
offset += odom->index[i];
}
return offset;
}
static float target_data[X][Y][Z] = {
1, 0.999950000416665, 0.999800006666578, 0.999550033748988,
0.999200106660978, 0.998750260394966, 0.998200539935204,
0.99755100025328, 0.996801706302619, 0.995952733011994,
0.995004165278026, 0.993956097956697, 0.992808635853866, 0.991561893714788,
0.990215996212637, 0.988771077936042, 0.987227283375627,
0.985584766909561, 0.983843692788121, 0.98200423511727,
0.980066577841242, 0.978030914724148, 0.975897449330606, 0.973666395005375,
0.97133797485203, 0.968912421710645, 0.966389978134513,
0.963770896365891, 0.961055438310771, 0.958243875512697,
0.955336489125606, 0.952333569885713, 0.949235418082441, 0.946042343528387,
0.942754665528346, 0.939372712847379, 0.935896823677935,
0.932327345606034, 0.92866463557651, 0.924909059857313,
0.921060994002885, 0.917120822816605, 0.913088940312308, 0.908965749674885,
0.904751663219963, 0.900447102352677, 0.896052497525525,
0.891568288195329, 0.886994922779284, 0.882332858610121,
0.877582561890373, 0.872744507645751, 0.86781917967765, 0.862807070514761,
0.857708681363824, 0.852524522059506, 0.847255111013416,
0.841900975162269, 0.836462649915187, 0.830940679100164,
0.825335614909678, 0.819648017845479, 0.813878456662534, 0.808027508312152,
0.802095757884293, 0.796083798549056, 0.789992231497365,
0.783821665880849, 0.777572718750928, 0.771246014997107,
0.764842187284488, 0.758361875990508, 0.751805729140895, 0.74517440234487,
0.738468558729588, 0.731688868873821, 0.724836010740905,
0.717910669610943, 0.710913538012277, 0.703845315652236,
0.696706709347165, 0.689498432951747, 0.682221207287614, 0.674875760071267,
0.667462825841308, 0.659983145884982, 0.652437468164052,
0.644826547240001, 0.63715114419858, 0.629412026573697,
0.621609968270664, 0.613745749488812, 0.605820156643463, 0.597833982287298,
0.589788025031098, 0.581683089463883, 0.573519986072457,
0.565299531160354, 0.557022546766217, 0.548689860581588,
0.54030230586814, 0.531860721374355, 0.52336595125165, 0.514818844969955,
0.506220257232778, 0.497571047891727, 0.488872081860527,
0.480124229028534, 0.47132836417374, 0.462485366875301,
0.453596121425577, 0.444661516741707, 0.435682446276712, 0.426659807930157,
0.417594503958358, 0.408487440884157, 0.399339529406273,
0.39015168430823, 0.380924824366882, 0.371659872260533,
0.362357754476674, 0.35301940121933, 0.343645746316047, 0.334237727124503,
0.324796284438776, 0.315322362395269, 0.305816908378289,
0.296280872925319, 0.286715209631956, 0.277120875056558,
0.267498828624587, 0.25785003253267, 0.248175451652373, 0.238476053433723,
0.228752807808459, 0.219006687093041, 0.209238665891419,
0.199449720997573, 0.189640831297834, 0.179812977673,
0.169967142900241, 0.160104311554831, 0.150225469911686, 0.140331605846737,
0.130423708738146, 0.120502769367367, 0.11056977982007,
0.100625733386932, 0.0906716244643097, 0.0807084484548006,
0.0707372016677029, 0.0607588812193859, 0.0507744849335792,
0.040785011241591, 0.0307914590824661, 0.0207948278030924,
0.0107961170582674, 0.000796326710733263, -0.00920354326880834,
-0.0192024929016926,
-0.0291995223012888, -0.0391936317729877, -0.0491838219141706,
-0.0591690937141481, -0.0691484486540619, -0.0791208888067339,
-0.089085416936459, -0.099041036598728, -0.108986752239871,
-0.118921569296612,
-0.128844494295525, -0.138754534952378, -0.148650700271364,
-0.158532000644198, -0.168397447949077, -0.178246055649492,
-0.18807683889288, -0.197888814609109, -0.207681001608784,
-0.217452420681365,
-0.227202094693087, -0.236929048684675, -0.246632309968834,
-0.256310908227523, -0.26596387560898, -0.275590246824513,
-0.285189059245021, -0.294759352997261, -0.304300171059833,
-0.313810559358882,
-0.323289566863503, -0.332736245680845, -0.342149651150898,
-0.35152884194096, -0.360872880139767, -0.370180831351287,
-0.379451764788155, -0.388684753364752, -0.397878873789916,
-0.407033206659266,
-0.416146836547142, -0.425218852098152, -0.4342483461183,
-0.443234415665709, -0.452176162140912, -0.461072691376713,
-0.469923113727602, -0.47872654415872, -0.487482102334359,
-0.496188912705999,
-0.504846104599858, -0.513452812303959, -0.522008175154707,
-0.530511337622945, -0.538961449399512, -0.547357665480271,
-0.555699146250613, -0.56398505756941, -0.572214570852437,
-0.580386863155222,
-0.588501117255346, -0.59655652173416, -0.60455227105793,
-0.612487565658385, -0.62036161201268, -0.628173622722739,
-0.635922816594002, -0.643608418713541, -0.651229660527546,
-0.658785779918188,
-0.666276021279824, -0.673699635594561, -0.681055880507152,
-0.688344020399238, -0.695563326462902, -0.702713076773554,
-0.70979255636212, -0.716801057286543, -0.723737878702569,
-0.730602326933837,
-0.737393715541245, -0.744111365391593, -0.750754604725491,
-0.757322769224544, -0.763815202077774, -0.770231254047307,
-0.776570283533293, -0.782831656638065, -0.789014747229531,
-0.795118937003784,
-0.801143615546934, -0.807088180396146, -0.81295203709989,
-0.818734599277382, -0.824435288677222, -0.830053535235222,
-0.835588777131408, -0.841040460846201, -0.846408041215776,
-0.851690981486566,
-0.856888753368947, -0.862000837090063, -0.867026721445802,
-0.871965903851917, -0.876817890394281, -0.881582195878286,
-0.886258343877352, -0.890845866780576, -0.895344305839492,
-0.899753211213941,
-0.904072142017061, -0.90830066635937, -0.912438361391958,
-0.916484813348769, -0.920439617587981, -0.924302378632464,
-0.928072710209333, -0.931750235288572, -0.935334586120739,
-0.938825404273736,
-0.942222340668658, -0.945525055614696, -0.948733218843107,
-0.951846509540242, -0.954864616379626, -0.95778723755309,
-0.960614080800952, -0.963344863441243, -0.965979312397975,
-0.968517164228447,
-0.970958165149591, -0.973302071063349, -0.975548647581083,
-0.977697670047013, -0.979748923560684, -0.981702202998454,
-0.983557313034006, -0.985314068157884, -0.986972292696038,
-0.988531820827396,
-0.989992496600445, -0.991354173948826, -0.992616716705937,
-0.993779998618556, -0.994843903359459, -0.995808324539061,
-0.996673165716047, -0.997438340407019, -0.998103772095146,
-0.998669394237814,
-0.999135150273279, -0.999500993626328, -0.999766887712928,
-0.999932805943894, -0.99999873172754, -0.999964658471342,
-0.999830589582598, -0.999596538468086, -0.999262528532721,
-0.998828593177219,
-0.998294775794753, -0.997661129766618, -0.996927718456887,
-0.996094615206081, -0.99516190332383, -0.994129676080546,
-0.992998036698093, -0.991767098339465, -0.990436984097473,
-0.989007826982433,
-0.987479769908865, -0.985852965681203, -0.984127576978514,
-0.982303776338232, -0.980381746138899, -0.978361678581934,
-0.97624377567241, -0.974028249198852, -0.971715320712062,
-0.969305221502961,
-0.966798192579461, -0.964194484642366, -0.961494358060299,
-0.958698082843669, -0.955805938617666, -0.952818214594305,
-0.949735209543496, -0.946557231763177, -0.943284599048476,
-0.939917638659938,
-0.936456687290796, -0.932902091033304, -0.929254205344123,
-0.925513395008784, -0.921680034105203, -0.917754505966276,
-0.913737203141545, -0.909628527357945, -0.90542888947963,
-0.901138709466889,
-0.896758416334147, -0.892288448107068, -0.88772925177875,
-0.883081283265026, -0.878345007358874, -0.873520897683938,
-0.868609436647165, -0.863611115390566, -0.858526433742102,
-0.8533559001657,
-0.848100031710408, -0.842759353958694, -0.83733440097388,
-0.831825715246746, -0.826233847641272, -0.820559357339561,
-0.814802811785913, -0.808964786630086, -0.803045865669731,
-0.797046640792012,
-0.790967711914417, -0.784809686924768, -0.778573181620433,
-0.772258819646744, -0.765867232434637, -0.759399059137508,
-0.752854946567295, -0.746235549129803, -0.739541528759258,
-0.73277355485212,
-0.72593230420014, -0.719018460922681, -0.71203271639831,
-0.704975769195658, -0.697848325003564, -0.690651096560507,
-0.683384803583336, -0.676050172695292, -0.668647937353351,
-0.66117883777488,
-0.653643620863612, -0.646043040134959, -0.63837785564066,
-0.630648833892775, -0.622856747787041, -0.615002376525574,
-0.607086505538955, -0.599109926407685, -0.591073436783031,
-0.582977840307259,
-0.574823946533269, -0.566612570843644, -0.55834453436911,
-0.550020663906425, -0.541641791835699, -0.533208756037154,
-0.524722399807346, -0.516183571774825, -0.507593125815277,
-0.49895192096614,
-0.490260821340699, -0.481520696041674, -0.47273241907431,
-0.46389686925898, -0.455014930143305, -0.446087489913793,
-0.437115441307028, -0.428099681520394, -0.419041112122356,
-0.409940638962306,
-0.400799172079975, -0.391617625614436, -0.38239691771268,
-0.373137970437818, -0.363841709676858, -0.354509065048132,
-0.345140969808323, -0.335738360759151, -0.326302178153684,
-0.316833365602319,
-0.307332869978419, -0.297801641323633, -0.288240632752882,
-0.278650800359055, -0.269033103117399, -0.259388502789626,
-0.249717963827731, -0.24002245327755, -0.230302940682059,
-0.220560397984419,
-0.21079579943078, -0.20101012147286, -0.191204342670302,
-0.181379443592811, -0.171536406722112, -0.161676216353687,
-0.151799858498356, -0.141908320783673, -0.13200259235517,
-0.122083663777433,
-0.112152526935055, -0.102210174933442, -0.0922576019995117,
-0.0822958033822624, -0.0723257752532545, -0.0623485146069917,
-0.0523650191612259, -0.0423762872571815, -0.0323833177597247,
-0.0223871099574771,
-0.0123886634628906, -0.00238897811228154, 0.0076109461341479,
0.0176101092923073, 0.0276075114542115, 0.0376021528879766,
0.0475930341377878, 0.057579156123846, 0.0675595202422752,
0.0775331284649787,
0.0874989834394464, 0.0974560885884857, 0.10740344820988,
0.117340067575955, 0.127264953033056, 0.137177112100907,
0.147075553571863, 0.156959287610023, 0.166827325850222, 0.176678681496857,
0.186512369422576, 0.196327406266778, 0.206122810533958, 0.215897602691854,
0.225650805269396, 0.235381442954451, 0.245088542691362,
0.254771133778243, 0.264428247964056, 0.274058919545427,
0.283662185463226, 0.293237085398863, 0.302782661870324, 0.312297960327916,
0.321782029249722, 0.331233920236754, 0.340652688107789,
0.350037390993891, 0.35938709043259, 0.368700851461733,
0.37797774271298, 0.387216836504937, 0.396417208935922, 0.405577939976361,
0.414698113560782, 0.423776817679428, 0.432813144469452,
0.441806190305705, 0.450755055891099, 0.459658846346532,
0.468516671300377, 0.477327644977522, 0.48609088628794, 0.494805518914805,
0.503470671402114, 0.512085477241841, 0.520649074960579,
0.529160608205695, 0.537619225830956, 0.546024081981648,
0.554374336179161, 0.562669153405032, 0.570907704184454, 0.57908916466921,
0.587212716720073, 0.595277547988606, 0.603282851998404,
0.611227828225735, 0.619111682179599, 0.626933625481169,
0.634692875942635, 0.642388657645414, 0.650020201017752, 0.657586742911669,
0.665087526679283, 0.672521802248466, 0.679888826197857,
0.687187861831201, 0.694418179251016, 0.701579055431586,
0.70866977429126, 0.715689626764061, 0.722637910870592, 0.729513931788232,
0.736317001920619, 0.74304644096641, 0.749701575987307,
0.756281741475356, 0.762786279419489, 0.769214539371333,
0.77556587851025, 0.781839661707619, 0.788035261590348, 0.794152058603611,
0.800189441072806, 0.806146805264716, 0.812023555447886,
0.817819103952194, 0.823532871227622, 0.829164285902202,
0.83471278483916, 0.840177813193225, 0.845558824466117, 0.850855280561193,
0.856066651837255, 0.861192417161521, 0.866232063961728,
0.871185088277397, 0.876050994810224, 0.880829296973609,
0.885519516941319, 0.890121185695265, 0.894633843072407, 0.899057037810768,
0.903390327594559, 0.907633279098413, 0.911785468030717,
0.915846479176035, 0.919815906436639, 0.92369335287311,
0.927478430744036, 0.931170761544783, 0.934769976045349, 0.938275714327283,
0.941687625819678, 0.945005369334228, 0.948228613099346,
0.951357034793342, 0.954390321576654, 0.957328170123131,
0.960170286650366, 0.962916386949075, 0.965566196411518, 0.968119450058955,
0.970575892568149, 0.972935278296897, 0.975197371308593,
0.977361945395819, 0.979428784102971, 0.981397680747901,
0.983268438442584, 0.985040870112812, 0.986714798516892, 0.98829005626338,
0.989766485827815, 0.991143939568469, 0.992422279741117,
0.993601378512806, 0.994681117974643, 0.99566139015358,
0.996542097023217, 0.997323150513601, 0.998004472520033, 0.998585994910881,
0.99906765953439, 0.999449418224499, 0.999731232805658,
0.999913075096642, 0.999994926913375, 0.999976780070743,
0.999858636383415, 0.999640507665662, 0.999322415730172, 0.998904392385876,
0.998386479434759, 0.997768728667684, 0.997051201859214,
0.996233970761431, 0.995317117096764, 0.994300732549815,
0.993184918758193, 0.991969787302346, 0.990655459694407, 0.989242067366043,
0.987729751655308, 0.986118663792513, 0.984408964885101,
0.982600825901538, 0.980694427654217, 0.978689960781373,
0.976587625728023, 0.974387632725921, 0.972090201772533, 0.96969556260904,
0.967203954697364, 0.964615627196218, 0.961930838936196,
0.959149858393887, 0.956272963665028, 0.953300442436693,
0.95023259195853, 0.947069719013028, 0.943812139884847, 0.940460180329185,
0.937014175539204, 0.933474470112512, 0.929841418016701,
0.926115382553955, 0.922296736324713, 0.918385861190416,
0.914383148235319, 0.910288997727383, 0.906103819078245, 0.901828030802283,
0.897462060474762, 0.893006344689077, 0.888461329013091,
0.883827467944587, 0.879105224865808, 0.874295071997128,
0.869397490349825, 0.864412969677983, 0.859342008429514, 0.854185113696319,
0.848942801163572, 0.84361559505816, 0.838204028096251,
0.832708641430035, 0.827129984593597, 0.821468615447972,
0.815725100125357, 0.809900012972498, 0.803993936493257, 0.798007461290359,
0.791941186006336, 0.785795717263661, 0.779571669604088,
0.773269665427194, 0.766890334928147, 0.760434316034681,
0.753902254343305, 0.747294803054744, 0.740612622908621, 0.733856382117381,
0.727026756299476, 0.720124428411794, 0.713150088681373,
0.706104434536373, 0.698988170536338, 0.691802008301737,
0.684546666442807, 0.677222870487685, 0.669831352809865, 0.662372852554955,
0.654848115566766, 0.647257894312724, 0.639602947808631,
0.631884041542758, 0.624101947399299, 0.616257443581182,
0.608351314532255, 0.600384350858831, 0.592357349250641, 0.584271112401154,
0.576126448927319, 0.567924173288695, 0.55966510570601,
0.551350072079141, 0.542979903904521, 0.534555438191992,
0.526077517381105, 0.517546989256877, 0.50896470686501, 0.500331528426593,
0.491648317252275, 0.482915941655938, 0.474135274867862,
0.465307194947413, 0.456432584695223, 0.447512331564922,
0.43854732757439, 0.429538469216557, 0.420486657369749, 0.411392797207609,
0.402257798108573, 0.393082573564941, 0.38386804109152,
0.374615122133879, 0.365324741976202, 0.355997829648764,
0.346635317835026, 0.337238142778366, 0.327807244188458, 0.318343565147303,
0.30884805201492, 0.299321654334707, 0.289765324738495,
0.280180018851278, 0.27056669519566, 0.260926315095994,
0.251259842582256, 0.241568244293641, 0.231852489381904, 0.222113549414439,
0.212352398277126, 0.202570012076944, 0.192767369044364,
0.182945449435517, 0.173105235434182, 0.163247711053556,
0.153373862037864, 0.14348467576378, 0.13358114114169, 0.123664248516802,
0.113734989570117, 0.103794357219253, 0.0938433455191623,
0.0838829495627223, 0.0739141653812273, 0.06393798984479,
0.0539554205626498, 0.0439674557834159, 0.0339750942952423,
0.0239793353259525, 0.0139811784431128, 0.00398162345407974,
-0.0060183296939816, -0.0160176810140879, -0.0260154305794408,
-0.0360105786234153,
-0.0460021256395369, -0.0559890724814288, -0.0659704204627299,
-0.0759451714569599, -0.0859123279973325, -0.0958708933764978,
-0.105819871746218, -0.115758268216946, -0.125685088957318,
-0.135599341293531,
-0.145500033808614, -0.155386176441565, -0.16525678058636,
-0.17511085919081, -0.184947426855267, -0.194765499931161,
-0.204564096619365, -0.214342237068377, -0.2240989434723,
-0.233833240168624,
-0.243544153735791, -0.253230713090538, -0.262891949585,
-0.272526897103582, -0.282134592159557, -0.291714073991427,
-0.301264384658992, -0.310784569139144, -0.320273675421368,
-0.329730754602944,
-0.339154860983835, -0.348545052161256, -0.357900389123914,
-0.367219936345908, -0.376502761880283, -0.385747937452222,
-0.394954538551871, -0.404121644526792, -0.413248338674028,
-0.422333708331768,
-0.431376844970621, -0.440376844284454, -0.449332806280839,
-0.458243835371038, -0.467109040459569, -0.47592753503331,
-0.484698437250152, -0.493420870027184, -0.502093961128397,
-0.510716843251906,
-0.519288654116686, -0.527808536548793, -0.536275638567084,
-0.544689113468413, -0.553048119912302, -0.561351822005071,
-0.569599389383433, -0.57778999729752, -0.585922826693367,
-0.593997064294812,
-0.602011902684824, -0.609966540386242, -0.617860181941925,
-0.625692037994295, -0.633461325364275, -0.641167267129602,
-0.648809092702519, -0.656386037906838, -0.663897345054353,
-0.671342263020609,
-0.678720047320012, -0.686029960180282, -0.693271270616224,
-0.700443254502829, -0.707545194647683, -0.714576380862692,
-0.721536110035093, -0.728423686197768, -0.735238420598841,
-0.741979631770551,
-0.748646645597399, -0.755238795383558, -0.76175542191954,
-0.768195873548125, -0.774559506229517, -0.780845683605749,
-0.787053777064324, -0.793183165801068, -0.799233236882215,
-0.8052033853057,
-0.811093014061656, -0.816901534192113, -0.8226283648499,
-0.828272933356724, -0.833834675260437, -0.839313034391484,
-0.844707462918517, -0.850017421403178, -0.855242378854046,
-0.860381812779727,
-0.865435209241112, -0.870402062902767, -0.875281877083464,
-0.880074163805853, -0.884778443845253, -0.889394246777581,
-0.893921111026392, -0.898358583909032, -0.90270622168191,
-0.906963589584872,
-0.911130261884677, -0.915205821917566, -0.919189862130932,
-0.923081984124074, -0.926881798688036, -0.930588925844528,
-0.934202994883924, -0.937723644402332, -0.941150522337732,
-0.944483286005189,
-0.947721602131112, -0.950865146886587, -0.953913605919758,
-0.956866674387264, -0.959724056984716, -0.962485467976237,
-0.965150631223029, -0.967719280210989, -0.970191158077357,
-0.972566017636408,
-0.974843621404164, -0.977023741622146, -0.97910616028015,
-0.981090669138045, -0.982977069746599, -0.984765173467324,
-0.986454801491336, -0.988045784857242, -0.989537964468031,
-0.990931191106986,
-0.992225325452603, -0.993420238092527, -0.994515809536489,
-0.995511930228257, -0.996408500556594, -0.997205430865212,
-0.997902641461745, -0.998500062625715, -0.998997634615504,
-0.999395307674325,
-0.999693042035206, -0.999890807924959, -0.999988585567158,
-0.999986365184122, -0.999884146997886, -0.999681941230185,
-0.999379768101426, -0.998977657828671, -0.998475650622611,
-0.99787379668355,
-0.997172156196378, -0.996370799324562, -0.995469806203119,
-0.994469266930611, -0.993369281560131, -0.992169960089301,
-0.990871422449267, -0.989473798492712, -0.987977227980866,
-0.986381860569534,
-0.984687855794127, -0.982895383053711, -0.981004621594066,
-0.979015760489763, -0.976928998625255, -0.974744544674989,
-0.97246261708254, -0.970083444038766, -0.967607263458988,
-0.965034322959201,
-0.96236487983131, -0.959599201017404, -0.95673756308306,
-0.953780252189686, -0.950727564065908, -0.947579803977993,
-0.944337286699328, -0.941000336478938, -0.937569287009064,
-0.934044481391795,
-0.930426272104753, -0.926715020965855, -0.922911099097119,
-0.919014886887564, -0.915026773955164, -0.910947159107888,
-0.906776450303821, -0.902515064610368, -0.898163428162546,
-0.893721976120377,
-0.889191152625361, -0.884571410756073, -0.879863212482849,
-0.875067028621594, -0.870183338786697, -0.865212631343072,
-0.86015540335732, -0.855012160548026, -0.849783417235186,
-0.844469696288772
};

71
libdap4/d4curlfunctions.c
View File

@ -26,19 +26,12 @@
#endif
#endif
#define CHECK(state,flag,value) {if(check(state,flag,(void*)value) != NC_NOERR) {goto done;}}
#define SETCURLOPT(state,flag,value) {if(set_curlopt(state,flag,(void*)value) != NC_NOERR) {goto done;}}
/* forward */
static int set_curlflag(NCD4INFO*, int flag);
static int set_curlopt(NCD4INFO*, int flag, void* value);
static int
check(NCD4INFO* info, int flag, void* value)
{
int ret = set_curlopt(info,flag,value);
return THROW(ret);
}
/*
Set a specific curl flag; primary wrapper for curl_easy_setopt
*/
@ -64,66 +57,66 @@ set_curlflag(NCD4INFO* state, int flag)
case CURLOPT_USERPWD: /* Do both user and pwd */
if(state->auth.creds.user != NULL
&& state->auth.creds.pwd != NULL) {
CHECK(state, CURLOPT_USERNAME, state->auth.creds.user);
CHECK(state, CURLOPT_PASSWORD, state->auth.creds.pwd);
CHECK(state, CURLOPT_HTTPAUTH, (OPTARG)CURLAUTH_ANY);
SETCURLOPT(state, CURLOPT_USERNAME, state->auth.creds.user);
SETCURLOPT(state, CURLOPT_PASSWORD, state->auth.creds.pwd);
SETCURLOPT(state, CURLOPT_HTTPAUTH, (OPTARG)CURLAUTH_ANY);
}
break;
case CURLOPT_COOKIEJAR: case CURLOPT_COOKIEFILE:
if(state->auth.curlflags.cookiejar) {
/* Assume we will read and write cookies to same place */
CHECK(state, CURLOPT_COOKIEJAR, state->auth.curlflags.cookiejar);
CHECK(state, CURLOPT_COOKIEFILE, state->auth.curlflags.cookiejar);
SETCURLOPT(state, CURLOPT_COOKIEJAR, state->auth.curlflags.cookiejar);
SETCURLOPT(state, CURLOPT_COOKIEFILE, state->auth.curlflags.cookiejar);
}
break;
case CURLOPT_NETRC: case CURLOPT_NETRC_FILE:
if(state->auth.curlflags.netrc) {
CHECK(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_REQUIRED);
CHECK(state, CURLOPT_NETRC_FILE, state->auth.curlflags.netrc);
SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_REQUIRED);
SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth.curlflags.netrc);
}
break;
case CURLOPT_VERBOSE:
if(state->auth.curlflags.verbose)
CHECK(state, CURLOPT_VERBOSE, (OPTARG)1L);
SETCURLOPT(state, CURLOPT_VERBOSE, (OPTARG)1L);
break;
case CURLOPT_TIMEOUT:
if(state->auth.curlflags.timeout)
CHECK(state, CURLOPT_TIMEOUT, (OPTARG)((long)state->auth.curlflags.timeout));
SETCURLOPT(state, CURLOPT_TIMEOUT, (OPTARG)((long)state->auth.curlflags.timeout));
break;
case CURLOPT_CONNECTTIMEOUT:
if(state->auth.curlflags.connecttimeout)
CHECK(state, CURLOPT_CONNECTTIMEOUT, (OPTARG)((long)state->auth.curlflags.connecttimeout));
SETCURLOPT(state, CURLOPT_CONNECTTIMEOUT, (OPTARG)((long)state->auth.curlflags.connecttimeout));
break;
case CURLOPT_USERAGENT:
if(state->auth.curlflags.useragent)
CHECK(state, CURLOPT_USERAGENT, state->auth.curlflags.useragent);
SETCURLOPT(state, CURLOPT_USERAGENT, state->auth.curlflags.useragent);
break;
case CURLOPT_FOLLOWLOCATION:
CHECK(state, CURLOPT_FOLLOWLOCATION, (OPTARG)1L);
SETCURLOPT(state, CURLOPT_FOLLOWLOCATION, (OPTARG)1L);
break;
case CURLOPT_MAXREDIRS:
CHECK(state, CURLOPT_MAXREDIRS, (OPTARG)MAX_REDIRECTS);
SETCURLOPT(state, CURLOPT_MAXREDIRS, (OPTARG)MAX_REDIRECTS);
break;
case CURLOPT_ERRORBUFFER:
CHECK(state, CURLOPT_ERRORBUFFER, state->curl->errdata.errorbuf);
SETCURLOPT(state, CURLOPT_ERRORBUFFER, state->curl->errdata.errorbuf);
break;
case CURLOPT_ENCODING:
#ifdef CURLOPT_ENCODING
if(state->auth.curlflags.compress) {
CHECK(state, CURLOPT_ENCODING,"deflate, gzip");
SETCURLOPT(state, CURLOPT_ENCODING,"deflate, gzip");
}
#endif
break;
case CURLOPT_PROXY:
if(state->auth.proxy.host != NULL) {
CHECK(state, CURLOPT_PROXY, state->auth.proxy.host);
CHECK(state, CURLOPT_PROXYPORT, (OPTARG)(long)state->auth.proxy.port);
SETCURLOPT(state, CURLOPT_PROXY, state->auth.proxy.host);
SETCURLOPT(state, CURLOPT_PROXYPORT, (OPTARG)(long)state->auth.proxy.port);
if(state->auth.proxy.user != NULL
&& state->auth.proxy.pwd != NULL) {
CHECK(state, CURLOPT_PROXYUSERNAME, state->auth.proxy.user);
CHECK(state, CURLOPT_PROXYPASSWORD, state->auth.proxy.pwd);
SETCURLOPT(state, CURLOPT_PROXYUSERNAME, state->auth.proxy.user);
SETCURLOPT(state, CURLOPT_PROXYPASSWORD, state->auth.proxy.pwd);
#ifdef CURLOPT_PROXYAUTH
CHECK(state, CURLOPT_PROXYAUTH, (long)CURLAUTH_ANY);
SETCURLOPT(state, CURLOPT_PROXYAUTH, (long)CURLAUTH_ANY);
#endif
}
}
@ -137,39 +130,39 @@ set_curlflag(NCD4INFO* state, int flag)
/* We need to have 2 states: default and a set value */
/* So -1 => default, >= 0 => use value; */
if(ssl->verifypeer >= 0)
CHECK(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(ssl->verifypeer));
SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(ssl->verifypeer));
#ifdef HAVE_LIBCURL_766
if(ssl->verifyhost >= 0)
CHECK(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(ssl->verifyhost));
SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(ssl->verifyhost));
#endif
if(ssl->certificate)
CHECK(state, CURLOPT_SSLCERT, ssl->certificate);
SETCURLOPT(state, CURLOPT_SSLCERT, ssl->certificate);
if(ssl->key)
CHECK(state, CURLOPT_SSLKEY, ssl->key);
SETCURLOPT(state, CURLOPT_SSLKEY, ssl->key);
if(ssl->keypasswd)
/* libcurl prior to 7.16.4 used 'CURLOPT_SSLKEYPASSWD' */
CHECK(state, CURLOPT_KEYPASSWD, ssl->keypasswd);
SETCURLOPT(state, CURLOPT_KEYPASSWD, ssl->keypasswd);
if(ssl->cainfo)
CHECK(state, CURLOPT_CAINFO, ssl->cainfo);
SETCURLOPT(state, CURLOPT_CAINFO, ssl->cainfo);
if(ssl->capath)
CHECK(state, CURLOPT_CAPATH, ssl->capath);
SETCURLOPT(state, CURLOPT_CAPATH, ssl->capath);
}
break;
#ifdef HAVE_CURLOPT_BUFFERSIZE
case CURLOPT_BUFFERSIZE:
CHECK(state, CURLOPT_BUFFERSIZE, (OPTARG)state->curl->buffersize);
SETCURLOPT(state, CURLOPT_BUFFERSIZE, (OPTARG)state->curl->buffersize);
break;
#endif
#ifdef HAVE_CURLOPT_KEEPALIVE
case CURLOPT_TCP_KEEPALIVE:
if(state->curl->keepalive.active != 0)
CHECK(state, CURLOPT_TCP_KEEPALIVE, (OPTARG)1L);
SETCURLOPT(state, CURLOPT_TCP_KEEPALIVE, (OPTARG)1L);
if(state->curl->keepalive.idle > 0)
CHECK(state, CURLOPT_TCP_KEEPIDLE, (OPTARG)state->curl->keepalive.idle);
SETCURLOPT(state, CURLOPT_TCP_KEEPIDLE, (OPTARG)state->curl->keepalive.idle);
if(state->curl->keepalive.interval > 0)
CHECK(state, CURLOPT_TCP_KEEPINTVL, (OPTARG)state->curl->keepalive.interval);
SETCURLOPT(state, CURLOPT_TCP_KEEPINTVL, (OPTARG)state->curl->keepalive.interval);
break;
#endif

13
libdap4/d4file.c
View File

@ -78,7 +78,7 @@ NCD4_open(const char * path, int mode,
/* fail if we are unconstrainable but have constraints */
if(FLAGSET(d4info->controls.flags,NCF_UNCONSTRAINABLE)) {
if(d4info->uri->query != NULL) {
if(d4info->uri != NULL && d4info->uri->query != NULL) {
nclog(NCLOGWARN,"Attempt to constrain an unconstrainable data source: %s",
d4info->uri->query);
ret = THROW(NC_EDAPCONSTRAINT);
@ -131,10 +131,7 @@ NCD4_open(const char * path, int mode,
}
/* Turn on logging; only do this after oc_open*/
if((value = ncurilookup(d4info->uri,"log")) != NULL) {
ncloginit();
if(nclogopen(value))
ncsetlogging(1);
if((value = ncurifragmentlookup(d4info->uri,"log")) != NULL) {
ncloginit();
if(nclogopen(value))
ncsetlogging(1);
@ -170,7 +167,7 @@ NCD4_open(const char * path, int mode,
/* if the url goes astray to a random web page, then try to just dump it */
{
char* response = ncbytescontents(d4info->curl->packet);
char* response = (char*)ncbytescontents(d4info->curl->packet);
size_t responselen = ncbyteslength(d4info->curl->packet);
/* Apply some heuristics to see what we have.
@ -178,7 +175,7 @@ NCD4_open(const char * path, int mode,
be less than 0x0f (for now). However, it will not be zero if
the data was little-endian
*/
if(responselen == 0 || response[0] >= ' ') {
if(responselen == 0 || ((unsigned char*)response)[0] > 0x0f) {
/* does not look like a chunk, so probable server failure */
if(responselen == 0)
nclog(NCLOGERR,"Empty DAP4 response");
@ -523,7 +520,7 @@ getparam(NCD4INFO* info, const char* key)
const char* value;
if(info == NULL || key == NULL) return NULL;
if((value=ncurilookup(info->uri,key)) == NULL)
if((value=ncurifragmentlookup(info->uri,key)) == NULL)
return NULL;
return value;
}

3
libdap4/d4http.c
View File

@ -144,7 +144,8 @@ fail:
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
ret = curlerrtoncerr(cstat);
} else switch (httpcode) {
case 401: ret = NC_EAUTH; break;
case 401: ret = NC_EACCESS; break;
case 403: ret = NC_EAUTH; break;
case 404: ret = ENOENT; break;
case 500: ret = NC_EDAPSVC; break;
case 200: break;

3
libdap4/d4includes.h
View File

@ -15,6 +15,9 @@
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include <curl/curl.h>

2
libdispatch/CMakeLists.txt
View File

@ -7,7 +7,7 @@
SET(libdispatch_SOURCES dparallel.c dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dwinpath.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c crc32.c daux.c dinfermodel.c)
# Netcdf-4 only functions. Must be defined even if not used
SET(libdispatch_SOURCES ${libdispatch_SOURCES} dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c)
SET(libdispatch_SOURCES ${libdispatch_SOURCES} dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c dfilterx.c)
IF(BUILD_V2)
SET(libdispatch_SOURCES ${libdispatch_SOURCES} dv2i.c)

2
libdispatch/Makefile.am
View File

@ -33,7 +33,7 @@ libdispatch_la_SOURCES += drc.c
# Add functions only found in netCDF-4.
# They are always defined, even if they just return an error
libdispatch_la_SOURCES += dgroup.c dvlen.c dcompound.c dtype.c denum.c \
dopaque.c dfilter.c
dopaque.c dfilter.c dfilterx.c
# Add V2 API convenience library if needed.
if BUILD_V2

2
libdispatch/datt.c
View File

@ -5,6 +5,8 @@
*
* These functions in this file rename and delete attributes.
*/
#include "config.h"
#include "ncdispatch.h"
/**

1
libdispatch/dauth.c
View File

@ -180,6 +180,7 @@ NC_authsetup(NCauth* auth, NCURI* uri)
void
NC_authclear(NCauth* auth)
{
if(auth == NULL) return;
if(auth->curlflags.cookiejarcreated) {
#ifdef _MSC_VER
DeleteFile(auth->curlflags.cookiejar);

537
libdispatch/daux.c
View File

@ -18,6 +18,7 @@ See COPYRIGHT for license information.
* access to either file, you may request a copy from help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include "config.h"
#include <stdlib.h>
#include <string.h>
#include <assert.h>
@ -26,6 +27,8 @@ See COPYRIGHT for license information.
#include "netcdf_aux.h"
#include "ncoffsets.h"
#include "nclog.h"
#include "netcdf_filter.h"
#include "ncfilter.h"
struct NCAUX_FIELD {
char* name;
@ -63,10 +66,10 @@ static int reclaim_compound(int ncid, int xtype, size_t size, size_t nfields, Po
static int reclaim_vlen(int ncid, int xtype, int basetype, Position* offset);
static int reclaim_enum(int ncid, int xtype, int basetype, size_t, Position* offset);
static int reclaim_opaque(int ncid, int xtype, size_t size, Position* offset);
static int computefieldinfo(struct NCAUX_CMPD* cmpd);
#endif /* USE_NETCDF4 */
static void ncaux_freestringvec(int n, char** vec);
/**************************************************/
/**
@ -407,7 +410,7 @@ done:
/**
@param ncclass - type class for which alignment is requested; excludes ENUM|COMPOUND
*/
EXTERNL size_t
size_t
ncaux_class_alignment(int ncclass)
{
if(ncclass <= NC_MAX_ATOMIC_TYPE || ncclass == NC_VLEN || ncclass == NC_OPAQUE)
@ -420,7 +423,7 @@ ncaux_class_alignment(int ncclass)
@param ncid - only needed for a compound type
@param xtype - type for which alignment is requested
*/
EXTERNL size_t
size_t
ncaux_type_alignment(int xtype, int ncid)
{
if(!ncaux_initialized) {
@ -536,3 +539,531 @@ done:
}
#endif /*USE_NETCDF4*/
/**************************************************/
/* Forward */
#define NUMCHAR "0123456789"
#define LPAREN '('
#define RPAREN ')'
#define LBRACK '['
#define RBRACK ']'
static int gettype(const int q0, const int q1, int* unsignedp);
const struct LegalFormat {
const char* tag;
int format;
} legalformats[] = {
{"hdf5", NC_FILTER_FORMAT_HDF5},
{NULL, 0},
};
void
ncaux_filterfix8(unsigned char* mem, int decode)
{
#ifdef WORDS_BIGENDIAN
if(decode) { /* Apply inverse of the encode case */
byteswap4(mem); /* step 1: byte-swap each piece */
byteswap4(mem+4);
byteswap8(mem); /* step 2: convert to little endian format */
} else { /* encode */
byteswap8(mem); /* step 1: convert to little endian format */
byteswap4(mem); /* step 2: byte-swap each piece */
byteswap4(mem+4);
}
#else /* Little endian */
/* No action is necessary */
#endif
}
/* Look at q0 and q1) to determine type */
static int
gettype(const int q0, const int q1, int* isunsignedp)
{
int type = 0;
int isunsigned = 0;
char typechar;
isunsigned = (q0 == 'u' || q0 == 'U');
if(q1 == '\0')
typechar = q0; /* we were given only a single char */
else if(isunsigned)
typechar = q1; /* we have something like Ux as the tag */
else
typechar = q1; /* look at last char for tag */
switch (typechar) {
case 'f': case 'F': case '.': type = 'f'; break; /* float */
case 'd': case 'D': type = 'd'; break; /* double */
case 'b': case 'B': type = 'b'; break; /* byte */
case 's': case 'S': type = 's'; break; /* short */
case 'l': case 'L': type = 'l'; break; /* long long */
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': type = 'i'; break;
case 'u': case 'U': type = 'i'; isunsigned = 1; break; /* unsigned int */
case '\0': type = 'i'; break;
default: break;
}
if(isunsignedp) *isunsignedp = isunsigned;
return type;
}
#ifdef WORDS_BIGENDIAN
/* Byte swap an 8-byte integer in place */
static void
byteswap8(unsigned char* mem)
{
unsigned char c;
c = mem[0];
mem[0] = mem[7];
mem[7] = c;
c = mem[1];
mem[1] = mem[6];
mem[6] = c;
c = mem[2];
mem[2] = mem[5];
mem[5] = c;
c = mem[3];
mem[3] = mem[4];
mem[4] = c;
}
/* Byte swap an 8-byte integer in place */
static void
byteswap4(unsigned char* mem)
{
unsigned char c;
c = mem[0];
mem[0] = mem[3];
mem[3] = c;
c = mem[1];
mem[1] = mem[2];
mem[2] = c;
}
#endif
/*
Parse a filter spec string into a NC_FILTER_SPEC*
@param txt - a string containing the spec as a sequence of
constants separated by commas.
@param specp - store the parsed filter here -- caller frees
@return NC_NOERR if parse succeeded
@return NC_EINVAL otherwise
*/
EXTERNL int
ncaux_filterspec_parse(const char* txt, NC_Filterspec** specp)
{
int i,stat = NC_NOERR;
char* p;
char* sdata0 = NULL; /* what to free */
char* sdata = NULL; /* sdata0 with leading prefix skipped */
size_t nparams; /* no. of comma delimited params */
char* filterid = NULL;
char** params = NULL;
size_t len;
NC_Filterspec* pfs = NULL;
if(txt == NULL)
{stat = NC_EINVAL; goto done;}
len = strlen(txt);
if(len == 0)
{stat = NC_EINVAL; goto done;}
if((sdata0 = (char*)calloc(1,len+1+1))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(sdata0,txt,len);
sdata = sdata0;
/* Count number of parameters + id and delimit */
p=sdata;
for(nparams=0;;nparams++) {
char* q = strchr(p,',');
if(q == NULL) break;
*q++ = '\0';
p = q;
}
nparams++; /* for final piece */
if(nparams == 0)
{stat = NC_EINVAL; goto done;} /* no id and no parameters */
p = sdata;
/* Extract the filter id */
if((filterid=strdup(p))==NULL)
{stat = NC_ENOMEM; goto done;}
nparams--;
/* skip past the filter id */
p = p + strlen(p) + 1;
/* Allocate the max needed space */
if((params = (char**)calloc(sizeof(char*),(nparams)))==NULL)
{stat = NC_ENOMEM; goto done;}
/* walk and capture */
for(i=0;i<nparams;i++) { /* step thru param strings */
len = strlen(p);
/* skip leading white space */
while(strchr(" ",*p) != NULL) {p++; len--;}
if((params[i] = strdup(p))==NULL) goto done;
p = p + strlen(p) + 1; /* move to next param */
}
/* Now return results */
if(*specp != NULL) abort();
{
pfs = calloc(1,sizeof(NC_Filterspec));
if(pfs == NULL) {stat = NC_ENOMEM; goto done;}
pfs->version = NCAUX_FILTERSPEC_VERSION;
pfs->filterid = filterid; filterid = NULL;
pfs->nparams = nparams;;
pfs->params = params; params = NULL;
*specp = pfs; pfs = NULL;
}
done:
ncaux_filterspec_free(pfs);
nullfree(sdata);
nullfree(filterid);
ncaux_freestringvec(nparams,params);
return stat;
}
/*
Parse a string containing multiple '|' separated filter specs.
@param spec0 - a string containing the list of filter specs.
@param nspecsp - # of parsed specs
@param specsp - pointer to hold vector of parsed specs. Caller frees
@return NC_NOERR if parse succeeded
@return NC_EINVAL if bad parameters or parse failed
*/
EXTERNL int
ncaux_filterspec_parselist(const char* txt0, char** formatp, size_t* nspecsp, NC_Filterspec*** vectorp)
{
int stat = NC_NOERR;
char* format = NULL;
size_t len = 0;
size_t nspecs = 0;
NC_Filterspec** vector = NULL;
char* spec0 = NULL; /* with prefix */
char* spec = NULL; /* without prefix */
char* p = NULL;
char* q = NULL;
if(txt0 == NULL) return NC_EINVAL;
/* Duplicate txt0 so we can modify it */
len = strlen(txt0);
if((spec = calloc(1,len+1+1)) == NULL) {stat = NC_ENOMEM; goto done;}
memcpy(spec,txt0,len); /* Note double ending nul */
spec0 = spec; /* Save for later free */
/* See if there is a prefix '[format]' tag */
if(spec[0] == LBRACK) {
p = spec + 1;
q = strchr(p,RBRACK);
if(q == NULL) {stat = NC_EINVAL; goto done;}
*q++ = '\0'; /* delimit tag */
if((format = strdup(p))==NULL)
{stat = NC_ENOMEM; goto done;}
spec = q; /* skip tag wrt later processing */
}
/* pass 1: count number of specs */
p = spec;
nspecs = 0;
while(*p) {
q = strchr(p,'|');
if(q == NULL) q = p + strlen(p); /* fake it */
nspecs++;
p = q + 1;
}
if(nspecs > 0) {
int count = 0;
if((vector = (NC_Filterspec**)calloc(sizeof(NC_Filterspec*),nspecs)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* pass 2: parse */
p = spec;
for(count=0;count<nspecs;count++) {
NC_Filterspec* aspec = NULL;
q = strchr(p,'|');
if(q == NULL) q = p + strlen(p); /* fake it */
*q = '\0';
if((stat=ncaux_filterspec_parse(p,&aspec))) goto done;
vector[count] = aspec; aspec = NULL;
p = q+1; /* ok because of double nul */
}
}
if(formatp) {*formatp = format; format = NULL;}
if(nspecsp) *nspecsp = nspecs;
if(vectorp) {*vectorp = vector; vector = NULL;}
done:
nullfree(spec0);
nullfree(format);
if(vector) {
int i;
for(i=0;i<nspecs;i++)
ncaux_filterspec_free(vector[i]);
nullfree(vector);
}
return stat;
}
void
ncaux_filterspec_free(NC_Filterspec* spec)
{
if(spec) {
nullfree(spec->filterid);
if(spec->params != NULL)
ncaux_freestringvec(spec->nparams,spec->params);
free(spec);
}
}
static void
ncaux_freestringvec(int n, char** vec)
{
int i;
if(vec) {
for(i=0;i<n;i++)
nullfree(vec[i]);
nullfree(vec);
}
}
/*
Convert an NC_Filterspec to equivalent NC_H5_Filterspec.
@param spec - (in) NC_Filterspec instance
@param spech5p - (out) NC_H5_Filterspec poinbter
@return NC_NOERR if parse succeeded
@return NC_EINVAL if bad parameters or parse failed
*/
EXTERNL int
ncaux_filterspec_cvt(const NC_Filterspec* spec, NC_H5_Filterspec** spech5p)
{
int i,stat = NC_NOERR;
NC_H5_Filterspec* h5spec = NULL;
unsigned int* h5params = NULL;
size_t nparams = 0; /*actual count*/
if((h5spec = calloc(1,sizeof(NC_H5_Filterspec))) == NULL)
{stat = NC_ENOMEM; goto done;}
/* Try to name convert the filter id */
if((stat = NC_cvtX2I_id(spec->filterid,&h5spec->filterid))) goto done;
if(spec->nparams > 0) {
if((h5params = (unsigned int*)calloc(sizeof(unsigned int),(spec->nparams)*2)) == NULL) /* Assume all are 8-byte */
{stat = NC_ENOMEM; goto done;}
/* Walk and convert */
for(nparams=0,i=0;i<spec->nparams;i++) {
unsigned long long val64u;
unsigned int val32u;
double vald;
float valf;
unsigned int *vector;
unsigned char mem[8];
int isunsigned = 0;
int isnegative = 0;
int type = 0;
char* q;
char* p = spec->params[i];
size_t len = strlen(p);
int sstat;
/* skip leading white space */
while(strchr(" ",*p) != NULL) {p++; len--;}
/* Get leading sign character, if any */
if(*p == '-') isnegative = 1;
/* Get trailing type tag characters */
switch (len) {
case 0: stat = NC_EINVAL; goto done; /* empty parameter */
case 1: case 2:
q = (p + len) - 1; /* point to last char */
type = gettype(*q,'\0',&isunsigned);
break;
default: /* > 2 => we might have a two letter tag */
q = (p + len) - 2;
type = gettype(*q,*(q+1),&isunsigned);
break;
}
/* Now parse */
switch (type) {
case 'b': case 's': case 'i':
/* special case for a positive integer;for back compatibility.*/
if(!isnegative)
sstat = sscanf(p,"%u",&val32u);
else
sstat = sscanf(p,"%d",(int*)&val32u);
if(sstat != 1) {stat = NC_EINVAL; goto done;}
switch(type) {
case 'b': val32u = (val32u & 0xFF); break;
case 's': val32u = (val32u & 0xFFFF); break;
}
h5params[nparams++] = val32u;
break;
case 'f':
sstat = sscanf(p,"%lf",&vald);
if(sstat != 1) {stat = NC_EINVAL; goto done;}
valf = (float)vald;
h5params[nparams++] = *(unsigned int*)&valf;
break;
/* The following are 8-byte values, so we must swap pieces if this
is a little endian machine */
case 'd':
sstat = sscanf(p,"%lf",&vald);
if(sstat != 1) {stat = NC_EINVAL; goto done;};
memcpy(mem,&vald,sizeof(mem));
ncaux_filterfix8(mem,0);
vector = (unsigned int*)mem;
h5params[nparams++] = vector[0];
h5params[nparams++] = vector[1];
break;
case 'l': /* long long */
if(isunsigned)
sstat = sscanf(p,"%llu",&val64u);
else
sstat = sscanf(p,"%lld",(long long*)&val64u);
if(sstat != 1) {stat = NC_EINVAL; goto done;};
memcpy(mem,&val64u,sizeof(mem));
ncaux_filterfix8(mem,0);
vector = (unsigned int*)&mem;
h5params[nparams++] = vector[0];
h5params[nparams++] = vector[1];
break;
default:
{stat = NC_EINVAL; goto done;};
}
}
h5spec->params = h5params;
h5params = NULL;
}
if(spech5p) {*spech5p = h5spec; h5spec = NULL;}
done:
if(h5spec) {
nullfree(h5spec->params);
nullfree(h5spec);
}
return stat;
}
#if 0
/*
Parse a filter spec string into a NC_H5_Filterspec*
@param txt - a string containing the spec as a sequence of
constants separated by commas.
@param specp - store the parsed filter here -- caller frees
@return NC_NOERR if parse succeeded
@return NC_EINVAL otherwise
*/
EXTERNL int
ncaux_filter_parsespec(const char* txt, NC_H5_Filterspec** h5specp)
{
int stat = NC_NOERR;
NC_Filterspec* spec = NULL;
NC_H5_Filterspec* h5spec = NULL;
size_t len;
if(txt == NULL)
{stat = NC_EINVAL; goto done;}
len = strlen(txt);
if(len == 0) {stat = NC_EINVAL; goto done;}
/* Parse as strings */
if((stat = ncaux_filterspec_parse(txt,&spec))) goto done;
/* walk and convert */
if((stat = ncaux_filterspec_cvt(spec,&h5spec))) goto done;
/* Now return results */
if(h5specp != NULL) {*h5specp = h5spec; h5spec = NULL;}
done:
ncaux_filterspec_free(spec);
if(h5spec) nullfree(h5spec->params);
nullfree(h5spec);
return stat;
}
/*
Parse a string containing multiple '|' separated filter specs.
@param spec0 - a string containing the list of filter specs.
@param nspecsp - # of parsed specs
@param specsp - pointer to hold vector of parsed specs. Caller frees
@return NC_NOERR if parse succeeded
@return NC_EINVAL if bad parameters or parse failed
*/
EXTERNL int
ncaux_filter_parselist(const char* txt0, size_t* nspecsp, NC_H5_Filterspec*** vectorp)
{
int stat = NC_NOERR;
size_t len = 0;
size_t nspecs = 0;
NC_H5_Filterspec** vector = NULL;
char* spec0 = NULL; /* with prefix */
char* spec = NULL; /* without prefix */
char* p = NULL;
char* q = NULL;
if(txt0 == NULL) return NC_EINVAL;
/* Duplicate txt0 so we can modify it */
len = strlen(txt0);
if((spec = calloc(1,len+1+1)) == NULL) return NC_ENOMEM;
memcpy(spec,txt0,len); /* Note double ending nul */
spec0 = spec; /* Save for later free */
/* See if there is a prefix '[format]' tag; ignore it */
if(spec[0] == LBRACK) {
spec = q; /* skip tag wrt later processing */
}
/* pass 1: count number of specs */
p = spec;
nspecs = 0;
while(*p) {
q = strchr(p,'|');
if(q == NULL) q = p + strlen(p); /* fake it */
nspecs++;
p = q + 1;
}
if(nspecs > 0) {
int count = 0;
if((vector = (NC_H5_Filterspec**)malloc(sizeof(NC_H5_Filterspec*)*nspecs)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* pass 2: parse */
p = spec;
for(count=0;count<nspecs;count++) {
NC_H5_Filterspec* aspec = NULL;
q = strchr(p,'|');
if(q == NULL) q = p + strlen(p); /* fake it */
*q = '\0';
if(ncaux_filter_parsespec(p,&aspec))
{stat = NC_EINVAL; goto done;}
vector[count] = aspec; aspec = NULL;
p = q+1; /* ok because of double nul */
}
}
if(nspecsp) *nspecsp = nspecs;
if(vectorp) *vectorp = (nspecs == 0 ? NULL : vector);
vector = NULL;
done:
nullfree(spec0);
if(vector != NULL) {
int k;
for(k=0;k<nspecs;k++) {
NC_H5_Filterspec* nfs = vector[k];
if(nfs->params) free(nfs->params);
nullfree(nfs);
}
free(vector);
}
return stat;
}
#endif

1
libdispatch/dcopy.c
View File

@ -7,6 +7,7 @@
*
* @author Dennis Heimbigner
*/
#include "config.h"
#include "ncdispatch.h"
#include "nc_logging.h"

1
libdispatch/ddispatch.c
View File

@ -27,6 +27,7 @@ See LICENSE.txt for license information.
#endif
/* Define vectors of zeros and ones for use with various nc_get_varX functions */
/* Note, this form of initialization fails under Cygwin */
size_t NC_coord_zero[NC_MAX_VAR_DIMS] = {0};
size_t NC_coord_one[NC_MAX_VAR_DIMS] = {1};
ptrdiff_t NC_stride_one[NC_MAX_VAR_DIMS] = {1};

9
libdispatch/derror.c
View File

@ -7,6 +7,7 @@ Copyright 2018 University Corporation for Atmospheric
Research/Unidata. See COPYRIGHT file for more info.
*/
#include "config.h"
#include "ncdispatch.h"
#ifdef USE_PNETCDF
#include <pnetcdf.h> /* for ncmpi_strerror() */
@ -171,7 +172,7 @@ const char *nc_strerror(int ncerr1)
case NC_EIO:
return "NetCDF: I/O failure";
case NC_ENODATA:
return "NetCDF: Variable has no data in DAP request";
return "NetCDF: Variable has no data";
case NC_EDAPSVC:
return "NetCDF: DAP server error";
case NC_EDAS:
@ -270,6 +271,12 @@ const char *nc_strerror(int ncerr1)
return "NetCDF: File fails strict Null-Byte Header check.";
case NC_EINMEMORY:
return "NetCDF: In-memory File operation failed.";
case NC_ENCZARR:
return "NetCDF: NCZarr error";
case NC_ES3:
return "NetCDF: AWS S3 error";
case NC_EEMPTY:
return "NetCDF: Attempt to read empty NCZarr map key";
default:
#ifdef USE_PNETCDF
/* The behavior of ncmpi_strerror here is to return

33
libdispatch/dfile.c
View File

@ -1906,6 +1906,11 @@ NC_create(const char *path0, int cmode, size_t initialsz,
dispatcher = UDF1_dispatch_table;
break;
#endif /* USE_NETCDF4 */
#ifdef ENABLE_NCZARR
case NC_FORMATX_NCZARR:
dispatcher = NCZ_dispatch_table;
break;
#endif
case NC_FORMATX_NC3:
dispatcher = NC3_dispatch_table;
break;
@ -2016,6 +2021,7 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
if(newpath) {
nullfree(path);
path = newpath;
newpath = NULL;
}
/* Still no implementation, give up */
@ -2027,12 +2033,14 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
}
/* Suppress unsupported formats */
/* (should be more compact, table-driven, way to do this) */
{
int hdf5built = 0;
int hdf4built = 0;
int cdf5built = 0;
int udf0built = 0;
int udf1built = 0;
int hdf5built = 0;
int hdf4built = 0;
int cdf5built = 0;
int udf0built = 0;
int udf1built = 0;
int nczarrbuilt = 0;
#ifdef USE_NETCDF4
hdf5built = 1;
#ifdef USE_HDF4
@ -2041,6 +2049,9 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
#endif
#ifdef ENABLE_CDF5
cdf5built = 1;
#endif
#ifdef ENABLE_NCZARR
nczarrbuilt = 1;
#endif
if(UDF0_dispatch_table != NULL)
udf0built = 1;
@ -2053,6 +2064,8 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
{stat = NC_ENOTBUILT; goto done;}
if(!cdf5built && model.impl == NC_FORMATX_NC3 && model.format == NC_FORMAT_CDF5)
{stat = NC_ENOTBUILT; goto done;}
if(!nczarrbuilt && model.impl == NC_FORMATX_NCZARR)
{stat = NC_ENOTBUILT; goto done;}
if(!udf0built && model.impl == NC_FORMATX_UDF0)
{stat = NC_ENOTBUILT; goto done;}
if(!udf1built && model.impl == NC_FORMATX_UDF1)
@ -2071,6 +2084,11 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
dispatcher = NCD4_dispatch_table;
break;
#endif
#ifdef ENABLE_NCZARR
case NC_FORMATX_NCZARR:
dispatcher = NCZ_dispatch_table;
break;
#endif
#ifdef USE_PNETCDF
case NC_FORMATX_PNETCDF:
dispatcher = NCP_dispatch_table;
@ -2098,8 +2116,8 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
dispatcher = NC3_dispatch_table;
break;
default:
nullfree(path);
return NC_ENOTNC;
stat = NC_ENOTNC;
goto done;
}
}
@ -2125,6 +2143,7 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
done:
nullfree(path);
nullfree(newpath);
return stat;
}

580
libdispatch/dfilter.c
View File

@ -15,6 +15,7 @@
#include "netcdf_filter.h"
#include "ncdispatch.h"
#include "nc4internal.h"
#include "ncfilter.h"
#ifdef USE_HDF5
#include "hdf5internal.h"
@ -29,370 +30,23 @@ Unified filter related code
#define H5Z_FILTER_SZIP 4
#endif
#define LPAREN '('
#define RPAREN ')'
#define LBRACK '['
#define RBRACK ']'
#define NUMCHAR "0123456789"
#define NAMECHAR1 "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
#define NAMECHARN (NAMECHAR1 NUMCHAR "_-")
/* Forward */
static int gettype(const int q0, const int q1, int* unsignedp);
#ifdef WORDS_BIGENDIAN
static void byteswap8(unsigned char* mem);
static void byteswap4(unsigned char* mem);
#endif
const struct LegalFormat {
const char* tag;
int format;
} legalformats[] = {
{"hdf5", NC_FILTER_FORMAT_HDF5},
{NULL, 0},
};
const struct FilterName {
const char* name; /* name or alias as assigned by HDF group*/
unsigned int id; /* id as assigned by HDF group*/
} known_filters[] = {
{"zip", 2}, /* Standard zlib compression */
{"zlib", 2}, /* alias */
{"deflate", 2}, /* alias */
{"szip", 4}, /* Standard szip compression */
{"bzip2", 307}, /* BZIP2 lossless compression used by PyTables */
{"lzf", 32000}, /* LZF lossless compression used by H5Py project */
{"blosc", 32001}, /* Blosc lossless compression used by PyTables */
{"mafisc", 32002}, /* Modified LZMA compression filter, MAFISC (Multidimensional Adaptive Filtering Improved Scientific data Compression) */
{"snappy", 32003}, /* Snappy lossless compression. */
{"lz4", 32004}, /* LZ4 fast lossless compression algorithm */
{"apax", 32005}, /* Samplify's APAX Numerical Encoding Technology */
{"cbf", 32006}, /* All imgCIF/CBF compressions and decompressions, including Canonical, Packed, Packed Vesrsion 2, Byte Offset and Nibble Offset. */
{"jpeg-xr", 32007}, /* Enables images to be compressed/decompressed with JPEG-XR compression */
{"bitshuffle", 32008}, /* Extreme version of shuffle filter that shuffles data at bit level instead of byte level. */
{"spdp", 32009}, /* SPDP fast lossless compression algorithm for single- and double-precision floating-point data. */
{"lpc-rice", 32010}, /* LPC-Rice multi-threaded lossless compression */
{"ccsds-123", 32011}, /* ESA CCSDS-123 multi-threaded compression filter */
{"jpeg-ls", 32012}, /* CharLS JPEG-LS multi-threaded compression filter */
{"zfp", 32013}, /* Rate, accuracy or precision bounded compression for floating-point arrays */
{"fpzip", 32014}, /* Fast and Efficient Lossy or Lossless Compressor for Floating-Point Data */
{"zstandard", 32015}, /* Real-time compression algorithm with wide range of compression / speed trade-off and fast decoder */
{"b3d", 32016}, /* GPU based image compression method developed for light-microscopy applications */
{"sz", 32017}, /* An error-bounded lossy compressor for scientific floating-point data */
{"fcidecomp", 32018}, /* EUMETSAT CharLS compression filter for use with netCDF */
{"user-defined", 32768}, /* First user-defined filter */
{NULL,0}
};
/*Mnemonic*/
#define USENAME 1
/**************************************************/
/*
Parse a filter spec string into a NC_FILTER_SPEC*
@param txt - a string containing the spec as a sequence of
constants separated by commas.
@param specp - store the parsed filter here -- caller frees
@return NC_NOERR if parse succeeded
@return NC_EINVAL otherwise
*/
EXTERNL int
NC_parsefilterspec(const char* txt, int format, NC_Filterspec** specp)
{
int stat = NC_NOERR;
int sstat; /* for scanf */
char* p;
char* sdata0 = NULL; /* what to free */
char* sdata = NULL; /* sdata0 with leading prefix skipped */
unsigned int id;
size_t count; /* no. of comma delimited params */
size_t nparams; /* final no. of unsigned ints */
size_t len;
int i;
unsigned int* ulist = NULL;
unsigned char mem[8];
if(txt == NULL) goto fail;
len = strlen(txt);
if(len == 0) goto fail;
sdata0 = (char*)calloc(1,len+1+1);
if(sdata0 == NULL) return 0;
memcpy(sdata0,txt,len);
sdata = sdata0;
/* Count number of parameters + id and delimit */
p=sdata;
for(count=0;;count++) {
char* q = strchr(p,',');
if(q == NULL) break;
*q++ = '\0';
p = q;
}
count++; /* for final piece */
if(count == 0)
goto fail; /* no id and no parameters */
/* Extract the filter id */
p = sdata;
if(strchr(NAMECHAR1,*p) != NULL) {
const struct FilterName* candidate = known_filters;
char* q = p+1;
while(*q && strchr(NAMECHARN,*q) != NULL) {
q++;
}
if(*q) goto fail; /*name has bad char*/
/* Lookup name */
id = 0; /* => not found */
for(;candidate->name;candidate++) {
if(strcasecmp(candidate->name,p)==0) {id = candidate->id; break;}
}
if(id == 0) goto fail; /* unknown name */
} else if(strchr(NUMCHAR,*p) != NULL) {
sstat = sscanf(p,"%u",&id);
if(sstat != 1) goto fail;
} else goto fail; /* unparseable id */
count--;
/* skip past the filter id */
p = p + strlen(p) + 1;
/* Allocate the max needed space; *2 in case the params are all doubles */
ulist = (unsigned int*)malloc(sizeof(unsigned int)*(count)*2);
if(ulist == NULL) goto fail;
/* walk and convert */
nparams = 0; /* actual count */
for(i=0;i<count;i++) { /* step thru param strings */
unsigned long long val64u;
unsigned int val32u;
double vald;
float valf;
unsigned int *vector;
int isunsigned = 0;
int isnegative = 0;
int type = 0;
char* q;
len = strlen(p);
/* skip leading white space */
while(strchr(" ",*p) != NULL) {p++; len--;}
/* Get leading sign character, if any */
if(*p == '-') isnegative = 1;
/* Get trailing type tag characters */
switch (len) {
case 0:
goto fail; /* empty parameter */
case 1:
case 2:
q = (p + len) - 1; /* point to last char */
type = gettype(*q,'\0',&isunsigned);
break;
default: /* > 2 => we might have a two letter tag */
q = (p + len) - 2;
type = gettype(*q,*(q+1),&isunsigned);
break;
}
/* Now parse */
switch (type) {
case 'b':
case 's':
case 'i':
/* special case for a positive integer;for back compatibility.*/
if(!isnegative)
sstat = sscanf(p,"%u",&val32u);
else
sstat = sscanf(p,"%d",(int*)&val32u);
if(sstat != 1) goto fail;
switch(type) {
case 'b': val32u = (val32u & 0xFF); break;
case 's': val32u = (val32u & 0xFFFF); break;
}
ulist[nparams++] = val32u;
break;
case 'f':
sstat = sscanf(p,"%lf",&vald);
if(sstat != 1) goto fail;
valf = (float)vald;
ulist[nparams++] = *(unsigned int*)&valf;
break;
/* The following are 8-byte values, so we must swap pieces if this
is a little endian machine */
case 'd':
sstat = sscanf(p,"%lf",&vald);
if(sstat != 1) goto fail;
memcpy(mem,&vald,sizeof(mem));
NC4_filterfix8(mem,0);
vector = (unsigned int*)mem;
ulist[nparams++] = vector[0];
ulist[nparams++] = vector[1];
break;
case 'l': /* long long */
if(isunsigned)
sstat = sscanf(p,"%llu",&val64u);
else
sstat = sscanf(p,"%lld",(long long*)&val64u);
if(sstat != 1) goto fail;
memcpy(mem,&val64u,sizeof(mem));
NC4_filterfix8(mem,0);
vector = (unsigned int*)&mem;
ulist[nparams++] = vector[0];
ulist[nparams++] = vector[1];
break;
default:
goto fail;
}
p = p + strlen(p) + 1; /* move to next param */
}
/* Now return results */
if(specp != NULL) {
NC4_Filterspec* pfs = calloc(1,sizeof(NC4_Filterspec));
if(pfs == NULL) {stat = NC_ENOMEM; goto done;}
pfs->hdr.hdr.format = format;
pfs->filterid = id;
pfs->nparams = nparams;
pfs->params = ulist; ulist = NULL;
*specp = (NC_Filterspec*)pfs;
}
done:
if(sdata) free(sdata);
if(ulist) free(ulist);
return stat;
fail:
stat = NC_EINVAL;
goto done;
}
/*
Parse a string containing multiple '|' separated filter specs.
@param spec0 - a string containing the list of filter specs.
@param nspecsp - # of parsed specs
@param specsp - pointer to hold vector of parsed specs. Caller frees
@return NC_NOERR if parse succeeded
@return NC_EINVAL if bad parameters or parse failed
*/
EXTERNL int
NC_parsefilterlist(const char* txt0, int* formatp, size_t* nspecsp, NC_Filterspec*** vectorp)
{
int stat = NC_NOERR;
int format = NC_FILTER_FORMAT_HDF5; /* default */
size_t len = 0;
size_t nspecs = 0;
NC4_Filterspec** vector = NULL;
char* spec0 = NULL; /* with prefix */
char* spec = NULL; /* without prefix */
char* p = NULL;
char* q = NULL;
if(txt0 == NULL) return NC_EINVAL;
/* Duplicate txt0 so we can modify it */
len = strlen(txt0);
if((spec = calloc(1,len+1+1)) == NULL) return NC_ENOMEM;
memcpy(spec,txt0,len); /* Note double ending nul */
spec0 = spec; /* Save for later free */
/* See if there is a prefix '[format]' tag */
if(spec[0] == LBRACK) {
int found = 0;
const struct LegalFormat* candidates = legalformats;
p = spec + 1;
q = strchr(p,RBRACK);
if(q == NULL) {stat = NC_EINVAL; goto done;}
*q++ = '\0'; /* delimit tag */
while(candidates->tag != NULL) {
if(strcasecmp(p,candidates->tag) == 0) {
found = 1;
format = candidates->format;
}
}
if(found == 0) {stat = NC_EINVAL; goto done;}
spec = q; /* skip tag wrt later processing */
}
if(formatp) *formatp = format;
if(format != NC_FILTER_FORMAT_HDF5) {stat = NC_EINVAL; goto done;} /* for now */
/* pass 1: count number of specs */
p = spec;
nspecs = 0;
while(*p) {
q = strchr(p,'|');
if(q == NULL) q = p + strlen(p); /* fake it */
nspecs++;
p = q + 1;
}
if(nspecs > 0) {
int count = 0;
if((vector = (NC4_Filterspec**)malloc(sizeof(NC4_Filterspec*)*nspecs)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* pass 2: parse */
p = spec;
for(count=0;count<nspecs;count++) {
NC4_Filterspec* aspec = NULL;
q = strchr(p,'|');
if(q == NULL) q = p + strlen(p); /* fake it */
*q = '\0';
if(NC_parsefilterspec(p,format,(NC_Filterspec**)&aspec))
{stat = NC_EINVAL; goto done;}
vector[count] = aspec; aspec = NULL;
p = q+1; /* ok because of double nul */
}
}
if(nspecsp) *nspecsp = nspecs;
if(vectorp) *vectorp = (nspecs == 0 ? NULL : (NC_Filterspec**)vector);
vector = NULL;
done:
nullfree(spec0);
if(vector != NULL) {
int k;
for(k=0;k<nspecs;k++) {
NC4_Filterspec* nfs = vector[k];
if(nfs->params) free(nfs->params);
nullfree(nfs);
}
free(vector);
}
return stat;
}
EXTERNL void
NC4_filterfix8(unsigned char* mem, int decode)
{
#ifdef WORDS_BIGENDIAN
if(decode) { /* Apply inverse of the encode case */
byteswap4(mem); /* step 1: byte-swap each piece */
byteswap4(mem+4);
byteswap8(mem); /* step 2: convert to little endian format */
} else { /* encode */
byteswap8(mem); /* step 1: convert to little endian format */
byteswap4(mem); /* step 2: byte-swap each piece */
byteswap4(mem+4);
}
#else /* Little endian */
/* No action is necessary */
#endif
}
/* Support direct user defined filters */
#ifdef ENABLE_CLIENTSIDE_FILTERS
/* Support direct user defined filters */
/* Use void* to avoid having to include hdf.h*/
EXTERNL int
nc_filter_client_register(unsigned int id, void* info)
{
int stat = NC_NOERR;
if(id == 0 ||info == NULL)
return NC_EINVAL;
#ifdef USE_HDF5
NC_FILTER_OBJ_HDF5 client;
if(id == 0 ||info == NULL)
return NC_EINVAL;
memset(&client,0,sizeof(client));
client.hdr.format = NC_FILTER_FORMAT_HDF5;
client.sort = NC_FILTER_SORT_CLIENT;
@ -409,7 +63,7 @@ nc_filter_client_register(unsigned int id, void* info)
EXTERNL int
nc_filter_client_unregister(unsigned int id)
{
int stat = NC_NOERR;
int stat = NC_NOERR;
#ifdef USE_HDF5
stat = nc4_global_filter_action(NCFILTER_CLIENT_UNREG, id, NULL);
#else
@ -422,7 +76,7 @@ nc_filter_client_unregister(unsigned int id)
EXTERNL int
nc_filter_client_inq(unsigned int id, void* infop)
{
int stat = NC_NOERR;
int stat = NC_NOERR;
#ifdef USE_HDF5
H5Z_class2_t* hct = (H5Z_class2_t*)infop;
NC_FILTER_OBJ_HDF5 client;
@ -445,6 +99,13 @@ nc_filter_client_inq(unsigned int id, void* infop)
}
#endif /*ENABLE_CLIENTSIDE_FILTERS*/
/**************************************************/
/* Per-variable filters */
/* The original HDF5-based functions are left
but are now wrappers around the filterx functions.
*/
/**
Find the set of filters (if any) associated with a variable.
@ -452,6 +113,7 @@ Find the set of filters (if any) associated with a variable.
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
@deprecated{Use nc_inq_filterx_ids}
\param varid Variable ID
\param nfilters return no. of filters
\param ids return the filter ids (caller allocates)
@ -467,24 +129,30 @@ nc_inq_ncid().
EXTERNL int
nc_inq_var_filterids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids)
{
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
NC_FILTER_OBJ_HDF5 ncids;
int stat = NC_NOERR;
size_t nfilters = 0;
char** xidlist = NULL;
if(stat != NC_NOERR) return stat;
TRACE(nc_inq_var_filterids);
memset(&ncids,0,sizeof(ncids));
ncids.hdr.format = NC_FILTER_FORMAT_HDF5;
ncids.sort = NC_FILTER_SORT_IDS;
ncids.u.ids.nfilters = (nfiltersp?*nfiltersp:0);
ncids.u.ids.filterids = ids;
if((stat = ncp->dispatch->filter_actions(ncid,varid, NCFILTER_FILTERIDS, (NC_Filterobject*)&ncids)) == NC_NOERR) {
if(nfiltersp) *nfiltersp = ncids.u.ids.nfilters;
if((stat = nc_inq_var_filterx_ids(ncid, varid, &nfilters, NULL))) goto done;
if(nfilters > 0) {
if((xidlist = malloc(sizeof(char*)*(nfilters+1)))==NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = nc_inq_var_filterx_ids(ncid, varid, &nfilters, xidlist))) goto done;
/* Convert out to HDF5 format */
if(ids != NULL) {
if((stat = NC_cvtX2I_idlist(nfilters,(const char**)xidlist,ids)))
goto done;
}
}
if(nfiltersp) *nfiltersp = nfilters;
done:
/* Cleanup */
NC_filterx_freestringvec(nfilters,xidlist);
return stat;
}
}
/**
Find the the param info about filter (if any)
@ -496,6 +164,7 @@ This is a wrapper for nc_inq_var_all().
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
@deprecated{Use nc_inq_filterx_info}
\param varid Variable ID
\param id The filter id of interest
\param formatp (Out) Storage for the filter format
@ -514,24 +183,29 @@ Note: the caller must allocate and free.
EXTERNL int
nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, unsigned int* params)
{
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
NC_FILTER_OBJ_HDF5 spec;
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
size_t nparams = 0;
char** xparamslist = NULL;
char* xid = NULL;
if(stat != NC_NOERR) return stat;
TRACE(nc_inq_var_filter_info_hdf5);
if(stat != NC_NOERR) return stat;
TRACE(nc_inq_var_filter_info);
memset(&spec,0,sizeof(spec));
spec.hdr.format = NC_FILTER_FORMAT_HDF5;
spec.sort = NC_FILTER_SORT_SPEC;
spec.u.spec.filterid = id;
spec.u.spec.nparams = (nparamsp?*nparamsp:0);
spec.u.spec.params = params;
if((stat = ncp->dispatch->filter_actions(ncid,varid,NCFILTER_INFO,(NC_Filterobject*)&spec)) == NC_NOERR) {
if(nparamsp) *nparamsp = spec.u.spec.nparams;
}
return stat;
if((stat = NC_cvtI2X_id(id,&xid,!USENAME))) goto done;
if((stat = nc_inq_var_filterx_info(ncid,varid,xid,&nparams,NULL)))
goto done;
if((xparamslist = malloc(sizeof(char*)*(nparams+1))) == NULL) goto done;
if((stat = nc_inq_var_filterx_info(ncid,varid,xid,&nparams,xparamslist)))
goto done;
if(nparamsp) *nparamsp = nparams;
if(params) {
if((stat = NC_cvtX2I_params(nparams,(const char**)xparamslist,params))) goto done;
}
done:
nullfree(xid);
NC_filterx_freestringvec(nparams,xparamslist);
return stat;
}
/**
@ -563,23 +237,27 @@ EXTERNL int
nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsigned int* params)
{
NC* ncp;
NC_FILTER_OBJ_HDF5 spec;
int stat = NC_check_id(ncid,&ncp);
size_t nfilters,nparams;
unsigned int* filterids = NULL;
if(stat != NC_NOERR) return stat;
TRACE(nc_inq_var_filter);
memset(&spec,0,sizeof(spec));
spec.hdr.format = NC_FILTER_FORMAT_HDF5;
spec.sort = NC_FILTER_SORT_SPEC;
spec.u.spec.filterid = (idp?*idp:0);
spec.u.spec.nparams = (nparamsp?*nparamsp:0);
spec.u.spec.params = params;
if(idp == NULL)
{stat = NC_ENOFILTER; goto done;}
if((stat = nc_inq_var_filterids(ncid,varid, &nfilters, NULL))) goto done;
if(nfilters == 0)
{stat = NC_ENOFILTER; goto done;}
if((filterids = malloc(sizeof(unsigned int)*nfilters))==NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = nc_inq_var_filterids(ncid,varid, &nfilters, filterids))) goto done;
if((stat = nc_inq_var_filter_info(ncid,varid,filterids[0],&nparams,params))) goto done;
if(idp) *idp = filterids[0];
if(nparamsp) *nparamsp = nparams;
if((stat=ncp->dispatch->filter_actions(ncid,varid,NCFILTER_INQ,(NC_Filterobject*)&spec)))
return stat;
if(idp) *idp = spec.u.spec.filterid;
if(nparamsp) *nparamsp = spec.u.spec.nparams;
done:
nullfree(filterids);
return stat;
}
@ -588,6 +266,7 @@ nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsi
Only variables with chunked storage can use filters.
@deprecated{Use nc_def_var_filterx}
@param ncid File and group ID.
@param varid Variable ID.
@param id Filter ID.
@ -604,24 +283,31 @@ EXTERNL int
nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params)
{
NC* ncp;
NC_FILTER_OBJ_HDF5 spec;
int stat = NC_check_id(ncid,&ncp);
char* xid = NULL;
char** xparams = NULL;
if(stat != NC_NOERR) return stat;
TRACE(nc_def_var_filter_hdf5);
TRACE(nc_def_var_filter);
memset(&spec,0,sizeof(spec));
spec.hdr.format = NC_FILTER_FORMAT_HDF5;
spec.sort = NC_FILTER_SORT_SPEC;
spec.u.spec.filterid = id;
spec.u.spec.nparams = nparams;
spec.u.spec.params = (unsigned int*)params; /* discard const */
return ncp->dispatch->filter_actions(ncid,varid,NCFILTER_DEF,(NC_Filterobject*)&spec);
if((stat = NC_cvtI2X_id(id,&xid,!USENAME))) goto done;
if(nparams > 0) {
if((xparams=malloc(nparams*sizeof(char*))) == NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = NC_cvtI2X_params(nparams,params,xparams))) goto done;
}
if((stat = nc_def_var_filterx(ncid,varid,xid,nparams,(const char**)xparams))) goto done;
done:
nullfree(xid);
NC_filterx_freestringvec(nparams,xparams);
return stat;
}
/**
Remove all filters with specified id from a variable
@deprecated{Use nc_var_filterx_remove}
@param ncid File and group ID.
@param varid Variable ID.
@param id filter to remove
@ -633,86 +319,14 @@ nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const un
EXTERNL int
nc_var_filter_remove(int ncid, int varid, unsigned int id)
{
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
NC_FILTER_OBJ_HDF5 spec;
int stat = NC_NOERR;
char* xid = NULL;
if(stat != NC_NOERR) return stat;
TRACE(nc_var_filter_hdf5_remove);
if((stat = NC_cvtI2X_id(id,&xid,!USENAME))) goto done;
if((stat = nc_var_filterx_remove(ncid,varid,xid))) goto done;
memset(&spec,0,sizeof(spec));
spec.hdr.format = NC_FILTER_FORMAT_HDF5;
spec.sort = NC_FILTER_SORT_SPEC;
spec.u.spec.filterid = id;
return ncp->dispatch->filter_actions(ncid,varid,NCFILTER_REMOVE,(NC_Filterobject*)&spec);
done:
nullfree(xid);
return stat;
}
/**************************************************/
/* Utilities */
/* Look at q0 and q1) to determine type */
static int
gettype(const int q0, const int q1, int* isunsignedp)
{
int type = 0;
int isunsigned = 0;
char typechar;
isunsigned = (q0 == 'u' || q0 == 'U');
if(q1 == '\0')
typechar = q0; /* we were given only a single char */
else if(isunsigned)
typechar = q1; /* we have something like Ux as the tag */
else
typechar = q1; /* look at last char for tag */
switch (typechar) {
case 'f': case 'F': case '.': type = 'f'; break; /* float */
case 'd': case 'D': type = 'd'; break; /* double */
case 'b': case 'B': type = 'b'; break; /* byte */
case 's': case 'S': type = 's'; break; /* short */
case 'l': case 'L': type = 'l'; break; /* long long */
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': type = 'i'; break;
case 'u': case 'U': type = 'i'; isunsigned = 1; break; /* unsigned int */
case '\0': type = 'i'; break;
default: break;
}
if(isunsignedp) *isunsignedp = isunsigned;
return type;
}
#ifdef WORDS_BIGENDIAN
/* Byte swap an 8-byte integer in place */
static void
byteswap8(unsigned char* mem)
{
unsigned char c;
c = mem[0];
mem[0] = mem[7];
mem[7] = c;
c = mem[1];
mem[1] = mem[6];
mem[6] = c;
c = mem[2];
mem[2] = mem[5];
mem[5] = c;
c = mem[3];
mem[3] = mem[4];
mem[4] = c;
}
/* Byte swap an 8-byte integer in place */
static void
byteswap4(unsigned char* mem)
{
unsigned char c;
c = mem[0];
mem[0] = mem[3];
mem[3] = c;
c = mem[1];
mem[1] = mem[2];
mem[2] = c;
}
#endif

405
libdispatch/dfilterx.c Normal file
View File

@ -0,0 +1,405 @@
/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef _MSC_VER
#include <io.h>
#endif
#include "netcdf.h"
#include "netcdf_filter.h"
#include "ncdispatch.h"
#include "nc4internal.h"
#include "ncfilter.h"
#ifdef USE_HDF5
#include "hdf5internal.h"
#endif
/*
Unified filter related code
*/
#ifndef H5Z_FILTER_SZIP
/** ID of HDF SZIP filter. */
#define H5Z_FILTER_SZIP 4
#endif
/*Mnemonic*/
#define USENAME 1
#define LPAREN '('
#define RPAREN ')'
#define LBRACK '['
#define RBRACK ']'
#define NUMCHAR "0123456789"
#define NAMECHAR1 "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
#define NAMECHARN (NAMECHAR1 NUMCHAR "_-")
const struct FilterName {
const char* name; /* name or alias as assigned by HDF group*/
unsigned int id; /* id as assigned by HDF group*/
} known_filters[] = {
{"zip", 2}, /* Standard zlib compression */
{"zlib", 2}, /* alias */
{"deflate", 2}, /* alias */
{"szip", 4}, /* Standard szip compression */
{"bzip2", 307}, /* BZIP2 lossless compression used by PyTables */
{"lzf", 32000}, /* LZF lossless compression used by H5Py project */
{"blosc", 32001}, /* Blosc lossless compression used by PyTables */
{"mafisc", 32002}, /* Modified LZMA compression filter, MAFISC (Multidimensional Adaptive Filtering Improved Scientific data Compression) */
{"snappy", 32003}, /* Snappy lossless compression. */
{"lz4", 32004}, /* LZ4 fast lossless compression algorithm */
{"apax", 32005}, /* Samplify's APAX Numerical Encoding Technology */
{"cbf", 32006}, /* All imgCIF/CBF compressions and decompressions, including Canonical, Packed, Packed Vesrsion 2, Byte Offset and Nibble Offset. */
{"jpeg-xr", 32007}, /* Enables images to be compressed/decompressed with JPEG-XR compression */
{"bitshuffle", 32008}, /* Extreme version of shuffle filter that shuffles data at bit level instead of byte level. */
{"spdp", 32009}, /* SPDP fast lossless compression algorithm for single- and double-precision floating-point data. */
{"lpc-rice", 32010}, /* LPC-Rice multi-threaded lossless compression */
{"ccsds-123", 32011}, /* ESA CCSDS-123 multi-threaded compression filter */
{"jpeg-ls", 32012}, /* CharLS JPEG-LS multi-threaded compression filter */
{"zfp", 32013}, /* Rate, accuracy or precision bounded compression for floating-point arrays */
{"fpzip", 32014}, /* Fast and Efficient Lossy or Lossless Compressor for Floating-Point Data */
{"zstandard", 32015}, /* Real-time compression algorithm with wide range of compression / speed trade-off and fast decoder */
{"b3d", 32016}, /* GPU based image compression method developed for light-microscopy applications */
{"sz", 32017}, /* An error-bounded lossy compressor for scientific floating-point data */
{"fcidecomp", 32018}, /* EUMETSAT CharLS compression filter for use with netCDF */
{"user-defined", 32768}, /* First user-defined filter */
{NULL,0}
};
/**************************************************/
/*Forward*/
static unsigned int NC_filterx_lookup(const char* filtername);
static const char* NC_filterx_toname(unsigned int id);
static int NC_filterx_transferstringvec(size_t n, char** vec, char** copy);
/**************************************************/
/* Per-variable filters extended string-based */
/**
Find the set of filters (if any) associated with a variable.
\param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
\param varid Variable ID
\param nfilters return no. of filters
\param ids return the filter ids (caller allocates)
\returns ::NC_NOERR No error.
\returns ::NC_ENOTNC4 Not a netCDF-4 file.
\returns ::NC_EBADID Bad ncid.
\returns ::NC_ENOTVAR Invalid variable ID.
\returns ::NC_EINVAL Invalid arguments
\ingroup variables
\author Dennis Heimbigner
*/
EXTERNL int
nc_inq_var_filterx_ids(int ncid, int varid, size_t* nfiltersp, char** ids)
{
NC* ncp;
int stat = NC_NOERR;
NC_FILTERX_OBJ ncids;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_inq_var_filterids);
memset(&ncids,0,sizeof(ncids));
ncids.usort = NC_FILTER_UNION_IDS;
ncids.u.ids.nfilters = 0;
ncids.u.ids.filterids = NULL;
if((stat = ncp->dispatch->filter_actions(ncid,varid, NCFILTER_FILTERIDS, &ncids))) goto done;
if(nfiltersp) *nfiltersp = ncids.u.ids.nfilters;
if(ids) {
if((stat = NC_filterx_transferstringvec(ncids.u.ids.nfilters, ncids.u.ids.filterids, ids))) goto done;
}
done:
NC_filterx_freestringvec(ncids.u.ids.nfilters,ncids.u.ids.filterids);
return stat;
}
/**
Find the the param info about filter (if any)
associated with a variable and with specified id.
This is a wrapper for nc_inq_var_all().
\param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
\param varid Variable ID
\param id The filter id of interest
\param formatp (Out) Storage for the filter format
\param nparamsp (Out) Storage which will get the number of parameters to the filter
\param params (Out) Storage which will get associated parameters.
Note: the caller must allocate and free.
\returns ::NC_NOERR No error.
\returns ::NC_ENOTNC4 Not a netCDF-4 file.
\returns ::NC_EBADID Bad ncid.
\returns ::NC_ENOTVAR Invalid variable ID.
\returns ::NC_ENOFILTER No filter defined.
\ingroup variables
\author Dennis Heimbigner
*/
EXTERNL int
nc_inq_var_filterx_info(int ncid, int varid, const char* id, size_t* nparamsp, char** params)
{
NC* ncp;
int stat = NC_NOERR;
NC_FILTERX_OBJ spec;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_inq_var_filter_info);
memset(&spec,0,sizeof(spec));
spec.usort = NC_FILTER_UNION_SPEC;
spec.u.spec.filterid = (char*)id;
spec.u.spec.params = NULL;
if((stat = ncp->dispatch->filter_actions(ncid,varid,NCFILTER_INFO,&spec))) goto done;
if(nparamsp) *nparamsp = spec.u.spec.nparams;
if(params) {
if((stat = NC_filterx_transferstringvec(spec.u.spec.nparams, spec.u.spec.params, params))) goto done;
}
done:
NC_filterx_freestringvec(spec.u.spec.nparams,spec.u.spec.params);
return stat;
}
/**
Define a new variable filter.
Only variables with chunked storage can use filters.
@param ncid File and group ID.
@param varid Variable ID.
@param id Filter ID.
@param nparams Number of filter parameters.
@param parms Filter parameters.
@return ::NC_NOERR No error.
@return ::NC_EINVAL Variable must be chunked.
@return ::NC_EBADID Bad ID.
@author Dennis Heimbigner
*/
EXTERNL int
nc_def_var_filterx(int ncid, int varid, const char* id, size_t nparams, const char** params)
{
NC* ncp;
NC_FILTERX_OBJ spec;
int stat = NC_NOERR;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_def_var_filter);
memset(&spec,0,sizeof(spec));
spec.usort = NC_FILTER_UNION_SPEC;
spec.u.spec.filterid = (char*)id;
spec.u.spec.nparams = nparams;
spec.u.spec.params = (char**)params; /* discard const */
if((stat=ncp->dispatch->filter_actions(ncid,varid,NCFILTER_DEF,&spec))) goto done;
done:
return stat;
}
/**
Remove all filters with specified id from a variable
@param ncid File and group ID.
@param varid Variable ID.
@param id filter to remove
@return ::NC_NOERR No error.
@return ::NC_EBADID Bad ID.
@author Dennis Heimbigner
*/
EXTERNL int
nc_var_filterx_remove(int ncid, int varid, const char* id)
{
NC* ncp;
int stat = NC_NOERR;
NC_FILTERX_OBJ spec;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_var_filter_remove);
memset(&spec,0,sizeof(spec));
spec.usort = NC_FILTER_UNION_SPEC;
spec.u.spec.filterid = (char*)id;
if((stat=ncp->dispatch->filter_actions(ncid,varid,NCFILTER_REMOVE,&spec))) goto done;
done:
return stat;
}
/**************************************************/
/* Utilities */
int
NC_cvtX2I_idlist(size_t n, const char** xidlist, unsigned int* ids)
{
int i,stat = NC_NOERR;
for(i=0;i<n;i++) {
if((stat = NC_cvtX2I_id(xidlist[i],&ids[i]))) break;
}
return stat;
}
int
NC_cvtX2I_params(size_t nparams, const char** xparamslist, unsigned int* params)
{
int i,stat = NC_NOERR;
unsigned int id;
for(i=0;i<nparams;i++) {
/* See if this param looks like an unsigned int */
if(sscanf(xparamslist[i],"%u",&id) == 1) {
params[i] = id;
}
}
return stat;
}
int
NC_cvtI2X_idlist(int n, const unsigned int* ids, char** xid)
{
int i,stat = NC_NOERR;
char sid[1024];
/* For now, do not attempt a name conversion */
for(i=0;i<n;i++) {
snprintf(sid,sizeof(sid),"%u",ids[i]);
if((xid[i] = strdup(sid)) == NULL)
{stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
int
NC_cvtI2X_params(int n, const unsigned int* ids, char** params)
{
int i,stat = NC_NOERR;
char sid[1024];
for(i=0;i<n;i++) {
snprintf(sid,sizeof(sid),"%u",ids[i]);
if((params[i] = strdup(sid))==NULL)
{stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
/* Convert an xid; allow a name */
int
NC_cvtX2I_id(const char* xid, unsigned int* idp)
{
unsigned int id;
/* See if this id looks like an unsigned int */
if(sscanf(xid,"%u",&id) != 1) {
id = NC_filterx_lookup(xid);
}
if(idp) *idp = id;
return (id > 0 ? NC_NOERR : NC_EINVAL);
}
/* Convert an int id to a string; optional name conversion */
int
NC_cvtI2X_id(unsigned int id, char** xidp, int usename)
{
char xid[NC_MAX_NAME];
snprintf(xid,sizeof(xid),"%u",id);
if(usename) {/* See if this id has a name */
const char* name = NC_filterx_toname(id);
if(name != NULL) {xid[0] = '\0'; strlcat(xid,name,sizeof(xid));}
}
if(xidp) {
if((*xidp = strdup(xid))==NULL) return NC_ENOMEM;
}
return NC_NOERR;
}
static unsigned int
NC_filterx_lookup(const char* filtername)
{
const struct FilterName* p = known_filters;
/* Try a name lookup */
for(;p->name;p++) {
if(strcasecmp(p->name,filtername)==0)
return p->id;
}
return 0; /* no match */
}
static const char*
NC_filterx_toname(unsigned int id)
{
const struct FilterName* p = known_filters;
for(;p->name;p++) {
if(p->id == id)
return p->name;
}
return NULL; /* no match */
}
void
NC_filterx_freestringvec(size_t n, char** vec)
{
int i;
if(vec != NULL) {
for(i=0;i<n;i++) {
if(vec[i]) free(vec[i]);
}
free(vec);
}
}
static int
NC_filterx_transferstringvec(size_t n, char** vec, char** copy)
{
int i, stat = NC_NOERR;
for(i=0;i<n;i++) {
copy[i] = vec[i];
vec[i] = NULL;
}
return stat;
}
int
NC_filterx_copy(size_t n, const char** vec, char*** copyp)
{
char** copy = NULL;
int i, stat = NC_NOERR;
if((copy = calloc(sizeof(char*),n+1))==NULL)
{stat = NC_ENOMEM; goto done;}
for(i=0;i<n;i++) {
char* s = nulldup(vec[i]);
if(s == NULL) {stat = NC_ENOMEM; goto done;}
copy[i] = s;
}
if(copyp) {*copyp = copy; copy = NULL;}
done:
if(copy) NC_filterx_freestringvec(n,copy);
return stat;
}

217
libdispatch/dhttp.c
View File

@ -32,11 +32,14 @@
#define GETCMD 0
#define HEADCMD 1
static const char* LENGTH_ACCEPT[] = {"content-length","accept-ranges",NULL};
static const char* CONTENTLENGTH[] = {"content-length",NULL};
/* Forward */
static int setupconn(CURL* curl, const char* objecturl, NCbytes* buf);
static int execute(CURL* curl, int headcmd, long* httpcodep);
static int headerson(CURL* curl, NClist* list);
static void headersoff(CURL* curl);
static int setupconn(NC_HTTP_STATE* state, const char* objecturl, NCbytes* buf);
static int execute(NC_HTTP_STATE* state, int headcmd);
static int headerson(NC_HTTP_STATE* state, const char** which);
static void headersoff(NC_HTTP_STATE* state);
#ifdef TRACE
static void
@ -49,7 +52,7 @@ static void
Trace(const char* fcn)
{
fprintf(stdout,"xxx: %s\n",fcn);
dbgflush();
dbgflush();
}
#else
#define dbgflush()
@ -65,58 +68,63 @@ Trace(const char* fcn)
*/
int
nc_http_open(const char* objecturl, void** curlp, long long* filelenp)
nc_http_open(const char* objecturl, NC_HTTP_STATE** statep, size64_t* filelenp)
{
int stat = NC_NOERR;
CURL* curl = NULL;
int i;
NClist* list = NULL;
NC_HTTP_STATE* state = NULL;
Trace("open");
if((state = calloc(1,sizeof(NC_HTTP_STATE))) == NULL)
{stat = NC_ENOMEM; goto done;}
/* initialize curl*/
curl = curl_easy_init();
if (curl == NULL) stat = NC_ECURL;
if(curlp && curl) *curlp = (void*)curl;
state->curl = curl_easy_init();
if (state->curl == NULL) {stat = NC_ECURL; goto done;}
if(filelenp) {
*filelenp = -1;
/* Attempt to get the file length using HEAD */
list = nclistnew();
if((stat = setupconn(curl,objecturl,NULL))) goto done;
if((stat = headerson(curl,list))) goto done;
if((stat = execute(curl,HEADCMD,NULL))) goto done;
headersoff(curl);
for(i=0;i<nclistlength(list);i+=2) {
char* s = nclistget(list,i);
if((stat = setupconn(state,objecturl,NULL))) goto done;
if((stat = headerson(state,LENGTH_ACCEPT))) goto done;
if((stat = execute(state,HEADCMD))) goto done;
for(i=0;i<nclistlength(state->headers);i+=2) {
char* s = nclistget(state->headers,i);
if(strcasecmp(s,"content-length")==0) {
s = nclistget(list,i+1);
s = nclistget(state->headers,i+1);
sscanf(s,"%lld",filelenp);
break;
}
/* Also check for the Accept-Ranges header */
if(strcasecmp(s,"accept-ranges")==0) {
s = nclistget(list,i+1);
s = nclistget(state->headers,i+1);
if(strcasecmp(s,"bytes")!=0) /* oops! */
{stat = NC_EACCESS; goto done;}
{stat = NC_EURL; goto done;}
}
}
headersoff(state);
}
if(statep) {*statep = state; state = NULL;}
done:
nclistfreeall(list);
nc_http_close(state);
dbgflush();
return stat;
}
int
nc_http_close(void* curl0)
nc_http_close(NC_HTTP_STATE* state)
{
int stat = NC_NOERR;
CURL* curl = curl0;
Trace("close");
if(curl != NULL)
(void)curl_easy_cleanup(curl);
if(state == NULL) return stat;
if(state->curl != NULL)
(void)curl_easy_cleanup(state->curl);
nclistfreeall(state->headers); state->headers = NULL;
if(state->buf != NULL)
abort();
nullfree(state);
dbgflush();
return stat;
}
@ -130,11 +138,10 @@ Assume URL etc has already been set.
*/
int
nc_http_read(CURL* curl, const char* objecturl, fileoffset_t start, fileoffset_t count, NCbytes* buf)
nc_http_read(NC_HTTP_STATE* state, const char* objecturl, size64_t start, size64_t count, NCbytes* buf)
{
int stat = NC_NOERR;
char range[64];
long httpcode = 200;
CURLcode cstat = CURLE_OK;
Trace("read");
@ -142,19 +149,19 @@ nc_http_read(CURL* curl, const char* objecturl, fileoffset_t start, fileoffset_t
if(count == 0)
goto done; /* do not attempt to read */
if((stat = setupconn(curl,objecturl,buf)))
if((stat = setupconn(state,objecturl,buf)))
goto fail;
/* Set to read byte range */
snprintf(range,sizeof(range),"%ld-%ld",(long)start,(long)((start+count)-1));
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_RANGE, range));
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_RANGE, range));
if(cstat != CURLE_OK)
{stat = NC_ECURL; goto done;}
if((stat = execute(curl,GETCMD,&httpcode)))
if((stat = execute(state,GETCMD)))
goto done;
done:
state->buf = NULL;
dbgflush();
return stat;
@ -163,16 +170,67 @@ fail:
goto done;
}
/**
Return length of an object.
Assume URL etc has already been set.
@param curl curl handle
*/
int
nc_http_size(NC_HTTP_STATE* state, const char* objecturl, size64_t* sizep)
{
int i,stat = NC_NOERR;
Trace("size");
if(sizep == NULL)
goto done; /* do not attempt to read */
if((stat = setupconn(state,objecturl,NULL)))
goto done;
/* Make sure we get headers */
if((stat = headerson(state,CONTENTLENGTH))) goto done;
state->httpcode = 200;
if((stat = execute(state,HEADCMD)))
goto done;
if(nclistlength(state->headers) == 0)
{stat = NC_EURL; goto done;}
/* Get the content length header */
for(i=0;i<nclistlength(state->headers);i+=2) {
char* s = nclistget(state->headers,i);
if(strcasecmp(s,"content-length")==0) {
s = nclistget(state->headers,i+1);
sscanf(s,"%llu",sizep);
break;
}
}
done:
headersoff(state);
dbgflush();
return stat;
}
int
nc_http_headers(NC_HTTP_STATE* state, const NClist** headersp)
{
if(headersp) *headersp = state->headers;
return NC_NOERR;
}
/**************************************************/
static size_t
WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
{
NCbytes* buf = data;
NC_HTTP_STATE* state = data;
size_t realsize = size * nmemb;
Trace("WriteMemoryCallback");
if(realsize == 0)
nclog(NCLOGWARN,"WriteMemoryCallback: zero sized chunk");
ncbytesappendn(buf, ptr, realsize);
ncbytesappendn(state->buf, ptr, realsize);
return realsize;
}
@ -208,13 +266,15 @@ trim(char* s)
static size_t
HeaderCallback(char *buffer, size_t size, size_t nitems, void *data)
{
NClist* list = data;
size_t realsize = size * nitems;
char* name = NULL;
char* value = NULL;
char* p = NULL;
size_t i;
int havecolon;
NC_HTTP_STATE* state = data;
int match;
const char** hdr;
Trace("HeaderCallback");
if(realsize == 0)
@ -228,6 +288,13 @@ HeaderCallback(char *buffer, size_t size, size_t nitems, void *data)
name = malloc(i+1);
memcpy(name,buffer,i);
name[i] = '\0';
if(state->headset != NULL) {
for(match=0,hdr=state->headset;*hdr;hdr++) {
if(strcasecmp(*hdr,name)==0) {match = 1; break;}
}
if(!match) goto done;
}
/* Capture this header */
value = NULL;
if(havecolon) {
size_t vlen = (realsize - i);
@ -237,77 +304,80 @@ HeaderCallback(char *buffer, size_t size, size_t nitems, void *data)
value[vlen] = '\0';
trim(value);
}
nclistpush(list,name);
if(state->headers == NULL)
state->headers = nclistnew();
nclistpush(state->headers,name);
name = NULL;
if(value == NULL) value = strdup("");
nclistpush(list,value);
nclistpush(state->headers,value);
value = NULL;
done:
nullfree(name);
return realsize;
}
static int
setupconn(CURL* curl, const char* objecturl, NCbytes* buf)
setupconn(NC_HTTP_STATE* state, const char* objecturl, NCbytes* buf)
{
int stat = NC_NOERR;
CURLcode cstat = CURLE_OK;
if(objecturl != NULL) {
/* Set the URL */
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_URL, (void*)objecturl));
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_URL, (void*)objecturl));
if (cstat != CURLE_OK) goto fail;
}
/* Set options */
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_TIMEOUT, 100)); /* 30sec timeout*/
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 100)); /* 30sec timeout*/
if (cstat != CURLE_OK) goto fail;
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 100));
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_CONNECTTIMEOUT, 100));
if (cstat != CURLE_OK) goto fail;
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1));
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOPROGRESS, 1));
if (cstat != CURLE_OK) goto fail;
cstat = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
cstat = curl_easy_setopt(state->curl, CURLOPT_FOLLOWLOCATION, 1);
if (cstat != CURLE_OK) goto fail;
state->buf = buf;
if(buf != NULL) {
/* send all data to this function */
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback));
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback));
if (cstat != CURLE_OK) goto fail;
/* Set argument for WriteMemoryCallback */
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)buf));
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void*)state));
if (cstat != CURLE_OK) goto fail;
} else {/* turn off data capture */
(void)CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, NULL));
(void)CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL));
(void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, NULL));
(void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, NULL));
}
done:
return stat;
fail:
/* Turn off header capture */
headersoff(curl);
done:
return stat;
fail:
headersoff(state);
stat = NC_ECURL;
goto done;
}
static int
execute(CURL* curl, int headcmd, long* httpcodep)
execute(NC_HTTP_STATE* state, int headcmd)
{
int stat = NC_NOERR;
CURLcode cstat = CURLE_OK;
long httpcode = 0;
if(headcmd) {
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_NOBODY, 1L));
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1L));
if(cstat != CURLE_OK) goto fail;
}
cstat = CURLERR(curl_easy_perform(curl));
cstat = CURLERR(curl_easy_perform(state->curl));
if(cstat != CURLE_OK) goto fail;
cstat = CURLERR(curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE,&httpcode));
if(cstat != CURLE_OK) httpcode = 0;
if(httpcodep) *httpcodep = httpcode;
cstat = CURLERR(curl_easy_getinfo(state->curl,CURLINFO_RESPONSE_CODE,&state->httpcode));
if(cstat != CURLE_OK) state->httpcode = 0;
if(headcmd) {
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_NOBODY, 0L));
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0L));
if(cstat != CURLE_OK) goto fail;
}
@ -319,14 +389,19 @@ fail:
}
static int
headerson(CURL* curl, NClist* list)
headerson(NC_HTTP_STATE* state, const char** headset)
{
int stat = NC_NOERR;
CURLcode cstat = CURLE_OK;
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, HeaderCallback));
if(state->headers != NULL)
nclistfreeall(state->headers);
state->headers = nclistnew();
state->headset = headset;
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERFUNCTION, HeaderCallback));
if(cstat != CURLE_OK) goto fail;
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void*)list));
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERDATA, (void*)state));
if (cstat != CURLE_OK) goto fail;
done:
@ -337,16 +412,10 @@ fail:
}
static void
headersoff(CURL* curl)
headersoff(NC_HTTP_STATE* state)
{
(void)CURLERR(curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, NULL));
(void)CURLERR(curl_easy_setopt(curl, CURLOPT_HEADERDATA, NULL));
nclistfreeall(state->headers);
state->headers = NULL;
(void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERFUNCTION, NULL));
(void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERDATA, NULL));
}
#ifdef IGNORE
static void
reset(CURL* curl)
{
(void)curl_easy_reset(curl);
}
#endif

571
libdispatch/dinfermodel.c
View File

@ -52,8 +52,8 @@ struct MagicFile {
MPI_File fh;
#endif
#ifdef ENABLE_BYTERANGE
void* curl; /* avoid need to include curl.h */
char* curlurl; /* url to use with CURLOPT_SET_URL */
NC_HTTP_STATE* state;
#endif
};
@ -96,23 +96,33 @@ take URLs are included.
static struct FORMATMODES {
const char* tag;
const int impl; /* NC_FORMATX_XXX value */
const int format; /* NC_FORMAT_XXX value */
} formatmodes[] = {
{"dap2",NC_FORMATX_DAP2},
{"dap4",NC_FORMATX_DAP4},
{"netcdf-3",NC_FORMATX_NC3},
{"classic",NC_FORMATX_NC3},
{"netcdf-4",NC_FORMATX_NC4},
{"enhanced",NC_FORMATX_NC4},
{"udf0",NC_FORMATX_UDF0},
{"udf1",NC_FORMATX_UDF1},
{"zarr",NC_FORMATX_ZARR},
{"dap2",NC_FORMATX_DAP2,NC_FORMAT_CLASSIC},
{"dap4",NC_FORMATX_DAP4,NC_FORMAT_NETCDF4},
{"netcdf-3",NC_FORMATX_NC3,0}, /* Might be e.g. cdf5 */
{"classic",NC_FORMATX_NC3,0}, /* ditto */
{"netcdf-4",NC_FORMATX_NC4,NC_FORMAT_NETCDF4},
{"enhanced",NC_FORMATX_NC4,NC_FORMAT_NETCDF4},
{"udf0",NC_FORMATX_UDF0,NC_FORMAT_NETCDF4},
{"udf1",NC_FORMATX_UDF1,NC_FORMAT_NETCDF4},
{"nczarr",NC_FORMATX_NCZARR,NC_FORMAT_NETCDF4},
{"zarr",NC_FORMATX_NCZARR,NC_FORMAT_NETCDF4},
{NULL,0},
};
/* Define the legal singleton mode tags;
thse should also appear in the above mode table. */
static const char* modesingles[] = {
"dap2", "dap4", "bytes", "zarr", NULL
/* For some reason, compiler complains */
static const struct MACRODEF {
char* name;
char* defkey;
char* defvalue;
} macrodefs[] = {
{"zarr","mode","nczarr,zarr"},
{"dap2","mode","dap2"},
{"dap4","mode","dap4"},
{"s3","mode","nczarr,s3"},
{"bytes","mode","bytes"},
{NULL,NULL,NULL}
};
/* Map FORMATX to readability to get magic number */
@ -128,7 +138,7 @@ static struct Readable {
{NC_FORMATX_DAP4,0},
{NC_FORMATX_UDF0,0},
{NC_FORMATX_UDF1,0},
{NC_FORMATX_ZARR,0},
{NC_FORMATX_NCZARR,0}, /* eventually make readable */
{0,0},
};
@ -136,25 +146,25 @@ static struct Readable {
static struct NCPROTOCOLLIST {
const char* protocol;
const char* substitute;
const char* mode;
const char* fragments; /* arbitrary fragment arguments */
} ncprotolist[] = {
{"http",NULL,NULL},
{"https",NULL,NULL},
{"file",NULL,NULL},
{"dods","http","dap2"},
{"dap4","http","dap4"},
{"s3","http","s3"},
{NULL,NULL,NULL} /* Terminate search */
};
/* Forward */
static int NC_omodeinfer(int useparallel, int omode, NCmodel*);
static int check_file_type(const char *path, int omode, int use_parallel, void *parameters, NCmodel* model, NCURI* uri);
static int parseurlmode(const char* modestr, NClist* list);
static int processuri(const char* path, NCURI** urip, char** newpathp, NClist* modeargs);
static char* list2string(NClist* modelist);
static char* envv2string(NClist* envv);
static int issingleton(const char* tag);;
static int processuri(const char* path, NCURI** urip, NClist* fraglist);
static int processmacros(NClist** fraglistp);
static char* envvlist2string(NClist* pairs, const char*);
static void set_default_mode(int* cmodep);
static int parseonchar(const char* s, int ch, NClist* segments);
static int openmagic(struct MagicFile* file);
static int readmagic(struct MagicFile* file, long pos, char* magic);
@ -162,30 +172,31 @@ static int closemagic(struct MagicFile* file);
static int NC_interpret_magic_number(char* magic, NCmodel* model);
#ifdef DEBUG
static void printmagic(const char* tag, char* magic,struct MagicFile*);
static void printlist(NClist* list, const char* tag);
#endif
static int isreadable(NCmodel*);
static char* list2string(NClist*);
static int parsepair(const char* pair, char** keyp, char** valuep);
/*
If the path looks like a URL, then parse it, reformat it,
and compute the mode= flags. Then return it and the the reformatted path.
If the path looks like a URL, then parse it, reformat it.
*/
static int
processuri(const char* path, NCURI** urip, char** newpathp, NClist* modeargs)
processuri(const char* path, NCURI** urip, NClist* fraglenv)
{
int i,j,stat = NC_NOERR;
int stat = NC_NOERR;
int found = 0;
const char** fragp = NULL;
NClist* fraglist = NULL;
NClist* tmp = NULL;
struct NCPROTOCOLLIST* protolist;
NCURI* uri = NULL;
size_t pathlen = strlen(path);
char* str = NULL;
const char** ufrags;
const char** p;
if(path == NULL || pathlen == 0) {stat = NC_EURL; goto done;}
/* Defaults */
if(newpathp) *newpathp = NULL;
if(urip) *urip = NULL;
if(ncuriparse(path,&uri)) goto done; /* not url */
@ -200,85 +211,82 @@ processuri(const char* path, NCURI** urip, char** newpathp, NClist* modeargs)
if(!found)
{stat = NC_EINVAL; goto done;} /* unrecognized URL form */
/* process the corresponding mode arg */
if(protolist->mode != NULL)
nclistpush(modeargs,strdup(protolist->mode));
/* process the corresponding fragments for that protocol */
if(protolist->fragments != NULL) {
int i;
tmp = nclistnew();
if((stat = parseonchar(protolist->fragments,'&',tmp))) goto done;
for(i=0;i<nclistlength(tmp);i++) {
char* key=NULL;
char* value=NULL;
if((stat = parsepair(nclistget(tmp,i),&key,&value))) goto done;
if(value == NULL) value = strdup("");
nclistpush(fraglenv,key);
nclistpush(fraglenv,value);
}
nclistfreeall(tmp); tmp = NULL;
}
/* Substitute the protocol in any case */
if(protolist->substitute) ncurisetprotocol(uri,protolist->substitute);
/* Iterate over the url fragment parameters and collect,
but remove mode= proto= and protocol= */
fraglist = nclistnew();
for(fragp=ncurifragmentparams(uri);fragp && *fragp;fragp+=2) {
int elide = 0;
const char* name = fragp[0];
const char* value = fragp[1];
if(strcmp(name,"protocol")==0
|| strcmp(name,"proto")==0) { /* for back compatibility */
nclistpush(modeargs,strdup(value));
elide = 1;
} else if(strcmp(name,"mode")==0) {
/* Capture the list of mode arguments */
if((stat = parseurlmode(value,modeargs))) goto done;
elide = 1;
} else if(issingleton(name) && (value == NULL || strlen(value)==0)) {
nclistpush(modeargs,strdup(name));
elide = 1;
} /*else ignore*/
if(!elide) {
/* Copy over */
nclistpush(fraglist,strdup(name));
if(value == NULL) value = "";
nclistpush(fraglist,strdup(value));
/* capture the fragments of the url */
ufrags = ncurifragmentparams(uri);
if(ufrags != NULL) {
for(p=ufrags;*p;p+=2) {
const char* key = p[0];
const char* value = p[1];
nclistpush(fraglenv,nulldup(key));
value = (value==NULL?"":value);
nclistpush(fraglenv,strdup(value));
}
}
/* At this point modeargs should contain all mode-like args from the URL */
/* Remove duplicates */
for(i=nclistlength(modeargs)-1;i>=0;i--) {
const char* mode = nclistget(modeargs,i);
for(j=0;j<i;j++) {
const char* other = nclistget(modeargs,i);
if(strcasecmp(mode,other)==0) {
nclistremove(modeargs,i); /* duplicate */
break;
}
}
}
/* Convert the modelist to a new mode= fragment */
if(nclistlength(modeargs) > 0) {
str = list2string(modeargs);
/* Re-insert mode into fraglist */
nclistinsert(fraglist,0,str);
nclistinsert(fraglist,0,strdup("mode"));
}
/* Convert frag list to a string */
str = envv2string(fraglist);
ncurisetfragments(uri,str);
/* Rebuild the path (including fragment)*/
if(newpathp)
*newpathp = ncuribuild(uri,NULL,NULL,NCURIALL);
if(urip) {
*urip = uri;
uri = NULL;
}
#ifdef DEBUG
fprintf(stderr,"newpath=|%s|\n",*newpathp); fflush(stderr);
#endif
done:
nclistfreeall(fraglist);
nclistfreeall(tmp);
nullfree(str);
if(uri != NULL) ncurifree(uri);
return check(stat);
}
/* Parse a mode string at the commas */
/* Split a key=value pair */
static int
parsepair(const char* pair, char** keyp, char** valuep)
{
const char* p;
char* key = NULL;
char* value = NULL;
if(pair == NULL)
return NC_EINVAL; /* empty pair */
if(pair[0] == '\0' || pair[0] == '=')
return NC_EINVAL; /* no key */
p = strchr(pair,'=');
if(p == NULL) {
value = NULL;
key = strdup(pair);
} else {
ptrdiff_t len = (p-pair);
if((key = malloc(len+1))==NULL) return NC_ENOMEM;
memcpy(key,pair,len);
key[len] = '\0';
if(p[1] == '\0')
value = NULL;
else
value = strdup(p+1);
}
if(keyp) {*keyp = key; key = NULL;};
if(valuep) {*valuep = value; value = NULL;};
nullfree(key);
nullfree(value);
return NC_NOERR;
}
#if 0
static int
parseurlmode(const char* modestr, NClist* list)
{
@ -307,10 +315,40 @@ parseurlmode(const char* modestr, NClist* list)
done:
return check(stat);
}
#endif
/* Convert an envv into a comma'd string*/
/* Split a string at a given char */
static int
parseonchar(const char* s, int ch, NClist* segments)
{
int stat = NC_NOERR;
const char* p = NULL;
const char* endp = NULL;
if(s == NULL || *s == '\0') goto done;
p = s;
for(;;) {
char* q;
ptrdiff_t slen;
endp = strchr(p,ch);
if(endp == NULL) endp = p + strlen(p);
slen = (endp - p);
if((q = malloc(slen+1)) == NULL) {stat = NC_ENOMEM; goto done;}
memcpy(q,p,slen);
q[slen] = '\0';
nclistpush(segments,q);
if(*endp == '\0') break;
p = endp+1;
}
done:
return check(stat);
}
/* Convert a key,value envv pairlist into a delimited string*/
static char*
envv2string(NClist* envv)
envvlist2string(NClist* envv, const char* delim)
{
int i;
NCbytes* buf = NULL;
@ -322,11 +360,13 @@ envv2string(NClist* envv)
const char* key = nclistget(envv,i);
const char* val = nclistget(envv,i+1);
if(key == NULL || strlen(key) == 0) continue;
if(val == NULL) val = "";
assert(val != NULL);
if(i > 0) ncbytescat(buf,"&");
ncbytescat(buf,key);
ncbytescat(buf,"=");
ncbytescat(buf,val);
if(val != NULL && val[0] != '\0') {
ncbytescat(buf,"=");
ncbytescat(buf,val);
}
}
result = ncbytesextract(buf);
ncbytesfree(buf);
@ -335,26 +375,27 @@ envv2string(NClist* envv)
/* Convert a list into a comma'd string */
static char*
list2string(NClist* modelist)
list2string(NClist* list)
{
int i;
NCbytes* buf = NULL;
char* result = NULL;
if(modelist == NULL || nclistlength(modelist)==0) return NULL;
if(list == NULL || nclistlength(list)==0) return strdup("");
buf = ncbytesnew();
for(i=0;i<nclistlength(modelist);i++) {
const char* m = nclistget(modelist,i);
for(i=0;i<nclistlength(list);i++) {
const char* m = nclistget(list,i);
if(m == NULL || strlen(m) == 0) continue;
if(i > 0) ncbytescat(buf,",");
ncbytescat(buf,m);
}
result = ncbytesextract(buf);
ncbytesfree(buf);
if(result == NULL) result = strdup("");
return result;
}
/* Given a mode= argument, fill in the impl and possibly mode flags */
/* Given a mode= argument, fill in the impl */
static int
processmodearg(const char* arg, NCmodel* model)
{
@ -362,24 +403,189 @@ processmodearg(const char* arg, NCmodel* model)
struct FORMATMODES* format = formatmodes;
for(;format->tag;format++) {
if(strcmp(format->tag,arg)==0) {
model->impl = format->impl;
model->impl = format->impl;
if(format->format != 0) model->format = format->format;
}
}
return check(stat);
}
/* Search singleton list */
/* Given an envv fragment list, do macro replacement */
static int
issingleton(const char* tag)
processmacros(NClist** fraglenvp)
{
const char** p;
for(p=modesingles;*p;p++) {
if(strcmp(*p,tag)==0) return 1;
int stat = NC_NOERR;
const struct MACRODEF* macros = NULL;
NClist* fraglenv = NULL;
NClist* expanded = NULL;
if(fraglenvp == NULL || nclistlength(*fraglenvp) == 0) goto done;
fraglenv = *fraglenvp;
expanded = nclistnew();
while(nclistlength(fraglenv) > 0) {
int found = 0;
char* key = NULL;
char* value = NULL;
key = nclistremove(fraglenv,0); /* remove from changing front */
value = nclistremove(fraglenv,0); /* remove from changing front */
if(strlen(value) == 0) { /* must be a singleton */
for(macros=macrodefs;macros->name;macros++) {
if(strcmp(macros->name,key)==0) {
nclistpush(expanded,strdup(macros->defkey));
nclistpush(expanded,strdup(macros->defvalue));
found = 1;
break;
}
}
}
if(!found) {/* pass thru */
nclistpush(expanded,strdup(key));
nclistpush(expanded,strdup(value));
}
nullfree(key);
nullfree(value);
}
*fraglenvp = expanded; expanded = NULL;
done:
nclistfreeall(expanded);
nclistfreeall(fraglenv);
return check(stat);
}
static int
mergekey(NClist** valuesp)
{
int i,j;
int stat = NC_NOERR;
NClist* values = *valuesp;
NClist* allvalues = nclistnew();
NClist* newvalues = nclistnew();
char* value = NULL;
for(i=0;i<nclistlength(values);i++) {
char* val1 = nclistget(values,i);
/* split on commas and put pieces into allvalues */
if((stat=parseonchar(val1,',',allvalues))) goto done;
}
/* Remove duplicates and "" */
while(nclistlength(allvalues) > 0) {
value = nclistremove(allvalues,0);
if(strlen(value) == 0) {
nullfree(value); value = NULL;
} else {
for(j=0;j<nclistlength(newvalues);j++) {
char* candidate = nclistget(newvalues,j);
if(strcasecmp(candidate,value)==0)
{nullfree(value); value = NULL; break;}
}
}
if(value != NULL) {nclistpush(newvalues,value); value = NULL;}
}
/* Make sure to have at least 1 value */
if(nclistlength(newvalues)==0) nclistpush(newvalues,strdup(""));
*valuesp = values; values = NULL;
done:
nclistfree(allvalues);
nclistfreeall(values);
nclistfreeall(newvalues);
return check(stat);
}
static int
lcontains(NClist* l, const char* key0)
{
int i;
for(i=0;i<nclistlength(l);i++) {
const char* key1 = nclistget(l,i);
if(strcasecmp(key0,key1)==0) return 1;
}
return 0;
}
/* Warning values should not use nclistfreeall */
static void
collectvaluesbykey(NClist* fraglenv, const char* key, NClist* values)
{
int i;
/* collect all the values with the same key (including this one) */
for(i=0;i<nclistlength(fraglenv);i+=2) {
const char* key2 = nclistget(fraglenv,i);
if(strcasecmp(key,key2)==0) {
const char* value2 = nclistget(fraglenv,i+1);
nclistpush(values,value2); value2 = NULL;
}
}
}
/* Warning allkeys should not use nclistfreeall */
static void
collectallkeys(NClist* fraglenv, NClist* allkeys)
{
int i;
/* collect all the distinct keys */
for(i=0;i<nclistlength(fraglenv);i+=2) {
char* key = nclistget(fraglenv,i);
if(!lcontains(allkeys,key)) {
nclistpush(allkeys,key);
}
}
}
/* Given a fragment envv list, coalesce duplicate keys and remove duplicate values*/
static int
cleanfragments(NClist** fraglenvp)
{
int i,stat = NC_NOERR;
NClist* fraglenv = NULL;
NClist* tmp = NULL;
NClist* allkeys = NULL;
NClist* newlist = NULL;
NCbytes* buf = NULL;
char* key = NULL;
char* value = NULL;
if(fraglenvp == NULL || nclistlength(*fraglenvp) == 0) return NC_NOERR;
fraglenv = *fraglenvp; /* take control of this list */
*fraglenvp = NULL;
newlist = nclistnew();
buf = ncbytesnew();
allkeys = nclistnew();
tmp = nclistnew();
/* collect all unique keys */
collectallkeys(fraglenv,allkeys);
/* Collect all values for same key across all fragments */
for(i=0;i<nclistlength(allkeys);i++) {
key = nclistget(allkeys,i);
collectvaluesbykey(fraglenv,key,tmp);
/* merge the key values, remove duplicate */
if((stat=mergekey(&tmp))) goto done;
/* Construct key,value pair and insert into newlist */
key = strdup(key);
nclistpush(newlist,key);
value = list2string(tmp);
nclistpush(newlist,value);
nclistclear(tmp);
}
*fraglenvp = newlist; newlist = NULL;
done:
nclistfree(allkeys);
nclistfree(tmp);
ncbytesfree(buf);
nclistfreeall(fraglenv);
nclistfreeall(newlist);
return check(stat);
}
/* process non-mode fragment keys in case they hold significance; currently not */
static int
processfragmentkeys(const char* key, const char* value, NCmodel* model)
{
return NC_NOERR;
}
/*
Infer from the mode + useparallel
only call if iscreate or file is not easily readable.
@ -478,45 +684,98 @@ set_default_mode(int* modep)
@param params
@param model
@param newpathp
*/
int
NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void* params, NCmodel* model, char** newpathp)
{
int i,stat = NC_NOERR;
char* newpath = NULL;
NCURI* uri = NULL;
int omode = *omodep;
NClist* fraglenv = nclistnew();
NClist* modeargs = nclistnew();
char* sfrag = NULL;
const char* modeval = NULL;
/* Phase 1: Reformat the uri to canonical form; store canonical form
into newpath. Return the "mode=" list in modeargs */
if((stat = processuri(path, &uri, &newpath, modeargs))) goto done;
if(newpath == NULL) newpath = strdup(path); /* No change */
/* Phase 1:
1. convert special protocols to http|https
2. begin collecting fragments
*/
if((stat = processuri(path, &uri, fraglenv))) goto done;
/* Phase 2: Process the modeargs list to see if we can tell the formatx */
/* Note that if the path was not a URL, then modeargs will be empty list*/
for(i=0;i<nclistlength(modeargs);i++) {
const char* arg = nclistget(modeargs,i);
if((stat=processmodearg(arg,model))) goto done;
}
if(uri != NULL) {
#ifdef DEBUG
printlist(fraglenv,"processuri");
#endif
/* Phase 2.5: Special case: if this is a URL, and there are no mode args
and model.impl is still not defined, default to DAP2 */
if(uri != NULL && nclistlength(modeargs) == 0 && !modelcomplete(model)) {
model->impl = NC_FORMATX_DAP2;
model->format = NC_FORMAT_NC3;
}
/* Phase 2: Expand macros and add to fraglenv */
if((stat = processmacros(&fraglenv))) goto done;
#ifdef DEBUG
printlist(fraglenv,"processmacros");
#endif
/* Phase 3: mode inference from mode flags */
/* Phase 3: coalesce duplicate fragment keys and remove duplicate values */
if((stat = cleanfragments(&fraglenv))) goto done;
#ifdef DEBUG
printlist(fraglenv,"cleanfragments");
#endif
/* Phase 4: Rebuild the url fragment and rebuilt the url */
sfrag = envvlist2string(fraglenv,"&");
nclistfreeall(fraglenv); fraglenv = NULL;
#ifdef DEBUG
fprintf(stderr,"frag final: %s\n",sfrag);
#endif
ncurisetfragments(uri,sfrag);
nullfree(sfrag); sfrag = NULL;
/* rebuild the path */
if(newpathp)
*newpathp = ncuribuild(uri,NULL,NULL,NCURIALL);
#ifdef DEBUG
fprintf(stderr,"newpath=|%s|\n",*newpathp); fflush(stderr);
#endif
/* Phase 5: Process the mode key to see if we can tell the formatx */
modeval = ncurifragmentlookup(uri,"mode");
if(modeval != NULL) {
if((stat = parseonchar(modeval,',',modeargs))) goto done;
for(i=0;i<nclistlength(modeargs);i++) {
const char* arg = nclistget(modeargs,i);
if((stat=processmodearg(arg,model))) goto done;
}
}
/* Phase 6: Process the non-mode keys to see if we can tell the formatx */
if(!modelcomplete(model)) {
const char** p = ncurifragmentparams(uri); /* envv format */
if(p != NULL) {
for(;*p;p++) {
const char* key = p[0];
const char* value = p[1];;
if((stat=processfragmentkeys(key,value,model))) goto done;
}
}
}
/* Phase 6: Special case: if this is a URL, and there are no mode args
and model.impl is still not defined, default to DAP2 */
if(nclistlength(modeargs) == 0 && !modelcomplete(model)) {
model->impl = NC_FORMATX_DAP2;
model->format = NC_FORMAT_NC3;
}
} else {/* Not URL */
if(*newpathp) *newpathp = NULL;
}
/* Phase 7: mode inference from mode flags */
/* The modeargs did not give us a model (probably not a URL).
So look at the combination of mode flags and the useparallel flag */
if(!modelcomplete(model)) {
if((stat = NC_omodeinfer(useparallel,omode,model))) goto done;
}
/* Phase 4: Infer from file content, if possible;
/* Phase 6: Infer from file content, if possible;
this has highest precedence, so it may override
previous decisions. Note that we do this last
because we need previously determined model info
@ -531,39 +790,43 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
if(!modelcomplete(model))
{stat = NC_ENOTNC; goto done;}
/* Force flag consistency */
/* Force flag consistency */
switch (model->impl) {
case NC_FORMATX_NC4:
case NC_FORMATX_NC_HDF4:
case NC_FORMATX_DAP4:
case NC_FORMATX_UDF0:
case NC_FORMATX_UDF1:
case NC_FORMATX_NCZARR:
/* Check for illegal flags */
if((omode & (NC_64BIT_OFFSET|NC_64BIT_DATA)) != 0) {stat = NC_EINVAL; goto done;}
omode |= NC_NETCDF4;
if(model->format == NC_FORMAT_NETCDF4_CLASSIC)
omode |= NC_CLASSIC_MODEL;
break;
case NC_FORMATX_NC3:
omode &= ~NC_NETCDF4; /* must be netcdf-3 (CDF-1, CDF-2, CDF-5) */
if((omode & (NC_NETCDF4)) != 0) {stat = NC_EINVAL; goto done;}
if(model->format == NC_FORMAT_64BIT_OFFSET) omode |= NC_64BIT_OFFSET;
else if(model->format == NC_FORMAT_64BIT_DATA) omode |= NC_64BIT_DATA;
break;
case NC_FORMATX_PNETCDF:
omode &= ~NC_NETCDF4; /* must be netcdf-3 (CDF-1, CDF-2, CDF-5) */
if((omode & (NC_NETCDF4)) != 0) {stat = NC_EINVAL; goto done;}
if(model->format == NC_FORMAT_64BIT_OFFSET) omode |= NC_64BIT_OFFSET;
else if(model->format == NC_FORMAT_64BIT_DATA) omode |= NC_64BIT_DATA;
break;
case NC_FORMATX_DAP2:
omode &= ~(NC_NETCDF4|NC_64BIT_OFFSET|NC_64BIT_DATA|NC_CLASSIC_MODEL);
if((omode & (NC_NETCDF4|NC_64BIT_OFFSET|NC_64BIT_DATA|NC_CLASSIC_MODEL)) != 0)
{stat = NC_EINVAL; goto done;}
break;
default:
{stat = NC_ENOTNC; goto done;}
}
done:
if(uri) ncurifree(uri);
nullfree(sfrag);
ncurifree(uri);
nclistfreeall(modeargs);
if(stat == NC_NOERR && newpathp) {*newpathp = newpath; newpath = NULL;}
nullfree(newpath);
nclistfreeall(fraglenv);
*omodep = omode; /* in/out */
return check(stat);
}
@ -579,6 +842,23 @@ isreadable(NCmodel* model)
return 0;
}
#if 0
static char*
emptyify(char* s)
{
if(s == NULL) s = strdup("");
return strdup(s);
}
static const char*
nullify(const char* s)
{
if(s != NULL && strlen(s) == 0)
return NULL;
return s;
}
#endif
/**************************************************/
#if 0
/* return 1 if path looks like a url; 0 otherwise */
@ -740,7 +1020,7 @@ openmagic(struct MagicFile* file)
/* Construct a URL minus any fragment */
file->curlurl = ncuribuild(file->uri,NULL,NULL,NCURISVC);
/* Open the curl handle */
if((status=nc_http_open(file->curlurl,&file->curl,&file->filelen))) goto done;
if((status=nc_http_open(file->curlurl,&file->state,&file->filelen))) goto done;
#endif
} else {
#ifdef USE_PARALLEL
@ -826,7 +1106,7 @@ readmagic(struct MagicFile* file, long pos, char* magic)
NCbytes* buf = ncbytesnew();
fileoffset_t start = (size_t)pos;
fileoffset_t count = MAGIC_NUMBER_LEN;
status = nc_http_read(file->curl,file->curlurl,start,count,buf);
status = nc_http_read(file->state,file->curlurl,start,count,buf);
if(status == NC_NOERR) {
if(ncbyteslength(buf) != count)
status = NC_EINVAL;
@ -881,7 +1161,7 @@ closemagic(struct MagicFile* file)
/* noop */
#ifdef ENABLE_BYTERANGE
} else if(file->uri != NULL) {
status = nc_http_close(file->curl);
status = nc_http_close(file->state);
nullfree(file->curlurl);
#endif
} else {
@ -993,4 +1273,17 @@ printmagic(const char* tag, char* magic, struct MagicFile* f)
fprintf(stderr,"\n");
fflush(stderr);
}
static void
printlist(NClist* list, const char* tag)
{
int i;
fprintf(stderr,"%s:",tag);
for(i=0;i<nclistlength(list);i++)
fprintf(stderr," %s",(char*)nclistget(list,i));
fprintf(stderr,"\n");
dbgflush();
}
#endif

18
libdispatch/dnotnc4.c
View File

@ -15,8 +15,10 @@
#include "netcdf.h"
#include "netcdf_filter.h"
#include "netcdf_aux.h"
#include "ncdispatch.h"
#include "nc4internal.h"
#include "ncfilter.h"
/**
* @internal Not implemented in some dispatch tables
@ -31,7 +33,7 @@
* @author Ed Hartnett
*/
int
NC_NOTNC4_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
NC_NOTNC4_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
const unsigned int* parms)
{
return NC_ENOTNC4;
@ -627,11 +629,13 @@ NC_NOTNC4_inq_typeid(int ncid, const char *name, nc_type *typeidp)
* @param varid Containing variable id
* @param action Action to perform
*
* @return ::NC_ENOTNC4 Not implemented for a dispatch table.
* @return ::NC_NOERR Implemented as a no-op.
* @return ::NC_ENOTNC4 Not implemented
* @return ::NC_ENOFILTER No filter defined
* @author D. Heimbigner
*/
int
NC_NOTNC4_filter_actions(int ncid, int varid, int action, struct NC_Filterobject* spec)
NC_NOTNC4_filter_actions(int ncid, int varid, int action, void* args)
{
return NC_ENOTNC4;
}
@ -649,17 +653,17 @@ NC_NOTNC4_filter_actions(int ncid, int varid, int action, struct NC_Filterobject
* @author D. Heimbigner
*/
int
NC_NOOP_filter_actions(int ncid, int varid, int action, struct NC_Filterobject* args)
NC_NOOP_filter_actions(int ncid, int varid, int action, void* args)
{
NC_FILTER_OBJ_HDF5* obj = (NC_FILTER_OBJ_HDF5*)args;
NC_FILTERX_OBJ* obj = (NC_FILTERX_OBJ*)args;
switch (action) {
case NCFILTER_FILTERIDS:
obj->u.ids.nfilters = 0;
return NC_NOERR;
case NCFILTER_INQ: /* fall thrue */
case NCFILTER_INFO:
return NC_ENOFILTER;
default:
return NC_ENOTNC4;
return NC_EFILTER;
}
}

4
libdispatch/drc.c
View File

@ -103,7 +103,7 @@ NC_rcload(void)
NCRCglobalstate* globalstate = ncrc_getglobalstate();
if(globalstate->rcinfo.ignore) {
nclog(NCLOGDBG,"No runtime configuration file specified; continuing");
nclog(NCLOGDBG,"No .daprc|.dodsrc runtime configuration file specified; continuing");
return (NC_NOERR);
}
if(globalstate->rcinfo.loaded) return (NC_NOERR);
@ -132,7 +132,7 @@ NC_rcload(void)
}
}
if(path == NULL) {
nclog(NCLOGDBG,"Cannot find runtime configuration file; continuing");
nclog(NCLOGDBG,"No .daprc|.dodsrc runtime configuration file specified; continuing");
} else {
#ifdef D4DEBUG
fprintf(stderr, "RC file: %s\n", path);

21
libdispatch/durlmodel.c
View File

@ -29,17 +29,15 @@ static struct LEGALMODES {
const char* tag;
const int format; /* NC_FORMAT_XXX value */
const int implementation; /* NC_FORMATX_XXX value */
const int iosp; /* NC_IOSP_XXX value */
} legalmodes[] = {
/* Format X Implementation */
{"netcdf-3",NC_FORMAT_CLASSIC,NC_FORMATX_NC3,0},
{"classic",NC_FORMAT_CLASSIC,NC_FORMATX_NC3,0},
{"netcdf-4",NC_FORMAT_NETCDF4,NC_FORMATX_NC4,0},
{"enhanced",NC_FORMAT_NETCDF4,NC_FORMATX_NC4,0},
{"dap2",NC_FORMAT_CLASSIC,NC_FORMATX_DAP2,0},
{"dap4",NC_FORMAT_NETCDF4,NC_FORMATX_DAP4,0},
/* IO Handlers */
{"zarr",0,0,NC_IOSP_ZARR},
{"netcdf-3",NC_FORMAT_CLASSIC,NC_FORMATX_NC3},
{"classic",NC_FORMAT_CLASSIC,NC_FORMATX_NC3},
{"netcdf-4",NC_FORMAT_NETCDF4,NC_FORMATX_NC4},
{"enhanced",NC_FORMAT_NETCDF4,NC_FORMATX_NC4},
{"dap2",NC_FORMAT_CLASSIC,NC_FORMATX_DAP2},
{"dap4",NC_FORMAT_NETCDF4,NC_FORMATX_DAP4},
{"nczarr",NC_FORMAT_NETCDF4,NC_FORMATX_DAP4},
{NULL,0,0},
};
@ -108,12 +106,9 @@ url_getmodel(const char* modestr, NCmode* model)
{stat = NC_EINVAL; goto done;}
if(model->implementation != 0 && legal->implementation != 0)
{stat = NC_EINVAL; goto done;}
if(model->iosp != 0 && legal->iosp != 0)
{stat = NC_EINVAL; goto done;}
if(legal->format != 0) model->format = legal->format;
if(legal->implementation != 0)
model->implementation = legal->implementation;
if(legal->iosp != 0) model->iosp = legal->iosp;
}
}
}
@ -233,7 +228,7 @@ NC_urlmodel(const char* path, int cmode, char** newurl, NCmode* model)
case NC_FORMATX_DAP4:
model->format = NC_FORMAT_NETCDF4;
break;
case NC_FORMATX_ZARR:
case NC_FORMATX_NCZARR:
model->format = NC_FORMAT_NETCDF4;
break;
default:

2
libdispatch/dutil.c
View File

@ -303,7 +303,7 @@ NC_getmodelist(const char* path, NClist** modelistp)
/* Get the mode= arg from the fragment */
modelist = nclistnew();
modestr = ncurilookup(uri,"mode");
modestr = ncurifragmentlookup(uri,"mode");
if(modestr == NULL || strlen(modestr) == 0) goto done;
/* Parse the mode string at the commas or EOL */
p = modestr;

98
libdispatch/dwinpath.c
View File

@ -15,7 +15,10 @@
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef _MSC_VER
#ifdef HAVE_DIRENT_H
#include <dirent.h>
#endif
#ifdef _WIN32
#include <io.h>
#endif
@ -135,14 +138,14 @@ slashtrans:
if(*p == '/') {*p = '\\';}
}
#ifdef PATHFORMAT
#ifndef _MSC_VER
#ifndef _WIN32
p = outpath;
/* Convert '\' back to '/' */
for(;*p;p++) {
if(*p == '\\') {*p = '/';}
}
}
#endif /*!_MSC_VER*/
#endif /*!_WIN32*/
#endif /*PATHFORMAT*/
done:
@ -168,6 +171,34 @@ makeabsolute(const char* relpath)
return path;
}
/* Fix up a path in case extra escapes were added by shell */
EXTERNL
char*
NCdeescape(const char* name)
{
char* ename = NULL;
const char* p;
char* q;
if(name == NULL) return NULL;
ename = strdup(name);
if(ename == NULL) return NULL;
for(p=name,q=ename;*p;) {
switch (*p) {
case '\0': break;
case '\\':
if(p[1] == '#') {
p++;
break;
}
/* fall thru */
default: *q++ = *p++; break;
}
}
*q++ = '\0';
return ename;
}
#ifdef WINPATH
/*
@ -205,6 +236,32 @@ NCopen2(const char *path, int flags)
return NCopen3(path,flags,0);
}
#ifdef HAVE_DIRENT_H
EXTERNL
DIR*
NCopendir(const char* path)
{
DIR* ent = NULL;
char* cvtname = NCpathcvt(path);
if(cvtname == NULL) return -1;
ent = opendir(cvtname);
free(cvtname);
return ent;
}
EXTERNL
int
NCclosedir(DIR* ent)
{
int stat = 0;
char* cvtname = NCpathcvt(path);
if(cvtname == NULL) return -1;
stat = closedir(cvtname);
free(cvtname);
return stat;
}
#endif
/*
Provide wrappers for other file system functions
*/
@ -217,7 +274,7 @@ NCaccess(const char* path, int mode)
int status = 0;
char* cvtname = NCpathcvt(path);
if(cvtname == NULL) return -1;
#ifdef _MSC_VER
#ifdef _WIN32
status = _access(cvtname,mode);
#else
status = access(cvtname,mode);
@ -238,4 +295,37 @@ NCremove(const char* path)
return status;
}
EXTERNL
int
NCmkdir(const char* path, int mode)
{
int status = 0;
char* cvtname = NCpathcvt(path);
if(cvtname == NULL) return -1;
status = _mkdir(cvtname,mode);
free(cvtname);
return status;
}
EXTERNL
char*
NCcwd(char* cwdbuf, size_t len)
{
int ret = 0;
char* cvtname = NULL;
errno = 0;
if(cwdbuf == NULL || len == 0) {errno = ENAMETOOLONG; goto done;}
if(getcwd(cwdbuf,len) == NULL) {goto done;}
cvtname = NCpathcvt(cwdbuf);
if(cvtname == NULL || strlen(cvtname)+1 > len)
{errno = ENAMETOOLONG; goto done;}
cwdbuf[0] = '\0';
strlcat(cwdbuf,cvtname,len);
done:
nullfree(cvtname);
if(errno) return NULL;
return cwdbuf;
}
#endif /*WINPATH*/

15
libdispatch/ncbytes.c
View File

@ -52,7 +52,7 @@ ncbytessetalloc(NCbytes* bb, unsigned long sz)
if(bb->alloc >= sz) return TRUE;
if(bb->nonextendible) return ncbytesfail();
newcontent=(char*)calloc(sz,sizeof(char));
if(newcontent == NULL) return FALSE;
if(newcontent == NULL) ncbytesfail();
if(bb->alloc > 0 && bb->length > 0 && bb->content != NULL) {
memcpy((void*)newcontent,(void*)bb->content,sizeof(char)*bb->length);
}
@ -110,12 +110,11 @@ ncbytesset(NCbytes* bb, unsigned long index, char elem)
int
ncbytesappend(NCbytes* bb, char elem)
{
char s[2];
if(bb == NULL) return ncbytesfail();
/* We need space for the char + null */
ncbytessetalloc(bb,bb->length+2);
bb->content[bb->length] = (char)(elem & 0xFF);
bb->length++;
bb->content[bb->length] = '\0';
s[0] = elem;
s[1] = '\0';
ncbytesappendn(bb,s,1);
return TRUE;
}
@ -138,9 +137,7 @@ ncbytesappendn(NCbytes* bb, const void* elem, unsigned long n)
{
if(bb == NULL || elem == NULL) return ncbytesfail();
if(n == 0) {n = strlen((char*)elem);}
while(!ncbytesavail(bb,n+1)) {
if(!ncbytessetalloc(bb,0)) return ncbytesfail();
}
ncbytessetalloc(bb,bb->length+n+1);
memcpy((void*)&bb->content[bb->length],(void*)elem,n);
bb->length += n;
bb->content[bb->length] = '\0';

18
libdispatch/nchashmap.c
View File

@ -271,7 +271,7 @@ NC_hashmapget(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t* data
return 0; /* not present */
h = &hash->table[index];
if(h->flags & ACTIVE) {
if(datap) *datap = h->data;
if(datap) *datap = h->data;
return 1;
} else /* Not found */
return 0;
@ -311,6 +311,22 @@ NC_hashmapcount(NC_hashmap* hash)
return hash->active;
}
int
NC_hashmapith(NC_hashmap* map, size_t i, uintptr_t* datap, const char** keyp)
{
NC_hentry* h = NULL;
if(map == NULL || i >= map->alloc) return NC_EINVAL;
h = &map->table[i];
if(h && (h->flags & ACTIVE)) {
if(datap) *datap = h->data;
if(keyp) *((char**)keyp) = h->key;
} else {
if(datap) *datap = 0;
if(keyp) *((char**)keyp) = NULL;
}
return NC_NOERR;
}
int
NC_hashmapfree(NC_hashmap* hash)
{

60
libdispatch/nclist.c
View File

@ -6,6 +6,10 @@
#include "nclist.h"
#ifdef _WIN32
#define strcasecmp _stricmp
#endif
int nclistnull(void* e) {return e == NULL;}
#ifndef TRUE
@ -134,11 +138,11 @@ nclistinsert(NClist* l, size_t index, void* elem)
}
int
nclistpush(NClist* l, void* elem)
nclistpush(NClist* l, const void* elem)
{
if(l == NULL) return FALSE;
if(l->length >= l->alloc) nclistsetalloc(l,0);
l->content[l->length] = elem;
l->content[l->length] = (void*)elem;
l->length++;
return TRUE;
}
@ -171,16 +175,6 @@ nclistremove(NClist* l, size_t i)
return elem;
}
/* Duplicate and return the content (null terminate) */
void**
nclistdup(NClist* l)
{
void** result = (void**)malloc(sizeof(void*)*(l->length+1));
memcpy((void*)result,(void*)l->content,sizeof(void*)*l->length);
result[l->length] = (void*)0;
return result;
}
int
nclistcontains(NClist* l, void* elem)
{
@ -191,6 +185,23 @@ nclistcontains(NClist* l, void* elem)
return 0;
}
/* Return 1/0 */
int
nclistmatch(NClist* l, const char* elem, int casesensitive)
{
size_t i;
for(i=0;i<nclistlength(l);i++) {
const char* candidate = (const char*)nclistget(l,i);
int match;
if(casesensitive)
match = strcmp(elem,candidate);
else
match = strcasecmp(elem,candidate);
if(match == 0) return 1;
}
return 0;
}
/* Remove element by value; only removes first encountered */
int
nclistelemremove(NClist* l, void* elem)
@ -238,15 +249,32 @@ nclistunique(NClist* l)
return 1;
}
/* Duplicate a list and if deep is true, assume the contents
are char** and duplicate those also */
NClist*
nclistclone(NClist* l)
nclistclone(NClist* l, int deep)
{
NClist* clone = nclistnew();
*clone = *l;
clone->content = nclistdup(l);
NClist* clone = NULL;
if(l == NULL) goto done;
clone = nclistnew();
nclistsetalloc(clone,l->length+1); /* make room for final null */
if(!deep) {
nclistsetlength(clone,l->length);
memcpy((void*)clone->content,(void*)l->content,sizeof(void*)*l->length);
} else { /*deep*/
int i;
for(i=0;i<nclistlength(l);i++) {
char* dups = strdup(nclistget(l,i));
if(dups == NULL) {nclistfreeall(clone); clone = NULL; goto done;}
nclistpush(clone,dups);
}
}
clone->content[l->length] = (void*)0;
done:
return clone;
}
void*
nclistextract(NClist* l)
{

22
libdispatch/nclog.c
View File

@ -35,7 +35,7 @@ static struct NCLOGGLOBAL {
FILE* nclogstream;
} nclog_global = {0,0,NULL,NULL};
static const char* nctagset[] = {"Warning","Error","Note","Debug"};
static const char* nctagset[] = {"Note","Warning","Error","Debug"};
static const int nctagsize = sizeof(nctagset)/sizeof(char*);
/* Forward */
@ -60,11 +60,12 @@ ncloginit(void)
nclog_global.nclogstream = NULL;
/* Use environment variables to preset nclogging state*/
/* I hope this is portable*/
file = getenv(NCENVFLAG);
if(getenv(NCENVLOGGING) != NULL) {
ncsetlogging(1);
}
file = getenv(NCENVLOGFILE);
if(file != NULL && strlen(file) > 0) {
if(nclogopen(file)) {
ncsetlogging(1);
}
nclogopen(file);
}
}
@ -164,10 +165,16 @@ nclog(int tag, const char* fmt, ...)
{
va_list args;
const char* prefix;
int was;
if(!nclogginginitialized) ncloginit();
if(!nclog_global.nclogging || nclog_global.nclogstream == NULL) return;
if(tag == NCLOGERR) was = ncsetlogging(1);
if(!nclog_global.nclogging) goto done;
if(nclog_global.nclogstream == NULL)
nclog_global.nclogstream = stderr;
prefix = nctagname(tag);
fprintf(nclog_global.nclogstream,"%s:",prefix);
@ -179,6 +186,9 @@ nclog(int tag, const char* fmt, ...)
}
fprintf(nclog_global.nclogstream, "\n" );
fflush(nclog_global.nclogstream);
done:
if(tag == NCLOGERR) ncsetlogging(was);
}
void

299
libdispatch/ncuri.c
View File

@ -89,6 +89,12 @@ static void freestringvec(char** list);
static int ncfind(char** params, const char* key);
static char* nclocate(char* p, const char* charlist);
static int parselist(const char* ptext, NClist* list);
static int unparselist(const char** vec, const char* prefix, int encode, char** svecp);
static int ensurefraglist(NCURI* uri);
static int ensurequerylist(NCURI* uri);
static void buildlist(const char** list, int encode, NCbytes* buf);
static void removedups(NClist* list);
static int extendenvv(char*** envvp, int amount, int* oldlen);
/**************************************************/
/*
@ -168,7 +174,7 @@ ncuriparse(const char* uri0, NCURI** durip)
/* break up the url into coarse pieces */
if(*p == LBRACKET) {
prefix = p;
ret = collectprefixparams(p,&next); /* collect the prefix */
ret = collectprefixparams(p,&next); /* collect the prefix; convert to & form */
if(ret != NC_NOERR)
{THROW(NC_EURL);}
p = next;
@ -202,11 +208,13 @@ ncuriparse(const char* uri0, NCURI** durip)
if(parselist(prefix,params) != NC_NOERR)
{THROW(NC_EURL);}
}
/* Parse the fragment parameters */
/* Parse the fragment parameters into the params list */
if(tmp.fragment != NULL) {
if(parselist(tmp.fragment,params) != NC_NOERR)
{THROW(NC_EURL);}
}
/* Remove duplicates */
removedups(params);
if(nclistlength(params) > 0) {
nclistpush(params,NULL);
tmp.fraglist = nclistextract(params);
@ -280,10 +288,10 @@ ncuriparse(const char* uri0, NCURI** durip)
} else if(l >= 4 && p[0] == '/' && p[1] == '/' && p[2] != '/') { /* case 5 */
p += 2; /* points to the start of the path */
} else /* everything else is illegal */
{THROW(NC_EACCESS);}
{THROW(NC_EURL);}
} else {
if(p[0] != '/' || p[1] != '/') /* must be proto:// */
{THROW(NC_EACCESS);}
{THROW(NC_EURL);}
p += 2;
hashost = 1; /* Assume we have a hostname */
}
@ -374,10 +382,15 @@ ncuriparse(const char* uri0, NCURI** durip)
*tmp.path = pathchar;
duri->path = nulldup(tmp.path);
}
duri->query = nulldup(tmp.query);
duri->fragment = nulldup(tmp.fragment);
duri->query = NULL; /* let ensurequery fix this */
duri->fragment = NULL; /* let ensurefrag fix this */
duri->fraglist = tmp.fraglist; tmp.fraglist = NULL;
duri->querylist = tmp.querylist; tmp.querylist = NULL;
/* make sure query and fragment strings are defined */
ensurequerylist(duri);
ensurefraglist(duri);
if(durip)
*durip = duri;
else
@ -495,15 +508,56 @@ ncurisetfragments(NCURI* duri,const char* fragments)
duri->fragment = NULL;
duri->fraglist = NULL;
if(fragments != NULL && strlen(fragments) > 0) {
NClist* params = nclistnew();
duri->fragment = strdup(fragments);
ret = parselist(duri->fragment,params);
if(ret != NC_NOERR)
{THROW(NC_EURL);}
nclistpush(params,NULL);
duri->fraglist = nclistextract(params);
nclistfree(params);
}
return ret;
}
/* Replace a specific fragment key*/
int
ncurisetfragmentkey(NCURI* duri,const char* key, const char* value)
{
int ret = NC_NOERR;
int pos = -1;
char* newlist = NULL;
ensurefraglist(duri);
pos = ncfind(duri->fraglist, key);
if(pos < 0) return NC_EINVAL; /* does not exist */
nullfree(duri->fraglist[pos+1]);
duri->fraglist[pos+1] = strdup(value);
/* Rebuild the fragment */
if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done;
nullfree(duri->fragment);
duri->fragment = newlist; newlist = NULL;
done:
return ret;
}
/* Replace or add a specific fragment key*/
int
ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value)
{
int ret = NC_NOERR;
int len;
int pos = -1;
char* newlist = NULL;
ensurefraglist(duri);
pos = ncfind(duri->fraglist, key);
if(pos < 0) { /* does not exist */
if((ret = extendenvv(&duri->fraglist,2,&len))) goto done;
duri->fraglist[len] = strdup(key);
duri->fraglist[len+1] = nulldup(value);
duri->fraglist[len+2] = NULL;
} else {
nullfree(duri->fraglist[pos+1]);
duri->fraglist[pos+1] = strdup(value);
}
/* Rebuild the fragment */
if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done;
nullfree(duri->fragment);
duri->fragment = newlist; newlist = NULL;
done:
return ret;
}
@ -607,43 +661,18 @@ ncuribuild(NCURI* duri, const char* prefix, const char* suffix, int flags)
ncbytescat(buf,suffix);
/* The query and the querylist are assumed to be unencoded */
if(flags & NCURIQUERY && duri->querylist != NULL) {
char** p;
int first = 1;
for(p=duri->querylist;*p;p+=2,first=0) {
ncbytescat(buf,(first?"?":"&"));
if(encode) {
char* encoded = ncuriencodeonly(p[0],queryallow);
ncbytescat(buf,encoded);
nullfree(encoded);
} else
ncbytescat(buf,p[0]);
if(p[1] != NULL && strlen(p[1]) > 0) {
ncbytescat(buf,"=");
if(encode) {
char* encoded = ncuriencodeonly(p[1],queryallow);
ncbytescat(buf,encoded);
nullfree(encoded);
} else
ncbytescat(buf,p[1]);
}
}
if(flags & NCURIQUERY) {
ensurequerylist(duri);
if(duri->query != NULL) {
ncbytescat(buf,"?");
ncbytescat(buf,duri->query);
}
}
if((flags & NCURIFRAG) && duri->fraglist != NULL) {
char** p;
int first = 1;
for(p=duri->fraglist;*p;p+=2,first=0) {
ncbytescat(buf,(first?"#":"&"));
ncbytescat(buf,p[0]);
if(p[1] != NULL && strlen(p[1]) > 0) {
ncbytescat(buf,"=");
if(encode) {
char* encoded = ncuriencodeonly(p[1],queryallow);
ncbytescat(buf,encoded);
nullfree(encoded);
} else
ncbytescat(buf,p[1]);
}
if(flags & NCURIFRAG) {
ensurefraglist(duri);
if(duri->fragment != NULL) {
ncbytescat(buf,"#");
ncbytescat(buf,duri->fragment);
}
}
ncbytesnull(buf);
@ -654,11 +683,12 @@ ncuribuild(NCURI* duri, const char* prefix, const char* suffix, int flags)
const char*
ncurilookup(NCURI* uri, const char* key)
ncurifragmentlookup(NCURI* uri, const char* key)
{
int i;
char* value = NULL;
if(uri == NULL || key == NULL || uri->fraglist == NULL) return NULL;
if(uri == NULL || key == NULL) return NULL;
ensurefraglist(uri);
i = ncfind(uri->fraglist,key);
if(i < 0)
return NULL;
@ -683,6 +713,7 @@ ncuriquerylookup(NCURI* uri, const char* key)
const char**
ncurifragmentparams(NCURI* uri)
{
ensurefraglist(uri);
return (const char**)uri->fraglist;
}
@ -690,6 +721,7 @@ ncurifragmentparams(NCURI* uri)
const char**
ncuriqueryparams(NCURI* uri)
{
ensurequerylist(uri);
return (const char**)uri->querylist;
}
@ -722,6 +754,8 @@ ncfind(char** params, const char* key)
{
int i;
char** p;
if(key == NULL) return -1;
if(params == NULL) return -1;
for(i=0,p=params;*p;p+=2,i++) {
if(strcasecmp(key,*p)==0) return i;
}
@ -809,7 +843,7 @@ fromHex(int c)
Support encode of user and password fields
*/
char*
ncuriencodeuserpwd(char* s)
ncuriencodeuserpwd(const char* s)
{
return ncuriencodeonly(s,userpwdallow);
}
@ -820,11 +854,11 @@ ncuriencodeuserpwd(char* s)
*/
char*
ncuriencodeonly(char* s, const char* allowable)
ncuriencodeonly(const char* s, const char* allowable)
{
size_t slen;
char* encoded;
char* inptr;
const char* inptr;
char* outptr;
if(s == NULL) return NULL;
@ -856,12 +890,12 @@ ncuriencodeonly(char* s, const char* allowable)
/* Return a string representing decoding of input; caller must free;*/
char*
ncuridecode(char* s)
ncuridecode(const char* s)
{
size_t slen;
char* decoded;
char* outptr;
char* inptr;
const char* inptr;
unsigned int c;
if (s == NULL) return NULL;
@ -894,12 +928,12 @@ Partially decode a string. Only characters in 'decodeset'
are decoded. Return decoded string; caller must free.
*/
char*
ncuridecodepartial(char* s, const char* decodeset)
ncuridecodepartial(const char* s, const char* decodeset)
{
size_t slen;
char* decoded;
char* outptr;
char* inptr;
const char* inptr;
unsigned int c;
if (s == NULL || decodeset == NULL) return NULL;
@ -1022,3 +1056,152 @@ parselist(const char* text, NClist* list)
nullfree(ptext);
return ret;
}
static int
unparselist(const char** vec, const char* prefix, int encode, char** svecp)
{
int stat = NC_NOERR;
NCbytes* buf = ncbytesnew();
const char** p;
int first = 1;
if(vec == NULL || vec[0] == NULL) goto done;
if(prefix != NULL) ncbytescat(buf,prefix);
for(p=vec;*p;p+=2,first=0) {
if(!first) ncbytescat(buf,"&");
if(encode) {
char* encoded = ncuriencodeonly(p[0],queryallow);
ncbytescat(buf,encoded);
nullfree(encoded);
} else
ncbytescat(buf,p[0]);
if(p[1] != NULL && strlen(p[1]) > 0) {
ncbytescat(buf,"=");
if(encode) {
char* encoded = ncuriencodeonly(p[1],queryallow);
ncbytescat(buf,encoded);
nullfree(encoded);
} else
ncbytescat(buf,p[1]);
}
}
if(svecp) {*svecp = ncbytesextract(buf);}
done:
ncbytesfree(buf);
return stat;
}
static int
ensurefraglist(NCURI* uri)
{
int stat = NC_NOERR;
int nofrag = 0;
int nolist = 0;
NClist* fraglist = NULL;
NCbytes* frag = NULL;
if(uri->fragment == NULL || strlen(uri->fragment) == 0)
{nullfree(uri->fragment); uri->fragment = NULL; nofrag=1;}
if(uri->fraglist == NULL) nolist = 1;
if(nolist && !nofrag) {
fraglist = nclistnew();
if((stat = parselist(uri->fragment,fraglist))) goto done;
removedups(fraglist);
uri->fraglist = nclistextract(fraglist);
} else if(!nolist && nofrag) {
/* Create the fragment string from fraglist */
frag = ncbytesnew();
buildlist((const char**)uri->fraglist,1,frag);
uri->fragment = ncbytesextract(frag);
}
done:
ncbytesfree(frag);
nclistfreeall(fraglist);
return stat;
}
static int
ensurequerylist(NCURI* uri)
{
int stat = NC_NOERR;
int noquery = 0;
int nolist = 0;
NClist* querylist = NULL;
NCbytes* query = NULL;
if(uri->query == NULL || strlen(uri->query) == 0)
{nullfree(uri->query); uri->query = NULL; noquery=1;}
if(uri->querylist == NULL) nolist = 1;
if(nolist && !noquery) {
querylist = nclistnew();
if((stat = parselist(uri->query,querylist))) goto done;
removedups(querylist);
uri->querylist = nclistextract(querylist);
} else if(!nolist && noquery) {
/* Create the query string from querylist */
query = ncbytesnew();
buildlist((const char**)uri->querylist,1,query);
uri->query = ncbytesextract(query);
}
done:
ncbytesfree(query);
nclistfreeall(querylist);
return stat;
}
static void
removedups(NClist* list)
{
int i,j;
if(nclistlength(list) <= 2) return; /* need at least 2 pairs */
for(i=0;i<nclistlength(list);i+=2) {
/* look for dups for this entry */
for(j=nclistlength(list)-2;j>i;j-=2) {
if(strcasecmp(nclistget(list,i),nclistget(list,j))==0
&& strcasecmp(nclistget(list,i+1),nclistget(list,j+1))) {
nclistremove(list,j+1); nclistremove(list,j);
}
}
}
/* NULL terminate the list */
nclistpush(list,NULL);
}
static void
buildlist(const char** list, int encode, NCbytes* buf)
{
const char** p;
int first = 1;
for(p=list;*p;p+=2,first=0) {
if(!first) ncbytescat(buf,"&");
ncbytescat(buf,p[0]);
if(p[1] != NULL && strlen(p[1]) > 0) {
ncbytescat(buf,"=");
if(encode) {
char* encoded = ncuriencodeonly(p[1],queryallow);
ncbytescat(buf,encoded);
nullfree(encoded);
} else
ncbytescat(buf,p[1]);
}
}
}
static int
extendenvv(char*** envvp, int amount, int* oldlenp)
{
char** envv = *envvp;
char** p;
int len;
for(len=0,p=envv;*p;p++) len++;
*oldlenp = len;
if((envv = (char**)malloc((amount+len+1)*sizeof(char*)))==NULL) return NC_ENOMEM;
memcpy(envv,*envvp,sizeof(char*)*len);
envv[len] = NULL;
nullfree(*envvp);
*envvp = envv; envv = NULL;
return NC_NOERR;
}

2
libhdf4/hdf4dispatch.c
View File

@ -103,7 +103,7 @@ static const NC_Dispatch HDF4_dispatcher = {
NC_NOTNC4_set_var_chunk_cache,
NC_NOTNC4_get_var_chunk_cache,
NC_NOOP_filter_actions,
NC_NOTNC4_filter_actions,
};
const NC_Dispatch *HDF4_dispatch_table = NULL;

2
libhdf4/hdf4file.c
View File

@ -425,6 +425,8 @@ nc4_var_list_add_full(NC_GRP_INFO_T* grp, const char* name, int ndims, nc_type x
if ((retval = nc4_set_var_type(xtype, endianness, type_size, type_name,
&(*var)->type_info)))
return retval;
/* Propate the endianness to the variable */
(*var)->endianness = (*var)->type_info->endianness;
(*var)->type_info->rc++;

21
libhdf5/H5FDhttp.c
View File

@ -69,6 +69,7 @@ Define a simple #ifdef test for the version of H5FD_class_t we are using
#include "netcdf.h"
#include "ncbytes.h"
#include "nclist.h"
#include "nchttp.h"
#include "H5FDhttp.h"
@ -104,7 +105,7 @@ typedef struct H5FD_http_t {
haddr_t pos; /* current file I/O position */
unsigned write_access; /* Flag to indicate the file was opened with write access */
H5FD_http_file_op op; /* last operation */
CURL* curl; /* Curl handle */
NC_HTTP_STATE* state; /* Curl handle + extra */
char* url; /* The URL (minus any fragment) for the dataset */
} H5FD_http_t;
@ -305,11 +306,11 @@ H5FD_http_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id,
haddr_t maxaddr)
{
unsigned write_access = 0; /* File opened with write access? */
H5FD_http_t *file = NULL;
H5FD_http_t *file = NULL;
static const char *func = "H5FD_http_open"; /* Function Name for error reporting */
CURL* curl = NULL;
long long len = -1;
int ncstat = NC_NOERR;
NC_HTTP_STATE* state = NULL;
/* Sanity check on file offsets */
assert(sizeof(file_offset_t) >= sizeof(size_t));
@ -332,13 +333,13 @@ H5FD_http_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id,
write_access = 0;
/* Open file in read-only mode, to check for existence and get length */
if((ncstat = nc_http_open(name,&curl,&len))) {
if((ncstat = nc_http_open(name,&state,&len))) {
H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_CANTOPENFILE, "cannot access object", NULL)
}
/* Build the return value */
if(NULL == (file = (H5FD_http_t *)H5allocate_memory(sizeof(H5FD_http_t),0))) {
nc_http_close(curl);
nc_http_close(state);
H5Epush_ret(func, H5E_ERR_CLS, H5E_RESOURCE, H5E_NOSPACE, "memory allocation failed", NULL)
} /* end if */
memset(file,0,sizeof(H5FD_http_t));
@ -347,10 +348,10 @@ H5FD_http_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id,
file->pos = HADDR_UNDEF;
file->write_access = write_access; /* Note the write_access for later */
file->eof = (haddr_t)len;
file->curl = curl; curl = NULL;
file->state = state; state = NULL;
file->url = H5allocate_memory(strlen(name+1),0);
if(file->url == NULL) {
nc_http_close(curl);
nc_http_close(state);
H5Epush_ret(func, H5E_ERR_CLS, H5E_RESOURCE, H5E_NOSPACE, "memory allocation failed", NULL)
}
memcpy(file->url,name,strlen(name)+1);
@ -385,7 +386,7 @@ H5FD_http_close(H5FD_t *_file)
H5Eclear2(H5E_DEFAULT);
/* Close the underlying curl handle*/
if(file->curl) nc_http_close(file->curl);
if(file->state) nc_http_close(file->state);
if(file->url) H5free_memory(file->url);
H5free_memory(file);
@ -634,7 +635,7 @@ H5FD_http_get_handle(H5FD_t *_file, hid_t /*UNUSED*/ fapl, void **file_handle)
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
*file_handle = file->curl;
*file_handle = file->state;
if(*file_handle == NULL)
H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_WRITEERROR, "get handle failed", -1)
@ -711,7 +712,7 @@ H5FD_http_read(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxpl_
{
NCbytes* bbuf = ncbytesnew();
if((ncstat = nc_http_read(file->curl,file->url,addr,size,bbuf))) {
if((ncstat = nc_http_read(file->state,file->url,addr,size,bbuf))) {
file->op = H5FD_HTTP_OP_UNKNOWN;
file->pos = HADDR_UNDEF;
ncbytesfree(bbuf); bbuf = NULL;

1
libhdf5/hdf5attr.c
View File

@ -22,6 +22,7 @@
*
* @return NC_NOERR No error.
* @author Dennis Heimbigner, Ed Hartnett
* [Candidate for moving to libsrc4]
*/
static int
getattlist(NC_GRP_INFO_T *grp, int varid, NC_VAR_INFO_T **varp,

48
libhdf5/hdf5file.c
View File

@ -23,58 +23,10 @@ static void dumpopenobjects(NC_FILE_INFO_T* h5);
we log them or print to stdout. Default is to log. */
#define LOGOPEN 1
/** @internal Number of reserved attributes. These attributes are
* hidden from the netcdf user, but exist in the HDF5 file to help
* netcdf read the file. */
#define NRESERVED 11 /*|NC_reservedatt|*/
/** @internal List of reserved attributes. This list must be in sorted
* order for binary search. */
static const NC_reservedatt NC_reserved[NRESERVED] = {
{NC_ATT_CLASS, READONLYFLAG|DIMSCALEFLAG}, /*CLASS*/
{NC_ATT_DIMENSION_LIST, READONLYFLAG|DIMSCALEFLAG}, /*DIMENSION_LIST*/
{NC_ATT_NAME, READONLYFLAG|DIMSCALEFLAG}, /*NAME*/
{NC_ATT_REFERENCE_LIST, READONLYFLAG|DIMSCALEFLAG}, /*REFERENCE_LIST*/
{NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/
{ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/
{NCPROPS, READONLYFLAG|NAMEONLYFLAG|MATERIALIZEDFLAG},/*_NCProperties*/
{NC_ATT_COORDINATES, READONLYFLAG|DIMSCALEFLAG|MATERIALIZEDFLAG},/*_Netcdf4Coordinates*/
{NC_DIMID_ATT_NAME, READONLYFLAG|DIMSCALEFLAG|MATERIALIZEDFLAG},/*_Netcdf4Dimid*/
{SUPERBLOCKATT, READONLYFLAG|NAMEONLYFLAG},/*_SuperblockVersion*/
{NC3_STRICT_ATT_NAME, READONLYFLAG|MATERIALIZEDFLAG}, /*_nc3_strict*/
};
/* Forward */
static int NC4_enddef(int ncid);
static void dumpopenobjects(NC_FILE_INFO_T* h5);
/**
* @internal Define a binary searcher for reserved attributes
* @param name for which to search
* @return pointer to the matchig NC_reservedatt structure.
* @return NULL if not found.
* @author Dennis Heimbigner
*/
const NC_reservedatt*
NC_findreserved(const char* name)
{
int n = NRESERVED;
int L = 0;
int R = (n - 1);
for(;;) {
if(L > R) break;
int m = (L + R) / 2;
const NC_reservedatt* p = &NC_reserved[m];
int cmp = strcmp(p->name,name);
if(cmp == 0) return p;
if(cmp < 0)
L = (m + 1);
else /*cmp > 0*/
R = (m - 1);
}
return NULL;
}
/**
* @internal Recursively determine if there is a mismatch between
* order of coordinate creation and associated dimensions in this

280
libhdf5/hdf5filter.c
View File

@ -16,6 +16,7 @@
#include <stdlib.h>
#include "hdf5internal.h"
#include "hdf5debug.h"
#include "ncfilter.h"
#define HAVE_H5_DEFLATE
@ -26,12 +27,15 @@
/**************************************************/
/* Filter registration support */
#ifdef ENABLE_CLIENTSIZE_FILTERS
#ifdef ENABLE_CLIENTSIDE_FILTERS
/* WARNING: GLOBAL VARIABLE */
/* Define list of registered filters */
static NClist* NC4_registeredfilters = NULL; /** List<NC_FILTER_CLIENT_HDF5*> */
/**************************************************/
/* Filter registration support */
static int
clientfilterlookup(unsigned int id)
{
@ -91,6 +95,8 @@ nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop)
NC_FILTER_CLIENT_HDF5* elem = NULL;
NC_FILTER_CLIENT_HDF5 ncf;
NC_UNUSED(format);
switch (op) {
case NCFILTER_CLIENT_REG: /* Ignore id argument */
if(infop == NULL) {stat = NC_EINVAL; goto done;}
@ -144,55 +150,26 @@ done:
#endif /*ENABLE_CLIENTSIDE_FILTERS*/
/**************************************************/
static int
filterlookup(NC_VAR_INFO_T* var, unsigned int id, NC_FILTER_SPEC_HDF5** fp)
{
int i;
if(var->filters == NULL)
var->filters = nclistnew();
for(i=0;i<nclistlength(var->filters);i++) {
NC_FILTER_SPEC_HDF5* x = nclistget(var->filters,i);
if(x != NULL && x->filterid == id) {
if(fp) *fp = x;
return i; /* return position */
}
}
return -1;
}
int
NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams,
unsigned int* inparams, NC_FILTER_SPEC_HDF5** filtspecp)
NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* inparams)
{
int stat = NC_NOERR;
NC_FILTER_SPEC_HDF5* fi = NULL;
unsigned int* params = NULL;
int pos;
char* idx = NULL;
char** paramsx = NULL;
if(var->filters == NULL) {
if((var->filters = nclistnew())==NULL) return THROW(NC_ENOMEM);
}
if(nparams > 0 && inparams == NULL)
if((nparams > 0 && inparams == NULL) || id == 0)
return THROW(NC_EINVAL);
if((stat=NC_cvtI2X_id(id,&idx,0))) goto done;
if(inparams != NULL) {
if((params = malloc(sizeof(unsigned int)*nparams)) == NULL)
if((paramsx = malloc(sizeof(char*)*nparams)) == NULL)
return THROW(NC_ENOMEM);
memcpy(params,inparams,sizeof(unsigned int)*nparams);
if((stat = NC_cvtI2X_params(nparams,inparams,paramsx))) goto done;
}
if((pos=filterlookup(var, id,&fi)) < 0) {
if((fi = calloc(1,sizeof(NC_FILTER_SPEC_HDF5))) == NULL)
{nullfree(params); return THROW(NC_ENOMEM);}
}
assert(fi != NULL);
fi->active = active;
fi->filterid = id;
fi->nparams = nparams;
if(fi->params) free(fi->params);
fi->params = params;
if(filtspecp) *filtspecp = fi;
if(pos < 0) nclistpush(var->filters,fi);
fi = NULL;
if((stat = NC4_filterx_add(var,active,idx,nparams,(const char**)paramsx))) goto done;
done:
nullfree(idx);
NC_filterx_freestringvec(nparams,paramsx);
return THROW(stat);
}
@ -216,30 +193,28 @@ NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t npara
* @author Dennis Heimbigner
*/
int
NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args)
NC4_filter_actions(int ncid, int varid, int op, void* args)
{
int stat = NC_NOERR;
NC_GRP_INFO_T *grp = NULL;
NC_FILE_INFO_T *h5 = NULL;
NC_VAR_INFO_T *var = NULL;
NC_FILTER_OBJ_HDF5* obj = (NC_FILTER_OBJ_HDF5*)args;
NC_FILTERX_OBJ* obj = (NC_FILTERX_OBJ*)args;
unsigned int id = 0;
size_t nparams = 0;
unsigned int* idp = NULL;
size_t* nparamsp = NULL;
size_t* nfiltersp = NULL;
unsigned int* params = NULL;
size_t nfilters = 0;
unsigned int* filterids = NULL;
char* idx = NULL;
NC_FILTERX_SPEC* oldspec = NULL;
int havedeflate, haveszip;
LOG((2, "%s: ncid 0x%x varid %d op=%d", __func__, ncid, varid, op));
if(args == NULL) return THROW(NC_EINVAL);
if(args->format != NC_FILTER_FORMAT_HDF5) return THROW(NC_EFILTER);
if(args == NULL) {stat = THROW(NC_EINVAL); goto done;}
/* Find info for this file and group and var, and set pointer to each. */
if ((stat = nc4_hdf5_find_grp_h5_var(ncid, varid, &h5, &grp, &var)))
return THROW(stat);
{stat = THROW(stat); goto done;}
assert(h5 && var && var->hdr.id == varid);
@ -247,67 +222,95 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args)
switch (op) {
case NCFILTER_DEF: {
if(obj->sort != NC_FILTER_SORT_SPEC) return THROW(NC_EFILTER);
if(obj->usort != NC_FILTER_UNION_SPEC)
{stat = THROW(NC_EFILTER); goto done;}
/* If the HDF5 dataset has already been created, then it is too
* late to set all the extra stuff. */
if (!(h5->flags & NC_INDEF)) return THROW(NC_EINDEFINE);
if (!var->ndims) return NC_EINVAL; /* For scalars, complain */
if (!(h5->flags & NC_INDEF))
{stat = THROW(NC_EINDEFINE); goto done;}
if (!var->ndims)
{stat = NC_EINVAL; goto done;} /* For scalars, complain */
if (var->created)
return THROW(NC_ELATEDEF);
{stat = THROW(NC_ELATEDEF); goto done;}
/* Can't turn on parallel and szip before HDF5 1.10.2. */
#ifdef USE_PARALLEL
#ifndef HDF5_SUPPORTS_PAR_FILTERS
if (h5->parallel == NC_TRUE)
return THROW(NC_EINVAL);
{stat = THROW(NC_EINVAL); goto done;}
#endif /* HDF5_SUPPORTS_PAR_FILTERS */
#endif /* USE_PARALLEL */
id = obj->u.spec.filterid;
nparams = obj->u.spec.nparams;
params = obj->u.spec.params;
/* Lookup incoming id to see if already defined */
if((stat = NC_cvtX2I_id(obj->u.spec.filterid,&id))) goto done;
if((params = calloc(sizeof(unsigned int),nparams))==NULL) {stat = NC_ENOMEM; goto done;}
if((stat = NC_cvtX2I_params(nparams,(const char**)obj->u.spec.params,params))) goto done;
oldspec = NULL;
switch((stat=NC4_filterx_lookup(var,obj->u.spec.filterid,&oldspec))) {
case NC_NOERR: break; /* already defined */
case NC_ENOFILTER: break; /*not defined*/
default: goto done;
}
/* See if deflate &/or szip is defined */
if((stat = NC_cvtI2X_id(H5Z_FILTER_DEFLATE,&idx,0))) goto done;
switch ((stat = NC4_filterx_lookup(var,idx,NULL))) {
case NC_NOERR: havedeflate = 1; break;
case NC_ENOFILTER: havedeflate = 0; break;
default: goto done;
}
nullfree(idx); idx = NULL;
if((stat = NC_cvtI2X_id(H5Z_FILTER_SZIP,&idx,0))) goto done;
switch ((stat = NC4_filterx_lookup(var,idx,NULL))) {
case NC_NOERR: haveszip = 1; break;
case NC_ENOFILTER: haveszip = 0; break;
default: goto done;
}
nullfree(idx); idx = NULL;
/* If incoming filter not already defined, then check for conflicts */
if(oldspec == NULL) {
#ifdef HAVE_H5_DEFLATE
if(id == H5Z_FILTER_DEFLATE) {
int level;
if(nparams != 1)
return THROW(NC_EFILTER); /* incorrect no. of parameters */
level = (int)params[0];
if (level < NC_MIN_DEFLATE_LEVEL ||
level > NC_MAX_DEFLATE_LEVEL)
return THROW(NC_EINVAL);
/* If szip compression is already applied, return error. */
if((filterlookup(var,H5Z_FILTER_SZIP,NULL)) >= 0)
return THROW(NC_EINVAL);
}
if(id == H5Z_FILTER_DEFLATE) {
int level;
if(nparams != 1)
{stat = THROW(NC_EFILTER); goto done;}/* incorrect no. of parameters */
level = (int)params[0];
if (level < NC_MIN_DEFLATE_LEVEL || level > NC_MAX_DEFLATE_LEVEL)
{stat = THROW(NC_EINVAL); goto done;}
/* If szip compression is already applied, return error. */
if(haveszip) {stat = THROW(NC_EINVAL); goto done;}
}
#else /*!HAVE_H5_DEFLATE*/
if(id == H5Z_FILTER_DEFLATE)
return THROW(NC_EFILTER); /* Not allowed */
if(id == H5Z_FILTER_DEFLATE)
{stat = THROW(NC_EFILTER); /* Not allowed */ goto done;}
#endif
#ifdef HAVE_H5Z_SZIP
if(id == H5Z_FILTER_SZIP) { /* Do error checking */
if(nparams != 2)
return THROW(NC_EFILTER); /* incorrect no. of parameters */
/* Pixels per block must be an even number, < 32. */
if (params[1] % 2 || params[1] > NC_MAX_PIXELS_PER_BLOCK)
return THROW(NC_EINVAL);
/* If zlib compression is already applied, return error. */
if((filterlookup(var,H5Z_FILTER_DEFLATE,NULL)) >= 0)
return THROW(NC_EINVAL);
}
if(id == H5Z_FILTER_SZIP) { /* Do error checking */
if(nparams != 2)
{stat = THROW(NC_EFILTER); goto done;}/* incorrect no. of parameters */
/* Pixels per block must be an even number, < 32. */
if (params[1] % 2 || params[1] > NC_MAX_PIXELS_PER_BLOCK)
{stat = THROW(NC_EINVAL); goto done;}
/* If zlib compression is already applied, return error. */
if(havedeflate) {stat = THROW(NC_EINVAL); goto done;}
}
#else /*!HAVE_H5Z_SZIP*/
if(id == H5Z_FILTER_SZIP)
return THROW(NC_EFILTER); /* Not allowed */
if(id == H5Z_FILTER_SZIP)
{stat = THROW(NC_EFILTER); goto done;} /* Not allowed */
#endif
/* Filter => chunking */
var->storage = NC_CHUNKED;
/* Determine default chunksizes for this variable unless already specified */
if(var->chunksizes && !var->chunksizes[0]) {
/* Should this throw error? */
if((stat = nc4_find_default_chunksizes2(grp, var)))
goto done;
/* Adjust the cache. */
if ((stat = nc4_adjust_var_cache(grp, var)))
goto done;
}
/* Filter => chunking */
var->storage = NC_CHUNKED;
/* Determine default chunksizes for this variable unless already specified */
if(var->chunksizes && !var->chunksizes[0]) {
/* Should this throw error? */
if((stat = nc4_find_default_chunksizes2(grp, var)))
goto done;
/* Adjust the cache. */
if ((stat = nc4_adjust_var_cache(grp, var)))
goto done;
}
}
#ifdef HAVE_H5Z_SZIP
/* More error checking */
if(id == H5Z_FILTER_SZIP) { /* szip X chunking error checking */
/* For szip, the pixels_per_block parameter must not be greater
* than the number of elements in a chunk of data. */
@ -317,12 +320,12 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args)
num_elem *= var->dim[d]->len;
/* Pixels per block must be <= number of elements. */
if (params[1] > num_elem)
return THROW(NC_EINVAL);
{stat = THROW(NC_EINVAL); goto done;}
}
#endif
if((stat = NC4_hdf5_addfilter(var,!FILTERACTIVE,id,nparams,params, NULL)))
goto done;
/* addfilter can handle case where filter is already defined, and will just replace parameters */
if((stat = NC4_hdf5_addfilter(var,!FILTERACTIVE,id,nparams,params)))
goto done;
#ifdef USE_PARALLEL
#ifdef HDF5_SUPPORTS_PAR_FILTERS
/* Switch to collective access. HDF5 requires collevtive access
@ -331,51 +334,37 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args)
var->parallel_access = NC_COLLECTIVE;
#else
if (h5->parallel)
return NC_EINVAL;
{stat = THROW(NC_EINVAL); goto done;}
#endif /* HDF5_SUPPORTS_PAR_FILTERS */
#endif /* USE_PARALLEL */
} break;
case NCFILTER_INQ: {
if (!var->ndims) return THROW(NC_EINVAL); /* For scalars, fail */
if(obj->sort != NC_FILTER_SORT_SPEC) return THROW(NC_EFILTER);
idp = &obj->u.spec.filterid;
nparamsp = &obj->u.spec.nparams;
params = obj->u.spec.params;
if(nfilters > 0) {
/* Return info about var->filters[0] */
NC_FILTER_SPEC_HDF5* f = (NC_FILTER_SPEC_HDF5*)nclistget(var->filters,0);
if(idp) *idp = f->filterid;
if(nparamsp) {
*nparamsp = (f->params == NULL ? 0 : f->nparams);
if(params && f->params != NULL && f->nparams > 0)
memcpy(params,f->params,f->nparams*sizeof(unsigned int));
}
} else {stat = NC_ENOFILTER; goto done;}
} break;
case NCFILTER_FILTERIDS: {
if(obj->sort != NC_FILTER_SORT_IDS) return THROW(NC_EFILTER);
nfiltersp = &obj->u.ids.nfilters;
filterids = obj->u.ids.filterids;
if(nfiltersp) *nfiltersp = nfilters;
if(filterids) filterids[0] = 0;
if(nfilters > 0 && filterids != NULL) {
if(obj->usort != NC_FILTER_UNION_IDS)
{stat = THROW(NC_EFILTER); goto done;}
nfilters = nclistlength(var->filters);
obj->u.ids.nfilters = nfilters;
if(nfilters > 0) {
int k;
if((obj->u.ids.filterids = calloc(sizeof(char*),nfilters+1))==NULL)
{stat = NC_ENOMEM; goto done;}
for(k=0;k<nfilters;k++) {
NC_FILTER_SPEC_HDF5* f = (NC_FILTER_SPEC_HDF5*)nclistget(var->filters,k);
filterids[k] = f->filterid;
NC_FILTERX_SPEC* f = (NC_FILTERX_SPEC*)nclistget(var->filters,k);
if((obj->u.ids.filterids[k] = strdup(f->filterid)) == NULL)
{stat = NC_ENOMEM; goto done;}
}
}
} break;
case NCFILTER_INFO: {
int k,found;
if(obj->sort != NC_FILTER_SORT_SPEC) return THROW(NC_EFILTER);
id = obj->u.spec.filterid;
if(obj->usort != NC_FILTER_UNION_SPEC)
{stat = THROW(NC_EFILTER); goto done;}
for(found=0,k=0;k<nfilters;k++) {
NC_FILTER_SPEC_HDF5* f = (NC_FILTER_SPEC_HDF5*)nclistget(var->filters,k);
if(f->filterid == id) {
NC_FILTERX_SPEC* f = (NC_FILTERX_SPEC*)nclistget(var->filters,k);
if(strcmp(f->filterid,obj->u.spec.filterid)==0) {
obj->u.spec.nparams = f->nparams;
if(obj->u.spec.params != NULL && f->params != NULL && f->nparams > 0)
memcpy(obj->u.spec.params,f->params,f->nparams*sizeof(unsigned int));
if(f->params != NULL && f->nparams > 0) {
if((stat=NC_filterx_copy(f->nparams,(const char**)f->params,&obj->u.spec.params))) goto done;
}
found = 1;
break;
}
@ -383,36 +372,31 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args)
if(!found) {stat = NC_ENOFILTER; goto done;}
} break;
case NCFILTER_REMOVE: {
int k;
if (!(h5->flags & NC_INDEF)) return THROW(NC_EINDEFINE);
if(obj->sort != NC_FILTER_SORT_SPEC) return THROW(NC_EFILTER);
id = obj->u.spec.filterid;
/* Walk backwards */
for(k=nfilters-1;k>=0;k--) {
NC_FILTER_SPEC_HDF5* f = (NC_FILTER_SPEC_HDF5*)nclistget(var->filters,k);
if(f->filterid == id) {
if(f->active) {
/* Remove from variable */
if((stat = NC4_hdf5_remove_filter(var,id))) {stat = NC_ENOFILTER; goto done;}
}
nclistremove(var->filters,k);
NC4_freefilterspec(f);
}
}
if (!(h5->flags & NC_INDEF))
{stat = THROW(NC_EINDEFINE); goto done;}
if(obj->usort != NC_FILTER_UNION_SPEC)
{stat = THROW(NC_EFILTER); goto done;}
/* Lookup filter */
if((stat = NC4_filterx_lookup(var,obj->u.spec.filterid,&oldspec))) goto done;
if(oldspec && oldspec->active)/* Cannot remove */
{stat = NC_EFILTER; goto done;}
if((stat = NC4_filterx_remove(var,obj->u.spec.filterid))) goto done;
} break;
default:
{stat = NC_EINTERNAL; goto done;}
}
done:
nullfree(idx);
nullfree(params);
return THROW(stat);
}
void
NC4_freefilterspec(NC_FILTER_SPEC_HDF5* f)
NC4_freefilterspec(NC_FILTERX_SPEC* f)
{
if(f) {
if(f->params != NULL) {free(f->params);}
if(f->params != NULL) {NC_filterx_freestringvec(f->nparams,f->params);}
free(f);
}
}

2
libhdf5/hdf5internal.c
View File

@ -16,6 +16,7 @@
#include "config.h"
#include "hdf5internal.h"
#include "ncfilter.h"
#undef DEBUGH5
@ -625,6 +626,7 @@ close_vars(NC_GRP_INFO_T *grp)
if (hdf5_var->dimscale_attached)
free(hdf5_var->dimscale_attached);
nullfree(hdf5_var);
}
return NC_NOERR;

42
libhdf5/hdf5open.c
View File

@ -13,6 +13,7 @@
#include "hdf5internal.h"
#include "ncrc.h"
#include "ncmodel.h"
#include "ncfilter.h"
#ifdef ENABLE_BYTERANGE
#include "H5FDhttp.h"
@ -967,21 +968,22 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var)
size_t cd_nelems;
int f;
int stat = NC_NOERR;
NC_FILTERX_SPEC* spec = NULL;
assert(var);
if ((num_filters = H5Pget_nfilters(propid)) < 0)
return NC_EHDFERR;
{stat = NC_EHDFERR; goto done;}
for (f = 0; f < num_filters; f++)
{
cd_nelems = 0;
if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, NULL, 0, NULL, NULL)) < 0)
return NC_EHDFERR;
{stat = NC_EHDFERR; goto done;}
if((cd_values = calloc(sizeof(unsigned int),cd_nelems))==NULL)
return NC_ENOMEM;
{stat = NC_EHDFERR; goto done;}
if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, cd_values, 0, NULL, NULL)) < 0)
return NC_EHDFERR;
{stat = NC_EHDFERR; goto done;}
switch (filter)
{
case H5Z_FILTER_SHUFFLE:
@ -995,43 +997,45 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var)
case H5Z_FILTER_DEFLATE:
if (cd_nelems != CD_NELEMS_ZLIB ||
cd_values[0] > NC_MAX_DEFLATE_LEVEL)
return NC_EHDFERR;
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values,NULL)))
return stat;
{stat = NC_EHDFERR; goto done;}
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values)))
goto done;
break;
case H5Z_FILTER_SZIP: {
/* Szip is tricky because the filter code expands the set of parameters from 2 to 4
and changes some of the parameter values; try to compensate */
if(cd_nelems == 0) {
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL,NULL)))
return stat;
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL)))
goto done;
} else {
/* fix up the parameters and the #params */
if(cd_nelems != 4)
return NC_EHDFERR;
{stat = NC_EHDFERR; goto done;}
cd_nelems = 2; /* ignore last two */
/* Fix up changed params */
cd_values[0] &= (H5_SZIP_ALL_MASKS);
/* Save info */
stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values,NULL);
if(stat) return stat;
stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values);
if(stat) goto done;
}
} break;
default:
if(cd_nelems == 0) {
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL,NULL))) return stat;
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL))) goto done;
} else {
stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values,NULL);
if(stat) return stat;
stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values);
if(stat) goto done;
}
break;
}
nullfree(cd_values); cd_values = NULL;
}
return NC_NOERR;
done:
nullfree(cd_values);
NC4_filterx_free(spec);
return stat;
}
/**
@ -1417,6 +1421,9 @@ read_var(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name,
/* Indicate that the variable has a pointer to the type */
var->type_info->rc++;
/* Transfer endianness */
var->endianness = var->type_info->endianness;
exit:
if (finalname)
free(finalname);
@ -2226,6 +2233,7 @@ nc4_read_atts(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
* @returns ::NC_NOERR No error.
* @return ::NC_EHDFERR HDF5 returned error.
* @author Ed Hartnett
* [Candidate for libsrc4]
*/
static int
read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name,

58
libhdf5/hdf5var.c
View File

@ -9,11 +9,13 @@
*/
#include "config.h"
#include <hdf5internal.h>
#include "nc4internal.h"
#include "hdf5internal.h"
#include <math.h> /* For pow() used below. */
#include "netcdf.h"
#include "netcdf_filter.h"
#include "ncfilter.h"
/** @internal Default size for unlimited dim chunksize. */
#define DEFAULT_1D_UNLIM_SIZE (4096)
@ -418,13 +420,16 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, int ndims,
if (xtype <= NC_STRING)
{
size_t len;
char name[NC_MAX_NAME];
/* Get type length. */
if ((retval = nc4_get_typelen_mem(h5, xtype, &len)))
BAIL(retval);
/* Create new NC_TYPE_INFO_T struct for this atomic type. */
if ((retval = nc4_type_new(len, nc4_atomic_name[xtype], xtype, &type)))
if((retval=NC4_inq_atomic_type(xtype,name,NULL)))
BAIL(retval);
if ((retval = nc4_type_new(len, name, xtype, &type)))
BAIL(retval);
type->endianness = NC_ENDIAN_NATIVE;
type->size = len;
@ -499,6 +504,9 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, int ndims,
var->type_info->rc++;
type = NULL;
/* Propagate the endianness */
var->endianness = var->type_info->endianness;
/* Set variables no_fill to match the database default unless the
* variable type is variable length (NC_STRING or NC_VLEN) or is
* user-defined type. */
@ -692,14 +700,16 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
if (shuffle)
{
var->shuffle = *shuffle;
var->storage = NC_CHUNKED;
if(var->shuffle)
var->storage = NC_CHUNKED;
}
/* Fletcher32 checksum error protection? */
if (fletcher32)
{
var->fletcher32 = *fletcher32;
var->storage = NC_CHUNKED;
if(var->fletcher32)
var->storage = NC_CHUNKED;
}
#ifdef USE_PARALLEL
@ -860,6 +870,8 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
return NC_EINVAL;
}
var->type_info->endianness = *endianness;
/* Propagate */
var->endianness = *endianness;
}
return NC_NOERR;
@ -894,11 +906,14 @@ NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
unsigned int level = (unsigned int)deflate_level;
/* Set shuffle first */
if((stat = nc_def_var_extra(ncid, varid, &shuffle, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL))) goto done;
if(deflate)
stat = nc_def_var_filter(ncid, varid, H5Z_FILTER_DEFLATE,1,&level);
else
stat = nc_var_filter_remove(ncid, varid, H5Z_FILTER_DEFLATE);
if(stat) goto done;
if(deflate) {
if((stat = nc_def_var_filter(ncid, varid, H5Z_FILTER_DEFLATE,1,&level))) goto done;
} else {
switch (stat = nc_var_filter_remove(ncid, varid, H5Z_FILTER_DEFLATE)) {
case NC_NOERR: case NC_ENOFILTER: stat = NC_NOERR; break; /* ok if not previously defined */
default: goto done;
}
}
done:
return stat;
}
@ -1071,21 +1086,30 @@ NC4_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
{
int retval = NC_NOERR;
NC *nc;
NC_FILTER_OBJ_HDF5 spec;
NC_FILTERX_OBJ obj;
char* xid = NULL;
char** xparams = NULL;
LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid));
if((retval = NC_check_id(ncid,&nc))) return retval;
assert(nc);
memset(&spec,0,sizeof(spec));
spec.hdr.format = NC_FILTER_FORMAT_HDF5;
spec.sort = NC_FILTER_SORT_SPEC;
spec.u.spec.filterid = id;
spec.u.spec.nparams = nparams;
spec.u.spec.params = (unsigned int*)params; /* Need to remove const */
if((retval=NC_cvtI2X_idlist(1,&id,&xid))) goto done;
if((xparams = malloc(nparams*sizeof(char*)))==NULL)
{retval = NC_ENOMEM; goto done;}
if((retval=NC_cvtI2X_params(nparams,params,xparams))) goto done;
return nc->dispatch->filter_actions(ncid,varid,NCFILTER_DEF,(NC_Filterobject*)&spec);
memset(&obj,0,sizeof(obj));
obj.usort = NC_FILTER_UNION_SPEC;
obj.u.spec.filterid = xid;
obj.u.spec.nparams = nparams;
obj.u.spec.params = (char**)xparams; /* Need to remove const */
if((retval=nc->dispatch->filter_actions(ncid,varid,NCFILTER_DEF,&obj))) goto done;
done:
return retval;
}
/**

156
libhdf5/nc4hdf.c
View File

@ -18,6 +18,7 @@
#include "netcdf.h"
#include "nc4internal.h"
#include "ncdispatch.h"
#include "ncfilter.h"
#include "hdf5internal.h"
#include "hdf5debug.h"
#include <math.h>
@ -228,87 +229,6 @@ nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset)
return NC_NOERR;
}
/**
* @internal What fill value should be used for a variable?
*
* @param h5 Pointer to HDF5 file info struct.
* @param var Pointer to variable info struct.
* @param fillp Pointer that gets pointer to fill value.
*
* @returns NC_NOERR No error.
* @returns NC_ENOMEM Out of memory.
* @author Ed Hartnett
*/
int
nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp)
{
size_t size;
int retval;
/* Find out how much space we need for this type's fill value. */
if (var->type_info->nc_type_class == NC_VLEN)
size = sizeof(nc_vlen_t);
else if (var->type_info->nc_type_class == NC_STRING)
size = sizeof(char *);
else
{
if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size)))
return retval;
}
assert(size);
/* Allocate the space. */
if (!((*fillp) = calloc(1, size)))
return NC_ENOMEM;
/* If the user has set a fill_value for this var, use, otherwise
* find the default fill value. */
if (var->fill_value)
{
LOG((4, "Found a fill value for var %s", var->hdr.name));
if (var->type_info->nc_type_class == NC_VLEN)
{
nc_vlen_t *in_vlen = (nc_vlen_t *)(var->fill_value), *fv_vlen = (nc_vlen_t *)(*fillp);
size_t basetypesize = 0;
if((retval=nc4_get_typelen_mem(h5, var->type_info->u.v.base_nc_typeid, &basetypesize)))
return retval;
fv_vlen->len = in_vlen->len;
if (!(fv_vlen->p = malloc(basetypesize * in_vlen->len)))
{
free(*fillp);
*fillp = NULL;
return NC_ENOMEM;
}
memcpy(fv_vlen->p, in_vlen->p, in_vlen->len * basetypesize);
}
else if (var->type_info->nc_type_class == NC_STRING)
{
if (*(char **)var->fill_value)
if (!(**(char ***)fillp = strdup(*(char **)var->fill_value)))
{
free(*fillp);
*fillp = NULL;
return NC_ENOMEM;
}
}
else
memcpy((*fillp), var->fill_value, size);
}
else
{
if (nc4_get_default_fill_value(var->type_info, *fillp))
{
/* Note: release memory, but don't return error on failure */
free(*fillp);
*fillp = NULL;
}
}
return NC_NOERR;
}
/**
* @internal Given a netcdf type, return appropriate HDF typeid. (All
* hdf_typeid's returned from this routine must be H5Tclosed by the
@ -835,6 +755,7 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
NC_DIM_INFO_T *dim = NULL;
char *name_to_use;
int retval;
unsigned int* params = NULL;
assert(grp && grp->format_grp_info && var && var->format_var_info);
@ -923,36 +844,46 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
if(var->filters != NULL) {
int j;
for(j=0;j<nclistlength(var->filters);j++) {
NC_FILTER_SPEC_HDF5* fi = (NC_FILTER_SPEC_HDF5*)nclistget(var->filters,j);
NC_FILTERX_SPEC* fi = (NC_FILTERX_SPEC*)nclistget(var->filters,j);
unsigned int filterid;
size_t nparams;
unsigned int* params;
nparams = fi->nparams;
params = fi->params;
if(fi->filterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */
unsigned level;
if(nparams != 1)
BAIL(NC_EFILTER);
level = (int)params[0];
if(H5Pset_deflate(plistid, level) < 0)
BAIL(NC_EFILTER);
} else if(fi->filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */
int options_mask;
int bits_per_pixel;
if(nparams != 2)
BAIL(NC_EFILTER);
options_mask = (int)params[0];
bits_per_pixel = (int)params[1];
if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
BAIL(NC_EFILTER);
} else {
herr_t code = H5Pset_filter(plistid, (unsigned int)fi->filterid, H5Z_FLAG_MANDATORY, nparams, params);
if(code < 0) {
BAIL(NC_EFILTER);
}
}
}
}
if(!fi->active) {
nparams = fi->nparams;
/* Do the conversions */
if((retval = NC_cvtX2I_id(fi->filterid,&filterid))) BAIL(retval);
if((params = malloc(nparams*sizeof(unsigned int)))==NULL)
BAIL(NC_ENOMEM);
if((retval = NC_cvtX2I_params(nparams,(const char**)fi->params,params))) BAIL(retval);
if(filterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */
unsigned level;
if(nparams != 1)
BAIL(NC_EFILTER);
level = (int)params[0];
if(H5Pset_deflate(plistid, level) < 0)
BAIL(NC_EFILTER);
fi->active = 1;
} else if(filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */
int options_mask;
int bits_per_pixel;
if(nparams != 2)
BAIL(NC_EFILTER);
options_mask = (int)params[0];
bits_per_pixel = (int)params[1];
if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
BAIL(NC_EFILTER);
fi->active = 1;
} else {
herr_t code = H5Pset_filter(plistid, filterid, H5Z_FLAG_MANDATORY, nparams, params);
if(code < 0)
BAIL(NC_EFILTER);
fi->active = 1;
}
}
/* reclaim */
nullfree(params); params = NULL;
}
}
/* If ndims non-zero, get info for all dimensions. We look up the
dimids and get the len of each dimension. We need this to create
@ -1098,6 +1029,7 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
var->attr_dirty = NC_FALSE;
exit:
nullfree(params);
if (typeid > 0 && H5Tclose(typeid) < 0)
BAIL2(NC_EHDFERR);
if (plistid > 0 && H5Pclose(plistid) < 0)
@ -2712,19 +2644,21 @@ NC4_walk(hid_t gid, int* countp)
}
int
NC4_hdf5_remove_filter(NC_VAR_INFO_T* var, unsigned int filterid)
NC4_hdf5_remove_filter(NC_VAR_INFO_T* var, const char* filterid)
{
int stat = NC_NOERR;
NC_HDF5_VAR_INFO_T *hdf5_var;
hid_t propid = -1;
herr_t herr = -1;
H5Z_filter_t hft;
unsigned int id;
hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0)
{stat = NC_EHDFERR; goto done;}
hft = filterid;
if((stat = NC_cvtX2I_id(filterid,&id))) goto done;
hft = id;
if((herr = H5Premove_filter(propid,hft)) < 0)
{stat = NC_EHDFERR; goto done;}
done:

9
liblib/CMakeLists.txt
View File

@ -32,6 +32,15 @@ IF(ENABLE_DAP4)
SET(liblib_LIBS ${liblib_LIBS} dap4)
ENDIF()
IF(ENABLE_NCZARR)
SET(liblib_LIBS ${liblib_LIBS} nczarr)
ENDIF()
IF(ENABLE_S3_SDK)
SET(liblib_LIBS ${liblib_LIBS} ${AWS_CPP_SDK_CORE_LIB} ${AWS_CPP_SDK_S3_LIB} ${AWS_C_COMMON_LIB})
MESSAGE(STATUS "xxx: ${AWS_CPP_SDK_CORE_LIB} ${AWS_CPP_SDK_S3_LIB} ${AWS_C_COMMON_LIB}")
ENDIF(ENABLE_S3_SDK)
FOREACH(LIBS ${liblib_LIBS})
SET(LARGS ${LARGS} $<TARGET_OBJECTS:${LIBS}>)
ENDFOREACH()

5
liblib/Makefile.am
View File

@ -70,6 +70,11 @@ AM_CPPFLAGS += -I${top_srcdir}/libsrc4
libnetcdf_la_LIBADD += ${top_builddir}/libsrc4/libnetcdf4.la
endif #USE_NETCDF4
if ENABLE_NCZARR
AM_CPPFLAGS += -I${top_srcdir}/libnczarr
libnetcdf_la_LIBADD += ${top_builddir}/libnczarr/libnczarr.la
endif #ENABLE_NCZARR
if ISCYGWIN
# Force binary mode for file read/write
AM_LDFLAGS += -lbinmode

7
liblib/nc_initialize.c
View File

@ -95,6 +95,9 @@ nc_initialize()
#ifdef USE_HDF4
if((stat = NC_HDF4_initialize())) goto done;
#endif
#ifdef ENABLE_NCZARR
if((stat = NCZ_initialize())) goto done;
#endif
done:
return stat;
@ -143,6 +146,10 @@ nc_finalize(void)
if((stat = NC_HDF5_finalize())) return stat;
#endif
#ifdef ENABLE_NCZARR
if((stat = NCZ_finalize())) return stat;
#endif
if((stat = NC3_finalize())) return stat;
/* Do general finalization */

62
libnczarr/CMakeLists.txt Normal file
View File

@ -0,0 +1,62 @@
## This is a CMake file, part of Unidata's netCDF package.
# Copyright 2018, see the COPYRIGHT file for more information.
#
# This builds the libzarr dispatch layer.
#
# Dennis Heimbigner
#add_compile_options("/showincludes")
# The source files for the HDF5 dispatch layer.
SET(libnczarr_SOURCES
zarr.c
zattr.c
zcache.c
zchunking.c
zclose.c
zcreate.c
zcvt.c
zdim.c
zdispatch.c
zfile.c
zfilter.c
zgrp.c
zinternal.c
zjson.c
zmap.c
zmap_nz4.c
zmap_nzf.c
zodom.c
zopen.c
zprov.c
zsync.c
ztype.c
zutil.c
zvar.c
zwalk.c
zdebug.c
zarr.h
zcache.h
zchunking.h
zdispatch.h
zincludes.h
zinternal.h
zjson.h
zmap.h
zodom.h
zprovenance.h
zdebug.h
)
IF(ENABLE_S3_SDK)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
SET(libnczarr_SOURCES ${libnczarr_SOURCES} zs3sdk.cpp zmap_s3sdk.c awsincludes.h zs3sdk.h)
ENDIF()
# Build the Zarr dispatch layer as a library that will be included in
# the netCDF library.
add_library(nczarr OBJECT ${libnczarr_SOURCES})
# Remember to package this file for CMake builds.
ADD_EXTRA_DIST(${libnczarr_SOURCES} CMakeLists.txt)

79
libnczarr/Makefile.am Normal file
View File

@ -0,0 +1,79 @@
# This is part of Unidata's netCDF package. Copyright 2018, see the
# COPYRIGHT file for more information.
# Build the ZARR dispatch layer.
# Dennis Heimbigner
# Get AM_CPPFLAGS and AM_LDFLAGS
include $(top_srcdir)/lib_flags.am
AM_CPPFLAGS += -D_LARGEFILE_SOURCE
AM_CPPFLAGS += -I$(top_srcdir)/libsrc4 -I$(top_srcdir)/libdap4
libnczarr_la_CPPFLAGS = ${AM_CPPFLAGS}
AM_CXXFLAGS =
libnczarr_la_LIBADD =
libnczarr_la_LDFLAGS =
# Remember ENABLE_NCZARR implies USE_NETCDF4
# We may have to add to these later.
DISTCLEANFILES =
MAINTAINERCLEANFILES =
CLEANFILES =
EXTRA_DIST =
LDADD=
# This is our output. The ZARR convenience library.
noinst_LTLIBRARIES = libnczarr.la
# The source files.
libnczarr_la_SOURCES = \
zarr.c \
zattr.c \
zcache.c \
zchunking.c \
zclose.c \
zcreate.c \
zcvt.c \
zdim.c \
zdispatch.c \
zfile.c \
zfilter.c \
zgrp.c \
zinternal.c \
zjson.c \
zmap.c \
zmap_nz4.c \
zmap_nzf.c \
zodom.c \
zopen.c \
zprov.c \
zsync.c \
ztype.c \
zutil.c \
zvar.c \
zwalk.c \
zdebug.c \
zarr.h \
zcache.h \
zchunking.h \
zdispatch.h \
zincludes.h \
zinternal.h \
zjson.h \
zmap.h \
zodom.h \
zprovenance.h \
zdebug.h
if ENABLE_S3_SDK
libnczarr_la_SOURCES += zmap_s3sdk.c
libnczarr_la_SOURCES += zs3sdk.cpp awsincludes.h zs3sdk.h
AM_CXXFLAGS += -std=c++11
endif
# For now, ignore these
IGNORED=ztype.c
# Package this for cmake build.
EXTRA_DIST += CMakeLists.txt

1107
libnczarr/SimpleBinStream.h Normal file
View File

File diff suppressed because it is too large Load Diff

28
libnczarr/awsincludes.h Executable file
View File

@ -0,0 +1,28 @@
/* Copyright 2018-2018 University Corporation for Atmospheric
Research/Unidata. */
/**
* @author Dennis Heimbigner
*/
#include <aws/core/Aws.h> /* Needed for InitAPI, SDKOptions, Logging,
and ShutdownAPI */
#include <aws/s3/S3Client.h>
#include <aws/s3/model/Object.h>
#include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/ListObjectsV2Request.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <aws/s3/model/DeleteObjectRequest.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <aws/s3/model/CreateBucketRequest.h>
#include <aws/s3/model/DeleteBucketRequest.h>
#include <aws/core/utils/memory/stl/AWSString.h>
#include <aws/core/utils/logging/DefaultLogSystem.h>
#include <aws/core/utils/logging/AWSLogging.h>
#include <ios>
#include <iostream>
#include <fstream>
#include <sstream>
#include <cstdio>
#include <sys/stat.h>

421
libnczarr/zarr.c Normal file
View File

@ -0,0 +1,421 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#include "zincludes.h"
#include "ncfilter.h"
/**************************************************/
/* Forwards */
static int applycontrols(NCZ_FILE_INFO_T* zinfo);
/***************************************************/
/* API */
/**
@internal Create the topmost dataset object and setup up
NCZ_FILE_INFO_T state.
@param zinfo - [in] the internal state
@return NC_NOERR
@author Dennis Heimbigner
*/
int
ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** controls)
{
int stat = NC_NOERR;
NCZ_FILE_INFO_T* zinfo = NULL;
NCZ_GRP_INFO_T* zgrp = NULL;
NCURI* uri = NULL;
NC* nc = NULL;
NCjson* json = NULL;
char* key = NULL;
ZTRACE("%s/%s %s",file->hdr.name,root->hdr.name,controls);
nc = (NC*)file->controller;
/* Add struct to hold NCZ-specific file metadata. */
if (!(zinfo = calloc(1, sizeof(NCZ_FILE_INFO_T))))
{stat = NC_ENOMEM; goto done;}
file->format_file_info = zinfo;
zinfo->common.file = file;
/* Add struct to hold NCZ-specific group info. */
if (!(zgrp = calloc(1, sizeof(NCZ_GRP_INFO_T))))
{stat = NC_ENOMEM; goto done;}
root->format_grp_info = zgrp;
zgrp->common.file = file;
/* Fill in NCZ_FILE_INFO_T */
zinfo->created = 1;
zinfo->common.file = file;
zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG);
if((zinfo->controls=NCZ_clonestringvec(0,controls)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* fill in some of the zinfo and zroot fields */
zinfo->zarr.zarr_version = ZARRVERSION;
sscanf(NCZARRVERSION,"%lu.%lu.%lu",
&zinfo->zarr.nczarr_version.major,
&zinfo->zarr.nczarr_version.minor,
&zinfo->zarr.nczarr_version.release);
/* Apply client controls */
if((stat = applycontrols(zinfo))) goto done;
/* Load auth info from rc file */
if((zinfo->auth = calloc(1,sizeof(NCauth)))==NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = ncuriparse(nc->path,&uri))) goto done;
if(uri) {
if((stat = NC_authsetup(zinfo->auth, uri)))
goto done;
}
/* initialize map handle*/
if((stat = nczmap_create(zinfo->features.mapimpl,nc->path,nc->mode,zinfo->features.flags,NULL,&zinfo->map)))
goto done;
/* Create super block (NCZMETAROOT) */
if((stat = ncz_create_superblock(zinfo))) goto done;
done:
ncurifree(uri);
NCJreclaim(json);
nullfree(key);
return stat;
}
/**
@internal Open the topmost dataset object.
@param file - [in] the file struct
@param controls - [in] the fragment list in envv form from uri
@return NC_NOERR
@author Dennis Heimbigner
*/
int
ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls)
{
int i,stat = NC_NOERR;
NC* nc = NULL;
NC_GRP_INFO_T* root = NULL;
NCURI* uri = NULL;
void* content = NULL;
NCjson* json = NULL;
NCZ_FILE_INFO_T* zinfo = NULL;
int mode;
NClist* modeargs = NULL;
ZTRACE("%s %s",file->hdr.name,controls);
/* Extract info reachable via file */
nc = (NC*)file->controller;
mode = nc->mode;
root = file->root_grp;
assert(root != NULL && root->hdr.sort == NCGRP);
/* Add struct to hold NCZ-specific file metadata. */
if (!(file->format_file_info = calloc(1, sizeof(NCZ_FILE_INFO_T))))
{stat = NC_ENOMEM; goto done;}
zinfo = file->format_file_info;
/* Fill in NCZ_FILE_INFO_T */
zinfo->created = 0;
zinfo->common.file = file;
zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG);
if((zinfo->controls = NCZ_clonestringvec(0,controls))==NULL) /*0=>envv style*/
{stat = NC_ENOMEM; goto done;}
/* Add struct to hold NCZ-specific group info. */
if (!(root->format_grp_info = calloc(1, sizeof(NCZ_GRP_INFO_T))))
{stat = NC_ENOMEM; goto done;}
((NCZ_GRP_INFO_T*)root->format_grp_info)->common.file = file;
/* Apply client controls */
if((stat = applycontrols(zinfo))) goto done;
/* initialize map handle*/
if((stat = nczmap_open(zinfo->features.mapimpl,nc->path,mode,zinfo->features.flags,NULL,&zinfo->map)))
goto done;
if(!(zinfo->features.flags & FLAG_PUREZARR)
&& (stat = NCZ_downloadjson(zinfo->map, NCZMETAROOT, &json)) == NC_NOERR) {
/* Extract the information from it */
for(i=0;i<nclistlength(json->contents);i+=2) {
const NCjson* key = nclistget(json->contents,i);
const NCjson* value = nclistget(json->contents,i+1);
if(strcmp(key->value,"zarr_format")==0) {
if(sscanf(value->value,"%d",&zinfo->zarr.zarr_version)!=1)
{stat = NC_ENOTNC; goto done;}
} else if(strcmp(key->value,"nczarr_version")==0) {
sscanf(value->value,"%lu.%lu.%lu",
&zinfo->zarr.nczarr_version.major,
&zinfo->zarr.nczarr_version.minor,
&zinfo->zarr.nczarr_version.release);
}
}
} else { /* zinfo->features.purezarr || no object */
zinfo->zarr.zarr_version = ZARRVERSION;
sscanf(NCZARRVERSION,"%lu.%lu.%lu",
&zinfo->zarr.nczarr_version.major,
&zinfo->zarr.nczarr_version.minor,
&zinfo->zarr.nczarr_version.release);
}
/* Load auth info from rc file */
if((zinfo->auth = calloc(1,sizeof(NCauth)))==NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = ncuriparse(nc->path,&uri))) goto done;
if(uri) {
if((stat = NC_authsetup(zinfo->auth, uri)))
goto done;
}
done:
ncurifree(uri);
nclistfreeall(modeargs);
if(json) NCJreclaim(json);
nullfree(content);
return stat;
}
/**
* @internal Determine whether file is netCDF-4.
*
* For libzarr, this is always true.
*
* @param h5 Pointer to HDF5 file info struct.
*
* @returns NC_NOERR No error.
* @author Dennis Heimbigner.
*/
int
NCZ_isnetcdf4(struct NC_FILE_INFO* h5)
{
int isnc4 = 1;
NC_UNUSED(h5);
return isnc4;
}
/**
* @internal Determine version info
*
* For libzarr, this is not well defined
*
* @param majorp Pointer to major version number
* @param minorp Pointer to minor version number
* @param releasep Pointer to release version number
*
* @returns NC_NOERR No error.
* @author Dennis Heimbigner.
*/
int
NCZ_get_libversion(unsigned long* majorp, unsigned long* minorp,unsigned long* releasep)
{
unsigned long m0,m1,m2;
sscanf(NCZARRVERSION,"%lu.%lu.%lu",&m0,&m1,&m2);
if(majorp) *majorp = m0;
if(minorp) *minorp = m1;
if(releasep) *releasep = m2;
return NC_NOERR;
}
/**
* @internal Determine "superblock" number.
*
* For libzarr, use the value of the major part of the nczarr version.
*
* @param superblocp Pointer to place to return superblock.
* use the nczarr format version major as the superblock number.
*
* @returns NC_NOERR No error.
* @author Dennis Heimbigner.
*/
int
NCZ_get_superblock(NC_FILE_INFO_T* file, int* superblockp)
{
NCZ_FILE_INFO_T* zinfo = file->format_file_info;
if(superblockp) *superblockp = zinfo->zarr.nczarr_version.major;
return NC_NOERR;
}
/**************************************************/
/* Utilities */
#if 0
/**
@internal Open the root group object
@param dataset - [in] the root dataset object
@param rootp - [out] created root group
@return NC_NOERR
@author Dennis Heimbigner
*/
static int
ncz_open_rootgroup(NC_FILE_INFO_T* dataset)
{
int stat = NC_NOERR;
int i;
NCZ_FILE_INFO_T* zfile = NULL;
NC_GRP_INFO_T* root = NULL;
void* content = NULL;
char* rootpath = NULL;
NCjson* json = NULL;
ZTRACE();
zfile = dataset->format_file_info;
/* Root should already be defined */
root = dataset->root_grp;
assert(root != NULL);
if((stat=nczm_concat(NULL,ZGROUP,&rootpath)))
goto done;
if((stat = NCZ_downloadjson(zfile->map, rootpath, &json)))
goto done;
/* Process the json */
for(i=0;i<nclistlength(json->contents);i+=2) {
const NCjson* key = nclistget(json->contents,i);
const NCjson* value = nclistget(json->contents,i+1);
if(strcmp(key->value,"zarr_format")==0) {
int zversion;
if(sscanf(value->value,"%d",&zversion)!=1)
{stat = NC_ENOTNC; goto done;}
/* Verify against the dataset */
if(zversion != zfile->zarr.zarr_version)
{stat = NC_ENOTNC; goto done;}
}
}
done:
if(json) NCJreclaim(json);
nullfree(rootpath);
nullfree(content);
return stat;
}
#endif
/**
@internal Rewrite attributes into a group or var
@param map - [in] the map object for storage
@param container - [in] the containing object
@param jattrs - [in] the json for .zattrs
@param jtypes - [in] the json for .ztypes
@return NC_NOERR
@author Dennis Heimbigner
*/
int
ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjson* jtypes)
{
int stat = NC_NOERR;
char* fullpath = NULL;
char* akey = NULL;
char* tkey = NULL;
NCZMAP* map = zinfo->map;
assert((jattrs->sort = NCJ_DICT));
assert((jtypes->sort = NCJ_DICT));
if(container->sort == NCGRP) {
NC_GRP_INFO_T* grp = (NC_GRP_INFO_T*)container;
/* Get grp's fullpath name */
if((stat = NCZ_grpkey(grp,&fullpath)))
goto done;
} else {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)container;
/* Get var's fullpath name */
if((stat = NCZ_varkey(var,&fullpath)))
goto done;
}
/* Construct the path to the .zattrs object */
if((stat = nczm_concat(fullpath,ZATTRS,&akey)))
goto done;
/* Upload the .zattrs object */
if((stat=NCZ_uploadjson(map,tkey,jattrs)))
goto done;
if(!(zinfo->features.flags & FLAG_PUREZARR)) {
/* Construct the path to the .nczattr object */
if((stat = nczm_concat(fullpath,NCZATTR,&tkey)))
goto done;
/* Upload the .nczattr object */
if((stat=NCZ_uploadjson(map,tkey,jtypes)))
goto done;
}
done:
if(stat) {
NCJreclaim(jattrs);
NCJreclaim(jtypes);
}
nullfree(fullpath);
nullfree(akey);
nullfree(tkey);
return stat;
}
static const char*
controllookup(const char** controls, const char* key)
{
const char** p;
for(p=controls;*p;p++) {
if(strcasecmp(key,*p)==0) {
return p[1];
}
}
return NULL;
}
static int
applycontrols(NCZ_FILE_INFO_T* zinfo)
{
int i,stat = NC_NOERR;
const char* value = NULL;
NClist* modelist = nclistnew();
if((value = controllookup((const char**)zinfo->controls,"mode")) != NULL) {
if((stat = NCZ_comma_parse(value,modelist))) goto done;
}
/* Process the modelist first */
zinfo->features.mapimpl = NCZM_DEFAULT;
for(i=0;i<nclistlength(modelist);i++) {
const char* p = nclistget(modelist,i);
if(strcasecmp(p,PUREZARR)==0) zinfo->features.flags |= FLAG_PUREZARR;
else if(strcasecmp(p,"bytes")==0) zinfo->features.flags |= FLAG_BYTERANGE;
else if(strcasecmp(p,"s3")==0) zinfo->features.mapimpl = NCZM_S3;
else if(strcasecmp(p,"nz4")==0) zinfo->features.mapimpl = NCZM_NC4;
else if(strcasecmp(p,"nzf")==0) zinfo->features.mapimpl = NCZM_FILE;
else if(strcasecmp(p,"nzrf")==0)
{zinfo->features.mapimpl = NCZM_FILE; zinfo->features.flags |= FLAG_BYTERANGE;}
else if(strcasecmp(p,"nzr4")==0)
{zinfo->features.mapimpl = NCZM_NC4; zinfo->features.flags |= FLAG_BYTERANGE;}
}
/* Process other controls */
if((value = controllookup((const char**)zinfo->controls,"log")) != NULL) {
zinfo->features.flags |= FLAG_LOGGING;
ncloginit();
if(nclogopen(value))
ncsetlogging(1);
ncloginit();
if(nclogopen(value))
ncsetlogging(1);
}
if((value = controllookup((const char**)zinfo->controls,"show")) != NULL) {
if(strcasecmp(value,"fetch")==0)
zinfo->features.flags |= FLAG_SHOWFETCH;
}
done:
nclistfreeall(modelist);
return stat;
}

68
libnczarr/zarr.h Normal file
View File

@ -0,0 +1,68 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
/**
* Provide the zarr specific code to implement the netcdf-4 code.
*
* @author Dennis Heimbigner
*/
#ifndef ZARR_H
#define ZARR_H
/* zarr.c */
extern int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, const char** controls);
extern int ncz_open_dataset(NC_FILE_INFO_T*, const char** controls);
extern int ncz_del_attr(NC_FILE_INFO_T* file, NC_OBJ* container, const char* name);
extern int NCZ_isnetcdf4(struct NC_FILE_INFO*);
extern int NCZ_get_libversion(unsigned long* majorp, unsigned long* minorp,unsigned long* releasep);
extern int NCZ_get_superblock(NC_FILE_INFO_T* file, int* superblockp);
extern int ncz_unload_jatts(NCZ_FILE_INFO_T*, NC_OBJ* container, NCjson* jattrs, NCjson* jtypes);
/* zclose.c */
extern int ncz_close_file(NC_FILE_INFO_T* file, int abort);
/* zcvt.c */
extern int NCZ_convert1(NCjson* jsrc, nc_type, char* memory0);
extern int NCZ_stringconvert1(nc_type typid, char* src, char** strp);
extern int NCZ_stringconvert(nc_type typid, size_t len, void* data0, NCjson** jdatap);
/* zsync.c */
extern int ncz_sync_file(NC_FILE_INFO_T* file);
extern int ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
extern int ncz_sync_atts(NC_FILE_INFO_T*, NC_OBJ* container, NCindex* attlist);
extern int ncz_read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
extern int ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container);
extern int ncz_read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
extern int ncz_read_file(NC_FILE_INFO_T* file);
extern int ncz_write_var(NC_VAR_INFO_T* var);
extern int ncz_create_superblock(NCZ_FILE_INFO_T* zinfo);
/* zutil.c */
extern int NCZ_grpkey(const NC_GRP_INFO_T* grp, char** pathp);
extern int NCZ_varkey(const NC_VAR_INFO_T* var, char** pathp);
extern int NCZ_dimkey(const NC_DIM_INFO_T* dim, char** pathp);
extern int ncz_splitkey(const char* path, NClist* segments);
extern int NCZ_readdict(NCZMAP* zmap, const char* key, NCjson** jsonp);
extern int NCZ_readarray(NCZMAP* zmap, const char* key, NCjson** jsonp);
extern int ncz_zarr_type_name(nc_type nctype, int little, const char** znamep);
extern int ncz_dtype2typeinfo(const char* dtype, nc_type* nctypep, int* endianness);
extern int ncz_fill_value_sort(nc_type nctype, int*);
extern int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size);
extern int NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json);
extern int NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp);
extern int NCZ_isLittleEndian(void);
extern int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, NClist* objlist);
extern int NCZ_grpname_full(int gid, char** pathp);
extern int ncz_get_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var);
extern int NCZ_comma_parse(const char* s, NClist* list);
extern int NCZ_swapatomicdata(size_t datalen, void* data, int typesize);
extern char** NCZ_clonestringvec(size_t len, const char** vec);
extern void NCZ_freestringvec(size_t len, char** vec);
extern int NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fillchunkp);
#endif /*ZARR_H*/

1042
libnczarr/zattr.c Normal file
View File

File diff suppressed because it is too large Load Diff

534
libnczarr/zcache.c Normal file
View File

@ -0,0 +1,534 @@
/* Copyright 2018, University Corporation for Atmospheric
* Research. See COPYRIGHT file for copying and redistribution
* conditions. */
/**
* @file @internal The functions which control NCZ
* caching. These caching controls allow the user to change the cache
* sizes of ZARR before opening files.
*
* @author Dennis Heimbigner, Ed Hartnett
*/
#include "zincludes.h"
#include "zcache.h"
#undef DEBUG
#undef FILLONREAD
#undef FLUSH
/* Forward */
static int get_chunk(NCZChunkCache* cache, const char* key, NCZCacheEntry* entry);
static int put_chunk(NCZChunkCache* cache, const char* key, const NCZCacheEntry*);
static int create_chunk(NCZChunkCache* cache, const char* key, NCZCacheEntry* entry);
static int buildchunkkey(size_t R, const size64_t* chunkindices, char** keyp);
static int makeroom(NCZChunkCache* cache);
/**************************************************/
/* Dispatch table per-var cache functions */
/**
* @internal Set chunk cache size for a variable. This is the internal
* function called by nc_set_var_chunk_cache().
*
* @param ncid File ID.
* @param varid Variable ID.
* @param size Size in bytes to set cache.
* @param nelems # of entries in cache
* @param preemption Controls cache swapping.
*
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADID Bad ncid.
* @returns ::NC_ENOTVAR Invalid variable ID.
* @returns ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict
* nc3 netcdf-4 file.
* @returns ::NC_EINVAL Invalid input.
* @returns ::NC_EHDFERR HDF5 error.
* @author Ed Hartnett
*/
int
NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, float preemption)
{
NC_GRP_INFO_T *grp;
NC_FILE_INFO_T *h5;
NC_VAR_INFO_T *var;
NCZ_VAR_INFO_T *zvar;
int retval;
/* Check input for validity. */
if (preemption < 0 || preemption > 1)
return NC_EINVAL;
/* Find info for this file and group, and set pointer to each. */
if ((retval = nc4_find_nc_grp_h5(ncid, NULL, &grp, &h5)))
return retval;
assert(grp && h5);
/* Find the var. */
if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
return NC_ENOTVAR;
assert(var && var->hdr.id == varid);
zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
assert(zvar != NULL && zvar->cache != NULL);
/* Set the values. */
var->chunk_cache_size = cachesize;
var->chunk_cache_nelems = nelems;
var->chunk_cache_preemption = preemption;
#ifdef LOOK
/* Reopen the dataset to bring new settings into effect. */
if ((retval = nc4_reopen_dataset(grp, var)))
return retval;
#endif
return NC_NOERR;
}
/**
* @internal Adjust the chunk cache of a var for better
* performance.
*
* @note For contiguous and compact storage vars, or when parallel I/O
* is in use, this function will do nothing and return ::NC_NOERR;
*
* @param grp Pointer to group info struct.
* @param var Pointer to var info struct.
*
* @return ::NC_NOERR No error.
* @author Ed Hartnett
*/
int
NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
{
return NC_NOERR;
}
/**************************************************/
/**
* Create a chunk cache object
*
* @param var containing var
* @param entrysize Size in bytes of an entry
* @param cachep return cache pointer
*
* @return ::NC_NOERR No error.
* @return ::NC_EINVAL Bad preemption.
* @author Dennis Heimbigner, Ed Hartnett
*/
int
NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, NCZChunkCache** cachep)
{
int stat = NC_NOERR;
NCZChunkCache* cache = NULL;
void* fill = NULL;
size_t nelems, cachesize;
if(chunksize == 0) return NC_EINVAL;
if((cache = calloc(1,sizeof(NCZChunkCache))) == NULL)
{stat = NC_ENOMEM; goto done;}
cache->var = var;
cache->chunksize = chunksize;
assert(cache->fillchunk == NULL);
cache->fillchunk = NULL;
/* Figure out the actual cache size */
cachesize = var->chunk_cache_size;
nelems = (cachesize / chunksize);
if(nelems == 0) nelems = 1;
/* Make consistent */
cachesize = nelems * chunksize;
cache->maxentries = nelems;
#ifdef FLUSH
cache->maxentries = 1;
#endif
#ifdef DEBUG
fprintf(stderr,"%s.cache: nelems=%ld size=%ld\n",
var->hdr.name,(unsigned long)cache->maxentries,(unsigned long)(cache->maxentries*cache->chunksize));
#endif
if((cache->entries = nclistnew()) == NULL)
{stat = NC_ENOMEM; goto done;}
nclistsetalloc(cache->entries,cache->maxentries);
if(cachep) {*cachep = cache; cache = NULL;}
done:
nullfree(fill);
nullfree(cache);
return THROW(stat);
}
void
NCZ_free_chunk_cache(NCZChunkCache* cache)
{
if(cache == NULL) return;
/* Iterate over the entries */
while(nclistlength(cache->entries) > 0) {
NCZCacheEntry* entry = nclistremove(cache->entries,0);
nullfree(entry->data); nullfree(entry->key); nullfree(entry);
}
#ifdef DEBUG
fprintf(stderr,"|cache.free|=%ld\n",nclistlength(cache->entries));
#endif
nclistfree(cache->entries);
cache->entries = NULL;
nullfree(cache->fillchunk);
nullfree(cache);
}
size64_t
NCZ_cache_entrysize(NCZChunkCache* cache)
{
assert(cache);
return cache->chunksize;
}
/* Return number of active entries in cache */
size64_t
NCZ_cache_size(NCZChunkCache* cache)
{
assert(cache);
return nclistlength(cache->entries);
}
int
NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap)
{
int stat = NC_NOERR;
char* key = NULL;
int rank = cache->var->ndims;
NC_FILE_INFO_T* file = cache->var->container->nc4_info;
NCZCacheEntry* entry = NULL;
int i;
/* Create the key for this cache */
if((stat = NCZ_buildchunkpath(cache,indices,&key))) goto done;
/* See if already in cache try MRU */
for(i=nclistlength(cache->entries)-1;i>=0;i--) {
entry = (NCZCacheEntry*)nclistget(cache->entries,i);
if(strcmp(key,entry->key)==0) {
if(datap) *datap = entry->data;
/* Move to keep MRU at end */
nclistremove(cache->entries,i);
break;
} else entry = NULL;
}
if(entry == NULL) { /*!found*/
/* Make room in the cache */
if((stat=makeroom(cache))) goto done;
/* Create a new entry */
if((entry = calloc(1,sizeof(NCZCacheEntry)))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(entry->indices,indices,rank*sizeof(size64_t));
/* Create the local copy space */
if((entry->data = calloc(1,cache->chunksize)) == NULL)
{stat = NC_ENOMEM; goto done;}
entry->key= key; key = NULL;
/* Try to read the object in toto */
stat=get_chunk(cache,entry->key,entry);
switch (stat) {
case NC_NOERR: break;
case NC_EEMPTY:
case NC_ENOTFOUND: /*signals the chunk needs to be created */
/* If the file is read-only, then fake the chunk */
entry->modified = (!file->no_write);
if(!file->no_write) {
if((stat = create_chunk(cache,entry->key,entry))) goto done;
}
#ifdef FILLONREAD
/* apply fill value */
memcpy(entry->data,cache->fillchunk,cache->chunksize);
#else
memset(entry->data,0,cache->chunksize);
#endif
break;
default: goto done;
}
}
nclistpush(cache->entries,entry);
#ifdef DEBUG
fprintf(stderr,"|cache.read.lru|=%ld\n",nclistlength(cache->entries));
#endif
if(datap) *datap = entry->data;
entry = NULL;
done:
if(entry) {nullfree(entry->data); nullfree(entry->key);}
nullfree(entry);
nullfree(key);
return THROW(stat);
}
int
NCZ_write_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap)
{
int stat = NC_NOERR;
char* key = NULL;
int i,rank = cache->var->ndims;
NCZCacheEntry* entry = NULL;
/* Create the key for this cache */
if((stat = NCZ_buildchunkpath(cache,indices,&key))) goto done;
/* See if already in cache try MRU */
for(i=nclistlength(cache->entries)-1;i>=0;i--) {
entry = (NCZCacheEntry*)nclistget(cache->entries,i);
if(strcmp(key,entry->key)==0) {
if(datap) *datap = entry->data;
/* Move to keep MRU at end */
nclistremove(cache->entries,i);
break;
} else entry = NULL;
}
if(entry == NULL) { /*!found*/
if((stat=makeroom(cache))) goto done;
/* Create a new entry */
if((entry = calloc(1,sizeof(NCZCacheEntry)))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(entry->indices,indices,rank*sizeof(size64_t));
/* Create the local copy space */
if((entry->data = calloc(1,cache->chunksize)) == NULL)
{stat = NC_ENOMEM; goto done;}
entry->key= key; key = NULL;
}
entry->modified = 1;
nclistpush(cache->entries,entry); /* MRU order */
#ifdef DEBUG
fprintf(stderr,"|cache.write|=%ld\n",nclistlength(cache->entries));
#endif
entry = NULL;
done:
if(entry) {nullfree(entry->data); nullfree(entry->key);}
nullfree(entry);
nullfree(key);
return THROW(stat);
}
static int
makeroom(NCZChunkCache* cache)
{
int stat = NC_NOERR;
/* Flush from LRU end if we are at capacity */
while(nclistlength(cache->entries) >= cache->maxentries) {
NCZCacheEntry* e = nclistremove(cache->entries,0);
assert(e != NULL);
if(e->modified) /* flush to file */
stat=put_chunk(cache,e->key,e);
/* reclaim */
nullfree(e->data); nullfree(e->key); nullfree(e);
}
#ifdef DEBUG
fprintf(stderr,"|cache.makeroom|=%ld\n",nclistlength(cache->entries));
#endif
return stat;
}
int
NCZ_flush_chunk_cache(NCZChunkCache* cache)
{
int stat = NC_NOERR;
size_t i;
if(NCZ_cache_size(cache) == 0) goto done;
/* Iterate over the entries in hashmap */
for(i=0;i<nclistlength(cache->entries);i++) {
NCZCacheEntry* entry = nclistget(cache->entries,i);
if(entry->modified) {
/* Write out this chunk in toto*/
if((stat=put_chunk(cache,entry->key,entry)))
goto done;
}
entry->modified = 0;
}
done:
return THROW(stat);
}
#if 0
int
NCZ_chunk_cache_modified(NCZChunkCache* cache, const size64_t* indices)
{
int stat = NC_NOERR;
char* key = NULL;
NCZCacheEntry* entry = NULL;
int rank = cache->var->ndims;
/* Create the key for this cache */
if((stat=buildchunkkey(rank, indices, &key))) goto done;
/* See if already in cache */
if(NC_hashmapget(cache->entries, key, strlen(key), (uintptr_t*)entry)) { /* found */
entry->modified = 1;
}
done:
nullfree(key);
return THROW(stat);
}
#endif
/**************************************************/
/*
From Zarr V2 Specification:
"The compressed sequence of bytes for each chunk is stored under
a key formed from the index of the chunk within the grid of
chunks representing the array. To form a string key for a
chunk, the indices are converted to strings and concatenated
with the period character (".") separating each index. For
example, given an array with shape (10000, 10000) and chunk
shape (1000, 1000) there will be 100 chunks laid out in a 10 by
10 grid. The chunk with indices (0, 0) provides data for rows
0-1000 and columns 0-1000 and is stored under the key "0.0"; the
chunk with indices (2, 4) provides data for rows 2000-3000 and
columns 4000-5000 and is stored under the key "2.4"; etc."
*/
/**
* @param R Rank
* @param chunkindices The chunk indices
* @param keyp Return the chunk key string
*/
static int
buildchunkkey(size_t R, const size64_t* chunkindices, char** keyp)
{
int stat = NC_NOERR;
int r;
NCbytes* key = ncbytesnew();
if(keyp) *keyp = NULL;
for(r=0;r<R;r++) {
char sindex[64];
if(r > 0) ncbytescat(key,".");
/* Print as decimal with no leading zeros */
snprintf(sindex,sizeof(sindex),"%lu",(unsigned long)chunkindices[r]);
ncbytescat(key,sindex);
}
ncbytesnull(key);
if(keyp) *keyp = ncbytesextract(key);
ncbytesfree(key);
return THROW(stat);
}
/**
* @internal Push data to chunk of a file.
* If chunk does not exist, create it
*
* @param file Pointer to file info struct.
* @param proj Chunk projection
* @param datalen size of data
* @param data Buffer containing the chunk data to write
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
put_chunk(NCZChunkCache* cache, const char* key, const NCZCacheEntry* entry)
{
int stat = NC_NOERR;
NCZ_FILE_INFO_T* zfile = NULL;
NCZMAP* map = NULL;
LOG((3, "%s: var: %p", __func__, cache->var));
zfile = ((cache->var->container)->nc4_info)->format_file_info;
map = zfile->map;
stat = nczmap_write(map,key,0,cache->chunksize,entry->data);
switch(stat) {
case NC_NOERR: break;
case NC_EEMPTY:
/* Create the chunk */
if((stat = nczmap_defineobj(map,key))) goto done;
/* write again */
if((stat = nczmap_write(map,key,0,cache->chunksize,entry->data)))
goto done;
break;
default: goto done;
}
done:
return THROW(stat);
}
/**
* @internal Push data from memory to file.
*
* @param cache Pointer to parent cache
* @param key chunk key
* @param entry cache entry to read into
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
get_chunk(NCZChunkCache* cache, const char* key, NCZCacheEntry* entry)
{
int stat = NC_NOERR;
NCZMAP* map = NULL;
NC_FILE_INFO_T* file = NULL;
NCZ_FILE_INFO_T* zfile = NULL;
LOG((3, "%s: file: %p", __func__, file));
file = (cache->var->container)->nc4_info;
zfile = file->format_file_info;
map = zfile->map;
assert(map && entry->data);
stat = nczmap_read(map,key,0,cache->chunksize,(char*)entry->data);
return THROW(stat);
}
static int
create_chunk(NCZChunkCache* cache, const char* key, NCZCacheEntry* entry)
{
int stat = NC_NOERR;
NC_FILE_INFO_T* file = NULL;
NCZ_FILE_INFO_T* zfile = NULL;
NCZMAP* map = NULL;
file = (cache->var->container)->nc4_info;
zfile = file->format_file_info;
map = zfile->map;
/* Create the chunk */
if((stat = nczmap_defineobj(map,key))) goto done;
entry->modified = 1; /* mark as modified */
/* let higher function decide on fill */
done:
return THROW(stat);
}
int
NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, char** keyp)
{
int stat = NC_NOERR;
char* chunkname = NULL;
char* varkey = NULL;
char* key = NULL;
/* Get the chunk object name */
if((stat = buildchunkkey(cache->var->ndims, chunkindices, &chunkname))) goto done;
/* Get the var object key */
if((stat = NCZ_varkey(cache->var,&varkey))) goto done;
/* Prefix the path to the containing variable object */
if((stat=nczm_concat(varkey,chunkname,&key))) goto done;
if(keyp) {*keyp = key; key = NULL;}
done:
nullfree(chunkname);
nullfree(varkey);
nullfree(key);
return THROW(stat);
}

38
libnczarr/zcache.h Normal file
View File

@ -0,0 +1,38 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#ifndef ZCACHE_H
#define ZCACHE_H
typedef struct NCZCacheEntry {
int modified;
size64_t indices[NC_MAX_VAR_DIMS];
char* key; /* as string */
void* data;
} NCZCacheEntry;
typedef struct NCZChunkCache {
const NC_VAR_INFO_T* var; /* backlink */
size64_t chunksize;
void* fillchunk; /* enough fillvalues to fill a chunk */
size_t maxentries; /* Max number of entries allowed */
NClist* entries; /* in LRU order */
} NCZChunkCache;
/**************************************************/
extern int NCZ_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption);
extern int NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
extern int NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t, NCZChunkCache** cachep);
extern void NCZ_free_chunk_cache(NCZChunkCache* cache);
extern int NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap);
extern int NCZ_flush_chunk_cache(NCZChunkCache* cache);
extern size64_t NCZ_cache_entrysize(NCZChunkCache* cache);
extern NCZCacheEntry* NCZ_cache_entry(NCZChunkCache* cache, const size64_t* indices);
extern size64_t NCZ_cache_size(NCZChunkCache* cache);
extern int NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, char** keyp);
#endif /*ZCACHE_H*/

227
libnczarr/zchunking.c Normal file
View File

@ -0,0 +1,227 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#include "zincludes.h"
static int pcounter = 0;
/* Forward */
static int compute_intersection(const NCZSlice* slice, const size64_t chunklen, NCZChunkRange* range);
/**************************************************/
/* Goal:create a vector of chunk ranges: one for each slice in
the top-level input. For each slice, compute the index (not
absolute position) of the first chunk that intersects the slice
and the index of the last chunk that intersects the slice.
In practice, the count = last - first + 1 is stored instead of the last index.
*/
int
NCZ_compute_chunk_ranges(
int rank, /* variable rank */
const NCZSlice* slices, /* the complete set of slices |slices| == R*/
const size64_t* chunklen, /* the chunk length corresponding to the dimensions */
NCZChunkRange* ncr)
{
int stat = NC_NOERR;
int i;
for(i=0;i<rank;i++) {
if((stat = compute_intersection(&slices[i],chunklen[i],&ncr[i])))
goto done;
}
done:
return stat;
}
static int
compute_intersection(
const NCZSlice* slice,
const size64_t chunklen,
NCZChunkRange* range)
{
range->start = floordiv(slice->start, chunklen);
range->stop = ceildiv(slice->stop, chunklen);
return NC_NOERR;
}
/**
Compute the projection of a slice as applied to n'th chunk.
This is somewhat complex because for the first projection, the
start is the slice start, but after that, we have to take into
account that for a non-one stride, the start point in a
projection may be offset by some value in the range of
0..(slice.stride-1).
*/
int
NCZ_compute_projections(size64_t dimlen, size64_t chunklen, size64_t chunkindex, const NCZSlice* slice, size_t n, NCZProjection* projections)
{
int stat = NC_NOERR;
size64_t offset,count,avail;
NCZProjection* projection;
projection = &projections[n];
projection->id = ++pcounter;
projection->chunkindex = chunkindex;
offset = chunklen * chunkindex; /* with respect to dimension (WRD) */
/* Actual limit of the n'th touched chunk, taking
dimlen and stride->stop into account. */
projection->limit = (chunkindex + 1) * chunklen; /* WRD */
if(projection->limit > dimlen) projection->limit = dimlen;
if(projection->limit > slice->stop) projection->limit = slice->stop;
/* Len is no. of touched indices along this dimension */
projection->len = projection->limit - offset;
if(n == 0) {
/*initial case: original slice start is in 1st projection */
projection->first = slice->start; /* WRD */
} else { /* n > 0 */
NCZProjection* prev = &projections[n-1];
/* prevunused is the amount unused at end of the previous chunk.
=> we need to skip (slice->stride-prevunused) in this chunk */
/* Compute limit of previous chunk */
size64_t prevunused = prev->limit - prev->last;
projection->first = offset + (slice->stride - prevunused); /* WRD */
}
/* Compute number of places touched in this chunk */
avail = projection->limit - projection->first; /*WRD*/
count = ceildiv(avail, slice->stride);
/* Last place to be touched */
projection->last = projection->first + ((slice->stride * count) - 1); /*WRD*/
/* Compute the slice relative to this chunk.
Recall the possibility that start+stride >= projection->limit */
projection->chunkslice.start = (projection->first - offset);
projection->chunkslice.stop = projection->chunkslice.start + (slice->stride * count);
//+1
if(slice->stop > projection->limit) {
projection->chunkslice.stop = projection->len;
}
projection->chunkslice.stride = slice->stride;
projection->chunkslice.len = chunklen;
/* compute the I/O position: the "location" in the memory
array to read/write items */
if(n == 0)
projection->iopos = 0;
else
projection->iopos = ceildiv(offset - slice->start, slice->stride);
/* And number of I/O items */
projection->iocount = count;
projection->memslice.start = projection->iopos;
projection->memslice.stop = projection->memslice.start + projection->iocount;
projection->memslice.stride = 1;
#if 0
projection->memslice.len = projection->memslice.stop;
#else
projection->memslice.len = dimlen;
#endif
return stat;
}
/* Goal:
Create a vector of projections wrt a slice and a sequence of chunks.
*/
int
NCZ_compute_per_slice_projections(
int r, /* which dimension are we projecting? */
const NCZSlice* slice, /* the slice for which projections are computed */
const NCZChunkRange* range, /* range */
size64_t dimlen, /* the dimension length for r'th dimension */
size64_t chunklen, /* the chunk length corresponding to the dimension */
NCZSliceProjections* slp)
{
int stat = NC_NOERR;
size64_t index,slicecount;
size_t n;
/* Part fill the Slice Projections */
slp->r = r;
slp->range = *range;
slp->count = range->stop - range->start;
if((slp->projections = calloc(slp->count,sizeof(NCZProjection))) == NULL)
{stat = NC_ENOMEM; goto done;}
/* Compute the total number of output items defined by this slice
(equivalent to count as used by nc_get_vars) */
slicecount = ceildiv((slice->stop - slice->start), slice->stride);
if(slicecount < 0) slicecount = 0;
/* Iterate over each chunk that intersects slice to produce projection */
for(n=0,index=range->start;index<range->stop;index++,n++) {
if((stat = NCZ_compute_projections(dimlen, chunklen, index, slice, n, slp->projections)))
goto done;
}
done:
return stat;
}
/* Goal:create a vector of SliceProjection instances: one for each
slice in the top-level input. For each slice, compute a set
of projections from it wrt a dimension and a chunk size
associated with that dimension.
*/
int
NCZ_compute_all_slice_projections(
int rank, /* variable rank */
const NCZSlice* slices, /* the complete set of slices |slices| == R*/
const size64_t* dimlen, /* the dimension lengths associated with a variable */
const size64_t* chunklen, /* the chunk length corresponding to the dimensions */
const NCZChunkRange* ranges,
NCZSliceProjections* results)
{
int stat = NC_NOERR;
size64_t r;
for(r=0;r<rank;r++) {
/* Compute each of the rank SliceProjections instances */
NCZSliceProjections* slp = &results[r];
if((stat=NCZ_compute_per_slice_projections(
r,
&slices[r],
&ranges[r],
dimlen[r],
chunklen[r],
slp))) goto done;
}
done:
return stat;
}
/**************************************************/
/* Utilities */
void
NCZ_clearsliceprojections(int count, NCZSliceProjections* slpv)
{
if(slpv != NULL) {
int i;
for(i=0;i<count;i++) {
NCZSliceProjections* slp = &slpv[i];
nullfree(slp->projections);
}
}
}
#if 0
static void
clearallprojections(NCZAllProjections* nap)
{
if(nap != NULL) {
int i;
for(i=0;i<nap->rank;i++)
nclistfreeall(&nap->allprojections[i].projections);
}
}
#endif

104
libnczarr/zchunking.h Normal file
View File

@ -0,0 +1,104 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#ifndef ZCHUNKING_H
#define ZCHUNKING_H
/* Callback functions so we can use with unit tests */
typedef int (*NCZ_reader)(void* source, size64_t* chunkindices, void** chunkdata);
struct Reader {void* source; NCZ_reader read;};
/* Define the intersecting set of chunks for a slice
in terms of chunk indices (not absolute positions)
*/
typedef struct NCZChunkRange {
size64_t start; /* index, not absolute */
size64_t stop;
} NCZChunkRange;
/* A per-dimension slice for the incoming hyperslab */
typedef struct NCZSlice {
size64_t start;
size64_t stop; /* start + (count*stride) */
size64_t stride;
size64_t len; /* full dimension length */
} NCZSlice;
typedef struct NCProjection {
int id;
size64_t chunkindex; /* which chunk are we projecting */
size64_t first; /* absolute first position to be touched in this chunk */
size64_t last; /* absolute position of last value touched */
size64_t len; /* Not last place touched, but the offset of last place
that could be touched */
size64_t limit; /* Actual limit of chunk = min(limit,dimlen) */
size64_t iopos; /* start point in the data memory to access the data */
size64_t iocount; /* no. of I/O items */
NCZSlice chunkslice; /* slice relative to this chunk */
NCZSlice memslice; /* slice relative to memory */
} NCZProjection;
/* Set of Projections for a slice */
typedef struct NCZSliceProjections {
int r; /* 0<=r<rank */
NCZChunkRange range; /* Chunk ranges covered by this set of projections */
size_t count; /* |projections| == (range.stop - range.start) */
NCZProjection* projections; /* Vector of projections derived from the
original slice when intersected across
the chunk */
} NCZSliceProjections;
/* Combine some values to simplify internal argument lists */
struct Common {
NC_FILE_INFO_T* file;
NC_VAR_INFO_T* var;
struct NCZChunkCache* cache;
int reading; /* 1=> read, 0 => write */
int rank;
size64_t* dimlens;
size64_t* chunklens;
void* memory;
size_t typesize;
void* fillvalue;
size64_t chunksize; /* computed product of chunklens */
int swap; /* var->format_info_file->native_endianness == var->endianness */
size64_t shape[NC_MAX_VAR_DIMS]; /* shape of the output hyperslab */
NCZSliceProjections* allprojections;
/* Parametric chunk reader so we can do unittests */
struct Reader reader;
};
/**************************************************/
/* From zchunking.c */
extern int NCZ_compute_chunk_ranges(int rank, const NCZSlice*, const size64_t*, NCZChunkRange* ncr);
extern int NCZ_compute_projections(size64_t dimlen, size64_t chunklen, size64_t chunkindex, const NCZSlice* slice, size_t n, NCZProjection* projections);
extern int NCZ_compute_per_slice_projections(int rank, const NCZSlice*, const NCZChunkRange*, size64_t dimlen, size64_t chunklen, NCZSliceProjections* slp);
extern int NCZ_compute_all_slice_projections(int rank, const NCZSlice* slices, const size64_t* dimlen, const size64_t* chunklen, const NCZChunkRange*, NCZSliceProjections*);
/* From zwalk.c */
extern int ncz_chunking_init(void);
extern int NCZ_transferslice(NC_VAR_INFO_T* var, int reading,
size64_t* start, size64_t* count, size64_t* stride,
void* memory, nc_type typecode);
extern int NCZ_transfer(struct Common* common, NCZSlice* slices);
extern size64_t NCZ_computelinearoffset(size_t, const size64_t*, const size64_t*);
/* Special entry points for unit testing */
struct Common;
struct NCZOdometer;
extern int NCZ_projectslices(size64_t* dimlens,
size64_t* chunklens,
NCZSlice* slices,
struct Common*, struct NCZOdometer**);
extern int NCZ_chunkindexodom(int rank, const NCZChunkRange* ranges, size64_t*, struct NCZOdometer** odom);
extern void NCZ_clearsliceprojections(int count, NCZSliceProjections* slpv);
extern void NCZ_clearcommon(struct Common* common);
#define floordiv(x,y) ((x) / (y))
#define ceildiv(x,y) (((x) % (y)) == 0 ? ((x) / (y)) : (((x) / (y)) + 1))
#endif /*ZCHUNKING_H*/

287
libnczarr/zclose.c Normal file
View File

@ -0,0 +1,287 @@
/*********************************************************************
* Copyright 1993, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#include "zincludes.h"
/* Forward */
static int zclose_group(NC_GRP_INFO_T*);
static int zclose_gatts(NC_GRP_INFO_T*);
static int zclose_vars(NC_GRP_INFO_T*);
static int zclose_dims(NC_GRP_INFO_T*);
static int zclose_types(NC_GRP_INFO_T*);
static int zclose_type(NC_TYPE_INFO_T* type);
static int zwrite_vars(NC_GRP_INFO_T *grp);
/**************************************************/
/**
* @internal This function will recurse through an open ZARR file and
* release resources. All ZARR annotations reclaimed
*
* @param file Pointer to ZARR file info struct.
* @param abort True if this is an abort.
*
* @return ::NC_NOERR No error.
* @return ::NC_ENCZARR could not close the file.
* @author Dennis Heimbigner
*/
int
ncz_close_file(NC_FILE_INFO_T* file, int abort)
{
int stat = NC_NOERR;
NCZ_FILE_INFO_T* zinfo = NULL;
if(!abort) {
/* Flush | create all chunks for all vars */
if((stat=zwrite_vars(file->root_grp))) goto done;
}
/* Internal close to reclaim zarr annotations */
if((stat = zclose_group(file->root_grp)))
goto done;
zinfo = file->format_file_info;
if((stat = nczmap_close(zinfo->map,(abort && zinfo->created)?1:0)))
goto done;
NCZ_freestringvec(0,zinfo->controls);
NC_authclear(zinfo->auth);
nullfree(zinfo->auth);
nullfree(zinfo);
done:
return stat;
}
/**************************************************/
/**
* @internal Recursively free zarr annotations for a group (and everything
* it contains).
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_group(NC_GRP_INFO_T *grp)
{
int stat = NC_NOERR;
NCZ_GRP_INFO_T* zgrp;
int i;
assert(grp && grp->format_grp_info != NULL);
LOG((3, "%s: grp->name %s", __func__, grp->hdr.name));
/* Recursively call this function for each child, if any, stopping
* if there is an error. */
for(i=0; i<ncindexsize(grp->children); i++) {
if ((stat = zclose_group((NC_GRP_INFO_T*)ncindexith(grp->children,i))))
goto done;
}
/* Close resources associated with global attributes. */
if ((stat = zclose_gatts(grp)))
goto done;
/* Close resources associated with vars. */
if ((stat = zclose_vars(grp)))
goto done;
/* Close resources associated with dims. */
if ((stat = zclose_dims(grp)))
goto done;
/* Close resources associated with types. */
if ((stat = zclose_types(grp)))
goto done;
/* Close the zgroup. */
zgrp = grp->format_grp_info;
LOG((4, "%s: closing group %s", __func__, grp->hdr.name));
nullfree(zgrp);
grp->format_grp_info = NULL; /* avoid memory errors */
done:
return stat;
}
/**
* @internal Close resources for global atts in a group.
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_gatts(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
NC_ATT_INFO_T *att;
int a;
for(a = 0; a < ncindexsize(grp->att); a++) {
NCZ_ATT_INFO_T* zatt = NULL;
att = (NC_ATT_INFO_T* )ncindexith(grp->att, a);
assert(att && att->format_att_info != NULL);
zatt = att->format_att_info;
nullfree(zatt);
att->format_att_info = NULL; /* avoid memory errors */
}
return stat;
}
/**
* @internal Close resources for vars in a group.
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_vars(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
NC_VAR_INFO_T* var;
NCZ_VAR_INFO_T* zvar;
NC_ATT_INFO_T* att;
int a, i;
for(i = 0; i < ncindexsize(grp->vars); i++) {
var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
assert(var && var->format_var_info);
zvar = var->format_var_info;;
for(a = 0; a < ncindexsize(var->att); a++) {
NCZ_ATT_INFO_T* zatt;
att = (NC_ATT_INFO_T*)ncindexith(var->att, a);
assert(att && att->format_att_info);
zatt = att->format_att_info;
nullfree(zatt);
att->format_att_info = NULL; /* avoid memory errors */
}
/* Reclaim the type */
(void)zclose_type(var->type_info);
NCZ_free_chunk_cache(zvar->cache);
nullfree(zvar);
var->format_var_info = NULL; /* avoid memory errors */
}
return stat;
}
/**
* @internal Close resources for dims in a group.
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_dims(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
NC_DIM_INFO_T* dim;
int i;
for(i = 0; i < ncindexsize(grp->dim); i++) {
NCZ_DIM_INFO_T* zdim;
dim = (NC_DIM_INFO_T*)ncindexith(grp->dim, i);
assert(dim && dim->format_dim_info);
zdim = dim->format_dim_info;
nullfree(zdim);
dim->format_dim_info = NULL; /* avoid memory errors */
}
return stat;
}
/**
* @internal Close resources for a single type. Set values to
* 0 after closing types. Because of type reference counters, these
* closes can be called multiple times.
*
* @param type Pointer to type struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_type(NC_TYPE_INFO_T* type)
{
int stat = NC_NOERR;
NCZ_TYPE_INFO_T* ztype;
assert(type && type->format_type_info != NULL);
/* Get Zarr-specific type info. */
ztype = type->format_type_info;
nullfree(ztype);
type->format_type_info = NULL; /* avoid memory errors */
return stat;
}
/**
* @internal Close resources for types in a group. Set values to
* 0 after closing types. Because of type reference counters, these
* closes can be called multiple times.
* Warning: note that atomic types are not covered here; this
* is only for user-defined types.
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_types(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
int i;
NC_TYPE_INFO_T* type;
for(i = 0; i < ncindexsize(grp->type); i++)
{
type = (NC_TYPE_INFO_T*)ncindexith(grp->type, i);
if((stat = zclose_type(type))) goto done;
}
done:
return stat;
}
/**
* @internal Recursively flush/create all data for all vars.
*
* @param grp Pointer to group info struct whose vars need to be written
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zwrite_vars(NC_GRP_INFO_T *grp)
{
int stat = NC_NOERR;
int i;
assert(grp && grp->format_grp_info != NULL);
LOG((3, "%s: grp->name %s", __func__, grp->hdr.name));
/* Write all vars for this group breadth first */
for(i = 0; i < ncindexsize(grp->vars); i++) {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
if((stat = ncz_write_var(var))) goto done;
}
/* Recursively call this function for each child group, if any, stopping
* if there is an error. */
for(i=0; i<ncindexsize(grp->children); i++) {
if ((stat = zwrite_vars((NC_GRP_INFO_T*)ncindexith(grp->children,i))))
goto done;
}
done:
return stat;
}

132
libnczarr/zcreate.c Normal file
View File

@ -0,0 +1,132 @@
/* Copyright 2003-2018, University Corporation for Atmospheric
* Research. See COPYRIGHT file for copying and redistribution
* conditions. */
/**
* @file
* @internal The netCDF-4 file functions relating to file creation.
*
* @author Dennis Heimbigner, Ed Hartnett
*/
#include "zincludes.h"
/** @internal These flags may not be set for create. */
static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_DISKLESS|NC_64BIT_OFFSET|NC_CDF5);
/**
* @internal Create a netCDF-4/NCZ file.
*
* @param path The file name of the new file.
* @param cmode The creation mode flag.
* @param initialsz The proposed initial file size (advisory)
* @param nc Pointer to an instance of NC.
*
* @return ::NC_NOERR No error.
* @return ::NC_EINVAL Invalid input (check cmode).
* @return ::NC_EEXIST File exists and NC_NOCLOBBER used.
* @return ::NC_EHDFERR ZARR returned error.
* @ingroup netcdf4
* @author Dennis Heimbigner, Ed Hartnett
*/
static int
ncz_create_file(const char *path, int cmode, size_t initialsz, const char** controls, int ncid)
{
int retval = NC_NOERR;
NC_FILE_INFO_T* h5 = NULL;
assert(path);
LOG((3, "%s: path %s mode 0x%x", __func__, path, cmode));
/* Add necessary structs to hold netcdf-4 file data. */
if ((retval = nc4_file_list_add(ncid, path, cmode, (void**)&h5)))
BAIL(retval);
assert(h5 && h5->root_grp);
h5->root_grp->atts_read = 1;
h5->mem.inmemory = ((cmode & NC_INMEMORY) == NC_INMEMORY);
h5->mem.diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS);
h5->mem.persist = ((cmode & NC_PERSIST) == NC_PERSIST);
/* Do format specific setup */
/* Should check if file already exists, and if NC_NOCLOBBER is specified,
return an error */
if((retval = ncz_create_dataset(h5,h5->root_grp,controls)))
BAIL(retval);
/* Define mode gets turned on automatically on create. */
h5->flags |= NC_INDEF;
/* Set provenance. */
if ((retval = NCZ_new_provenance(h5)))
BAIL(retval);
return NC_NOERR;
exit: /*failure exit*/
if(!h5) return retval;
#ifdef LOOK
ncz_close_ncz_file(h5, 1, NULL); /* treat like abort */
#endif
return retval;
}
/**
* @internal Create a netCDF-4/NCZ file.
*
* @param path The file name of the new file.
* @param cmode The creation mode flag.
* @param initialsz Ignored by this function.
* @param basepe Ignored by this function.
* @param chunksizehintp Ignored by this function.
* @param parameters pointer to struct holding extra data (e.g. for
* parallel I/O) layer. Ignored if NULL.
* @param dispatch Pointer to the dispatch table for this file.
* @param nc_file Pointer to an instance of NC.
*
* @return ::NC_NOERR No error.
* @return ::NC_EINVAL Invalid input (check cmode).
* @ingroup netcdf4
* @author Dennis Heimbigner, Ed Hartnett
*/
int
NCZ_create(const char* path, int cmode, size_t initialsz, int basepe,
size_t *chunksizehintp, void *parameters,
const NC_Dispatch *dispatch, int ncid)
{
int stat = NC_NOERR;
NCURI* uri = NULL;
NC_UNUSED(parameters);
assert(path);
LOG((1, "%s: path %s cmode 0x%x ncid %d",
__func__, path, cmode ,ncid));
/* If this is our first file, initialize */
if (!ncz_initialized) NCZ_initialize();
#ifdef LOGGING
/* If nc logging level has changed, see if we need to turn on
* NCZ's error messages. */
NCZ_set_log_level();
#endif /* LOGGING */
/* Check the cmode for validity. */
if((cmode & ILLEGAL_CREATE_FLAGS) != 0)
{stat = NC_EINVAL; goto done;}
/* Turn on NC_WRITE */
cmode |= NC_WRITE;
/* Get the controls */
if(ncuriparse(path,&uri)) goto done;
/* Create the file */
stat = ncz_create_file(path, cmode, initialsz, ncurifragmentparams(uri), ncid);
done:
ncurifree(uri);
return stat;
}

356
libnczarr/zcvt.c Normal file
View File

@ -0,0 +1,356 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#include "zincludes.h"
#include <math.h>
#ifdef _MSC_VER
#include <crtdbg.h>
#endif
/*
Code taken directly from libdap4/d4cvt.c
*/
/* Intermediate results */
struct ZCVT {
signed long long int64v;
unsigned long long uint64v;
double float64v;
};
int
NCZ_convert1(NCjson* jsrc, nc_type dsttype, char* memory)
{
int stat = NC_NOERR;
nc_type srctype;
struct ZCVT zcvt;
int outofrange = 0;
/* Convert the incoming jsrc string to a restricted set of values */
switch (jsrc->sort) {
case NCJ_INT: /* convert to (u)int64 */
if(jsrc->value[0] == '-') {
if(sscanf(jsrc->value,"%lld",&zcvt.int64v) != 1)
{stat = NC_EINVAL; goto done;}
srctype = NC_INT64;
} else {
if(sscanf(jsrc->value,"%llu",&zcvt.uint64v) != 1)
{stat = NC_EINVAL; goto done;}
srctype = NC_UINT64;
}
break;
case NCJ_DOUBLE:
/* Capture nan and infinity values */
if(strcasecmp(jsrc->value,"nan")==0)
zcvt.float64v = NAN;
else if(strcasecmp(jsrc->value,"infinity")==0)
zcvt.float64v = INFINITY;
else if(strcasecmp(jsrc->value,"-infinity")==0)
zcvt.float64v = (- INFINITY);
else {
if(sscanf(jsrc->value,"%lg",&zcvt.float64v) != 1)
{stat = NC_EINVAL; goto done;}
}
srctype = NC_DOUBLE;
break;
case NCJ_BOOLEAN:
srctype = NC_UINT64;
if(strcasecmp(jsrc->value,"false")==0)
zcvt.uint64v = 0;
else
zcvt.uint64v = 1;
break;
default: stat = NC_EINTERNAL; goto done;
}
/* Now, do the down conversion into memory */
switch (dsttype) {
case NC_BYTE: {
signed char* p = (signed char*)memory;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < NC_MIN_BYTE || zcvt.int64v > NC_MAX_BYTE) outofrange = 1;
*p = (signed char)zcvt.int64v;
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_BYTE) outofrange = 1;
*p = (signed char)zcvt.uint64v;
break;
}
} break;
case NC_UBYTE: {
unsigned char* p = (unsigned char*)memory;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < 0 || zcvt.int64v > NC_MAX_BYTE) outofrange = 1;
*p = (unsigned char)zcvt.int64v;
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_UBYTE) outofrange = 1;
*p = (unsigned char)zcvt.uint64v;
break;
}
} break;
case NC_SHORT: {
signed short* p = (signed short*)memory;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < NC_MIN_SHORT || zcvt.int64v > NC_MAX_SHORT) outofrange = 1;
*p = (signed short)zcvt.int64v;
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_SHORT) outofrange = 1;
*p = (signed short)zcvt.uint64v;
break;
}
} break;
case NC_USHORT: {
unsigned short* p = (unsigned short*)memory;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < 0 || zcvt.int64v > NC_MAX_USHORT) outofrange = 1;
*p = (unsigned short)zcvt.int64v;
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_USHORT) outofrange = 1;
*p = (unsigned short)zcvt.uint64v;
break;
}
} break;
case NC_INT: {
signed int* p = (signed int*)memory;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < NC_MIN_INT || zcvt.int64v > NC_MAX_INT) outofrange = 1;
*p = (signed int)zcvt.int64v;
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_INT) outofrange = 1;
*p = (signed int)zcvt.uint64v;
break;
}
} break;
case NC_UINT: {
unsigned int* p = (unsigned int*)memory;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < 0 || zcvt.int64v > NC_MAX_UINT) outofrange = 1;
*p = (unsigned int)zcvt.int64v;
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_UINT) outofrange = 1;
*p = (unsigned int)zcvt.uint64v;
break;
}
} break;
case NC_INT64: {
signed long long* p = (signed long long*)memory;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
*p = (signed long long)zcvt.int64v;
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_INT64) outofrange = 1;
*p = (signed long long)zcvt.uint64v;
break;
}
} break;
case NC_UINT64: {
unsigned long long* p = (unsigned long long*)memory;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (signed long long)zcvt.float64v;
/* fall thru */
case NC_INT64:
if(zcvt.int64v < 0) outofrange = 1;
*p = (unsigned long long)zcvt.int64v;
break;
case NC_UINT64:
*p = (unsigned long long)zcvt.uint64v;
break;
}
} break;
case NC_FLOAT: {
float* p = (float*)memory;
switch (srctype) {
case NC_DOUBLE:
*p = (float)zcvt.float64v;
break;
case NC_INT64:
*p = (float)zcvt.int64v;
break;
case NC_UINT64:
*p = (float)zcvt.uint64v;
break;
}
} break;
case NC_DOUBLE: {
double* p = (double*)memory;
switch (srctype) {
case NC_DOUBLE:
*p = (double)zcvt.float64v;
break;
case NC_INT64:
*p = (double)zcvt.int64v;
break;
case NC_UINT64:
*p = (double)zcvt.uint64v;
break;
}
} break;
default: stat = NC_EINTERNAL; goto done;
}
done:
if(stat == NC_NOERR && outofrange) stat = NC_ERANGE;
return stat;
}
int
NCZ_stringconvert1(nc_type srctype, char* src, char** strp)
{
int stat = NC_NOERR;
struct ZCVT zcvt;
nc_type dsttype = NC_NAT;
char s[1024];
assert(srctype >= NC_NAT && srctype != NC_CHAR && srctype < NC_STRING);
/* Convert to a restricted set of values */
switch (srctype) {
case NC_BYTE: {
zcvt.int64v = (signed long long)(*((signed char*)src));
dsttype = NC_INT64;
} break;
case NC_UBYTE: {
zcvt.uint64v = (unsigned long long)(*((unsigned char*)src));
dsttype = NC_UINT64;
} break;
case NC_SHORT: {
zcvt.int64v = (signed long long)(*((signed short*)src));
dsttype = NC_INT64;
} break;
case NC_USHORT: {
zcvt.uint64v = (unsigned long long)(*((unsigned short*)src));
dsttype = NC_UINT64;
} break;
case NC_INT: {
zcvt.int64v = (signed long long)(*((signed int*)src));
dsttype = NC_INT64;
} break;
case NC_UINT: {
zcvt.uint64v = (unsigned long long)(*((unsigned int*)src));
dsttype = NC_UINT64;
} break;
case NC_INT64: {
zcvt.int64v = (signed long long)(*((signed long long*)src));
dsttype = NC_INT64;
} break;
case NC_UINT64: {
zcvt.uint64v = (unsigned long long)(*((unsigned long long*)src));
dsttype = NC_UINT64;
} break;
case NC_FLOAT: {
zcvt.float64v = (double)(*((float*)src));
dsttype = NC_DOUBLE;
} break;
case NC_DOUBLE: {
zcvt.float64v= (double)(*((double*)src));
dsttype = NC_DOUBLE;
} break;
default: stat = NC_EINTERNAL; goto done;
}
/* Convert from restricted set of values to standardized string form*/
switch (dsttype) {
case NC_INT64: {
snprintf(s,sizeof(s),"%lld",zcvt.int64v);
} break;
case NC_UINT64: {
snprintf(s,sizeof(s),"%llu",zcvt.uint64v);
} break;
case NC_DOUBLE: {
snprintf(s,sizeof(s),"%lg",zcvt.float64v); /* handles NAN? */
} break;
default: stat = NC_EINTERNAL; goto done;
}
if(strp) *strp = strdup(s);
done:
return stat;
}
int
NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap)
{
int stat = NC_NOERR;
int i;
char* src = data0; /* so we can do arithmetic on it */
size_t typelen;
char* str = NULL;
NCjson* jvalue = NULL;
NCjson* jdata = NULL;
if((stat = NC4_inq_atomic_type(typeid, NULL, &typelen)))
goto done;
/* Handle char type specially */
if(typeid == NC_CHAR) {
/* Create a string valued json object */
if((stat = NCJnewstringn(NCJ_STRING,len,src,&jdata)))
goto done;
} else { /* for all other values, create an array of values */
if((stat = NCJnew(NCJ_ARRAY,&jdata))) goto done;
for(i=0;i<len;i++) {
if((stat = NCZ_stringconvert1(typeid, src, &str)))
goto done;
switch (typeid) {
case NC_BYTE: case NC_SHORT: case NC_INT: case NC_INT64:
case NC_UBYTE: case NC_USHORT: case NC_UINT: case NC_UINT64:
if((stat=NCJnew(NCJ_INT,&jvalue))) goto done;
break;
case NC_FLOAT: case NC_DOUBLE:
if((stat=NCJnew(NCJ_DOUBLE,&jvalue))) goto done;
break;
case NC_CHAR:
if((stat=NCJnew(NCJ_STRING,&jvalue))) goto done;
break;
default: stat = NC_EINTERNAL; goto done;
}
jvalue->value = str;
str = NULL;
nclistpush(jdata->contents,jvalue);
jvalue = NULL;
src += typelen;
}
}
if(jdatap) {*jdatap = jdata; jdata = NULL;}
done:
nullfree(str);
NCJreclaim(jvalue);
NCJreclaim(jdata);
return stat;
}

Some files were not shown because too many files have changed in this diff Show More