mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-25 17:40:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into bloscfix.dmh

Browse Source
This commit is contained in:
Ward Fisher 2022-09-06 13:49:18 -06:00 committed by GitHub
commit 0d17edf2ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
58 changed files with 1981 additions and 854 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/run_tests_osx.yml vendored
View File

@ -7,7 +7,7 @@
name: Run macOS-based netCDF Tests
on: [pull_request, workflow_dispatch]
on: [push,pull_request, workflow_dispatch]
jobs:

6
.github/workflows/run_tests_ubuntu.yml vendored
View File

@ -4,7 +4,7 @@
name: Run Ubuntu/Linux netCDF Tests
on: [pull_request, workflow_dispatch]
on: [push,pull_request, workflow_dispatch]
jobs:
@ -42,7 +42,7 @@ jobs:
wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2
tar -jxf hdf-4.2.15.tar.bz2
pushd hdf-4.2.15
./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib
./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib
make -j
make install -j
popd
@ -164,7 +164,7 @@ jobs:
- name: Configure
shell: bash -l {0}
run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf4 --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-doxygen
run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf4 --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-doxygen --enable-external-server-tests
if: ${{ success() }}
- name: Look at config.log if error

2
.github/workflows/run_tests_win_mingw.yml vendored
View File

@ -7,7 +7,7 @@
name: Run MSYS2, MinGW64-based Tests
on: [pull_request, workflow_dispatch]
on: [push,pull_request, workflow_dispatch]
jobs:

15
CMakeLists.txt
View File

@ -1078,17 +1078,26 @@ OPTION(ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." OFF)
# Check for the math library so it can be explicitly linked.
IF(NOT WIN32)
FIND_LIBRARY(HAVE_LIBM NAMES math m libm)
MESSAGE(STATUS "Found Math library: ${HAVE_LIBM}")
IF(NOT HAVE_LIBM)
MESSAGE(FATAL_ERROR "Unable to find the math library.")
CHECK_FUNCTION_EXISTS(exp HAVE_LIBM_FUNC)
IF(NOT HAVE_LIBM_FUNC)
MESSAGE(FATAL_ERROR "Unable to find the math library.")
ELSE(NOT HAVE_LIBM_FUNC)
SET(HAVE_LIBM "")
ENDIF()
ELSE(NOT HAVE_LIBM)
MESSAGE(STATUS "Found Math library: ${HAVE_LIBM}")
ENDIF()
ENDIF()
# Option to Enable DAP long tests, remote tests.
OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON)
OPTION(ENABLE_EXTERNAL_SERVER_TESTS "Enable external Server remote tests." OFF)
OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF)
SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
# See if we have zlib
FIND_PACKAGE(ZLIB)
@ -1725,7 +1734,7 @@ ENDIF()
# Set some of the options as advanced.
MARK_AS_ADVANCED(ENABLE_INTERNAL_DOCS VALGRIND_TESTS ENABLE_COVERAGE_TESTS )
MARK_AS_ADVANCED(ENABLE_DAP_REMOTE_TESTS ENABLE_DAP_LONG_TESTS USE_REMOTE_CDASH)
MARK_AS_ADVANCED(ENABLE_DAP_REMOTE_TESTS ENABLE_DAP_LONG_TESTS USE_REMOTE_CDASH ENABLE_EXTERNAL_SERVER_TESTS)
MARK_AS_ADVANCED(ENABLE_DOXYGEN_BUILD_RELEASE_DOCS DOXYGEN_ENABLE_TASKS ENABLE_DOXYGEN_SERVER_SIDE_SEARCH)
MARK_AS_ADVANCED(ENABLE_SHARED_LIBRARY_VERSION)

5
RELEASE_NOTES.md
View File

@ -9,6 +9,9 @@ This file contains a high-level description of this package's evolution. Release
* [Bug Fix] Fix blosc plugin errors. See [Github #2461](https://github.com/Unidata/netcdf-c/pull/2461).
* [Bug Fix] Fix support for reading arrays of HDF5 fixed size strings. See [Github #2466](https://github.com/Unidata/netcdf-c/pull/2466).
* [Enhancement] Add support for Zarr (fixed length) string type in nczarr. See [Github #2492](https://github.com/Unidata/netcdf-c/pull/2492).
* [Bug Fix] Split the remote tests into two parts: one for the remotetest server and one for all other external servers. Also add a configure option to enable the latter set. See [Github #2491](https://github.com/Unidata/netcdf-c/pull/2491).
* [Bug Fix] Fix support for reading arrays of HDF5 fixed size strings. See [Github #2462](https://github.com/Unidata/netcdf-c/pull/2466).
* [Bug Fix] Provide a default enum const when fill value does not match any enum constant for the value zero. See [Github #2462](https://github.com/Unidata/netcdf-c/pull/2462).
* [Bug Fix] Fix the json submodule symbol conflicts between libnetcdf and the plugin specific netcdf_json.h. See [Github #2448](https://github.com/Unidata/netcdf-c/pull/2448).
* [Bug Fix] Fix quantize with CLASSIC_MODEL files. See [Github #2405](https://github.com/Unidata/netcdf-c/pull/2445).
@ -28,7 +31,7 @@ This file contains a high-level description of this package's evolution. Release
* [Enhancement] Allow the read/write of JSON-valued Zarr attributes to allow
for domain specific info such as used by GDAL/Zarr. See [Github #2278](https://github.com/Unidata/netcdf-c/pull/2278).
* [Enhancement] Turn on the XArray convention for NCZarr files by default. WARNING, this means that the mode should explicitly specify "nczarr" or "zarr" even if "xarray" or "noxarray" is specified. See [Github #2257](https://github.com/Unidata/netcdf-c/pull/2257).
* [Enhancement] Update the documentation to match the current filter capabilities See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249).
* [Enhancement] Update the documentation to match the current filter capabilities. See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249).
* [Enhancement] Support installation of pre-built standard filters into user-specified location. See [Github #2318](https://github.com/Unidata/netcdf-c/pull/2318).
* [Enhancement] Improve filter support. More specifically (1) add nc_inq_filter_avail to check if a filter is available, (2) add the notion of standard filters, (3) cleanup szip support to fix interaction with NCZarr. See [Github #2245](https://github.com/Unidata/netcdf-c/pull/2245).

3
config.h.cmake.in
View File

@ -441,6 +441,9 @@ with zip */
/* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */
#cmakedefine HDF5_UTF8_PATHS 1
/* if true, backtrace support will be used. */
#cmakedefine HAVE_EXECINFO_H 1
/* if true, include JNA bug fix */
#cmakedefine JNA 1

56
configure.ac
View File

@ -603,32 +603,53 @@ AM_CONDITIONAL(ENABLE_QUANTIZE, [test x$enable_quantize = xyes])
# --enable-dap => enable-dap4
enable_dap4=$enable_dap
AC_MSG_CHECKING([whether dap remote testing should be enabled])
AC_MSG_CHECKING([whether dap use of remotetest server should be enabled])
AC_ARG_ENABLE([dap-remote-tests],
[AS_HELP_STRING([--enable-dap-remote-tests],
[enable dap remote tests])])
test "x$enable_dap_remote_tests" = xno || enable_dap_remote_tests=yes
[AS_HELP_STRING([--disable-dap-remote-tests],
[disable dap remote tests])])
# Default off
test "x$enable_dap_remote_tests" = xyes || enable_dap_remote_tests=no
if test "x$enable_dap" = "xno" ; then
enable_dap_remote_tests=no
fi
AC_MSG_RESULT($enable_dap_remote_tests)
AC_MSG_CHECKING([whether dap use of remotetest server should be enabled])
AC_ARG_ENABLE([dap-remote-tests],
[AS_HELP_STRING([--disable-dap-remote-tests],
[disable dap remote tests])])
test "x$enable_dap_remote_tests" = xno || enable_dap_remote_tests=yes
AC_MSG_RESULT($enable_dap_remote_tests)
AC_MSG_CHECKING([whether use of external servers should be enabled])
AC_ARG_ENABLE([external-server-tests],
[AS_HELP_STRING([--enable-external-server-tests (default off)],
[enable external server tests])])
test "x$enable_external_server_tests" = xyes || enable_external_server_tests=no
AC_MSG_RESULT($enable_external_server_tests)
if test "x$enable_dap_remote_tests" = "xno" ; then
AC_MSG_NOTICE([--disable-dap_remote_tests => --disable-external-server-tests])
enable_external_server_tests=no
fi
# Default is not to do the remote authorization tests.
AC_MSG_CHECKING([whether dap remote authorization testing should be enabled (default off)])
AC_MSG_CHECKING([whether dap authorization testing should be enabled (default off)])
AC_ARG_ENABLE([dap-auth-tests],
[AS_HELP_STRING([--enable-dap-auth-tests],
[enable dap remote authorization tests])])
test "x$enable_dap_auth_tests" = xyes || enable_dap_auth_tests=no
# dap must be enabled
if test "x$enable_dap" = "xno" ; then
enable_dap_auth_tests=no
fi
# if remote tests are disabled, then so is this
if test "x$enable_dap_remote_tests" = "xno" ; then
enable_dap_remote_tests=no
fi
AC_MSG_RESULT($enable_dap_auth_tests)
# dap must be enabled
if test "x$enable_dap" = "xno" ; then
AC_MSG_NOTICE([--disable-dap => --disable-dap-remote-tests --disable-auth-tests --disable-external-server-tests])
enable_dap_remote_tests=no
enable_dap_auth_tests=no
enable_external_server_tests=no
fi
# Did the user specify a list of test servers to try for remote tests?
AC_MSG_CHECKING([which remote test server(s) to use])
AC_ARG_WITH([testservers],
@ -652,16 +673,20 @@ fi
if test "x$enable_dap_remote_tests" = xyes; then
AC_DEFINE([ENABLE_DAP_REMOTE_TESTS], [1], [if true, do remote tests])
fi
if test "x$enable_external_server_tests" = xyes; then
AC_DEFINE([ENABLE_EXTERNAL_SERVER_TESTS], [1], [if true, do remote external tests])
fi
AC_MSG_CHECKING([whether the time-consuming dap tests should be enabled (default off)])
AC_ARG_ENABLE([dap-long-tests],
[AS_HELP_STRING([--enable-dap-long-tests],
[enable dap long tests])])
test "x$enable_dap_long_tests" = xyes || enable_dap_long_tests=no
if test "x$enable_dap_remote_tests" = "xno" ; then
AC_MSG_RESULT([$enable_dap_long_tests])
if test "x$enable_dap_remote_tests" = "xno" || test "x$enable_external_server_tests" = "xno" ; then
AC_MSG_NOTICE([--disable-dap-remote|external-server-tests => --disable_dap_long_tests])
enable_dap_long_tests=no
fi
AC_MSG_RESULT([$enable_dap_long_tests])
# Control zarr storage
if test "x$enable_nczarr" = xyes ; then
@ -1757,6 +1782,7 @@ AM_CONDITIONAL(ENABLE_DAP4, [test "x$enable_dap4" = xyes])
AM_CONDITIONAL(USE_STRICT_NULL_BYTE_HEADER_PADDING, [test x$enable_strict_null_byte_header_padding = xyes])
AM_CONDITIONAL(ENABLE_CDF5, [test "x$enable_cdf5" = xyes])
AM_CONDITIONAL(ENABLE_DAP_REMOTE_TESTS, [test "x$enable_dap_remote_tests" = xyes])
AM_CONDITIONAL(ENABLE_EXTERNAL_SERVER_TESTS, [test "x$enable_external_server_tests" = xyes])
AM_CONDITIONAL(ENABLE_DAP_AUTH_TESTS, [test "x$enable_dap_auth_tests" = xyes])
AM_CONDITIONAL(ENABLE_DAP_LONG_TESTS, [test "x$enable_dap_long_tests" = xyes])
AM_CONDITIONAL(USE_PNETCDF_DIR, [test ! "x$PNETCDFDIR" = x])

2
dap4_test/test_thredds.sh
View File

@ -36,6 +36,7 @@ failure() {
setresultdir results_test_thredds
if test "x${RESET}" = x1 ; then rm -fr ${BASELINEH}/*.thredds ; fi
if test "x$FEATURE_THREDDSTEST" = x1 ; then
for f in $F ; do
makeurl "dap4://thredds-test.unidata.ucar.edu/thredds/dap4/casestudies" "$f"
echo "testing: $URL"
@ -51,6 +52,7 @@ for f in $F ; do
cp ./results_test_thredds/${base}.thredds ${BASELINETH}/${base}.thredds
fi
done
fi # FEATURE_THREDDSTEST
#rm -fr "${GOESLONG}"

220
docs/nczarr.md
View File

@ -13,12 +13,10 @@ This extension provides a mapping from a subset of the full netCDF Enhanced (aka
The NetCDF version of this storage format is called NCZarr <a href="#ref_nczarr">[4]</a>.
A note on terminology in this document.
1. The term "dataset" is used to refer to all of the Zarr objects constituting
the meta-data and data.
There are some important "caveats" of which to be aware when using this software.
1. NCZarr currently is not thread-safe. So any attempt to use it with parallelism, including MPIO, is likely to fail.
# The NCZarr Data Model {#nczarr_data_model}
@ -35,28 +33,29 @@ Specifically the XArray ''\_ARRAY\_DIMENSIONS'' attribute is one such.
There are two other, secondary assumption:
1. The actual storage format in which the dataset is stored -- a zip file, for example -- can be read by the _Zarr_ implementation.
2. The filters used by the dataset can be encoded/decoded by the implementation.
2. The compressors (aka filters) used by the dataset can be encoded/decoded by the implementation. NCZarr uses HDF5-style filters, so ensuring access to such filters is somewhat complicated. See [the companion document on
filters](./md_filters.html "filters") for details.
Briefly, the data model supported by NCZarr is netcdf-4 minus the user-defined types and the String type.
As with netcdf-4 chunking is supported.
Filters and compression are supported, but
[the companion document on filters](./md_filters.html "filters")
should be consulted for the details.
Briefly, the data model supported by NCZarr is netcdf-4 minus
the user-defined types. However, a restricted form of String type
is supported (see Appendix H).
As with netcdf-4 chunking is supported. Filters and compression
are also [supported](./md_filters.html "filters").
Specifically, the model supports the following.
- "Atomic" types: char, byte, ubyte, short, ushort, int, uint, int64, uint64.
- "Atomic" types: char, byte, ubyte, short, ushort, int, uint, int64, uint64, string.
- Shared (named) dimensions
- Attributes with specified types -- both global and per-variable
- Chunking
- Fill values
- Groups
- N-Dimensional variables
- Scalar variables
- Per-variable endianness (big or little)
- Filters (including compression)
With respect to full netCDF-4, the following concepts are
currently unsupported.
- String type
- User-defined types (enum, opaque, VLEN, and Compound)
- Unlimited dimensions
- Contiguous or compact storage
@ -66,6 +65,15 @@ because they are HDF5 specific.
When specified, they are treated as chunked where the file consists of only one chunk.
This means that testing for contiguous or compact is not possible; the _nc_inq_var_chunking_ function will always return NC_CHUNKED and the chunksizes will be the same as the dimension sizes of the variable's dimensions.
Additionally, it should be noted that NCZarr supports scalar variables,
but Zarr does not; Zarr only supports dimensioned variables.
In order to support interoperability, NCZarr does the following.
1. A scalar variable is recorded in the Zarr metadata as if it has a shape of **[1]**.
2. A note is stored in the NCZarr metadata that this is actually a netCDF scalar variable.
These actions allow NCZarr to properly show scalars in its API while still
maintaining compatibility with Zarr.
# Enabling NCZarr Support {#nczarr_enable}
NCZarr support is enabled by default.
@ -322,7 +330,6 @@ aws_secret_access_key=YYYY...
```
See Appendix E for additional information.
## Addressing Style
The notion of "addressing style" may need some expansion.
@ -378,14 +385,14 @@ of NCZarr specific information.
These keys are as follows:
_\_NCZARR_SUPERBLOCK\__ -- this is in the top level group -- key _/.zarr_.
_\_nczarr_superblock\__ -- this is in the top level group -- key _/.zarr_.
It is in effect the "superblock" for the dataset and contains
any netcdf specific dataset level information.
It is also used to verify that a given key is the root of a dataset.
Currently it contains the following key(s):
* "version" -- the NCZarr version defining the format of the dataset.
_\_NCZARR_GROUP\__ -- this key appears in every _.zgroup_ object.
_\_nczarr_group\__ -- this key appears in every _.zgroup_ object.
It contains any netcdf specific group information.
Specifically it contains the following keys:
* "dims" -- the name and size of shared dimensions defined in this group.
@ -393,13 +400,13 @@ Specifically it contains the following keys:
* "groups" -- the name of sub-groups defined in this group.
These lists allow walking the NCZarr dataset without having to use the potentially costly search operation.
_\_NCZARR_ARRAY\__ -- this key appears in every _.zarray_ object.
_\_nczarr_array\__ -- this key appears in every _.zarray_ object.
It contains netcdf specific array information.
Specifically it contains the following keys:
* dimrefs -- the names of the shared dimensions referenced by the variable.
* storage -- indicates if the variable is chunked vs contiguous in the netcdf sense.
_\_NCZARR_ATTR\__ -- this key appears in every _.zattr_ object.
_\_nczarr_attr\__ -- this key appears in every _.zattr_ object.
This means that technically, it is attribute, but one for which access
is normally surpressed .
Specifically it contains the following keys:
@ -412,17 +419,17 @@ The latter case, zarr reading nczarr is possible if the zarr library is willing
The former case, nczarr reading zarr is also possible if the nczarr can simulate or infer the contents of the missing _\_NCZARR\_XXX_ objects.
As a rule this can be done as follows.
1. _\_NCZARR_GROUP\__ -- The list of contained variables and sub-groups can be computed using the search API to list the keys "contained" in the key for a group.
1. _\_nczarr_group\__ -- The list of contained variables and sub-groups can be computed using the search API to list the keys "contained" in the key for a group.
The search looks for occurrences of _.zgroup_, _.zattr_, _.zarray_ to infer the keys for the contained groups, attribute sets, and arrays (variables).
Constructing the set of "shared dimensions" is carried out
by walking all the variables in the whole dataset and collecting
the set of unique integer shapes for the variables.
For each such dimension length, a top level dimension is created
named ".zdim_<len>" where len is the integer length.
2. _\_NCZARR_ARRAY\__ -- The dimrefs are inferred by using the shape
2. _\_nczarr_array\__ -- The dimrefs are inferred by using the shape
in _.zarray_ and creating references to the simulated shared dimension.
netcdf specific information.
3. _\_NCZARR_ATTR\__ -- The type of each attribute is inferred by trying to parse the first attribute value string.
3. _\_nczarr_attr\__ -- The type of each attribute is inferred by trying to parse the first attribute value string.
# Compatibility {#nczarr_compatibility}
@ -434,7 +441,7 @@ The Xarray <a href="#ref_xarray">[7]</a> Zarr implementation uses its own mechan
It uses a special attribute named ''_ARRAY_DIMENSIONS''.
The value of this attribute is a list of dimension names (strings).
An example might be ````["time", "lon", "lat"]````.
It is essentially equivalent to the ````_NCZARR_ARRAY "dimrefs" list````, except that the latter uses fully qualified names so the referenced dimensions can be anywhere in the dataset.
It is essentially equivalent to the ````_nczarr_array "dimrefs" list````, except that the latter uses fully qualified names so the referenced dimensions can be anywhere in the dataset.
As of _netcdf-c_ version 4.8.2, The Xarray ''_ARRAY_DIMENSIONS'' attribute is supported for both NCZarr and pure Zarr.
If possible, this attribute will be read/written by default,
@ -778,34 +785,169 @@ The version 1 format defines three specific objects: _.nczgroup_, _.nczarray_,_.
These are stored in parallel with the corresponding Zarr objects. So if there is a key of the form "/x/y/.zarray", then there is also a key "/x/y/.nczarray".
The content of these objects is the same as the contents of the corresponding keys. So the value of the ''_NCZARR_ARRAY'' key is the same as the content of the ''.nczarray'' object. The list of connections is as follows:
* ''.nczarr'' <=> ''_NCZARR_SUPERBLOCK_''
* ''.nczgroup <=> ''_NCZARR_GROUP_''
* ''.nczarray <=> ''_NCZARR_ARRAY_''
* ''.nczattr <=> ''_NCZARR_ATTR_''
* ''.nczarr'' <=> ''_nczarr_superblock_''
* ''.nczgroup <=> ''_nczarr_group_''
* ''.nczarray <=> ''_nczarr_array_''
* ''.nczattr <=> ''_nczarr_attr_''
# Appendix G. JSON Attribute Convention. {#nczarr_json}
An attribute may be encountered on read whose value when parsed
by JSON is a dictionary. As a special conventions, the value
converted to a string and stored as the value of the attribute
and the type of the attribute is treated as char.
The Zarr V2 specification is somewhat vague on what is a legal
value for an attribute. The examples all show one of two cases:
1. A simple JSON scalar atomic values (e.g. int, float, char, etc), or
2. A JSON array of such values.
When writing a character valued attribute, it's value is examined
to see if it looks like a JSON dictionary (i.e. "{...}")
and is parseable as JSON.
If so, then the attribute value is treated as one long string,
parsed as JSON, and stored in the .zattr file in JSON form.
These conventions are intended to help support various
However, the Zarr specification can be read to infer that the value
can in fact be any legal JSON expression.
This "convention" is currently used routinely to help support various
attributes created by other packages where the attribute is a
complex JSON dictionary. An example is the GDAL Driver
convention <a href="#ref_gdal">[12]</a>. The value is a complex
JSON dictionary and it is desirable to both read and write that kind of
information through the netcdf API.
complex JSON expression. An example is the GDAL Driver
convention <a href="#ref_gdal">[12]</a>, where the value is a complex
JSON dictionary.
In order for NCZarr to be as consistent as possible with Zarr Version 2,
it is desirable to support this convention for attribute values.
This means that there must be some way to handle an attribute
whose value is not either of the two cases above. That is, its value
is some more complex JSON expression. Ideally both reading and writing
of such attributes should be supported.
One more point. NCZarr attempts to record the associated netcdf
attribute type (encoded in the form of a NumPy "dtype") for each
attribute. This information is stored as NCZarr-specific
metadata. Note that pure Zarr makes no attempt to record such
type information.
The current algorithm to support JSON valued attributes
operates as follows.
## Writing an attribute:
There are mutiple cases to consider.
1. The netcdf attribute **is not** of type NC_CHAR and its value is a single atomic value.
* Convert to an equivalent JSON atomic value and write that JSON expression.
* Compute the Zarr equivalent dtype and store in the NCZarr metadata.
2. The netcdf attribute **is not** of type NC_CHAR and its value is a vector of atomic values.
* Convert to an equivalent JSON array of atomic values and write that JSON expression.
* Compute the Zarr equivalent dtype and store in the NCZarr metadata.
3. The netcdf attribute **is** of type NC_CHAR and its value &ndash; taken as a single sequence of characters &ndash;
**is** parseable as a legal JSON expression.
* Parse to produce a JSON expression and write that expression.
* Use "|S1" as the dtype and store in the NCZarr metadata.
4. The netcdf attribute **is** of type NC_CHAR and its value &ndash; taken as a single sequence of characters &ndash;
**is not** parseable as a legal JSON expression.
* Convert to a JSON string and write that expression
* Use "|S1" as the dtype and store in the NCZarr metadata.
## Reading an attribute:
The process of reading and interpreting an attribute value requires two
pieces of information.
* The value of the attribute as a JSON expression, and
* The optional associated dtype of the attribute; note that this may not exist
if, for example, the file is pure zarr.
Given these two pieces of information, the read process is as follows.
1. The JSON expression is a simple JSON atomic value.
* If the dtype is defined, then convert the JSON to that type of data,
and then store it as the equivalent netcdf vector of size one.
* If the dtype is not defined, then infer the dtype based on the the JSON value,
and then store it as the equivalent netcdf vector of size one.
2. The JSON expression is an array of simple JSON atomic values.
* If the dtype is defined, then convert each JSON value in the array to that type of data,
and then store it as the equivalent netcdf vector.
* If the dtype is not defined, then infer the dtype based on the first JSON value in the array,
and then store it as the equivalent netcdf vector.
3. The JSON expression is an array some of whose values are dictionaries or (sub-)arrays.
* Un-parse the expression to an equivalent sequence of characters, and then store it as of type NC_CHAR.
3. The JSON expression is a dictionary.
* Un-parse the expression to an equivalent sequence of characters, and then store it as of type NC_CHAR.
## Notes
1. If a character valued attributes's value can be parsed as a legal JSON expression, then it will be stored as such.
2. Reading and writing are *almost* idempotent in that the sequence of
actions "read-write-read" is equivalent to a single "read" and "write-read-write" is equivalent to a single "write".
The "almost" caveat is necessary because (1) whitespace may be added or lost during the sequence of operations,
and (2) numeric precision may change.
# Appendix H. Support for string types
Zarr supports a string type, but it is restricted to
fixed size strings. NCZarr also supports such strings,
but there are some differences in order to interoperate
with the netcdf-4/HDF5 variable length strings.
The primary issue to be addressed is to provide a way for user
to specify the maximum size of the fixed length strings. This is
handled by providing the following new attributes:
1. **_nczarr_default_maxstrlen** &mdash;
This is an attribute of the root group. It specifies the default
maximum string length for string types. If not specified, then
it has the value of 64 characters.
2. **_nczarr_maxstrlen** &mdash;
This is a per-variable attribute. It specifies the maximum
string length for the string type associated with the variable.
If not specified, then it is assigned the value of
**_nczarr_default_maxstrlen**.
Note that when accessing a string through the netCDF API, the
fixed length strings appear as variable length strings. This
means that they are stored as pointers to the string
(i.e. **char\***) and with a trailing nul character.
One consequence is that if the user writes a variable length
string through the netCDF API, and the length of that string
is greater than the maximum string length for a variable,
then the string is silently truncated.
Another consequence is that the user must reclaim the string storage.
Adding strings also requires some hacking to handle the existing
netcdf-c NC_CHAR type, which does not exist in Zarr. The goal
was to choose NumPY types for both the netcdf-c NC_STRING type
and the netcdf-c NC_CHAR type such that if a pure zarr
implementation reads them, it will still work.
For writing variables and NCZarr attributes, the type mapping is as follows:
* "|S1" for NC_CHAR.
* ">S1" for NC_STRING && MAXSTRLEN==1
* ">Sn" for NC_STRING && MAXSTRLEN==n
Note that it is a bit of a hack to use endianness, but it should be ok since for
string/char, the endianness has no meaning.
So when reading data with a pure zarr implementaion
the above types should always appear as strings,
and the type that signals NC_CHAR (in NCZarr)
would be handled by Zarr as a string of length 1.
# Change Log {#nczarr_changelog}
Note, this log was only started as of 8/11/2022 and is not
intended to be a detailed chronology. Rather, it provides highlights
that will be of interest to NCZarr users. In order to see exact changes,
It is necessary to use the 'git diff' command.
## 8/29/2022
1. Zarr fixed-size string types are now supported.
## 8/11/2022
1. The NCZarr specific keys have been converted to lower-case
(e.g. "_nczarr_attr" instead of "_NCZARR_ATTR"). Upper case is
accepted for back compatibility.
2. The legal values of an attribute has been extended to
include arbitrary JSON expressions; see Appendix G for more details.
# Point of Contact {#nczarr_poc}
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu<br>
__Initial Version__: 4/10/2020<br>
__Last Revised__: 7/16/2021
__Last Revised__: 8/27/2022

9
include/nc4internal.h
View File

@ -106,9 +106,6 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP, NCFIL} NC_SORT;
/** Subset of readonly flags; readable by name only thru the API. */
#define NAMEONLYFLAG 4
/** Subset of readonly flags; Value is actually in file. */
#define MATERIALIZEDFLAG 8
/** Per-variable attribute, as opposed to global */
#define VARFLAG 16
@ -492,9 +489,13 @@ extern void NC_freeglobalstate(void);
#define NC_ATT_COORDINATES "_Netcdf4Coordinates" /*see hdf5internal.h:COORDINATES*/
#define NC_ATT_FORMAT "_Format"
#define NC_ATT_DIMID_NAME "_Netcdf4Dimid"
#define NC_ATT_FILLVALUE "_FillValue"
#define NC_ATT_NC3_STRICT_NAME "_nc3_strict"
#define NC_XARRAY_DIMS "_ARRAY_DIMENSIONS"
#define NC_ATT_CODECS "_Codecs"
#define NC_NCZARR_ATTR "_NCZARR_ATTR"
#define NC_NCZARR_ATTR "_nczarr_attr"
#define NC_NCZARR_ATTR_UC "_NCZARR_ATTR"
#define NC_NCZARR_MAXSTRLEN_ATTR "_nczarr_maxstrlen"
#define NC_NCZARR_DEFAULT_MAXSTRLEN_ATTR "_nczarr_default_maxstrlen"
#endif /* _NC4INTERNAL_ */

7
include/ncconfigure.h
View File

@ -50,9 +50,10 @@ extern "C" {
#endif
/* WARNING: in some systems, these functions may be defined as macros, so check */
#ifndef HAVE_STRDUP
#ifndef strdup
#ifndef HAVE_STRDUP
char* strdup(const char*);
#define HAVE_STRDUP
#endif
#endif
@ -120,11 +121,7 @@ unsigned long long int strtoull(const char*, char**, int);
/* handle null arguments */
#ifndef nulldup
#ifdef HAVE_STRDUP
#define nulldup(s) ((s)==NULL?NULL:strdup(s))
#else
extern char *nulldup(const char* s);
#endif
#endif
#ifndef nulllen

3
include/ncjson.h
View File

@ -64,6 +64,7 @@ typedef struct NCjson {
don't use union so we can know when to reclaim sval
*/
struct NCJconst {int bval; long long ival; double dval; char* sval;};
#define NCJconst_empty {0,0,0.0,NULL}
/**************************************************/
/* Extended API */
@ -116,6 +117,8 @@ OPTEXPORT int NCJclone(const NCjson* json, NCjson** clonep);
#ifndef NETCDF_JSON_H
/* dump NCjson* object to output file */
OPTEXPORT void NCJdump(const NCjson* json, unsigned flags, FILE*);
/* convert NCjson* object to output string */
OPTEXPORT const char* NCJtotext(const NCjson* json);
#endif
#if defined(__cplusplus)

63
include/netcdf_json.h
View File

@ -2,9 +2,19 @@
See the COPYRIGHT file for more information.
*/
#ifndef NETCDF_JSON_H
#define NETCDF_JSON_H 1
/*
WARNING:
If you modify this file,
then you need to got to
the include/ directory
and do the command:
make makepluginjson
*/
/* Inside libnetcdf and for plugins, export the json symbols */
#ifndef DLLEXPORT
#ifdef _WIN32
@ -54,6 +64,7 @@ typedef struct NCjson {
don't use union so we can know when to reclaim sval
*/
struct NCJconst {int bval; long long ival; double dval; char* sval;};
#define NCJconst_empty {0,0,0.0,NULL}
/**************************************************/
/* Extended API */
@ -106,6 +117,8 @@ OPTEXPORT int NCJclone(const NCjson* json, NCjson** clonep);
#ifndef NETCDF_JSON_H
/* dump NCjson* object to output file */
OPTEXPORT void NCJdump(const NCjson* json, unsigned flags, FILE*);
/* convert NCjson* object to output string */
OPTEXPORT const char* NCJtotext(const NCjson* json);
#endif
#if defined(__cplusplus)
@ -140,6 +153,18 @@ OPTEXPORT void NCJdump(const NCjson* json, unsigned flags, FILE*);
TODO: make utf8 safe
*/
/*
WARNING:
If you modify this file,
then you need to got to
the include/ directory
and do the command:
make makenetcdfjson
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
@ -235,7 +260,7 @@ static int bytesappendquoted(NCJbuf* buf, const char* s);
static int bytesappend(NCJbuf* buf, const char* s);
static int bytesappendc(NCJbuf* bufp, const char c);
/* Hide these for plugins */
/* Hide everything for plugins */
#ifdef NETCDF_JSON_H
#define OPTSTATIC static
static int NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp);
@ -266,10 +291,6 @@ NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp)
NCJparser* parser = NULL;
NCjson* json = NULL;
/* Need at least 1 character of input */
if(len == 0 || text == NULL)
{stat = NCJTHROW(NCJ_ERR); goto done;}
if(jsonp == NULL) goto done;
parser = calloc(1,sizeof(NCJparser));
if(parser == NULL)
{stat = NCJTHROW(NCJ_ERR); goto done;}
@ -277,6 +298,16 @@ NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp)
if(parser->text == NULL)
{stat = NCJTHROW(NCJ_ERR); goto done;}
memcpy(parser->text,text,len);
/* trim trailing whitespace */
if(len > 0) {
char* p;
for(p=parser->text+(len-1);p >= parser->text;p--) {
if(*p > ' ') break;
}
len = (size_t)((p - parser->text) + 1);
}
if(len == 0)
{stat = NCJTHROW(NCJ_ERR); goto done;}
parser->text[len] = '\0';
parser->text[len+1] = '\0';
parser->pos = &parser->text[0];
@ -285,6 +316,8 @@ NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp)
fprintf(stderr,"json: |%s|\n",parser->text);
#endif
if((stat=NCJparseR(parser,&json))==NCJ_ERR) goto done;
/* Must consume all of the input */
if(parser->pos != (parser->text+len)) {stat = NCJ_ERR; goto done;}
*jsonp = json;
json = NULL;
@ -1186,16 +1219,32 @@ NCJdump(const NCjson* json, unsigned flags, FILE* out)
nullfree(text);
}
OPTSTATIC const char*
NCJtotext(const NCjson* json)
{
static char outtext[4096];
char* text = NULL;
if(json == NULL) {strcpy(outtext,"<null>"); goto done;}
(void)NCJunparse(json,0,&text);
outtext[0] = '\0';
strlcat(outtext,text,sizeof(outtext));
nullfree(text);
done:
return outtext;
}
/* Hack to avoid static unused warning */
void
static void
netcdf_supresswarnings(void)
{
void* ignore;
ignore = (void*)NCJdump;
ignore = (void*)netcdf_supresswarnings;
ignore = (void*)NCJinsert;
ignore = (void*)NCJaddstring;
ignore = (void*)NCJcvt;
ignore = (void*)NCJdictget;
ignore = (void*)NCJparse;
ignore = (void*)NCJdump;
ignore = (void*)NCJtotext;
ignore = ignore;
}

2
libdap4/ncd4dispatch.c
View File

@ -43,7 +43,7 @@ static const NC_reservedatt NCD4_reserved[] = {
{D4CHECKSUMATTR, READONLYFLAG|NAMEONLYFLAG}, /*_DAP4_Checksum_CRC32*/
{D4LEATTR, READONLYFLAG|NAMEONLYFLAG}, /*_DAP4_Little_Endian*/
/* Also need to include the provenance attributes */
{NCPROPS, READONLYFLAG|NAMEONLYFLAG|MATERIALIZEDFLAG}, /*_NCProperties*/
{NCPROPS, READONLYFLAG|NAMEONLYFLAG}, /*_NCProperties*/
{NULL, 0}
};

2
libdispatch/dinfermodel.c
View File

@ -907,7 +907,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
if(!modelcomplete(model)) {
const char** p = ncurifragmentparams(uri); /* envv format */
if(p != NULL) {
for(;*p;p++) {
for(;*p;p+=2) {
const char* key = p[0];
const char* value = p[1];;
if((stat=processfragmentkeys(key,value,model))) goto done;

38
libdispatch/ncjson.c
View File

@ -15,6 +15,9 @@ and do the command:
make makenetcdfjson
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
@ -111,7 +114,7 @@ static int bytesappendquoted(NCJbuf* buf, const char* s);
static int bytesappend(NCJbuf* buf, const char* s);
static int bytesappendc(NCJbuf* bufp, const char c);
/* Hide these for plugins */
/* Hide everything for plugins */
#ifdef NETCDF_JSON_H
#define OPTSTATIC static
static int NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp);
@ -142,10 +145,6 @@ NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp)
NCJparser* parser = NULL;
NCjson* json = NULL;
/* Need at least 1 character of input */
if(len == 0 || text == NULL)
{stat = NCJTHROW(NCJ_ERR); goto done;}
if(jsonp == NULL) goto done;
parser = calloc(1,sizeof(NCJparser));
if(parser == NULL)
{stat = NCJTHROW(NCJ_ERR); goto done;}
@ -153,6 +152,16 @@ NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp)
if(parser->text == NULL)
{stat = NCJTHROW(NCJ_ERR); goto done;}
memcpy(parser->text,text,len);
/* trim trailing whitespace */
if(len > 0) {
char* p;
for(p=parser->text+(len-1);p >= parser->text;p--) {
if(*p > ' ') break;
}
len = (size_t)((p - parser->text) + 1);
}
if(len == 0)
{stat = NCJTHROW(NCJ_ERR); goto done;}
parser->text[len] = '\0';
parser->text[len+1] = '\0';
parser->pos = &parser->text[0];
@ -161,6 +170,8 @@ NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp)
fprintf(stderr,"json: |%s|\n",parser->text);
#endif
if((stat=NCJparseR(parser,&json))==NCJ_ERR) goto done;
/* Must consume all of the input */
if(parser->pos != (parser->text+len)) {stat = NCJ_ERR; goto done;}
*jsonp = json;
json = NULL;
@ -1062,17 +1073,32 @@ NCJdump(const NCjson* json, unsigned flags, FILE* out)
nullfree(text);
}
OPTSTATIC const char*
NCJtotext(const NCjson* json)
{
static char outtext[4096];
char* text = NULL;
if(json == NULL) {strcpy(outtext,"<null>"); goto done;}
(void)NCJunparse(json,0,&text);
outtext[0] = '\0';
strlcat(outtext,text,sizeof(outtext));
nullfree(text);
done:
return outtext;
}
/* Hack to avoid static unused warning */
static void
netcdf_supresswarnings(void)
{
void* ignore;
ignore = (void*)netcdf_supresswarnings;
ignore = (void*)NCJdump;
ignore = (void*)NCJinsert;
ignore = (void*)NCJaddstring;
ignore = (void*)NCJcvt;
ignore = (void*)NCJdictget;
ignore = (void*)NCJparse;
ignore = (void*)NCJdump;
ignore = (void*)NCJtotext;
ignore = ignore;
}

2
libdispatch/nclog.c
View File

@ -32,7 +32,7 @@
#define MAXTAGS 256
#define NCTAGDFALT "Log";
#define NC_MAX_FRAMES 256
#define NC_MAX_FRAMES 1024
static int nclogginginitialized = 0;

6
libhdf5/hdf5debug.c
View File

@ -5,7 +5,7 @@
#include "config.h"
#include <stdarg.h>
#include <stdio.h>
#if !defined _WIN32 && !defined __CYGWIN__
#ifdef HAVE_EXECINFO_H
#include <execinfo.h>
#endif
@ -15,15 +15,18 @@
#define STSIZE 1000
#ifdef HAVE_EXECINFO_H
#ifdef H5BACKTRACE
# if !defined _WIN32 && !defined __CYGWIN__
static void* stacktrace[STSIZE];
# endif
#endif
#endif
int
nch5breakpoint(int err)
{
#ifdef HAVE_EXECINFO_H
#ifdef H5BACKTRACE
# if !defined _WIN32 && !defined __CYGWIN__
int count = 0;
@ -39,6 +42,7 @@ nch5breakpoint(int err)
if(trace != NULL) free(trace);
# endif
# endif
#endif
#endif
return err;
}

2
liblib/CMakeLists.txt
View File

@ -135,7 +135,7 @@ IF(ENABLE_PNETCDF AND PNETCDF)
ENDIF()
IF(ENABLE_S3_SDK)
# TARGET_LINK_DIRECTORIES(netcdf PUBLIC ${AWSSDK_LIB_DIR})
TARGET_LINK_DIRECTORIES(netcdf PUBLIC ${AWSSDK_LIB_DIR})
TARGET_LINK_LIBRARIES(netcdf ${AWS_LINK_LIBRARIES})
ENDIF()

144
libnczarr/zarr.c
View File

@ -62,6 +62,8 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr
&zinfo->zarr.nczarr_version.minor,
&zinfo->zarr.nczarr_version.release);
zinfo->default_maxstrlen = NCZ_MAXSTR_DEFAULT;
/* Apply client controls */
if((stat = applycontrols(zinfo))) goto done;
@ -126,6 +128,7 @@ ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls)
zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG);
if((zinfo->envv_controls = NCZ_clonestringvec(0,controls))==NULL) /*0=>envv style*/
{stat = NC_ENOMEM; goto done;}
zinfo->default_maxstrlen = NCZ_MAXSTR_DEFAULT;
/* Add struct to hold NCZ-specific group info. */
if (!(root->format_grp_info = calloc(1, sizeof(NCZ_GRP_INFO_T))))
@ -288,75 +291,6 @@ done:
}
#endif
#if 0
/**
@internal Rewrite attributes into a group or var
@param map - [in] the map object for storage
@param container - [in] the containing object
@param jattrs - [in] the json for .zattrs
@param jtypes - [in] the json for .ztypes
@return NC_NOERR
@author Dennis Heimbigner
*/
int
ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjson* jtypes)
{
int stat = NC_NOERR;
char* fullpath = NULL;
char* akey = NULL;
char* tkey = NULL;
NCZMAP* map = zinfo->map;
assert((NCJsort(jattrs) == NCJ_DICT));
assert((NCJsort(jtypes) == NCJ_DICT));
if(container->sort == NCGRP) {
NC_GRP_INFO_T* grp = (NC_GRP_INFO_T*)container;
/* Get grp's fullpath name */
if((stat = NCZ_grpkey(grp,&fullpath)))
goto done;
} else {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)container;
/* Get var's fullpath name */
if((stat = NCZ_varkey(var,&fullpath)))
goto done;
}
/* Construct the path to the .zattrs object */
if((stat = nczm_concat(fullpath,ZATTRS,&akey)))
goto done;
/* Always write as V2 */
{
NCjson* k = NULL;
NCjson* v = NULL;
/* remove any previous version */
if(!NCJremove(jattrs,NCZ_V2_ATTRS,&k,&v)) {
NCJreclaim(k); NCJreclaim(v);
}
}
if(!(zinfo->controls.flags & FLAG_PUREZARR)) {
/* Insert the jtypes into the set of attributes */
if((stat = NCJinsert(jattrs,NCZ_V2_ATTRS,jtypes))) goto done;
}
/* Upload the .zattrs object */
if((stat=NCZ_uploadjson(map,tkey,jattrs)))
goto done;
done:
if(stat) {
NCJreclaim(jattrs);
NCJreclaim(jtypes);
}
nullfree(fullpath);
nullfree(akey);
nullfree(tkey);
return stat;
}
#endif
static const char*
controllookup(const char** envv_controls, const char* key)
@ -415,3 +349,75 @@ done:
return stat;
}
#if 0
/**
@internal Rewrite attributes into a group or var
@param map - [in] the map object for storage
@param container - [in] the containing object
@param jattrs - [in] the json for .zattrs
@param jtypes - [in] the json for .ztypes
@return NC_NOERR
@author Dennis Heimbigner
*/
int
ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjson* jtypes)
{
int stat = NC_NOERR;
char* fullpath = NULL;
char* akey = NULL;
char* tkey = NULL;
NCZMAP* map = zinfo->map;
assert((NCJsort(jattrs) == NCJ_DICT));
assert((NCJsort(jtypes) == NCJ_DICT));
if(container->sort == NCGRP) {
NC_GRP_INFO_T* grp = (NC_GRP_INFO_T*)container;
/* Get grp's fullpath name */
if((stat = NCZ_grpkey(grp,&fullpath)))
goto done;
} else {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)container;
/* Get var's fullpath name */
if((stat = NCZ_varkey(var,&fullpath)))
goto done;
}
/* Construct the path to the .zattrs object */
if((stat = nczm_concat(fullpath,ZATTRS,&akey)))
goto done;
/* Always write as V2 */
{
NCjson* k = NULL;
NCjson* v = NULL;
/* remove any previous version */
if(!NCJremove(jattrs,NCZ_V2_ATTRS,1,&k,&v)) {
NCJreclaim(k); NCJreclaim(v);
}
}
if(!(zinfo->controls.flags & FLAG_PUREZARR)) {
/* Insert the jtypes into the set of attributes */
if((stat = NCJinsert(jattrs,NCZ_V2_ATTRS,jtypes))) goto done;
}
/* Upload the .zattrs object */
if((stat=NCZ_uploadjson(map,tkey,jattrs)))
goto done;
done:
if(stat) {
NCJreclaim(jattrs);
NCJreclaim(jtypes);
}
nullfree(fullpath);
nullfree(akey);
nullfree(tkey);
return stat;
}
#endif

28
libnczarr/zarr.h
View File

@ -15,6 +15,16 @@
struct ChunkKey;
struct S3credentials;
/* Intermediate results */
struct ZCVT {
signed long long int64v;
unsigned long long uint64v;
double float64v;
char* strv; /* null terminated utf-8 */
};
#define zcvt_empty {0,0,0.0,NULL}
/* zarr.c */
EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, const char** controls);
EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, const char** controls);
@ -31,8 +41,9 @@ EXTERNL int ncz_unload_jatts(NCZ_FILE_INFO_T*, NC_OBJ* container, NCjson* jattrs
EXTERNL int ncz_close_file(NC_FILE_INFO_T* file, int abort);
/* zcvt.c */
EXTERNL int NCZ_convert1(NCjson* jsrc, nc_type, unsigned char* memory0);
EXTERNL int NCZ_stringconvert1(nc_type typid, size_t len, char* src, NCjson* jvalue);
EXTERNL int NCZ_json2cvt(NCjson* jsrc, struct ZCVT* zcvt, nc_type* typeidp);
EXTERNL int NCZ_convert1(NCjson* jsrc, nc_type, NCbytes*);
EXTERNL int NCZ_stringconvert1(nc_type typid, char* src, NCjson* jvalue);
EXTERNL int NCZ_stringconvert(nc_type typid, size_t len, void* data0, NCjson** jdatap);
/* zsync.c */
@ -53,9 +64,11 @@ EXTERNL int NCZ_dimkey(const NC_DIM_INFO_T* dim, char** pathp);
EXTERNL int ncz_splitkey(const char* path, NClist* segments);
EXTERNL int NCZ_readdict(NCZMAP* zmap, const char* key, NCjson** jsonp);
EXTERNL int NCZ_readarray(NCZMAP* zmap, const char* key, NCjson** jsonp);
EXTERNL int ncz_zarr_type_name(nc_type nctype, int little, const char** znamep);
EXTERNL int ncz_nctype2typeinfo(const char* snctype, nc_type* nctypep);
EXTERNL int ncz_dtype2typeinfo(const char* dtype, nc_type* nctypep, int* endianness);
EXTERNL int ncz_nctypedecode(const char* snctype, nc_type* nctypep);
EXTERNL int ncz_nctype2dtype(nc_type nctype, int endianness, int purezarr,int len, char** dnamep);
EXTERNL int ncz_dtype2nctype(const char* dtype, nc_type typehint, int purezarr, nc_type* nctypep, int* endianp, int* typelenp);
EXTERNL int NCZ_inferattrtype(NCjson* value, nc_type typehint, nc_type* typeidp);
EXTERNL int NCZ_inferinttype(unsigned long long u64, int negative);
EXTERNL int ncz_fill_value_sort(nc_type nctype, int*);
EXTERNL int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size);
EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json);
@ -73,6 +86,11 @@ EXTERNL int NCZ_ischunkname(const char* name,char dimsep);
EXTERNL char* NCZ_chunkpath(struct ChunkKey key);
EXTERNL int NCZ_reclaim_fill_value(NC_VAR_INFO_T* var);
EXTERNL int NCZ_copy_fill_value(NC_VAR_INFO_T* var, void** dstp);
EXTERNL int NCZ_get_maxstrlen(NC_OBJ* obj);
EXTERNL int NCZ_fixed2char(const void* fixed, char** charp, size_t count, int maxstrlen);
EXTERNL int NCZ_char2fixed(const char** charp, void* fixed, size_t count, int maxstrlen);
EXTERNL int NCZ_copy_data(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* xtype, const void* memory, size_t count, int nofill, void* copy);
EXTERNL int NCZ_iscomplexjson(NCjson* value, nc_type typehint);
/* zwalk.c */
EXTERNL int NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata);

39
libnczarr/zattr.c
View File

@ -68,9 +68,11 @@ ncz_getattlist(NC_GRP_INFO_T *grp, int varid, NC_VAR_INFO_T **varp, NCindex **at
}
/**
* @internal Get one of three special attributes, NCPROPS,
* ISNETCDF4ATT, and SUPERBLOCKATT. These atts are not all really in
* the file, they are constructed on the fly.
* @internal Get one of the special attributes:
* See the reserved attribute table in libsrc4/nc4internal.c.
* The special attributes are the ones marked with NAMEONLYFLAG.
* For example: NCPROPS, ISNETCDF4ATT, and SUPERBLOCKATT, and CODECS.
* These atts are not all really in the file, they are constructed on the fly.
*
* @param h5 Pointer to ZARR file info struct.
* @param var Pointer to var info struct; NULL signals global.
@ -323,8 +325,9 @@ NCZ_del_att(int ncid, int varid, const char *name)
return NC_ENOTATT;
/* Reclaim the content of the attribute */
if(att->data)
if(att->data) {
if((retval = nc_reclaim_data_all(ncid,att->nc_typeid,att->data,att->len))) return retval;
}
att->data = NULL;
att->len = 0;
@ -426,7 +429,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
size_t len, const void *data, nc_type mem_type, int force)
{
NC* nc;
NC_FILE_INFO_T *h5;
NC_FILE_INFO_T *h5 = NULL;
NC_VAR_INFO_T *var = NULL;
NCindex* attlist = NULL;
NC_ATT_INFO_T* att;
@ -575,7 +578,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
}
/* If this is the _FillValue attribute, then we will also have to
* copy the value to the fill_vlue pointer of the NC_VAR_INFO_T
* copy the value to the fill_value pointer of the NC_VAR_INFO_T
* struct for this var. (But ignore a global _FillValue
* attribute). Also kill the cache fillchunk as no longer valid */
if (!strcmp(att->hdr.name, _FillValue) && varid != NC_GLOBAL)
@ -670,6 +673,23 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
att->data = copy; copy = NULL;
}
}
/* If this is a maxstrlen attribute, then we will also have to
* sync the value to NCZ_VAR_INFO_T or NCZ_FILE_INFO_T structure */
{
if(strcmp(att->hdr.name,NC_NCZARR_DEFAULT_MAXSTRLEN_ATTR)==0 && varid == NC_GLOBAL && len == 1) {
NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)h5->format_file_info;
if((retval = nc4_convert_type(att->data, &zfile->default_maxstrlen, file_type, NC_INT,
len, &range_error, NULL, NC_CLASSIC_MODEL, NC_NOQUANTIZE, 0)))
BAIL(retval);
} else if(strcmp(att->hdr.name,NC_NCZARR_MAXSTRLEN_ATTR)==0 && varid != NC_GLOBAL && len == 1) {
NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
if((retval = nc4_convert_type(att->data, &zvar->maxstrlen, file_type, NC_INT,
len, &range_error, NULL, NC_CLASSIC_MODEL, NC_NOQUANTIZE, 0)))
BAIL(retval);
}
}
att->dirty = NC_TRUE;
att->created = NC_FALSE;
att->len = len;
@ -994,14 +1014,19 @@ ncz_makeattr(NC_OBJ* container, NCindex* attlist, const char* name, nc_type type
NCZ_ATT_INFO_T* zatt = NULL;
void* clone = NULL;
size_t typesize, clonesize;
int ncid;
NC* nc = NULL;
NC_GRP_INFO_T* grp = (container->sort == NCGRP ? (NC_GRP_INFO_T*)container
: ((NC_VAR_INFO_T*)container)->container);
nc = grp->nc4_info->controller;
ncid = nc->ext_ncid | grp->hdr.id;
/* Duplicate the values */
if ((stat = nc4_get_typelen_mem(grp->nc4_info, typeid, &typesize))) goto done;
clonesize = len*typesize;
if((clone = malloc(clonesize))==NULL) {stat = NC_ENOMEM; goto done;}
memcpy(clone,values,clonesize);
if((stat = nc_copy_data(ncid, typeid, values, len, clone))) goto done;
if((stat=nc4_att_list_add(attlist,name,&att)))
goto done;

1
libnczarr/zcache.h
View File

@ -32,6 +32,7 @@ typedef struct NCZCacheEntry {
} key;
size64_t hashkey;
int isfiltered; /* 1=>data contains filtered data else real data */
int isfixedstring; /* 1 => data contains the fixed strings, 0 => data contains pointers to strings */
size64_t size; /* |data| */
void* data; /* contains either filtered or real data */
} NCZCacheEntry;

12
libnczarr/zclose.c
View File

@ -172,10 +172,10 @@ zclose_vars(NC_GRP_INFO_T* grp)
var->filters = NULL;
#endif
/* Reclaim the type */
(void)zclose_type(var->type_info);
NCZ_free_chunk_cache(zvar->cache);
if(var->type_info) (void)zclose_type(var->type_info);
if(zvar->cache) NCZ_free_chunk_cache(zvar->cache);
/* reclaim xarray */
nclistfreeall(zvar->xarray);
if(zvar->xarray) nclistfreeall(zvar->xarray);
nullfree(zvar);
var->format_var_info = NULL; /* avoid memory errors */
}
@ -223,13 +223,9 @@ static int
zclose_type(NC_TYPE_INFO_T* type)
{
int stat = NC_NOERR;
NCZ_TYPE_INFO_T* ztype;
assert(type && type->format_type_info != NULL);
/* Get Zarr-specific type info. */
ztype = type->format_type_info;
nullfree(ztype);
type->format_type_info = NULL; /* avoid memory errors */
nullfree(type->format_type_info);
return stat;
}

276
libnczarr/zcvt.c
View File

@ -15,51 +15,121 @@
Code taken directly from libdap4/d4cvt.c
*/
/* Intermediate results */
struct ZCVT {
signed long long int64v;
unsigned long long uint64v;
double float64v;
static const int ncz_type_size[NC_MAX_ATOMIC_TYPE+1] = {
0, /*NC_NAT*/
sizeof(char), /*NC_BYTE*/
sizeof(char), /*NC_CHAR*/
sizeof(short), /*NC_SHORT*/
sizeof(int), /*NC_INT*/
sizeof(float), /*NC_FLOAT*/
sizeof(double), /*NC_DOUBLE*/
sizeof(unsigned char), /*NC_UBYTE*/
sizeof(unsigned short), /*NC_USHORT*/
sizeof(unsigned int), /*NC_UINT*/
sizeof(long long), /*NC_INT64*/
sizeof(unsigned long long), /*NC_UINT64*/
sizeof(char *), /*NC_STRING*/
};
/* Forward */
static int typeid2jtype(nc_type typeid);
/* Convert an NCJ_STRING to a memory equivalent value of specified dsttype */
#if 0
/* Convert a JSON value to a struct ZCVT value and also return the type */
int
NCZ_convert1(NCjson* jsrc, nc_type dsttype, unsigned char* memory)
NCZ_string2cvt(char* src, nc_type srctype, struct ZCVT* zcvt, nc_type* typeidp)
{
int stat = NC_NOERR;
nc_type srctype;
struct ZCVT zcvt;
int outofrange = 0;
nc_type dsttype = NC_NAT;
/* Convert the incoming jsrc string to a restricted set of values */
assert(zcvt);
/* Convert to a restricted set of values */
switch (srctype) {
case NC_BYTE: {
zcvt->int64v = (signed long long)(*((signed char*)src));
dsttype = NC_INT64;
} break;
case NC_UBYTE: {
zcvt->uint64v = (unsigned long long)(*((unsigned char*)src));
dsttype = NC_UINT64;
} break;
case NC_SHORT: {
zcvt->int64v = (signed long long)(*((signed short*)src));
dsttype = NC_INT64;
} break;
case NC_USHORT: {
zcvt->uint64v = (unsigned long long)(*((unsigned short*)src));
dsttype = NC_UINT64;
} break;
case NC_INT: {
zcvt->int64v = (signed long long)(*((signed int*)src));
dsttype = NC_INT64;
} break;
case NC_UINT: {
zcvt->uint64v = (unsigned long long)(*((unsigned int*)src));
dsttype = NC_UINT64;
} break;
case NC_INT64: {
zcvt->int64v = (signed long long)(*((signed long long*)src));
dsttype = NC_INT64;
} break;
case NC_UINT64: {
zcvt->uint64v = (unsigned long long)(*((unsigned long long*)src));
dsttype = NC_UINT64;
} break;
case NC_FLOAT: {
zcvt->float64v = (double)(*((float*)src));
dsttype = NC_DOUBLE;
} break;
case NC_DOUBLE: {
dsttype = NC_DOUBLE;
zcvt->float64v= (double)(*((double*)src));
} break;
case NC_STRING: {
dsttype = NC_STRING;
zcvt->strv= *((char**)src);
} break;
default: stat = NC_EINTERNAL; goto done;
}
if(typeidp) *typeidp = dsttype;
done:
return stat;
}
#endif
/* Warning: not free returned zcvt.strv; it may point into a string in jsrc */
int
NCZ_json2cvt(NCjson* jsrc, struct ZCVT* zcvt, nc_type* typeidp)
{
int stat = NC_NOERR;
nc_type srctype = NC_NAT;
/* Convert the incoming jsrc to a restricted set of values */
switch (NCJsort(jsrc)) {
case NCJ_INT: /* convert to (u)int64 */
if(NCJstring(jsrc)[0] == '-') {
if(sscanf(NCJstring(jsrc),"%lld",&zcvt.int64v) != 1)
if(sscanf(NCJstring(jsrc),"%lld",&zcvt->int64v) != 1)
{stat = NC_EINVAL; goto done;}
srctype = NC_INT64;
} else {
if(sscanf(NCJstring(jsrc),"%llu",&zcvt.uint64v) != 1)
if(sscanf(NCJstring(jsrc),"%llu",&zcvt->uint64v) != 1)
{stat = NC_EINVAL; goto done;}
srctype = NC_UINT64;
}
break;
case NCJ_STRING:
case NCJ_DOUBLE:
/* Capture nan and infinity values */
if(strcasecmp(NCJstring(jsrc),"nan")==0)
zcvt.float64v = NAN;
zcvt->float64v = NAN;
else if(strcasecmp(NCJstring(jsrc),"-nan")==0)
zcvt.float64v = - NAN;
zcvt->float64v = - NAN;
else if(strcasecmp(NCJstring(jsrc),"infinity")==0)
zcvt.float64v = INFINITY;
zcvt->float64v = INFINITY;
else if(strcasecmp(NCJstring(jsrc),"-infinity")==0)
zcvt.float64v = (- INFINITY);
zcvt->float64v = (- INFINITY);
else {
if(sscanf(NCJstring(jsrc),"%lg",&zcvt.float64v) != 1)
if(sscanf(NCJstring(jsrc),"%lg",&zcvt->float64v) != 1)
{stat = NC_EINVAL; goto done;}
}
srctype = NC_DOUBLE;
@ -67,169 +137,233 @@ NCZ_convert1(NCjson* jsrc, nc_type dsttype, unsigned char* memory)
case NCJ_BOOLEAN:
srctype = NC_UINT64;
if(strcasecmp(NCJstring(jsrc),"false")==0)
zcvt.uint64v = 0;
zcvt->uint64v = 0;
else
zcvt.uint64v = 1;
zcvt->uint64v = 1;
break;
case NCJ_STRING:
srctype = NC_STRING;
zcvt->strv = NCJstring(jsrc);
break;
default: stat = NC_EINTERNAL; goto done;
}
if(typeidp) *typeidp = srctype;
done:
return stat;
}
/* Convert a singleton NCjson value to a memory equivalent value of specified dsttype; */
int
NCZ_convert1(NCjson* jsrc, nc_type dsttype, NCbytes* buf)
{
int stat = NC_NOERR;
nc_type srctype;
struct ZCVT zcvt = zcvt_empty;
int outofrange = 0;
size_t len = 0;
assert(dsttype != NC_NAT && dsttype <= NC_MAX_ATOMIC_TYPE && buf);
switch (NCJsort(jsrc)) {
case NCJ_STRING: case NCJ_INT: case NCJ_DOUBLE: case NCJ_BOOLEAN:
if((stat = NCZ_json2cvt(jsrc,&zcvt,&srctype))) goto done;
break;
default: stat = NC_EINVAL; goto done; /* Illegal JSON */
}
len = ncz_type_size[dsttype]; /* may change later */
/* Now, do the down conversion */
switch (dsttype) {
case NC_BYTE: {
signed char* p = (signed char*)memory;
signed char c = 0;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < NC_MIN_BYTE || zcvt.int64v > NC_MAX_BYTE) outofrange = 1;
*p = (signed char)zcvt.int64v;
c = (signed char)zcvt.int64v;
ncbytesappend(buf,(char)c);
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_BYTE) outofrange = 1;
*p = (signed char)zcvt.uint64v;
c = (signed char)zcvt.uint64v;
ncbytesappend(buf,(char)c);
break;
}
} break;
case NC_UBYTE: {
unsigned char* p = (unsigned char*)memory;
unsigned char c = 0;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < 0 || zcvt.int64v > NC_MAX_BYTE) outofrange = 1;
*p = (unsigned char)zcvt.int64v;
c = (unsigned char)zcvt.int64v;
ncbytesappend(buf,(char)c);
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_UBYTE) outofrange = 1;
*p = (unsigned char)zcvt.uint64v;
c = (unsigned char)zcvt.uint64v;
ncbytesappend(buf,(char)c);
break;
}
} break;
case NC_SHORT: {
signed short* p = (signed short*)memory;
signed short s = 0;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < NC_MIN_SHORT || zcvt.int64v > NC_MAX_SHORT) outofrange = 1;
*p = (signed short)zcvt.int64v;
s = (signed short)zcvt.int64v;
ncbytesappendn(buf,(char*)&s,sizeof(s));
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_SHORT) outofrange = 1;
*p = (signed short)zcvt.uint64v;
s = (signed short)zcvt.uint64v;
ncbytesappendn(buf,(char*)&s,sizeof(s));
break;
}
} break;
case NC_USHORT: {
unsigned short* p = (unsigned short*)memory;
unsigned short s = 0;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < 0 || zcvt.int64v > NC_MAX_USHORT) outofrange = 1;
*p = (unsigned short)zcvt.int64v;
s = (unsigned short)zcvt.int64v;
ncbytesappendn(buf,(char*)&s,sizeof(s));
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_USHORT) outofrange = 1;
*p = (unsigned short)zcvt.uint64v;
s = (unsigned short)zcvt.uint64v;
ncbytesappendn(buf,(char*)&s,sizeof(s));
break;
}
} break;
case NC_INT: {
signed int* p = (signed int*)memory;
signed int ii = 0;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < NC_MIN_INT || zcvt.int64v > NC_MAX_INT) outofrange = 1;
*p = (signed int)zcvt.int64v;
ii = (signed int)zcvt.int64v;
ncbytesappendn(buf,(char*)&ii,sizeof(ii));
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_INT) outofrange = 1;
*p = (signed int)zcvt.uint64v;
ii = (signed int)zcvt.uint64v;
ncbytesappendn(buf,(char*)&ii,sizeof(ii));
break;
}
} break;
case NC_UINT: {
unsigned int* p = (unsigned int*)memory;
unsigned int ii = 0;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
if(zcvt.int64v < 0 || zcvt.int64v > NC_MAX_UINT) outofrange = 1;
*p = (unsigned int)zcvt.int64v;
ii = (unsigned int)zcvt.int64v;
ncbytesappendn(buf,(char*)&ii,sizeof(ii));
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_UINT) outofrange = 1;
*p = (unsigned int)zcvt.uint64v;
ii = (unsigned int)zcvt.uint64v;
ncbytesappendn(buf,(char*)&ii,sizeof(ii));
break;
}
} break;
case NC_INT64: {
signed long long* p = (signed long long*)memory;
signed long long ll = 0;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */
/* fall thru */
case NC_INT64:
*p = (signed long long)zcvt.int64v;
ll = (signed long long)zcvt.int64v;
ncbytesappendn(buf,(char*)&ll,sizeof(ll));
break;
case NC_UINT64:
if(zcvt.uint64v > NC_MAX_INT64) outofrange = 1;
*p = (signed long long)zcvt.uint64v;
ll = (signed long long)zcvt.uint64v;
ncbytesappendn(buf,(char*)&ll,sizeof(ll));
break;
}
} break;
case NC_UINT64: {
unsigned long long* p = (unsigned long long*)memory;
unsigned long long ll = 0;
switch (srctype) {
case NC_DOUBLE:
zcvt.int64v = (signed long long)zcvt.float64v;
/* fall thru */
case NC_INT64:
if(zcvt.int64v < 0) outofrange = 1;
*p = (unsigned long long)zcvt.int64v;
ll = (unsigned long long)zcvt.int64v;
ncbytesappendn(buf,(char*)&ll,sizeof(ll));
break;
case NC_UINT64:
*p = (unsigned long long)zcvt.uint64v;
ll = (unsigned long long)zcvt.uint64v;
ncbytesappendn(buf,(char*)&ll,sizeof(ll));
break;
}
} break;
case NC_FLOAT: {
float* p = (float*)memory;
float f = 0;
switch (srctype) {
case NC_DOUBLE:
*p = (float)zcvt.float64v;
f = (float)zcvt.float64v;
ncbytesappendn(buf,(char*)&f,sizeof(f));
break;
case NC_INT64:
*p = (float)zcvt.int64v;
f = (float)zcvt.int64v;
ncbytesappendn(buf,(char*)&f,sizeof(f));
break;
case NC_UINT64:
*p = (float)zcvt.uint64v;
f = (float)zcvt.uint64v;
ncbytesappendn(buf,(char*)&f,sizeof(f));
break;
}
} break;
case NC_DOUBLE: {
double* p = (double*)memory;
double d = 0;
switch (srctype) {
case NC_DOUBLE:
*p = (double)zcvt.float64v;
d = (double)zcvt.float64v;
ncbytesappendn(buf,(char*)&d,sizeof(d));
break;
case NC_INT64:
*p = (double)zcvt.int64v;
break;
case NC_UINT64:
*p = (double)zcvt.uint64v;
d = (double)zcvt.int64v;
ncbytesappendn(buf,(char*)&d,sizeof(d));
case NC_UINT64:
d = (double)zcvt.uint64v;
ncbytesappendn(buf,(char*)&d,sizeof(d));
break;
}
} break;
case NC_STRING: {
char* scopy = NULL;
if(srctype != NC_STRING) {stat = NC_EINVAL; goto done;}
/* Need to append the pointer and not what it points to */
scopy = nulldup(zcvt.strv);
ncbytesappendn(buf,(void*)&scopy,sizeof(scopy));
scopy = NULL;
} break;
case NC_CHAR: {
if(srctype != NC_STRING) {stat = NC_EINVAL; goto done;}
len = strlen(zcvt.strv);
ncbytesappendn(buf,zcvt.strv,len);
} break;
default: stat = NC_EINTERNAL; goto done;
}
@ -240,14 +374,15 @@ done:
/* Convert a memory value to a JSON string value */
int
NCZ_stringconvert1(nc_type srctype, size_t len, char* src, NCjson* jvalue)
NCZ_stringconvert1(nc_type srctype, char* src, NCjson* jvalue)
{
int stat = NC_NOERR;
struct ZCVT zcvt;
nc_type dsttype = NC_NAT;
char s[1024];
char* p = NULL;
assert(srctype >= NC_NAT && srctype != NC_CHAR && srctype < NC_STRING);
assert(srctype >= NC_NAT && srctype != NC_CHAR && srctype <= NC_STRING);
/* Convert to a restricted set of values */
switch (srctype) {
case NC_BYTE: {
@ -290,6 +425,10 @@ NCZ_stringconvert1(nc_type srctype, size_t len, char* src, NCjson* jvalue)
dsttype = NC_DOUBLE;
zcvt.float64v= (double)(*((double*)src));
} break;
case NC_STRING: {
dsttype = NC_STRING;
zcvt.strv= *((char**)src);
} break;
default: stat = NC_EINTERNAL; goto done;
}
@ -326,13 +465,21 @@ NCZ_stringconvert1(nc_type srctype, size_t len, char* src, NCjson* jvalue)
}
#endif
} break;
case NC_STRING: {
p = nulldup(zcvt.strv);
} break;
default: stat = NC_EINTERNAL; goto done;
}
NCJsetstring(jvalue,strdup(s));
if(p == NULL)
p = strdup(s);
NCJsetstring(jvalue,p);
p = NULL;
done:
nullfree(p);
return stat;
}
/* Convert arbitrary netcdf attribute vector to equivalent JSON */
int
NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap)
{
@ -352,17 +499,19 @@ NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap)
/* Handle char type specially */
if(typeid == NC_CHAR) {
/* Create a string valued json object */
if((stat = NCJnewstringn(NCJ_STRING,len,src,&jdata)))
goto done;
/* Apply the JSON write convention */
if((stat = NCJparsen(len,src,0,&jdata))) { /* !parseable */
/* Create a string valued json object */
if((stat = NCJnewstringn(NCJ_STRING,len,src,&jdata))) goto done;
}
} else if(len == 1) { /* create singleton */
if((stat = NCJnew(jtype,&jdata))) goto done;
if((stat = NCZ_stringconvert1(typeid, len, src, jdata))) goto done;
if((stat = NCZ_stringconvert1(typeid, src, jdata))) goto done;
} else { /* len > 1 create array of values */
if((stat = NCJnew(NCJ_ARRAY,&jdata))) goto done;
for(i=0;i<len;i++) {
if((stat = NCJnew(jtype,&jvalue))) goto done;
if((stat = NCZ_stringconvert1(typeid, len, src, jvalue))) goto done;
if((stat = NCZ_stringconvert1(typeid, src, jvalue))) goto done;
NCJappend(jdata,jvalue);
jvalue = NULL;
src += typelen;
@ -388,6 +537,7 @@ typeid2jtype(nc_type typeid)
case NC_DOUBLE:
return NCJ_DOUBLE;
case NC_CHAR:
case NC_STRING:
return NCJ_STRING;
default: break;
}

4
libnczarr/zdebug.h
View File

@ -8,8 +8,8 @@
#undef ZDEBUG /* general debug */
#undef ZDEBUG1 /* detailed debug */
#undef ZCATCH /* Warning: significant performance impact */
#undef ZTRACING /* Warning: significant performance impact */
#define ZCATCH /* Warning: significant performance impact */
#define ZTRACING /* Warning: significant performance impact */
#include "ncexternl.h"
#include "nclog.h"

11
libnczarr/zfile.c
View File

@ -116,17 +116,6 @@ NCZ_enddef(NC_FILE_INFO_T* h5)
assert(var);
var->written_to = NC_TRUE; /* mark it written */
var->created = 1;
#if 0
/* set the fill value and _FillValue attribute */
if((stat = NCZ_ensure_fill_value(var))) goto done; /* ensure var->fill_value is set */
assert(var->no_fill || var->fill_value != NULL);
/* rebuild the fill chunk */
if((stat = NCZ_adjust_var_cache(var))) goto done;
#ifdef ENABLE_NCZARR_FILTERS
/* Build the filter working parameters for any filters */
if((stat = NCZ_filter_setup(var))) goto done;
#endif
#endif /*0|1*/
}
}
if((stat = ncz_enddef_netcdf4_file(h5))) goto done;

38
libnczarr/zinternal.h
View File

@ -22,9 +22,6 @@
#define NCZ_CHUNKSIZE_FACTOR (10)
#define NCZ_MIN_CHUNK_SIZE (2)
/* An attribute in the ZARR root group of this name means that the
* file must follow strict netCDF classic format rules. */
#define NCZ_NC3_STRICT_ATT_NAME "_nc3_strict"
/**************************************************/
/* Constants */
@ -62,43 +59,54 @@
/* V2 Reserved Attributes */
/*
Inserted into /.zgroup
_NCZARR_SUPERBLOCK: {"version": "2.0.0"}
_nczarr_superblock: {"version": "2.0.0"}
Inserted into any .zgroup
"_NCZARR_GROUP": "{
"_nczarr_group": "{
\"dimensions\": {\"d1\": \"1\", \"d2\": \"1\",...}
\"variables\": [\"v1\", \"v2\", ...]
\"groups\": [\"g1\", \"g2\", ...]
}"
Inserted into any .zarray
"_NCZARR_ARRAY": "{
"_nczarr_array": "{
\"dimensions\": [\"/g1/g2/d1\", \"/d2\",...]
\"storage\": \"scalar\"|\"contiguous\"|\"compact\"|\"chunked\"
}"
Inserted into any .zattrs ? or should it go into the container?
"_NCZARR_ATTRS": "{
"_nczarr_attrs": "{
\"types\": {\"attr1\": \"<i4\", \"attr2\": \"<i1\",...}
}
+
+Note: _nczarr_attrs type include non-standard use of a zarr type "|U1" => NC_CHAR.
+
*/
#define NCZ_V2_SUPERBLOCK "_NCZARR_SUPERBLOCK"
#define NCZ_V2_GROUP "_NCZARR_GROUP"
#define NCZ_V2_ARRAY "_NCZARR_ARRAY"
#define NCZ_V2_SUPERBLOCK "_nczarr_superblock"
#define NCZ_V2_GROUP "_nczarr_group"
#define NCZ_V2_ARRAY "_nczarr_array"
#define NCZ_V2_ATTR NC_NCZARR_ATTR
#define NCZ_V2_SUPERBLOCK_UC "_NCZARR_SUPERBLOCK"
#define NCZ_V2_GROUP_UC "_NCZARR_GROUP"
#define NCZ_V2_ARRAY_UC "_NCZARR_ARRAY"
#define NCZ_V2_ATTR_UC NC_NCZARR_ATTR_UC
#define NCZARRCONTROL "nczarr"
#define PUREZARRCONTROL "zarr"
#define XARRAYCONTROL "xarray"
#define NOXARRAYCONTROL "noxarray"
#define XARRAYSCALAR "_scalar_"
#define LEGAL_DIM_SEPARATORS "./"
#define DFALT_DIM_SEPARATOR '.'
/* Default max string length for fixed length strings */
#define NCZ_MAXSTR_DEFAULT 64
#define islegaldimsep(c) ((c) != '\0' && strchr(LEGAL_DIM_SEPARATORS,(c)) != NULL)
/* Mnemonics */
#define ZCLOSE 1 /* this is closeorabort as opposed to enddef */
/* Mnemonics */
#define ZCLOSE 1 /* this is closeorabort as opposed to enddef */
#define ZCLEAR 0 /* For NCZ_copy_data */
#define ZCLOSE 1 /* this is closeorabort as opposed to enddef */
/* Useful macro */
#define ncidforx(file,grpid) ((file)->controller->ext_ncid | (grpid))
@ -146,6 +154,7 @@ typedef struct NCZ_FILE_INFO {
# define FLAG_NCZARR_V1 16
NCZM_IMPL mapimpl;
} controls;
int default_maxstrlen; /* default max str size for variables of type string */
} NCZ_FILE_INFO_T;
/* This is a struct to handle the dim metadata. */
@ -186,6 +195,7 @@ typedef struct NCZ_VAR_INFO {
struct NClist* xarray; /* names from _ARRAY_DIMENSIONS */
char dimension_separator; /* '.' | '/' */
NClist* incompletefilters;
int maxstrlen; /* max length of strings for this variable */
} NCZ_VAR_INFO_T;
/* Struct to hold ZARR-specific info for a field. */

853
libnczarr/zsync.c
View File

File diff suppressed because it is too large Load Diff

2
libnczarr/ztype.c
View File

@ -147,7 +147,7 @@ NCZ_inq_typeids(int ncid, int *ntypes, int *typeids)
* @param size Size in bytes of new type.
* @param name Name of new type.
* @param base_typeid Base type ID.
* @param type_class NC_VLEN, NC_ENUM, or NC_STRING
* @param type_class NC_VLEN, NC_ENUM
* @param typeidp Pointer that gets new type ID.
*
* @return ::NC_NOERR No error.

444
libnczarr/zutil.c
View File

@ -17,20 +17,57 @@
/**************************************************/
/* Static zarr type name table */
static const char* znames_little[NUM_ATOMIC_TYPES] = {
NULL, /*NC_NAT*/
"<i1", /*NC_BYTE*/ "<U1", /*NC_CHAR*/ "<i2", /*NC_SHORT*/
"<i4", /*NC_INT*/ "<f4", /*NC_FLOAT*/ "<f8", /*NC_DOUBLE*/ "<u1", /*NC_UBYTE*/
"<u2", /*NC_USHORT*/ "<u4", /*NC_UINT*/ "<i8", /*NC_INT64*/ "<u8", /*NC_UINT64*/
NULL, /*NC_STRING*/
};
/* Table of nc_type X {Zarr,NCZarr} X endianness
Issue: Need to distinquish NC_STRING && MAXSTRLEN==1 from NC_CHAR
in a way that allows other Zarr implementations to read the data.
static const char* znames_big[NUM_ATOMIC_TYPES] = {
NULL, /*NC_NAT*/
">i1", /*NC_BYTE*/ ">U1", /*NC_CHAR*/ ">i2", /*NC_SHORT*/ ">i4", /*NC_INT*/
">f4", /*NC_FLOAT*/ ">f8", /*NC_DOUBLE*/ ">u1", /*NC_UBYTE*/
">u2", /*NC_USHORT*/ ">u4", /*NC_UINT*/ ">i8", /*NC_INT64*/ ">u8", /*NC_UINT64*/
NULL, /*NC_STRING*/
Available info:
Write: we have the netcdf type, so there is no ambiguity.
Read: we have the variable type and also any attribute dtype,
but those types are ambiguous.
We also have the attribute vs variable type problem.
For pure zarr, we have to infer the type of an attribute,
so if we have "var:strattr = \"abcdef\"", then we need
to decide how to infer the type: NC_STRING vs NC_CHAR.
Solution:
For variables and for NCZarr type attributes, distinquish by using:
* "|S1" for NC_CHAR.
* ">S1" for NC_STRING && MAXSTRLEN==1
It is a bit of a hack to use endianness, but it should be ok since for
string/char, the endianness has no meaning.
Note that we could use "|U1", but since this is utf-16 or utf-32
in python, it may cause problems when reading what amounts to utf-8.
For attributes, we infer:
* NC_CHAR if the hint is 0
- e.g. var:strattr = 'abcdef'" => NC_CHAR
* NC_STRING if hint is NC_STRING.
- e.g. string var:strattr = \"abc\", \"def\"" => NC_STRING
Note also that if we read a pure zarr file we will probably always
see "|S1", so we will never see a variable of type NC_STRING with length == 1.
We might however see an attribute of type string.
*/
static const struct ZTYPES {
char* zarr[3];
char* nczarr[3];
} znames[NUM_ATOMIC_TYPES] = {
/* nc_type Pure Zarr NCZarr
NE LE BE NE LE BE*/
/*NC_NAT*/ {{NULL,NULL,NULL}, {NULL,NULL,NULL}},
/*NC_BYTE*/ {{"|i1","<i1",">i1"},{"|i1","<i1",">i1"}},
/*NC_CHAR*/ {{"|S1","|S1","|S1"},{"|S1","|S1","|S1"}},
/*NC_SHORT*/ {{"|i2","<i2",">i2"},{"|i2","<i2",">i2"}},
/*NC_INT*/ {{"|i4","<i4",">i4"},{"|i4","<i4",">i4"}},
/*NC_FLOAT*/ {{"|f4","<f4",">f4"},{"|f4","<f4",">f4"}},
/*NC_DOUBLE*/ {{"|f8","<f8",">f8"},{"|f8","<f8",">f8"}},
/*NC_UBYTE*/ {{"|u1","<u1",">u1"},{"|u1","<u1",">u1"}},
/*NC_USHORT*/ {{"|u2","<u2",">u2"},{"|u2","<u2",">u2"}},
/*NC_UINT*/ {{"|u4","<u4",">u4"},{"|u4","<u4",">u4"}},
/*NC_INT64*/ {{"|i8","<i8",">i8"},{"|i8","<i8",">i8"}},
/*NC_UINT64*/ {{"|u8","<u8",">u8"},{"|u8","<u8",">u8"}},
/*NC_STRING*/ {{">S%d",">S%d",">S%d"},{">S%d",">S%d",">S%d"}},
};
#if 0
@ -47,6 +84,7 @@ NULL, /*NC_NAT*/
"4294967295", /*NC_UINT*/
"-9223372036854775806", /*NC_INT64*/
"18446744073709551614", /*NC_UINT64*/
"", /*NC_STRING*/
};
#endif
@ -64,6 +102,7 @@ NCJ_INT, /*NC_USHORT*/
NCJ_INT, /*NC_UINT*/
NCJ_INT, /*NC_INT64*/
NCJ_INT, /*NC_UINT64*/
NCJ_STRING, /*NC_STRING*/
};
/* Forward */
@ -204,8 +243,8 @@ NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp)
goto done;
content[len] = '\0';
if((stat = NCJparse(content,0,&json)))
goto done;
if((stat = NCJparse(content,0,&json)) < 0)
{stat = NC_ENCZARR; goto done;}
if(jsonp) {*jsonp = json; json = NULL;}
@ -370,28 +409,6 @@ done:
return stat;
}
/**
@internal Given an nc_type+other, produce the corresponding
zarr type name.
@param nctype - [in] nc_type
@param little - [in] 1=>little, 0 => big
@param namep - [out] pointer to hold pointer to the name
@return NC_NOERR
@author Dennis Heimbigner
*/
int
ncz_zarr_type_name(nc_type nctype, int little, const char** znamep)
{
if(nctype <= 0 || nctype >= NC_STRING) return NC_EINVAL;
if(little) {
if(znamep) *znamep = znames_little[nctype];
} else {
if(znamep) *znamep = znames_big[nctype];
}
return NC_NOERR;
}
#if 0
/**
@internal Given an nc_type, produce the corresponding
@ -405,7 +422,7 @@ default fill value as a string.
int
ncz_default_fill_value(nc_type nctype, const char** dfaltp)
{
if(nctype <= 0 || nctype >= NC_STRING) return NC_EINVAL;
if(nctype <= 0 || nctype > NC_MAX_ATOMIC_TYPE) return NC_EINVAL;
if(dfaltp) *dfaltp = zfillvalue[nctype];
return NC_NOERR;
}
@ -423,7 +440,7 @@ fill value JSON type
int
ncz_fill_value_sort(nc_type nctype, int* sortp)
{
if(nctype <= 0 || nctype >= NC_STRING) return NC_EINVAL;
if(nctype <= 0 || nctype > NC_MAX_ATOMIC_TYPE) return NC_EINVAL;
if(sortp) *sortp = zjsonsort[nctype];
return NC_NOERR;
}
@ -486,57 +503,123 @@ done:
return stat;
}
#if 0
/* Convert a netcdf-4 type integer */
int
ncz_nctype2typeinfo(const char* snctype, nc_type* nctypep)
ncz_nctypedecode(const char* snctype, nc_type* nctypep)
{
unsigned nctype = 0;
if(sscanf(snctype,"%u",&nctype)!=1) return NC_EINVAL;
if(nctypep) *nctypep = nctype;
return NC_NOERR;
}
#endif
/**
@internal Given an nc_type+other, produce the corresponding dtype string.
@param nctype - [in] nc_type
@param endianness - [in] endianness
@param purezarr - [in] 1=>pure zarr, 0 => nczarr
@param strlen - [in] max string length
@param namep - [out] pointer to hold pointer to the dtype; user frees
@return NC_NOERR
@return NC_EINVAL
@author Dennis Heimbigner
*/
int
ncz_dtype2typeinfo(const char* dtype, nc_type* nctypep, int* endianp)
ncz_nctype2dtype(nc_type nctype, int endianness, int purezarr, int strlen, char** dnamep)
{
char dname[64];
char* format = NULL;
if(nctype <= NC_NAT || nctype > NC_MAX_ATOMIC_TYPE) return NC_EINVAL;
if(purezarr)
format = znames[nctype].zarr[endianness];
else
format = znames[nctype].nczarr[endianness];
snprintf(dname,sizeof(dname),format,strlen);
if(dnamep) *dnamep = strdup(dname);
return NC_NOERR;
}
/*
@internal Convert a numcodecs dtype spec to a corresponding nc_type.
@param nctype - [in] dtype the dtype to convert
@param nctype - [in] typehint help disambiguate char vs string
@param purezarr - [in] 1=>pure zarr, 0 => nczarr
@param nctypep - [out] hold corresponding type
@param endianp - [out] hold corresponding endianness
@param typelenp - [out] hold corresponding type size (for fixed length strings)
@return NC_NOERR
@return NC_EINVAL
@author Dennis Heimbigner
*/
int
ncz_dtype2nctype(const char* dtype, nc_type typehint, int purezarr, nc_type* nctypep, int* endianp, int* typelenp)
{
int stat = NC_NOERR;
int typelen = 0;
int count;
char tchar;
nc_type nctype = NC_NAT;
int endianness = 0;
int endianness = -1;
const char* p;
int n;
if(endianness) *endianp = NC_ENDIAN_NATIVE;
if(endianp) *endianp = NC_ENDIAN_NATIVE;
if(nctypep) *nctypep = NC_NAT;
if(dtype == NULL) goto zerr;
if(strlen(dtype) < 3) goto zerr;
switch (dtype[0]) {
p = dtype;
switch (*p++) {
case '<': endianness = NC_ENDIAN_LITTLE; break;
case '>': endianness = NC_ENDIAN_BIG; break;
case '=': endianness = NC_ENDIAN_NATIVE; break;
case '|': endianness = NC_ENDIAN_NATIVE; break;
default: goto zerr;
default: p--; endianness = NC_ENDIAN_NATIVE; break;
}
tchar = *p++; /* get the base type */
/* Decode the type length */
count = sscanf(dtype+2,"%d",&typelen);
if(count != 1) goto zerr;
tchar = dtype[1];
switch(typelen) {
case 1:
switch (tchar) {
case 'i': nctype = NC_BYTE; break;
case 'u': nctype = NC_UBYTE; break;
case 'U': nctype = NC_CHAR; break;
default: goto zerr;
count = sscanf(p,"%d%n",&typelen,&n);
if(count == 0) goto zerr;
p += n;
/* Short circuit fixed length strings */
if(tchar == 'S') {
/* Fixed length string */
switch (typehint) {
case NC_CHAR: nctype = NC_CHAR; typelen = 1; break;
case NC_STRING: nctype = NC_STRING; break;
default:
if(typelen == 1) {/* so |S1 => NC_CHAR */
if(purezarr || endianness == NC_ENDIAN_NATIVE) nctype = NC_CHAR;
} else
nctype = NC_STRING;
}
break;
case 2:
#if 0
} else if(tchar == 'U') {/*back compatibility*/
if(purezarr || typelen != 1) goto zerr;
nctype = NC_CHAR;
#endif
} else {
switch(typelen) {
case 1:
switch (tchar) {
case 'i': nctype = NC_BYTE; break;
case 'u': nctype = NC_UBYTE; break;
default: goto zerr;
}
break;
case 2:
switch (tchar) {
case 'i': nctype = NC_SHORT; break;
case 'u': nctype = NC_USHORT; break;
default: goto zerr;
}
break;
case 4:
case 4:
switch (tchar) {
case 'i': nctype = NC_INT; break;
case 'u': nctype = NC_UINT; break;
@ -544,7 +627,7 @@ ncz_dtype2typeinfo(const char* dtype, nc_type* nctypep, int* endianp)
default: goto zerr;
}
break;
case 8:
case 8:
switch (tchar) {
case 'i': nctype = NC_INT64; break;
case 'u': nctype = NC_UINT64; break;
@ -552,10 +635,16 @@ ncz_dtype2typeinfo(const char* dtype, nc_type* nctypep, int* endianp)
default: goto zerr;
}
break;
default: goto zerr;
default: goto zerr;
}
}
/* Convert NC_ENDIAN_NATIVE and NC_ENDIAN_NA */
if(endianness == NC_ENDIAN_NATIVE)
endianness = (NC_isLittleEndian()?NC_ENDIAN_LITTLE:NC_ENDIAN_BIG);
if(nctypep) *nctypep = nctype;
if(typelenp) *typelenp = typelen;
if(endianp) *endianp = endianness;
done:
@ -565,7 +654,104 @@ zerr:
goto done;
}
/* Infer the attribute's type based
primarily on the first atomic value encountered
recursively.
*/
int
NCZ_inferattrtype(NCjson* value, nc_type typehint, nc_type* typeidp)
{
int i,stat = NC_NOERR;
nc_type typeid;
NCjson* j = NULL;
unsigned long long u64;
long long i64;
int negative = 0;
if(NCJsort(value) == NCJ_ARRAY && NCJlength(value) == 0)
{typeid = NC_NAT; goto done;} /* Empty array is illegal */
if(NCJsort(value) == NCJ_NULL)
{typeid = NC_NAT; goto done;} /* NULL is also illegal */
if(NCJsort(value) == NCJ_DICT) /* Complex JSON expr -- a dictionary */
{typeid = NC_NAT; goto done;}
/* If an array, make sure all the elements are simple */
if(value->sort == NCJ_ARRAY) {
for(i=0;i<NCJlength(value);i++) {
j=NCJith(value,i);
if(!NCJisatomic(j))
{typeid = NC_NAT; goto done;}
}
}
/* Infer from first element */
if(value->sort == NCJ_ARRAY) {
j=NCJith(value,0);
return NCZ_inferattrtype(j,typehint,typeidp);
}
/* At this point, value is a primitive JSON Value */
switch (NCJsort(value)) {
case NCJ_NULL:
typeid = NC_NAT;
return NC_NOERR;
case NCJ_DICT:
typeid = NC_CHAR;
goto done;
case NCJ_UNDEF:
return NC_EINVAL;
default: /* atomic */
break;
}
if(NCJstring(value) != NULL)
negative = (NCJstring(value)[0] == '-');
switch (value->sort) {
case NCJ_INT:
if(negative) {
sscanf(NCJstring(value),"%lld",&i64);
u64 = (unsigned long long)i64;
} else
sscanf(NCJstring(value),"%llu",&u64);
typeid = NCZ_inferinttype(u64,negative);
break;
case NCJ_DOUBLE:
typeid = NC_DOUBLE;
break;
case NCJ_BOOLEAN:
typeid = NC_UBYTE;
break;
case NCJ_STRING: /* requires special handling as an array of characters */
typeid = NC_CHAR;
break;
default:
stat = NC_ENCZARR;
}
done:
if(typeidp) *typeidp = typeid;
return stat;
}
/* Infer the int type from the value;
minimum type will be int.
*/
int
NCZ_inferinttype(unsigned long long u64, int negative)
{
long long i64 = (long long)u64; /* keep bit pattern */
if(!negative && u64 >= NC_MAX_INT64) return NC_UINT64;
if(i64 < 0) {
if(i64 >= NC_MIN_INT) return NC_INT;
return NC_INT64;
}
if(i64 <= NC_MAX_INT) return NC_INT;
if(i64 <= NC_MAX_UINT) return NC_UINT;
return NC_INT64;
}
/**
@internal Similar to NCZ_grppath, but using group ids.
@param gid - [in] group id
@ -788,3 +974,139 @@ done:
if(dst) (void)nc_reclaim_data_all(ncid,tid,dst,1);
return stat;
}
/* Get max str len for a variable or grp */
/* Has side effect of setting values in the
internal data structures */
int
NCZ_get_maxstrlen(NC_OBJ* obj)
{
int maxstrlen = 0;
assert(obj->sort == NCGRP || obj->sort == NCVAR);
if(obj->sort == NCGRP) {
NC_GRP_INFO_T* grp = (NC_GRP_INFO_T*)obj;
NC_FILE_INFO_T* file = grp->nc4_info;
NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info;
if(zfile->default_maxstrlen == 0)
zfile->default_maxstrlen = NCZ_MAXSTR_DEFAULT;
maxstrlen = zfile->default_maxstrlen;
} else { /*(obj->sort == NCVAR)*/
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)obj;
NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
if(zvar->maxstrlen == 0)
zvar->maxstrlen = NCZ_get_maxstrlen((NC_OBJ*)var->container);
maxstrlen = zvar->maxstrlen;
}
return maxstrlen;
}
int
NCZ_fixed2char(const void* fixed, char** charp, size_t count, int maxstrlen)
{
size_t i;
unsigned char* sp = NULL;
const unsigned char* p = fixed;
memset((void*)charp,0,sizeof(char*)*count);
for(i=0;i<count;i++,p+=maxstrlen) {
if(p[0] == '\0') {
sp = NULL;
} else {
if((sp = (unsigned char*)malloc(maxstrlen+1))==NULL) /* ensure null terminated */
return NC_ENOMEM;
memcpy(sp,p,maxstrlen);
sp[maxstrlen] = '\0';
}
charp[i] = sp;
sp = NULL;
}
return NC_NOERR;
}
int
NCZ_char2fixed(const char** charp, void* fixed, size_t count, int maxstrlen)
{
size_t i;
unsigned char* p = fixed;
memset(fixed,0,maxstrlen*count); /* clear target */
for(i=0;i<count;i++,p+=maxstrlen) {
size_t len;
if(charp[i] != NULL) {
len = strlen(charp[i]);
if(len > maxstrlen) len = maxstrlen;
memcpy(p,charp[i],len);
} else {
memset(p,'\0',maxstrlen);
}
}
return NC_NOERR;
}
/*
Wrap NC_copy_data, but take string value into account when overwriting
*/
int
NCZ_copy_data(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* xtype, const void* memory, size_t count, int noclear, void* copy)
{
if(xtype->hdr.id == NC_STRING && !noclear) {
size_t i;
char** scopy = (char**)copy;
/* Reclaim any string fill values in copy */
for(i=0;i<count;i++) {
nullfree(scopy[i]);
scopy[i] = NULL;
}
}
return nc_copy_data(file->controller->ext_ncid,xtype->hdr.id,memory,count,copy);
}
#if 0
/* Recursive helper */
static int
checksimplejson(NCjson* json, int depth)
{
int i;
switch (NCJsort(json)) {
case NCJ_ARRAY:
if(depth > 0) return 0; /* e.g. [...,[...],...] or [...,{...},...] */
for(i=0;i < NCJlength(json);i++) {
NCjson* j = NCJith(json,i);
if(!checksimplejson(j,depth+1)) return 0;
}
break;
case NCJ_DICT:
case NCJ_NULL:
case NCJ_UNDEF:
return 0;
default: break;
}
return 1;
}
#endif
/* Return 1 if the attribute will be stored as a complex JSON valued attribute; return 0 otherwise */
int
NCZ_iscomplexjson(NCjson* json, nc_type typehint)
{
int i, stat = 0;
switch (NCJsort(json)) {
case NCJ_ARRAY:
/* If the typehint is NC_CHAR, then always treat it as complex */
if(typehint == NC_CHAR) {stat = 1; goto done;}
/* Otherwise see if it is a simple vector of atomic values */
for(i=0;i < NCJlength(json);i++) {
NCjson* j = NCJith(json,i);
if(!NCJisatomic(j)) {stat = 1; goto done;}
}
break;
case NCJ_DICT:
case NCJ_NULL:
case NCJ_UNDEF:
stat = 1; goto done;
default: break;
}
done:
return stat;
}

13
libnczarr/zvar.c
View File

@ -12,6 +12,11 @@
#include "zincludes.h"
#include <math.h> /* For pow() used below. */
/* Mnemonics */
#define CREATE 0
#define NOCREATE 1
#ifdef LOGGING
static void
reportchunking(const char* title, NC_VAR_INFO_T* var)
@ -107,10 +112,7 @@ ncz_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
double total_chunk_size;
#endif
if (var->type_info->nc_type_class == NC_STRING)
type_size = sizeof(char *);
else
type_size = var->type_info->size;
type_size = var->type_info->size;
#ifdef LOGGING
/* Later this will become the total number of bytes in the default
@ -406,7 +408,7 @@ var->type_info->rc++;
/* Set variables no_fill to match the database default unless the
* variable type is variable length (NC_STRING or NC_VLEN) or is
* user-defined type. */
if (var->type_info->nc_type_class < NC_STRING)
if (var->type_info->nc_type_class <= NC_STRING)
var->no_fill = (h5->fill_mode == NC_NOFILL);
/* Assign dimensions to the variable. At the same time, check to
@ -473,7 +475,6 @@ exit:
if (type)
if ((retval = nc4_type_free(type)))
BAILLOG(retval);
return ZUNTRACE(retval);
}

41
libnczarr/zwalk.c
View File

@ -128,10 +128,16 @@ NCZ_transferslice(NC_VAR_INFO_T* var, int reading,
common.swap = (zfile->native_endianness == var->endianness ? 0 : 1);
common.chunkcount = 1;
for(r=0;r<common.rank+common.scalar;r++) {
if(common.scalar)
dimlens[r] = 1;
else
if(common.scalar) {
dimlens[0] = 1;
chunklens[0] = var->chunksizes[0];
slices[0].start = 0;
slices[0].stride = 1;
slices[0].stop = 0;
slices[0].len = 1;
common.chunkcount = 1;
memshape[0] = 1;
} else for(r=0;r<common.rank;r++) {
dimlens[r] = var->dim[r]->len;
chunklens[r] = var->chunksizes[r];
slices[r].start = start[r];
@ -221,7 +227,7 @@ NCZ_transfer(struct Common* common, NCZSlice* slices)
if((stat=wholechunk_indices(common,slices,chunkindices))) goto done;
if(wdebug >= 1)
fprintf(stderr,"case: wholechunk: chunkindices: %s\n",nczprint_vector(common->rank,chunkindices));
/* Read the chunk */
/* Read the chunk; handles fixed vs char* strings*/
switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) {
case NC_EEMPTY: /* cache created the chunk */
break;
@ -232,9 +238,9 @@ NCZ_transfer(struct Common* common, NCZSlice* slices)
memptr = ((unsigned char*)common->memory);
slpptr = ((unsigned char*)chunkdata);
if(common->reading) {
memcpy(memptr,slpptr,common->chunkcount*common->typesize);
if((stat=NCZ_copy_data(common->file,common->var->type_info,slpptr,common->chunkcount,!ZCLEAR,memptr))) goto done;
} else {
memcpy(slpptr,memptr,common->chunkcount*common->typesize);
if((stat=NCZ_copy_data(common->file,common->var->type_info,memptr,common->chunkcount,ZCLEAR,slpptr))) goto done;
}
// transfern(common,slpptr,memptr,common->chunkcount,1,chunkdata);
if(zutest && zutest->tests & UTEST_WHOLECHUNK)
@ -410,15 +416,16 @@ NCZ_walk(NCZProjection** projv, NCZOdometer* chunkodom, NCZOdometer* slpodom, NC
if(slpavail > 0) {
if(wdebug > 0) wdebug2(common,slpptr0,memptr0,slpavail,laststride,chunkdata);
if(common->reading) {
memcpy(memptr0,slpptr0,slpavail*common->typesize);
if((stat=NCZ_copy_data(common->file,common->var->type_info,slpptr0,slpavail,!ZCLEAR,memptr0))) goto done;
} else {
memcpy(slpptr0,memptr0,slpavail*common->typesize);
if((stat=NCZ_copy_data(common->file,common->var->type_info,memptr0,slpavail,ZCLEAR,slpptr0))) goto done;
}
}
// if((stat = transfern(common,slpptr0,memptr0,avail,nczodom_laststride(slpodom),chunkdata)))goto done;
nczodom_next(memodom);
nczodom_next(slpodom);
}
done:
return stat;
}
@ -734,10 +741,11 @@ NCZ_transferscalar(struct Common* common)
/* Figure out memory address */
memptr = ((unsigned char*)common->memory);
slpptr = ((unsigned char*)chunkdata);
if(common->reading)
memcpy(memptr,slpptr,common->chunkcount*common->typesize);
else
memcpy(slpptr,memptr,common->chunkcount*common->typesize);
if(common->reading) {
if((stat=NCZ_copy_data(common->file,common->var->type_info,slpptr,common->chunkcount,!ZCLEAR,memptr))) goto done;
} else {
if((stat=NCZ_copy_data(common->file,common->var->type_info,memptr,common->chunkcount,ZCLEAR,slpptr))) goto done;
}
done:
return stat;
@ -749,7 +757,7 @@ NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata)
{
int stat = NC_NOERR;
NC_VAR_INFO_T* var = NULL;
NCZ_VAR_INFO_T* zvar;
NCZ_VAR_INFO_T* zvar = NULL;
struct NCZChunkCache* cache = NULL;
void* cachedata = NULL;
@ -759,8 +767,9 @@ NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata)
cache = zvar->cache;
if((stat = NCZ_read_cache_chunk(cache,zindices,&cachedata))) goto done;
if(chunkdata)
memcpy(chunkdata,cachedata,cache->chunksize);
if(chunkdata) {
if((stat = nc_copy_data(ncid,var->type_info->hdr.id,cachedata,cache->chunkcount,chunkdata))) goto done;
}
done:
return stat;

90
libnczarr/zxcache.c
View File

@ -202,9 +202,14 @@ done:
}
static void
free_cache_entry(NCZCacheEntry* entry)
free_cache_entry(NCZChunkCache* cache, NCZCacheEntry* entry)
{
if(entry) {
int tid = cache->var->type_info->hdr.id;
if(tid == NC_STRING && !entry->isfixedstring) {
int ncid = cache->var->container->nc4_info->controller->ext_ncid;
nc_reclaim_data(ncid,tid,entry->data,cache->chunkcount);
}
nullfree(entry->data);
nullfree(entry->key.varkey);
nullfree(entry->key.chunkkey);
@ -225,7 +230,7 @@ NCZ_free_chunk_cache(NCZChunkCache* cache)
NCZCacheEntry* entry = nclistremove(cache->mru,0);
(void)ncxcacheremove(cache->xcache,entry->hashkey,&ptr);
assert(ptr == entry);
free_cache_entry(entry);
free_cache_entry(cache,entry);
}
#ifdef DEBUG
fprintf(stderr,"|cache.free|=%ld\n",nclistlength(cache->mru));
@ -303,7 +308,7 @@ fprintf(stderr,"|cache.read.lru|=%ld\n",nclistlength(cache->mru));
done:
if(created && stat == NC_NOERR) stat = NC_EEMPTY; /* tell upper layers */
if(entry) free_cache_entry(entry);
if(entry) free_cache_entry(cache,entry);
return THROW(stat);
}
@ -343,7 +348,7 @@ fprintf(stderr,"|cache.write|=%ld\n",nclistlength(cache->mru));
if((stat=makeroom(cache))) goto done;
done:
if(entry) free_cache_entry(entry);
if(entry) free_cache_entry(cache,entry);
return THROW(stat);
}
#endif
@ -449,6 +454,7 @@ NCZ_ensure_fill_chunk(NCZChunkCache* cache)
{
int i, stat = NC_NOERR;
NC_VAR_INFO_T* var = cache->var;
nc_type typeid = var->type_info->hdr.id;
size_t typesize = var->type_info->size;
if(cache->fillchunk) goto done;
@ -461,6 +467,11 @@ NCZ_ensure_fill_chunk(NCZChunkCache* cache)
goto done;
}
if((stat = NCZ_ensure_fill_value(var))) goto done;
if(typeid == NC_STRING) {
char* src = *((char**)(var->fill_value));
char** dst = (char**)(cache->fillchunk);
for(i=0;i<cache->chunkcount;i++) dst[i] = strdup(src);
} else
switch (typesize) {
case 1: {
unsigned char c = *((unsigned char*)var->fill_value);
@ -597,6 +608,9 @@ put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
NCZ_FILE_INFO_T* zfile = NULL;
NCZMAP* map = NULL;
char* path = NULL;
nc_type tid = NC_NAT;
void* strchunk = NULL;
int ncid = 0;
ZTRACE(5,"cache.var=%s entry.key=%s",cache->var->hdr.name,entry->key);
LOG((3, "%s: var: %p", __func__, cache->var));
@ -605,6 +619,26 @@ put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
zfile = file->format_file_info;
map = zfile->map;
/* Collect some info */
ncid = file->controller->ext_ncid;
tid = cache->var->type_info->hdr.id;
if(tid == NC_STRING && !entry->isfixedstring) {
/* Convert from char* to char[strlen] format */
int maxstrlen = NCZ_get_maxstrlen((NC_OBJ*)cache->var);
assert(maxstrlen > 0);
if((strchunk = malloc(cache->chunkcount*maxstrlen))==NULL) {stat = NC_ENOMEM; goto done;}
/* copy char* to char[] format */
if((stat = NCZ_char2fixed((const char**)entry->data,strchunk,cache->chunkcount,maxstrlen))) goto done;
/* Reclaim the old chunk */
if((stat = nc_reclaim_data_all(ncid,tid,entry->data,cache->chunkcount))) goto done;
entry->data = NULL;
entry->data = strchunk; strchunk = NULL;
entry->size = cache->chunkcount * maxstrlen;
entry->isfixedstring = 1;
}
#ifdef ENABLE_NCZARR_FILTERS
/* Make sure the entry is in filtered state */
if(!entry->isfiltered) {
@ -636,6 +670,7 @@ put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
default: goto done;
}
done:
nullfree(strchunk);
nullfree(path);
return ZUNTRACE(stat);
}
@ -657,9 +692,12 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
NCZMAP* map = NULL;
NC_FILE_INFO_T* file = NULL;
NCZ_FILE_INFO_T* zfile = NULL;
NC_TYPE_INFO_T* xtype = NULL;
char** strchunk = NULL;
size64_t size;
int empty = 0;
char* path = NULL;
int tid;
ZTRACE(5,"cache.var=%s entry.key=%s sep=%d",cache->var->hdr.name,entry->key,cache->dimension_separator);
@ -670,22 +708,24 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
map = zfile->map;
assert(map);
/* Collect some info */
xtype = cache->var->type_info;
tid = xtype->hdr.id;
/* get size of the "raw" data on "disk" */
path = NCZ_chunkpath(entry->key);
stat = nczmap_len(map,path,&size);
nullfree(path); path = NULL;
switch(stat) {
case NC_NOERR: break;
case NC_NOERR: entry->size = size; break;
case NC_EEMPTY: empty = 1; stat = NC_NOERR; break;
default: goto done;
}
if(!empty) {
/* Make sure we have a place to read it */
entry->size = size;
entry->isfiltered = FILTERED(cache); /* Is the data being read filtered? */
if((entry->data = (void*)malloc(entry->size)) == NULL)
{stat = NC_ENOMEM; goto done;}
if((entry->data = (void*)calloc(1,entry->size)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* Read the raw data */
path = NCZ_chunkpath(entry->key);
stat = nczmap_read(map,path,0,entry->size,(char*)entry->data);
@ -695,27 +735,32 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
case NC_EEMPTY: empty = 1; stat = NC_NOERR;break;
default: goto done;
}
entry->isfiltered = FILTERED(cache); /* Is the data being read filtered? */
if(tid == NC_STRING)
entry->isfixedstring = 1; /* fill cache is in char[maxstrlen] format */
}
if(empty) {
/* fake the chunk */
entry->modified = (file->no_write?0:1);
entry->size = cache->chunksize;
if((entry->data = (void*)malloc(entry->size)) == NULL)
{stat = NC_ENOMEM; goto done;}
entry->data = NULL;
entry->isfixedstring = 0;
entry->isfiltered = 0;
/* apply fill value */
if(cache->fillchunk == NULL)
{if((stat = NCZ_ensure_fill_chunk(cache))) goto done;}
memcpy(entry->data,cache->fillchunk,entry->size);
entry->isfiltered = 0;
if((entry->data = calloc(1,entry->size))==NULL) {stat = NC_ENOMEM; goto done;}
if((stat = NCZ_copy_data(file,xtype,cache->fillchunk,cache->chunkcount,!ZCLEAR,entry->data))) goto done;
stat = NC_NOERR;
}
#ifdef ENABLE_NCZARR_FILTERS
/* Make sure the entry is in unfiltered state */
if(entry->isfiltered) {
if(!empty && entry->isfiltered) {
NC_VAR_INFO_T* var = cache->var;
void* unfiltered = NULL; /* pointer to the unfiltered data */
void* filtered = NULL; /* pointer to the filtered data */
size_t unflen; /* length of unfiltered data */
assert(tid != NC_STRING || entry->isfixedstring);
/* Get the filter chain to apply */
NClist* filterchain = (NClist*)var->filters;
if(nclistlength(filterchain) == 0) {stat = NC_EFILTER; goto done;}
@ -730,7 +775,24 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
}
#endif
if(tid == NC_STRING && entry->isfixedstring) {
/* Convert from char[strlen] to char* format */
int maxstrlen = NCZ_get_maxstrlen((NC_OBJ*)cache->var);
assert(maxstrlen > 0);
/* copy char[] to char* format */
if((strchunk = (char**)malloc(sizeof(char*)*cache->chunkcount))==NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = NCZ_fixed2char(entry->data,strchunk,cache->chunkcount,maxstrlen))) goto done;
/* Reclaim the old chunk */
nullfree(entry->data);
entry->data = NULL;
entry->data = strchunk; strchunk = NULL;
entry->size = cache->chunkcount * sizeof(char*);
entry->isfixedstring = 0;
}
done:
nullfree(strchunk);
nullfree(path);
return ZUNTRACE(stat);
}

5
libsrc/posixio.c
View File

@ -15,6 +15,7 @@
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <stdint.h>
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
@ -120,7 +121,7 @@ static off_t nc_get_filelen(const int fd) {
off_t flen;
#ifdef HAVE_FILE_LENGTH_I64
__int64 file_len = 0;
int64_t file_len = 0;
if ((file_len = _filelengthi64(fd)) < 0) {
return file_len;
}
@ -1829,7 +1830,7 @@ ncio_px_filesize(ncio *nciop, off_t *filesizep)
Use _filelengthi64 isntead. */
#ifdef HAVE_FILE_LENGTH_I64
__int64 file_len = 0;
int64_t file_len = 0;
if( (file_len = _filelengthi64(nciop->fd)) < 0) {
return errno;
}

20
libsrc4/nc4internal.c
View File

@ -35,22 +35,24 @@
*/
/** @internal List of reserved attributes.
WARNING: This list must be in sorted order for binary search. */
WARNING: This list must be in (strcmp) sorted order for binary search. */
static const NC_reservedatt NC_reserved[] = {
{NC_ATT_CLASS, READONLYFLAG|HIDDENATTRFLAG}, /*CLASS*/
{NC_ATT_DIMENSION_LIST, READONLYFLAG|HIDDENATTRFLAG}, /*DIMENSION_LIST*/
{NC_ATT_NAME, READONLYFLAG|HIDDENATTRFLAG}, /*NAME*/
{NC_ATT_REFERENCE_LIST, READONLYFLAG|HIDDENATTRFLAG}, /*REFERENCE_LIST*/
{NC_XARRAY_DIMS, READONLYFLAG|HIDDENATTRFLAG}, /*_ARRAY_DIMENSIONS*/
{NC_ATT_CODECS, VARFLAG|READONLYFLAG|NAMEONLYFLAG|HIDDENATTRFLAG}, /*_Codecs*/
{NC_XARRAY_DIMS, READONLYFLAG|NAMEONLYFLAG|HIDDENATTRFLAG}, /*_ARRAY_DIMENSIONS*/
{NC_ATT_CODECS, VARFLAG|READONLYFLAG|NAMEONLYFLAG}, /*_Codecs*/
{NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/
{ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/
{NCPROPS, READONLYFLAG|NAMEONLYFLAG|MATERIALIZEDFLAG}, /*_NCProperties*/
{NC_NCZARR_ATTR, READONLYFLAG|HIDDENATTRFLAG}, /*_NCZARR_ATTR*/
{NC_ATT_COORDINATES, READONLYFLAG|HIDDENATTRFLAG|MATERIALIZEDFLAG}, /*_Netcdf4Coordinates*/
{NC_ATT_DIMID_NAME, READONLYFLAG|HIDDENATTRFLAG|MATERIALIZEDFLAG}, /*_Netcdf4Dimid*/
{NCPROPS,READONLYFLAG|NAMEONLYFLAG|HIDDENATTRFLAG}, /*_NCProperties*/
{NC_NCZARR_ATTR_UC, READONLYFLAG|HIDDENATTRFLAG}, /*_NCZARR_ATTR */
{NC_ATT_COORDINATES, READONLYFLAG|HIDDENATTRFLAG}, /*_Netcdf4Coordinates*/
{NC_ATT_DIMID_NAME, READONLYFLAG|HIDDENATTRFLAG}, /*_Netcdf4Dimid*/
{SUPERBLOCKATT, READONLYFLAG|NAMEONLYFLAG}, /*_SuperblockVersion*/
{NC_ATT_NC3_STRICT_NAME, READONLYFLAG|MATERIALIZEDFLAG}, /*_nc3_strict*/
{NC_ATT_NC3_STRICT_NAME, READONLYFLAG}, /*_nc3_strict*/
{NC_ATT_NC3_STRICT_NAME, READONLYFLAG}, /*_nc3_strict*/
{NC_NCZARR_ATTR, READONLYFLAG|HIDDENATTRFLAG}, /*_nczarr_attr */
};
#define NRESERVED (sizeof(NC_reserved) / sizeof(NC_reservedatt)) /*|NC_reservedatt|*/
@ -1592,7 +1594,7 @@ nc4_rec_grp_del_att_data(NC_GRP_INFO_T *grp)
LOG((3, "%s: grp->name %s", __func__, grp->hdr.name));
/* Recursively call this function for each child, if any, stopping
* if there is an error. */
* if there is an error. */
for (i = 0; i < ncindexsize(grp->children); i++)
if ((retval = nc4_rec_grp_del_att_data((NC_GRP_INFO_T *)ncindexith(grp->children, i))))
return retval;

2
nc_test/CMakeLists.txt
View File

@ -107,8 +107,10 @@ IF(BUILD_UTILITIES)
ENDIF()
IF(ENABLE_BYTERANGE)
IF(ENABLE_EXTERNAL_SERVER_TESTS)
build_bin_test_no_prefix(tst_byterange)
add_sh_test(nc_test test_byterange)
ENDIF()
ENDIF()
IF(BUILD_MMAP)

9
nc_test/Makefile.am
View File

@ -75,26 +75,27 @@ endif
TESTS = $(TESTPROGRAMS)
if ENABLE_BYTERANGE
if BUILD_UTILITIES
if ENABLE_BYTERANGE
if ENABLE_EXTERNAL_SERVER_TESTS
tst_byterange_SOURCES = tst_byterange.c
check_PROGRAMS += tst_byterange
TESTS += test_byterange.sh
endif
endif
if BUILD_UTILITIES
TESTS += run_diskless.sh run_diskless5.sh run_inmemory.sh
if LARGE_FILE_TESTS
if ! ENABLE_PARALLEL
TESTS += run_diskless2.sh
endif
endif
if BUILD_MMAP
TESTS += run_mmap.sh
run_mmap.log: run_diskless.log
endif
endif
# If pnetcdf is enabled, these tests are run by a test
# script. Otherwise, the are run by automake in the usual way.
@ -104,6 +105,8 @@ else
TESTS += t_nc tst_atts3 tst_nofill nc_test tst_small
endif
endif # BUILD_UTILITIES
# The .c files that are generated with m4 are already distributed, but
# we also include the original m4 files, plus test scripts data.
EXTRA_DIST = test_get.m4 test_put.m4 run_diskless.sh run_diskless2.sh \

19
nc_test/test_byterange.sh
View File

@ -5,11 +5,14 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
set -e
# Note: thredds-test is currently down and the URLs need to be replaced
# Test Urls
if test "x$FEATURE_THREDDSTEST" = x1 ; then
URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes&aws.profile=none"
#URL3a="https://remotetest.unidata.ucar.edu/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes&aws.profile=none"
URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none"
URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes&aws.profile=none"
fi
URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none"
URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none"
# Requires auth
URL3b="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes&aws.profile=unidata"
@ -81,12 +84,16 @@ testsetup https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data
echo "*** Testing reading NetCDF-3 file with http"
echo "***Test remote classic file"
testbytes nc3 classic "$URL3"
if test "x$FEATURE_THREDDSTEST" = x1 ; then
echo "***Test remote classic file"
testbytes nc3 classic "$URL3"
fi
if test "x$FEATURE_HDF5" = xyes ; then
echo "***Test remote netcdf-4 file: non-s3"
testbytes nc4b netCDF-4 "$URL4b"
if test "x$FEATURE_THREDDSTEST" = x1 ; then
echo "***Test remote netcdf-4 file: non-s3"
testbytes nc4b netCDF-4 "$URL4b"
fi
fi
if test "x$FEATURE_S3TESTS" = xyes && test "x$FEATURE_HDF5" = xyes ; then
echo "***Test remote netdf-4 file: s3"

12
ncdap_test/CMakeLists.txt
View File

@ -52,14 +52,14 @@ IF(ENABLE_TESTS)
# not yet add_sh_test(ncdap tst_hyrax)
add_sh_test(ncdap tst_fillmismatch)
IF(ENABLE_DAP_LONG_TESTS)
add_sh_test(ncdap tst_longremote3)
add_bin_test(ncdap test_manyurls)
SET_TESTS_PROPERTIES(ncdap_tst_longremote3 ncdap_test_manyurls PROPERTIES RUN_SERIAL TRUE)
add_sh_test(ncdap tst_longremote3)
SET_TESTS_PROPERTIES(ncdap_tst_longremote3 PROPERTIES RUN_SERIAL TRUE)
ENDIF(ENABLE_DAP_LONG_TESTS)
ENDIF(BUILD_UTILITIES)
IF(ENABLE_EXTERNAL_SERVER_TESTS)
add_bin_test(ncdap test_manyurls)
SET_TESTS_PROPERTIES(ncdap_test_manyurls PROPERTIES RUN_SERIAL TRUE)
ENDIF()
IF(ENABLE_DAP_AUTH_TESTS)
##obsolete add_bin_test(ncdap t_auth)

4
ncdap_test/Makefile.am
View File

@ -52,10 +52,12 @@ TESTS += test_partvar
if ENABLE_DAP_LONG_TESTS
TESTS += tst_longremote3.sh
tst_longremote3.log: tst_remote3.log
if ENABLE_EXTERNAL_SERVER_TESTS
test_manyurls_SOURCES = test_manyurls.c manyurls.h
check_PROGRAMS += test_manyurls
TESTS += test_manyurls
test_manyurls.log: tst_longremote3.log
TESTS += test_manyurls
endif
endif
test_partvar_SOURCES = test_partvar.c

2
ncgen/Makefile.am
View File

@ -61,7 +61,7 @@ CLEANFILES = c0.nc c0_64.nc c0_4.nc c0_4c.nc ref_camrun.c \
makeparser::
rm -f ncgenl.c lex.ncg.c
flex -L -Pncg -8 ncgen.l
flex -Pncg -8 ncgen.l
sed -e s/lex.ncg.c/ncgenl.c/g <lex.ncg.c >ncgenl.c
bison -pncg -t -d ncgen.y
rm -f ncgeny.c ncgeny.h

4
ncgen/bindata.c
View File

@ -422,6 +422,7 @@ done:
return stat;
}
#if 0
/**
Internal equivalent of ncaux_reclaim_data.
*/
@ -591,5 +592,8 @@ done:
}
#endif /*USE_NETCDF4*/
#endif /*0*/
#endif /*ENABLE_BINARY*/

8
ncgen/ncgen.l
View File

@ -481,7 +481,8 @@ done: return 0;
if(tag == NC_NAT) {
sprintf(errstr,"Illegal integer suffix: %s",stag);
yyerror(errstr);
goto done;
//goto done;
return 0;
}
yytext[slen - strlen(stag)] = '\0';
if(xlen > 16) { /* truncate hi order digits */
@ -527,7 +528,7 @@ done: return 0;
}
return lexdebug(FLOAT_CONST);
}
\'[^\\]\' {
\'[^\\]\' {
(void) sscanf((char*)&yytext[1],"%c",&byte_val);
return lexdebug(BYTE_CONST);
}
@ -559,8 +560,9 @@ done: return 0;
case 'r': byte_val = '\r'; break;
case 't': byte_val = '\t'; break;
case 'v': byte_val = '\v'; break;
case '0' : byte_val = '\0'; break;
case '\\': byte_val = '\\'; break;
case '?': byte_val = '\177'; break;
case '?': byte_val = '\177'; break;
case '\'': byte_val = '\''; break;
default: byte_val = (char)yytext[2];
}

75
ncgen/ncgenl.c
View File

@ -1,4 +1,6 @@
#line 3 "ncgenl.c"
#define YY_INT_ALIGNED short int
/* A lexical scanner generated by flex */
@ -1582,6 +1584,8 @@ int yy_flex_debug = 0;
#define YY_MORE_ADJ 0
#define YY_RESTORE_YY_MORE_OFFSET
char *yytext;
#line 1 "ncgen.l"
#line 2 "ncgen.l"
/*********************************************************************
* Copyright 1993, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
@ -1727,6 +1731,8 @@ struct Specialtoken specials[] = {
{NULL,0} /* null terminate */
};
#line 1735 "ncgenl.c"
/* The most correct (validating) version of UTF8 character set
(Taken from: http://www.w3.org/2005/03/23-lex-U)
@ -1768,6 +1774,7 @@ ID ([A-Za-z_]|{UTF8})([A-Z.@#\[\]a-z_0-9+-]|{UTF8})*
/* Note: this definition of string will work for utf8 as well,
although it is a very relaxed definition
*/
#line 1778 "ncgenl.c"
#define INITIAL 0
#define ST_C_COMMENT 1
@ -1986,6 +1993,9 @@ YY_DECL
}
{
#line 226 "ncgen.l"
#line 1999 "ncgenl.c"
while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */
{
@ -2044,12 +2054,14 @@ do_action: /* This label is used only to access EOF actions. */
case 1:
YY_RULE_SETUP
#line 227 "ncgen.l"
{ /* whitespace */
break;
}
YY_BREAK
case 2:
YY_RULE_SETUP
#line 231 "ncgen.l"
{ /* comment */
break;
}
@ -2057,6 +2069,7 @@ YY_RULE_SETUP
case 3:
/* rule 3 can match eol */
YY_RULE_SETUP
#line 235 "ncgen.l"
{int len; char* s = NULL;
/* In netcdf4, this will be used in a variety
of places, so only remove escapes */
@ -2080,6 +2093,7 @@ yytext[MAXTRST-1] = '\0';
YY_BREAK
case 4:
YY_RULE_SETUP
#line 256 "ncgen.l"
{ /* drop leading 0x; pad to even number of chars */
char* p = yytext+2;
int len = yyleng - 2;
@ -2094,115 +2108,143 @@ YY_RULE_SETUP
YY_BREAK
case 5:
YY_RULE_SETUP
#line 268 "ncgen.l"
{return lexdebug(COMPOUND);}
YY_BREAK
case 6:
YY_RULE_SETUP
#line 269 "ncgen.l"
{return lexdebug(ENUM);}
YY_BREAK
case 7:
YY_RULE_SETUP
#line 270 "ncgen.l"
{return lexdebug(OPAQUE_);}
YY_BREAK
case 8:
YY_RULE_SETUP
#line 272 "ncgen.l"
{return lexdebug(FLOAT_K);}
YY_BREAK
case 9:
YY_RULE_SETUP
#line 273 "ncgen.l"
{return lexdebug(DOUBLE_K);}
YY_BREAK
case 10:
YY_RULE_SETUP
#line 274 "ncgen.l"
{return lexdebug(CHAR_K);}
YY_BREAK
case 11:
YY_RULE_SETUP
#line 275 "ncgen.l"
{return lexdebug(BYTE_K);}
YY_BREAK
case 12:
YY_RULE_SETUP
#line 276 "ncgen.l"
{return lexdebug(SHORT_K);}
YY_BREAK
case 13:
YY_RULE_SETUP
#line 277 "ncgen.l"
{return lexdebug(INT_K);}
YY_BREAK
case 14:
YY_RULE_SETUP
#line 278 "ncgen.l"
{return lexdebug(identcheck(UBYTE_K));}
YY_BREAK
case 15:
YY_RULE_SETUP
#line 279 "ncgen.l"
{return lexdebug(identcheck(USHORT_K));}
YY_BREAK
case 16:
YY_RULE_SETUP
#line 280 "ncgen.l"
{return lexdebug(identcheck(UINT_K));}
YY_BREAK
case 17:
YY_RULE_SETUP
#line 281 "ncgen.l"
{return lexdebug(identcheck(INT64_K));}
YY_BREAK
case 18:
YY_RULE_SETUP
#line 282 "ncgen.l"
{return lexdebug(identcheck(UINT64_K));}
YY_BREAK
case 19:
YY_RULE_SETUP
#line 283 "ncgen.l"
{return lexdebug(identcheck(STRING_K));}
YY_BREAK
case 20:
YY_RULE_SETUP
#line 285 "ncgen.l"
{return lexdebug(FLOAT_K);}
YY_BREAK
case 21:
YY_RULE_SETUP
#line 286 "ncgen.l"
{return lexdebug(INT_K);}
YY_BREAK
case 22:
YY_RULE_SETUP
#line 287 "ncgen.l"
{return lexdebug(INT_K);}
YY_BREAK
case 23:
YY_RULE_SETUP
#line 288 "ncgen.l"
{return lexdebug(identcheck(UINT_K));}
YY_BREAK
case 24:
YY_RULE_SETUP
#line 289 "ncgen.l"
{return lexdebug(identcheck(UINT_K));}
YY_BREAK
case 25:
YY_RULE_SETUP
#line 292 "ncgen.l"
{int32_val = -1;
return lexdebug(NC_UNLIMITED_K);}
YY_BREAK
case 26:
YY_RULE_SETUP
#line 295 "ncgen.l"
{return lexdebug(TYPES);}
YY_BREAK
case 27:
YY_RULE_SETUP
#line 296 "ncgen.l"
{return lexdebug(DIMENSIONS);}
YY_BREAK
case 28:
YY_RULE_SETUP
#line 297 "ncgen.l"
{return lexdebug(VARIABLES);}
YY_BREAK
case 29:
YY_RULE_SETUP
#line 298 "ncgen.l"
{return lexdebug(DATA);}
YY_BREAK
case 30:
YY_RULE_SETUP
#line 299 "ncgen.l"
{return lexdebug(GROUP);}
YY_BREAK
case 31:
YY_RULE_SETUP
#line 301 "ncgen.l"
{BEGIN(TEXT);return lexdebug(NETCDF);}
YY_BREAK
case 32:
YY_RULE_SETUP
#line 303 "ncgen.l"
{ /* missing value (pre-2.4 backward compatibility) */
if (yytext[0] == '-') {
double_val = -INFINITY;
@ -2215,6 +2257,7 @@ YY_RULE_SETUP
YY_BREAK
case 33:
YY_RULE_SETUP
#line 312 "ncgen.l"
{ /* missing value (pre-2.4 backward compatibility) */
double_val = NAN;
specialconstants = 1;
@ -2223,6 +2266,7 @@ YY_RULE_SETUP
YY_BREAK
case 34:
YY_RULE_SETUP
#line 318 "ncgen.l"
{/* missing value (pre-2.4 backward compatibility)*/
if (yytext[0] == '-') {
float_val = -INFINITYF;
@ -2235,6 +2279,7 @@ YY_RULE_SETUP
YY_BREAK
case 35:
YY_RULE_SETUP
#line 327 "ncgen.l"
{ /* missing value (pre-2.4 backward compatibility) */
float_val = NANF;
specialconstants = 1;
@ -2243,6 +2288,7 @@ YY_RULE_SETUP
YY_BREAK
case 36:
YY_RULE_SETUP
#line 333 "ncgen.l"
{
#ifdef USE_NETCDF4
if(l_flag == L_C || l_flag == L_BINARY)
@ -2255,6 +2301,7 @@ YY_RULE_SETUP
YY_BREAK
case 37:
YY_RULE_SETUP
#line 343 "ncgen.l"
{
bbClear(lextext);
bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */
@ -2265,6 +2312,7 @@ YY_RULE_SETUP
YY_BREAK
case 38:
YY_RULE_SETUP
#line 352 "ncgen.l"
{struct Specialtoken* st;
bbClear(lextext);
bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */
@ -2278,6 +2326,7 @@ YY_RULE_SETUP
case 39:
/* rule 39 can match eol */
YY_RULE_SETUP
#line 362 "ncgen.l"
{
int c;
char* p; char* q;
@ -2297,6 +2346,7 @@ YY_RULE_SETUP
YY_BREAK
case 40:
YY_RULE_SETUP
#line 379 "ncgen.l"
{ char* id = NULL; int len;
len = strlen(yytext);
len = unescape(yytext,len,ISIDENT,&id);
@ -2311,6 +2361,7 @@ YY_RULE_SETUP
YY_BREAK
case 41:
YY_RULE_SETUP
#line 391 "ncgen.l"
{
/*
We need to try to see what size of integer ((u)int).
@ -2391,6 +2442,7 @@ done: return 0;
YY_BREAK
case 42:
YY_RULE_SETUP
#line 469 "ncgen.l"
{
int c;
int token = 0;
@ -2406,7 +2458,8 @@ YY_RULE_SETUP
if(tag == NC_NAT) {
sprintf(errstr,"Illegal integer suffix: %s",stag);
yyerror(errstr);
goto done;
//goto done;
return 0;
}
yytext[slen - strlen(stag)] = '\0';
if(xlen > 16) { /* truncate hi order digits */
@ -2441,6 +2494,7 @@ YY_RULE_SETUP
YY_BREAK
case 43:
YY_RULE_SETUP
#line 517 "ncgen.l"
{
if (sscanf((char*)yytext, "%le", &double_val) != 1) {
sprintf(errstr,"bad long or double constant: %s",(char*)yytext);
@ -2451,6 +2505,7 @@ YY_RULE_SETUP
YY_BREAK
case 44:
YY_RULE_SETUP
#line 524 "ncgen.l"
{
if (sscanf((char*)yytext, "%e", &float_val) != 1) {
sprintf(errstr,"bad float constant: %s",(char*)yytext);
@ -2462,6 +2517,7 @@ YY_RULE_SETUP
case 45:
/* rule 45 can match eol */
YY_RULE_SETUP
#line 531 "ncgen.l"
{
(void) sscanf((char*)&yytext[1],"%c",&byte_val);
return lexdebug(BYTE_CONST);
@ -2469,6 +2525,7 @@ YY_RULE_SETUP
YY_BREAK
case 46:
YY_RULE_SETUP
#line 535 "ncgen.l"
{
int oct = unescapeoct(&yytext[2]);
if(oct < 0) {
@ -2481,6 +2538,7 @@ YY_RULE_SETUP
YY_BREAK
case 47:
YY_RULE_SETUP
#line 544 "ncgen.l"
{
int hex = unescapehex(&yytext[3]);
if(byte_val < 0) {
@ -2493,6 +2551,7 @@ YY_RULE_SETUP
YY_BREAK
case 48:
YY_RULE_SETUP
#line 553 "ncgen.l"
{
switch ((char)yytext[2]) {
case 'a': byte_val = '\007'; break; /* not everyone under-
@ -2503,8 +2562,9 @@ YY_RULE_SETUP
case 'r': byte_val = '\r'; break;
case 't': byte_val = '\t'; break;
case 'v': byte_val = '\v'; break;
case '0' : byte_val = '\0'; break;
case '\\': byte_val = '\\'; break;
case '?': byte_val = '\177'; break;
case '?': byte_val = '\177'; break;
case '\'': byte_val = '\''; break;
default: byte_val = (char)yytext[2];
}
@ -2514,6 +2574,7 @@ YY_RULE_SETUP
case 49:
/* rule 49 can match eol */
YY_RULE_SETUP
#line 572 "ncgen.l"
{
lineno++ ;
break;
@ -2521,6 +2582,7 @@ YY_RULE_SETUP
YY_BREAK
case 50:
YY_RULE_SETUP
#line 577 "ncgen.l"
{/*initial*/
BEGIN(ST_C_COMMENT);
break;
@ -2529,18 +2591,21 @@ YY_RULE_SETUP
case 51:
/* rule 51 can match eol */
YY_RULE_SETUP
#line 582 "ncgen.l"
{/* continuation */
break;
}
YY_BREAK
case 52:
YY_RULE_SETUP
#line 586 "ncgen.l"
{/* final */
BEGIN(INITIAL);
break;
}
YY_BREAK
case YY_STATE_EOF(ST_C_COMMENT):
#line 591 "ncgen.l"
{/* final, error */
fprintf(stderr,"unterminated /**/ comment");
BEGIN(INITIAL);
@ -2549,14 +2614,17 @@ case YY_STATE_EOF(ST_C_COMMENT):
YY_BREAK
case 53:
YY_RULE_SETUP
#line 597 "ncgen.l"
{/* Note: this next rule will not work for UTF8 characters */
return lexdebug(yytext[0]) ;
}
YY_BREAK
case 54:
YY_RULE_SETUP
#line 600 "ncgen.l"
ECHO;
YY_BREAK
#line 2628 "ncgenl.c"
case YY_STATE_EOF(INITIAL):
case YY_STATE_EOF(TEXT):
yyterminate();
@ -3562,6 +3630,8 @@ void yyfree (void * ptr )
#define YYTABLES_NAME "yytables"
#line 600 "ncgen.l"
static int
lexdebug(int token)
{
@ -3660,6 +3730,7 @@ parseULL(int radix, char* text, int* failp)
return uint64;
}
/**
Given the raw bits, the sign char, the tag, and hasU
fill in the appropriate *_val field

1
nczarr_test/CMakeLists.txt
View File

@ -116,6 +116,7 @@ IF(ENABLE_TESTS)
add_sh_test(nczarr_test run_misc)
add_sh_test(nczarr_test run_nczarr_fill)
add_sh_test(nczarr_test run_jsonconvention)
add_sh_test(nczarr_test run_strings)
BUILD_BIN_TEST(test_quantize ${TSTCOMMONSRC})
add_sh_test(nczarr_test run_quantize)

5
nczarr_test/Makefile.am
View File

@ -62,6 +62,7 @@ TESTS += run_interop.sh
TESTS += run_misc.sh
TESTS += run_nczarr_fill.sh
TESTS += run_jsonconvention.sh
TESTS += run_strings.sh
endif
@ -149,7 +150,9 @@ ref_xarray.cdl ref_purezarr.cdl ref_purezarr_base.cdl ref_nczarr2zarr.cdl \
ref_bzip2.cdl ref_filtered.cdl ref_multi.cdl \
ref_any.cdl ref_oldformat.cdl ref_oldformat.zip ref_newformatpure.cdl \
ref_groups.h5 ref_byte.zarr.zip ref_byte_fill_value_null.zarr.zip \
ref_groups_regular.cdl ref_byte.cdl ref_byte_fill_value_null.cdl ref_jsonconvention.cdl
ref_groups_regular.cdl ref_byte.cdl ref_byte_fill_value_null.cdl \
ref_jsonconvention.cdl ref_jsonconvention.zmap \
ref_string.cdl
# Interoperability files
EXTRA_DIST += ref_power_901_constants_orig.zip ref_power_901_constants.cdl ref_quotes_orig.zip ref_quotes.cdl

10
nczarr_test/ref_jsonconvention.cdl
View File

@ -3,10 +3,16 @@ dimensions:
d1 = 1 ;
variables:
int v(d1) ;
v:varconvention = "{\n\"key1\": [1,2,3], \"key2\": {\"key3\": \"abc\"}}" ;
v:varjson1 = "{\"key1\": [1,2,3], \"key2\": {\"key3\": \"abc\"}}" ;
v:varjson2 = "[[1.0,0.0,0.0],[0.0,1.0,0.0],[0.0,0.0,1.0]]" ;
v:varvec1 = "1.0, 0.0, 0.0" ;
v:varvec2 = "[0.,0.,1.]" ;
// global attributes:
:grpconvention = "{\"key1\": [1,2,3], \n\"key2\": {\"key3\": \"abc\"}}" ;
:globalfloat = 1. ;
:globalfloatvec = 1., 2. ;
:globalchar = "abc" ;
:globalillegal = "[ [ 1.0, 0.0, 0.0 ], [ 0.0, 1.0, 0.0 ], [ 0.0, 0.0, 1.0 " ;
data:
v = _ ;

5
nczarr_test/ref_jsonconvention.zmap Normal file
View File

@ -0,0 +1,5 @@
[0] /.zattrs : (354) |{"globalfloat": 1, "globalfloatvec": [1,2], "globalchar": "abc", "globalillegal": "[ [ 1.0, 0.0, 0.0 ], [ 0.0, 1.0, 0.0 ], [ 0.0, 0.0, 1.0 ", "_NCProperties": "version=2,netcdf=4.9.1-development,nczarr=2.0.0", "_nczarr_attr": {"types": {"globalfloat": "<f8", "globalfloatvec": "<f8", "globalchar": "|S1", "globalillegal": "|S1", "_NCProperties": "|S1"}}}|
[1] /.zgroup : (129) |{"zarr_format": 2, "_nczarr_superblock": {"version": "2.0.0"}, "_nczarr_group": {"dims": {"d1": 1}, "vars": ["v"], "groups": []}}|
[3] /v/.zarray : (202) |{"zarr_format": 2, "shape": [1], "dtype": "<i4", "chunks": [1], "fill_value": -2147483647, "order": "C", "compressor": null, "filters": null, "_nczarr_array": {"dimrefs": ["/d1"], "storage": "chunked"}}|
[4] /v/.zattrs : (296) |{"varjson1": {"key1": [1,2,3], "key2": {"key3": "abc"}}, "varjson2": [[1.0,0.0,0.0],[0.0,1.0,0.0],[0.0,0.0,1.0]], "varvec1": "1.0, 0.0, 0.0", "varvec2": [0.,0.,1.], "_ARRAY_DIMENSIONS": ["d1"], "_nczarr_attr": {"types": {"varjson1": "|S1", "varjson2": "|S1", "varvec1": "|S1", "varvec2": "|S1"}}}|
[5] /v/0 : (4) (ubyte) |...|

BIN
nczarr_test/ref_oldformat.zip
View File

Binary file not shown.

2
nczarr_test/ref_quotes.cdl
View File

@ -5,7 +5,7 @@ dimensions:
lon = 30 ;
variables:
float fractional_snow_cover(time, lat, lon) ;
fractional_snow_cover:ID = 68b ;
fractional_snow_cover:ID = 68 ;
fractional_snow_cover:esa_cci_path = NaN ;
fractional_snow_cover:long_name = "Surface Fraction Covered by Snow" ;
fractional_snow_cover:orig_attrs = "{\'comment\': \'Grid cell fractional snow cover based on the Globsnow CCI product.\', \'long_name\': \'Surface fraction covered by snow.\', \'project_name\': \'GlobSnow\', \'references\': \'Luojus, Kari, et al. \"ESA DUE Globsnow-Global Snow Database for Climate Research.\" ESA Special Publication. Vol. 686. 2010.\', \'source_name\': \'MFSC\', \'standard_name\': \'surface_snow_area_fraction\', \'units\': \'percent\', \'url\': \'http://www.globsnow.info/\'}" ;

17
nczarr_test/ref_string.cdl Normal file
View File

@ -0,0 +1,17 @@
netcdf ref_string {
dimensions:
d = 2 ;
variables:
char c(d);
string v(d) ;
// global attributes:
string :stringattr = "abc", "def" ;
:charattr = "ghi", "jkl" ;
:_nczarr_default_maxstrlen = 6 ;
data:
c = "a", "b" ;
v = "uvw", "xyz" ;
}

BIN
nczarr_test/ref_zarr_test_data.cdl.gz
View File

Binary file not shown.

3
nczarr_test/run_jsonconvention.sh
View File

@ -17,11 +17,14 @@ echo "*** Test: write then read using json convention"
fileargs tmp_jsonconvention "mode=nczarr,$zext"
deletemap $zext $file
${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_jsonconvention.cdl
# Clean up extraneous changes wrt _NCProperties
${ZMD} -h $fileurl | sed -e 's/,key1=value1|key2=value2//' -e '/"_NCProperties"/ s/(378)/(354)/' > tmp_jsonconvention_${zext}.txt
${NCDUMP} $fileurl > tmp_jsonconvention_${zext}.cdl
# remove '\n' from ref file before comparing
rm -f tmp_jsonconvention.cdl
sed -e 's|\\n||g' < ${srcdir}/ref_jsonconvention.cdl > tmp_jsonconvention.cdl
diff -b tmp_jsonconvention.cdl tmp_jsonconvention_${zext}.cdl
diff -b ${srcdir}/ref_jsonconvention.zmap tmp_jsonconvention_${zext}.txt
}
testcase file

1
nczarr_test/run_newformat.sh
View File

@ -3,7 +3,6 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -x
. "$srcdir/test_nczarr.sh"
set -e

88
nczarr_test/run_strings.sh Executable file
View File

@ -0,0 +1,88 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. "$srcdir/test_nczarr.sh"
# This shell script tests support for the NC_STRING type
set -e
# Cvt stringattr to single char string
stringfixsa() {
rm -f $2
sed -e '/:stringattr/ s|string :|:|' -e '/:stringattr/ s|", "||g' < $1 > $2
}
# Cvt stringattr to JSON format string
stringfixjsa() {
rm -f $2
sed -e '/:stringattr/ s|string :|:|' -e '/:stringattr/ s|"|\\"|g' -e '/:stringattr/ s|= \(.*\);|= "\[\1\]" ;|' < $1 > $2
}
# Cvt v var data to single char string
stringfixv() {
rm -f $2
sed -e '/v = / s|", "||g' < $1 > $2
}
# Cvt charattr to single char string
stringfixca() {
rm -f $2
sed -e '/:charattr/ s|", "||g' <$1 > $2
}
# Cvt c var data to single char string
stringfixc() {
rm -f $2
sed -e '/c = / s|", "||g' < $1 > $2
}
testcase() {
zext=$1
echo "*** Test: nczarr string write then read; format=$zext"
# Get pure zarr args
fileargs tmp_string_zarr "mode=zarr,$zext"
zarrurl="$fileurl"
zarrfile="$file"
# Get nczarr args
fileargs tmp_string_nczarr "mode=nczarr,$zext"
nczarrurl="$fileurl"
nczarrfile="$file"
# setup
deletemap $zext $zarrfile
deletemap $zext $nczarrfile
# Create alternate ref files
echo "*** create pure zarr file"
${NCGEN} -4 -b -o "$zarrurl" $srcdir/ref_string.cdl
echo "*** create nczarr file"
${NCGEN} -4 -b -o "$nczarrurl" $srcdir/ref_string.cdl
echo "*** read purezarr"
${NCDUMP} -n ref_string $zarrurl > tmp_string_zarr_${zext}.cdl
${ZMD} -h $zarrurl > tmp_string_zarr_${zext}.txt
echo "*** read nczarr"
${NCDUMP} -n ref_string $nczarrurl > tmp_string_nczarr_${zext}.cdl
${ZMD} -h $nczarrurl > tmp_string_nczarr_${zext}.txt
echo "*** convert for nczarr comparison"
stringfixca ${srcdir}/ref_string.cdl tmp_ref_string_ca.cdl
stringfixc tmp_ref_string_ca.cdl tmp_ref_string_cac.cdl
echo "*** convert for zarr comparison"
stringfixjsa tmp_ref_string_cac.cdl tmp_ref_string_cacsa.cdl
echo "*** verify"
diff -bw tmp_ref_string_cac.cdl tmp_string_nczarr_${zext}.cdl
diff -bw tmp_ref_string_cacsa.cdl tmp_string_zarr_${zext}.cdl
}
testcase file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi
exit 0

90
nczarr_test/zmapio.c
View File

@ -55,18 +55,21 @@ static struct Type {
const char* typename;
nc_type nctype;
int typesize;
const char format[16];
} types[] = {
{"ubyte",NC_UBYTE,1},
{"byte",NC_BYTE,1},
{"ushort",NC_USHORT,2},
{"short",NC_SHORT,2},
{"uint",NC_UINT,4},
{"int",NC_INT,4},
{"uint64",NC_UINT64,8},
{"int64",NC_INT64,8},
{"float",NC_FLOAT,4},
{"double",NC_DOUBLE,8},
{NULL,NC_NAT,0}
{"ubyte",NC_UBYTE,1,"%u"},
{"byte",NC_BYTE,1,"%d"},
{"ushort",NC_USHORT,2,"%u"},
{"short",NC_SHORT,2,"%d"},
{"uint",NC_UINT,4,"%u"},
{"int",NC_INT,4,"%d"},
{"uint64",NC_UINT64,8,"%llu"},
{"int64",NC_INT64,8,"%lld"},
{"float",NC_FLOAT,4,"%f"},
{"double",NC_DOUBLE,8,"%lf"},
{"char",NC_CHAR,1,"'%c'"},
{"string",NC_STRING,sizeof(char*),"%*s"},
{NULL,NC_NAT,0,""}
};
/* Command line options */
@ -78,8 +81,10 @@ struct Dumpptions {
NCZM_IMPL impl;
char* rootpath;
const struct Type* nctype;
char format[16];
int xflags;
# define XNOZMETADATA 1
int strlen;
} dumpoptions;
/* Forward */
@ -120,9 +125,15 @@ decodeop(const char* name)
}
static const struct Type*
decodetype(const char* name)
decodetype(const char* name, int* strlenp)
{
struct Type* p = types;
if(strncmp(name,"string/",strlen("string/"))==0) {
*strlenp = atoi(name+strlen("string/"));
name = "string";
}
for(;p->typename != NULL;p++) {
if(strcasecmp(p->typename,name)==0) return p;
}
@ -138,9 +149,10 @@ main(int argc, char** argv)
nc_initialize();
/* Init options */
memset((void*)&dumpoptions,0,sizeof(dumpoptions));
while ((c = getopt(argc, argv, "dhvx:t:T:X:")) != EOF) {
while ((c = getopt(argc, argv, "dhvx:t:F:T:X:")) != EOF) {
switch(c) {
case 'd':
dumpoptions.debug = 1;
@ -148,17 +160,20 @@ main(int argc, char** argv)
case 'h':
dumpoptions.meta_only = 1;
break;
case 'v':
zmapusage();
goto done;
case 't':
dumpoptions.nctype = decodetype(optarg);
dumpoptions.nctype = decodetype(optarg,&dumpoptions.strlen);
if(dumpoptions.nctype == NULL) zmapusage();
break;
case 'x':
dumpoptions.mop = decodeop(optarg);
if(dumpoptions.mop == MOP_NONE) zmapusage();
break;
case 'v':
zmapusage();
goto done;
case 'F':
strcpy(dumpoptions.format,optarg);
break;
case 'T':
nctracelevel(atoi(optarg));
break;
@ -344,7 +359,7 @@ objdump(void)
len = ceildiv(len,dumpoptions.nctype->typesize);
}
printf("[%d] %s : (%llu)",depth,obj,len);
if(kind == OK_CHUNK)
if(kind == OK_CHUNK && dumpoptions.nctype->nctype != NC_STRING)
printf(" (%s)",dumpoptions.nctype->typename);
printf(" |");
switch(kind) {
@ -434,25 +449,40 @@ static char hex[16] = "0123456789abcdef";
static void
printcontent(size64_t len, const char* content, OBJKIND kind)
{
size64_t i;
size64_t i, count;
unsigned int c0,c1;
for(i=0;i<len;i++) {
const char* format = NULL;
int strlen = 1;
format = dumpoptions.nctype->format;
if(dumpoptions.format[0] != '\0')
format = dumpoptions.format;
if(dumpoptions.strlen > 0) {
strlen = dumpoptions.strlen;
count = ((len+strlen)-1)/strlen;
} else
count = len;
for(i=0;i<count;i++) {
/* If kind is chunk, then len is # of values, not # of bytes */
switch(kind) {
case OK_CHUNK:
if(i > 0) printf(", ");
switch(dumpoptions.nctype->nctype) {
case NC_BYTE: printf("%d",((char*)content)[i]); break;
case NC_SHORT: printf("%d",((short*)content)[i]); break;
case NC_INT: printf("%d",((int*)content)[i]); break;
case NC_INT64: printf("%lld",((long long*)content)[i]); break;
case NC_UBYTE: printf("%u",((unsigned char*)content)[i]); break;
case NC_USHORT: printf("%u",((unsigned short*)content)[i]); break;
case NC_UINT: printf("%u",((unsigned int*)content)[i]); break;
case NC_UINT64: printf("%llu",((unsigned long long*)content)[i]); break;
case NC_FLOAT: printf("%f",((float*)content)[i]); break;
case NC_DOUBLE: printf("%lf",((double*)content)[i]); break;
case NC_BYTE: printf(format,((char*)content)[i]); break;
case NC_SHORT: printf(format,((short*)content)[i]); break;
case NC_INT: printf(format,((int*)content)[i]); break;
case NC_INT64: printf(format,((long long*)content)[i]); break;
case NC_UBYTE: printf(format,((unsigned char*)content)[i]); break;
case NC_USHORT: printf(format,((unsigned short*)content)[i]); break;
case NC_UINT: printf(format,((unsigned int*)content)[i]); break;
case NC_UINT64: printf(format,((unsigned long long*)content)[i]); break;
case NC_FLOAT: printf(format,((float*)content)[i]); break;
case NC_DOUBLE: printf(format,((double*)content)[i]); break;
case NC_CHAR: printf(format,((char*)content)[i]); break;
case NC_STRING: printf(format,(int)strlen,((char*)(&content[i*strlen]))); break;
default: abort();
}
break;

12
test_common.in
View File

@ -5,6 +5,10 @@
# See netcdf-c/COPYRIGHT file for more info.
# Load only once
if test "x$TEST_COMMON_SH" = x ; then
export TEST_COMMON_SH=1
# Define location of execution
TOPSRCDIR='@abs_top_srcdir@'
TOPBUILDDIR='@abs_top_builddir@'
@ -32,6 +36,9 @@ FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@
FEATURE_FILTERTESTS=@DO_FILTER_TESTS@
FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@
# Thredds-test server is currently disabled
#FEATURE_THREDDSTEST=1
set -e
# Figure out various locations in the src/build tree.
@ -170,5 +177,10 @@ avail() {
if test yes = `${execdir}/../ncdump/ncfilteravail $1` ; then return 0 ; else echo "filter $1 not available" ; return 1; fi
}
# Thredds-test is no longer available
#FEATURE_THREDDSTEST=1
# Make sure we are in builddir (not execdir)
cd $builddir
fi #TEST_COMMON_SH