mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-02-23 16:59:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into ncdumptests.dmh

Browse Source
This commit is contained in:
Ward Fisher 2023-06-07 14:55:11 -06:00 committed by GitHub
commit 3695cc0668
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
54 changed files with 1769 additions and 870 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/run_tests_win_cygwin.yml vendored
View File

@ -15,6 +15,10 @@ jobs:
shell: bash -eo pipefail -o igncr "{0}"
name: Cygwin-based Autotools tests
strategy:
matrix:
plugin_dir_option: ["", "--without-plugin-dir"]
steps:
- name: Fix line endings
@ -41,9 +45,12 @@ jobs:
- name: (Autotools) Configure in-tree build
run: >-
/bin/dash ./configure --enable-hdf5 --enable-shared
--disable-static --enable-dap --disable-dap-remote-tests
--enable-plugins --enable-nczarr
/bin/dash ./configure --enable-hdf5 --enable-shared
--disable-static --enable-dap --disable-dap-remote-tests
--enable-plugins ${{ matrix.plugin_dir_option }}
--disable-nczarr --disable-nczarr-filters
--disable-nczarr-s3 --disable-nczarr-s3-tests
- name: Look at config.log if error
if: ${{ failure() }}

38
CMakeLists.txt
View File

@ -1636,7 +1636,7 @@ IF (ENABLE_PARALLEL_TESTS AND NOT USE_PARALLEL)
ENDIF()
# Enable special filter test; experimental when using cmake.
OPTION(ENABLE_FILTER_TESTING "Enable filter testing. Ignored if shared libraries or netCDF4 are not enabled" yes)
OPTION(ENABLE_FILTER_TESTING "Enable filter testing. Ignored if shared libraries or netCDF4 are not enabled" ON)
IF(ENABLE_FILTER_TESTING)
if(NOT ENABLE_HDF5 AND NOT ENABLE_NCZARR)
@ -1650,8 +1650,8 @@ IF(NOT BUILD_SHARED_LIBS)
SET(ENABLE_FILTER_TESTING OFF)
ENDIF()
OPTION(ENABLE_NCZARR_FILTERS "Enable NCZarr filters" yes)
OPTION(ENABLE_NCZARR_FILTERS_TESTING "Enable NCZarr filter testing." yes)
OPTION(ENABLE_NCZARR_FILTERS "Enable NCZarr filters" ON)
OPTION(ENABLE_NCZARR_FILTERS_TESTING "Enable NCZarr filter testing." ON)
# Constraints
IF (NOT ENABLE_PLUGINS)
@ -1968,20 +1968,20 @@ ENDIF(ENABLE_MMAP)
# Used in the `configure_file` calls below
SET(ISCMAKE "yes")
IF(MSVC)
SET(ISMSVC yes CACHE BOOL "" FORCE)
SET(REGEDIT yes CACHE BOOL "" FORCE)
# Get windows major version and build number
EXECUTE_PROCESS(COMMAND "systeminfo" OUTPUT_VARIABLE WININFO)
IF(WININFO STREQUAL "")
SET(WVM 0)
SET(WVB 0)
ELSE()
STRING(REGEX MATCH "\nOS Version:[ \t]+[0-9.]+" WINVERLINE "${WININFO}")
STRING(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\1" WVM "${WINVERLINE}")
STRING(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\3" WVB "${WINVERLINE}")
ENDIF()
SET(WINVERMAJOR ${WVM} CACHE STRING "" FORCE)
SET(WINVERBUILD ${WVB} CACHE STRING "" FORCE)
SET(ISMSVC ON CACHE BOOL "" FORCE)
SET(REGEDIT ON CACHE BOOL "" FORCE)
# Get windows major version and build number
EXECUTE_PROCESS(COMMAND "systeminfo" OUTPUT_VARIABLE WININFO)
IF(WININFO STREQUAL "")
SET(WVM 0)
SET(WVB 0)
ELSE()
STRING(REGEX MATCH "\nOS Version:[ \t]+[0-9.]+" WINVERLINE "${WININFO}")
STRING(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\1" WVM "${WINVERLINE}")
STRING(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\3" WVB "${WINVERLINE}")
ENDIF()
SET(WINVERMAJOR ${WVM} CACHE STRING "" FORCE)
SET(WINVERBUILD ${WVB} CACHE STRING "" FORCE)
ENDIF()
#####
@ -2205,7 +2205,7 @@ MACRO(print_conf_summary)
MESSAGE("")
ENDMACRO()
##
##specific
# Shell script Macro
##
# Determine if 'bash' is on the system.
@ -2232,7 +2232,7 @@ ENDIF(ENABLE_BASH_SCRIPT_TESTING)
MACRO(add_sh_test prefix F)
IF(HAVE_BASH)
ADD_TEST(${prefix}_${F} bash "-c" "export srcdir=${CMAKE_CURRENT_SOURCE_DIR};export TOPSRCDIR=${CMAKE_SOURCE_DIR};${CMAKE_CURRENT_BINARY_DIR}/${F}.sh ${ARGN}")
ADD_TEST(${prefix}_${F} bash "-c" "export srcdir=${CMAKE_CURRENT_SOURCE_DIR};export TOPSRCDIR=${CMAKE_SOURCE_DIR};bash ${CMAKE_CURRENT_BINARY_DIR}/${F}.sh ${ARGN}")
ENDIF()
ENDMACRO()

2
RELEASE_NOTES.md
View File

@ -10,6 +10,8 @@ This file contains a high-level description of this package's evolution. Release
* Fix a race condition in some ncdump tests. See [Github #2682](https://github.com/Unidata/netcdf-c/pull/2682).
* Fix a minor bug in reporting the use of szip. See [Github #2679](https://github.com/Unidata/netcdf-c/pull/2679).
* Simplify the handling of XGetopt. See [Github #2678](https://github.com/Unidata/netcdf-c/pull/2678).
* Improve performance and testing of the new nc_reclaim/copy functions. See [Github #2699](https://github.com/Unidata/netcdf-c/pull/2699).
* [Bug Fix] Provide a partial fix to the libcurl certificates problem. See [Github #2690](https://github.com/Unidata/netcdf-c/pull/2690).
* Improve S3 documentation, testing, and support See [Github #2686](https://github.com/Unidata/netcdf-c/pull/2686).
* Remove obsolete code. See [Github #2680](https://github.com/Unidata/netcdf-c/pull/2680).
* [Bug Fix] Add a crude test to see if an NCZarr path looks like a valid NCZarr/Zarr file. See [Github #2658](https://github.com/Unidata/netcdf-c/pull/2658).

28
configure.ac
View File

@ -530,19 +530,19 @@ AC_MSG_RESULT([${havecurloption}])
if test $havecurloption = yes; then
AC_DEFINE([HAVE_CURLOPT_KEEPALIVE],[1],[Is CURLOPT_TCP_KEEPALIVE defined])
fi
# CURLOPT_VERIFYHOST semantics differ depending on version
AC_MSG_CHECKING([whether libcurl is version 7.66 or later?])
AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[#include "curl/curl.h"],
[[
#if LIBCURL_VERSION_NUM < 0x074200
#if !CURL_AT_LEAST_VERSION(7,66,0)
error "<7.66";
#endif
]])], [libcurl766=yes], [libcurl766=no])
AC_MSG_RESULT([$libcurl766])
if test x$libcurl66 = xno; then
AC_DEFINE([HAVE_LIBCURL_766],[1],[Is libcurl version 7.66 or later])
if test x$libcurl766 = xyes; then
AC_DEFINE([HAVE_LIBCURL_766],[1],[libcurl version is 7.66 or later])
fi
CFLAGS="$SAVECFLAGS"
@ -1488,10 +1488,14 @@ AC_CHECK_SIZEOF(ssize_t)
$SLEEPCMD
AC_CHECK_SIZEOF([void*])
if test "x$enable_hdf5" = xyes || test "x$enable_dap" = xyes ; then
AC_SEARCH_LIBS([deflate], [zlibwapi zlibstat zlib zlib1 z], [], [
AC_MSG_ERROR([Can't find or link to the z library. Turn off netCDF-4 and \
DAP clients with --disable-hdf5 --disable-dap, or see config.log for errors.])])
# Check for deflate library
AC_SEARCH_LIBS([deflate], [zlibwapi zlibstat zlib zlib1 z], [have_deflate=yes], [have_deflate=no])
AC_MSG_CHECKING([whether deflate library is available])
AC_MSG_RESULT([${have_deflate}])
if test "x$have_deflate" = xno ; then
AC_MSG_ERROR([Can't find or link to the z library. Turn off netCDF-4 and \
DAP and NCZarr clients with --disable-hdf5 --disable-dap --disable-nczarr, or see config.log for errors.])
fi
# We need the math library
@ -1857,6 +1861,7 @@ AM_CONDITIONAL(ENABLE_S3_TESTPUB, [test "x$with_s3_testing" != xno]) # all => pu
AM_CONDITIONAL(ENABLE_S3_TESTALL, [test "x$with_s3_testing" = xyes])
AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
AM_CONDITIONAL(HAVE_DEFLATE, [test "x$have_deflate" = xyes])
AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes])
AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes])
AM_CONDITIONAL(HAVE_BLOSC, [test "x$have_blosc" = xyes])
@ -1971,6 +1976,7 @@ AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
AC_SUBST(HAS_QUANTIZE,[$enable_quantize])
AC_SUBST(HAS_LOGGING,[$enable_logging])
AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing])
AC_SUBST(HAS_DEFLATE,[$have_deflate])
AC_SUBST(HAS_SZLIB,[$have_sz])
AC_SUBST(HAS_SZLIB_WRITE, [$have_sz])
AC_SUBST(HAS_ZSTD,[$have_zstd])
@ -1987,8 +1993,10 @@ AC_SUBST(WHICH_S3_SDK,[none])
fi
# Always available
std_filters="deflate bz2"
std_filters="bz2"
if test "x$have_deflate" = xyes ; then
std_filters="${std_filters} deflate"
fi
if test "x$have_sz" = xyes ; then
std_filters="${std_filters} szip"
fi

2
docs/cloud.md
View File

@ -258,7 +258,7 @@ This is an experimental SDK provided internally in the netcdf-c library.
* It is written in C
* It provides the minimum functionality necessary to read/write/search an Amazon S3 bucket.
* It was constructed by heavily modifying the HDF5 *H5FDros3* Virtual File Driver and combining it with crypto code wrappers provided by libcurl. The resulting file was then modified to fit into the netcdf coding style.
* It was constructed by heavily modifying the HDF5 *H5Fs3commons.c* file and combining it with crypto code wrappers provided by libcurl. The resulting file was then modified to fit into the netcdf coding style.
* The resulting code is rather ugly, but appears to work under at least Linux and under Windows (using Visual C++).
### Dependencies

129
docs/internal.md
View File

@ -4,10 +4,14 @@ Notes On the Internals of the NetCDF-C Library
# Notes On the Internals of the NetCDF-C Library {#intern_head}
\tableofcontents
This document attempts to record important information about
the internal architecture and operation of the netcdf-c library.
It covers the following issues.
* [Including C++ Code in the netcdf-c Library](#intern_c++)
* [Managing instances of variable-length data types](#intern_vlens)
* [Inferring File Types](#intern_infer)
* [Adding a Standard Filter](#intern_filters)
# 1. Including C++ Code in the netcdf-c Library {#intern_c++}
@ -41,46 +45,71 @@ AM_CXXFLAGS = -std=c++11
````
It is possible that other values (e.g. *-std=c++14*) may also work.
# 2. Managing instances of complex data types
# 2. Managing instances of variable-length data types {#intern_vlens}
For a long time, there have been known problems with the
management of complex types containing VLENs. This also
involves the string type because it is stored as a VLEN of
chars.
involves the string type because it is stored as a VLEN of chars.
The term complex type refers to any type that directly or
recursively references a VLEN type. So an array of VLENS, a
compound with a VLEN field, and so on.
The term "variable-length" refers to any
type that directly or recursively references a VLEN type. So an
array of VLENS, a compound with a VLEN field, and so on.
In order to properly handle instances of these complex types, it
In order to properly handle instances of these variable-length types, it
is necessary to have function that can recursively walk
instances of such types to perform various actions on them. The
term "deep" is also used to mean recursive.
Two deep walking operations are provided by the netcdf-c library
to aid in managing instances of complex structures.
- free'ing an instance of the complex type
- copying an instance of the complex type.
Two primary deep walking operations are provided by the netcdf-c library
to aid in managing instances of variable-length types.
- free'ing an instance of the type
- copying an instance of the type.
Previously The netcdf-c library only did shallow free and shallow copy of
complex types. This meant that only the top level was properly
free'd or copied, but deep internal blocks in the instance were
not touched. This led to a host of memory leaks and failures
when the deep data was effectively shared between the netcdf-c library
internally and the user's data.
Note that the term "vector" is used in the following text to
mean a contiguous (in memory) sequence of instances of some
type. Given an array with, say, dimensions 2 X 3 X 4, this will
be stored in memory as a vector of length 2\*3\*4=24 instances.
Note that the term "vector" is used to mean a contiguous (in
memory) sequence of instances of some type. Given an array with,
say, dimensions 2 X 3 X 4, this will be stored in memory as a
vector of length 2\*3\*4=24 instances.
## Special case reclamation functions
The use cases are primarily these.
Previously The netcdf-c library provided functions to reclaim an
instance of a subset of the possible variable-length types. It
provided no specialized support for copying instances of
variable-length types.
## nc\_get\_vars
These specialized functions are still available and can be used
when their pre-conditions are met. They have the advantage of
being faster than the general purpose functions described
later in this document.
These functions, and their pre and post conditions, are as follows.
* *int nc_free_string(size_t len, char\* data[])*
<u>Action</u>: reclaim a vector of variable length strings.
<u>Pre-condition(s)</u>: the data argument is a vector containing *len* pointers to variable length, nul-terminated strings.
<u>Post-condition(s)</u>: only the strings in the vector are reclaimed -- the vector itself is not reclaimed.
* int nc_free_vlens(size_t len, nc_vlen_t vlens[])
<u>Action</u>: reclaim a vector of VLEN instances.
<u>Pre-condition(s)</u>: the data argument is a vector containing *len* pointers to variable length, nul-terminated strings.
<u>Post-condition(s)</u>:
* only the data pointed to by the VLEN instances (i.e. *nc_vlen_t.p* in the vector are reclaimed -- the vector itself is not reclaimed.
* the base type of the VLEN must be a fixed-size type -- this means atomic types in the range NC_CHAR thru NC_UINT64, and compound types where the basetype of each field is itself fixed-size.
* *int nc_free_vlen(nc_vlen_t \*vl)*
<u>Action</u>: this is equivalent to calling *nc_free_vlens(1,&vl)*.
If the pre and post conditions are not met, then using these
functions can lead to a host of memory leaks and failures
because the deep data variable-length data is in effect
shared between the netcdf-c library internally and the user's data.
## Typical use cases
### *nc\_get\_vars*
Suppose one is reading a vector of instances using nc\_get\_vars
(or nc\_get\_vara or nc\_get\_var, etc.). These functions will
return the vector in the top-level memory provided. All
interior blocks (form nested VLEN or strings) will have been
interior blocks (from nested VLEN or strings) will have been
dynamically allocated. Note that computing the size of the vector
may be tricky because the strides must be taken into account.
@ -90,14 +119,7 @@ memory leak occurs. So, the recursive reclaim function is used
to walk the returned instance vector and do a deep reclaim of
the data.
Currently functions are defined in netcdf.h that are supposed to
handle this: nc\_free\_vlen(), nc\_free\_vlens(), and
nc\_free\_string(). Unfortunately, these functions only do a
shallow free, so deeply nested instances are not properly
handled by them. They are marked in the description as
deprecated in favor of the newer recursive function.
## nc\_put\_vars
### *nc\_put\_vars*
Suppose one is writing a vector of instances using nc\_put\_vars
(or nc\_put\_vara or nc\_put\_var, etc.). These functions will
@ -113,7 +135,10 @@ So again, the recursive reclaim function can be used
to walk the returned instance vector and do a deep reclaim of
the data.
## nc\_put\_att
WARNING: If the data passed into these functions contains statically
allocated data, then using any of the reclaim functions will fail.
### *nc\_put\_att*
Suppose one is writing a vector of instances as the data of an attribute
using, say, nc\_put\_att.
@ -122,15 +147,11 @@ so that changes/reclamation of the input data will not affect
the attribute. Note that this copying behavior is different from
writing to a variable, where the data is written immediately.
Again, the code inside the netcdf library used to use only shallow copying
rather than deep copy. As a result, one saw effects such as described
in Github Issue https://github.com/Unidata/netcdf-c/issues/2143.
Also, after defining the attribute, it may be necessary for the user
After defining the attribute, it may be necessary for the user
to free the data that was provided as input to nc\_put\_att() as in the
nc\_put\_xxx functions (previously described).
## nc\_get\_att
### *nc\_get\_att*
Suppose one is reading a vector of instances as the data of an attribute
using, say, nc\_get\_att.
@ -138,10 +159,6 @@ Internally, the existing attribute data must be copied and returned
to the caller, and the caller is responsible for reclaiming
the returned data.
Again, the code inside the netcdf library used to only do shallow copying
rather than deep copy. So this could lead to memory leaks and errors
because the deep data was shared between the library and the user.
## New Instance Walking API
Proper recursive functions were added to the netcdf-c library to
@ -156,14 +173,14 @@ int nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t count);
int nc_copy_data(int ncid, nc_type xtypeid, const void* memory, size_t count, void* copy);
int nc_copy_data_all(int ncid, nc_type xtypeid, const void* memory, size_t count, void** copyp);
````
There are two variants. The first two, nc\_reclaim\_data() and
There are two variants. The two functions, nc\_reclaim\_data() and
nc\_copy\_data(), assume the top-level vector is managed by the
caller. For reclaim, this is so the user can use, for example, a
statically allocated vector. For copy, it assumes the user
provides the space into which the copy is stored.
The second two, nc\_reclaim\_data\_all() and
nc\_copy\_data\_all(), allows the functions to manage the
The other two, nc\_reclaim\_data\_all() and
nc\_copy\_data\_all(), allow the called functions to manage the
top-level. So for nc\_reclaim\_data\_all, the top level is
assumed to be dynamically allocated and will be free'd by
nc\_reclaim\_data\_all(). The nc\_copy\_data\_all() function
@ -171,12 +188,10 @@ will allocate the top level and return a pointer to it to the
user. The user can later pass that pointer to
nc\_reclaim\_data\_all() to reclaim the instance(s).
# Internal Changes
## Internal Changes
The netcdf-c library internals are changed to use the proper reclaim
and copy functions. This also allows some simplification of the code
since the stdata and vldata fields of NC\_ATT\_INFO are no longer needed.
Currently this is commented out using the SEPDATA \#define macro.
When the bugs are found and fixed, all this code will be removed.
## Optimizations
@ -202,7 +217,7 @@ The classification of types can be made at the time the type is defined
or is read in from some existing file. The reclaim and copy functions
use this information to speed up the handling of fixed size types.
# Warnings
## Warnings
1. The new API functions require that the type information be
accessible. This means that you cannot use these functions
@ -228,7 +243,7 @@ use this information to speed up the handling of fixed size types.
}
````
# 3. Inferring File Types
# 3. Inferring File Types {#intern_infer}
As described in the companion document -- docs/dispatch.md --
when nc\_create() or nc\_open() is called, it must figure out what
@ -478,7 +493,7 @@ So for example, if DAP2 is the model, then all netcdf-4 mode flags
and some netcdf-3 flags are removed from the set of mode flags
because DAP2 provides only a standard netcdf-classic format.
# 4. Adding a Standard Filter
# 4. Adding a Standard Filter {#intern_filters}
The standard filter system extends the netcdf-c library API to
support a fixed set of "standard" filters. This is similar to the
@ -534,7 +549,7 @@ and CMake (CMakeLists.txt)
Configure.ac must have a block that similar to this that locates
the implementing library.
````
# See if we have libzstd
\# See if we have libzstd
AC_CHECK_LIB([zstd],[ZSTD_compress],[have_zstd=yes],[have_zstd=no])
if test "x$have_zstd" = "xyes" ; then
AC_SEARCH_LIBS([ZSTD_compress],[zstd zstd.dll cygzstd.dll], [], [])
@ -559,7 +574,7 @@ endif
````
````
# Need our version of szip if libsz available and we are not using HDF5
\# Need our version of szip if libsz available and we are not using HDF5
if HAVE_SZ
noinst_LTLIBRARIES += libh5szip.la
libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h
@ -637,4 +652,4 @@ done:
*Author*: Dennis Heimbigner<br>
*Email*: dmh at ucar dot edu<br>
*Initial Version*: 12/22/2021<br>
*Last Revised*: 01/25/2022
*Last Revised*: 5/16/2023

60
include/nc.h
View File

@ -77,4 +77,64 @@ extern int iterate_NCList(int i,NC**); /* Walk from 0 ...; ERANGE return => stop
extern void free_NC(NC*);
extern int new_NC(const struct NC_Dispatch*, const char*, int, NC**);
/* Defined in dinstance_intern.c */
/**************************************************/
/**
Following are the internal implementation signatures upon which
are built the API functions: nc_reclaim_data, nc_reclaim_data_all,
nc_copy_data, and nc_copy_data_all.
These functions access internal data structures instead of using e.g. ncid.
This is to improve performance.
*/
/*
Reclaim a vector of instances of arbitrary type.
This recursively walks the top-level instances to reclaim any
nested data such as vlen or strings or such.
Assumes it is passed a pointer to count instances of xtype.
Reclaims any nested data.
WARNING: nc_reclaim_data does not reclaim the top-level
memory because we do not know how it was allocated. However
nc_reclaim_data_all does reclaim top-level memory assuming that it
was allocated using malloc().
WARNING: all data blocks below the top-level (e.g. string
instances) will be reclaimed, so do not call if there is any
static data in the instance.
Should work for any netcdf format.
*/
EXTERNL int NC_reclaim_data(NC* nc, nc_type xtypeid, void* memory, size_t count);
EXTERNL int NC_reclaim_data_all(NC* nc, nc_type xtypeid, void* memory, size_t count);
/**
Copy vector of arbitrary type instances. This recursively walks
the top-level instances to copy any nested data such as vlen or
strings or such.
Assumes it is passed a pointer to count instances of xtype.
WARNING: nc_copy_data does not copy the top-level memory, but
assumes a block of proper size was passed in. However
nc_copy_data_all does allocate top-level memory copy.
Should work for any netcdf format.
*/
EXTERNL int NC_copy_data(NC* nc, nc_type xtypeid, const void* memory, size_t count, void* copy);
EXTERNL int NC_copy_data_all(NC* nc, nc_type xtypeid, const void* memory, size_t count, void** copyp);
/* Macros to map NC_FORMAT_XX to metadata structure (e.g. NC_FILE_INFO_T) */
/* Fast test for what file structure is used */
#define NC3INFOFLAGS ((1<<NC_FORMATX_NC3)|(1<<NC_FORMATX_PNETCDF)|(1<<NC_FORMATX_DAP2))
#define FILEINFOFLAGS ((1<<NC_FORMATX_NC_HDF5)|(1<<NC_FORMATX_NC_HDF4)|(1<<NC_FORMATX_DAP4)|(1<<NC_FORMATX_UDF1)|(1<<NC_FORMATX_UDF0)|(1<<NC_FORMATX_NCZARR))
#define USENC3INFO(nc) ((1<<(nc->dispatch->model)) & NC3INFOFLAGS)
#define USEFILEINFO(nc) ((1<<(nc->dispatch->model)) & FILEINFOFLAGS)
#define USED2INFO(nc) ((1<<(nc->dispatch->model)) & (1<<NC_FORMATX_DAP2))
#define USED4INFO(nc) ((1<<(nc->dispatch->model)) & (1<<NC_FORMATX_DAP4))
#endif /* _NC_H_ */

7
include/nc4internal.h
View File

@ -239,6 +239,7 @@ typedef struct NC_TYPE_INFO
nc_bool_t committed; /**< True when datatype is committed in the file */
nc_type nc_type_class; /**< NC_VLEN, NC_COMPOUND, NC_OPAQUE, NC_ENUM, NC_INT, NC_FLOAT, or NC_STRING. */
void *format_type_info; /**< HDF5-specific type info. */
int varsized; /**< <! 1 if this type is (recursively) variable sized; 0 if fixed size */
/** Information for each type or class */
union {
@ -248,7 +249,6 @@ typedef struct NC_TYPE_INFO
} e; /**< Enum */
struct Fields {
NClist* field; /**< <! NClist<NC_FIELD_INFO_T*> */
int varsized; /**< <! 1 if this compound is variable sized; 0 if fixed size */
} c; /**< Compound */
struct {
nc_type base_nc_typeid; /**< Typeid of the base type. */
@ -446,8 +446,11 @@ extern int nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_
const char *name, nc_type *xtype, nc_type mem_type,
size_t *lenp, int *attnum, void *data);
/* Get variable/fixed size flag for type */
/* Get variable/fixed size flag for type (ncid API level)*/
extern int NC4_inq_type_fixed_size(int ncid, nc_type xtype, int* isfixedsizep);
/* Manage the fixed/var sized'ness of a type */
extern int NC4_recheck_varsize(NC_TYPE_INFO_T* parenttype, nc_type addedtype);
extern int NC4_set_varsize(NC_TYPE_INFO_T* parenttype);
/* Close the file. */
extern int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio);

10
include/ncoffsets.h
View File

@ -57,4 +57,14 @@ EXTERNL void NC_compute_alignments(void);
/* From libdispatch/dinstance.c */
EXTERNL int NC_type_alignment(int ncid, nc_type xtype, size_t*);
/* From libdispatch/dinstance_intern.c */
/**
* Internal version of NC_type_alignment
*/
struct NC_FILE_INFO; /* forward */
struct NC_TYPE_INFO; /* forward */
EXTERNL int NC_type_alignment_internal(struct NC_FILE_INFO* file, nc_type xtype, struct NC_TYPE_INFO* utype, size_t* alignp);
EXTERNL uintptr_t NC_read_align(uintptr_t addr, size_t alignment);
#endif /*NCOFFSETS_H*/

77
include/netcdf.h
View File

@ -763,19 +763,6 @@ EXTERNL int
nc_inq_vlen(int ncid, nc_type xtype, char *name, size_t *datum_sizep,
nc_type *base_nc_typep);
/* When you read VLEN type the library will actually allocate the
* storage space for the data. This storage space must be freed, so
* pass the pointer back to this function, when you're done with the
* data, and it will free the vlen memory.
* These two functions are deprecated in favor of the nc_reclaim_data function.
*/
EXTERNL int
nc_free_vlen(nc_vlen_t *vl);
EXTERNL int
nc_free_vlens(size_t len, nc_vlen_t vlens[]);
/* Put or get one element in a vlen array. */
EXTERNL int
nc_put_vlen_element(int ncid, int typeid1, void *vlen_element,
@ -785,15 +772,6 @@ EXTERNL int
nc_get_vlen_element(int ncid, int typeid1, const void *vlen_element,
size_t *len, void *data);
/* When you read the string type the library will allocate the storage
* space for the data. This storage space must be freed, so pass the
* pointer back to this function, when you're done with the data, and
* it will free the string memory.
* This function is deprecated in favor of the nc_reclaim_data function.
*/
EXTERNL int
nc_free_string(size_t len, char **data);
/* Find out about a user defined type. */
EXTERNL int
nc_inq_user_type(int ncid, nc_type xtype, char *name, size_t *size,
@ -1785,20 +1763,49 @@ nc_put_var_string(int ncid, int varid, const char **op);
EXTERNL int
nc_get_var_string(int ncid, int varid, char **ip);
/* Begin recursive instance walking functions */
/* Begin instance walking functions */
/* When you read an array of string typed instances, the library will allocate
* the storage space for the strings in the array (but not the array itself).
* The strings must be freed eventually, so pass the pointer to the array plus
* the number of elements in the array to this function when you're done with
* the data, and it will free the all the string instances.
* The caller is still responsible for free'ing the array itself,
* if it was dynamically allocated.
*/
EXTERNL int
nc_free_string(size_t nelems, char **data);
/* When you read an array of VLEN typed instances, the library will allocate
* the storage space for the data in each VLEN in the array (but not the array itself).
* That VLEN data must be freed eventually, so pass the pointer to the array plus
* the number of elements in the array to this function when you're done with
* the data, and it will free the all the VLEN instances.
* The caller is still responsible for free'ing the array itself,
* if it was dynamically allocated.
*
* WARNING: this function only works if the basetype of the vlen type
* is fixed size. This means it is an atomic type except NC_STRING,
* or an NC_ENUM, or and NC_OPAQUE, or an NC_COMPOUND where all
* the fields of the compound type are themselves fixed size.
*/
EXTERNL int
nc_free_vlens(size_t nelems, nc_vlen_t vlens[]);
/* This function is a special case of "nc_free_vlens" where nelem == 1 */
EXTERNL int
nc_free_vlen(nc_vlen_t *vl);
/**
Reclaim a vector of instances of arbitrary type. Intended for
use with e.g. nc_get_vara or the input to e.g. nc_put_vara.
This recursively walks the top-level instances to reclaim any
nested data such as vlen or strings or such.
Assumes it is passed a pointer to count instances of xtype.
Reclaims any nested data.
Reclaim an array of instances of an arbitrary type.
This function is intended for use with e.g. nc_get_vara
or the input to e.g. nc_put_vara.
This function recursively walks the top-level instances to
reclaim any nested data such as vlen or strings or such.
WARNING: nc_reclaim_data does not reclaim the top-level
memory because we do not know how it was allocated. However
nc_reclaim_data_all does reclaim top-level memory.
nc_reclaim_data_all does attempt to reclaim top-level memory.
WARNING: all data blocks below the top-level (e.g. string
instances) will be reclaimed, so do not call if there is any
@ -1807,11 +1814,10 @@ static data in the instance.
Should work for any netcdf format.
*/
EXTERNL int nc_reclaim_data(int ncid, nc_type xtypeid, void* memory, size_t count);
EXTERNL int nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t count);
EXTERNL int nc_reclaim_data(int ncid, nc_type xtypeid, void* memory, size_t nelems);
EXTERNL int nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t nelems);
/**
Copy vector of arbitrary type instances. This recursively walks
the top-level instances to copy any nested data such as vlen or
strings or such.
@ -1827,9 +1833,6 @@ Should work for any netcdf format.
EXTERNL int nc_copy_data(int ncid, nc_type xtypeid, const void* memory, size_t count, void* copy);
EXTERNL int nc_copy_data_all(int ncid, nc_type xtypeid, const void* memory, size_t count, void** copyp);
/* Instance dumper for debugging */
EXTERNL int nc_dump_data(int ncid, nc_type xtypeid, void* memory, size_t count, char** buf);
/* end recursive instance walking functions */
/* Begin Deprecated, same as functions with "_ubyte" replaced by "_uchar" */

7
include/netcdf_aux.h
View File

@ -48,6 +48,13 @@ are here for back compatibilty.
EXTERNL int ncaux_reclaim_data(int ncid, int xtype, void* memory, size_t count);
EXTERNL int ncaux_reclaim_data_all(int ncid, int xtype, void* memory, size_t count);
EXTERNL int ncaux_copy_data(int ncid, int xtype, void* memory, size_t count, void* copy);
EXTERNL int ncaux_copy_data_all(int ncid, int xtype, void* memory, size_t count, void** copyp);
EXTERNL int ncaux_dump_data(int ncid, nc_type xtypeid, void* memory, size_t count, char** buf);
EXTERNL int ncaux_inq_any_type(int ncid, nc_type typeid, char *name, size_t *size, nc_type *basetypep, size_t *nfieldsp, int *classp);
/**************************************************/
/* Capture the id and parameters for a filter

20
libdap2/ncd2dispatch.c
View File

@ -2864,3 +2864,23 @@ NCD2_get_var_chunk_cache(int ncid, int p2, size_t* p3, size_t* p4, float* p5)
return THROW(ret);
}
/* Get substrate NC* object.
This function breaks the abstraction, but is necessary for
code that accesses the underlying netcdf-4 metadata objects.
Used in: NC_reclaim_data[_all]
NC_copy_data[_all]
*/
NC*
NCD2_get_substrate(NC* nc)
{
NC* subnc = NULL;
/* Iff nc->dispatch is the DAP2 dispatcher, then do a level of indirection */
if(USED2INFO(nc)) {
NCDAPCOMMON* d2 = (NCDAPCOMMON*)nc->dispatchdata;
/* Find pointer to NC struct for this file. */
(void)NC_check_id(d2->substrate.nc3id,&subnc);
} else subnc = nc;
return subnc;
}

5
libdap4/d4debug.c
View File

@ -10,6 +10,8 @@
#include "ncdispatch.h"
#include "netcdf_aux.h"
extern NC* NCD4_get_substrate(NC* nc);
#ifdef D4CATCH
/* Place breakpoint here to catch errors close to where they occur*/
int
@ -112,7 +114,6 @@ NCD4_debugcopy(NCD4INFO* info)
int varid = var->meta.id;
d4size_t varsize;
size_t dimprod = NCD4_dimproduct(var);
int ncid = info->substrate.nc4id;
varsize = type->meta.memsize * dimprod;
memory = d4alloc(varsize);
@ -141,7 +142,7 @@ NCD4_debugcopy(NCD4INFO* info)
if((ret=nc_put_vara(grpid,varid,NC_coord_zero,edges,memory)))
goto done;
}
if((ret=ncaux_reclaim_data(ncid,type->meta.id,memory,dimprod)))
if((ret=NC_reclaim_data(NCD4_get_substrate(ncp),type->meta.id,memory,dimprod)))
goto done;
nullfree(memory); memory = NULL;
}

28
libdap4/d4file.c
View File

@ -275,11 +275,9 @@ freeInfo(NCD4INFO* d4info)
when aborted, it should be deleted. But that is not working
for some reason, so we delete it ourselves.
*/
#if 0
if(d4info->substrate.filename != NULL) {
unlink(d4info->substrate.filename);
}
#endif
}
nullfree(d4info->substrate.filename); /* always reclaim */
NCD4_reclaimMeta(d4info->substrate.metadata);
@ -580,15 +578,21 @@ makesubstrate(NCD4INFO* d4info)
return THROW(ret);
}
int
NCD4_get_substrate(int ncid)
/* This function breaks the abstraction, but is necessary for
code that accesses the underlying netcdf-4 metadata objects.
Used in: NC_reclaim_data[_all]
NC_copy_data[_all]
*/
NC*
NCD4_get_substrate(NC* nc)
{
NC* nc = NULL;
NCD4INFO* d4 = NULL;
int subncid = 0;
/* Find pointer to NC struct for this file. */
(void)NC_check_id(ncid,&nc);
d4 = (NCD4INFO*)nc->dispatchdata;
subncid = d4->substrate.nc4id;
return subncid;
NC* subnc = NULL;
/* Iff nc->dispatch is the DAP4 dispatcher, then do a level of indirection */
if(USED4INFO(nc)) {
NCD4INFO* d4 = (NCD4INFO*)nc->dispatchdata;
/* Find pointer to NC struct for this file. */
(void)NC_check_id(d4->substrate.nc4id,&subnc);
} else subnc = nc;
return subnc;
}

2
libdap4/ncd4.h
View File

@ -253,7 +253,7 @@ EXTERNL d4size_t NCD4_getcounter(NCD4offset* p);
*/
#define HYRAXHACK
EXTERNL int NCD4_get_substrate(int ncid);
EXTERNL NC* NCD4_get_substrate_nc(NC* nc);
#endif /*NCD4_H*/

2
libdispatch/CMakeLists.txt
View File

@ -5,7 +5,7 @@
# See netcdf-c/COPYRIGHT file for more info.
SET(libdispatch_SOURCES dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dpathmgr.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c
daux.c dinstance.c
daux.c dinstance.c dinstance_intern.c
dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c ncjson.c ds3util.c dparallel.c dmissing.c)
# Netcdf-4 only functions. Must be defined even if not used

2
libdispatch/Makefile.am
View File

@ -21,7 +21,7 @@ dinternal.c ddispatch.c dutf8.c nclog.c dstring.c ncuri.c nclist.c \
ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c dauth.c doffsets.c \
dpathmgr.c dutil.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c \
daux.c dinstance.c dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c \
ncjson.c ds3util.c dparallel.c dmissing.c
ncjson.c ds3util.c dparallel.c dmissing.c dinstance_intern.c
# Add the utf8 codebase
libdispatch_la_SOURCES += utf8proc.c utf8proc.h

28
libdispatch/daux.c
View File

@ -25,6 +25,7 @@ See COPYRIGHT for license information.
#include "config.h"
#include "netcdf.h"
#include "netcdf_aux.h"
#include "nc4internal.h"
#include "ncoffsets.h"
#include "nclog.h"
#include "ncrc.h"
@ -915,6 +916,14 @@ ncaux_reclaim_data_all(int ncid, int xtype, void* memory, size_t count)
return nc_reclaim_data_all(ncid, xtype, memory, count);
}
EXTERNL int NC_inq_any_type(int ncid, nc_type typeid, char *name, size_t *size, nc_type *basetypep, size_t *nfieldsp, int *classp);
EXTERNL int
ncaux_inq_any_type(int ncid, nc_type typeid, char *name, size_t *sizep, nc_type *basetypep, size_t *nfieldsp, int *classp)
{
return NC_inq_any_type(ncid, typeid, name, sizep, basetypep, nfieldsp, classp);
}
/**
@param ncid - only needed for a compound type
@param xtype - type for which alignment is requested
@ -925,3 +934,22 @@ ncaux_type_alignment(int xtype, int ncid, size_t* alignp)
/* Defer to the internal version */
return NC_type_alignment(ncid, xtype, alignp);
}
/**
Dump the output tree of data from a call
to e.g. nc_get_vara or the input to e.g. nc_put_vara.
This function is just a wrapper around nc_dump__data.
@param ncid file ncid
@param xtype type id
@param memory to print
@param count number of instances of the type in memory
@return error code
*/
EXTERNL int
ncaux_dump_data(int ncid, int xtype, void* memory, size_t count, char** bufp)
{
EXTERNL int nc_dump_data(int ncid, nc_type xtype, void* memory, size_t count, char** bufp);
return nc_dump_data(ncid, xtype, memory, count, bufp);
}

525
libdispatch/dinstance.c
View File

@ -16,6 +16,7 @@ Currently two operations are defined:
#include <string.h>
#include <assert.h>
#include "netcdf.h"
#include "netcdf_aux.h"
#include "nc4internal.h"
#include "nc4dispatch.h"
#include "ncoffsets.h"
@ -24,42 +25,46 @@ Currently two operations are defined:
#undef REPORT
#undef DEBUG
/* DAP2 and DAP4 currently defer most of their API to a substrate NC
that hold the true metadata. So there is a level of indirection
necessary in order to get to the right NC* instance.
*/
#if defined(ENABLE_DAP4) || defined(ENABLE_DAP2)
EXTERNL NC* NCD4_get_substrate(NC* nc);
EXTERNL NC* NCD2_get_substrate(NC* nc);
static NC*
DAPSUBSTRATE(NC* nc)
{
if(USED2INFO(nc) != 0)
return NCD2_get_substrate(nc);
else if(USED4INFO(nc) != 0)
return NCD4_get_substrate(nc);
return nc;
}
#else
#define DAPSUBSTRATE(nc) (nc)
#endif
/* It is helpful to have a structure that contains memory and an offset */
typedef struct Position{char* memory; ptrdiff_t offset;} Position;
static int type_alignment_initialized = 0;
/* Forward */
#ifdef USE_NETCDF4
static int reclaim_datar(int ncid, nc_type xtype, Position*);
static int reclaim_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* offset);
static int reclaim_vlen(int ncid, nc_type xtype, nc_type basetype, Position* offset);
static int reclaim_enum(int ncid, nc_type xtype, nc_type basetype, Position* offset);
static int reclaim_opaque(int ncid, nc_type xtype, size_t size, Position* offset);
static int copy_datar(int ncid, nc_type xtype, Position* src, Position* dst);
static int copy_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* src, Position* dst);
static int copy_vlen(int ncid, nc_type xtype, nc_type basetype, Position* src, Position* dst);
static int copy_enum(int ncid, nc_type xtype, nc_type basetype, Position* src, Position* dst);
static int copy_opaque(int ncid, nc_type xtype, size_t size, Position* src,Position* dst);
static int dump_datar(int ncid, nc_type xtype, Position*, NCbytes* buf);
static int dump_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* offset, NCbytes* buf);
static int dump_vlen(int ncid, nc_type xtype, nc_type basetype, Position* offset, NCbytes* buf);
static int dump_enum(int ncid, nc_type xtype, nc_type basetype, Position* offset, NCbytes* buf);
static int dump_opaque(int ncid, nc_type xtype, size_t size, Position* offset, NCbytes* buf);
static ptrdiff_t read_align(ptrdiff_t offset, size_t alignment);
#endif
static int NC_inq_any_type(int ncid, nc_type typeid, char *name, size_t *size, nc_type *basetypep, size_t *nfieldsp, int *classp);
/**
Reclaim a vector of instances of a type. This improves upon
e.g. nc_free_vlen. This recursively walks the top-level
instances to reclaim any nested data such as vlen or strings or such.
\ingroup user_types
Reclaim an array of instances of an arbitrary type.
This function should be used when the other simpler functions
such as *nc_free_vlens* or *nc_free_string* cannot be used.
This recursively walks the top-level instances to reclaim
any nested data such as vlen or strings or such.
Assumes it is passed a pointer to count instances of xtype.
Reclaims any nested data.
@ -88,10 +93,8 @@ int
nc_reclaim_data(int ncid, nc_type xtype, void* memory, size_t count)
{
int stat = NC_NOERR;
size_t i;
Position offset;
int isf;
NC* nc = NULL;
if(ncid < 0 || xtype <= 0)
{stat = NC_EINVAL; goto done;}
if(memory == NULL && count > 0)
@ -102,35 +105,41 @@ nc_reclaim_data(int ncid, nc_type xtype, void* memory, size_t count)
fprintf(stderr,">>> reclaim: memory=%p count=%lu ncid=%d xtype=%d\n",memory,(unsigned long)count,ncid,xtype);
#endif
/* Optimizations */
/* 1. Vector of fixed size types */
if((stat = NC4_inq_type_fixed_size(ncid,xtype,&isf))) goto done;
if(isf) goto done; /* no need to reclaim anything */
if((stat = NC_check_id(ncid,&nc))) goto done;
nc = DAPSUBSTRATE(nc);
/* Call internal version */
stat = NC_reclaim_data(nc,xtype,memory,count);
#if 0
#ifdef USE_NETCDF4
/* 2.Vector of strings */
if(xtype == NC_STRING) {
char** ss = (char**)memory;
for(i=0;i<count;i++) {
nullfree(ss[i]);
}
goto done;
}
offset.memory = (char*)memory; /* use char* so we can do pointer arithmetic */
offset.offset = 0;
for(i=0;i<count;i++) {
if((stat=reclaim_datar(ncid,xtype,&offset))) /* reclaim one instance */
break;
}
#else
stat = NC_EBADTYPE;
NC_FILE_INFO_T* file = NULL;
/* Find info for this file and group and var, and set pointer to each. */
if ((stat = nc4_find_grp_h5(ncid, NULL, &file))) return stat;
/* Call internal version */
stat = NC_reclaim_data(file,xtype,memory,count);
#endif
#endif
done:
return stat;
}
/* Alternate entry point: includes recovering the top-level memory */
/**
\ingroup user_types
Reclaim the memory allocated for an array of instances of an arbitrary type.
This recursively walks the top-level instances to reclaim any nested data such as vlen or strings or such.
This function differs from *nc_reclaim_data* in that it also reclaims the top-level memory.
Assumes it is passed a count and a pointer to the top-level memory.
Should work for any netcdf format.
@param ncid root id
@param xtype type id
@param memory ptr to top-level memory to reclaim
@param count number of instances of the type in memory block
@return error code
*/
int
nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t count)
{
@ -141,146 +150,12 @@ nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t count)
return stat;
}
#ifdef USE_NETCDF4
/* Recursive type walker: reclaim a single instance */
static int
reclaim_datar(int ncid, nc_type xtype, Position* offset)
{
int stat = NC_NOERR;
size_t xsize;
nc_type basetype;
size_t nfields;
int klass, isf;
if((stat = NC4_inq_type_fixed_size(ncid,xtype,&isf))) goto done;
/* Get relevant type info */
if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&klass))) goto done;
if(isf) { /* no need to reclaim anything */
offset->offset += xsize;
goto done;
}
switch (xtype) {
case NC_STRING: {
char** sp = (char**)(offset->memory + offset->offset);
/* Need to reclaim string */
if(*sp != NULL) free(*sp);
offset->offset += xsize;
} break;
default:
/* reclaim a user type */
switch (klass) {
case NC_OPAQUE: stat = reclaim_opaque(ncid,xtype,xsize,offset); break;
case NC_ENUM: stat = reclaim_enum(ncid,xtype,basetype,offset); break;
case NC_COMPOUND: stat = reclaim_compound(ncid,xtype,xsize,nfields,offset); break;
case NC_VLEN:
stat = reclaim_vlen(ncid,xtype,basetype,offset);
break;
default:
stat = NC_EINVAL;
break;
}
break;
}
done:
return stat;
}
static int
reclaim_vlen(int ncid, nc_type xtype, nc_type basetype, Position* offset)
{
int stat = NC_NOERR;
size_t i;
nc_vlen_t* vl = (nc_vlen_t*)(offset->memory+offset->offset);
if(vl->len > 0 && vl->p == NULL)
{stat = NC_EINVAL; goto done;}
/* Free up each entry in the vlen list */
if(vl->len > 0) {
Position voffset;
size_t alignment = 0;
if((stat = NC_type_alignment(ncid,basetype,&alignment))) goto done;;
voffset.memory = vl->p;
voffset.offset = 0;
for(i=0;i<vl->len;i++) {
voffset.offset = read_align(voffset.offset,alignment);
if((stat = reclaim_datar(ncid,basetype,&voffset))) goto done;
}
free(vl->p);
}
offset->offset += sizeof(nc_vlen_t);
done:
return stat;
}
static int
reclaim_enum(int ncid, nc_type xtype, nc_type basetype, Position* offset)
{
int stat = NC_NOERR;
abort();
/* basically same as an instance of the enum's integer basetype */
stat = reclaim_datar(ncid,basetype,offset);
return stat;
}
static int
reclaim_opaque(int ncid, nc_type xtype, size_t size, Position* offset)
{
abort();
/* basically a fixed size sequence of bytes */
offset->offset += size;
return NC_NOERR;
}
static int
reclaim_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* offset)
{
int stat = NC_NOERR;
size_t fid, i, arraycount;
ptrdiff_t saveoffset;
int ndims;
int dimsizes[NC_MAX_VAR_DIMS];
saveoffset = offset->offset;
/* Get info about each field in turn and reclaim it */
for(fid=0;fid<nfields;fid++) {
size_t fieldalignment;
nc_type fieldtype;
/* Get all relevant info about the field */
if((stat = nc_inq_compound_field(ncid,xtype,fid,NULL,&fieldalignment,&fieldtype,&ndims,dimsizes))) goto done;
if(ndims == 0) {ndims=1; dimsizes[0]=1;} /* fake the scalar case */
/* Align to this field */
offset->offset = saveoffset + fieldalignment;
/* compute the total number of elements in the field array */
arraycount = 1;
for(i=0;i<ndims;i++) arraycount *= dimsizes[i];
for(i=0;i<arraycount;i++) {
if((stat = reclaim_datar(ncid, fieldtype, offset))) goto done;
}
}
/* Return to beginning of the compound and move |compound| */
offset->offset = saveoffset;
offset->offset += size;
done:
return stat;
}
#endif
/**************************************************/
/**
Copy a vector of instances of a type. This recursively walks
the top-level instances to copy any nested data such as vlen or
strings or such.
\ingroup user_types
Copy an array of instances of an arbitrary type. This recursively walks
the top-level instances to copy any nested data such as vlen or strings or such.
Assumes it is passed a pointer to count instances of xtype and a
space into which to copy the instance. Copys any nested data.
@ -307,12 +182,8 @@ int
nc_copy_data(int ncid, nc_type xtype, const void* memory, size_t count, void* copy)
{
int stat = NC_NOERR;
size_t i;
Position src;
Position dst;
size_t xsize;
int isf;
NC* nc = NULL;
if(ncid < 0 || xtype <= 0)
{stat = NC_EINVAL; goto done;}
if(memory == NULL && count > 0)
@ -326,34 +197,32 @@ nc_copy_data(int ncid, nc_type xtype, const void* memory, size_t count, void* co
fprintf(stderr,">>> copy : copy =%p memory=%p count=%lu ncid=%d xtype=%d\n",copy,memory,(unsigned long)count,ncid,xtype);
#endif
/* Get type size */
if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,NULL,NULL,NULL))) goto done;
if((stat = NC_check_id(ncid,&nc))) goto done;
nc = DAPSUBSTRATE(nc);
/* Optimizations */
/* 1. Vector of fixed sized objects */
if((stat = NC4_inq_type_fixed_size(ncid,xtype,&isf))) goto done;
if(isf) {
memcpy(copy,memory,xsize*count);
goto done;
}
#ifdef USE_NETCDF4
src.memory = (char*)memory; /* use char* so we can do pointer arithmetic */
src.offset = 0;
dst.memory = (char*)copy; /* use char* so we can do pointer arithmetic */
dst.offset = 0;
for(i=0;i<count;i++) {
if((stat=copy_datar(ncid,xtype,&src,&dst))) /* copy one instance copy_datar will increment src and dst*/
break;
}
#else
stat = NC_EBADTYPE;
#endif
/* Call internal version */
stat = NC_copy_data(nc,xtype,memory,count,copy);
done:
return stat;
}
/* Alternate entry point: includes recovering the top-level memory */
/**
\ingroup user_types
Copy an array of instances of an arbitrary type. This recursively walks
the top-level instances to copy any nested data such as vlen or strings or such.
This function differs from *nc_copy_data* in that it also allocates
the top-level memory.
Assumes it is passed a pointer to count instances of xtype and a pointer
into which the top-level allocated space is stored.
@param ncid root id
@param xtype type id
@param memory ptr to top-level memory to copy
@param count number of instances of the type in memory block
@param copyp pointer into which the allocated top-level space is stored.
@return error code
*/
int
nc_copy_data_all(int ncid, nc_type xtype, const void* memory, size_t count, void** copyp)
{
@ -362,7 +231,7 @@ nc_copy_data_all(int ncid, nc_type xtype, const void* memory, size_t count, void
void* copy = NULL;
/* Get type size */
if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,NULL,NULL,NULL))) goto done;
if((stat = ncaux_inq_any_type(ncid,xtype,NULL,&xsize,NULL,NULL,NULL))) goto done;
/* allocate the top-level */
if(count > 0) {
@ -379,191 +248,10 @@ done:
return stat;
}
#ifdef USE_NETCDF4
/* Recursive type walker: copy a single instance */
static int
copy_datar(int ncid, nc_type xtype, Position* src, Position* dst)
{
int stat = NC_NOERR;
size_t xsize;
nc_type basetype;
size_t nfields;
int xclass,isf;
if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&xclass))) goto done;
/* Optimizations */
/* 1. Vector of fixed size types */
if((stat = NC4_inq_type_fixed_size(ncid,xtype,&isf))) goto done;
if(isf) {
memcpy(dst->memory+dst->offset,src->memory+src->offset,xsize*1);
src->offset += xsize;
dst->offset += xsize;
goto done;
}
switch (xtype) {
case NC_STRING: {
char** sp = (char**)(src->memory + src->offset);
char* copy = NULL;
/* Need to copy string */
if(*sp != NULL) {
if((copy = strdup(*sp))==NULL) {stat = NC_ENOMEM; goto done;}
}
memcpy(dst->memory+dst->offset,(void*)&copy,sizeof(char*));
src->offset += xsize;
dst->offset += xsize;
} break;
default:
/* copy a user type */
switch (xclass) {
case NC_OPAQUE: stat = copy_opaque(ncid,xtype,xsize,src,dst); break;
case NC_ENUM: stat = copy_enum(ncid,xtype,basetype,src,dst); break;
case NC_COMPOUND: stat = copy_compound(ncid,xtype,xsize,nfields,src,dst); break;
case NC_VLEN: stat = copy_vlen(ncid,xtype,basetype,src,dst); break;
default: stat = NC_EINVAL; break;
}
break;
}
done:
return stat;
}
static int
copy_vlen(int ncid, nc_type xtype, nc_type basetype, Position* src, Position* dst)
{
int stat = NC_NOERR;
size_t i, basetypesize;
nc_vlen_t* vl = (nc_vlen_t*)(src->memory+src->offset);
nc_vlen_t copy = {0,NULL};
if(vl->len > 0 && vl->p == NULL)
{stat = NC_EINVAL; goto done;}
/* Get basetype info */
if((stat = NC_inq_any_type(ncid,basetype,NULL,&basetypesize,NULL,NULL,NULL))) goto done;
/* Make space in the copy vlen */
if(vl->len > 0) {
copy.len = vl->len;
if((copy.p = calloc(copy.len,basetypesize))==NULL) {stat = NC_ENOMEM; goto done;}
}
/* Copy each entry in the vlen list */
if(vl->len > 0) {
Position vsrc, vdst;
size_t alignment = 0;
if((stat = NC_type_alignment(ncid,basetype,&alignment))) goto done;;
vsrc.memory = vl->p;
vsrc.offset = 0;
vdst.memory = copy.p;
vdst.offset = 0;
for(i=0;i<vl->len;i++) {
vsrc.offset= read_align(vsrc.offset,alignment);
vdst.offset= read_align(vdst.offset,alignment);
if((stat = copy_datar(ncid,basetype,&vsrc,&vdst))) goto done;
}
}
/* Move into place */
memcpy(dst->memory+dst->offset,&copy,sizeof(nc_vlen_t));
src->offset += sizeof(nc_vlen_t);
dst->offset += sizeof(nc_vlen_t);
done:
if(stat) {
nullfree(copy.p);
}
return stat;
}
static int
copy_enum(int ncid, nc_type xtype, nc_type basetype, Position* src, Position* dst)
{
int stat = NC_NOERR;
abort();
/* basically same as an instance of the enum's integer basetype */
stat = copy_datar(ncid,basetype,src,dst);
return stat;
}
static int
copy_opaque(int ncid, nc_type xtype, size_t size, Position* src, Position* dst)
{
abort();
/* basically a fixed size sequence of bytes */
memcpy(dst->memory+dst->offset,src->memory+src->offset,size);
src->offset += size;
dst->offset += size;
return NC_NOERR;
}
static int
copy_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* src, Position* dst)
{
int stat = NC_NOERR;
size_t fid, i, arraycount;
ptrdiff_t savesrcoffset, savedstoffset;
int ndims;
int dimsizes[NC_MAX_VAR_DIMS];
savesrcoffset = src->offset;
savedstoffset = dst->offset;
/* Get info about each field in turn and copy it */
for(fid=0;fid<nfields;fid++) {
size_t fieldoffset;
nc_type fieldtype;
char name[NC_MAX_NAME];
/* Get all relevant info about the field */
if((stat = nc_inq_compound_field(ncid,xtype,fid,name,&fieldoffset,&fieldtype,&ndims,dimsizes))) goto done;
if(ndims == 0) {ndims=1; dimsizes[0]=1;} /* fake the scalar case */
/* Set offset for this field */
#ifdef DEBUG
fprintf(stderr,"before: offset = %d after: offset = %d\n",(int)src->offset,(int)(savesrcoffset+fieldoffset));
#endif
src->offset = savesrcoffset+fieldoffset;
dst->offset = savedstoffset+fieldoffset;
#ifdef DEBUG
fprintf(stderr,"field %s(%d) = %d\n",name,(int)fieldoffset,(int)src->offset);
#endif
/* compute the total number of elements in the field array */
arraycount = 1;
for(i=0;i<ndims;i++) arraycount *= dimsizes[i];
for(i=0;i<arraycount;i++) {
if((stat = copy_datar(ncid, fieldtype, src, dst))) goto done;
}
#ifdef DEBUG
fprintf(stderr,"src=(%d,%p)\n",(int)src->offset,src->memory);
#endif
}
#ifdef DEBUG
fprintf(stderr,"\n");
#endif
/* Return to beginning of the compound and move |compound| */
src->offset = savesrcoffset;
dst->offset = savedstoffset;
src->offset += size;
dst->offset += size;
done:
return stat;
}
#endif
/**************************************************/
/* Alignment functions */
#ifdef USE_NETCDF4
static ptrdiff_t
read_align(ptrdiff_t offset, size_t alignment)
{
size_t loc_align = (alignment == 0 ? 1 : alignment);
size_t delta = (offset % loc_align);
if(delta == 0) return offset;
return offset + (alignment - delta);
}
/**
@param ncid - only needed for a compound type
@ -574,36 +262,15 @@ int
NC_type_alignment(int ncid, nc_type xtype, size_t* alignp)
{
int stat = NC_NOERR;
size_t align = 0;
int klass;
NC* nc = NULL;
if(!type_alignment_initialized) {
NC_compute_alignments();
type_alignment_initialized = 1;
}
if(xtype <= NC_MAX_ATOMIC_TYPE)
{stat = NC_class_alignment(xtype,&align); goto done;}
else {/* Presumably a user type */
if((stat = NC_inq_any_type(ncid,xtype,NULL,NULL,NULL,NULL,&klass))) goto done;
switch(klass) {
case NC_VLEN: stat = NC_class_alignment(klass,&align); break;
case NC_OPAQUE: stat = NC_class_alignment(klass,&align); break;
case NC_COMPOUND: {/* get alignment of the first field of the compound */
nc_type fieldtype;
/* Get all relevant info about the first field */
if((stat = nc_inq_compound_field(ncid,xtype,0,NULL,NULL,&fieldtype,NULL,NULL))) goto done;
stat = NC_type_alignment(ncid,fieldtype,&align); /* may recurse repeatedly */
} break;
default: break;
}
}
if(alignp) *alignp = align;
if((stat = NC_check_id(ncid,&nc))) goto done;
nc = DAPSUBSTRATE(nc);
if(USENC3INFO(nc)) goto done;
/* Call internal version */
stat = NC_type_alignment_internal((NC_FILE_INFO_T*)nc->dispatchdata,xtype,NULL,alignp);
done:
#if 0
Why was this here?
if(stat == NC_NOERR && align == 0) stat = NC_EINVAL;
#endif
return stat;
}
#endif
@ -619,7 +286,7 @@ Why was this here?
*/
int
nc_dump_data(int ncid, nc_type xtype, void* memory, size_t count, char** bufp)
nc_dump_data(int ncid, nc_type xtype, const void* memory, size_t count, char** bufp)
{
int stat = NC_NOERR;
size_t i;
@ -651,7 +318,7 @@ done:
}
int
nc_print_data(int ncid, nc_type xtype, void* memory, size_t count)
nc_print_data(int ncid, nc_type xtype, const void* memory, size_t count)
{
char* s = NULL;
int stat = NC_NOERR;
@ -673,7 +340,7 @@ dump_datar(int ncid, nc_type xtype, Position* offset, NCbytes* buf)
char s[128];
/* Get relevant type info */
if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&klass))) goto done;
if((stat = ncaux_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&klass))) goto done;
switch (xtype) {
case NC_CHAR:
@ -773,7 +440,7 @@ dump_vlen(int ncid, nc_type xtype, nc_type basetype, Position* offset, NCbytes*
voffset.offset = 0;
for(i=0;i<vl->len;i++) {
if(i > 0) ncbytescat(buf," ");
voffset.offset = read_align(voffset.offset,alignment);
voffset.offset = NC_read_align(voffset.offset,alignment);
if((stat = dump_datar(ncid,basetype,&voffset,buf))) goto done;
}
}
@ -835,13 +502,15 @@ dump_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* of
if((stat = nc_inq_compound_field(ncid,xtype,fid,name,&fieldalignment,&fieldtype,&ndims,dimsizes))) goto done;
if(fid > 0) ncbytescat(buf,";");
ncbytescat(buf,name);
ncbytescat(buf,"(");
if(ndims > 0) {
int j;
for(j=0;j<ndims;j++) {
snprintf(sd,sizeof(sd),"[%d]",(int)dimsizes[j]);
snprintf(sd,sizeof(sd),"%s%d",(j==0?"":","),(int)dimsizes[j]);
ncbytescat(buf,sd);
}
}
ncbytescat(buf,")");
if(ndims == 0) {ndims=1; dimsizes[0]=1;} /* fake the scalar case */
/* Align to this field */
offset->offset = saveoffset + fieldalignment;

610
libdispatch/dinstance_intern.c Normal file
View File

@ -0,0 +1,610 @@
/*
Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT for license information.
*/
/** \internal
This file contains various instance operations that operate
on a deep level rather than the shallow level of e.g. nc_free_vlen_t.
Currently two operations are defined:
1. reclaim a vector of instances
2. copy a vector of instances
*/
#include "config.h"
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "netcdf.h"
#include "nc4internal.h"
#include "nc4dispatch.h"
#include "ncoffsets.h"
#include "ncbytes.h"
#undef REPORT
#undef DEBUG
/* It is helpful to have a structure that identifies a pointer into the overall memory */
typedef struct Position{char* memory;} Position;
static int type_alignment_initialized = 0;
/* Forward */
#ifdef USE_NETCDF4
static int reclaim_datar(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* utype, Position instance);
static int copy_datar(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* utype, Position src, Position dst);
#endif
int NC_print_data(NC_FILE_INFO_T* file, nc_type xtype, const void* memory, size_t count);
/**
Reclaim a vector of instances of a type. This recursively walks the
top-level instances to reclaim any nested data such as vlen or strings
or such.
Assumes it is passed a pointer to count instances of xtype.
Reclaims any nested data.
These are the internal equivalents of nc_reclaim_data[_all] and
nc_copy_data[_all] and as such operate using internal data structures.
WARNING: DOES NOT RECLAIM THE TOP-LEVEL MEMORY.
The reason is that we do not know how it was allocated (e.g. static vs
dynamic); only the caller can know that. But note that it assumes all
memory blocks other than the top were dynamically allocated, so they
will be free'd.
Should work for any netcdf type.
Note that this has been optimized significantly, largely
by unwinding various reclaim cliche procedures.
@param nc NC* structure
@param xtype type id
@param memory ptr to top-level memory to reclaim
@param count number of instances of the type in memory block
@return error code
*/
int
NC_reclaim_data(NC* nc, nc_type xtype, void* memory, size_t count)
{
int stat = NC_NOERR;
size_t i;
Position instance;
NC_FILE_INFO_T* file = NULL;
NC_TYPE_INFO_T* utype = NULL;
assert(nc != NULL);
assert((memory == NULL && count == 0) || (memory != NULL || count > 0));
/* Process atomic types */
/* Optimize: Vector of fixed size atomic types (always the case for netcdf-3)*/
if(xtype < NC_STRING) goto done;
#ifdef USE_NETCDF4
/* Optimize: Vector of strings */
if(xtype == NC_STRING) {
char** ss = (char**)memory;
for(i=0;i<count;i++) {
nullfree(ss[i]);
}
goto done;
}
/* Process User types */
assert(USEFILEINFO(nc) != 0);
file = (NC_FILE_INFO_T*)(nc)->dispatchdata;
if((stat = nc4_find_type(file,xtype,&utype))) goto done;
/* Optimize: vector of fixed sized compound type instances */
if(!utype->varsized) goto done; /* no need to reclaim anything */
/* Remaining cases: vector of VLEN and vector of (transitive) variable sized compound types.
These all require potential recursion.
*/
/* Build a memory walker object */
instance.memory = (char*)memory; /* use char* so we can do pointer arithmetic */
/* Walk each vector instance */
/* Note that we avoid reclaiming the top level memory */
for(i=0;i<count;i++) {
if((stat=reclaim_datar(file,utype,instance))) goto done;
instance.memory += utype->size; /* move to next entry */
}
#else
stat = NC_EBADTYPE;
#endif
done:
return stat;
}
#ifdef USE_NETCDF4
/* Recursive type walker: reclaim a single instance of a variable-sized user-defined type;
specifically a vlen or a variable-sized compound type instance
*/
static int
reclaim_datar(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* utype, Position instance)
{
int i,stat = NC_NOERR;
nc_type basetypeid;
NC_TYPE_INFO_T* basetype = NULL;
size_t nfields;
nc_vlen_t* vlen;
size_t fid, arraycount;
int ndims;
int dimsizes[NC_MAX_VAR_DIMS];
size_t alignment = 0;
Position vinstance; /* walk the vlen instance memory */
assert(utype->varsized); /* All fixed size cases are optimized out */
/* Leaving VLEN or Compound */
if(utype->nc_type_class == NC_VLEN) {
basetypeid = utype->u.v.base_nc_typeid; /* Get basetype */
vlen = (nc_vlen_t*)instance.memory;/* memory as vector of nc_vlen_t instances */
/* Optimize: basetype is atomic fixed size */
if(basetypeid < NC_STRING) {
goto out;
}
/* Optimize: basetype is string */
if(basetypeid == NC_STRING) {
if(vlen->len > 0 && vlen->p != NULL) {
char** slist = (char**)vlen->p; /* vlen instance is a vector of string pointers */
for(i=0;i<vlen->len;i++) {if(slist[i] != NULL) free(slist[i]);}
}
goto out;
}
/* Optimize: vlen basetype is a fixed-size user-type */
if((stat = nc4_find_type(file,basetypeid,&basetype))) goto done;
if(!basetype->varsized) {
goto out;
}
/* Remaining case: basetype is itself variable size => recurse */
if((stat = NC_type_alignment_internal(file,basetypeid,basetype,&alignment))) goto done;;
vinstance.memory = (char*)vlen->p; /* use char* so we can do pointer arithmetic */
vinstance.memory = (void*)NC_read_align((uintptr_t)vinstance.memory,alignment);
for(i=0;i<vlen->len;i++) {
if((stat=reclaim_datar(file,basetype,vinstance))) goto done; /* reclaim one basetype instance */
vinstance.memory += basetype->size; /* move to next base instance */
}
out:
if(vlen->len > 0 && vlen->p != NULL) {free(vlen->p);}
goto done;
} else if(utype->nc_type_class == NC_COMPOUND) {
Position finstance; /* mark the fields's instance */
nfields = nclistlength(utype->u.c.field);
/* Get info about each field in turn and reclaim it */
for(fid=0;fid<nfields;fid++) {
NC_FIELD_INFO_T* field = NULL;
/* Get field's dimension sizes */
field = (NC_FIELD_INFO_T*)nclistget(utype->u.c.field,fid);
ndims = field->ndims;
arraycount = 1;
for(i=0;i<ndims;i++) {dimsizes[i] = field->dim_size[i]; arraycount *= dimsizes[i];}
if(field->ndims == 0) {ndims=1; dimsizes[0]=1;} /* fake the scalar case */
/* "Move" to start of this field's instance */
finstance.memory = instance.memory + field->offset; /* includes proper alignment */
/* optimize: fixed length atomic type */
if(field->nc_typeid < NC_STRING) continue;
/* optimize: string field type */
if(field->nc_typeid == NC_STRING) {
char** strvec = (char**)finstance.memory;
for(i=0;i<arraycount;i++) {
if(strvec[i] != NULL) free(strvec[i]);
}
continue; /* do next field */
}
/* optimize: fixed length compound base type */
if((stat = nc4_find_type(file,field->nc_typeid,&basetype))) goto done;
if(!basetype->varsized) continue;
/* Field is itself variable length (possibly transitively) */
for(i=0;i<arraycount;i++) {
if((stat = reclaim_datar(file, basetype, finstance))) goto done;
finstance.memory += basetype->size;
}
}
goto done;
} else {stat = NC_EBADTYPE; goto done;}
done:
return stat;
}
#endif
/**************************************************/
/**
Copy a vector of instances of a type. This recursively walks
the top-level instances to copy any nested data such as vlen or
strings or such.
Assumes it is passed a pointer to count instances of xtype and a
space into which to copy the instance. Copys any nested data
by calling malloc().
WARNING: DOES NOT ALLOCATE THE TOP-LEVEL MEMORY (see the
nc_copy_data_all function). Note that all memory blocks other
than the top are dynamically allocated.
Should work for any netcdf type.
@param file NC_FILE_INFO_T* structure
@param xtype type id
@param memory ptr to top-level memory to reclaim
@param count number of instances of the type in memory block
@param copy top-level space into which to copy the instance
@return error code
*/
int
NC_copy_data(NC* nc, nc_type xtype, const void* memory, size_t count, void* copy)
{
int stat = NC_NOERR;
size_t i;
Position src;
Position dst;
NC_FILE_INFO_T* file = NULL;
NC_TYPE_INFO_T* utype = NULL;
size_t typesize = 0;
if(memory == NULL || count == 0)
goto done; /* ok, do nothing */
assert(nc != NULL);
assert(memory != NULL || count > 0);
assert(copy != NULL || count == 0);
/* Process atomic types */
/* Optimize: Vector of fixed size atomic types */
if(xtype < NC_STRING) {
typesize = NC_atomictypelen(xtype);
memcpy(copy,memory,count*typesize);
goto done;
}
#ifdef USE_NETCDF4
/* Optimize: Vector of strings */
if(xtype == NC_STRING) {
char** svec = (char**)memory;
char** dvec = (char**)copy;
size_t len;
for(i=0;i<count;i++) {
const char* ssrc = svec[i];
char* sdst;
if(ssrc == NULL)
sdst = NULL;
else {
len = nulllen(ssrc);
if((sdst = (char*)malloc(len+1))==NULL) {stat = NC_ENOMEM; goto done;}
memcpy(sdst,ssrc,len+1); /* +1 for trailing nul */
}
dvec[i] = sdst;
}
goto done;
}
assert(USEFILEINFO(nc) != 0);
file = (NC_FILE_INFO_T*)(nc)->dispatchdata;
/* Process User types */
if((stat = nc4_find_type(file,xtype,&utype))) goto done;
/* Optimize: vector of fixed sized compound type instances */
if(!utype->varsized) {
memcpy(copy,memory,count*utype->size);
goto done;
}
/* Remaining cases: vector of VLEN and vector of variable sized compound types.
These all require potential recursion.
*/
src.memory = (char*)memory; /* use char* so we can do pointer arithmetic */
dst.memory = (char*)copy; /* use char* so we can do pointer arithmetic */
/* Walk each vector instance */
for(i=0;i<count;i++) {
if((stat=copy_datar(file,utype,src,dst))) goto done;
src.memory += utype->size;
dst.memory += utype->size;
}
#else
stat = NC_EBADTYPE;
#endif
done:
return stat;
}
#ifdef USE_NETCDF4
/* Recursive type walker: reclaim an instance of variable-sized user-defined types;
specifically a vlen or a variable-sized compound type instance
*/
static int
copy_datar(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* utype, Position src, Position dst)
{
int i, stat = NC_NOERR;
nc_type basetypeid;
NC_TYPE_INFO_T* basetype = NULL;
size_t nfields;
nc_vlen_t* srcvlens = NULL;
nc_vlen_t* dstvlens = NULL;
size_t fid, arraycount;
int ndims;
int dimsizes[NC_MAX_VAR_DIMS];
Position vsrc, vdst;
size_t alignment = 0;
assert(utype->varsized); /* All fixed size cases are optimized out */
/* Leaving VLEN or Compound */
if(utype->nc_type_class == NC_VLEN) {
size_t basetypesize = 0;
size_t copycount = 0;
basetypeid = utype->u.v.base_nc_typeid; /* Get basetype */
srcvlens = (nc_vlen_t*)src.memory;
dstvlens = (nc_vlen_t*)dst.memory;
dstvlens->len = srcvlens->len; /* always */
if(srcvlens->len == 0) {
dstvlens->p = NULL;
goto done;
}
if(basetypeid <= NC_MAX_ATOMIC_TYPE)
basetypesize = NC_atomictypelen(basetypeid);
copycount = srcvlens->len*basetypesize;
/* Optimize: basetype is atomic fixed size */
if(basetypeid < NC_STRING) {
if((dstvlens->p = (void*)malloc(copycount))==NULL) {stat = NC_ENOMEM; goto done;}
memcpy(dstvlens->p,srcvlens->p,copycount);
goto done;
}
/* Optimize: basetype is string */
if(basetypeid == NC_STRING) {
char** srcstrvec = (char**)srcvlens->p;
char** dststrvec = NULL;
if((dststrvec = (void*)malloc(copycount))==NULL) {stat = NC_ENOMEM; goto done;}
dstvlens->p = (void*)dststrvec;
for(i=0;i<srcvlens->len;i++) {
if((dststrvec[i] = strdup(srcstrvec[i]))==NULL) {stat = NC_ENOMEM; goto done;}
}
goto done;
}
/* User-defined type */
/* Recompute base type size */
if((stat = nc4_find_type(file,basetypeid,&basetype))) goto done;
basetypesize = basetype->size;
copycount = srcvlens->len*basetypesize;
/* Optimize: basetype is user-type fixed size */
if(!basetype->varsized) {
if((dstvlens->p = (void*)malloc(copycount))==NULL) {stat = NC_ENOMEM; goto done;}
memcpy(dstvlens->p,srcvlens->p,copycount);
goto done;
}
/* Remaining case: basetype is itself variable size => recurse */
if((stat = NC_type_alignment_internal(file,basetypeid,basetype,&alignment))) goto done;;
vsrc.memory = (char*)srcvlens->p; /* use char* so we can do pointer arithmetic */
if((vdst.memory = (char*)malloc(copycount))==NULL) {stat = NC_ENOMEM; goto done;}
dstvlens->p = vdst.memory; /* don't lose it */
vsrc.memory = (void*)NC_read_align((uintptr_t)vsrc.memory,alignment);
vdst.memory = (void*)NC_read_align((uintptr_t)vdst.memory,alignment);
for(i=0;i<srcvlens->len;i++) {
if((stat=copy_datar(file,basetype,vsrc,vdst))) goto done;
vsrc.memory += basetype->size;
vdst.memory += basetype->size;
}
goto done;
} else if(utype->nc_type_class == NC_COMPOUND) {
Position fsrc; /* mark the src fields's instance */
Position fdst; /* mark the dst fields's instance */
nfields = nclistlength(utype->u.c.field);
/* Get info about each field in turn and copy it */
for(fid=0;fid<nfields;fid++) {
NC_FIELD_INFO_T* field = NULL;
field = (NC_FIELD_INFO_T*)nclistget(utype->u.c.field,fid);
ndims = field->ndims;
arraycount = 1;
for(i=0;i<ndims;i++) {dimsizes[i] = field->dim_size[i]; arraycount *= dimsizes[i];}
if(field->ndims == 0) {ndims=1; dimsizes[0]=1;} /* fake the scalar case */
/* "move" to this field */
fsrc.memory = src.memory + field->offset;
fdst.memory = dst.memory + field->offset;
/* optimize: fixed length atomic type */
if(field->nc_typeid < NC_STRING) {
size_t typesize = NC_atomictypelen(field->nc_typeid);
memcpy(fdst.memory,fsrc.memory,arraycount*typesize);
continue; /* move to next field */
}
/* optimize: string field type */
if(field->nc_typeid == NC_STRING) {
char** srcstrvec = (char**)src.memory;
char** dststrvec = (char**)dst.memory;
for(i=0;i<arraycount;i++)
{if(srcstrvec[i] != NULL) {dststrvec[i] = strdup(srcstrvec[i]);} else {dststrvec[i] = NULL;}}
continue; /* move to next field */
}
/* optimize: fixed length compound base type */
if((stat = nc4_find_type(file,field->nc_typeid,&basetype))) goto done;
if(!basetype->varsized) {
memcpy(fdst.memory,fsrc.memory,arraycount*basetype->size);
continue; /* move to next field */
}
/* Remaining case; field type is variable type */
for(i=0;i<arraycount;i++) {
if((stat = copy_datar(file, basetype, fsrc, fdst))) goto done;
fsrc.memory += basetype->size;
fdst.memory += basetype->size;
}
}
goto done;
} else {stat = NC_EBADTYPE; goto done;}
done:
return stat;
}
#endif
/**************************************************/
/* Alignment functions */
#ifdef USE_NETCDF4
uintptr_t
NC_read_align(uintptr_t addr, size_t alignment)
{
size_t loc_align = (alignment == 0 ? 1 : alignment);
size_t delta = (addr % loc_align);
if(delta == 0) return addr;
return (addr + (alignment - delta));
}
/**
* Compute proper data alignment for a type
* @param file - only needed for a compound type
* @param xtype - type for which alignment is requested
* @param utype - if known
* @return 0 if not found
*/
int
NC_type_alignment_internal(NC_FILE_INFO_T* file, nc_type xtype, NC_TYPE_INFO_T* utype, size_t* alignp)
{
int stat = NC_NOERR;
size_t align = 0;
int klass;
if(!type_alignment_initialized) {
NC_compute_alignments();
type_alignment_initialized = 1;
}
if(xtype <= NC_MAX_ATOMIC_TYPE)
{stat = NC_class_alignment(xtype,&align); goto done;}
else {/* Presumably a user type */
if(utype == NULL) {
if((stat = nc4_find_type(file,xtype,&utype))) goto done;
}
klass = utype->nc_type_class;
switch(klass) {
case NC_VLEN: stat = NC_class_alignment(klass,&align); break;
case NC_OPAQUE: stat = NC_class_alignment(klass,&align); break;
case NC_COMPOUND: {/* get alignment of the first field of the compound */
NC_FIELD_INFO_T* field = NULL;
NC_TYPE_INFO_T* basetype = NULL;
if(nclistlength(utype->u.c.field) == 0) {stat = NC_EINVAL; goto done;}
field = (NC_FIELD_INFO_T*)nclistget(utype->u.c.field,0);
if(field->nc_typeid <= NC_MAX_ATOMIC_TYPE) {
if((stat = nc4_find_type(file,field->nc_typeid,&basetype))) goto done;
}
stat = NC_type_alignment_internal(file,field->nc_typeid,basetype,&align); /* may recurse repeatedly */
} break;
default: break;
}
}
if(alignp) *alignp = align;
done:
#if 0
Why was this here?
if(stat == NC_NOERR && align == 0) stat = NC_EINVAL;
#endif
return stat;
}
#endif
/**************************************************/
/* Internal versions of the XX_all functions */
/* Alternate entry point: includes recovering the top-level memory */
int
NC_reclaim_data_all(NC* nc, nc_type xtypeid, void* memory, size_t count)
{
int stat = NC_NOERR;
assert(nc != NULL);
stat = NC_reclaim_data(nc,xtypeid,memory,count);
if(stat == NC_NOERR && memory != NULL)
free(memory);
return stat;
}
/* Alternate entry point: includes recovering the top-level memory */
int
NC_copy_data_all(NC* nc, nc_type xtype, const void* memory, size_t count, void** copyp)
{
int stat = NC_NOERR;
size_t xsize = 0;
void* copy = NULL;
NC_FILE_INFO_T* file = NULL;
NC_TYPE_INFO_T* utype = NULL;
assert(nc != NULL);
if(xtype <= NC_STRING && count > 0) {
xsize = NC_atomictypelen(xtype);
if((copy = calloc(xsize,count))==NULL) {stat = NC_ENOMEM; goto done;}
if(xtype < NC_STRING) /* fixed-size atomic type */
memcpy(copy,memory,xsize*count);
else { /* string type */
size_t i;
char** strvec = (char**)memory;
char** scopyvec = (char**)copy;
for(i=0;i<count;i++) {
char* s = strvec[i];
char* sdup = NULL;
if(s != NULL) sdup = strdup(s);
scopyvec[i] = sdup;
}
}
}
#ifdef USE_NETCDF4
else {
file = (NC_FILE_INFO_T*)(nc)->dispatchdata;
if((stat = nc4_find_type(file,xtype,&utype))) goto done;
xsize = utype->size;
/* allocate the top-level */
if(count > 0) {
if((copy = calloc(xsize,count))==NULL) {stat = NC_ENOMEM; goto done;}
}
if((stat = NC_copy_data(nc,xtype,memory,count,copy)))
(void)NC_reclaim_data_all(nc,xtype,copy,count);
}
#endif
if(copyp) {*copyp = copy; copy = NULL;}
done:
return stat;
}
/* Alternate entry point: includes recovering the top-level memory */
int
NC_print_data(NC_FILE_INFO_T* file, nc_type xtype, const void* memory, size_t count)
{
EXTERNL int nc_print_data(int ncid, nc_type xtype, const void* memory, size_t count);
return nc_print_data(file->controller->ext_ncid,xtype,memory,count);
}

13
libdispatch/drc.c
View File

@ -569,11 +569,12 @@ rcequal(NCRCentry* e1, NCRCentry* e2)
nulltest = 0;
if(e1->host == NULL) nulltest |= 1;
if(e2->host == NULL) nulltest |= 2;
/* Use host to decide if entry applies */
switch (nulltest) {
case 0: if(strcmp(e1->host,e2->host) != 0) {return 0;} break;
case 1: return 0;
case 2: return 0;
case 3: break;
case 1: break; /* .rc->host == NULL && candidate->host != NULL */
case 2: return 0; /* .rc->host != NULL && candidate->host == NULL */
case 3: break; /* .rc->host == NULL && candidate->host == NULL */
default: return 0;
}
/* test urlpath take NULL into account*/
@ -582,9 +583,9 @@ rcequal(NCRCentry* e1, NCRCentry* e2)
if(e2->urlpath == NULL) nulltest |= 2;
switch (nulltest) {
case 0: if(strcmp(e1->urlpath,e2->urlpath) != 0) {return 0;} break;
case 1: return 0;
case 2: return 0;
case 3: break;
case 1: break; /* .rc->urlpath == NULL && candidate->urlpath != NULL */
case 2: return 0; /* .rc->urlpath != NULL && candidate->urlpath == NULL */
case 3: break; /* .rc->urlpath == NULL && candidate->urlpath == NULL */
default: return 0;
}
return 1;

13
libdispatch/dvar.c
View File

@ -1291,19 +1291,18 @@ NC_check_nulls(int ncid, int varid, const size_t *start, size_t **count,
/**
@name Free String Resources
Use this functions to free resources associated with ::NC_STRING
data.
Use these functions to free resources associated with ::NC_STRING data.
*/
/*! @{ */
/**
Free string space allocated by the library.
When you read string type the library will allocate the storage
space for the data. This storage space must be freed, so pass the
pointer back to this function, when you're done with the data, and
it will free the string memory.
When you read an array string typed data the library will allocate the storage
space for the data. The allocated strings must be freed, so pass the
pointer to the array plus a count of the number of elements in the array to this function,
when you're done with the data, and it will free the allocated string memory.
WARNING: This does not free the data vector itself, only
WARNING: This does not free the top-level array itself, only
the strings to which it points.
@param len The number of character arrays in the array.

92
libdispatch/dvlen.c
View File

@ -13,30 +13,48 @@
/**
\ingroup user_types
Free memory in a VLEN object.
Free an array of vlens given the number of elements and an array.
When you read VLEN type the library will actually allocate the storage
space for the data. This storage space must be freed, so pass the
pointer back to this function, when you're done with the data, and it
will free the vlen memory.
When you read an array of VLEN typed instances, the library will allocate
the storage space for the data in each VLEN in the array (but not the array itself).
That VLEN data must be freed eventually, so pass the pointer to the array plus
the number of elements in the array to this function when you're done with
the data, and it will free the all the VLEN instances.
The caller is still responsible for free'ing the array itself,
if it was dynamically allocated.
The function nc_free_vlens() is more useful than this function,
because it can free an array of VLEN objects.
WARNING: this function only works if the basetype of the vlen type
is fixed size. This means it is an atomic type except NC_STRING,
or an NC_ENUM, or and NC_OPAQUE, or an NC_COMPOUND where all
the fields of the compound type are themselves fixed size.
WARNING: this code is incorrect because it will only
work if the basetype of the vlen is
- atomic (excluding string basetype)
- + enum
- + opaque
If you have a more complex VLEN base-type, then it is better to call
the "nc_reclaim_data" function.
The reason is that to operate properly, it needs to recurse when
the basetype is a complex object such as another vlen or compound.
\param nelems number of elements in the array.
\param vlens pointer to the vlen object.
This function is deprecated in favor of the function "nc_reclaim_data".
See include/netcdf.h.
\returns ::NC_NOERR No error.
*/
int
nc_free_vlens(size_t nelems, nc_vlen_t vlens[])
{
int ret;
size_t i;
for(i = 0; i < nelems; i++)
if ((ret = nc_free_vlen(&vlens[i])))
return ret;
return NC_NOERR;
}
/**
\ingroup user_types
Free memory in a single VLEN object.
This function is equivalent to calling *nc_free_vlens* with nelems == 1.
\param vl pointer to the vlen object.
\returns ::NC_NOERR No error.
*/
int
@ -46,46 +64,6 @@ nc_free_vlen(nc_vlen_t *vl)
return NC_NOERR;
}
/**
\ingroup user_types
Free an array of vlens given the number of elements and an array.
When you read VLEN type the library will actually allocate the storage
space for the data. This storage space must be freed, so pass the
pointer back to this function, when you're done with the data, and it
will free the vlen memory.
WARNING: this code is incorrect because it will only
work if the basetype of the vlen is
- atomic
- + enum
- + opaque
- excluding string basetype,
The reason is that to operate properly, it needs to recurse when
the basetype is a complex object such as another vlen or compound.
This function is deprecated in favor of the function "nc_reclaim_data".
See include/netcdf.h.
\param len number of elements in the array.
\param vlens pointer to the vlen object.
\returns ::NC_NOERR No error.
*/
int
nc_free_vlens(size_t len, nc_vlen_t vlens[])
{
int ret;
size_t i;
for(i = 0; i < len; i++)
if ((ret = nc_free_vlen(&vlens[i])))
return ret;
return NC_NOERR;
}
/**
\ingroup user_types
Use this function to define a variable length array type.

7
libdispatch/ncs3sdk_aws.cpp
View File

@ -24,11 +24,13 @@
#define NCTRACEMORE(level,fmt,...) nctracemore((level),fmt,##__VA_ARGS__)
#define NCUNTRACE(e) ncuntrace(__func__,THROW(e),NULL)
#define NCUNTRACEX(e,fmt,...) ncuntrace(__func__,THROW(e),fmt,##__VA_ARGS__)
#define NCUNTRACENOOP(e) NCUNTRACE(e)
#else
#define NCTRACE(level,fmt,...)
#define NCTRACEMORE(level,fmt,...)
#define NCUNTRACE(e) (e)
#define NCUNTRACEX(e,fmt,...) (e)
#define NCUNTRACENOOP(e)
#endif
#ifdef __CYGWIN__
@ -150,7 +152,6 @@ makeerrmsg(const Aws::Client::AWSError<Aws::S3::S3Errors> err, const char* key="
static Aws::Client::ClientConfiguration
s3sdkcreateconfig(NCS3INFO* info)
{
int stat = NC_NOERR;
NCTRACE(11,"info=%s", dumps3info(info));
Aws::Client::ClientConfiguration config;
@ -164,7 +165,7 @@ s3sdkcreateconfig(NCS3INFO* info)
if(info->host) config.endpointOverride = info->host;
config.enableEndpointDiscovery = true;
config.followRedirects = Aws::Client::FollowRedirectsPolicy::ALWAYS;
stat = NCUNTRACE(stat);
NCUNTRACENOOP(NC_NOERR);
return config;
}
@ -190,7 +191,7 @@ NC_s3sdkcreateclient(NCS3INFO* info)
false);
}
// delete config;
stat = NCUNTRACE(stat);
NCUNTRACENOOP(NC_NOERR);
return (void*)s3client;
}

16
libhdf5/hdf5attr.c
View File

@ -316,7 +316,7 @@ NC4_HDF5_del_att(int ncid, int varid, const char *name)
/* Reclaim the content of the attribute */
if(att->data)
if((retval = nc_reclaim_data_all(ncid,att->nc_typeid,att->data,att->len))) return retval;
if((retval = NC_reclaim_data_all(h5->controller,att->nc_typeid,att->data,att->len))) return retval;
att->data = NULL;
att->len = 0;
@ -618,7 +618,7 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
} else { /* no conversion */
/* Still need a copy of the input data */
copy = NULL;
if((retval = nc_copy_data_all(h5->controller->ext_ncid, mem_type, data, 1, &copy)))
if((retval = NC_copy_data_all(h5->controller, mem_type, data, 1, &copy)))
BAIL(retval);
}
var->fill_value = copy;
@ -652,7 +652,7 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
NC_NOQUANTIZE, 0)))
BAIL(retval);
} else if(mem_type == file_type) { /* General case: no conversion */
if((retval = nc_copy_data(h5->controller->ext_ncid,file_type,data,len,copy)))
if((retval = NC_copy_data(h5->controller,file_type,data,len,copy)))
BAIL(retval);
} else
BAIL(NC_EURL);
@ -670,30 +670,30 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
/* Reclaim saved data */
if(attsave.data != NULL) {
assert(attsave.len > 0);
(void)nc_reclaim_data_all(h5->controller->ext_ncid,attsave.type,attsave.data,attsave.len);
(void)NC_reclaim_data_all(h5->controller,attsave.type,attsave.data,attsave.len);
attsave.len = 0; attsave.data = NULL;
}
if(fillsave.data != NULL) {
assert(fillsave.len > 0);
(void)nc_reclaim_data_all(h5->controller->ext_ncid,fillsave.type,fillsave.data,fillsave.len);
(void)NC_reclaim_data_all(h5->controller,fillsave.type,fillsave.data,fillsave.len);
fillsave.len = 0; fillsave.data = NULL;
}
exit:
if(copy)
(void)nc_reclaim_data_all(h5->controller->ext_ncid,file_type,copy,len);
(void)NC_reclaim_data_all(h5->controller,file_type,copy,len);
if(retval) {
/* Rollback */
if(attsave.data != NULL) {
assert(attsave.len > 0);
if(att->data)
(void)nc_reclaim_data_all(h5->controller->ext_ncid,attsave.type,att->data,att->len);
(void)NC_reclaim_data_all(h5->controller,attsave.type,att->data,att->len);
att->len = attsave.len; att->data = attsave.data;
}
if(fillsave.data != NULL) {
assert(fillsave.len > 0);
if(att->data)
(void)nc_reclaim_data_all(h5->controller->ext_ncid,fillsave.type,var->fill_value,1);
(void)NC_reclaim_data_all(h5->controller,fillsave.type,var->fill_value,1);
var->fill_value = fillsave.data;
}
}

2
libhdf5/hdf5internal.c
View File

@ -614,7 +614,7 @@ close_vars(NC_GRP_INFO_T *grp)
if (var->type_info)
{
int stat = NC_NOERR;
if((stat = nc_reclaim_data(grp->nc4_info->controller->ext_ncid,var->type_info->hdr.id,var->fill_value,1)))
if((stat = NC_reclaim_data(grp->nc4_info->controller,var->type_info->hdr.id,var->fill_value,1)))
return stat;
nullfree(var->fill_value);
}

17
libhdf5/hdf5open.c
View File

@ -183,6 +183,7 @@ get_type_info2(NC_FILE_INFO_T *h5, hid_t datasetid, NC_TYPE_INFO_T **type_info)
/* Set a class for the type */
t = NUM_TYPES - 1;
(*type_info)->nc_type_class = NC_STRING;
NC4_set_varsize(*type_info);
}
else
{
@ -2014,6 +2015,7 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name)
{
case H5T_STRING:
type->nc_type_class = NC_STRING;
if((retval = NC4_set_varsize(type))) return retval;
break;
case H5T_COMPOUND:
@ -2026,6 +2028,7 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name)
#endif
type->nc_type_class = NC_COMPOUND;
if((retval = NC4_set_varsize(type))) return retval;
if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0)
return NC_EHDFERR;
@ -2045,7 +2048,6 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name)
* compound type. */
if ((member_hdf_typeid = H5Tget_member_type(native_typeid, m)) < 0)
return NC_EHDFERR;
if ((member_native_typeid = H5Tget_native_type(member_hdf_typeid,
H5T_DIR_DEFAULT)) < 0)
return NC_EHDFERR;
@ -2107,11 +2109,8 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name)
return retval;
}
{ /* See if this changes from fixed size to variable size */
int fixedsize;
if((retval = NC4_inq_type_fixed_size(grp->nc4_info->controller->ext_ncid,member_xtype,&fixedsize))) return retval;
if(!fixedsize) type->u.c.varsized = 1;
}
/* See if this changes from fixed size to variable size */
if((retval=NC4_recheck_varsize(type,member_xtype))) return retval;
hdf5free(member_name);
}
@ -2156,11 +2155,13 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name)
/* Remember the base type for this vlen. */
type->u.v.base_nc_typeid = base_nc_type;
}
if((retval = NC4_set_varsize(type))) return retval;
}
break;
case H5T_OPAQUE:
type->nc_type_class = NC_OPAQUE;
if((retval = NC4_set_varsize(type))) return retval;
break;
case H5T_ENUM:
@ -2175,7 +2176,8 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name)
#endif
type->nc_type_class = NC_ENUM;
if((retval = NC4_set_varsize(type))) return retval;
/* Find the base type of this enum (i.e. what is this a
* enum of?) */
if (!(base_hdf_typeid = H5Tget_super(hdf_typeid)))
@ -2236,6 +2238,7 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name)
LOG((0, "unknown class"));
return NC_EBADCLASS;
}
return retval;
}

24
libhdf5/hdf5type.c
View File

@ -176,13 +176,22 @@ add_user_type(int ncid, size_t size, const char *name, nc_type base_typeid,
/* Remember info about this type. */
type->nc_type_class = type_class;
if (type_class == NC_VLEN)
type->u.v.base_nc_typeid = base_typeid;
else if (type_class == NC_ENUM) {
switch(type_class) {
case NC_ENUM:
type->u.e.base_nc_typeid = base_typeid;
type->u.e.enum_member = nclistnew();
} else if (type_class == NC_COMPOUND)
break;
case NC_OPAQUE:
break;
case NC_VLEN:
type->u.v.base_nc_typeid = base_typeid;
break;
case NC_COMPOUND:
type->u.c.field = nclistnew();
break;
default: break;
}
if((retval=NC4_set_varsize(type))) return retval;
/* Return the typeid to the user. */
if (typeidp)
@ -263,7 +272,6 @@ NC4_insert_array_compound(int ncid, int typeid1, const char *name,
NC_TYPE_INFO_T *type;
char norm_name[NC_MAX_NAME + 1];
int retval;
int fixedsize = 0;
LOG((2, "nc_insert_array_compound: ncid 0x%x, typeid %d name %s "
"offset %d field_typeid %d ndims %d", ncid, typeid1,
@ -296,11 +304,7 @@ NC4_insert_array_compound(int ncid, int typeid1, const char *name,
return retval;
/* See if this changes from fixed size to variable size */
if((retval = NC4_inq_type_fixed_size(ncid,field_typeid,&fixedsize)))
return retval;
if(!fixedsize)
type->u.c.varsized = 1;
if((retval=NC4_recheck_varsize(type,field_typeid))) return retval;
return NC_NOERR;
}

4
libhdf5/hdf5var.c
View File

@ -684,7 +684,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
/* If there's a _FillValue attribute, delete it. */
retval = NC4_HDF5_del_att(ncid, varid, _FillValue);
if (retval && retval != NC_ENOTATT) return retval;
if((retval = nc_reclaim_data_all(ncid,var->type_info->hdr.id,var->fill_value,1))) return retval;
if((retval = NC_reclaim_data_all(h5->controller,var->type_info->hdr.id,var->fill_value,1))) return retval;
var->fill_value = NULL;
}
@ -2186,7 +2186,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
{
/* Copy one instance of the fill_value */
if((retval = nc_copy_data(ncid,var->type_info->hdr.id,fillvalue,1,filldata)))
if((retval = NC_copy_data(h5->controller,var->type_info->hdr.id,fillvalue,1,filldata)))
BAIL(retval);
}
filldata = (char *)filldata + file_type_size;

24
libnczarr/zattr.c
View File

@ -326,7 +326,7 @@ NCZ_del_att(int ncid, int varid, const char *name)
/* Reclaim the content of the attribute */
if(att->data) {
if((retval = nc_reclaim_data_all(ncid,att->nc_typeid,att->data,att->len))) return retval;
if((retval = NC_reclaim_data_all(h5->controller,att->nc_typeid,att->data,att->len))) return retval;
}
att->data = NULL;
att->len = 0;
@ -628,7 +628,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
} else { /* no conversion */
/* Still need a copy of the input data */
copy = NULL;
if((retval = nc_copy_data_all(h5->controller->ext_ncid, mem_type, data, 1, &copy)))
if((retval = NC_copy_data_all(h5->controller, mem_type, data, 1, &copy)))
BAIL(retval);
}
var->fill_value = copy;
@ -665,7 +665,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
NC_NOQUANTIZE, 0)))
BAIL(retval);
} else if(mem_type == file_type) { /* General case: no conversion */
if((retval = nc_copy_data(h5->controller->ext_ncid,file_type,data,len,copy)))
if((retval = NC_copy_data(h5->controller,file_type,data,len,copy)))
BAIL(retval);
} else
BAIL(NC_EURL);
@ -700,30 +700,30 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
/* Reclaim saved data */
if(attsave.data != NULL) {
assert(attsave.len > 0);
(void)nc_reclaim_data_all(h5->controller->ext_ncid,attsave.type,attsave.data,attsave.len);
(void)NC_reclaim_data_all(h5->controller,attsave.type,attsave.data,attsave.len);
attsave.len = 0; attsave.data = NULL;
}
if(fillsave.data != NULL) {
assert(fillsave.len > 0);
(void)nc_reclaim_data_all(h5->controller->ext_ncid,fillsave.type,fillsave.data,fillsave.len);
(void)NC_reclaim_data_all(h5->controller,fillsave.type,fillsave.data,fillsave.len);
fillsave.len = 0; fillsave.data = NULL;
}
exit:
if(copy)
(void)nc_reclaim_data_all(h5->controller->ext_ncid,file_type,copy,len);
(void)NC_reclaim_data_all(h5->controller,file_type,copy,len);
if(retval) {
/* Rollback */
if(attsave.data != NULL) {
assert(attsave.len > 0);
if(att->data)
(void)nc_reclaim_data_all(h5->controller->ext_ncid,attsave.type,att->data,att->len);
(void)NC_reclaim_data_all(h5->controller,attsave.type,att->data,att->len);
att->len = attsave.len; att->data = attsave.data;
}
if(fillsave.data != NULL) {
assert(fillsave.len > 0);
if(att->data)
(void)nc_reclaim_data_all(h5->controller->ext_ncid,fillsave.type,var->fill_value,1);
(void)NC_reclaim_data_all(h5->controller,fillsave.type,var->fill_value,1);
var->fill_value = fillsave.data;
}
}
@ -1014,20 +1014,14 @@ ncz_makeattr(NC_OBJ* container, NCindex* attlist, const char* name, nc_type type
NCZ_ATT_INFO_T* zatt = NULL;
void* clone = NULL;
size_t typesize, clonesize;
int ncid;
NC* nc = NULL;
NC_GRP_INFO_T* grp = (container->sort == NCGRP ? (NC_GRP_INFO_T*)container
: ((NC_VAR_INFO_T*)container)->container);
nc = grp->nc4_info->controller;
ncid = nc->ext_ncid | grp->hdr.id;
/* Duplicate the values */
if ((stat = nc4_get_typelen_mem(grp->nc4_info, typeid, &typesize))) goto done;
clonesize = len*typesize;
if((clone = malloc(clonesize))==NULL) {stat = NC_ENOMEM; goto done;}
if((stat = nc_copy_data(ncid, typeid, values, len, clone))) goto done;
if((stat = NC_copy_data(grp->nc4_info->controller, typeid, values, len, clone))) goto done;
if((stat=nc4_att_list_add(attlist,name,&att)))
goto done;
if((zatt = calloc(1,sizeof(NCZ_ATT_INFO_T))) == NULL)

2
libnczarr/zinternal.c
View File

@ -330,7 +330,7 @@ close_vars(NC_GRP_INFO_T *grp)
if (var->type_info)
{
int stat = NC_NOERR;
if((stat = nc_reclaim_data(grp->nc4_info->controller->ext_ncid,var->type_info->hdr.id,var->fill_value,1)))
if((stat = NC_reclaim_data(grp->nc4_info,var->type_info->hdr.id,var->fill_value,1)))
return stat;
nullfree(var->fill_value);
}

4
libnczarr/zsync.c
View File

@ -1341,7 +1341,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container)
if((stat = ncz_makeattr(container,attlist,aname,typeid,len,data,&att)))
goto done;
/* No longer need this copy of the data */
if((stat = nc_reclaim_data_all(file->controller->ext_ncid,att->nc_typeid,data,len))) goto done;
if((stat = NC_reclaim_data_all(file->controller,att->nc_typeid,data,len))) goto done;
data = NULL;
if(isfillvalue)
fillvalueatt = att;
@ -1365,7 +1365,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container)
done:
if(data != NULL)
stat = nc_reclaim_data(file->controller->ext_ncid,att->nc_typeid,data,len);
stat = NC_reclaim_data(file->controller,att->nc_typeid,data,len);
NCJreclaim(jattrs);
nclistfreeall(atypes);
nullfree(fullpath);

4
libnczarr/ztype.c
View File

@ -228,6 +228,8 @@ add_user_type(int ncid, size_t size, const char *name, nc_type base_typeid,
} else if (type_class == NC_COMPOUND)
type->u.c.field = nclistnew();
(void)NC4_set_varsize(type);
/* Return the typeid to the user. */
if (typeidp)
*typeidp = type->hdr.id;
@ -338,6 +340,8 @@ NCZ_insert_array_compound(int ncid, int typeid1, const char *name,
ndims, dim_sizesp)))
return retval;
NC4_recheck_varsize(type,field_typeid);
return NC_NOERR;
}

10
libnczarr/zutil.c
View File

@ -948,9 +948,8 @@ NCZ_reclaim_fill_value(NC_VAR_INFO_T* var)
{
int stat = NC_NOERR;
if(var->fill_value) {
int ncid = var->container->nc4_info->controller->ext_ncid;
int tid = var->type_info->hdr.id;
stat = nc_reclaim_data_all(ncid,tid,var->fill_value,1);
stat = NC_reclaim_data_all(var->container->nc4_info->controller,tid,var->fill_value,1);
var->fill_value = NULL;
}
/* Reclaim any existing fill_chunk */
@ -962,16 +961,15 @@ int
NCZ_copy_fill_value(NC_VAR_INFO_T* var, void** dstp)
{
int stat = NC_NOERR;
int ncid = var->container->nc4_info->controller->ext_ncid;
int tid = var->type_info->hdr.id;
void* dst = NULL;
if(var->fill_value) {
if((stat = nc_copy_data_all(ncid,tid,var->fill_value,1,&dst))) goto done;
if((stat = NC_copy_data_all(var->container->nc4_info->controller,tid,var->fill_value,1,&dst))) goto done;
}
if(dstp) {*dstp = dst; dst = NULL;}
done:
if(dst) (void)nc_reclaim_data_all(ncid,tid,dst,1);
if(dst) (void)NC_reclaim_data_all(var->container->nc4_info->controller,tid,dst,1);
return stat;
}
@ -1057,7 +1055,7 @@ NCZ_copy_data(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* xtype, const void* memory, s
scopy[i] = NULL;
}
}
return nc_copy_data(file->controller->ext_ncid,xtype->hdr.id,memory,count,copy);
return NC_copy_data(file->controller,xtype->hdr.id,memory,count,copy);
}
#if 0

2
libnczarr/zvar.c
View File

@ -2127,7 +2127,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
for (i = 0; i < fill_len; i++)
{
/* Copy one instance of the fill_value */
if((retval = nc_copy_data(ncid,var->type_info->hdr.id,var->fill_value,1,filldata)))
if((retval = NC_copy_data(h5->controller,var->type_info->hdr.id,var->fill_value,1,filldata)))
BAIL(retval);
filldata = (char *)filldata + file_type_size;
}

5
libnczarr/zwalk.c
View File

@ -755,19 +755,20 @@ EXTERNL int
NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata)
{
int stat = NC_NOERR;
NC_FILE_INFO_T* h5 = NULL;
NC_VAR_INFO_T* var = NULL;
NCZ_VAR_INFO_T* zvar = NULL;
struct NCZChunkCache* cache = NULL;
void* cachedata = NULL;
if ((stat = nc4_find_grp_h5_var(ncid, varid, NULL, NULL, &var)))
if ((stat = nc4_find_grp_h5_var(ncid, varid, &h5, NULL, &var)))
return THROW(stat);
zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
cache = zvar->cache;
if((stat = NCZ_read_cache_chunk(cache,zindices,&cachedata))) goto done;
if(chunkdata) {
if((stat = nc_copy_data(ncid,var->type_info->hdr.id,cachedata,cache->chunkcount,chunkdata))) goto done;
if((stat = NC_copy_data(h5->controller,var->type_info->hdr.id,cachedata,cache->chunkcount,chunkdata))) goto done;
}
done:

10
libnczarr/zxcache.c
View File

@ -207,8 +207,7 @@ free_cache_entry(NCZChunkCache* cache, NCZCacheEntry* entry)
if(entry) {
int tid = cache->var->type_info->hdr.id;
if(tid == NC_STRING && !entry->isfixedstring) {
int ncid = cache->var->container->nc4_info->controller->ext_ncid;
nc_reclaim_data(ncid,tid,entry->data,cache->chunkcount);
NC_reclaim_data(cache->var->container->nc4_info->controller,tid,entry->data,cache->chunkcount);
}
nullfree(entry->data);
nullfree(entry->key.varkey);
@ -508,10 +507,9 @@ NCZ_reclaim_fill_chunk(NCZChunkCache* zcache)
int stat = NC_NOERR;
if(zcache && zcache->fillchunk) {
NC_VAR_INFO_T* var = zcache->var;
int ncid = var->container->nc4_info->controller->ext_ncid;
int tid = var->type_info->hdr.id;
size_t chunkcount = zcache->chunkcount;
stat = nc_reclaim_data_all(ncid,tid,zcache->fillchunk,chunkcount);
stat = NC_reclaim_data_all(var->container->nc4_info->controller,tid,zcache->fillchunk,chunkcount);
zcache->fillchunk = NULL;
}
return stat;
@ -610,7 +608,6 @@ put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
char* path = NULL;
nc_type tid = NC_NAT;
void* strchunk = NULL;
int ncid = 0;
ZTRACE(5,"cache.var=%s entry.key=%s",cache->var->hdr.name,entry->key);
LOG((3, "%s: var: %p", __func__, cache->var));
@ -620,7 +617,6 @@ put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
map = zfile->map;
/* Collect some info */
ncid = file->controller->ext_ncid;
tid = cache->var->type_info->hdr.id;
if(tid == NC_STRING && !entry->isfixedstring) {
@ -631,7 +627,7 @@ put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
/* copy char* to char[] format */
if((stat = NCZ_char2fixed((const char**)entry->data,strchunk,cache->chunkcount,maxstrlen))) goto done;
/* Reclaim the old chunk */
if((stat = nc_reclaim_data_all(ncid,tid,entry->data,cache->chunkcount))) goto done;
if((stat = NC_reclaim_data_all(file->controller,tid,entry->data,cache->chunkcount))) goto done;
entry->data = NULL;
entry->data = strchunk; strchunk = NULL;
entry->size = cache->chunkcount * maxstrlen;

2
libsrc4/nc4attr.c
View File

@ -140,7 +140,7 @@ nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var,
if (data)
{
{
if((retval = nc_copy_data(h5->controller->ext_ncid,mem_type,bufr,att->len,data)))
if((retval = NC_copy_data(h5->controller,mem_type,bufr,att->len,data)))
BAIL(retval);
}
}

12
libsrc4/nc4internal.c
View File

@ -1109,6 +1109,9 @@ nc4_type_list_add(NC_GRP_INFO_T *grp, size_t size, const char *name,
ncindexadd(grp->type, (NC_OBJ *)new_type);
obj_track(grp->nc4_info,(NC_OBJ*)new_type);
/* back link */
new_type->container = grp;
/* Return a pointer to the new type. */
*type = new_type;
@ -1337,7 +1340,7 @@ nc4_att_free(NC_ATT_INFO_T *att)
assert(parent->sort == NCGRP);
h5 = ((NC_GRP_INFO_T*)parent)->nc4_info;
/* Reclaim the attribute data */
if((stat = nc_reclaim_data(h5->controller->ext_ncid,att->nc_typeid,att->data,att->len))) goto done;
if((stat = NC_reclaim_data(h5->controller,att->nc_typeid,att->data,att->len))) goto done;
free(att->data); /* reclaim top level */
att->data = NULL;
}
@ -1386,9 +1389,8 @@ var_free(NC_VAR_INFO_T *var)
/* Delete any fill value allocation. */
if (var->fill_value) {
int ncid = var->container->nc4_info->controller->ext_ncid;
int tid = var->type_info->hdr.id;
if((retval = nc_reclaim_data_all(ncid, tid, var->fill_value, 1))) return retval;
if((retval = NC_reclaim_data_all(var->container->nc4_info->controller, tid, var->fill_value, 1))) return retval;
var->fill_value = NULL;
}
@ -1563,7 +1565,7 @@ nc4_rec_grp_del_att_data(NC_GRP_INFO_T *grp)
/* Free attribute data in this group */
for (i = 0; i < ncindexsize(grp->att); i++) {
NC_ATT_INFO_T * att = (NC_ATT_INFO_T*)ncindexith(grp->att, i);
if((retval = nc_reclaim_data_all(grp->nc4_info->controller->ext_ncid,att->nc_typeid,att->data,att->len)))
if((retval = NC_reclaim_data_all(grp->nc4_info->controller,att->nc_typeid,att->data,att->len)))
return retval;
att->data = NULL;
att->len = 0;
@ -1576,7 +1578,7 @@ nc4_rec_grp_del_att_data(NC_GRP_INFO_T *grp)
NC_VAR_INFO_T* v = (NC_VAR_INFO_T *)ncindexith(grp->vars, i);
for(j=0;j<ncindexsize(v->att);j++) {
NC_ATT_INFO_T* att = (NC_ATT_INFO_T*)ncindexith(v->att, j);
if((retval = nc_reclaim_data_all(grp->nc4_info->controller->ext_ncid,att->nc_typeid,att->data,att->len)))
if((retval = NC_reclaim_data_all(grp->nc4_info->controller,att->nc_typeid,att->data,att->len)))
return retval;
att->data = NULL;
att->len = 0;

85
libsrc4/nc4type.c
View File

@ -14,8 +14,10 @@
#include "nc4internal.h"
#include "nc4dispatch.h"
#if 0
#ifdef ENABLE_DAP4
EXTERNL int NCD4_get_substrate(int ncid);
EXTERNL NC* NCD4_get_substrate_nc(int ncid);
#endif
#endif
/* The sizes of types may vary from platform to platform, but within
@ -729,38 +731,71 @@ NC4_inq_type_fixed_size(int ncid, nc_type xtype, int* fixedsizep)
{
int stat = NC_NOERR;
int f = 0;
int xclass;
NC_FILE_INFO_T* h5 = NULL;
NC_TYPE_INFO_T* typ = NULL;
if(xtype < NC_STRING) {f = 1; goto done;}
if(xtype == NC_STRING) {f = 0; goto done;}
#ifdef USE_NETCDF4
/* Must be user type */
if((stat = nc_inq_user_type(ncid,xtype,NULL,NULL,NULL,NULL,&xclass))) goto done;
switch (xclass) {
case NC_ENUM: case NC_OPAQUE: f = 1; break;
case NC_VLEN: f = 0; break;
case NC_COMPOUND: {
NC_FILE_INFO_T* h5 = NULL;
NC_TYPE_INFO_T* typ = NULL;
#ifdef ENABLE_DAP4
NC* nc = NULL;
int xformat;
if ((stat = NC_check_id(ncid, &nc))) goto done;
xformat = nc->dispatch->model;
if(xformat == NC_FORMATX_DAP4) {
ncid = NCD4_get_substrate(ncid);
} /* Fall thru */
#endif
if ((stat = nc4_find_grp_h5(ncid, NULL, &h5)))
goto done;
if((stat = nc4_find_type(h5,xtype,&typ))) goto done;
f = !typ->u.c.varsized;
} break;
default: stat = NC_EBADTYPE; goto done;
}
if ((stat = nc4_find_grp_h5(ncid, NULL, &h5)))
goto done;
if((stat = nc4_find_type(h5,xtype,&typ))) goto done;
f = !typ->varsized;
#endif
done:
if(fixedsizep) *fixedsizep = f;
return stat;
}
/**
For types with one or more subtypes (e.g. basetype or
fieldtype), determine the varsizedness of the type based on the
basetype. The idea is to inform the code of the fact that
parenttype has addedtype "inserted" into it.
@param parenttype
@param subtype
*/
int
NC4_recheck_varsize(NC_TYPE_INFO_T* parenttype, nc_type subtype)
{
int stat = NC_NOERR;
NC_FILE_INFO_T* file = NULL;
NC_TYPE_INFO_T* utype = NULL;
if(subtype < NC_STRING) goto done; /* will not change the "variable-sizedness" of parenttype */
if(subtype == NC_STRING) {parenttype->varsized = 1; goto done;}
/* Get the inferred user-type */
file = parenttype->container->nc4_info;
if((stat = nc4_find_type(file,subtype,&utype))) goto done;
switch (utype->nc_type_class) {
case NC_OPAQUE: case NC_ENUM: break; /* no change */
case NC_VLEN: parenttype->varsized = 1; break;
case NC_COMPOUND: if(utype->varsized) parenttype->varsized = 1; break;
}
done:
return stat;
}
/**
When creating a type, mark it as variable-sized if known for sure.
@param typ
*/
int
NC4_set_varsize(NC_TYPE_INFO_T* typ)
{
int stat = NC_NOERR;
if(typ->hdr.id < NC_STRING) goto done; /* will not change the "variable-sizedness" of typ */
if(typ->hdr.id == NC_STRING) {typ->varsized = 1; goto done;}
switch (typ->nc_type_class) {
case NC_OPAQUE: case NC_ENUM: break; /* no change */
case NC_VLEN: typ->varsized = 1; break;
case NC_COMPOUND: typ->varsized = 0; break; /* until proven otherwise */
}
done:
return stat;
}

2
libsrc4/nc4var.c
View File

@ -265,7 +265,7 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
if (var->fill_value)
{
int xtype = var->type_info->hdr.id;
if((retval = nc_copy_data(ncid,xtype,var->fill_value,1,fill_valuep))) return retval;
if((retval = NC_copy_data(h5->controller,xtype,var->fill_value,1,fill_valuep))) return retval;
}
else
{

40
nc_test4/tst_vlenstr.c
View File

@ -87,6 +87,17 @@ writeVariable(int dimlength, int ncid, nc_type vlen_typeID)
free(data);
}
static void
createFruitsData(int elemno, int vlensize, int* stringIndexp, nc_vlen_t* data)
{
data[elemno].len = vlensize;
data[elemno].p = NULL;
if(vlensize > 0) {
data[elemno].p = charPointers+(*stringIndexp);
(*stringIndexp) += vlensize;
}
}
void
writeAttribute(int len, int ncid, nc_type vlen_typeID)
{
@ -98,29 +109,12 @@ writeAttribute(int len, int ncid, nc_type vlen_typeID)
/* create six variable length arrays of strings */
int stringIndex = 0;
data[0].len = first_size;
data[0].p = charPointers+stringIndex;
stringIndex += first_size;
data[1].len = second_size;
data[1].p = charPointers+stringIndex;
stringIndex += second_size;
data[2].len = third_size;
data[2].p = charPointers+stringIndex;
stringIndex += third_size;
data[3].len = fourth_size;
data[3].p = charPointers+stringIndex;
stringIndex += fourth_size;
data[4].len = fifth_size;
data[4].p = charPointers+stringIndex;
stringIndex += fifth_size;
data[5].len = sixth_size;
data[5].p = charPointers+stringIndex;
stringIndex += sixth_size;
createFruitsData(0,first_size,&stringIndex,data);
createFruitsData(1,second_size,&stringIndex,data);
createFruitsData(2,third_size,&stringIndex,data);
createFruitsData(3,fourth_size,&stringIndex,data);
createFruitsData(4,fifth_size,&stringIndex,data);
createFruitsData(5,sixth_size,&stringIndex,data);
#ifdef DEBUG
if(buf) {free(buf); buf = NULL;}

127
ncgen/genbin.c
View File

@ -14,11 +14,12 @@
#undef TRACE
/* Forward*/
static int genbin_defineattr(Symbol* asym);
static int genbin_definevardata(Symbol* vsym);
static int genbin_write(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
static int genbin_defineattr(int ncid, Symbol* asym);
static int genbin_definevardata(int ncid, Symbol* vsym);
static int genbin_write(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
static int genbin_writevar(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
static int genbin_writeattr(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
#ifdef USE_NETCDF4
static int genbin_deftype(Symbol* tsym);
static int genbin_definespecialattributes(Symbol* var);
@ -67,7 +68,7 @@ genbin_netcdf(void)
rootgroup->nc_id = ncid;
if (nofill_flag) {
stat = nc_set_fill(rootgroup->nc_id, NC_NOFILL, 0);
stat = nc_set_fill(ncid, NC_NOFILL, 0);
CHECK_ERR(stat);
}
@ -146,7 +147,7 @@ genbin_netcdf(void)
if(ngatts > 0) {
for(iatt = 0; iatt < ngatts; iatt++) {
Symbol* gasym = (Symbol*)listget(gattdefs,iatt);
genbin_defineattr(gasym);
genbin_defineattr(ncid,gasym);
}
}
@ -154,17 +155,17 @@ genbin_netcdf(void)
if(natts > 0) {
for(iatt = 0; iatt < natts; iatt++) {
Symbol* asym = (Symbol*)listget(attdefs,iatt);
genbin_defineattr(asym);
genbin_defineattr(ncid,asym);
}
}
if (nofill_flag) {
stat = nc_set_fill(rootgroup->nc_id, NC_NOFILL, 0);
stat = nc_set_fill(ncid, NC_NOFILL, 0);
CHECK_ERR(stat);
}
/* leave define mode */
stat = nc_enddef(rootgroup->nc_id);
stat = nc_enddef(ncid);
CHECK_ERR(stat);
if(!header_only) {
@ -173,7 +174,7 @@ genbin_netcdf(void)
for(ivar = 0; ivar < nvars; ivar++) {
Symbol* vsym = (Symbol*)listget(vardefs,ivar);
if(vsym->data != NULL) {
genbin_definevardata(vsym);
genbin_definevardata(ncid,vsym);
}
}
}
@ -316,7 +317,7 @@ genbin_deftype(Symbol* tsym)
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* econst = (Symbol*)listget(tsym->subnodes,i);
ASSERT(econst->subclass == NC_ECONST);
generator_reset(bin_generator,NULL);
generator_reset(bin_generator,rootgroup);
bbClear(datum);
generate_basetype(econst->typ.basetype,econst->typ.econst,datum,NULL,bin_generator);
stat = nc_insert_enum(tsym->container->nc_id,
@ -378,13 +379,13 @@ genbin_deftype(Symbol* tsym)
#endif /*USE_NETCDF4*/
static int
genbin_defineattr(Symbol* asym)
genbin_defineattr(int ncid, Symbol* asym)
{
int stat = NC_NOERR;
Bytebuffer* databuf = bbNew();
generator_reset(bin_generator,NULL);
generator_reset(bin_generator,rootgroup);
generate_attrdata(asym,bin_generator,(Writer)genbin_write,databuf);
stat = nc_reclaim_data(asym->container->nc_id,asym->typ.basetype->nc_id,bbContents(databuf),datalistlen(asym->data));
stat = nc_reclaim_data(ncid,asym->typ.basetype->nc_id,bbContents(databuf),datalistlen(asym->data));
bbFree(databuf);
return stat;
}
@ -392,15 +393,15 @@ genbin_defineattr(Symbol* asym)
/* Following is patterned after the walk functions in semantics.c */
static int
genbin_definevardata(Symbol* vsym)
genbin_definevardata(int ncid, Symbol* vsym)
{
int stat = NC_NOERR;
Bytebuffer* databuf = NULL;
if(vsym->data == NULL) goto done;
databuf = bbNew();
generator_reset(bin_generator,NULL);
generator_reset(bin_generator,rootgroup);
generate_vardata(vsym,bin_generator,(Writer)genbin_write,databuf);
stat = nc_reclaim_data_all(vsym->container->nc_id,vsym->typ.basetype->nc_id,bbExtract(databuf),datalistlen(vsym->data));
stat = nc_reclaim_data(ncid,vsym->typ.basetype->nc_id,bbContents(databuf),datalistlen(vsym->data));
done:
bbFree(databuf);
return stat;
@ -416,6 +417,7 @@ genbin_write(Generator* generator, Symbol* sym, Bytebuffer* memory,
return genbin_writevar(generator,sym,memory,rank,start,count);
else
PANIC("illegal symbol for genbin_write");
return NC_EINVAL;
}
@ -453,12 +455,6 @@ genbin_writevar(Generator* generator, Symbol* vsym, Bytebuffer* memory,
stat = nc_put_vara(vsym->container->nc_id, vsym->nc_id, start, count, data);
}
CHECK_ERR(stat);
#if 0
/* Reclaim the data */
stat = nc_reclaim_data(vsym->container->nc_id, vsym->typ.basetype->nc_id, data, nelems);
CHECK_ERR(stat);
bbClear(memory); /* reclaim top-level memory */
#endif
return stat;
}
@ -489,9 +485,9 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
} break;
case NC_CHAR: {
char* data = (char*)bbContents(databuf);
size_t slen = bbLength(databuf);
/* Revise length if slen == 0 */
if(slen == 0) {
size_t slen = bbLength(databuf);
/* Revise length if slen == 0 */
if(slen == 0) {
bbAppend(databuf,'\0');
/* bbAppend frees the memory pointed to by char* data,
so re-assign. See Coverity issue: 1265731.*/
@ -506,55 +502,55 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
stat = nc_put_att_short(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
} break;
case NC_INT: {
int* data = (int*)bbContents(databuf);
stat = nc_put_att_int(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
case NC_INT: {
int* data = (int*)bbContents(databuf);
stat = nc_put_att_int(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
} break;
case NC_FLOAT: {
float* data = (float*)bbContents(databuf);
stat = nc_put_att_float(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
case NC_FLOAT: {
float* data = (float*)bbContents(databuf);
stat = nc_put_att_float(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
} break;
case NC_DOUBLE: {
double* data = (double*)bbContents(databuf);
stat = nc_put_att_double(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
} break;
case NC_STRING: {
const char** data;
data = (const char**)bbContents(databuf);
stat = nc_put_att_string(grpid,varid,asym->name,
case NC_DOUBLE: {
double* data = (double*)bbContents(databuf);
stat = nc_put_att_double(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
} break;
case NC_STRING: {
const char** data;
data = (const char**)bbContents(databuf);
stat = nc_put_att_string(grpid,varid,asym->name,
bbLength(databuf)/sizeof(char*),
data);
} break;
case NC_UBYTE: {
unsigned char* data = (unsigned char*)bbContents(databuf);
stat = nc_put_att_uchar(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
} break;
case NC_UBYTE: {
unsigned char* data = (unsigned char*)bbContents(databuf);
stat = nc_put_att_uchar(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
} break;
case NC_USHORT: {
unsigned short* data = (unsigned short*)bbContents(databuf);
stat = nc_put_att_ushort(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
case NC_USHORT: {
unsigned short* data = (unsigned short*)bbContents(databuf);
stat = nc_put_att_ushort(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
} break;
case NC_UINT: {
unsigned int* data = (unsigned int*)bbContents(databuf);
stat = nc_put_att_uint(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
case NC_UINT: {
unsigned int* data = (unsigned int*)bbContents(databuf);
stat = nc_put_att_uint(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
} break;
case NC_INT64: {
long long* data = (long long*)bbContents(databuf);
stat = nc_put_att_longlong(grpid,varid,asym->name,typid,len,data);
CHECK_ERR2(stat,asym->lineno);
case NC_INT64: {
long long* data = (long long*)bbContents(databuf);
stat = nc_put_att_longlong(grpid,varid,asym->name,typid,len,data);
CHECK_ERR2(stat,asym->lineno);
} break;
case NC_UINT64: {
unsigned long long* data = (unsigned long long*)bbContents(databuf);
stat = nc_put_att_ulonglong(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
case NC_UINT64: {
unsigned long long* data = (unsigned long long*)bbContents(databuf);
stat = nc_put_att_ulonglong(grpid,varid,asym->name,typid,len,data);
CHECK_ERR(stat);
} break;
default: PANIC1("genbin_defineattr: unexpected basetype: %d",basetype->typ.typecode);
}
default: PANIC1("genbin_defineattr: unexpected basetype: %d",basetype->typ.typecode);
}
} else { /* use the generic put_attribute for user defined types*/
const char* data;
data = (const char*)bbContents(databuf);
@ -570,6 +566,7 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
}
#endif
}
CHECK_ERR(stat);
return stat;
}

3
ncgen/generate.c
View File

@ -180,7 +180,7 @@ generate_basetype(Symbol* tsym, NCConstant* con, Bytebuffer* codebuf, Datalist*
} else {
generator->listbegin(generator,tsym,NULL,LISTVLEN,data->length,codebuf,&uid);
for(count=0;count<data->length;count++) {
NCConstant* con;
NCConstant* con;
generator->list(generator,tsym,NULL,LISTVLEN,uid,count,vlenbuf);
con = datalistith(data,count);
generate_basetype(tsym->typ.basetype,con,vlenbuf,NULL,generator);
@ -399,6 +399,7 @@ generate_arrayR(struct Args* args, int dimindex, size_t* index, Datalist* data)
args->generator->listend(args->generator,args->vsym,NULL,LISTDATA,uid,counter,args->code);
memcpy(count,onesvector,sizeof(size_t)*dimindex);
count[dimindex] = stop;
/* Write the data; also reclaims written data */
args->writer(args->generator,args->vsym,args->code,args->rank,index,count);
bbClear(args->code);
} else {

2
ncgen/generate.h
View File

@ -55,7 +55,7 @@ extern int generator_reset(Generator*,void*);
typedef int (*Writer)(Generator*,struct Symbol*,Bytebuffer*,int,const size_t*,const size_t*);
extern void generate_attrdata(struct Symbol*, Generator*, Writer writer, Bytebuffer*);
extern void generate_vardata(struct Symbol*, Generator*, Writer writer,Bytebuffer*);
extern void generate_vardata(struct Symbol*, Generator*, Writer writer, Bytebuffer*);
extern void generate_basetype(struct Symbol*,NCConstant*,Bytebuffer*,Datalist*,Generator*);
#endif /*DATA_H*/

40
nczarr_test/CMakeLists.txt
View File

@ -21,8 +21,10 @@ FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_specific_filters.sh SPSOUR
STRING(PREPEND SPSOURCE "#!/bin/bash\n")
STRING(PREPEND SPSOURCE "TESTNCZARR=1\n")
# Replace with FILE(CONFIGURE) when cmake 3.18 is in common use
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/run_specific_filters.1 "${SPSOURCE}")
CONFIGURE_FILE(${CMAKE_CURRENT_BINARY_DIR}/run_specific_filters.1 ${CMAKE_CURRENT_BINARY_DIR}/run_specific_filters.sh FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE @ONLY NEWLINE_STYLE LF)
FILE(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/run_specific_filters.1)
FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_unknown.sh SPSOURCE)
@ -130,26 +132,26 @@ IF(ENABLE_TESTS)
add_sh_test(nczarr_test run_s3_cleanup)
ENDIF()
IF(ENABLE_FILTER_TESTING)
build_bin_test(tst_nczfilter)
build_bin_test(testfilter)
build_bin_test(testfilter_misc)
build_bin_test(testfilter_multi)
build_bin_test(testfilter_order)
build_bin_test(testfilter_repeat)
build_bin_test(test_filter_avail)
ADD_SH_TEST(nczarr_test run_nczfilter)
ADD_SH_TEST(nczarr_test run_filter)
ADD_SH_TEST(nczarr_test run_specific_filters)
IF(FALSE)
IF(ENABLE_FILTER_TESTING)
build_bin_test(tst_nczfilter)
build_bin_test(testfilter)
build_bin_test(testfilter_misc)
build_bin_test(testfilter_multi)
build_bin_test(testfilter_order)
build_bin_test(testfilter_repeat)
build_bin_test(test_filter_avail)
ADD_SH_TEST(nczarr_test run_nczfilter)
ADD_SH_TEST(nczarr_test run_filter)
ADD_SH_TEST(nczarr_test run_specific_filters)
IF(FALSE)
# This test is too dangerous to run in a parallel make environment.
# It causes race conditions. So suppress and only test by hand.
ADD_SH_TEST(nczarr_test run_unknown)
ENDIF(FALSE)
ENDIF(ENABLE_FILTER_TESTING)
if(ENABLE_NCZARR_ZIP)
add_sh_test(nczarr_test run_newformat)
endif()
# It causes race conditions. So suppress and only test by hand.
ADD_SH_TEST(nczarr_test run_unknown)
ENDIF(FALSE)
ENDIF(ENABLE_FILTER_TESTING)
if(ENABLE_NCZARR_ZIP)
add_sh_test(nczarr_test run_newformat)
endif()
ENDIF(BUILD_UTILITIES)

6
nczarr_test/zmapio.c
View File

@ -277,7 +277,6 @@ rootpathfor(const char* path)
NCURI* uri = NULL;
char* rootpath = NULL;
NClist* segments = nclistnew();
char* p = NULL;
ncuriparse(path,&uri);
if(uri == NULL) goto done;
@ -287,7 +286,8 @@ rootpathfor(const char* path)
rootpath = strdup("/"); /*constant*/
break;
#ifdef ENABLE_S3
case NCZM_S3:
case NCZM_S3: {
char* p = NULL;
/* Split the path part */
if((stat = nczm_split(uri->path,segments))) goto done;
/* remove the bucket name */
@ -295,7 +295,7 @@ rootpathfor(const char* path)
nullfree(p); p = NULL;
/* Put it back together */
if((stat = nczm_join(segments,&rootpath))) goto done;
break;
} break;
#endif
default:
stat = NC_EINVAL;

1
oc2/occurlfunctions.c
View File

@ -130,6 +130,7 @@ ocset_curlflag(OCstate* state, int flag)
case CURLOPT_USE_SSL:
case CURLOPT_SSLCERT: case CURLOPT_SSLKEY:
case CURLOPT_SSL_VERIFYPEER: case CURLOPT_SSL_VERIFYHOST:
case CURLOPT_CAINFO: case CURLOPT_CAPATH:
{
struct ssl* ssl = &state->auth->ssl;
/* VERIFYPEER == 0 => VERIFYHOST == 0 */

5
plugins/Makefile.am
View File

@ -65,10 +65,13 @@ EXTRA_DIST += H5checksum.c
if ENABLE_FILTER_TESTING
if ENABLE_NCZARR_FILTERS
plugins_to_install += lib__nch5fletcher32.la lib__nch5shuffle.la lib__nch5deflate.la
plugins_to_install += lib__nch5fletcher32.la lib__nch5shuffle.la
lib__nch5shuffle_la_SOURCES = H5Zshuffle.c
lib__nch5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c
if HAVE_DEFLATE
plugins_to_install += lib__nch5deflate.la
lib__nch5deflate_la_SOURCES = H5Zdeflate.c
endif
# Need our version of szip if libsz available and we are not using HDF5
if HAVE_SZ

21
unit_test/CMakeLists.txt
View File

@ -19,15 +19,14 @@ IF(USE_X_GETOPT)
SET(XGETOPTSRC "${CMAKE_CURRENT_SOURCE_DIR}/../libdispatch/XGetopt.c")
ENDIF()
IF(NOT MSVC)
IF(ENABLE_NETCDF_4)
SET(UNIT_TESTS ${UNIT_TESTS} tst_nclist tst_nc4internal)
ENDIF(ENABLE_NETCDF_4)
ENDIF(NOT MSVC)
FOREACH(CTEST ${UNIT_TESTS})
add_bin_test(unit_test ${CTEST})
ENDFOREACH()
IF(ENABLE_HDF5)
IF(NOT MSVC)
add_bin_test(unit_test tst_nclist)
add_bin_test(unit_test tst_nc4internal)
ENDIF(NOT MSVC)
build_bin_test(tst_reclaim ${XGETOPTSRC})
add_sh_test(unit_test run_reclaim_tests)
ENDIF(ENABLE_HDF5)
# Path convert test(s)
add_bin_test(unit_test test_pathcvt)
@ -35,8 +34,8 @@ add_bin_test(unit_test test_pathcvt)
IF(BUILD_UTILITIES)
IF(ENABLE_S3 AND WITH_S3_TESTING)
# SDK Test
BUILD_BIN_TEST(test_s3sdk ${XGETOPTSRC})
ADD_SH_TEST(unit_test run_s3sdk)
build_bin_test(test_s3sdk ${XGETOPTSRC})
add_sh_test(unit_test run_s3sdk)
ENDIF()
ENDIF()

13
unit_test/Makefile.am
View File

@ -32,10 +32,11 @@ tst_xcache_SOURCES = tst_xcache.c timer_utils.c timer_utils.h
TESTS += tst_nclist test_ncuri test_pathcvt tst_exhash tst_xcache
if USE_NETCDF4
check_PROGRAMS += tst_nc4internal
if USE_HDF5
check_PROGRAMS += tst_nc4internal tst_reclaim
TESTS += tst_nc4internal
endif # USE_NETCDF4
TESTS += run_reclaim_tests.sh
endif # USE_HDF5
if ENABLE_S3
if ENABLE_S3_TESTALL
@ -44,8 +45,10 @@ TESTS += run_s3sdk.sh
endif
endif
EXTRA_DIST = CMakeLists.txt run_s3sdk.sh
EXTRA_DIST += nctest_netcdf4_classic.nc
EXTRA_DIST = CMakeLists.txt run_s3sdk.sh run_reclaim.sh
EXTRA_DIST += nctest_netcdf4_classic.nc reclaim_tests.cdl
CLEANFILES = reclaim_tests*.txt reclaim_tests.nc
# If valgrind is present, add valgrind targets.
@VALGRIND_CHECK_RULES@

75
unit_test/reclaim_tests.cdl Normal file
View File

@ -0,0 +1,75 @@
netcdf reclaim_tests {
types:
int(*) vint_t;
string(*) vstr_t;
vint_t(*) vvint_t;
vstr_t(*) vvstr_t;
compound cmpd_atomic_t { /* compound with all atomic fields */
int f1;
float f2(2,2);
};
compound cmpd_str_t { /* compound with string typed field(s) */
string f1(3);
};
compound cmpd_vlen_t { /* compound with vlen typed field(s) */
vint_t f1(2);
};
compound cmpd_cmpd_t { /* compound with nested variable length compound field(s) */
vstr_t f1(2) ;
cmpd_vlen_t f2(2);
};
cmpd_atomic_t(*) vcmpd_atomic_t;
cmpd_str_t(*) vcmpd_str_t;
cmpd_vlen_t(*) vcmpd_vlen_t;
cmpd_cmpd_t(*) vcmpd_cmpd_t;
dimensions:
d1 = 1;
d2 = 2;
d3 = 3;
d4 = 4;
variables:
/* atomic types */
int intvar(d4); /* fixed-size atomic type */
string strvar(d4); /* string type */
/* vlen types */
vint_t vintvar(d4);
vstr_t vstrvar(d4);
vvint_t vvintvar(d4);
vvstr_t vvstrvar(d4);
/* compound types */
cmpd_atomic_t catomvar(d2);
cmpd_str_t cstrvar(d2);
cmpd_vlen_t cvlenvar(d2);
cmpd_cmpd_t ccmpdvar(d2);
/* vlen of compound types */
vcmpd_atomic_t vcatomvar;
vcmpd_str_t vcstrvar(d2);
vcmpd_vlen_t vcvlenvar(d3);
vcmpd_cmpd_t vccmpdvar;
data:
intvar = 17, 13, 11, 7 ;
strvar = "a", "ab", "abc", "abcd" ;
vintvar = {1}, {1,2}, {1,2,3}, {1,2,3,4} ;
vstrvar = {"a","ab","abc","abcd"}, {"abcd","abc","ab"}, {"a","ab"}, {"abcd"} ;
vvintvar = {{1}, {1,2}}, {{1,2,3}}, {{1,2,3,4}}, {{17,13},{11,7},{5,3},{2,1}} ;
vvstrvar = {{"1"}, {"1","2"}}, {{"1","2","3"}}, {{"1","2","3","4"}}, {{"17","13"},{"11","7"},{5,"3"},{"2","1"}} ;
catomvar = {17, {1.1, 2.2, 3.3, 4.4}}, {27, {4.1, 3.2, 3.3, 1.4}} ;
cstrvar = {{"a","abc","abcd"}}, {{"x","yz", "ijkl"}} ;
cvlenvar = {{{17,13},{11}}}, {{{3,5},{7,11}}} ;
ccmpdvar = {{{"xxx"},{"yyy"}},{{{{17,13},{12,9}}},{{{1,3},{2,11}}}}}, {{{"uv"},{"w"}},{{{{117,113},{112,119}}},{{{111,113},{112,1111}}}}} ;
vcatomvar = {{17, {1.1, 2.2, 3.3, 4.4}}, {27, {4.1, 3.2, 3.3, 1.4}}} ;
vcstrvar = {{{"c11a","c12a","c13a"}},{{"c11b","c12b","c13b"}},{{"c11c","c12c","c13c"}},{{"c11d","c12d","c13d"}}},{{{"c21a","c22a","c23a"}},{{"c21b","c22b","c23b"}},{{"c21c","c22c","c23c"}},{{"c21d","c22d","c23d"}}};
vcvlenvar = {{{{117,113},{111}}}, {{{13,15},{17,111}}},{{{217,213},{211}}}, {{{23,25},{27,211}}}}, {{{{2117,2113},{2211}}}, {{{2223,2215},{2227,22111}}},{{{22217,22213},{22211}}}, {{{2223,2225},{2227,22211}}}}, {{{{33117,33113},{33111}}}, {{{3313,3315},{3317,33111}}},{{{33217,33213},{33211}}}, {{{3323,3325},{3327,33211}}}};
vccmpdvar = {{{{"xxx"},{"yyy"}},{{{{17,13},{12,9}}},{{{1,3},{2,11}}}}}, {{{"uv"},{"w"}},{{{{117,113},{112,119}}},{{{111,113},{112,1111}}}}}};
} //reclaim_tests

16
unit_test/run_reclaim_tests.sh Executable file
View File

@ -0,0 +1,16 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
rm -f reclaim_tests.nc reclaim_tests*.txt
${NCGEN} -4 ${srcdir}/reclaim_tests.cdl
${execdir}/tst_reclaim > reclaim_tests.txt
sed -e '/^(o)/p' -ed reclaim_tests.txt | sed -e 's/^(o) //' > reclaim_tests_o.txt
sed -e '/^(c)/p' -ed reclaim_tests.txt | sed -e 's/^(c) //' > reclaim_tests_c.txt
diff reclaim_tests_o.txt reclaim_tests_c.txt

345
unit_test/tst_reclaim.c Normal file
View File

@ -0,0 +1,345 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
/**
Test the various type walker functions:
nc_reclaim_data => NC_reclaim_data
nc_reclaim_data_all => NC_reclaim_data_all => NC_reclaim_data
nc_copy_data => NC_copy_data
nc_copy_data_all => NC_copy_data_all => NC_copy_data
nc_dump_data => NC_dump_data
See the file "reclaim_tests.cdl" to see the input file semantics.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include "netcdf.h"
#include "netcdf_aux.h"
#define NCCATCH
#include "nclog.h"
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#if defined(_WIN32) && !defined(__MINGW32__)
#include "XGetopt.h"
#endif
#define DEBUG
#define FILE "reclaim_tests.nc"
#define MAXOBJECTS 1024
struct Options {
int debug;
} dumpoptions;
struct Type {
char name[NC_MAX_NAME];
int tid;
size_t size;
};
struct Dim {
char name[NC_MAX_NAME];
int did;
size_t size;
};
struct Var {
char name[NC_MAX_NAME];
int vid;
int tid;
size_t dimprod;
void* data;
};
struct Metadata {
int ncid;
/* type ids */
struct Type vint_t;
struct Type vstr_t;
struct Type vvint_t;
struct Type vvstr_t;
struct Type cmpd_atomic_t;
struct Type cmpd_str_t;
struct Type cmpd_vlen_t;
struct Type cmpd_cmpd_t;
struct Type vcmpd_atomic_t;
struct Type vcmpd_str_t;
struct Type vcmpd_vlen_t;
struct Type vcmpd_cmpd_t;
/* dim ids */
struct Dim d1;
struct Dim d2;
struct Dim d4;
/* var ids */
struct Var intvar;
struct Var strvar;
struct Var vintvar;
struct Var vstrvar;
struct Var vvintvar;
struct Var vvstrvar;
struct Var catomvar;
struct Var cstrvar;
struct Var cvlenvar;
struct Var ccmpdvar;
struct Var vcatomvar;
struct Var vcstrvar;
struct Var vcvlenvar;
struct Var vccmpdvar;
} metadata;
/* atomic Types */
static struct Type atomics[NC_MAX_ATOMIC_TYPE+1];
/* Track the metadata objects */
static struct Type* typemap[MAXOBJECTS];
static struct Dim* dimmap[MAXOBJECTS];
static struct Var* varmap[MAXOBJECTS];
#define CHECK(code) do {stat = check(code,__func__,__LINE__); if(stat) {goto done;}} while(0)
static int
check(int code, const char* fcn, int line)
{
if(code == NC_NOERR) return code;
fprintf(stderr,"***fail: (%d) %s @ %s:%d\n",code,nc_strerror(code),fcn,line);
#ifdef debug
abort();
#endif
exit(1);
}
static int
initialize(void)
{
int i,stat = NC_NOERR;
memset(atomics,0,sizeof(atomics));
memset(typemap,0,sizeof(typemap));
memset(dimmap,0,sizeof(dimmap));
memset(varmap,0,sizeof(varmap));
for(i=1;i<=NC_MAX_ATOMIC_TYPE;i++) {
struct Type* t = &atomics[i];
t->tid = i;
if((stat=ncaux_inq_any_type(0,i,t->name,&t->size,NULL,NULL,NULL))) return NCTHROW(stat);
typemap[i] = t;
}
return NCTHROW(stat);
}
static void
cleanup(void)
{
int i;
for(i=0;i<MAXOBJECTS;i++) {
if(varmap[i] == NULL) break;
if(varmap[i]->data != NULL) free(varmap[i]->data);
}
}
static int
setuptype(int ncid, const char* name, struct Type* t)
{
int stat = NC_NOERR;
strcpy(t->name,name);
if((stat=nc_inq_typeid(ncid,name,&t->tid))) return NCTHROW(stat);
if((stat=ncaux_inq_any_type(ncid,t->tid,NULL,&t->size,NULL,NULL,NULL))) return NCTHROW(stat);
typemap[t->tid] = t;
return NCTHROW(stat);
}
static int
setupdim(int ncid, const char* name, struct Dim* dim)
{
int stat = NC_NOERR;
strcpy(dim->name,name);
if((stat = nc_inq_dimid(ncid,name,&dim->did))) return NCTHROW(stat);
if((nc_inq_dimlen(ncid,dim->did,&dim->size))) return NCTHROW(stat);
dimmap[dim->did] = dim;
return NCTHROW(stat);
}
static int
setupvar(int ncid, const char* name, struct Var* var)
{
int stat = NC_NOERR;
int i,ndims;
int dimids[NC_MAX_VAR_DIMS];
size_t product,dimsizes[NC_MAX_VAR_DIMS];
strcpy(var->name,name);
if((stat=nc_inq_varid(ncid,name,&var->vid))) return NCTHROW(stat);
if((stat=nc_inq_vartype(ncid,var->vid,&var->tid))) return NCTHROW(stat);
if((stat=nc_inq_varndims(ncid,var->vid,&ndims))) return NCTHROW(stat);
if((stat=nc_inq_vardimid(ncid,var->vid,dimids))) return NCTHROW(stat);
for(product=1,i=0;i<ndims;i++) {
if((stat=nc_inq_dimlen(ncid,dimids[i],&dimsizes[i]))) return NCTHROW(stat);
product *= dimsizes[i];
}
var->dimprod = product;
varmap[var->vid] = var;
return NCTHROW(stat);
}
static void
setup(struct Metadata* md)
{
int stat = NC_NOERR;
CHECK(nc_open(FILE,NC_NETCDF4,&md->ncid));
CHECK(setupdim(md->ncid,"d1",&md->d1));
CHECK(setupdim(md->ncid,"d2",&md->d2));
CHECK(setupdim(md->ncid,"d4",&md->d4));
CHECK(setuptype(md->ncid,"vint_t",&md->vint_t));
CHECK(setuptype(md->ncid,"vstr_t",&md->vstr_t));
CHECK(setuptype(md->ncid,"vvint_t",&md->vvint_t));
CHECK(setuptype(md->ncid,"vvstr_t",&md->vvstr_t));
CHECK(setuptype(md->ncid,"cmpd_atomic_t",&md->cmpd_atomic_t));
CHECK(setuptype(md->ncid,"cmpd_str_t",&md->cmpd_str_t));
CHECK(setuptype(md->ncid,"cmpd_vlen_t",&md->cmpd_vlen_t));
CHECK(setuptype(md->ncid,"cmpd_cmpd_t",&md->cmpd_cmpd_t));
CHECK(setuptype(md->ncid,"vcmpd_atomic_t",&md->vcmpd_atomic_t));
CHECK(setuptype(md->ncid,"vcmpd_str_t",&md->vcmpd_str_t));
CHECK(setuptype(md->ncid,"vcmpd_vlen_t",&md->vcmpd_vlen_t));
CHECK(setuptype(md->ncid,"vcmpd_cmpd_t",&md->vcmpd_cmpd_t));
CHECK(setupvar(md->ncid,"intvar",&md->intvar));
CHECK(setupvar(md->ncid,"strvar",&md->strvar));
CHECK(setupvar(md->ncid,"vintvar",&md->vintvar));
CHECK(setupvar(md->ncid,"vstrvar",&md->vstrvar));
CHECK(setupvar(md->ncid,"vvintvar",&md->vvintvar));
CHECK(setupvar(md->ncid,"vvstrvar",&md->vvstrvar));
CHECK(setupvar(md->ncid,"catomvar",&md->catomvar));
CHECK(setupvar(md->ncid,"cstrvar",&md->cstrvar));
CHECK(setupvar(md->ncid,"cvlenvar",&md->cvlenvar));
CHECK(setupvar(md->ncid,"ccmpdvar",&md->ccmpdvar));
CHECK(setupvar(md->ncid,"vcatomvar",&md->vcatomvar));
CHECK(setupvar(md->ncid,"vcstrvar",&md->vcstrvar));
CHECK(setupvar(md->ncid,"vcvlenvar",&md->vcvlenvar));
CHECK(setupvar(md->ncid,"vccmpdvar",&md->vccmpdvar));
done:
assert(stat == NC_NOERR);
}
static int
readvar(int ncid, struct Var* v)
{
int stat = NC_NOERR;
size_t size;
struct Type* basetype = typemap[v->tid];
size = (basetype->size * v->dimprod);
v->data = calloc(1,size);
if((stat=nc_get_var(ncid,v->vid,v->data)))
return NCTHROW(stat);
return NCTHROW(stat);
}
static int
dumpvar(int ncid, struct Var* v, void* data, char** bufp)
{
int stat = NC_NOERR;
if((stat=ncaux_dump_data(ncid,v->tid,data,v->dimprod,bufp))) return NCTHROW(stat);
return NCTHROW(stat);
}
static int
testvar(int ncid, struct Var* v)
{
int stat = NC_NOERR;
char* buforig = NULL;
char* bufcopy = NULL;
void* copy = NULL;
if((stat=readvar(ncid,v))) return NCTHROW(stat);
if((stat=dumpvar(ncid,v,v->data,&buforig))) return NCTHROW(stat);
printf("(o) %s: %s\n",v->name,buforig);
// Test copying
if((stat=nc_copy_data_all(ncid,v->tid,v->data,v->dimprod,&copy))) return NCTHROW(stat);
// Print copy
if((stat=dumpvar(ncid,v,copy,&bufcopy))) return NCTHROW(stat);
printf("(c) %s: %s\n",v->name,bufcopy);
/* Compare */
if(strcmp(buforig,bufcopy) != 0)
fprintf(stderr,"*** orig != copy\n");
if(buforig) free(buforig);
if(bufcopy) free(bufcopy);
// reclaim original
if((stat=nc_reclaim_data_all(ncid,v->tid,v->data,v->dimprod))) return NCTHROW(stat);
// reclaim copy
if((stat=nc_reclaim_data_all(ncid,v->tid,copy,v->dimprod))) return NCTHROW(stat);
v->data = NULL;
return NCTHROW(stat);
}
static void
test(struct Metadata* md)
{
int stat = NC_NOERR;
CHECK(testvar(md->ncid,&md->intvar));
CHECK(testvar(md->ncid,&md->strvar));
CHECK(testvar(md->ncid,&md->vintvar));
CHECK(testvar(md->ncid,&md->vstrvar));
CHECK(testvar(md->ncid,&md->vvintvar));
CHECK(testvar(md->ncid,&md->vvstrvar));
CHECK(testvar(md->ncid,&md->catomvar));
CHECK(testvar(md->ncid,&md->cstrvar));
CHECK(testvar(md->ncid,&md->cvlenvar));
CHECK(testvar(md->ncid,&md->ccmpdvar));
CHECK(testvar(md->ncid,&md->vcatomvar));
CHECK(testvar(md->ncid,&md->vcstrvar));
CHECK(testvar(md->ncid,&md->vcvlenvar));
CHECK(testvar(md->ncid,&md->vccmpdvar));
done:
return;
}
int
main(int argc, char** argv)
{
int c,stat = NC_NOERR;
CHECK(nc_initialize());
CHECK(initialize());
/* Init options */
memset((void*)&dumpoptions,0,sizeof(dumpoptions));
while ((c = getopt(argc, argv, "dhk:tu:")) != EOF) {
switch(c) {
case 'd':
dumpoptions.debug = 1;
break;
case '?':
fprintf(stderr,"unknown option\n");
stat = NC_EINVAL;
goto done;
}
}
setup(&metadata);
test(&metadata);
done:
cleanup();
if(metadata.ncid) nc_close(metadata.ncid);
(void)nc_finalize();
exit(stat?1:0);
}