mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-04-12 18:10:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Provide a Number of fixes/improvements to NCZarr

Browse Source 
Primary changes:
* Add an improved cache system to speed up performance.
* Fix NCZarr to properly handle scalar variables.

Misc. Related Changes:
* Added unit tests for extendible hash and for the generic cache.
* Add config parameter to set size of the NCZarr cache.
* Add initial performance tests but leave them unused.
* Add CRC64 support.
* Move location of ncdumpchunks utility from /ncgen to /ncdump.
* Refactor auth support.

Misc. Unrelated Changes:
* More cleanup of the S3 support
* Add support for S3 authentication in .rc files: HTTP.S3.ACCESSID and HTTP.S3.SECRETKEY.
* Remove the hashkey from the struct OBJHDR since it is never used.
This commit is contained in:
Dennis Heimbigner 2020-11-19 17:01:04 -07:00
parent 8279a078b0
commit eb3d9eb0c9
92 changed files with 4399 additions and 707 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -314,6 +314,7 @@ SET(DEFAULT_CHUNKS_IN_CACHE 10 CACHE STRING "Default number of chunks in cache."
SET(CHUNK_CACHE_SIZE 16777216 CACHE STRING "Default Chunk Cache Size.")
SET(CHUNK_CACHE_NELEMS 4133 CACHE STRING "Default maximum number of elements in cache.")
SET(CHUNK_CACHE_PREEMPTION 0.75 CACHE STRING "Default file chunk cache preemption policy for HDf5 files(a number between 0 and 1, inclusive.")
SET(CHUNK_CACHE_SIZE_NCZARR 4194304 CACHE STRING "Default NCZarr Chunk Cache Size.")
SET(MAX_DEFAULT_CACHE_SIZE 67108864 CACHE STRING "Default maximum cache size.")
SET(NETCDF_LIB_NAME "" CACHE STRING "Default name of the netcdf library.")
SET(TEMP_LARGE "." CACHE STRING "Where to put large temp files if large file tests are run.")
@ -1004,7 +1005,7 @@ ENDIF(ENABLE_NCZARR_S3)
IF(NOT ENABLE_S3_SDK)
IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS)
message(FATAL_ERROR "AWS S3 libraries not found; please specify options DENABLE_NCZARR_S3=NO AND -DENABLE-NCZARR-S3-TESTS=NO")
message(FATAL_ERROR "AWS S3 libraries not found; please specify option DENABLE_NCZARR_S3=NO")
SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE)
SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
ENDIF()
@ -1412,6 +1413,7 @@ CHECK_INCLUDE_FILE("ftw.h" HAVE_FTW_H)
CHECK_INCLUDE_FILE("libgen.h" HAVE_LIBGEN_H)
CHECK_INCLUDE_FILE("execinfo.h" HAVE_EXECINFO_H)
CHECK_INCLUDE_FILE("dirent.h" HAVE_DIRENT_H)
CHECK_INCLUDE_FILE("time.h" HAVE_TIME_H)
# Symbol Exists
CHECK_SYMBOL_EXISTS(isfinite "math.h" HAVE_DECL_ISFINITE)
@ -1493,6 +1495,7 @@ ENDIF(SIZEOF_UINTPTR_T)
# __int64 is used on Windows for large file support.
CHECK_TYPE_SIZE("__int64" SIZEOF___INT_64)
CHECK_TYPE_SIZE("int64_t" SIZEOF_INT64_T)
CHECK_TYPE_SIZE("uint64" SIZEOF_UINT64)
CHECK_TYPE_SIZE("uint64_t" SIZEOF_UINT64_T)
CHECK_TYPE_SIZE("unsigned char" SIZEOF_UCHAR)
CHECK_TYPE_SIZE("unsigned short int" SIZEOF_UNSIGNED_SHORT_INT)

4
RELEASE_NOTES.md
View File

@ -7,7 +7,9 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.0 - TBD
* [Enhancement] Give the client control over what parts of a DAP2 URL are URL encoded (i.e. %xx). This is to support the different decoding rules that servers apply to incoming URLS.
* [Bug Fix] Implement a better chunk cache system for NCZarr. The cache now uses extendible hashing plus a linked list for provide a combination of expandibility, fast access, and LRU behavior. See [Github #1845](https://github.com/Unidata/netcdf-c/pull/1845) for more information.
* [Enhancement] Provide .rc fields for S3 authentication: HTTP.S3.ACCESSID and HTTP.S3.SECRETKEY.
* [Enhancement] Give the client control over what parts of a DAP2 URL are URL encoded (i.e. %xx). This is to support the different decoding rules that servers apply to incoming URLS. See [Github #1884](https://github.com/Unidata/netcdf-c/pull/1844) for more information.
* [Bug Fix] Fix incorrect time offsets from `ncdump -t`, in some cases when the time `units` attribute contains both a **non-zero** time-of-day, and a time zone suffix containing the letter "T", such as "UTC". See [Github #1866](https://github.com/Unidata/netcdf-c/pull/1866) for more information.
* [Bug Fix] Cleanup the NCZarr S3 build options. See [Github #1869](https://github.com/Unidata/netcdf-c/pull/1869) for more information.
* [Bug Fix] Support aligned access for selected ARM processors. See [Github #1871](https://github.com/Unidata/netcdf-c/pull/1871) for more information.

9
config.h.cmake.in
View File

@ -101,6 +101,9 @@ are set when opening a binary file on Windows. */
/* default file chunk cache size in bytes. */
#cmakedefine CHUNK_CACHE_SIZE ${CHUNK_CACHE_SIZE}
/* default nczarr chunk cache size in bytes. */
#cmakedefine CHUNK_CACHE_SIZE_NCZARR ${CHUNK_CACHE_SIZE_NCZARR}
/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP
systems. This function is required for `alloca.c' support on those systems.
*/
@ -383,6 +386,9 @@ are set when opening a binary file on Windows. */
/* Define to 1 if you have the <sys/types.h> header file. */
#cmakedefine HAVE_SYS_TYPES_H 1
/* Define to 1 if you have the <time.h> header file. */
#cmakedefine HAVE_TIME_H 1
/* Define to 1 if the system has the type `uchar'. */
#cmakedefine HAVE_UCHAR 1
@ -392,6 +398,9 @@ are set when opening a binary file on Windows. */
/* Define to 1 if the system has the type `uint64'. */
#cmakedefine HAVE_UINT64 1
/* Define to 1 if the system has the type `uint64_t'. */
#cmakedefine HAVE_UINT64_t 1
/* Define to 1 if you have the <unistd.h> header file. */
#cmakedefine HAVE_UNISTD_H 1
#cmakedefine YY_NO_UNISTD_H 1

15
configure.ac
View File

@ -347,7 +347,7 @@ AC_ARG_WITH([default-chunks-in-cache],
AC_MSG_RESULT([$DEFAULT_CHUNKS_IN_CACHE])
AC_DEFINE_UNQUOTED([DEFAULT_CHUNKS_IN_CACHE], [$DEFAULT_CHUNKS_IN_CACHE], [num chunks in default per-var chunk cache.])
# Did the user specify a default cache size?
# Did the user specify a default cache size for HDF5?
AC_MSG_CHECKING([whether a default file cache size for HDF5 was specified])
AC_ARG_WITH([chunk-cache-size],
[AS_HELP_STRING([--with-chunk-cache-size=<integer>],
@ -625,6 +625,15 @@ if test "x$enable_nczarr_s3_tests" = xyes ; then
fi
AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes])
# Did the user specify a default cache size for NCZarr?
AC_MSG_CHECKING([whether a default file cache size for NCZarr was specified])
AC_ARG_WITH([chunk-cache-size-nczarr],
[AS_HELP_STRING([--with-chunk-cache-size-nczarr=<integer>],
[Specify default maximum space used by the chunk cache NCZarr.])],
[CHUNK_CACHE_SIZE_NCZARR=$with_chunk_cache_size_nczarr], [CHUNK_CACHE_SIZE_NCZARR=4194304])
AC_MSG_RESULT([$CHUNK_CACHE_SIZE_NCZARR])
AC_DEFINE_UNQUOTED([CHUNK_CACHE_SIZE_NCZARR], [$CHUNK_CACHE_SIZE_NCZARR], [default nczarr chunk cache size.])
# This has multiversion capability
AC_MSG_CHECKING([whether multi-filter support is enabled])
has_multifilters=yes
@ -941,7 +950,7 @@ AC_CHECK_HEADERS([sys/param.h])
AC_CHECK_HEADERS([libgen.h])
#AC_CHECK_HEADERS([locale.h])
AC_HEADER_STDC
AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h dirent.h])
AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h])
# Do sys/resource.h separately
#AC_CHECK_HEADERS([sys/resource.h],[havesysresource=1],[havesysresource=0])
@ -1036,7 +1045,7 @@ AC_FUNC_ALLOCA
AC_CHECK_DECLS([isnan, isinf, isfinite],,,[#include <math.h>])
AC_STRUCT_ST_BLKSIZE
UD_CHECK_IEEE
AC_CHECK_TYPES([size_t, ssize_t, schar, uchar, longlong, ushort, uint, int64, uint64, size64_t, ssize64_t, _off64_t])
AC_CHECK_TYPES([size_t, ssize_t, schar, uchar, longlong, ushort, uint, int64, uint64, size64_t, ssize64_t, _off64_t, uint64_t])
AC_TYPE_OFF_T
AC_TYPE_UINTPTR_T
AC_C_CHAR_UNSIGNED

171
docs/indexing.dox
View File

@ -2,7 +2,7 @@
\internal
\page nchashmap Indexed Access to Metadata Objects
\page Hashed + Indexed Access to Metadata Objects
\tableofcontents
@ -32,7 +32,7 @@ internal object lookup mechanisms now in place.
\section Sindexed_searches Indexed Searches
There are, as a rule, two searches that are used to locate
metadata object: (1) search by name and (2) search by
metadata objects: (1) search by name and (2) search by
externally visible id (e.g. dimid or varid).
It is currently the case that after all the metadata is read or
@ -85,51 +85,37 @@ typedef struct NClist {
} NClist;
\endcode
\subsection Snc_hashmap NC_hashmap
\subsection Snc_exhashmap NCexhashmap
The NC_hashmap type is a hash table mapping a string
(the key) to a data item. As a rule, the data item is a pointer to a
metadata object. The current implementation supports table
expansion when the # of entries in the table starts to get too
large. A simple linear rehash is used for collisions
and no separate hash-chain is used. This means that when
expanded, it must be completely rebuilt. The performance hit for
this has yet to be determined. The alternative is to move to some
form of extendible hashing as used in databases.
The NCexhashmap type is a hash table mapping a key
to a data item. As a rule, the data item is a pointer to a
metadata object. This hash map uses Fagin's extendible hashing
algorithm. The current implementation supports table
expansion as described for that algorithm.
The hashtable definition is as follows.
\code
typedef struct NC_hashmap {
size_t size;
size_t count;
NC_hentry* table;
} NC_hashmap;
typedef struct NCexhashmap {
int depth; /* Global depth */
int nactive; /* # of active entries in whole table */
NCexleaf** directory; /* |directory| == 2^depth */
...
} NCexhashmap;
\endcode
where size is the current allocated size and count is the
number of active entries in the table. The "table" field is
a vector of entries of this form.
where the directory (of leaf pointers) has (1<<depth) pointers,
and where nactive is the total number of entries. There are other
fields not relevant to this discussion.
An entry in a leaf has this form:
\code
typedef struct NC_hentry {
int flags;
typedef struct NCexentry {
ncexhashkey_t hashkey; /* Hash id */
uintptr_t data;
unsigned int hashkey;
size_t keysize;
char* key;
} NC_hentry;
} NCexentry;
\endcode
The _flags_ field indicates the state of the entry and can be
in one of three disjoint states:
1. ACTIVE - there is an object referenced in this entry
2. DELETED - an entry was deleted, but must be marked so
that linear rehash will work.
3. EMPTY - unused
The "data" field is of type "uintptr_t". Note that we assume
The _data_ field is of type _uintptr\_t_. Note that we assume
that sizeof(unintptr_t) == sizeof(void*). This is very
important. It is supposed to be the case that a value of type
uintptr_t is an integer of sufficient size to hold a void* pointer.
@ -138,20 +124,24 @@ This means that the data field can hold an unsigned integer or a
void* pointer. As a pointer, it often points to an instance of a
variable, or dimension, or other object.
The hashkey field is a CRC32 hash of the key. Note that comparing
hashkeys is not sufficient to ensure that the corresponding keys are
the same because hash collisions are possible. Even moving to, say,
64 bit keys, is probably not sufficient to avoid hash collisions.
A 128 bit key (e.g. MD5) might be sufficient but mathematical
investigation would be required.
The hashkey field is a CRC64 hash of the key. It is assumed that
64 bits is sufficient to guarantee that the hash of the key (a
string typically) is always unique.
Since comparing only hash keys is not sufficient, it is necessary to store
a copy of the actual key. The key and keysize fields are used for this.
This means that comparing hashkeys should be sufficient to
ensure that the corresponding keys are the same because hash
collisions will not occur. If a violation of this assumption
is detected, then a netcdf library internal error code is returned.
\subsection Sncindex NCindex
Finally, a leaf holds a vector of entries, and keeps them in
hashkey sorted order. Lookup within the leaf uses binary search.
This differs form Fagin's original algorithm, which used
secondary hashing within the leaf.
An "index" \(aka instance of type "NCindex"\) is a combination
of one NClist instance plus one NC_hashmap instance.
\subsection Sncindex NCxindex
An "index" \(aka instance of type "NCxindex"\) is a combination
of one NClist instance plus one NCexhashmap instance.
The hashmap maps a name to the position of the correspondingly
named object in the NClist part of the NCindex.
@ -167,14 +157,14 @@ provides these capabilities. The NCindex object contains:
and iterated over.
2. A map from name to the corresponding object index in the vector.
Note that currently, NCindex is only used in libsrc4 and libhdf4.
But if performance issues warrant, it will also be used in
libsrc.
Note that currently, NCxindex is only used in libsrc4 and
libhdf5 and libnczarr. But if performance issues warrant, it
will eventually also be used in libsrc.
Note also that alternative implementations are feasible that do not
use a hash table for name indexing, but rather keep a list sorted by name
and use binary search to do name-based lookup. If this alternative were
implemented, then it is probable that we could get rid of using the NC_hashmap
implemented, then it is probable that we could get rid of using the NCexhashmap
structure altogether for netcdf-4. There is a performance cost since binary
search is O(log n). In practice, it is probable that this is of negligible
effect. The advantage is that rename operations become considerably simpler.
@ -191,17 +181,19 @@ Note that this was already the case for netcdf-3 (libsrc) so
this is a change for libsrc4 only.
The global set of dimensions, types, and groups is maintained by
three instances of NClist in the NC_FILE_INFO structure:
namely _alldims_, _alltypes_, and _allgroups_.
The position of the object within the corresponding list determines
the object's external id. Thus, a position of a dimension object within the
"alldims" field of the file structure determines its dimid. Similarly
for types and groups.
three instances of NClist (within an index object) in the
NC_FILE_INFO structure: namely _alldims_, _alltypes_, and
_allgroups_. The position of the object within the
corresponding list determines the object's external id. Thus, a
position of a dimension object within the "alldims" field of the
file structure determines its dimid. Similarly for types and groups.
Note that _alldims_ is an _NClist_ rather than an _NCxindex_ because
the dimension names are not unique unless they are full names (i.e. /g1/g2/dim).
\section Sper_group_object_access Per-Group Object Access
Each group object (NC_GRP_INFO_T) contains five instances of
NCindex. One is for dimensions, one is for types, one is for
NCxindex. One is for dimensions, one is for types, one is for
subgroups, one is for variables, and one is for attributes. An
index is used for two reasons. First, it allows name-based lookup
for these items. Second, the declaration order is maintained by
@ -232,13 +224,13 @@ typedef struct NC_OBJ {
NC_SORT sort;
char* name; /* assumed to be null terminated */
size_t id;
unsigned int hashkey;
ncexhashkey_t hashkey;
} NC_OBJ;
\endcode
The sort is one of the values _NCVAR_, _NCDIM_, _NCATT_, _NCTYP_, or _NCGRP_.
The name is assumed to be nul terminated. The id is the assigned id
for the object. The hashkey is the same hash value as used in nchashtable.c.
for the object. The hashkey is the same hash value as used in NCexhashmap.
\section Scliches Programming cliches
@ -263,12 +255,12 @@ of the grp. It does so by walking the linked list of child groups.
It does a name comparison in order to find the group with the desired name.
In the new code, this iteration cliche is replaced by something
that will look like this.
that looks like this.
\code
NC_GRP_INFO_T* grp = ...;
NC_GRP_INFO_T* g;
...
g = ncindexlookup(grp->children,name);
g = ncxindexlookup(grp->children,name);
if(g != NULL)
... code to process matching grp by name
}
@ -297,7 +289,7 @@ It does an id comparison in order to find the group with the desired
dimension.
In the new code, this iteration cliche is replaced by something
that will look like this.
that looks like this.
\code
NC_FILE_INFO_T* h5 = ...;
NC_DIM_INFO_T* d;;
@ -314,13 +306,13 @@ This approach works for dimension ids, group ids, and type ids
because they are globally unique.
For variables and attributes, we have to use the containing group's
NCindex, such as grp->vars. In this case, the varid, is mapped using
Ncxindex, such as grp->vars. In this case, the varid, is mapped using
code like this.
\code
NC_GRP_INFO_T* grp = ...;
NC_VAR_INFO_T* v;
...
v = ncindexith(grp->vars,id);
v = ncxindexith(grp->vars,id);
if(v != NULL)
... code to process matching variable by id
}
@ -344,11 +336,11 @@ Similar loops are used to walk a list of dimensions, variables, types,
or attributes.
In the new code, this iteration cliche is replaced by something
that will look like this.
that looks like this.
\code
NC_GRP_INFO_T* grp;
...
for(i=0;i<ncindexsize(grp->children);i++) {
for(i=0;i<ncxindexsize(grp->children);i++) {
NC_GRP_INFO_T* g = nclistith(grp->children,i);
...
}
@ -361,23 +353,21 @@ The initial impetus for this change was to improve the performance
of netcdf-4 metadata loading by replacing linear searches with O(1)
searches.
In fact, this goal has not been met. It appears to be the case
In fact, this goal was not initially met. It appears to be the case
that the metadata loading costs are entirely dominated by the
performance of the HDF5 library. The reason for this is that
the netcdf-c library loads all the metadata immediately
when a file is opened. This in turn means that all of the metadata
is immediately extracted from the underlying HDF5 file. So, there is
the netcdf-c library used to load all the metadata immediately
when a file was opened. This in turn meant that all of the metadata
was immediately extracted from the underlying HDF5 file. So, there was
no opportunity for lazy loading to be used.
The remedys of which I can conceive are these.
1. Modify the netcdf-c library to also do lazy loading
(work on this is under way).
2. Store a single metadata object into the file so it can
be read at one shot. This object would then be processed
in-memory to construct the internal metadata. The costs for
this approach are space in the file plus the need to keep it
consistent with the actual metadata stored by HDF5.
The netcdf library has since been modified to do lazy loading
of variables and attributes. A not pursued alternative would
have been to store a single metadata object into the file so it could
be read at one shot. This object would then be processed
in-memory to construct the internal metadata. The costs for
this approach are space in the file plus the need to keep it
consistent with the actual metadata stored by HDF5.
It should be noted that there is an effect from this change.
Using gprof, one can see that in the original code the obj_list_add
@ -398,27 +388,18 @@ Some other observations:
\section Snotes_and_warnings Notes and Warning
1. NCindex is currently not used for enum constants and compound fields.
1. Ncxindex is currently not used for enum constants and compound fields.
Additionally, it is not used for listing the dimensions associated
with a variable.
with a variable. Small size is the reason.
2. References between meta-data objects (e.g. group parent or
containing group) are stored directly and not using any kind
of vector or hashtable.
3. Attribute deletion is still a costly operation because it causes
the whole attribute index to be rebuilt.
4. Renaming is still a costly operation because it causes
the whole containing index to be rebuilt.
5. As in the original code, object ids (dimid, etc) are assigned
3. Attribute rename and deletion are still moderately costly operations.
4. As in the original code, object ids (dimid, etc) are assigned
explicitly using counters within the NC_FILE_INFO_T object.
When stored into, for example, "alldims", the position of the
object is forcibly made to match the value of the assigned id.
6. The file nchashmap.c has a constant, SMALLTABLE, that controls
the size of the default hash table. Setting this constant
may be useful in debugging by reducing the default table size.
7. The file ncindex.c has a constant, SMALLTABLE, that controls
the size of the default hash table. Setting this constant
may be useful in debugging by reducing the default table size.
8. The file ncindex.c has a constant, NCNOHASH, that controls
5. The file ncxindex.c has a constant, NCNOHASH, that controls
if the index uses that hash table versus just searching the
index's vector. This is for experimental purposes.
@ -426,6 +407,6 @@ Some other observations:
__Author__: Dennis Heimbigner<br>
__Initial Version__: 01/10/2018<br>
__Last Revised__: 03/15/2018
__Last Revised__: 10/30/2020
*/

2
include/Makefile.am
View File

@ -19,7 +19,7 @@ ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h \
nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \
ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \
ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \
hdf5dispatch.h ncmodel.h isnan.h
hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h
if USE_DAP
noinst_HEADERS += ncdap.h

6
include/hdf5internal.h
View File

@ -54,13 +54,17 @@
/** This is the name of the name HDF5 dimension scale attribute. */
#define HDF5_DIMSCALE_NAME_ATT_NAME NC_ATT_NAME
/* forward */
struct NCauth;
/** Struct to hold HDF5-specific info for the file. */
typedef struct NC_HDF5_FILE_INFO {
hid_t hdfid;
#ifdef ENABLE_BYTERANGE
#if defined(ENABLE_BYTERANGE) || defined(ENABLE_HDF5_ROS3) || defined(ENABLE_S3_SDK)
struct HTTP {
NCURI* uri; /* Parse of the incoming path, if url */
int iosp; /* We are using the S3 rawvirtual file driver */
struct NCauth* auth;
} http;
#endif
} NC_HDF5_FILE_INFO_T;

3
include/nc4internal.h
View File

@ -21,7 +21,7 @@
#include "nc_logging.h"
#include "ncindex.h"
#include "nc_provenance.h"
#include "nchashmap.h"
#include "netcdf_f.h"
#include "netcdf_mem.h"
@ -133,7 +133,6 @@ typedef struct NC_OBJ
NC_SORT sort; /**< Type of object. */
char* name; /**< Name, assumed to be null terminated. */
size_t id; /**< This objects ID. */
unsigned int hashkey; /**< The hash key, crc32(name). */
} NC_OBJ;
/**

6
include/nc_provenance.h
View File

@ -13,7 +13,7 @@
*
* For netcdf4 files, capture state information about the following:
* - Global: netcdf library version
* - Global: hdf5 library version
* - Global: hdf5 library version | nczarr library versions
* - Global: any parameters specified by the --with-ncproperties option for ./configure
* - Per file: superblock version
* - Per File: was it created by netcdf-4?
@ -21,7 +21,6 @@
*
* @author Dennis Heimbigner, Ward Fisher
*
* [This file is too hdf5 specific, need to clean so we can use with Zarr]
*/
#ifndef _NCPROVENANCE_
@ -49,7 +48,7 @@ struct NC_FILE_INFO;
/**
For netcdf4 files, capture state information about the following:
1. Global: netcdf library version
2. Global: hdf5 library version
2. Global: hdf5 library version | nczarr library version
3. Per file: superblock version
4. Per File: was it created by netcdf-4?
5. Per file: _NCProperties attribute
@ -70,6 +69,7 @@ extern int NC4_provenance_init(void);
extern int NC4_provenance_finalize(void);
/* Read and store the provenance from an existing file */
extern int NC4_read_provenance(struct NC_FILE_INFO* file);
/* Write the provenance into a newly created file */

8
include/ncauth.h
View File

@ -49,10 +49,14 @@ typedef struct NCauth {
char *user; /*CURLOPT_USERNAME*/
char *pwd; /*CURLOPT_PASSWORD*/
} creds;
struct s3credentials {
char *accessid;
char *secretkey;
} s3creds;
} NCauth;
extern int NC_authsetup(NCauth*, NCURI*);
extern void NC_authclear(NCauth*);
extern int NC_authsetup(NCauth**, NCURI*);
extern void NC_authfree(NCauth*);
extern char* NC_combinehostport(NCURI*);
extern int NC_parsecredentials(const char* userpwd, char** userp, char** pwdp);

8
include/ncconfigure.h
View File

@ -104,6 +104,14 @@ typedef unsigned short ushort;
typedef unsigned int uint;
#endif
#ifndef HAVE_UINT64
typedef unsigned long long uint64;
#endif
#ifndef HAVE_UINT64_T
typedef unsigned long long uint64_t;
#endif
#ifndef _WIN32
#ifndef HAVE_UINTPTR_T
#if SIZEOF_VOIDP == 8

16
include/nccrc.h Normal file
View File

@ -0,0 +1,16 @@
/*
Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT for license information.
*/
/*
CRC32 and CRC64 implementations from Mark Adler
*/
#ifndef NCCRC_H
#define NCCRC_H 1
EXTERNL unsigned int NC_crc32(unsigned int crc, const void* buf, unsigned int len);
EXTERNL unsigned long long NC_crc64(unsigned long long crc, void* buf, unsigned int len);
#endif /*NCCRC_H*/

129
include/ncexhash.h Normal file
View File

@ -0,0 +1,129 @@
/*
Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
See LICENSE.txt for license information.
*/
#ifndef NCEXHASH_H
#define NCEXHASH_H
#if HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdint.h>
#include <netcdf.h>
#if 0
#ifndef EXTERNL
# ifdef _WIN32
# ifdef DLL_EXPORT /* define when building the library */
# define EXTERNL __declspec(dllexport) extern
# else
# define EXTERNL __declspec(dllimport) extern
# endif
# else /*!_WIN32*/
# define EXTERNL extern
# endif /* _WIN32 */
#endif
#endif
/*
Implement extendible hashing as defined in:
````
R. Fagin, J. Nievergelt, N. Pippenger, and H. Strong, "Extendible Hashing -·a fast access method for dynamic files", ACM Transactions on Database Systems, vol. 4, No. 3, pp. 315-344, 1979.
````
*/
/*! Hashmap-related structs.
NOTES:
1. 'data' is the an arbitrary uintptr_t integer or void* pointer.
2. hashkey is a crc64 hash of key -- it is assumed to be unique for keys.
WARNINGS:
1. It is critical that |uintptr_t| == |void*|
*/
#define ncexhashkey_t unsigned long long
#define NCEXHASHKEYBITS 64
typedef struct NCexentry {
ncexhashkey_t hashkey; /* Hash id */
uintptr_t data;
} NCexentry;
typedef struct NCexleaf {
int uid; /* primarily for debug */
struct NCexleaf* next; /* linked list of all leaves for cleanup */
int depth; /* local depth */
int active; /* index of the first emptry slot */
NCexentry* entries; /* |entries| == leaflen*/
} NCexleaf;
/* Top Level Vector */
typedef struct NCexhashmap {
int leaflen; /* # entries a leaf can store */
int depth; /* Global depth */
NCexleaf* leaves; /* head of the linked list of leaves */
int nactive; /* # of active entries in whole table */
NCexleaf** directory; /* |directory| == 2^depth */
int uid; /* unique id counter */
/* Allow a single iterator over the entries */
struct {
int walking; /* 0=>not in use */
int index; /* index of current entry in leaf */
NCexleaf* leaf; /* leaf we are walking */
} iterator;
} NCexhashmap;
/** Creates a new exhash using LSB */
EXTERNL NCexhashmap* ncexhashnew(int leaflen);
/** Reclaims the exhash structure. */
EXTERNL void ncexhashmapfree(NCexhashmap*);
/** Returns the number of active elements. */
EXTERNL int ncexhashcount(NCexhashmap*);
/* Hash key based API */
/* Lookup by Hash Key */
EXTERNL int ncexhashget(NCexhashmap*, ncexhashkey_t hkey, uintptr_t*);
/* Insert by Hash Key */
EXTERNL int ncexhashput(NCexhashmap*, ncexhashkey_t hkey, uintptr_t data);
/* Remove by Hash Key */
EXTERNL int ncexhashremove(NCexhashmap*, ncexhashkey_t hkey, uintptr_t* datap);
/** Change the data for the specified key; takes hashkey. */
EXTERNL int ncexhashsetdata(NCexhashmap*, ncexhashkey_t hkey, uintptr_t newdata, uintptr_t* olddatap);
/** Get map parameters */
EXTERNL int ncexhashinqmap(NCexhashmap* map, int* leaflenp, int* depthp, int* nactivep, int* uidp, int* walkingp);
/* Return the hash key for specified key; takes key+size*/
EXTERNL ncexhashkey_t ncexhashkey(const unsigned char* key, size_t size);
/* Walk the entries in some order */
/*
@return NC_NOERR if keyp and datap are valid
@return NC_ERANGE if iteration is finished
@return NC_EINVAL for all other errors
*/
EXTERNL int ncexhashiterate(NCexhashmap* map, ncexhashkey_t* keyp, uintptr_t* datap);
/* Debugging */
EXTERNL void ncexhashprint(NCexhashmap*);
EXTERNL void ncexhashprintstats(NCexhashmap*);
EXTERNL void ncexhashprintdir(NCexhashmap*, NCexleaf** dir);
EXTERNL void ncexhashprintleaf(NCexhashmap*, NCexleaf* leaf);
EXTERNL void ncexhashprintentry(NCexhashmap* map, NCexentry* entry);
EXTERNL char* ncexbinstr(ncexhashkey_t hkey, int depth);
/* Macro defined functions */
/** Get map parameters */
#define ncexhashmaplength(map) ((map)==NULL?0:(map)->nactive)
#endif /*NCEXHASH_H*/

10
include/nchashmap.h
View File

@ -37,10 +37,16 @@ e.g. crc64 or such. Needs some thought.
1. It is critical that |uintptr_t| == |void*|
*/
/** The type and # bits in a hashkey */
#ifndef nchashkey_t
#define nchashkey_t unsigned
#define NCHASHKEYBITS (sizeof(nchashkey_t)*8)
#endif
typedef struct NC_hentry {
int flags;
uintptr_t data;
unsigned int hashkey; /* Hash id */
nchashkey_t hashkey; /* Hash id (crc)*/
size_t keysize;
char* key; /* copy of the key string; kept as unsigned char */
} NC_hentry;
@ -92,7 +98,7 @@ extern size_t NC_hashmapcount(NC_hashmap*);
extern int NC_hashmapfree(NC_hashmap*);
/* Return the hash key for specified key; takes key+size*/
extern unsigned int NC_hashmapkey(const char* key, size_t size);
extern nchashkey_t NC_hashmapkey(const char* key, size_t size);
/* Return the ith entry info:
@param map

68
include/ncxcache.h Normal file
View File

@ -0,0 +1,68 @@
/*
Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT for license information.
*/
#ifndef NCXCACHE_H
#define NCXCACHE_H
#include "nclist.h"
#include "ncexhash.h" /* Also includes name map and id map */
/* Define the implementation.
if defined, then the user's object
is assumed to hold the double linked list node,
otherwise, it is created here.
*/
#define NCXUSER
/*
This cache data structure is an ordered list of objects. It is
used to create an LRU cache of arbitrary objects.
*/
/* Doubly linked list element */
typedef struct NCxnode {
struct NCxnode* next;
struct NCxnode* prev;
void* content; /* associated data of some kind may be unused*/
} NCxnode;
typedef struct NCxcache {
NCxnode lru;
NCexhashmap* map;
} NCxcache;
/* Locate object by hashkey */
EXTERNL int ncxcachelookup(NCxcache* cache, ncexhashkey_t hkey, void** objp);
/* Insert object into the cache >*/
EXTERNL int ncxcacheinsert(NCxcache* cache, ncexhashkey_t hkey, void* obj);
/* Bring to front of the LRU queue */
EXTERNL int ncxcachetouch(NCxcache* cache, ncexhashkey_t hkey);
/* "Remove" object from the cache; return object */
EXTERNL int ncxcacheremove(NCxcache* cache, ncexhashkey_t hkey, void** obj);
/* Free cache. */
EXTERNL void ncxcachefree(NCxcache* cache);
/* Create a cache: size == 0 => use defaults */
EXTERNL int ncxcachenew(size_t initsize, NCxcache**) ;
/* Macro function */
/* Get the number of entries in an NCxcache */
#define ncxcachecount(cache) (cache == NULL ? 0 : ncexhashcount((cache)->map))
EXTERNL void* ncxcachefirst(NCxcache* cache);
EXTERNL void* ncxcachelast(NCxcache* cache);
/* Return the hash key for specified key; takes key+size; an alias for the one in ncexhash */
EXTERNL ncexhashkey_t ncxcachekey(const void* key, size_t size);
/* Debugging */
EXTERNL void ncxcacheprint(NCxcache* cache);
#endif /*NCXCACHE_H*/

1
libdap4/Makefile.am
View File

@ -18,7 +18,6 @@ LDADD=
#d4curlflags.c
SRC= \
d4crc32.c \
d4curlfunctions.c \
d4fix.c \
d4data.c \

56
libdap4/d4curlfunctions.c
View File

@ -55,41 +55,41 @@ set_curlflag(NCD4INFO* state, int flag)
int ret = NC_NOERR;
switch (flag) {
case CURLOPT_USERPWD: /* Do both user and pwd */
if(state->auth.creds.user != NULL
&& state->auth.creds.pwd != NULL) {
SETCURLOPT(state, CURLOPT_USERNAME, state->auth.creds.user);
SETCURLOPT(state, CURLOPT_PASSWORD, state->auth.creds.pwd);
if(state->auth->creds.user != NULL
&& state->auth->creds.pwd != NULL) {
SETCURLOPT(state, CURLOPT_USERNAME, state->auth->creds.user);
SETCURLOPT(state, CURLOPT_PASSWORD, state->auth->creds.pwd);
SETCURLOPT(state, CURLOPT_HTTPAUTH, (OPTARG)CURLAUTH_ANY);
}
break;
case CURLOPT_COOKIEJAR: case CURLOPT_COOKIEFILE:
if(state->auth.curlflags.cookiejar) {
if(state->auth->curlflags.cookiejar) {
/* Assume we will read and write cookies to same place */
SETCURLOPT(state, CURLOPT_COOKIEJAR, state->auth.curlflags.cookiejar);
SETCURLOPT(state, CURLOPT_COOKIEFILE, state->auth.curlflags.cookiejar);
SETCURLOPT(state, CURLOPT_COOKIEJAR, state->auth->curlflags.cookiejar);
SETCURLOPT(state, CURLOPT_COOKIEFILE, state->auth->curlflags.cookiejar);
}
break;
case CURLOPT_NETRC: case CURLOPT_NETRC_FILE:
if(state->auth.curlflags.netrc) {
if(state->auth->curlflags.netrc) {
SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_REQUIRED);
SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth.curlflags.netrc);
SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc);
}
break;
case CURLOPT_VERBOSE:
if(state->auth.curlflags.verbose)
if(state->auth->curlflags.verbose)
SETCURLOPT(state, CURLOPT_VERBOSE, (OPTARG)1L);
break;
case CURLOPT_TIMEOUT:
if(state->auth.curlflags.timeout)
SETCURLOPT(state, CURLOPT_TIMEOUT, (OPTARG)((long)state->auth.curlflags.timeout));
if(state->auth->curlflags.timeout)
SETCURLOPT(state, CURLOPT_TIMEOUT, (OPTARG)((long)state->auth->curlflags.timeout));
break;
case CURLOPT_CONNECTTIMEOUT:
if(state->auth.curlflags.connecttimeout)
SETCURLOPT(state, CURLOPT_CONNECTTIMEOUT, (OPTARG)((long)state->auth.curlflags.connecttimeout));
if(state->auth->curlflags.connecttimeout)
SETCURLOPT(state, CURLOPT_CONNECTTIMEOUT, (OPTARG)((long)state->auth->curlflags.connecttimeout));
break;
case CURLOPT_USERAGENT:
if(state->auth.curlflags.useragent)
SETCURLOPT(state, CURLOPT_USERAGENT, state->auth.curlflags.useragent);
if(state->auth->curlflags.useragent)
SETCURLOPT(state, CURLOPT_USERAGENT, state->auth->curlflags.useragent);
break;
case CURLOPT_FOLLOWLOCATION:
SETCURLOPT(state, CURLOPT_FOLLOWLOCATION, (OPTARG)1L);
@ -102,19 +102,19 @@ set_curlflag(NCD4INFO* state, int flag)
break;
case CURLOPT_ENCODING:
#ifdef CURLOPT_ENCODING
if(state->auth.curlflags.compress) {
if(state->auth->curlflags.compress) {
SETCURLOPT(state, CURLOPT_ENCODING,"deflate, gzip");
}
#endif
break;
case CURLOPT_PROXY:
if(state->auth.proxy.host != NULL) {
SETCURLOPT(state, CURLOPT_PROXY, state->auth.proxy.host);
SETCURLOPT(state, CURLOPT_PROXYPORT, (OPTARG)(long)state->auth.proxy.port);
if(state->auth.proxy.user != NULL
&& state->auth.proxy.pwd != NULL) {
SETCURLOPT(state, CURLOPT_PROXYUSERNAME, state->auth.proxy.user);
SETCURLOPT(state, CURLOPT_PROXYPASSWORD, state->auth.proxy.pwd);
if(state->auth->proxy.host != NULL) {
SETCURLOPT(state, CURLOPT_PROXY, state->auth->proxy.host);
SETCURLOPT(state, CURLOPT_PROXYPORT, (OPTARG)(long)state->auth->proxy.port);
if(state->auth->proxy.user != NULL
&& state->auth->proxy.pwd != NULL) {
SETCURLOPT(state, CURLOPT_PROXYUSERNAME, state->auth->proxy.user);
SETCURLOPT(state, CURLOPT_PROXYPASSWORD, state->auth->proxy.pwd);
#ifdef CURLOPT_PROXYAUTH
SETCURLOPT(state, CURLOPT_PROXYAUTH, (long)CURLAUTH_ANY);
#endif
@ -125,7 +125,7 @@ set_curlflag(NCD4INFO* state, int flag)
case CURLOPT_SSLCERT: case CURLOPT_SSLKEY:
case CURLOPT_SSL_VERIFYPEER: case CURLOPT_SSL_VERIFYHOST:
{
struct ssl* ssl = &state->auth.ssl;
struct ssl* ssl = &state->auth->ssl;
/* VERIFYPEER == 0 => VERIFYHOST == 0 */
/* We need to have 2 states: default and a set value */
/* So -1 => default, >= 0 => use value; */
@ -284,7 +284,7 @@ cvt(char* value, enum CURLFLAGTYPE type)
void
NCD4_curl_debug(NCD4INFO* state)
{
state->auth.curlflags.verbose = 1;
state->auth->curlflags.verbose = 1;
set_curlflag(state,CURLOPT_VERBOSE);
set_curlflag(state,CURLOPT_ERRORBUFFER);
}
@ -301,10 +301,10 @@ NCD4_curl_protocols(NCD4INFO* state)
curl_version_info_data* curldata;
curldata = curl_version_info(CURLVERSION_NOW);
for(proto=curldata->protocols;*proto;proto++) {
if(strcmp("http",*proto)==0) {state->auth.curlflags.proto_https=1;}
if(strcmp("http",*proto)==0) {state->auth->curlflags.proto_https=1;}
}
#ifdef D4DEBUG
nclog(NCLOGNOTE,"Curl https:// support = %d",state->auth.curlflags.proto_https);
nclog(NCLOGNOTE,"Curl https:// support = %d",state->auth->curlflags.proto_https);
#endif
}

5
libdap4/d4data.c
View File

@ -9,6 +9,7 @@
#include "ezxml.h"
#include "d4includes.h"
#include "d4odom.h"
#include "nccrc.h"
/**
This code serves two purposes
@ -39,11 +40,11 @@ static unsigned int debugcrc32(unsigned int crc, const void *buf, size_t size)
fprintf(stderr,"crc32: ");
for(i=0;i<size;i++) {fprintf(stderr,"%02x",((unsigned char*)buf)[i]);}
fprintf(stderr,"\n");
return NCD4_crc32(crc,buf,size);
return NC_crc32(crc,buf,size);
}
#define CRC32 debugcrc32
#else
#define CRC32 NCD4_crc32
#define CRC32 NC_crc32
#endif
#define ISTOPLEVEL(var) ((var)->container == NULL || (var)->container->sort == NCD4_GROUP)

24
libdap4/d4file.c
View File

@ -315,7 +315,7 @@ freeInfo(NCD4INFO* d4info)
}
nullfree(d4info->substrate.filename); /* always reclaim */
NCD4_reclaimMeta(d4info->substrate.metadata);
NC_authclear(&d4info->auth);
NC_authfree(d4info->auth);
nclistfree(d4info->blobs);
free(d4info);
}
@ -353,26 +353,26 @@ set_curl_properties(NCD4INFO* d4info)
{
int ret = NC_NOERR;
if(d4info->auth.curlflags.useragent == NULL) {
if(d4info->auth->curlflags.useragent == NULL) {
char* agent;
size_t len = strlen(DFALTUSERAGENT) + strlen(VERSION);
len++; /*strlcat nul*/
agent = (char*)malloc(len+1);
strncpy(agent,DFALTUSERAGENT,len);
strlcat(agent,VERSION,len);
d4info->auth.curlflags.useragent = agent;
d4info->auth->curlflags.useragent = agent;
}
/* Some servers (e.g. thredds and columbia) appear to require a place
to put cookies in order for some security functions to work
*/
if(d4info->auth.curlflags.cookiejar != NULL
&& strlen(d4info->auth.curlflags.cookiejar) == 0) {
free(d4info->auth.curlflags.cookiejar);
d4info->auth.curlflags.cookiejar = NULL;
if(d4info->auth->curlflags.cookiejar != NULL
&& strlen(d4info->auth->curlflags.cookiejar) == 0) {
free(d4info->auth->curlflags.cookiejar);
d4info->auth->curlflags.cookiejar = NULL;
}
if(d4info->auth.curlflags.cookiejar == NULL) {
if(d4info->auth->curlflags.cookiejar == NULL) {
/* If no cookie file was defined, define a default */
char* path = NULL;
char* newpath = NULL;
@ -395,16 +395,16 @@ set_curl_properties(NCD4INFO* d4info)
fprintf(stderr,"Cannot create cookie file\n");
goto fail;
}
d4info->auth.curlflags.cookiejar = newpath;
d4info->auth.curlflags.cookiejarcreated = 1;
d4info->auth->curlflags.cookiejar = newpath;
d4info->auth->curlflags.cookiejarcreated = 1;
errno = 0;
}
assert(d4info->auth.curlflags.cookiejar != NULL);
assert(d4info->auth->curlflags.cookiejar != NULL);
/* Make sure the cookie jar exists and can be read and written */
{
FILE* f = NULL;
char* fname = d4info->auth.curlflags.cookiejar;
char* fname = d4info->auth->curlflags.cookiejar;
/* See if the file exists already */
f = fopen(fname,"r");
if(f == NULL) {

3
libdap4/ncd4.h
View File

@ -165,9 +165,6 @@ extern int NCD4_rcdefault(NCD4INFO*);
/* From d4cvt.c */
extern int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char* value0, size_t count);
/* From d4crc32.c */
extern unsigned int NCD4_crc32(unsigned int crc, const void *buf, size_t size);
/* d4file.c */
extern void NCD4_applyclientparamcontrols(NCD4INFO*);

2
libdap4/ncd4types.h
View File

@ -319,7 +319,7 @@ struct NCD4INFO {
char substratename[NC_MAX_NAME];
size_t opaquesize; /* default opaque size */
} controls;
NCauth auth;
NCauth* auth;
struct {
char* filename;
} fileproto;

3
libdispatch/CMakeLists.txt
View File

@ -4,7 +4,8 @@
# University Corporation for Atmospheric Research/Unidata.
# See netcdf-c/COPYRIGHT file for more info.
SET(libdispatch_SOURCES dparallel.c dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dpathmgr.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c crc32.c daux.c dinfermodel.c)
SET(libdispatch_SOURCES dparallel.c dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dpathmgr.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c daux.c dinfermodel.c
dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c)
# Netcdf-4 only functions. Must be defined even if not used
SET(libdispatch_SOURCES ${libdispatch_SOURCES} dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c)

7
libdispatch/Makefile.am
View File

@ -19,8 +19,9 @@ libdispatch_la_SOURCES = dparallel.c dcopy.c dfile.c ddim.c datt.c \
dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c \
dvarinq.c dinternal.c ddispatch.c dutf8.c nclog.c dstring.c ncuri.c \
nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c dauth.c \
doffsets.c dpathmgr.c dutil.c dreadonly.c dnotnc4.c dnotnc3.c crc32.c \
crc32.h daux.c dinfermodel.c
doffsets.c dpathmgr.c dutil.c dreadonly.c dnotnc4.c dnotnc3.c \
daux.c dinfermodel.c \
dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c
# Add the utf8 codebase
libdispatch_la_SOURCES += utf8proc.c utf8proc.h
@ -28,7 +29,7 @@ libdispatch_la_SOURCES += utf8proc.c utf8proc.h
# The rc code is currently only used by libdap2 and libdap4,
# but in the future, it will be expanded to be used as the
# general .rc file for the netcdf-c library. So, always compile it.
libdispatch_la_SOURCES += drc.c
libdispatch_la_SOURCES += drc.c
# Add functions only found in netCDF-4.
# They are always defined, even if they just return an error

27
libdispatch/dauth.c
View File

@ -88,15 +88,19 @@ NC_combinehostport(NCURI* uri)
}
int
NC_authsetup(NCauth* auth, NCURI* uri)
NC_authsetup(NCauth** authp, NCURI* uri)
{
int ret = NC_NOERR;
char* uri_hostport = NULL;
NCauth* auth = NULL;
if(uri != NULL)
uri_hostport = NC_combinehostport(uri);
else
return NC_EDAP; /* Generic EDAP error. */
if((auth=calloc(1,sizeof(NCauth)))==NULL)
return NC_ENOMEM;
setdefaults(auth);
/* Note, we still must do this function even if
@ -145,6 +149,10 @@ NC_authsetup(NCauth* auth, NCURI* uri)
NC_rclookup("HTTP.SSL.VALIDATE",uri_hostport));
setauthfield(auth,"HTTP.NETRC",
NC_rclookup("HTTP.NETRC",uri_hostport));
setauthfield(auth,"HTTP.S3.ACCESSID",
NC_rclookup("HTTP.S3.ACCESSID",uri_hostport));
setauthfield(auth,"HTTP.S3.SECRETKEY",
NC_rclookup("HTTP.S3.SECRETKEY",uri_hostport));
{ /* Handle various cases for user + password */
/* First, see if the user+pwd was in the original url */
@ -174,11 +182,12 @@ NC_authsetup(NCauth* auth, NCURI* uri)
nullfree(pwd);
nullfree(uri_hostport);
}
if(authp) {*authp = auth; auth = NULL;}
return (ret);
}
void
NC_authclear(NCauth* auth)
NC_authfree(NCauth* auth)
{
if(auth == NULL) return;
if(auth->curlflags.cookiejarcreated) {
@ -201,6 +210,9 @@ NC_authclear(NCauth* auth)
nullfree(auth->proxy.pwd);
nullfree(auth->creds.user);
nullfree(auth->creds.pwd);
nullfree(auth->s3creds.accessid);
nullfree(auth->s3creds.secretkey);
nullfree(auth);
}
/**************************************************/
@ -350,6 +362,17 @@ setauthfield(NCauth* auth, const char* flag, const char* value)
MEMCHECK(auth->creds.pwd);
}
if(strcmp(flag,"HTTP.S3.ACCESSID")==0) {
nullfree(auth->s3creds.accessid);
auth->s3creds.accessid = strdup(value);
MEMCHECK(auth->s3creds.accessid);
}
if(strcmp(flag,"HTTP.S3.SECRETKEY")==0) {
nullfree(auth->s3creds.secretkey);
auth->s3creds.secretkey = strdup(value);
MEMCHECK(auth->s3creds.secretkey);
}
done:
return (ret);

24
libdispatch/crc32.c → libdispatch/dcrc32.c
View File

@ -65,12 +65,10 @@ UCAR
first call get_crc_table() to initialize the tables before allowing more than
one thread to use crc32().
DYNAMIC_CRC_TABLE and MAKECRCH can be #defined to write out crc32.h.
DYNAMIC_CRC_TABLE and MAKECRCH can be #defined to write out dcrc32.h.
*/
/* Prototype for the crc32 function
extern unsigned int NC_crc32(unsigned int crc, const unsigned char* buf, unsigned int len);
*/
/* Prototype for the crc32 function is defined in include/nccrc.h */
/* Define some of the macros used here */
#define FAR
@ -109,6 +107,8 @@ extern unsigned int NC_crc32(unsigned int crc, const unsigned char* buf, unsigne
# define TBLS 1
#endif /* BYFOUR */
local const z_crc_t FAR crc_table[TBLS][256];
#ifdef DYNAMIC_CRC_TABLE
local volatile int crc_table_empty = 1;
@ -194,14 +194,14 @@ local void make_crc_table()
}
#ifdef MAKECRCH
/* write out CRC tables to crc32.h */
/* write out CRC tables to dcrc32.h */
{
FILE *out;
out = fopen("crc32.h", "w");
out = fopen("dcrc32.h", "w");
if (out == NULL) return;
fprintf(out, "/* crc32.h -- tables for rapid CRC calculation\n");
fprintf(out, " * Generated automatically by crc32.c\n */\n\n");
fprintf(out, "/* dcrc32.h -- tables for rapid CRC calculation\n");
fprintf(out, " * Generated automatically by dcrc32.c\n */\n\n");
fprintf(out, "local const z_crc_t FAR ");
fprintf(out, "crc_table[TBLS][256] =\n{\n {\n");
write_table(out, crc_table[0]);
@ -237,7 +237,7 @@ local void write_table(out, table)
/* ========================================================================
* Tables of CRC-32s of all single-byte values, made by make_crc_table().
*/
#include "crc32.h"
#include "dcrc32.h"
#endif /* DYNAMIC_CRC_TABLE */
/* =========================================================================
@ -294,10 +294,12 @@ local unsigned long ZEXPORT crc32_z(crc, buf, len)
}
/* ========================================================================= */
unsigned int ZEXPORT NC_crc32(unsigned int crc, const unsigned char* buf, unsigned int len)
unsigned int ZEXPORT
NC_crc32(unsigned int crc, const void* buf, unsigned int len)
{
unsigned long value = (unsigned long)crc;
value = crc32_z(value, buf, len);
unsigned char* cbuf = (unsigned char*)buf;
value = crc32_z(value, cbuf, len);
return (unsigned int)(value & 0xFFFFFFFF); /* in case |long| is 64 bits */
}

0
libdispatch/crc32.h → libdispatch/dcrc32.h Normal file → Executable file
View File

344
libdispatch/dcrc64.c Normal file
View File

@ -0,0 +1,344 @@
/* crc64.c -- compute CRC-64
* Copyright (C) 2013 Mark Adler
* Version 1.4 16 Dec 2013 Mark Adler
*/
/*
This software is provided 'as-is', without any express or implied
warranty. In no event will the author be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
Mark Adler
madler@alumni.caltech.edu
*/
/* Compute CRC-64 in the manner of xz, using the ECMA-182 polynomial,
bit-reversed, with one's complement pre and post processing. Provide a
means to combine separately computed CRC-64's. */
/* Version history:
1.0 13 Dec 2013 First version
1.1 13 Dec 2013 Fix comments in test code
1.2 14 Dec 2013 Determine endianess at run time
1.3 15 Dec 2013 Add eight-byte processing for big endian as well
Make use of the pthread library optional
1.4 16 Dec 2013 Make once variable volatile for limited thread protection
*/
#include "config.h"
#include <stdio.h>
#include <inttypes.h>
#include <assert.h>
/* The include of pthread.h below can be commented out in order to not use the
pthread library for table initialization. In that case, the initialization
will not be thread-safe. That's fine, so long as it can be assured that
there is only one thread using crc64(). */
#if 0
#include <pthread.h> /* link with -lpthread */
#endif
/* 64-bit CRC polynomial with these coefficients, but reversed:
64, 62, 57, 55, 54, 53, 52, 47, 46, 45, 40, 39, 38, 37, 35, 33, 32,
31, 29, 27, 24, 23, 22, 21, 19, 17, 13, 12, 10, 9, 7, 4, 1, 0 */
#define POLY UINT64_C(0xc96c5795d7870f42)
/* Tables for CRC calculation -- filled in by initialization functions that are
called once. These could be replaced by constant tables generated in the
same way. There are two tables, one for each endianess. Since these are
static, i.e. local, one should be compiled out of existence if the compiler
can evaluate the endianess check in crc64() at compile time. */
static uint64 crc64_little_table[8][256];
static uint64 crc64_big_table[8][256];
/* Fill in the CRC-64 constants table. */
static void crc64_init(uint64 table[][256])
{
unsigned n, k;
uint64 crc;
/* generate CRC-64's for all single byte sequences */
for (n = 0; n < 256; n++) {
crc = n;
for (k = 0; k < 8; k++)
crc = crc & 1 ? POLY ^ (crc >> 1) : crc >> 1;
table[0][n] = crc;
}
/* generate CRC-64's for those followed by 1 to 7 zeros */
for (n = 0; n < 256; n++) {
crc = table[0][n];
for (k = 1; k < 8; k++) {
crc = table[0][crc & 0xff] ^ (crc >> 8);
table[k][n] = crc;
}
}
}
/* This function is called once to initialize the CRC-64 table for use on a
little-endian architecture. */
static void crc64_little_init(void)
{
crc64_init(crc64_little_table);
}
/* Reverse the bytes in a 64-bit word. */
static inline uint64 rev8(uint64 a)
{
uint64 m;
m = UINT64_C(0xff00ff00ff00ff);
a = ((a >> 8) & m) | (a & m) << 8;
m = UINT64_C(0xffff0000ffff);
a = ((a >> 16) & m) | (a & m) << 16;
return a >> 32 | a << 32;
}
/* This function is called once to initialize the CRC-64 table for use on a
big-endian architecture. */
static void crc64_big_init(void)
{
unsigned k, n;
crc64_init(crc64_big_table);
for (k = 0; k < 8; k++)
for (n = 0; n < 256; n++)
crc64_big_table[k][n] = rev8(crc64_big_table[k][n]);
}
/* Run the init() function exactly once. If pthread.h is not included, then
this macro will use a simple static state variable for the purpose, which is
not thread-safe. The init function must be of the type void init(void). */
#ifdef PTHREAD_ONCE_INIT
# define ONCE(init) \
do { \
static pthread_once_t once = PTHREAD_ONCE_INIT; \
pthread_once(&once, init); \
} while (0)
#else
# define ONCE(init) \
do { \
static volatile int once = 1; \
if (once) { \
if (once++ == 1) { \
init(); \
once = 0; \
} \
else \
while (once) \
; \
} \
} while (0)
#endif
/* Calculate a CRC-64 eight bytes at a time on a little-endian architecture. */
static inline uint64 crc64_little(uint64 crc, void *buf, size_t len)
{
unsigned char *next = buf;
ONCE(crc64_little_init);
crc = ~crc;
while (len && ((uintptr_t)next & 7) != 0) {
crc = crc64_little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
len--;
}
while (len >= 8) {
crc ^= *(uint64 *)next;
crc = crc64_little_table[7][crc & 0xff] ^
crc64_little_table[6][(crc >> 8) & 0xff] ^
crc64_little_table[5][(crc >> 16) & 0xff] ^
crc64_little_table[4][(crc >> 24) & 0xff] ^
crc64_little_table[3][(crc >> 32) & 0xff] ^
crc64_little_table[2][(crc >> 40) & 0xff] ^
crc64_little_table[1][(crc >> 48) & 0xff] ^
crc64_little_table[0][crc >> 56];
next += 8;
len -= 8;
}
while (len) {
crc = crc64_little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
len--;
}
return ~crc;
}
/* Calculate a CRC-64 eight bytes at a time on a big-endian architecture. */
static inline uint64 crc64_big(uint64 crc, void *buf, size_t len)
{
unsigned char *next = buf;
ONCE(crc64_big_init);
crc = ~rev8(crc);
while (len && ((uintptr_t)next & 7) != 0) {
crc = crc64_big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
len--;
}
while (len >= 8) {
crc ^= *(uint64 *)next;
crc = crc64_big_table[0][crc & 0xff] ^
crc64_big_table[1][(crc >> 8) & 0xff] ^
crc64_big_table[2][(crc >> 16) & 0xff] ^
crc64_big_table[3][(crc >> 24) & 0xff] ^
crc64_big_table[4][(crc >> 32) & 0xff] ^
crc64_big_table[5][(crc >> 40) & 0xff] ^
crc64_big_table[6][(crc >> 48) & 0xff] ^
crc64_big_table[7][crc >> 56];
next += 8;
len -= 8;
}
while (len) {
crc = crc64_big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
len--;
}
return ~rev8(crc);
}
/* Return the CRC-64 of buf[0..len-1] with initial crc, processing eight bytes
at a time. This selects one of two routines depending on the endianess of
the architecture. A good optimizing compiler will determine the endianess
at compile time if it can, and get rid of the unused code and table. If the
endianess can be changed at run time, then this code will handle that as
well, initializing and using two tables, if called upon to do so. */
static int littleendian = -1;
uint64
NC_crc64(uint64 crc, void *buf, unsigned int len)
{
/* Is this machine big vs little endian? */
if(littleendian < 0) {
unsigned char* p = (void*)&littleendian;
littleendian = 1;
if(*p == 0) littleendian = 0; /* big endian */
}
return littleendian ? crc64_little(crc, buf, (size_t)len) :
crc64_big(crc, buf, (size_t)len);
}
#define GF2_DIM 64 /* dimension of GF(2) vectors (length of CRC) */
static uint64 gf2_matrix_times(uint64 *mat, uint64 vec)
{
uint64 sum;
sum = 0;
while (vec) {
if (vec & 1)
sum ^= *mat;
vec >>= 1;
mat++;
}
return sum;
}
static void gf2_matrix_square(uint64 *square, uint64 *mat)
{
unsigned n;
for (n = 0; n < GF2_DIM; n++)
square[n] = gf2_matrix_times(mat, mat[n]);
}
/* Return the CRC-64 of two sequential blocks, where crc1 is the CRC-64 of the
first block, crc2 is the CRC-64 of the second block, and len2 is the length
of the second block. */
uint64 crc64_combine(uint64 crc1, uint64 crc2, uintmax_t len2)
{
unsigned n;
uint64 row;
uint64 even[GF2_DIM]; /* even-power-of-two zeros operator */
uint64 odd[GF2_DIM]; /* odd-power-of-two zeros operator */
/* degenerate case */
if (len2 == 0)
return crc1;
/* put operator for one zero bit in odd */
odd[0] = POLY; /* CRC-64 polynomial */
row = 1;
for (n = 1; n < GF2_DIM; n++) {
odd[n] = row;
row <<= 1;
}
/* put operator for two zero bits in even */
gf2_matrix_square(even, odd);
/* put operator for four zero bits in odd */
gf2_matrix_square(odd, even);
/* apply len2 zeros to crc1 (first square will put the operator for one
zero byte, eight zero bits, in even) */
do {
/* apply zeros operator for this bit of len2 */
gf2_matrix_square(even, odd);
if (len2 & 1)
crc1 = gf2_matrix_times(even, crc1);
len2 >>= 1;
/* if no more bits set, then done */
if (len2 == 0)
break;
/* another iteration of the loop with odd and even swapped */
gf2_matrix_square(odd, even);
if (len2 & 1)
crc1 = gf2_matrix_times(odd, crc1);
len2 >>= 1;
/* if no more bits set, then done */
} while (len2 != 0);
/* return combined crc */
crc1 ^= crc2;
return crc1;
}
#if 0
/* Test crc64() on vector[0..len-1] which should have CRC-64 crc. Also test
crc64_combine() on vector[] split in two. */
static void crc64_test(void *vector, size_t len, uint64 crc)
{
uint64 crc1, crc2;
/* test crc64() */
crc1 = crc64(0, vector, len);
if (crc1 ^ crc)
printf("mismatch: %" PRIx64 ", should be %" PRIx64 "n", (ulong)crc1, (ulong)crc);
/* test crc64_combine() */
crc1 = crc64(0, vector, (len + 1) >> 1);
crc2 = crc64(0, vector + ((len + 1) >> 1), len >> 1);
crc1 = crc64_combine(crc1, crc2, len >> 1);
if (crc1 ^ crc)
printf("mismatch: %" PRIx64 ", should be %" PRIx64 "n", (ulong)crc1, (ulong)crc);
}
/* Test vectors. */
#define TEST1 "123456789"
#define TESTLEN1 9
#define TESTCRC1 UINT64_C(0x995dc9bbdf1939fa)
#define TEST2 "This is a test of the emergency broadcast system."
#define TESTLEN2 49
#define TESTCRC2 UINT64_C(0x27db187fc15bbc72)
int main(void)
{
crc64_test(TEST1, TESTLEN1, TESTCRC1);
crc64_test(TEST2, TESTLEN2, TESTCRC2);
return 0;
}
#endif

802
libdispatch/ncexhash.c Normal file
View File

@ -0,0 +1,802 @@
/*
Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
See LICENSE.txt for license information.
*/
#include "config.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "netcdf.h"
#include "ncexhash.h"
#include "nccrc.h"
/* 0 => no debug */
#define DEBUG 0
#undef DEBUGTRACE
#undef CATCH
#ifdef CATCH
#define THROW(x) throw(x)
static void breakpoint(void) {}
static int ignore[] = {NC_ENOTFOUND, 0};
static int throw(int x)
{
int* p;
if(x != 0) {
for(p=ignore;*p;p++) {if(x == *p) break;}
if(*p == 0) breakpoint();
}
return x;
}
#else
#define THROW(x) (x)
#endif
/**
@Internal
*/
#ifdef DEBUGTRACE
#define TRACE(x) {fprintf(stderr,">>>> %s\n",x); fflush(stderr);}
#else
#define TRACE(x)
#endif
/* Minimum table size is 2 */
#define MINDEPTH 1
/* Minimum leaf size is 2 entries */
#define MINLEAFLEN 2
#define MAX(a,b) ((a) > (b) ? (a) : (b))
static int ncexinitialized = 0;
/* Define a vector of bit masks */
ncexhashkey_t bitmasks[NCEXHASHKEYBITS];
/* Extract the leftmost n bits from h */
#if 1
#define MSB(h,d) (((h) >> (NCEXHASHKEYBITS - (d))) & bitmasks[d])
#else
static ncexhashkey_t
MSB(ncexhashkey_t h, int d)
{
ncexhashkey_t bm = bitmasks[d];
ncexhashkey_t hkey = h >> (NCEXHASHKEYBITS - d);
hkey = hkey & bm;
return hkey;
}
#endif
/* Provide a mask to get the rightmost bit
of the left n bits of our hash key */
#define MSBMASK(d) (1 << (NCEXHASHKEYBITS - d))
static int exhashlookup(NCexhashmap* map, ncexhashkey_t hkey, NCexleaf** leafp, int* indexp);
static int exhashlocate(NCexhashmap* map, ncexhashkey_t hkey, NCexleaf** leafp, int* indexp);
static int exhashsplit(NCexhashmap* map, ncexhashkey_t hkey, NCexleaf* leaf);
static int exhashdouble(NCexhashmap* map);
static int exbinsearch(ncexhashkey_t hkey, NCexleaf* leaf, int* indexp);
static void exhashnewentry(NCexhashmap* map, NCexleaf* leaf, ncexhashkey_t hkey, int* indexp);
/**************************************************/
static void
ncexinit(void)
{
int i;
bitmasks[0] = 0;
for(i=1;i<NCEXHASHKEYBITS;i++)
bitmasks[i] = (1 << i) - 1;
ncexinitialized = 1;
}
/**************************************************/
/** Creates a new exhash using DEPTH */
NCexhashmap*
ncexhashnew(int leaflen)
{
NCexhashmap* map = NULL;
NCexleaf* leaf[2] = {NULL,NULL};
NCexleaf** topvector = NULL;
int i;
int gdepth;
TRACE("ncexhashnew");
if(!ncexinitialized) ncexinit();
gdepth = MINDEPTH;
if(leaflen < MINLEAFLEN) leaflen = MINLEAFLEN;
/* Create 2 leaves so we can set the initial leaf depth to one */
if((leaf[0] = calloc(1,sizeof(NCexleaf))) == NULL)
goto done;
if((leaf[1] = calloc(1,sizeof(NCexleaf))) == NULL)
goto done;
/* Fill in the leaves */
if((leaf[0]->entries = calloc(leaflen,sizeof(NCexentry))) == NULL)
goto done;
if((leaf[1]->entries = calloc(leaflen,sizeof(NCexentry))) == NULL)
goto done;
leaf[0]->depth = 1;
leaf[1]->depth = 1;
/* Create the top level vector */
if((topvector = calloc(1<<gdepth,sizeof(NCexleaf*))) == NULL)
goto done;
/* Fill it in */
for(i=0;i<(1<<gdepth);i++) topvector[i] = (i & 0x1?leaf[1]:leaf[0]);
/* Create the table */
if((map = (NCexhashmap*)calloc(1,sizeof(NCexhashmap))) == NULL)
goto done;
/* leaf chain */
map->leaves = leaf[1];
leaf[1]->next = leaf[0];
map->depth = gdepth;
map->directory = topvector;
map->leaflen = leaflen;
topvector = NULL;
leaf[0]->uid = map->uid++;
leaf[1]->uid = map->uid++;
leaf[0] = leaf[1] = NULL;
done:
if(leaf[0]) nullfree(leaf[0]->entries);
if(leaf[1]) nullfree(leaf[1]->entries);
if(leaf[0]) free(leaf[0]);
if(leaf[1]) free(leaf[1]);
if(topvector) free(topvector);
return map;
}
/** Reclaims the exhash structure. */
void
ncexhashmapfree(NCexhashmap* map)
{
NCexleaf* current = NULL;
NCexleaf* next= NULL;
if(map == NULL) return;
/* Walk the leaf chain to free leaves */
current = map->leaves; next = NULL;
while(current) {
next = current->next;
nullfree(current->entries);
free(current);
current = next;
}
nullfree(map->directory);
free(map);
}
/** Returns the number of active elements. */
int
ncexhashcount(NCexhashmap* map)
{
return map->nactive;
}
/* Hash key based API */
/* Lookup by Hash Key */
int
ncexhashget(NCexhashmap* map, ncexhashkey_t hkey, uintptr_t* datap)
{
int stat = NC_NOERR;
NCexleaf* leaf;
NCexentry* entry;
int index;
/* Do internal lookup */
if((stat = exhashlookup(map,hkey,&leaf,&index)))
return THROW(stat);
entry = &leaf->entries[index];
assert(entry->hashkey == hkey);
if(datap) *datap = entry->data;
return THROW(stat);
}
/* Insert by Hash Key */
int
ncexhashput(NCexhashmap* map, ncexhashkey_t hkey, uintptr_t data)
{
int stat = NC_NOERR;
NCexleaf* leaf;
NCexentry* entry;
int index;
if(map->iterator.walking) return THROW(NC_EPERM);
/* Do internal lookup */
if((stat = exhashlookup(map,hkey,&leaf,&index)) == NC_ENOTFOUND) {
/* We have to add an entry (sigh!) so find where it goes */
if((stat = exhashlocate(map,hkey,&leaf,&index)))
return THROW(stat);
}
entry = &leaf->entries[index];
entry->hashkey = hkey;
assert(entry->hashkey == hkey);
entry->data = data;
return THROW(stat);
}
static int
exhashlookup(NCexhashmap* map, ncexhashkey_t hkey, NCexleaf** leafp, int* indexp)
{
int stat = NC_NOERR;
NCexleaf* leaf;
ncexhashkey_t offset;
int index;
TRACE("exhashlookup");
/* Extract global depth least significant bits of hkey */
offset = MSB(hkey,map->depth);
/* Get corresponding leaf from directory */
leaf = map->directory[offset];
if(leafp) *leafp = leaf; /* always return this if possible */
/* Binary search the leaf entries looking for hkey */
stat = exbinsearch(hkey,leaf,&index);
if(indexp) *indexp = index;
#if DEBUG >= 3
fprintf(stderr,"lookup: found=%s offset=%x leaf=%d index=%d\n",
(stat == NC_NOERR ? "true" : "false"),
offset,leaf->uid,index);
#endif
return THROW(stat);
}
static int
exhashlocate(NCexhashmap* map, ncexhashkey_t hkey, NCexleaf** leafp, int* indexp)
{
int stat = NC_NOERR;
ncexhashkey_t offset;
NCexleaf* leaf = NULL;
int index = -1;
int iter;
TRACE("exhashlocate");
#if DEBUG >= 2
fprintf(stderr,"locate.before: "); ncexhashprint(map);
#endif
/* Setup */
*leafp = NULL;
*indexp = -1;
if(map->iterator.walking) return THROW(NC_EPERM);
/* Repeatedly split and/or double until the target leaf has room */
for(iter=0;;iter++) {
/* Extract global depth least significant bits of hkey */
offset = MSB(hkey,map->depth);
/* Get corresponding leaf from directory */
leaf = map->directory[offset];
/* Is there room in the leaf to add an entry? */
#if DEBUG >= 3
fprintf(stderr,"locate: iter=%d offset=%x leaf=(%d)%p active=%d\n",iter,offset,leaf->uid,leaf,(int)leaf->active);
#endif
if(leaf->active < map->leaflen) break; /* yes, there is room */
/* Not Enough room, so we need to split this leaf */
/* Split this leaf and loop to verify we have room */
#if DEBUG >= 3
fprintf(stderr,"locate.split.loop: uid=%d\n",leaf->uid);
#endif
if((stat = exhashsplit(map,hkey,leaf))) return THROW(stat); /* failed */
}
/* We now now that there is room in the leaf */
/* Insert into this leaf */
exhashnewentry(map,leaf,hkey,&index);
#if DEBUG >= 3
fprintf(stderr,"locate.inserted: index=%d\n",index);
#endif
#if DEBUG >= 1
fprintf(stderr,"locate.inserted.after: "); ncexhashprint(map);
#endif
*leafp = leaf;
*indexp = index;
return THROW(stat);
}
static int
exhashdouble(NCexhashmap* map)
{
NCexleaf** olddir = NULL;
NCexleaf** newdir = NULL;
size_t oldcount,newcount;
ncexhashkey_t iold, inew;
TRACE("exhashdouble");
if(map->iterator.walking) return THROW(NC_EPERM);
#if DEBUG >= 1
fprintf(stderr,"double.before: "); ncexhashprint(map);
#endif
olddir = map->directory;
/* Attempt to double the directory count */
oldcount = (1<<map->depth);
newcount = 2 * oldcount;
newdir = (NCexleaf**)malloc(newcount*sizeof(NCexleaf*));
if(newdir == NULL) return THROW(NC_ENOMEM);
/* Note that newdir == olddir is possible because realloc */
/* Walk the old directory from top to bottom to double copy
into newspace */
assert(oldcount >= 1 && newcount >= 2);
iold = oldcount;
inew = newcount;
do {
iold -= 1;
inew -= 2;
newdir[inew] = olddir[iold];
newdir[inew+1] = olddir[iold];
} while(iold > 0);
assert(iold == 0 && inew == 0);
/* Fix up */
map->directory = newdir;
map->depth++;
#if DEBUG >= 1
fprintf(stderr,"double.after: "); ncexhashprint(map);
#endif
nullfree(olddir);
return THROW(NC_NOERR);
}
static int
exhashsplit(NCexhashmap* map, ncexhashkey_t hkey, NCexleaf* leaf)
{
int stat = NC_NOERR;
NCexleaf* newleaf = NULL;
NCexleaf* leafptr = leaf;
NCexleaf entries;
int i, index;
TRACE("exhashsplit");
if(map->iterator.walking) {stat = NC_EPERM; goto done;}
#if DEBUG >= 1
fprintf(stderr,"split.before: leaf=%d",leaf->uid); ncexhashprint(map);
#endif
/* Save the old leaf's entries */
entries = *leaf;
/* bump leaf depth */
leaf->depth++;
/* May require doubling of the map directory */
#if DEBUG >= 3
fprintf(stderr,"split: leaf.depth=%d map.depth=%d\n",leaf->depth,map->depth);
#endif
if(leaf->depth > map->depth) {
/* double the directory */
if((stat = exhashdouble(map))) return THROW(stat); /* failed */
}
/* Re-build the old leaf; keep same uid */
if((leaf->entries = (NCexentry*)calloc(map->leaflen,sizeof(NCexentry))) == NULL)
{stat = NC_ENOMEM; goto done;}
leaf->active = 0;
/* Allocate the new leaf */
if((newleaf = (NCexleaf*)calloc(1,sizeof(NCexleaf)))==NULL)
{stat = NC_ENOMEM; goto done;}
if((newleaf->entries = (NCexentry*)calloc(map->leaflen,sizeof(NCexentry))) == NULL)
{stat = NC_ENOMEM; goto done;}
newleaf->uid = map->uid++;
newleaf->depth = leaf->depth;
#if DEBUG >= 3
fprintf(stderr,"split.split: newleaf=");ncexhashprintleaf(map,newleaf);
#endif
/* Now walk the directory to locate all occurrences of old
leaf and replace with newleaf in those cases where the
directory index % 2 == 1
*/
for(i=0;i<(1<<map->depth);i++) {
if(map->directory[i] == leafptr) {
if(i % 2 == 1) { /* entries should be newleaf */
#if DEBUG >= 3
fprintf(stderr,"split.directory[%d]=%d (newleaf)\n",(int)i,newleaf->uid);
#endif
map->directory[i] = newleaf;
}
}
}
#if DEBUG >= 1
fprintf(stderr,"split.after: leaf=%d newleaf=%d",leaf->uid,newleaf->uid); ncexhashprint(map);
#endif
/* link in newleaf and release local pointer */
newleaf->next = map->leaves; /* add to leaf list */
map->leaves = newleaf;
newleaf = NULL; /* no longer needed */
/* Now re-insert the entries */
/* Should not cause splits or doubles */
for(i=0;i<entries.active;i++) {
NCexentry* e = &entries.entries[i];
switch (stat = exhashlookup(map,e->hashkey,&leaf,&index)) {
case NC_NOERR: /* Already exists, so fail */
stat = NC_EINTERNAL;
goto done;
default:
stat = NC_NOERR;
break;
}
assert(leaf != NULL);
/* Insert in the proper leaf */
#if DEBUG >= 3
fprintf(stderr,"split.reinsert: entry.hashkey=%x leaf.uid=%d index=%d\n",
e->hashkey,leaf->uid,index);
#endif
leaf->entries[index] = *e;
leaf->active++;
#if DEBUG >= 1
fprintf(stderr,"split.reinsert.after: "); ncexhashprint(map);
#endif
}
done:
if(stat) { /* unwind */
nullfree(leaf->entries);
*leaf = entries;
if(newleaf)
map->leaves = newleaf->next;
} else {
nullfree(entries.entries);
}
if(newleaf) {
nullfree(newleaf->entries);
nullfree(newleaf);
}
return THROW(stat);
}
/**
* @internal Define a binary searcher for hash keys in leaf
* @param hkey for which to search
* @param leaf to search
* @param indexp store index of the match or if no match, the index
* where new value should be stored (0 <= *indexp <= leaf->active)
* @return NC_NOERR if found
* @return NC_ENOTFOUND if not found, indexp points to insertion location
* @author Dennis Heimbigner
*/
static int
exbinsearch(ncexhashkey_t hkey, NCexleaf* leaf, int* indexp)
{
int stat = NC_NOERR;
int n = leaf->active;
int L = 0;
int R = (n - 1);
if(n == 0) {
if(indexp) *indexp = 0; /* Insert at 0 */
return THROW(NC_ENOTFOUND);
}
while(L != R) {
int m = (L + R);
if((m & 0x1)) /* ceiling */
m = (m / 2) + 1;
else
m = (m / 2);
if(leaf->entries[m].hashkey > hkey)
R = m - 1;
else
L = m;
}
/* Return match index or insertion index */
if(leaf->entries[L].hashkey == hkey) {
/* L = L; */
/* stat = NC_NOERR; */
} else if(leaf->entries[L].hashkey > hkey) {
/* L = L; */
stat = NC_ENOTFOUND;
} else {/* (leaf->entries[m].hashkey < hkey) */
L = L+1;
stat = NC_ENOTFOUND;
}
if(indexp) *indexp = L;
return THROW(stat);
}
/* Create new entry at position index */
static void
exhashnewentry(NCexhashmap* map, NCexleaf* leaf, ncexhashkey_t hkey, int* indexp)
{
int stat;
int dst,src;
int index;
/* Figure out where the key should be inserted (*indexp + 1)*/
stat = exbinsearch(hkey,leaf,indexp);
assert(stat != NC_NOERR); /* already present */
index = *indexp;
assert(index >= 0 && index <= leaf->active);
assert(index == leaf->active || leaf->entries[index].hashkey > hkey);
dst = leaf->active;
src = leaf->active - 1;
for(;src >= index;src--,dst--)
leaf->entries[dst] = leaf->entries[src];
#if 0
leaf->entries[index].hashkey = hkey;
#else
leaf->entries[index].hashkey = (ncexhashkey_t)0xffffffffffffffff;
#endif
leaf->entries[index].data = 0;
leaf->active++;
map->nactive++;
}
/**
* Remove by Hash Key
* @param map
* @param hkey
* @param datap Store the data of the removed hkey
* @return NC_NOERR if found
* @return NC_ENOTFOUND if not found
* @return NC_EINVAL for other errors
* @author Dennis Heimbigner
*/
int
ncexhashremove(NCexhashmap* map, ncexhashkey_t hkey, uintptr_t* datap)
{
int stat = NC_NOERR;
NCexleaf* leaf;
int src,dst;
if(map->iterator.walking) return THROW(NC_EPERM);
if((stat = exhashlookup(map,hkey,&leaf,&dst)))
return THROW(stat);
if(datap) *datap = leaf->entries[dst].data;
/* Compress out the index'th entry */
for(src=dst+1;src<leaf->active;src++,dst++)
leaf->entries[dst] = leaf->entries[src];
leaf->active--;
map->nactive--;
return THROW(stat);
}
/**
* Change data associated with key; do not insert if not found
* @param map
* @param hkey
* @param newdata new data
* @param olddatap Store previous value
* @return NC_NOERR if found
* @return NC_ENOTFOUND if not found
* @return NC_EINVAL for other errors
* @author Dennis Heimbigner
*/
int
ncexhashsetdata(NCexhashmap* map, ncexhashkey_t hkey, uintptr_t newdata, uintptr_t* olddatap)
{
int stat = NC_NOERR;
NCexleaf* leaf = NULL;
NCexentry* e = NULL;
int index;
if(map->iterator.walking) return THROW(NC_EPERM);
if((stat = exhashlookup(map,hkey,&leaf,&index)))
return THROW(stat);
e = &leaf->entries[index];
if(olddatap) *olddatap = e->data;
e->data = newdata;
return THROW(stat);
}
/**
* Inquire map-related values
* @param map
* @param leaflenp Store map->leaflen
* @param depthp Store map->depth
* @param nactivep Store map->nactive
* @param uidp Store map->uid
* @param walkingp Store map->iterator.walking
*
* @return NC_NOERR if found
* @return NC_EINVAL for other errors
* @author Dennis Heimbigner
*/
int
ncexhashinqmap(NCexhashmap* map, int* leaflenp, int* depthp, int* nactivep, int* uidp, int* walkingp)
{
if(map == NULL) return NC_EINVAL;
if(leaflenp) *leaflenp = map->leaflen;
if(depthp) *depthp = map->depth;
if(nactivep) *nactivep = map->nactive;
if(uidp) *uidp = map->uid;
if(walkingp) *walkingp = map->iterator.walking;
return NC_NOERR;
}
/* Return the hash key for specified key; takes key+size*/
ncexhashkey_t
ncexhashkey(const unsigned char* key, size_t size)
{
return NC_crc64(0,(unsigned char*)key,(unsigned int)size);
}
/* Walk the entries in some order */
/*
@return NC_NOERR if keyp and datap are valid
@return NC_ERANGE if iteration is finished
@return NC_EINVAL for all other errors
*/
int
ncexhashiterate(NCexhashmap* map, ncexhashkey_t* keyp, uintptr_t* datap)
{
int stat = NC_NOERR;
if(!map->iterator.walking) {
map->iterator.leaf = map->leaves;
map->iterator.index = 0;
map->iterator.walking = 1;
}
for(;;) {
if(map->iterator.leaf == NULL)
{stat = NC_ERANGE; break;}
if(map->iterator.index >= map->iterator.leaf->active) {
map->iterator.leaf = map->iterator.leaf->next;
map->iterator.index = 0;
} else {
assert(map->iterator.leaf != NULL && map->iterator.index < map->iterator.leaf->active);
/* Return data from this entry */
if(keyp) *keyp = map->iterator.leaf->entries[map->iterator.index].hashkey;
if(datap) *datap = map->iterator.leaf->entries[map->iterator.index].data;
map->iterator.index++;
break;
}
}
if(stat != NC_NOERR) { /* stop */
map->iterator.walking = 0;
map->iterator.leaf = NULL;
map->iterator.index = 0;
}
return THROW(stat);
}
/**************************************************/
/* Debug support */
void
ncexhashprint(NCexhashmap* hm)
{
int dirindex,index;
if(hm == NULL) {fprintf(stderr,"NULL"); fflush(stderr); return;}
fprintf(stderr,"{depth=%u leaflen=%u",hm->depth,hm->leaflen);
if(hm->iterator.walking) {
fprintf(stderr," iterator=(leaf=%p index=%u)",
hm->iterator.leaf,hm->iterator.index);
}
fprintf(stderr,"\n");
for(dirindex=0;dirindex<(1<<hm->depth);dirindex++) {
NCexleaf* leaf = hm->directory[dirindex];
fprintf(stderr,"\tdirectory[%03d|%sb]=(%04x)[(%u)^%d|%d|",
dirindex,ncexbinstr(dirindex,hm->depth),
leaf->active,
(unsigned)(0xffff & (uintptr_t)leaf),
leaf->uid, leaf->depth);
for(index=0;index<leaf->active;index++) {
ncexhashkey_t hkey, bits;
const char* s;
hkey = leaf->entries[index].hashkey;
/* Reduce to the leaf->hash MSB */
bits = MSB(hkey,hm->depth);
s = ncexbinstr(bits,hm->depth);
fprintf(stderr,"%s(%s/",(index==0?":":" "),s);
bits = MSB(hkey,leaf->depth);
s = ncexbinstr(bits,leaf->depth);
fprintf(stderr,"%s|0x%llx,%lu)",
s,
(unsigned long long)hkey,
(uintptr_t)leaf->entries[index].data);
}
fprintf(stderr,"]\n");
}
fprintf(stderr,"}\n");
fflush(stderr);
}
void
ncexhashprintdir(NCexhashmap* map, NCexleaf** dir)
{
int dirindex;
for(dirindex=0;dirindex<(1<<map->depth);dirindex++) {
NCexleaf* leaf = dir[dirindex];
fprintf(stderr,"\tdirectory[%03d|%sb]=%d/%p\n",
dirindex,ncexbinstr(dirindex,map->depth),leaf->uid,leaf);
}
fflush(stderr);
}
void
ncexhashprintleaf(NCexhashmap* map, NCexleaf* leaf)
{
int index;
fprintf(stderr,"(%04x)[(%u)^%d|%d|",
(unsigned)(0xffff & (uintptr_t)leaf),
leaf->uid, leaf->depth,leaf->active);
for(index=0;index<leaf->active;index++) {
ncexhashkey_t hkey, bits;
const char* s;
hkey = leaf->entries[index].hashkey;
/* Reduce to the leaf->hash MSB */
bits = MSB(hkey,map->depth);
s = ncexbinstr(bits,map->depth);
fprintf(stderr,"%s(%s/",(index==0?":":" "),s);
bits = MSB(hkey,leaf->depth);
s = ncexbinstr(bits,leaf->depth);
fprintf(stderr,"%s|0x%llx,%lu)",
s, (unsigned long long)hkey, (uintptr_t)leaf->entries[index].data);
}
fprintf(stderr,"]\n");
}
void
ncexhashprintentry(NCexhashmap* map, NCexentry* entry)
{
fprintf(stderr,"{0x%llx,%lu)",(unsigned long long)entry->hashkey,(uintptr_t)entry->data);
}
char*
ncexbinstr(ncexhashkey_t hkey, int depth)
{
int i;
static char bits[NCEXHASHKEYBITS+1];
memset(bits,'0',NCEXHASHKEYBITS+1);
bits[NCEXHASHKEYBITS] = '\0';
for(i=0;i<depth;i++)
bits[(depth-1)-i] = ((hkey >> i) & 0x1) == 0 ? '0' : '1';
bits[depth] = '\0';
return bits;
}
void
ncexhashprintstats(NCexhashmap* map)
{
int nactive, nleaves;
NCexleaf* leaf = NULL;
double leafavg = 0.0;
double leafload = 0.0;
unsigned long long dirsize, leafsize, total;
nactive = 0;
nleaves = 0;
for(leaf=map->leaves;leaf;leaf=leaf->next) {
nleaves++;
nactive += leaf->active;
}
leafavg = ((double)nactive)/((double)nleaves);
leafload = leafavg / ((double)map->leaflen);
if(nactive != map->nactive) {
fprintf(stderr,"nactive mismatch: map->active=%d actual=%d\n",map->nactive,nactive);
}
fprintf(stderr,"|directory|=%llu nleaves=%d nactive=%d",
(unsigned long long)(1<<(map->depth)),nleaves,nactive);
fprintf(stderr," |leaf|=%d nactive/nleaves=%g", map->leaflen, leafavg);
fprintf(stderr," load=%g",leafload);
fprintf(stderr,"]\n");
dirsize = (1<<(map->depth)*((unsigned long long)sizeof(void*)));
leafsize = (nleaves)*((unsigned long long)sizeof(NCexleaf));
total = dirsize + leafsize;
fprintf(stderr,"\tsizeof(directory)=%llu sizeof(leaves)=%lld total=%lld\n",
dirsize,leafsize,total);
}

31
libdispatch/nchashmap.c
View File

@ -12,7 +12,7 @@ See LICENSE.txt for license information.
#include <stdint.h>
#endif
#include "ncdispatch.h"
#include "nccrc.h"
#include "nchashmap.h"
#include "nc3internal.h"
@ -32,9 +32,6 @@ entry. Since we also keep the hashkey, the probability is high that
we will only do string comparisons when they will match.
*/
/* Prototype for the crc32 function */
extern unsigned int NC_crc32(unsigned int crc, const unsigned char* buf, unsigned int len);
#undef SMALLTABLE
#ifdef ASSERTIONS
@ -52,7 +49,7 @@ extern unsigned int NC_crc32(unsigned int crc, const unsigned char* buf, unsigne
#define SEED 37
/* See lookup3.c */
extern unsigned int hash_fast(const char*, size_t length);
extern nchashkey_t hash_fast(const char*, size_t length);
/* this should be prime */
#ifdef SMALLTABLE
@ -111,7 +108,7 @@ rehash(NC_hashmap* hm)
return invariant: return == 0 || *indexp is defined
*/
static int
locate(NC_hashmap* hash, unsigned int hashkey, const char* key, size_t keysize, size_t* indexp, int deletedok)
locate(NC_hashmap* hash, nchashkey_t hashkey, const char* key, size_t keysize, size_t* indexp, int deletedok)
{
size_t i;
size_t index;
@ -155,11 +152,11 @@ locate(NC_hashmap* hash, unsigned int hashkey, const char* key, size_t keysize,
}
/* Return the hash key for specified key; takes key+size*/
/* Wrapper for crc32 */
unsigned int
/* Wrapper for crc64*/
nchashkey_t
NC_hashmapkey(const char* key, size_t size)
{
return NC_crc32(0,(unsigned char*)key,(unsigned int)size);
return NC_crc64(0,(void*)key,(unsigned int)size);
}
NC_hashmap*
@ -188,13 +185,13 @@ int
NC_hashmapadd(NC_hashmap* hash, uintptr_t data, const char* key, size_t keysize)
{
NC_hentry* entry;
unsigned int hashkey;
nchashkey_t hashkey;
Trace("NC_hashmapadd");
if(key == NULL || keysize == 0)
return 0;
hashkey = NC_crc32(0,(unsigned char*)key,(unsigned int)keysize);
hashkey = NC_hashmapkey(key,keysize);
if(hash->alloc*3/4 <= hash->active)
rehash(hash);
@ -229,7 +226,7 @@ NC_hashmapadd(NC_hashmap* hash, uintptr_t data, const char* key, size_t keysize)
int
NC_hashmapremove(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t* datap)
{
unsigned int hashkey;
nchashkey_t hashkey;
size_t index;
NC_hentry* h;
@ -238,7 +235,7 @@ NC_hashmapremove(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t* d
if(key == NULL || keysize == 0)
return 0;
hashkey = NC_crc32(0,(unsigned char*)key,(unsigned int)keysize);
hashkey = NC_hashmapkey(key,keysize);
if(!locate(hash,hashkey,key,keysize,&index,0))
return 0; /* not present */
h = &hash->table[index];
@ -257,13 +254,13 @@ NC_hashmapremove(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t* d
int
NC_hashmapget(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t* datap)
{
unsigned int hashkey;
nchashkey_t hashkey;
Trace("NC_hashmapget");
if(key == NULL || keysize == 0)
return 0;
hashkey = NC_crc32(0,(unsigned char*)key,(unsigned int)keysize);
hashkey = NC_hashmapkey(key,keysize);
if(hash->active) {
size_t index;
NC_hentry* h;
@ -287,13 +284,13 @@ NC_hashmapsetdata(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t n
{
size_t index;
NC_hentry* h;
unsigned int hashkey;
nchashkey_t hashkey;
Trace("NC_hashmapsetdata");
if(key == NULL || keysize == 0)
return 0;
hashkey = NC_crc32(0,(unsigned char*)key,(unsigned int)keysize);
hashkey = NC_hashmapkey(key,keysize);
if(hash == NULL || hash->active == 0)
return 0; /* no such entry */
if(!locate(hash,hashkey,key,keysize,&index,0))

277
libdispatch/ncxcache.c Normal file
View File

@ -0,0 +1,277 @@
/*
Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
See LICENSE.txt for license information.
*/
/** \file \internal
Internal netcdf-4 functions.
This file contains functions for manipulating NCxcache objects.
Warning: This code depends critically on the assumption that
|void*| == |uintptr_t|
*/
/* 0 => no debug */
#define DEBUG 0
#define CATCH
/* Define this for debug so that table sizes are small */
#define SMALLTABLE
#ifdef CATCH
#define THROW(x) throw(x)
static void breakpoint(void) {}
static int ignore[] = {0};
static int throw(int x)
{
int* p;
if(x != 0) {
for(p=ignore;*p;p++) {if(x == *p) break;}
if(*p == 0) breakpoint();
}
return x;
}
#else
#define THROW(x) (x)
#endif
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include <assert.h>
#include "nc4internal.h"
#include "ncexhash.h"
#include "ncxcache.h"
#ifdef SMALLTABLE
/* Keep the table sizes small initially */
#define DFALTTABLESIZE 4
#define DFALTLEAFLEN 4
#else
#define DFALTTABLESIZE 32
#define DFALTLEAFLEN 12
#endif
static void insertafter(NCxnode* current, NCxnode* node);
static void unlinknode(NCxnode* node);
/* Locate object by hashkey in an NCxcache */
int
ncxcachelookup(NCxcache* NCxcache, ncexhashkey_t hkey, void** op)
{
int stat = NC_NOERR;
uintptr_t inode = 0;
NCxnode* node = NULL;
if(NCxcache == NULL) return THROW(NC_EINVAL);
assert(NCxcache->map != NULL);
if((stat=ncexhashget(NCxcache->map,hkey,&inode)))
{stat = THROW(NC_ENOTFOUND); goto done;} /* not present */
node = (void*)inode;
if(op) *op = node->content;
done:
return stat;
}
/* Move object to the front of the LRU list */
int
ncxcachetouch(NCxcache* cache, ncexhashkey_t hkey)
{
int stat = NC_NOERR;
uintptr_t inode = 0;
NCxnode* node = NULL;
if(cache == NULL) return THROW(NC_EINVAL);
if((stat=ncexhashget(cache->map,hkey,&inode)))
{stat = THROW(NC_ENOTFOUND); goto done;} /* not present */
node = (void*)inode;
/* unlink */
unlinknode(node);
/* Relink into front of chain */
insertafter(&cache->lru,node);
done:
return stat;
}
/* Add object to the cache */
int
ncxcacheinsert(NCxcache* cache, const ncexhashkey_t hkey, void* o)
{
int stat = NC_NOERR;
uintptr_t inode = 0;
NCxnode* node = NULL;
if(cache == NULL) return THROW(NC_EINVAL);
#ifndef NCXUSER
node = calloc(1,sizeof(NCxnode));
#else
node = (NCxnode*)o;
#endif
node->content = o; /* Cheat and make content point to the node part*/
inode = (uintptr_t)node;
if((stat = ncexhashput(cache->map,hkey,inode)))
goto done;
/* link into the LRU chain at front */
insertafter(&cache->lru,node);
node = NULL;
done:
#ifndef NCXUSER
if(node) nullfree(node);
#endif
return THROW(stat);
}
/* Remove object from the index;*/
int
ncxcacheremove(NCxcache* cache, ncexhashkey_t hkey, void** op)
{
int stat = NC_NOERR;
uintptr_t inode = 0;
NCxnode* node = NULL;
if(cache == NULL) return THROW(NC_EINVAL);
/* Remove from the hash map */
if((stat=ncexhashremove(cache->map,hkey,&inode)))
{stat = NC_ENOTFOUND; goto done;} /* not present */
node = (NCxnode*)inode;
/* unlink */
unlinknode(node);
if(op) {
*op = node->content;
}
#ifndef NCXUSER
nullfree(node);
#endif
done:
return THROW(stat);
}
/* Free a cache */
void
ncxcachefree(NCxcache* cache)
{
NCxnode* lru = NULL;
if(cache == NULL) return;
lru = &cache->lru;
#ifndef NCXUSER
{
NCxnode* p = NULL;
NCxnode* next = NULL;
/* walk the lru chain */
next = NULL;
for(p=lru->next;p != lru;) {
next = p->next;
nullfree(p);
p = next;
}
}
#endif
lru->next = (lru->prev = lru); /*reset*/
ncexhashmapfree(cache->map);
free(cache);
}
/* Create a new cache holding at least size objects */
int
ncxcachenew(size_t leaflen, NCxcache** cachep)
{
int stat = NC_NOERR;
NCxcache* cache = NULL;
if(leaflen == 0) leaflen = DFALTLEAFLEN;
cache = calloc(1,sizeof(NCxcache));
if(cache == NULL)
{stat = NC_ENOMEM; goto done;}
cache->map = ncexhashnew(leaflen);
if(cache->map == NULL)
{stat = NC_ENOMEM; goto done;}
cache->lru.next = &cache->lru;
cache->lru.prev = &cache->lru;
if(cachep) {*cachep = cache; cache = NULL;}
done:
ncxcachefree(cache);
return THROW(stat);
}
void
ncxcacheprint(NCxcache* cache)
{
int i;
NCxnode* p = NULL;
fprintf(stderr,"NCxcache: lru=");
fprintf(stderr,"{");
for(i=0,p=cache->lru.next;p != &cache->lru;p=p->next,i++) {
if(i>0) fprintf(stderr,",");
fprintf(stderr,"%p:%p",p,p->content);
}
fprintf(stderr,"}\n");
ncexhashprint(cache->map);
}
void*
ncxcachefirst(NCxcache* cache)
{
if(cache == NULL) return NULL;
if(ncexhashcount(cache->map) == 0)
return NULL;
return cache->lru.next->content;
}
void*
ncxcachelast(NCxcache* cache)
{
if(cache == NULL) return NULL;
if(ncexhashcount(cache->map) == 0)
return NULL;
return cache->lru.prev->content;
}
/* Insert node after current */
static void
insertafter(NCxnode* current, NCxnode* node)
{
NCxnode* curnext = current->next;
current->next = node;
node->prev = current;
node->next = curnext;
curnext->prev = node;
}
/* Remove node from chain */
static void
unlinknode(NCxnode* node)
{
NCxnode* next = node->next;
NCxnode* prev = node->prev;
node->next = node->prev = NULL;
/* repair the chain */
next->prev = prev;
prev->next = next;
}
ncexhashkey_t
ncxcachekey(const void* key, size_t size)
{
return ncexhashkey((unsigned char*)key,size);
}

1
libhdf4/hdf4file.c
View File

@ -243,7 +243,6 @@ nc4_set_var_type(nc_type xtype, int endianness, size_t type_size, char *type_nam
type->endianness = endianness;
type->size = type_size;
type->hdr.id = (size_t)xtype;
type->hdr.hashkey = NC_hashmapkey(type->hdr.name, strlen(type->hdr.name));
/* Return to caller. */
*typep = type;

1
libhdf5/hdf5attr.c
View File

@ -230,7 +230,6 @@ NC4_HDF5_rename_att(int ncid, int varid, const char *name, const char *newname)
if(att->hdr.name) free(att->hdr.name);
if (!(att->hdr.name = strdup(norm_newname)))
return NC_ENOMEM;
att->hdr.hashkey = NC_hashmapkey(att->hdr.name,strlen(att->hdr.name)); /* Fix hash key */
att->dirty = NC_TRUE;

3
libhdf5/hdf5dim.c
View File

@ -275,8 +275,7 @@ HDF5_rename_dim(int ncid, int dimid, const char *name)
return NC_ENOMEM;
LOG((3, "dim is now named %s", dim->hdr.name));
/* Fix hash key and rebuild index. */
dim->hdr.hashkey = NC_hashmapkey(dim->hdr.name,strlen(dim->hdr.name));
/* rebuild index. */
if (!ncindexrebuild(grp->dim))
return NC_EINTERNAL;

5
libhdf5/hdf5file.c
View File

@ -14,6 +14,7 @@
#include "config.h"
#include "hdf5internal.h"
#include "ncrc.h"
#include "ncauth.h"
extern int NC4_extract_file_image(NC_FILE_INFO_T* h5); /* In nc4memcb.c */
@ -222,6 +223,10 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio)
* hidden attribute. */
NC4_clear_provenance(&h5->provenance);
/* Free the http info */
ncurifree(hdf5_info->http.uri);
NC_authfree(hdf5_info->http.auth);
/* Close hdf file. It may not be open, since this function is also
* called by NC_create() when a file opening is aborted. */
if (hdf5_info->hdfid > 0 && H5Fclose(hdf5_info->hdfid) < 0)

3
libhdf5/hdf5grp.c
View File

@ -166,8 +166,7 @@ NC4_rename_grp(int grpid, const char *name)
if (!(grp->hdr.name = strdup(norm_name)))
return NC_ENOMEM;
/* Update the hash and rebuild index. */
grp->hdr.hashkey = NC_hashmapkey(grp->hdr.name,strlen(grp->hdr.name));
/* Rebuild index. */
if(!ncindexrebuild(grp->parent->children))
return NC_EINTERNAL;

22
libhdf5/hdf5open.c
View File

@ -12,6 +12,7 @@
#include "config.h"
#include "hdf5internal.h"
#include "ncrc.h"
#include "ncauth.h"
#include "ncmodel.h"
#ifdef ENABLE_BYTERANGE
@ -72,7 +73,7 @@ static hid_t nc4_H5Fopen(const char *filename, unsigned flags, hid_t fapl_id);
#endif
#ifdef ENABLE_HDF5_ROS3
static int ros3info(NCURI* uri, char** hostportp, char** regionp);
static int ros3info(NCauth** auth, NCURI* uri, char** hostportp, char** regionp);
#endif
/**
@ -858,29 +859,24 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
H5FD_ros3_fapl_t fa;
char* hostport = NULL;
char* region = NULL;
char* accessid = NULL;
char* secretkey = NULL;
ncuriparse(path,&uri);
if(uri == NULL)
BAIL(NC_EINVAL);
/* Extract auth related info */
if((ros3info(uri,&hostport,&region)))
if((ros3info(&h5->http.auth,uri,&hostport,&region)))
BAIL(NC_EINVAL);
ncurifree(uri); uri = NULL;
accessid = NC_rclookup("HTTP.CREDENTIALS.USER",hostport);
secretkey = NC_rclookup("HTTP.CREDENTIALS.PASSWORD",hostport);
fa.version = 1;
fa.aws_region[0] = '\0';
fa.secret_id[0] = '\0';
fa.secret_key[0] = '\0';
if(accessid == NULL || secretkey == NULL) {
if(h5->http.auth->s3creds.accessid == NULL || h5->http.auth->s3creds.secretkey == NULL) {
/* default, non-authenticating, "anonymous" fapl configuration */
fa.authenticate = (hbool_t)0;
} else {
fa.authenticate = (hbool_t)1;
strlcat(fa.aws_region,region,H5FD_ROS3_MAX_REGION_LEN);
strlcat(fa.secret_id, accessid, H5FD_ROS3_MAX_SECRET_ID_LEN);
strlcat(fa.secret_key, secretkey, H5FD_ROS3_MAX_SECRET_KEY_LEN);
strlcat(fa.secret_id, h5->http.auth->s3creds.accessid, H5FD_ROS3_MAX_SECRET_ID_LEN);
strlcat(fa.secret_key, h5->http.auth->s3creds.secretkey, H5FD_ROS3_MAX_SECRET_KEY_LEN);
}
nullfree(region);
nullfree(hostport);
@ -2764,7 +2760,7 @@ exit:
#ifdef ENABLE_HDF5_ROS3
static int
ros3info(NCURI* uri, char** hostportp, char** regionp)
ros3info(NCauth** authp, NCURI* uri, char** hostportp, char** regionp)
{
int stat = NC_NOERR;
size_t len;
@ -2799,6 +2795,10 @@ ros3info(NCURI* uri, char** hostportp, char** regionp)
if(hostportp) {*hostportp = hostport; hostport = NULL;}
if(regionp) {*regionp = region; region = NULL;}
/* Extract auth related info */
if((stat=NC_authsetup(authp, uri)))
goto done;
done:
nullfree(hostport);
nullfree(region);

3
libhdf5/hdf5var.c
View File

@ -1113,8 +1113,7 @@ NC4_rename_var(int ncid, int varid, const char *name)
return NC_ENOMEM;
LOG((3, "var is now %s", var->hdr.name));
/* Fix hash key and rebuild index. */
var->hdr.hashkey = NC_hashmapkey(var->hdr.name, strlen(var->hdr.name));
/* rebuild index. */
if (!ncindexrebuild(grp->vars))
return NC_EINTERNAL;

2
libnczarr/CMakeLists.txt
View File

@ -11,7 +11,7 @@
SET(libnczarr_SOURCES
zarr.c
zattr.c
zcache.c
zxcache.c
zchunking.c
zclose.c
zcreate.c

2
libnczarr/Makefile.am
View File

@ -30,7 +30,7 @@ noinst_LTLIBRARIES = libnczarr.la
libnczarr_la_SOURCES = \
zarr.c \
zattr.c \
zcache.c \
zxcache.c \
zchunking.c \
zclose.c \
zcreate.c \

8
libnczarr/zarr.c
View File

@ -66,11 +66,9 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr
if((stat = applycontrols(zinfo))) goto done;
/* Load auth info from rc file */
if((zinfo->auth = calloc(1,sizeof(NCauth)))==NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = ncuriparse(nc->path,&uri))) goto done;
if(uri) {
if((stat = NC_authsetup(zinfo->auth, uri)))
if((stat = NC_authsetup(&zinfo->auth, uri)))
goto done;
}
@ -167,11 +165,9 @@ ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls)
}
/* Load auth info from rc file */
if((zinfo->auth = calloc(1,sizeof(NCauth)))==NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = ncuriparse(nc->path,&uri))) goto done;
if(uri) {
if((stat = NC_authsetup(zinfo->auth, uri)))
if((stat = NC_authsetup(&zinfo->auth, uri)))
goto done;
}

3
libnczarr/zarr.h
View File

@ -65,4 +65,7 @@ extern char** NCZ_clonestringvec(size_t len, const char** vec);
extern void NCZ_freestringvec(size_t len, char** vec);
extern int NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fillchunkp);
/* zwalk.c */
extern int NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata);
#endif /*ZARR_H*/

5
libnczarr/zattr.c
View File

@ -213,7 +213,6 @@ NCZ_rename_att(int ncid, int varid, const char *name, const char *newname)
if(att->hdr.name) free(att->hdr.name);
if (!(att->hdr.name = strdup(norm_newname)))
return NC_ENOMEM;
att->hdr.hashkey = NC_hashmapkey(att->hdr.name,strlen(att->hdr.name)); /* Fix hash key */
att->dirty = NC_TRUE;
@ -707,7 +706,9 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
/* Just copy the data, for non-atomic types */
if (type_class == NC_OPAQUE || type_class == NC_COMPOUND || type_class == NC_ENUM)
memcpy(att->data, data, len * type_size);
else
else if(mem_type == file_type) {
memcpy(att->data, data, len * type_size);
} else /* need to convert */
{
/* Data types are like religions, in that one can convert. */
if ((retval = nc4_convert_type(data, att->data, mem_type, file_type,

15
libnczarr/zcache.c
View File

@ -103,6 +103,9 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl
int
NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
{
/* Reset the cache parameters since var chunking may have changed */
return NC_NOERR;
}
@ -126,12 +129,16 @@ NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, NCZChunkCache** c
NCZChunkCache* cache = NULL;
void* fill = NULL;
size_t nelems, cachesize;
NCZ_VAR_INFO_T* zvar = NULL;
if(chunksize == 0) return NC_EINVAL;
zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
if((cache = calloc(1,sizeof(NCZChunkCache))) == NULL)
{stat = NC_ENOMEM; goto done;}
cache->var = var;
cache->ndims = var->ndims + zvar->scalar;
cache->chunksize = chunksize;
assert(cache->fillchunk == NULL);
cache->fillchunk = NULL;
@ -199,7 +206,7 @@ NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap
{
int stat = NC_NOERR;
char* key = NULL;
int rank = cache->var->ndims;
int rank = cache->ndims;
NC_FILE_INFO_T* file = cache->var->container->nc4_info;
NCZCacheEntry* entry = NULL;
int i;
@ -268,7 +275,7 @@ NCZ_write_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** data
{
int stat = NC_NOERR;
char* key = NULL;
int i,rank = cache->var->ndims;
int i,rank = cache->ndims;
NCZCacheEntry* entry = NULL;
/* Create the key for this cache */
@ -358,7 +365,7 @@ NCZ_chunk_cache_modified(NCZChunkCache* cache, const size64_t* indices)
int stat = NC_NOERR;
char* key = NULL;
NCZCacheEntry* entry = NULL;
int rank = cache->var->ndims;
int rank = cache->ndims;
/* Create the key for this cache */
if((stat=buildchunkkey(rank, indices, &key))) goto done;
@ -518,7 +525,7 @@ NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, char** ke
char* key = NULL;
/* Get the chunk object name */
if((stat = buildchunkkey(cache->var->ndims, chunkindices, &chunkname))) goto done;
if((stat = buildchunkkey(cache->ndims, chunkindices, &chunkname))) goto done;
/* Get the var object key */
if((stat = NCZ_varkey(cache->var,&varkey))) goto done;
/* Prefix the path to the containing variable object */

14
libnczarr/zcache.h
View File

@ -6,19 +6,29 @@
#ifndef ZCACHE_H
#define ZCACHE_H
/* This holds all the fields
to support either impl of cache
*/
struct NCxcache;
typedef struct NCZCacheEntry {
struct List {void* next; void* prev; void* unused;} list;
int modified;
size64_t indices[NC_MAX_VAR_DIMS];
char* key; /* as string */
char* key;
size64_t hashkey;
void* data;
} NCZCacheEntry;
typedef struct NCZChunkCache {
const NC_VAR_INFO_T* var; /* backlink */
size64_t ndims; /* true ndims == var->ndims + scalar */
size64_t chunksize;
void* fillchunk; /* enough fillvalues to fill a chunk */
size_t maxentries; /* Max number of entries allowed */
NClist* entries; /* in LRU order */
NClist* mru; /* all cache entries in mru order */
struct NCxcache* xcache;
} NCZChunkCache;
/**************************************************/

11
libnczarr/zchunking.h
View File

@ -34,9 +34,8 @@ typedef struct NCProjection {
size64_t chunkindex; /* which chunk are we projecting */
size64_t first; /* absolute first position to be touched in this chunk */
size64_t last; /* absolute position of last value touched */
size64_t len; /* Not last place touched, but the offset of last place
that could be touched */
size64_t limit; /* Actual limit of chunk = min(limit,dimlen) */
size64_t limit; /* Actual limit of chunk WRT start of chunk */
size64_t len; /* Actual len of chunk WRT start of chunk */
size64_t iopos; /* start point in the data memory to access the data */
size64_t iocount; /* no. of I/O items */
NCZSlice chunkslice; /* slice relative to this chunk */
@ -60,12 +59,13 @@ struct Common {
struct NCZChunkCache* cache;
int reading; /* 1=> read, 0 => write */
int rank;
int scalar; /* 1 => scalar variable */
size64_t* dimlens;
size64_t* chunklens;
void* memory;
size_t typesize;
void* fillvalue;
size64_t chunksize; /* computed product of chunklens */
size64_t chunkcount; /* computed product of chunklens; warning indices, not bytes */
int swap; /* var->format_info_file->native_endianness == var->endianness */
size64_t shape[NC_MAX_VAR_DIMS]; /* shape of the output hyperslab */
NCZSliceProjections* allprojections;
@ -86,6 +86,7 @@ EXTERNL int NCZ_transferslice(NC_VAR_INFO_T* var, int reading,
size64_t* start, size64_t* count, size64_t* stride,
void* memory, nc_type typecode);
EXTERNL int NCZ_transfer(struct Common* common, NCZSlice* slices);
EXTERNL int NCZ_transferscalar(struct Common* common);
EXTERNL size64_t NCZ_computelinearoffset(size_t, const size64_t*, const size64_t*);
/* Special entry points for unit testing */
@ -103,4 +104,6 @@ EXTERNL void NCZ_clearcommon(struct Common* common);
#define ceildiv(x,y) (((x) % (y)) == 0 ? ((x) / (y)) : (((x) / (y)) + 1))
#define minimum(x,y) ((x) > (y) ? (y) : (x))
#endif /*ZCHUNKING_H*/

3
libnczarr/zclose.c
View File

@ -47,8 +47,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort)
if((stat = nczmap_close(zinfo->map,(abort && zinfo->created)?1:0)))
goto done;
NCZ_freestringvec(0,zinfo->controls);
NC_authclear(zinfo->auth);
nullfree(zinfo->auth);
NC_authfree(zinfo->auth);
nullfree(zinfo);
done:

35
libnczarr/zdebug.c
View File

@ -69,13 +69,13 @@ nczprint_reclaim(void)
}
char*
nczprint_slice(NCZSlice slice)
nczprint_slice(const NCZSlice slice)
{
return nczprint_slicex(slice,!RAW);
}
char*
nczprint_slicex(NCZSlice slice, int raw)
nczprint_slicex(const NCZSlice slice, int raw)
{
char* result = NULL;
NCbytes* buf = ncbytesnew();
@ -108,13 +108,13 @@ nczprint_slicex(NCZSlice slice, int raw)
}
char*
nczprint_slices(int rank, NCZSlice* slices)
nczprint_slices(int rank, const NCZSlice* slices)
{
return nczprint_slicesx(rank, slices, !RAW);
}
char*
nczprint_slicesx(int rank, NCZSlice* slices, int raw)
nczprint_slicesx(int rank, const NCZSlice* slices, int raw)
{
int i;
char* result = NULL;
@ -135,20 +135,20 @@ nczprint_slicesx(int rank, NCZSlice* slices, int raw)
}
char*
nczprint_slab(int rank, NCZSlice* slices)
nczprint_slab(int rank, const NCZSlice* slices)
{
return nczprint_slicesx(rank,slices,RAW);
}
char*
nczprint_odom(NCZOdometer* odom)
nczprint_odom(const NCZOdometer* odom)
{
char* result = NULL;
NCbytes* buf = ncbytesnew();
char value[128];
char* txt = NULL;
snprintf(value,sizeof(value),"Odometer{rank=%d,",odom->rank);
snprintf(value,sizeof(value),"Odometer{rank=%d optimized=%d",odom->rank,odom->properties.optimized);
ncbytescat(buf,value);
ncbytescat(buf," start=");
@ -169,6 +169,9 @@ nczprint_odom(NCZOdometer* odom)
ncbytescat(buf," offset=");
snprintf(value,sizeof(value),"%llu",nczodom_offset(odom));
ncbytescat(buf,value);
ncbytescat(buf," avail=");
snprintf(value,sizeof(value),"%llu",nczodom_avail(odom));
ncbytescat(buf,value);
ncbytescat(buf,"}");
result = ncbytesextract(buf);
@ -177,13 +180,13 @@ nczprint_odom(NCZOdometer* odom)
}
char*
nczprint_projection(NCZProjection proj)
nczprint_projection(const NCZProjection proj)
{
return nczprint_projectionx(proj,!RAW);
}
char*
nczprint_projectionx(NCZProjection proj, int raw)
nczprint_projectionx(const NCZProjection proj, int raw)
{
char* result = NULL;
NCbytes* buf = ncbytesnew();
@ -218,7 +221,7 @@ nczprint_projectionx(NCZProjection proj, int raw)
}
char*
nczprint_allsliceprojections(int r, NCZSliceProjections* slp)
nczprint_allsliceprojections(int r, const NCZSliceProjections* slp)
{
int i;
char* s;
@ -233,13 +236,13 @@ nczprint_allsliceprojections(int r, NCZSliceProjections* slp)
}
char*
nczprint_sliceprojections(NCZSliceProjections slp)
nczprint_sliceprojections(const NCZSliceProjections slp)
{
return nczprint_sliceprojectionsx(slp,!RAW);
}
char*
nczprint_sliceprojectionsx(NCZSliceProjections slp, int raw)
nczprint_sliceprojectionsx(const NCZSliceProjections slp, int raw)
{
char* result = NULL;
NCbytes* buf = ncbytesnew();
@ -266,7 +269,7 @@ nczprint_sliceprojectionsx(NCZSliceProjections slp, int raw)
}
char*
nczprint_chunkrange(NCZChunkRange range)
nczprint_chunkrange(const NCZChunkRange range)
{
char* result = NULL;
NCbytes* buf = ncbytesnew();
@ -285,7 +288,7 @@ nczprint_chunkrange(NCZChunkRange range)
}
char*
nczprint_vector(size_t len, size64_t* vec)
nczprint_vector(size_t len, const size64_t* vec)
{
char* result = NULL;
int i;
@ -304,9 +307,8 @@ nczprint_vector(size_t len, size64_t* vec)
return capture(result);
}
#ifdef ZDEBUG
void
zdumpcommon(struct Common* c)
zdumpcommon(const struct Common* c)
{
int r;
fprintf(stderr,"Common:\n");
@ -329,7 +331,6 @@ zdumpcommon(struct Common* c)
fprintf(stderr,"\t\t[%d] %s\n",r,nczprint_sliceprojectionsx(c->allprojections[r],RAW));
fflush(stderr);
}
#endif
#ifdef HAVE_EXECINFO_H
#define MAXSTACKDEPTH 100

30
libnczarr/zdebug.h
View File

@ -8,7 +8,7 @@
#undef ZDEBUG /* general debug */
#undef ZDEBUG1 /* detailed debug */
#undef ZCATCH /* Warning: significant performance impact */
#define ZCATCH /* Warning: significant performance impact */
#undef ZTRACING /* Warning: significant performance impact */
#include "ncexternl.h"
@ -36,22 +36,20 @@ EXTERNL int zthrow(int err, const char* fname, int line);
/* printers */
EXTERNL void nczprint_reclaim(void);
EXTERNL char* nczprint_slice(NCZSlice);
EXTERNL char* nczprint_slices(int rank, NCZSlice*);
EXTERNL char* nczprint_slab(int rank, NCZSlice*);
EXTERNL char* nczprint_odom(NCZOdometer*);
EXTERNL char* nczprint_chunkrange(NCZChunkRange);
EXTERNL char* nczprint_projection(NCZProjection);
EXTERNL char* nczprint_sliceprojections(NCZSliceProjections);
EXTERNL char* nczprint_allsliceprojections(int r, NCZSliceProjections* slp);
EXTERNL char* nczprint_vector(size_t,size64_t*);
EXTERNL char* nczprint_slicex(NCZSlice slice, int raw);
EXTERNL char* nczprint_slicesx(int rank, NCZSlice* slices, int raw);
EXTERNL char* nczprint_projectionx(NCZProjection proj, int raw);
EXTERNL char* nczprint_sliceprojectionsx(NCZSliceProjections slp, int raw);
EXTERNL char* nczprint_slices(int rank, const NCZSlice*);
EXTERNL char* nczprint_slab(int rank, const NCZSlice*);
EXTERNL char* nczprint_odom(const NCZOdometer*);
EXTERNL char* nczprint_chunkrange(const NCZChunkRange);
EXTERNL char* nczprint_projection(const NCZProjection);
EXTERNL char* nczprint_sliceprojections(const NCZSliceProjections);
EXTERNL char* nczprint_allsliceprojections(int r, const NCZSliceProjections* slp);
EXTERNL char* nczprint_vector(size_t,const size64_t*);
EXTERNL char* nczprint_slicex(const NCZSlice slice, int raw);
EXTERNL char* nczprint_slicesx(int rank, const NCZSlice* slices, int raw);
EXTERNL char* nczprint_projectionx(const NCZProjection proj, int raw);
EXTERNL char* nczprint_sliceprojectionsx(const NCZSliceProjections slp, int raw);
#ifdef ZDEBUG
EXTERNL void zdumpcommon(struct Common*);
#endif
EXTERNL void zdumpcommon(const struct Common*);
#ifdef HAVE_EXECINFO_H
EXTERNL void NCZbacktrace(void);

11
libnczarr/zdim.c
View File

@ -46,7 +46,6 @@ NCZ_def_dim(int ncid, const char *name, size_t len, int *idp)
NC_DIM_INFO_T *dim;
char norm_name[NC_MAX_NAME + 1];
int stat = NC_NOERR;
int i;
LOG((2, "%s: ncid 0x%x name %s len %d", __func__, ncid, name,
(int)len));
@ -63,8 +62,10 @@ NCZ_def_dim(int ncid, const char *name, size_t len, int *idp)
/* Check some stuff if strict nc3 rules are in effect. */
if (h5->cmode & NC_CLASSIC_MODEL)
{
#ifdef LOOK
/* Only one limited dimenson for strict nc3. */
if (len == NC_UNLIMITED) {
int i;
for(i=0;i<ncindexsize(grp->dim);i++) {
dim = (NC_DIM_INFO_T*)ncindexith(grp->dim,i);
if(dim == NULL) continue;
@ -75,12 +76,17 @@ NCZ_def_dim(int ncid, const char *name, size_t len, int *idp)
/* Must be in define mode for stict nc3. */
if (!(h5->flags & NC_INDEF))
return NC_ENOTINDEFINE;
#endif
}
/* Make sure this is a valid netcdf name. */
if ((stat = nc4_check_name(name, norm_name)))
return stat;
/* Since unlimited is not supported, len > 0 */
if(len <= 0)
return NC_EDIMSIZE;
/* For classic model: dim length has to fit in a 32-bit unsigned
* int, as permitted for 64-bit offset format. */
if (h5->cmode & NC_CLASSIC_MODEL)
@ -257,8 +263,7 @@ NCZ_rename_dim(int ncid, int dimid, const char *name)
return NC_ENOMEM;
LOG((3, "dim is now named %s", dim->hdr.name));
/* Fix hash key and rebuild index. */
dim->hdr.hashkey = NC_hashmapkey(dim->hdr.name,strlen(dim->hdr.name));
/* rebuild index. */
if (!ncindexrebuild(grp->dim))
return NC_EINTERNAL;

4
libnczarr/zdispatch.c
View File

@ -55,8 +55,8 @@ static const NC_Dispatch NCZ_dispatcher = {
NCZ_rename_var,
NCZ_get_vara,
NCZ_put_vara,
NCZ_get_vars,
NCZ_put_vars,
NCDEFAULT_get_vars,
NCDEFAULT_put_vars,
NCDEFAULT_get_varm,
NCDEFAULT_put_varm,

3
libnczarr/zgrp.c
View File

@ -165,8 +165,7 @@ NCZ_rename_grp(int grpid, const char *name)
if (!(grp->hdr.name = strdup(norm_name)))
return NC_ENOMEM;
/* Update the hash and rebuild index. */
grp->hdr.hashkey = NC_hashmapkey(grp->hdr.name,strlen(grp->hdr.name));
/* rebuild index. */
if(!ncindexrebuild(grp->parent->children))
return NC_EINTERNAL;

4
libnczarr/zinternal.h
View File

@ -126,10 +126,10 @@ typedef struct NCZ_GRP_INFO {
typedef struct NCZ_VAR_INFO {
NCZcommon common;
size64_t chunkproduct; /* product of chunksizes */
size64_t chunksize; /* chunkproduct * typesize */
int order; /* 1=>column major, 0=>row major (default); not currently enforced */
size_t scalar;
struct NCZChunkCache* cache;
/* Following are duplicates of NC_VAR_INFO_T equivalents */
size_t chunk_cache_nelems; /**< Number of slots in var chunk cache. */
} NCZ_VAR_INFO_T;
/* Struct to hold ZARR-specific info for a field. */

70
libnczarr/zodom.c
View File

@ -21,15 +21,19 @@ nczodom_new(int rank, const size64_t* start, const size64_t* stop, const size64_
int i;
NCZOdometer* odom = NULL;
if(buildodom(rank,&odom)) return NULL;
odom->properties.stride1 = 1; /* assume */
odom->properties.start0 = 1; /* assume */
for(i=0;i<rank;i++) {
odom->start[i] = (size64_t)start[i];
odom->stop[i] = (size64_t)stop[i];
odom->stride[i] = (size64_t)stride[i];
odom->max[i] = (size64_t)len[i];
if(odom->start[i] != 0) odom->properties.start0 = 0;
if(odom->stride[i] != 1) odom->properties.stride1 = 0;
}
nczodom_reset(odom);
for(i=0;i<rank;i++)
assert(stop[i] > 0 && stride[i] > 0 && len[i] >= stop[i]);
assert(stop[i] >= start[i] && stride[i] > 0 && (len[i]+1) >= stop[i]);
return odom;
}
@ -40,15 +44,20 @@ nczodom_fromslices(int rank, const NCZSlice* slices)
NCZOdometer* odom = NULL;
if(buildodom(rank,&odom)) return NULL;
odom->properties.stride1 = 1; /* assume */
odom->properties.start0 = 1; /* assume */
for(i=0;i<rank;i++) {
odom->start[i] = slices[i].start;
odom->stop[i] = slices[i].stop;
odom->stride[i] = slices[i].stride;
odom->max[i] = slices[i].len;
if(odom->start[i] != 0) odom->properties.start0 = 0;
if(odom->stride[i] != 1) odom->properties.stride1 = 0;
}
nczodom_reset(odom);
for(i=0;i<rank;i++)
assert(slices[i].stop > 0 && slices[i].stride > 0 && slices[i].len >= slices[i].stop);
if(!(slices[i].stop >= slices[i].start && slices[i].stride > 0 && (slices[i].len+1) >= slices[i].stop))
assert(slices[i].stop >= slices[i].start && slices[i].stride > 0 && (slices[i].len+1) >= slices[i].stop);
return odom;
}
@ -65,7 +74,7 @@ nczodom_free(NCZOdometer* odom)
}
int
nczodom_more(NCZOdometer* odom)
nczodom_more(const NCZOdometer* odom)
{
return (odom->index[0] < odom->stop[0]);
}
@ -77,7 +86,10 @@ nczodom_next(NCZOdometer* odom)
int rank;
rank = odom->rank;
for(i=rank-1;i>=0;i--) {
odom->index[i] += odom->stride[i];
if(i == rank-1 && odom->properties.optimized) {
odom->index[i] = odom->stop[i];
} else
odom->index[i] += odom->stride[i];
if(odom->index[i] < odom->stop[i]) break;
if(i == 0) goto done; /* leave the 0th entry if it overflows */
odom->index[i] = odom->start[i]; /* reset this position */
@ -88,13 +100,13 @@ done:
/* Get the value of the odometer */
size64_t*
nczodom_indices(NCZOdometer* odom)
nczodom_indices(const NCZOdometer* odom)
{
return odom->index;
}
size64_t
nczodom_offset(NCZOdometer* odom)
nczodom_offset(const NCZOdometer* odom)
{
int i;
size64_t offset;
@ -133,17 +145,50 @@ nomem:
}
size64_t
nczodom_avail(NCZOdometer* odom)
nczodom_avail(const NCZOdometer* odom)
{
size64_t avail;
/* The best we can do is compute the count for the rightmost index */
if(odom->stride[odom->rank-1] == 1)
avail = (odom->stop[odom->rank-1] - odom->start[odom->rank-1]);
if(odom->properties.optimized)
avail = (odom->stop[odom->rank-1] - odom->start[odom->rank-1]);
else
avail = 1;
return avail;
}
size64_t
nczodom_laststride(const NCZOdometer* odom)
{
return odom->stride[odom->rank-1];
}
size64_t
nczodom_lastlen(const NCZOdometer* odom)
{
return odom->max[odom->rank-1];
}
/**
Do limited amount of optimization:
assert:
odom->stride[odom->rank-1] == 1
odom->stop[odom->rank-1] == 0
then
odom->stride[odom->rank-1] = odom->stop[odom->rank-1]
*/
void
nczodom_optimize(NCZOdometer* odom)
{
if(odom) {
#if 0
if(odom->stride[odom->rank-1] == 1)
#endif
odom->properties.optimized = 1;
}
}
#if 0
void
nczodom_incr(NCZOdometer* odom, size64_t count)
{
@ -152,3 +197,10 @@ nczodom_incr(NCZOdometer* odom, size64_t count)
}
}
void
nczodom_reducerank(NCZOdometer* odom)
{
odom->rank--;
}
#endif

20
libnczarr/zodom.h
View File

@ -15,19 +15,31 @@ typedef struct NCZOdometer {
size64_t* stride;
size64_t* max; /* full dimension length */
size64_t* index; /* current value of the odometer*/
struct NCZOprop {
int stride1; /* all strides == 1 */
int start0; /* all starts == 0 */
int optimized; /* stride[rank-1]==1 && start[rank-1]==0 */
} properties;
} NCZOdometer;
/**************************************************/
/* From zodom.c */
extern NCZOdometer* nczodom_new(int rank, const size64_t*, const size64_t*, const size64_t*, const size64_t*);
extern NCZOdometer* nczodom_fromslices(int rank, const struct NCZSlice* slices);
extern int nczodom_more(NCZOdometer*);
extern int nczodom_more(const NCZOdometer*);
extern void nczodom_next(NCZOdometer*);
extern size64_t* nczodom_indices(NCZOdometer*);
extern size64_t nczodom_offset(NCZOdometer*);
extern size64_t* nczodom_indices(const NCZOdometer*);
extern size64_t nczodom_offset(const NCZOdometer*);
extern void nczodom_reset(NCZOdometer* odom);
extern void nczodom_free(NCZOdometer*);
extern size64_t nczodom_avail(NCZOdometer*);
extern size64_t nczodom_avail(const NCZOdometer*);
extern size64_t nczodom_laststride(const NCZOdometer* odom);
extern size64_t nczodom_lastlen(const NCZOdometer* odom);
extern void nczodom_optimize(NCZOdometer*);
#if 0
extern void nczodom_incr(NCZOdometer*,size64_t);
extern void nczodom_reducerank(NCZOdometer* odom);
#endif
#endif /*ZODOM_H*/

5
libnczarr/zprov.c
View File

@ -51,6 +51,7 @@ NCZ_provenance_init(void)
int stat = NC_NOERR;
char* name = NULL;
char* value = NULL;
unsigned long major, minor, release;
NCbytes* buffer = NULL; /* for constructing the global _NCProperties */
char printbuf[1024];
@ -78,16 +79,14 @@ NCZ_provenance_init(void)
ncbytescat(buffer,"=");
ncbytescat(buffer,PACKAGE_VERSION);
#if 0
/* This should be redundant since netcdf version => zarr format */
/* Insert the ZARR as underlying storage format library */
ncbytesappend(buffer,NCPROPSSEP2);
ncbytescat(buffer,NCPNCZLIB);
ncbytescat(buffer,"=");
if((stat = NCZ_get_libversion(&major,&minor,&release))) goto done;
if((stat = NCZ_get_libversion(&major,&minor,&release))) return stat;
snprintf(printbuf,sizeof(printbuf),"%lu.%lu.%lu",major,minor,release);
ncbytescat(buffer,printbuf);
#endif
#ifdef NCPROPERTIES_EXTRA
if(NCPROPERTIES_EXTRA != NULL && strlen(NCPROPERTIES_EXTRA) > 0)

2
libnczarr/zs3sdk.cpp
View File

@ -67,7 +67,7 @@ NCZ_s3sdkcreateconfig(const char* host, const char* region, void** configp)
int stat = NC_NOERR;
Aws::Client::ClientConfiguration *config = new Aws::Client::ClientConfiguration();
config->scheme = Aws::Http::Scheme::HTTPS;
config->connectTimeoutMs = 30000;
config->connectTimeoutMs = 300000;
config->requestTimeoutMs = 600000;
if(region) config->region = region;
if(host) config->endpointOverride = host;

42
libnczarr/zsync.c
View File

@ -257,12 +257,15 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
NC_DIM_INFO_T* dim = var->dim[i];
shape[i] = dim->len;
}
/* but might be scalar */
if(var->ndims == 0)
shape[0] = 1;
/* shape key */
/* Integer list defining the length of each dimension of the array.*/
/* Create the list */
if((stat = NCJnew(NCJ_ARRAY,&jtmp))) goto done;
for(i=0;i<var->ndims;i++) {
for(i=0;i<var->ndims+zvar->scalar;i++) {
snprintf(number,sizeof(number),"%llu",shape[i]);
NCJaddstring(jtmp,NCJ_INT,number);
}
@ -297,7 +300,7 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
if((stat = NCJaddstring(jvar,NCJ_STRING,"chunks"))) goto done;
/* Create the list */
if((stat = NCJnew(NCJ_ARRAY,&jtmp))) goto done;
for(i=0;i<var->ndims;i++) {
for(i=0;i<(var->ndims+zvar->scalar);i++) {
size64_t len = (var->storage == NC_CONTIGUOUS ? shape[i] : var->chunksizes[i]);
snprintf(number,sizeof(number),"%lld",len);
NCJaddstring(jtmp,NCJ_INT,number);
@ -377,13 +380,17 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
/* Create the NCZVAR json object */
if((stat = NCJnew(NCJ_DICT,&jncvar)))
goto done;
/* Insert dimrefs */
if((stat = NCJinsert(jncvar,"dimrefs",jdimrefs)))
goto done;
jdimrefs = NULL; /* Avoid memory problems */
/* Add the _Storage flag */
if(var->storage == NC_CONTIGUOUS) {
/* Record if this is a scalar; use the storage field */
if(var->ndims == 0) {
if((stat = NCJnewstring(NCJ_STRING,"scalar",&jtmp)))goto done;
} else if(var->storage == NC_CONTIGUOUS) {
if((stat = NCJnewstring(NCJ_STRING,"contiguous",&jtmp)))goto done;
} else if(var->storage == NC_COMPACT) {
if((stat = NCJnewstring(NCJ_STRING,"compact",&jtmp)))goto done;
@ -451,14 +458,22 @@ ncz_write_var(NC_VAR_INFO_T* var)
size64_t stop[NC_MAX_VAR_DIMS];
size64_t stride[NC_MAX_VAR_DIMS];
char* key = NULL;
if(var->ndims == 0) { /* scalar */
start[i] = 0;
stop[i] = 1;
stride[i] = 1;
} else {
for(i=0;i<var->ndims;i++) {
size64_t nchunks = ceildiv(var->dim[i]->len,var->chunksizes[i]);
start[i] = 0;
stop[i] = nchunks;
stride[i] = 1;
}
}
/* Iterate over all the chunks to create missing ones */
if((chunkodom = nczodom_new(var->ndims,start,stop,stride,stop))==NULL)
if((chunkodom = nczodom_new(var->ndims+zvar->scalar,start,stop,stride,stop))==NULL)
{stat = NC_ENOMEM; goto done;}
for(;nczodom_more(chunkodom);nczodom_next(chunkodom)) {
size64_t* indices = nczodom_indices(chunkodom);
@ -1152,7 +1167,8 @@ define_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* diminfo)
/* Create the NC_DIM_INFO_T object */
sscanf(value,"%lld",&len); /* Get length */
if(len < 0) {stat = NC_EDIMSIZE; goto done;}
if(len <= 0)
{stat = NC_EDIMSIZE; goto done;}
if((stat = nc4_dim_list_add(grp, name, (size_t)len, -1, &dim)))
goto done;
if((dim->format_dim_info = calloc(1,sizeof(NCZ_DIM_INFO_T))) == NULL)
@ -1240,9 +1256,12 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
goto done;
if(jvalue != NULL) {
if(strcmp(jvalue->value,"chunked") == 0) {
var->storage = NC_CHUNKED;
var->storage = NC_CHUNKED;
} else if(strcmp(jvalue->value,"compact") == 0) {
var->storage = NC_COMPACT;
} else if(strcmp(jvalue->value,"scalar") == 0) {
var->storage = NC_CONTIGUOUS;
zvar->scalar = 1;
} else { /*storage = NC_CONTIGUOUS;*/
var->storage = NC_CONTIGUOUS;
}
@ -1302,7 +1321,10 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
if((stat = NCJdictget(jvar,"shape",&jvalue))) goto done;
if(jvalue->sort != NCJ_ARRAY) {stat = THROW(NC_ENCZARR); goto done;}
/* Verify the rank */
rank = nclistlength(jvalue->contents);
if(zvar->scalar)
rank = 0;
else
rank = nclistlength(jvalue->contents);
if(hasdimrefs) { /* verify rank consistency */
if(nclistlength(dimrefs) != rank)
{stat = THROW(NC_ENCZARR); goto done;}
@ -1333,7 +1355,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
rank = nclistlength(jvalue->contents);
if(rank > 0) {
var->storage = NC_CHUNKED;
if(var->ndims != rank)
if(var->ndims+zvar->scalar != rank)
{stat = THROW(NC_ENCZARR); goto done;}
if((var->chunksizes = malloc(sizeof(size_t)*rank)) == NULL)
{stat = NC_ENOMEM; goto done;}
@ -1347,8 +1369,8 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
var->chunksizes[j] = (size_t)chunks[j];
zvar->chunkproduct *= chunks[j];
}
zvar->chunksize = zvar->chunkproduct * var->type_info->size;
/* Create the cache */
zvar->chunk_cache_nelems = var->chunk_cache_nelems;
if((stat = NCZ_create_chunk_cache(var,var->type_info->size*zvar->chunkproduct,&zvar->cache)))
goto done;
}
@ -1466,7 +1488,7 @@ parse_group_content(NCjson* jcontent, NClist* dimdefs, NClist* varnames, NClist*
{stat = NC_EBADNAME; goto done;}
/* check the length */
sscanf(jlen->value,"%lld",&len);
if(len <= 0)
if(len < 0)
{stat = NC_EDIMSIZE; goto done;}
nclistpush(dimdefs,strdup(norm_name));
nclistpush(dimdefs,strdup(jlen->value));

107
libnczarr/zvar.c
View File

@ -368,7 +368,7 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims,
if((retval = ncz_gettype(h5,grp,xtype,&type)))
BAIL(retval);
/* Create a new var and fill in some ZARR cache setting values. */
/* Create a new var and fill in some cache setting values. */
if ((retval = nc4_var_list_add(grp, norm_name, ndims, &var)))
BAIL(retval);
@ -377,6 +377,7 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims,
BAIL(NC_ENOMEM);
zvar = var->format_var_info;
zvar->common.file = h5;
zvar->scalar = (ndims == 0 ? 1 : 0);
/* Set these state flags for the var. */
var->is_new_var = NC_TRUE;
@ -443,10 +444,13 @@ var->type_info->rc++;
/* Compute the chunksize cross product */
zvar->chunkproduct = 1;
for(d=0;d<var->ndims;d++) {zvar->chunkproduct *= var->chunksizes[d];}
for(d=0;d<var->ndims+zvar->scalar;d++) {zvar->chunkproduct *= var->chunksizes[d];}
zvar->chunksize = zvar->chunkproduct * var->type_info->size;
/* Copy cache parameters */
zvar->chunk_cache_nelems = var->chunk_cache_nelems;
/* Override the cache setting to use NCZarr defaults */
var->chunk_cache_size = CHUNK_CACHE_SIZE_NCZARR;
var->chunk_cache_nelems = ceildiv(var->chunk_cache_size,zvar->chunksize);
var->chunk_cache_preemption = 1; /* not used */
/* Create the cache */
if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,&zvar->cache)))
@ -655,11 +659,10 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
zvar->chunkproduct = 1;
for (d = 0; d < var->ndims; d++)
zvar->chunkproduct *= var->chunksizes[d];
#ifdef LOOK
zvar->chunksize = zvar->chunkproduct * var->type_info->size;
/* Adjust the cache. */
if ((retval = ncz_adjust_var_cache(grp, var)))
if ((retval = NCZ_adjust_var_cache(grp, var)))
return THROW(retval);
#endif
}
#ifdef LOGGING
@ -1096,8 +1099,7 @@ NCZ_rename_var(int ncid, int varid, const char *name)
return NC_ENOMEM;
LOG((3, "var is now %s", var->hdr.name));
/* Fix hash key and rebuild index. */
var->hdr.hashkey = NC_hashmapkey(var->hdr.name, strlen(var->hdr.name));
/* rebuild index. */
if (!ncindexrebuild(grp->vars))
return NC_EINTERNAL;
@ -1297,6 +1299,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
int zero_count = 0; /* true if a count is zero */
size_t len = 1;
size64_t fmaxdims[NC_MAX_VAR_DIMS];
NCZ_VAR_INFO_T* zvar;
NC_UNUSED(fmaxdims);
@ -1312,6 +1315,8 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
LOG((3, "%s: var->hdr.name %s mem_nc_type %d", __func__,
var->hdr.name, mem_nc_type));
zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
/* Cannot convert to user-defined types. */
if (mem_nc_type >= NC_FIRSTUSERTYPEID)
return THROW(NC_EINVAL);
@ -1324,21 +1329,26 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
/* Convert from size_t and ptrdiff_t to size64_t */
/* Also do sanity checks */
for (i = 0; i < var->ndims; i++)
{
/* Check for non-positive stride. */
if (stridep && stridep[i] <= 0)
return NC_ESTRIDE;
if(var->ndims == 0) { /* scalar */
start[0] = 0;
count[0] = 1;
stride[0] = 1;
} else {
for (i = 0; i < var->ndims; i++)
{
/* Check for non-positive stride. */
if (stridep && stridep[i] <= 0)
return NC_ESTRIDE;
start[i] = startp[i];
count[i] = countp ? countp[i] : var->dim[i]->len;
stride[i] = stridep ? stridep[i] : 1;
/* Check to see if any counts are zero. */
if (!count[i])
zero_count++;
fdims[i] = var->dim[i]->len;
start[i] = startp[i];
count[i] = countp ? countp[i] : var->dim[i]->len;
stride[i] = stridep ? stridep[i] : 1;
/* Check to see if any counts are zero. */
if (!count[i])
zero_count++;
fdims[i] = var->dim[i]->len;
}
}
#ifdef LOOK
@ -1408,9 +1418,8 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
/* We must convert - allocate a buffer. */
need_to_convert++;
if (var->ndims)
for (d2=0; d2<var->ndims; d2++)
len *= countp[d2];
for (d2=0; d2<(var->ndims+zvar->scalar); d2++)
len *= countp[d2];
LOG((4, "converting data for var %s type=%d len=%d", var->hdr.name,
var->type_info->hdr.id, len));
@ -1596,10 +1605,11 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
void *fillvalue = NULL;
int no_read = 0, provide_fill = 0;
int fill_value_size[NC_MAX_VAR_DIMS];
int scalar = 0, retval, range_error = 0, i, d2;
int retval, range_error = 0, i, d2;
void *bufr = NULL;
int need_to_convert = 0;
size_t len = 1;
NCZ_VAR_INFO_T* zvar = NULL;
NC_UNUSED(fmaxdims);
@ -1613,6 +1623,8 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
LOG((3, "%s: var->hdr.name %s mem_nc_type %d", __func__,
var->hdr.name, mem_nc_type));
zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
/* Check some stuff about the type and the file. Also end define
* mode, if needed. */
if ((retval = check_for_vara(&mem_nc_type, var, h5)))
@ -1621,20 +1633,26 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
/* Convert from size_t and ptrdiff_t to size64_t. Also do sanity
* checks. */
for (i = 0; i < var->ndims; i++)
{
/* If any of the stride values are non-positive, fail. */
if (stridep && stridep[i] <= 0)
return NC_ESTRIDE;
start[i] = startp[i];
count[i] = countp[i];
stride[i] = stridep ? stridep[i] : 1;
/* if any of the count values are zero don't actually read. */
if (count[i] == 0)
no_read++;
/* Get dimension sizes also */
fdims[i] = var->dim[i]->len;
fmaxdims[i] = fdims[i];
if(var->ndims == 0) { /* scalar */
start[0] = 0;
count[0] = 1;
stride[0] = 1;
} else {
for (i = 0; i < var->ndims; i++)
{
/* If any of the stride values are non-positive, fail. */
if (stridep && stridep[i] <= 0)
return NC_ESTRIDE;
start[i] = startp[i];
count[i] = countp[i];
stride[i] = stridep ? stridep[i] : 1;
/* if any of the count values are zero don't actually read. */
if (count[i] == 0)
no_read++;
/* Get dimension sizes also */
fdims[i] = var->dim[i]->len;
fmaxdims[i] = fdims[i];
}
}
#ifdef LOOK
@ -1668,11 +1686,10 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
{
/* We must convert - allocate a buffer. */
need_to_convert++;
if (var->ndims)
for (d2 = 0; d2 < var->ndims; d2++)
len *= countp[d2];
for (d2 = 0; d2 < (var->ndims+zvar->scalar); d2++)
len *= countp[d2];
LOG((4, "converting data for var %s type=%d len=%d", var->hdr.name,
var->type_info->hdr.id, len));
var->type_info->hdr.id, len));
/* If we're reading, we need bufr to have enough memory to store
* the data in the file. If we're writing, we need bufr to be
@ -1812,7 +1829,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
/* Now we need to fake up any further data that was asked for,
using the fill values instead. First skip past the data we
just read, if any. */
if (!scalar && provide_fill)
if (!zvar->scalar && provide_fill)
{
void *filldata;
size_t real_data_size = 0;

189
libnczarr/zwalk.c
View File

@ -8,13 +8,14 @@
static int initialized = 0;
static unsigned int optimized = 0;
/* Forward */
static int NCZ_walk(NCZProjection** projv, NCZOdometer* chunkodom, NCZOdometer* slpodom, NCZOdometer* memodom, const struct Common* common, void* chunkdata);
static int rangecount(NCZChunkRange range);
static int readfromcache(void* source, size64_t* chunkindices, void** chunkdata);
static int NCZ_fillchunk(void* chunkdata, struct Common* common);
static int transfern(NCZOdometer* slpodom, NCZOdometer* memodom, const struct Common* common, unsigned char* slpptr0, unsigned char* memptr0);
static int transfern(const struct Common* common, unsigned char* slpptr, unsigned char* memptr);
const char*
astype(int typesize, void* ptr)
@ -34,6 +35,8 @@ astype(int typesize, void* ptr)
int
ncz_chunking_init(void)
{
const char* eval = getenv("NCZ_NOOPTIMIZATION");
optimized = (eval == NULL ? 1 : 0);
initialized = 1;
return NC_NOERR;
}
@ -73,16 +76,6 @@ NCZ_transferslice(NC_VAR_INFO_T* var, int reading,
if((stat = NC4_inq_atomic_type(typecode, NULL, &typesize))) goto done;
for(r=0;r<var->ndims;r++) {
dimlens[r] = var->dim[r]->len;
chunklens[r] = var->chunksizes[r];
slices[r].start = start[r];
slices[r].stride = stride[r];
slices[r].stop = start[r]+(count[r]*stride[r]);
slices[r].len = dimlens[r];
}
/* Fill in common */
memset(&common,0,sizeof(common));
common.var = var;
@ -97,15 +90,37 @@ NCZ_transferslice(NC_VAR_INFO_T* var, int reading,
if((stat = ncz_get_fill_value(common.file, common.var, &common.fillvalue))) goto done;
common.rank = var->ndims;
/* We need to talk scalar into account */
common.rank = var->ndims + zvar->scalar;
common.scalar = zvar->scalar;
common.swap = (zfile->native_endianness == var->endianness ? 0 : 1);
common.chunkcount = 1;
for(r=0;r<common.rank+common.scalar;r++) {
if(common.scalar)
dimlens[r] = 1;
else
dimlens[r] = var->dim[r]->len;
chunklens[r] = var->chunksizes[r];
slices[r].start = start[r];
slices[r].stride = stride[r];
slices[r].stop = start[r]+(count[r]*stride[r]);
slices[r].len = dimlens[r];
common.chunkcount *= chunklens[r];
}
common.dimlens = dimlens;
common.chunklens = chunklens;
common.reader.source = ((NCZ_VAR_INFO_T*)(var->format_var_info))->cache;
common.reader.read = readfromcache;
if((stat = NCZ_transfer(&common, slices))) goto done;
if(common.scalar) {
if((stat = NCZ_transferscalar(&common))) goto done;
}
else {
if((stat = NCZ_transfer(&common, slices))) goto done;
}
done:
NCZ_clearcommon(&common);
return stat;
@ -127,7 +142,7 @@ NCZ_transfer(struct Common* common, NCZSlice* slices)
NCZOdometer* slpodom = NULL;
NCZOdometer* memodom = NULL;
void* chunkdata = NULL;
/*
We will need three sets of odometers.
1. Chunk odometer to walk the chunk ranges to get all possible
@ -146,6 +161,39 @@ NCZ_transfer(struct Common* common, NCZSlice* slices)
fprintf(stderr,"allprojections:\n%s",nczprint_allsliceprojections(common->rank,common->allprojections)); fflush(stderr);
#endif
#if 0
if(optimized && wholevar) {
size64_t* chunkindices = nczodom_indices(chunkodom);
/* Implement a whole var read optimization; this is a rare occurrence
where the variable has a single chunk and we are reading the whole chunk
*/
#if WDEBUG >= 1
fprintf(stderr,"case: optimized+wholevar:\n");
#endif
/* we are transfering the whole singular chunk */
/* Read the chunk */
#if WDEBUG >= 1
fprintf(stderr,"chunkindices: %s\n",nczprint_vector(common->rank,chunkindices));
#endif
switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) {
case NC_ENOTFOUND: /* cache created the chunk */
if((stat = NCZ_fillchunk(chunkdata,common))) goto done;
break;
case NC_NOERR: break;
default: goto done;
}
/* Figure out memory address */
unsigned char* memptr = ((unsigned char*)common->memory);
unsigned char* slpptr = ((unsigned char*)chunkdata);
transfern(common,slpptr,memptr);
goto done;
}
#endif
/* iterate over the odometer: all combination of chunk
indices in the projections */
for(;nczodom_more(chunkodom);) {
@ -175,23 +223,23 @@ fprintf(stderr,"allprojections:\n%s",nczprint_allsliceprojections(common->rank,c
if(zutest && zutest->tests & UTEST_TRANSFER)
zutest->print(UTEST_TRANSFER, common, chunkodom, slpslices, memslices);
slpodom = nczodom_fromslices(common->rank,slpslices);
memodom = nczodom_fromslices(common->rank,memslices);
/* Read from cache */
switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) {
case NC_EEMPTY: /* cache created the chunk */
case NC_ENOTFOUND: /* cache created the chunk */
if((stat = NCZ_fillchunk(chunkdata,common))) goto done;
break;
case NC_NOERR: break;
default: goto done;
}
/* This is the key action: walk this set of slices and transfer data */
slpodom = nczodom_fromslices(common->rank,slpslices);
memodom = nczodom_fromslices(common->rank,memslices);
/* This is the key action: walk this set of slices and transfer data */
if((stat = NCZ_walk(proj,chunkodom,slpodom,memodom,common,chunkdata))) goto done;
nczodom_free(slpodom); slpodom = NULL;
nczodom_free(memodom); memodom = NULL;
nczodom_next(chunkodom);
}
@ -224,7 +272,7 @@ NCZ_walk(NCZProjection** projv, NCZOdometer* chunkodom, NCZOdometer* slpodom, NC
unsigned char* memptr0 = NULL;
unsigned char* slpptr0 = NULL;
/* Convert the indices to a linear offset WRT to chunk */
/* Convert the indices to a linear offset WRT to chunk indices */
slpoffset = nczodom_offset(slpodom);
memoffset = nczodom_offset(memodom);
@ -251,7 +299,7 @@ fflush(stderr);
LOG((1,"%s: slpptr0=%p memptr0=%p slpoffset=%llu memoffset=%lld",__func__,slpptr0,memptr0,slpoffset,memoffset));
if(zutest && zutest->tests & UTEST_WALK)
zutest->print(UTEST_WALK, common, chunkodom, slpodom, memodom);
if((stat = transfern(slpodom,memodom,common,slpptr0,memptr0))) goto done;
if((stat = transfern(common,slpptr0,memptr0))) goto done;
nczodom_next(memodom);
} else break; /* slpodom exhausted */
nczodom_next(slpodom);
@ -261,17 +309,17 @@ done:
}
static int
transfern(NCZOdometer* slpodom, NCZOdometer* memodom, const struct Common* common, unsigned char* slpptr0, unsigned char* memptr0)
transfern(const struct Common* common, unsigned char* slpptr, unsigned char* memptr)
{
int stat = NC_NOERR;
if(common->reading) {
memcpy(memptr0,slpptr0,common->typesize);
memcpy(memptr,slpptr,common->typesize);
if(common->swap)
NCZ_swapatomicdata(common->typesize,memptr0,common->typesize);
NCZ_swapatomicdata(common->typesize,memptr,common->typesize);
} else { /*writing*/
memcpy(slpptr0,memptr0,common->typesize);
memcpy(slpptr,memptr,common->typesize);
if(common->swap)
NCZ_swapatomicdata(common->typesize,slpptr0,common->typesize);
NCZ_swapatomicdata(common->typesize,slpptr,common->typesize);
}
return THROW(stat);
}
@ -283,7 +331,7 @@ NCZ_fillchunk(void* chunkdata, struct Common* common)
int stat = NC_NOERR;
if(common->fillvalue == NULL) {
memset(chunkdata,0,common->chunksize*common->typesize);
memset(chunkdata,0,common->chunkcount*common->typesize);
goto done;
}
@ -453,3 +501,86 @@ NCZ_clearcommon(struct Common* common)
nullfree(common->allprojections);
nullfree(common->fillvalue);
}
#if 0
/* Does the User want the whole variable? */
static int
iswholevar(struct Common* common, NCZSlice* slices)
{
int i;
/* Check that slices cover whole file */
for(i=0;i<common->rank;i++) {
if(slices[i].start != 0
|| slices[i].stop != common->dimlens[i]
|| slices[i].stride != 1)
return 0; /* slices do not cover the whole file */
}
/* Check that there is only one chunk */
for(i=0;i<common->rank;i++) {
if(common->dimlens[i] != common->chunklens[i])
return 0; /* must be more than one chunk */
}
return 1;
}
#endif
/**************************************************/
/* Scalar variable support */
/*
@param common common parameters
*/
int
NCZ_transferscalar(struct Common* common)
{
int stat = NC_NOERR;
void* chunkdata = NULL;
size64_t chunkindices[NC_MAX_VAR_DIMS];
unsigned char* memptr, *slpptr;
/* Read from single chunk from cache */
chunkindices[0] = 0;
switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) {
case NC_ENOTFOUND: /* cache created the chunk */
if((stat = NCZ_fillchunk(chunkdata,common))) goto done;
break;
case NC_NOERR: break;
default: goto done;
}
/* Figure out memory address */
memptr = ((unsigned char*)common->memory);
slpptr = ((unsigned char*)chunkdata);
if(common->reading)
memcpy(memptr,slpptr,common->chunkcount*common->typesize);
else
memcpy(slpptr,memptr,common->chunkcount*common->typesize);
done:
return stat;
}
/* Debugging Interface: return the contents of a specified chunk */
int
NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata)
{
int stat = NC_NOERR;
NC_VAR_INFO_T* var = NULL;
NCZ_VAR_INFO_T* zvar;
struct NCZChunkCache* cache = NULL;
void* cachedata = NULL;
if ((stat = nc4_find_grp_h5_var(ncid, varid, NULL, NULL, &var)))
return THROW(stat);
zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
cache = zvar->cache;
if((stat = NCZ_read_cache_chunk(cache,zindices,&cachedata))) goto done;
if(chunkdata)
memcpy(chunkdata,cachedata,cache->chunksize);
done:
return stat;
}

575
libnczarr/zxcache.c Normal file
View File

@ -0,0 +1,575 @@
/* Copyright 2018, University Corporation for Atmospheric
* Research. See COPYRIGHT file for copying and redistribution
* conditions. */
/**
* @file @internal The functions which control NCZ
* caching. These caching controls allow the user to change the cache
* sizes of ZARR before opening files.
*
* @author Dennis Heimbigner, Ed Hartnett
*/
#include "zincludes.h"
#include "zcache.h"
#include "ncxcache.h"
#undef DEBUG
#undef FILLONREAD
#undef FLUSH
#define LEAFLEN 32
/* Forward */
static int get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry);
static int put_chunk(NCZChunkCache* cache, const NCZCacheEntry*);
static int create_chunk(NCZChunkCache* cache, NCZCacheEntry* entry);
static int buildchunkkey(size_t R, const size64_t* chunkindices, char** keyp);
static int makeroom(NCZChunkCache* cache);
/**************************************************/
/* Dispatch table per-var cache functions */
/**
* @internal Set chunk cache size for a variable. This is the internal
* function called by nc_set_var_chunk_cache().
*
* @param ncid File ID.
* @param varid Variable ID.
* @param size Size in bytes to set cache.
* @param nelems # of entries in cache
* @param preemption Controls cache swapping.
*
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADID Bad ncid.
* @returns ::NC_ENOTVAR Invalid variable ID.
* @returns ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict
* nc3 netcdf-4 file.
* @returns ::NC_EINVAL Invalid input.
* @returns ::NC_EHDFERR HDF5 error.
* @author Ed Hartnett
*/
int
NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, float preemption)
{
NC_GRP_INFO_T *grp;
NC_FILE_INFO_T *h5;
NC_VAR_INFO_T *var;
NCZ_VAR_INFO_T *zvar;
int retval;
/* Check input for validity. */
if (preemption < 0 || preemption > 1)
return NC_EINVAL;
/* Find info for this file and group, and set pointer to each. */
if ((retval = nc4_find_nc_grp_h5(ncid, NULL, &grp, &h5)))
return retval;
assert(grp && h5);
/* Find the var. */
if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
return NC_ENOTVAR;
assert(var && var->hdr.id == varid);
zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
assert(zvar != NULL && zvar->cache != NULL);
/* Set the values. */
var->chunk_cache_size = cachesize;
var->chunk_cache_nelems = nelems;
var->chunk_cache_preemption = preemption;
#ifdef LOOK
/* Reopen the dataset to bring new settings into effect. */
if ((retval = nc4_reopen_dataset(grp, var)))
return retval;
#endif
return NC_NOERR;
}
/**
* @internal Adjust the chunk cache of a var for better
* performance.
*
* @note For contiguous and compact storage vars, or when parallel I/O
* is in use, this function will do nothing and return ::NC_NOERR;
*
* @param grp Pointer to group info struct.
* @param var Pointer to var info struct.
*
* @return ::NC_NOERR No error.
* @author Ed Hartnett
*/
int
NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
{
size64_t cachesize,nelems;
NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
/* empty the cache */
zvar->cache->maxentries = 0;
makeroom(zvar->cache);
/* Reset the parameters */
/* The total cache size is considered fixed here, so modify nelems */
cachesize = var->chunk_cache_size;
nelems = floordiv(cachesize , zvar->chunksize);
if(nelems == 0) nelems = 1;
zvar->cache->maxentries = nelems;
#ifdef DEBUG
fprintf(stderr,"%s.cache.adjust: size=%ld nelems=%ld\n",
var->hdr.name,(unsigned long)cachesize,(unsigned long)zvar->cache->maxentries);
#endif
/* One more thing, adjust the chunksize */
zvar->cache->chunksize = zvar->chunksize;
/* and also free the fillchunk */
nullfree(zvar->cache->fillchunk);
zvar->cache->fillchunk = NULL;
return NC_NOERR;
}
/**************************************************/
/**
* Create a chunk cache object
*
* @param var containing var
* @param entrysize Size in bytes of an entry
* @param cachep return cache pointer
*
* @return ::NC_NOERR No error.
* @return ::NC_EINVAL Bad preemption.
* @author Dennis Heimbigner, Ed Hartnett
*/
int
NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, NCZChunkCache** cachep)
{
int stat = NC_NOERR;
NCZChunkCache* cache = NULL;
void* fill = NULL;
size_t nelems, cachesize;
NCZ_VAR_INFO_T* zvar = NULL;
if(chunksize == 0) return NC_EINVAL;
zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
if((cache = calloc(1,sizeof(NCZChunkCache))) == NULL)
{stat = NC_ENOMEM; goto done;}
cache->var = var;
cache->ndims = var->ndims + zvar->scalar;
assert(cache->fillchunk == NULL);
cache->fillchunk = NULL;
cache->chunksize = chunksize;
/* Figure out the actual cache size */
cachesize = var->chunk_cache_size;
nelems = (cachesize / chunksize);
if(nelems == 0) nelems = 1;
/* Make consistent */
cachesize = nelems * chunksize;
cache->maxentries = nelems;
#ifdef FLUSH
cache->maxentries = 1;
#endif
#ifdef DEBUG
fprintf(stderr,"%s.cache: nelems=%ld size=%ld\n",
var->hdr.name,(unsigned long)cache->maxentries,(unsigned long)(cache->maxentries*cache->chunksize));
#endif
if((stat = ncxcachenew(LEAFLEN,&cache->xcache))) goto done;
if((cache->mru = nclistnew()) == NULL)
{stat = NC_ENOMEM; goto done;}
nclistsetalloc(cache->mru,cache->maxentries);
if(cachep) {*cachep = cache; cache = NULL;}
done:
nullfree(fill);
NCZ_free_chunk_cache(cache);
return THROW(stat);
}
void
NCZ_free_chunk_cache(NCZChunkCache* cache)
{
if(cache == NULL) return;
/* Iterate over the entries */
while(nclistlength(cache->mru) > 0) {
void* ptr;
NCZCacheEntry* entry = nclistremove(cache->mru,0);
(void)ncxcacheremove(cache->xcache,entry->hashkey,&ptr);
assert(ptr == entry);
nullfree(entry->data); nullfree(entry->key); nullfree(entry);
}
#ifdef DEBUG
fprintf(stderr,"|cache.free|=%ld\n",nclistlength(cache->mru));
#endif
ncxcachefree(cache->xcache);
nclistfree(cache->mru);
cache->mru = NULL;
nullfree(cache->fillchunk);
nullfree(cache);
}
size64_t
NCZ_cache_entrysize(NCZChunkCache* cache)
{
assert(cache);
return cache->chunksize;
}
/* Return number of active entries in cache */
size64_t
NCZ_cache_size(NCZChunkCache* cache)
{
assert(cache);
return nclistlength(cache->mru);
}
int
NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap)
{
int stat = NC_NOERR;
int rank = cache->ndims;
NC_FILE_INFO_T* file = cache->var->container->nc4_info;
NCZCacheEntry* entry = NULL;
ncexhashkey_t hkey = 0;
int created = 0;
/* the hash key */
hkey = ncxcachekey(indices,sizeof(size64_t)*cache->ndims);
/* See if already in cache */
switch (stat = ncxcachelookup(cache->xcache,hkey,(void**)&entry)) {
case NC_NOERR:
/* Move to front of the lru */
(void)ncxcachetouch(cache->xcache,hkey);
break;
case NC_ENOTFOUND:
entry = NULL; /* not found; */
break;
default: goto done;
}
if(entry == NULL) { /*!found*/
/* Make room in the cache */
if((stat=makeroom(cache))) goto done;
/* Create a new entry */
if((entry = calloc(1,sizeof(NCZCacheEntry)))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(entry->indices,indices,rank*sizeof(size64_t));
/* Create the local copy space */
if((entry->data = calloc(1,cache->chunksize)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* Create the key for this cache */
if((stat = NCZ_buildchunkpath(cache,indices,&entry->key))) goto done;
entry->hashkey = hkey;
/* Try to read the object in toto */
stat=get_chunk(cache,entry);
switch (stat) {
case NC_NOERR: break;
case NC_EEMPTY:
case NC_ENOTFOUND: /*signals the chunk needs to be created */
/* If the file is read-only, then fake the chunk */
entry->modified = (!file->no_write);
if(!file->no_write) {
if((stat = create_chunk(cache,entry))) goto done;
}
#ifdef FILLONREAD
/* apply fill value */
memcpy(entry->data,cache->fillchunk,cache->chunksize);
#else
memset(entry->data,0,cache->chunksize);
#endif
created = 1;
break;
default: goto done;
}
nclistpush(cache->mru,entry);
if((stat = ncxcacheinsert(cache->xcache,entry->hashkey,entry))) goto done;
}
#ifdef DEBUG
fprintf(stderr,"|cache.read.lru|=%ld\n",nclistlength(cache->mru));
#endif
if(datap) *datap = entry->data;
entry = NULL;
if(created) stat = NC_ENOTFOUND;
done:
if(entry) {nullfree(entry->data); nullfree(entry->key);}
nullfree(entry);
return THROW(stat);
}
int
NCZ_write_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap)
{
int stat = NC_NOERR;
int rank = cache->ndims;
NCZCacheEntry* entry = NULL;
ncexhashkey_t hkey;
/* and the hash key */
hkey = ncxcachekey(indices,sizeof(size64_t)*cache->ndims);
if(entry == NULL) { /*!found*/
if((stat=makeroom(cache))) goto done;
/* Create a new entry */
if((entry = calloc(1,sizeof(NCZCacheEntry)))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(entry->indices,indices,rank*sizeof(size64_t));
/* Create the local copy space */
if((entry->data = calloc(1,cache->chunksize)) == NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = NCZ_buildchunkpath(cache,indices,&entry->key))) goto done;
entry->hashkey = hkey;
}
entry->modified = 1;
nclistpush(cache->mru,entry); /* MRU order */
#ifdef DEBUG
fprintf(stderr,"|cache.write|=%ld\n",nclistlength(cache->mru));
#endif
entry = NULL;
done:
if(entry) {nullfree(entry->data); nullfree(entry->key);}
nullfree(entry);
return THROW(stat);
}
static int
makeroom(NCZChunkCache* cache)
{
int stat = NC_NOERR;
/* Flush from LRU end if we are at capacity */
while(nclistlength(cache->mru) > cache->maxentries) {
int i;
void* ptr;
NCZCacheEntry* e = ncxcachelast(cache->xcache); /* last entry is the least recently used */
if((stat = ncxcacheremove(cache->xcache,e->hashkey,&ptr))) goto done;
assert(e == ptr);
for(i=0;i<nclistlength(cache->mru);i++) {
e = nclistget(cache->mru,i);
if(ptr == e) break;
}
assert(e != NULL);
assert(i >= 0 && i < nclistlength(cache->mru));
nclistremove(cache->mru,i);
if(e->modified) /* flush to file */
stat=put_chunk(cache,e);
/* reclaim */
nullfree(e->data); nullfree(e->key); nullfree(e);
}
#ifdef DEBUG
fprintf(stderr,"|cache.makeroom|=%ld\n",nclistlength(cache->mru));
#endif
done:
return stat;
}
int
NCZ_flush_chunk_cache(NCZChunkCache* cache)
{
int stat = NC_NOERR;
size_t i;
if(NCZ_cache_size(cache) == 0) goto done;
/* Iterate over the entries in hashmap */
for(i=0;i<nclistlength(cache->mru);i++) {
NCZCacheEntry* entry = nclistget(cache->mru,i);
if(entry->modified) {
/* Write out this chunk in toto*/
if((stat=put_chunk(cache,entry)))
goto done;
}
entry->modified = 0;
}
done:
return THROW(stat);
}
#if 0
int
NCZ_chunk_cache_modified(NCZChunkCache* cache, const size64_t* indices)
{
int stat = NC_NOERR;
char* key = NULL;
NCZCacheEntry* entry = NULL;
int rank = cache->ndims;
/* Create the key for this cache */
if((stat=buildchunkkey(rank, indices, &key))) goto done;
/* See if already in cache */
if(NC_hashmapget(cache->mru, key, strlen(key), (uintptr_t*)entry)) { /* found */
entry->modified = 1;
}
done:
nullfree(key);
return THROW(stat);
}
#endif
/**************************************************/
/*
From Zarr V2 Specification:
"The compressed sequence of bytes for each chunk is stored under
a key formed from the index of the chunk within the grid of
chunks representing the array. To form a string key for a
chunk, the indices are converted to strings and concatenated
with the period character (".") separating each index. For
example, given an array with shape (10000, 10000) and chunk
shape (1000, 1000) there will be 100 chunks laid out in a 10 by
10 grid. The chunk with indices (0, 0) provides data for rows
0-1000 and columns 0-1000 and is stored under the key "0.0"; the
chunk with indices (2, 4) provides data for rows 2000-3000 and
columns 4000-5000 and is stored under the key "2.4"; etc."
*/
/**
* @param R Rank
* @param chunkindices The chunk indices
* @param keyp Return the chunk key string
*/
static int
buildchunkkey(size_t R, const size64_t* chunkindices, char** keyp)
{
int stat = NC_NOERR;
int r;
NCbytes* key = ncbytesnew();
if(keyp) *keyp = NULL;
for(r=0;r<R;r++) {
char sindex[64];
if(r > 0) ncbytescat(key,".");
/* Print as decimal with no leading zeros */
snprintf(sindex,sizeof(sindex),"%lu",(unsigned long)chunkindices[r]);
ncbytescat(key,sindex);
}
ncbytesnull(key);
if(keyp) *keyp = ncbytesextract(key);
ncbytesfree(key);
return THROW(stat);
}
/**
* @internal Push data to chunk of a file.
* If chunk does not exist, create it
*
* @param file Pointer to file info struct.
* @param proj Chunk projection
* @param datalen size of data
* @param data Buffer containing the chunk data to write
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
put_chunk(NCZChunkCache* cache, const NCZCacheEntry* entry)
{
int stat = NC_NOERR;
NCZ_FILE_INFO_T* zfile = NULL;
NCZMAP* map = NULL;
LOG((3, "%s: var: %p", __func__, cache->var));
zfile = ((cache->var->container)->nc4_info)->format_file_info;
map = zfile->map;
stat = nczmap_write(map,entry->key,0,cache->chunksize,entry->data);
switch(stat) {
case NC_NOERR: break;
case NC_EEMPTY:
/* Create the chunk */
if((stat = nczmap_defineobj(map,entry->key))) goto done;
/* write again */
if((stat = nczmap_write(map,entry->key,0,cache->chunksize,entry->data)))
goto done;
break;
default: goto done;
}
done:
return THROW(stat);
}
/**
* @internal Push data from memory to file.
*
* @param cache Pointer to parent cache
* @param key chunk key
* @param entry cache entry to read into
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
{
int stat = NC_NOERR;
NCZMAP* map = NULL;
NC_FILE_INFO_T* file = NULL;
NCZ_FILE_INFO_T* zfile = NULL;
LOG((3, "%s: file: %p", __func__, file));
file = (cache->var->container)->nc4_info;
zfile = file->format_file_info;
map = zfile->map;
assert(map && entry->data);
stat = nczmap_read(map,entry->key,0,cache->chunksize,(char*)entry->data);
return THROW(stat);
}
static int
create_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
{
int stat = NC_NOERR;
NC_FILE_INFO_T* file = NULL;
NCZ_FILE_INFO_T* zfile = NULL;
NCZMAP* map = NULL;
file = (cache->var->container)->nc4_info;
zfile = file->format_file_info;
map = zfile->map;
/* Create the chunk */
if((stat = nczmap_defineobj(map,entry->key))) goto done;
entry->modified = 1; /* mark as modified */
/* let higher function decide on fill */
done:
return THROW(stat);
}
int
NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, char** keyp)
{
int stat = NC_NOERR;
char* chunkname = NULL;
char* varkey = NULL;
char* key = NULL;
/* Get the chunk object name */
if((stat = buildchunkkey(cache->ndims, chunkindices, &chunkname))) goto done;
/* Get the var object key */
if((stat = NCZ_varkey(cache->var,&varkey))) goto done;
/* Prefix the path to the containing variable object */
if((stat=nczm_concat(varkey,chunkname,&key))) goto done;
if(keyp) {*keyp = key; key = NULL;}
done:
nullfree(chunkname);
nullfree(varkey);
nullfree(key);
return THROW(stat);
}

11
libsrc4/nc4internal.c
View File

@ -687,9 +687,6 @@ nc4_var_list_add2(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var)
return NC_ENOMEM;
}
new_var->hdr.hashkey = NC_hashmapkey(new_var->hdr.name,
strlen(new_var->hdr.name));
/* Create an indexed list for the attributes. */
new_var->att = ncindexnew(0);
@ -807,8 +804,6 @@ nc4_dim_list_add(NC_GRP_INFO_T *grp, const char *name, size_t len,
return NC_ENOMEM;
}
new_dim->hdr.hashkey = NC_hashmapkey(new_dim->hdr.name,
strlen(new_dim->hdr.name));
/* Is dimension unlimited? */
new_dim->len = len;
@ -859,8 +854,6 @@ nc4_att_list_add(NCindex *list, const char *name, NC_ATT_INFO_T **att)
free(new_att);
return NC_ENOMEM;
}
/* Create a hash of the name. */
new_att->hdr.hashkey = NC_hashmapkey(name, strlen(name));
/* Add object to list as specified by its number */
ncindexadd(list, (NC_OBJ *)new_att);
@ -915,8 +908,6 @@ nc4_grp_list_add(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *parent, char *name,
free(new_grp);
return NC_ENOMEM;
}
new_grp->hdr.hashkey = NC_hashmapkey(new_grp->hdr.name,
strlen(new_grp->hdr.name));
/* Set up new indexed lists for stuff this group can contain. */
new_grp->children = ncindexnew(0);
@ -1003,7 +994,6 @@ nc4_type_new(size_t size, const char *name, int assignedid,
if (!(new_type = calloc(1, sizeof(NC_TYPE_INFO_T))))
return NC_ENOMEM;
new_type->hdr.sort = NCTYP;
new_type->hdr.hashkey = NC_hashmapkey(name, strlen(name));
new_type->hdr.id = assignedid;
/* Remember info about this type. */
@ -1097,7 +1087,6 @@ nc4_field_list_add(NC_TYPE_INFO_T *parent, const char *name,
free(field);
return NC_ENOMEM;
}
field->hdr.hashkey = NC_hashmapkey(field->hdr.name,strlen(field->hdr.name));
field->nc_typeid = xtype;
field->offset = offset;
field->ndims = ndims;

4
libsrc4/ncindex.c
View File

@ -379,8 +379,8 @@ printindexlist(NClist* lm)
if(o == NULL)
fprintf(stderr,"[%ld] <null>\n",(unsigned long)i);
else
fprintf(stderr,"[%ld] sort=%s name=|%s| id=%lu hashkey=%lu\n",
(unsigned long)i,sortname(o->sort),o->name,(unsigned long)o->id,(unsigned long)o->hashkey);
fprintf(stderr,"[%ld] sort=%s name=|%s| id=%lu\n",
(unsigned long)i,sortname(o->sort),o->name,(unsigned long)o->id);
}
}

19
ncdump/CMakeLists.txt
View File

@ -29,6 +29,25 @@ IF(USE_HDF5)
ADD_EXECUTABLE(nc4print nc4print.c nc4printer.c)
ENDIF(USE_HDF5)
# Given a netcdf4 file, dump the actual chunk contents.
# Used to validate nczarr chunking code.
IF(USE_HDF5 OR ENABLE_NCZARR)
SET(ncdumpchunks_FILES ncdumpchunks.c)
ADD_EXECUTABLE(ncdumpchunks ${ncdumpchunks_FILES})
TARGET_LINK_LIBRARIES(ncdumpchunks netcdf ${ALL_TLL_LIBS})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
IF(MSVC)
SET_TARGET_PROPERTIES(ncdumpchunks
PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT"
)
ENDIF(MSVC)
ENDIF(USE_HDF5 OR ENABLE_NCZARR)
IF(ENABLE_DAP)
ADD_EXECUTABLE(ocprint ${ocprint_FILES})
ENDIF(ENABLE_DAP)

12
ncdump/Makefile.am
View File

@ -49,6 +49,18 @@ noinst_PROGRAMS += nc4print
nc4print_SOURCES = nc4print.c nc4printer.c
endif
# Given a netcdf4|NCZarr file, dump the actual chunk contents.
# Used to validate nczarr chunking code.
if ENABLE_NCZARR
AM_CPPFLAGS += -I$(top_srcdir)/libnczarr
noinst_PROGRAMS = ncdumpchunks
else
if USE_HDF5
noinst_PROGRAMS = ncdumpchunks
endif
endif
ncdumpchunks_SOURCES = ncdumpchunks.c
# Conditionally build the ocprint program, but do not install
if ENABLE_DAP
bin_PROGRAMS += ocprint

365
ncdump/ncdumpchunks.c Executable file
View File

@ -0,0 +1,365 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <getopt.h>
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "netcdf.h"
#ifdef HAVE_HDF5_H
#include <hdf5.h>
#include <H5DSpublic.h>
#endif
#ifdef ENABLE_NCZARR
#include "zincludes.h"
#endif
typedef struct Odometer {
size_t rank; /*rank */
size_t stop[NC_MAX_VAR_DIMS];
size_t max[NC_MAX_VAR_DIMS]; /* max size of ith index */
size_t index[NC_MAX_VAR_DIMS]; /* current value of the odometer*/
} Odometer;
#define floordiv(x,y) ((x) / (y))
#define ceildiv(x,y) (((x) % (y)) == 0 ? ((x) / (y)) : (((x) / (y)) + 1))
Odometer* odom_new(size_t rank, const size_t* stop, const size_t* max);
void odom_free(Odometer* odom);
int odom_more(Odometer* odom);
int odom_next(Odometer* odom);
size_t* odom_indices(Odometer* odom);
size_t odom_offset(Odometer* odom);
static void
usage(int err)
{
if(err != 0) {
fprintf(stderr,"Error: (%d) %s\n",err,nc_strerror(err));
}
fprintf(stderr,"usage: ncdumpchunks <file> <var>\n");
fflush(stderr);
exit(1);
}
Odometer*
odom_new(size_t rank, const size_t* stop, const size_t* max)
{
int i;
Odometer* odom = NULL;
if((odom = calloc(1,sizeof(Odometer))) == NULL)
return NULL;
odom->rank = rank;
for(i=0;i<rank;i++) {
odom->stop[i] = stop[i];
odom->max[i] = max[i];
odom->index[i] = 0;
}
return odom;
}
void
odom_free(Odometer* odom)
{
if(odom) free(odom);
}
int
odom_more(Odometer* odom)
{
return (odom->index[0] < odom->stop[0]);
}
int
odom_next(Odometer* odom)
{
size_t i;
for(i=odom->rank-1;i>=0;i--) {
odom->index[i]++;
if(odom->index[i] < odom->stop[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows */
odom->index[i] = 0; /* reset this position */
}
return 1;
}
/* Get the value of the odometer */
size_t*
odom_indices(Odometer* odom)
{
return odom->index;
}
size_t
odom_offset(Odometer* odom)
{
size_t offset;
int i;
offset = 0;
for(i=0;i<odom->rank;i++) {
offset *= odom->max[i];
offset += odom->index[i];
}
return offset;
}
char*
printvector(int rank, size_t* vec)
{
char svec[NC_MAX_VAR_DIMS*3+1];
int i;
svec[0] = '\0';
for(i=0;i<rank;i++) {
char s[3+1];
if(i > 0) strlcat(svec,",",sizeof(svec));
snprintf(s,sizeof(s),"%u",(unsigned)vec[i]);
strlcat(svec,s,sizeof(svec));
}
return strdup(svec);
}
#ifdef USE_HDF5
void
hdf5_setoffset(Odometer* odom, size_t* chunksizes, hsize_t* offset)
{
int i;
for(i=0;i<odom->rank;i++)
offset[i] = odom->index[i] * chunksizes[i];
}
int
hdf5(const char* file_name, const char* var_name, int debug,
int rank, size_t* dimlens, size_t* chunklens, size_t* chunkcounts
)
{
int i;
hid_t fileid, grpid, datasetid;
int* chunkdata = NULL; /*[CHUNKPROD];*/
size_t chunkprod;
Odometer* odom = NULL;
hsize_t offset[NC_MAX_VAR_DIMS];
#ifdef HDF5_SUPPORTS_PAR_FILTERS
int r;
hid_t dxpl_id = H5P_DEFAULT; /*data transfer property list */
unsigned int filter_mask = 0;
#endif
if(debug) {
H5Eset_auto2(H5E_DEFAULT,(H5E_auto2_t)H5Eprint,stderr);
}
chunkprod = 1;
for(i=0;i<rank;i++)
chunkprod *= chunklens[i];
if ((fileid = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) usage(NC_EHDFERR);
if ((grpid = H5Gopen1(fileid, "/")) < 0) usage(NC_EHDFERR);
if ((datasetid = H5Dopen1(grpid, var_name)) < 0) usage(NC_EHDFERR);
if((odom = odom_new(rank,chunkcounts,dimlens))==NULL) usage(NC_ENOMEM);
if((chunkdata = calloc(sizeof(int),chunkprod))==NULL) usage(NC_ENOMEM);
printf("rank=%d dims=(%s) chunks=(%s)\n",rank,printvector(rank,dimlens),printvector(rank,chunklens));
while(odom_more(odom)) {
hdf5_setoffset(odom,chunklens,offset);
#ifdef DEBUG
fprintf(stderr,"(");
for(i=0;i<rank;i++)
fprintf(stderr,"%s%lu",(i > 0 ? "," : ""),(unsigned long)offset[i]);
fprintf(stderr,")\n");
fflush(stderr);
#endif
memset(chunkdata,0,sizeof(int)*chunkprod);
if(H5Dread_chunk(datasetid, dxpl_id, offset, &filter_mask, chunkdata) < 0) abort();
for(r=0;r<rank;r++)
printf("[%lu/%lu]",(unsigned long)odom->index[r],(unsigned long)offset[r]);
printf(" =");
for(r=0;r<chunkprod;r++)
printf(" %02d", chunkdata[r]);
printf("\n");
fflush(stdout);
odom_next(odom);
}
/* Close up. */
if (H5Dclose(datasetid) < 0) abort();
if (H5Gclose(grpid) < 0) abort();
if (H5Fclose(fileid) < 0) abort();
/* Cleanup */
free(chunkdata);
odom_free(odom);
return 0;
}
#endif
#ifdef ENABLE_NCZARR
static void
nczarr_setoffset(Odometer* odom, size_t* chunksizes, size64_t* offset)
{
int i;
for(i=0;i<odom->rank;i++)
offset[i] = odom->index[i] * chunksizes[i];
}
char*
chunk_key(int rank, size_t* indices)
{
char key[NC_MAX_VAR_DIMS*3+1];
int i;
key[0] = '\0';
for(i=0;i<rank;i++) {
char s[3+1];
if(i > 0) strlcat(key,".",sizeof(key));
snprintf(s,sizeof(s),"%u",(unsigned)indices[i]);
strlcat(key,s,sizeof(key));
}
return strdup(key);
}
int
nczarr(const char* file_name, const char* var_name, int debug,
int rank, size_t* dimlens, size_t* chunklens, size_t* chunkcounts
)
{
int r,stat = NC_NOERR;
int* chunkdata = NULL; /*[CHUNKPROD];*/
size_t chunkprod;
Odometer* odom = NULL;
size64_t zindices[NC_MAX_VAR_DIMS];
size64_t offset[NC_MAX_VAR_DIMS];
int ncid, varid;
NC_UNUSED(debug);
if((stat=nc_open(file_name,0,&ncid))) usage(stat);
if((stat=nc_inq_varid(ncid,var_name,&varid))) usage(stat);
chunkprod = 1;
for(r=0;r<rank;r++)
chunkprod *= chunklens[r];
if((odom = odom_new(rank,chunkcounts,dimlens))==NULL) usage(NC_ENOMEM);
if((chunkdata = calloc(sizeof(int),chunkprod))==NULL) usage(NC_ENOMEM);
printf("rank=%d dims=(%s) chunks=(%s)\n",rank,printvector(rank,dimlens),printvector(rank,chunklens));
while(odom_more(odom)) {
memset(chunkdata,0,sizeof(int)*chunkprod);
nczarr_setoffset(odom,chunklens,offset);
for(r=0;r<rank;r++)
zindices[r] = (size64_t)odom->index[r];
if((stat=NCZ_read_chunk(ncid, varid, zindices, chunkdata) < 0)) usage(stat);
for(r=0;r<rank;r++)
printf("[%lu/%lu]",(unsigned long)odom->index[r],(unsigned long)offset[r]);
printf(" =");
for(r=0;r<chunkprod;r++)
printf(" %02d", chunkdata[r]);
printf("\n");
fflush(stdout);
odom_next(odom);
}
/* Cleanup */
free(chunkdata);
odom_free(odom);
if((stat=nc_close(ncid))) usage(stat);
return 0;
}
#endif
int
main(int argc, char** argv)
{
int i,stat = NC_NOERR;
const char* file_name = NULL;
const char* var_name = NULL;
int ncid, varid, dimids[NC_MAX_VAR_DIMS];
int rank, vtype, storage;
size_t dimlens[NC_MAX_VAR_DIMS];
size_t chunklens[NC_MAX_VAR_DIMS];
size_t chunkcounts[NC_MAX_VAR_DIMS];
int debug = 0;
int format,mode;
int c;
while ((c = getopt(argc, argv, "dv:")) != EOF) {
switch(c) {
case 'd':
debug = 1;
break;
case 'v':
var_name = strdup(optarg);
break;
case '?':
fprintf(stderr,"unknown option: '%c'\n",c);
exit(1);
}
}
/* get file argument */
argc -= optind;
argv += optind;
if (argc == 0) {
fprintf(stderr, "no input file specified\n");
exit(1);
}
file_name = strdup(argv[0]);
if(file_name == NULL || strlen(file_name) == 0) {
fprintf(stderr, "no input file specified\n");
exit(1);
}
if(var_name == NULL || strlen(var_name) == 0) {
fprintf(stderr, "no input var specified\n");
exit(1);
}
/* Get info about the file type */
if((stat=nc_open(file_name,0,&ncid))) usage(stat);
if((stat=nc_inq_format_extended(ncid,&format,&mode))) usage(stat);
/* Get the info about the var */
if((stat=nc_inq_varid(ncid,var_name,&varid))) usage(stat);
if((stat=nc_inq_var(ncid,varid,NULL,&vtype,&rank,dimids,NULL))) usage(stat);
if(rank == 0) usage(NC_EDIMSIZE);
if((stat=nc_inq_var_chunking(ncid,varid,&storage,chunklens))) usage(stat);
if(storage != NC_CHUNKED) usage(NC_EBADCHUNK);
for(i=0;i<rank;i++) {
if((stat=nc_inq_dimlen(ncid,dimids[i],&dimlens[i]))) usage(stat);
chunkcounts[i] = ceildiv(dimlens[i],chunklens[i]);
}
if((stat=nc_close(ncid))) usage(stat);
switch (format) {
#ifdef USE_HDF5
case NC_FORMATX_NC_HDF5:
hdf5(file_name, var_name, debug, rank, dimlens, chunklens, chunkcounts);
break;
#endif
#ifdef ENABLE_NCZARR
case NC_FORMATX_NCZARR:
nczarr(file_name, var_name, debug, rank, dimlens, chunklens, chunkcounts);
break;
#endif
default:
usage(NC_EINVAL);
break;
}
return 0;
}

18
ncgen/CMakeLists.txt
View File

@ -34,24 +34,6 @@ TARGET_LINK_LIBRARIES(ncgen netcdf ${ALL_TLL_LIBS})
# Given a netcdf4 file, dump the actual chunk contents.
# Used to validate nczarr chunking code.
IF(USE_HDF5)
IF(ENABLE_NCDUMPCHUNKS)
SET(ncdumpchunks_FILES ncdumpchunks.c)
ADD_EXECUTABLE(ncdumpchunks ${ncdumpchunks_FILES})
TARGET_LINK_LIBRARIES(ncdumpchunks netcdf ${ALL_TLL_LIBS})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
IF(MSVC)
SET_TARGET_PROPERTIES(ncdumpchunks
PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT"
)
ENDIF()
ENDIF()
ENDIF()
####
# We have to do a little tweaking

7
ncgen/Makefile.am
View File

@ -21,13 +21,6 @@ ncgen.h ncgeny.h util.h
# Obsolete
OBSOLETE = odom.c odom.h jdatastd.c jdatajni.c genjni.c cdfdata.c cmldata.c
# Given a netcdf4 file, dump the actual chunk contents.
# Used to validate nczarr chunking code.
if USE_HDF5
noinst_PROGRAMS = ncdumpchunks
ncdumpchunks_SOURCES = ncdumpchunks.c
endif
# This is the man page.
man_MANS = ncgen.1

202
ncgen/ncdumpchunks.c
View File

@ -1,202 +0,0 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <netcdf.h>
#include <hdf5.h>
#include <H5DSpublic.h>
#if 0
#define FILE_NAME "tst_vars.nc"
#define VAR "v1"
#define RANK 3
static const size_t dimlens[RANK] = {4,4,2};
static const size_t chunksize[RANK] = {2,2,2};
static const size_t chunkcount[RANK] = {2,2,1};
static const size_t CHUNKPROD = 8;
#endif
typedef struct Odometer {
size_t rank; /*rank */
size_t stop[NC_MAX_VAR_DIMS];
size_t max[NC_MAX_VAR_DIMS]; /* max size of ith index */
size_t index[NC_MAX_VAR_DIMS]; /* current value of the odometer*/
} Odometer;
#define floordiv(x,y) ((x) / (y))
#define ceildiv(x,y) (((x) % (y)) == 0 ? ((x) / (y)) : (((x) / (y)) + 1))
Odometer* odom_new(size_t rank, const size_t* stop, const size_t* max);
void odom_free(Odometer* odom);
int odom_more(Odometer* odom);
int odom_next(Odometer* odom);
size_t* odom_indices(Odometer* odom);
size_t odom_offset(Odometer* odom);
void setoffset(Odometer* odom, size_t* chunksizes, hsize_t* offset);
static void
usage(int err)
{
if(err != 0) {
fprintf(stderr,"Error: (%d) %s\n",err,nc_strerror(err));
}
fprintf(stderr,"usage: ncdumpchunks <file> <var>\n");
fflush(stderr);
exit(1);
}
int
main(int argc, char** argv)
{
int i,stat = NC_NOERR;
hid_t fileid, grpid, datasetid;
int* chunkdata = NULL; /*[CHUNKPROD];*/
#ifdef HDF5_SUPPORTS_PAR_FILTERS
int r;
hid_t dxpl_id = H5P_DEFAULT; /*data transfer property list */
unsigned int filter_mask = 0;
#endif
const char* file_name = NULL;
const char* var_name = NULL;
int ncid, varid, dimids[NC_MAX_VAR_DIMS];
int rank, vtype, storage;
size_t dimlens[NC_MAX_VAR_DIMS];
size_t chunklens[NC_MAX_VAR_DIMS];
size_t chunkcounts[NC_MAX_VAR_DIMS];
size_t chunkprod;
Odometer* odom = NULL;
hsize_t offset[NC_MAX_VAR_DIMS];
if(argc < 3)
usage(0);
file_name = argv[1];
var_name = argv[2];
/* Get the info about the var */
if((stat=nc_open(file_name,0,&ncid))) usage(stat);
if((stat=nc_inq_varid(ncid,var_name,&varid))) usage(stat);
if((stat=nc_inq_var(ncid,varid,NULL,&vtype,&rank,dimids,NULL))) usage(stat);
if(rank == 0) usage(NC_EDIMSIZE);
if((stat=nc_inq_var_chunking(ncid,varid,&storage,chunklens))) usage(stat);
if(storage != NC_CHUNKED) usage(NC_EBADCHUNK);
chunkprod = 1;
for(i=0;i<rank;i++) {
if((stat=nc_inq_dimlen(ncid,dimids[i],&dimlens[i]))) usage(stat);
chunkcounts[i] = ceildiv(dimlens[i],chunklens[i]);
chunkprod *= chunklens[i];
}
if((stat=nc_close(ncid))) usage(stat);
if ((fileid = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) usage(NC_EHDFERR);
if ((grpid = H5Gopen(fileid, "/", H5P_DEFAULT)) < 0) usage(NC_EHDFERR);
if ((datasetid = H5Dopen1(grpid, var_name)) < 0) usage(NC_EHDFERR);
if((odom = odom_new(rank,chunkcounts,dimlens))==NULL) usage(NC_ENOMEM);
if((chunkdata = calloc(sizeof(int),chunkprod))==NULL) usage(NC_ENOMEM);
while(odom_more(odom)) {
setoffset(odom,chunklens,offset);
#ifdef DEBUG
fprintf(stderr,"(");
for(i=0;i<rank;i++)
fprintf(stderr,"%s%lu",(i > 0 ? "," : ""),(unsigned long)offset[i]);
fprintf(stderr,")\n");
fflush(stderr);
#endif
memset(chunkdata,0,sizeof(int)*chunkprod);
#ifdef HDF5_SUPPORTS_PAR_FILTERS
if(H5Dread_chunk(datasetid, dxpl_id, offset, &filter_mask, chunkdata) < 0) abort();
for(r=0;r<rank;r++)
printf("[%lu/%lu]",(unsigned long)odom->index[r],(unsigned long)offset[r]);
printf(" =");
for(r=0;r<chunkprod;r++)
printf(" %02d", chunkdata[r]);
printf("\n");
fflush(stdout);
#endif
odom_next(odom);
}
/* Close up. */
if (H5Dclose(datasetid) < 0) abort();
if (H5Gclose(grpid) < 0) abort();
if (H5Fclose(fileid) < 0) abort();
/* Cleanup */
free(chunkdata);
odom_free(odom);
return 0;
}
Odometer*
odom_new(size_t rank, const size_t* stop, const size_t* max)
{
int i;
Odometer* odom = NULL;
if((odom = calloc(1,sizeof(Odometer))) == NULL)
return NULL;
odom->rank = rank;
for(i=0;i<rank;i++) {
odom->stop[i] = stop[i];
odom->max[i] = max[i];
odom->index[i] = 0;
}
return odom;
}
void
odom_free(Odometer* odom)
{
if(odom) free(odom);
}
int
odom_more(Odometer* odom)
{
return (odom->index[0] < odom->stop[0]);
}
int
odom_next(Odometer* odom)
{
size_t i;
for(i=odom->rank-1;i>=0;i--) {
odom->index[i]++;
if(odom->index[i] < odom->stop[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows */
odom->index[i] = 0; /* reset this position */
}
return 1;
}
/* Get the value of the odometer */
size_t*
odom_indices(Odometer* odom)
{
return odom->index;
}
size_t
odom_offset(Odometer* odom)
{
size_t offset;
int i;
offset = 0;
for(i=0;i<odom->rank;i++) {
offset *= odom->max[i];
offset += odom->index[i];
}
return offset;
}
void
setoffset(Odometer* odom, size_t* chunksizes, hsize_t* offset)
{
int i;
for(i=0;i<odom->rank;i++)
offset[i] = odom->index[i] * chunksizes[i];
}

44
nczarr_test/Makefile.am
View File

@ -13,9 +13,9 @@ TESTS_ENVIRONMENT =
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#TESTS_ENVIRONMENT = export SETX=1;
#TESTS_ENVIRONMENT += export SETX=1;
AM_CFLAGS += -I${top_srcdir} -I${top_srcdir}/libnczarr
AM_CPPFLAGS += -I${top_srcdir} -I${top_srcdir}/libnczarr
AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la
LDADD = ${top_builddir}/liblib/libnetcdf.la
@ -23,7 +23,6 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la
check_PROGRAMS =
TESTS =
# Unit Tests
check_PROGRAMS += ut_map ut_mapapi ut_json ut_projections ut_chunking
commonsrc = ut_util.c ut_test.c ut_includes.h ut_test.h ut_util.h test_nczarr_utils.h
@ -42,13 +41,15 @@ TESTS += run_ut_misc.sh
TESTS += run_ut_chunk.sh
if BUILD_UTILITIES
if USE_HDF5
TESTS += tst_nccopyz.sh
TESTS += run_it_test1.sh
TESTS += test_fillonlyz.sh
endif
TESTS += run_it_test2.sh
endif
endif #BUILD_UTILITIES
ignorec = ut_allslices.c ut_transfer.c ut_vars.c ut_walk.c
ignoresh = run_meta_tests.sh run_unittests.sh
@ -64,7 +65,6 @@ zisjson_SOURCES = zisjson.c
noinst_PROGRAMS += zs3parse
zs3parse_SOURCES = zs3parse.c
EXTRA_DIST = \
run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_it_test1.sh run_it_test2.sh \
test_nczarr.sh tst_nccopyz.sh \
@ -108,6 +108,40 @@ tst_chunks.nz4 tst_chunks2.nc.nz4 \
ut_*.txt ut*.cdl tmp*.nc tmp*.cdl *.dmp test.nzf \
tst_chunks3.nc
if AX_IGNORE
if BUILD_BENCHMARKS
BMCOMMON=bm_utils.c ut_util.c ut_test.c
if AX_IGNORE
BENCHMARKS = bm_chunks3 bm_many_atts bm_many_objs
bm_chunks3_SOURCES = bm_chunks3.c ${BMCOMMON}
bm_many_atts_SOURCES = bm_many_atts.c ${BMCOMMON}
bm_many_objs_SOURCES = bm_many_objs.c ${BMCOMMON}
else
BENCHMARKS = bm_many_atts bm_many_objs
bm_many_atts_SOURCES = bm_many_atts.c ${BMCOMMON}
bm_many_objs_SOURCES = bm_many_objs.c ${BMCOMMON}
endif
#PG = -pg
AM_CFLAGS += ${PG}
AM_LDFLAGS += ${PG}
check_PROGRAMS += ${BENCHMARKS}
TESTS += run_bm_many_atts.sh
TESTS += run_bm_many_objs.sh
EXTRA_DIST += bm_common.sh run_bm_metatest.sh run_bm_many_atts.sh run_bm_many_objs.sh
CLEANFILES += bm_many_atts.nc4 bm_many_atts.nzf
CLEANFILES += bm_many_objs.nc4 bm_many_objs.nzf
endif
endif
# Remove directories
clean-local:
rm -fr tst_chunks.nzf tst_chunks2.nzf tst_perdimspecs.nzf testmap.nzf testmapapi.nzf test.nzf
rm -fr bm_many_*.nzf

43
nczarr_test/perf.sh Executable file
View File

@ -0,0 +1,43 @@
#!/bin/sh
# This is a metadata performance test for nczarr
# Dennis Heimbigner
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. ./bm_common.sh
echo "Testing performance of nc_create and nc_open on file with large metadata"
MEDARGS="--treedepth=2 \
--ngroups=2 \
--ngroupattrs=100 \
--ndims=100 \
--ntypes=10 \
--nvars=100 \
--varrank=2 \
--nvarattrs=500"
BIGARGS="--treedepth=6 \
--ngroups=2 \
--ngroupattrs=100 \
--ndims=100 \
--ntypes=10 \
--nvars=100 \
--varrank=2 \
--nvarattrs=500"
SMALLARGS="--treedepth=2 \
--ngroups=2 \
--ngroupattrs=100 \
--ndims=100 \
--ntypes=10 \
--nvars=10 \
--varrank=2 \
--nvarattrs=100"
ARGS="$SMALLARGS"
bmtest bm_bigmeta $ARGS
bmtest bm_openmeta $ARGS

4
nczarr_test/ref_perdimspecs.cdl
View File

@ -1,8 +1,8 @@
netcdf ref_tst_perdimspecs {
dimensions:
lon = 40 ;
lat = 41 ;
time = 42;
lat = 40 ;
time = 40;
variables:
float lon(lon) ;
float lat(lat) ;

2
nczarr_test/ref_ut_mapapi_data_nz4.cdl
View File

@ -2,7 +2,7 @@
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[2] /data1 : (100) |000000000100000002000000030000000400000005000000060000000700000008000000090000000a0000000b0000000c0000000d0000000e0000000f000000100000001100000012000000130000001400000015000000160000001700000018000000|
[2] /data1 : (100) (ubyte) |0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, 12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 18, 0, 0, 0, 19, 0, 0, 0, 20, 0, 0, 0, 21, 0, 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, 24, 0, 0, 0|
[4] /meta1/.zarray : (34) |{
"shape": [1,2,3],
"dtype": "<1"}|

2
nczarr_test/ref_ut_mapapi_data_nzf.cdl
View File

@ -2,7 +2,7 @@
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[2] /data1 : (100) |000000000100000002000000030000000400000005000000060000000700000008000000090000000a0000000b0000000c0000000d0000000e0000000f000000100000001100000012000000130000001400000015000000160000001700000018000000|
[2] /data1 : (100) (ubyte) |0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, 12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 18, 0, 0, 0, 19, 0, 0, 0, 20, 0, 0, 0, 21, 0, 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, 24, 0, 0, 0|
[4] /meta1/.zarray : (34) |{
"shape": [1,2,3],
"dtype": "<1"}|

23
nczarr_test/run_bm_many_atts.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/sh
# This is a metadata performance test for nczarr
# Dennis Heimbigner
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. ./bm_common.sh
echo "Testing performance of nc_create and nc_open on file with large # of attributes"
CMD=bm_many_atts
EXC=${execdir}/bm_many_atts
ARGS="--ngroups=100 --ngroupattrs=100 --X=1"
echo "***Testing format: nc4"
FMT=nc4 ; $EXC --format=$FMT --f=$CMD $ARGS
echo "***Testing format: nzf"
FMT=nzf ; $EXC --format=$FMT --f=$CMD $ARGS
if test "x$FEATURE_S3TESTS" = xyes ; then
echo "***Testing format: s3"
FMT=s3 ; $EXC --format=$FMT --f=$CMD $ARGS
fi

23
nczarr_test/run_bm_many_objs.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/sh
# This is a metadata performance test for nczarr
# Dennis Heimbigner
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. ./bm_common.sh
echo "Testing performance of nc_create and nc_open on file with large # of variables"
CMD=bm_many_objs
EXC=${execdir}/bm_many_objs
ARGS="--ngroups=100 --ngroupattrs=100 --nvars=100"
echo "***Testing format: nc4"
FMT=nc4 ; $EXC --format=$FMT --f=$CMD $ARGS
echo "***Testing format: nzf"
FMT=nzf ; $EXC --format=$FMT --f=$CMD $ARGS
if test "x$FEATURE_S3TESTS" = xyes ; then
echo "***Testing format: s3"
FMT=s3 ; $EXC --format=$FMT --f=$CMD $ARGS
fi

43
nczarr_test/run_bm_metatest.sh Executable file
View File

@ -0,0 +1,43 @@
#!/bin/sh
# This is a metadata performance test for nczarr
# Dennis Heimbigner
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. ./bm_common.sh
echo "Testing performance of nc_create and nc_open on file with large metadata"
MEDARGS="--treedepth=2 \
--ngroups=2 \
--ngroupattrs=100 \
--ndims=100 \
--ntypes=10 \
--nvars=100 \
--varrank=2 \
--nvarattrs=500"
BIGARGS="--treedepth=6 \
--ngroups=2 \
--ngroupattrs=100 \
--ndims=100 \
--ntypes=10 \
--nvars=100 \
--varrank=2 \
--nvarattrs=500"
SMALLARGS="--treedepth=2 \
--ngroups=2 \
--ngroupattrs=100 \
--ndims=100 \
--ntypes=10 \
--nvars=10 \
--varrank=2 \
--nvarattrs=100"
ARGS="$SMALLARGS"
bmtest bm_bigmeta $ARGS
#reclaim

44
nczarr_test/run_bm_perf.sh Executable file
View File

@ -0,0 +1,44 @@
#!/bin/sh
# This is a metadata performance test for nczarr
# Dennis Heimbigner
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. ./bm_common.sh
echo "Testing performance of nc_create and nc_open on file with large metadata"
MEDARGS="--treedepth=2 \
--ngroups=2 \
--ngroupattrs=100 \
--ndims=100 \
--ntypes=10 \
--nvars=100 \
--varrank=2 \
--nvarattrs=500"
BIGARGS="--treedepth=6 \
--ngroups=2 \
--ngroupattrs=100 \
--ndims=100 \
--ntypes=10 \
--nvars=100 \
--varrank=2 \
--nvarattrs=500"
SMALLARGS="--treedepth=2 \
--ngroups=2 \
--ngroupattrs=100 \
--ndims=100 \
--ntypes=10 \
--nvars=10 \
--varrank=2 \
--nvarattrs=100"
ARGS="$SMALLARGS"
bmtest bm_bigmeta $ARGS
bmtest bm_openmeta $ARGS
#reclaim

32
nczarr_test/run_tst_chunks.sh Executable file
View File

@ -0,0 +1,32 @@
#!/bin/sh
# This shell just tests the tst_chunks3 program by running it a few
# times to generate a simple test file. Then it uses ncdump -s to
# check that the output is what it should be.
# Russ Rew, Dennis Heimbigner
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
test() {
FMT=$1
DIMS=$1
CHUNKS=$2
${execdir}/bm_chunks3 --format=$FMT --X=1 --f=bm_chunks3 --dims="$DIMS" chunks="$CHUNKS"
}
echo ""
echo "*** Running benchmarking program tst_chunks3 for tiny test file"
test "6,12,4" 2,3,1"
echo '*** SUCCESS!!!'
exit 0
echo ""
echo "*** Testing the benchmarking program tst_chunks3 for larger variables ..."
#cachesize=10000000; cachehash=10000; cachepre=0.0
test "32,90,91" "8,10,13"
echo '*** SUCCESS!!!'
exit 0

0
nczarr_test/test_nczarr.sh Normal file → Executable file
View File

8
nczarr_test/tst_chunks.c
View File

@ -341,7 +341,7 @@ main(int argc, char **argv)
if (nc_def_var(ncid, VAR_NAME_CACHE_CHUNK_2, NC_INT64, NDIM2, dimid, &varid2)) ERR;
if (nc_def_var(ncid, VAR_NAME_CACHE_CHUNK_3, NC_INT64, NDIM2, dimid, &varid3)) ERR;
/* Set the var cache. */
/* Set the var cache to something arbitrary but small */
if (nc_set_var_chunk_cache(ncid, varid, cache_size, cache_nelems,
cache_preemption)) ERR;
@ -363,9 +363,10 @@ main(int argc, char **argv)
cache_preemption_in != cache_preemption) ERR;
if (nc_get_var_chunk_cache(ncid, varid2, &cache_size_in, &cache_nelems_in,
&cache_preemption_in)) ERR;
if (cache_size_in != CHUNK_CACHE_SIZE || cache_nelems_in != CHUNK_CACHE_NELEMS ||
cache_preemption_in != CHUNK_CACHE_PREEMPTION) ERR;
if (cache_size_in != CHUNK_CACHE_SIZE_NCZARR) ERR;
#if 0
/* Inapplicable to zarr */
/* The cache_size has been increased due to larger chunksizes
* for varid3. */
if (nc_get_var_chunk_cache(ncid, varid3, &cache_size_in, &cache_nelems_in,
@ -373,6 +374,7 @@ main(int argc, char **argv)
if (cache_nelems_in != CHUNK_CACHE_NELEMS ||
cache_preemption_in != CHUNK_CACHE_PREEMPTION) ERR;
/* printf("cache_size_in %ld\n", cache_size_in); */
#endif
/* Close the file. */
if (nc_close(ncid)) ERR;

32
nczarr_test/tst_ncgen4.sh Executable file
View File

@ -0,0 +1,32 @@
#!/bin/sh
# Tests for ncgen4 using list of test cdl files from the cdl4
# directory, and comparing output to expected results in the expected4
# directory. Note that these tests are run for classic files in
# tst_ncgen4_classic.sh
# Dennis Heimbigner
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
# To add a new test,
# 1. put the .cdl file in the 'cdl4' directory
# 2. put the result of running ncgen then ncdump
# into the directory 'expected4' as .dmp
# 3. Modify the file tst_ncgen_shared.sh to add
# the test to the end of the TESTS4 variable
# 4. Add the new files into cdl4/Makefile.am
# and expected4/Makefile.am
verbose=1
export verbose
KFLAG=4 ; export KFLAG
echo "*** Performing diff tests: k=4"
bash ${srcdir}/tst_ncgen4_diff.sh
echo "*** Performing cycle tests: k=4"
bash ${srcdir}/tst_ncgen4_cycle.sh
rm -rf ${RESULTSDIR}
echo "SUCCESS!!"
exit 0

133
nczarr_test/tst_ncgen4_diff.sh Executable file
View File

@ -0,0 +1,133 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
TESTSET="\
ref_dimscope \
ref_typescope \
ref_tst_string_data \
ref_tst_comp \
ref_tst_comp2 \
ref_tst_comp3 \
ref_tst_group_data \
ref_tst_opaque_data \
ref_tst_solar_1 \
ref_tst_solar_2 \
ref_tst_enum_data \
ref_tst_special_atts \
ref_tst_nans \
ref_solar \
unlimtest2 \
ref_niltest \
ref_tst_h_scalar \
ref_tst_nul4 \
"
# Functions
extfor() {
case "$1" in
nc4) zext="nz4" ;;
nz4) zext="nz4" ;;
nzf) zext="nzf" ;;
s3) zext="s3" ;;
*) echo "unknown kind: $1" ; exit 1;;
esac
}
deletefile() {
case "$1" in
nc4) rm -fr $2;;
nz4) rm -fr $2;;
nzf) rm -fr $2;;
esac
}
mapexists() {
mapexists=1
case "$1" in
nz4) if test -f $file; then mapexists=0; fi ;;
nzf) if test -f $file; then mapexists=0; fi ;;
s3)
if "./zmapdump $fileurl" ; then mapexists=1; else mapexists=0; fi
;;
*) echo unknown format: $1 : abort ; exit 1 ;;
esac
if test $mapexists = 1 ; then
echo "delete did not delete $1"
fi
}
fileargs() {
if test "x$zext" = xs3 ; then
fileurl="https://stratus.ucar.edu/unidata-netcdf-zarr-testing/test$tag#mode=nczarr,$zext"
file=$fileurl
else
file="test$tag.$zext"
fileurl="file://test$tag.$zext#mode=$zext"
fi
}
checkxfail() {
# determine if this is an xfailtest
isxfail=
for t in ${ALLXFAIL} ; do
if test "x${t}" = "x${x}" ; then isxfail=1; fi
done
}
diffcycle() {
echo ""; echo "*** Test cycle zext=$1"
for x in ${TESTSET} ; do
if test $verbose = 1 ; then echo "*** Testing: ${x}" ; fi
# determine if we need the specflag set
specflag=
headflag=
for s in ${SPECIALTESTS} ; do
if test "x${s}" = "x${x}" ; then specflag="-s"; headflag="-h"; fi
done
# determine if this is an xfailtest
checkxfail ${x}
deletefile ${x}
rm -f ${x}.dmp
fileargs
${NCGEN} -b -k${KFLAG} -o ${fileurl} ${cdl}/${x}.cdl
${NCDUMP} ${headflag} ${specflag} -n ${x} ${fileurl} > ${x}.dmp
# compare the expected (silently if XFAIL)
if test "x$isxfail" = "x1" -a "x$SHOWXFAILS" = "x" ; then
if diff -b -bw ${expected}/${x}.dmp ${x}.dmp >/dev/null 2>&1; then ok=1; else ok=0; fi
else
if diff -b -w ${expected}/${x}.dmp ${x}.dmp ; then ok=1; else ok=0; fi
fi
if test "x$ok" = "x1" ; then
test $verbose = 1 && echo "*** SUCCEED: ${x}"
passcount=`expr $passcount + 1`
elif test "x${isxfail}" = "x1" ; then
echo "*** XFAIL : ${x}"
xfailcount=`expr $xfailcount + 1`
else
echo "*** FAIL: ${x}"
failcount=`expr $failcount + 1`
fi
done
echo "*** Testing ncgen with -k${KFLAG} and zmap=${zext}"
main() {
extfor $1
mkdir ${RESULTSDIR}.${zext}
cd ${RESULTSDIR}.${zext}
diffcycle
cd ..
totalcount=`expr $passcount + $failcount + $xfailcount`
okcount=`expr $passcount + $xfailcount`
echo "*** PASSED: zext=${zext} ${okcount}/${totalcount} ; ${xfailcount} expected failures ; ${failcount} unexpected failures"
}
rm -rf ${RESULTSDIR}
main nz4
if test $failcount -gt 0 ; then exit 1; else exit 0; fi

4
nczarr_test/ut_includes.h
View File

@ -12,10 +12,6 @@
#include <unistd.h>
#endif
#ifdef HAVE_STDARG_H
#include <stdarg.h>
#endif
#include "netcdf.h"
#include "nclist.h"
#include "ncbytes.h"

2
nczarr_test/ut_test.h
View File

@ -6,6 +6,8 @@
#ifndef ZTEST_H
#define ZTEST_H
#include "nclist.h"
typedef struct Dimdef {
char* name;
size64_t size;

125
nczarr_test/zmapio.c
View File

@ -28,6 +28,12 @@ MOP_OBJDUMP=1,
MOP_CLEAR=2
} Mapop;
typedef enum OBJKIND {
OK_NONE=0,
OK_META=1,
OK_CHUNK=2
} OBJKIND;
static struct Mops {
Mapop mapop;
const char* opname;
@ -38,6 +44,24 @@ static struct Mops {
{MOP_NONE,NULL}
};
static struct Type {
const char* typename;
nc_type nctype;
int typesize;
} types[] = {
{"ubyte",NC_UBYTE,1},
{"byte",NC_BYTE,1},
{"ushort",NC_USHORT,2},
{"short",NC_SHORT,2},
{"uint",NC_UINT,4},
{"int",NC_INT,4},
{"uint64",NC_UINT64,8},
{"int64",NC_INT64,8},
{"float",NC_FLOAT,4},
{"double",NC_DOUBLE,8},
{NULL,NC_NAT,0}
};
/* Command line options */
struct Dumpptions {
int debug;
@ -45,15 +69,16 @@ struct Dumpptions {
char* infile;
NCZM_IMPL impl;
char* rootpath;
const struct Type* nctype;
} dumpoptions;
/* Forward */
static int objdump(void);
static NCZM_IMPL implfor(const char* path);
static void printcontent(size64_t len, const char* content,int ismeta);
static void printcontent(size64_t len, const char* content, OBJKIND kind);
static int depthR(NCZMAP* map, char* key, NClist* stack);
static char* rootpathfor(const char* path);
static int ismetakey(const char* key);
static OBJKIND keykind(const char* key);
static void sortlist(NClist* l);
#define NCCHECK(expr) nccheck((expr),__LINE__)
@ -83,6 +108,16 @@ decodeop(const char* name)
return MOP_NONE;
}
static const struct Type*
decodetype(const char* name)
{
struct Type* p = types;
for(;p->typename != NULL;p++) {
if(strcasecmp(p->typename,name)==0) return p;
}
return NULL;
}
int
main(int argc, char** argv)
{
@ -91,10 +126,7 @@ main(int argc, char** argv)
memset((void*)&dumpoptions,0,sizeof(dumpoptions));
/* Set defaults */
dumpoptions.mop = MOP_OBJDUMP;
while ((c = getopt(argc, argv, "dvx:")) != EOF) {
while ((c = getopt(argc, argv, "dvx:t:")) != EOF) {
switch(c) {
case 'd':
dumpoptions.debug = 1;
@ -102,6 +134,10 @@ main(int argc, char** argv)
case 'v':
zmapusage();
goto done;
case 't':
dumpoptions.nctype = decodetype(optarg);
if(dumpoptions.nctype == NULL) zmapusage();
break;
case 'x':
dumpoptions.mop = decodeop(optarg);
if(dumpoptions.mop == MOP_NONE) zmapusage();
@ -112,6 +148,12 @@ main(int argc, char** argv)
}
}
/* Default the kind */
if(dumpoptions.nctype == NULL) {
dumpoptions.nctype = &types[0];
fprintf(stderr,"Default type: %s\n",dumpoptions.nctype->typename);
}
/* get file argument */
argc -= optind;
argv += optind;
@ -133,12 +175,12 @@ main(int argc, char** argv)
zmapusage();
switch (dumpoptions.mop) {
default:
fprintf(stderr,"Default action: objdump\n");
/* fall thru */
case MOP_OBJDUMP:
if((stat = objdump())) goto done;
break;
default:
fprintf(stderr,"Unimplemented action\n");
goto fail;
}
done:
@ -246,7 +288,7 @@ objdump(void)
}
for(depth=0;nclistlength(stack) > 0;depth++) {
size64_t len = 0;
int ismeta = 0;
OBJKIND kind = 0;
int hascontent = 0;
obj = nclistremove(stack,0); /* zero pos is always top of stack */
/* Now print info for this obj key */
@ -267,10 +309,12 @@ objdump(void)
if(hascontent) {
if(len > 0) {
assert(content != NULL);
printf("[%d] %s : (%llu) |",depth,obj,len);
if(ismetakey(obj))
ismeta = 1;
printcontent(len,content,ismeta);
kind = keykind(obj);
if(kind == OK_CHUNK) len /= dumpoptions.nctype->typesize;
printf("[%d] %s : (%llu)",depth,obj,len);
if(kind == OK_CHUNK) printf(" (%s)",dumpoptions.nctype->typename);
printf(" |");
printcontent(len,content,kind);
printf("|\n");
} else {
printf("[%d] %s : (%llu) ||\n",depth,obj,len);
@ -315,14 +359,34 @@ done:
static char hex[16] = "0123456789abcdef";
static void
printcontent(size64_t len, const char* content, int ismeta)
printcontent(size64_t len, const char* content, OBJKIND kind)
{
size64_t i;
unsigned int c0,c1;
for(i=0;i<len;i++) {
if(ismeta) {
/* If kind is chunk, then len is # of values, not # of bytes */
switch(kind) {
case OK_CHUNK:
if(i > 0) printf(", ");
switch(dumpoptions.nctype->nctype) {
case NC_BYTE: printf("%d",((char*)content)[i]); break;
case NC_SHORT: printf("%d",((short*)content)[i]); break;
case NC_INT: printf("%d",((int*)content)[i]); break;
case NC_INT64: printf("%lld",((long long*)content)[i]); break;
case NC_UBYTE: printf("%u",((unsigned char*)content)[i]); break;
case NC_USHORT: printf("%u",((unsigned short*)content)[i]); break;
case NC_UINT: printf("%u",((unsigned int*)content)[i]); break;
case NC_UINT64: printf("%llu",((unsigned long long*)content)[i]); break;
case NC_FLOAT: printf("%f",((float*)content)[i]); break;
case NC_DOUBLE: printf("%lf",((double*)content)[i]); break;
default: abort();
}
break;
case OK_META:
printf("%c",content[i]);
} else {
unsigned int c0,c1;
break;
default:
c1 = (unsigned char)(content[i]);
c0 = c1 & 0xf;
c1 = (c1 >> 4);
@ -333,17 +397,30 @@ printcontent(size64_t len, const char* content, int ismeta)
}
}
static int
ismetakey(const char* key)
static char chunkchars[] = ".0123456789";
static OBJKIND
keykind(const char* key)
{
OBJKIND kind = OK_NONE;
char* suffix = NULL;
int ismeta = 0;
if(nczm_divide_at(key,-1,NULL,&suffix) == NC_NOERR) {
if(suffix && suffix[0] == '/' && suffix[1] == '.')
ismeta = 1;
if(suffix) {
if(suffix[0] != '/')
kind = OK_NONE;
else if(suffix[1] == '.')
kind = OK_META;
else {
char* p = suffix+1;
for(;*p;p++) {
if(strchr(chunkchars,*p) == NULL) break;
}
kind = OK_CHUNK;
}
}
}
nullfree(suffix);
return ismeta;
return kind;
}
/* bubble sort a list of strings */

52
oc2/occurlfunctions.c
View File

@ -49,46 +49,46 @@ ocset_curlflag(OCstate* state, int flag)
switch (flag) {
case CURLOPT_USERPWD: /* Does both user and pwd */
if(state->auth.creds.user != NULL && state->auth.creds.pwd != NULL) {
SETCURLOPT(state, CURLOPT_USERNAME, state->auth.creds.user);
SETCURLOPT(state, CURLOPT_PASSWORD, state->auth.creds.pwd);
if(state->auth->creds.user != NULL && state->auth->creds.pwd != NULL) {
SETCURLOPT(state, CURLOPT_USERNAME, state->auth->creds.user);
SETCURLOPT(state, CURLOPT_PASSWORD, state->auth->creds.pwd);
SETCURLOPT(state, CURLOPT_HTTPAUTH, (OPTARG)CURLAUTH_ANY);
}
break;
case CURLOPT_COOKIEJAR: case CURLOPT_COOKIEFILE:
if(state->auth.curlflags.cookiejar) {
if(state->auth->curlflags.cookiejar) {
/* Assume we will read and write cookies to same place */
SETCURLOPT(state, CURLOPT_COOKIEJAR, state->auth.curlflags.cookiejar);
SETCURLOPT(state, CURLOPT_COOKIEFILE, state->auth.curlflags.cookiejar);
SETCURLOPT(state, CURLOPT_COOKIEJAR, state->auth->curlflags.cookiejar);
SETCURLOPT(state, CURLOPT_COOKIEFILE, state->auth->curlflags.cookiejar);
}
break;
case CURLOPT_NETRC: case CURLOPT_NETRC_FILE:
if(state->auth.curlflags.netrc) {
if(state->auth->curlflags.netrc) {
SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_REQUIRED);
SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth.curlflags.netrc);
SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc);
}
break;
case CURLOPT_VERBOSE:
if(state->auth.curlflags.verbose)
if(state->auth->curlflags.verbose)
SETCURLOPT(state, CURLOPT_VERBOSE, (OPTARG)1L);
break;
case CURLOPT_TIMEOUT:
if(state->auth.curlflags.timeout)
SETCURLOPT(state, CURLOPT_TIMEOUT, (OPTARG)((long)state->auth.curlflags.timeout));
if(state->auth->curlflags.timeout)
SETCURLOPT(state, CURLOPT_TIMEOUT, (OPTARG)((long)state->auth->curlflags.timeout));
break;
case CURLOPT_CONNECTTIMEOUT:
if(state->auth.curlflags.connecttimeout)
SETCURLOPT(state, CURLOPT_CONNECTTIMEOUT, (OPTARG)((long)state->auth.curlflags.connecttimeout));
if(state->auth->curlflags.connecttimeout)
SETCURLOPT(state, CURLOPT_CONNECTTIMEOUT, (OPTARG)((long)state->auth->curlflags.connecttimeout));
break;
case CURLOPT_USERAGENT:
if(state->auth.curlflags.useragent)
SETCURLOPT(state, CURLOPT_USERAGENT, state->auth.curlflags.useragent);
if(state->auth->curlflags.useragent)
SETCURLOPT(state, CURLOPT_USERAGENT, state->auth->curlflags.useragent);
break;
case CURLOPT_FOLLOWLOCATION:
@ -105,19 +105,19 @@ ocset_curlflag(OCstate* state, int flag)
case CURLOPT_ENCODING:
#ifdef CURLOPT_ENCODING
if(state->auth.curlflags.compress) {
if(state->auth->curlflags.compress) {
SETCURLOPT(state, CURLOPT_ENCODING,"deflate, gzip");
}
#endif
break;
case CURLOPT_PROXY:
if(state->auth.proxy.host != NULL) {
SETCURLOPT(state, CURLOPT_PROXY, state->auth.proxy.host);
SETCURLOPT(state, CURLOPT_PROXYPORT, (OPTARG)(long)state->auth.proxy.port);
if(state->auth.proxy.user != NULL && state->auth.proxy.pwd != NULL) {
SETCURLOPT(state, CURLOPT_PROXYUSERNAME, state->auth.proxy.user);
SETCURLOPT(state, CURLOPT_PROXYPASSWORD, state->auth.proxy.pwd);
if(state->auth->proxy.host != NULL) {
SETCURLOPT(state, CURLOPT_PROXY, state->auth->proxy.host);
SETCURLOPT(state, CURLOPT_PROXYPORT, (OPTARG)(long)state->auth->proxy.port);
if(state->auth->proxy.user != NULL && state->auth->proxy.pwd != NULL) {
SETCURLOPT(state, CURLOPT_PROXYUSERNAME, state->auth->proxy.user);
SETCURLOPT(state, CURLOPT_PROXYPASSWORD, state->auth->proxy.pwd);
#ifdef CURLOPT_PROXYAUTH
SETCURLOPT(state, CURLOPT_PROXYAUTH, (long)CURLAUTH_ANY);
#endif
@ -129,7 +129,7 @@ ocset_curlflag(OCstate* state, int flag)
case CURLOPT_SSLCERT: case CURLOPT_SSLKEY:
case CURLOPT_SSL_VERIFYPEER: case CURLOPT_SSL_VERIFYHOST:
{
struct ssl* ssl = &state->auth.ssl;
struct ssl* ssl = &state->auth->ssl;
/* VERIFYPEER == 0 => VERIFYHOST == 0 */
/* We need to have 2 states: default and a set value */
/* So -1 => default >= 0 => use value */
@ -228,7 +228,7 @@ ocset_flags_perlink(OCstate* state)
void
oc_curl_debug(OCstate* state)
{
state->auth.curlflags.verbose = 1;
state->auth->curlflags.verbose = 1;
ocset_curlflag(state,CURLOPT_VERBOSE);
ocset_curlflag(state,CURLOPT_ERRORBUFFER);
}
@ -239,7 +239,7 @@ int
ocrc_netrc_required(OCstate* state)
{
char* netrcfile = NC_rclookup(NETRCFILETAG,state->uri->uri);
return (netrcfile != NULL || state->auth.curlflags.netrc != NULL ? 0 : 1);
return (netrcfile != NULL || state->auth->curlflags.netrc != NULL ? 0 : 1);
}
void
@ -257,6 +257,6 @@ oc_curl_protocols(OCstate* state)
curldata = curl_version_info(CURLVERSION_NOW);
for(proto=curldata->protocols;*proto;proto++) {
if(strcmp("http",*proto)==0)
state->auth.curlflags.proto_https=1;
state->auth->curlflags.proto_https=1;
}
}

42
oc2/ocinternal.c
View File

@ -376,7 +376,7 @@ occlose(OCstate* state)
ocfree(state->error.code);
ocfree(state->error.message);
if(state->curl != NULL) occurlclose(state->curl);
NC_authclear(&state->auth);
NC_authfree(state->auth);
ocfree(state);
}
@ -528,11 +528,11 @@ ocset_curlproperties(OCstate* state)
OCerror stat = OC_NOERR;
NCRCglobalstate* globalstate = ncrc_getglobalstate();
if(state->auth.curlflags.useragent == NULL) {
if(state->auth->curlflags.useragent == NULL) {
size_t len = strlen(DFALTUSERAGENT) + strlen(VERSION) + 1;
char* agent = (char*)malloc(len+1);
if(occopycat(agent,len,2,DFALTUSERAGENT,VERSION))
state->auth.curlflags.useragent = agent;
state->auth->curlflags.useragent = agent;
else
free(agent);
}
@ -540,13 +540,13 @@ ocset_curlproperties(OCstate* state)
/* Some servers (e.g. thredds and columbia) appear to require a place
to put cookies in order for some security functions to work
*/
if(state->auth.curlflags.cookiejar != NULL
&& strlen(state->auth.curlflags.cookiejar) == 0) {
free(state->auth.curlflags.cookiejar);
state->auth.curlflags.cookiejar = NULL;
if(state->auth->curlflags.cookiejar != NULL
&& strlen(state->auth->curlflags.cookiejar) == 0) {
free(state->auth->curlflags.cookiejar);
state->auth->curlflags.cookiejar = NULL;
}
if(state->auth.curlflags.cookiejar == NULL) {
if(state->auth->curlflags.cookiejar == NULL) {
/* If no cookie file was defined, define a default */
int stat = NC_NOERR;
char* path = NULL;
@ -563,20 +563,20 @@ ocset_curlproperties(OCstate* state)
occopycat(path,len,3,globalstate->tempdir,"/","occookies");
tmppath = NC_mktmp(path);
free(path);
state->auth.curlflags.cookiejar = tmppath;
state->auth.curlflags.cookiejarcreated = 1;
state->auth->curlflags.cookiejar = tmppath;
state->auth->curlflags.cookiejarcreated = 1;
if(stat != OC_NOERR && errno != EEXIST) {
fprintf(stderr,"Cannot create cookie file\n");
goto fail;
}
errno = 0;
}
OCASSERT(state->auth.curlflags.cookiejar != NULL);
OCASSERT(state->auth->curlflags.cookiejar != NULL);
/* Make sure the cookie jar exists and can be read and written */
{
FILE* f = NULL;
char* fname = state->auth.curlflags.cookiejar;
char* fname = state->auth->curlflags.cookiejar;
/* See if the file exists already */
f = NCfopen(fname,"r");
if(f == NULL) {
@ -603,7 +603,7 @@ ocset_curlproperties(OCstate* state)
if(ocrc_netrc_required(state)) {
/* WARNING: it appears that a user+pwd was specified specifically, then
the netrc file will be completely disabled. */
if(state->auth.creds.userpwd != NULL) {
if(state->auth->creds.userpwd != NULL) {
nclog(NCLOGWARN,"The rc file specifies both netrc and user+pwd; this will cause curl to ignore the netrc file");
}
stat = oc_build_netrc(state);
@ -673,10 +673,10 @@ OCerror
ocset_useragent(OCstate* state, const char* agent)
{
OCerror stat = OC_NOERR;
if(state->auth.curlflags.useragent != NULL)
free(state->auth.curlflags.useragent);
state->auth.curlflags.useragent = strdup(agent);
if(state->auth.curlflags.useragent == NULL)
if(state->auth->curlflags.useragent != NULL)
free(state->auth->curlflags.useragent);
state->auth->curlflags.useragent = strdup(agent);
if(state->auth->curlflags.useragent == NULL)
return OCTHROW(OC_ENOMEM);
stat = ocset_curlflag(state,CURLOPT_USERAGENT);
return stat;
@ -686,10 +686,10 @@ OCerror
ocset_netrc(OCstate* state, const char* path)
{
OCerror stat = OC_NOERR;
if(state->auth.curlflags.netrc != NULL)
free(state->auth.curlflags.netrc);
state->auth.curlflags.netrc = strdup(path);
if(state->auth.curlflags.netrc == NULL)
if(state->auth->curlflags.netrc != NULL)
free(state->auth->curlflags.netrc);
state->auth->curlflags.netrc = strdup(path);
if(state->auth->curlflags.netrc == NULL)
return OCTHROW(OC_ENOMEM);
stat = ocset_curlflag(state,CURLOPT_NETRC);
return stat;

2
oc2/ocinternal.h
View File

@ -174,7 +174,7 @@ struct OCstate {
CURL* curl; /* curl handle*/
char curlerror[CURL_ERROR_SIZE];
void* usercurldata;
NCauth auth; /* curl auth data */
NCauth* auth; /* curl auth data */
long ddslastmodified;
long datalastmodified;
long curlbuffersize;

2
unit_test/CMakeLists.txt
View File

@ -10,7 +10,7 @@
# Some unit testing
SET(UNIT_TESTS tst_nclist test_ncuri test_pathcvt)
SET(UNIT_TESTS tst_nclist test_ncuri test_pathcvt tst_exhash tst_xcache)
IF(ENABLE_NETCDF_4)
SET(UNIT_TESTS ${UNIT_TESTS} tst_nc4internal)

4
unit_test/Makefile.am
View File

@ -18,8 +18,8 @@ if USE_NETCDF4
NC4_TESTS = tst_nc4internal
endif # USE_NETCDF4
check_PROGRAMS = tst_nclist test_ncuri test_pathcvt $(NC4_TESTS)
TESTS = tst_nclist test_ncuri test_pathcvt $(NC4_TESTS)
check_PROGRAMS = tst_nclist test_ncuri test_pathcvt tst_exhash tst_xcache $(NC4_TESTS)
TESTS = ${check_PROGRAMS}
EXTRA_DIST = CMakeLists.txt

292
unit_test/tst_exhash.c Normal file
View File

@ -0,0 +1,292 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
/**
Test the Extendible Hash Implementation of ncexhash
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef HAVE_TIME_H
#include <time.h>
#endif
#ifdef HAVE_SYS_TIME_H
#include <sys/time.h>
#endif
#ifdef HAVE_SYS_RESOURCE_H
#include <sys/resource.h>
#endif
#include <assert.h>
#include "netcdf.h"
#include "ncexhash.h"
#include "nccrc.h"
#define LEAFN 16
#define HMODE 3
#undef VERBOSE
#if NCEXHASHKEYBITS == 64
#define CRC NC_crc64
#else
#define CRC NC_crc32
#endif
static unsigned N[] = {1000, 10000, 100000, 1000000, 0};
#define CHECK(expr) check((expr),__LINE__)
void check(int stat, int line)
{
if(stat) {
fprintf(stderr,"%d: (%d)%s\n",line,stat,nc_strerror(stat));
fflush(stderr);
exit(1);
}
}
static ncexhashkey_t
hkeyfor(unsigned key)
{
ncexhashkey_t hashkey = 0;
int i;
switch (HMODE) {
case 1:
hashkey = ncexhashkey((char*)&key,sizeof(key));
break;
case 2:
for(i=0;i<NCEXHASHKEYBITS;i++) {
hashkey |= (key & 0x1) << (31-i);
key = key >> 1;
}
break;
case 3: /* Convert key to a random number using crc */
hashkey = CRC(0,(void*)&key,sizeof(key));
break;
default:
abort();
}
return hashkey;
}
static void
reporttime(unsigned nelems, long* times, const char* tag)
{
NC_UNUSED(nelems);
NC_UNUSED(times);
NC_UNUSED(tag);
#if 0
double delta;
double deltasec;
delta = (double)(times[1] - times[0]);
deltasec = delta / 1000000.0;
fprintf(stderr,"\t%s:\t%5.1lf sec",tag,deltasec);
fprintf(stderr," avg=%5.1lf usec\n",delta/nelems);
#endif
}
static void
xreporttime(unsigned nelems, struct timespec* times, const char* tag)
{
double delta;
double deltasec;
long long nsec[2];
nsec[0] = times[0].tv_nsec+(1000000000 * times[0].tv_sec);
nsec[1] = times[1].tv_nsec+(1000000000 * times[1].tv_sec);
delta = (double)(nsec[1] - nsec[0]);
deltasec = delta / 1000000000.0;
fprintf(stderr,"\t%s:\t%8.6lf sec",tag,deltasec);
fprintf(stderr," avg=%5.2lf nsec\n",delta/nelems);
}
int
main(int argc, char** argv)
{
int stat = NC_NOERR;
NCexhashmap* map = NULL;
unsigned key;
struct rusage ru;
clockid_t clk_id = CLOCK_MONOTONIC;
struct timespec xinserttime[2];
struct timespec xreadtime[2];
struct timespec xremtime[2];
long inserttime[2], readtime[2], remtime[2]; /* elapsed time in microseconds */
unsigned* np;
uintptr_t data;
ncexhashkey_t hashkey;
fprintf(stderr,"insert:\n");
#ifdef VERBOSE
{
long microcvt, seccvt;
struct timespec res;
if(clock_getres(clk_id, &res) < 0)
abort();
fprintf(stderr,"xxx: tv_sec=%lld tv_nsec=%ld\n",(long long)res.tv_sec,res.tv_nsec);
microcvt = res.tv_nsec;
seccvt = microcvt * 1000000;
fprintf(stderr,"xxx: seccvt=%lld microcvt=%lld\n",
(long long)seccvt,(long long)microcvt);
}
#endif
for(np=N;*np;np++) {
getrusage(RUSAGE_SELF, &ru);
inserttime[0] = (1000000*(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec)
+ ru.ru_utime.tv_usec + ru.ru_stime.tv_usec);
clock_gettime(clk_id,&xinserttime[0]);
map=ncexhashnew(LEAFN);
if(map == NULL) CHECK(NC_EINVAL);
#ifdef VERBOSE
fprintf(stderr,"new:\n"); ncexhashprint(map);
#endif
for(key=0;key<*np;key++) {
ncexhashkey_t hashkey = hkeyfor(key);
#ifdef VERBOSE
fprintf(stderr,"insert[%08llx|%s->%u]:\n",hashkey,ncexbinstr(hashkey,NCEXHASHKEYBITS),key);
#endif
CHECK(ncexhashput(map,hashkey,(uintptr_t)key));
}
#ifdef VERBOSE
fprintf(stderr,"insert.after:");ncexhashprint(map);
#endif
clock_gettime(clk_id,&xinserttime[1]);
getrusage(RUSAGE_SELF, &ru);
inserttime[1] = (1000000*(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec)
+ ru.ru_utime.tv_usec + ru.ru_stime.tv_usec);
fprintf(stderr,"read:\n");
getrusage(RUSAGE_SELF, &ru);
readtime[0] = (1000000*(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec)
+ ru.ru_utime.tv_usec + ru.ru_stime.tv_usec);
clock_gettime(clk_id,&xreadtime[0]);
for(key=0;key<*np;key++) {
uintptr_t data = 0;
ncexhashkey_t hashkey = hkeyfor(key);
CHECK(ncexhashget(map,hashkey,&data));
#ifdef VERBOSE
fprintf(stderr,"read[%08llx|%s->%u]:\n",hashkey,ncexbinstr(hashkey,NCEXHASHKEYBITS),(unsigned)data);
#endif
if(data != key) fprintf(stderr,"\tMISMATCH\n");
}
clock_gettime(clk_id,&xreadtime[1]);
getrusage(RUSAGE_SELF, &ru);
readtime[1] = (1000000*(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec)
+ ru.ru_utime.tv_usec + ru.ru_stime.tv_usec);
fprintf(stderr,"statistics:\n"); ncexhashprintstats(map);
fprintf(stderr,"times: N=%u\n",*np);
reporttime(*np, inserttime, "insert");
reporttime(*np, readtime, "read");
xreporttime(*np, xinserttime, "insert");
xreporttime(*np, xreadtime, "read");
/* Test iterator */
{
unsigned char* found = NULL;
int nmatches = 0;
fprintf(stderr,"iterating: %u\n",*np);
if((found = (unsigned char*)calloc(1,*np))==NULL) CHECK(NC_ENOMEM);
for(;;) {
if((stat = ncexhashiterate(map, &hashkey, &data)) == NC_EINVAL) CHECK(stat);
if(stat == NC_ERANGE) break;
assert(data >= 0 && data < *np);
if(found[data] != 0)
fprintf(stderr,"iterator duplicate: data=%lu\n",(unsigned long)data);
found[data] = 1;
nmatches++;
}
for(key=0;key<*np;key++) {
if(found[key] == 0) fprintf(stderr,"iterator missing: %u\n",key);
}
fprintf(stderr,"iterating: |keys|=%u |matches|=%u\n",*np,nmatches);
if(found) free(found);
}
/* Test setdata */
{
fprintf(stderr,"setdata: %u\n",*np);
for(key=0;key<*np;key++) {
ncexhashkey_t hashkey = hkeyfor(key);
uintptr_t newdata = (uintptr_t)(key+(*np));
stat = ncexhashsetdata(map, hashkey, newdata, &data);
assert(stat == NC_NOERR);
assert(data == key);
}
for(;;) {
ncexhashkey_t rehash;
if((stat = ncexhashiterate(map, &hashkey, &data)) == NC_EINVAL) CHECK(stat);
if(stat == NC_ERANGE) break;
rehash = hkeyfor(data - (*np));
assert(hashkey == rehash);
}
}
{
int leaflen = 0;
int depth = 0;
int nactive = 0;
int uid = 0;
int walking = 0;
CHECK(ncexhashinqmap(map,&leaflen,&depth,&nactive,&uid,&walking));
fprintf(stderr,"map parameters: leaflen=%d depth=%d nactive=%d uid=%d walking=%d\n",
leaflen,depth,nactive,uid,walking);
}
/* Test removal */
{
int nactive = 0;
fprintf(stderr,"removing: %u\n",*np);
getrusage(RUSAGE_SELF, &ru);
remtime[0] = (1000000*(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec)
+ ru.ru_utime.tv_usec + ru.ru_stime.tv_usec);
clock_gettime(clk_id,&xremtime[0]);
for(key=0;key<*np;key++) {
ncexhashkey_t hashkey = hkeyfor(key);
switch (stat = ncexhashremove(map, hashkey, &data)) {
case NC_NOERR:
break;
case NC_ENOTFOUND:
fprintf(stderr,"remove: missing: key=%u hashkey=%llu\n",key,hashkey);
break;
default: CHECK(stat);
}
}
CHECK(ncexhashinqmap(map,NULL,NULL,&nactive,NULL,NULL));
fprintf(stderr,"removal: final nactive=%d\n",nactive);
clock_gettime(clk_id,&xremtime[1]);
getrusage(RUSAGE_SELF, &ru);
remtime[1] = (1000000*(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec)
+ ru.ru_utime.tv_usec + ru.ru_stime.tv_usec);
reporttime(*np, remtime, "removal");
xreporttime(*np, xremtime, "removal");
}
ncexhashmapfree(map);
}
return 0;
}

246
unit_test/tst_xcache.c Normal file
View File

@ -0,0 +1,246 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
/**
Test the NCxcache data structure
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef HAVE_TIME_H
#include <time.h>
#endif
#ifdef HAVE_SYS_TIME_H
#include <sys/time.h>
#endif
#ifdef HAVE_SYS_RESOURCE_H
#include <sys/resource.h>
#endif
#include <assert.h>
#include "netcdf.h"
#include "ncexhash.h"
#include "ncxcache.h"
#define DEBUG 0
static struct rusage ru;
static clockid_t clk_id = CLOCK_MONOTONIC;
static struct timespec xinserttime[2];
static long inserttime[2];
static struct timespec xreadtime[2];
static long readtime[2];
#if 0
static struct timespec xremtime[2];
static long readtime[2];
static long remtime[2];
#endif
#define CHECK(expr) check((expr),__LINE__)
void check(int stat, int line)
{
if(stat) {
fprintf(stderr,"%d: (%d)%s\n",line,stat,nc_strerror(stat));
fflush(stderr);
exit(1);
}
}
#if 0
static void
reporttime(unsigned nelems, long* times, const char* tag)
{
NC_UNUSED(nelems);
NC_UNUSED(times);
NC_UNUSED(tag);
#if 0
double delta;
double deltasec;
delta = (double)(times[1] - times[0]);
deltasec = delta / 1000000.0;
fprintf(stderr,"\t%s:\t%5.1lf sec",tag,deltasec);
fprintf(stderr," avg=%5.1lf usec\n",delta/nelems);
#endif
}
#endif
static void
xreporttime(unsigned nelems, struct timespec* times, const char* tag)
{
double delta;
double deltasec;
long long nsec[2];
nsec[0] = times[0].tv_nsec+(1000000000 * times[0].tv_sec);
nsec[1] = times[1].tv_nsec+(1000000000 * times[1].tv_sec);
delta = (double)(nsec[1] - nsec[0]);
deltasec = delta / 1000000000.0;
fprintf(stderr,"\t%s:\t%8.6lf sec",tag,deltasec);
fprintf(stderr," avg=%5.2lf nsec\n",delta/nelems);
}
/*
Test set 1:
- insert string with obj
- lookup by various methods
- remove
- verify remove
*/
#define MAXSTRLEN 25
#define DEFAULTSEED 1
typedef struct NC_OBJ {
int sort;
char* name;
size_t id;
} NC_OBJ;
typedef struct NCXSTR {
void* next;
void* prev;
void* ptr;
char* string;
} NCXSTR;
//static int N[] = {10, 100, 1000, 0};
static int N[] = {4,0};
/* Generate random ascii strings */
static NCXSTR* strings = NULL;
static void
generatestrings(int n, unsigned seed)
{
int i,k;
long rnd;
int len;
char* s = NULL;
srandom(seed);
strings = (NCXSTR*)calloc(sizeof(NCXSTR),(n+1));
if(strings == NULL) abort();
for(i=0;i<n;i++) {
/* Generate one random string */
if((s = (char*)malloc(1+MAXSTRLEN))==NULL) abort();
rnd = random();
len = rnd % MAXSTRLEN;
/* generate the characters */
for(k=0;k<len;k++) {
do {rnd = random() % 127;} while(rnd < ' ');
assert(rnd > ' ' && rnd < 127);
s[k] = (char)rnd;
}
s[len] = '\0';
strings[i].string = s;
#if DEBUG >= 3
fprintf(stderr,"strings[%d] = |%s|\n",i,strings[i].string);
#endif
}
}
static void
freestrings(void)
{
if(strings) {
NCXSTR* p = strings;
for(;p->string;p++) {nullfree(p->string);}
nullfree(strings);
strings = NULL;
}
}
int
main(int argc, char** argv)
{
int stat = NC_NOERR;
NCxcache* cache = NULL;
ncexhashkey_t hkey;
int* np = NULL;
void* content = NULL;
for(np=N;*np;np++) {
int ns,i;
ns = *np;
generatestrings(ns,DEFAULTSEED);
fprintf(stderr,"insert:\n");
if((stat = ncxcachenew(2,&cache))) goto done;
getrusage(RUSAGE_SELF, &ru);
inserttime[0] = (1000000*(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec)
+ ru.ru_utime.tv_usec + ru.ru_stime.tv_usec);
clock_gettime(clk_id,&xinserttime[0]);
for(i=0;i<ns;i++) {
hkey = ncexhashkey(strings[i].string,strlen(strings[i].string));
if((stat=ncxcacheinsert(cache,hkey,&strings[i]))) goto done;
}
assert(ncxcachecount(cache) == ns);
clock_gettime(clk_id,&xinserttime[1]);
getrusage(RUSAGE_SELF, &ru);
inserttime[1] = (1000000*(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec)
+ ru.ru_utime.tv_usec + ru.ru_stime.tv_usec);
#if DEBUG > 0
ncxcacheprint(cache);
#endif
xreporttime(ns, xinserttime, "insert");
/* Try to touch and extract all the entries */
fprintf(stderr,"read:\n");
getrusage(RUSAGE_SELF, &ru);
readtime[0] = (1000000*(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec)
+ ru.ru_utime.tv_usec + ru.ru_stime.tv_usec);
clock_gettime(clk_id,&xreadtime[0]);
for(i=0;i<ns;i++) {
void* top = NULL;
hkey = ncexhashkey(strings[i].string,strlen(strings[i].string));
if((stat=ncxcachelookup(cache,hkey,&content))) goto done;
if((stat=ncxcachetouch(cache,hkey))) goto done;
top = ncxcachefirst(cache);
if(top != content) {stat = NC_EINTERNAL; goto done;}
}
clock_gettime(clk_id,&xreadtime[1]);
getrusage(RUSAGE_SELF, &ru);
readtime[1] = (1000000*(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec)
+ ru.ru_utime.tv_usec + ru.ru_stime.tv_usec);
#if DEBUG > 0
ncxcacheprint(cache);
#endif
xreporttime(ns, xreadtime, "read");
for(i=0;i<ns;i++) {
void* top = NULL;
hkey = ncexhashkey(strings[i].string,strlen(strings[i].string));
if((stat=ncxcachetouch(cache,hkey))) goto done;
top = ncxcachefirst(cache);
if(top != &strings[i])
fprintf(stderr,"touch failure: top=%p strings[%d]=%p\n",top,i,&strings[i]);
}
fprintf(stderr,"touch: passed\n");
freestrings();
ncxcachefree(cache);
}
done:
if(stat) fprintf(stderr,"***fail: %s\n",nc_strerror(stat));
return 0;
}