mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-12-21 08:39:46 +08:00
3ffe7be446
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
72 lines
2.9 KiB
C
72 lines
2.9 KiB
C
/*********************************************************************
|
|
* Copyright 2018, UCAR/Unidata
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
*********************************************************************/
|
|
|
|
#ifndef ZCACHE_H
|
|
#define ZCACHE_H
|
|
|
|
/* This holds all the fields
|
|
to support either impl of cache
|
|
*/
|
|
|
|
struct NCxcache;
|
|
|
|
/* Note in the following: the term "real"
|
|
refers to the unfiltered/uncompressed data
|
|
The term filtered refers to the result of running
|
|
the real data through the filter chain. Note that the
|
|
sizeof the filtered data might be larger than the size of
|
|
the real data.
|
|
The term "raw" is used to refer to the data on disk and it may either
|
|
be real or filtered.
|
|
*/
|
|
|
|
typedef struct NCZCacheEntry {
|
|
struct List {void* next; void* prev; void* unused;} list;
|
|
int modified;
|
|
size64_t indices[NC_MAX_VAR_DIMS];
|
|
struct ChunkKey {
|
|
char* varkey; /* key to the containing variable */
|
|
char* chunkkey; /* name of the chunk */
|
|
} key;
|
|
size64_t hashkey;
|
|
int isfiltered; /* 1=>data contains filtered data else real data */
|
|
size64_t size; /* |data| */
|
|
void* data; /* contains either filtered or real data */
|
|
} NCZCacheEntry;
|
|
|
|
typedef struct NCZChunkCache {
|
|
int valid; /* 0 => following fields need to be re-calculated */
|
|
NC_VAR_INFO_T* var; /* backlink */
|
|
size64_t ndims; /* true ndims == var->ndims + scalar */
|
|
size64_t chunksize; /* for real data */
|
|
size64_t chunkcount; /* cross product of chunksizes */
|
|
void* fillchunk; /* enough fillvalues to fill a real chunk */
|
|
size_t maxentries; /* Max number of entries allowed; maxsize can override */
|
|
size_t maxsize; /* Maximum space used by cache; 0 => nolimit */
|
|
size_t used; /* How much total space is being used */
|
|
NClist* mru; /* NClist<NCZCacheEntry> all cache entries in mru order */
|
|
struct NCxcache* xcache;
|
|
char dimension_separator;
|
|
} NCZChunkCache;
|
|
|
|
/**************************************************/
|
|
|
|
#define FILTERED(cache) (nclistlength((NClist*)(cache)->var->filters))
|
|
|
|
extern int NCZ_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption);
|
|
extern int NCZ_adjust_var_cache(NC_VAR_INFO_T *var);
|
|
extern int NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t, char dimsep, NCZChunkCache** cachep);
|
|
extern void NCZ_free_chunk_cache(NCZChunkCache* cache);
|
|
extern int NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap);
|
|
extern int NCZ_flush_chunk_cache(NCZChunkCache* cache);
|
|
extern size64_t NCZ_cache_entrysize(NCZChunkCache* cache);
|
|
extern NCZCacheEntry* NCZ_cache_entry(NCZChunkCache* cache, const size64_t* indices);
|
|
extern size64_t NCZ_cache_size(NCZChunkCache* cache);
|
|
extern int NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, struct ChunkKey* key);
|
|
extern int NCZ_ensure_fill_chunk(NCZChunkCache* cache);
|
|
extern int NCZ_reclaim_fill_chunk(NCZChunkCache* cache);
|
|
|
|
#endif /*ZCACHE_H*/
|