netcdf-c/libsrc/nc3dispatch.c

517 lines
12 KiB
C
Raw Normal View History

2010-06-03 21:24:43 +08:00
/*********************************************************************
Copyright 2018, UCAR/Unidata See netcdf/COPYRIGHT file for
2010-06-03 21:24:43 +08:00
copying and redistribution conditions.
$Id: nc3dispatch.c,v 2.8 2010/05/26 11:11:26 ed Exp $
*********************************************************************/
#include "config.h"
#include <stdlib.h>
#include <string.h>
#include "netcdf.h"
#include "nc3internal.h"
2010-06-03 21:24:43 +08:00
#include "nc3dispatch.h"
#ifndef NC_CONTIGUOUS
#define NC_CONTIGUOUS 1
#endif
#ifndef NC_ENOTNC4
#define NC_ENOTNC4 (-111)
#endif
#ifndef NC_ENOGRP
#define NC_ENOGRP (-125)
#endif
#ifndef NC_STRING
#define NC_STRING (12)
#endif
static int NC3_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *ndimsp, int *dimidsp, int *nattsp,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
unsigned int* idp, size_t* nparamsp, unsigned int* params
);
2010-06-03 21:24:43 +08:00
2013-03-16 04:31:07 +08:00
static int NC3_var_par_access(int,int,int);
2010-06-03 21:24:43 +08:00
static int NC3_show_metadata(int);
static int NC3_inq_unlimdims(int,int*,int*);
static int NC3_inq_ncid(int,const char*,int*);
static int NC3_inq_grps(int,int*,int*);
static int NC3_inq_grpname(int,char*);
static int NC3_inq_grpname_full(int,size_t*,char*);
static int NC3_inq_grp_parent(int,int*);
static int NC3_inq_grp_full_ncid(int,const char*,int*);
static int NC3_inq_varids(int,int* nvars,int*);
static int NC3_inq_dimids(int,int* ndims,int*,int);
static int NC3_inq_typeids(int,int* ntypes,int*);
static int NC3_inq_type_equal(int,nc_type,int,nc_type,int*);
static int NC3_def_grp(int,const char*,int*);
static int NC3_rename_grp(int,const char*);
2010-06-03 21:24:43 +08:00
static int NC3_inq_user_type(int,nc_type,char*,size_t*,nc_type*,size_t*,int*);
static int NC3_inq_typeid(int,const char*,nc_type*);
static int NC3_def_compound(int,size_t,const char*,nc_type*);
static int NC3_insert_compound(int,nc_type,const char*,size_t,nc_type);
static int NC3_insert_array_compound(int,nc_type,const char*,size_t,nc_type,int,const int*);
static int NC3_inq_compound_field(int,nc_type,int,char*,size_t*,nc_type*,int*,int*);
static int NC3_inq_compound_fieldindex(int,nc_type,const char*,int*);
static int NC3_def_vlen(int,const char*,nc_type base_typeid,nc_type*);
static int NC3_put_vlen_element(int,int,void*,size_t,const void*);
static int NC3_get_vlen_element(int,int,const void*,size_t*,void*);
static int NC3_def_enum(int,nc_type,const char*,nc_type*);
static int NC3_insert_enum(int,nc_type,const char*,const void*);
static int NC3_inq_enum_member(int,nc_type,int,char*,void*);
static int NC3_inq_enum_ident(int,nc_type,long long,char*);
static int NC3_def_opaque(int,size_t,const char*,nc_type*);
static int NC3_def_var_deflate(int,int,int,int,int);
static int NC3_def_var_fletcher32(int,int,int);
static int NC3_def_var_chunking(int,int,int,const size_t*);
static int NC3_def_var_endian(int,int,int);
static int NC3_def_var_filter(int, int, unsigned int, size_t, const unsigned int*);
2010-06-03 21:24:43 +08:00
static int NC3_set_var_chunk_cache(int,int,size_t,size_t,float);
static int NC3_get_var_chunk_cache(int,int,size_t*,size_t*,float*);
static const NC_Dispatch NC3_dispatcher = {
2010-06-03 21:24:43 +08:00
2015-08-16 06:26:35 +08:00
NC_FORMATX_NC3,
2020-02-28 05:06:45 +08:00
NC_DISPATCH_VERSION,
2010-06-03 21:24:43 +08:00
NC3_create,
NC3_open,
NC3_redef,
NC3__enddef,
NC3_sync,
NC3_abort,
NC3_close,
NC3_set_fill,
NC3_inq_format,
NC3_inq_format_extended,
2010-06-03 21:24:43 +08:00
NC3_inq,
NC3_inq_type,
NC3_def_dim,
NC3_inq_dimid,
NC3_inq_dim,
NC3_inq_unlimdim,
NC3_rename_dim,
NC3_inq_att,
NC3_inq_attid,
NC3_inq_attname,
NC3_rename_att,
NC3_del_att,
NC3_get_att,
NC3_put_att,
NC3_def_var,
NC3_inq_varid,
NC3_rename_var,
NC3_get_vara,
NC3_put_vara,
NCDEFAULT_get_vars,
NCDEFAULT_put_vars,
NCDEFAULT_get_varm,
NCDEFAULT_put_varm,
2010-06-03 21:24:43 +08:00
NC3_inq_var_all,
2013-03-16 04:31:07 +08:00
NC3_var_par_access,
2017-12-21 10:53:30 +08:00
NC3_def_var_fill,
2013-03-16 04:31:07 +08:00
2010-06-03 21:24:43 +08:00
NC3_show_metadata,
NC3_inq_unlimdims,
NC3_inq_ncid,
NC3_inq_grps,
NC3_inq_grpname,
NC3_inq_grpname_full,
NC3_inq_grp_parent,
NC3_inq_grp_full_ncid,
NC3_inq_varids,
NC3_inq_dimids,
NC3_inq_typeids,
NC3_inq_type_equal,
NC3_def_grp,
NC3_rename_grp,
2010-06-03 21:24:43 +08:00
NC3_inq_user_type,
NC3_inq_typeid,
NC3_def_compound,
NC3_insert_compound,
NC3_insert_array_compound,
NC3_inq_compound_field,
NC3_inq_compound_fieldindex,
NC3_def_vlen,
NC3_put_vlen_element,
NC3_get_vlen_element,
NC3_def_enum,
NC3_insert_enum,
NC3_inq_enum_member,
NC3_inq_enum_ident,
NC3_def_opaque,
NC3_def_var_deflate,
NC3_def_var_fletcher32,
NC3_def_var_chunking,
NC3_def_var_endian,
NC3_def_var_filter,
2010-06-03 21:24:43 +08:00
NC3_set_var_chunk_cache,
NC3_get_var_chunk_cache,
Mostly revert the filter code to reduce its complexity of use. re: https://github.com/Unidata/netcdf-c/issues/1836 Revert the internal filter code to simplify it. From the user's point of view, the only visible changes should be: 1. The functions that convert text to filter specs have had their signature reverted and have been moved to netcdf_aux.h 2. Some filter API functions now return NC_ENOFILTER when inquiry is made about some filter. Internally,the dispatch table has been modified to get rid of the filter_actions entry and associated complex structures. It has been replaced with inq_var_filter_ids and inq_var_filter_info entries and the dispatch table version has been bumped to 3. Corresponding NOOP and NOTNC4 functions were added to libdispatch/dnotnc4.c. Also, the filter_action entries in dispatch tables were replaced for all dispatch code bases (HDF5, DAP2, etc). This should only impact UDF users. In the process, it became clear that the form of the filters field in NC_VAR_INFO_T was format dependent, so I converted it to be of type void* and pushed its management into the various dispatch code bases. Specifically libhdf5 and libnczarr now manage the filters field in their own way. The auxilliary functions for parsing textual filter specifications were moved to netcdf_aux.h and were renamed to the following: * ncaux_h5filterspec_parse * ncaux_h5filterspec_parselist * ncaux_h5filterspec_free * ncaux_h5filter_fix8 Misc. Other Changes: 1. Document NUG/filters.md updated to reflect the changes above. 2. All the old data types (structs and enums) used by filter_actions actions were deleted. The exception is the NC_H5_Filterspec because it is needed by ncaux_h5filterspec_parselist. 3. Clientside filters were removed -- another enhancement for which no-one ever asked. 4. The ability to remove filters was itself removed. 5. Some functionality needed by nczarr was moved from libhdf5 to libsrc4 e.g. nc4_find_default_chunksizes 6. All the filterx code was removed 7. ncfilter.h and nc4filter.c no longer used Misc. Unrelated Changes: 1. The nczarr_test makefile clean was leaving some directories; so add clean-local to take care of them.
2020-09-28 02:43:46 +08:00
NC_NOOP_inq_var_filter_ids,
NC_NOOP_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
NC_NOOP_inq_filter_avail,
2010-06-03 21:24:43 +08:00
};
const NC_Dispatch* NC3_dispatch_table = NULL; /*!< NC3 Dispatch table, moved here from ddispatch.c */
2011-09-21 01:30:02 +08:00
2010-06-03 21:24:43 +08:00
int
NC3_initialize(void)
{
NC3_dispatch_table = &NC3_dispatcher;
return NC_NOERR;
}
2015-08-16 06:26:35 +08:00
int
NC3_finalize(void)
{
return NC_NOERR;
}
2010-06-03 21:24:43 +08:00
static int
NC3_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *ndimsp, int *dimidsp, int *nattsp,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
unsigned int* idp, size_t* nparamsp, unsigned int* params
)
2010-06-03 21:24:43 +08:00
{
2017-12-21 10:53:30 +08:00
int stat = NC3_inq_var(ncid,varid,name,xtypep,ndimsp,dimidsp,nattsp,no_fill,fill_valuep);
2010-06-03 21:24:43 +08:00
if(stat) return stat;
if(shufflep) *shufflep = 0;
if(deflatep) *deflatep = 0;
if(fletcher32p) *fletcher32p = 0;
if(contiguousp) *contiguousp = NC_CONTIGUOUS;
if(endiannessp) return NC_ENOTNC4;
if(idp) return NC_ENOTNC4;
if(nparamsp) return NC_ENOTNC4;
if(params) return NC_ENOTNC4;
2010-06-03 21:24:43 +08:00
return NC_NOERR;
}
2013-03-16 04:31:07 +08:00
static int
NC3_var_par_access(int ncid, int varid, int par_access)
{
return NC_NOERR; /* no-op for netcdf classic */
}
2010-06-03 21:24:43 +08:00
static int
NC3_inq_unlimdims(int ncid, int *ndimsp, int *unlimdimidsp)
{
int retval;
int unlimid;
if ((retval = NC3_inq_unlimdim(ncid, &unlimid)))
2010-06-03 21:24:43 +08:00
return retval;
if (unlimid != -1) {
if(ndimsp) *ndimsp = 1;
if (unlimdimidsp)
unlimdimidsp[0] = unlimid;
} else
if(ndimsp) *ndimsp = 0;
return NC_NOERR;
}
static int
NC3_def_grp(int parent_ncid, const char *name, int *new_ncid)
{
return NC_ENOTNC4;
}
static int
NC3_rename_grp(int grpid, const char *name)
{
return NC_ENOTNC4;
}
2010-06-03 21:24:43 +08:00
static int
NC3_inq_ncid(int ncid, const char *name, int *grp_ncid)
{
if(grp_ncid) *grp_ncid = ncid;
2010-06-03 21:24:43 +08:00
return NC_NOERR;
}
static int
NC3_inq_grps(int ncid, int *numgrps, int *ncids)
{
if (numgrps)
2010-06-03 21:24:43 +08:00
*numgrps = 0;
return NC_NOERR;
}
static int
NC3_inq_grpname(int ncid, char *name)
{
if (name)
strcpy(name, "/");
return NC_NOERR;
}
static int
NC3_inq_grpname_full(int ncid, size_t *lenp, char *full_name)
{
if (full_name)
strcpy(full_name, "/");
if(lenp) *lenp = 1;
return NC_NOERR;
}
static int
NC3_inq_grp_parent(int ncid, int *parent_ncid)
{
return NC_ENOGRP;
}
static int
NC3_inq_grp_full_ncid(int ncid, const char *full_name, int *grp_ncid)
{
return NC_ENOGRP;
}
static int
NC3_inq_varids(int ncid, int *nvarsp, int *varids)
{
int retval,v,nvars;
/* This is a netcdf-3 file, there is only one group, the root
2010-06-03 21:24:43 +08:00
group, and its vars have ids 0 thru nvars - 1. */
if ((retval = NC3_inq(ncid, NULL, &nvars, NULL, NULL)))
2010-06-03 21:24:43 +08:00
return retval;
if(nvarsp) *nvarsp = nvars;
if (varids)
for (v = 0; v < nvars; v++)
varids[v] = v;
return NC_NOERR;
}
static int
NC3_inq_dimids(int ncid, int *ndimsp, int *dimids, int include_parents)
{
int retval,d,ndims;
/* If this is a netcdf-3 file, then the dimids are going to be 0
thru ndims-1, so just provide them. */
if ((retval = NC3_inq(ncid, &ndims, NULL, NULL, NULL)))
2010-06-03 21:24:43 +08:00
return retval;
if(ndimsp) *ndimsp = ndims;
if (dimids)
for (d = 0; d < ndims; d++)
dimids[d] = d;
return NC_NOERR;
}
static int
NC3_show_metadata(int ncid)
{
return NC_NOERR;
}
static int
NC3_inq_type_equal(int ncid1, nc_type typeid1, int ncid2, nc_type typeid2, int* equalp)
{
/* Check input. */
if(equalp == NULL) return NC_NOERR;
if (typeid1 <= NC_NAT || typeid2 <= NC_NAT)
return NC_EINVAL;
*equalp = 0; /* assume */
/* If one is atomic, and the other user-defined, the types are not equal */
if ((typeid1 <= NC_STRING && typeid2 > NC_STRING) ||
(typeid2 <= NC_STRING && typeid1 > NC_STRING)) {
if (equalp) *equalp = 0;
return NC_NOERR;
}
/* If both are atomic types, the answer is easy. */
2015-08-16 06:26:35 +08:00
if (typeid1 <= ATOMICTYPEMAX3) {
2010-06-03 21:24:43 +08:00
if (equalp) {
if (typeid1 == typeid2)
*equalp = 1;
else
*equalp = 0;
}
return NC_NOERR;
}
return NC_NOERR;
}
static int
NC3_inq_typeid(int ncid, const char *name, nc_type *typeidp)
{
int i;
2015-08-16 06:26:35 +08:00
for (i = 0; i <= ATOMICTYPEMAX3; i++)
2010-06-03 21:24:43 +08:00
if (!strcmp(name, NC_atomictypename(i))) {
if (typeidp) *typeidp = i;
return NC_NOERR;
}
return NC_ENOTNC4;
}
static int
NC3_inq_typeids(int ncid, int *ntypes, int *typeids)
{
if(ntypes) *ntypes = 0;
return NC_NOERR;
}
static int
NC3_inq_user_type(int ncid, nc_type typeid, char *name, size_t *size,
nc_type *base_nc_typep, size_t *nfieldsp, int *classp)
{
return NC_ENOTNC4;
}
static int
NC3_def_compound(int ncid, size_t size, const char *name, nc_type *typeidp)
{
return NC_ENOTNC4;
}
static int
NC3_insert_compound(int ncid, nc_type typeid, const char *name, size_t offset,
nc_type field_typeid)
{
return NC_ENOTNC4;
}
static int
NC3_insert_array_compound(int ncid, nc_type typeid, const char *name,
size_t offset, nc_type field_typeid,
int ndims, const int *dim_sizes)
{
return NC_ENOTNC4;
}
static int
NC3_inq_compound_field(int ncid, nc_type typeid, int fieldid, char *name,
size_t *offsetp, nc_type *field_typeidp, int *ndimsp,
int *dim_sizesp)
{
return NC_ENOTNC4;
}
static int
NC3_inq_compound_fieldindex(int ncid, nc_type typeid, const char *name, int *fieldidp)
{
return NC_ENOTNC4;
}
static int
NC3_def_opaque(int ncid, size_t datum_size, const char *name, nc_type* xtypep)
{
return NC_ENOTNC4;
}
static int
NC3_def_vlen(int ncid, const char *name, nc_type base_typeid, nc_type* xtypep)
{
return NC_ENOTNC4;
}
static int
NC3_def_enum(int ncid, nc_type base_typeid, const char *name,
nc_type *typeidp)
{
return NC_ENOTNC4;
}
static int
NC3_inq_enum_ident(int ncid, nc_type xtype, long long value, char *identifier)
{
return NC_ENOTNC4;
}
static int
NC3_inq_enum_member(int ncid, nc_type typeid, int idx, char *identifier,
void *value)
{
return NC_ENOTNC4;
}
static int
NC3_insert_enum(int ncid, nc_type typeid, const char *identifier,
const void *value)
{
return NC_ENOTNC4;
}
static int
NC3_put_vlen_element(int ncid, int typeid, void *vlen_element,
size_t len, const void *data)
{
return NC_ENOTNC4;
}
static int
NC3_get_vlen_element(int ncid, int typeid, const void *vlen_element,
size_t *len, void *data)
{
return NC_ENOTNC4;
}
static int
NC3_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption)
{
return NC_ENOTNC4;
}
static int
NC3_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp, float *preemptionp)
{
return NC_ENOTNC4;
}
static int
NC3_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
int deflate_level)
{
return NC_ENOTNC4;
}
static int
NC3_def_var_fletcher32(int ncid, int varid, int fletcher32)
{
return NC_ENOTNC4;
}
static int
NC3_def_var_chunking(int ncid, int varid, int contiguous, const size_t *chunksizesp)
{
return NC_ENOTNC4;
}
static int
NC3_def_var_endian(int ncid, int varid, int endianness)
{
return NC_ENOTNC4;
}
static int
NC3_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms)
{
return NC_ENOTNC4;
}
2010-06-03 21:24:43 +08:00