mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-02-05 16:20:10 +08:00
db772ce34c
re: PR https://github.com/Unidata/netcdf-c/pull/2716). re: Issue https://github.com/Unidata/netcdf-c/issues/2189 The basic change is to make use of the fact that HDF5 automatically suppresses optional filters when an attempt is made to apply them to variable-length typed arrays. This means that e.g. ncdump or nccopy will properly see meaningful data. Note that if a filter is defined as HDF5 mandatory, then the corresponding variable will be suppressed and will be invisible to ncdump and nccopy. This functionality is also propagated to NCZarr. This PR makes some minor changes to PR https://github.com/Unidata/netcdf-c/pull/2716 as follows: * Move the test for filter X variable-length from dfilter.c down into the dispatch table functions. * Make all filters for HDF5 optional rather than mandatory so that the built-in HDF5 test for filter X variable-length will be invoked. The test case for this was expanded to verify that the filters are defined, but suppressed.
551 lines
16 KiB
C
551 lines
16 KiB
C
/*
|
|
* Copyright 2018, University Corporation for Atmospheric Research
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
*/
|
|
/**
|
|
* @file
|
|
* Functions for working with filters.
|
|
*/
|
|
|
|
#include "config.h"
|
|
#include <stdlib.h>
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
#ifdef _MSC_VER
|
|
#include <io.h>
|
|
#endif
|
|
|
|
#include "netcdf.h"
|
|
#include "netcdf_filter.h"
|
|
#include "ncdispatch.h"
|
|
#include "nc4internal.h"
|
|
#include "nclog.h"
|
|
|
|
#ifdef USE_HDF5
|
|
#include "hdf5internal.h"
|
|
#endif
|
|
|
|
#ifdef ENABLE_NCZARR
|
|
#include "zdispatch.h"
|
|
#endif
|
|
|
|
/*
|
|
Unified filter related code
|
|
*/
|
|
|
|
/**************************************************/
|
|
/* Per-variable filters */
|
|
|
|
/**
|
|
Find the set of filters (if any) associated with a variable.
|
|
Assumes HDF5 format using unsigned ints.
|
|
|
|
@param ncid NetCDF or group ID, from a previous call to nc_open(),
|
|
nc_create(), nc_def_grp(), or associated inquiry functions such as
|
|
nc_inq_ncid().
|
|
@param varid Variable ID
|
|
@param nfiltersp Pointer that gets the number of filters; may be zero.
|
|
@param ids return the filter ids (caller allocates)
|
|
|
|
@returns ::NC_NOERR No error.
|
|
@returns ::NC_ENOTNC4 Not a netCDF-4 file.
|
|
@returns ::NC_EBADID Bad ncid
|
|
@returns ::NC_ENOTVAR Invalid variable ID.
|
|
@returns ::NC_EINVAL Invalid arguments
|
|
@ingroup variables
|
|
@author Dennis Heimbigner
|
|
*/
|
|
EXTERNL int
|
|
nc_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids)
|
|
{
|
|
NC* ncp;
|
|
int stat = NC_check_id(ncid,&ncp);
|
|
if(stat != NC_NOERR) return stat;
|
|
TRACE(nc_inq_var_filter_ids);
|
|
if((stat = ncp->dispatch->inq_var_filter_ids(ncid,varid,nfiltersp,ids))) goto done;
|
|
|
|
done:
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
Find the the param info about filter (if any)
|
|
associated with a variable and with specified id.
|
|
Assumes HDF5 format using unsigned ints.
|
|
|
|
@param ncid NetCDF or group ID, from a previous call to nc_open(),
|
|
nc_create(), nc_def_grp(), or associated inquiry functions such as
|
|
nc_inq_ncid().
|
|
|
|
@param varid Variable ID
|
|
@param id The filter id of interest
|
|
@param nparamsp (Out) Storage which will get the number of parameters to the filter
|
|
@param params (Out) Storage which will get associated parameters.
|
|
Note: the caller must allocate and free.
|
|
|
|
@returns ::NC_NOERR No error.
|
|
@returns ::NC_ENOTNC4 Not a netCDF-4 file.
|
|
@returns ::NC_EBADID Bad ncid.
|
|
@returns ::NC_ENOTVAR Invalid variable ID.
|
|
@returns ::NC_ENOFILTER Specified filter not defined for this variable.
|
|
@ingroup variables
|
|
@author Dennis Heimbigner
|
|
*/
|
|
EXTERNL int
|
|
nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, unsigned int* params)
|
|
{
|
|
NC* ncp;
|
|
int stat = NC_check_id(ncid,&ncp);
|
|
if(stat != NC_NOERR) return stat;
|
|
TRACE(nc_inq_var_filter_info);
|
|
if((stat = ncp->dispatch->inq_var_filter_info(ncid,varid,id,nparamsp,params))) goto done;
|
|
|
|
done:
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
Define a new variable filter
|
|
Assumes HDF5 format using unsigned ints.
|
|
Only variables with chunked storage can use filters.
|
|
|
|
@param ncid File and group ID.
|
|
@param varid Variable ID.
|
|
@param id Filter ID.
|
|
@param nparams Number of filter parameters.
|
|
@param params Filter parameters.
|
|
|
|
@return ::NC_NOERR No error.
|
|
@return ::NC_EINVAL Variable must be chunked.
|
|
@return ::NC_EBADID Bad ID.
|
|
@author Dennis Heimbigner
|
|
*/
|
|
|
|
EXTERNL int
|
|
nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params)
|
|
{
|
|
int stat = NC_NOERR;
|
|
NC* ncp;
|
|
|
|
TRACE(nc_inq_var_filter);
|
|
if((stat = NC_check_id(ncid,&ncp))) return stat;
|
|
if((stat = ncp->dispatch->def_var_filter(ncid,varid,id,nparams,params))) goto done;
|
|
done:
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
Find the first filter (if any) associated with a variable.
|
|
Assumes HDF5 format using unsigned int.
|
|
|
|
@param ncid NetCDF or group ID, from a previous call to nc_open(),
|
|
nc_create(), nc_def_grp(), or associated inquiry functions such as
|
|
nc_inq_ncid().
|
|
|
|
@param varid Variable ID
|
|
|
|
@param idp Storage which will get the filter id; a return value of zero means variable has no filters.
|
|
|
|
@param nparamsp Storage which will get the number of parameters to the filter
|
|
|
|
@param params Storage which will get associated parameters (call allocates and frees).
|
|
|
|
This is redundant over the multi-filter API, so
|
|
it can be implemented in terms of those functions.
|
|
|
|
@returns ::NC_NOERR No error.
|
|
@returns ::NC_ENOTNC4 Not a netCDF-4 file.
|
|
@returns ::NC_EBADID Bad ncid.
|
|
@returns ::NC_ENOTVAR Invalid variable ID.
|
|
|
|
@ingroup variables
|
|
@author Dennis Heimbigner
|
|
*/
|
|
EXTERNL int
|
|
nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsigned int* params)
|
|
{
|
|
NC* ncp;
|
|
size_t nfilters;
|
|
unsigned int* ids = NULL;
|
|
int stat = NC_check_id(ncid,&ncp);
|
|
|
|
if(stat != NC_NOERR) return stat;
|
|
TRACE(nc_inq_var_filter);
|
|
|
|
/* Get the number of filters on this variable */
|
|
if((stat = nc_inq_var_filter_ids(ncid,varid,&nfilters, NULL))) goto done;
|
|
/* If no filters, then return zero */
|
|
if(nfilters == 0) {
|
|
if(idp) *idp = 0;
|
|
goto done;
|
|
}
|
|
/* Get the filter ids */
|
|
if((ids = calloc(sizeof(unsigned int),nfilters)) == NULL) {stat = NC_ENOMEM; goto done;}
|
|
if((stat = nc_inq_var_filter_ids(ncid,varid,&nfilters, ids))) goto done;
|
|
/* Get params for the first filter */
|
|
if((stat = nc_inq_var_filter_info(ncid,varid,ids[0],nparamsp,params))) goto done;
|
|
if(idp) *idp = ids[0];
|
|
done:
|
|
nullfree(ids);
|
|
return stat;
|
|
}
|
|
|
|
/** Test if filter is available. Would prefer
|
|
* returning a list of all available filters, but HDF5
|
|
* does not support that capability.
|
|
*
|
|
* @param ncid ID of file for which filter list is desired
|
|
* @param id filter id of interest
|
|
*
|
|
* @return NC_NOERR if the filter is available
|
|
* @return NC_EBADID if ncid is invalid
|
|
* @return NC_ENOFILTER if filter is not available.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
|
|
EXTERNL int
|
|
nc_inq_filter_avail(int ncid, unsigned id)
|
|
{
|
|
int stat = NC_NOERR;
|
|
NC* ncp;
|
|
|
|
stat = NC_check_id(ncid,&ncp);
|
|
if(stat != NC_NOERR) return stat;
|
|
if((stat = ncp->dispatch->inq_filter_avail(ncid,id))) goto done;
|
|
done:
|
|
return stat;
|
|
}
|
|
|
|
/**************************************************/
|
|
/* Support direct user defined filters */
|
|
|
|
#ifdef ENABLE_CLIENTSIDE_FILTERS
|
|
|
|
/** Register filer client.
|
|
* @note Use void* to avoid having to include hdf.h
|
|
*
|
|
* @param id Filter ID
|
|
* @param info Pointer that gets info.
|
|
*
|
|
* @return NC_NOERR if the filter is available
|
|
* @return NC_EBADID if ncid is invalid
|
|
* @return NC_ENOFILTER if filter is not available.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
EXTERNL int
|
|
nc_filter_client_register(unsigned int id, void* info)
|
|
{
|
|
int stat = NC_NOERR;
|
|
#ifdef USE_HDF5
|
|
NC_FILTER_OBJ_HDF5 client;
|
|
if(id == 0 ||info == NULL)
|
|
return NC_EINVAL;
|
|
memset(&client,0,sizeof(client));
|
|
client.hdr.format = NC_FILTER_FORMAT_HDF5;
|
|
client.sort = NC_FILTER_SORT_CLIENT;
|
|
client.u.client.id = id;
|
|
client.u.client.info = info;
|
|
/* Note use of a global function, not part of the dispatch table */
|
|
stat = nc4_global_filter_action(NCFILTER_CLIENT_REG, id, &client);
|
|
#else
|
|
stat = NC_ENOTBUILT;
|
|
#endif
|
|
return stat;
|
|
}
|
|
|
|
/** Unregister filer client.
|
|
* @note Use void* to avoid having to include hdf.h
|
|
*
|
|
* @param id Filter ID
|
|
*
|
|
* @return NC_NOERR if the filter is available
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
EXTERNL int
|
|
nc_filter_client_unregister(unsigned int id)
|
|
{
|
|
int stat = NC_NOERR;
|
|
#ifdef USE_HDF5
|
|
stat = nc4_global_filter_action(NCFILTER_CLIENT_UNREG, id, NULL);
|
|
#else
|
|
stat = NC_ENOTBUILT;
|
|
#endif
|
|
return stat;
|
|
}
|
|
|
|
/** Inquire about filer client.
|
|
* @note Use void* to avoid having to include hdf.h
|
|
*
|
|
* @param id Filter ID
|
|
* @param infop Pointer that gets info.
|
|
*
|
|
* @return NC_NOERR if the filter is available
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
EXTERNL int
|
|
nc_filter_client_inq(unsigned int id, void* infop)
|
|
{
|
|
int stat = NC_NOERR;
|
|
#ifdef USE_HDF5
|
|
H5Z_class2_t* hct = (H5Z_class2_t*)infop;
|
|
NC_FILTER_OBJ_HDF5 client;
|
|
if(id == 0 ||infop == NULL)
|
|
return NC_EINVAL;
|
|
memset(&client,0,sizeof(client));
|
|
client.hdr.format = NC_FILTER_FORMAT_HDF5;
|
|
client.sort = NC_FILTER_SORT_CLIENT;
|
|
client.u.client.id = id;
|
|
client.u.client.info = hct;
|
|
/* Note use of a global function, not part of the dispatch table */
|
|
stat = nc4_global_filter_action(NCFILTER_CLIENT_INQ, id, &client);
|
|
if(stat == NC_NOERR) {
|
|
*hct = *(H5Z_class2_t*)client.u.client.info;
|
|
}
|
|
#else
|
|
stat = NC_ENOTBUILT;
|
|
#endif
|
|
return stat;
|
|
}
|
|
#endif /*ENABLE_CLIENTSIDE_FILTERS*/
|
|
|
|
/**************************************************/
|
|
/* Functions for accessing standardized filters */
|
|
|
|
/**
|
|
* Turn on bzip2 compression for a variable.
|
|
*
|
|
* @param ncid File ID.
|
|
* @param varid Variable ID.
|
|
* @param level From 1 to 9. Set the block size to 100k, 200k ... 900k
|
|
* when compressing. (bzip2 default level is 9).
|
|
*
|
|
* @return 0 for success, error code otherwise.
|
|
* @author Dennis Heimbigner, Ed Hartnett
|
|
*/
|
|
int
|
|
nc_def_var_bzip2(int ncid, int varid, int level)
|
|
{
|
|
int stat = NC_NOERR;
|
|
unsigned ulevel;
|
|
|
|
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2))) goto done;
|
|
/* Filter is available */
|
|
/* 1 <= Level <= 9 */
|
|
if (level < 1 || level > 9)
|
|
return NC_EINVAL;
|
|
ulevel = (unsigned) level; /* Keep bit pattern */
|
|
if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_BZIP2,1,&ulevel))) goto done;
|
|
done:
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
* Learn whether bzip2 compression is on for a variable, and, if so,
|
|
* the level setting.
|
|
*
|
|
* @param ncid File ID.
|
|
* @param varid Variable ID.
|
|
* @param hasfilterp Pointer that gets a 0 if bzip2 is not in use for this
|
|
* var, and a 1 if it is. Ignored if NULL.
|
|
* @param levelp Pointer that gets the level setting (from 1 to 9), if
|
|
* bzip2 is in use. Ignored if NULL.
|
|
*
|
|
* @return 0 for success, error code otherwise.
|
|
* @author Dennis Heimbigner, Ed Hartnett
|
|
*/
|
|
int
|
|
nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
size_t nparams;
|
|
unsigned params = 0;
|
|
int hasfilter = 0;
|
|
|
|
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2))) goto done;
|
|
/* Filter is available */
|
|
/* Get filter info */
|
|
stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BZIP2,&nparams,NULL);
|
|
if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;}
|
|
if(stat != NC_NOERR) goto done;
|
|
hasfilter = 1;
|
|
if(nparams != 1) {stat = NC_EFILTER; goto done;}
|
|
if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BZIP2,&nparams,¶ms))) goto done;
|
|
done:
|
|
if(levelp) *levelp = (int)params;
|
|
if(hasfilterp) *hasfilterp = hasfilter;
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
* Turn on Zstandard compression for a variable.
|
|
*
|
|
* @param ncid File ID.
|
|
* @param varid Variable ID.
|
|
* @param level From -131072 to 22 (depends on Zstandard version).
|
|
* when compressing. Regular compression levels are from 1 up to 19.
|
|
* Use levels >= 20, labeled `--ultra`, cautiously: they require more memory.
|
|
* Negative compression levels that extend the range of speed vs. ratio preferences.
|
|
* The lower the level, the faster the speed (at the cost of compression).
|
|
*
|
|
* @return 0 for success, error code otherwise.
|
|
* @author Charlie Zender, Dennis Heimbigner, Ed Hartnett
|
|
*/
|
|
int
|
|
nc_def_var_zstandard(int ncid, int varid, int level)
|
|
{
|
|
#ifdef HAVE_ZSTD
|
|
int stat = NC_NOERR;
|
|
unsigned ulevel;
|
|
|
|
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_ZSTD))) goto done;
|
|
/* Filter is available */
|
|
/* Level must be between -131072 and 22 on Zstandard v. 1.4.5 (~202009)
|
|
Earlier versions have fewer levels (especially fewer negative levels) */
|
|
if (level < -131072 || level > 22)
|
|
return NC_EINVAL;
|
|
ulevel = (unsigned) level; /* Keep bit pattern */
|
|
if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_ZSTD,1,&ulevel))) goto done;
|
|
done:
|
|
return stat;
|
|
#else
|
|
return NC_NOERR;
|
|
#endif /*HAVE_ZSTD*/
|
|
}
|
|
|
|
/**
|
|
* Learn whether Zstandard compression is on for a variable, and, if so,
|
|
* the level setting.
|
|
*
|
|
* @param ncid File ID.
|
|
* @param varid Variable ID.
|
|
* @param hasfilterp Pointer that gets a 0 if Zstandard is not in use for this
|
|
* var, and a 1 if it is. Ignored if NULL.
|
|
* @param levelp Pointer that gets the level setting (from -131072 to 22), if
|
|
* Zstandard is in use. Ignored if NULL.
|
|
*
|
|
* @return 0 for success, error code otherwise.
|
|
* @author Charlie Zender, Dennis Heimbigner, Ed Hartnett
|
|
*/
|
|
int
|
|
nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp)
|
|
{
|
|
#ifdef HAVE_ZSTD
|
|
int stat = NC_NOERR;
|
|
size_t nparams;
|
|
unsigned params = 0;
|
|
int hasfilter = 0;
|
|
|
|
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_ZSTD))) goto done;
|
|
/* Filter is available */
|
|
/* Get filter info */
|
|
stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_ZSTD,&nparams,NULL);
|
|
if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;}
|
|
if(stat != NC_NOERR) goto done;
|
|
hasfilter = 1;
|
|
if(nparams != 1) {stat = NC_EFILTER; goto done;}
|
|
if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_ZSTD,&nparams,¶ms))) goto done;
|
|
done:
|
|
if(levelp) *levelp = (int)params;
|
|
if(hasfilterp) *hasfilterp = hasfilter;
|
|
return stat;
|
|
#else
|
|
return NC_NOERR;
|
|
#endif /*HAVE_ZSTD*/
|
|
}
|
|
|
|
/**
|
|
* Turn on blosc for a variable.
|
|
*
|
|
* @param ncid File ID.
|
|
* @param varid Variable ID.
|
|
* @param subcompressor The subcompressor.
|
|
* @param level The level setting.
|
|
* @param blocksize The block size.
|
|
* @param addshuffle If non-zero, turn on shuffle.
|
|
*
|
|
* @return 0 for success, error code otherwise.
|
|
* @author Dennis Heimbigner
|
|
* @ingroup variables
|
|
*/
|
|
int
|
|
nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle)
|
|
{
|
|
#ifdef HAVE_BLOSC
|
|
int stat = NC_NOERR;
|
|
unsigned params[7];;
|
|
|
|
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BLOSC))) goto done;
|
|
/* Filter is available */
|
|
|
|
/* Verify parameters */
|
|
if(addshuffle > (unsigned)BLOSC_BITSHUFFLE) {stat = NC_EINVAL; goto done;}
|
|
if(subcompressor > (unsigned)BLOSC_ZSTD) {stat = NC_EINVAL; goto done;}
|
|
|
|
/* Set the parameters */
|
|
params[0] = 0;
|
|
params[1] = 0;
|
|
params[2] = 0;
|
|
params[3] = blocksize;
|
|
params[4] = level;
|
|
params[5] = addshuffle;
|
|
params[6] = subcompressor;
|
|
if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_BLOSC,7,params))) goto done;
|
|
done:
|
|
return stat;
|
|
#else
|
|
return NC_NOERR;
|
|
#endif
|
|
}
|
|
|
|
/**
|
|
* Learn whether Blosc compression is on for a variable, and, if so,
|
|
* the settings.
|
|
*
|
|
* @param ncid File ID.
|
|
* @param varid Variable ID.
|
|
* @param hasfilterp Pointer that gets a 0 if blosc is not in use for this
|
|
* var, and a 1 if it is. Ignored if NULL.
|
|
* @param subcompressorp Pointer that gets the subcompressor, if
|
|
* blosc is in use. Ignored if NULL.
|
|
* @param levelp Pointer that gets the level setting, if
|
|
* blosc is in use. Ignored if NULL.
|
|
* @param blocksizep Pointer that gets the block size, if
|
|
* blosc is in use. Ignored if NULL.
|
|
* @param addshufflep Pointer that gets non-zero value if shuffle is
|
|
* in use, if blosc is in use. Ignored if NULL.
|
|
*
|
|
* @return 0 for success, error code otherwise.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
int
|
|
nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp,
|
|
unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep)
|
|
{
|
|
#ifdef HAVE_BLOSC
|
|
int stat = NC_NOERR;
|
|
size_t nparams;
|
|
unsigned params[7];
|
|
int hasfilter = 0;
|
|
|
|
if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BLOSC))) goto done;
|
|
/* Filter is available */
|
|
|
|
/* Get filter info */
|
|
stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BLOSC,&nparams,NULL);
|
|
if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;}
|
|
if(stat != NC_NOERR) goto done;
|
|
hasfilter = 1;
|
|
if(nparams != 7) {stat = NC_EFILTER; goto done;}
|
|
if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BLOSC,&nparams,params))) goto done;
|
|
if(blocksizep) *blocksizep = params[3];
|
|
if(levelp) *levelp = params[4];
|
|
if(addshufflep) *addshufflep = params[5];
|
|
if(subcompressorp) *subcompressorp = params[6];
|
|
done:
|
|
if(hasfilterp) *hasfilterp = hasfilter;
|
|
return stat;
|
|
#else
|
|
return NC_NOERR;
|
|
#endif
|
|
}
|