netcdf-c/nc_test4/tst_vars3.c

763 lines
30 KiB
C
Raw Normal View History

2010-06-03 21:24:43 +08:00
/* This is part of the netCDF package.
Copyright 2020 University Corporation for Atmospheric Research/Unidata
2010-06-03 21:24:43 +08:00
See COPYRIGHT file for conditions of use.
Test netcdf-4 variables.
Ed Hartnett, Russ Rew, Dennis Heimbigner, Ward Fisher
2010-06-03 21:24:43 +08:00
*/
#include <nc_tests.h>
#include "err_macros.h"
2010-06-03 21:24:43 +08:00
#include "netcdf.h"
Add support for multiple filters per variable. re: https://github.com/Unidata/netcdf-c/issues/1584 Support has been added for multiple filters per variable. This affects a number of components in netcdf. The new APIs are documented in NUG/filters.md. The primary changes are: * A set of new functions are provided (see __include/netcdf_filter.h__). - Obtain a list of the filters associated with a variable - Obtain the parameters for a specific filter. * The existing __nc_inq_var_filter__ function now returns info about the first defined filter. * The utilities (ncgen, ncdump, and nccopy) now support an extended format for specifying a sequence of filters. The general form is __<filter>|<filter>..._. * The ncdump **_Filter** attribute now dumps a list of all the filters associated with a variable using the above new format. * Filter specifications can now use a filter name instead of number for filters known to the netcdf library, which in turn is taken from the HDF5 filter registration page. * New errors are defined: NC_EFILTER and NC_ENOFILTER. The latter is returned if an attempt is made to access an unknown filter. * Internally, the dispatch table has been extended to add a function to handle all of the filter functions. * New, filter-related, tests were added to nc_test4. * A new plugin was added to the plugins directory to help with testing. Notes: 1. The shuffle and fletcher32 filters are not part of the multifilter system. Misc. changes: 1. A debug module was added to libhdf5 to help catch error locations.
2020-02-17 03:59:33 +08:00
#include "netcdf_filter.h"
#define NC_SZIP_EC_BPP_IN 32 /**< @internal bits per pixel input. */
#define NC_SZIP_EC_BPP_OUT 64 /**< @internal bits per pixel output. */
2010-06-03 21:24:43 +08:00
#define FILE_NAME "tst_vars3.nc"
#define NDIMS1 1
2017-12-02 06:08:22 +08:00
#define NDIMS2 2
2010-06-03 21:24:43 +08:00
#define D_SMALL "small_dim"
#define D_SMALL_LEN 16
#define D_MEDIUM "medium_dim"
#define D_MEDIUM_LEN 65546
#define D_LARGE "large_dim"
#define D_LARGE_LEN 1048586
#define V_SMALL "small_var"
#define V_MEDIUM "medium_var"
#define V_LARGE "large_var"
#define D_MAX_ONE_D 16384
2020-01-06 22:57:50 +08:00
#define DIM_NAME_1 "one_dim"
#define DIM_LEN_1 100
#define H5_FILTER_SZIP 4
#define VAR_NAME "var1"
#define NUM_PARAMS_IN 2
#define NUM_PARAMS_OUT 4
2010-06-03 21:24:43 +08:00
int
main(int argc, char **argv)
{
2020-01-06 22:58:11 +08:00
printf("\n*** Testing netcdf-4 variable functions, some more.\n");
printf("**** testing definition of coordinate variable after endef/redef...");
{
#define NX 6
#define NY 36
#define ZD1_NAME "zD1"
#define D2_NAME "D2"
2020-01-06 22:58:11 +08:00
int ncid, x_dimid, y_dimid, varid2;
char name_in[NC_MAX_NAME + 1];
/* Create file with two dims, two 1D vars. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, ZD1_NAME, NX, &x_dimid)) ERR;
if (nc_def_dim(ncid, D2_NAME, NY, &y_dimid)) ERR;
if (nc_enddef(ncid)) ERR;
/* Go back into define mode and add a coordinate variable. Now
* dimensions will be out of order. Thanks for confusing my poor
* library. Why can't you just make up your bloody mind? */
if (nc_redef(ncid)) ERR;
if (nc_def_var(ncid, ZD1_NAME, NC_DOUBLE, NDIMS1, &x_dimid, &varid2)) ERR;
if (nc_close(ncid)) ERR;
/* Reopen file and check the name of the first dimension. Even
* though you've changed the order after doing a redef, you will
* still expect to get D1_NAME. I sure hope you appreciate how
* hard you are making life for a poor C library, just trying to
* do its best in a demanding world. Next time, why don't you
* try and be a little bit more considerate? Jerk. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq_dimname(ncid, 0, name_in)) ERR;
if (strcmp(name_in, ZD1_NAME)) ERR;
if (nc_inq_dimname(ncid, 1, name_in)) ERR;
if (strcmp(name_in, D2_NAME)) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing definition of coordinate variable with some data...");
{
#define NX 6
#define NY 36
#define V1_NAME "V1"
#define D1_NAME "D1"
#define D2_NAME "D2"
2020-01-06 22:58:11 +08:00
int ncid, x_dimid, y_dimid, varid1, varid2;
int nvars, ndims, ngatts, unlimdimid, dimids_in[2], natts;
size_t len_in;
char name_in[NC_MAX_NAME + 1];
nc_type xtype_in;
#if 0
2020-01-06 22:58:11 +08:00
int x, y;
double data_outx[NX], data_outy[NY];
int retval;
#endif
#if 0
2020-01-06 22:58:11 +08:00
/* Create some pretend data. */
for (x = 0; x < NX; x++)
data_outx[x] = x;
for (y = 0; y < NY; y++)
data_outy[y] = y;
#endif
2020-01-06 22:58:11 +08:00
/* Create file with two dims, two 1D vars. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, D1_NAME, NX, &x_dimid)) ERR;
if (nc_def_dim(ncid, D2_NAME, NY, &y_dimid)) ERR;
if (nc_def_var(ncid, V1_NAME, NC_DOUBLE, NDIMS1, &y_dimid, &varid1)) ERR;
if (nc_enddef(ncid)) ERR;
if (nc_redef(ncid)) ERR;
if (nc_def_var(ncid, D1_NAME, NC_DOUBLE, NDIMS1, &x_dimid, &varid2)) ERR;
/* if (nc_put_var_double(ncid, varid1, &data_outy[0])) ERR; */
/* if (nc_put_var_double(ncid, varid2, &data_outx[0])) ERR; */
/* if (nc_sync(ncid)) ERR; */
2010-06-03 21:24:43 +08:00
2020-01-06 22:58:11 +08:00
/* Check the file. */
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (nvars != 2 || ndims != 2 || ngatts != 0 || unlimdimid != -1) ERR;
/* Check the dimensions. */
if (nc_inq_dimids(ncid, &ndims, dimids_in, 1)) ERR;
if (ndims != 2 || dimids_in[0] != x_dimid || dimids_in[1] != y_dimid) ERR;
if (nc_inq_dim(ncid, dimids_in[0], name_in, &len_in)) ERR;
if (strcmp(name_in, D1_NAME) || len_in != NX) ERR;
if (nc_inq_dim(ncid, dimids_in[1], name_in, &len_in)) ERR;
if (strcmp(name_in, D2_NAME) || len_in != NY) ERR;
/* Check the variables. */
if (nc_inq_var(ncid, 0, name_in, &xtype_in, &ndims, dimids_in, &natts)) ERR;
if (strcmp(name_in, V1_NAME) || xtype_in != NC_DOUBLE || ndims != 1 ||
natts != 0 || dimids_in[0] != y_dimid) ERR;
if (nc_inq_var(ncid, 1, name_in, &xtype_in, &ndims, dimids_in, &natts)) ERR;
if (strcmp(name_in, D1_NAME) || xtype_in != NC_DOUBLE || ndims != 1 ||
natts != 0 || dimids_in[0] != x_dimid) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
/* Reopen and check the file. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (nvars != 2 || ndims != 2 || ngatts != 0 || unlimdimid != -1) ERR;
/* Check the dimensions. */
if (nc_inq_dimids(ncid, &ndims, dimids_in, 1)) ERR;
if (ndims != 2 || dimids_in[0] != x_dimid || dimids_in[1] != y_dimid) ERR;
if (nc_inq_dim(ncid, dimids_in[0], name_in, &len_in)) ERR;
if (strcmp(name_in, D1_NAME) || len_in != NX) ERR;
if (nc_inq_dim(ncid, dimids_in[1], name_in, &len_in)) ERR;
if (strcmp(name_in, D2_NAME) || len_in != NY) ERR;
/* Check the variables. */
if (nc_inq_var(ncid, 0, name_in, &xtype_in, &ndims, dimids_in, &natts)) ERR;
if (strcmp(name_in, V1_NAME) || xtype_in != NC_DOUBLE || ndims != 1 ||
natts != 0 || dimids_in[0] != y_dimid) ERR;
if (nc_inq_var(ncid, 1, name_in, &xtype_in, &ndims, dimids_in, &natts)) ERR;
if (strcmp(name_in, D1_NAME) || xtype_in != NC_DOUBLE || ndims != 1 ||
natts != 0 || dimids_in[0] != x_dimid) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing endianness of compound type variable...");
{
2010-06-03 21:24:43 +08:00
#define COMPOUND_NAME "Billy-Bob"
#define BILLY "Billy"
#define BOB "Bob"
#define VAR_NAME1 "Buddy-Joe"
#define NDIMS 2
#define TEXT_LEN 15
2020-01-06 22:58:11 +08:00
int ncid, nvars_in, varids_in[1], typeid, varid;
int nvars, ndims, ngatts, unlimdimid;
int ndims_in, natts_in, dimids_in[NDIMS];
char var_name_in[NC_MAX_NAME + 1];
nc_type xtype_in;
struct billy_bob
{
int billy;
int bob;
};
/* Create a netcdf-4 file with scalar compound var. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_compound(ncid, sizeof(struct billy_bob), COMPOUND_NAME, &typeid)) ERR;
if (nc_insert_compound(ncid, typeid, BILLY, NC_COMPOUND_OFFSET(struct billy_bob, billy), NC_INT)) ERR;
if (nc_insert_compound(ncid, typeid, BOB, NC_COMPOUND_OFFSET(struct billy_bob, bob), NC_INT)) ERR;
if (nc_def_var(ncid, VAR_NAME1, typeid, 0, NULL, &varid)) ERR;
if (nc_def_var_endian(ncid, varid, NC_ENDIAN_BIG) != NC_EINVAL) ERR;
if (nc_close(ncid)) ERR;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (nvars != 1 || ndims != 0 || ngatts != 0 || unlimdimid != -1) ERR;
if (nc_inq_varids(ncid, &nvars_in, varids_in)) ERR;
if (nvars_in != 1 || varids_in[0] != 0) ERR;
if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims_in, dimids_in, &natts_in)) ERR;
if (strcmp(var_name_in, VAR_NAME1) || xtype_in <= NC_STRING || ndims_in != 0 ||
natts_in != 0) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing that fixed vars with no filter end up being contiguous...");
{
2010-06-03 21:24:43 +08:00
#define VAR_NAME2 "Yoman_of_the_Guard"
#define NDIMS 2
#define D0_NAME1 "Tower_warders_under_orders"
#define D0_LEN 55
#define D1_NAME1 "When_our_gallent_Norman_Foes"
#define D1_LEN 99
2020-01-06 22:58:11 +08:00
int ncid, varid;
int nvars, ndims, ngatts, unlimdimid;
int dimids[NDIMS], contig;
int ndims_in, natts_in, dimids_in[NDIMS];
char var_name_in[NC_MAX_NAME + 1];
nc_type xtype_in;
/* Create a netcdf-4 file with 2D fixed var. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, D0_NAME1, D0_LEN, &dimids[0])) ERR;
if (nc_def_dim(ncid, D1_NAME1, D1_LEN, &dimids[1])) ERR;
if (nc_def_var(ncid, VAR_NAME2, NC_UINT64, NDIMS, dimids, &varid)) ERR;
if (nc_def_var_endian(ncid, varid, NC_ENDIAN_BIG)) ERR;
if (nc_close(ncid)) ERR;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (nvars != 1 || ndims != 2 || ngatts != 0 || unlimdimid != -1) ERR;
if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims_in, dimids_in, &natts_in)) ERR;
if (strcmp(var_name_in, VAR_NAME2) || xtype_in != NC_UINT64 || ndims_in != 2 ||
natts_in != 0) ERR;
if (nc_inq_var_chunking(ncid, varid, &contig, NULL)) ERR;
if (!contig) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing typeless access for classic model...");
{
2010-06-03 21:24:43 +08:00
#define RANK_P 3
#define LEN 4
2020-01-06 22:58:11 +08:00
int ncid, dimids[RANK_P], time_id, p_id;
int ndims, dimids_in[RANK_P];
double data[1] = {3.14159};
size_t start[1] = {0}, count[1] = {1};
static float P_data[LEN];
size_t cor[RANK_P] = {0, 1, 0};
size_t edg[RANK_P] = {1, 1, LEN};
int i;
/* Create a 3D test file. */
if (nc_create(FILE_NAME, NC_CLASSIC_MODEL|NC_NETCDF4, &ncid)) ERR;
/* define dimensions */
if (nc_def_dim(ncid, "Time", NC_UNLIMITED, &dimids[0])) ERR;
if (nc_def_dim(ncid, "X", 4, &dimids[2])) ERR;
if (nc_def_dim(ncid, "Y", 3, &dimids[1])) ERR;
/* define variables */
if (nc_def_var(ncid, "Time", NC_DOUBLE, 1, dimids, &time_id)) ERR;
if (nc_def_var(ncid, "P", NC_FLOAT, RANK_P, dimids, &p_id)) ERR;
if (nc_enddef(ncid)) ERR;
/* Add one record in coordinate variable. */
if (nc_put_vara(ncid, time_id, start, count, data)) ERR;
/* The other variable should show an increase in size, since it
* uses the unlimited dimension. */
if (nc_inq_var(ncid, 1, NULL, NULL, &ndims, dimids_in, NULL)) ERR;
if (ndims != 3 || dimids_in[0] != 0 || dimids_in[1] != 2 || dimids_in[2] != 1) ERR;
/* These will not work due to bad parameters. */
if (nc_get_vara(ncid + MILLION, 1, cor, edg, P_data) != NC_EBADID) ERR;
if (nc_get_vara(ncid + TEST_VAL_42, 1, cor, edg, P_data) != NC_EBADID) ERR;
/* Read the record of non-existent data. */
if (nc_get_vara(ncid, 1, cor, edg, P_data)) ERR;
for (i = 0; i < LEN; i++)
if (P_data[i] != NC_FILL_FLOAT) ERR;
/* That's it! */
if (nc_close(ncid)) ERR;
/* Reopen the file and read the second slice. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq_var(ncid, 1, NULL, NULL, &ndims, dimids_in, NULL)) ERR;
if (ndims != 3 || dimids_in[0] != 0 || dimids_in[1] != 2 || dimids_in[2] != 1) ERR;
if (nc_get_vara(ncid, 1, cor, edg, P_data)) ERR;
for (i = 0; i < LEN; i++)
if (P_data[i] != NC_FILL_FLOAT) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing large number of vars with unlimited dimension...");
{
2010-06-03 21:24:43 +08:00
#define D0_NAME3 "dim0"
#define NUM_VARS 1000
2020-01-06 22:58:11 +08:00
int ncid, varid;
int nvars, ndims, ngatts, unlimdimid;
int dimid;
char var_name[NC_MAX_NAME + 1];
int v;
/* Create a netcdf-4 file with lots of 1D unlim dim vars. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, D0_NAME3, NC_UNLIMITED, &dimid)) ERR;
for (v = 0; v < NUM_VARS; v++)
{
sprintf(var_name, "var_%d", v);
if (nc_def_var(ncid, var_name, NC_INT, 1, &dimid, &varid)) ERR_RET;
if (nc_set_var_chunk_cache(ncid, varid, 0, 0, 0.75)) ERR_RET;
}
if (nc_close(ncid)) ERR;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (nvars != NUM_VARS || ndims != 1 || ngatts != 0 || unlimdimid != 0) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing fix of bug in checking coordinate variables out of order...");
{
#define GRP_NAME "group_G"
#define TIME_DIMNAME "time"
#define B_DIMNAME "bDim"
#define C_DIMNAME "cDim"
#define C_VARNAME "c"
#define D_VARNAME "dd"
#define E_VARNAME "ee"
2020-01-06 22:58:11 +08:00
int ncid, grpid;
int timeDimId, bDimId, cDimId, dimidIn;
size_t timeDimSize = 2, bDimSize = 3, cDimSize = 1;
int cNdims = 1, eeNdims = 1, ddNdims = 1 ;
int cVarId, eeVarId, ddVarId ;
size_t index = 0;
double s1Data = 10;
const double timeVar[] = {1.3, 4.6 };
if ( nc_create(FILE_NAME, NC_NETCDF4, &ncid) ) ERR;
if ( nc_def_grp(ncid, GRP_NAME, &grpid) ) ERR;
if ( nc_def_dim(ncid, TIME_DIMNAME, timeDimSize, &timeDimId) ) ERR;
if ( nc_def_dim(grpid, B_DIMNAME, bDimSize, &bDimId) ) ERR;
if ( nc_def_dim(grpid, C_DIMNAME, cDimSize, &cDimId) ) ERR;
if ( nc_def_var(grpid, C_VARNAME, NC_DOUBLE, cNdims, &cDimId, &cVarId) ) ERR;
if ( nc_def_var(ncid, E_VARNAME, NC_DOUBLE, eeNdims, &timeDimId, &eeVarId) ) ERR;
/* worked without this enddef, but in 4.2.1.1 and earlier, inserting this caused failure */
if ( nc_enddef(ncid)) ERR;
if ( nc_def_var(grpid, D_VARNAME, NC_DOUBLE, ddNdims, &timeDimId, &ddVarId)) ERR;
if ( nc_put_var1(grpid, cVarId, &index, &s1Data) ) ERR;
if ( nc_put_var_double(grpid, ddVarId, timeVar)) ERR;
if ( nc_close(ncid)) ERR;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_grp_ncid(ncid, GRP_NAME, &grpid)) ERR;
if (nc_inq_varid(grpid, D_VARNAME, &ddVarId)) ERR;
if (nc_inq_vardimid(grpid, ddVarId, &dimidIn)) ERR;
if (nc_inq_dimid(ncid, TIME_DIMNAME, &timeDimId)) ERR;
if (nc_inq_dimid(grpid, C_DIMNAME, &cDimId)) ERR;
if (dimidIn == cDimId || cDimId == timeDimId) ERR; /* bug in 4.2.1.1 and earlier */
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing fix of bug in non-coordinate scalar variable with same name as dimension ...");
{
#define DIMNAME "abc"
#define SCALAR_VARNAME DIMNAME
2020-01-06 22:58:11 +08:00
int ncid;
int dimid, varid;
int ndims = 1;
size_t dimsize = 3;
char varname_in[NC_MAX_NAME];
if ( nc_create(FILE_NAME, NC_NETCDF4, &ncid) ) ERR;
if ( nc_def_dim(ncid, DIMNAME, dimsize, &dimid) ) ERR;
if ( nc_def_var(ncid, SCALAR_VARNAME, NC_FLOAT, ndims, &dimid, &varid) ) ERR;
if ( nc_close(ncid)) ERR;
/* Open the file and check varname. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq_varid(ncid, SCALAR_VARNAME, &varid)) ERR;
if (nc_inq_varname(ncid, varid, varname_in)) ERR;
if (strcmp(varname_in, SCALAR_VARNAME) != 0) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing bad inputs to put/get_vara calls...");
{
int ncid, dimid[NDIMS2], varid;
size_t start[NDIMS2] = {0, 0}, count[NDIMS2] = {NX, NY};
ptrdiff_t stride[NDIMS2] = {1, 1};
double double_data[NX * NY];
/* Create file with two dims, one 2D var. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, ZD1_NAME, NX, &dimid[0])) ERR;
if (nc_def_dim(ncid, D2_NAME, NY, &dimid[1])) ERR;
if (nc_def_var(ncid, ZD1_NAME, NC_DOUBLE, NDIMS2, dimid, &varid)) ERR;
if (nc_enddef(ncid)) ERR;
/* Try to write some data, but fail. */
if (nc_put_vara_double(ncid + MILLION, 0, start, count, double_data) != NC_EBADID) ERR;
if (nc_put_vara_double(ncid + TEST_VAL_42, 0, start, count, double_data) != NC_EBADID) ERR;
/* Write some data. */
if (nc_put_vars_double(ncid, varid, start, count, NULL, double_data)) ERR;
if (nc_put_vars_double(ncid, varid, start, count, stride, double_data)) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
#ifdef HAVE_H5Z_SZIP
2020-01-06 22:58:11 +08:00
printf("**** testing simple szip filter setup...");
{
int ncid;
int dimid;
int varid;
unsigned int params[NUM_PARAMS_IN];
2020-02-06 22:36:10 +08:00
int options_mask, pixels_per_block;
2020-01-06 22:58:11 +08:00
size_t nparams;
unsigned int filterid;
unsigned int params_out[NUM_PARAMS_OUT];
2020-01-06 23:01:28 +08:00
/* Create a netcdf-4 file with one dimensions. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_1, &dimid)) ERR;
2020-02-07 19:37:04 +08:00
/* Add a var. */
2020-01-06 23:01:28 +08:00
if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &dimid, &varid)) ERR;
2020-02-07 19:37:04 +08:00
/* Turn on szip filter. */
params[0] = NC_SZIP_NN; /* options_mask */
2020-02-06 22:36:10 +08:00
params[1] = NC_SZIP_EC_BPP_IN; /* pixels_per_block */
2020-01-06 23:01:28 +08:00
if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
if (nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN, params)) ERR;
if (nc_close(ncid)) ERR;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
2020-02-07 19:37:04 +08:00
/* Check szip settings. */
2020-02-06 22:36:10 +08:00
if (nc_inq_var_szip(ncid, varid, &options_mask, &pixels_per_block)) ERR;
/* H5Zszip code will sometimes bump the pixels_per_block from 32 to 64
2020-01-06 23:01:28 +08:00
and may add other flags to the options_mask */
if (!(options_mask & NC_SZIP_NN)) ERR;
2020-02-06 22:36:10 +08:00
if (pixels_per_block != NC_SZIP_EC_BPP_IN && pixels_per_block != NC_SZIP_EC_BPP_OUT)
2020-01-06 23:01:28 +08:00
ERR;
/* Also check using nc_inq_var_filter */
if (nc_inq_var_filter(ncid, varid, &filterid, &nparams, params_out)) ERR;
Add support for multiple filters per variable. re: https://github.com/Unidata/netcdf-c/issues/1584 Support has been added for multiple filters per variable. This affects a number of components in netcdf. The new APIs are documented in NUG/filters.md. The primary changes are: * A set of new functions are provided (see __include/netcdf_filter.h__). - Obtain a list of the filters associated with a variable - Obtain the parameters for a specific filter. * The existing __nc_inq_var_filter__ function now returns info about the first defined filter. * The utilities (ncgen, ncdump, and nccopy) now support an extended format for specifying a sequence of filters. The general form is __<filter>|<filter>..._. * The ncdump **_Filter** attribute now dumps a list of all the filters associated with a variable using the above new format. * Filter specifications can now use a filter name instead of number for filters known to the netcdf library, which in turn is taken from the HDF5 filter registration page. * New errors are defined: NC_EFILTER and NC_ENOFILTER. The latter is returned if an attempt is made to access an unknown filter. * Internally, the dispatch table has been extended to add a function to handle all of the filter functions. * New, filter-related, tests were added to nc_test4. * A new plugin was added to the plugins directory to help with testing. Notes: 1. The shuffle and fletcher32 filters are not part of the multifilter system. Misc. changes: 1. A debug module was added to libhdf5 to help catch error locations.
2020-02-17 03:59:33 +08:00
if (filterid != H5_FILTER_SZIP || nparams != 2) ERR;
2020-01-06 23:01:28 +08:00
/* According to H5Zszip, the mapping should be as follows */
Add support for multiple filters per variable. re: https://github.com/Unidata/netcdf-c/issues/1584 Support has been added for multiple filters per variable. This affects a number of components in netcdf. The new APIs are documented in NUG/filters.md. The primary changes are: * A set of new functions are provided (see __include/netcdf_filter.h__). - Obtain a list of the filters associated with a variable - Obtain the parameters for a specific filter. * The existing __nc_inq_var_filter__ function now returns info about the first defined filter. * The utilities (ncgen, ncdump, and nccopy) now support an extended format for specifying a sequence of filters. The general form is __<filter>|<filter>..._. * The ncdump **_Filter** attribute now dumps a list of all the filters associated with a variable using the above new format. * Filter specifications can now use a filter name instead of number for filters known to the netcdf library, which in turn is taken from the HDF5 filter registration page. * New errors are defined: NC_EFILTER and NC_ENOFILTER. The latter is returned if an attempt is made to access an unknown filter. * Internally, the dispatch table has been extended to add a function to handle all of the filter functions. * New, filter-related, tests were added to nc_test4. * A new plugin was added to the plugins directory to help with testing. Notes: 1. The shuffle and fletcher32 filters are not part of the multifilter system. Misc. changes: 1. A debug module was added to libhdf5 to help catch error locations.
2020-02-17 03:59:33 +08:00
if (!(options_mask & NC_SZIP_NN)) ERR;
if (pixels_per_block != NC_SZIP_EC_BPP_IN && pixels_per_block != NC_SZIP_EC_BPP_OUT)
ERR;
2020-01-06 23:01:28 +08:00
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing simple szip function use...");
{
int ncid;
int dimid;
int varid;
2020-02-06 22:36:10 +08:00
int options_mask, pixels_per_block;
2020-01-06 23:01:28 +08:00
size_t nparams;
unsigned int filterid;
unsigned int params_out[NUM_PARAMS_OUT];
unsigned int tmp;
2020-01-06 22:58:11 +08:00
/* Create a netcdf-4 file with one dimensions. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_1, &dimid)) ERR;
/* Add a var. */
2020-01-06 22:58:11 +08:00
if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &dimid, &varid)) ERR;
/* Check szip filter settings. */
2020-02-06 22:36:10 +08:00
if (nc_inq_var_szip(ncid, varid, &options_mask, &pixels_per_block)) ERR;
if (options_mask != 0 || pixels_per_block != 0) ERR;
/* Turn on szip filter. */
if (nc_def_var_szip(ncid, varid, NC_SZIP_NN, NC_SZIP_EC_BPP_IN)) ERR;
/* Check szip filter settings. */
2020-02-06 22:36:10 +08:00
if (nc_inq_var_szip(ncid, varid, &options_mask, &pixels_per_block)) ERR;
if (!(options_mask & NC_SZIP_NN)) ERR;
2020-01-06 22:58:11 +08:00
if (nc_close(ncid)) ERR;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
/* The following code should work, but doesn't. See issue 972 in github. */
2020-02-06 22:36:10 +08:00
if (nc_inq_var_szip(ncid, varid, &options_mask, &pixels_per_block)) ERR;
/* H5Zszip code will sometimes bump the pixels_per_block from 32 to 64
2020-01-06 22:58:11 +08:00
and may add other flags to the options_mask */
tmp = options_mask & NC_SZIP_NN;
if (tmp != NC_SZIP_NN) ERR;
2020-02-06 22:36:10 +08:00
if (pixels_per_block != NC_SZIP_EC_BPP_IN && pixels_per_block != NC_SZIP_EC_BPP_OUT)
2020-01-06 22:58:11 +08:00
ERR;
/* Also check using nc_inq_var_filter */
if (nc_inq_var_filter(ncid, varid, &filterid, &nparams, params_out)) ERR;
Add support for multiple filters per variable. re: https://github.com/Unidata/netcdf-c/issues/1584 Support has been added for multiple filters per variable. This affects a number of components in netcdf. The new APIs are documented in NUG/filters.md. The primary changes are: * A set of new functions are provided (see __include/netcdf_filter.h__). - Obtain a list of the filters associated with a variable - Obtain the parameters for a specific filter. * The existing __nc_inq_var_filter__ function now returns info about the first defined filter. * The utilities (ncgen, ncdump, and nccopy) now support an extended format for specifying a sequence of filters. The general form is __<filter>|<filter>..._. * The ncdump **_Filter** attribute now dumps a list of all the filters associated with a variable using the above new format. * Filter specifications can now use a filter name instead of number for filters known to the netcdf library, which in turn is taken from the HDF5 filter registration page. * New errors are defined: NC_EFILTER and NC_ENOFILTER. The latter is returned if an attempt is made to access an unknown filter. * Internally, the dispatch table has been extended to add a function to handle all of the filter functions. * New, filter-related, tests were added to nc_test4. * A new plugin was added to the plugins directory to help with testing. Notes: 1. The shuffle and fletcher32 filters are not part of the multifilter system. Misc. changes: 1. A debug module was added to libhdf5 to help catch error locations.
2020-02-17 03:59:33 +08:00
if (filterid != H5_FILTER_SZIP || nparams != 2) ERR;
2020-01-06 22:58:11 +08:00
/* According to H5Zszip, the mapping should be as follows */
if(params_out[0] != options_mask) ERR;
2020-02-06 22:36:10 +08:00
if(params_out[1] != pixels_per_block) ERR;
2020-01-06 22:58:11 +08:00
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing more complex use of szip...");
{
2018-05-13 21:31:46 +08:00
#define D_SMALL "small_dim"
#define D_SMALL_LEN1 100
#define D_MEDIUM "medium_dim"
#define D_MEDIUM_LEN1 D_SMALL_LEN1 * 2
#define D_LARGE "large_dim"
#define D_LARGE_LEN1 D_SMALL_LEN1 * 4
#define V_SMALL "small_var"
#define V_MEDIUM "medium_var"
#define V_LARGE "large_var"
#define NUM_PARAMS 2
Add support for multiple filters per variable. re: https://github.com/Unidata/netcdf-c/issues/1584 Support has been added for multiple filters per variable. This affects a number of components in netcdf. The new APIs are documented in NUG/filters.md. The primary changes are: * A set of new functions are provided (see __include/netcdf_filter.h__). - Obtain a list of the filters associated with a variable - Obtain the parameters for a specific filter. * The existing __nc_inq_var_filter__ function now returns info about the first defined filter. * The utilities (ncgen, ncdump, and nccopy) now support an extended format for specifying a sequence of filters. The general form is __<filter>|<filter>..._. * The ncdump **_Filter** attribute now dumps a list of all the filters associated with a variable using the above new format. * Filter specifications can now use a filter name instead of number for filters known to the netcdf library, which in turn is taken from the HDF5 filter registration page. * New errors are defined: NC_EFILTER and NC_ENOFILTER. The latter is returned if an attempt is made to access an unknown filter. * Internally, the dispatch table has been extended to add a function to handle all of the filter functions. * New, filter-related, tests were added to nc_test4. * A new plugin was added to the plugins directory to help with testing. Notes: 1. The shuffle and fletcher32 filters are not part of the multifilter system. Misc. changes: 1. A debug module was added to libhdf5 to help catch error locations.
2020-02-17 03:59:33 +08:00
2020-01-06 22:58:11 +08:00
int ncid;
int nvars, ndims, ngatts, unlimdimid;
int ndims_in, natts_in, dimids_in;
int small_dimid, medium_dimid, large_dimid;
int small_varid, medium_varid, large_varid;
char var_name_in[NC_MAX_NAME + 1];
nc_type xtype_in;
int options_mask_in, pixels_per_block_in;
2020-01-06 22:58:11 +08:00
long long small_data[D_SMALL_LEN1], small_data_in[D_SMALL_LEN1];
long long medium_data[D_MEDIUM_LEN1], medium_data_in[D_MEDIUM_LEN1];
long long large_data[D_LARGE_LEN1], large_data_in[D_LARGE_LEN1];
unsigned int params[NUM_PARAMS];
int i;
for (i = 0; i < D_SMALL_LEN1; i++)
small_data[i] = i;
for (i = 0; i < D_MEDIUM_LEN1; i++)
medium_data[i] = i;
for (i = 0; i < D_LARGE_LEN1; i++)
large_data[i] = i;
/* Create a netcdf-4 file with three dimensions. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, D_SMALL, D_SMALL_LEN1, &small_dimid)) ERR;
if (nc_def_dim(ncid, D_MEDIUM, D_MEDIUM_LEN1, &medium_dimid)) ERR;
if (nc_def_dim(ncid, D_LARGE, D_LARGE_LEN1, &large_dimid)) ERR;
/* Add three vars. Turn on szip for two of them. */
if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &small_dimid, &small_varid)) ERR;
if (nc_def_var(ncid, V_MEDIUM, NC_INT64, NDIMS1, &medium_dimid, &medium_varid)) ERR;
params[0] = NC_SZIP_NN;
2020-01-06 22:58:11 +08:00
params[1] = 32;
if (nc_def_var_chunking(ncid, medium_varid, NC_CHUNKED, NULL)) ERR;
if (nc_def_var_filter(ncid, medium_varid, H5_FILTER_SZIP, NUM_PARAMS, params)) ERR;
if (nc_def_var(ncid, V_LARGE, NC_INT64, NDIMS1, &large_dimid, &large_varid)) ERR;
params[1] = 32;
if (nc_def_var_chunking(ncid, large_varid, NC_CHUNKED, NULL)) ERR;
if (nc_def_var_filter(ncid, large_varid, H5_FILTER_SZIP, NUM_PARAMS, params)) ERR;
/* Write data. */
if (nc_put_var_longlong(ncid, small_varid, small_data)) ERR;
if (nc_put_var_longlong(ncid, medium_varid, medium_data)) ERR;
if (nc_put_var_longlong(ncid, large_varid, large_data)) ERR;
if (nc_close(ncid)) ERR;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
if (nvars != 3 || ndims != 3 || ngatts != 0 || unlimdimid != -1) ERR;
if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims_in, &dimids_in, &natts_in)) ERR;
if (strcmp(var_name_in, V_SMALL) || xtype_in != NC_INT64 || ndims_in != 1 ||
natts_in != 0) ERR;
/* Make sure we have the szip settings we expect. */
if (nc_inq_var_szip(ncid, small_varid, &options_mask_in, &pixels_per_block_in)) ERR;
if (options_mask_in != 0 || pixels_per_block_in !=0) ERR;
if (nc_inq_var_szip(ncid, medium_varid, &options_mask_in, &pixels_per_block_in)) ERR;
if (!(options_mask_in & NC_SZIP_NN)) ERR;
if (nc_inq_var_szip(ncid, large_varid, &options_mask_in, &pixels_per_block_in)) ERR;
if (!(options_mask_in & NC_SZIP_NN)) ERR;
2020-01-06 22:58:11 +08:00
/* Read data. */
if (nc_get_var_longlong(ncid, small_varid, small_data_in)) ERR;
if (nc_get_var_longlong(ncid, medium_varid, medium_data_in)) ERR;
if (nc_get_var_longlong(ncid, large_varid, large_data_in)) ERR;
/* Check data. */
for (i = 0; i < D_SMALL_LEN1; i++)
if (small_data[i] != small_data_in[i]) ERR;
for (i = 0; i < D_MEDIUM_LEN1; i++)
if (medium_data[i] != medium_data_in[i]) ERR;
for (i = 0; i < D_LARGE_LEN1; i++)
if (large_data[i] != large_data_in[i]) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
2020-02-07 20:01:06 +08:00
printf("**** testing attempts to use both zlib and szip...");
{
int ncid;
int dimid;
int varid;
unsigned int params[NUM_PARAMS_IN] = {NC_SZIP_NN,
2020-02-07 20:01:06 +08:00
NC_SZIP_EC_BPP_IN};
/* Create a netcdf-4 file with one dimensions. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_1, &dimid)) ERR;
/* Add a var. */
if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &dimid, &varid)) ERR;
if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
/* Turn on zlib. */
if (nc_def_var_deflate(ncid, varid, 0, 1, 3)) ERR;
/* Try to turn on szip filter - it will fail. */
if (nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN,
params) != NC_EINVAL) ERR;
2020-02-07 20:01:06 +08:00
if (nc_close(ncid)) ERR;
/* Create a netcdf-4 file with one dimensions. */
2020-02-07 20:01:06 +08:00
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_1, &dimid)) ERR;
/* Add a var. */
if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &dimid, &varid)) ERR;
if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
/* Turn on szip. */
if (nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN,
params)) ERR;
2020-02-07 20:01:06 +08:00
/* Try to turn on zlib filter - it will fail. */
2020-02-07 20:01:06 +08:00
if (nc_def_var_deflate(ncid, varid, 0, 1, 3) != NC_EINVAL) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
#define SHORT_DIM_LEN_1 12
#define NUM_MASK 2
2020-02-07 22:05:29 +08:00
#define NUM_PPB 10
#define FIRST_VALID 5
printf("**** testing different values for szip params...");
{
int ncid;
int dimid;
int varid;
int option_mask[NUM_MASK] = {NC_SZIP_NN, NC_SZIP_EC};
2020-02-07 22:05:29 +08:00
int pixels_per_block[NUM_PPB] = {1, 34, 14, 16, 24, 2, 4, 6, 10, 12};
int option_mask_in, pixels_per_block_in;
int m, p, ret;
/* Try different option masks. */
for (m = 0; m < NUM_MASK; m++)
{
/* Try different option masks. */
for (p = 0; p < NUM_PPB; p++)
{
/* Create a netcdf-4 file with one dimensions. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, SHORT_DIM_LEN_1, &dimid)) ERR;
/* Add a var. */
if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &dimid, &varid)) ERR;
/* Try to turn on szip filter. */
ret = nc_def_var_szip(ncid, varid, option_mask[m], pixels_per_block[p]);
if (ret != (p < FIRST_VALID ? NC_EINVAL : NC_NOERR)) ERR;
/* Check szip filter settings. */
if (nc_inq_var_szip(ncid, varid, &option_mask_in, &pixels_per_block_in)) ERR;
if (p < FIRST_VALID)
{
if (option_mask_in) ERR;
}
else
{
if (!(option_mask[m] & option_mask_in)) ERR;
}
if (nc_close(ncid)) ERR;
{
unsigned int params_in[NUM_PARAMS_OUT];
size_t nparams;
unsigned int filterid;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
Add support for multiple filters per variable. re: https://github.com/Unidata/netcdf-c/issues/1584 Support has been added for multiple filters per variable. This affects a number of components in netcdf. The new APIs are documented in NUG/filters.md. The primary changes are: * A set of new functions are provided (see __include/netcdf_filter.h__). - Obtain a list of the filters associated with a variable - Obtain the parameters for a specific filter. * The existing __nc_inq_var_filter__ function now returns info about the first defined filter. * The utilities (ncgen, ncdump, and nccopy) now support an extended format for specifying a sequence of filters. The general form is __<filter>|<filter>..._. * The ncdump **_Filter** attribute now dumps a list of all the filters associated with a variable using the above new format. * Filter specifications can now use a filter name instead of number for filters known to the netcdf library, which in turn is taken from the HDF5 filter registration page. * New errors are defined: NC_EFILTER and NC_ENOFILTER. The latter is returned if an attempt is made to access an unknown filter. * Internally, the dispatch table has been extended to add a function to handle all of the filter functions. * New, filter-related, tests were added to nc_test4. * A new plugin was added to the plugins directory to help with testing. Notes: 1. The shuffle and fletcher32 filters are not part of the multifilter system. Misc. changes: 1. A debug module was added to libhdf5 to help catch error locations.
2020-02-17 03:59:33 +08:00
if (nc_inq_var_szip(ncid, varid, &option_mask_in, &pixels_per_block_in))
ERR;
if (p < FIRST_VALID)
{
if (option_mask_in) ERR;
}
else
{
if (!(option_mask[m] & option_mask_in)) ERR;
}
/* Also check using nc_inq_var_filter */
Add support for multiple filters per variable. re: https://github.com/Unidata/netcdf-c/issues/1584 Support has been added for multiple filters per variable. This affects a number of components in netcdf. The new APIs are documented in NUG/filters.md. The primary changes are: * A set of new functions are provided (see __include/netcdf_filter.h__). - Obtain a list of the filters associated with a variable - Obtain the parameters for a specific filter. * The existing __nc_inq_var_filter__ function now returns info about the first defined filter. * The utilities (ncgen, ncdump, and nccopy) now support an extended format for specifying a sequence of filters. The general form is __<filter>|<filter>..._. * The ncdump **_Filter** attribute now dumps a list of all the filters associated with a variable using the above new format. * Filter specifications can now use a filter name instead of number for filters known to the netcdf library, which in turn is taken from the HDF5 filter registration page. * New errors are defined: NC_EFILTER and NC_ENOFILTER. The latter is returned if an attempt is made to access an unknown filter. * Internally, the dispatch table has been extended to add a function to handle all of the filter functions. * New, filter-related, tests were added to nc_test4. * A new plugin was added to the plugins directory to help with testing. Notes: 1. The shuffle and fletcher32 filters are not part of the multifilter system. Misc. changes: 1. A debug module was added to libhdf5 to help catch error locations.
2020-02-17 03:59:33 +08:00
ret = nc_inq_var_filter(ncid, varid, &filterid, &nparams, params_in);
if (p < FIRST_VALID) {
Mostly revert the filter code to reduce its complexity of use. re: https://github.com/Unidata/netcdf-c/issues/1836 Revert the internal filter code to simplify it. From the user's point of view, the only visible changes should be: 1. The functions that convert text to filter specs have had their signature reverted and have been moved to netcdf_aux.h 2. Some filter API functions now return NC_ENOFILTER when inquiry is made about some filter. Internally,the dispatch table has been modified to get rid of the filter_actions entry and associated complex structures. It has been replaced with inq_var_filter_ids and inq_var_filter_info entries and the dispatch table version has been bumped to 3. Corresponding NOOP and NOTNC4 functions were added to libdispatch/dnotnc4.c. Also, the filter_action entries in dispatch tables were replaced for all dispatch code bases (HDF5, DAP2, etc). This should only impact UDF users. In the process, it became clear that the form of the filters field in NC_VAR_INFO_T was format dependent, so I converted it to be of type void* and pushed its management into the various dispatch code bases. Specifically libhdf5 and libnczarr now manage the filters field in their own way. The auxilliary functions for parsing textual filter specifications were moved to netcdf_aux.h and were renamed to the following: * ncaux_h5filterspec_parse * ncaux_h5filterspec_parselist * ncaux_h5filterspec_free * ncaux_h5filter_fix8 Misc. Other Changes: 1. Document NUG/filters.md updated to reflect the changes above. 2. All the old data types (structs and enums) used by filter_actions actions were deleted. The exception is the NC_H5_Filterspec because it is needed by ncaux_h5filterspec_parselist. 3. Clientside filters were removed -- another enhancement for which no-one ever asked. 4. The ability to remove filters was itself removed. 5. Some functionality needed by nczarr was moved from libhdf5 to libsrc4 e.g. nc4_find_default_chunksizes 6. All the filterx code was removed 7. ncfilter.h and nc4filter.c no longer used Misc. Unrelated Changes: 1. The nczarr_test makefile clean was leaving some directories; so add clean-local to take care of them.
2020-09-28 02:43:46 +08:00
if(ret != NC_NOERR || filterid != 0) ERR;
}
else
{
Add support for multiple filters per variable. re: https://github.com/Unidata/netcdf-c/issues/1584 Support has been added for multiple filters per variable. This affects a number of components in netcdf. The new APIs are documented in NUG/filters.md. The primary changes are: * A set of new functions are provided (see __include/netcdf_filter.h__). - Obtain a list of the filters associated with a variable - Obtain the parameters for a specific filter. * The existing __nc_inq_var_filter__ function now returns info about the first defined filter. * The utilities (ncgen, ncdump, and nccopy) now support an extended format for specifying a sequence of filters. The general form is __<filter>|<filter>..._. * The ncdump **_Filter** attribute now dumps a list of all the filters associated with a variable using the above new format. * Filter specifications can now use a filter name instead of number for filters known to the netcdf library, which in turn is taken from the HDF5 filter registration page. * New errors are defined: NC_EFILTER and NC_ENOFILTER. The latter is returned if an attempt is made to access an unknown filter. * Internally, the dispatch table has been extended to add a function to handle all of the filter functions. * New, filter-related, tests were added to nc_test4. * A new plugin was added to the plugins directory to help with testing. Notes: 1. The shuffle and fletcher32 filters are not part of the multifilter system. Misc. changes: 1. A debug module was added to libhdf5 to help catch error locations.
2020-02-17 03:59:33 +08:00
if (filterid != H5_FILTER_SZIP || nparams != 2) ERR;
/* According to H5Zszip, the mapping should be as follows */
if(params_in[0] != option_mask_in) ERR;
if(params_in[1] != pixels_per_block_in) ERR;
}
if (nc_close(ncid)) ERR;
}
} /* next PPB */
} /* next mask */
}
SUMMARIZE_ERR;
#else
2020-02-07 19:32:57 +08:00
/* This code is run if szip is not present in HDF5. It checks that
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
* nc_def_var_szip() returns NC_ENOFILTER in that case. */
2020-01-06 22:58:11 +08:00
printf("**** testing szip handling when szip not built...");
{
int ncid;
int dimid;
int varid;
unsigned int params[NUM_PARAMS_IN];
/* Create a netcdf-4 file with one dimensions. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_1, &dimid)) ERR;
/* Add a var. Try to turn on szip filter, but it will return
* error. */
if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &dimid, &varid)) ERR;
params[0] = NC_SZIP_NN; /* options_mask */
2020-02-06 22:36:10 +08:00
params[1] = NC_SZIP_EC_BPP_IN; /* pixels_per_block */
2020-01-06 22:58:11 +08:00
if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
{ int stat;
if ((stat = nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN,
params)) != NC_ENOFILTER)
Add filter support to NCZarr Filter support has three goals: 1. Use the existing HDF5 filter implementations, 2. Allow filter metadata to be stored in the NumCodecs metadata format used by Zarr, 3. Allow filters to be used even when HDF5 is disabled Detailed usage directions are define in docs/filters.md. For now, the existing filter API is left in place. So filters are defined using ''nc_def_var_filter'' using the HDF5 style where the id and parameters are unsigned integers. This is a big change since filters affect many parts of the code. In the following, the terms "compressor" and "filter" and "codec" are generally used synonomously. ### Filter-Related Changes: * In order to support dynamic loading of shared filter libraries, a new library was added in the libncpoco directory; it helps to isolate dynamic loading across multiple platforms. * Provide a json parsing library for use by plugins; this is created by merging libdispatch/ncjson.c with include/ncjson.h. * Add a new _Codecs attribute to allow clients to see what codecs are being used; let ncdump -s print it out. * Provide special headers to help support compilation of HDF5 filters when HDF5 is not enabled: netcdf_filter_hdf5_build.h and netcdf_filter_build.h. * Add a number of new test to test the new nczarr filters. * Let ncgen parse _Codecs attribute, although it is ignored. ### Plugin directory changes: * Add support for the Blosc compressor; this is essential because it is the most common compressor used in Zarr datasets. This also necessitated adding a CMake FindBlosc.cmake file * Add NCZarr support for the big-four filters provided by HDF5: shuffle, fletcher32, deflate (zlib), and szip * Add a Codec defaulter (see docs/filters.md) for the big four filters. * Make plugins work with windows by properly adding __declspec declaration. ### Misc. Non-Filter Changes * Replace most uses of USE_NETCDF4 (deprecated) with USE_HDF5. * Improve support for caching * More fixes for path conversion code * Fix misc. memory leaks * Add new utility -- ncdump/ncpathcvt -- that does more or less the same thing as cygpath. * Add a number of new test to test the non-filter fixes. * Update the parsers * Convert most instances of '#ifdef _MSC_VER' to '#ifdef _WIN32'
2021-09-03 07:04:26 +08:00
ERR;
}
if (nc_def_var_szip(ncid, varid, NC_SZIP_NN,
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
NC_SZIP_EC_BPP_IN) != NC_ENOFILTER) ERR;
2020-01-06 22:58:11 +08:00
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
2022-03-15 02:39:37 +08:00
#endif /* HAVE_H5Z_SZIP */
2020-01-06 22:58:11 +08:00
FINAL_RESULTS;
2010-06-03 21:24:43 +08:00
}