mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-04-18 18:20:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Allow in-line definition of filters

Browse Source 
Priority: Low

re: issue https://github.com/Unidata/netcdf-c/issues/1329

HDF5 has the ability to programmatically define new filters,
as opposed to using HDF5_PLUGIN_PATH env variable.
This PR adds support for that feature.
Not clear how useful this is, though.
See docs/filters.md for details.
This commit is contained in:
Dennis Heimbigner 2019-03-21 11:33:27 -06:00
parent a25236eac4
commit 8d0bced60d
12 changed files with 1016 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
docs/filters.md
View File

@ -508,6 +508,85 @@ for use by client programs and by filter implementations.
Examples of the use of these functions can be seen in the test program
*nc_test4/tst_filterparser.c*.
Appendix B. Programmatic Filter Definition {#filters_programmatic}
==========
HDF5 provides an API [6] to allow for the programmatic definition
of filters -- as opposed to using the HDF5_PLUGIN_PATH environment variable.
The idea is that instead of using dynamic shared libraries, the filter code
is compiled into the application and the relevant information
(namely an instance of *H5Z_class2_t*) is passed to the HDF5 library API.
Because it is anticipated that in the future, other plugin formats
will be used, this netcdf-c API is deliberately more general than
strictly required by HDF5.
## API Concepts
Three concepts are used in this API.
1. Format - this is an integer defining the format of the plugin.
Currently, only *NC_FILTER_FORMAT_HDF5* is defined and corresponds
to the existing HDF5 plugin format.
2. ID - this is an integer that is a unique identifier for the filter.
This value is interpreted in the context of the format, so the same
id might be assigned to different filters if the format is different.
3. The structure *NC_FILTER_INFO* that provides generic information
to the API and has a placeholder for format-specific information.
typedef struct NC_FILTER_INFO {
int version; /* Of this structure */
int format; /* Controls actual type of this structure */
int id; /* Must be unique WRT format */
void* info; /* The filter info as defined by the format. */
} NC_FILTER_INFO;
When the format is the value NC_FILTER_FORMAT_HDF5,
then the info field is a pointer to an instance of
H5Z_class2_t as define in H5Zpublic.h.
The use of void* is, of course, to allow for passing arbitrary objects.
### NetCDF API
The following function signatures are provided (see *netcdf_filter.h*).
1. Register a filter
int nc_filter_register(NC_FILTER_INFO* filter_info);
Register a filter whose format and ID are specified in the 'filter_info'
argument.
2. Unregister a filter
int nc_filter_unregister(int format, int id);
Unregister the filter specified by the id. Note that only
filters registered using 'nc_filter_register' can be unregistered.
3. Inquire about a filter
int nc_filter_inq(int format, int id, NC_FILTER_INFO* filter_info);
Unregister the filter specified by the id. Note that only
filters registered using 'nc_filter_register' can be inquired.
The 'filter_info' is filled with a copy of the original argument to
'nc_filter_register'.
### Example
static const H5Z_class2_t H5Z_REG[1] = {
...
};
...
NC_FILTER_INFO info;
...
info.version = NC_FILTER_INFO_VERSION;
info.format = NC_FILTER_FORMAT_HDF5;
info.id = FILTER_ID;
info.info = (void*)&H5Z_REG[0];
stat = nc_filter_register(&info);
...
memset(&info,0,sizeof(NC_FILTER_INFO));
stat = nc_filter_inq(NC_FILTER_FORMAT_HDF5, FILTER_ID, &info);
...
stat = nc_filter_unregister(NC_FILTER_FORMAT_HDF5, FILTER_ID);
# References {#filters_References}
1. https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf
@ -515,6 +594,7 @@ Examples of the use of these functions can be seen in the test program
3. https://portal.hdfgroup.org/display/support/Contributions#Contributions-filters
4. https://support.hdfgroup.org/services/contributions.html#filters
5. https://support.hdfgroup.org/HDF5/doc/RM/RM_H5.html
6. https://confluence.hdfgroup.org/display/HDF5/Filters
# Point of Contact

9
include/hdf5internal.h
View File

@ -18,6 +18,7 @@
#include "ncdimscale.h"
#include "nc4dispatch.h"
#include "hdf5dispatch.h"
#include "netcdf_filter.h"
#define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10)
@ -51,6 +52,11 @@
/** This is the name of the name HDF5 dimension scale attribute. */
#define HDF5_DIMSCALE_NAME_ATT_NAME "NAME"
/** Define Filter API Operations */
#define FILTER_REG 1
#define FILTER_UNREG 2
#define FILTER_INQ 3
/** Struct to hold HDF5-specific info for the file. */
typedef struct NC_HDF5_FILE_INFO {
hid_t hdfid;
@ -193,4 +199,7 @@ int nc4_hdf5_find_grp_h5_var(int ncid, int varid, NC_FILE_INFO_T **h5,
/* Perform lazy read of the rest of the metadata for a var. */
int nc4_get_var_meta(NC_VAR_INFO_T *var);
/* Define Filter API Function */
int nc4_filter_action(int action, int formatx, int id, NC_FILTER_INFO* info);
#endif /* _HDF5INTERNAL_ */

24
include/netcdf_filter.h
View File

@ -11,6 +11,25 @@
#define H5Z_FILTER_SZIP 4
#endif
/* Define the known filter formats */
#define NC_FILTER_FORMAT_HDF5 1 /* Use the H5Z_class2_t format */
/* Note that this structure can be extended
in the usual C way if the first field of the extended
struct is of type NC_FILTER_INFO
*/
typedef struct NC_FILTER_INFO {
int version; /* Of this structure */
# define NC_FILTER_INFO_VERSION 1
int format; /* Controls actual type of this structure */
int id; /* Must be unique WRT format */
void* info; /* The filter info as defined by the format.
For format == NC_FILTER_FORMAT_HDF5,
this must conform to H5Z_class2_t in H5Zpublic.h;
Defined as void* to avoid specifics.
*/
} NC_FILTER_INFO;
#if defined(__cplusplus)
extern "C" {
#endif
@ -20,6 +39,11 @@ EXTERNL int NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* npar
EXTERNL void NC_filterfix8(unsigned char* mem, int decode);
/* Support direct user defined filters */
EXTERNL int nc_filter_register(NC_FILTER_INFO* filter_info);
EXTERNL int nc_filter_unregister(int format, int id);
EXTERNL int nc_filter_inq(int format, int id, NC_FILTER_INFO* filter_info);
#if defined(__cplusplus)
}
#endif

63
libdispatch/dfilter.c
View File

@ -14,6 +14,10 @@
#include "netcdf.h"
#include "netcdf_filter.h"
#ifdef USE_NETCDF4
#include "hdf5internal.h"
#endif
/*
Common utilities related to filters.
*/
@ -262,3 +266,62 @@ NC_filterfix8(unsigned char* mem, int decode)
/* No action is necessary */
#endif
}
/**************************************************/
/* Support direct user defined filters */
EXTERNL int
nc_filter_register(NC_FILTER_INFO* filter)
{
int stat = NC_NOERR;
if(filter == NULL)
return NC_EINVAL;
switch (filter->format) {
case NC_FILTER_FORMAT_HDF5:
#ifdef USE_NETCDF4
stat = nc4_filter_action(FILTER_REG, filter->format, filter->id, filter);
#else
stat = NC_ENOTBUILT;
#endif
break;
default:
stat = NC_EINVAL;
}
return stat;
}
EXTERNL int
nc_filter_unregister(int fformat, int id)
{
int stat = NC_NOERR;
switch (fformat) {
case NC_FILTER_FORMAT_HDF5:
#ifdef USE_NETCDF4
stat = nc4_filter_action(FILTER_UNREG, fformat, id, NULL);
#else
stat = NC_ENOTBUILT;
#endif
break;
default:
stat = NC_EINVAL;
}
return stat;
}
EXTERNL int
nc_filter_inq(int fformat, int id, NC_FILTER_INFO* filter_info)
{
int stat = NC_NOERR;
switch (fformat) {
case NC_FILTER_FORMAT_HDF5:
#ifdef USE_NETCDF4
stat = nc4_filter_action(FILTER_INQ, fformat, id, filter_info);
#else
stat = NC_ENOTBUILT;
#endif
break;
default:
stat = NC_EINVAL;
}
return stat;
}

124
libhdf5/nc4hdf.c
View File

@ -25,6 +25,11 @@
#define NC_HDF5_MAX_NAME 1024 /**< @internal Max size of HDF5 name. */
/* WARNING: GLOBAL VARIABLE */
/* Define list of registered filters */
static NClist* filters = NULL;
/**
* @internal Flag attributes in a linked list as dirty.
*
@ -2638,3 +2643,122 @@ NC4_walk(hid_t gid, int* countp)
}
return ncstat;
}
/**************************************************/
/* Filter registration support */
static int
filterlookup(int id)
{
int i;
if(filters == NULL)
filters = nclistnew();
for(i=0;i<nclistlength(filters);i++) {
NC_FILTER_INFO* x = nclistget(filters,i);
if(x != NULL && x->id == id) return i; /* return position */
}
return -1;
}
static void
reclaiminfo(NC_FILTER_INFO* info)
{
if(info != NULL)
nullfree(info->info);
nullfree(info);
}
static int
filterremove(int pos)
{
NC_FILTER_INFO* info = NULL;
if(filters == NULL)
filters = nclistnew();
if(pos < 0 || pos >= nclistlength(filters))
return NC_EINVAL;
info = nclistget(filters,pos);
reclaiminfo(info);
nclistremove(filters,pos);
return NC_NOERR;
}
static NC_FILTER_INFO*
dupfilterinfo(NC_FILTER_INFO* info)
{
NC_FILTER_INFO* dup = NULL;
if(info == NULL) goto fail;
if(info->info == NULL) goto fail;
if((dup = calloc(1,sizeof(NC_FILTER_INFO))) == NULL) goto fail;
*dup = *info;
if((dup->info = calloc(1,sizeof(H5Z_class2_t))) == NULL) goto fail;
{
H5Z_class2_t* h5dup = (H5Z_class2_t*)dup->info;
H5Z_class2_t* h5info = (H5Z_class2_t*)info->info;
*h5dup = *h5info;
}
return dup;
fail:
reclaiminfo(dup);
return NULL;
}
int
nc4_filter_action(int op, int format, int id, NC_FILTER_INFO* info)
{
int stat = NC_NOERR;
H5Z_class2_t* h5filterinfo = NULL;
herr_t herr;
int pos = -1;
NC_FILTER_INFO* dup = NULL;
if(format != NC_FILTER_FORMAT_HDF5)
{stat = NC_ENOTNC4; goto done;}
switch (op) {
case FILTER_REG: /* Ignore id argument */
if(info == NULL || info->info == NULL)
{stat = NC_EINVAL; goto done;}
if(info->version != NC_FILTER_INFO_VERSION
|| info->format != NC_FILTER_FORMAT_HDF5)
{stat = NC_ENOTNC4; goto done;}
h5filterinfo = info->info;
/* Another sanity check */
if(info->id != h5filterinfo->id)
{stat = NC_EINVAL; goto done;}
/* See if this filter is already defined */
if((pos = filterlookup(id)) >= 0)
{stat = NC_ENAMEINUSE; goto done;} /* Already defined */
if((herr = H5Zregister(h5filterinfo)) < 0)
{stat = NC_EFILTER; goto done;}
/* Save a copy of the passed in info */
if((dup = dupfilterinfo(info)) == NULL)
{stat = NC_ENOMEM; goto done;}
nclistpush(filters,dup);
break;
case FILTER_UNREG:
if(id <= 0)
{stat = NC_ENOTNC4; goto done;}
/* See if this filter is already defined */
if((pos = filterlookup(id)) < 0)
{stat = NC_EFILTER; goto done;} /* Not defined */
if((herr = H5Zunregister(id)) < 0)
{stat = NC_EFILTER; goto done;}
if((stat=filterremove(pos))) goto done;
break;
case FILTER_INQ:
if(id <= 0)
{stat = NC_ENOTNC4; goto done;}
/* Look up the id in our local table */
if((pos = filterlookup(id)) < 0)
{stat = NC_EFILTER; goto done;} /* Not defined */
if(info != NULL) {
*info = *((NC_FILTER_INFO*)nclistget(filters,pos));
}
break;
default:
{stat = NC_EINTERNAL; goto done;}
}
done:
return stat;
}

3
nc_test4/CMakeLists.txt
View File

@ -28,8 +28,9 @@ IF(BUILD_UTILITIES)
IF(ENABLE_FILTER_TESTING)
build_bin_test(test_filter)
build_bin_test(test_filter_misc)
build_bin_test(test_filter_reg)
ADD_SH_TEST(nc_test4 tst_filter)
SET(NC4_TESTS ${NC4_TESTS} tst_filterparser)
SET(NC4_TESTS ${NC4_TESTS} tst_filterparser test_filter_reg)
ENDIF(ENABLE_FILTER_TESTING)
ENDIF(BUILD_UTILITIES)

8
nc_test4/Makefile.am
View File

@ -109,7 +109,8 @@ if ENABLE_FILTER_TESTING
if BUILD_UTILITIES
extradir =
extra_PROGRAMS = test_filter test_filter_misc
TESTS += tst_filter.sh
check_PROGRAMS = test_filter_reg
TESTS += tst_filter.sh test_filter_reg
endif
endif
@ -144,8 +145,9 @@ tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl tst_elena_*.cdl \
tst_simple*.cdl tst_chunks.cdl pr_A1.* tauu_A1.* usi_01.* thetau_01.* \
tst_*.h5 tst_grp_rename.cdl tst_grp_rename.dmp ref_grp_rename.cdl \
foo1.nc tst_*.h4 test.nc testszip.nc test.h5 szip_dump.cdl \
perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \
floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl
perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \
floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \
testfilter_reg.nc
DISTCLEANFILES = findplugin.sh run_par_test.sh

119
nc_test4/test_filter_misc.c
View File

@ -12,6 +12,8 @@
#include "netcdf.h"
#include "netcdf_filter.h"
#undef TESTODDSIZE
#undef DEBUG
/* The C standard apparently defines all floating point constants as double;
@ -31,25 +33,27 @@ static unsigned int baseline[NPARAMS];
#define MAXDIMS 8
#define DEFAULTACTUALDIMS 4
#define DEFAULTDIMSIZE 4
#define DEFAULTCHUNKSIZE 4
#define TESTFILE "testmisc.nc"
#define spec "32768, -17b, 23ub, -25S, 27US, 77, 93U, 789f, 12345678.12345678d, -9223372036854775807L, 18446744073709551615UL"
static size_t dimsize = DEFAULTDIMSIZE;
static size_t chunksize = DEFAULTCHUNKSIZE;
static size_t actualdims = DEFAULTACTUALDIMS;
static size_t pattern[MAXDIMS];
#ifdef TESTODDSIZE
#define NDIMS 1
static size_t dimsize[NDIMS] = {4};
static size_t chunksize[NDIMS] = {3};
#else
#define NDIMS 4
static size_t dimsize[NDIMS] = {4,4,4,4};
static size_t chunksize[NDIMS] = {4,4,4,4};
#endif
static size_t ndims = NDIMS;
static size_t totalproduct = 1; /* x-product over max dims */
static size_t actualproduct = 1; /* x-product over actualdims */
static size_t chunkproduct = 1; /* x-product over actual chunks */
static size_t dims[MAXDIMS];
static size_t chunks[MAXDIMS];
static size_t pattern[MAXDIMS];
static int nerrs = 0;
@ -105,15 +109,15 @@ verifychunks(void)
{
int i;
int store = -1;
size_t chunksizes[MAXDIMS];
memset(chunksizes,0,sizeof(chunksizes));
CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes));
size_t localchunks[MAXDIMS];
memset(localchunks,0,sizeof(localchunks));
CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks));
if(store != NC_CHUNKED) {
fprintf(stderr,"bad chunk store\n");
return 0;
}
for(i=0;i<actualdims;i++) {
if(chunksizes[i] != chunks[i]) {
for(i=0;i<ndims;i++) {
if(chunksize[i] != localchunks[i]) {
fprintf(stderr,"bad chunk size: %d\n",i);
return 0;
}
@ -126,15 +130,15 @@ create(void)
{
int i;
/* Create a file with one big variable. */
/* Create a file with one big variable, but whose dimensions arte not a multiple of chunksize (to see what happens) */
CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
for(i=0;i<actualdims;i++) {
for(i=0;i<ndims;i++) {
char dimname[1024];
snprintf(dimname,sizeof(dimname),"dim%d",i);
CHECK(nc_def_dim(ncid, dimname, dims[i], &dimids[i]));
CHECK(nc_def_dim(ncid, dimname, dimsize[i], &dimids[i]));
}
CHECK(nc_def_var(ncid, "var", NC_FLOAT, actualdims, dimids, &varid));
CHECK(nc_def_var(ncid, "var", NC_FLOAT, ndims, dimids, &varid));
return NC_NOERR;
}
@ -207,7 +211,7 @@ setchunking(void)
int store;
store = NC_CHUNKED;
CHECK(nc_def_var_chunking(ncid,varid,store,chunks));
CHECK(nc_def_var_chunking(ncid,varid,store,chunksize));
if(!verifychunks())
return NC_EINVAL;
return NC_NOERR;
@ -281,8 +285,19 @@ showparameters(void)
fprintf(stderr," %u",params[i]);
}
fprintf(stderr,"\n");
for(i=0;i<actualdims;i++)
fprintf(stderr,"%s%ld",(i==0?" chunks=":","),(unsigned long)chunks[i]);
for(i=0;i<ndims;i++) {
if(i==0)
fprintf(stderr,"dimsizes=%ld",(unsigned long)dimsize[i]);
else
fprintf(stderr,",%ld",(unsigned long)dimsize[i]);
}
fprintf(stderr,"\n");
for(i=0;i<ndims;i++) {
if(i==0)
fprintf(stderr,"chunksizes=%ld",(unsigned long)chunksize[i]);
else
fprintf(stderr,",%ld",(unsigned long)chunksize[i]);
}
fprintf(stderr,"\n");
fflush(stderr);
}
@ -332,6 +347,8 @@ buildbaseline(unsigned int testcasenumber)
val8 = 18446744073709551615UL;
insert(12,&val8,sizeof(val8)); /* 12 unsigned long long */
break;
case 2:
break;
default:
fprintf(stderr,"Unknown testcase number: %d\n",testcasenumber);
abort();
@ -369,6 +386,37 @@ test_test1(void)
return ok;
}
static int
test_test2(void)
{
int ok = 1;
reset();
buildbaseline(2);
fprintf(stderr,"test2: dimsize %% chunksize != 0: compress.\n");
create();
setchunking();
setvarfilter();
showparameters();
CHECK(nc_enddef(ncid));
/* Fill in the array */
fill();
/* write array */
CHECK(nc_put_var(ncid,varid,expected));
CHECK(nc_close(ncid));
fprintf(stderr,"test2: dimsize %% chunksize != 0: decompress.\n");
reset();
openfile();
CHECK(nc_get_var_float(ncid, varid, array));
ok = compare();
CHECK(nc_close(ncid));
return ok;
}
/**************************************************/
/* Utilities */
@ -387,16 +435,16 @@ odom_reset(void)
static int
odom_more(void)
{
return (odom[0] < dims[0]);
return (odom[0] < dimsize[0]);
}
static int
odom_next(void)
{
int i; /* do not make unsigned */
for(i=actualdims-1;i>=0;i--) {
for(i=ndims-1;i>=0;i--) {
odom[i] += 1;
if(odom[i] < dims[i]) break;
if(odom[i] < dimsize[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows*/
odom[i] = 0; /* reset this position*/
}
@ -408,8 +456,8 @@ odom_offset(void)
{
int i;
int offset = 0;
for(i=0;i<actualdims;i++) {
offset *= dims[i];
for(i=0;i<ndims;i++) {
offset *= dimsize[i];
offset += odom[i];
}
return offset;
@ -421,8 +469,8 @@ expectedvalue(void)
int i;
float offset = 0;
for(i=0;i<actualdims;i++) {
offset *= dims[i];
for(i=0;i<ndims;i++) {
offset *= dimsize[i];
offset += odom[i];
}
return offset;
@ -437,12 +485,12 @@ init(int argc, char** argv)
actualproduct = 1;
chunkproduct = 1;
for(i=0;i<MAXDIMS;i++) {
dims[i] = dimsize;
chunks[i] = (pattern[i] == 1 ? 1 : chunksize);
totalproduct *= dims[i];
if(i < actualdims) {
actualproduct *= dims[i];
chunkproduct *= chunks[i];
if(pattern[i] == 1)
chunksize[i] = 1;
totalproduct *= dimsize[i];
if(i < ndims) {
actualproduct *= dimsize[i];
chunkproduct *= chunksize[i];
}
}
/* Allocate max size */
@ -460,5 +508,6 @@ main(int argc, char **argv)
#endif
init(argc,argv);
if(!test_test1()) ERRR;
if(!test_test2()) ERRR;
exit(nerrs > 0?1:0);
}

613
nc_test4/test_filter_reg.c Normal file
View File

@ -0,0 +1,613 @@
/*
Copyright 2018, UCAR/Unidata
See COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <sys/types.h>
#include <hdf5.h>
/* Older versions of the hdf library may define H5PL_type_t here */
#include <H5PLextern.h>
#include "netcdf.h"
#include "netcdf_filter.h"
#undef TESTODDSIZE
#undef DEBUG
#define FILTER_ID 32768
#define MAXERRS 8
#define MAXPARAMS 32
#define MAXDIMS 8
#define TESTFILE "testfilter_reg.nc"
#define NPARAMS 1
#define PARAMVAL 17
#define NDIMS 4
static size_t dimsize[NDIMS] = {4,4,4,4};
static size_t chunksize[NDIMS] = {4,4,4,4};
static size_t ndims = NDIMS;
static size_t totalproduct = 1; /* x-product over max dims */
static size_t actualproduct = 1; /* x-product over actualdims */
static size_t chunkproduct = 1; /* x-product over actual chunks */
static int nerrs = 0;
static int ncid, varid;
static int dimids[MAXDIMS];
static size_t odom[MAXDIMS];
static float* array = NULL;
static float* expected = NULL;
static unsigned int filterid = 0;
static size_t nparams = 0;
static unsigned int params[MAXPARAMS];
static unsigned int baseline[NPARAMS] = {PARAMVAL};
static NC_FILTER_INFO baseinfo;
static const H5Z_class2_t H5Z_REG[1];
/* Forward */
static int filter_test1(void);
static void init(int argc, char** argv);
static void reset(void);
static void odom_reset(void);
static int odom_more(void);
static int odom_next(void);
static int odom_offset(void);
static float expectedvalue(void);
static void verifyparams(void);
#define ERRR do { \
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \
__FILE__, __LINE__); \
nerrs++;\
} while (0)
static int
check(int err,int line)
{
if(err != NC_NOERR) {
fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err));
}
return NC_NOERR;
}
static void
report(const char* msg, int lineno)
{
fprintf(stderr,"fail: line=%d %s\n",lineno,msg);
exit(1);
}
#define CHECK(x) check(x,__LINE__)
#define REPORT(x) report(x,__LINE__)
static int
verifychunks(void)
{
int i;
int store = -1;
size_t localchunks[MAXDIMS];
memset(localchunks,0,sizeof(localchunks));
CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks));
if(store != NC_CHUNKED) {
fprintf(stderr,"bad chunk store\n");
return 0;
}
for(i=0;i<ndims;i++) {
if(chunksize[i] != localchunks[i]) {
fprintf(stderr,"bad chunk size: %d\n",i);
return 0;
}
}
return 1;
}
static int
create(void)
{
int i;
/* Create a file with one big variable */
CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
for(i=0;i<ndims;i++) {
char dimname[1024];
snprintf(dimname,sizeof(dimname),"dim%d",i);
CHECK(nc_def_dim(ncid, dimname, dimsize[i], &dimids[i]));
}
CHECK(nc_def_var(ncid, "var", NC_FLOAT, ndims, dimids, &varid));
return NC_NOERR;
}
static int
verifyfilterinfo(NC_FILTER_INFO* info, NC_FILTER_INFO* base)
{
int stat = NC_NOERR;
if(info->version != base->version)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: version mismatch\n");}
if(info->format != base->format)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: format mismatch\n");}
if(info->id != base->id)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: id mismatch\n");}
if(info->format == NC_FILTER_FORMAT_HDF5) {
#ifdef USE_HDF5
H5Z_class2_t* h5info = (H5Z_class2_t*)info->info;
H5Z_class2_t* h5base = (H5Z_class2_t*)base->info;
if(h5info->version != h5base->version)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: version mismatch\n");}
if(h5info->id != h5base->id)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: id mismatch\n");}
if(h5info->encoder_present != h5base->encoder_present)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: encoder_present mismatch\n");}
if(h5info->decoder_present != h5base->decoder_present)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: decoder_present mismatch\n");}
if(h5info->decoder_present != h5base->decoder_present)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: decoder_present mismatch\n");}
if(strcmp(h5info->name,h5base->name) != 0)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: name mismatch\n");}
if(h5info->can_apply != h5base->can_apply)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: can_apply mismatch\n");}
if(h5info->set_local != h5base->set_local)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: set_local mismatch\n");}
if(h5info->filter != h5base->filter)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: filter mismatch\n");}
#else
stat = NC_ENOTBUILT; fprintf(stderr,"Unknown format\n")}
#endif
} else
{stat = NC_EINVAL; fprintf(stderr,"Unknown format\n");}
return stat;
}
static void
registerfilter(void)
{
NC_FILTER_INFO inqinfo;
baseinfo.version = NC_FILTER_INFO_VERSION;
baseinfo.format = NC_FILTER_FORMAT_HDF5;
baseinfo.id = FILTER_ID;
baseinfo.info = (void*)&H5Z_REG[0];
CHECK(nc_filter_register(&baseinfo));
/* Verify by inquiry */
memset(&inqinfo,0,sizeof(NC_FILTER_INFO));
CHECK((nc_filter_inq(NC_FILTER_FORMAT_HDF5, FILTER_ID, &inqinfo)));
CHECK((verifyfilterinfo(&inqinfo,&baseinfo)));
}
static void
unregisterfilter(void)
{
int stat = NC_NOERR;
NC_FILTER_INFO inqinfo;
/* Verify that the filter info is still good */
memset(&inqinfo,0,sizeof(NC_FILTER_INFO));
CHECK((stat = nc_filter_inq(NC_FILTER_FORMAT_HDF5, FILTER_ID, &inqinfo)));
CHECK((verifyfilterinfo(&inqinfo,&baseinfo)));
/* Unregister */
CHECK((stat = nc_filter_unregister(NC_FILTER_FORMAT_HDF5, FILTER_ID)));
/* Inq again to verify unregistered */
stat = nc_filter_inq(NC_FILTER_FORMAT_HDF5, FILTER_ID, NULL);
if(stat != NC_EFILTER) {
fprintf(stderr,"unregister: failed\n");
CHECK(NC_EFILTER);
}
}
static void
setvarfilter(void)
{
/* NOOP the params */
CHECK(nc_def_var_filter(ncid,varid,FILTER_ID,NPARAMS,baseline));
verifyparams();
}
static void
verifyparams(void)
{
int i;
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params));
if(filterid != FILTER_ID) REPORT("id mismatch");
if(nparams != NPARAMS) REPORT("nparams mismatch");
for(i=0;i<nparams;i++) {
if(params[i] != baseline[i])
REPORT("param mismatch");
}
}
static int
openfile(void)
{
unsigned int* params = NULL;
/* Open the file and check it. */
CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid));
CHECK(nc_inq_varid(ncid, "var", &varid));
/* Check the compression algorithm */
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,NULL));
if(nparams > 0) {
params = (unsigned int*)malloc(sizeof(unsigned int)*nparams);
if(params == NULL)
return NC_ENOMEM;
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params));
}
if(filterid != FILTER_ID) {
fprintf(stderr,"open: test id mismatch: %d\n",filterid);
return NC_EFILTER;
}
if(nparams != NPARAMS) {
size_t i;
unsigned int inqparams[MAXPARAMS];
fprintf(stderr,"nparams mismatch\n");
for(nerrs=0,i=0;i<nparams;i++) {
if(inqparams[i] != baseline[i]) {
fprintf(stderr,"open: testparam mismatch: %ld\n",(unsigned long)i);
nerrs++;
}
}
}
if(nerrs > 0) return NC_EFILTER;
if(params) free(params);
/* Verify chunking */
if(!verifychunks())
return 0;
fflush(stderr);
return 1;
}
static int
setchunking(void)
{
int store;
store = NC_CHUNKED;
CHECK(nc_def_var_chunking(ncid,varid,store,chunksize));
if(!verifychunks())
return NC_EINVAL;
return NC_NOERR;
}
static void
fill(void)
{
odom_reset();
if(1) {
int i;
if(actualproduct <= 1) abort();
for(i=0;i<actualproduct;i++)
expected[i] = (float)i;
} else {
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
expected[offset] = expect;
odom_next();
}
}
}
static int
compare(void)
{
int errs = 0;
fprintf(stderr,"data comparison: |array|=%ld\n",(unsigned long)actualproduct);
if(1)
{
int i;
for(i=0;i<actualproduct;i++) {
if(expected[i] != array[i]) {
fprintf(stderr,"data mismatch: array[%d]=%f expected[%d]=%f\n",
i,array[i],i,expected[i]);
errs++;
if(errs >= MAXERRS)
break;
}
}
} else
{
odom_reset();
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
if(array[offset] != expect) {
fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n",
offset,array[offset],expect);
errs++;
if(errs >= MAXERRS)
break;
}
odom_next();
}
}
if(errs == 0)
fprintf(stderr,"no data errors\n");
return (errs == 0);
}
static void
showparameters(void)
{
int i;
fprintf(stderr,"test: nparams=%ld: params=",(unsigned long)nparams);
for(i=0;i<nparams;i++) {
fprintf(stderr," %u",params[i]);
}
fprintf(stderr,"\n");
for(i=0;i<ndims;i++) {
if(i==0)
fprintf(stderr,"dimsizes=%ld",(unsigned long)dimsize[i]);
else
fprintf(stderr,",%ld",(unsigned long)dimsize[i]);
}
fprintf(stderr,"\n");
for(i=0;i<ndims;i++) {
if(i==0)
fprintf(stderr,"chunksizes=%ld",(unsigned long)chunksize[i]);
else
fprintf(stderr,",%ld",(unsigned long)chunksize[i]);
}
fprintf(stderr,"\n");
fflush(stderr);
}
static int
filter_test1(void)
{
int ok = 1;
reset();
fprintf(stderr,"test1: compression.\n");
create();
setchunking();
setvarfilter();
showparameters();
CHECK(nc_enddef(ncid));
/* Fill in the array */
fill();
/* write array */
CHECK(nc_put_var(ncid,varid,expected));
CHECK(nc_close(ncid));
fprintf(stderr,"test1: decompression.\n");
reset();
openfile();
CHECK(nc_get_var_float(ncid, varid, array));
ok = compare();
CHECK(nc_close(ncid));
return ok;
}
/**************************************************/
/* Utilities */
static void
reset()
{
memset(array,0,sizeof(float)*actualproduct);
}
static void
odom_reset(void)
{
memset(odom,0,sizeof(odom));
}
static int
odom_more(void)
{
return (odom[0] < dimsize[0]);
}
static int
odom_next(void)
{
int i; /* do not make unsigned */
for(i=ndims-1;i>=0;i--) {
odom[i] += 1;
if(odom[i] < dimsize[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows*/
odom[i] = 0; /* reset this position*/
}
return 1;
}
static int
odom_offset(void)
{
int i;
int offset = 0;
for(i=0;i<ndims;i++) {
offset *= dimsize[i];
offset += odom[i];
}
return offset;
}
static float
expectedvalue(void)
{
int i;
float offset = 0;
for(i=0;i<ndims;i++) {
offset *= dimsize[i];
offset += odom[i];
}
return offset;
}
static void
init(int argc, char** argv)
{
int i;
/* Setup various variables */
totalproduct = 1;
actualproduct = 1;
chunkproduct = 1;
for(i=0;i<MAXDIMS;i++) {
totalproduct *= dimsize[i];
if(i < ndims) {
actualproduct *= dimsize[i];
chunkproduct *= chunksize[i];
}
}
/* Allocate max size */
array = (float*)calloc(1,sizeof(float)*actualproduct);
expected = (float*)calloc(1,sizeof(float)*actualproduct);
/* Register the filter */
registerfilter();
}
/**************************************************/
int
main(int argc, char **argv)
{
#ifdef DEBUG
H5Eprint(stderr);
nc_set_log_level(1);
#endif
init(argc,argv);
if(!filter_test1()) ERRR;
/* Unregister filter */
unregisterfilter();
exit(nerrs > 0?1:0);
}
/**************************************************/
/* In-line filter code */
#define H5Z_FILTER_REG FILTER_ID
#ifndef DLL_EXPORT
#define DLL_EXPORT
#endif
static int paramcheck(size_t nparams, const unsigned int* params);
/* Forward */
static int paramcheck(size_t nparams, const unsigned int* params);
/* Make this explicit */
/*
* The "can_apply" callback returns positive a valid combination, zero for an
* invalid combination and negative for an error.
*/
static htri_t
H5Z_reg_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id)
{
return 1; /* Assume it can always apply */
}
/*
As a filter, it is the identity function,
passing input to output unchanged.
*/
size_t
H5Z_filter_reg(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
void* newbuf;
if(cd_nelmts == 0)
goto fail;
if(!paramcheck(cd_nelmts,cd_values))
goto fail;
fprintf(stderr,"nbytes=%ld\n",(long)nbytes);
if (flags & H5Z_FLAG_REVERSE) {
/* Replace buffer */
#ifdef HDF5_HAS_ALLOCATE_MEMORY
newbuf = H5allocate_memory(*buf_size,0);
#else
newbuf = malloc(*buf_size * sizeof(void));
#endif
if(newbuf == NULL) abort();
memcpy(newbuf,*buf,*buf_size);
/* reclaim old buffer */
#ifdef HDF5_HAS_H5FREE
H5free_memory(*buf);
#else
free(*buf);
#endif
*buf = newbuf;
} else {
/* Replace buffer */
#ifdef HDF5_HAS_ALLOCATE_MEMORY
newbuf = H5allocate_memory(*buf_size,0);
#else
newbuf = malloc(*buf_size * sizeof(void));
#endif
if(newbuf == NULL) abort();
memcpy(newbuf,*buf,*buf_size);
/* reclaim old buffer */
#ifdef HDF5_HAS_H5FREE
H5free_memory(*buf);
#else
free(*buf);
#endif
*buf = newbuf;
}
return *buf_size;
fail:
return 0;
}
static int
paramcheck(size_t nparams, const unsigned int* params)
{
if(nparams != 1) {
fprintf(stderr,"Incorrect parameter count: need=1 sent=%ld\n",(unsigned long)nparams);
goto fail;
}
return 1;
fail:
return 0;
}
static const H5Z_class2_t H5Z_REG[1] = {{
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)(H5Z_FILTER_REG), /* Filter id number */
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"registered", /* Filter name for debugging */
(H5Z_can_apply_func_t)H5Z_reg_can_apply, /* The "can apply" callback */
NULL, /* The "set local" callback */
(H5Z_func_t)H5Z_filter_reg, /* The actual filter function */
}};

2
nc_test4/tst_filter.sh
View File

@ -78,7 +78,7 @@ rm -f ./tst_filter.txt
trimleft ./tst_filter2.txt ./tst_filter.txt
rm -f ./tst_filter2.txt
cat >./tst_filter2.txt <<EOF
var:_Filter = "32768,1,239,23,65511,27,77,93,1145389056,3287505826,1097305129,1,2147483648,4294967295,4294967295" ;
var:_Filter = "32768,2,239,23,65511,27,77,93,1145389056,3287505826,1097305129,1,2147483648,4294967295,4294967295" ;
EOF
diff -b -w ./tst_filter.txt ./tst_filter2.txt
echo "*** Pass: parameter passing"

11
plugins/H5Zmisc.c
View File

@ -82,6 +82,7 @@ that the parameters passed to the filter
are correct. Specifically, that endian-ness
is correct. As a filter, it is the identify
function, passing input to output unchanged.
It also prints out the size of each chunk.
Test cases format:
1.The first param is the test index i.e. which test to execute.
@ -102,9 +103,17 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts,
testcase = cd_values[0];
if(testcase == TC_ENDIAN) {
switch (testcase) {
case TC_ENDIAN:
if(!paramcheck(cd_nelmts,cd_values))
goto fail;
break;
case TC_ODDSIZE:
/* Print out the chunk size */
fprintf(stderr,"nbytes = %lld chunk size = %lld\n",(long long)nbytes,(long long)*buf_size);
fflush(stderr);
break;
default: break;
}
if (flags & H5Z_FLAG_REVERSE) {

1
plugins/h5misc.h
View File

@ -19,6 +19,7 @@
typedef enum H5testcase {
TC_NONE = 0,
TC_ENDIAN = 1,
TC_ODDSIZE = 2,
} H5testcase;
/* declare the hdf5 interface */