mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-01-18 15:55:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Allow redefinition of variable filters

Browse Source 
re: Github issue https://github.com/Unidata/netcdf-c/issues/1713

If nc_def_var_filter or nc_def_var_deflate or nc_def_var_szip is
called multiple times with the same filter id, but possibly with
different sets of parameters, then the first invocation is
sticky and later invocations are ignored. The desired behavior
is to have the last invocation be used.

This PR implements that desired behavior, with some special
cases.  If you call nc_def_var_deflate multiple times, then the
last invocation rule applies with respect to deflate. However,
the shuffle filter, if enabled, is always applied just before
applying deflate.

Misc unrelated changes:
1. Make client-side filters be disabled by default
2. Fix the definition of uintptr_t and use in oc2 and libdap4
3. Add some test cases
4. modify filter order tests to use plugin filters rather
   than client-side filters
This commit is contained in:
Dennis Heimbigner 2020-05-11 09:42:31 -06:00
parent e47a2a3037
commit 84c69afca7
26 changed files with 652 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CMakeLists.txt
View File

@ -1182,6 +1182,11 @@ IF(NOT BUILD_SHARED_LIBS)
MESSAGE(WARNING "ENABLE_FILTER_TESTING requires shared libraries. Disabling.")
SET(ENABLE_FILTER_TESTING OFF)
ENDIF()
OPTION(ENABLE_CLIENTSIDE_FILTERS "Enable client-side filter registration." OFF)
IF(NOT ENABLE_FILTER_TESTING)
SET(ENABLE_CLIENTSIDE_FILTERS OFF)
ENDIF()
# Determine whether or not to generate documentation.
OPTION(ENABLE_DOXYGEN "Enable generation of doxygen-based documentation." OFF)
@ -1776,6 +1781,7 @@ ENDIF()
add_subdirectory(liblib)
IF(ENABLE_FILTER_TESTING)
CONFIGURE_FILE(plugins/H5Znoop.c ${CMAKE_SOURCE_DIR}/plugins/H5Znoop1.c COPYONLY)
add_subdirectory(plugins)
ENDIF()
@ -2110,8 +2116,6 @@ install(
# End export files
####
# CPack inclusion must come last.
# INCLUDE(CPack)
INCLUDE(CMakeInstallation.cmake)

28
NUG/filters.md
View File

@ -482,6 +482,34 @@ demonstrate how to build the hdf5 plugin for bzip2.
Notes
==========
Order of Invocation for Multiple Filters
-----------
When multiple filters are defined on a variable, the
order of application, when writing data to the file,
is same as the order in which _nc_def_var_filter_ is called.
When reading a file the order of application is of necessity
the reverse.
There are some special cases.
1. The fletcher32 filter is always applied first, if enabled.
1. If _nc_def_var_filter_ or _nc_def_var_deflate_ or _nc_def_var_szip_
is called multiple times with the same filter id, but possibly
with different sets of parameters, then the position of that filter
in the sequence of applictions does not change. However the last set
of parameters specified is used when actually writing the dataset.
1. Deflate and shuffle -- these two are inextricably linked in the
current API, but have quite different semantics.
If you call _nc_def_var_deflate_ multiple times, then
the previous rule applies with respect to deflate. However,
the shuffle filter, if enabled, is ''always'' applied before
applying any other filters, except fletcher32.
1. If you want to move the location of a filter in the application
sequence, then you must remove it using _nc_var_filter_remove_
or using _nc_def_var_deflate_. The next time you add the filter
back, its position will be at the end of the current sequence.
Memory Allocation Issues
-----------

3
RELEASE_NOTES.md
View File

@ -7,6 +7,9 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.0 - TBD
* [Enhancement] When a filter is applied twice with different
parameters, then the second set is used for writing the dataset
[https://github.com/Unidata/netcdf-c/issues/1713].
* [Bug Fix] Now larger cache settings are used for sequential HDF5 file creates/opens on parallel I/O capable builds; see [Github #1716](https://github.com/Unidata/netcdf-c/issues/1716) for more information.
* [Bug Fix] Add functions to libdispatch/dnotnc4.c to support
dispatch table operations that should work for any dispatch

14
configure.ac
View File

@ -1428,6 +1428,20 @@ enable_filter_testing=no
fi
AM_CONDITIONAL(ENABLE_FILTER_TESTING, [test x$enable_filter_testing = xyes])
# Enable client side filter registration
AC_MSG_CHECKING([If client-side filters are enabled (default off)])
AC_ARG_ENABLE([clientside-filters],
[AS_HELP_STRING([--enable-clientside-filters],
[enable client side filters])],
[],
[enable_clientside_filters=no])
test "x$enable_clientside_filters" = xyes || enable_clientside_filters=no
AC_MSG_RESULT($enable_clientside_filters)
if test "x$enable_clientside_filters" = xyes ; then
AC_DEFINE([ENABLE_CLIENTSIDE_FILTERS], [1], [if true, enable client-side filters])
fi
AM_CONDITIONAL(ENABLE_CLIENTSIDE_FILTERS, [test x$enable_clientside_filters = xyes])
AC_SUBST(NC_LIBS,[$NC_LIBS])
AC_SUBST(HAS_DAP,[$enable_dap])
AC_SUBST(HAS_DAP2,[$enable_dap])

2
include/hdf5internal.h
View File

@ -183,7 +183,7 @@ int nc4_hdf5_get_chunk_cache(int ncid, size_t *sizep, size_t *nelemsp,
/* Define Filter API Function */
int nc4_global_filter_action(int action, unsigned int id, struct NC_FILTER_OBJ_HDF5* infop);
int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* params);
int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* params, struct NC_FILTER_SPEC_HDF5**);
int NC4_hdf5_remove_filter(NC_VAR_INFO_T* var, unsigned int filterid);
/* Support functions for provenance info (defined in nc4hdf.c) */

7
include/ncconfigure.h
View File

@ -104,6 +104,13 @@ typedef unsigned short ushort;
typedef unsigned int uint;
#endif
#ifndef HAVE_UINTPTR_T
#if SIZEOF_VOIDP == 8
typedef unsigned long uintptr_t;
#else
typedef unsigned int uintptr_t;
#endif
#endif
/* Provide a fixed size alternative to off_t or off64_t */
typedef long long fileoffset_t;

2
libdap4/d4curlfunctions.c
View File

@ -9,7 +9,7 @@
#define MAX_REDIRECTS 20L
/* Mnemonic */
#define OPTARG void*
#define OPTARG uintptr_t
/* Condition on libcurl version */
/* Set up an alias as needed */

8
libdispatch/dfile.c
View File

@ -1855,9 +1855,9 @@ NC_create(const char *path0, int cmode, size_t initialsz,
for(p=(const unsigned char*)path0;*p;p++) {if(*p > ' ') break;}
#ifdef WINPATH
/* Need to do path conversion */
path = NCpathcvt(p);
path = NCpathcvt((const char*)p);
#else
path = nulldup(p);
path = nulldup((const char*)p);
#endif
}
@ -1999,8 +1999,8 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
{
/* Skip past any leading whitespace in path */
const unsigned char* p;
for(p=(const unsigned char*)path0;*p;p++) {if(*p > ' ') break;}
const char* p;
for(p=(const char*)path0;*p;p++) {if(*p < 0 || *p > ' ') break;}
#ifdef WINPATH
/* Need to do path conversion */
path = NCpathcvt(p);

2
libdispatch/dfilter.c
View File

@ -381,6 +381,7 @@ NC4_filterfix8(unsigned char* mem, int decode)
}
#ifdef ENABLE_CLIENTSIDE_FILTERS
/* Support direct user defined filters */
/* Use void* to avoid having to include hdf.h*/
@ -442,6 +443,7 @@ nc_filter_client_inq(unsigned int id, void* infop)
#endif
return stat;
}
#endif /*ENABLE_CLIENTSIDE_FILTERS*/
/**
Find the set of filters (if any) associated with a variable.

125
libhdf5/hdf5filter.c
View File

@ -23,23 +23,26 @@
#define FILTERACTIVE 1
/* WARNING: GLOBAL VARIABLE */
/* Define list of registered filters */
static NClist* NC4_registeredfilters = NULL; /** List<NC_FILTER_CLIENT_HDF5*> */
/**************************************************/
/* Filter registration support */
#ifdef ENABLE_CLIENTSIZE_FILTERS
/* WARNING: GLOBAL VARIABLE */
/* Define list of registered filters */
static NClist* NC4_registeredfilters = NULL; /** List<NC_FILTER_CLIENT_HDF5*> */
static int
filterlookup(unsigned int id)
clientfilterlookup(unsigned int id)
{
int i;
if(NC4_registeredfilters == NULL)
NC4_registeredfilters = nclistnew();
for(i=0;i<nclistlength(NC4_registeredfilters);i++) {
NC_FILTER_CLIENT_HDF5* x = nclistget(NC4_registeredfilters,i);
if(x != NULL && x->id == id) return i; /* return position */
if(x != NULL && x->id == id) {
return i; /* return position */
}
}
return -1;
}
@ -77,36 +80,6 @@ fail:
return NULL;
}
int
NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams, unsigned int* inparams)
{
int stat = NC_NOERR;
NC_FILTER_SPEC_HDF5* fi = NULL;
unsigned int* params = NULL;
if(var->filters == NULL) {
if((var->filters = nclistnew())==NULL) return THROW(NC_ENOMEM);
}
if(nparams > 0 && inparams == NULL)
return THROW(NC_EINVAL);
if(inparams != NULL) {
if((params = malloc(sizeof(unsigned int)*nparams)) == NULL)
return THROW(NC_ENOMEM);
memcpy(params,inparams,sizeof(unsigned int)*nparams);
}
if((fi = calloc(1,sizeof(NC_FILTER_SPEC_HDF5))) == NULL)
{nullfree(params); return THROW(NC_ENOMEM);}
fi->active = active;
fi->filterid = id;
fi->nparams = nparams;
fi->params = params;
nclistpush(var->filters,fi);
return THROW(stat);
}
int
nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop)
{
@ -129,7 +102,7 @@ nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop)
if(id != h5filterinfo->id)
{stat = NC_EINVAL; goto done;}
/* See if this filter is already defined */
if((pos = filterlookup(id)) >= 0)
if((pos = clientfilterlookup(id)) >= 0)
{stat = NC_ENAMEINUSE; goto done;} /* Already defined */
if((herr = H5Zregister(h5filterinfo)) < 0)
{stat = NC_EFILTER; goto done;}
@ -144,7 +117,7 @@ nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop)
if(id <= 0)
{stat = NC_ENOTNC4; goto done;}
/* See if this filter is already defined */
if((pos = filterlookup(id)) < 0)
if((pos = clientfilterlookup(id)) < 0)
{stat = NC_ENOFILTER; goto done;} /* Not defined */
if((herr = H5Zunregister(id)) < 0)
{stat = NC_EFILTER; goto done;}
@ -153,7 +126,7 @@ nc4_global_filter_action(int op, unsigned int id, NC_FILTER_OBJ_HDF5* infop)
case NCFILTER_CLIENT_INQ:
if(infop == NULL) goto done;
/* Look up the id in our local table */
if((pos = filterlookup(id)) < 0)
if((pos = clientfilterlookup(id)) < 0)
{stat = NC_ENOFILTER; goto done;} /* Not defined */
elem = (NC_FILTER_CLIENT_HDF5*)nclistget(NC4_registeredfilters,pos);
if(elem == NULL) {stat = NC_EINTERNAL; goto done;}
@ -168,6 +141,61 @@ done:
return THROW(stat);
}
#endif /*ENABLE_CLIENTSIDE_FILTERS*/
/**************************************************/
static int
filterlookup(NC_VAR_INFO_T* var, unsigned int id, NC_FILTER_SPEC_HDF5** fp)
{
int i;
if(var->filters == NULL)
var->filters = nclistnew();
for(i=0;i<nclistlength(var->filters);i++) {
NC_FILTER_SPEC_HDF5* x = nclistget(var->filters,i);
if(x != NULL && x->filterid == id) {
if(fp) *fp = x;
return i; /* return position */
}
}
return -1;
}
int
NC4_hdf5_addfilter(NC_VAR_INFO_T* var, int active, unsigned int id, size_t nparams,
unsigned int* inparams, NC_FILTER_SPEC_HDF5** filtspecp)
{
int stat = NC_NOERR;
NC_FILTER_SPEC_HDF5* fi = NULL;
unsigned int* params = NULL;
int pos;
if(var->filters == NULL) {
if((var->filters = nclistnew())==NULL) return THROW(NC_ENOMEM);
}
if(nparams > 0 && inparams == NULL)
return THROW(NC_EINVAL);
if(inparams != NULL) {
if((params = malloc(sizeof(unsigned int)*nparams)) == NULL)
return THROW(NC_ENOMEM);
memcpy(params,inparams,sizeof(unsigned int)*nparams);
}
if((pos=filterlookup(var, id,&fi)) < 0) {
if((fi = calloc(1,sizeof(NC_FILTER_SPEC_HDF5))) == NULL)
{nullfree(params); return THROW(NC_ENOMEM);}
}
assert(fi != NULL);
fi->active = active;
fi->filterid = id;
fi->nparams = nparams;
if(fi->params) free(fi->params);
fi->params = params;
if(filtspecp) *filtspecp = fi;
if(pos < 0) nclistpush(var->filters,fi);
fi = NULL;
return THROW(stat);
}
/**
* @internal Define filter settings. Called by nc_def_var_filter().
*
@ -238,7 +266,6 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args)
params = obj->u.spec.params;
#ifdef HAVE_H5_DEFLATE
if(id == H5Z_FILTER_DEFLATE) {
int k;
int level;
if(nparams != 1)
return THROW(NC_EFILTER); /* incorrect no. of parameters */
@ -247,11 +274,8 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args)
level > NC_MAX_DEFLATE_LEVEL)
return THROW(NC_EINVAL);
/* If szip compression is already applied, return error. */
for(k=0;k<nclistlength(var->filters);k++) {
NC_FILTER_SPEC_HDF5* f = nclistget(var->filters,k);
if (f->filterid == H5Z_FILTER_SZIP)
if((filterlookup(var,H5Z_FILTER_SZIP,NULL)) >= 0)
return THROW(NC_EINVAL);
}
}
#else /*!HAVE_H5_DEFLATE*/
if(id == H5Z_FILTER_DEFLATE)
@ -259,18 +283,14 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args)
#endif
#ifdef HAVE_H5Z_SZIP
if(id == H5Z_FILTER_SZIP) { /* Do error checking */
int k;
if(nparams != 2)
return THROW(NC_EFILTER); /* incorrect no. of parameters */
/* Pixels per block must be an even number, < 32. */
if (params[1] % 2 || params[1] > NC_MAX_PIXELS_PER_BLOCK)
return THROW(NC_EINVAL);
/* If zlib compression is already applied, return error. */
for(k=0;k<nclistlength(var->filters);k++) {
NC_FILTER_SPEC_HDF5* f = nclistget(var->filters,k);
if (f->filterid == H5Z_FILTER_DEFLATE)
if((filterlookup(var,H5Z_FILTER_DEFLATE,NULL)) >= 0)
return THROW(NC_EINVAL);
}
}
#else /*!HAVE_H5Z_SZIP*/
if(id == H5Z_FILTER_SZIP)
@ -300,8 +320,9 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args)
return THROW(NC_EINVAL);
}
#endif
if((stat = NC4_hdf5_addfilter(var,!FILTERACTIVE,id,nparams,params)))
goto done;
if((stat = NC4_hdf5_addfilter(var,!FILTERACTIVE,id,nparams,params, NULL)))
goto done;
#ifdef USE_PARALLEL
#ifdef HDF5_SUPPORTS_PAR_FILTERS
/* Switch to collective access. HDF5 requires collevtive access

46
libhdf5/hdf5open.c
View File

@ -963,8 +963,8 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var)
{
H5Z_filter_t filter;
int num_filters;
unsigned int cd_values_zip[CD_NELEMS_ZLIB];
size_t cd_nelems = CD_NELEMS_ZLIB;
unsigned int* cd_values = NULL;
size_t cd_nelems;
int f;
int stat = NC_NOERR;
@ -975,8 +975,12 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var)
for (f = 0; f < num_filters; f++)
{
if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, cd_values_zip,
0, NULL, NULL)) < 0)
cd_nelems = 0;
if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, NULL, 0, NULL, NULL)) < 0)
return NC_EHDFERR;
if((cd_values = calloc(sizeof(unsigned int),cd_nelems))==NULL)
return NC_ENOMEM;
if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, cd_values, 0, NULL, NULL)) < 0)
return NC_EHDFERR;
switch (filter)
{
@ -990,36 +994,27 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var)
case H5Z_FILTER_DEFLATE:
if (cd_nelems != CD_NELEMS_ZLIB ||
cd_values_zip[0] > NC_MAX_DEFLATE_LEVEL)
cd_values[0] > NC_MAX_DEFLATE_LEVEL)
return NC_EHDFERR;
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values_zip)))
return stat;
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values,NULL)))
return stat;
break;
case H5Z_FILTER_SZIP: {
/* Szip is tricky because the filter code expands the set of parameters from 2 to 4
and changes some of the parameter values; try to compensate */
if(cd_nelems == 0) {
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL)))
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL,NULL)))
return stat;
} else {
/* We have to re-read the parameters based on actual nparams,
which in the case of szip, differs from users original nparams */
unsigned int* realparams = (unsigned int*)calloc(1,sizeof(unsigned int)*cd_nelems);
if(realparams == NULL)
return NC_ENOMEM;
if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems,
realparams, 0, NULL, NULL)) < 0)
return NC_EHDFERR;
/* fix up the parameters and the #params */
if(cd_nelems != 4)
return NC_EHDFERR;
cd_nelems = 2; /* ignore last two */
/* Fix up changed params */
realparams[0] &= (H5_SZIP_ALL_MASKS);
cd_values[0] &= (H5_SZIP_ALL_MASKS);
/* Save info */
stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,realparams);
nullfree(realparams);
stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values,NULL);
if(stat) return stat;
}
@ -1027,21 +1022,14 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var)
default:
if(cd_nelems == 0) {
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL))) return stat;
if((stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,0,NULL,NULL))) return stat;
} else {
/* We have to re-read the parameters based on actual nparams */
unsigned int* realparams = (unsigned int*)calloc(1,sizeof(unsigned int)*cd_nelems);
if(realparams == NULL)
return NC_ENOMEM;
if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems,
realparams, 0, NULL, NULL)) < 0)
return NC_EHDFERR;
stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,realparams);
nullfree(realparams);
stat = NC4_hdf5_addfilter(var,FILTERACTIVE,filter,cd_nelems,cd_values,NULL);
if(stat) return stat;
}
break;
}
nullfree(cd_values); cd_values = NULL;
}
return NC_NOERR;
}

10
libhdf5/nc4hdf.c
View File

@ -902,6 +902,12 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
}
}
/* If the user wants to fletcher error correction, set that up now. */
/* Since it is a checksum of sorts, flatcher is always applied first */
if (var->fletcher32)
if (H5Pset_fletcher32(plistid) < 0)
BAIL(NC_EHDFERR);
/* If the user wants to shuffle the data, set that up now. */
if (var->shuffle) {
if (H5Pset_shuffle(plistid) < 0)
@ -947,10 +953,6 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
}
}
/* If the user wants to fletcher error correction, set that up now. */
if (var->fletcher32)
if (H5Pset_fletcher32(plistid) < 0)
BAIL(NC_EHDFERR);
/* If ndims non-zero, get info for all dimensions. We look up the
dimids and get the len of each dimension. We need this to create

3
libhdf5/nc4info.c
View File

@ -56,7 +56,6 @@ NC4_provenance_init(void)
unsigned major,minor,release;
NCbytes* buffer = NULL; /* for constructing the global _NCProperties */
char printbuf[1024];
const char* p = NULL;
if(globalpropinitialized)
return stat;
@ -91,11 +90,13 @@ NC4_provenance_init(void)
ncbytescat(buffer,printbuf);
#ifdef NCPROPERTIES_EXTRA
{ const char* p;
/* Add any extra fields */
p = NCPROPERTIES_EXTRA;
if(p[0] == NCPROPSSEP2) p++; /* If leading separator */
ncbytesappend(buffer,NCPROPSSEP2);
ncbytescat(buffer,p);
}
#endif
ncbytesnull(buffer);
globalprovenance.ncproperties = ncbytesextract(buffer);

7
nc_test4/CMakeLists.txt
View File

@ -30,13 +30,16 @@ IF(BUILD_UTILITIES)
add_sh_test(nc_test4 run_grp_rename)
ADD_SH_TEST(nc_test4 tst_misc)
IF(ENABLE_FILTER_TESTING)
build_bin_test(tst_filterparser)
build_bin_test(test_filter)
build_bin_test(test_filter_misc)
build_bin_test(test_filter_reg)
build_bin_test(tst_multifilter)
build_bin_test(test_filter_order)
build_bin_test(test_filter_repeat)
ADD_SH_TEST(nc_test4 tst_filter)
SET(NC4_TESTS ${NC4_TESTS} tst_filterparser test_filter_reg)
IF(ENABLE_CLIENTSIDE_FILTERS)
add_bin_test(nc_test4 test_filter_reg)
ENDIF(ENABLE_CLIENTSIDE_FILTERS)
ENDIF(ENABLE_FILTER_TESTING)
ENDIF(BUILD_UTILITIES)

16
nc_test4/Makefile.am
View File

@ -72,10 +72,13 @@ endif
# Filter Tests (requires ncdump and ncgen)
if ENABLE_FILTER_TESTING
extradir =
extra_PROGRAMS = test_filter test_filter_misc test_filter_order
check_PROGRAMS += test_filter_reg
check_PROGRAMS += test_filter test_filter_misc test_filter_order test_filter_repeat
check_PROGRAMS += tst_multifilter
TESTS += tst_filter.sh test_filter_reg
TESTS += tst_filter.sh
if ENABLE_CLIENTSIDE_FILTERS
#check_PROGRAMS += test_filter_reg
TESTS += test_filter_reg
endif
endif
endif # BUILD_UTILITIES
@ -95,8 +98,9 @@ run_empty_vlen_test.sh ref_hdf5_compat1.nc ref_hdf5_compat2.nc \
ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5 \
ref_szip.cdl tst_filter.sh bzip2.cdl ref_filtered.cdl \
ref_unfiltered.cdl ref_bzip2.c findplugin.in ref_unfilteredvv.cdl \
ref_filteredvv.cdl ref_multi.cdl ref_filter_order.txt \
ref_ncgenF.cdl ref_nccopyF.cdl
ref_filteredvv.cdl ref_multi.cdl \
ref_ncgenF.cdl ref_nccopyF.cdl \
ref_filter_order.txt ref_filter_repeat.txt
CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc \
@ -106,7 +110,7 @@ tst_*.h5 tst_grp_rename.cdl tst_grp_rename.dmp ref_grp_rename.cdl \
foo1.nc tst_*.h4 test.nc testszip.nc test.h5 szip_dump.cdl \
perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \
floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \
testfilter_reg.nc
testfilter_reg.nc filterorder.txt filterrepeat.txt
DISTCLEANFILES = findplugin.sh run_par_test.sh

16
nc_test4/ref_filter_order.txt
View File

@ -1,17 +1,9 @@
test1: filter order.
set var filter 0
test: nparams=1: params= 0
dimsizes=4,4,4,4
chunksizes=4,4,4,4
set var filter 1
test: nparams=1: params= 1
dimsizes=4,4,4,4
chunksizes=4,4,4,4
test1: compression.
Apply filter: 0
Apply filter: 1
cd_nelmts=1 cd_values= 0
cd_nelmts=1 cd_values= 1
test1: decompression.
Apply filter: 1
Apply filter: 0
cd_nelmts=1 cd_values= 1
cd_nelmts=1 cd_values= 0
data comparison: |array|=256
no data errors

8
nc_test4/ref_filter_repeat.txt Normal file
View File

@ -0,0 +1,8 @@
test1: def filter repeat .
test1: compression.
cd_nelmts=2 cd_values= 0 18
test1: decompression.
filter(40000): |params|=2 params= 0 18
cd_nelmts=2 cd_values= 0 18
data comparison: |array|=256
no data errors

280
nc_test4/test_filter_order.c
View File

@ -10,10 +10,6 @@
#include <assert.h>
#include <sys/types.h>
#include <hdf5.h>
/* Older versions of the hdf library may define H5PL_type_t here */
#include <H5PLextern.h>
#include "netcdf.h"
#include "netcdf_filter.h"
@ -21,7 +17,7 @@
#undef DEBUG
#define FILTER_ID 32768
#define FILTER_ID 40000
#define MAXERRS 8
@ -52,17 +48,6 @@ static size_t odom[MAXDIMS];
static float* array = NULL;
static float* expected = NULL;
static size_t nparams = 0;
static unsigned int params[MAXPARAMS];
static unsigned int baseline[2][NPARAMS] = {{0},{1}};
static struct Base {
unsigned int id;
H5Z_class2_t* info;
} baseinfo;
static const H5Z_class2_t H5Z_REG[2];
/* Forward */
static int filter_test1(void);
static void init(int argc, char** argv);
@ -72,7 +57,6 @@ static int odom_more(void);
static int odom_next(void);
static int odom_offset(void);
static float expectedvalue(void);
static void verifyparams(int);
#define ERRR do { \
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
@ -138,87 +122,29 @@ create(void)
return NC_NOERR;
}
static int
verifyfilterinfo(struct Base* info, struct Base* base)
{
int stat = NC_NOERR;
if(info->id != base->id)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: id mismatch\n");}
#ifdef USE_HDF5
H5Z_class2_t* h5info = info->info;
H5Z_class2_t* h5base = base->info;
if(info->id != base->id)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: id mismatch\n");}
if(h5info->encoder_present != h5base->encoder_present)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: encoder_present mismatch\n");}
if(h5info->decoder_present != h5base->decoder_present)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: decoder_present mismatch\n");}
if(h5info->decoder_present != h5base->decoder_present)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: decoder_present mismatch\n");}
if(strcmp(h5info->name,h5base->name) != 0)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: name mismatch\n");}
if(h5info->can_apply != h5base->can_apply)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: can_apply mismatch\n");}
if(h5info->set_local != h5base->set_local)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: set_local mismatch\n");}
if(h5info->filter != h5base->filter)
{stat = NC_EINVAL; fprintf(stderr,"verifyfilterinfo: H5Z_class_t: filter mismatch\n");}
#else
stat = NC_ENOTBUILT; fprintf(stderr,"Unknown format\n")}
#endif
return stat;
}
static void
registerfilters(void)
deffilters(void)
{
struct Base inqinfo;
unsigned int params[1];
unsigned int filterids[2];
size_t nfilters = 0;
size_t nparams = 0;
/* Register filter 0 */
baseinfo.id = FILTER_ID;
baseinfo.info = (H5Z_class2_t*)&H5Z_REG[0];
CHECK(nc_filter_client_register(baseinfo.id,baseinfo.info));
/* Verify by inquiry */
memset(&inqinfo,0,sizeof(struct Base));
inqinfo.id = FILTER_ID;
if((inqinfo.info = (H5Z_class2_t*)calloc(1,sizeof(H5Z_class2_t))) == NULL)
CHECK(NC_ENOMEM);
CHECK((nc_filter_client_inq(inqinfo.id,(void*)inqinfo.info)));
CHECK((verifyfilterinfo(&inqinfo,&baseinfo)));
nullfree(inqinfo.info);
/* Register filter 1 */
baseinfo.id = FILTER_ID+1;
baseinfo.info = (H5Z_class2_t*)&H5Z_REG[1];
CHECK(nc_filter_client_register(baseinfo.id,baseinfo.info));
/* Verify by inquiry */
memset(&inqinfo,0,sizeof(struct Base));
inqinfo.id = FILTER_ID+1;
if((inqinfo.info = (H5Z_class2_t*)calloc(1,sizeof(H5Z_class2_t))) == NULL)
CHECK(NC_ENOMEM);
CHECK((nc_filter_client_inq(inqinfo.id,(void*)inqinfo.info)));
CHECK((verifyfilterinfo(&inqinfo,&baseinfo)));
nullfree(inqinfo.info);
}
static void
setvarfilter(int index)
{
/* NOOP the params */
CHECK(nc_def_var_filter(ncid,varid,FILTER_ID+index,NPARAMS,baseline[index]));
verifyparams(index);
}
static void
verifyparams(int index)
{
int i;
CHECK(nc_inq_var_filter_info(ncid,varid,FILTER_ID+index,&nparams,params));
if(nparams != NPARAMS) REPORT("nparams mismatch");
for(i=0;i<nparams;i++) {
if(params[i] != baseline[index][i])
REPORT("param mismatch");
}
params[0] = 0;
CHECK(nc_def_var_filter(ncid,varid,FILTER_ID,1,params));
params[0] = 1;
CHECK(nc_def_var_filter(ncid,varid,FILTER_ID+1,1,params));
CHECK(nc_inq_var_filterids(ncid,varid,&nfilters,filterids));
if(nfilters != 2) REPORT("nfilters mismatch");
if(filterids[0] != FILTER_ID+0) REPORT("0: filterids mismatch");
if(filterids[1] != FILTER_ID+1) REPORT("1: filterids mismatch");
CHECK(nc_inq_var_filter_info(ncid,varid,FILTER_ID+0,&nparams,params));;
if(nparams != 1) REPORT("0: nparams mismatch");
if(params[0] != 0) REPORT("0: param mismatch");
CHECK(nc_inq_var_filter_info(ncid,varid,FILTER_ID+1,&nparams,params));
if(nparams != 1) REPORT("1: nparams mismatch");
if(params[0] != 1) REPORT("1: param mismatch");
}
static int
@ -227,6 +153,7 @@ openfile(void)
unsigned int filterids[2];
unsigned int params[1];
size_t nfilters = 0;
size_t nparams = 0;
int k;
/* Open the file and check it. */
@ -322,32 +249,6 @@ compare(void)
return (errs == 0);
}
static void
showparameters(void)
{
int i;
printf("test: nparams=%ld: params=",(unsigned long)nparams);
for(i=0;i<nparams;i++) {
printf(" %u",params[i]);
}
printf("\n");
for(i=0;i<ndims;i++) {
if(i==0)
printf("dimsizes=%ld",(unsigned long)dimsize[i]);
else
printf(",%ld",(unsigned long)dimsize[i]);
}
printf("\n");
for(i=0;i<ndims;i++) {
if(i==0)
printf("chunksizes=%ld",(unsigned long)chunksize[i]);
else
printf(",%ld",(unsigned long)chunksize[i]);
}
printf("\n");
fflush(stderr);
}
static int
filter_test1(void)
{
@ -358,12 +259,7 @@ filter_test1(void)
printf("test1: filter order.\n");
create();
setchunking();
printf("set var filter 0\n");
setvarfilter(0);
showparameters();
printf("set var filter 1\n");
setvarfilter(1);
showparameters();
deffilters();
CHECK(nc_enddef(ncid));
/* Fill in the array */
@ -461,8 +357,6 @@ init(int argc, char** argv)
/* Allocate max size */
array = (float*)calloc(1,sizeof(float)*actualproduct);
expected = (float*)calloc(1,sizeof(float)*actualproduct);
/* Register the filter */
registerfilters();
}
/**************************************************/
@ -477,133 +371,3 @@ main(int argc, char **argv)
if(!filter_test1()) ERRR;
exit(nerrs > 0?1:0);
}
/**************************************************/
/* In-line filter code */
#define H5Z_FILTER_REG0 FILTER_ID
#define H5Z_FILTER_REG1 (FILTER_ID+1)
#ifndef DLL_EXPORT
#define DLL_EXPORT
#endif
static int paramcheck(size_t nparams, const unsigned int* params);
/* Forward */
static int paramcheck(size_t nparams, const unsigned int* params);
/* Make this explicit */
/*
* The "can_apply" callback returns positive a valid combination, zero for an
* invalid combination and negative for an error.
*/
static htri_t
H5Z_reg_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id)
{
return 1; /* Assume it can always apply */
}
/*
As a filter, it is the identity function,
passing input to output unchanged.
*/
size_t
H5Z_filter_reg(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
void* newbuf;
if(cd_nelmts == 0)
goto fail;
if(!paramcheck(cd_nelmts,cd_values))
goto fail;
printf("Apply filter: %u\n",cd_values[0]);
if (flags & H5Z_FLAG_REVERSE) {
/* Replace buffer */
#ifdef HAVE_H5ALLOCATE_MEMORY
newbuf = H5allocate_memory(*buf_size,0);
#else
newbuf = malloc(*buf_size);
#endif
if(newbuf == NULL) abort();
if(*buf != NULL) {
memcpy(newbuf,*buf,*buf_size);
/* reclaim old buffer */
#ifdef HAVE_H5FREE_MEMORY
H5free_memory(*buf);
#else
free(*buf);
#endif
}
*buf = newbuf;
} else {
/* Replace buffer */
#ifdef HAVE_H5ALLOCATE_MEMORY
newbuf = H5allocate_memory(*buf_size,0);
#else
newbuf = malloc(*buf_size);
#endif
if(newbuf == NULL) abort();
if(*buf != NULL) {
memcpy(newbuf,*buf,*buf_size);
/* reclaim old buffer */
#ifdef HAVE_H5FREE_MEMORY
H5free_memory(*buf);
#else
free(*buf);
#endif
}
*buf = newbuf;
}
return *buf_size;
fail:
return 0;
}
static int
paramcheck(size_t nparams, const unsigned int* params)
{
if(nparams != 1) {
fprintf(stderr,"Incorrect parameter count: need=1 sent=%ld\n",(unsigned long)nparams);
goto fail;
}
return 1;
fail:
return 0;
}
static const H5Z_class2_t H5Z_REG[2] = {
{
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)(H5Z_FILTER_REG0), /* Filter id number */
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"registered0", /* Filter name for debugging */
(H5Z_can_apply_func_t)H5Z_reg_can_apply, /* The "can apply" callback */
NULL, /* The "set local" callback */
(H5Z_func_t)H5Z_filter_reg, /* The actual filter function */
},
{
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)(H5Z_FILTER_REG1), /* Filter id number */
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"registered1", /* Filter name for debugging */
(H5Z_can_apply_func_t)H5Z_reg_can_apply, /* The "can apply" callback */
NULL, /* The "set local" callback */
(H5Z_func_t)H5Z_filter_reg, /* The actual filter function */
}
};

368
nc_test4/test_filter_repeat.c Normal file
View File

@ -0,0 +1,368 @@
/*
Copyright 2018, UCAR/Unidata
See COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <sys/types.h>
#include "netcdf.h"
#include "netcdf_filter.h"
#undef TESTODDSIZE
#undef DEBUG
#define FILTER_ID 40000
#define MAXERRS 8
#define MAXPARAMS 32
#define MAXDIMS 8
#define TESTFILE "testfilter_reg.nc"
#define NPARAMS 1
#define PARAMVAL 17
#define NDIMS 4
static size_t dimsize[NDIMS] = {4,4,4,4};
static size_t chunksize[NDIMS] = {4,4,4,4};
static size_t ndims = NDIMS;
static size_t totalproduct = 1; /* x-product over max dims */
static size_t actualproduct = 1; /* x-product over actualdims */
static size_t chunkproduct = 1; /* x-product over actual chunks */
static int nerrs = 0;
static int ncid, varid;
static int dimids[MAXDIMS];
static size_t odom[MAXDIMS];
static float* array = NULL;
static float* expected = NULL;
/* Forward */
static int filter_test1(void);
static void init(int argc, char** argv);
static void reset(void);
static void odom_reset(void);
static int odom_more(void);
static int odom_next(void);
static int odom_offset(void);
static float expectedvalue(void);
#define ERRR do { \
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \
__FILE__, __LINE__); \
nerrs++;\
} while (0)
static int
check(int err,int line)
{
if(err != NC_NOERR) {
fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err));
}
return NC_NOERR;
}
static void
report(const char* msg, int lineno)
{
fprintf(stderr,"fail: line=%d %s\n",lineno,msg);
exit(1);
}
#define CHECK(x) check(x,__LINE__)
#define REPORT(x) report(x,__LINE__)
static int
verifychunks(void)
{
int i;
int store = -1;
size_t localchunks[MAXDIMS];
memset(localchunks,0,sizeof(localchunks));
CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks));
if(store != NC_CHUNKED) {
fprintf(stderr,"bad chunk store\n");
return 0;
}
for(i=0;i<ndims;i++) {
if(chunksize[i] != localchunks[i]) {
fprintf(stderr,"bad chunk size: %d\n",i);
return 0;
}
}
return 1;
}
static int
create(void)
{
int i;
/* Create a file with one big variable */
CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
for(i=0;i<ndims;i++) {
char dimname[1024];
snprintf(dimname,sizeof(dimname),"dim%d",i);
CHECK(nc_def_dim(ncid, dimname, dimsize[i], &dimids[i]));
}
CHECK(nc_def_var(ncid, "var", NC_FLOAT, ndims, dimids, &varid));
return NC_NOERR;
}
static void
deffilter(unsigned int id, size_t nparams, unsigned int* params)
{
/* Register filter */
CHECK(nc_def_var_filter(ncid,varid,id,nparams,params));
}
static void
printfilter(unsigned int id)
{
size_t i,nparams = 0;
unsigned int params[MAXPARAMS];
CHECK(nc_inq_var_filter_info(ncid,varid,id,&nparams,params));;
if(nparams == 0) REPORT("no nparams");
printf("filter(%u): |params|=%u params=",id,(unsigned)nparams);
for(i=0;i<nparams;i++)
printf(" %u",params[i]);
printf("\n");
}
static int
openfile(void)
{
unsigned int filterids[MAXPARAMS];
size_t nfilters = 0;
int k;
/* Open the file and check it. */
CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid));
CHECK(nc_inq_varid(ncid, "var", &varid));
/* Verify chunking */
if(!verifychunks())
return 0;
/* Check the compression algorithms */
CHECK(nc_inq_var_filterids(ncid,varid,&nfilters,filterids));
for(k=0;k<nfilters;k++)
printfilter(filterids[k]);
fflush(stderr);
return 1;
}
static int
setchunking(void)
{
int store;
store = NC_CHUNKED;
CHECK(nc_def_var_chunking(ncid,varid,store,chunksize));
if(!verifychunks())
return NC_EINVAL;
return NC_NOERR;
}
static void
fill(void)
{
odom_reset();
if(1) {
int i;
if(actualproduct <= 1) abort();
for(i=0;i<actualproduct;i++)
expected[i] = (float)i;
} else {
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
expected[offset] = expect;
odom_next();
}
}
}
static int
compare(void)
{
int errs = 0;
printf("data comparison: |array|=%ld\n",(unsigned long)actualproduct);
if(1)
{
int i;
for(i=0;i<actualproduct;i++) {
if(expected[i] != array[i]) {
fprintf(stderr,"data mismatch: array[%d]=%f expected[%d]=%f\n",
i,array[i],i,expected[i]);
errs++;
if(errs >= MAXERRS)
break;
}
}
} else
{
odom_reset();
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
if(array[offset] != expect) {
fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n",
offset,array[offset],expect);
errs++;
if(errs >= MAXERRS)
break;
}
odom_next();
}
}
if(errs == 0)
printf("no data errors\n");
return (errs == 0);
}
static int
filter_test1(void)
{
int ok = 1;
unsigned int params[MAXPARAMS];
reset();
printf("test1: def filter repeat .\n");
create();
setchunking();
params[0] = 1;
params[1] = 17;
deffilter(FILTER_ID,2,params);
params[0] = 0;
params[1] = 18;
deffilter(FILTER_ID,2,params);
CHECK(nc_enddef(ncid));
/* Fill in the array */
fill();
printf("test1: compression.\n");
/* write array */
CHECK(nc_put_var(ncid,varid,expected));
CHECK(nc_close(ncid));
printf("test1: decompression.\n");
reset();
openfile();
CHECK(nc_get_var_float(ncid, varid, array));
ok = compare();
CHECK(nc_close(ncid));
return ok;
}
/**************************************************/
/* Utilities */
static void
reset()
{
memset(array,0,sizeof(float)*actualproduct);
}
static void
odom_reset(void)
{
memset(odom,0,sizeof(odom));
}
static int
odom_more(void)
{
return (odom[0] < dimsize[0]);
}
static int
odom_next(void)
{
int i; /* do not make unsigned */
for(i=ndims-1;i>=0;i--) {
odom[i] += 1;
if(odom[i] < dimsize[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows*/
odom[i] = 0; /* reset this position*/
}
return 1;
}
static int
odom_offset(void)
{
int i;
int offset = 0;
for(i=0;i<ndims;i++) {
offset *= dimsize[i];
offset += odom[i];
}
return offset;
}
static float
expectedvalue(void)
{
int i;
float offset = 0;
for(i=0;i<ndims;i++) {
offset *= dimsize[i];
offset += odom[i];
}
return offset;
}
static void
init(int argc, char** argv)
{
int i;
/* Setup various variables */
totalproduct = 1;
actualproduct = 1;
chunkproduct = 1;
for(i=0;i<NDIMS;i++) {
totalproduct *= dimsize[i];
if(i < ndims) {
actualproduct *= dimsize[i];
chunkproduct *= chunksize[i];
}
}
/* Allocate max size */
array = (float*)calloc(1,sizeof(float)*actualproduct);
expected = (float*)calloc(1,sizeof(float)*actualproduct);
}
/**************************************************/
int
main(int argc, char **argv)
{
#ifdef DEBUG
H5Eprint(stderr);
nc_set_log_level(1);
#endif
init(argc,argv);
if(!filter_test1()) ERRR;
exit(nerrs > 0?1:0);
}

13
nc_test4/tst_filter.sh
View File

@ -12,6 +12,7 @@ NGC=1
MISC=1
MULTI=1
ORDER=1
REP=1
# Load the findplugins function
. ${builddir}/findplugin.sh
@ -219,13 +220,23 @@ diff -b -w ${srcdir}/ref_nccopyF.cdl ./ncgenFs.cdl
echo "*** Pass: multiple filters"
fi
if test "x$MULTI" = x1 ; then
if test "x$ORDER" = x1 ; then
echo "*** Testing multiple filter order of invocation"
rm -f filterorder.txt
${execdir}/test_filter_order >filterorder.txt
diff -b -w ${srcdir}/ref_filter_order.txt filterorder.txt
fi
if test "x$REP" = x1 ; then
echo "*** Testing filter re-definition invocation"
rm -f filterrepeat.txt
${execdir}/test_filter_repeat >filterrepeat.txt
pwd
ls -l *.txt
ls -l ${srcdir}/*.txt
diff -b -w ${srcdir}/ref_filter_repeat.txt filterrepeat.txt
fi
echo "*** Pass: all selected tests passed"
#cleanup

3
ncdump/Makefile.am
View File

@ -50,9 +50,6 @@ noinst_PROGRAMS += ocprint
ocprint_SOURCES = ocprint.c
endif
noinst_PROGRAMS += ncdumpchunks
ncdumpchunks_SOURCES = ncdumpchunks.c
# This is the man page.
man_MANS = ncdump.1 nccopy.1

7
oc2/occurlfunctions.c
View File

@ -2,6 +2,11 @@
See the COPYRIGHT file for more information. */
#include "config.h"
#include <stdlib.h>
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include "ncrc.h"
#include "ocinternal.h"
#include "ocdebug.h"
@ -10,7 +15,7 @@
#define OC_MAX_REDIRECTS 20L
/* Mnemonic */
#define OPTARG void*
#define OPTARG uintptr_t
/* Define some .rc file entries of interest*/
#define NETRCFILETAG "HTTP.NETRC"

8
plugins/CMakeLists.txt
View File

@ -11,6 +11,7 @@ SET(libh5bzip2_SOURCES blocksort.c huffman.c crctable.c randtable.c compress.c d
SET(libmisc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h)
SET(libnoop_SOURCES H5Znoop.c H5Zutil.c h5noop.h)
SET(libnoop1_SOURCES H5Znoop1.c H5Zutil.c h5noop.h)
IF(ENABLE_FILTER_TESTING)
IF(BUILD_UTILITIES)
@ -38,12 +39,19 @@ SET_TARGET_PROPERTIES(misc PROPERTIES ARCHIVE_OUTPUT_NAME "misc")
SET_TARGET_PROPERTIES(misc PROPERTIES RUNTIME_OUTPUT_NAME "misc")
TARGET_LINK_LIBRARIES(misc ${ALL_TLL_LIBS})
SET_SOURCE_FILES_PROPERTIES(H5Znoop.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=0)
ADD_LIBRARY(noop MODULE ${libnoop_SOURCES})
SET_TARGET_PROPERTIES(noop PROPERTIES LIBRARY_OUTPUT_NAME "noop")
SET_TARGET_PROPERTIES(noop PROPERTIES ARCHIVE_OUTPUT_NAME "noop")
SET_TARGET_PROPERTIES(noop PROPERTIES RUNTIME_OUTPUT_NAME "noop")
TARGET_LINK_LIBRARIES(noop ${ALL_TLL_LIBS})
SET_SOURCE_FILES_PROPERTIES(H5Znoop1.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=1)
ADD_LIBRARY(noop1 MODULE ${libnoop1_SOURCES})
SET_TARGET_PROPERTIES(noop1 PROPERTIES LIBRARY_OUTPUT_NAME "noop1")
SET_TARGET_PROPERTIES(noop1 PROPERTIES ARCHIVE_OUTPUT_NAME "noop1")
SET_TARGET_PROPERTIES(noop1 PROPERTIES RUNTIME_OUTPUT_NAME "noop1")
TARGET_LINK_LIBRARIES(noop1 ${ALL_TLL_LIBS})
ENDIF(BUILD_UTILITIES)
ENDIF(ENABLE_FILTER_TESTING)

39
plugins/H5Znoop.c
View File

@ -7,12 +7,22 @@
#include <hdf5.h>
/* Older versions of the hdf library may define H5PL_type_t here */
#include <H5PLextern.h>
#include "h5noop.h"
#ifndef DLL_EXPORT
#define DLL_EXPORT
#endif
#if NOOP_INSTANCE == 1
const static int instance1 = 1;
#endif
#if NOOP_INSTANCE == 0
const static int instance0 = 1;
#endif
/* use a temporary */
#define H5Z_FILTER_NOOP 40000
/* WARNING:
Starting with HDF5 version 1.10.x, the plugin code MUST be
careful when using the standard *malloc()*, *realloc()*, and
@ -33,12 +43,23 @@ will generate an error.
extern void NC_filterfix8(void* mem, int decode);
const H5Z_class2_t H5Z_NOOP[1] = {{
static htri_t H5Z_noop_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id);
static size_t H5Z_filter_noop(unsigned int, size_t, const unsigned int cd_values[], size_t, size_t*, void**);
static H5Z_class2_t H5Z_NOOP[1] = {{
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
#if NOOP_INSTANCE == 0
(H5Z_filter_t)(H5Z_FILTER_NOOP), /* Filter id number */
#else
(H5Z_filter_t)(H5Z_FILTER_NOOP+1), /* Filter id number */
#endif
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"noop", /* Filter name for debugging */
#if NOOP_INSTANCE == 0
"noop", /* Filter name for debugging */
#else
"noop1", /* Filter name for debugging */
#endif
(H5Z_can_apply_func_t)H5Z_noop_can_apply, /* The "can apply" callback */
NULL, /* The "set local" callback */
(H5Z_func_t)H5Z_filter_noop, /* The actual filter function */
@ -62,21 +83,26 @@ H5PLget_plugin_info(void)
* The "can_apply" callback returns positive a valid combination, zero for an
* invalid combination and negative for an error.
*/
htri_t
static htri_t
H5Z_noop_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id)
{
return 1; /* Assume it can always apply */
}
size_t
static size_t
H5Z_filter_noop(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
void* newbuf;
size_t i;
printf("cd_nelmts=%lu cd_values=",(unsigned long)cd_nelmts);
for(i=0;i<cd_nelmts;i++)
printf(" %u",cd_values[i]);
printf("\n");
if (flags & H5Z_FLAG_REVERSE) {
/* Replace buffer */
#ifdef HAVE_H5ALLOCATE_MEMORY
newbuf = H5allocate_memory(*buf_size,0);
@ -94,7 +120,6 @@ H5Z_filter_noop(unsigned int flags, size_t cd_nelmts,
*buf = newbuf;
} else {
/* Replace buffer */
#ifdef HAVE_H5ALLOCATE_MEMORY
newbuf = H5allocate_memory(*buf_size,0);

19
plugins/Makefile.am
View File

@ -4,6 +4,8 @@
# Put Together AM_CPPFLAGS and AM_LDFLAGS
include $(top_srcdir)/lib_flags.am
LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -rpath ${abs_builddir}
BZIP2HDRS=bzlib.h bzlib_private.h
BZIP2SRC= blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c
@ -11,7 +13,7 @@ PLUGINSRC=H5Zbzip2.c
PLUGINHDRS=h5bzip2.h
EXTRA_DIST=${PLUGINSRC} ${BZIP2SRC} ${PLUGINHDRS} ${BZIP2HDRS} \
H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c CMakeLists.txt
H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h CMakeLists.txt
# WARNING: This list must be kept consistent with the corresponding
# AC_CONFIG_LINK commands near the end of configure.ac.
@ -19,17 +21,22 @@ HDF5PLUGINSRC=${PLUGINSRC} ${BZIP2SRC} ${PLUGINHDRS} ${BZIP2HDRS}
if ENABLE_FILTER_TESTING
noinst_LTLIBRARIES = libmisc.la libnoop.la
noinst_LTLIBRARIES = libmisc.la libnoop.la libnoop1.la
lib_LTLIBRARIES = libh5bzip2.la
libh5bzip2_la_SOURCES = ${HDF5PLUGINSRC}
libh5bzip2_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined
libmisc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h
libmisc_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -rpath ${abs_builddir}
# The noop filter is to allow testing of multifilters
# The noop filter is to allow testing of multifilters and filter order
# Need two distinct instances
libnoop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h
libnoop_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -rpath ${abs_builddir}
libnoop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h
endif #ENABLE_FILTER_TESTING
BUILT_SOURCES = H5Znoop1.c
CLEANFILES = H5Znoop1.c
H5Znoop1.c: Makefile H5Znoop.c
echo '#define NOOP_INSTANCE 1' > $@
cat ${srcdir}/H5Znoop.c >> $@

9
plugins/h5noop.h
View File

@ -11,17 +11,8 @@
#define DECLSPEC extern
#endif
/* use an integer greater than 256 to be id of the registered filter. */
#define H5Z_FILTER_NOOP 40000
/* declare the hdf5 interface */
DECLSPEC H5PL_type_t H5PLget_plugin_type(void);
DECLSPEC const void* H5PLget_plugin_info(void);
DECLSPEC const H5Z_class2_t H5Z_NOOP[1];
/* Declare filter specific functions */
DECLSPEC htri_t H5Z_noop_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id);
DECLSPEC size_t H5Z_filter_noop(unsigned flags,size_t cd_nelmts,const unsigned cd_values[],
size_t nbytes,size_t *buf_size,void**buf);
#endif /*H5MISC_H*/