mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-03-31 17:50:26 +08:00
Improve filter installation process to avoid use of an extra shell script
re: https://github.com/Unidata/netcdf-c/issues/2338 re: https://github.com/Unidata/netcdf-c/issues/2294 In issue https://github.com/Unidata/netcdf-c/issues/2338, Ed Hartnett suggested a better way to install filters to a user defined location -- for Automake, anyway. This PR implements that suggestion. It turns out to be more complicated than it appears, so there are fair number of changes; mostly to shell scripts. Most of the change is in plugins/Makefile.am. NOTE: this PR still does NOT address the use of HDF5_PLUGIN_PATH as the default; this turns out to be complex when dealing with NCZarr. So this will be addressed in a subsequent post 4.9.0 PR. ## Misc. Changes 1. Record the occurrences of incomplete codecs in libnczarr so that they can be included in _Codecs attribute correctly. This allows users to see what missing filters are referenced in the Zarr file. Primarily affects libnczarr/zfilter.[ch]. Also required creating a new no-effect filter: H5Zunknown.c. 2. Move the unknown filter test to a separate test file. 3. Incorporates PR https://github.com/Unidata/netcdf-c/pull/2343
This commit is contained in:
parent
0dd076919f
commit
7b09290a3a
@ -1862,10 +1862,10 @@ ENDIF(ENABLE_MMAP)
|
||||
#CHECK_FUNCTION_EXISTS(alloca HAVE_ALLOCA)
|
||||
|
||||
# Used in the `configure_file` calls below
|
||||
SET(ISCMAKE "1")
|
||||
SET(ISCMAKE "yes")
|
||||
IF(MSVC)
|
||||
SET(ISMSVC ON CACHE BOOL "" FORCE)
|
||||
SET(REGEDIT ON CACHE BOOL "" FORCE)
|
||||
SET(ISMSVC yes CACHE BOOL "" FORCE)
|
||||
SET(REGEDIT yes CACHE BOOL "" FORCE)
|
||||
# Get windows major version and build number
|
||||
EXECUTE_PROCESS(COMMAND "systeminfo" OUTPUT_VARIABLE WININFO)
|
||||
IF(WININFO STREQUAL "")
|
||||
@ -2509,25 +2509,15 @@ configure_file(
|
||||
SET(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_CURRENT_SOURCE_DIR}/test_common.in)
|
||||
SET(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
SET(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}")
|
||||
SET(ISMSVC "${MSVC}")
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY_DIR}/test_common.sh @ONLY NEWLINE_STYLE LF)
|
||||
|
||||
#####
|
||||
# Build nc_test4/findplugin.sh
|
||||
# Build and copy nc_test4/findplugin.sh to various places
|
||||
#####
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF)
|
||||
|
||||
IF(ENABLE_NCZARR)
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nczarr_test/findplugin.sh @ONLY NEWLINE_STYLE LF)
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_PLUGINS)
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/plugins/findplugin.sh @ONLY NEWLINE_STYLE LF)
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_EXAMPLES)
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF)
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_TESTS)
|
||||
#####
|
||||
|
@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release
|
||||
|
||||
## 4.8.2 - TBD
|
||||
|
||||
* [Enhancement] Improve filter installation process to avoid use of an extra shell script. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????).
|
||||
* [Bug Fix] Get "make distcheck" to work See [Github #/2343](https://github.com/Unidata/netcdf-c/pull/2343).
|
||||
* [Enhancement] Update the documentation to match the current filter capabilities See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249).
|
||||
* [Enhancement] Support installation of pre-built standard filters into user-specified location. See [Github #2318](https://github.com/Unidata/netcdf-c/pull/2318).
|
||||
* [Enhancement] Improve filter support. More specifically (1) add nc_inq_filter_avail to check if a filter is available, (2) add the notion of standard filters, (3) cleanup szip support to fix interaction with NCZarr. See [Github #2245](https://github.com/Unidata/netcdf-c/pull/2245).
|
||||
|
@ -1993,10 +1993,10 @@ AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in)
|
||||
AC_CONFIG_FILES(plugins/findplugin.sh:nc_test4/findplugin.in)
|
||||
AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in)
|
||||
AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in)
|
||||
AC_CONFIG_FILES([nc_test/run_pnetcdf_tests.sh:nc_test/run_pnetcdf_tests.sh.in],[chmod ugo+x nc_test/run_pnetcdf_tests.sh])
|
||||
AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)
|
||||
AC_CONFIG_FILES(dap4_test/pingurl4.c:ncdap_test/pingurl.c)
|
||||
AC_CONFIG_FILES([h5_test/run_par_tests.sh], [chmod ugo+x h5_test/run_par_tests.sh])
|
||||
AC_CONFIG_FILES([nc_test/run_pnetcdf_tests.sh], [chmod ugo+x nc_test/run_pnetcdf_tests.sh])
|
||||
AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh])
|
||||
AC_CONFIG_FILES([nc_perf/run_par_bm_test.sh], [chmod ugo+x nc_perf/run_par_bm_test.sh])
|
||||
AC_CONFIG_FILES([nc_perf/run_gfs_test.sh], [chmod ugo+x nc_perf/run_gfs_test.sh])
|
||||
|
@ -19,7 +19,7 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la
|
||||
AM_CPPFLAGS += -I$(top_srcdir)/liblib
|
||||
AM_CPPFLAGS += -I$(top_srcdir)/libdap4
|
||||
AM_CPPFLAGS += -DTOPSRCDIR=${abs_top_srcdir}
|
||||
AM_CPPFLAGS += -DTOPBINDIR=${abs_top_bindir}
|
||||
AM_CPPFLAGS += -DTOPBINDIR=${abs_top_builddir}
|
||||
|
||||
# Set up the tests; do the .sh first, then .c
|
||||
check_PROGRAMS =
|
||||
|
@ -15,11 +15,14 @@ set -e
|
||||
|
||||
echo "*** running test_filter example..."
|
||||
. ${builddir}/findplugin.sh
|
||||
echo "findplugin.sh loaded"
|
||||
|
||||
# Locate the plugin path and the library names; argument order is critical
|
||||
# Find bzip2 and capture
|
||||
findplugin h5bzip2
|
||||
BZIP2PATH="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
|
||||
BZIP2LIB="${HDF5_PLUGIN_LIB}"
|
||||
BZIP2DIR="${HDF5_PLUGIN_DIR}"
|
||||
BZIP2PATH="${BZIP2DIR}/${BZIP2LIB}"
|
||||
|
||||
# Verify
|
||||
if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
|
||||
|
@ -167,7 +167,7 @@ zclose_vars(NC_GRP_INFO_T* grp)
|
||||
#ifdef ENABLE_NCZARR_FILTERS
|
||||
/* Reclaim filters */
|
||||
if(var->filters != NULL) {
|
||||
(void)NCZ_filter_freelist(var);
|
||||
(void)NCZ_filter_freelists(var);
|
||||
}
|
||||
var->filters = NULL;
|
||||
#endif
|
||||
|
@ -65,6 +65,7 @@
|
||||
|
||||
/* Hold the loaded filter plugin information */
|
||||
typedef struct NCZ_Plugin {
|
||||
int incomplete;
|
||||
struct HDF5API {
|
||||
const H5Z_class2_t* filter;
|
||||
NCPSharedLib* hdf5lib; /* source of the filter */
|
||||
@ -134,11 +135,15 @@ typedef struct NCZ_Filter {
|
||||
# define FLAG_CODEC 4 /* If set, then visbile parameters come from an existing codec string */
|
||||
# define FLAG_HDF5 8 /* If set, => visible parameters came from nc_def_var_filter */
|
||||
# define FLAG_NEWVISIBLE 16 /* If set, => visible parameters were modified */
|
||||
# define FLAG_INCOMPLETE 32 /* If set, => filter has no complete matching plugin */
|
||||
NCZ_HDF5 hdf5;
|
||||
NCZ_Codec codec;
|
||||
struct NCZ_Plugin* plugin; /**< Implementation of this filter. */
|
||||
int chainindex; /* Position in original chain */
|
||||
} NCZ_Filter;
|
||||
|
||||
#define FILTERINCOMPLETE(f) ((f)->flags & FLAG_INCOMPLETE?1:0)
|
||||
|
||||
/* WARNING: GLOBAL DATA */
|
||||
/* TODO: move to common global state */
|
||||
|
||||
@ -280,12 +285,14 @@ static int pluginnamecheck(const char* name);
|
||||
*/
|
||||
|
||||
int
|
||||
NCZ_filter_freelist(NC_VAR_INFO_T* var)
|
||||
NCZ_filter_freelists(NC_VAR_INFO_T* var)
|
||||
{
|
||||
int i, stat=NC_NOERR;
|
||||
NClist* filters = (NClist*)var->filters;
|
||||
NClist* filters = NULL;
|
||||
NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
|
||||
|
||||
ZTRACE(6,"var=%s",var->hdr.name);
|
||||
filters = (NClist*)var->filters;
|
||||
if(filters == NULL) goto done;
|
||||
/* Free the filter list elements */
|
||||
for(i=0;i<nclistlength(filters);i++) {
|
||||
@ -294,6 +301,16 @@ NCZ_filter_freelist(NC_VAR_INFO_T* var)
|
||||
}
|
||||
nclistfree(filters);
|
||||
var->filters = NULL;
|
||||
/* Free the incomplete filters */
|
||||
filters = (NClist*)zvar->incompletefilters;
|
||||
if(filters == NULL) goto done;
|
||||
/* Free the filter list elements */
|
||||
for(i=0;i<nclistlength(filters);i++) {
|
||||
struct NCZ_Filter* spec = nclistget(filters,i);
|
||||
if((stat = NCZ_filter_free(spec))) goto done;
|
||||
}
|
||||
nclistfree(filters);
|
||||
zvar->incompletefilters = NULL;
|
||||
done:
|
||||
return ZUNTRACE(stat);
|
||||
}
|
||||
@ -337,7 +354,7 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t
|
||||
int stat = NC_NOERR;
|
||||
struct NCZ_Filter* fi = NULL;
|
||||
NCZ_Plugin* plugin = NULL;
|
||||
NCZ_HDF5 hdf5 = hdf5_empty;
|
||||
NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
|
||||
|
||||
ZTRACE(6,"file=%s var=%s id=%u nparams=%u params=%p",file->hdr.name,var->hdr.name,id,nparams,params);
|
||||
|
||||
@ -345,24 +362,16 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t
|
||||
{stat = NC_EINVAL; goto done;}
|
||||
|
||||
if(var->filters == NULL) var->filters = (void*)nclistnew();
|
||||
if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew();
|
||||
|
||||
/* Before anything else, find the matching plugin */
|
||||
if((stat = NCZ_plugin_loaded(id,&plugin))) goto done;
|
||||
if(plugin == NULL || plugin->codec.codec == NULL) { /* fail */
|
||||
if(plugin == NULL) {
|
||||
ZLOG(NCLOGWARN,"no such plugin: %u",(unsigned)id);
|
||||
stat = NC_ENOFILTER;
|
||||
goto done;
|
||||
}
|
||||
|
||||
/* Fill in the hdf5 */
|
||||
hdf5 = hdf5_empty;
|
||||
hdf5.id = id;
|
||||
/* Capture the visible parameters */
|
||||
hdf5.visible.nparams = nparams;
|
||||
if(nparams > 0) {
|
||||
if((stat = paramclone(nparams,&hdf5.visible.params,params))) goto done;
|
||||
}
|
||||
|
||||
/* Find the NCZ_Filter */
|
||||
if((stat=NCZ_filter_lookup(var,id,&fi))) goto done;
|
||||
if(fi != NULL) {
|
||||
@ -373,19 +382,31 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t
|
||||
if((fi = calloc(1,sizeof(struct NCZ_Filter))) == NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
fi->plugin = plugin;
|
||||
nclistpush((NClist*)var->filters, fi);
|
||||
}
|
||||
if(plugin->incomplete) {
|
||||
fi->flags |= (FLAG_INCOMPLETE);
|
||||
nclistpush((NClist*)zvar->incompletefilters, fi);
|
||||
} else
|
||||
nclistpush((NClist*)var->filters, fi);
|
||||
}
|
||||
|
||||
/* (over)write the HDF5 parameters */
|
||||
nullfree(fi->hdf5.visible.params);
|
||||
nullfree(fi->hdf5.working.params);
|
||||
fi->hdf5.working.nparams = 0;
|
||||
fi->hdf5.working.params = NULL;
|
||||
fi->hdf5 = hdf5;
|
||||
hdf5 = hdf5_empty;
|
||||
fi->flags |= FLAG_VISIBLE;
|
||||
if(!FILTERINCOMPLETE(fi)) {
|
||||
/* (over)write the HDF5 parameters */
|
||||
nullfree(fi->hdf5.visible.params);
|
||||
nullfree(fi->hdf5.working.params);
|
||||
/* Fill in the hdf5 */
|
||||
fi->hdf5 = hdf5_empty; /* struct copy */
|
||||
fi->hdf5.id = id;
|
||||
/* Capture the visible parameters */
|
||||
fi->hdf5.visible.nparams = nparams;
|
||||
if(nparams > 0) {
|
||||
if((stat = paramclone(nparams,&fi->hdf5.visible.params,params))) goto done;
|
||||
}
|
||||
fi->hdf5.working.nparams = 0;
|
||||
fi->hdf5.working.params = NULL;
|
||||
fi->flags |= FLAG_VISIBLE;
|
||||
}
|
||||
|
||||
fi = NULL; /* either way,its in the var->filters list */
|
||||
fi = NULL; /* either way,its in a filters list */
|
||||
|
||||
done:
|
||||
if(fi) NCZ_filter_free(fi);
|
||||
@ -422,7 +443,6 @@ NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp
|
||||
int i;
|
||||
NClist* flist = (NClist*)var->filters;
|
||||
|
||||
|
||||
ZTRACE(6,"var=%s id=%u",var->hdr.name,id);
|
||||
|
||||
if(specp) *specp = NULL;
|
||||
@ -434,7 +454,7 @@ NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp
|
||||
for(i=0;i<nclistlength(flist);i++) {
|
||||
NCZ_Filter* spec = nclistget(flist,i);
|
||||
assert(spec != NULL);
|
||||
if(spec->hdf5.id == id) {
|
||||
if(spec->hdf5.id == id && !FILTERINCOMPLETE(spec)) {
|
||||
if(specp) *specp = spec;
|
||||
break;
|
||||
}
|
||||
@ -606,7 +626,7 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids
|
||||
NC_GRP_INFO_T* grp = NULL;
|
||||
NC_VAR_INFO_T* var = NULL;
|
||||
NClist* flist = NULL;
|
||||
size_t nfilters = 0;
|
||||
size_t nfilters;
|
||||
|
||||
ZTRACE(1,"ncid=%d varid=%d",ncid,varid);
|
||||
|
||||
@ -624,12 +644,12 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids
|
||||
|
||||
flist = var->filters;
|
||||
|
||||
nfilters = nclistlength(flist);
|
||||
nfilters = nclistlength(flist); /* including incomplets */
|
||||
if(nfilters > 0 && ids != NULL) {
|
||||
int k;
|
||||
for(k=0;k<nfilters;k++) {
|
||||
struct NCZ_Filter* f = (struct NCZ_Filter*)nclistget(flist,k);
|
||||
ids[k] = f->hdf5.id;
|
||||
ids[k] = f->hdf5.id;
|
||||
}
|
||||
}
|
||||
if(nfiltersp) *nfiltersp = nfilters;
|
||||
@ -710,7 +730,7 @@ NCZ_inq_filter_avail(int ncid, unsigned id)
|
||||
if((stat = NCZ_filter_initialize())) goto done;
|
||||
/* Check the available filters list */
|
||||
if((stat = NCZ_plugin_loaded((int)id, &plug))) goto done;
|
||||
if(plug == NULL)
|
||||
if(plug == NULL || plug->incomplete)
|
||||
stat = NC_ENOFILTER;
|
||||
done:
|
||||
return ZUNTRACE(stat);
|
||||
@ -770,6 +790,9 @@ NCZ_filter_finalize(void)
|
||||
if(nclistlength(default_libs) > 0) {
|
||||
for(i=0;i<nclistlength(default_libs);i++) {
|
||||
NCPSharedLib* l = (NCPSharedLib*)nclistget(default_libs,i);
|
||||
#ifdef DEBUGL
|
||||
fprintf(stderr,">>> DEBUGL: NCZ_filter_finalize: reclaim default_lib[i]=%p\n",l);
|
||||
#endif
|
||||
if(l != NULL) (void)ncpsharedlibfree(l);
|
||||
}
|
||||
}
|
||||
@ -822,7 +845,9 @@ NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* cha
|
||||
/* Make sure all the filters are loaded && setup */
|
||||
for(i=0;i<nclistlength(chain);i++) {
|
||||
struct NCZ_Filter* f = (struct NCZ_Filter*)nclistget(chain,i);
|
||||
assert(f != NULL && f->hdf5.id > 0 && f->plugin != NULL);
|
||||
assert(f != NULL);
|
||||
if(FILTERINCOMPLETE(f)) {stat = NC_ENOFILTER; goto done;}
|
||||
assert(f->hdf5.id > 0 && f->plugin != NULL);
|
||||
if(!(f->flags & FLAG_WORKING)) {/* working not yet available */
|
||||
if((stat = ensure_working(var,f))) goto done;
|
||||
}
|
||||
@ -938,10 +963,9 @@ done:
|
||||
return ZUNTRACEX(stat,"codec=%s",NULLIFY(filter->codec.codec));
|
||||
}
|
||||
|
||||
|
||||
/* Build filter from parsed Zarr metadata */
|
||||
int
|
||||
NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* jfilter)
|
||||
NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex)
|
||||
{
|
||||
int i,stat = NC_NOERR;
|
||||
NCZ_Filter* filter = NULL;
|
||||
@ -949,10 +973,12 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j
|
||||
NCZ_Plugin* plugin = NULL;
|
||||
NCZ_Codec codec = codec_empty;
|
||||
NCZ_HDF5 hdf5 = hdf5_empty;
|
||||
NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
|
||||
|
||||
ZTRACE(6,"file=%s var=%s jfilter=%s",file->hdr.name,var->hdr.name,NCJtrace(jfilter));
|
||||
|
||||
if(var->filters == NULL) var->filters = nclistnew();
|
||||
if(zvar->incompletefilters == NULL) zvar->incompletefilters = nclistnew();
|
||||
|
||||
/* Get the id of this codec filter */
|
||||
if(NCJdictget(jfilter,"id",&jvalue)<0) {stat = NC_EFILTER; goto done;}
|
||||
@ -973,6 +999,10 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j
|
||||
{plugin = loaded_plugins[i]; break;}
|
||||
}
|
||||
|
||||
/* Will always have a filter; possibly unknown */
|
||||
if((filter = calloc(1,sizeof(NCZ_Filter)))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
filter->chainindex = chainindex;
|
||||
|
||||
if(plugin != NULL) {
|
||||
/* Save the hdf5 id */
|
||||
hdf5.id = plugin->codec.codec->hdf5id;
|
||||
@ -984,13 +1014,16 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j
|
||||
#endif
|
||||
if(stat) goto done;
|
||||
}
|
||||
|
||||
if((filter = calloc(1,sizeof(NCZ_Filter)))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
filter->flags |= FLAG_VISIBLE;
|
||||
filter->hdf5 = hdf5; hdf5 = hdf5_empty;
|
||||
filter->codec = codec; codec = codec_empty;
|
||||
filter->flags |= FLAG_CODEC;
|
||||
filter->plugin = plugin; plugin = NULL;
|
||||
} else {
|
||||
/* Create a fake filter so we do not forget about this codec */
|
||||
filter->hdf5 = hdf5_empty;
|
||||
filter->codec = codec; codec = codec_empty;
|
||||
filter->flags |= (FLAG_INCOMPLETE|FLAG_CODEC);
|
||||
}
|
||||
|
||||
if(filter != NULL) {
|
||||
@ -1199,20 +1232,19 @@ NCZ_load_all_plugins(void)
|
||||
}
|
||||
}
|
||||
|
||||
/* Expunge all plugins for which we do not have both HDF5 and codec */
|
||||
/* Mark all plugins for which we do not have both HDF5 and codec */
|
||||
{
|
||||
int i;
|
||||
NCZ_Plugin* p;
|
||||
for(i=0;i<loaded_plugins_max;i++) {
|
||||
if((p = loaded_plugins[i]) != NULL) {
|
||||
if(p->hdf5.filter == NULL || p->codec.codec == NULL) {
|
||||
/* mark this entry as incomplete */
|
||||
p->incomplete = 1;
|
||||
#ifdef DEBUGL
|
||||
fprintf(stderr,">>> DEBUGL: Incomplete plugin: expunged: id=%u; reasons: %s %s\n",i,
|
||||
fprintf(stderr,">>> DEBUGL: Incomplete plugin: id=%u; reasons: %s %s\n",i,
|
||||
(p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":""));
|
||||
#endif
|
||||
/* expunge this entry */
|
||||
(void)NCZ_unload_plugin(p);
|
||||
loaded_plugins[i] = NULL;
|
||||
}
|
||||
#ifdef DEBUGL
|
||||
else
|
||||
@ -1227,6 +1259,7 @@ NCZ_load_all_plugins(void)
|
||||
NCZ_Plugin* p;
|
||||
for(i=0;i<loaded_plugins_max;i++) {
|
||||
if((p = loaded_plugins[i]) != NULL) {
|
||||
if(p->incomplete) continue;
|
||||
if(p->hdf5.filter != NULL && p->codec.codec != NULL) {
|
||||
if(p->codec.codec && p->codec.codec->NCZ_codec_initialize)
|
||||
p->codec.codec->NCZ_codec_initialize();
|
||||
@ -1347,12 +1380,38 @@ done:
|
||||
return ZUNTRACE(stat);
|
||||
}
|
||||
|
||||
int
|
||||
loadcodecdefaults(const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
int lib_used = 0;
|
||||
|
||||
nclistpush(default_libs,lib);
|
||||
for(;*cp;cp++) {
|
||||
struct CodecAPI* c0;
|
||||
#ifdef DEBUGL
|
||||
fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id);
|
||||
#endif
|
||||
c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI));
|
||||
if(c0 == NULL) {stat = NC_ENOMEM; goto done;}
|
||||
c0->codec = *cp;
|
||||
c0->codeclib = lib;
|
||||
lib_used = 1; /* remember */
|
||||
nclistpush(codec_defaults,c0); c0 = NULL;
|
||||
}
|
||||
done:
|
||||
if(lib_usedp) *lib_usedp = lib_used;
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCZ_Plugin* plugin = NULL;
|
||||
const H5Z_class2_t* h5class = NULL;
|
||||
H5PL_type_t h5type = 0;
|
||||
const NCZ_codec_t** cp = NULL;
|
||||
const NCZ_codec_t* codec = NULL;
|
||||
NCPSharedLib* lib = NULL;
|
||||
int flags = NCP_GLOBAL;
|
||||
@ -1374,7 +1433,7 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
|
||||
if((stat = ncpload(lib,path,flags))) goto done;
|
||||
|
||||
#ifdef DEBUGL
|
||||
fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s\n",path);
|
||||
fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s lib=%p\n",path,lib);
|
||||
#endif
|
||||
|
||||
/* See what we have */
|
||||
@ -1384,49 +1443,27 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp)
|
||||
const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info");
|
||||
const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults");
|
||||
|
||||
if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL)
|
||||
if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL)
|
||||
{stat = NC_ENOFILTER; goto done;}
|
||||
|
||||
/* We can have cpd or we can have (gpt && gpi && npi) but not both sets */
|
||||
if(cpd != NULL) {
|
||||
/* Deal with defaults first */
|
||||
const NCZ_codec_t** cp = NULL;
|
||||
nclistpush(default_libs,lib);
|
||||
cp = (const NCZ_codec_t**)cpd();
|
||||
#ifdef DEBUGL
|
||||
fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp);
|
||||
#endif
|
||||
for(;*cp;cp++) {
|
||||
struct CodecAPI* c0;
|
||||
#ifdef DEBUGL
|
||||
fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id);
|
||||
#endif
|
||||
c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI));
|
||||
if(c0 == NULL) {stat = NC_ENOMEM; goto done1;}
|
||||
c0->codec = *cp;
|
||||
c0->codeclib = lib;
|
||||
nclistpush(codec_defaults,c0); c0 = NULL;
|
||||
} else {/* cpd => !gpt && !gpi && !npi */
|
||||
if(gpt != NULL && gpi != NULL) { /* get HDF5 info */
|
||||
h5type = gpt();
|
||||
h5class = gpi();
|
||||
/* Verify */
|
||||
if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;}
|
||||
if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;}
|
||||
}
|
||||
if(npi != NULL) {/* get Codec info */
|
||||
codec = npi();
|
||||
/* Verify */
|
||||
if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;}
|
||||
if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;}
|
||||
}
|
||||
done1:
|
||||
lib = NULL;
|
||||
goto done;
|
||||
}
|
||||
|
||||
if(gpt != NULL && gpi != NULL) {
|
||||
/* get HDF5 info */
|
||||
H5PL_type_t h5type = gpt();
|
||||
h5class = gpi();
|
||||
/* Verify */
|
||||
if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;}
|
||||
if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;}
|
||||
}
|
||||
|
||||
if(npi != NULL) {
|
||||
/* get Codec info */
|
||||
codec = npi();
|
||||
/* Verify */
|
||||
if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;}
|
||||
if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef DEBUGL
|
||||
@ -1436,6 +1473,17 @@ if(codec) fprintf(stderr,">>> %u/%s",codec->hdf5id,codec->codecid);
|
||||
fprintf(stderr,">>> \n");
|
||||
#endif
|
||||
|
||||
/* Handle defaults separately */
|
||||
if(cp != NULL) {
|
||||
int used = 0;
|
||||
#ifdef DEBUGL
|
||||
fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp);
|
||||
#endif
|
||||
if((stat = loadcodecdefaults(path,cp,lib,&used))) goto done;
|
||||
if(used) lib = NULL;
|
||||
goto done;
|
||||
}
|
||||
|
||||
if(h5class != NULL && codec != NULL) {
|
||||
/* Verify consistency of the HDF5 and the Codec */
|
||||
if(h5class->id != codec->hdf5id) goto done; /* ignore */
|
||||
@ -1461,33 +1509,29 @@ fprintf(stderr,">>> \n");
|
||||
}
|
||||
|
||||
/* Fill in the plugin */
|
||||
if(plugin->hdf5.filter == NULL) {
|
||||
if(h5class != NULL && plugin->hdf5.filter == NULL) {
|
||||
plugin->hdf5.filter = h5class;
|
||||
plugin->hdf5.hdf5lib = lib;
|
||||
lib = NULL;
|
||||
}
|
||||
if(plugin->codec.codec == NULL) {
|
||||
if(codec != NULL && plugin->codec.codec == NULL) {
|
||||
plugin->codec.codec = codec;
|
||||
plugin->codec.codeclib = lib;
|
||||
lib = NULL;
|
||||
}
|
||||
|
||||
#ifdef DEBUGL
|
||||
if(plugin)
|
||||
if(plugin)
|
||||
fprintf(stderr,">>> DEBUGL: load_plugin: %s\n",printplugin(plugin));
|
||||
#endif
|
||||
|
||||
|
||||
/* Cleanup */
|
||||
if(plugin->hdf5.hdf5lib == plugin->codec.codeclib)
|
||||
plugin->codec.codeclib = NULL;
|
||||
if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) /* Works for NULL case also */
|
||||
plugin->codec.codeclib = NULL;
|
||||
if((stat=NCZ_plugin_save(h5id,plugin))) goto done;
|
||||
plugin = NULL;
|
||||
|
||||
done:
|
||||
if(lib) {
|
||||
(void)ncpsharedlibfree(lib);
|
||||
}
|
||||
if(lib)
|
||||
(void)ncpsharedlibfree(lib);
|
||||
if(plugin) NCZ_unload_plugin(plugin);
|
||||
return ZUNTRACEX(stat,"plug=%p",*plugp);
|
||||
}
|
||||
@ -1541,7 +1585,6 @@ pluginnamecheck(const char* name)
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************************/
|
||||
/* _Codecs attribute */
|
||||
|
||||
@ -1552,23 +1595,51 @@ NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data)
|
||||
size_t len;
|
||||
char* contents = NULL;
|
||||
NCbytes* buf = NULL;
|
||||
NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
|
||||
NClist* filters = (NClist*)var->filters;
|
||||
|
||||
NClist* incfilters = (NClist*)zvar->incompletefilters;
|
||||
NCZ_Filter** chain = NULL;
|
||||
size_t nfilters;
|
||||
|
||||
ZTRACE(6,"var=%s",var->hdr.name);
|
||||
if(nclistlength(filters) == 0) {stat = NC_ENOTATT; goto done;}
|
||||
|
||||
nfilters = nclistlength(filters) + nclistlength(incfilters);
|
||||
|
||||
if(nfilters == 0)
|
||||
{stat = NC_ENOTATT; goto done;}
|
||||
|
||||
/* Collect all filters in chain order */
|
||||
chain = (NCZ_Filter**)calloc(sizeof(NCZ_Filter*),nfilters);
|
||||
if(chain == NULL) {stat = NC_ENOMEM; goto done;}
|
||||
|
||||
/* Sort the complete and incomplete filters in order in the chain */
|
||||
for(i=0;i<nclistlength(filters);i++) {
|
||||
NCZ_Filter* f = nclistget(filters,i);
|
||||
assert(chain[f->chainindex] == NULL);
|
||||
chain[f->chainindex] = f;
|
||||
}
|
||||
for(i=0;i<nclistlength(incfilters);i++) {
|
||||
NCZ_Filter* f = nclistget(incfilters,i);
|
||||
assert(chain[f->chainindex] == NULL);
|
||||
chain[f->chainindex] = f;
|
||||
}
|
||||
|
||||
/* Now construct the attribute */
|
||||
buf = ncbytesnew(); ncbytessetalloc(buf,1024);
|
||||
ncbytescat(buf,"[");
|
||||
for(i=0;i<nclistlength(filters);i++) {
|
||||
NCZ_Filter* spec = nclistget(filters,i);
|
||||
for(i=0;i<nfilters;i++) {
|
||||
NCZ_Filter* spec = chain[i];
|
||||
if(i > 0) ncbytescat(buf,",");
|
||||
ncbytescat(buf,spec->codec.codec);
|
||||
}
|
||||
ncbytescat(buf,"]");
|
||||
|
||||
len = ncbyteslength(buf);
|
||||
contents = nclistcontents(buf);
|
||||
if(lenp) *lenp = len;
|
||||
if(data) strncpy((char*)data,contents,len+1);
|
||||
done:
|
||||
nullfree(chain);
|
||||
ncbytesfree(buf);
|
||||
return ZUNTRACEX(stat,"len=%u data=%p",(unsigned)len,data);
|
||||
}
|
||||
@ -1577,6 +1648,7 @@ static int
|
||||
ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
if(FILTERINCOMPLETE(filter)) {stat = NC_ENOFILTER; goto done;}
|
||||
if(!(filter->flags & FLAG_WORKING)) {
|
||||
const size_t oldnparams = filter->hdf5.visible.nparams;
|
||||
const unsigned* oldparams = filter->hdf5.visible.params;
|
||||
@ -1683,7 +1755,9 @@ NCZ_filter_setup(NC_VAR_INFO_T* var)
|
||||
filters = (NClist*)var->filters;
|
||||
for(i=0;i<nclistlength(filters);i++) {
|
||||
NCZ_Filter* filter = (NCZ_Filter*)nclistget(filters,i);
|
||||
assert(filter != NULL && filter->plugin != NULL);
|
||||
assert(filter != NULL);
|
||||
if(FILTERINCOMPLETE(filter)) continue; /* ignore these */
|
||||
assert(filter->plugin != NULL);
|
||||
assert((filter->flags & FLAG_VISIBLE)); /* Assume visible params are defined */
|
||||
/* verify */
|
||||
assert(filter->hdf5.id > 0 && (filter->hdf5.visible.nparams == 0 || filter->hdf5.visible.params != NULL));
|
||||
|
@ -32,11 +32,11 @@ int NCZ_filter_initialize(void);
|
||||
int NCZ_filter_finalize(void);
|
||||
int NCZ_addfilter(NC_FILE_INFO_T*, NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params);
|
||||
int NCZ_filter_setup(NC_VAR_INFO_T* var);
|
||||
int NCZ_filter_freelist(NC_VAR_INFO_T* var);
|
||||
int NCZ_filter_freelists(NC_VAR_INFO_T* var);
|
||||
int NCZ_codec_freelist(NCZ_VAR_INFO_T* zvar);
|
||||
int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, size_t insize, void* indata, size_t* outlen, void** outdata, int encode);
|
||||
int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**);
|
||||
int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter);
|
||||
int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex);
|
||||
int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data);
|
||||
|
||||
#endif /*ZFILTER_H*/
|
||||
|
@ -358,7 +358,7 @@ close_vars(NC_GRP_INFO_T *grp)
|
||||
|
||||
/* Reclaim filters */
|
||||
if(var->filters != NULL) {
|
||||
(void)NCZ_filter_freelist(var);
|
||||
(void)NCZ_filter_freelists(var);
|
||||
}
|
||||
var->filters = NULL;
|
||||
|
||||
|
@ -185,6 +185,7 @@ typedef struct NCZ_VAR_INFO {
|
||||
struct NCZChunkCache* cache;
|
||||
struct NClist* xarray; /* names from _ARRAY_DIMENSIONS */
|
||||
char dimension_separator; /* '.' | '/' */
|
||||
NClist* incompletefilters;
|
||||
} NCZ_VAR_INFO_T;
|
||||
|
||||
/* Struct to hold ZARR-specific info for a field. */
|
||||
|
@ -1446,6 +1446,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
|
||||
NClist* dimnames = nclistnew();
|
||||
#ifdef ENABLE_NCZARR_FILTERS
|
||||
NCjson* jfilter = NULL;
|
||||
int chainindex;
|
||||
#endif
|
||||
|
||||
zinfo = file->format_file_info;
|
||||
@ -1614,8 +1615,10 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
|
||||
/* Do filters key before compressor key so final filter chain is in correct order */
|
||||
{
|
||||
if(var->filters == NULL) var->filters = (void*)nclistnew();
|
||||
if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew();
|
||||
#ifdef ENABLE_NCZARR_FILTERS
|
||||
{ int k;
|
||||
chainindex = 0; /* track location of filter in the chain */
|
||||
if((stat = NCZ_filter_initialize())) goto done;
|
||||
if((stat = NCJdictget(jvar,"filters",&jvalue))) goto done;
|
||||
if(jvalue != NULL && NCJsort(jvalue) != NCJ_NULL) {
|
||||
@ -1625,7 +1628,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
|
||||
jfilter = NCJith(jvalue,k);
|
||||
if(jfilter == NULL) break; /* done */
|
||||
if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;}
|
||||
if((stat = NCZ_filter_build(file,var,jfilter))) goto done;
|
||||
if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1642,7 +1645,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
|
||||
if((stat = NCJdictget(jvar,"compressor",&jfilter))) goto done;
|
||||
if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) {
|
||||
if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;}
|
||||
if((stat = NCZ_filter_build(file,var,jfilter))) goto done;
|
||||
if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
@ -16,7 +16,7 @@
|
||||
include $(top_srcdir)/lib_flags.am
|
||||
AM_CPPFLAGS += -I$(top_srcdir)/libsrc
|
||||
AM_CPPFLAGS += -DTOPSRCDIR=${abs_top_srcdir}
|
||||
AM_CPPFLAGS += -DTOPBINDIR=${abs_top_bindir}
|
||||
AM_CPPFLAGS += -DTOPBINDIR=${abs_top_builddir}
|
||||
LDADD = ${top_builddir}/liblib/libnetcdf.la
|
||||
AM_CPPFLAGS += -I$(top_builddir)/liblib -I$(top_builddir)/include -I$(top_srcdir)/libsrc
|
||||
|
||||
@ -107,7 +107,7 @@ endif
|
||||
# The .c files that are generated with m4 are already distributed, but
|
||||
# we also include the original m4 files, plus test scripts data.
|
||||
EXTRA_DIST = test_get.m4 test_put.m4 run_diskless.sh run_diskless2.sh \
|
||||
run_diskless5.sh run_mmap.sh run_pnetcdf_test.sh test_read.m4 \
|
||||
run_diskless5.sh run_mmap.sh test_read.m4 \
|
||||
test_write.m4 ref_tst_diskless2.cdl tst_diskless5.cdl \
|
||||
ref_tst_diskless3_create.cdl ref_tst_diskless3_open.cdl \
|
||||
run_inmemory.sh run_mmap.sh f03tst_open_mem.nc test_byterange.sh \
|
||||
|
@ -3,6 +3,7 @@
|
||||
# This script runs some PnetCDF I/O tests
|
||||
|
||||
set -e
|
||||
|
||||
echo
|
||||
echo "Testing file created with PnetCDF is modifiable with netCDF..."
|
||||
./tst_pnetcdf
|
||||
|
@ -50,6 +50,7 @@ IF(USE_HDF5 AND ENABLE_FILTER_TESTING)
|
||||
build_bin_test(tst_filter_avail)
|
||||
build_bin_test(test_filter_vlen)
|
||||
ADD_SH_TEST(nc_test4 tst_filter)
|
||||
ADD_SH_TEST(nc_test4 tst_unknown)
|
||||
ADD_SH_TEST(nc_test4 tst_specific_filters)
|
||||
IF(ENABLE_CLIENTSIDE_FILTERS)
|
||||
add_bin_test(nc_test4 test_filter_reg)
|
||||
|
@ -83,6 +83,7 @@ extradir =
|
||||
check_PROGRAMS += test_filter test_filter_misc test_filter_order test_filter_repeat test_filter_vlen
|
||||
check_PROGRAMS += tst_multifilter tst_filter_avail
|
||||
TESTS += tst_filter.sh
|
||||
TESTS += tst_unknown.sh
|
||||
TESTS += tst_specific_filters.sh
|
||||
endif
|
||||
endif # USE_HDF5
|
||||
@ -109,8 +110,8 @@ ref_filteredvv.cdl ref_multi.cdl \
|
||||
ref_ncgenF.cdl ref_nccopyF.cdl \
|
||||
ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
|
||||
ref_filter_order_create.txt ref_filter_order_read.txt \
|
||||
ref_any.cdl tst_specific_filters.sh \
|
||||
tst_virtual_datasets.c
|
||||
ref_any.cdl tst_specific_filters.sh tst_unknown.sh \
|
||||
tst_virtual_datasets.c noop1.cdl unknown.cdl
|
||||
|
||||
# The tst_filterinstall test can only be run after an install
|
||||
# occurred with --with-plugin-dir enabled. So there is no point
|
||||
@ -128,7 +129,8 @@ perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \
|
||||
floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \
|
||||
testfilter_reg.nc filterrepeat.txt tmp_fillonly.nc \
|
||||
testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1 \
|
||||
tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp tmp_bzip2.c bzip2.nc
|
||||
tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp \
|
||||
tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp
|
||||
|
||||
DISTCLEANFILES = findplugin.sh run_par_test.sh
|
||||
|
||||
|
81
nc_test4/findplugin.in
Normal file → Executable file
81
nc_test4/findplugin.in
Normal file → Executable file
@ -20,27 +20,43 @@
|
||||
# is obviated by setting the LIBRARY_OUTPUT_DIRECTORY
|
||||
# variables: see hdf5plugins/CMakeLists.txt
|
||||
|
||||
findplugin() {
|
||||
# Define location of execution
|
||||
TOPSRCDIR='@abs_top_srcdir@'
|
||||
TOPBUILDDIR='@abs_top_builddir@'
|
||||
|
||||
FP_NAME="$1"
|
||||
# Need info from test_common
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ${TOPBUILDDIR}/test_common.sh
|
||||
|
||||
FP_PLUGIN_LIB=
|
||||
findpluginext() {
|
||||
# Infer the expected plugin shared library extension
|
||||
# Note: will fail if used before plugins is built
|
||||
# Also assumes that misc filter is always built
|
||||
# Approach is to use find to see what is in plugins directory.
|
||||
TSO=`find ${TOPBUILDDIR}/plugins -name '*misc.so'`
|
||||
TDY=`find ${TOPBUILDDIR}/plugins -name '*misc.0.dylib'`
|
||||
TCYG=`find ${TOPBUILDDIR}/plugins -name 'cyg*misc.dll'`
|
||||
TDLL=`find ${TOPBUILDDIR}/plugins -name '*misc.dll'`
|
||||
if test "x$TSO" != x ; then
|
||||
FP_PLUGIN_EXT="so"
|
||||
FP_PLUGIN_PRE="lib"
|
||||
elif test "x$TDY" != x ; then
|
||||
FP_PLUGIN_EXT="0.dylib"
|
||||
FP_PLUGIN_PRE="lib"
|
||||
elif test "x$TCYG" != x ; then
|
||||
FP_PLUGIN_EXT="dll"
|
||||
FP_PLUGIN_PRE="cyg"
|
||||
elif test "x$TDLL" != x ; then
|
||||
FP_PLUGIN_EXT="dll"
|
||||
FP_PLUGIN_PRE=""
|
||||
else # unknown
|
||||
unset FP_PLUGIN_EXT
|
||||
unset FP_PLUGIN_PRE
|
||||
fi
|
||||
}
|
||||
|
||||
findplugindir() {
|
||||
FP_PLUGIN_DIR=
|
||||
|
||||
# Figure out the plugin file name
|
||||
# Test for visual studio before cygwin since both might be true
|
||||
if test "x$FP_ISMSVC" != x ; then
|
||||
FP_PLUGIN_LIB="${FP_NAME}.dll"
|
||||
elif test "x$FP_ISMINGW" != x || test "x$FP_ISMSYS" != x ; then
|
||||
FP_PLUGIN_LIB="${FP_NAME}.dll"
|
||||
elif test "x$FP_ISCYGWIN" != x ; then
|
||||
FP_PLUGIN_LIB="cyg${FP_NAME}.dll"
|
||||
elif test "x$FP_ISOSX" != x ; then
|
||||
FP_PLUGIN_LIB="lib${FP_NAME}.so" # Should this include the version number in the name?
|
||||
else # Presumably some form on *nix"
|
||||
FP_PLUGIN_LIB="lib${FP_NAME}.so"
|
||||
fi
|
||||
|
||||
# Figure out the path to where the lib is stored
|
||||
# This can probably be simplified
|
||||
|
||||
@ -52,15 +68,15 @@ cd ${CURWD}
|
||||
# Case 1: Cmake with Visual Studio
|
||||
if test "x$FP_ISCMAKE" != x -a "x${FP_ISMSVC}" != x ; then
|
||||
# Case 1a: ignore the build type directory
|
||||
if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
|
||||
if test -e "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
|
||||
FP_PLUGIN_DIR="${FP_PLUGINS}"
|
||||
fi
|
||||
else # Case 2: automake
|
||||
# Case 2a: look in .libs
|
||||
if test -f "${FP_PLUGINS}/.libs/${FP_PLUGIN_LIB}" ; then
|
||||
if test -e "${FP_PLUGINS}/.libs" ; then
|
||||
FP_PLUGIN_DIR="${FP_PLUGINS}/.libs"
|
||||
else # Case 2: look in FP_PLUGINS directly
|
||||
if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then
|
||||
if test -e "${FP_PLUGINS}" ; then
|
||||
FP_PLUGIN_DIR="${FP_PLUGINS}"
|
||||
fi
|
||||
fi
|
||||
@ -71,16 +87,35 @@ if test "x$FP_PLUGIN_DIR" = x ; then
|
||||
echo "***Fail: Could not locate a usable HDF5_PLUGIN_DIR"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Make local path
|
||||
FP_PLUGIN_DIR=`${NCPATHCVT} -F $FP_PLUGIN_DIR`
|
||||
HDF5_PLUGIN_DIR="$FP_PLUGIN_DIR"
|
||||
}
|
||||
|
||||
findplugin() {
|
||||
|
||||
FP_NAME="$1"
|
||||
|
||||
FP_PLUGIN_LIB=
|
||||
|
||||
# Figure out the plugin file name
|
||||
FP_PLUGIN_LIB="${FP_PLUGIN_PRE}${FP_NAME}.${FP_PLUGIN_EXT}"
|
||||
|
||||
# Verify
|
||||
if ! test -f "$FP_PLUGIN_DIR/$FP_PLUGIN_LIB" ; then
|
||||
echo "***Fail: Could not locate a usable HDF5_PLUGIN_LIB"
|
||||
return 1
|
||||
fi
|
||||
|
||||
FP_PLUGIN_DIR=`${NCPATHCVT} $FP_PLUGIN_DIR`
|
||||
|
||||
# Set the final output variables
|
||||
HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB"
|
||||
HDF5_PLUGIN_DIR="$FP_PLUGIN_DIR"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# init
|
||||
unset HDF5_PLUGIN_DIR
|
||||
findpluginext
|
||||
findplugindir
|
||||
|
83
nc_test4/noop1.cdl
Normal file
83
nc_test4/noop1.cdl
Normal file
@ -0,0 +1,83 @@
|
||||
netcdf bzip2 {
|
||||
dimensions:
|
||||
dim0 = 4 ;
|
||||
dim1 = 4 ;
|
||||
dim2 = 4 ;
|
||||
dim3 = 4 ;
|
||||
variables:
|
||||
float var(dim0, dim1, dim2, dim3) ;
|
||||
var:_Storage = "chunked" ;
|
||||
var:_ChunkSizes = 4, 4, 4, 4 ;
|
||||
var:_Filter = "40001,17" ;
|
||||
var:_NoFill = "true" ;
|
||||
|
||||
// global attributes:
|
||||
:_Format = "netCDF-4" ;
|
||||
data:
|
||||
|
||||
var =
|
||||
0, 1, 2, 3,
|
||||
4, 5, 6, 7,
|
||||
8, 9, 10, 11,
|
||||
12, 13, 14, 15,
|
||||
16, 17, 18, 19,
|
||||
20, 21, 22, 23,
|
||||
24, 25, 26, 27,
|
||||
28, 29, 30, 31,
|
||||
32, 33, 34, 35,
|
||||
36, 37, 38, 39,
|
||||
40, 41, 42, 43,
|
||||
44, 45, 46, 47,
|
||||
48, 49, 50, 51,
|
||||
52, 53, 54, 55,
|
||||
56, 57, 58, 59,
|
||||
60, 61, 62, 63,
|
||||
64, 65, 66, 67,
|
||||
68, 69, 70, 71,
|
||||
72, 73, 74, 75,
|
||||
76, 77, 78, 79,
|
||||
80, 81, 82, 83,
|
||||
84, 85, 86, 87,
|
||||
88, 89, 90, 91,
|
||||
92, 93, 94, 95,
|
||||
96, 97, 98, 99,
|
||||
100, 101, 102, 103,
|
||||
104, 105, 106, 107,
|
||||
108, 109, 110, 111,
|
||||
112, 113, 114, 115,
|
||||
116, 117, 118, 119,
|
||||
120, 121, 122, 123,
|
||||
124, 125, 126, 127,
|
||||
128, 129, 130, 131,
|
||||
132, 133, 134, 135,
|
||||
136, 137, 138, 139,
|
||||
140, 141, 142, 143,
|
||||
144, 145, 146, 147,
|
||||
148, 149, 150, 151,
|
||||
152, 153, 154, 155,
|
||||
156, 157, 158, 159,
|
||||
160, 161, 162, 163,
|
||||
164, 165, 166, 167,
|
||||
168, 169, 170, 171,
|
||||
172, 173, 174, 175,
|
||||
176, 177, 178, 179,
|
||||
180, 181, 182, 183,
|
||||
184, 185, 186, 187,
|
||||
188, 189, 190, 191,
|
||||
192, 193, 194, 195,
|
||||
196, 197, 198, 199,
|
||||
200, 201, 202, 203,
|
||||
204, 205, 206, 207,
|
||||
208, 209, 210, 211,
|
||||
212, 213, 214, 215,
|
||||
216, 217, 218, 219,
|
||||
220, 221, 222, 223,
|
||||
224, 225, 226, 227,
|
||||
228, 229, 230, 231,
|
||||
232, 233, 234, 235,
|
||||
236, 237, 238, 239,
|
||||
240, 241, 242, 243,
|
||||
244, 245, 246, 247,
|
||||
248, 249, 250, 251,
|
||||
252, 253, 254, 255 ;
|
||||
}
|
@ -9,7 +9,6 @@ set -e
|
||||
API=1
|
||||
NG=1
|
||||
NCP=1
|
||||
UNK=1
|
||||
NGC=1
|
||||
MISC=1
|
||||
MULTI=1
|
||||
@ -41,18 +40,6 @@ trimleft() {
|
||||
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
|
||||
}
|
||||
|
||||
# Hide/unhide the noop filter
|
||||
hidenoop() {
|
||||
rm -fr ${HDF5_PLUGIN_DIR}/save
|
||||
mkdir ${HDF5_PLUGIN_DIR}/save
|
||||
mv ${NOOPDIR} ${HDF5_PLUGIN_DIR}/save
|
||||
}
|
||||
|
||||
unhidenoop() {
|
||||
mv ${HDF5_PLUGIN_DIR}/save/${NOOPLIB} ${HDF5_PLUGIN_DIR}
|
||||
rm -fr ${HDF5_PLUGIN_DIR}/save
|
||||
}
|
||||
|
||||
# Locate the plugin dir and the library names; argument order is critical
|
||||
# Find bzip2 and capture
|
||||
findplugin h5bzip2
|
||||
@ -180,36 +167,6 @@ echo " *** Pass: -F var,none"
|
||||
echo "*** Pass: all nccopy filter tests"
|
||||
fi
|
||||
|
||||
if test "x$UNK" = x1 ; then
|
||||
echo "*** Testing access to filter info when filter dll is not available"
|
||||
rm -f noop.nc ./tmp_filter.txt
|
||||
# xfail build noop.nc
|
||||
hidenoop
|
||||
if ${NCGEN} -lb -4 -o noop.nc ${srcdir}/noop.cdl ; then
|
||||
echo "*** FAIL: ncgen"
|
||||
else
|
||||
echo "*** XFAIL: ncgen"
|
||||
fi
|
||||
unhidenoop
|
||||
# build noop.nc
|
||||
${NCGEN} -lb -4 -o noop.nc ${srcdir}/noop.cdl
|
||||
# Now hide the filter code
|
||||
hidenoop
|
||||
rm -f ./tmp_filter.txt
|
||||
# This will xfail
|
||||
if ${NCDUMP} -s noop.nc > ./tmp_filter.txt ; then
|
||||
echo "*** FAIL: ncdump -hs noop.nc"
|
||||
else
|
||||
echo "*** XFAIL: ncdump -hs noop.nc"
|
||||
fi
|
||||
# Restore the filter code
|
||||
unhidenoop
|
||||
# Verify we can see filter when using -h
|
||||
rm -f ./tmp_filter.txt
|
||||
${NCDUMP} -hs noop.nc > ./tmp_filter.txt
|
||||
echo "*** Pass: unknown filter"
|
||||
fi
|
||||
|
||||
if test "x$NGC" = x1 ; then
|
||||
rm -f ./tmp_bzip2.c
|
||||
echo "*** Testing dynamic filters using ncgen with -lc"
|
||||
|
109
nc_test4/tst_unknown.sh
Executable file
109
nc_test4/tst_unknown.sh
Executable file
@ -0,0 +1,109 @@
|
||||
#!/bin/bash
|
||||
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
. "$srcdir/test_nczarr.sh"
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
# Load the findplugins function
|
||||
. ${builddir}/findplugin.sh
|
||||
echo "findplugin.sh loaded"
|
||||
|
||||
# Set up HDF5_PLUGIN_PATH
|
||||
export HDF5_PLUGIN_PATH=${HDF5_PLUGIN_DIR}
|
||||
|
||||
# Test operation with an unknown filter
|
||||
|
||||
# Function to remove selected -s attributes from file;
|
||||
# These attributes might be platform dependent
|
||||
sclean() {
|
||||
cat $1 \
|
||||
| sed -e '/:_IsNetcdf4/d' \
|
||||
| sed -e '/:_Endianness/d' \
|
||||
| sed -e '/_NCProperties/d' \
|
||||
| sed -e '/_SuperblockVersion/d' \
|
||||
| cat > $2
|
||||
}
|
||||
|
||||
# Function to extract _Filter attribute from a file
|
||||
# These attributes might be platform dependent
|
||||
getfilterattr() {
|
||||
sed -e '/var.*:_Filter/p' -ed <$1 >$2
|
||||
}
|
||||
|
||||
# Function to extract _Codecs attribute from a file
|
||||
# These attributes might be platform dependent
|
||||
getcodecsattr() {
|
||||
sed -e '/var.*:_Codecs/p' -ed <$1 >$2
|
||||
}
|
||||
|
||||
trimleft() {
|
||||
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
|
||||
}
|
||||
|
||||
# Locate the plugin path and the library names; argument order is critical
|
||||
|
||||
# Find noop and capture
|
||||
findplugin h5unknown
|
||||
UNKNOWNDIR="${HDF5_PLUGIN_DIR}"
|
||||
UNKNOWNLIB="${HDF5_PLUGIN_LIB}"
|
||||
UNKNOWNFILTER="${HDF5_PLUGIN_DIR}/${UNKNOWNLIB}"
|
||||
|
||||
# Verify
|
||||
if ! test -f ${UNKNOWNFILTER} ; then echo "Unable to locate ${UNKNOWNFILTER}"; exit 1; fi
|
||||
|
||||
testunk() {
|
||||
zext=$1
|
||||
echo "*** Testing access to filter info when filter implementation is not available for map $zext"
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
fileargs tmp_known
|
||||
deletemap $zext $file
|
||||
else
|
||||
file="tmp_known_${zfilt}.nc"
|
||||
rm -f $file
|
||||
fileurl="$file"
|
||||
fi
|
||||
printenv HDF5_PLUGIN_PATH
|
||||
# build .nc file using unknown
|
||||
${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/unknown.cdl
|
||||
# dump and clean file when filter is avail
|
||||
${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt
|
||||
# Remove irrelevant -s output
|
||||
sclean ./tmp_known_$zext.txt tmp_known_$zext.dump
|
||||
# Hide the filter
|
||||
rm -fr ${UNKNOWNDIR}/save
|
||||
mkdir -p ${UNKNOWNDIR}/save
|
||||
# Figure out all matching libs; make sure to remove .so, so.0, etc
|
||||
LSRC=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNFILTER}`
|
||||
LDST=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNDIR}/save`
|
||||
mv ${LSRC}* ${LDST}
|
||||
# Verify that the filter is no longer defined
|
||||
# Try to read the data; should xfail
|
||||
if ${NCDUMP} -s $fileurl > ./tmp_unk_$zext.dmp ; then
|
||||
echo "*** FAIL: filter found"
|
||||
found=1
|
||||
else
|
||||
echo "*** XFAIL: filter not found"
|
||||
found=0
|
||||
fi
|
||||
# Restore the filter
|
||||
LSRC="${LDST}"
|
||||
LDST=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNDIR}`
|
||||
mv ${LSRC}/* ${LDST}
|
||||
rm -fr ${UNKNOWNDIR}/save
|
||||
if test "x$found" = x1 ; then exit 1; fi
|
||||
}
|
||||
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
testunk file
|
||||
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testunk zip ; fi
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then testunk s3 ; fi
|
||||
else
|
||||
testunk nc
|
||||
fi
|
||||
|
||||
exit 0
|
83
nc_test4/unknown.cdl
Normal file
83
nc_test4/unknown.cdl
Normal file
@ -0,0 +1,83 @@
|
||||
netcdf bzip2 {
|
||||
dimensions:
|
||||
dim0 = 4 ;
|
||||
dim1 = 4 ;
|
||||
dim2 = 4 ;
|
||||
dim3 = 4 ;
|
||||
variables:
|
||||
float var(dim0, dim1, dim2, dim3) ;
|
||||
var:_Storage = "chunked" ;
|
||||
var:_ChunkSizes = 4, 4, 4, 4 ;
|
||||
var:_Filter = "33000" ;
|
||||
var:_NoFill = "true" ;
|
||||
|
||||
// global attributes:
|
||||
:_Format = "netCDF-4" ;
|
||||
data:
|
||||
|
||||
var =
|
||||
0, 1, 2, 3,
|
||||
4, 5, 6, 7,
|
||||
8, 9, 10, 11,
|
||||
12, 13, 14, 15,
|
||||
16, 17, 18, 19,
|
||||
20, 21, 22, 23,
|
||||
24, 25, 26, 27,
|
||||
28, 29, 30, 31,
|
||||
32, 33, 34, 35,
|
||||
36, 37, 38, 39,
|
||||
40, 41, 42, 43,
|
||||
44, 45, 46, 47,
|
||||
48, 49, 50, 51,
|
||||
52, 53, 54, 55,
|
||||
56, 57, 58, 59,
|
||||
60, 61, 62, 63,
|
||||
64, 65, 66, 67,
|
||||
68, 69, 70, 71,
|
||||
72, 73, 74, 75,
|
||||
76, 77, 78, 79,
|
||||
80, 81, 82, 83,
|
||||
84, 85, 86, 87,
|
||||
88, 89, 90, 91,
|
||||
92, 93, 94, 95,
|
||||
96, 97, 98, 99,
|
||||
100, 101, 102, 103,
|
||||
104, 105, 106, 107,
|
||||
108, 109, 110, 111,
|
||||
112, 113, 114, 115,
|
||||
116, 117, 118, 119,
|
||||
120, 121, 122, 123,
|
||||
124, 125, 126, 127,
|
||||
128, 129, 130, 131,
|
||||
132, 133, 134, 135,
|
||||
136, 137, 138, 139,
|
||||
140, 141, 142, 143,
|
||||
144, 145, 146, 147,
|
||||
148, 149, 150, 151,
|
||||
152, 153, 154, 155,
|
||||
156, 157, 158, 159,
|
||||
160, 161, 162, 163,
|
||||
164, 165, 166, 167,
|
||||
168, 169, 170, 171,
|
||||
172, 173, 174, 175,
|
||||
176, 177, 178, 179,
|
||||
180, 181, 182, 183,
|
||||
184, 185, 186, 187,
|
||||
188, 189, 190, 191,
|
||||
192, 193, 194, 195,
|
||||
196, 197, 198, 199,
|
||||
200, 201, 202, 203,
|
||||
204, 205, 206, 207,
|
||||
208, 209, 210, 211,
|
||||
212, 213, 214, 215,
|
||||
216, 217, 218, 219,
|
||||
220, 221, 222, 223,
|
||||
224, 225, 226, 227,
|
||||
228, 229, 230, 231,
|
||||
232, 233, 234, 235,
|
||||
236, 237, 238, 239,
|
||||
240, 241, 242, 243,
|
||||
244, 245, 246, 247,
|
||||
248, 249, 250, 251,
|
||||
252, 253, 254, 255 ;
|
||||
}
|
@ -18,7 +18,7 @@ TEST_EXTENSIONS = .sh
|
||||
LDADD = ${top_builddir}/liblib/libnetcdf.la
|
||||
AM_CPPFLAGS += -I$(top_srcdir)/liblib
|
||||
AM_CPPFLAGS += -DTOPSRCDIR=${abs_top_srcdir}
|
||||
AM_CPPFLAGS += -DTOPBINDIR=${abs_top_bindir}
|
||||
AM_CPPFLAGS += -DTOPBINDIR=${abs_top_builddir}
|
||||
|
||||
# Set up the tests; do the .sh first, then .c
|
||||
check_PROGRAMS =
|
||||
@ -52,6 +52,7 @@ TESTS += test_partvar
|
||||
if ENABLE_DAP_LONG_TESTS
|
||||
TESTS += tst_longremote3.sh
|
||||
tst_longremote3.log: tst_remote3.log
|
||||
test_manyurls_SOURCES = test_manyurls.c manyurls.h
|
||||
check_PROGRAMS += test_manyurls
|
||||
TESTS += test_manyurls
|
||||
test_manyurls.log: tst_longremote3.log
|
||||
|
@ -245,7 +245,7 @@ tst_roman_szip_unlim.cdl tst_perdimpspecs.nc tmppds.* \
|
||||
keyword1.nc keyword2.nc keyword3.nc keyword4.nc \
|
||||
tmp_keyword1.cdl tmp_keyword2.cdl tmp_keyword3.cdl tmp_keyword4.cdl \
|
||||
type_*.nc copy_type_*.cdl \
|
||||
scope_*.nc copy_scope_*.cdl
|
||||
scope_*.nc copy_scope_*.cdl keyword5.nc
|
||||
|
||||
# Remove directories
|
||||
clean-local:
|
||||
|
@ -29,12 +29,13 @@
|
||||
#include "ncpathmgr.h"
|
||||
|
||||
static const char* USAGE =
|
||||
"ncpathcvt [-c|-C|-m|-u|-w] [-h] [-e] [-d <driveletter>] [-B<char>] [-k] [-p] PATH\n"
|
||||
"ncpathcvt [-c|-C|-m|-u|-w] [-h] [-e] [-F] [-d <driveletter>] [-B<char>] [-k] [-p] PATH\n"
|
||||
"Options\n"
|
||||
" -h help"
|
||||
" -e add backslash escapes to '\' and ' '\n"
|
||||
" -d <driveletter> use driveletter when needed; defaults to 'c'\n"
|
||||
" -B <char> convert occurrences of <char> to ' '\n"
|
||||
" -B <char> convert occurrences of <char> to blank\n"
|
||||
" -F convert occurrences of '\\' to '/'"
|
||||
"Output type options:\n"
|
||||
" -c convert to Cygwin form of path\n"
|
||||
" -C return canonical form of path\n"
|
||||
@ -57,6 +58,7 @@ struct Options {
|
||||
int debug;
|
||||
int canon;
|
||||
int blank;
|
||||
int slash;
|
||||
int pathkind;
|
||||
} cvtoptions;
|
||||
|
||||
@ -93,6 +95,27 @@ escape(const char* path)
|
||||
return epath;
|
||||
}
|
||||
|
||||
static char*
|
||||
slash(const char* path)
|
||||
{
|
||||
size_t slen = strlen(path);
|
||||
const char* p;
|
||||
char* q;
|
||||
char* epath = NULL;
|
||||
|
||||
epath = (char*)malloc(slen + 1);
|
||||
if(epath == NULL) usage("out of memtory");
|
||||
p = path;
|
||||
q = epath;
|
||||
for(;*p;p++) {
|
||||
if(*p == '\\')
|
||||
*q++ = '/';
|
||||
else *q++ = *p;
|
||||
}
|
||||
*q = '\0';
|
||||
return epath;
|
||||
}
|
||||
|
||||
void
|
||||
printlocalkind(void)
|
||||
{
|
||||
@ -173,7 +196,7 @@ main(int argc, char** argv)
|
||||
memset((void*)&cvtoptions,0,sizeof(cvtoptions));
|
||||
cvtoptions.drive = 'c';
|
||||
|
||||
while ((c = getopt(argc, argv, "B:CcD:d:ehkmpuwX")) != EOF) {
|
||||
while ((c = getopt(argc, argv, "B:CFcD:d:ehkmpuwX")) != EOF) {
|
||||
switch(c) {
|
||||
case 'c': cvtoptions.target = NCPD_CYGWIN; break;
|
||||
case 'd': cvtoptions.drive = optarg[0]; break;
|
||||
@ -190,6 +213,7 @@ main(int argc, char** argv)
|
||||
usage("Bad -B argument");
|
||||
break;
|
||||
case 'C': cvtoptions.canon = 1; break;
|
||||
case 'F': cvtoptions.slash = 1; break;
|
||||
case 'D':
|
||||
sscanf(optarg,"%d",&cvtoptions.debug);
|
||||
break;
|
||||
@ -245,6 +269,11 @@ main(int argc, char** argv)
|
||||
cvtpath = escape(path);
|
||||
free(path);
|
||||
}
|
||||
if(cvtpath && cvtoptions.slash) {
|
||||
char* path = cvtpath; cvtpath = NULL;
|
||||
cvtpath = slash(path);
|
||||
free(path);
|
||||
}
|
||||
printf("%s",cvtpath);
|
||||
done:
|
||||
if(canon) free(canon);
|
||||
|
@ -31,7 +31,6 @@ diff -wB ${srcdir}/$1.cdl ${execdir}/copy_$1.cdl
|
||||
}
|
||||
|
||||
typescope() {
|
||||
echo ">>>"
|
||||
ls -l ${execdir}/printfqn* ${execdir}/$1.nc ${execdir}/$1_copy.nc
|
||||
REFT=`${execdir}/printfqn -f ${execdir}/$1.nc -v test_variable -t`
|
||||
COPYT=`${execdir}/printfqn -f ${execdir}/$1_copy.nc -v test_variable -t`
|
||||
|
@ -8,7 +8,6 @@
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
set -x
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
|
@ -4,7 +4,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
# This shell script tests the output from several previous tests.
|
||||
set -x
|
||||
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
|
@ -5,7 +5,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
|
||||
set -e
|
||||
|
||||
WS=`${NCPATHCVT} ${srcdir}`
|
||||
WS=`${NCPATHCVT} -F ${srcdir}`
|
||||
|
||||
cmp nctest_classic.nc "$WS/ref_nctest_classic.nc"
|
||||
cmp nctest_64bit_offset.nc "$WS/ref_nctest_64bit_offset.nc"
|
||||
|
@ -26,6 +26,15 @@ FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh DESTINATION ${CMAK
|
||||
FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.1)
|
||||
FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh)
|
||||
|
||||
FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_unknown.sh SPSOURCE)
|
||||
STRING(PREPEND SPSOURCE "#!/bin/bash\n")
|
||||
STRING(PREPEND SPSOURCE "TESTNCZARR=1\n")
|
||||
FILE(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.1 "${SPSOURCE}")
|
||||
CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.1 ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.sh @ONLY NEWLINE_STYLE LF)
|
||||
FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR} FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE)
|
||||
FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.1)
|
||||
FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.sh)
|
||||
|
||||
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ref*.cdl
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ref*.txt)
|
||||
@ -126,6 +135,7 @@ IF(ENABLE_TESTS)
|
||||
build_bin_test(test_filter_avail)
|
||||
ADD_SH_TEST(nczarr_test run_nczfilter)
|
||||
ADD_SH_TEST(nczarr_test run_filter)
|
||||
ADD_SH_TEST(nczarr_test run_unknown)
|
||||
ADD_SH_TEST(nczarr_test run_specific_filters)
|
||||
ENDIF(ENABLE_FILTER_TESTING)
|
||||
if(ENABLE_NCZARR_ZIP)
|
||||
|
@ -93,7 +93,8 @@ TESTS += run_nczfilter.sh
|
||||
|
||||
# Echo filter tests from nc_test4
|
||||
check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi test_filter_avail
|
||||
TESTS += run_filter.sh
|
||||
TESTS += run_filter.sh
|
||||
TESTS += run_unknown.sh
|
||||
TESTS += run_specific_filters.sh
|
||||
|
||||
endif #ENABLE_FILTER_TESTING
|
||||
@ -127,7 +128,8 @@ run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_ncgen4.sh \
|
||||
run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh run_s3_cleanup.sh \
|
||||
run_purezarr.sh run_interop.sh run_misc.sh \
|
||||
run_filter.sh run_specific_filters.sh \
|
||||
run_newformat.sh run_nczarr_fill.sh run_quantize.sh
|
||||
run_newformat.sh run_nczarr_fill.sh run_quantize.sh run_nczfilter.sh \
|
||||
run_unknown.sh
|
||||
|
||||
EXTRA_DIST += \
|
||||
ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \
|
||||
@ -145,14 +147,13 @@ ref_avail1.cdl ref_avail1.dmp ref_avail1.txt \
|
||||
ref_xarray.cdl ref_purezarr.cdl ref_purezarr_base.cdl ref_nczarr2zarr.cdl \
|
||||
ref_bzip2.cdl ref_filtered.cdl ref_multi.cdl \
|
||||
ref_any.cdl ref_oldformat.cdl ref_oldformat.zip ref_newformatpure.cdl \
|
||||
ref_quotes.zip ref_quotes.cdl \
|
||||
ref_groups.h5 ref_byte.zarr.zip ref_byte_fill_value_null.zarr.zip \
|
||||
ref_groups_regular.cdl ref_byte.cdl ref_byte_fill_value_null.cdl
|
||||
|
||||
# Interoperability files
|
||||
EXTRA_DIST += ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes.zip ref_quotes.cdl
|
||||
EXTRA_DIST += ref_power_901_constants_orig.zip ref_power_901_constants.cdl ref_quotes_orig.zip ref_quotes.cdl
|
||||
|
||||
CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl
|
||||
CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip
|
||||
|
||||
BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh run_filterinstall.sh
|
||||
test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c
|
||||
@ -165,6 +166,15 @@ test_filter_avail.c: $(top_srcdir)/nc_test4/tst_filter_avail.c
|
||||
echo "#define TESTNCZARR" > $@
|
||||
cat $(top_srcdir)/nc_test4/tst_filter_avail.c >> $@
|
||||
|
||||
run_unknown.sh: $(top_srcdir)/nc_test4/tst_unknown.sh
|
||||
rm -f $@ run_unknown.tmp
|
||||
echo "#!/bin/bash" > run_unknown.tmp
|
||||
echo "TESTNCZARR=1" >> run_unknown.tmp
|
||||
cat $(top_srcdir)/nc_test4/tst_unknown.sh >> run_unknown.tmp
|
||||
tr -d '\r' < run_unknown.tmp > $@
|
||||
chmod a+x $@
|
||||
rm -f run_unknown.tmp
|
||||
|
||||
run_specific_filters.sh: $(top_srcdir)/nc_test4/tst_specific_filters.sh
|
||||
rm -f $@ run_specific_filters.tmp
|
||||
echo "#!/bin/bash" > run_specific_filters.tmp
|
||||
@ -185,10 +195,11 @@ run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh
|
||||
|
||||
# Remove directories
|
||||
clean-local:
|
||||
rm -fr tmp*.file results.file results.s3 results.zip
|
||||
rm -fr rcmiscdir
|
||||
rm -fr tmp_*.nc tmp_*.zarr tst_quantize*.zarr tmp*.file results.file results.s3 results.zip
|
||||
rm -fr rcmiscdir ref_power_901_constants.file
|
||||
|
||||
DISTCLEANFILES = findplugin.sh test_quantize.c run_specific_filters.sh
|
||||
|
||||
DISTCLEANFILES = findplugin.sh test_quantize.c run_specific_filters.sh run_filterinstall.sh
|
||||
|
||||
# If valgrind is present, add valgrind targets.
|
||||
@VALGRIND_CHECK_RULES@
|
||||
|
@ -9,7 +9,7 @@ variables:
|
||||
var:_Storage = "chunked" ;
|
||||
var:_ChunkSizes = 4, 4, 4, 4 ;
|
||||
var:_Filter = "307,9|1,2|40000" ;
|
||||
var:_Codecs = "[{\"id\": \"bz2\",\"level\": \"9\"},{\"id\": \"zlib\",\"level\": \"2\"},{\"id\": \"noop0\"}]" ;
|
||||
var:_Codecs = "[{\"id\": \"bz2\",\"level\": \"9\"},{\"id\": \"zlib\",\"level\": \"2\"},{\"id\": \"noop\"}]" ;
|
||||
var:_NoFill = "true" ;
|
||||
|
||||
// global attributes:
|
||||
|
@ -12,7 +12,6 @@ testset() {
|
||||
testapi $1
|
||||
testng $1
|
||||
testncp $1
|
||||
testunk $1
|
||||
testngc $1
|
||||
testmisc $1
|
||||
testmulti $1
|
||||
@ -51,21 +50,19 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
|
||||
|
||||
# Find misc and capture
|
||||
findplugin h5misc
|
||||
MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}"
|
||||
|
||||
# Find noop and capture
|
||||
findplugin h5noop
|
||||
NOOPLIB="${HDF5_PLUGIN_LIB}"
|
||||
NOOPDIR="${HDF5_PLUGIN_DIR}/${NOOPLIB}"
|
||||
MISCLIB="${HDF5_PLUGIN_LIB}"
|
||||
MISCDIR="${HDF5_PLUGIN_DIR}"
|
||||
MISCPATH="${MISCDIR}/${MISCLIB}"
|
||||
|
||||
# Find bzip2 and capture
|
||||
findplugin h5bzip2
|
||||
BZIP2LIB="${HDF5_PLUGIN_LIB}"
|
||||
BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}"
|
||||
BZIP2DIR="${HDF5_PLUGIN_DIR}"
|
||||
BZIP2PATH="${BZIP2DIR}/${BZIP2LIB}"
|
||||
|
||||
# Verify
|
||||
if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi
|
||||
if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi
|
||||
if ! test -f ${BZIP2path} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi
|
||||
if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi
|
||||
|
||||
# Execute the specified tests
|
||||
|
||||
@ -132,29 +129,6 @@ diff -b -w ${srcdir}/ref_filtered.cdl ./tmp_ncp_$zext.dump
|
||||
echo " *** Pass: nccopy simple filter for map $zext"
|
||||
}
|
||||
|
||||
testunk() {
|
||||
zext=$1
|
||||
echo "*** Testing access to filter info when filter implementation is not available for map $zext"
|
||||
fileargs tmp_known
|
||||
deletemap $zext $file
|
||||
# build noop.nc
|
||||
${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/noop.cdl
|
||||
# dump and clean noop.nc header when filter is avail
|
||||
${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt
|
||||
# Remove irrelevant -s output
|
||||
sclean ./tmp_known_$zext.txt tmp_known_$zext.dump
|
||||
# Now hide the filter code
|
||||
mv ${NOOPDIR} ./${NOOPLIB}.save
|
||||
# dump and clean noop.nc header when filter is not avail
|
||||
${NCDUMP} -hs $fileurl > ./tmp_unk_$zext.txt
|
||||
# Restore the filter code
|
||||
mv ./${NOOPLIB}.save ${NOOPDIR}
|
||||
# Verify that the filter is no longer defined
|
||||
UNK=`sed -e '/var:_Filter/p' -e d ./tmp_unk_$zext.txt`
|
||||
test "x$UNK" = x
|
||||
echo "*** Pass: ncgen dynamic filter for map $zext"
|
||||
}
|
||||
|
||||
testngc() {
|
||||
zext=$1
|
||||
echo "*** Testing dynamic filters using ncgen with -lc for map $zext"
|
||||
|
@ -54,16 +54,14 @@ case "$zext" in
|
||||
file)
|
||||
# need to unpack
|
||||
rm -fr ref_power_901_constants ref_power_901_constants.file
|
||||
unzip ${srcdir}/ref_power_901_constants.zip > /dev/null
|
||||
unzip ${srcdir}/ref_power_901_constants_orig.zip > /dev/null
|
||||
mv ref_power_901_constants ref_power_901_constants.file
|
||||
testcasefile ref_power_901_constants zarr metaonly; # test xarray as default
|
||||
;;
|
||||
zip)
|
||||
# Move into position
|
||||
if test "x$srcdir" != "x$execdir" ; then
|
||||
cp ${srcdir}/ref_power_901_constants.zip ${execdir}
|
||||
cp ${srcdir}/ref_quotes.zip ${execdir}
|
||||
fi
|
||||
cp ${srcdir}/ref_power_901_constants_orig.zip ${execdir}/ref_power_901_constants.zip
|
||||
cp ${srcdir}/ref_quotes_orig.zip ${execdir}/ref_quotes.zip
|
||||
testcasezip ref_power_901_constants xarray metaonly
|
||||
# Test large constant interoperability
|
||||
testcasezip ref_quotes zarr metaonly
|
||||
|
@ -18,6 +18,8 @@ SET_SOURCE_FILES_PROPERTIES(H5Znoop.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE
|
||||
SET(h5noop1_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c H5Zutil.c h5noop.h)
|
||||
SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=1)
|
||||
|
||||
SET(h5unknown_SOURCES H5Zunknown.c)
|
||||
|
||||
SET(h5shuffle_SOURCES H5Zshuffle.c)
|
||||
SET(h5fletcher32_SOURCES H5Zfletcher32.c H5checksum.c)
|
||||
SET(h5deflate_SOURCES H5Zdeflate.c)
|
||||
@ -51,6 +53,7 @@ ENDMACRO()
|
||||
buildplugin(h5misc "h5misc")
|
||||
buildplugin(h5noop "h5noop")
|
||||
buildplugin(h5noop1 "h5noop1")
|
||||
buildplugin(h5unknown "h5unknown")
|
||||
|
||||
buildplugin(h5shuffle "h5shuffle")
|
||||
buildplugin(h5fletcher32 "h5fletcher32")
|
||||
|
@ -154,7 +154,7 @@ static NCZ_codec_t NCZ_noop_codec = {/* NCZ_codec_t codec fields */
|
||||
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
||||
NCZ_CODEC_HDF5, /* Struct sort */
|
||||
#if NOOP_INSTANCE == 0
|
||||
"noop0", /* Standard name/id of the codec */
|
||||
"noop", /* Standard name/id of the codec */
|
||||
H5Z_FILTER_NOOP, /* HDF5 alias for noop */
|
||||
#else
|
||||
"noop1", /* Standard name/id of the codec */
|
||||
|
147
plugins/H5Zunknown.c
Normal file
147
plugins/H5Zunknown.c
Normal file
@ -0,0 +1,147 @@
|
||||
#include "config.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
#include "netcdf_filter_build.h"
|
||||
|
||||
/* use a temporary filter id*/
|
||||
#define H5Z_FILTER_UNKNOWN 33000
|
||||
|
||||
#undef DEBUG
|
||||
|
||||
static htri_t H5Z_unknown_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id);
|
||||
static size_t H5Z_filter_unknown(unsigned int, size_t, const unsigned int cd_values[], size_t, size_t*, void**);
|
||||
|
||||
static H5Z_class2_t H5Z_UNKNOWN[1] = {{
|
||||
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
|
||||
(H5Z_filter_t)(H5Z_FILTER_UNKNOWN), /* Filter id number */
|
||||
1, /* encoder_present flag (set to true) */
|
||||
1, /* decoder_present flag (set to true) */
|
||||
"unknown", /* Filter name for debugging */
|
||||
(H5Z_can_apply_func_t)H5Z_unknown_can_apply, /* The "can apply" callback */
|
||||
NULL, /* The "set local" callback */
|
||||
(H5Z_func_t)H5Z_filter_unknown, /* The actual filter function */
|
||||
}};
|
||||
|
||||
/* External Discovery Functions */
|
||||
DLLEXPORT
|
||||
H5PL_type_t
|
||||
H5PLget_plugin_type(void)
|
||||
{
|
||||
return H5PL_TYPE_FILTER;
|
||||
}
|
||||
|
||||
DLLEXPORT
|
||||
const void*
|
||||
H5PLget_plugin_info(void)
|
||||
{
|
||||
return H5Z_UNKNOWN;
|
||||
}
|
||||
|
||||
/* Make this explicit */
|
||||
/*
|
||||
* The "can_apply" callback returns positive a valid combination, zero for an
|
||||
* invalid combination and negative for an error.
|
||||
*/
|
||||
static htri_t
|
||||
H5Z_unknown_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id)
|
||||
{
|
||||
return 1; /* Assume it can always apply */
|
||||
}
|
||||
|
||||
static size_t
|
||||
H5Z_filter_unknown(unsigned int flags, size_t cd_nelmts,
|
||||
const unsigned int cd_values[], size_t nbytes,
|
||||
size_t *buf_size, void **buf)
|
||||
{
|
||||
void* newbuf;
|
||||
|
||||
if (flags & H5Z_FLAG_REVERSE) {
|
||||
/* Replace buffer */
|
||||
newbuf = H5allocate_memory(*buf_size,0);
|
||||
if(newbuf == NULL) abort();
|
||||
memcpy(newbuf,*buf,*buf_size);
|
||||
/* reclaim old buffer */
|
||||
H5free_memory(*buf);
|
||||
*buf = newbuf;
|
||||
|
||||
} else {
|
||||
/* Replace buffer */
|
||||
newbuf = H5allocate_memory(*buf_size,0);
|
||||
if(newbuf == NULL) abort();
|
||||
memcpy(newbuf,*buf,*buf_size);
|
||||
/* reclaim old buffer */
|
||||
H5free_memory(*buf);
|
||||
*buf = newbuf;
|
||||
|
||||
}
|
||||
return *buf_size;
|
||||
}
|
||||
|
||||
/**************************************************/
|
||||
/* NCZarr Codec API */
|
||||
|
||||
/* Codec Format
|
||||
{
|
||||
"id": "unknown",
|
||||
}
|
||||
*/
|
||||
|
||||
/* Forward */
|
||||
static int NCZ_unknown_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
||||
static int NCZ_unknown_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
||||
|
||||
/* Structure for NCZ_PLUGIN_CODEC */
|
||||
static NCZ_codec_t NCZ_unknown_codec = {/* NCZ_codec_t codec fields */
|
||||
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
||||
NCZ_CODEC_HDF5, /* Struct sort */
|
||||
"unknown", /* Standard name/id of the codec */
|
||||
H5Z_FILTER_UNKNOWN, /* HDF5 alias for unknown */
|
||||
NULL, /*NCZ_unknown_codec_initialize*/
|
||||
NULL, /*NCZ_unknown_codec_finalize*/
|
||||
NCZ_unknown_codec_to_hdf5,
|
||||
NCZ_unknown_hdf5_to_codec,
|
||||
NULL, /*NCZ_unknown_modify_parameters*/
|
||||
};
|
||||
|
||||
/* External Export API */
|
||||
DLLEXPORT
|
||||
const void*
|
||||
NCZ_get_codec_info(void)
|
||||
{
|
||||
return (void*)&NCZ_unknown_codec;
|
||||
}
|
||||
|
||||
/* NCZarr Interface Functions */
|
||||
|
||||
static int
|
||||
NCZ_unknown_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
|
||||
*nparamsp = 0;
|
||||
*paramsp = NULL;
|
||||
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_unknown_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
char json[8192];
|
||||
|
||||
if(nparams != 0 || params != NULL)
|
||||
{stat = NC_EINVAL; goto done;}
|
||||
|
||||
snprintf(json,sizeof(json),"{\"id\": \"%s\"}",NCZ_unknown_codec.codecid);
|
||||
if(codecp) {
|
||||
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
}
|
||||
|
||||
done:
|
||||
return stat;
|
||||
}
|
@ -7,8 +7,28 @@ include $(top_srcdir)/lib_flags.am
|
||||
AM_LDFLAGS += -module -avoid-version -shared -export-dynamic \
|
||||
-rpath ${abs_builddir} ${NOUNDEFINED}
|
||||
|
||||
lib_LTLIBRARIES =
|
||||
noinst_LTLIBRARIES =
|
||||
# Create an alternate directory if not installing or for noinst installs.
|
||||
ALTPLUGINDIR = ${abs_top_builddir}/plugins/plugindir
|
||||
|
||||
# This is where the plugins are to be installed
|
||||
if ENABLE_PLUGIN_DIR
|
||||
plugindir = @PLUGIN_INSTALL_DIR@
|
||||
else
|
||||
plugindir = ${ALTPLUGINDIR}
|
||||
endif
|
||||
|
||||
plugin_LTLIBRARIES =
|
||||
|
||||
# Apparently one cannot have plugin_LTLIBRARIES and also noinst_LTLIBRARIES.
|
||||
# So create a tmp location for "noinst" shared libraries.
|
||||
tmpdir = ${ALTPLUGINDIR}
|
||||
|
||||
tmp_LTLIBRARIES =
|
||||
|
||||
# This linker flag specifies libtool version info.
|
||||
# See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning
|
||||
# for information regarding incrementing `-version-info`.
|
||||
plugin_version_info = -version-info 0:0:0
|
||||
|
||||
if ISMINGW
|
||||
LDADD = ${top_builddir}/liblib/libnetcdf.la
|
||||
@ -17,7 +37,7 @@ endif
|
||||
EXTRA_DIST = CMakeLists.txt
|
||||
|
||||
# The HDF5 filter wrappers
|
||||
EXTRA_DIST += stdinstall.sh \
|
||||
EXTRA_DIST += \
|
||||
H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \
|
||||
H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \
|
||||
H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h H5Zzstd.c H5Zzstd.h
|
||||
@ -29,55 +49,66 @@ EXTRA_DIST += H5checksum.c
|
||||
|
||||
if ENABLE_FILTER_TESTING
|
||||
|
||||
noinst_LTLIBRARIES += libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la
|
||||
|
||||
if ENABLE_NCZARR_FILTERS
|
||||
noinst_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la
|
||||
plugin_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la
|
||||
libh5shuffle_la_SOURCES = H5Zshuffle.c
|
||||
libh5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c
|
||||
libh5deflate_la_SOURCES = H5Zdeflate.c
|
||||
|
||||
libh5shuffle_la_LDFLAGS = ${plugin_version_info}
|
||||
libh5deflate_la_LDFLAGS = ${plugin_version_info}
|
||||
libh5fletcher32_la_LDFLAGS = ${plugin_version_info}
|
||||
|
||||
# Need our version of szip if libsz available and we are not using HDF5
|
||||
if HAVE_SZ
|
||||
noinst_LTLIBRARIES += libh5szip.la
|
||||
plugin_LTLIBRARIES += libh5szip.la
|
||||
libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h
|
||||
libh5szip_la_LDFLAGS = ${plugin_version_info}
|
||||
endif
|
||||
|
||||
endif # ENABLE_NCZARR_FILTERS
|
||||
|
||||
if ENABLE_PLUGINS
|
||||
|
||||
# The NCZarr codec libraries
|
||||
libnczstdfilters_la_SOURCES = NCZstdfilters.c
|
||||
|
||||
libnczhdf5filters_la_SOURCES = NCZhdf5filters.c
|
||||
|
||||
plugin_LTLIBRARIES += libnczhdf5filters.la
|
||||
plugin_LTLIBRARIES += libnczstdfilters.la
|
||||
|
||||
if HAVE_BLOSC
|
||||
libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h
|
||||
libh5blosc_la_LDFLAGS = ${plugin_version_info}
|
||||
plugin_LTLIBRARIES += libh5blosc.la
|
||||
endif
|
||||
|
||||
if HAVE_ZSTD
|
||||
libh5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h
|
||||
endif
|
||||
|
||||
noinst_LTLIBRARIES += libnczhdf5filters.la
|
||||
noinst_LTLIBRARIES += libnczstdfilters.la
|
||||
|
||||
if HAVE_BLOSC
|
||||
noinst_LTLIBRARIES += libh5blosc.la
|
||||
endif
|
||||
if HAVE_ZSTD
|
||||
noinst_LTLIBRARIES += libh5zstd.la
|
||||
libh5zstd_la_LDFLAGS = ${plugin_version_info}
|
||||
plugin_LTLIBRARIES += libh5zstd.la
|
||||
endif
|
||||
|
||||
endif #ENABLE_PLUGINS
|
||||
|
||||
libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h
|
||||
libnczmisc_la_SOURCES = NCZmisc.c
|
||||
|
||||
# The noop filter is to allow testing of multifilters and filter order
|
||||
# Need two distinct instances
|
||||
libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h
|
||||
libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h
|
||||
libh5noop_la_LDFLAGS = ${plugin_version_info}
|
||||
libh5noop1_la_LDFLAGS = ${plugin_version_info}
|
||||
|
||||
# The misc filter is to allow testing of filter arguments
|
||||
libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h
|
||||
libh5misc_la_LDFLAGS = ${plugin_version_info}
|
||||
libnczmisc_la_SOURCES = NCZmisc.c
|
||||
libnczmisc_la_LDFLAGS = ${plugin_version_info}
|
||||
|
||||
# Provide a filter to test missing filter
|
||||
libh5unknown_la_SOURCES = H5Zunknown.c
|
||||
libh5unknown_la_LDFLAGS = ${plugin_version_info}
|
||||
|
||||
tmp_LTLIBRARIES += libh5noop.la libh5noop1.la libh5misc.la libnczmisc.la libh5unknown.la
|
||||
|
||||
# Bzip2 is used to test more complex filters
|
||||
libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h
|
||||
@ -86,17 +117,17 @@ EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE
|
||||
if HAVE_LOCAL_BZ2
|
||||
libh5bzip2_la_SOURCES += ${BZIP2SRC}
|
||||
endif
|
||||
libh5bzip2_la_LDFLAGS = ${plugin_version_info}
|
||||
plugin_LTLIBRARIES += libh5bzip2.la
|
||||
|
||||
endif #ENABLE_FILTER_TESTING
|
||||
|
||||
BUILT_SOURCES = H5Znoop1.c
|
||||
DISTCLEANFILES = H5Znoop1.c ncjson.h
|
||||
DISTCLEANFILES = H5Znoop1.c H5Znoop2.c ncjson.h
|
||||
H5Znoop1.c: Makefile H5Znoop.c
|
||||
echo '#define NOOP_INSTANCE 1' > $@
|
||||
cat ${srcdir}/H5Znoop.c >> $@
|
||||
|
||||
noinst_LTLIBRARIES += libh5bzip2.la
|
||||
|
||||
# Record where bzip2 came from; may be out of date
|
||||
BZIP2VER = 1.0.8
|
||||
BZIP2DIR = bzip2-${BZIP2VER}
|
||||
@ -108,6 +139,7 @@ bzip2::
|
||||
cd ${BZIP2DIR}; cp ${BZIP2SRC} ..; cp LICENSE ../BZIP2_LICENSE ; cd ..
|
||||
rm -fr ./${BZIP2DIR}
|
||||
|
||||
# Custom install
|
||||
install-exec-hook:
|
||||
sh ./stdinstall.sh
|
||||
# Custom clean
|
||||
clean-local:
|
||||
rm -fr ${ALTPLUGINDIR}
|
||||
|
||||
|
179
plugins/NCZtest.c
Normal file
179
plugins/NCZtest.c
Normal file
@ -0,0 +1,179 @@
|
||||
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
|
||||
* Copyright by The HDF Group. *
|
||||
* Copyright by the Board of Trustees of the University of Illinois. *
|
||||
* All rights reserved. *
|
||||
* *
|
||||
* This file is part of HDF5. The full HDF5 copyright notice, including *
|
||||
* terms governing use, modification, and redistribution, is contained in *
|
||||
* the COPYING file, which can be found at the root of the source code *
|
||||
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
|
||||
* If you do not have access to either file, you may request a copy from *
|
||||
* help@hdfgroup.org. *
|
||||
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
|
||||
|
||||
/*
|
||||
* Programmer: Robb Matzke
|
||||
* Friday, August 27, 1999
|
||||
*/
|
||||
|
||||
/* Converted to NCZarr support by Dennis Heimbigner 5/1/2021 */
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "netcdf_json.h"
|
||||
|
||||
#include "netcdf.h"
|
||||
#include "netcdf_filter.h"
|
||||
#include "netcdf_filter_build.h"
|
||||
|
||||
#include "h5misc.h"
|
||||
|
||||
/**************************************************/
|
||||
/* NCZarr Filter Objects */
|
||||
/* Codec Format
|
||||
{
|
||||
"id": "test",
|
||||
"testcase": "n",
|
||||
"byte": "<unsigned int>",
|
||||
"ubyte": "<unsigned int>",
|
||||
"short": "<unsigned int>",
|
||||
"ushort": "<unsigned int>",
|
||||
"int": "<unsigned int>",
|
||||
"uint": "<unsigned int>",
|
||||
"float": "<unsigned int>",
|
||||
"double0": "<unsigned int>",
|
||||
"double1": "<unsigned int>",
|
||||
"int640": "<unsigned int>",
|
||||
"int641": "<unsigned int>",
|
||||
"uint640": "<unsigned int>",
|
||||
"uint641": "<unsigned int>",
|
||||
}
|
||||
*/
|
||||
|
||||
/* Give unique dict key names for parameters */
|
||||
static const char* fields[14] = {
|
||||
"testcase",
|
||||
"byte",
|
||||
"ubyte",
|
||||
"short",
|
||||
"ushort",
|
||||
"int",
|
||||
"uint",
|
||||
"float",
|
||||
"double_0",
|
||||
"double_1",
|
||||
"int64_0",
|
||||
"int64_1",
|
||||
"uint64_0",
|
||||
"uint64_1"
|
||||
};
|
||||
|
||||
/* Forward */
|
||||
static int NCZ_misc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
||||
static int NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
||||
|
||||
/* Structure for NCZ_PLUGIN_CODEC */
|
||||
static NCZ_codec_t NCZ_misc_codec = {/* NCZ_codec_t codec fields */
|
||||
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
||||
NCZ_CODEC_HDF5, /* Struct sort */
|
||||
"test", /* Standard name/id of the codec */
|
||||
H5Z_FILTER_TEST, /* HDF5 alias for misc */
|
||||
NULL, /*NCZ_misc_codec_initialize*/
|
||||
NULL, /*NCZ_misc_codec_finalize*/
|
||||
NCZ_misc_codec_to_hdf5,
|
||||
NCZ_misc_hdf5_to_codec,
|
||||
NULL, /*NCZ_misc_modify_parameters*/
|
||||
};
|
||||
|
||||
/* External Export API */
|
||||
DLLEXPORT
|
||||
const void*
|
||||
NCZ_get_codec_info(void)
|
||||
{
|
||||
return (void*)&NCZ_misc_codec;
|
||||
}
|
||||
|
||||
/* NCZarr Interface Functions */
|
||||
|
||||
static int
|
||||
NCZ_misc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCjson* jcodec = NULL;
|
||||
NCjson* jtmp = NULL;
|
||||
size_t i,nparams = 0;
|
||||
unsigned* params = NULL;
|
||||
|
||||
/* parse the JSON */
|
||||
if(NCJparse(codec_json,0,&jcodec))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
|
||||
|
||||
/* Verify the codec ID */
|
||||
if(NCJdictget(jcodec,"id",&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;}
|
||||
if(strcmp(NCJstring(jtmp),NCZ_misc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
|
||||
|
||||
/* The codec will have (2*14 + 1) +1 = 29 dict entries + id*/
|
||||
nparams = (2*14 + 1) + 1;
|
||||
if(NCJlength(jcodec) != nparams) {
|
||||
fprintf(stderr,"Incorrect no. of codec parameters: need=29 sent=%ld\n",(unsigned long)(nparams-1));
|
||||
stat = NC_EINVAL;
|
||||
goto done;
|
||||
}
|
||||
|
||||
/* Actual # of parameters is 14 (ignoring the testcase number) */
|
||||
nparams = 14;
|
||||
if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
|
||||
for(i=0;i<nparams;i++) {
|
||||
struct NCJconst jc;
|
||||
if(NCJdictget(jcodec,fields[i],&jtmp))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(NCJcvt(jtmp,NCJ_INT,&jc))
|
||||
{stat = NC_EFILTER; goto done;}
|
||||
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
|
||||
params[i] = (unsigned)jc.ival;
|
||||
}
|
||||
if(nparamsp) *nparamsp = nparams;
|
||||
if(paramsp) {*paramsp = params; params = NULL;}
|
||||
|
||||
done:
|
||||
if(params) free(params);
|
||||
NCJreclaim(jcodec);
|
||||
return stat;
|
||||
}
|
||||
|
||||
static int
|
||||
NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
|
||||
{
|
||||
int i,stat = NC_NOERR;
|
||||
char json[4096];
|
||||
char value[1024];
|
||||
|
||||
if(nparams == 0 || params == NULL)
|
||||
{stat = NC_EINVAL; goto done;}
|
||||
if(nparams != 14) {
|
||||
fprintf(stderr,"Incorrect no. of parameters: need=14 sent=%ld\n",(unsigned long)nparams);
|
||||
stat = NC_EINVAL;
|
||||
goto done;
|
||||
}
|
||||
snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_misc_codec.codecid);
|
||||
for(i=0;i<14;i++) {
|
||||
snprintf(value,sizeof(value),", \"%s\": \"%u\"",fields[i],params[i]);
|
||||
strlcat(json,value,sizeof(json));
|
||||
}
|
||||
strlcat(json,"}",sizeof(json));
|
||||
if(codecp) {
|
||||
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
|
||||
}
|
||||
|
||||
done:
|
||||
return stat;
|
||||
}
|
@ -1,44 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
set -x
|
||||
|
||||
INSTALLDIR=@PLUGIN_INSTALL_DIR@
|
||||
|
||||
# Load the findplugins function
|
||||
. ${builddir}/findplugin.sh
|
||||
echo "findplugin.sh loaded"
|
||||
|
||||
installplugin() {
|
||||
PLUG="$1"
|
||||
# Locate the plugin path and the library name; argument order is critical
|
||||
findplugin ${PLUG}
|
||||
if ! test -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" ; then
|
||||
echo "Not exists: ${HDF5_PLUGIN_DIR}/$HDF5_PLUGIN_LIB ; ignored"
|
||||
return
|
||||
fi
|
||||
if ! test -d "${INSTALLDIR}" ; then
|
||||
echo "Not exists: ${INSTALLDIR} ; creating"
|
||||
mkdir ${INSTALLDIR}
|
||||
fi
|
||||
echo "Installing: $HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB into $INSTALLDIR"
|
||||
cp -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" $INSTALLDIR
|
||||
}
|
||||
|
||||
if test "x$USEPLUGINS" != x ; then
|
||||
if test "x$INSTALLDIR" != x ; then
|
||||
installplugin h5bzip2
|
||||
installplugin h5zstd
|
||||
installplugin h5blosc
|
||||
if test "x$FEATURE_NCZARR" ; then
|
||||
installplugin h5fletcher32
|
||||
installplugin h5shuffle
|
||||
installplugin h5deflate
|
||||
installplugin h5szip
|
||||
installplugin nczdefaults
|
||||
installplugin nczszip
|
||||
fi
|
||||
fi
|
||||
fi
|
@ -15,12 +15,10 @@ FP_ISMSVC=@ISMSVC@
|
||||
FP_ISCYGWIN=@ISCYGWIN@
|
||||
FP_ISMINGW=@ISMINGW@
|
||||
FP_ISMSYS=@ISMSYS@
|
||||
|
||||
FP_ISOSX=@ISOSX@
|
||||
FP_ISREGEDIT=@ISREGEDIT@
|
||||
|
||||
FP_USEPLUGINS=@USEPLUGINS@
|
||||
FP_ISREGEDIT=@ISREGEDIT@
|
||||
FP_USEPLUGINS=@USEPLUGINS@
|
||||
|
||||
# Feature flags
|
||||
FEATURE_HDF5=@HAS_HDF5@
|
||||
@ -156,11 +154,13 @@ ncgen3c0="${top_srcdir}/ncgen3/c0.cdl"
|
||||
ncgenc0="${top_srcdir}/ncgen/c0.cdl"
|
||||
ncgenc04="${top_srcdir}/ncgen/c0_4.cdl"
|
||||
|
||||
# Set LC_ALL
|
||||
if test "x$FP_ISMSVC" = xyes || test "x$FP_ISCYGWIN" = xyes; then export LC_ALL="en_US.utf8"; fi
|
||||
|
||||
# Test for filter availability
|
||||
avail() {
|
||||
if test yes = `${execdir}/../ncdump/ncfilteravail $1` ; then return 0 ; else echo "filter $1 not available" ; return 1; fi
|
||||
}
|
||||
if test "x$FP_ISMSVC" = xyes || test "x$FP_ISCYGWIN" = xyes; then export LC_ALL="en_US.utf8"; fi
|
||||
|
||||
# Make sure we are in builddir (not execdir)
|
||||
cd $builddir
|
||||
|
Loading…
x
Reference in New Issue
Block a user