mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-01-12 15:45:21 +08:00
3ffe7be446
re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing.
130 lines
4.9 KiB
Makefile
130 lines
4.9 KiB
Makefile
# This is part of the netCDF package. Copyright 2005-2019 University
|
|
# Corporation for Atmospheric Research/Unidata See COPYRIGHT file for
|
|
# conditions of use.
|
|
#
|
|
# This directory holds tests for netCDF-4. It is skipped if netCDF-4
|
|
# is not enabled.
|
|
#
|
|
# Ed Hartnett, Ward Fisher
|
|
|
|
# Put together AM_CPPFLAGS and AM_LDFLAGS.
|
|
include $(top_srcdir)/lib_flags.am
|
|
|
|
# Un comment to use a more verbose test driver
|
|
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
|
#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
|
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
|
#TESTS_ENVIRONMENT = export SETX=1;
|
|
|
|
TEST_EXTENSIONS = .sh
|
|
|
|
# Link to our assembled library.
|
|
LDADD = ${top_builddir}/liblib/libnetcdf.la
|
|
|
|
# These are netCDF-4 C test programs which are built and run.
|
|
NC4_TESTS = tst_dims tst_dims2 tst_dims3 tst_files tst_files4 \
|
|
tst_vars tst_varms tst_unlim_vars tst_converts tst_converts2 tst_grps \
|
|
tst_grps2 tst_compounds tst_compounds2 tst_compounds3 tst_opaques \
|
|
tst_strings tst_strings2 tst_interops tst_interops4 tst_interops5 \
|
|
tst_interops6 tst_interops_dims tst_enums tst_coords tst_coords2 \
|
|
tst_coords3 tst_vars3 tst_vars4 tst_chunks tst_chunks2 tst_utf8 \
|
|
tst_fills tst_fills2 tst_fillbug tst_xplatform tst_xplatform2 \
|
|
tst_endian_fill tst_atts t_type cdm_sea_soundings tst_camrun tst_vl \
|
|
tst_atts1 tst_atts2 tst_vars2 tst_files5 tst_files6 tst_sync \
|
|
tst_h_scalar tst_rename tst_rename2 tst_rename3 tst_h5_endians \
|
|
tst_atts_string_rewrite tst_hdf5_file_compat tst_fill_attr_vanish \
|
|
tst_rehash tst_filterparser tst_bug324 tst_types tst_atts3 \
|
|
tst_put_vars tst_elatefill tst_udf tst_put_vars_two_unlim_dim \
|
|
tst_bug1442 tst_quantize
|
|
|
|
if HAS_PAR_FILTERS
|
|
NC4_TESTS += tst_alignment
|
|
endif
|
|
|
|
# Temporary I hoped, but hoped in vain.
|
|
if !ISCYGWIN
|
|
NC4_TESTS += tst_h_strbug tst_h_refs
|
|
endif
|
|
|
|
# Build test programs plus programs used in test scripts.
|
|
check_PROGRAMS = $(NC4_TESTS) tst_empty_vlen_unlim tst_charvlenbug tst_vlenstr
|
|
TESTS = $(NC4_TESTS) run_empty_vlen_test.sh
|
|
|
|
# Add these if large file tests are turned on.
|
|
if LARGE_FILE_TESTS
|
|
check_PROGRAMS += tst_large tst_large2
|
|
TESTS += tst_large tst_large2
|
|
endif
|
|
|
|
# If the v2 API was built, add its test program.
|
|
if BUILD_V2
|
|
check_PROGRAMS += tst_v2
|
|
TESTS += tst_v2
|
|
endif # BUILD_V2
|
|
|
|
# The rest of the tests require ncgen/ncdump.
|
|
if BUILD_UTILITIES
|
|
check_PROGRAMS += renamegroup
|
|
TESTS += run_grp_rename.sh tst_misc.sh
|
|
|
|
check_PROGRAMS += tst_fillonly
|
|
TESTS += test_fillonly.sh
|
|
|
|
# Szip Tests (requires ncdump)
|
|
if HAVE_H5Z_SZIP
|
|
check_PROGRAMS += test_szip h5testszip
|
|
TESTS += tst_szip.sh
|
|
endif
|
|
|
|
# Filter Tests (requires ncdump and ncgen and HDF5)
|
|
if USE_HDF5
|
|
if ENABLE_FILTER_TESTING
|
|
extradir =
|
|
check_PROGRAMS += test_filter test_filter_misc test_filter_order test_filter_repeat test_filter_vlen
|
|
check_PROGRAMS += tst_multifilter tst_filter_avail
|
|
TESTS += tst_filter.sh
|
|
TESTS += tst_specific_filters.sh
|
|
endif
|
|
endif # USE_HDF5
|
|
endif # BUILD_UTILITIES
|
|
|
|
# This are extra tests that will only be run if netcdf-4 is configured
|
|
# with --enable-parallel-tests.
|
|
if TEST_PARALLEL4
|
|
check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3 \
|
|
tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r \
|
|
tst_mode tst_parallel_zlib tst_parallel_compress tst_quantize_par
|
|
TESTS += run_par_test.sh
|
|
endif # TEST_PARALLEL4
|
|
|
|
EXTRA_DIST = run_par_test.sh.in ref_tst_compounds.nc \
|
|
ref_tst_xplatform2_1.nc ref_tst_xplatform2_2.nc ref_tst_dims.nc \
|
|
ref_tst_interops4.nc CMakeLists.txt run_grp_rename.sh \
|
|
run_empty_vlen_test.sh ref_hdf5_compat1.nc ref_hdf5_compat2.nc \
|
|
ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5 \
|
|
ref_szip.cdl tst_filter.sh bzip2.cdl noop.cdl ref_filtered.cdl \
|
|
ref_unfiltered.cdl ref_bzip2.c findplugin.in ref_unfilteredvv.cdl \
|
|
ref_filteredvv.cdl ref_multi.cdl \
|
|
ref_ncgenF.cdl ref_nccopyF.cdl \
|
|
ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
|
|
ref_filter_order_create.txt ref_filter_order_read.txt \
|
|
ref_any.cdl tst_specific_filters.sh \
|
|
tst_virtual_datasets.c
|
|
|
|
CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
|
|
tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt \
|
|
tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl tst_elena_*.cdl \
|
|
tst_simple*.cdl tst_chunks.cdl pr_A1.* tauu_A1.* usi_01.* thetau_01.* \
|
|
tst_*.h5 tst_grp_rename.cdl tst_grp_rename.dmp ref_grp_rename.cdl \
|
|
foo1.nc tst_*.h4 test.nc testszip.nc test.h5 szip_dump.cdl \
|
|
perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \
|
|
floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \
|
|
testfilter_reg.nc filterrepeat.txt tmp_fillonly.nc \
|
|
testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1 \
|
|
tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp tmp_bzip2.c bzip2.nc
|
|
|
|
DISTCLEANFILES = findplugin.sh run_par_test.sh
|
|
|
|
# If valgrind is present, add valgrind targets.
|
|
@VALGRIND_CHECK_RULES@
|