Merge branch 'master' into ejh_move_tests

CMakeLists.txt

@@ -1622,6 +1622,10 @@ add_subdirectory("include")
 add_subdirectory(libdispatch)
 add_subdirectory(libsrc)
 
+IF(ENABLE_FILTER_TESTING)
+add_subdirectory(plugins)
+ENDIF()
+
 IF(USE_PNETCDF)
   add_subdirectory(libsrcp)
 ENDIF(USE_PNETCDF)
@@ -1764,18 +1768,6 @@ STRING(REPLACE ";" " " LINKFLAGS "${LINKFLAGS}")
 LIST(REMOVE_DUPLICATES NC_LIBS)
 LIST(REMOVE_DUPLICATES LINKFLAGS)
 
-IF(ENABLE_FILTER_TESTING)
-IF(ENABLE_EXAMPLES)
-MESSAGE(STATUS "copying filter plugin building files")
-FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/findplugin.in
-     DESTINATION ${netCDF_BINARY_DIR}/examples/C/)
-FILE(GLOB PLUGINFILES ${netCDF_SOURCE_DIR}/nc_test4/hdf5plugins/*.c
-		      ${netCDF_SOURCE_DIR}/nc_test4/hdf5plugins/*.h)
-FILE(COPY ${PLUGINFILES}
-     DESTINATION ${netCDF_BINARY_DIR}/examples/C/hdf5plugins/)
-ENDIF()
-ENDIF()
-
 configure_file(
   ${netCDF_SOURCE_DIR}/netcdf.pc.in
   ${netCDF_BINARY_DIR}/netcdf.pc @ONLY)
@@ -1900,11 +1892,9 @@ configure_file(${CMAKE_SOURCE_DIR}/test_common.in ${CMAKE_BINARY_DIR}/test_commo
 SET(ISCMAKE "1")
 configure_file(${CMAKE_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF)
 
-IF(ENABLE_FILTER_TESTING)
 IF(ENABLE_EXAMPLES)
 configure_file(${CMAKE_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF)
 ENDIF()
-ENDIF()
 
 ####
 # Export files

Makefile.am

@@ -71,6 +71,11 @@ HDF4_TEST_DIR = hdf4_test
 LIBHDF4 = libhdf4
 endif
 
+# Optionally build plugins
+if ENABLE_FILTER_TESTING
+PLUGIN_DIR = plugins
+endif
+
 # Define Test directories
 if BUILD_TESTSETS
 TESTDIRS = $(V2_TEST) nc_test $(NC_TEST4) $(HDF4_TEST_DIR)	\
@@ -82,7 +87,7 @@ endif
 # depend on it.
 SUBDIRS = include $(H5_TEST_DIR) libdispatch libsrc $(LIBSRC4_DIR)	\
 $(LIBSRCP) $(LIBHDF4) $(OCLIB) $(DAP2) ${DAP4} liblib $(NCGEN3)		\
-$(NCGEN) $(NCDUMP) $(TESTDIRS) docs $(EXAMPLES)
+$(NCGEN) $(NCDUMP) ${PLUGIN_DIR} $(TESTDIRS) docs $(EXAMPLES)
 
 # Remove these generated files, for a distclean.
 DISTCLEANFILES = VERSION comps.txt test_prog libnetcdf.settings	\

cf

@@ -2,6 +2,7 @@
 #NB=1
 DB=1
 #X=-x
+
 #FAST=1
 #PROF=1
 
@@ -117,7 +118,7 @@ FLAGS="$FLAGS --enable-logging"
 #FLAGS="$FLAGS --disable-properties-attribute"
 #FLAGS="$FLAGS --disable-silent-rules"
 #FLAGS="$FLAGS --with-testservers=remotestserver.localhost:8083"
-FLAGS="$FLAGS --disable-filter-testing"
+#FLAGS="$FLAGS --disable-filter-testing"
 #FLAGS="$FLAGS --enable-metadata-perf"
 
 if test "x$PAR4" != x1 ; then
@@ -179,6 +180,7 @@ CFLAGS="${CFLAGS} -pg"
 LDFLAGS="${LDFLAGS} -pg"
 fi
 
+
 #FLAGS="${FLAGS} --enable-stdio"
 
 export PATH

cf.cmake

@@ -47,7 +47,7 @@ FLAGS="$FLAGS -DENABLE_EXAMPLES=false"
 FLAGS="$FLAGS -DENABLE_DYNAMIC_LOADING=false"
 FLAGS="$FLAGS -DENABLE_WINSOCK2=false"
 #FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true"
-#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
+FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
 
 rm -fr build
 mkdir build

configure.ac

@@ -1432,27 +1432,6 @@ AC_SUBST([MSVC], [])
 AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in)
 AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in)
 
-if test "x$enable_filter_testing" = xyes ; then
-if test "x$nc_build_examples" = xyes ; then
-AC_MSG_NOTICE([copying filter plugin building files])
-# Must be a simpler way; all my attempts to use some kind of wildcard failed,
-# so falling back on enumeration
-# WARNING: This enumeration must be kept consistent with the
-# HDF5PLUGINSRC list in nc_test4/hdf5plugins/Makefile.am
-AC_CONFIG_LINKS([examples/C/hdf5plugins/bzlib.h:nc_test4/hdf5plugins/bzlib.h])
-AC_CONFIG_LINKS([examples/C/hdf5plugins/bzlib_private.h:nc_test4/hdf5plugins/bzlib_private.h])
-AC_CONFIG_LINKS([examples/C/hdf5plugins/blocksort.c:nc_test4/hdf5plugins/blocksort.c])
-AC_CONFIG_LINKS([examples/C/hdf5plugins/huffman.c:nc_test4/hdf5plugins/huffman.c])
-AC_CONFIG_LINKS([examples/C/hdf5plugins/crctable.c:nc_test4/hdf5plugins/crctable.c])
-AC_CONFIG_LINKS([examples/C/hdf5plugins/randtable.c:nc_test4/hdf5plugins/randtable.c])
-AC_CONFIG_LINKS([examples/C/hdf5plugins/compress.c:nc_test4/hdf5plugins/compress.c])
-AC_CONFIG_LINKS([examples/C/hdf5plugins/decompress.c:nc_test4/hdf5plugins/decompress.c])
-AC_CONFIG_LINKS([examples/C/hdf5plugins/bzlib.c:nc_test4/hdf5plugins/bzlib.c])
-AC_CONFIG_LINKS([examples/C/hdf5plugins/H5Zbzip2.c:nc_test4/hdf5plugins/H5Zbzip2.c])
-AC_CONFIG_LINKS([examples/C/hdf5plugins/h5bzip2.h:nc_test4/hdf5plugins/h5bzip2.h])
-fi
-fi
-
 #####
 # End netcdf_meta.h definitions.
 #####
@@ -1475,7 +1454,6 @@ AC_CONFIG_FILES([Makefile
                  ncgen/Makefile
                  examples/Makefile
                  examples/C/Makefile
-                 examples/C/hdf5plugins/Makefile
                  examples/CDL/Makefile
                  oc2/Makefile
                  libdap2/Makefile
@@ -1489,13 +1467,13 @@ AC_CONFIG_FILES([Makefile
                  docs/images/Makefile
                  nctest/Makefile
                  nc_test4/Makefile
-                 nc_test4/hdf5plugins/Makefile
                  nc_test/Makefile
                  ncdap_test/Makefile
                  ncdap_test/testdata3/Makefile
                  ncdap_test/expected3/Makefile
                  ncdap_test/expectremote3/Makefile
                  dap4_test/Makefile
+                 plugins/Makefile
 		 ],
                  [test -f  nc-config && chmod 755 nc-config])
 

examples/C/CMakeLists.txt

@@ -12,9 +12,5 @@ SET_TESTS_PROPERTIES(C_tests_simple_xy_rd PROPERTIES DEPENDS C_tests_simple_xy_w
 SET_TESTS_PROPERTIES(C_tests_sfc_pres_temp_rd PROPERTIES DEPENDS C_tests_sfc_pres_temp_wr)
 SET_TESTS_PROPERTIES(C_tests_pres_temp_4D_rd PROPERTIES DEPENDS C_tests_pres_temp_4D_wr)
 
-IF(ENABLE_FILTER_TESTING)
-  ADD_SUBDIRECTORY(hdf5plugins)
-ENDIF(ENABLE_FILTER_TESTING)
-
 SET(CLEANFILES sfc_pres_temp.nc simple_xy.nc pres_temp_4D.nc simple_nc4.nc simple_xy_nc4.nc)
 SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${CLEANFILES}")

examples/C/Makefile.am

@@ -25,9 +25,6 @@ if USE_NETCDF4
 check_PROGRAMS += simple_nc4_wr simple_nc4_rd simple_xy_nc4_wr	\
 simple_xy_nc4_rd
 
-# This subdir should be same as nc_test4/hdf5plugins
-SUBDIRS = hdf5plugins
-
 if ENABLE_FILTER_TESTING
 # filter_example.c should be same as nc_test4/test_filter.c
 check_PROGRAMS += filter_example

examples/C/hdf5plugins/CMakeLists.txt

@@ -1,31 +0,0 @@
-SET(CMAKE_BUILD_TYPE "")
-
-SET(libbzip2_SOURCES bzlib.h bzlib_private.h
-         blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c
-         H5Zbzip2.c h5bzip2.h
-)
-
-# Get the compilable sources from nc_test4/hdf5plugins
-FOREACH(S ${libbzip2_SOURCES})
-  FILE(COPY ${CMAKE_SOURCE_DIR}/nc_test4/hdf5plugins/${S} DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
-ENDFOREACH(S)
-
-IF(ENABLE_FILTER_TESTING)
-IF(BUILD_UTILITIES)
-
-# LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined
-
-ADD_LIBRARY(bzip2 MODULE ${libbzip2_SOURCES})
-SET_TARGET_PROPERTIES(bzip2 PROPERTIES LIBRARY_OUTPUT_NAME "bzip2")
-SET_TARGET_PROPERTIES(bzip2 PROPERTIES ARCHIVE_OUTPUT_NAME "bzip2")
-SET_TARGET_PROPERTIES(bzip2 PROPERTIES RUNTIME_OUTPUT_NAME "bzip2")
-TARGET_LINK_LIBRARIES(bzip2 ${HDF5_HL_LIBRARIES} ${HDF5_C_LIBRARIES})
-
-ENDIF(BUILD_UTILITIES)
-ENDIF(ENABLE_FILTER_TESTING)
-
-# Copy some test files from current source dir to out-of-tree build dir.
-FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
-IF(MSVC)
-  FILE(COPY ${COPY_FILES} DESTINATION ${RUNTIME_OUTPUT_DIRECTORY}/)
-ENDIF()

examples/C/hdf5plugins/Makefile.am

@@ -1,32 +0,0 @@
-#    Copyright 2018, UCAR/Unidata
-#    See netcdf/COPYRIGHT file for copying and redistribution conditions.
-
-BZIP2SRC = bzlib.h bzlib_private.h blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c
-
-PLUGINSRC = H5Zbzip2.c h5bzip2.h
-
-TARGETS = ${PLUGINSRC} ${BZIP2SRC}
-
-HDF5PLUGINSRC=${PLUGINSRC} ${BZIP2SRC}
-BUILT_SOURCES = ${TARGETS}
-
-${TARGETS}:
-	for x in ${TARGETS} ; do \
-	rm -f ./$$x; \
-	'cp' -f ${top_srcdir}/nc_test4/hdf5plugins/$$x . ; \
-	done
-
-CLEANFILES = ${TARGETS}
-
-
-if ENABLE_FILTER_TESTING
-
-lib_LTLIBRARIES = libbzip2.la
-
-libbzip2_la_SOURCES = ${HDF5PLUGINSRC}
-libbzip2_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined
-
-endif #ENABLE_FILTER_TESTING
-
-EXTRA_DIST = CMakeLists.txt H5Zbzip2.c Makefile.am blocksort.c bzlib.c bzlib.h bzlib_private.h compress.c \
-crctable.c decompress.c h5bzip2.h huffman.c randtable.c bzip2.nc

libdispatch/crc32.c

@@ -109,13 +109,6 @@ extern unsigned int NC_crc32(unsigned int crc, const unsigned char* buf, unsigne
 #  define TBLS 1
 #endif /* BYFOUR */
 
-/* Local functions for crc concatenation */
-local unsigned long gf2_matrix_times OF((unsigned long *mat,
-                                         unsigned long vec));
-local void gf2_matrix_square OF((unsigned long *square, unsigned long *mat));
-local uLong crc32_combine_ OF((uLong crc1, uLong crc2, z_off64_t len2));
-
-
 #ifdef DYNAMIC_CRC_TABLE
 
 local volatile int crc_table_empty = 1;
@@ -250,7 +243,8 @@ local void write_table(out, table)
 /* =========================================================================
  * This function can be used by asm versions of crc32()
  */
-const z_crc_t FAR * ZEXPORT get_crc_table()
+#if 0 /* Unused */
+local const z_crc_t FAR * ZEXPORT get_crc_table()
 {
 #ifdef DYNAMIC_CRC_TABLE
     if (crc_table_empty)
@@ -258,13 +252,14 @@ const z_crc_t FAR * ZEXPORT get_crc_table()
 #endif /* DYNAMIC_CRC_TABLE */
     return (const z_crc_t FAR *)crc_table;
 }
+#endif
 
 /* ========================================================================= */
 #define DO1 crc = crc_table[0][((int)crc ^ (*buf++)) & 0xff] ^ (crc >> 8)
 #define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1
 
 /* ========================================================================= */
-unsigned long ZEXPORT crc32_z(crc, buf, len)
+local unsigned long ZEXPORT crc32_z(crc, buf, len)
     unsigned long crc;
     const unsigned char FAR *buf;
     z_size_t len;
@@ -405,6 +400,7 @@ local unsigned long crc32_big(crc, buf, len)
 #define GF2_DIM 32      /* dimension of GF(2) vectors (length of CRC) */
 
 /* ========================================================================= */
+#if 0 /* Unused */
 local unsigned long gf2_matrix_times(mat, vec)
     unsigned long *mat;
     unsigned long vec;
@@ -433,6 +429,7 @@ local void gf2_matrix_square(square, mat)
 }
 
 /* ========================================================================= */
+
 local uLong crc32_combine_(crc1, crc2, len2)
     uLong crc1;
     uLong crc2;
@@ -489,7 +486,7 @@ local uLong crc32_combine_(crc1, crc2, len2)
 }
 
 /* ========================================================================= */
-uLong ZEXPORT crc32_combine(crc1, crc2, len2)
+local uLong ZEXPORT crc32_combine(crc1, crc2, len2)
     uLong crc1;
     uLong crc2;
     z_off_t len2;
@@ -497,10 +494,11 @@ uLong ZEXPORT crc32_combine(crc1, crc2, len2)
     return crc32_combine_(crc1, crc2, len2);
 }
 
-uLong ZEXPORT crc32_combine64(crc1, crc2, len2)
+local uLong ZEXPORT crc32_combine64(crc1, crc2, len2)
     uLong crc1;
     uLong crc2;
     z_off64_t len2;
 {
     return crc32_combine_(crc1, crc2, len2);
 }
+#endif

libdispatch/nchashmap.c

@@ -33,7 +33,7 @@ we will only do string comparisons when they will match.
 */
 
 /* Prototype for the crc32 function */
-extern unsigned int NC_crc32(unsigned int crc, const char* buf, unsigned int len);
+extern unsigned int NC_crc32(unsigned int crc, const unsigned char* buf, unsigned int len);
 
 #undef SMALLTABLE
 

nc_test4/CMakeLists.txt

@@ -94,5 +94,3 @@ IF(TEST_PARALLEL4)
 ENDIF()
 
 ADD_EXTRA_DIST(findplugin.in)
-
-ADD_SUBDIRECTORY(hdf5plugins)

nc_test4/Makefile.am

@@ -154,7 +154,5 @@ szip_dump.cdl perftest.txt bigmeta.nc
 
 DISTCLEANFILES = findplugin.sh
 
-SUBDIRS=hdf5plugins
-
 # If valgrind is present, add valgrind targets.
 @VALGRIND_CHECK_RULES@

nc_test4/findplugin.in

@@ -3,7 +3,7 @@
 # Define a function that attempts to locate a
 # plugin with a given canonical name.
 # Assumptions:
-#   1. hdf5plugins is a directory in this current directory
+#   1. plugins is a top-level directory
 # Inputs:
 #   $1 is the canonical name
 #   $2 is 1 if we are running under cmake
@@ -28,6 +28,9 @@ FP_NAME="$1"
 FP_ISCMAKE=@ISCMAKE@
 FP_ISMSVC=@MSVC@
 
+# And topsrcdir
+topbuilddir='@abs_top_builddir@'
+
 # Are we operating under OS-X? (test using uname)
 FP_OS=`uname | cut -d '_'  -f 1`
 if test "x$FP_OS" = xDarwin ; then FP_ISOSX=1; fi
@@ -36,8 +39,7 @@ if test "x$FP_OS" = xDarwin ; then FP_ISOSX=1; fi
 FP_OS=`uname | cut -d '_'  -f 1`
 if test "x$FP_OS" = xCYGWIN ; then FP_ISCYGWIN=1; fi
 
-FP_PLUGINS=`pwd`
-FP_PLUGINS="$FP_PLUGINS/hdf5plugins"
+FP_PLUGINS="$topbuilddir/plugins"
 
 FP_PLUGIN_LIB=
 FP_PLUGIN_PATH=

nc_test4/perftest.sh

@@ -1,6 +1,5 @@
 #!/bin/sh
 
-BUILD=1
 #PROF=1
 #DEBUG=1
 #MEM=1
@@ -21,35 +20,12 @@ ARGS="--treedepth=6 \
 --varrank=2 \
 --nvarattrs=500"
 
-if test "x$MEM" = x1 ; then
-CMD="valgrind --leak-check=full"
-elif test "x$DEBUG" = x1 ; then
-CMD="gdb --args "
-fi
-
-if test "x$BUILD" = x1 ; then
-CFLAGS="-Wall -Wno-unused-variable -Wno-unused-function -I.. -I../include"
-LDFLAGS="../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm -lmfhdf -ldf"
-CC=gcc
-if test "x$PROF" = x1 ; then
-CFLAGS="-pg $CFLAGS"
-LDFLAGS="-pg $LDFLAGS"
-fi
-if test "x$DEBUG" = x1 ; then
-CFLAGS="-g -O0 $CFLAGS"
-LDFLAGS="-g -O0 $LDFLAGS"
-fi
-LLP="/usr/local/lib:${LD_LIBRARY_PATH}"
-export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS
-${CC} -o bigmeta ${CFLAGS} bigmeta.c  ${LDFLAGS}
-${CC} -o openbigmeta ${CFLAGS} openbigmeta.c  ${LDFLAGS}
-fi
-
 echo "timing bigmeta:"
-${CMD} ./bigmeta $ARGS
+${execdir}/bigmeta $ARGS
 echo "timing openbigmeta:"
-${CMD} ./openbigmeta
+${execdir}/openbigmeta
 if test "x$PROF" = x1 ; then
 rm -f perftest.txt
 gprof openbigmeta gmon.out >perftest.txt
 fi
+

nc_test4/tst_filter.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+export SETX=1
+
 if test "x$srcdir" = x ; then srcdir=`pwd`; fi
 . ../test_common.sh
 

plugins/Makefile.am

@@ -1,7 +1,6 @@
 #    Copyright 2018, UCAR/Unidata
 #    See netcdf/COPYRIGHT file for copying and redistribution conditions.
 
-
 BZIP2HDRS=bzlib.h bzlib_private.h
 BZIP2SRC= blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c