Merge branch 'master' into gh1868.allured

CMakeLists.txt

@@ -566,6 +566,9 @@ IF(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC)
   SET(ENABLE_SET_LOG_LEVEL ON)
 ENDIF()
 
+# This has multiversion capability
+SET(HAS_MULTIFILTERS yes CACHE BOOL "")
+
 # Option to allow for strict null file padding.
 # See https://github.com/Unidata/netcdf-c/issues/657 for more information
 OPTION(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING "Enable strict null byte header padding." OFF)
@@ -779,7 +782,7 @@ IF(USE_HDF5)
     SET(ENABLE_NCDUMPCHUNKS ON)
   ENDIF()
 
-  # Check to see if this is hdf5-1.10.6 or later.
+  # Record if ROS3 Driver is available
   IF(HAS_HDF5_ROS3)
     SET(ENABLE_HDF5_ROS3 ON)
   ENDIF()
@@ -821,6 +824,19 @@ IF(USE_HDF5)
   #option to include HDF5 High Level header file (hdf5_hl.h) in case we are not doing a make install
   INCLUDE_DIRECTORIES(${HDF5_HL_INCLUDE_DIR})
 
+  # Check to see if we have libcurl 7.66 or later
+  CHECK_C_SOURCE_COMPILES("
+  #include <H5public.h>
+  int main() {
+#if (H5_VERS_MAJOR*10000 + H5_VERS_MINOR*100 + H5_VERS_RELEASE < 11006)
+      choke me;
+#endif
+  }" HDF5_VERSION_1106)
+
+  IF(HDF5_VERSION_1106)
+   SET(HDF5_UTF8_PATHS ON)
+  ENDIF()
+
 ENDIF(USE_HDF5)
 
 # See if we have libcurl
@@ -962,24 +978,36 @@ IF(NOT WIN32)
 ENDIF()
 
 # Options to Extend NCZarr support
-OPTION(ENABLE_S3_SDK "Enable Search for AWS libraries." OFF)
+OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support." OFF)
-OPTION(ENABLE_S3_TESTS "Enable NCZarr S3 tests." OFF)
+OPTION(ENABLE_NCZARR_S3_TESTS "Enable NCZarr S3 tests." OFF)
 
-IF(NOT ENABLE_S3_SDK)
+IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_NCZARR_S3)
-  IF(ENABLE_S3_TESTS)
+  message(FATAL_ERROR "NCZarr S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=no")
-     message(WARNING "DISABLE_S3_SDK => DISABLE_S3_TESTS")
+  SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCARR S3 TESTS" FORCE)  
-     SET(ENABLE_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
-  ENDIF()
 ENDIF()
 
-IF(ENABLE_S3_SDK)
 # See if aws-s3-sdk is available
+# But only if enabled
+IF(ENABLE_NCZARR_S3)
 find_package(AWSSDK REQUIRED)
 set(SERVICE s3)
 AWSSDK_DETERMINE_LIBS_TO_LINK(SERVICE AWSSDK_LINK_LIBRARIES)
-IF(NOT AWSSDK_FOUND)
+IF(AWSSDK_FOUND)
+  SET(ENABLE_S3_SDK ON CACHE BOOL "S3 SDK" FORCE)  
+ELSE()
   SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
 ENDIF()
+ELSE(ENABLE_NCZARR_S3)
+# Unconditionally disable
+SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
+ENDIF(ENABLE_NCZARR_S3)
+
+IF(NOT ENABLE_S3_SDK)
+  IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS)
+    message(FATAL_ERROR "AWS S3 libraries not found; please specify options DENABLE_NCZARR_S3=NO AND -DENABLE-NCZARR-S3-TESTS=NO")
+    SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE)
+    SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
+  ENDIF()
 ENDIF()
 
 ##
@@ -2076,7 +2104,7 @@ is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
 is_enabled(HAVE_H5Z_SZIP HAS_SZLIB)
 is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
 is_enabled(ENABLE_NCZARR HAS_NCZARR)
-is_enabled(ENABLE_S3_TESTS DO_S3_TESTS)
+is_enabled(ENABLE_NCZARR_S3_TESTS DO_S3_TESTS)
 
 # Generate file from template.
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
@@ -2120,6 +2148,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY
 # Build nc_test4/findplugin.sh
 #####
 SET(ISCMAKE "1")
+IF(MSVC)
+SET(ISMSVC "1")
+ENDIF()
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF)
 
 IF(ENABLE_EXAMPLES)

RELEASE_NOTES.md

@@ -8,6 +8,7 @@ This file contains a high-level description of this package's evolution. Release
 ## 4.8.0 - TBD
 
 * [Bug Fix] Fix incorrect time offsets from `ncdump -t`, in some cases when the time `units` attribute contains both a **non-zero** time-of-day, and a time zone suffix containing the letter "T", such as "UTC".  See [Github #1866](https://github.com/Unidata/netcdf-c/pull/1866) for more information.
+* [Bug Fix] Cleanup the NCZarr S3 build options. See [Github #1869](https://github.com/Unidata/netcdf-c/pull/1869) for more information.
 * [Bug Fix] Support aligned access for selected ARM processors.  See [Github #1871](https://github.com/Unidata/netcdf-c/pull/1871) for more information.
 * [Bug Fix] Fix time zone parser bug.  See more information at [Github #1866](https://github.com/Unidata/netcdf-c/pull/1866) for more information.
 * [Documentation] Migrated the documents in the NUG/ directory to the dedicated NUG repository found at https://github.com/Unidata/netcdf

config.h.cmake.in

@@ -149,10 +149,13 @@ are set when opening a binary file on Windows. */
 #cmakedefine ENABLE_NCZARR 1
 
 /* if true, enable S3 support */
-#cmakedefine ENABLE_S3_SDK 1
+#cmakedefine ENABLE_NCZARR_S3 1
 
 /* if true, enable S3 testing*/
-#cmakedefine ENABLE_S3_TESTS 1
+#cmakedefine ENABLE_NCZARR_S3_TESTS 1
+
+/* if true, S3 SDK is available */
+#cmakedefine ENABLE_S3_SDK 1
 
 /* define the possible sources for remote test servers */
 #cmakedefine REMOTETESTSERVERS	"${REMOTETESTSERVERS}"
@@ -413,6 +416,9 @@ are set when opening a binary file on Windows. */
 with zip */
 #cmakedefine HDF5_SUPPORTS_PAR_FILTERS 1
 
+/* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */
+#cmakedefine HDF5_UTF8_PATHS 1
+
 /* if true, include JNA bug fix */
 #cmakedefine JNA 1
 

configure.ac

@@ -572,42 +572,64 @@ if test "x$enable_nczarr" = xyes; then
 fi
 AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
 
-# Check for enabling S3 support
+# Check for enabling of S3 support
 AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled])
-AC_ARG_ENABLE([s3-sdk],
+AC_ARG_ENABLE([nczarr-s3],
-              [AS_HELP_STRING([--enable-s3-sdk],
+              [AS_HELP_STRING([--enable-nczarr-s3],
                               [enable netcdf zarr S3 support; make sure to set LDFLAGS])])
-test "x$enable_s3_sdk" = xyes || enable_s3_sdk=no
+test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no
-have_aws=${enable_s3_sdk}
+AC_MSG_RESULT($enable_nczarr_s3)
-AC_MSG_RESULT($enable_s3_sdk)
+
+# Check for enabling S3 testing
+AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
+AC_ARG_ENABLE([nczarr-s3-tests],
+              [AS_HELP_STRING([--enable-nczarr-s3-tests],
+                              [enable netcdf zarr S3 testing])])
+test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no
+AC_MSG_RESULT($enable_nczarr_s3_tests)
+
+# Disable S3 tests if S3 support is disabled
+if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then
+    AC_MSG_ERROR([NCZarr S3 support is disabled; please specify option --disable-nczarr-s3-tests])
+    enable_nczarr_s3_tests=no
+fi
+
+# See if we have the s3 aws library
+# Check for the AWS S3 SDK library
+AC_LANG_PUSH([C++])
+AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
+AC_LANG_POP
+AC_MSG_CHECKING([whether AWS S3 SDK library is available])
+AC_MSG_RESULT([$have_aws])
+
 if test "x$have_aws" = xyes ; then
 AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
 fi
 AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
 
-# Check for enabling S3 testing
-AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
-AC_ARG_ENABLE([s3-tests],
-              [AS_HELP_STRING([--enable-s3-tests],
-                              [enable netcdf zarr S3 testing])])
-test "x$enable_s3_tests" = xyes || enable_s3_tests=no
-AC_MSG_RESULT($enable_s3_tests)
-
 if test "x$have_aws" = xno ; then
-  AC_MSG_WARN([aws libraries not found, so default to --disable-s3-tests])
+  if test "x$enable_nczarr_s3" = xyes || test "x$enable_nczarr_s3_tests" = xyes ; then
-  enable_s3_tests=no
+    AC_MSG_ERROR([AWS S3 libraries not found; please specify options --disable-nczarr-s3 and --disable-nczarr-s3-tests])
+    enable_nczarr_s3_tests=no
+    enable_nczarr_s3=no
+  fi
 fi
 
-if test "x$enable_s3_tests" = xyes ; then
+if test "x$enable_nczarr_s3" = xyes ; then
-  AC_DEFINE([ENABLE_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled])
+  AC_DEFINE([ENABLE_NCZARR_S3], [1], [if true, build libnczarr with S3 support enabled])
 fi
-AM_CONDITIONAL(ENABLE_S3_TESTS, [test "x$enable_s3_tests" = xyes])
+AM_CONDITIONAL(ENABLE_NCZARR_S3, [test "x$enable_nczarr_s3" = xyes])
+
+if test "x$enable_nczarr_s3_tests" = xyes ; then
+  AC_DEFINE([ENABLE_NCZARR_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled])
+fi
+AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes])
 
 # This has multiversion capability
 AC_MSG_CHECKING([whether multi-filter support is enabled])
-have_multifilters=yes
+has_multifilters=yes
-AC_MSG_RESULT($have_multifilters)
+AC_MSG_RESULT($has_multifilters)
-AC_DEFINE([HAVE_MULTIFILTERS], [1], [if true, multi-filter support enabled])
+AC_DEFINE([HAS_MULTIFILTERS], [1], [if true, multi-filter support enabled])
 
 # Check whether we want to enable strict null byte header padding.
 # See https://github.com/Unidata/netcdf-c/issues/657 for more information.
@@ -1020,18 +1042,23 @@ AC_TYPE_UINTPTR_T
 AC_C_CHAR_UNSIGNED
 AC_C_BIGENDIAN
 
+# Figure out platforms of special interest
+case "`uname`" in
+  CYGWIN*) ISCYGWIN=yes;;
+  WIN*) ISMSVC=yes;;
+esac
+AM_CONDITIONAL(ISCYGWIN, [test "x$ISCYGWIN" = xyes])
+AM_CONDITIONAL(ISMSVC, [test "x$ISMSVC" = xyes])
+
 ###
 # Crude hack to work around an issue
 # in Cygwin.
 ###
 SLEEPCMD=""
-PLTFORMOUT="$(uname | cut -d '_' -f 1)"
+if test "x$ISCYGWIN" = "xyes"; then
-if test "$PLTFORMOUT" = "CYGWIN"; then
-   ISCYGWIN=yes
    SLEEPCMD="sleep 5"
    AC_MSG_NOTICE([Pausing between sizeof() checks to mitigate a Cygwin issue.])
 fi
-AM_CONDITIONAL(ISCYGWIN, [test "x$ISCYGWIN" = xyes])
 
 $SLEEPCMD
 AC_CHECK_SIZEOF(short)
@@ -1200,7 +1227,20 @@ if test "x$enable_hdf5" = xyes; then
    fi
    AC_MSG_RESULT([$enable_szlib])
 
+   # Check to see if HDF5 library is 1.10.6 or greater.
+   # Used to control path name conversion
+   AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <H5public.h>]], [[
+#if (H5_VERS_MAJOR*10000 + H5_VERS_MINOR*100 + H5_VERS_RELEASE < 11006)
+      choke me
+#endif
+  ]])], [hdf5_version_1106=yes], [hdf5_version_1106=no])
+  AC_MSG_CHECKING([whether HDF5 library is version 1.10.6 or later])
+  AC_MSG_RESULT([$hdf5_version_1106])
+  if test "x$hdf5_version_1106" = xyes; then
+      AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8])
+   fi
 fi
+
 AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ])
 
 # If the user wants hdf4 built in, check it out.
@@ -1406,7 +1446,7 @@ AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes])
 AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
 AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
 AM_CONDITIONAL(HAVE_AWS, [test "x$have_aws" = xyes])
-AM_CONDITIONAL(HAVE_MULTIFILTERS, [test "x$have_multifilters" = xyes])
+AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
 
 # If the machine doesn't have a long long, and we want netCDF-4, then
 # we've got problems!
@@ -1529,8 +1569,8 @@ AC_SUBST(HAS_BYTERANGE,[$enable_byterange])
 AC_SUBST(RELAX_COORD_BOUND,[yes])
 AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters])
 AC_SUBST(HAS_NCZARR,[$enable_nczarr])
-AC_SUBST(DO_S3_TESTS,[$enable_s3_tests])
+AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
-AC_SUBST(HAS_MULTIFILTERS,[$have_multifilters])
+AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
 
 # Include some specifics for netcdf on windows.
 #AH_VERBATIM([_WIN32_STRICMP],
@@ -1602,7 +1642,7 @@ AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes])
 AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes])
 AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes])
 AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
-AX_SET_META([NC_HAS_MULTIFILTERS],[$have_multifilters],[yes])
+AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes])
 
 # This is the version of the dispatch table. If the dispatch table is
 # changed, this should be incremented, so that user-defined format
@@ -1615,11 +1655,13 @@ AC_SUBST([NC_DISPATCH_VERSION], [2])
 # End netcdf_meta.h definitions.
 #####
 
+# This would be true for a visual studio build.
+AC_SUBST([ISMSVC], [${ISMSVC}])
 # This would be true for a cmake build.
 AC_SUBST([ISCMAKE], [])
 
-# This would be true for a visual studio build.
+# Provide  conditional to temporarily suppress tests and such
-AC_SUBST([MSVC], [])
+AM_CONDITIONAL([AX_IGNORE], [test xno = xyes])
 
 AC_MSG_NOTICE([generating header files and makefiles])
 AC_CONFIG_FILES(test_common.sh:test_common.in)

libhdf5/hdf5internal.c

@@ -1041,12 +1041,11 @@ nc4_ndf5_ansi_to_utf8(pathbuf_t *pb, const char *path)
     int n;
 
     if (hdf5_encoding == UNDEF) {
-        uint majnum, minnum, relnum;
+#ifdef HDF5_UTF8_PATHS
-        H5get_libversion(&majnum, &minnum, &relnum);
+	hdf5_encoding = UTF8;
-        hdf5_encoding = (((majnum == UTF8_MAJNUM && minnum == UTF8_MINNUM && relnum >= UTF8_RELNUM)
+#else
-                          || (majnum == UTF8_MAJNUM && minnum > UTF8_MINNUM)
+	hdf5_encoding = ANSI;
-                          || majnum > UTF8_MAJNUM)
+#endif
-                         ? UTF8 : ANSI);
     }
     if (hdf5_encoding == ANSI) {
         pb->ptr = NULL;

nc-config.cmake.in

@@ -101,6 +101,13 @@ else
     has_parallel="yes"
 fi
 
+has_nczarr="@ENABLE_NCZARR@"
+if [ -z "$has_nczarr" -o "$has_nczarr" = "OFF" -o "$has_nczarr" = "FALSE" ]; then
+    has_nczarr="no"
+else
+    has_nczarr="yes"
+fi
+
 version="@PACKAGE@ @VERSION@"
 
 
@@ -164,6 +171,7 @@ Available values for OPTION include:
   --has-cdf5      whether cdf5 support is included in build
   --has-parallel4 whether has parallel IO support via HDF5
   --has-parallel  whether has parallel IO support via HDF5 or PnetCDF
+  --has-nczarr    whether nczarr enabled
   --libs          library linking information for netcdf
   --static        library linking information for statically-compiled netcdf
   --prefix        Install prefix
@@ -245,6 +253,7 @@ fi
         echo "  --has-cdf5      -> $has_cdf5"
         echo "  --has-parallel4 -> $has_parallel4"
         echo "  --has-parallel  -> $has_parallel"
+        echo "  --has-nczarr    -> $has_nczarr"
         echo
         echo "  --prefix        -> $prefix"
         echo "  --includedir    -> $includedir"
@@ -346,6 +355,10 @@ while test $# -gt 0; do
         echo $has_parallel
         ;;
 
+    --has-nczarr)
+        echo $has_nczarr
+        ;;
+
     --libs)
         if [ "x$HAS_STATIC" = "xTRUE" ]; then
             echo $libs $libsprivate

nc-config.in

@@ -28,7 +28,6 @@ has_szlib="@HAS_SZLIB@"
 has_parallel="@HAS_PARALLEL@"
 has_parallel4="@HAS_PARALLEL4@"
 has_nczarr="@HAS_NCZARR@"
-has_s3_sdk="@HAS_S3_SDK@"
 version="@PACKAGE_NAME@ @PACKAGE_VERSION@"
 HAS_STATIC=""
 
@@ -94,7 +93,6 @@ Available values for OPTION include:
   --has-parallel4 whether has parallel IO support via HDF5
   --has-parallel  whether has parallel IO support via HDF5 or PnetCDF
   --has-nczarr    whether nczarr enabled
-  --has-s3-sdk    whether AWS S3 support enabled
   --libs          library linking information for netcdf
   --static        library linking information for statically-compiled netcdf
   --prefix        Install prefix
@@ -177,7 +175,6 @@ fi
         echo "  --has-parallel4 -> $has_parallel4"
         echo "  --has-parallel  -> $has_parallel"
         echo "  --has-nczarr    -> $has_nczarr"
-        echo "  --has-s3-sdk    -> $has_s3_sdk"
         echo
         echo "  --prefix        -> $prefix"
         echo "  --includedir    -> $includedir"
@@ -284,10 +281,6 @@ while test $# -gt 0; do
         echo $has_nczarr
         ;;
 
-    --has-s3-sdk)
-        echo $has_s3_sdk
-        ;;
-
     --libs)
         if [ "x$HAS_STATIC" = "xTRUE" ]; then
             echo $libs $libsprivate

nc_test4/findplugin.in

@@ -26,7 +26,7 @@ FP_NAME="$1"
 
 # Figure out the compiler (some values from ./configure)
 FP_ISCMAKE=@ISCMAKE@
-FP_ISMSVC=@MSVC@
+FP_ISMSVC=@ISMSVC@
 
 # Are we operating under OS-X? (test using uname)
 FP_OS=`uname | cut -d '_'  -f 1`

test_common.in

@@ -11,7 +11,7 @@ TOPBUILDDIR='@abs_top_builddir@'
 
 # Define selected features of the build
 FEATURE_HDF5=@HAS_HDF5@
-FEATURE_S3TESTS=@DO_S3_TESTS@
+FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@
 
 set -e