Improve the building of NCZARR S3 support in CMake and Autoconf

There were some irregularities in the flags for handling NCZarr S3 support.

The primary change is to regularize the flags controlling this to the following.

1. Automake: --enable-nczarr-s3 and CMake: ENABLE_NCZARR_S3
2. Automake: --enable-nczarr-s3-tests and CMake: ENABLE_NCZARR_S3_TESTS

Flag 1 indicates that NCZarr should be built with S3 support enabled.
Flag 2 indicates that the NCZarr S3 tests should be run

These two flags are separate because running the NCZarr S3 tests
requires access to protected S3 resources. Currently, running
these tests is restricted to Unidata personnel. However, users
may want to enable S3 support even if they cannot run the tests.
It is, of course, an error to specify 2 without specifying 1.

Additionally, if the AWS S3 SDK library is not found, then the NCZARR S3
support and testing must be disabled. Otherwise an error is signaled
during the build.

Some of these NCZarr and S3 changes are propagated to nc-config.

Misc. Other Changes:

1. Allow testing for CYGWIN or MSVC in shell scripts.
2. Add specific test for HDF5 library version 1.10.6.
   This is encoded as "HDF5_UTF8_PATHS" because that is the first
   version where HDF5 properly supports it under Windows. This is used
   in hdf5internal/nc4_ndf5_ansi_to_utf8.
3. Add a AM Conditional -- AX_IGNORE -- for use in testing
   when it is desirable to temporarily suppress Makefile code.
4. Add MULTIFILTER flag to CMakeLists.txt

CMakeLists.txt

@@ -566,6 +566,9 @@ IF(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC)
   SET(ENABLE_SET_LOG_LEVEL ON)
 ENDIF()
 
+# This has multiversion capability
+SET(HAS_MULTIFILTERS yes CACHE BOOL "")
+
 # Option to allow for strict null file padding.
 # See https://github.com/Unidata/netcdf-c/issues/657 for more information
 OPTION(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING "Enable strict null byte header padding." OFF)
@@ -779,7 +782,7 @@ IF(USE_HDF5)
     SET(ENABLE_NCDUMPCHUNKS ON)
   ENDIF()
 
-  # Check to see if this is hdf5-1.10.6 or later.
+  # Record if ROS3 Driver is available
   IF(HAS_HDF5_ROS3)
     SET(ENABLE_HDF5_ROS3 ON)
   ENDIF()
@@ -821,6 +824,19 @@ IF(USE_HDF5)
   #option to include HDF5 High Level header file (hdf5_hl.h) in case we are not doing a make install
   INCLUDE_DIRECTORIES(${HDF5_HL_INCLUDE_DIR})
 
+  # Check to see if we have libcurl 7.66 or later
+  CHECK_C_SOURCE_COMPILES("
+  #include <H5public.h>
+  int main() {
+#if (H5_VERS_MAJOR*10000 + H5_VERS_MINOR*100 + H5_VERS_RELEASE < 11006)
+      choke me;
+#endif
+  }" HDF5_VERSION_1106)
+
+  IF(HDF5_VERSION_1106)
+   SET(HDF5_UTF8_PATHS ON)
+  ENDIF()
+
 ENDIF(USE_HDF5)
 
 # See if we have libcurl
@@ -962,24 +978,36 @@ IF(NOT WIN32)
 ENDIF()
 
 # Options to Extend NCZarr support
-OPTION(ENABLE_S3_SDK "Enable Search for AWS libraries." OFF)
-OPTION(ENABLE_S3_TESTS "Enable NCZarr S3 tests." OFF)
+OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support." OFF)
+OPTION(ENABLE_NCZARR_S3_TESTS "Enable NCZarr S3 tests." OFF)
 
-IF(NOT ENABLE_S3_SDK)
-  IF(ENABLE_S3_TESTS)
-     message(WARNING "DISABLE_S3_SDK => DISABLE_S3_TESTS")
-     SET(ENABLE_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
-  ENDIF()
+IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_NCZARR_S3)
+  message(FATAL_ERROR "NCZarr S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=no")
+  SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCARR S3 TESTS" FORCE)  
 ENDIF()
 
-IF(ENABLE_S3_SDK)
 # See if aws-s3-sdk is available
+# But only if enabled
+IF(ENABLE_NCZARR_S3)
 find_package(AWSSDK REQUIRED)
 set(SERVICE s3)
 AWSSDK_DETERMINE_LIBS_TO_LINK(SERVICE AWSSDK_LINK_LIBRARIES)
-IF(NOT AWSSDK_FOUND)
+IF(AWSSDK_FOUND)
+  SET(ENABLE_S3_SDK ON CACHE BOOL "S3 SDK" FORCE)  
+ELSE()
   SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
 ENDIF()
+ELSE(ENABLE_NCZARR_S3)
+# Unconditionally disable
+SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
+ENDIF(ENABLE_NCZARR_S3)
+
+IF(NOT ENABLE_S3_SDK)
+  IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS)
+    message(FATAL_ERROR "AWS S3 libraries not found; please specify options DENABLE_NCZARR_S3=NO AND -DENABLE-NCZARR-S3-TESTS=NO")
+    SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE)
+    SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
+  ENDIF()
 ENDIF()
 
 ##
@@ -2076,7 +2104,7 @@ is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
 is_enabled(HAVE_H5Z_SZIP HAS_SZLIB)
 is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
 is_enabled(ENABLE_NCZARR HAS_NCZARR)
-is_enabled(ENABLE_S3_TESTS DO_S3_TESTS)
+is_enabled(ENABLE_NCZARR_S3_TESTS DO_S3_TESTS)
 
 # Generate file from template.
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
@@ -2120,6 +2148,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY
 # Build nc_test4/findplugin.sh
 #####
 SET(ISCMAKE "1")
+IF(MSVC)
+SET(ISMSVC "1")
+ENDIF()
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF)
 
 IF(ENABLE_EXAMPLES)

config.h.cmake.in

@@ -149,10 +149,13 @@ are set when opening a binary file on Windows. */
 #cmakedefine ENABLE_NCZARR 1
 
 /* if true, enable S3 support */
-#cmakedefine ENABLE_S3_SDK 1
+#cmakedefine ENABLE_NCZARR_S3 1
 
 /* if true, enable S3 testing*/
-#cmakedefine ENABLE_S3_TESTS 1
+#cmakedefine ENABLE_NCZARR_S3_TESTS 1
+
+/* if true, S3 SDK is available */
+#cmakedefine ENABLE_S3_SDK 1
 
 /* define the possible sources for remote test servers */
 #cmakedefine REMOTETESTSERVERS	"${REMOTETESTSERVERS}"
@@ -413,6 +416,9 @@ are set when opening a binary file on Windows. */
 with zip */
 #cmakedefine HDF5_SUPPORTS_PAR_FILTERS 1
 
+/* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */
+#cmakedefine HDF5_UTF8_PATHS 1
+
 /* if true, include JNA bug fix */
 #cmakedefine JNA 1
 

configure.ac

@@ -572,42 +572,64 @@ if test "x$enable_nczarr" = xyes; then
 fi
 AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
 
-# Check for enabling S3 support
+# Check for enabling of S3 support
 AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled])
-AC_ARG_ENABLE([s3-sdk],
-              [AS_HELP_STRING([--enable-s3-sdk],
+AC_ARG_ENABLE([nczarr-s3],
+              [AS_HELP_STRING([--enable-nczarr-s3],
                               [enable netcdf zarr S3 support; make sure to set LDFLAGS])])
-test "x$enable_s3_sdk" = xyes || enable_s3_sdk=no
-have_aws=${enable_s3_sdk}
-AC_MSG_RESULT($enable_s3_sdk)
+test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no
+AC_MSG_RESULT($enable_nczarr_s3)
+
+# Check for enabling S3 testing
+AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
+AC_ARG_ENABLE([nczarr-s3-tests],
+              [AS_HELP_STRING([--enable-nczarr-s3-tests],
+                              [enable netcdf zarr S3 testing])])
+test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no
+AC_MSG_RESULT($enable_nczarr_s3_tests)
+
+# Disable S3 tests if S3 support is disabled
+if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then
+    AC_MSG_ERROR([NCZarr S3 support is disabled; please specify option --disable-nczarr-s3-tests])
+    enable_nczarr_s3_tests=no
+fi
+
+# See if we have the s3 aws library
+# Check for the AWS S3 SDK library
+AC_LANG_PUSH([C++])
+AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
+AC_LANG_POP
+AC_MSG_CHECKING([whether AWS S3 SDK library is available])
+AC_MSG_RESULT([$have_aws])
+
 if test "x$have_aws" = xyes ; then
 AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
 fi
 AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
 
-# Check for enabling S3 testing
-AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
-AC_ARG_ENABLE([s3-tests],
-              [AS_HELP_STRING([--enable-s3-tests],
-                              [enable netcdf zarr S3 testing])])
-test "x$enable_s3_tests" = xyes || enable_s3_tests=no
-AC_MSG_RESULT($enable_s3_tests)
-
 if test "x$have_aws" = xno ; then
-  AC_MSG_WARN([aws libraries not found, so default to --disable-s3-tests])
-  enable_s3_tests=no
+  if test "x$enable_nczarr_s3" = xyes || test "x$enable_nczarr_s3_tests" = xyes ; then
+    AC_MSG_ERROR([AWS S3 libraries not found; please specify options --disable-nczarr-s3 and --disable-nczarr-s3-tests])
+    enable_nczarr_s3_tests=no
+    enable_nczarr_s3=no
+  fi
 fi
 
-if test "x$enable_s3_tests" = xyes ; then
-  AC_DEFINE([ENABLE_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled])
+if test "x$enable_nczarr_s3" = xyes ; then
+  AC_DEFINE([ENABLE_NCZARR_S3], [1], [if true, build libnczarr with S3 support enabled])
 fi
-AM_CONDITIONAL(ENABLE_S3_TESTS, [test "x$enable_s3_tests" = xyes])
+AM_CONDITIONAL(ENABLE_NCZARR_S3, [test "x$enable_nczarr_s3" = xyes])
+
+if test "x$enable_nczarr_s3_tests" = xyes ; then
+  AC_DEFINE([ENABLE_NCZARR_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled])
+fi
+AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes])
 
 # This has multiversion capability
 AC_MSG_CHECKING([whether multi-filter support is enabled])
-have_multifilters=yes
-AC_MSG_RESULT($have_multifilters)
-AC_DEFINE([HAVE_MULTIFILTERS], [1], [if true, multi-filter support enabled])
+has_multifilters=yes
+AC_MSG_RESULT($has_multifilters)
+AC_DEFINE([HAS_MULTIFILTERS], [1], [if true, multi-filter support enabled])
 
 # Check whether we want to enable strict null byte header padding.
 # See https://github.com/Unidata/netcdf-c/issues/657 for more information.
@@ -1020,18 +1042,23 @@ AC_TYPE_UINTPTR_T
 AC_C_CHAR_UNSIGNED
 AC_C_BIGENDIAN
 
+# Figure out platforms of special interest
+case "`uname`" in
+  CYGWIN*) ISCYGWIN=yes;;
+  WIN*) ISMSVC=yes;;
+esac
+AM_CONDITIONAL(ISCYGWIN, [test "x$ISCYGWIN" = xyes])
+AM_CONDITIONAL(ISMSVC, [test "x$ISMSVC" = xyes])
+
 ###
 # Crude hack to work around an issue
 # in Cygwin.
 ###
 SLEEPCMD=""
-PLTFORMOUT="$(uname | cut -d '_' -f 1)"
-if test "$PLTFORMOUT" = "CYGWIN"; then
-   ISCYGWIN=yes
+if test "x$ISCYGWIN" = "xyes"; then
    SLEEPCMD="sleep 5"
    AC_MSG_NOTICE([Pausing between sizeof() checks to mitigate a Cygwin issue.])
 fi
-AM_CONDITIONAL(ISCYGWIN, [test "x$ISCYGWIN" = xyes])
 
 $SLEEPCMD
 AC_CHECK_SIZEOF(short)
@@ -1200,7 +1227,20 @@ if test "x$enable_hdf5" = xyes; then
    fi
    AC_MSG_RESULT([$enable_szlib])
 
+   # Check to see if HDF5 library is 1.10.6 or greater.
+   # Used to control path name conversion
+   AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <H5public.h>]], [[
+#if (H5_VERS_MAJOR*10000 + H5_VERS_MINOR*100 + H5_VERS_RELEASE < 11006)
+      choke me
+#endif
+  ]])], [hdf5_version_1106=yes], [hdf5_version_1106=no])
+  AC_MSG_CHECKING([whether HDF5 library is version 1.10.6 or later])
+  AC_MSG_RESULT([$hdf5_version_1106])
+  if test "x$hdf5_version_1106" = xyes; then
+      AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8])
+   fi
 fi
+
 AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ])
 
 # If the user wants hdf4 built in, check it out.
@@ -1406,7 +1446,7 @@ AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes])
 AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
 AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
 AM_CONDITIONAL(HAVE_AWS, [test "x$have_aws" = xyes])
-AM_CONDITIONAL(HAVE_MULTIFILTERS, [test "x$have_multifilters" = xyes])
+AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
 
 # If the machine doesn't have a long long, and we want netCDF-4, then
 # we've got problems!
@@ -1529,8 +1569,8 @@ AC_SUBST(HAS_BYTERANGE,[$enable_byterange])
 AC_SUBST(RELAX_COORD_BOUND,[yes])
 AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters])
 AC_SUBST(HAS_NCZARR,[$enable_nczarr])
-AC_SUBST(DO_S3_TESTS,[$enable_s3_tests])
-AC_SUBST(HAS_MULTIFILTERS,[$have_multifilters])
+AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
+AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
 
 # Include some specifics for netcdf on windows.
 #AH_VERBATIM([_WIN32_STRICMP],
@@ -1602,7 +1642,7 @@ AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes])
 AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes])
 AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes])
 AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
-AX_SET_META([NC_HAS_MULTIFILTERS],[$have_multifilters],[yes])
+AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes])
 
 # This is the version of the dispatch table. If the dispatch table is
 # changed, this should be incremented, so that user-defined format
@@ -1615,11 +1655,13 @@ AC_SUBST([NC_DISPATCH_VERSION], [2])
 # End netcdf_meta.h definitions.
 #####
 
+# This would be true for a visual studio build.
+AC_SUBST([ISMSVC], [${ISMSVC}])
 # This would be true for a cmake build.
 AC_SUBST([ISCMAKE], [])
 
-# This would be true for a visual studio build.
-AC_SUBST([MSVC], [])
+# Provide  conditional to temporarily suppress tests and such
+AM_CONDITIONAL([AX_IGNORE], [test xno = xyes])
 
 AC_MSG_NOTICE([generating header files and makefiles])
 AC_CONFIG_FILES(test_common.sh:test_common.in)

libhdf5/hdf5internal.c

@@ -1041,12 +1041,11 @@ nc4_ndf5_ansi_to_utf8(pathbuf_t *pb, const char *path)
     int n;
 
     if (hdf5_encoding == UNDEF) {
-        uint majnum, minnum, relnum;
-        H5get_libversion(&majnum, &minnum, &relnum);
-        hdf5_encoding = (((majnum == UTF8_MAJNUM && minnum == UTF8_MINNUM && relnum >= UTF8_RELNUM)
-                          || (majnum == UTF8_MAJNUM && minnum > UTF8_MINNUM)
-                          || majnum > UTF8_MAJNUM)
-                         ? UTF8 : ANSI);
+#ifdef HDF5_UTF8_PATHS
+	hdf5_encoding = UTF8;
+#else
+	hdf5_encoding = ANSI;
+#endif
     }
     if (hdf5_encoding == ANSI) {
         pb->ptr = NULL;

nc-config.cmake.in

@@ -101,6 +101,13 @@ else
     has_parallel="yes"
 fi
 
+has_nczarr="@ENABLE_NCZARR@"
+if [ -z "$has_nczarr" -o "$has_nczarr" = "OFF" -o "$has_nczarr" = "FALSE" ]; then
+    has_nczarr="no"
+else
+    has_nczarr="yes"
+fi
+
 version="@PACKAGE@ @VERSION@"
 
 
@@ -164,6 +171,7 @@ Available values for OPTION include:
   --has-cdf5      whether cdf5 support is included in build
   --has-parallel4 whether has parallel IO support via HDF5
   --has-parallel  whether has parallel IO support via HDF5 or PnetCDF
+  --has-nczarr    whether nczarr enabled
   --libs          library linking information for netcdf
   --static        library linking information for statically-compiled netcdf
   --prefix        Install prefix
@@ -245,6 +253,7 @@ fi
         echo "  --has-cdf5      -> $has_cdf5"
         echo "  --has-parallel4 -> $has_parallel4"
         echo "  --has-parallel  -> $has_parallel"
+        echo "  --has-nczarr    -> $has_nczarr"
         echo
         echo "  --prefix        -> $prefix"
         echo "  --includedir    -> $includedir"
@@ -346,6 +355,10 @@ while test $# -gt 0; do
         echo $has_parallel
         ;;
 
+    --has-nczarr)
+        echo $has_nczarr
+        ;;
+
     --libs)
         if [ "x$HAS_STATIC" = "xTRUE" ]; then
             echo $libs $libsprivate

nc-config.in

@@ -28,7 +28,6 @@ has_szlib="@HAS_SZLIB@"
 has_parallel="@HAS_PARALLEL@"
 has_parallel4="@HAS_PARALLEL4@"
 has_nczarr="@HAS_NCZARR@"
-has_s3_sdk="@HAS_S3_SDK@"
 version="@PACKAGE_NAME@ @PACKAGE_VERSION@"
 HAS_STATIC=""
 
@@ -94,7 +93,6 @@ Available values for OPTION include:
   --has-parallel4 whether has parallel IO support via HDF5
   --has-parallel  whether has parallel IO support via HDF5 or PnetCDF
   --has-nczarr    whether nczarr enabled
-  --has-s3-sdk    whether AWS S3 support enabled
   --libs          library linking information for netcdf
   --static        library linking information for statically-compiled netcdf
   --prefix        Install prefix
@@ -177,7 +175,6 @@ fi
         echo "  --has-parallel4 -> $has_parallel4"
         echo "  --has-parallel  -> $has_parallel"
         echo "  --has-nczarr    -> $has_nczarr"
-        echo "  --has-s3-sdk    -> $has_s3_sdk"
         echo
         echo "  --prefix        -> $prefix"
         echo "  --includedir    -> $includedir"
@@ -284,10 +281,6 @@ while test $# -gt 0; do
         echo $has_nczarr
         ;;
 
-    --has-s3-sdk)
-        echo $has_s3_sdk
-        ;;
-
     --libs)
         if [ "x$HAS_STATIC" = "xTRUE" ]; then
             echo $libs $libsprivate

nc_test4/findplugin.in

@@ -26,7 +26,7 @@ FP_NAME="$1"
 
 # Figure out the compiler (some values from ./configure)
 FP_ISCMAKE=@ISCMAKE@
-FP_ISMSVC=@MSVC@
+FP_ISMSVC=@ISMSVC@
 
 # Are we operating under OS-X? (test using uname)
 FP_OS=`uname | cut -d '_'  -f 1`

test_common.in

@@ -11,7 +11,7 @@ TOPBUILDDIR='@abs_top_builddir@'
 
 # Define selected features of the build
 FEATURE_HDF5=@HAS_HDF5@
-FEATURE_S3TESTS=@DO_S3_TESTS@
+FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@
 
 set -e