diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index e7cf2f052..bc084a401 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -49,7 +49,7 @@ jobs: --disable-static --enable-dap --disable-dap-remote-tests --enable-plugins ${{ matrix.plugin_dir_option }} --disable-nczarr --disable-nczarr-filters - --disable-nczarr-s3 --disable-nczarr-s3-tests + --disable-s3 --with-s3-testing=no - name: Look at config.log if error diff --git a/CMakeLists.txt b/CMakeLists.txt index 6630a1635..1b17bf6be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -523,6 +523,15 @@ if(ENABLE_HDF5 OR ENABLE_HDF4 OR ENABLE_NCZARR) SET(ENABLE_NETCDF4 ON CACHE BOOL "Enable netCDF4 Alias" FORCE) endif() +# enable|disable all forms of network access +OPTION(ENABLE_NETWORK_ACCESS "Enable|disable all forms of network access" ON) +MESSAGE(">>> ENABLE_NETWORK_ACCESS=${ENABLE_NETWORK_ACCESS}") +if(NOT ENABLE_NETWORK_ACCESS) +MESSAGE(WARNING "ENABLE_NETWORK_ACCESS=NO => ENABLE_DAP[4]=NO") +SET(ENABLE_DAP OFF CACHE BOOL "ENABLE_NETWORK_ACCESS=NO => ENABLE_DAP=NO" FORCE) +SET(ENABLE_DAP4 OFF CACHE BOOL "ENABLE_NETWORK_ACCESS=NO => ENABLE_DAP4=NO" FORCE) +ENDIF() + IF(ENABLE_HDF4) SET(USE_HDF4 ON) # Check for include files, libraries. @@ -622,9 +631,6 @@ IF(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) SET(ENABLE_SET_LOG_LEVEL ON) ENDIF() -# This has multiversion capability -SET(ENABLE_MULTIFILTERS yes CACHE BOOL "") - # Option to allow for strict null file padding. # See https://github.com/Unidata/netcdf-c/issues/657 for more information OPTION(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING "Enable strict null byte header padding." OFF) @@ -633,12 +639,6 @@ IF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING) SET(USE_STRICT_NULL_BYTE_HEADER_PADDING ON CACHE BOOL "") ENDIF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING) -# Option for building RPC -#OPTION(ENABLE_RPC "Enable RPC Client and Server." OFF) -#IF(ENABLE_RPC) -# SET(BUILD_RPC ON CACHE BOOL "") -#ENDIF() - # Note that szip management is tricky. # This is because we have three things to consider: # 1. is libsz available? @@ -995,6 +995,12 @@ ELSE() SET(FOUND_CURL FALSE) ENDIF() +# Start disabling if curl not found +IF(NOT FOUND_CURL) + MESSAGE(WARNING "ENABLE_NETWORK_ACCESS requires libcurl; disabling") + SET(ENABLE_NETWORK_ACCESS OFF CACHE BOOL "ENABLE_NETWORK_ACCESS requires libcurl; disabling" FORCE) +ENDIF() + set (CMAKE_REQUIRED_INCLUDES ${CURL_INCLUDE_DIRS}) # Check to see if we have libcurl 7.66 or later CHECK_C_SOURCE_COMPILES(" @@ -1078,6 +1084,11 @@ ENDIF() # Option to support byte-range reading of remote datasets OPTION(ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." ON) +if(NOT ENABLE_NETWORK_ACCESS) +MESSAGE(WARNING "ENABLE_NETWORK_ACCESS=NO => ENABLE_BYTERANGE=NO") +SET(ENABLE_BYTERANGE OFF CACHE BOOL "ENABLE_NETWORK_ACCESS=NO => ENABLE_BYTERANGE=NO" FORCE) +ENDIF() + # Check for the math library so it can be explicitly linked. IF(NOT WIN32) @@ -1098,8 +1109,14 @@ ENDIF() OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON) OPTION(ENABLE_EXTERNAL_SERVER_TESTS "Enable external Server remote tests." OFF) OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF) -SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test") +if(NOT ENABLE_DAP) +SET(ENABLE_DAP_REMOTE_TESTS OFF CACHE BOOL "" FORCE) +SET(ENABLE_EXTERNAL_SERVER_TESTS OFF CACHE BOOL "" FORCE) +SET(ENABLE_DAP_LONG_TESTS OFF CACHE BOOL "" FORCE) +ENDIF() + +SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test") SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test") # See if we have zlib @@ -1273,6 +1290,12 @@ OPTION(ENABLE_S3 "Enable S3 support." OFF) OPTION(ENABLE_S3_INTERNAL "Enable S3 Internal support." OFF) OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support; Deprecated in favor of ENABLE_S3" OFF) +IF(NOT ENABLE_NETWORK_ACCESS) + SET(ENABLE_S3 OFF CACHE BOOL "" FORCE) + SET(ENABLE_S3_INTERNAL OFF CACHE BOOL "" FORCE) + SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "" FORCE) +ENDIF() + # Control S3 Testing: Multi-valued option SET(WITH_S3_TESTING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none) PUBLIC") SET_PROPERTY(CACHE WITH_S3_TESTING PROPERTY STRINGS ON OFF PUBLIC) # @@ -1290,6 +1313,15 @@ if (NOT ENABLE_S3 AND ENABLE_NCZARR_S3) ENDIF() UNSET(ENABLE_NCZARR_S3) +IF(NOT ENABLE_NETWORK_ACCESS) + MESSAGE(WARNING "ENABLE_NETWORK_ACCESS=NO => disable all s3 functionality") + SET(ENABLE_S3 OFF CACHE BOOL "" FORCE) + SET(ENABLE_S3_INTERNAL OFF CACHE BOOL "" FORCE) + SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "" FORCE) + SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE) + SET(WITH_S3_TESTING OFF CACHE STRING "" FORCE) +ENDIF() + # Note we check for the library after checking for enable_s3 # because for some reason this screws up if we unconditionally test for sdk # and it is not available. Fix someday @@ -1332,29 +1364,6 @@ IF(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING) ENDIF() ENDIF() -# Start disabling if curl not found -IF(NOT FOUND_CURL) - IF(ENABLE_BYTERANGE) - MESSAGE(FATAL_ERROR "Byte-range support specified, CURL libraries are not found.") - ENDIF() - - IF(ENABLE_DAP2 OR ENABLE_DAP4) - MESSAGE(WARNING "CURL libraries are not found; DAP support disabled") - SET(ENABLE_DAP2 OFF CACHE BOOL "Use DAP2" FORCE) - SET(ENABLE_DAP4 OFF CACHE BOOL "Use DAP4" FORCE) - ENDIF() - - IF(ENABLE_HDF5_ROS3) - MESSAGE(WARNING "CURL libraries are not found; ROS3 support disabled.") - SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE) - ENDIF() - - IF(ENABLE_S3) - MESSAGE(FATAL_ERROR "S3 support specified, but CURL libraries are not found.") - ENDIF() -ENDIF(NOT FOUND_CURL) - - OPTION(ENABLE_LIBXML2 "Link against libxml2 if it is available, use the packaged tinyxml2 parser otherwise." ON) SET(XMLPARSER "tinyxml2 (bundled)") @@ -2569,7 +2578,6 @@ is_enabled(ENABLE_S3_INTERNAL HAS_S3_INTERNAL) is_enabled(HAS_HDF5_ROS3 HAS_HDF5_ROS3) is_enabled(ENABLE_NCZARR HAS_NCZARR) is_enabled(ENABLE_NCZARR_ZIP HAS_NCZARR_ZIP) -is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS) is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS) is_enabled(ENABLE_QUANTIZE HAS_QUANTIZE) is_enabled(ENABLE_LOGGING HAS_LOGGING) @@ -2579,6 +2587,7 @@ is_enabled(HAVE_SZ HAS_SZLIB_WRITE) is_enabled(HAVE_ZSTD HAS_ZSTD) is_enabled(HAVE_BLOSC HAS_BLOSC) is_enabled(HAVE_BZ2 HAS_BZ2) +is_enabled(ENABLE_NETWORK_ACCESS DO_NETWORK_ACCESS) if(ENABLE_S3_INTERNAL) SET(WHICH_S3_SDK "internal") diff --git a/configure.ac b/configure.ac index 1c49a0cb2..c28c9f299 100644 --- a/configure.ac +++ b/configure.ac @@ -178,6 +178,12 @@ AC_ARG_ENABLE([pnetcdf], [AS_HELP_STRING([--enable-pnetcdf], test "x$enable_pnetcdf" = xyes || enable_pnetcdf=no AC_MSG_RESULT($enable_pnetcdf) +AC_MSG_CHECKING([whether any network access should be allowed]) +AC_ARG_ENABLE([network-access], [AS_HELP_STRING([--enable-network-access], + [enable|disable all forms of network access (default enabled)])]) +test "x$enable_network_access" = xno || enable_network_access=yes +AC_MSG_RESULT($enable_network_access) + # We need curl for remote operations AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no]) if test "x$found_curl" = "xyes" ; then @@ -192,6 +198,11 @@ AC_ARG_ENABLE([dap], test "x$enable_dap" = xno || enable_dap=yes AC_MSG_RESULT($enable_dap) +if test "x$enable_network_access" = xno ; then + AC_MSG_WARN([All network access is disabled => DAP support disabled.]) + enable_dap=no +fi + if test "x$enable_dap" = xyes & test "x$found_curl" = xno ; then AC_MSG_WARN([curl required for dap access. DAP support disabled.]) enable_dap=no @@ -641,7 +652,7 @@ AC_ARG_ENABLE([dap-auth-tests], test "x$enable_dap_auth_tests" = xyes || enable_dap_auth_tests=no AC_MSG_RESULT($enable_dap_auth_tests) -# dap must be enabled +# dap must be enabled in order to use various other flags if test "x$enable_dap" = "xno" ; then AC_MSG_NOTICE([--disable-dap => --disable-dap-remote-tests --disable-auth-tests --disable-external-server-tests]) @@ -738,7 +749,6 @@ if test "x$have_zstd" = "xyes" ; then fi fi - # See if we have libbz2 AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bz2=yes],[have_bz2=no]) if test "x$have_bz2" = "xyes" ; then @@ -799,6 +809,11 @@ AC_ARG_ENABLE([s3], test "x$enable_s3" = xyes || enable_s3=no AC_MSG_RESULT($enable_s3) +if test "x$enable_network_access" = xno ; then + AC_MSG_WARN([--disable-network-access => --disable-s3]) + enable_s3=no +fi + # --enable-nczarr-s3 is synonym for --enable-s3 (but...) AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled]) AC_ARG_ENABLE([nczarr-s3], @@ -807,7 +822,7 @@ AC_ARG_ENABLE([nczarr-s3], AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)]) # Set enable_s3 instead of enable_nczarr_s3 -if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes ; then +if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_network_access" = xyes; then enable_s3=yes # back compatibility fi unset enable_nczarr_s3 @@ -898,12 +913,6 @@ AC_ARG_WITH([chunk-cache-size-nczarr], AC_MSG_RESULT([$CHUNK_CACHE_SIZE_NCZARR]) AC_DEFINE_UNQUOTED([CHUNK_CACHE_SIZE_NCZARR], [$CHUNK_CACHE_SIZE_NCZARR], [default nczarr chunk cache size.]) -# This has multiversion capability -AC_MSG_CHECKING([whether multi-filter support is enabled]) -has_multifilters=yes -AC_MSG_RESULT($has_multifilters) -AC_DEFINE([HAS_MULTIFILTERS], [1], [if true, multi-filter support enabled]) - # Check whether we want to enable strict null byte header padding. # See https://github.com/Unidata/netcdf-c/issues/657 for more information. AC_MSG_CHECKING([whether to enable strict null-byte header padding when reading (default off)]) @@ -1310,6 +1319,11 @@ AC_ARG_ENABLE([byterange], test "x$enable_byterange" = xno || enable_byterange=yes AC_MSG_RESULT($enable_byterange) +if test "x$enable_network_access" = xno ; then + AC_MSG_WARN([--disable-network-access => --disable-byterange]) + enable_byterange=no +fi + # Need curl for byte ranges if test "x$found_curl" = xno && test "x$enable_byterange" = xyes ; then AC_MSG_ERROR([curl required for byte range support. Install curl or build without --enable-byterange.]) @@ -1433,7 +1447,6 @@ AC_CHECK_SIZEOF(size_t) $SLEEPCMD AC_CHECK_SIZEOF(unsigned long long) - if test "$ac_cv_sizeof_size_t" -lt "8" ; then if test "x${enable_cdf5}" = xyes ; then dnl unable to support CDF5, but --enable-cdf5 is explicitly set @@ -1860,7 +1873,6 @@ AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes]) AM_CONDITIONAL(ENABLE_S3_TESTPUB, [test "x$with_s3_testing" != xno]) # all => public AM_CONDITIONAL(ENABLE_S3_TESTALL, [test "x$with_s3_testing" = xyes]) AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes]) -AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes]) AM_CONDITIONAL(HAVE_DEFLATE, [test "x$have_deflate" = xyes]) AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes]) AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes]) @@ -1971,7 +1983,6 @@ AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3]) AC_SUBST(HAS_NCZARR,[$enable_nczarr]) AC_SUBST(DO_S3_TESTING,[$with_s3_testing]) AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip]) -AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters]) AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip]) AC_SUBST(HAS_QUANTIZE,[$enable_quantize]) AC_SUBST(HAS_LOGGING,[$enable_logging]) @@ -1981,6 +1992,7 @@ AC_SUBST(HAS_SZLIB,[$have_sz]) AC_SUBST(HAS_SZLIB_WRITE, [$have_sz]) AC_SUBST(HAS_ZSTD,[$have_zstd]) AC_SUBST(DO_LARGE_TESTS,[$enable_large_file_tests]) +AC_SUBST(DO_NETWORK_ACCESS,[$enable_network_access]) if test "x$enable_s3_aws" = xyes ; then AC_SUBST(WHICH_S3_SDK,[aws-sdk-cpp]) @@ -2115,7 +2127,6 @@ AX_SET_META([NC_HAS_S3_AWS],[$enable_s3_aws],[yes]) AX_SET_META([NC_HAS_S3_INTERNAL],[$enable_s3_internal],[yes]) AX_SET_META([NC_HAS_HDF5_ROS3],[$has_hdf5_ros3],[yes]) AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes]) -AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes]) AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes]) AX_SET_META([NC_HAS_QUANTIZE],[$enable_quantize],[yes]) AX_SET_META([NC_HAS_SZIP],[$enable_hdf5_szip],[yes]) diff --git a/include/netcdf_meta.h.in b/include/netcdf_meta.h.in index e1f7fbe1a..4d49f49b5 100644 --- a/include/netcdf_meta.h.in +++ b/include/netcdf_meta.h.in @@ -60,7 +60,6 @@ #define NC_RELAX_COORD_BOUND 1 /*!< Always allow 0 counts in parallel I/O. */ #define NC_DISPATCH_VERSION @NC_DISPATCH_VERSION@ /*!< Dispatch table version. */ #define NC_HAS_PAR_FILTERS @NC_HAS_PAR_FILTERS@ /* Parallel I/O with filter support. */ -#define NC_HAS_MULTIFILTERS @NC_HAS_MULTIFILTERS@ /*!< Nczarr support. */ #define NC_HAS_LOGGING @NC_HAS_LOGGING@ /*!< Logging support. */ #define NC_HAS_QUANTIZE @NC_HAS_QUANTIZE@ /*!< Quantization support. */ #define NC_HAS_ZSTD @NC_HAS_ZSTD@ /*!< Zstd support. */ diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index 2f769dd3d..a018fc297 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -150,10 +150,10 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne /* Construct the revised path */ ncbytesclear(buf); - ncbytescat(buf,"/"); - if(bucket == NULL) - {stat = NC_EURL; goto done;} - ncbytescat(buf,bucket); + if(bucket != NULL) { + ncbytescat(buf,"/"); + ncbytescat(buf,bucket); + } for(i=0;i> $HOME/.ncrc + echo "ncrc=ncrc1" >> $RCHOME/.ncrc echo "ncrcx=ncrcx" >> $RCHOME/.ncrc echo "ncrc=ncrc2" >> $RCHOME/.dodsrc echo "daprc=daprc" >> $RCHOME/.daprc @@ -138,7 +138,4 @@ mergecase3 # Test the .rc api rcapi1 -# Test the .rc api -rcapi1 - resetrc