mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-04-06 18:00:24 +08:00
Merge branch 'main' into encode.dmh
This commit is contained in:
Ward Fisher
GitHub
commit
8d51666d04
@ -1214,16 +1214,12 @@ ELSE() # No option specified
|
||||
SET(PLUGIN_INSTALL_DIR_SETTING "N.A.")
|
||||
ENDIF()
|
||||
|
||||
# See if we have libzip
|
||||
FIND_PACKAGE(Zip)
|
||||
|
||||
# Define a test flag for have zip library
|
||||
IF(Zip_FOUND)
|
||||
# Try to enable NCZarr zip support
|
||||
OPTION(ENABLE_NCZARR_ZIP "Enable NCZarr ZIP support." OFF)
|
||||
IF (ENABLE_NCZARR_ZIP)
|
||||
FIND_PACKAGE(Zip REQUIRED)
|
||||
INCLUDE_DIRECTORIES(${Zip_INCLUDE_DIRS})
|
||||
SET(ENABLE_NCZARR_ZIP TRUE)
|
||||
ELSE()
|
||||
SET(ENABLE_NCZARR_ZIP FALSE)
|
||||
ENDIF()
|
||||
ENDIF ()
|
||||
|
||||
# libdl is always available; built-in in Windows and OSX
|
||||
OPTION(ENABLE_PLUGINS "Enable dynamically loaded plugins (default on)." ON)
|
||||
@ -1284,19 +1280,26 @@ IF(NOT WIN32)
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
|
||||
# Options to Extend NCZarr support
|
||||
OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support." OFF)
|
||||
# Options for S3 Support
|
||||
OPTION(ENABLE_S3 "Enable S3 support." OFF)
|
||||
OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support; Deprecated in favor of ENABLE_S3" OFF)
|
||||
OPTION(ENABLE_NCZARR_S3_TESTS "Enable NCZarr S3 tests." OFF)
|
||||
|
||||
IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_NCZARR_S3)
|
||||
message(FATAL_ERROR "NCZarr S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=no")
|
||||
SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCARR S3 TESTS" FORCE)
|
||||
# ENABLE_NCZARR_S3 is now an alias for ENABLE_S3 (but...)
|
||||
if (NOT ENABLE_S3 AND ENABLE_NCZARR_S3)
|
||||
SET(ENABLE_S3 ON CACHE BOOL "NCARR S3" FORCE) # For back compatibility
|
||||
ENDIF()
|
||||
UNSET(ENABLE_NCZARR_S3)
|
||||
|
||||
IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_S3)
|
||||
message(FATAL_ERROR "S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=NO")
|
||||
SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCZARR S3 TESTS" FORCE)
|
||||
ENDIF()
|
||||
|
||||
# Note we check for the library after checking for enable_nczarr_s3
|
||||
# Note we check for the library after checking for enable_s3
|
||||
# because for some reason this screws up if we unconditionally test for sdk
|
||||
# and it is not available. Fix someday
|
||||
IF(ENABLE_NCZARR_S3)
|
||||
IF(ENABLE_S3)
|
||||
# See if aws-s3-sdk is available
|
||||
find_package(AWSSDK REQUIRED COMPONENTS s3;core)
|
||||
IF(AWSSDK_FOUND)
|
||||
@ -1311,9 +1314,9 @@ ELSE()
|
||||
ENDIF()
|
||||
|
||||
IF(NOT ENABLE_S3_SDK)
|
||||
IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS)
|
||||
message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_NCZARR_S3=NO")
|
||||
SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE)
|
||||
IF(ENABLE_S3)
|
||||
message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_S3=NO")
|
||||
SET(ENABLE_S3 OFF CACHE BOOL "S3 support" FORCE)
|
||||
SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
@ -1339,8 +1342,8 @@ IF(NOT FOUND_CURL)
|
||||
SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE)
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_NCZARR_S3)
|
||||
MESSAGE(FATAL_ERROR "NCZarr S3 support specified, CURL libraries are not found.")
|
||||
IF(ENABLE_S3)
|
||||
MESSAGE(FATAL_ERROR "S3 support specified, but CURL libraries are not found.")
|
||||
ENDIF()
|
||||
ENDIF(NOT FOUND_CURL)
|
||||
|
||||
@ -1411,7 +1414,14 @@ IF(ENABLE_TESTS)
|
||||
# Option to turn on unit testing. See
|
||||
# https://github.com/Unidata/netcdf-c/pull/1472 for more information.
|
||||
###
|
||||
OPTION(ENABLE_UNIT_TESTS "Run Unit Tests." OFF)
|
||||
OPTION(ENABLE_UNIT_TESTS "Run Unit Tests." ON)
|
||||
|
||||
###
|
||||
# Option to turn on performance testing.
|
||||
# See https://github.com/Unidata/netcdf-c/issues/2627 for more information.
|
||||
###
|
||||
OPTION(ENABLE_BENCHMARKS "Run benchmark Tests." OFF)
|
||||
|
||||
|
||||
###
|
||||
# End known-failures.
|
||||
@ -2005,7 +2015,7 @@ MACRO(GEN_m4 filename)
|
||||
IF(NOT EXISTS ${fallbackdest})
|
||||
MESSAGE(FATAL_ERROR "m4 is required to generate ${filename}.c. Please install m4 so that it is on the PATH and try again.")
|
||||
ELSE()
|
||||
SET(${dest} ${fallbackdest})
|
||||
SET(dest ${fallbackdest})
|
||||
ENDIF()
|
||||
ELSE()
|
||||
ADD_CUSTOM_COMMAND(
|
||||
@ -2364,6 +2374,9 @@ IF(ENABLE_TESTS)
|
||||
IF(ENABLE_EXAMPLES)
|
||||
ADD_SUBDIRECTORY(examples)
|
||||
ENDIF()
|
||||
IF(ENABLE_BENCHMARKS)
|
||||
ADD_SUBDIRECTORY(nc_perf)
|
||||
ENDIF(ENABLE_BENCHMARKS)
|
||||
IF(ENABLE_UNIT_TESTS)
|
||||
ADD_SUBDIRECTORY(unit_test)
|
||||
ENDIF(ENABLE_UNIT_TESTS)
|
||||
@ -2538,9 +2551,11 @@ is_enabled(ENABLE_ZERO_LENGTH_COORD_BOUND RELAX_COORD_BOUND)
|
||||
is_enabled(USE_CDF5 HAS_CDF5)
|
||||
is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
|
||||
is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
|
||||
is_enabled(ENABLE_NCZARR_S3 HAS_NCZARR_S3)
|
||||
is_enabled(ENABLE_S3 HAS_S3)
|
||||
is_enabled(ENABLE_S3_SDK HAS_S3_SDK)
|
||||
is_enabled(ENABLE_NCZARR HAS_NCZARR)
|
||||
is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS)
|
||||
is_enabled(ENABLE_NCZARR_ZIP HAS_NCZARR_ZIP)
|
||||
is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)
|
||||
is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
|
||||
is_enabled(ENABLE_QUANTIZE HAS_QUANTIZE)
|
||||
@ -2552,6 +2567,7 @@ is_enabled(HAVE_ZSTD HAS_ZSTD)
|
||||
is_enabled(HAVE_BLOSC HAS_BLOSC)
|
||||
is_enabled(HAVE_BZ2 HAS_BZ2)
|
||||
|
||||
|
||||
# Generate file from template.
|
||||
CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/libnetcdf.settings"
|
||||
@ -2604,6 +2620,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURREN
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/plugins/findplugin.sh @ONLY NEWLINE_STYLE LF)
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF)
|
||||
|
||||
IF(ENABLE_BENCHMARKS)
|
||||
IF(ENABLE_PARALLEL4)
|
||||
CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/nc_perf/run_par_bm_test.sh.in ${CMAKE_CURRENT_BINARY_DIR}/nc_perf/run_par_bm_test.sh @ONLY NEWLINE_STYLE LF)
|
||||
CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/nc_perf/run_gfs_test.sh.in ${CMAKE_CURRENT_BINARY_DIR}/nc_perf/run_gfs_test.sh @ONLY NEWLINE_STYLE LF)
|
||||
ENDIF(ENABLE_PARALLEL4)
|
||||
ENDIF(ENABLE_BENCHMARKS)
|
||||
|
||||
IF(ENABLE_TESTS)
|
||||
#####
|
||||
# Build ncdap_test|dap4_test/findtestserver[4].c
|
||||
|
||||
@ -8,8 +8,10 @@ This file contains a high-level description of this package's evolution. Release
|
||||
## 4.9.2 - TBD
|
||||
|
||||
* [Bug Fix] Update DAP code to enable CURLOPT_ACCEPT_ENCODING by default. See [Github #2630](https://github.com/Unidata/netcdf-c/pull/2630).
|
||||
* Update `nc-config` to remove inclusion from automatically-detected `nf-config` and `ncxx-config` files, as the wrong files could be included in the output. This is in support of [GitHub #2274](https://github.com/Unidata/netcdf-c/issues/2274).
|
||||
* [Bug Fix] Update H5FDhttp.[ch] to work with HDF5 version 1.14.0. See [Github #2615](https://github.com/Unidata/netcdf-c/pull/2615).
|
||||
* [Bug Fix] Fix byterange failures for certain URLs. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????).
|
||||
* [Bug Fix] Fix 'make distcheck' error in run_interop.sh. See [Github #2631](https://github.com/Unidata/netcdf-c/pull/2631).
|
||||
* [Enhancement] Update `nc-config` to remove inclusion from automatically-detected `nf-config` and `ncxx-config` files, as the wrong files could be included in the output. This is in support of [GitHub #2274](https://github.com/Unidata/netcdf-c/issues/2274).
|
||||
* [Enhancement] Update H5FDhttp.[ch] to work with HDF5 version 1.14.0. See [Github #2615](https://github.com/Unidata/netcdf-c/pull/2615).
|
||||
|
||||
## 4.9.1 - February 2, 2023
|
||||
|
||||
|
||||
@ -136,23 +136,23 @@ are set when opening a binary file on Windows. */
|
||||
/* if true, do remote tests */
|
||||
#cmakedefine ENABLE_DAP_REMOTE_TESTS 1
|
||||
|
||||
/* if true, enable S3 support */
|
||||
#cmakedefine ENABLE_S3 1
|
||||
|
||||
/* if true, S3 SDK is available */
|
||||
#cmakedefine ENABLE_S3_SDK 1
|
||||
|
||||
/* if true, enable NCZARR */
|
||||
#cmakedefine ENABLE_NCZARR 1
|
||||
|
||||
/* if true, enable nczarr filter support */
|
||||
#cmakedefine ENABLE_NCZARR_FILTERS 1
|
||||
|
||||
/* if true, enable S3 support */
|
||||
#cmakedefine ENABLE_NCZARR_S3 1
|
||||
|
||||
/* if true, enable nczarr zip support */
|
||||
#cmakedefine ENABLE_NCZARR_ZIP 1
|
||||
|
||||
/* if true, enable S3 testing*/
|
||||
#cmakedefine ENABLE_NCZARR_S3_TESTS 1
|
||||
|
||||
/* if true, S3 SDK is available */
|
||||
#cmakedefine ENABLE_S3_SDK 1
|
||||
/* if true, enable nczarr zip support */
|
||||
#cmakedefine ENABLE_NCZARR_ZIP 1
|
||||
|
||||
/* if true, Allow dynamically loaded plugins */
|
||||
#cmakedefine ENABLE_PLUGINS 1
|
||||
|
||||
88
configure.ac
88
configure.ac
@ -325,13 +325,13 @@ fi
|
||||
|
||||
# Does the user want to turn off unit tests (useful for test coverage
|
||||
# analysis).
|
||||
AC_MSG_CHECKING([if unit tests should be disabled])
|
||||
AC_MSG_CHECKING([if unit tests should be enabled])
|
||||
AC_ARG_ENABLE([unit-tests],
|
||||
[AS_HELP_STRING([--disable-unit-tests],
|
||||
[AS_HELP_STRING([--enable-unit-tests],
|
||||
[Disable tests in unit_test directory. Other tests still run.])])
|
||||
test "x$enable_unit_tests" = xno || enable_unit_tests=yes
|
||||
AC_MSG_RESULT($enable_unit_tests)
|
||||
AM_CONDITIONAL([BUILD_UNIT_TESTS], [test "x$enable_unit_tests" = xyes])
|
||||
AM_CONDITIONAL(BUILD_UNIT_TESTS, [test "x$enable_unit_tests" = xyes])
|
||||
|
||||
# Does the user require dynamic loading?
|
||||
# This is only for those hdf5 installs that support it.
|
||||
@ -565,7 +565,7 @@ if test "x$enable_libxml2" = xyes; then
|
||||
if test -z "$NC_XML2_CONFIG"; then
|
||||
AC_MSG_ERROR([Cannot find xml2-config utility. Either install the libxml2 development package, or re-run configure with --disable-libxml2 to use the bundled xml2 parser])
|
||||
fi
|
||||
# We can optionally use libxml2 for DAP4, if available
|
||||
# We can optionally use libxml2 for DAP4
|
||||
AC_CHECK_LIB([xml2],[xmlReadMemory],[have_libxml2=yes],[have_libxml2=no])
|
||||
if test "x$have_libxml2" = "xyes" ; then
|
||||
AC_SEARCH_LIBS([xmlReadMemory],[xml2 xml2.dll cygxml2.dll], [],[])
|
||||
@ -779,64 +779,63 @@ AC_MSG_RESULT([${have_sz}])
|
||||
##########
|
||||
|
||||
# See if we have libzip for NCZarr
|
||||
AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
|
||||
if test "x$have_zip" = "xyes" ; then
|
||||
AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [], [])
|
||||
fi
|
||||
AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll],[have_zip=yes],[have_zip=no])
|
||||
AC_MSG_CHECKING([whether libzip library is available])
|
||||
AC_MSG_RESULT([${have_zip}])
|
||||
|
||||
enable_nczarr_zip=${have_zip} # alias
|
||||
|
||||
if test "x$enable_nczarr" = xno ; then
|
||||
enable_nczarr_zip=no
|
||||
enable_nczarr_zip=no
|
||||
fi
|
||||
|
||||
|
||||
AC_MSG_CHECKING([whether nczarr zip support is enabled])
|
||||
AC_MSG_RESULT([${enable_nczarr_zip}])
|
||||
|
||||
if test "x$enable_nczarr_zip" = xyes ; then
|
||||
AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
|
||||
fi
|
||||
AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
|
||||
|
||||
# Check for enabling of S3 support
|
||||
AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled])
|
||||
AC_MSG_CHECKING([whether netcdf S3 support should be enabled])
|
||||
AC_ARG_ENABLE([s3],
|
||||
[AS_HELP_STRING([--enable-s3],
|
||||
[enable netcdf S3 support])])
|
||||
test "x$enable_s3" = xyes || enable_s3=no
|
||||
AC_MSG_RESULT($enable_s3)
|
||||
|
||||
# --enable-nczarr-s3 is synonym for --enable-s3 (but...)
|
||||
AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled])
|
||||
AC_ARG_ENABLE([nczarr-s3],
|
||||
[AS_HELP_STRING([--enable-nczarr-s3],
|
||||
[enable netcdf zarr S3 support; make sure to set LDFLAGS])])
|
||||
test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no
|
||||
if test "x$enable_nczarr" = xno ; then
|
||||
enable_nczarr_s3=no
|
||||
[enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])])
|
||||
# Set enable_s3 instead of enable_nczarr_s3
|
||||
if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes ; then
|
||||
enable_s3=yes # back compatibility
|
||||
fi
|
||||
AC_MSG_RESULT($enable_nczarr_s3)
|
||||
unset enable_nczarr_s3
|
||||
AC_MSG_RESULT($enable_s3)
|
||||
|
||||
# Note we check for the library after checking for enable_nczarr_s3
|
||||
# because for some reason this screws up if we unconditionally test for sdk
|
||||
# Note we check for the library after checking for enable_s3
|
||||
# because for some reason this fails if we unconditionally test for sdk
|
||||
# and it is not available. Fix someday
|
||||
enable_s3_sdk=no
|
||||
if test "x$enable_nczarr_s3" = xyes ; then
|
||||
if test "x$enable_s3" = xyes ; then
|
||||
# See if we have the s3 aws library
|
||||
# Check for the AWS S3 SDK library
|
||||
AC_LANG_PUSH([C++])
|
||||
AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [enable_s3_sdk=yes],[enable_s3_sdk=no])
|
||||
AC_LANG_POP
|
||||
else
|
||||
enable_s3_sdk=no
|
||||
fi
|
||||
|
||||
AC_MSG_CHECKING([whether AWS S3 SDK library is available])
|
||||
AC_MSG_RESULT([$enable_s3_sdk])
|
||||
|
||||
if test "x$enable_s3_sdk" = xno ; then
|
||||
AC_MSG_WARN([No S3 library available; disabling S3 support])
|
||||
enable_nczarr_s3=no
|
||||
AC_MSG_WARN([No S3 library available => S3 support disabled])
|
||||
enable_S3=no
|
||||
fi
|
||||
|
||||
if test "x$enable_s3_sdk" = xyes ; then
|
||||
AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
|
||||
fi
|
||||
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
|
||||
|
||||
# Check for enabling S3 testing
|
||||
AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
|
||||
AC_ARG_ENABLE([nczarr-s3-tests],
|
||||
@ -846,20 +845,22 @@ test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no
|
||||
AC_MSG_RESULT($enable_nczarr_s3_tests)
|
||||
|
||||
# Disable S3 tests if S3 support is disabled
|
||||
if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then
|
||||
AC_MSG_ERROR([NCZarr S3 support is disabled; please remove option --enable-nczarr-s3-tests])
|
||||
if test "x$enable_nczarr_s3_tests" = xyes ; then
|
||||
if test "x$enable_s3" = xno ; then
|
||||
AC_MSG_ERROR([S3 support is disabled => no testing])
|
||||
enable_nczarr_s3_tests=no
|
||||
fi
|
||||
fi
|
||||
|
||||
if test "x$enable_nczarr_s3" = xyes ; then
|
||||
AC_DEFINE([ENABLE_NCZARR_S3], [1], [if true, build libnczarr with S3 support enabled])
|
||||
if test "x$enable_s3" = xyes ; then
|
||||
AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled])
|
||||
fi
|
||||
AM_CONDITIONAL(ENABLE_NCZARR_S3, [test "x$enable_nczarr_s3" = xyes])
|
||||
|
||||
if test "x$enable_nczarr_s3_tests" = xyes ; then
|
||||
AC_DEFINE([ENABLE_NCZARR_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled])
|
||||
fi
|
||||
AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes])
|
||||
if test "x$enable_s3_sdk" = xyes ; then
|
||||
AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
|
||||
fi
|
||||
|
||||
if test "x$enable_nczarr_s3_tests" = xyes ; then
|
||||
AC_MSG_WARN([*** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***])
|
||||
@ -1193,7 +1194,7 @@ AC_CHECK_HEADERS([sys/param.h])
|
||||
AC_CHECK_HEADERS([libgen.h])
|
||||
#AC_CHECK_HEADERS([locale.h])
|
||||
AC_HEADER_STDC
|
||||
AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h])
|
||||
AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h ctype.h])
|
||||
|
||||
# Do sys/resource.h separately
|
||||
#AC_CHECK_HEADERS([sys/resource.h],[havesysresource=1],[havesysresource=0])
|
||||
@ -1224,7 +1225,7 @@ if test "x$enable_hdf5" = "xno" ; then
|
||||
enable_dap4=no
|
||||
fi
|
||||
|
||||
if test "x$ISOSX" = xyes && "x$have_libxml2" = xno ; then
|
||||
if test "x$ISOSX" = xyes && test "x$have_libxml2" = xno ; then
|
||||
AC_MSG_ERROR([Error: OSX requires libxml2 => --disable-dap4.])
|
||||
enable_dap4=no
|
||||
fi
|
||||
@ -1286,6 +1287,7 @@ AC_ARG_ENABLE([byterange],
|
||||
[allow byte-range I/O])])
|
||||
test "x$enable_byterange" = xno || enable_byterange=yes
|
||||
AC_MSG_RESULT($enable_byterange)
|
||||
|
||||
# Need curl for byte ranges
|
||||
if test "x$found_curl" = xno && test "x$enable_byterange" = xyes ; then
|
||||
AC_MSG_ERROR([curl required for byte range support. Install curl or build without --enable-byterange.])
|
||||
@ -1824,8 +1826,12 @@ AM_CONDITIONAL(ENABLE_METADATA_PERF, [test x$enable_metadata_perf = xyes])
|
||||
AM_CONDITIONAL(ENABLE_BYTERANGE, [test "x$enable_byterange" = xyes])
|
||||
AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes])
|
||||
AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
|
||||
AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
|
||||
# We need to simplify the set of S3 and Zarr flag combinations
|
||||
AM_CONDITIONAL(ENABLE_S3, [test "x$enable_s3" = xyes])
|
||||
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
|
||||
AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
|
||||
AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes])
|
||||
AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
|
||||
AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
|
||||
AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes])
|
||||
AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes])
|
||||
@ -1929,9 +1935,10 @@ AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill])
|
||||
AC_SUBST(HAS_BYTERANGE,[$enable_byterange])
|
||||
AC_SUBST(RELAX_COORD_BOUND,[yes])
|
||||
AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters])
|
||||
AC_SUBST(HAS_NCZARR_S3,[$enable_nczarr_s3])
|
||||
AC_SUBST(HAS_S3,[$enable_s3])
|
||||
AC_SUBST(HAS_NCZARR,[$enable_nczarr])
|
||||
AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
|
||||
AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip])
|
||||
AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
|
||||
AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
|
||||
AC_SUBST(HAS_QUANTIZE,[$enable_quantize])
|
||||
@ -2058,6 +2065,7 @@ AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes])
|
||||
AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes])
|
||||
AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes])
|
||||
AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
|
||||
AX_SET_META([NC_HAS_S3],[$enable_s3],[yes])
|
||||
AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes])
|
||||
AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes])
|
||||
AX_SET_META([NC_HAS_QUANTIZE],[$enable_quantize],[yes])
|
||||
|
||||
@ -61,7 +61,7 @@ EXTRA_DIST = test_parse.sh test_meta.sh test_data.sh \
|
||||
CLEANFILES = *.exe
|
||||
# This should only be left behind if using parallel io
|
||||
CLEANFILES += tmp_*
|
||||
CLEANFILES += ${execdir}/baselinethredds/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds
|
||||
CLEANFILES += ${srcdir}/baselinethredds/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds
|
||||
|
||||
DISTCLEANFILES = findtestserver4.c pingurl4.c
|
||||
|
||||
|
||||
@ -288,7 +288,7 @@ typedef struct NC_FILE_INFO
|
||||
{
|
||||
NC_OBJ hdr;
|
||||
NC *controller; /**< Pointer to containing NC. */
|
||||
#ifdef USE_PARALLEL4
|
||||
#ifdef USE_PARALLEL
|
||||
MPI_Comm comm; /**< Copy of MPI Communicator used to open the file. */
|
||||
MPI_Info info; /**< Copy of MPI Information Object used to open the file. */
|
||||
#endif
|
||||
|
||||
@ -66,7 +66,7 @@
|
||||
#define NC_HAS_QUANTIZE @NC_HAS_QUANTIZE@ /*!< Quantization support. */
|
||||
#define NC_HAS_ZSTD @NC_HAS_ZSTD@ /*!< Zstd support. */
|
||||
#define NC_HAS_BENCHMARKS @NC_HAS_BENCHMARKS@ /*!< Benchmarks. */
|
||||
|
||||
#define NC_HAS_S3 @NC_HAS_S3@ /*!< Amazon S3 Support. */
|
||||
#define NC_HAS_BLOSC @NC_HAS_BLOSC@ /*!< Blosc Support. */
|
||||
#define NC_HAS_BZ2 @NC_HAS_BZ2@ /*!< bzip2 support */
|
||||
|
||||
|
||||
@ -494,6 +494,7 @@ parseVlenField(NCD4parser* parser, NCD4node* container, ncxml_t xml, NCD4node**
|
||||
if((ret = parseVariable(parser,container,x,&field)))
|
||||
goto done;
|
||||
}
|
||||
if(field == NULL) {ret = NC_EBADTYPE; goto done;}
|
||||
if(fieldp) *fieldp = field;
|
||||
done:
|
||||
return THROW(ret);
|
||||
|
||||
@ -135,7 +135,7 @@ nc_http_reset(NC_HTTP_STATE* state)
|
||||
if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;}
|
||||
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0L));
|
||||
if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;}
|
||||
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_PUT, 0L));
|
||||
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_UPLOAD, 0L));
|
||||
if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;}
|
||||
cstat = curl_easy_setopt(state->curl, CURLOPT_CUSTOMREQUEST, NULL);
|
||||
if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;}
|
||||
@ -168,7 +168,7 @@ nc_http_set_method(NC_HTTP_STATE* state, HTTPMETHOD method)
|
||||
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1L));
|
||||
break;
|
||||
case HTTPPUT:
|
||||
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_PUT, 1L));
|
||||
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_UPLOAD, 1L));
|
||||
break;
|
||||
case HTTPDELETE:
|
||||
cstat = curl_easy_setopt(state->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
|
||||
@ -258,7 +258,7 @@ nc_http_write(NC_HTTP_STATE* state, const char* objecturl, NCbytes* payload)
|
||||
Trace("write");
|
||||
|
||||
if((stat = nc_http_set_payload(state,ncbyteslength(payload),ncbytescontents(payload)))) goto fail;
|
||||
if((stat = nc_http_set_method(state,HTTPPUT)));
|
||||
if((stat = nc_http_set_method(state,HTTPPUT))) goto fail;
|
||||
if((stat = setupconn(state,objecturl))) goto fail;
|
||||
if((stat = execute(state)))
|
||||
goto done;
|
||||
|
||||
@ -484,7 +484,8 @@ rccompile(const char* filepath)
|
||||
nclog(NCLOGERR, "Malformed [url] in %s entry: %s",filepath,line);
|
||||
continue;
|
||||
}
|
||||
{ NCURI* newuri = NULL;
|
||||
if(NC_iss3(uri)) {
|
||||
NCURI* newuri = NULL;
|
||||
/* Rebuild the url to S3 "path" format */
|
||||
nullfree(bucket);
|
||||
if((ret = NC_s3urlrebuild(uri,&newuri,&bucket,NULL))) goto done;
|
||||
|
||||
@ -76,41 +76,48 @@ NC_s3urlrebuild(NCURI* url, NCURI** newurlp, char** bucketp, char** outregionp)
|
||||
if((stat = NC_split_delim(url->path,'/',pathsegments))) goto done;
|
||||
|
||||
/* Distinguish path-style from virtual-host style from s3: and from other.
|
||||
Virtual: https://bucket-name.s3.Region.amazonaws.com/<path>
|
||||
Path: https://s3.Region.amazonaws.com/bucket-name/<path>
|
||||
S3: s3://bucket-name/<path>
|
||||
Other: https://<host>/bucketname/<path>
|
||||
Virtual: https://bucket-name.s3.Region.amazonaws.com/<path> (1)
|
||||
or: https://bucket-name.s3.amazonaws.com/<path> -- region defaults to us-east-1 (2)
|
||||
Path: https://s3.Region.amazonaws.com/bucket-name/<path> (3)
|
||||
or: https://s3.amazonaws.com/bucket-name/<path> -- region defaults to us-east-1 (4)
|
||||
S3: s3://bucket-name/<path> (5)
|
||||
Other: https://<host>/bucketname/<path> (6)
|
||||
*/
|
||||
if(url->host == NULL || strlen(url->host) == 0)
|
||||
{stat = NC_EURL; goto done;}
|
||||
if(strcmp(url->protocol,"s3")==0 && nclistlength(hostsegments)==1) {
|
||||
bucket = strdup(url->host);
|
||||
region = NULL; /* unknown at this point */
|
||||
if(strcmp(url->protocol,"s3")==0 && nclistlength(hostsegments)==1) { /* Case (5) */
|
||||
bucket = nclistremove(hostsegments,0);
|
||||
/* region unknown at this point */
|
||||
} else if(endswith(url->host,AWSHOST)) { /* Virtual or path */
|
||||
/* If we find a bucket as part of the host, then remove it */
|
||||
switch (nclistlength(hostsegments)) {
|
||||
default: stat = NC_EURL; goto done;
|
||||
case 4:
|
||||
if(strcasecmp(nclistget(hostsegments,0),"s3")!=0)
|
||||
{stat = NC_EURL; goto done;}
|
||||
region = strdup(nclistget(hostsegments,1));
|
||||
if(nclistlength(pathsegments) > 0)
|
||||
bucket = nclistremove(pathsegments,0);
|
||||
case 3: /* Case (4) */
|
||||
/* region unknown at this point */
|
||||
/* bucket unknown at this point */
|
||||
break;
|
||||
case 5:
|
||||
case 4: /* Case (2) or (3) */
|
||||
if(strcasecmp(nclistget(hostsegments,1),"s3")==0) { /* Case (2) */
|
||||
/* region unknown at this point */
|
||||
bucket = nclistremove(hostsegments,0); /* Note removal */
|
||||
} else if(strcasecmp(nclistget(hostsegments,0),"s3")==0) { /* Case (3) */
|
||||
region = strdup(nclistget(hostsegments,1));
|
||||
/* bucket unknown at this point */
|
||||
} else /* ! (2) and !(3) => error */
|
||||
{stat = NC_EURL; goto done;}
|
||||
break;
|
||||
case 5: /* Case (1) */
|
||||
if(strcasecmp(nclistget(hostsegments,1),"s3")!=0)
|
||||
{stat = NC_EURL; goto done;}
|
||||
region = strdup(nclistget(hostsegments,2));
|
||||
bucket = strdup(nclistget(hostsegments,0));
|
||||
bucket = strdup(nclistremove(hostsegments,0));
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
} else { /* Presume Case (6) */
|
||||
if((host = strdup(url->host))==NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
/* region is unknown */
|
||||
region = NULL;
|
||||
/* bucket is assumed to be start of the path */
|
||||
if(nclistlength(pathsegments) > 0)
|
||||
bucket = nclistremove(pathsegments,0);
|
||||
/* bucket is unknown */
|
||||
}
|
||||
/* If region is null, use default */
|
||||
if(region == NULL) {
|
||||
@ -119,6 +126,14 @@ NC_s3urlrebuild(NCURI* url, NCURI** newurlp, char** bucketp, char** outregionp)
|
||||
if((stat = NC_getdefaults3region(url,®ion0))) goto done;
|
||||
region = strdup(region0);
|
||||
}
|
||||
/* if bucket is null, use first segment of the path, if any */
|
||||
if(bucket == NULL) {
|
||||
if(nclistlength(pathsegments) > 0)
|
||||
bucket = nclistremove(pathsegments,0);
|
||||
}
|
||||
assert(bucket != NULL);
|
||||
/* bucket may still be null */
|
||||
|
||||
if(host == NULL) { /* Construct the revised host */
|
||||
ncbytescat(buf,"s3.");
|
||||
ncbytescat(buf,region);
|
||||
|
||||
@ -635,16 +635,20 @@ s3commonprefixes(Aws::Vector<Aws::S3::Model::CommonPrefix> list, char*** keysp)
|
||||
if(!stat) {
|
||||
i = 0;
|
||||
for (auto const &s3prefix : list) {
|
||||
char* p; char* p1;
|
||||
size_t len;
|
||||
char* p; const char* px; char* p1;
|
||||
size_t len, alloc;
|
||||
const Aws::String& prefix = s3prefix.GetPrefix();
|
||||
len = prefix.length();
|
||||
if((p = (char*) malloc(len+1+1))==NULL) /* for nul + leading '/' */
|
||||
alloc = len + 1 + 1; /* for nul + leading '/' */
|
||||
if((p = (char*) malloc(alloc))==NULL)
|
||||
stat = NC_ENOMEM;
|
||||
if(stat == NC_NOERR) {
|
||||
if(*p == '/') {p1 = p;} else {*p = '/'; p1 = p+1;}
|
||||
memcpy(p1,prefix.c_str(),len);
|
||||
p1[len] = '\0';
|
||||
px = prefix.c_str();
|
||||
p1 = p;
|
||||
if(*px != '/') *p1++ = '/';
|
||||
memcpy(p1,px,len);
|
||||
p1 += len;
|
||||
*p1 = '\0';
|
||||
keys[i++] = p;
|
||||
}
|
||||
}
|
||||
|
||||
@ -27,10 +27,12 @@
|
||||
#ifndef H5FDHTTP_H
|
||||
#define H5FDHTTP_H
|
||||
|
||||
#define S3_REGION_DEFAULT "us-east-1"
|
||||
|
||||
#include "H5Ipublic.h"
|
||||
|
||||
#if H5_VERSION_GE(1,14,0)
|
||||
#define H5_VFD_HTTP ((H5FD_class_value_t)(H5_VFD_MAX - 2))
|
||||
#define H5_VFD_HTTP ((H5FD_class_value_t)(514))
|
||||
#define H5FD_HTTP (H5FDperform_init(H5FD_http_init))
|
||||
#else
|
||||
#define H5FD_HTTP (H5FD_http_init())
|
||||
|
||||
@ -44,7 +44,7 @@ nch5breakpoint(int err)
|
||||
# endif
|
||||
#endif
|
||||
#endif
|
||||
return err;
|
||||
return ncbreakpoint(err);
|
||||
}
|
||||
|
||||
int
|
||||
|
||||
@ -871,12 +871,18 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
|
||||
char* awsregion0 = NULL;
|
||||
#ifdef ENABLE_HDF5_ROS3
|
||||
H5FD_ros3_fapl_t fa;
|
||||
char* hostport = NULL;
|
||||
const char* profile0 = NULL;
|
||||
const char* awsaccessid0 = NULL;
|
||||
const char* awssecretkey0 = NULL;
|
||||
int iss3 = NC_iss3(h5->uri);
|
||||
|
||||
if(NC_iss3(h5->uri)) {
|
||||
fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION;
|
||||
fa.authenticate = (hbool_t)0;
|
||||
fa.aws_region[0] = '\0';
|
||||
fa.secret_id[0] = '\0';
|
||||
fa.secret_key[0] = '\0';
|
||||
|
||||
if(iss3) {
|
||||
/* Rebuild the URL */
|
||||
NCURI* newuri = NULL;
|
||||
if((retval = NC_s3urlrebuild(h5->uri,&newuri,NULL,&awsregion0))) goto exit;
|
||||
@ -884,38 +890,35 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
|
||||
{retval = NC_EURL; goto exit;}
|
||||
ncurifree(h5->uri);
|
||||
h5->uri = newuri;
|
||||
}
|
||||
hostport = NC_combinehostport(h5->uri);
|
||||
if((retval = NC_getactives3profile(h5->uri,&profile0)))
|
||||
BAIL(retval);
|
||||
|
||||
fa.version = 1;
|
||||
fa.aws_region[0] = '\0';
|
||||
fa.secret_id[0] = '\0';
|
||||
fa.secret_key[0] = '\0';
|
||||
if((retval = NC_s3profilelookup(profile0,AWS_ACCESS_KEY_ID,&awsaccessid0)))
|
||||
BAIL(retval);
|
||||
if((retval = NC_s3profilelookup(profile0,AWS_SECRET_ACCESS_KEY,&awssecretkey0)))
|
||||
BAIL(retval);
|
||||
if(awsaccessid0 == NULL || awssecretkey0 == NULL) {
|
||||
/* default, non-authenticating, "anonymous" fapl configuration */
|
||||
fa.authenticate = (hbool_t)0;
|
||||
} else {
|
||||
fa.authenticate = (hbool_t)1;
|
||||
if(awsregion0)
|
||||
if((retval = NC_getactives3profile(h5->uri,&profile0)))
|
||||
BAIL(retval);
|
||||
if((retval = NC_s3profilelookup(profile0,AWS_ACCESS_KEY_ID,&awsaccessid0)))
|
||||
BAIL(retval);
|
||||
if((retval = NC_s3profilelookup(profile0,AWS_SECRET_ACCESS_KEY,&awssecretkey0)))
|
||||
BAIL(retval);
|
||||
if(awsregion0 == NULL)
|
||||
awsregion0 = strdup(S3_REGION_DEFAULT);
|
||||
if(awsaccessid0 == NULL || awssecretkey0 == NULL ) {
|
||||
/* default, non-authenticating, "anonymous" fapl configuration */
|
||||
fa.authenticate = (hbool_t)0;
|
||||
} else {
|
||||
fa.authenticate = (hbool_t)1;
|
||||
assert(awsregion0 != NULL && strlen(awsregion0) > 0);
|
||||
assert(awsaccessid0 != NULL && strlen(awsaccessid0) > 0);
|
||||
assert(awssecretkey0 != NULL && strlen(awssecretkey0) > 0);
|
||||
strlcat(fa.aws_region,awsregion0,H5FD_ROS3_MAX_REGION_LEN);
|
||||
strlcat(fa.secret_id, awsaccessid0, H5FD_ROS3_MAX_SECRET_ID_LEN);
|
||||
strlcat(fa.secret_key, awssecretkey0, H5FD_ROS3_MAX_SECRET_KEY_LEN);
|
||||
}
|
||||
nullfree(hostport);
|
||||
/* create and set fapl entry */
|
||||
if(H5Pset_fapl_ros3(fapl_id, &fa) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
#else
|
||||
/* Configure FAPL to use our byte-range file driver */
|
||||
if (H5Pset_fapl_http(fapl_id) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
strlcat(fa.secret_id, awsaccessid0, H5FD_ROS3_MAX_SECRET_ID_LEN);
|
||||
strlcat(fa.secret_key, awssecretkey0, H5FD_ROS3_MAX_SECRET_KEY_LEN);
|
||||
}
|
||||
/* create and set fapl entry */
|
||||
if(H5Pset_fapl_ros3(fapl_id, &fa) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
} else
|
||||
#endif /*ENABLE_ROS3*/
|
||||
{/* Configure FAPL to use our byte-range file driver */
|
||||
if (H5Pset_fapl_http(fapl_id) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
}
|
||||
/* Open the HDF5 file. */
|
||||
if ((h5->hdfid = nc4_H5Fopen((newpath?newpath:path), flags, fapl_id)) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
|
||||
@ -51,7 +51,7 @@ const char*
|
||||
ncxml_name(ncxml_t xml0)
|
||||
{
|
||||
xmlNode* xml = (xmlNode*)xml0;
|
||||
return (xml?xml->name:(const char*)NULL);
|
||||
return (const char*)(xml?xml->name:NULL);
|
||||
}
|
||||
|
||||
char*
|
||||
|
||||
@ -19,7 +19,7 @@ struct ZUTEST* zutest = NULL;
|
||||
int
|
||||
zbreakpoint(int err)
|
||||
{
|
||||
return err;
|
||||
return ncbreakpoint(err);
|
||||
}
|
||||
|
||||
int
|
||||
|
||||
@ -39,8 +39,11 @@ PnetCDF Support: @HAS_PNETCDF@
|
||||
DAP2 Support: @HAS_DAP2@
|
||||
DAP4 Support: @HAS_DAP4@
|
||||
Byte-Range Support: @HAS_BYTERANGE@
|
||||
|
||||
S3 Support: @HAS_S3@
|
||||
|
||||
NCZarr Support: @HAS_NCZARR@
|
||||
NCZarr S3 Support: @HAS_NCZARR_S3@
|
||||
NCZarr Zip Support: @HAS_NCZARR_ZIP@
|
||||
|
||||
Diskless Support: @HAS_DISKLESS@
|
||||
MMap Support: @HAS_MMAP@
|
||||
@ -48,7 +51,6 @@ JNA Support: @HAS_JNA@
|
||||
ERANGE Fill Support: @HAS_ERANGE_FILL@
|
||||
Relaxed Boundary Check: @RELAX_COORD_BOUND@
|
||||
|
||||
NCZarr Support: @HAS_NCZARR@
|
||||
Multi-Filter Support: @HAS_MULTIFILTERS@
|
||||
Quantization: @HAS_QUANTIZE@
|
||||
Logging: @HAS_LOGGING@
|
||||
|
||||
10
libsrc/var.c
10
libsrc/var.c
@ -175,10 +175,12 @@ dup_NC_var(const NC_var *rvarp)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
(void) memcpy(varp->shape, rvarp->shape,
|
||||
rvarp->ndims * sizeof(size_t));
|
||||
(void) memcpy(varp->dsizes, rvarp->dsizes,
|
||||
rvarp->ndims * sizeof(off_t));
|
||||
if(rvarp->shape != NULL)
|
||||
(void) memcpy(varp->shape, rvarp->shape,
|
||||
rvarp->ndims * sizeof(size_t));
|
||||
if(rvarp->dsizes != NULL)
|
||||
(void) memcpy(varp->dsizes, rvarp->dsizes,
|
||||
rvarp->ndims * sizeof(off_t));
|
||||
varp->xsz = rvarp->xsz;
|
||||
varp->len = rvarp->len;
|
||||
varp->begin = rvarp->begin;
|
||||
|
||||
@ -24,8 +24,10 @@ add_bin_test(nc_perf tst_files3 tst_utils.c)
|
||||
add_bin_test(nc_perf tst_mem tst_utils.c)
|
||||
add_bin_test(nc_perf tst_wrf_reads tst_utils.c)
|
||||
add_bin_test(nc_perf tst_attsperf tst_utils.c)
|
||||
add_bin_test(nc_perf tst_bm_rando tst_utils.c)
|
||||
add_bin_test(nc_perf tst_compress tst_utils.c)
|
||||
|
||||
add_sh_test(nc_perf run_knmi_bm)
|
||||
#add_sh_test(nc_perf run_knmi_bm)
|
||||
add_sh_test(nc_perf perftest)
|
||||
add_sh_test(nc_perf run_tst_chunks)
|
||||
add_sh_test(nc_perf run_bm_elena)
|
||||
@ -36,9 +38,11 @@ add_sh_test(nc_perf run_bm_test2)
|
||||
|
||||
# This will run a parallel I/O benchmark for parallel builds.
|
||||
IF(TEST_PARALLEL4)
|
||||
add_bin_test(nc_perf tst_compress_par tst_utils.c)
|
||||
add_sh_test(nc_perf run_par_bm_test)
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
ENDIF(TEST_PARALLEL4)
|
||||
ENDIF(BUILD_UTILITIES)
|
||||
|
||||
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh)
|
||||
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE)
|
||||
|
||||
ADD_EXTRA_DIST(run_par_bm_test.sh.in run_knmi_bm.sh CMakeLists.txt
|
||||
perftest.sh run_bm_test1.sh run_bm_test2.sh)
|
||||
|
||||
0
nc_perf/run_gfs_test.sh.in
Normal file → Executable file
0
nc_perf/run_gfs_test.sh.in
Normal file → Executable file
0
nc_perf/run_par_bm_test.sh.in
Normal file → Executable file
0
nc_perf/run_par_bm_test.sh.in
Normal file → Executable file
@ -23,7 +23,6 @@
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include <netcdf.h>
|
||||
#include <netcdf_par.h>
|
||||
#include <netcdf_meta.h>
|
||||
|
||||
#define TEST_NAME "tst_compress"
|
||||
|
||||
@ -116,7 +116,7 @@ ref_tst_diskless3_create.cdl ref_tst_diskless3_open.cdl \
|
||||
run_inmemory.sh run_mmap.sh f03tst_open_mem.nc test_byterange.sh \
|
||||
ref_tst_http_nc3.cdl ref_tst_http_nc4a.cdl ref_tst_http_nc4b.cdl \
|
||||
ref_tst_http_nc4c.cdl CMakeLists.txt run_pnetcdf_tests.sh.in \
|
||||
tst_misc.c tst_norm.c
|
||||
tst_misc.c tst_norm.c ref_tst_http_nc4e.cdl ref_tst_http_nc4f.cdl
|
||||
|
||||
# These files are created by the tests.
|
||||
CLEANFILES = nc_test_*.nc tst_*.nc t_nc.nc large_files.nc \
|
||||
|
||||
331
nc_test/ref_tst_http_nc4e.cdl
Normal file
331
nc_test/ref_tst_http_nc4e.cdl
Normal file
@ -0,0 +1,331 @@
|
||||
netcdf OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562 {
|
||||
dimensions:
|
||||
y = 10848 ;
|
||||
x = 10848 ;
|
||||
number_of_time_bounds = 2 ;
|
||||
band = 1 ;
|
||||
number_of_image_bounds = 2 ;
|
||||
num_star_looks = 24 ;
|
||||
variables:
|
||||
short Rad(y, x) ;
|
||||
Rad:_FillValue = 1023s ;
|
||||
Rad:long_name = "ABI L1b Radiances" ;
|
||||
Rad:standard_name = "toa_outgoing_radiance_per_unit_wavelength" ;
|
||||
Rad:_Unsigned = "true" ;
|
||||
Rad:sensor_band_bit_depth = 10b ;
|
||||
Rad:valid_range = 0s, 1022s ;
|
||||
Rad:scale_factor = 0.8121064f ;
|
||||
Rad:add_offset = -25.93665f ;
|
||||
Rad:units = "W m-2 sr-1 um-1" ;
|
||||
Rad:resolution = "y: 0.000028 rad x: 0.000028 rad" ;
|
||||
Rad:coordinates = "band_id band_wavelength t y x" ;
|
||||
Rad:grid_mapping = "goes_imager_projection" ;
|
||||
Rad:cell_methods = "t: point area: point" ;
|
||||
Rad:ancillary_variables = "DQF" ;
|
||||
byte DQF(y, x) ;
|
||||
DQF:_FillValue = -1b ;
|
||||
DQF:long_name = "ABI L1b Radiances data quality flags" ;
|
||||
DQF:standard_name = "status_flag" ;
|
||||
DQF:_Unsigned = "true" ;
|
||||
DQF:valid_range = 0b, 4b ;
|
||||
DQF:units = "1" ;
|
||||
DQF:coordinates = "band_id band_wavelength t y x" ;
|
||||
DQF:grid_mapping = "goes_imager_projection" ;
|
||||
DQF:cell_methods = "t: point area: point" ;
|
||||
DQF:flag_values = 0b, 1b, 2b, 3b, 4b ;
|
||||
DQF:flag_meanings = "good_pixel_qf conditionally_usable_pixel_qf out_of_range_pixel_qf no_value_pixel_qf focal_plane_temperature_threshold_exceeded_qf" ;
|
||||
DQF:number_of_qf_values = 5b ;
|
||||
DQF:percent_good_pixel_qf = 0.998999f ;
|
||||
DQF:percent_conditionally_usable_pixel_qf = 5.e-07f ;
|
||||
DQF:percent_out_of_range_pixel_qf = 0.0009925f ;
|
||||
DQF:percent_no_value_pixel_qf = 8.e-06f ;
|
||||
DQF:percent_focal_plane_temperature_threshold_exceeded_qf = 0.f ;
|
||||
double t ;
|
||||
t:long_name = "J2000 epoch mid-point between the start and end image scan in seconds" ;
|
||||
t:standard_name = "time" ;
|
||||
t:units = "seconds since 2000-01-01 12:00:00" ;
|
||||
t:axis = "T" ;
|
||||
t:bounds = "time_bounds" ;
|
||||
short y(y) ;
|
||||
y:scale_factor = -2.8e-05f ;
|
||||
y:add_offset = 0.151858f ;
|
||||
y:units = "rad" ;
|
||||
y:axis = "Y" ;
|
||||
y:long_name = "GOES fixed grid projection y-coordinate" ;
|
||||
y:standard_name = "projection_y_coordinate" ;
|
||||
short x(x) ;
|
||||
x:scale_factor = 2.8e-05f ;
|
||||
x:add_offset = -0.151858f ;
|
||||
x:units = "rad" ;
|
||||
x:axis = "X" ;
|
||||
x:long_name = "GOES fixed grid projection x-coordinate" ;
|
||||
x:standard_name = "projection_x_coordinate" ;
|
||||
double time_bounds(number_of_time_bounds) ;
|
||||
time_bounds:long_name = "Scan start and end times in seconds since epoch (2000-01-01 12:00:00)" ;
|
||||
int goes_imager_projection ;
|
||||
goes_imager_projection:long_name = "GOES-R ABI fixed grid projection" ;
|
||||
goes_imager_projection:grid_mapping_name = "geostationary" ;
|
||||
goes_imager_projection:perspective_point_height = 35786023. ;
|
||||
goes_imager_projection:semi_major_axis = 6378137. ;
|
||||
goes_imager_projection:semi_minor_axis = 6356752.31414 ;
|
||||
goes_imager_projection:inverse_flattening = 298.2572221 ;
|
||||
goes_imager_projection:latitude_of_projection_origin = 0. ;
|
||||
goes_imager_projection:longitude_of_projection_origin = -75. ;
|
||||
goes_imager_projection:sweep_angle_axis = "x" ;
|
||||
float y_image ;
|
||||
y_image:long_name = "GOES-R fixed grid projection y-coordinate center of image" ;
|
||||
y_image:standard_name = "projection_y_coordinate" ;
|
||||
y_image:units = "rad" ;
|
||||
y_image:axis = "Y" ;
|
||||
float y_image_bounds(number_of_image_bounds) ;
|
||||
y_image_bounds:long_name = "GOES-R fixed grid projection y-coordinate north/south extent of image" ;
|
||||
y_image_bounds:units = "rad" ;
|
||||
float x_image ;
|
||||
x_image:long_name = "GOES-R fixed grid projection x-coordinate center of image" ;
|
||||
x_image:standard_name = "projection_x_coordinate" ;
|
||||
x_image:units = "rad" ;
|
||||
x_image:axis = "X" ;
|
||||
float x_image_bounds(number_of_image_bounds) ;
|
||||
x_image_bounds:long_name = "GOES-R fixed grid projection x-coordinate west/east extent of image" ;
|
||||
x_image_bounds:units = "rad" ;
|
||||
float nominal_satellite_subpoint_lat ;
|
||||
nominal_satellite_subpoint_lat:long_name = "nominal satellite subpoint latitude (platform latitude)" ;
|
||||
nominal_satellite_subpoint_lat:standard_name = "latitude" ;
|
||||
nominal_satellite_subpoint_lat:_FillValue = -999.f ;
|
||||
nominal_satellite_subpoint_lat:units = "degrees_north" ;
|
||||
float nominal_satellite_subpoint_lon ;
|
||||
nominal_satellite_subpoint_lon:long_name = "nominal satellite subpoint longitude (platform longitude)" ;
|
||||
nominal_satellite_subpoint_lon:standard_name = "longitude" ;
|
||||
nominal_satellite_subpoint_lon:_FillValue = -999.f ;
|
||||
nominal_satellite_subpoint_lon:units = "degrees_east" ;
|
||||
float nominal_satellite_height ;
|
||||
nominal_satellite_height:long_name = "nominal satellite height above GRS 80 ellipsoid (platform altitude)" ;
|
||||
nominal_satellite_height:standard_name = "height_above_reference_ellipsoid" ;
|
||||
nominal_satellite_height:_FillValue = -999.f ;
|
||||
nominal_satellite_height:units = "km" ;
|
||||
float geospatial_lat_lon_extent ;
|
||||
geospatial_lat_lon_extent:long_name = "geospatial latitude and longitude references" ;
|
||||
geospatial_lat_lon_extent:geospatial_westbound_longitude = -156.2995f ;
|
||||
geospatial_lat_lon_extent:geospatial_northbound_latitude = 81.3282f ;
|
||||
geospatial_lat_lon_extent:geospatial_eastbound_longitude = 6.2995f ;
|
||||
geospatial_lat_lon_extent:geospatial_southbound_latitude = -81.3282f ;
|
||||
geospatial_lat_lon_extent:geospatial_lat_center = 0.f ;
|
||||
geospatial_lat_lon_extent:geospatial_lon_center = -75.f ;
|
||||
geospatial_lat_lon_extent:geospatial_lat_nadir = 0.f ;
|
||||
geospatial_lat_lon_extent:geospatial_lon_nadir = -75.f ;
|
||||
geospatial_lat_lon_extent:geospatial_lat_units = "degrees_north" ;
|
||||
geospatial_lat_lon_extent:geospatial_lon_units = "degrees_east" ;
|
||||
byte yaw_flip_flag ;
|
||||
yaw_flip_flag:long_name = "Flag indicating the spacecraft is operating in yaw flip configuration" ;
|
||||
yaw_flip_flag:_Unsigned = "true" ;
|
||||
yaw_flip_flag:_FillValue = -1b ;
|
||||
yaw_flip_flag:valid_range = 0b, 1b ;
|
||||
yaw_flip_flag:units = "1" ;
|
||||
yaw_flip_flag:coordinates = "t" ;
|
||||
yaw_flip_flag:flag_values = 0b, 1b ;
|
||||
yaw_flip_flag:flag_meanings = "false true" ;
|
||||
byte band_id(band) ;
|
||||
band_id:long_name = "ABI band number" ;
|
||||
band_id:standard_name = "sensor_band_identifier" ;
|
||||
band_id:units = "1" ;
|
||||
float band_wavelength(band) ;
|
||||
band_wavelength:long_name = "ABI band central wavelength" ;
|
||||
band_wavelength:standard_name = "sensor_band_central_radiation_wavelength" ;
|
||||
band_wavelength:units = "um" ;
|
||||
float esun ;
|
||||
esun:long_name = "bandpass-weighted solar irradiance at the mean Earth-Sun distance" ;
|
||||
esun:standard_name = "toa_shortwave_irradiance_per_unit_wavelength" ;
|
||||
esun:_FillValue = -999.f ;
|
||||
esun:units = "W m-2 um-1" ;
|
||||
esun:coordinates = "band_id band_wavelength t" ;
|
||||
esun:cell_methods = "t: mean" ;
|
||||
float kappa0 ;
|
||||
kappa0:long_name = "Inverse of the incoming top of atmosphere radiance at current earth-sun distance (PI d2 esun-1)-1, where d is the ratio of instantaneous Earth-Sun distance divided by the mean Earth-Sun distance, esun is the bandpass-weighted solar irradiance and PI is a standard constant used to convert ABI L1b radiance to reflectance" ;
|
||||
kappa0:_FillValue = -999.f ;
|
||||
kappa0:units = "(W m-2 um-1)-1" ;
|
||||
kappa0:coordinates = "band_id band_wavelength t" ;
|
||||
kappa0:cell_methods = "t: mean" ;
|
||||
float planck_fk1 ;
|
||||
planck_fk1:long_name = "wavenumber-dependent coefficient (2 h c2/ nu3) used in the ABI emissive band monochromatic brightness temperature computation, where nu =central wavenumber and h and c are standard constants" ;
|
||||
planck_fk1:_FillValue = -999.f ;
|
||||
planck_fk1:units = "W m-1" ;
|
||||
planck_fk1:coordinates = "band_id band_wavelength" ;
|
||||
float planck_fk2 ;
|
||||
planck_fk2:long_name = "wavenumber-dependent coefficient (h c nu/b) used in the ABI emissive band monochromatic brightness temperature computation, where nu = central wavenumber and h, c, and b are standard constants" ;
|
||||
planck_fk2:_FillValue = -999.f ;
|
||||
planck_fk2:units = "K" ;
|
||||
planck_fk2:coordinates = "band_id band_wavelength" ;
|
||||
float planck_bc1 ;
|
||||
planck_bc1:long_name = "spectral bandpass correction offset for brightness temperature (B(nu) - bc_1)/bc_2 where B()=planck_function() and nu=wavenumber" ;
|
||||
planck_bc1:_FillValue = -999.f ;
|
||||
planck_bc1:units = "K" ;
|
||||
planck_bc1:coordinates = "band_id band_wavelength" ;
|
||||
float planck_bc2 ;
|
||||
planck_bc2:long_name = "spectral bandpass correction scale factor for brightness temperature (B(nu) - bc_1)/bc_2 where B()=planck_function() and nu=wavenumber" ;
|
||||
planck_bc2:_FillValue = -999.f ;
|
||||
planck_bc2:units = "1" ;
|
||||
planck_bc2:coordinates = "band_id band_wavelength" ;
|
||||
int valid_pixel_count ;
|
||||
valid_pixel_count:long_name = "number of good and conditionally usable pixels" ;
|
||||
valid_pixel_count:_FillValue = -1 ;
|
||||
valid_pixel_count:units = "count" ;
|
||||
valid_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
|
||||
valid_pixel_count:grid_mapping = "goes_imager_projection" ;
|
||||
valid_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000028 rad comment: good and conditionally usable quality pixels only)" ;
|
||||
int missing_pixel_count ;
|
||||
missing_pixel_count:long_name = "number of missing pixels" ;
|
||||
missing_pixel_count:_FillValue = -1 ;
|
||||
missing_pixel_count:units = "count" ;
|
||||
missing_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
|
||||
missing_pixel_count:grid_mapping = "goes_imager_projection" ;
|
||||
missing_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000028 rad comment: missing ABI fixed grid pixels only)" ;
|
||||
int saturated_pixel_count ;
|
||||
saturated_pixel_count:long_name = "number of saturated pixels" ;
|
||||
saturated_pixel_count:_FillValue = -1 ;
|
||||
saturated_pixel_count:units = "count" ;
|
||||
saturated_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
|
||||
saturated_pixel_count:grid_mapping = "goes_imager_projection" ;
|
||||
saturated_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000028 rad comment: radiometrically saturated geolocated/not missing pixels only)" ;
|
||||
int undersaturated_pixel_count ;
|
||||
undersaturated_pixel_count:long_name = "number of undersaturated pixels" ;
|
||||
undersaturated_pixel_count:_FillValue = -1 ;
|
||||
undersaturated_pixel_count:units = "count" ;
|
||||
undersaturated_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
|
||||
undersaturated_pixel_count:grid_mapping = "goes_imager_projection" ;
|
||||
undersaturated_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000028 rad comment: radiometrically undersaturated geolocated/not missing pixels only)" ;
|
||||
int focal_plane_temperature_threshold_exceeded_count ;
|
||||
focal_plane_temperature_threshold_exceeded_count:long_name = "number of pixels whose temperatures exceeded the threshold" ;
|
||||
focal_plane_temperature_threshold_exceeded_count:_FillValue = -1 ;
|
||||
focal_plane_temperature_threshold_exceeded_count:units = "count" ;
|
||||
focal_plane_temperature_threshold_exceeded_count:coordinates = "band_id band_wavelength t y_image x_image" ;
|
||||
focal_plane_temperature_threshold_exceeded_count:grid_mapping = "goes_imager_projection" ;
|
||||
focal_plane_temperature_threshold_exceeded_count:cell_methods = "t: sum area: sum (interval: 0.000028 rad comment: temperature exceeded pixels only)" ;
|
||||
float min_radiance_value_of_valid_pixels ;
|
||||
min_radiance_value_of_valid_pixels:long_name = "minimum radiance value of pixels" ;
|
||||
min_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavelength" ;
|
||||
min_radiance_value_of_valid_pixels:_FillValue = -999.f ;
|
||||
min_radiance_value_of_valid_pixels:valid_range = -25.93665f, 804.0361f ;
|
||||
min_radiance_value_of_valid_pixels:units = "W m-2 sr-1 um-1" ;
|
||||
min_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
|
||||
min_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
|
||||
min_radiance_value_of_valid_pixels:cell_methods = "t: sum area: minimum (interval: 0.000028 rad comment: good and conditionally usable quality pixels only)" ;
|
||||
float max_radiance_value_of_valid_pixels ;
|
||||
max_radiance_value_of_valid_pixels:long_name = "maximum radiance value of pixels" ;
|
||||
max_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavelength" ;
|
||||
max_radiance_value_of_valid_pixels:_FillValue = -999.f ;
|
||||
max_radiance_value_of_valid_pixels:valid_range = -25.93665f, 804.0361f ;
|
||||
max_radiance_value_of_valid_pixels:units = "W m-2 sr-1 um-1" ;
|
||||
max_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
|
||||
max_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
|
||||
max_radiance_value_of_valid_pixels:cell_methods = "t: sum area: maximum (interval: 0.000028 rad comment: good and conditionally usable quality pixels only)" ;
|
||||
float mean_radiance_value_of_valid_pixels ;
|
||||
mean_radiance_value_of_valid_pixels:long_name = "mean radiance value of pixels" ;
|
||||
mean_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavelength" ;
|
||||
mean_radiance_value_of_valid_pixels:_FillValue = -999.f ;
|
||||
mean_radiance_value_of_valid_pixels:valid_range = -25.93665f, 804.0361f ;
|
||||
mean_radiance_value_of_valid_pixels:units = "W m-2 sr-1 um-1" ;
|
||||
mean_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
|
||||
mean_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
|
||||
mean_radiance_value_of_valid_pixels:cell_methods = "t: sum area: mean (interval: 0.000028 rad comment: good and conditionally usable quality pixels only)" ;
|
||||
float std_dev_radiance_value_of_valid_pixels ;
|
||||
std_dev_radiance_value_of_valid_pixels:long_name = "standard deviation of radiance values of pixels" ;
|
||||
std_dev_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavelength" ;
|
||||
std_dev_radiance_value_of_valid_pixels:_FillValue = -999.f ;
|
||||
std_dev_radiance_value_of_valid_pixels:units = "W m-2 sr-1 um-1" ;
|
||||
std_dev_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
|
||||
std_dev_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
|
||||
std_dev_radiance_value_of_valid_pixels:cell_methods = "t: sum area: standard_deviation (interval: 0.000028 rad comment: good and conditionally usable quality pixels only)" ;
|
||||
float maximum_focal_plane_temperature ;
|
||||
maximum_focal_plane_temperature:long_name = "maximum focal plane temperature value" ;
|
||||
maximum_focal_plane_temperature:_FillValue = -999.f ;
|
||||
maximum_focal_plane_temperature:valid_range = 0.f, 999.f ;
|
||||
maximum_focal_plane_temperature:units = "K" ;
|
||||
float focal_plane_temperature_threshold_increasing ;
|
||||
focal_plane_temperature_threshold_increasing:long_name = "focal plane temperature threshold increasing bounds value" ;
|
||||
focal_plane_temperature_threshold_increasing:_FillValue = -999.f ;
|
||||
focal_plane_temperature_threshold_increasing:valid_range = 0.f, 999.f ;
|
||||
focal_plane_temperature_threshold_increasing:units = "K" ;
|
||||
float focal_plane_temperature_threshold_decreasing ;
|
||||
focal_plane_temperature_threshold_decreasing:long_name = "focal plane temperature threshold decreasing bounds value" ;
|
||||
focal_plane_temperature_threshold_decreasing:_FillValue = -999.f ;
|
||||
focal_plane_temperature_threshold_decreasing:valid_range = 0.f, 999.f ;
|
||||
focal_plane_temperature_threshold_decreasing:units = "K" ;
|
||||
float percent_uncorrectable_L0_errors ;
|
||||
percent_uncorrectable_L0_errors:long_name = "percent data lost due to uncorrectable L0 errors" ;
|
||||
percent_uncorrectable_L0_errors:_FillValue = -999.f ;
|
||||
percent_uncorrectable_L0_errors:valid_range = 0.f, 1.f ;
|
||||
percent_uncorrectable_L0_errors:units = "percent" ;
|
||||
percent_uncorrectable_L0_errors:coordinates = "t y_image x_image" ;
|
||||
percent_uncorrectable_L0_errors:grid_mapping = "goes_imager_projection" ;
|
||||
percent_uncorrectable_L0_errors:cell_methods = "t: sum area: sum (uncorrectable L0 errors only)" ;
|
||||
float earth_sun_distance_anomaly_in_AU ;
|
||||
earth_sun_distance_anomaly_in_AU:long_name = "earth sun distance anomaly in astronomical units" ;
|
||||
earth_sun_distance_anomaly_in_AU:_FillValue = -999.f ;
|
||||
earth_sun_distance_anomaly_in_AU:units = "ua" ;
|
||||
earth_sun_distance_anomaly_in_AU:coordinates = "t" ;
|
||||
earth_sun_distance_anomaly_in_AU:cell_methods = "t: mean" ;
|
||||
int algorithm_dynamic_input_data_container ;
|
||||
algorithm_dynamic_input_data_container:long_name = "container for filenames of dynamic algorithm input data" ;
|
||||
algorithm_dynamic_input_data_container:input_ABI_L0_data = "OR_ABI-L0-F-M6_G16_s20220011800205_e20220011809513_c*.nc" ;
|
||||
int processing_parm_version_container ;
|
||||
processing_parm_version_container:long_name = "container for processing parameter filenames" ;
|
||||
processing_parm_version_container:L1b_processing_parm_version = "OR-PARM-RAD_G16_v01r00.zip" ;
|
||||
int algorithm_product_version_container ;
|
||||
algorithm_product_version_container:long_name = "container for algorithm package filename and product version" ;
|
||||
algorithm_product_version_container:algorithm_version = "OR_ABI-L1b-ALG-RAD_v01r00.zip" ;
|
||||
algorithm_product_version_container:product_version = "v01r00" ;
|
||||
double t_star_look(num_star_looks) ;
|
||||
t_star_look:long_name = "J2000 epoch time of star observed in seconds" ;
|
||||
t_star_look:standard_name = "time" ;
|
||||
t_star_look:units = "seconds since 2000-01-01 12:00:00" ;
|
||||
t_star_look:axis = "T" ;
|
||||
float band_wavelength_star_look(num_star_looks) ;
|
||||
band_wavelength_star_look:long_name = "ABI band central wavelength associated with observed star" ;
|
||||
band_wavelength_star_look:standard_name = "sensor_band_central_radiation_wavelength" ;
|
||||
band_wavelength_star_look:units = "um" ;
|
||||
short star_id(num_star_looks) ;
|
||||
star_id:long_name = "ABI star catalog identifier associated with observed star" ;
|
||||
star_id:_Unsigned = "true" ;
|
||||
star_id:_FillValue = -1s ;
|
||||
star_id:coordinates = "band_id band_wavelength_star_look t_star_look" ;
|
||||
int channel_integration_time ;
|
||||
channel_integration_time:long_name = "Channel-dependent Channel Integration Time, as defined in the VNIR or IR Channel Configuration Table Telemetry" ;
|
||||
channel_integration_time:_FillValue = -1 ;
|
||||
channel_integration_time:units = "count" ;
|
||||
int channel_gain_field ;
|
||||
channel_gain_field:long_name = "Channel-dependent Gain Field, as defined in the VNIR or IR Channel Configuration Table Telemetry" ;
|
||||
channel_gain_field:_FillValue = -1 ;
|
||||
channel_gain_field:units = "1" ;
|
||||
|
||||
// global attributes:
|
||||
:naming_authority = "gov.nesdis.noaa" ;
|
||||
:Conventions = "CF-1.7" ;
|
||||
:standard_name_vocabulary = "CF Standard Name Table (v35, 20 July 2016)" ;
|
||||
:institution = "DOC/NOAA/NESDIS > U.S. Department of Commerce, National Oceanic and Atmospheric Administration, National Environmental Satellite, Data, and Information Services" ;
|
||||
:project = "GOES" ;
|
||||
:production_site = "WCDAS" ;
|
||||
:production_environment = "OE" ;
|
||||
:spatial_resolution = "1km at nadir" ;
|
||||
:Metadata_Conventions = "Unidata Dataset Discovery v1.0" ;
|
||||
:orbital_slot = "GOES-East" ;
|
||||
:platform_ID = "G16" ;
|
||||
:instrument_type = "GOES-R Series Advanced Baseline Imager (ABI)" ;
|
||||
:scene_id = "Full Disk" ;
|
||||
:instrument_ID = "FM1" ;
|
||||
:title = "ABI L1b Radiances" ;
|
||||
:summary = "Single reflective band ABI L1b Radiance Products are digital maps of outgoing radiance values at the top of the atmosphere for visible and near-IR bands." ;
|
||||
:keywords = "SPECTRAL/ENGINEERING > VISIBLE WAVELENGTHS > VISIBLE RADIANCE" ;
|
||||
:keywords_vocabulary = "NASA Global Change Master Directory (GCMD) Earth Science Keywords, Version 7.0.0.0.0" ;
|
||||
:iso_series_metadata_id = "a70be540-c38b-11e0-962b-0800200c9a66" ;
|
||||
:license = "Unclassified data. Access is restricted to approved users only." ;
|
||||
:processing_level = "National Aeronautics and Space Administration (NASA) L1b" ;
|
||||
:cdm_data_type = "Image" ;
|
||||
:dataset_name = "OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc" ;
|
||||
:production_data_source = "Realtime" ;
|
||||
:timeline_id = "ABI Mode 6" ;
|
||||
:date_created = "2022-01-01T18:09:56.2Z" ;
|
||||
:time_coverage_start = "2022-01-01T18:00:20.5Z" ;
|
||||
:time_coverage_end = "2022-01-01T18:09:51.3Z" ;
|
||||
:LUT_Filenames = "SpaceLookParams(FM1A_CDRL79RevP_PR_09_00_02)-637827000.0.h5 QTableBand01(FM1A_CDRL79RevH_DO_07_00_00)-582860861.0.h5 CalTargetTimeIntervals(FM1A_CDRL79RevP_DO_08_00_01)-611906620.0.h5 BandSaturationLimits(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.h5 SolarSpaceLookParams(FM1A_CDRL79RevH_DO_09_00_00)-600765435.0.h5 DeadRowListParams(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.h5 Mirror_Record(FM1A_CDRL79RevG_DO_07_00_00)-582860861.0.h5 KalmanAstroConsts(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.xml KalmanFilterControls(FM1A_PR_09_08_02)-677650371.0.xml KalmanMeasMaxSensibles(FMAA_INT_ONLY_DO_09_01_00)-652936814.0.xml KalmanPreprocessorControls(FM1A_CDRL79RevJ_PR_09_06_02)-657795700.0.xml KalmanReferenceData(FM1A_CDRL79RevH_DO_08_00_00)-888.0.xml KalmanStarCatalogs(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.xml ABI_NavigationRDP_Band01(FM1A_CDRL79RevJ_DO_07_00_00)-582860861.0.xml ABI_NavigationParameters_Band01(FM1A_CDRL79RevH_DO_07_00_00)-582860861.0.xml ABI_ResamplingImplementation_Band01(FM1A_CDRL79RevH_DO_07_02_00)-602129336.0.xml ABI_ResamplingParameters_Band01(FM1A_CDRL79RevJ_DO_07_00_00)-582860861.0.xml StarLookParams(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.h5 StarDetectionParams(FM1A_CDRL79RevJ_DO_07_00_00)-582860861.0.xml ResamplingScaledConversion(FMAA_INT_ONLY_DO_08_00_00)-1111.0.xml BlockReleaseRegions(FMAA_INT_ONLY_DO_08_00_00)-2222.0.csv VNIR_RetrievalParameters(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.h5 SCT_Record(FM1A_CDRL79RevM_DO_09_00_00)-600765435.0.h5 ICM_ConversionConsts(FM1A_CDRL43-18_DO_09_01_00)-652936750.0.h5 ICM_SensorCoefficients(FM1A_TMABI_18_159_TMABI_18_533_DO_09_05_00)-676949608.0.h5" ;
|
||||
:id = "75de858d-c386-4159-a95e-bce8a0d3d61e" ;
|
||||
}
|
||||
50
nc_test/ref_tst_http_nc4f.cdl
Normal file
50
nc_test/ref_tst_http_nc4f.cdl
Normal file
@ -0,0 +1,50 @@
|
||||
netcdf HadCRUT.4.6.0.0.median {
|
||||
dimensions:
|
||||
latitude = 36 ;
|
||||
longitude = 72 ;
|
||||
field_status_string_length = 1 ;
|
||||
time = UNLIMITED ; // (2062 currently)
|
||||
variables:
|
||||
float latitude(latitude) ;
|
||||
latitude:standard_name = "latitude" ;
|
||||
latitude:long_name = "latitude" ;
|
||||
latitude:point_spacing = "even" ;
|
||||
latitude:units = "degrees_north" ;
|
||||
latitude:axis = "Y" ;
|
||||
float longitude(longitude) ;
|
||||
longitude:standard_name = "longitude" ;
|
||||
longitude:long_name = "longitude" ;
|
||||
longitude:point_spacing = "even" ;
|
||||
longitude:units = "degrees_east" ;
|
||||
longitude:axis = "X" ;
|
||||
float time(time) ;
|
||||
time:standard_name = "time" ;
|
||||
time:long_name = "time" ;
|
||||
time:units = "days since 1850-1-1 00:00:00" ;
|
||||
time:calendar = "gregorian" ;
|
||||
time:start_year = 1850s ;
|
||||
time:end_year = 2021s ;
|
||||
time:start_month = 1s ;
|
||||
time:end_month = 10s ;
|
||||
time:axis = "T" ;
|
||||
float temperature_anomaly(time, latitude, longitude) ;
|
||||
temperature_anomaly:long_name = "near_surface_temperature_anomaly" ;
|
||||
temperature_anomaly:units = "K" ;
|
||||
temperature_anomaly:missing_value = -1.e+30f ;
|
||||
temperature_anomaly:_FillValue = -1.e+30f ;
|
||||
temperature_anomaly:reference_period = 1961s, 1990s ;
|
||||
char field_status(time, field_status_string_length) ;
|
||||
field_status:long_name = "field_status" ;
|
||||
|
||||
// global attributes:
|
||||
:title = "HadCRUT4 near-surface temperature ensemble data - ensemble median" ;
|
||||
:institution = "Met Office Hadley Centre / Climatic Research Unit, University of East Anglia" ;
|
||||
:history = "Updated at 15/12/2021 19:37:52" ;
|
||||
:source = "CRUTEM.4.6.0.0, HadSST.3.1.1.0" ;
|
||||
:comment = "" ;
|
||||
:reference = "Morice, C. P., J. J. Kennedy, N. A. Rayner, and P. D. Jones (2012), Quantifying uncertainties in global and regional temperature change using an ensemble of observational estimates: The HadCRUT4 dataset, J. Geophys. Res., doi:10.1029/2011JD017187" ;
|
||||
:version = "HadCRUT.4.6.0.0" ;
|
||||
:Conventions = "CF-1.0" ;
|
||||
:ensemble_members = 100s ;
|
||||
:ensemble_member_index = 0s ;
|
||||
}
|
||||
@ -9,17 +9,25 @@ set -e
|
||||
|
||||
# Test Urls
|
||||
if test "x$FEATURE_THREDDSTEST" = x1 ; then
|
||||
URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes&aws.profile=none"
|
||||
URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes&aws.profile=none"
|
||||
URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes"
|
||||
URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes"
|
||||
fi
|
||||
URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none"
|
||||
URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none"
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then
|
||||
URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
|
||||
URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
|
||||
# Requires auth
|
||||
URL3b="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes"
|
||||
# Requires auth
|
||||
URL3b="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes&aws.profile=unidata"
|
||||
URL4d="s3://unidata-zarr-test-data/byterangefiles/upload4.nc#bytes&aws.profile=unidata"
|
||||
# Test alternate URL with no specified region
|
||||
URL4e="http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3"
|
||||
fi
|
||||
URL4f="https://crudata.uea.ac.uk/cru/data/temperature/HadCRUT.4.6.0.0.median.nc#mode=bytes"
|
||||
|
||||
# Do not use unless we know it has some permanence (note the segment 'testing' in the URL);
|
||||
URL4x="https://s3.us-west-2.amazonaws.com/coawst-public/testing/HadCRUT.4.6.0.0.median.nc#mode=bytes,&aws.profile=none"
|
||||
|
||||
|
||||
echo ""
|
||||
|
||||
testsetup() {
|
||||
@ -90,22 +98,27 @@ if test "x$FEATURE_THREDDSTEST" = x1 ; then
|
||||
fi
|
||||
|
||||
if test "x$FEATURE_HDF5" = xyes ; then
|
||||
echo "***Test remote netcdf-4 files: non-s3"
|
||||
if test "x$FEATURE_THREDDSTEST" = x1 ; then
|
||||
echo "***Test remote netcdf-4 file: non-s3"
|
||||
testbytes nc4b netCDF-4 "$URL4b"
|
||||
fi
|
||||
testbytes nc4f netCDF-4 "$URL4f"
|
||||
fi
|
||||
if test "x$FEATURE_S3TESTS" = xyes && test "x$FEATURE_HDF5" = xyes ; then
|
||||
|
||||
if test "x$FEATURE_S3" = xyes ; then
|
||||
echo "***Test remote netcdf-3 file: s3 auth"
|
||||
tests3auth nc3b classic "$URL3b"
|
||||
fi
|
||||
|
||||
if test "x$FEATURE_S3" = xyes && test "x$FEATURE_HDF5" = xyes ; then
|
||||
echo "***Test remote netdf-4 file: s3"
|
||||
testbytes nc4a netCDF-4 "$URL4a"
|
||||
echo "***Test remote netcdf-4 file: s3"
|
||||
testbytes nc4c netCDF-4 "$URL4c"
|
||||
echo "***Test remote netcdf-4 file: s3 auth"
|
||||
tests3auth nc4d netCDF-4 "$URL4d"
|
||||
fi
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then
|
||||
echo "***Test remote netcdf-3 file: s3 auth"
|
||||
tests3auth nc3b classic "$URL3b"
|
||||
echo "***Test remote netcdf-4 file: s3 noauth"
|
||||
testbytes nc4e netCDF-4 "$URL4e"
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
|
||||
@ -170,8 +170,8 @@ equal(const double x,
|
||||
/* because in-memory data type char can be signed or unsigned,
|
||||
* type cast the value from external NC_CHAR before the comparison
|
||||
*/
|
||||
char x2 = (char) x;
|
||||
char y2 = (char) y;
|
||||
char x2 = *(char *)&x;
|
||||
char y2 = *(char *)&y;
|
||||
return ABS(x2-y2) <= epsilon * MAX( ABS(x2), ABS(y2));
|
||||
}
|
||||
|
||||
@ -194,8 +194,8 @@ equal2(const double x,
|
||||
/* because in-memory data type char can be signed or unsigned,
|
||||
* type cast the value from external NC_CHAR before the comparison
|
||||
*/
|
||||
char x2 = (char) x;
|
||||
char y2 = (char) y;
|
||||
char x2 = *(char *)&x;
|
||||
char y2 = *(char *)&y;
|
||||
return ABS(x2-y2) <= epsilon * MAX( ABS(x2), ABS(y2));
|
||||
}
|
||||
|
||||
@ -343,7 +343,7 @@ int dbl2nc ( const double d, const nc_type xtype, void *p)
|
||||
* reporting it as a range error.
|
||||
*/
|
||||
if ( r < X_CHAR_MIN || r > X_CHAR_MAX ) return 2;
|
||||
*((signed char*) p) = (signed char)r;
|
||||
*((unsigned char*) p) = (unsigned char)r;
|
||||
break;
|
||||
case NC_BYTE:
|
||||
r = floor(0.5+d);
|
||||
@ -413,8 +413,8 @@ int dbl2nc ( const double d, const nc_type xtype, void *p)
|
||||
double
|
||||
hash( const nc_type xtype, const int rank, const size_t *index )
|
||||
{
|
||||
double base;
|
||||
double result;
|
||||
double base = 0;
|
||||
double result = 0;
|
||||
int d; /* index of dimension */
|
||||
|
||||
/* If vector then elements 0 & 1 are min & max. Elements 2 & 3 are */
|
||||
@ -841,7 +841,7 @@ put_atts(int ncid)
|
||||
for (j = 0; j < NATTS(i); j++) {
|
||||
if (ATT_TYPE(i,j) == NC_CHAR) {
|
||||
for (k = 0; k < ATT_LEN(i,j); k++) {
|
||||
catt[k] = (char) hash(ATT_TYPE(i,j), -1, &k);
|
||||
catt[k] = (unsigned char) hash(ATT_TYPE(i,j), -1, &k);
|
||||
}
|
||||
err = nc_put_att_text(ncid, i, ATT_NAME(i,j),
|
||||
ATT_LEN(i,j), catt);
|
||||
@ -969,7 +969,7 @@ check_dims(int ncid)
|
||||
void
|
||||
check_vars(int ncid)
|
||||
{
|
||||
size_t index[MAX_RANK];
|
||||
size_t index[MAX_RANK] = {0};
|
||||
char text, name[NC_MAX_NAME];
|
||||
int i, err; /* status */
|
||||
size_t j;
|
||||
@ -1006,7 +1006,7 @@ check_vars(int ncid)
|
||||
err = nc_get_var1_text(ncid, i, index, &text);
|
||||
IF (err)
|
||||
error("nc_get_var1_text: %s", nc_strerror(err));
|
||||
IF (text != (char)expect) {
|
||||
IF ((unsigned char)text != (unsigned char)expect) {
|
||||
error("Var %s [%lu] value read %hhd not that expected %g ",
|
||||
var_name[i], j, text, expect);
|
||||
print_n_size_t(var_rank[i], index);
|
||||
@ -1073,8 +1073,9 @@ check_atts(int ncid)
|
||||
error("nc_get_att_text: %s", nc_strerror(err));
|
||||
for (k = 0; k < ATT_LEN(i,j); k++) {
|
||||
expect = hash(xtype, -1, &k);
|
||||
IF (text[k] != (char)expect) {
|
||||
error("nc_get_att_text: unexpected value");
|
||||
IF ((unsigned char)text[k] != (unsigned char)expect) {
|
||||
error("Var %s [%lu] value read %hhd not that expected %g ",
|
||||
var_name[i], j, text, expect);
|
||||
} else {
|
||||
nok++;
|
||||
}
|
||||
|
||||
@ -515,6 +515,7 @@ main(int argc, char **argv)
|
||||
union DU dfin;
|
||||
union DU dfout;
|
||||
union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5];
|
||||
NC_UNUSED(fout); NC_UNUSED(dfout);
|
||||
|
||||
switch (quantize_mode[q])
|
||||
{
|
||||
@ -633,6 +634,7 @@ main(int argc, char **argv)
|
||||
union FU fout;
|
||||
union DU dfout;
|
||||
int nsd_att_in;
|
||||
NC_UNUSED(fout); NC_UNUSED(dfout);
|
||||
|
||||
/* Open the file and check metadata. */
|
||||
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
|
||||
@ -752,6 +754,7 @@ main(int argc, char **argv)
|
||||
union FU fout;
|
||||
union DU dfout;
|
||||
union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5];
|
||||
NC_UNUSED(fout); NC_UNUSED(dfout);
|
||||
|
||||
switch (quantize_mode[q])
|
||||
{
|
||||
@ -875,6 +878,8 @@ main(int argc, char **argv)
|
||||
union FU fout;
|
||||
union DU dfout;
|
||||
union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5];
|
||||
NC_UNUSED(fout); NC_UNUSED(dfout);
|
||||
|
||||
switch (quantize_mode[q])
|
||||
{
|
||||
case NC_QUANTIZE_BITGROOM:
|
||||
@ -1003,6 +1008,8 @@ main(int argc, char **argv)
|
||||
union FU fout;
|
||||
union DU dfout;
|
||||
union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5];
|
||||
NC_UNUSED(fout); NC_UNUSED(dfout);
|
||||
|
||||
switch (quantize_mode[q])
|
||||
{
|
||||
case NC_QUANTIZE_BITGROOM:
|
||||
|
||||
@ -85,7 +85,7 @@ IF(ENABLE_TESTS)
|
||||
TARGET_INCLUDE_DIRECTORIES(zisjson PUBLIC ../libnczarr)
|
||||
BUILD_BIN_TEST(zs3parse ${COMMONSRC})
|
||||
TARGET_INCLUDE_DIRECTORIES(zs3parse PUBLIC ../libnczarr)
|
||||
if(ENABLE_NCZARR_S3)
|
||||
if(ENABLE_S3)
|
||||
BUILD_BIN_TEST(s3util ${COMMONSRC})
|
||||
TARGET_INCLUDE_DIRECTORIES(s3util PUBLIC ../libnczarr)
|
||||
endif()
|
||||
@ -124,7 +124,7 @@ IF(ENABLE_TESTS)
|
||||
BUILD_BIN_TEST(test_quantize ${TSTCOMMONSRC})
|
||||
add_sh_test(nczarr_test run_quantize)
|
||||
|
||||
if(ENABLE_NCZARR_S3)
|
||||
if(ENABLE_S3)
|
||||
add_sh_test(nczarr_test run_s3_cleanup)
|
||||
ENDIF()
|
||||
|
||||
|
||||
@ -117,7 +117,7 @@ zisjson_SOURCES = zisjson.c
|
||||
noinst_PROGRAMS += zs3parse
|
||||
zs3parse_SOURCES = zs3parse.c
|
||||
|
||||
if ENABLE_NCZARR_S3
|
||||
if ENABLE_S3
|
||||
noinst_PROGRAMS += s3util
|
||||
s3util_SOURCES = s3util.c
|
||||
TESTS += run_s3_cleanup.sh
|
||||
@ -164,7 +164,8 @@ ref_string.cdl ref_string_nczarr.baseline ref_string_zarr.baseline ref_scalar.cd
|
||||
ref_nulls_nczarr.baseline ref_nulls_zarr.baseline ref_nulls.cdl
|
||||
|
||||
# Interoperability files
|
||||
EXTRA_DIST += ref_power_901_constants_orig.zip ref_power_901_constants.cdl ref_quotes_orig.zip ref_quotes.cdl
|
||||
EXTRA_DIST += ref_power_901_constants_orig.zip ref_power_901_constants.cdl ref_quotes_orig.zip ref_quotes.cdl \
|
||||
ref_zarr_test_data.cdl.gz
|
||||
|
||||
CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip
|
||||
|
||||
|
||||
@ -14,38 +14,39 @@ set -e
|
||||
UH="${NCZARR_S3_TEST_HOST}"
|
||||
UB="${NCZARR_S3_TEST_BUCKET}"
|
||||
|
||||
RESULTSDIR="${builddir}/tmp_interop"
|
||||
|
||||
testcasefile() {
|
||||
zext=file
|
||||
ref=$1
|
||||
base=$1
|
||||
mode=$2
|
||||
metaonly=$3
|
||||
if test "x$metaonly" = xmetaonly ; then flags="-h"; fi
|
||||
fileargs ${execdir}/$ref "mode=$mode,$zext"
|
||||
rm -f tmp_${ref}_${zext}.cdl
|
||||
${NCDUMP} $flags $fileurl > tmp_${ref}_${zext}.cdl
|
||||
diff -b ${srcdir}/${ref}.cdl tmp_${ref}_${zext}.cdl
|
||||
fileargs ${RESULTSDIR}/ref_$base "mode=$mode,$zext"
|
||||
rm -f tmp_${base}_${zext}.cdl
|
||||
${NCDUMP} $flags $fileurl > tmp_${base}_${zext}.cdl
|
||||
diff -b ${srcdir}/ref_${base}.cdl tmp_${base}_${zext}.cdl
|
||||
}
|
||||
|
||||
testcasezip() {
|
||||
zext=zip
|
||||
ref=$1
|
||||
base=$1
|
||||
mode=$2
|
||||
fileargs $ref "mode=$mode,$zext"
|
||||
rm -f tmp_${ref}_${zext}.cdl
|
||||
${NCDUMP} -h $flags $fileurl > tmp_${ref}_${zext}.cdl
|
||||
diff -b ${srcdir}/${ref}.cdl tmp_${ref}_${zext}.cdl
|
||||
fileargs ${RESULTSDIR}/ref_$base "mode=$mode,$zext"
|
||||
rm -f tmp_${base}_${zext}.cdl
|
||||
${NCDUMP} -h $flags $fileurl > tmp_${base}_${zext}.cdl
|
||||
diff -b ${srcdir}/ref_${base}.cdl tmp_${base}_${zext}.cdl
|
||||
}
|
||||
|
||||
testcases3() {
|
||||
zext=s3
|
||||
zarr=$1
|
||||
ref=$2
|
||||
mode=$3
|
||||
rm -f tmp_${zarr}_${zext}.cdl
|
||||
url="https://${UH}/${UB}/${zarr}#mode=${mode},s3"
|
||||
${NCDUMP} $url > tmp_${zarr}_${zext}.cdl
|
||||
diff -b ${srcdir}/${ref}.cdl tmp_${zarr}_${zext}.cdl
|
||||
base=$1
|
||||
mode=$2
|
||||
rm -f tmp_${base}_${zext}.cdl
|
||||
url="https://${UH}/${UB}/${base}.zarr#mode=${mode},s3"
|
||||
${NCDUMP} $url > tmp_${base}_${zext}.cdl
|
||||
# Find the proper ref file
|
||||
diff -b ${RESULTSDIR}/ref_${base}.cdl tmp_${base}_${zext}.cdl
|
||||
}
|
||||
|
||||
testallcases() {
|
||||
@ -53,39 +54,39 @@ zext=$1
|
||||
case "$zext" in
|
||||
file)
|
||||
# need to unpack
|
||||
rm -fr ref_power_901_constants ref_power_901_constants.file
|
||||
unzip ${srcdir}/ref_power_901_constants_orig.zip > /dev/null
|
||||
mv ref_power_901_constants ref_power_901_constants.file
|
||||
testcasefile ref_power_901_constants zarr metaonly; # test xarray as default
|
||||
mv ${RESULTSDIR}/ref_power_901_constants ${RESULTSDIR}/ref_power_901_constants.file
|
||||
testcasefile power_901_constants zarr metaonly; # test xarray as default
|
||||
;;
|
||||
zip)
|
||||
# Move into position
|
||||
cp ${srcdir}/ref_power_901_constants_orig.zip ${execdir}/ref_power_901_constants.zip
|
||||
cp ${srcdir}/ref_quotes_orig.zip ${execdir}/ref_quotes.zip
|
||||
testcasezip ref_power_901_constants xarray metaonly
|
||||
cp -f ${srcdir}/ref_power_901_constants_orig.zip ${RESULTSDIR}/ref_power_901_constants.zip
|
||||
cp -f ${srcdir}/ref_quotes_orig.zip ${RESULTSDIR}/ref_quotes.zip
|
||||
testcasezip power_901_constants xarray metaonly
|
||||
# Test large constant interoperability
|
||||
testcasezip ref_quotes zarr metaonly
|
||||
testcasezip quotes zarr metaonly
|
||||
;;
|
||||
s3)
|
||||
# Read a test case created by netcdf-java zarr.
|
||||
# Move into position
|
||||
rm -f ${execdir}/ref_zarr_test_data.cdl
|
||||
# unpack
|
||||
# Use gunzip because it always appears to be available
|
||||
if ! test -f ${srcdir}/ref_zarr_test_data.cdl ; then
|
||||
gunzip -c ${srcdir}/ref_zarr_test_data.cdl.gz > ${srcdir}/ref_zarr_test_data.cdl
|
||||
fi
|
||||
testcases3 zarr_test_data.zarr ref_zarr_test_data xarray
|
||||
gunzip -c ${srcdir}/ref_zarr_test_data.cdl.gz > ${RESULTSDIR}/ref_zarr_test_data.cdl
|
||||
testcases3 zarr_test_data xarray
|
||||
;;
|
||||
*) echo "unimplemented kind: $1" ; exit 1;;
|
||||
esac
|
||||
}
|
||||
|
||||
THISDIR=`pwd`
|
||||
rm -fr ${RESULTSDIR}
|
||||
mkdir -p ${RESULTSDIR}
|
||||
cd ${RESULTSDIR}
|
||||
testallcases file
|
||||
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testallcases zip; fi
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then testallcases s3; fi
|
||||
exit
|
||||
cd ${THISDIR}
|
||||
|
||||
# Cleanup
|
||||
rm -fr ${execdir}/ref_power_901_constants.file
|
||||
rm -f ${execdir}/ref_zarr_test_data.cdl
|
||||
rm -fr ${RESULTSDIR}
|
||||
|
||||
exit 0
|
||||
|
||||
@ -7,7 +7,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
|
||||
# This shell script tests support for the NC_STRING type
|
||||
|
||||
set -x
|
||||
set -e
|
||||
|
||||
zarrscalar() {
|
||||
|
||||
@ -128,7 +128,7 @@ main(int argc, char** argv)
|
||||
|
||||
dumpoptions.nctype = NC_UBYTE; /* default */
|
||||
|
||||
while ((c = getopt(argc, argv, "df:k:p:t:T:u:v")) != EOF) {
|
||||
while ((c = getopt(argc, argv, "df:hk:p:t:T:u:v")) != EOF) {
|
||||
switch(c) {
|
||||
case 'd':
|
||||
dumpoptions.debug = 1;
|
||||
@ -136,6 +136,9 @@ main(int argc, char** argv)
|
||||
case 'f':
|
||||
dumpoptions.filename = strdup(optarg);
|
||||
break;
|
||||
case 'h':
|
||||
usage();
|
||||
return 0;
|
||||
case 'k': {
|
||||
size_t len = strlen(optarg);
|
||||
dumpoptions.key = (char*)malloc(len+1+1);
|
||||
|
||||
@ -30,11 +30,14 @@ FEATURE_PARALLEL=@HAS_PARALLEL@
|
||||
|
||||
# Define selected features of the build
|
||||
FEATURE_HDF5=@HAS_HDF5@
|
||||
FEATURE_FILTERTESTS=@DO_FILTER_TESTS@
|
||||
FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@
|
||||
FEATURE_BYTERANGE=@HAS_BYTERANGE@
|
||||
FEATURE_S3_SDK=@HAS_S3_SDK@
|
||||
FEATURE_S3=@HAS_S3@
|
||||
FEATURE_NCZARR=@HAS_NCZARR@
|
||||
FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@
|
||||
FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@
|
||||
FEATURE_FILTERTESTS=@DO_FILTER_TESTS@
|
||||
FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@
|
||||
|
||||
# Thredds-test server is currently disabled
|
||||
#FEATURE_THREDDSTEST=1
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user