mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-11-21 03:13:42 +08:00
## Addendum [5/9/23]
It turns out that attempting to test S3 using a github action secret is a very complex process. So, this was disabled for github actions. However, a new *run_tests_s3.yml* action file was added that will eventually encapsulate S3 testing.
This commit is contained in:
Dennis Heimbigner
parent
d79f08176a
commit
98477b9f25
2
.github/workflows/run_tests_osx.yml
vendored
2
.github/workflows/run_tests_osx.yml
vendored
@ -7,7 +7,7 @@
|
||||
name: Run macOS-based netCDF Tests
|
||||
|
||||
|
||||
on: [pull_request,workflow_dispatch]
|
||||
on: [push,pull_request,workflow_dispatch]
|
||||
|
||||
jobs:
|
||||
|
||||
|
||||
153
.github/workflows/run_tests_s3.yml
vendored
Normal file
153
.github/workflows/run_tests_s3.yml
vendored
Normal file
@ -0,0 +1,153 @@
|
||||
###
|
||||
# Test S3 Support
|
||||
# -- derived from run_tests_ubuntu.yml
|
||||
###
|
||||
|
||||
###
|
||||
# Build hdf5 dependencies and cache them in a combined directory.
|
||||
###
|
||||
|
||||
name: Run S3 netCDF Tests (under Ubuntu Linux)
|
||||
|
||||
on: [push,pull_request, workflow_dispatch]
|
||||
|
||||
jobs:
|
||||
|
||||
build-deps-serial:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
hdf5: [ 1.10.8, 1.12.2, 1.14.0 ]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install System dependencies
|
||||
shell: bash -l {0}
|
||||
run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev
|
||||
|
||||
###
|
||||
# Installing libhdf5
|
||||
###
|
||||
- name: Cache libhdf5-${{ matrix.hdf5 }}
|
||||
id: cache-hdf5
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/environments/${{ matrix.hdf5 }}
|
||||
key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }}
|
||||
|
||||
|
||||
- name: Build libhdf5-${{ matrix.hdf5 }}
|
||||
if: steps.cache-hdf5.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
set -x
|
||||
|
||||
wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2
|
||||
tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2
|
||||
pushd hdf5-${{ matrix.hdf5 }}
|
||||
./configure --disable-static --enable-shared --prefix=${HOME}/environments/${{ matrix.hdf5 }} --enable-hl --with-szlib
|
||||
make -j
|
||||
make install -j
|
||||
popd
|
||||
|
||||
|
||||
#####
|
||||
# S3 Autotools-based tests.
|
||||
#####
|
||||
##
|
||||
# Serial
|
||||
##
|
||||
nc-ac-tests-s3-serial:
|
||||
|
||||
needs: build-deps-serial
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
hdf5: [ 1.14.0 ]
|
||||
steps:
|
||||
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install System dependencies
|
||||
shell: bash -l {0}
|
||||
run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev openssl libssl-dev
|
||||
|
||||
###
|
||||
# Set Environmental Variables
|
||||
###
|
||||
|
||||
- run: echo "CFLAGS=-I${HOME}/environments/${{ matrix.hdf5 }}/include" >> $GITHUB_ENV
|
||||
- run: echo "LDFLAGS=-L${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV
|
||||
- run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV
|
||||
|
||||
|
||||
###
|
||||
# Fetch Cache
|
||||
###
|
||||
|
||||
- name: Fetch HDF Cache
|
||||
id: cache-hdf
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/environments/${{ matrix.hdf5 }}
|
||||
key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }}
|
||||
|
||||
- name: Check Cache
|
||||
shell: bash -l {0}
|
||||
run: ls ${HOME}/environments && ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib
|
||||
|
||||
###
|
||||
# Configure and build
|
||||
###
|
||||
|
||||
- name: Run autoconf
|
||||
shell: bash -l {0}
|
||||
run: autoreconf -if
|
||||
|
||||
- name: Configure
|
||||
shell: bash -l {0}
|
||||
run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf5 --disable-dap --enable-external-server-tests --enable-s3 --enable-s3-internal --with-s3-testing=public
|
||||
if: ${{ success() }}
|
||||
|
||||
- name: Look at config.log if error
|
||||
shell: bash -l {0}
|
||||
run: cat config.log
|
||||
if: ${{ failure() }}
|
||||
|
||||
- name: Print Summary
|
||||
shell: bash -l {0}
|
||||
run: cat libnetcdf.settings
|
||||
|
||||
- name: Build Library and Utilities
|
||||
shell: bash -l {0}
|
||||
run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make -j
|
||||
if: ${{ success() }}
|
||||
|
||||
- name: Build Tests
|
||||
shell: bash -l {0}
|
||||
run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check TESTS="" -j
|
||||
if: ${{ success() }}
|
||||
|
||||
- name: Run Tests
|
||||
shell: bash -l {0}
|
||||
env:
|
||||
AWS_PROFILE: ${{ secrets.DEFAULT_PROFILE }}
|
||||
run: |
|
||||
mkdir -p ~/.aws
|
||||
echo "" > ~/.aws/config
|
||||
chmod go-x ~/.aws/config
|
||||
echo "${AWS_PROFILE}" >> ~/.aws/config
|
||||
LD_LIBRARY_PATH="/home/runner/work/netcdf-c/netcdf-c/liblib/.libs:${LD_LIBRARY_PATH}"
|
||||
CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j
|
||||
if: ${{ success() }}
|
||||
|
||||
#####
|
||||
# S3 CMake-based tests.
|
||||
#####
|
||||
##
|
||||
# Serial
|
||||
##
|
||||
# T.B.D. nc-cmake-tests-s3-serial:
|
||||
4
.github/workflows/run_tests_ubuntu.yml
vendored
4
.github/workflows/run_tests_ubuntu.yml
vendored
@ -4,7 +4,7 @@
|
||||
|
||||
name: Run Ubuntu/Linux netCDF Tests
|
||||
|
||||
on: [pull_request, workflow_dispatch]
|
||||
on: [push,pull_request, workflow_dispatch]
|
||||
|
||||
jobs:
|
||||
|
||||
@ -167,7 +167,7 @@ jobs:
|
||||
|
||||
- name: Configure
|
||||
shell: bash -l {0}
|
||||
run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-doxygen --enable-external-server-tests --disable-s3
|
||||
run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-doxygen --enable-external-server-tests
|
||||
if: ${{ success() }}
|
||||
|
||||
- name: Look at config.log if error
|
||||
|
||||
4
.github/workflows/run_tests_win_cygwin.yml
vendored
4
.github/workflows/run_tests_win_cygwin.yml
vendored
@ -1,6 +1,6 @@
|
||||
name: Run Cygwin-based tests
|
||||
|
||||
on: [pull_request,workflow_dispatch]
|
||||
on: [push,pull_request,workflow_dispatch]
|
||||
|
||||
env:
|
||||
SHELLOPTS: igncr
|
||||
@ -43,7 +43,7 @@ jobs:
|
||||
run: >-
|
||||
/bin/dash ./configure --enable-hdf5 --enable-shared
|
||||
--disable-static --enable-dap --disable-dap-remote-tests
|
||||
--enable-plugins --enable-nczarr --disable-s3
|
||||
--enable-plugins --enable-nczarr
|
||||
|
||||
- name: Look at config.log if error
|
||||
if: ${{ failure() }}
|
||||
|
||||
2
.github/workflows/run_tests_win_mingw.yml
vendored
2
.github/workflows/run_tests_win_mingw.yml
vendored
@ -9,7 +9,7 @@ name: Run MSYS2, MinGW64-based Tests
|
||||
env:
|
||||
CPPFLAGS: "-D_BSD_SOURCE"
|
||||
|
||||
on: [pull_request,workflow_dispatch]
|
||||
on: [push,pull_request,workflow_dispatch]
|
||||
|
||||
jobs:
|
||||
|
||||
|
||||
@ -1272,7 +1272,17 @@ ENDIF()
|
||||
OPTION(ENABLE_S3 "Enable S3 support." OFF)
|
||||
OPTION(ENABLE_S3_INTERNAL "Enable S3 Internal support." OFF)
|
||||
OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support; Deprecated in favor of ENABLE_S3" OFF)
|
||||
OPTION(ENABLE_NCZARR_S3_TESTS "Enable NCZarr S3 tests." OFF)
|
||||
|
||||
# Control S3 Testing: Multi-valued option
|
||||
SET(WITH_S3_TESTING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none) PUBLIC")
|
||||
SET_PROPERTY(CACHE WITH_S3_TESTING PROPERTY STRINGS ON OFF PUBLIC) #
|
||||
IF(WITH_S3_TESTING STREQUAL "")
|
||||
SET(WITH_S3_TESTING OFF CACHE STRING "") # Default
|
||||
ENDIF()
|
||||
|
||||
IF(WITH_S3_TESTING)
|
||||
message(WARNING "**** DO NOT USE WITH_S3_TESTING=ON UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***")
|
||||
ENDIF()
|
||||
|
||||
# ENABLE_NCZARR_S3 is now an alias for ENABLE_S3 (but...)
|
||||
if (NOT ENABLE_S3 AND ENABLE_NCZARR_S3)
|
||||
@ -1284,14 +1294,16 @@ UNSET(ENABLE_NCZARR_S3)
|
||||
# because for some reason this screws up if we unconditionally test for sdk
|
||||
# and it is not available. Fix someday
|
||||
IF(ENABLE_S3)
|
||||
# See if aws-s3-sdk is available
|
||||
find_package(AWSSDK REQUIRED COMPONENTS s3;core)
|
||||
IF(AWSSDK_FOUND)
|
||||
SET(service s3;core)
|
||||
AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES)
|
||||
SET(ENABLE_S3_AWS ON CACHE BOOL "S3 AWS" FORCE)
|
||||
ELSE()
|
||||
SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE)
|
||||
IF(NOT ENABLE_S3_INTERNAL)
|
||||
# See if aws-s3-sdk is available
|
||||
find_package(AWSSDK REQUIRED COMPONENTS s3;core)
|
||||
IF(AWSSDK_FOUND)
|
||||
SET(service s3;core)
|
||||
AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES)
|
||||
SET(ENABLE_S3_AWS ON CACHE BOOL "S3 AWS" FORCE)
|
||||
ELSE()
|
||||
SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE)
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
ELSE()
|
||||
SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE)
|
||||
@ -1313,13 +1325,11 @@ IF(ENABLE_S3)
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_S3)
|
||||
message(FATAL_ERROR "S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=NO")
|
||||
SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCZARR S3 TESTS" FORCE)
|
||||
IF(NOT ENABLE_S3)
|
||||
IF(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING)
|
||||
message(WARNING "S3 support is disabled => WITH_S3_TESTING=OFF")
|
||||
SET(WITH_S3_TESTING OFF CACHE STRING "" FORCE)
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_NCZARR_S3_TESTS)
|
||||
message(WARNING "**** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***")
|
||||
ENDIF()
|
||||
|
||||
# Start disabling if curl not found
|
||||
@ -2558,7 +2568,6 @@ is_enabled(ENABLE_S3_AWS HAS_S3_AWS)
|
||||
is_enabled(ENABLE_S3_INTERNAL HAS_S3_INTERNAL)
|
||||
is_enabled(HAS_HDF5_ROS3 HAS_HDF5_ROS3)
|
||||
is_enabled(ENABLE_NCZARR HAS_NCZARR)
|
||||
is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS)
|
||||
is_enabled(ENABLE_NCZARR_ZIP HAS_NCZARR_ZIP)
|
||||
is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)
|
||||
is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
|
||||
@ -2570,17 +2579,26 @@ is_enabled(HAVE_SZ HAS_SZLIB_WRITE)
|
||||
is_enabled(HAVE_ZSTD HAS_ZSTD)
|
||||
is_enabled(HAVE_BLOSC HAS_BLOSC)
|
||||
is_enabled(HAVE_BZ2 HAS_BZ2)
|
||||
if(ENABLE_S3_AWS)
|
||||
SET(WHICH_S3_SDK "aws-sdk-cpp")
|
||||
SET(NC_WHICH_S3_SDK "aws-sdk-cpp")
|
||||
else()
|
||||
if(ENABLE_S3_AWS)
|
||||
|
||||
if(ENABLE_S3_INTERNAL)
|
||||
SET(WHICH_S3_SDK "internal")
|
||||
SET(NC_WHICH_S3_SDK "internal")
|
||||
elseif(ENABLE_S3_AWS)
|
||||
SET(WHICH_S3_SDK "aws-sdk-cpp")
|
||||
SET(NC_WHICH_S3_SDK "aws-sdk-cpp")
|
||||
else()
|
||||
SET(WHICH_S3_SDK "none")
|
||||
SET(NC_WHICH_S3_SDK "none")
|
||||
endif()
|
||||
|
||||
if(WITH_S3_TESTING STREQUAL PUBLIC)
|
||||
SET(DO_S3_TESTING "public")
|
||||
elseif(WITH_S3_TESTING)
|
||||
SET(DO_S3_TESTING "yes")
|
||||
elseif(NOT WITH_S3_TESTING)
|
||||
SET(DO_S3_TESTING "no")
|
||||
else()
|
||||
SET(DO_S3_TESTING "no")
|
||||
endif()
|
||||
|
||||
# Generate file from template.
|
||||
|
||||
@ -142,9 +142,6 @@ are set when opening a binary file on Windows. */
|
||||
/* if true, enable nczarr filter support */
|
||||
#cmakedefine ENABLE_NCZARR_FILTERS 1
|
||||
|
||||
/* if true, enable S3 testing*/
|
||||
#cmakedefine ENABLE_NCZARR_S3_TESTS 1
|
||||
|
||||
/* if true, enable nczarr zip support */
|
||||
#cmakedefine ENABLE_NCZARR_ZIP 1
|
||||
|
||||
@ -160,6 +157,9 @@ are set when opening a binary file on Windows. */
|
||||
/* if true, Force use of S3 internal library */
|
||||
#cmakedefine ENABLE_S3_INTERNAL 1
|
||||
|
||||
/* if true, enable S3 testing*/
|
||||
#cmakedefine WITH_S3_TESTING "PUBLIC"
|
||||
|
||||
/* if true, run extra tests which may not work yet */
|
||||
#cmakedefine EXTRA_TESTS 1
|
||||
|
||||
|
||||
43
configure.ac
43
configure.ac
@ -855,18 +855,18 @@ fi
|
||||
fi
|
||||
|
||||
# Check for enabling S3 testing
|
||||
AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
|
||||
AC_ARG_ENABLE([nczarr-s3-tests],
|
||||
[AS_HELP_STRING([--enable-nczarr-s3-tests],
|
||||
[enable netcdf zarr S3 testing])])
|
||||
test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no
|
||||
AC_MSG_RESULT($enable_nczarr_s3_tests)
|
||||
AC_MSG_CHECKING([what level of netcdf S3 testing should be enabled])
|
||||
AC_ARG_WITH([s3-testing],
|
||||
[AS_HELP_STRING([--with-s3-testing=yes|no|public],
|
||||
[control netcdf S3 testing])],
|
||||
[], [with_s3_testing=public])
|
||||
AC_MSG_RESULT($with_s3_testing)
|
||||
|
||||
# Disable S3 tests if S3 support is disabled
|
||||
if test "x$enable_nczarr_s3_tests" = xyes ; then
|
||||
if test "x$enable_s3" = xno ; then
|
||||
AC_MSG_ERROR([S3 support is disabled => no testing])
|
||||
enable_nczarr_s3_tests=no
|
||||
if test "x$enable_s3" = xno ; then
|
||||
if test "x$with_s3_testing" != xno ; then
|
||||
AC_MSG_WARN([S3 support is disabled => no testing])
|
||||
with_s3_testing=no
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -882,12 +882,10 @@ if test "x$enable_s3_internal" = xyes ; then
|
||||
AC_DEFINE([ENABLE_S3_INTERNAL], [1], [If true, then use internal S3 library])
|
||||
fi
|
||||
|
||||
if test "x$enable_nczarr_s3_tests" = xyes ; then
|
||||
AC_DEFINE([ENABLE_NCZARR_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled])
|
||||
fi
|
||||
AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.])
|
||||
|
||||
if test "x$enable_nczarr_s3_tests" = xyes ; then
|
||||
AC_MSG_WARN([*** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***])
|
||||
if test "x$with_s3_testing" = xyes ; then
|
||||
AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***])
|
||||
fi
|
||||
|
||||
# Set default
|
||||
@ -1855,7 +1853,8 @@ AM_CONDITIONAL(ENABLE_S3, [test "x$enable_s3" = xyes])
|
||||
AM_CONDITIONAL(ENABLE_S3_AWS, [test "x$enable_s3_aws" = xyes])
|
||||
AM_CONDITIONAL(ENABLE_S3_INTERNAL, [test "x$enable_s3_internal" = xyes])
|
||||
AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
|
||||
AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes])
|
||||
AM_CONDITIONAL(ENABLE_S3_TESTPUB, [test "x$with_s3_testing" != xno]) # all => public
|
||||
AM_CONDITIONAL(ENABLE_S3_TESTALL, [test "x$with_s3_testing" = xyes])
|
||||
AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
|
||||
AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
|
||||
AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes])
|
||||
@ -1965,7 +1964,7 @@ AC_SUBST(HAS_S3_AWS,[$enable_s3_aws])
|
||||
AC_SUBST(HAS_S3_INTERNAL,[$enable_s3_internal])
|
||||
AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3])
|
||||
AC_SUBST(HAS_NCZARR,[$enable_nczarr])
|
||||
AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
|
||||
AC_SUBST(DO_S3_TESTING,[$with_s3_testing])
|
||||
AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip])
|
||||
AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
|
||||
AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
|
||||
@ -2075,7 +2074,7 @@ AC_SUBST([NOUNDEFINED])
|
||||
# example: AX_SET_META([NC_HAS_NC2],[$nc_build_v2],[]) # Because it checks for no.
|
||||
# AX_SET_META([NC_HAS_HDF4],[$enable_hdf4],[yes])
|
||||
AC_DEFUN([AX_SET_META],[
|
||||
if [ test "x$2" = x$3 ]; then
|
||||
if [ test "x$2" = "x$3" ]; then
|
||||
AC_SUBST([$1]) $1=1
|
||||
else
|
||||
AC_SUBST([$1]) $1=0
|
||||
@ -2103,10 +2102,10 @@ AX_SET_META([NC_HAS_CDF5],[$enable_cdf5],[yes])
|
||||
AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes])
|
||||
AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes])
|
||||
AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes])
|
||||
AX_SET_META([NC_HAS_S3],[$enable_s3],[no])
|
||||
AX_SET_META([NC_HAS_S3_AWS],[$enable_s3_aws],[no])
|
||||
AX_SET_META([HAS_HAS_S3_INTERNAL],[$enable_s3_internal],[no])
|
||||
AX_SET_META([HAS_HDF5_ROS3],[$has_hdf5_ros3],[no])
|
||||
AX_SET_META([NC_HAS_S3],[$enable_s3],[yes])
|
||||
AX_SET_META([NC_HAS_S3_AWS],[$enable_s3_aws],[yes])
|
||||
AX_SET_META([NC_HAS_S3_INTERNAL],[$enable_s3_internal],[yes])
|
||||
AX_SET_META([NC_HAS_HDF5_ROS3],[$has_hdf5_ros3],[yes])
|
||||
AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
|
||||
AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes])
|
||||
AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes])
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
|
||||
Cloud Storage Access Using The NetCDF-C Library
|
||||
============================
|
||||
<!-- double header is needed to workaround doxygen bug -->
|
||||
@ -98,11 +99,12 @@ Currently the following build cases are known to work.
|
||||
|
||||
## Automake
|
||||
|
||||
There are several options relevant to NCZarr support and to Amazon S3 support.
|
||||
There are several options relevant to Amazon S3 support.
|
||||
These are as follows.
|
||||
|
||||
1. _--enable-s3_ -- Enable S3 support.
|
||||
2. _--enable-s3-internal_ -- Force use of the *nch5s3comms* SDK instead of the *aws-cpp-sdk* (assuming it is available).
|
||||
3. _--with-s3-testing_=yes|no|public -- "yes" means do all S3 tests, "no" means do no S3 testing, "public" means do only those tests that involve publically accessible S3 data.
|
||||
|
||||
__A note about using S3 with Automake.__
|
||||
If S3 support is desired, and using the Amazon "aws-sdk-cpp" SDK, and using Automake, then LDFLAGS must be properly set, namely to this.
|
||||
@ -117,8 +119,9 @@ Note also that if S3 support is enabled, then you need to have a C++ compiler in
|
||||
|
||||
The necessary CMake flags are as follows (with defaults)
|
||||
|
||||
1. -DENABLE_S3 -- Enable S3 support, including NCZarr support if NCZarr is enabled
|
||||
2. -DENABLE_S3_INTERNAL -- Force use of the *nch5s3comms* SDK instead of the *aws-cpp-sdk*.
|
||||
1. *-DENABLE_S3* -- Controll S3 support
|
||||
2. *-DENABLE_S3_INTERNAL* -- Force use of the *nch5s3comms* SDK instead of the *aws-cpp-sdk*.
|
||||
3. *-DWITH-S3-TESTING_=ON|OFF|PUBLIC -- "ON" means do all S3 tests, "OFF" means do no S3 testing, "PUBLIC" means do only those tests that involve publically accessible S3 data.
|
||||
|
||||
Note that unlike Automake, CMake can properly locate C++ libraries, so it should not be necessary to specify _-laws-cpp-sdk-s3_ assuming that the aws s3 libraries are installed in the default location.
|
||||
For CMake with Visual Studio, the default location is here:
|
||||
@ -167,7 +170,7 @@ has a number of properties of interest:
|
||||
For linux, the following context works. Of course your mileage may vary.
|
||||
* OS: ubuntu 21
|
||||
* aws-sdk-cpp version 1.9.96 (or later)
|
||||
* Dependencies: openssl, libcurl, cmake, ninja (ninja-build in apt)
|
||||
* Dependencies: openssl, libcurl, cmake, ninja (ninja-build using *apt-get*)
|
||||
|
||||
#### AWS-SDK-CPP CMake Build Recipe
|
||||
````
|
||||
@ -249,14 +252,13 @@ Then the following options must be specified for cmake.
|
||||
-DAWSSDK_ROOT_DIR=${AWSSDK_ROOT_DIR}
|
||||
-DAWSSDK_DIR=${AWSSDK_ROOT_DIR}/lib/cmake/AWSSDK"
|
||||
````
|
||||
|
||||
## Building ``nch5s3comms''
|
||||
|
||||
This is an experimental SDK provided internally in the netcdf-c library.
|
||||
|
||||
* It is written in C
|
||||
* It provides the minimum functionality necessary to read/write/search an Amazon S3 bucket.
|
||||
* It was constructed by heavily modifying the HDF5 *H5FDros3* Virtual File Driver and combining it with crypto code wrappers provided by libcurl.
|
||||
* It was constructed by heavily modifying the HDF5 *H5FDros3* Virtual File Driver and combining it with crypto code wrappers provided by libcurl. The resulting file was then modified to fit into the netcdf coding style.
|
||||
* The resulting code is rather ugly, but appears to work under at least Linux and under Windows (using Visual C++).
|
||||
|
||||
### Dependencies
|
||||
@ -303,12 +305,12 @@ The algorithm for choosing the active profile to use is as follows:
|
||||
https://...#mode=nczarr,s3&aws.profile=xxx
|
||||
````
|
||||
2. If the "AWS.PROFILE" entry in the .rc file (i.e. .netrc or .dodsrc) is set, then it is used.
|
||||
3. Otherwise the profile "default" is used.
|
||||
3. If defined, then profile "default" is used.
|
||||
4. Otherwise the profile "no" is used.
|
||||
|
||||
The profile named "none" is a special profile that the netcdf-c library automatically defines.
|
||||
The profile named "no" is a special profile that the netcdf-c library automatically defines.
|
||||
It should not be defined anywhere else. It signals to the library that no credentialas are to used.
|
||||
It is equivalent to the "--no-sign-request" option in the AWS CLI.
|
||||
Also, it must be explicitly specified by name. Otherwise "default" will be used.
|
||||
|
||||
## Region Selection
|
||||
|
||||
@ -332,7 +334,7 @@ The algorithm for picking an region is as follows.
|
||||
|
||||
Picking an access-key/secret-key pair is always determined
|
||||
by the current active profile. To choose to not use keys
|
||||
requires that the active profile must be "none".
|
||||
requires that the active profile must be "no".
|
||||
|
||||
# Change Log {#nccloud_changelog}
|
||||
[Note: minor text changes are not included.]
|
||||
|
||||
143
docs/nczarr.md
143
docs/nczarr.md
@ -46,7 +46,7 @@ filters](./md_filters.html "filters") for details.
|
||||
|
||||
Briefly, the data model supported by NCZarr is netcdf-4 minus
|
||||
the user-defined types. However, a restricted form of String type
|
||||
is supported (see Appendix H).
|
||||
is supported (see Appendix E).
|
||||
As with netcdf-4 chunking is supported. Filters and compression
|
||||
are also [supported](./md_filters.html "filters").
|
||||
|
||||
@ -455,11 +455,10 @@ Here are a couple of examples using the _ncgen_ and _ncdump_ utilities.
|
||||
```
|
||||
ncgen -4 -lb -o "s3://datasetbucket/rootkey#mode=nczarr,awsprofile=unidata" dataset.cdl
|
||||
```
|
||||
Note that the URLis internally translated to this
|
||||
Note that the URL is internally translated to this
|
||||
````
|
||||
https://s2.<region>.amazonaws.com/datasetbucket/rootkey#mode=nczarr,awsprofile=unidata" dataset.cdl
|
||||
````
|
||||
The region is from the algorithm described in Appendix E1.
|
||||
|
||||
# References {#nczarr_bib}
|
||||
|
||||
@ -499,153 +498,37 @@ A separate tabulation of S3 support is in the document cloud.md.
|
||||
The relevant ./configure options are as follows.
|
||||
|
||||
1. *--disable-nczarr* -- disable the NCZarr support.
|
||||
2. *--enable-nczarr-s3-tests* -- the NCZarr S3 tests are currently only usable by Unidata personnel, so they are disabled by default.
|
||||
|
||||
### A note about using S3 with Automake.
|
||||
If S3 support is desired, and using Automake, then LDFLAGS must be properly set, namely to this.
|
||||
````
|
||||
LDFLAGS="$LDFLAGS -L/usr/local/lib -laws-cpp-sdk-s3"
|
||||
````
|
||||
The above assumes that these libraries were installed in '/usr/local/lib', so the above requires modification if they were installed elsewhere.
|
||||
|
||||
## CMake
|
||||
|
||||
The relevant CMake flags are as follows.
|
||||
|
||||
1. *-DENABLE_NCZARR=off* -- equivalent to the Automake *--disable-nczarr* option.
|
||||
2. *-DENABLE_NCZARR_S3_TESTS=off* -- equivalent to the Automake *--enable-nczarr-s3-tests* option.
|
||||
|
||||
Note that unlike Automake, CMake can properly locate C++ libraries, so it should not be necessary to specify *-laws-cpp-sdk-s3*, assuming that the aws s3 libraries are installed in the default location.
|
||||
For CMake with Visual Studio, the default location is here:
|
||||
|
||||
````
|
||||
C:/Program Files (x86)/aws-cpp-sdk-all
|
||||
````
|
||||
|
||||
It is possible to install the sdk library in another location.
|
||||
In this case, one must add the following flag to the cmake command.
|
||||
````
|
||||
cmake ... -DAWSSDK_DIR=\<awssdkdir\>
|
||||
````
|
||||
where "awssdkdir" is the path to the sdk installation.
|
||||
For example, this might be as follows.
|
||||
````
|
||||
cmake ... -DAWSSDK_DIR="c:\tools\aws-cpp-sdk-all"
|
||||
````
|
||||
This can be useful if blanks in path names cause problems in your build environment.
|
||||
|
||||
## Testing S3 Support {#nczarr_testing_S3_support}
|
||||
## Testing NCZarr S3 Support {#nczarr_testing_S3_support}
|
||||
|
||||
The relevant tests for S3 support are in the _nczarr_test_ directory.
|
||||
Currently, by default, testing of S3 with NCZarr is supported only for Unidata members of the NetCDF Development Group.
|
||||
This is because it uses a Unidata-specific bucket is inaccessible to the general user.
|
||||
|
||||
# Appendix B. Building aws-sdk-cpp {#nczarr_s3sdk}
|
||||
|
||||
In order to use the S3 storage driver, it is necessary to install the Amazon [aws-sdk-cpp library](https://github.com/aws/aws-sdk-cpp.git).
|
||||
|
||||
Building this package from scratch has proven to be a formidable task.
|
||||
This appears to be due to dependencies on very specific versions of,
|
||||
for example, openssl.
|
||||
|
||||
## *\*nix\** Build
|
||||
|
||||
For linux, the following context works. Of course your mileage may vary.
|
||||
* OS: ubuntu 21
|
||||
* aws-sdk-cpp version 1.9.96 (or later?)
|
||||
* Required installed libraries: openssl, libcurl, cmake, ninja (ninja-build in apt)
|
||||
|
||||
### AWS-SDK-CPP Build Recipe
|
||||
|
||||
````
|
||||
git clone --recurse-submodules https://www.github.com/aws/aws-sdk-cpp.git
|
||||
cd aws-sdk-cpp
|
||||
mkdir build
|
||||
cd build
|
||||
PREFIX=/usr/local
|
||||
FLAGS="-DCMAKE_INSTALL_PREFIX=${PREFIX} \
|
||||
-DCMAKE_INSTALL_LIBDIR=lib \
|
||||
-DCMAKE_MODULE_PATH=${PREFIX}/lib/cmake \
|
||||
-DCMAKE_POLICY_DEFAULT_CMP0075=NEW \
|
||||
-DBUILD_ONLY=s3 \
|
||||
-DENABLE_UNITY_BUILD=ON \
|
||||
-DENABLE_TESTING=OFF \
|
||||
-DCMAKE_BUILD_TYPE=$CFG \
|
||||
-DSIMPLE_INSTALL=ON \
|
||||
_DMY_ASSEMBLER_IS_TOO_OLD_FOR_AVX=ON"
|
||||
cmake -GNinja $FLAGS ..
|
||||
ninja all
|
||||
sudo ninja install
|
||||
cd ..
|
||||
cd ..
|
||||
````
|
||||
This is because it uses a Unidata-specific bucket that is inaccessible to the general user.
|
||||
|
||||
### NetCDF Build
|
||||
|
||||
In order to build netcdf-c with S3 sdk support,
|
||||
the following options must be specified for ./configure.
|
||||
````
|
||||
--enable-nczarr-s3
|
||||
--enable-s3
|
||||
````
|
||||
If you have access to the Unidata bucket on Amazon, then you can
|
||||
also test S3 support with this option.
|
||||
````
|
||||
--enable-nczarr-s3-tests
|
||||
--with-s3-testing=yes
|
||||
````
|
||||
|
||||
## Windows build
|
||||
It is possible to build and install aws-sdk-cpp. It is also possible
|
||||
to build netcdf-c using cmake. Unfortunately, testing currently fails.
|
||||
|
||||
For Windows, the following context work. Of course your mileage may vary.
|
||||
* OS: Windows 10 64-bit with Visual Studio community edition 2019.
|
||||
* aws-sdk-cpp version 1.9.96 (or later?)
|
||||
* Required installed libraries: openssl, libcurl, cmake
|
||||
|
||||
### AWS-SDK-CPP Build Recipe
|
||||
|
||||
This command-line build assumes one is using Cygwin or Mingw to provide
|
||||
tools such as bash.
|
||||
|
||||
````
|
||||
git clone --recurse-submodules https://www.github.com/aws/aws-sdk-cpp
|
||||
pushd aws-sdk-cpp
|
||||
mkdir build
|
||||
cd build
|
||||
CFG="Release"
|
||||
PREFIX="c:/tools/aws-sdk-cpp"
|
||||
|
||||
FLAGS="-DCMAKE_INSTALL_PREFIX=${PREFIX} \
|
||||
-DCMAKE_INSTALL_LIBDIR=lib" \
|
||||
-DCMAKE_MODULE_PATH=${PREFIX}/cmake \
|
||||
-DCMAKE_POLICY_DEFAULT_CMP0075=NEW \
|
||||
-DBUILD_ONLY=s3 \
|
||||
-DENABLE_UNITY_BUILD=ON \
|
||||
-DCMAKE_BUILD_TYPE=$CFG \
|
||||
-DSIMPLE_INSTALL=ON"
|
||||
|
||||
rm -fr build
|
||||
mkdir -p build
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=${CFG} $FLAGS ..
|
||||
cmake --build . --config ${CFG}
|
||||
cmake --install . --config ${CFG}
|
||||
cd ..
|
||||
popd
|
||||
````
|
||||
Notice that the sdk is being installed in the directory "c:\tools\aws-sdk-cpp"
|
||||
rather than the default location "c:\Program Files (x86)/aws-sdk-cpp-all"
|
||||
This is because when using a command line, an install path that contains
|
||||
blanks may not work.
|
||||
|
||||
### NetCDF CMake Build
|
||||
|
||||
Enabling S3 support is controlled by these two cmake options:
|
||||
Enabling S3 support is controlled by this cmake option:
|
||||
````
|
||||
-DENABLE_NCZARR_S3=ON
|
||||
-DENABLE_NCZARR_S3_TESTS=OFF
|
||||
-DENABLE_S3=ON
|
||||
````
|
||||
|
||||
However, to find the aws sdk libraries,
|
||||
the following environment variables must be set:
|
||||
````
|
||||
@ -658,7 +541,7 @@ Then the following options must be specified for cmake.
|
||||
-DAWSSDK_ROOT_DIR=${AWSSDK_ROOT_DIR}
|
||||
-DAWSSDK_DIR=${AWSSDK_ROOT_DIR}/lib/cmake/AWSSDK"
|
||||
````
|
||||
# Appendix C. Amazon S3 Imposed Limits {#nczarr_s3limits}
|
||||
# Appendix B. Amazon S3 Imposed Limits {#nczarr_s3limits}
|
||||
|
||||
The Amazon S3 cloud storage imposes some significant limits that are inherited by NCZarr (and Zarr also, for that matter).
|
||||
|
||||
@ -668,7 +551,7 @@ Some of the relevant limits are as follows:
|
||||
Note that the limit is defined in terms of bytes and not (Unicode) characters.
|
||||
This affects the depth to which groups can be nested because the key encodes the full path name of a group.
|
||||
|
||||
# Appendix F. NCZarr Version 1 Meta-Data Representation. {#nczarr_version1}
|
||||
# Appendix C. NCZarr Version 1 Meta-Data Representation. {#nczarr_version1}
|
||||
|
||||
In NCZarr Version 1, the NCZarr specific metadata was represented using new objects rather than as keys in existing Zarr objects.
|
||||
Due to conflicts with the Zarr specification, that format is deprecated in favor of the one described above.
|
||||
@ -683,7 +566,7 @@ The content of these objects is the same as the contents of the corresponding ke
|
||||
* ''.nczarray <=> ''_nczarr_array_''
|
||||
* ''.nczattr <=> ''_nczarr_attr_''
|
||||
|
||||
# Appendix G. JSON Attribute Convention. {#nczarr_json}
|
||||
# Appendix D. JSON Attribute Convention. {#nczarr_json}
|
||||
|
||||
The Zarr V2 specification is somewhat vague on what is a legal
|
||||
value for an attribute. The examples all show one of two cases:
|
||||
@ -771,7 +654,7 @@ actions "read-write-read" is equivalent to a single "read" and "write-read-write
|
||||
The "almost" caveat is necessary because (1) whitespace may be added or lost during the sequence of operations,
|
||||
and (2) numeric precision may change.
|
||||
|
||||
# Appendix H. Support for string types
|
||||
# Appendix E. Support for string types
|
||||
|
||||
Zarr supports a string type, but it is restricted to
|
||||
fixed size strings. NCZarr also supports such strings,
|
||||
@ -839,7 +722,7 @@ It is necessary to use the 'git diff' command.
|
||||
accepted for back compatibility.
|
||||
|
||||
2. The legal values of an attribute has been extended to
|
||||
include arbitrary JSON expressions; see Appendix G for more details.
|
||||
include arbitrary JSON expressions; see Appendix D for more details.
|
||||
|
||||
# Point of Contact {#nczarr_poc}
|
||||
|
||||
|
||||
@ -50,7 +50,7 @@ endif # ENABLE_BYTERANGE
|
||||
if ENABLE_S3
|
||||
if ENABLE_S3_INTERNAL
|
||||
# Renamed to avoid conflicts with the HDF5 files
|
||||
libdispatch_la_SOURCES += ncs3sdk_h5.c nch5s3comms.c nch5s3comms.h nccurl_setup.h \
|
||||
libdispatch_la_SOURCES += ncs3sdk_h5.c nch5s3comms.c nch5s3comms.h ncutil.h nccurl_setup.h \
|
||||
nccurl_sha256.c nccurl_sha256.h nccurl_hmac.c nccurl_hmac.h
|
||||
AM_CPPFLAGS += -I$(top_srcdir)/libncxml
|
||||
libdispatch_la_CPPFLAGS += ${AM_CPPFLAGS}
|
||||
|
||||
@ -94,6 +94,7 @@ NC_authsetup(NCauth** authp, NCURI* uri)
|
||||
int ret = NC_NOERR;
|
||||
char* uri_hostport = NULL;
|
||||
NCauth* auth = NULL;
|
||||
struct AWSprofile* ap = NULL;
|
||||
|
||||
if(uri != NULL)
|
||||
uri_hostport = NC_combinehostport(uri);
|
||||
@ -175,8 +176,15 @@ NC_authsetup(NCauth** authp, NCURI* uri)
|
||||
nullfree(user);
|
||||
nullfree(pwd);
|
||||
}
|
||||
|
||||
/* Get the Default profile */
|
||||
auth->s3profile = strdup("default");
|
||||
if((ret=NC_authgets3profile("no",&ap))) goto done;
|
||||
if(ap == NULL)
|
||||
if((ret=NC_authgets3profile("default",&ap))) goto done;
|
||||
if(ap != NULL)
|
||||
auth->s3profile = strdup(ap->name);
|
||||
else
|
||||
auth->s3profile = NULL;
|
||||
|
||||
if(authp) {*authp = auth; auth = NULL;}
|
||||
done:
|
||||
|
||||
@ -155,9 +155,11 @@ nc_http_close(NC_HTTP_STATE* state)
|
||||
break;
|
||||
#ifdef ENABLE_S3
|
||||
case HTTPS3: {
|
||||
NC_s3sdkclose(state->s3.s3client, state->s3.info, 0, &state->errmsg);
|
||||
if(state->s3.s3client)
|
||||
NC_s3sdkclose(state->s3.s3client, state->s3.info, 0, NULL);
|
||||
NC_s3clear(state->s3.info);
|
||||
nullfree(state->s3.info);
|
||||
state->s3.s3client = NULL;
|
||||
} break;
|
||||
#endif
|
||||
default: stat = NCTHROW(NC_ENOTBUILT); goto done;
|
||||
|
||||
@ -762,6 +762,7 @@ Get the current active profile. The priority order is as follows:
|
||||
1. aws.profile key in mode flags
|
||||
2. aws.profile in .rc entries
|
||||
4. "default"
|
||||
5. "no" -- meaning do not use any profile => no secret key
|
||||
|
||||
@param uri uri with mode flags, may be NULL
|
||||
@param profilep return profile name here or NULL if none found
|
||||
@ -774,16 +775,27 @@ NC_getactives3profile(NCURI* uri, const char** profilep)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
const char* profile = NULL;
|
||||
struct AWSprofile* ap = NULL;
|
||||
|
||||
profile = ncurifragmentlookup(uri,"aws.profile");
|
||||
if(profile == NULL)
|
||||
profile = NC_rclookupx(uri,"AWS.PROFILE");
|
||||
if(profile == NULL)
|
||||
profile = "default";
|
||||
|
||||
if(profile == NULL) {
|
||||
if((stat=NC_authgets3profile("default",&ap))) goto done;
|
||||
if(ap) profile = "default";
|
||||
}
|
||||
|
||||
if(profile == NULL) {
|
||||
if((stat=NC_authgets3profile("no",&ap))) goto done;
|
||||
if(ap) profile = "no";
|
||||
}
|
||||
|
||||
#ifdef AWSDEBUG
|
||||
fprintf(stderr,">>> activeprofile = %s\n",(profile?profile:"null"));
|
||||
#endif
|
||||
if(profilep) *profilep = profile;
|
||||
done:
|
||||
return stat;
|
||||
}
|
||||
|
||||
@ -811,7 +823,7 @@ NC_getdefaults3region(NCURI* uri, const char** regionp)
|
||||
if(region == NULL)
|
||||
region = NC_rclookupx(uri,"AWS.REGION");
|
||||
if(region == NULL) {/* See if we can find a profile */
|
||||
if((stat = NC_getactives3profile(uri,&profile))==NC_NOERR) {
|
||||
if(NC_getactives3profile(uri,&profile)==NC_NOERR) {
|
||||
if(profile)
|
||||
(void)NC_s3profilelookup(profile,"aws_region",®ion);
|
||||
}
|
||||
@ -1122,6 +1134,14 @@ aws_load_credentials(NCglobalstate* gstate)
|
||||
NCbytes* buf = ncbytesnew();
|
||||
char path[8192];
|
||||
|
||||
/* add a "no" credentials */
|
||||
{
|
||||
struct AWSprofile* noprof = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile));
|
||||
noprof->name = strdup("no");
|
||||
noprof->entries = nclistnew();
|
||||
nclistpush(profiles,noprof); noprof = NULL;
|
||||
}
|
||||
|
||||
for(;*awscfg;awscfg++) {
|
||||
/* Construct the path ${HOME}/<file> or Windows equivalent. */
|
||||
const char* cfg = *awscfg;
|
||||
@ -1140,14 +1160,6 @@ aws_load_credentials(NCglobalstate* gstate)
|
||||
}
|
||||
}
|
||||
|
||||
/* add a "none" credentials */
|
||||
{
|
||||
struct AWSprofile* noprof = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile));
|
||||
noprof->name = strdup("none");
|
||||
noprof->entries = nclistnew();
|
||||
nclistpush(profiles,noprof); noprof = NULL;
|
||||
}
|
||||
|
||||
if(gstate->rcinfo->s3profiles)
|
||||
freeprofilelist(gstate->rcinfo->s3profiles);
|
||||
gstate->rcinfo->s3profiles = profiles; profiles = NULL;
|
||||
@ -1177,6 +1189,13 @@ done:
|
||||
return stat;
|
||||
}
|
||||
|
||||
/* Lookup a profile by name;
|
||||
@param profilename to lookup
|
||||
@param profilep return the matching profile; null if profile not found
|
||||
@return NC_NOERR if no error
|
||||
@return other error
|
||||
*/
|
||||
|
||||
int
|
||||
NC_authgets3profile(const char* profilename, struct AWSprofile** profilep)
|
||||
{
|
||||
@ -1197,25 +1216,23 @@ done:
|
||||
/**
|
||||
@param profile name of profile
|
||||
@param key key to search for in profile
|
||||
@param valup place to store the value if key is found.
|
||||
@return NC_NOERR if key is found, NC_NOOBJECT if key is not found, or other error.
|
||||
@param value place to store the value if key is found; NULL if not found
|
||||
@return NC_NOERR if key is found, Some other error otherwise.
|
||||
*/
|
||||
|
||||
int
|
||||
NC_s3profilelookup(const char* profile, const char* key, const char** valuep)
|
||||
{
|
||||
int i,stat = NC_ENOOBJECT;
|
||||
int i,stat = NC_NOERR;
|
||||
struct AWSprofile* awsprof = NULL;
|
||||
const char* value = NULL;
|
||||
|
||||
if(profile == NULL) return NC_ES3;
|
||||
stat = NC_authgets3profile(profile,&awsprof);
|
||||
if(stat == NC_NOERR && awsprof != NULL) {
|
||||
if((stat=NC_authgets3profile(profile,&awsprof))==NC_NOERR && awsprof != NULL) {
|
||||
for(i=0;i<nclistlength(awsprof->entries);i++) {
|
||||
struct AWSentry* entry = (struct AWSentry*)nclistget(awsprof->entries,i);
|
||||
if(strcasecmp(entry->key,key)==0) {
|
||||
value = entry->value;
|
||||
stat = NC_NOERR;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -214,7 +214,7 @@ NC_s3urlprocess(NCURI* url, NCS3INFO* s3)
|
||||
{stat = NC_EURL; goto done;}
|
||||
/* Get current profile */
|
||||
if((stat = NC_getactives3profile(url,&profile0))) goto done;
|
||||
if(profile0 == NULL) profile0 = "none";
|
||||
if(profile0 == NULL) profile0 = "no";
|
||||
s3->profile = strdup(profile0);
|
||||
|
||||
/* Rebuild the URL to path format and get a usable region and optional bucket*/
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -74,6 +74,7 @@
|
||||
/* Opaque Handles */
|
||||
struct CURL;
|
||||
struct NCURI;
|
||||
struct VString;
|
||||
|
||||
/*****************
|
||||
* PUBLIC MACROS *
|
||||
@ -184,11 +185,11 @@ struct NCURI;
|
||||
*---------------------------------------------------------------------------
|
||||
*/
|
||||
#define S3COMMS_FORMAT_CREDENTIAL(dest, access, iso8601_date, region, service) \
|
||||
ncbytescat((dest),(access)); ncbytescat((dest),"/"); \
|
||||
ncbytescat((dest),(iso8601_date)); ncbytescat((dest),"/"); \
|
||||
ncbytescat((dest),(region)); ncbytescat((dest),"/"); \
|
||||
ncbytescat((dest),(service)); ncbytescat((dest),"/"); \
|
||||
ncbytescat((dest),"aws4_request");
|
||||
vscat((dest),(access)); vscat((dest),"/"); \
|
||||
vscat((dest),(iso8601_date)); vscat((dest),"/"); \
|
||||
vscat((dest),(region)); vscat((dest),"/"); \
|
||||
vscat((dest),(service)); vscat((dest),"/"); \
|
||||
vscat((dest),"aws4_request");
|
||||
|
||||
#if 0
|
||||
snprintf((dest), S3COMMS_MAX_CREDENTIAL_SIZE, "%s/%s/%s/%s/aws4_request", (access), (iso8601_date), \
|
||||
@ -274,11 +275,6 @@ struct NCURI;
|
||||
* as the field would appear in an HTTP request.
|
||||
* e.g., "Range: bytes=0-9"
|
||||
*
|
||||
* `next` (hrb_node_t *)
|
||||
*
|
||||
* Pointers to next node in the list, or NULL sentinel as end of list.
|
||||
* Next node must have a greater `lowername` as determined by strcmp().
|
||||
*
|
||||
*----------------------------------------------------------------------------
|
||||
*/
|
||||
typedef struct hrb_node_t {
|
||||
@ -358,12 +354,11 @@ typedef struct hrb_node_t {
|
||||
*----------------------------------------------------------------------------
|
||||
*/
|
||||
typedef struct {
|
||||
unsigned long magic;
|
||||
char *body;
|
||||
size_t body_len;
|
||||
hrb_node_t *first_header;
|
||||
char *resource;
|
||||
char *version;
|
||||
unsigned long magic;
|
||||
struct VString *body;
|
||||
struct VList *headers;
|
||||
char *resource;
|
||||
char *version;
|
||||
} hrb_t;
|
||||
#define S3COMMS_HRB_MAGIC 0x6DCC84UL
|
||||
|
||||
@ -466,12 +461,19 @@ typedef struct {
|
||||
char *accessid;
|
||||
char *accesskey;
|
||||
char httpverb[S3COMMS_VERB_MAX];
|
||||
unsigned char signing_key[SHA256_DIGEST_LENGTH];
|
||||
unsigned char *signing_key; /*|signing_key| = SHA256_DIGEST_LENGTH*/
|
||||
char iso8601now[ISO8601_SIZE];
|
||||
char *reply;
|
||||
struct curl_slist *curlheaders;
|
||||
} s3r_t;
|
||||
|
||||
/* Combined storage for space + size */
|
||||
typedef struct s3r_buf_t {
|
||||
unsigned long long count; /* |content| */
|
||||
void* content;
|
||||
} s3r_buf_t;
|
||||
|
||||
|
||||
#define S3COMMS_S3R_MAGIC 0x44d8d79
|
||||
|
||||
typedef enum HTTPVerb {
|
||||
@ -492,7 +494,7 @@ EXTERNL int NCH5_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const ch
|
||||
* DECLARATION OF HTTP REQUEST BUFFER ROUTINES *
|
||||
***********************************************/
|
||||
|
||||
EXTERNL int NCH5_s3comms_hrb_destroy(hrb_t **buf);
|
||||
EXTERNL int NCH5_s3comms_hrb_destroy(hrb_t *buf);
|
||||
|
||||
EXTERNL hrb_t *NCH5_s3comms_hrb_init_request(const char *resource, const char *host);
|
||||
|
||||
@ -504,13 +506,11 @@ EXTERNL s3r_t *NCH5_s3comms_s3r_open(const char* root, const char* region, const
|
||||
|
||||
EXTERNL int NCH5_s3comms_s3r_close(s3r_t *handle);
|
||||
|
||||
EXTERNL int NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, NCbytes* dest);
|
||||
EXTERNL int NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* data);
|
||||
|
||||
EXTERNL int NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, NCbytes* data);
|
||||
EXTERNL int NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data);
|
||||
|
||||
EXTERNL int NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, NCbytes* response);
|
||||
|
||||
EXTERNL int NCH5_s3comms_s3r_execute(s3r_t *handle, const char* url, HTTPVerb verb, const char* byterange, const char* header, const char** otherheaders, long* httpcodep, NCbytes* data);
|
||||
EXTERNL int NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response);
|
||||
|
||||
EXTERNL int NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long * sizep);
|
||||
|
||||
@ -524,8 +524,8 @@ EXTERNL int NCH5_s3comms_s3r_head(s3r_t *handle, const char* url, const char* he
|
||||
|
||||
EXTERNL struct tm *gmnow(void);
|
||||
|
||||
EXTERNL int NCH5_s3comms_aws_canonical_request(NCbytes* canonical_request_dest,
|
||||
NCbytes* signed_headers_dest,
|
||||
EXTERNL int NCH5_s3comms_aws_canonical_request(struct VString* canonical_request_dest,
|
||||
struct VString* signed_headers_dest,
|
||||
HTTPVerb verb,
|
||||
const char* query,
|
||||
const char* payloadsha256,
|
||||
@ -544,15 +544,15 @@ EXTERNL int NCH5_s3comms_nlowercase(char *dest, const char *s, size_t len);
|
||||
|
||||
EXTERNL int NCH5_s3comms_percent_encode_char(char *repr, const unsigned char c, size_t *repr_len);
|
||||
|
||||
EXTERNL int NCH5_s3comms_signing_key(unsigned char *md, const char *secret, const char *region,
|
||||
EXTERNL int NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *region,
|
||||
const char *iso8601now);
|
||||
|
||||
EXTERNL int NCH5_s3comms_tostringtosign(NCbytes* dest, const char *req_str, const char *now,
|
||||
EXTERNL int NCH5_s3comms_tostringtosign(struct VString* dest, const char *req_str, const char *now,
|
||||
const char *region);
|
||||
|
||||
EXTERNL int NCH5_s3comms_trim(char *dest, char *s, size_t s_len, size_t *n_written);
|
||||
|
||||
EXTERNL int NCH5_s3comms_uriencode(NCbytes* dest, const char *s, size_t s_len, int encode_slash, size_t *n_written);
|
||||
EXTERNL int NCH5_s3comms_uriencode(char** destp, const char *s, size_t s_len, int encode_slash, size_t *n_written);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
@ -89,7 +89,7 @@ static int ncs3_finalized = 0;
|
||||
static void s3client_destroy(NCS3CLIENT* s3client);
|
||||
static char* makes3rooturl(NCS3INFO* info);
|
||||
static int makes3fullpath(const char* pathkey, const char* bucket, const char* prefix, const char* key, NCbytes* url);
|
||||
static int parse_listbucketresult(char* xml, size_t xmllen, struct LISTOBJECTSV2**);
|
||||
static int parse_listbucketresult(char* xml, unsigned long long xmllen, struct LISTOBJECTSV2**);
|
||||
static int parse_object(ncxml_t root, NClist* objects);
|
||||
static int parse_owner(ncxml_t root, struct Owner* ownerp);
|
||||
static int parse_prefix(ncxml_t root, NClist* prefixes);
|
||||
@ -103,7 +103,7 @@ static int makes3prefix(const char* prefix, char** prefixdirp);
|
||||
static int s3objectsinfo(NClist* contents, NClist* keys, NClist* lens);
|
||||
static int s3commonprefixes(NClist* list, NClist* keys);
|
||||
static int mergekeysets(NClist*,NClist*,NClist*);
|
||||
static int rawtokeys(NCbytes* response, NClist* keys, NClist* lengths, struct LISTOBJECTSV2** listv2p);
|
||||
static int rawtokeys(s3r_buf_t* response, NClist* keys, NClist* lengths, struct LISTOBJECTSV2** listv2p);
|
||||
|
||||
static int queryadd(NClist* query, const char* key, const char* value);
|
||||
static int queryend(NClist* query, char** querystring);
|
||||
@ -172,7 +172,8 @@ NC_s3sdkcreateclient(NCS3INFO* info)
|
||||
char* urlroot = NULL;
|
||||
NCS3CLIENT* s3client = NULL;
|
||||
|
||||
NCTRACE(11,NULL);
|
||||
NCTRACE(11,"info=%s",NC_s3dumps3info(info));
|
||||
|
||||
s3client = (NCS3CLIENT*)calloc(1,sizeof(NCS3CLIENT));
|
||||
if(s3client == NULL) goto done;
|
||||
if(info->profile != NULL) {
|
||||
@ -281,23 +282,18 @@ NC_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t
|
||||
int stat = NC_NOERR;
|
||||
NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
|
||||
NCbytes* url = ncbytesnew();
|
||||
NCbytes* wrap = ncbytesnew();
|
||||
struct s3r_buf_t data = {0,NULL};
|
||||
|
||||
NCTRACE(11,"bucket=%s pathkey=%s start=%llu count=%llu content=%p",bucket,pathkey,start,count,content);
|
||||
|
||||
if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done;
|
||||
|
||||
/* Wrap contents as a byte buffer */
|
||||
ncbytessetcontents(wrap,content,count);
|
||||
ncbytessetlength(wrap,0); /* prepare for writing */
|
||||
|
||||
/* Read the data */
|
||||
if((stat = NCH5_s3comms_s3r_read(s3client->h5s3client,ncbytescontents(url),(size_t)start,(size_t)count,wrap))) goto done;
|
||||
assert(ncbyteslength(wrap) == count);
|
||||
data.count = count;
|
||||
data.content = content;
|
||||
if((stat = NCH5_s3comms_s3r_read(s3client->h5s3client,ncbytescontents(url),(size_t)start,(size_t)count,&data))) goto done;
|
||||
|
||||
done:
|
||||
(void)ncbytesextract(wrap); /* reset the wrapper */
|
||||
ncbytesfree(wrap);
|
||||
ncbytesfree(url);
|
||||
return NCUNTRACE(stat);
|
||||
}
|
||||
@ -312,21 +308,18 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey, s
|
||||
int stat = NC_NOERR;
|
||||
NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
|
||||
NCbytes* url = ncbytesnew();
|
||||
NCbytes* wrap = ncbytesnew();
|
||||
s3r_buf_t data;
|
||||
|
||||
NCTRACE(11,"bucket=%s pathkey=%s count=%llu content=%p",bucket,pathkey,count,content);
|
||||
|
||||
if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done;
|
||||
|
||||
/* Wrap contents as a byte buffer */
|
||||
ncbytessetcontents(wrap,(void*)content,count);
|
||||
|
||||
/* Write the data */
|
||||
if((stat = NCH5_s3comms_s3r_write(s3client->h5s3client,ncbytescontents(url),wrap))) goto done;
|
||||
data.count = count;
|
||||
data.content = (void*)content;
|
||||
if((stat = NCH5_s3comms_s3r_write(s3client->h5s3client,ncbytescontents(url),&data))) goto done;
|
||||
|
||||
done:
|
||||
(void)ncbytesextract(wrap); /* reset the wrapper */
|
||||
ncbytesfree(wrap);
|
||||
ncbytesfree(url);
|
||||
return NCUNTRACE(stat);
|
||||
}
|
||||
@ -337,7 +330,7 @@ NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** errmsgp)
|
||||
int stat = NC_NOERR;
|
||||
NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
|
||||
|
||||
NCTRACE(11,"info=%s rootkey=%s deleteit=%d",NC_s3dumps3info(info),deleteit);
|
||||
NCTRACE(11,"info=%s deleteit=%d",NC_s3dumps3info(info),deleteit);
|
||||
|
||||
if(deleteit) {
|
||||
/* Delete the root key; ok it if does not exist */
|
||||
@ -347,7 +340,6 @@ NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** errmsgp)
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
stat = NCH5_s3comms_s3r_close(s3client->h5s3client);
|
||||
s3client_destroy(s3client);
|
||||
return NCUNTRACE(stat);
|
||||
}
|
||||
@ -363,7 +355,6 @@ getkeys(void* s3client0, const char* bucket, const char* prefixkey0, const char*
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0;
|
||||
NCbytes* response = ncbytesnew();
|
||||
char* prefixdir = NULL;
|
||||
NClist* query = NULL;
|
||||
char* querystring = NULL;
|
||||
@ -373,6 +364,7 @@ getkeys(void* s3client0, const char* bucket, const char* prefixkey0, const char*
|
||||
struct LISTOBJECTSV2* listv2 = NULL;
|
||||
int istruncated = 0;
|
||||
char* continuetoken = NULL;
|
||||
s3r_buf_t response = {0,NULL};
|
||||
|
||||
NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
|
||||
|
||||
@ -385,7 +377,7 @@ getkeys(void* s3client0, const char* bucket, const char* prefixkey0, const char*
|
||||
nullfree(querystring);
|
||||
querystring = NULL;
|
||||
ncbytesclear(listurl);
|
||||
ncbytesclear(response);
|
||||
nullfree(response.content); response.content = NULL; response.count = 0;
|
||||
/* Make sure order is sorted (after encoding) */
|
||||
if((stat = queryadd(query,"list-type","2"))) goto done;
|
||||
if((stat = queryadd(query,"prefix",prefixdir))) goto done;
|
||||
@ -403,9 +395,8 @@ getkeys(void* s3client0, const char* bucket, const char* prefixkey0, const char*
|
||||
ncbytescat(listurl,"?");
|
||||
ncbytescat(listurl,querystring);
|
||||
|
||||
if((stat = NCH5_s3comms_s3r_getkeys(s3client->h5s3client, ncbytescontents(listurl), response))) goto done;
|
||||
ncbytesnull(response);
|
||||
if((stat = rawtokeys(response,allkeys,NULL,&listv2))) goto done;
|
||||
if((stat = NCH5_s3comms_s3r_getkeys(s3client->h5s3client, ncbytescontents(listurl), &response))) goto done;
|
||||
if((stat = rawtokeys(&response,allkeys,NULL,&listv2))) goto done;
|
||||
istruncated = (strcasecmp(listv2->istruncated,"true")==0?1:0);
|
||||
nullfree(continuetoken);
|
||||
continuetoken = nulldup(listv2->nextcontinuationtoken);
|
||||
@ -422,7 +413,7 @@ done:
|
||||
nullfree(querystring);
|
||||
ncurifree(purl);
|
||||
ncbytesfree(listurl);
|
||||
ncbytesfree(response);
|
||||
nullfree(response.content);
|
||||
if(prefixdir) free(prefixdir);
|
||||
return NCUNTRACEX(stat,"nkeys=%u",PTRVAL(unsigned,nkeysp,0));
|
||||
}
|
||||
@ -459,7 +450,7 @@ NC_s3sdkdeletekey(void* s3client0, const char* bucket, const char* pathkey, char
|
||||
NCbytes* url = ncbytesnew();
|
||||
long httpcode = 0;
|
||||
|
||||
NCTRACE(11,"bucket=%s pathkey=%s",bucket,pathkey);
|
||||
NCTRACE(11,"s3client0=%p bucket=%s pathkey=%s",s3client0,bucket,pathkey);
|
||||
|
||||
if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done;
|
||||
|
||||
@ -477,14 +468,14 @@ done:
|
||||
Convert raw getkeys response to vector of keys
|
||||
*/
|
||||
static int
|
||||
rawtokeys(NCbytes* response, NClist* allkeys, NClist* lengths, struct LISTOBJECTSV2** listv2p)
|
||||
rawtokeys(s3r_buf_t* response, NClist* allkeys, NClist* lengths, struct LISTOBJECTSV2** listv2p)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
struct LISTOBJECTSV2* listv2 = NULL;
|
||||
NClist* realkeys = nclistnew();
|
||||
NClist* commonkeys = nclistnew();
|
||||
|
||||
if((stat = parse_listbucketresult(ncbytescontents(response),ncbyteslength(response),&listv2))) goto done;
|
||||
if((stat = parse_listbucketresult(response->content,response->count,&listv2))) goto done;
|
||||
|
||||
if(nclistlength(listv2->contents) > 0) {
|
||||
if((stat = s3objectsinfo(listv2->contents,realkeys,lengths))) goto done;
|
||||
@ -589,6 +580,7 @@ s3client_destroy(NCS3CLIENT* s3client)
|
||||
{
|
||||
if(s3client) {
|
||||
nullfree(s3client->rooturl);
|
||||
(void)NCH5_s3comms_s3r_close(s3client->h5s3client);
|
||||
free(s3client);
|
||||
}
|
||||
}
|
||||
@ -636,7 +628,7 @@ HTTP/1.1 200
|
||||
*/
|
||||
|
||||
static int
|
||||
parse_listbucketresult(char* xml, size_t xmllen, struct LISTOBJECTSV2** resultp)
|
||||
parse_listbucketresult(char* xml, unsigned long long xmllen, struct LISTOBJECTSV2** resultp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
ncxml_doc_t doc = NULL;
|
||||
@ -990,19 +982,14 @@ static int
|
||||
queryadd(NClist* query, const char* key, const char* value)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCbytes* buf = NULL;
|
||||
char* ekey = NULL;
|
||||
char* evalue = NULL;
|
||||
|
||||
if(key == NULL) {stat = NC_EINVAL; goto done;}
|
||||
buf = ncbytesnew();
|
||||
if((stat = NCH5_s3comms_uriencode(buf, key, strlen(key), 1/*true*/, NULL))) goto done;
|
||||
ekey = ncbytesextract(buf);
|
||||
if((stat = NCH5_s3comms_uriencode(&ekey, key, strlen(key), 1/*true*/, NULL))) goto done;
|
||||
evalue = NULL;
|
||||
if(value != NULL) {
|
||||
ncbytesclear(buf);
|
||||
if((stat = NCH5_s3comms_uriencode(buf, value, strlen(value), 1/*true*/, NULL))) goto done;
|
||||
evalue = ncbytesextract(buf);
|
||||
if((stat = NCH5_s3comms_uriencode(&evalue, value, strlen(value), 1/*true*/, NULL))) goto done;
|
||||
}
|
||||
/* Insert encoded key+value keeping sorted order */
|
||||
if((stat = queryinsert(query, ekey, evalue))) goto done;
|
||||
@ -1011,7 +998,6 @@ queryadd(NClist* query, const char* key, const char* value)
|
||||
done:
|
||||
nullfree(ekey);
|
||||
nullfree(evalue);
|
||||
ncbytesfree(buf);
|
||||
return NCTHROW(stat);
|
||||
}
|
||||
|
||||
|
||||
242
libdispatch/ncutil.h
Normal file
242
libdispatch/ncutil.h
Normal file
@ -0,0 +1,242 @@
|
||||
/* Copyright 2018, UCAR/Unidata and OPeNDAP, Inc.
|
||||
See the COPYRIGHT file for more information. */
|
||||
|
||||
#ifndef UTILS_H
|
||||
#define UTILS_H 1
|
||||
|
||||
/* Define a header-only simple version of a dynamically expandable list and byte buffer */
|
||||
/* To be used in code that should be independent of libnetcdf */
|
||||
|
||||
typedef struct VList {
|
||||
unsigned alloc;
|
||||
unsigned length;
|
||||
void** content;
|
||||
} VList;
|
||||
|
||||
typedef struct VString {
|
||||
int nonextendible; /* 1 => fail if an attempt is made to extend this string*/
|
||||
unsigned int alloc;
|
||||
unsigned int length;
|
||||
char* content;
|
||||
} VString;
|
||||
|
||||
/* VString has a fixed expansion size */
|
||||
#define VSTRALLOC 64
|
||||
|
||||
#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__)
|
||||
#define EXTERNC extern "C"
|
||||
#else
|
||||
#define EXTERNC extern
|
||||
#endif
|
||||
|
||||
static int util_initialized = 0;
|
||||
|
||||
static void util_initialize(void);
|
||||
|
||||
static VList*
|
||||
vlistnew(void)
|
||||
{
|
||||
VList* l;
|
||||
if(!util_initialized) util_initialize();
|
||||
l = (VList*)calloc(1,sizeof(VList));
|
||||
assert(l != NULL);
|
||||
return l;
|
||||
}
|
||||
|
||||
static void
|
||||
vlistfree(VList* l)
|
||||
{
|
||||
if(l == NULL) return;
|
||||
if(l->content != NULL) {free(l->content); l->content = NULL;}
|
||||
free(l);
|
||||
}
|
||||
|
||||
static void
|
||||
vlistexpand(VList* l)
|
||||
{
|
||||
void** newcontent = NULL;
|
||||
size_t newsz;
|
||||
|
||||
if(l == NULL) return;
|
||||
newsz = (l->length * 2) + 1; /* basically double allocated space */
|
||||
if(l->alloc >= newsz) return; /* space already allocated */
|
||||
newcontent=(void**)calloc(newsz,sizeof(void*));
|
||||
assert(newcontent != NULL);
|
||||
if(l->alloc > 0 && l->length > 0 && l->content != NULL) { /* something to copy */
|
||||
memcpy((void*)newcontent,(void*)l->content,sizeof(void*)*l->length);
|
||||
}
|
||||
if(l->content != NULL) free(l->content);
|
||||
l->content=newcontent;
|
||||
l->alloc=newsz;
|
||||
/* size is the same */
|
||||
}
|
||||
|
||||
static void*
|
||||
vlistget(VList* l, unsigned index) /* Return the ith element of l */
|
||||
{
|
||||
if(l == NULL || l->length == 0) return NULL;
|
||||
assert(index < l->length);
|
||||
return l->content[index];
|
||||
}
|
||||
|
||||
static void
|
||||
vlistpush(VList* l, void* elem)
|
||||
{
|
||||
if(l == NULL) return;
|
||||
while(l->length >= l->alloc) vlistexpand(l);
|
||||
l->content[l->length] = elem;
|
||||
l->length++;
|
||||
}
|
||||
|
||||
static void*
|
||||
vlistfirst(VList* l) /* remove first element */
|
||||
{
|
||||
unsigned i,len;
|
||||
void* elem;
|
||||
if(l == NULL || l->length == 0) return NULL;
|
||||
elem = l->content[0];
|
||||
len = l->length;
|
||||
for(i=1;i<len;i++) l->content[i-1] = l->content[i];
|
||||
l->length--;
|
||||
return elem;
|
||||
}
|
||||
|
||||
static void
|
||||
vlistfreeall(VList* l) /* call free() on each list element*/
|
||||
{
|
||||
unsigned i;
|
||||
if(l == NULL || l->length == 0) return;
|
||||
for(i=0;i<l->length;i++) if(l->content[i] != NULL) {free(l->content[i]);}
|
||||
vlistfree(l);
|
||||
}
|
||||
|
||||
static VString*
|
||||
vsnew(void)
|
||||
{
|
||||
VString* vs = NULL;
|
||||
if(!util_initialized) util_initialize();
|
||||
vs = (VString*)calloc(1,sizeof(VString));
|
||||
assert(vs != NULL);
|
||||
return vs;
|
||||
}
|
||||
|
||||
static void
|
||||
vsfree(VString* vs)
|
||||
{
|
||||
if(vs == NULL) return;
|
||||
if(vs->content != NULL) free(vs->content);
|
||||
free(vs);
|
||||
}
|
||||
|
||||
static void
|
||||
vsexpand(VString* vs)
|
||||
{
|
||||
char* newcontent = NULL;
|
||||
size_t newsz;
|
||||
|
||||
if(vs == NULL) return;
|
||||
assert(vs->nonextendible == 0);
|
||||
newsz = (vs->alloc + VSTRALLOC); /* basically double allocated space */
|
||||
if(vs->alloc >= newsz) return; /* space already allocated */
|
||||
newcontent=(char*)calloc(1,newsz+1);/* always room for nul term */
|
||||
assert(newcontent != NULL);
|
||||
if(vs->alloc > 0 && vs->length > 0 && vs->content != NULL) /* something to copy */
|
||||
memcpy((void*)newcontent,(void*)vs->content,vs->length);
|
||||
newcontent[vs->length] = '\0'; /* ensure null terminated */
|
||||
if(vs->content != NULL) free(vs->content);
|
||||
vs->content=newcontent;
|
||||
vs->alloc=newsz;
|
||||
/* length is the same */
|
||||
}
|
||||
|
||||
static void
|
||||
vsappendn(VString* vs, const char* elem, unsigned n)
|
||||
{
|
||||
size_t need;
|
||||
assert(vs != NULL && elem != NULL);
|
||||
if(n == 0) {n = strlen(elem);}
|
||||
need = vs->length + n;
|
||||
if(vs->nonextendible) {
|
||||
/* Space must already be available */
|
||||
assert(vs->alloc >= need);
|
||||
} else {
|
||||
while(vs->alloc < need)
|
||||
vsexpand(vs);
|
||||
}
|
||||
memcpy(&vs->content[vs->length],elem,n);
|
||||
vs->length += n;
|
||||
if(!vs->nonextendible)
|
||||
vs->content[vs->length] = '\0';
|
||||
}
|
||||
|
||||
static void
|
||||
vsappend(VString* vs, char elem)
|
||||
{
|
||||
char s[2];
|
||||
s[0] = elem;
|
||||
s[1] = '\0';
|
||||
vsappendn(vs,s,1);
|
||||
}
|
||||
|
||||
/* Set unexpandible contents */
|
||||
static void
|
||||
vssetcontents(VString* vs, char* contents, unsigned alloc)
|
||||
{
|
||||
assert(vs != NULL && contents != NULL);
|
||||
vs->length = 0;
|
||||
if(!vs->nonextendible && vs->content != NULL) free(vs->content);
|
||||
vs->content = contents;
|
||||
vs->length = alloc;
|
||||
vs->alloc = alloc;
|
||||
vs->nonextendible = 1;
|
||||
}
|
||||
|
||||
/* Extract the content and leave content null */
|
||||
static char*
|
||||
vsextract(VString* vs)
|
||||
{
|
||||
char* x = NULL;
|
||||
if(vs == NULL || vs->content == NULL) return NULL;
|
||||
x = vs->content;
|
||||
vs->content = NULL;
|
||||
vs->length = 0;
|
||||
vs->alloc = 0;
|
||||
return x;
|
||||
}
|
||||
|
||||
static void
|
||||
util_initialize(void)
|
||||
{
|
||||
/* quiet compiler */
|
||||
void* f = NULL;
|
||||
f = f;
|
||||
f = (void*)vlistnew;
|
||||
f = (void*)vlistfree;
|
||||
f = (void*)vlistexpand;
|
||||
f = (void*)vlistget;
|
||||
f = (void*)vlistpush;
|
||||
f = (void*)vlistfirst;
|
||||
f = (void*)vlistfreeall;
|
||||
f = (void*)vsnew;
|
||||
f = (void*)vsfree;
|
||||
f = (void*)vsexpand;
|
||||
f = (void*)vssetcontents;
|
||||
f = (void*)vsappendn;
|
||||
f = (void*)vsappend;
|
||||
f = (void*)vsextract;
|
||||
util_initialized = 1;
|
||||
}
|
||||
|
||||
/* Following are always "in-lined"*/
|
||||
#define vlistcontents(l) ((l)==NULL?NULL:(l)->content)
|
||||
#define vlistlength(l) ((l)==NULL?0:(int)(l)->length)
|
||||
#define vlistclear(l) vlistsetlength(l,0)
|
||||
#define vlistsetlength(l,len) do{if((l)!=NULL) (l)->length=len;} while(0)
|
||||
|
||||
#define vscontents(vs) ((vs)==NULL?NULL:(vs)->content)
|
||||
#define vslength(vs) ((vs)==NULL?0:(int)(vs)->length)
|
||||
#define vscat(vs,s) vsappendn(vs,s,0)
|
||||
#define vsclear(vs) vssetlength(vs,0)
|
||||
#define vssetlength(vs,len) do{if((vs)!=NULL) (vs)->length=len;} while(0)
|
||||
|
||||
#endif /*UTILS_H*/
|
||||
@ -12,15 +12,17 @@ if test "x$FEATURE_THREDDSTEST" = x1 ; then
|
||||
URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes"
|
||||
URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes"
|
||||
fi
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then
|
||||
if test "x$FEATURE_S3TESTS" != xno ; then
|
||||
URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
|
||||
URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
|
||||
# Test alternate URL with no specified region
|
||||
URL4e="http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3"
|
||||
fi
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then
|
||||
# Requires auth
|
||||
URL3b="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes"
|
||||
# Requires auth
|
||||
URL4d="s3://unidata-zarr-test-data/byterangefiles/upload4.nc#bytes&aws.profile=unidata"
|
||||
# Test alternate URL with no specified region
|
||||
URL4e="http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3"
|
||||
fi
|
||||
URL4f="https://crudata.uea.ac.uk/cru/data/temperature/HadCRUT.4.6.0.0.median.nc#mode=bytes"
|
||||
|
||||
@ -33,7 +35,6 @@ echo ""
|
||||
testsetup() {
|
||||
U=$1
|
||||
# Create and upload test files
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then
|
||||
rm -f upload4.nc upload3.nc
|
||||
${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles clear
|
||||
${NCGEN} -lb -3 ${srcdir}/nc_enddef.cdl
|
||||
@ -44,16 +45,13 @@ ${NCGEN} -lb -4 ${srcdir}/nc_enddef.cdl
|
||||
mv nc_enddef.nc upload4.nc
|
||||
${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles/upload4.nc -f upload4.nc upload
|
||||
fi
|
||||
fi
|
||||
rm -f tst_http_nc3.cdl tst_http_nc4?.cdl
|
||||
}
|
||||
|
||||
testcleanup() {
|
||||
U=$1
|
||||
rm -f upload4.nc upload3.nc
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then
|
||||
${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles clear
|
||||
fi
|
||||
}
|
||||
|
||||
testbytes() {
|
||||
@ -105,23 +103,31 @@ if test "x$FEATURE_HDF5" = xyes ; then
|
||||
testbytes nc4f netCDF-4 "$URL4f"
|
||||
fi
|
||||
|
||||
if test "x$FEATURE_S3" = xyes ; then
|
||||
if test "x$URL3B" != x ; then
|
||||
echo "***Test remote netcdf-3 file: s3 auth"
|
||||
tests3auth nc3b classic "$URL3b"
|
||||
fi
|
||||
|
||||
if test "x$FEATURE_S3" = xyes && test "x$FEATURE_HDF5" = xyes ; then
|
||||
if test "x$URL4a" != x ; then
|
||||
echo "***Test remote netdf-4 file: s3"
|
||||
testbytes nc4a netCDF-4 "$URL4a"
|
||||
fi
|
||||
if test "x$URL4c" != x ; then
|
||||
echo "***Test remote netcdf-4 file: s3"
|
||||
testbytes nc4c netCDF-4 "$URL4c"
|
||||
fi
|
||||
if test "x$URL4d" != x ; then
|
||||
echo "***Test remote netcdf-4 file: s3 auth"
|
||||
tests3auth nc4d netCDF-4 "$URL4d"
|
||||
fi
|
||||
if test "x$URL4e" != x ; then
|
||||
echo "***Test remote netcdf-4 file: s3 noauth"
|
||||
testbytes nc4e netCDF-4 "$URL4e"
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then
|
||||
testcleanup https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data
|
||||
fi
|
||||
|
||||
exit
|
||||
|
||||
@ -76,16 +76,18 @@ IF(ENABLE_TESTS)
|
||||
TARGET_INCLUDE_DIRECTORIES(tst_fillonlyz PUBLIC ../libnczarr)
|
||||
|
||||
# Helper programs for testing
|
||||
BUILD_BIN_TEST(zmapio ${COMMONSRC})
|
||||
TARGET_INCLUDE_DIRECTORIES(zmapio PUBLIC ../libnczarr)
|
||||
BUILD_BIN_TEST(zhex)
|
||||
BUILD_BIN_TEST(zisjson ${COMMONSRC})
|
||||
TARGET_INCLUDE_DIRECTORIES(zisjson PUBLIC ../libnczarr)
|
||||
BUILD_BIN_TEST(zs3parse ${COMMONSRC})
|
||||
TARGET_INCLUDE_DIRECTORIES(zs3parse PUBLIC ../libnczarr)
|
||||
if(ENABLE_S3)
|
||||
BUILD_BIN_TEST(s3util ${COMMONSRC})
|
||||
TARGET_INCLUDE_DIRECTORIES(s3util PUBLIC ../libnczarr)
|
||||
BUILD_BIN_TEST(zmapio ${COMMONSRC})
|
||||
TARGET_INCLUDE_DIRECTORIES(zmapio PUBLIC ../libnczarr)
|
||||
|
||||
IF(ENABLE_S3 AND NOT WITH_S3_TESTING STREQUAL "NO")
|
||||
# Helper programs for testing
|
||||
BUILD_BIN_TEST(s3util ${COMMONSRC})
|
||||
TARGET_INCLUDE_DIRECTORIES(s3util PUBLIC ../libnczarr)
|
||||
endif()
|
||||
|
||||
SET(ncdumpchunks_SOURCE ncdumpchunks.c)
|
||||
|
||||
@ -113,12 +113,12 @@ endif #ENABLE_FILTER_TESTING
|
||||
endif #BUILD_UTILITIES
|
||||
|
||||
# These programs are used by the test cases
|
||||
noinst_PROGRAMS = zmapio
|
||||
zmapio_SOURCES = zmapio.c
|
||||
noinst_PROGRAMS += zhex
|
||||
noinst_PROGRAMS = zhex
|
||||
zhex_SOURCES = zhex.c
|
||||
noinst_PROGRAMS += zisjson
|
||||
zisjson_SOURCES = zisjson.c
|
||||
noinst_PROGRAMS += zmapio
|
||||
zmapio_SOURCES = zmapio.c
|
||||
noinst_PROGRAMS += zs3parse
|
||||
zs3parse_SOURCES = zs3parse.c
|
||||
|
||||
|
||||
@ -131,8 +131,6 @@ echo "Test miscellaneous 1"
|
||||
makefile tmp_misc1
|
||||
rm -f tmp_misc1_${zext}.txt tmp_misc1_${zext}.cdl
|
||||
$TC -d 6,12,4 -c 2,3,1 -f 0,0,0 -e 6,1,4 -Ow $F
|
||||
ls
|
||||
ls -1 ${execdir}
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then
|
||||
${S3UTIL} -u 'https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data' -k '/netcdf-c' list
|
||||
fi
|
||||
|
||||
@ -90,4 +90,3 @@ if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testallcases zip; fi
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then testallcases s3; fi
|
||||
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup
|
||||
|
||||
|
||||
@ -182,6 +182,8 @@ atexit() {
|
||||
trap atexit_cleanup EXIT
|
||||
}
|
||||
|
||||
GDBB="gdb -batch -ex r -ex bt -ex q --args"
|
||||
|
||||
resetrc
|
||||
atexit
|
||||
|
||||
|
||||
@ -286,6 +286,7 @@ rootpathfor(const char* path)
|
||||
case NCZM_ZIP:
|
||||
rootpath = strdup("/"); /*constant*/
|
||||
break;
|
||||
#ifdef ENABLE_S3
|
||||
case NCZM_S3:
|
||||
/* Split the path part */
|
||||
if((stat = nczm_split(uri->path,segments))) goto done;
|
||||
@ -295,6 +296,7 @@ rootpathfor(const char* path)
|
||||
/* Put it back together */
|
||||
if((stat = nczm_join(segments,&rootpath))) goto done;
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
stat = NC_EINVAL;
|
||||
goto done;
|
||||
|
||||
@ -38,7 +38,7 @@ FEATURE_S3_AWS=@HAS_S3_AWS@
|
||||
FEATURE_S3_INTERNAL=@HAS_S3_INTERNAL@
|
||||
FEATURE_S3=@HAS_S3@
|
||||
FEATURE_NCZARR=@HAS_NCZARR@
|
||||
FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@
|
||||
FEATURE_S3TESTS=@DO_S3_TESTING@
|
||||
FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@
|
||||
FEATURE_LARGE_TESTS=@DO_LARGE_TESTS@
|
||||
|
||||
|
||||
@ -33,7 +33,7 @@ ENDFOREACH()
|
||||
add_bin_test(unit_test test_pathcvt)
|
||||
|
||||
IF(BUILD_UTILITIES)
|
||||
IF(ENABLE_S3)
|
||||
IF(ENABLE_S3 AND WITH_S3_TESTING)
|
||||
# SDK Test
|
||||
BUILD_BIN_TEST(test_s3sdk ${XGETOPTSRC})
|
||||
ADD_SH_TEST(unit_test run_s3sdk)
|
||||
|
||||
@ -8,9 +8,9 @@
|
||||
|
||||
# Ed Hartnett 8/9/19
|
||||
|
||||
SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
|
||||
#TESTS_ENVIRONMENT = export SETX=1;
|
||||
|
||||
# Put together AM_CPPFLAGS and AM_LDFLAGS.
|
||||
@ -38,9 +38,11 @@ TESTS += tst_nc4internal
|
||||
endif # USE_NETCDF4
|
||||
|
||||
if ENABLE_S3
|
||||
if ENABLE_S3_TESTALL
|
||||
check_PROGRAMS += test_s3sdk
|
||||
TESTS += run_s3sdk.sh
|
||||
endif
|
||||
endif
|
||||
|
||||
EXTRA_DIST = CMakeLists.txt run_s3sdk.sh
|
||||
EXTRA_DIST += nctest_netcdf4_classic.nc
|
||||
|
||||
@ -362,7 +362,6 @@ testsearch(void)
|
||||
|
||||
done:
|
||||
cleanup();
|
||||
NC_s3sdkclose(s3client,&s3info,0,NULL); s3client = NULL;
|
||||
for(i=0;i<nkeys;i++) nullfree(keys[i]);
|
||||
nullfree(keys);
|
||||
return stat;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user