netcdf-c/nczarr_test/test_nczarr.sh
Dennis Heimbigner 6b69b9c52c Significantly Improve Amazon S3 Cloud Storage Support
## S3 Related Fixes

* Add comprehensive support for specifying AWS profiles to provide access credentials.
* Parse the files "~/.aws/config" and "~/.aws/credentials to provide credentials for the HDF5 ROS3 driver and to locate default region.
* Add a function to obtain the currently active S3 credentials. The search rules are defined in docs/nczarr.md.
* Provide documentation for the new features.
* Modify the struct NCauth (in include/ncauth.h) to replace specific S3 credentials with a profile name.
* Add a unit test to test the operation of profile and credentials management.
* Add support for URLS of the form "s3://<bucket>/<key>"; this requires obtaining a default region.
* Allows the specification of profile and/or region in a URL of the form "#mode=nczarr,...&aws.region=...&aws.profile=..."

## Misc. Fixes

* Move the ezxml code to libdispatch so that it can be used both by DAP4 and nczarr.
* Modify nclist to provide a deep clone operation.
* Modify ncuri to provide a deep clone operation.
* Modify the .rc file format to allow the specification of a path to be tested when looking for an entry in the .rc file.
* Ensure that the NC_rcload function is called.
* Modify nchttp to support setting request headers.
2021-09-27 18:36:33 -06:00

153 lines
3.5 KiB
Bash
Executable File

#!/bin/sh
if test "x$SETX" != x; then set -x; fi
# Figure out which cloud repo to use
if test "x$NCZARR_S3_TEST_HOST" = x ; then
# export NCZARR_S3_TEST_HOST=stratus.ucar.edu
export NCZARR_S3_TEST_HOST=s3.us-east-1.amazonaws.com
fi
if test "x$NCZARR_S3_TEST_BUCKET" = x ; then
# export NCZARR_S3_TEST_BUCKET=unidata-netcdf-zarr-testing
export NCZARR_S3_TEST_BUCKET=unidata-zarr-test-data
fi
export NCZARR_S3_TEST_URL="https://${NCZARR_S3_TEST_HOST}/${NCZARR_S3_TEST_BUCKET}"
ZMD="${execdir}/zmapio"
awsdelete() {
${execdir}/s3util ${PROFILE} -u "${NCZARR_S3_TEST_URL}" -k "$1" clear
# aws s3api delete-object --endpoint-url=https://${NCZARR_S3_TEST_HOST} --bucket=${NCZARR_S3_TEST_BUCKET} --key="netcdf-c/$1"
X=
}
# Check settings
checksetting() {
if test -f ${TOPBUILDDIR}/libnetcdf.settings ; then
local PATTERN
PATTERN="${1}:[ ]*yes"
if grep "$PATTERN" <${TOPBUILDDIR}/libnetcdf.settings ; then
HAVE_SETTING=1
else
unset HAVE_SETTING
fi
fi
}
checkprops() {
specflag=
headflag=
isxfail=
# determine if this is an xfailtest
for t in ${XFAILTESTS} ; do
if test "x${t}" = "x${x}" ; then isxfail=1; fi
done
for t in ${SPECTESTS} ; do
if test "x${t}" = "x${f}" ; then specflag="-s"; fi
done
for t in ${HEADTESTS} ; do
if test "x${t}" = "x${f}" ; then headflag="-h"; fi
done
}
extfor() {
case "$1" in
file) zext="file" ;;
zip) zext="zip" ;;
s3) zext="s3" ;;
*) echo "unknown kind: $1" ; exit 1;;
esac
}
deletemap() {
case "$1" in
file) rm -fr $2;;
zip) rm -f $2;;
s3) S3KEY=`${execdir}/zs3parse -k $2`; awsdelete $S3KEY;;
*) echo "unknown kind: $1" ; exit 1;;
esac
}
mapstillexists() {
mapstillexists=0
if "./zmapio $fileurl" ; then
echo "delete failed: $1"
mapstillexists=1
fi
}
fileargs() {
f="$1"
frag="$2"
if test "x$frag" = x ; then frag="mode=nczarr,$zext" ; fi
case "$zext" in
s3)
S3PATH="${NCZARR_S3_TEST_URL}/netcdf-c"
fileurl="${S3PATH}/${f}#${frag}"
file=$fileurl
S3HOST=`${execdir}/zs3parse -h $S3PATH`
S3BUCKET=`${execdir}/zs3parse -b $S3PATH`
S3PREFIX=`${execdir}/zs3parse -k $S3PATH`
;;
*)
file="${f}.$zext"
fileurl="file://${f}.$zext#${frag}"
;;
esac
}
dumpmap() {
zext=$1
zbase=`basename $2 ".$zext"`
fileargs $zbase
${execdir}/zmapio -t int -x objdump $fileurl > $3
}
# Function to remove selected -s attributes from file;
# These attributes might be platform dependent
sclean() {
cat $1 \
| sed -e '/:_IsNetcdf4/d' \
| sed -e '/:_Endianness/d' \
| sed -e '/_NCProperties/d' \
| sed -e '/_SuperblockVersion/d' \
| cat > $2
}
# Make sure execdir and srcdir absolute paths are available
WD=`pwd`
cd $srcdir ; abs_srcdir=`pwd` ; cd $WD
cd $execdir ; abs_execdir=`pwd` ; cd $WD
# Clear out any existing .rc files
WD=`pwd`
if test "x$NCAUTH_HOMETEST" != x ; then RCHOME=1; fi
# Set plugin path
#cd ../plugins; make clean all >/dev/null; cd ../nczarr_test
# Load the findplugins function
. ${builddir}/findplugin.sh
echo "findplugin.sh loaded"
# Locate the plugin path and the library names; argument order is critical
# Find bzip2 and capture
# Assume all test filters are in same plugin dir
findplugin h5misc
echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}"
export HDF5_PLUGIN_PATH
resetrc() {
if test "x$RCHOME" = x1 ; then
rm -f ${HOME}/.dodsrc ${HOME}/.daprc ${HOME}/.ncrc
fi
rm -f ${WD}/.dodsrc ${WD}/.daprc ${WD}/.ncrc
unset NCRCENV_IGNORE
unset NCRCENV_RC
unset DAPRCFILE
}
resetrc