mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-02-05 16:20:10 +08:00
df3636b959
This PR started as an attempt to add unlimited dimensions to NCZarr. It did that, but this exposed significant problems with test interference. So this PR is mostly about fixing -- well mitigating anyway -- test interference. The problem of test interference is now documented in the document docs/internal.md. The solutions implemented here are also describe in that document. The solution is somewhat fragile but multiple cleanup mechanisms are provided. Note that this feature requires that the AWS command line utility must be installed. ## Unlimited Dimensions. The existing NCZarr extensions to Zarr are modified to support unlimited dimensions. NCzarr extends the Zarr meta-data for the ".zgroup" object to include netcdf-4 model extensions. This information is stored in ".zgroup" as dictionary named "_nczarr_group". Inside "_nczarr_group", there is a key named "dims" that stores information about netcdf-4 named dimensions. The value of "dims" is a dictionary whose keys are the named dimensions. The value associated with each dimension name has one of two forms Form 1 is a special case of form 2, and is kept for backward compatibility. Whenever a new file is written, it uses format 1 if possible, otherwise format 2. * Form 1: An integer representing the size of the dimension, which is used for simple named dimensions. * Form 2: A dictionary with the following keys and values" - "size" with an integer value representing the (current) size of the dimension. - "unlimited" with a value of either "1" or "0" to indicate if this dimension is an unlimited dimension. For Unlimited dimensions, the size is initially zero, and as variables extend the length of that dimension, the size value for the dimension increases. That dimension size is shared by all arrays referencing that dimension, so if one array extends an unlimited dimension, it is implicitly extended for all other arrays that reference that dimension. This is the standard semantics for unlimited dimensions. Adding unlimited dimensions required a number of other changes to the NCZarr code-base. These included the following. * Did a partial refactor of the slice handling code in zwalk.c to clean it up. * Added a number of tests for unlimited dimensions derived from the same test in nc_test4. * Added several NCZarr specific unlimited tests; more are needed. * Add test of endianness. ## Misc. Other Changes * Modify libdispatch/ncs3sdk_aws.cpp to optionally support use of the AWS Transfer Utility mechanism. This is controlled by the ```#define TRANSFER```` command in that file. It defaults to being disabled. * Parameterize both the standard Unidata S3 bucket (S3TESTBUCKET) and the netcdf-c test data prefix (S3TESTSUBTREE). * Fixed an obscure memory leak in ncdump. * Removed some obsolete unit testing code and test cases. * Uncovered a bug in the netcdf-c handling of big-endian floats and doubles. Have not fixed yet. See tst_h5_endians.c. * Renamed some nczarr_tests testcases to avoid name conflicts with nc_test4. * Modify the semantics of zmap\#ncsmap_write to only allow total rewrite of objects. * Modify the semantics of zodom to properly handle stride > 1. * Add a truncate operation to the libnczarr zmap code.
230 lines
7.7 KiB
Plaintext
230 lines
7.7 KiB
Plaintext
# Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
|
|
# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
|
|
# 2015, 2016, 2017, 2018
|
|
# University Corporation for Atmospheric Research/Unidata.
|
|
|
|
# See netcdf-c/COPYRIGHT file for more info.
|
|
|
|
# Load only once
|
|
if test "x$TEST_COMMON_SH" = x ; then
|
|
export TEST_COMMON_SH=1
|
|
|
|
# Define various global constants
|
|
|
|
# Define location of execution
|
|
TOPSRCDIR='@abs_top_srcdir@'
|
|
TOPBUILDDIR='@abs_top_builddir@'
|
|
FP_ISCMAKE=@ISCMAKE@
|
|
FP_ISMSVC=@ISMSVC@
|
|
FP_WINVERMAJOR=@WINVERMAJOR@
|
|
FP_WINVERBUILD=@WINVERBUILD@
|
|
FP_ISCYGWIN=@ISCYGWIN@
|
|
FP_ISMINGW=@ISMINGW@
|
|
FP_ISMSYS=@ISMSYS@
|
|
FP_ISOSX=@ISOSX@
|
|
|
|
FP_ISREGEDIT=@ISREGEDIT@
|
|
FP_USEPLUGINS=@USEPLUGINS@
|
|
|
|
# Feature flags
|
|
FEATURE_HDF5=@HAS_HDF5@
|
|
FEATURE_PARALLEL=@HAS_PARALLEL@
|
|
|
|
# Define selected features of the build
|
|
FEATURE_HDF5=@HAS_HDF5@
|
|
FEATURE_FILTERTESTS=@DO_FILTER_TESTS@
|
|
FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@
|
|
FEATURE_BYTERANGE=@HAS_BYTERANGE@
|
|
FEATURE_ROS3=@HAS_HDF5_ROS3@
|
|
FEATURE_S3_AWS=@HAS_S3_AWS@
|
|
FEATURE_S3_INTERNAL=@HAS_S3_INTERNAL@
|
|
FEATURE_S3=@HAS_S3@
|
|
FEATURE_NCZARR=@HAS_NCZARR@
|
|
FEATURE_S3TESTS=@ENABLE_S3_TESTING@
|
|
FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@
|
|
FEATURE_LARGE_TESTS=@DO_LARGE_TESTS@
|
|
|
|
# Thredds-test server is currently disabled
|
|
#FEATURE_THREDDSTEST=1
|
|
|
|
# This is the Unidata S3 test bucket
|
|
# All S3 tests should use this to store intermediate results.
|
|
S3TESTBUCKET=@S3TESTBUCKET@
|
|
|
|
# This is the s3 path within the Unidata bucket;
|
|
# All S3 tests should use this to store intermediate results.
|
|
S3TESTSUBTREE=@S3TESTSUBTREE@
|
|
|
|
TESTUID=@TESTUID@
|
|
|
|
set -e
|
|
|
|
# Figure out various locations in the src/build tree.
|
|
# This is relatively fragile code and is essentially
|
|
# specific to netcdf-c. It does, however, have the virtue
|
|
# of isolating all this nonsense into one place.
|
|
# This will get somewhat simplified (I hope) when
|
|
# we move to a separate test_utilities directory
|
|
|
|
# This code is intended to provide constants
|
|
# for accessing various objects in the src/build
|
|
# tree(s) across multiple ways of building netcdf-c.
|
|
# Currently, the following build situations are supported.
|
|
# 1. Autoconf with make check: the src and build trees are the same
|
|
# 2. Autoconf with make distcheck: the src and build trees are distinct
|
|
# 3. Cmake on a *nix platform using e.g. gcc:
|
|
# the src and build trees are distinct.
|
|
# 4. Cmake on windows using cygwin or msys.
|
|
# The src and build trees are distinct.
|
|
#
|
|
# For now, an explicit build using the Visual C(++) compiler
|
|
# is not supported. The big issue is the handling of executables
|
|
# and the notion of a VS configuration/build type like Debug or Release.
|
|
# When using VS, executables are placed in a subdirectory of the
|
|
# build directory. That subdirectory is named by the configuration type.
|
|
# Thus one finds ncdump.exe in $top_builddir/ncdump/Debug instead of
|
|
# $top_builddir/ncdump. An additional issue is the extension of an
|
|
# executable: .exe vs nothing. This code attempts to figure out which is used.
|
|
#
|
|
# For possible future fixes, a placeholder is left in place in the
|
|
# following code named VS. If it were set to the build type, then,
|
|
# in theory, this code would work with Visual C. It is disabled for now.
|
|
#
|
|
# The goal, then, of this common code is to set up some useful
|
|
#constants for use in test shell scripts.
|
|
# 1. srcdir - absolute path to the source dir (e.g. ${top_srcdir}/ncgen)
|
|
# 2. top_srcdir - absolute path to the root of the source
|
|
# 3. top_builddir - absolute path to the root of the build directory;
|
|
# may be same as top_srcdir (e.g. #1).
|
|
# 4. builddir - absolute path of th the directory into which generated
|
|
# stuff (.nc, .cdl, etc) is stored.
|
|
# 5. execdir - absolute path of the directory into which executables are
|
|
# placed. For all but the VS case, execdir == builddir.
|
|
#
|
|
# The following are defined to support inter-directory references.
|
|
# 6. NCDUMP - absolute path to the ncdump.exe executable
|
|
# 7. NCCOPY - absolute path to the nccopy.exe executable
|
|
# 8. NCGEN - absolute path to ncgen.exe
|
|
# 9. NCGEN3 - absolute path to ncgen3.exe
|
|
#10. NCPATHCVT - absolute path to ncpathcvt.exe
|
|
|
|
# Allow global set -x mechanism for debugging.
|
|
if test "x$SETX" = x1 ; then set -x ; fi
|
|
|
|
# On MINGW, bash and other POSIX utilities use a mounted root directory,
|
|
# but executables compiled for Windows do not recognise the mount point.
|
|
# Here we ensure that Windows paths are used in tests of Windows executables.
|
|
|
|
system=`uname`
|
|
if test "x${system##MINGW*}" = x; then
|
|
alias pwd='pwd -W'
|
|
|
|
fi
|
|
|
|
# We assume that TOPSRCDIR and TOPBUILDDIR are defined
|
|
# At the top of this shell script
|
|
top_srcdir="$TOPSRCDIR"
|
|
top_builddir="$TOPBUILDDIR"
|
|
|
|
# Currently not used, but left as a Visual Studio placeholder.
|
|
# VS=Debug
|
|
|
|
# srcdir may or may not be defined, but if not, then create it
|
|
if test "x$srcdir" = x ; then
|
|
# we need to figure out our directory
|
|
# pick off the last component as the relative name of this directory
|
|
srcdir=`pwd`
|
|
if test "x$srcdir" != "x$top_srcdir" ; then
|
|
current=`basename $srcdir`
|
|
srcdir="${top_srcdir}/$current"
|
|
fi
|
|
fi
|
|
|
|
# We also assume we are executing in builddir
|
|
builddir=`pwd`
|
|
|
|
# execdir is an alias for builddir
|
|
execdir="${builddir}"
|
|
|
|
# pick off the last component as the relative name of this directory
|
|
thisdir=`basename $srcdir`
|
|
|
|
WD=`pwd`
|
|
# Absolutize paths of interest
|
|
cd $srcdir; srcdir=`pwd` ; cd $WD
|
|
cd $top_srcdir; top_srcdir=`pwd` ; cd $WD
|
|
cd $builddir; builddir=`pwd` ; cd $WD
|
|
cd $top_builddir; top_builddir=`pwd` ; cd $WD
|
|
cd $execdir; execdir=`pwd` ; cd $WD
|
|
|
|
# For sun os
|
|
export srcdir top_srcdir builddir top_builddir execdir
|
|
|
|
# Figure out executable extension (probably a better way)
|
|
if test -e "${top_builddir}/ncdump${VS}/ncdump.exe" ; then
|
|
ext=".exe"
|
|
else
|
|
ext=""
|
|
fi
|
|
|
|
# We need to locate certain executables (and other things),
|
|
# capture absolute paths, and make visible
|
|
export NCDUMP="${top_builddir}/ncdump${VS}/ncdump${ext}"
|
|
export NCCOPY="${top_builddir}/ncdump${VS}/nccopy${ext}"
|
|
export NCGEN="${top_builddir}/ncgen${VS}/ncgen${ext}"
|
|
export NCGEN3="${top_builddir}/ncgen3${VS}/ncgen3${ext}"
|
|
export NCPATHCVT="${top_builddir}/ncdump${VS}/ncpathcvt${ext}"
|
|
|
|
# Temporary hacks (until we have a test_utils directory)
|
|
# to locate certain specific test files
|
|
ncgen3c0="${top_srcdir}/ncgen3/c0.cdl"
|
|
ncgenc0="${top_srcdir}/ncgen/c0.cdl"
|
|
ncgenc04="${top_srcdir}/ncgen/c0_4.cdl"
|
|
|
|
# Set LC_ALL
|
|
if test "x$FP_ISMSVC" = xyes || test "x$FP_ISCYGWIN" = xyes; then export LC_ALL="en_US.utf8"; fi
|
|
|
|
# Set HOME
|
|
if test "x$FP_ISMSVC" = xyes || test "x$FP_MINGW" = xyes; then
|
|
if test "x$HOME" = x ; then
|
|
HOME=`echo $USERPROFILE |tr '\\\' '/'`
|
|
export HOME
|
|
fi
|
|
fi
|
|
|
|
# OS/X apparently has no echo -n option, so fake it
|
|
echon() { echo "$@" | tr -d '\r\n' ; }
|
|
|
|
# Test for filter availability
|
|
avail() {
|
|
if test yes = `${execdir}/../ncdump/ncfilteravail $1` ; then return 0 ; else echo "filter $1 not available" ; return 1; fi
|
|
}
|
|
|
|
|
|
# Make sure we are in builddir (not execdir)
|
|
cd $builddir
|
|
|
|
|
|
# Parallel make can cause inter-test interference (mostly because of historical naming issues).
|
|
# As a protection against this, the isolate() function supports the creation of an
|
|
# isolation directory in which created products are stored.
|
|
# Cleanup can be accomplished by deleting the whole isolation directory.
|
|
# The name of the isolation directory is by convention "testdir_<random>".
|
|
# The isolation dir is created within the ${builddir} directory.
|
|
# The <random> is a generated 32 bit unsigned random integer to make the chance
|
|
# of collision very small.
|
|
# Process ID was not used because of the small but real chance of collision.
|
|
|
|
isolate() {
|
|
if test "x$ISOPATH" = x ; then
|
|
ISOTESTSET="${builddir}/testset_"
|
|
if test "x$NOISOPATH" = x ; then ISOTESTSET="${ISOTESTSET}${TESTUID}"; fi
|
|
ISODIR="$1"
|
|
ISOPATH="${ISOTESTSET}/$ISODIR"
|
|
rm -fr $ISOPATH
|
|
mkdir -p $ISOPATH
|
|
fi
|
|
}
|
|
|
|
fi #TEST_COMMON_SH
|