mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-02-23 16:59:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into deflate.dmh

Browse Source
This commit is contained in:
Ward Fisher 2021-02-08 11:10:06 -06:00 committed by GitHub
commit 0b6f5c7a27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
94 changed files with 2886 additions and 1271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
CMakeLists.txt
View File

@ -834,7 +834,7 @@ IF(USE_HDF5)
#option to include HDF5 High Level header file (hdf5_hl.h) in case we are not doing a make install
INCLUDE_DIRECTORIES(${HDF5_HL_INCLUDE_DIR})
# Check to see if we have libcurl 7.66 or later
# Check HDF5 version
CHECK_C_SOURCE_COMPILES("
#include <H5public.h>
int main() {
@ -942,6 +942,17 @@ OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF)
OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON)
SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
# See if we have libzip
FIND_PACKAGE(Zip)
# Define a test flag for have curl library
IF(Zip_FOUND)
INCLUDE_DIRECTORIES(${Zip_INCLUDE_DIRS})
SET(ENABLE_NCZARR_ZIP TRUE)
ELSE()
SET(ENABLE_NCZARR_ZIP FALSE)
ENDIF()
# Enable some developer-only tests
OPTION(ENABLE_EXTRA_TESTS "Enable Extra tests. Some may not work because of known issues. Developers only." OFF)
IF(ENABLE_EXTRA_TESTS)
@ -1017,6 +1028,10 @@ IF(NOT ENABLE_S3_SDK)
ENDIF()
ENDIF()
IF(ENABLE_NCZARR_S3_TESTS)
message(WARNING "**** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***")
ENDIF()
# Start disabling if curl not found
IF(NOT FOUND_CURL)
IF(ENABLE_BYTERANGE)
@ -1036,6 +1051,11 @@ IF(NOT FOUND_CURL)
ENDIF()
ENDIF(NOT FOUND_CURL)
IF(NOT ENABLE_BYTERANGE AND ENABLE_HDF5_ROS3)
MESSAGE(WARNING "ROS3 support requires ENABLE_BYTERANGE=TRUE; disabling ROS3 support")
SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "ROS3 support" FORCE)
ENDIF()
##
# Enable Tests
##
@ -2145,8 +2165,9 @@ is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
is_enabled(HAVE_H5Z_SZIP HAS_SZLIB)
is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
is_enabled(ENABLE_NCZARR HAS_NCZARR)
is_enabled(ENABLE_NCZARR_S3_TESTS DO_S3_TESTS)
is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS)
is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)
is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
# Generate file from template.
CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"

1
RELEASE_NOTES.md
View File

@ -8,6 +8,7 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.0 - TBD
* [Bug Fixes] The nccopy program was treating -d0 as turning deflation on rather than interpreting it as "turn off deflation". See [Github #1944](https://github.com/Unidata/netcdf-c/pull/1944) for more information.
* [Enhancement] Add support for storing NCZarr data in zip files. See [Github #1942](https://github.com/Unidata/netcdf-c/pull/1942) for more information.
* [Bug Fixes] Make fillmismatch the default for DAP2 and DAP4; too many servers ignore this requirement.
* [Bug Fixes] Fix some memory leaks in NCZarr, fix a bug with long strides in NCZarr. See [Github #1913](https://github.com/Unidata/netcdf-c/pull/1913) for more information.
* [Enhancement] Add some optimizations to NCZarr, dosome cleanup of code cruft, add some NCZarr test cases, add a performance test to NCZarr. See [Github #1908](https://github.com/Unidata/netcdf-c/pull/1908) for more information.

64
cmake/modules/FindZip.cmake Executable file
View File

@ -0,0 +1,64 @@
# Searches for an installation of the zip library. On success, it sets the following variables:
#
# Zip_FOUND Set to true to indicate the zip library was found
# Zip_INCLUDE_DIRS The directory containing the header file zip/zip.h
# Zip_LIBRARIES The libraries needed to use the zip library
#
# To specify an additional directory to search, set Zip_ROOT.
#
# Author: Siddhartha Chaudhuri, 2009
#
# Look for the header, first in the user-specified location and then in the system locations
SET(Zip_INCLUDE_DOC "The directory containing the header file zip/zip.h")
FIND_PATH(Zip_INCLUDE_DIRS NAMES zip.h zip/zip.h PATHS ${Zip_ROOT} ${Zip_ROOT}/include DOC ${Zip_INCLUDE_DOC} NO_DEFAULT_PATH)
IF(NOT Zip_INCLUDE_DIRS) # now look in system locations
FIND_PATH(Zip_INCLUDE_DIRS NAMES zip.h zip/zip.h DOC ${Zip_INCLUDE_DOC})
ENDIF(NOT Zip_INCLUDE_DIRS)
SET(Zip_FOUND FALSE)
IF(Zip_INCLUDE_DIRS)
SET(Zip_LIBRARY_DIRS ${Zip_INCLUDE_DIRS})
IF("${Zip_LIBRARY_DIRS}" MATCHES "/include$")
# Strip off the trailing "/include" in the path.
GET_FILENAME_COMPONENT(Zip_LIBRARY_DIRS ${Zip_LIBRARY_DIRS} PATH)
ENDIF("${Zip_LIBRARY_DIRS}" MATCHES "/include$")
IF(EXISTS "${Zip_LIBRARY_DIRS}/lib")
SET(Zip_LIBRARY_DIRS ${Zip_LIBRARY_DIRS}/lib)
ENDIF(EXISTS "${Zip_LIBRARY_DIRS}/lib")
# Find Zip libraries
FIND_LIBRARY(Zip_DEBUG_LIBRARY NAMES zipd zip_d libzipd libzip_d libzip
PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug
PATHS ${Zip_LIBRARY_DIRS} NO_DEFAULT_PATH)
FIND_LIBRARY(Zip_RELEASE_LIBRARY NAMES zip libzip
PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release
PATHS ${Zip_LIBRARY_DIRS} NO_DEFAULT_PATH)
SET(Zip_LIBRARIES )
IF(Zip_DEBUG_LIBRARY AND Zip_RELEASE_LIBRARY)
SET(Zip_LIBRARIES debug ${Zip_DEBUG_LIBRARY} optimized ${Zip_RELEASE_LIBRARY})
ELSEIF(Zip_DEBUG_LIBRARY)
SET(Zip_LIBRARIES ${Zip_DEBUG_LIBRARY})
ELSEIF(Zip_RELEASE_LIBRARY)
SET(Zip_LIBRARIES ${Zip_RELEASE_LIBRARY})
ENDIF(Zip_DEBUG_LIBRARY AND Zip_RELEASE_LIBRARY)
IF(Zip_LIBRARIES)
SET(Zip_FOUND TRUE)
ENDIF(Zip_LIBRARIES)
ENDIF(Zip_INCLUDE_DIRS)
IF(Zip_FOUND)
# IF(NOT Zip_FIND_QUIETLY)
MESSAGE(STATUS "Found Zip: headers at ${Zip_INCLUDE_DIRS}, libraries at ${Zip_LIBRARY_DIRS}")
MESSAGE(STATUS " library is ${Zip_LIBRARIES}")
# ENDIF(NOT Zip_FIND_QUIETLY)
ELSE(Zip_FOUND)
IF(Zip_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Zip library not found")
ENDIF(Zip_FIND_REQUIRED)
ENDIF(Zip_FOUND)

3
config.h.cmake.in
View File

@ -156,6 +156,9 @@ are set when opening a binary file on Windows. */
/* if true, enable S3 support */
#cmakedefine ENABLE_NCZARR_S3 1
/* if true, enable nczarr zip support */
#cmakedefine ENABLE_NCZARR_ZIP 1
/* if true, enable S3 testing*/
#cmakedefine ENABLE_NCZARR_S3_TESTS 1

22
configure.ac
View File

@ -629,6 +629,27 @@ if test "x$enable_nczarr_s3_tests" = xyes ; then
fi
AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes])
if test "x$enable_nczarr_s3_tests" = xyes ; then
AC_MSG_WARN([*** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***])
fi
# Set default
# See if we have libzip
AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
if test "x$have_zip" = "xyes" ; then
AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [],
[AC_MSG_ERROR([libzip search failed.])])
fi
AC_MSG_CHECKING([whether libzip library is available])
AC_MSG_RESULT([${have_zip}])
enable_nczarr_zip=${have_zip} # alias
if test "x$enable_nczarr_zip" = xyes ; then
AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
fi
AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
# Did the user specify a default cache size for NCZarr?
AC_MSG_CHECKING([whether a default file cache size for NCZarr was specified])
AC_ARG_WITH([chunk-cache-size-nczarr],
@ -1589,6 +1610,7 @@ AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters])
AC_SUBST(HAS_NCZARR,[$enable_nczarr])
AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
# Include some specifics for netcdf on windows.
#AH_VERBATIM([_WIN32_STRICMP],

2
dap4_test/Makefile.am
View File

@ -73,7 +73,7 @@ clean-local: clean-local-check
clean-local-check:
-rm -rf results results_test_parse results_test_data \
results_test_hyrax results_test_meta
results_test_hyrax results_test_meta results_test_raw
-rm -f .dodsrc .daprc
# The shell file maketests.sh is used to build the testdata

9
dap4_test/dump.c
View File

@ -21,12 +21,12 @@
#endif
#include "netcdf.h"
#include "netcdf_aux.h"
#include "ncbytes.h"
#include "ncpathmgr.h"
extern void NCD4_dumpbytes(size_t size, const void* data0, int swap);
extern void NCD4_tagdump(size_t size, const void* data0, int swap, const char* tag);
extern int NC_readfile(const char* filename, NCbytes* content);
static char* progname = NULL;
@ -42,7 +42,6 @@ main(int argc, char *argv[])
{
int c;
char* fname = NULL;
NCbytes* contents = ncbytesnew();
char* tag = NULL;
size_t offset = 0;
size_t len = 0;
@ -81,12 +80,10 @@ main(int argc, char *argv[])
}
if(tag == NULL) tag = strdup(progname);
if(NC_readfile(fname,contents)) usage();
len = ncbyteslength(contents) - offset;
data = (char*)ncbytescontents(contents);
if(ncaux_readfile(fname,&len,&((void*)data))) usage();
data += offset;
NCD4_tagdump(len,data,swap,tag);
nullfree(data);
return 0;
}

55
debug/cf.cmake
View File

@ -1,20 +1,22 @@
# Visual Studio
NCC="c:/tools/hdf5"
#export SETX=1
NCC="c:/tools/hdf5-1.10.6"
HDF5_DIR="$NCC/cmake/hdf5"
AWSSDK_DIR="c:/tools/aws-cpp-sdk-all"
# Is netcdf-4 and/or DAP enabled?
NCZARR=1
HDF5=1
DAP=1
S3=1
#S3TEST=1
CDF5=1
HDF4=1
S3TEST=1
#CDF5=1
#HDF4=1
#TR=--trace
export SETX=1
for arg in "$@" ; do
case "$arg" in
vs|VS) VS=1 ;;
@ -25,23 +27,13 @@ notest|nt) NOTEST=1 ;;
esac
done
if test "x$VS" = x1 ; then
if test "x$2" = xsetup ; then
VSSETUP=1
else
unset VSSETUP
fi
fi
#TESTSERVERS="localhost:8080,remotetest.unidata.ucar.edu"
#export NCPATHDEBUG=1
if test "x$VSSETUP" = x1 ; then
CFG="Debug"
else
#CFG="Debug"
CFG="Release"
fi
FLAGS=
@ -68,9 +60,9 @@ ignore=1
#FLAGS="$FLAGS -DDEFAULT_API_VERSION:STRING=v110"
#FLAGS="$FLAGS -DHDF5_ROOT=c:/tools/hdf5"
#FLAGS="$FLAGS -DHDF5_ROOT_DIR_HINT=c:/tools/hdf5/cmake/hdf5/hdf5-config.cmake"
FLAGS="$FLAGS -DHDF5_DIR=c:/tools/hdf5/cmake/hdf5"
#FLAGS="$FLAGS -DHDF5_DIR=/home/dmh/tools/share/cmake/hdf5"
#hdf5-config.cmake
#FLAGS="-DHDF5_LIBRARIES=${NCC}/lib/hdf5 -DHDF5_HL_LIBRARY=${NCC}/lib/hdf5_hl -DHDF5_INCLUDE_DIR=${NCC}/include"
#FLAGS="-DHDF5_C_LIBRARY=${NCC}/lib/libhdf5.so -DHDF5_HL_LIBRARY=${NCC}/lib/libhdf5_hl.so -DHDF5_INCLUDE_DIR=${NCC}/include"
fi
if test "x$CDF5" != x ; then
FLAGS="$FLAGS -DENABLE_CDF5=true"
@ -89,31 +81,28 @@ fi
if test "x$S3" != x ; then
FLAGS="$FLAGS -DENABLE_NCZARR_S3=true"
FLAGS="$FLAGS -DAWSSDK_DIR=${AWSSDK_DIR}"
if test "x$S3TEST" != x ; then
FLAGS="$FLAGS -DENABLE_NCZARR_S3_TESTS=true"
fi
else
FLAGS="$FLAGS -DENABLE_S3_SDK=false"
FLAGS="$FLAGS -DENABLE_S3_TESTS=false"
FLAGS="$FLAGS -DENABLE_NCZARR_S3=false"
FLAGS="$FLAGS -DENABLE_NCZARR_S3_TESTS=false"
fi
# Enables
# Misc.
FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true"
FLAGS="$FLAGS -DENABLE_LOGGING=true"
#FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true"
#FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true"
FLAGS="$FLAGS -DENABLE_BENCHMARKS=true"
#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
# Disables
#FLAGS="$FLAGS -DENABLE_BENCHMARKS=true"
FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
FLAGS="$FLAGS -DENABLE_BYTERANGE=true"
FLAGS="$FLAGS -DBUILD_UTILITIES=true"
FLAGS="$FLAGS -DENABLE_EXAMPLES=false"
FLAGS="$FLAGS -DENABLE_CONVERSION_WARNINGS=false"
#FLAGS="$FLAGS -DENABLE_TESTS=false"
#FLAGS="$FLAGS -DENABLE_DISKLESS=false"
FLAGS="$FLAGS -DBUILD_UTILITIES=true"
FLAGS="$FLAGS -DENABLE_FILTER_TESTING=false"
FLAGS="$FLAGS -DCURL_NO_CURL_CMAKE=TRUE"
FLAGS="$FLAGS -DENABLE_UNIT_TESTS=TRUE"
# Withs
@ -124,6 +113,9 @@ mkdir build
cd build
NCLIB=`pwd`
NCCYGLIB=`cygpath -u ${NCLIB} |tr -d ''`
NCCBIN=`cygpath -u "${NCC}/bin" |tr -d ''`
AWSSDKBIN="/cygdrive/c/tools/aws-cpp-sdk-all/bin"
if test "x$VS" != x ; then
@ -131,6 +123,7 @@ if test "x$VS" != x ; then
CFG="Release"
NCLIB="${NCLIB}/liblib"
export PATH="${NCLIB}:${PATH}"
export LD_LIBRARY_PATH="${NCCBIN}:$LD_LIBRARY_PATH:${AWSSDKBIN}:${NCCYGLIB}"
#G=
#TR=--trace
cmake ${TR} "$G" -DCMAKE_BUILD_TYPE=${CFG} $FLAGS ..
@ -147,7 +140,7 @@ else
NCLIB="${NCLIB}/build/liblib"
#G="-GUnix Makefiles"
#T="--trace-expand"
cmake "${G}" $FLAGS ..
cmake --loglevel=VERBOSE "${G}" $FLAGS ..
if test "x$NOBUILD" = x ; then
make all
fi

4
h5_test/tst_h_atts3.c
View File

@ -209,7 +209,11 @@ main()
hid_t file_typeid1[NUM_OBJ_1], native_typeid1[NUM_OBJ_1];
hid_t file_typeid2, native_typeid2;
hsize_t num_obj;
#if H5_VERSION_GE(1,12,0)
H5O_info2_t obj_info;
#else
H5O_info_t obj_info;
#endif
char obj_name[STR_LEN + 1];
hsize_t dims[1] = {ATT_LEN}; /* netcdf attributes always 1-D. */

1
include/netcdf.h
View File

@ -480,6 +480,7 @@ by the desired type. */
#define NC_ENCZARR (-137) /**< Error at NCZarr layer. */
#define NC_ES3 (-138) /**< Generic S3 error */
#define NC_EEMPTY (-139) /**< Attempt to read empty NCZarr map key */
#define NC_EFOUND (-140) /**< Some object exists when it should not */
#define NC4_LAST_ERROR (-140) /**< @internal All netCDF errors > this. */

5
include/netcdf_aux.h
View File

@ -69,6 +69,11 @@ EXTERNL int ncaux_h5filterspec_parselist(const char* txt0, int* formatp, size_t*
EXTERNL void ncaux_h5filterspec_free(struct NC_H5_Filterspec* f);
EXTERNL void ncaux_h5filterspec_fix8(unsigned char* mem, int decode);
/* Wrappers to export selected functions from libnetcdf */
EXTERNL int ncaux_readfile(const char* filename, size_t* sizep, void** content);
EXTERNL int ncaux_writefile(const char* filename, size_t size, void* content);
#if defined(__cplusplus)
}
#endif

22
libdispatch/daux.c
View File

@ -27,6 +27,7 @@ See COPYRIGHT for license information.
#include "netcdf_aux.h"
#include "ncoffsets.h"
#include "nclog.h"
#include "ncrc.h"
#include "netcdf_filter.h"
struct NCAUX_FIELD {
@ -1049,3 +1050,24 @@ done:
return stat;
}
#endif
/**************************************************/
/* Wrappers to export selected functions from libnetcdf */
EXTERNL int
ncaux_readfile(const char* filename, size_t* sizep, void** contentp)
{
int stat = NC_NOERR;
NCbytes* content = ncbytesnew();
stat = NC_readfile(filename,content);
if(stat == NC_NOERR && contentp)
*contentp = ncbytesextract(content);
ncbytesfree(content);
return stat;
}
EXTERNL int
ncaux_writefile(const char* filename, size_t size, void* content)
{
return NC_writefile(filename,size,content);
}

2
libdispatch/derror.c
View File

@ -277,6 +277,8 @@ const char *nc_strerror(int ncerr1)
return "NetCDF: AWS S3 error";
case NC_EEMPTY:
return "NetCDF: Attempt to read empty NCZarr map key";
case NC_EFOUND:
return "NetCDF: Some object exists when it should not";
default:
#ifdef USE_PNETCDF
/* The behavior of ncmpi_strerror here is to return

7
libdispatch/nclog.c
View File

@ -189,7 +189,9 @@ Send trace messages.
int
nctracelevel(int level)
{
int oldlevel = nclog_global.tracelevel;
int oldlevel;
if(!nclogginginitialized) ncloginit();
oldlevel = nclog_global.tracelevel;
if(level < 0) {
nclog_global.tracelevel = level;
ncsetlogging(0);
@ -198,7 +200,6 @@ nctracelevel(int level)
ncsetlogging(1);
nclogopen(NULL); /* use stderr */
}
fprintf(stderr,"XXX: level=%d\n",nclog_global.tracelevel);
return oldlevel;
}
@ -260,7 +261,7 @@ ncuntrace(const char* fcn, int err, const char* fmt, ...)
goto done;
}
if(frame->level <= nclog_global.tracelevel) {
fprintf(nclog_global.nclogstream,"%s: (%d): %s: ","Exit",frame->depth,frame->fcn);
fprintf(nclog_global.nclogstream,"%s: (%d): %s: ","Exit",frame->level,frame->fcn);
if(err)
fprintf(nclog_global.nclogstream,"err=(%d) '%s':",err,nc_strerror(err));
if(fmt != NULL)

4
liblib/CMakeLists.txt
View File

@ -108,6 +108,10 @@ IF(ENABLE_PNETCDF AND PNETCDF)
SET(TLL_LIBS ${TLL_LIBS} ${PNETCDF})
ENDIF()
IF(ENABLE_NCZARR_ZIP)
SET(TLL_LIBS ${TLL_LIBS} ${Zip_LIBRARIES})
ENDIF()
IF(ENABLE_S3_SDK)
TARGET_LINK_DIRECTORIES(netcdf PUBLIC ${AWSSDK_LIB_DIR})
TARGET_LINK_LIBRARIES(netcdf ${AWSSDK_LINK_LIBRARIES})

8
libnczarr/CMakeLists.txt
View File

@ -24,8 +24,7 @@ zgrp.c
zinternal.c
zjson.c
zmap.c
zmap_nz4.c
zmap_nzf.c
zmap_file.c
zodom.c
zopen.c
zprov.c
@ -48,12 +47,17 @@ zprovenance.h
zdebug.h
)
IF(ENABLE_NCZARR_ZIP)
SET(libnczarr_SOURCES ${libnczarr_SOURCES} zmap_zip.c)
ENDIF()
IF(ENABLE_S3_SDK)
SET(libnczarr_SOURCES ${libnczarr_SOURCES} zs3sdk.cpp zmap_s3sdk.c awsincludes.h zs3sdk.h)
ENDIF()
# Build the Zarr dispatch layer as a library that will be included in
# the netCDF library.
add_library(nczarr OBJECT ${libnczarr_SOURCES})
IF(MPI_C_INCLUDE_PATH)

7
libnczarr/Makefile.am
View File

@ -43,8 +43,7 @@ zgrp.c \
zinternal.c \
zjson.c \
zmap.c \
zmap_nz4.c \
zmap_nzf.c \
zmap_file.c \
zodom.c \
zopen.c \
zprov.c \
@ -67,6 +66,10 @@ zodom.h \
zprovenance.h \
zdebug.h
if ENABLE_NCZARR_ZIP
libnczarr_la_SOURCES += zmap_zip.c
endif
if ENABLE_S3_SDK
libnczarr_la_SOURCES += zmap_s3sdk.c
libnczarr_la_SOURCES += zs3sdk.cpp awsincludes.h zs3sdk.h

27
libnczarr/zmap_nz4.c → libnczarr/obsolete/zmap_nz4.c
View File

@ -52,6 +52,7 @@ static int parseurl(const char* path0, NCURI** urip);
static void nc4ify(const char* zname, char* nc4name);
static void zify(const char* nc4name, char* zname);
static int testcontentbearing(int grpid);
static int errno2ncerr(int err);
/* Define the Dataset level API */
@ -143,7 +144,7 @@ znc4open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
truepath = NULL;
if((stat=nc_open(local,mode,&ncid)))
{stat = NC_EEMPTY; goto done;} /* could not open */
goto done; /* could not open */
z4map->ncid = ncid;
if(mapp) *mapp = (NCZMAP*)z4map;
@ -153,7 +154,7 @@ done:
nullfree(local);
ncurifree(url);
if(stat) znc4close((NCZMAP*)z4map,0);
return (stat);
return errno2ncerr(stat);
}
/**************************************************/
@ -340,7 +341,6 @@ znc4search(NCZMAP* map, const char* prefix, NClist* matches)
int* vars = NULL;
int i;
NCbytes* key = ncbytesnew();
int trailing = 0;
if((stat=nczm_split(prefix,segments)))
goto done;
@ -375,17 +375,13 @@ znc4search(NCZMAP* map, const char* prefix, NClist* matches)
if((stat = nc_inq_grps(grpid,&ngrps,subgrps)))
goto done;
/* Add the subgroup keys to the list of matches (zified) */
trailing = (prefix[strlen(prefix)-1] == '/'); /* does prefix end with '/' */
for(i=0;i<ngrps;i++) {
char gname[NC_MAX_NAME];
char zname[NC_MAX_NAME];
/* See if this group is content-bearing */
if((stat = nc_inq_grpname(subgrps[i],gname))) goto done;
zify(gname,zname);
ncbytescat(key,prefix);
if(!trailing) ncbytescat(key,"/");
ncbytescat(key,zname);
nclistpush(matches,ncbytesextract(key));
nclistpush(matches,strdup(zname));
}
done:
@ -591,11 +587,26 @@ nc4ify(const char* zname, char* nc4name)
if(nc4name[0] == NCZM_DOT) nc4name[0] = ZDOTNC4;
}
/* Convert errno to closest NC_EXXX error */
static int
errno2ncerr(int err)
{
switch (err) {
case ENOENT: err = NC_ENOTFOUND; break; /* File does not exist */
case ENOTDIR: err = NC_EEMPTY; break; /* no content */
case EACCES: err = NC_EAUTH; break; /* file permissions */
case EPERM: err = NC_EAUTH; break; /* ditto */
default: break;
}
return err;
}
/**************************************************/
/* External API objects */
NCZMAP_DS_API zmap_nz4 = {
NCZM_NC4_V1,
0,
znc4create,
znc4open,
};

12
libnczarr/zarr.c
View File

@ -76,9 +76,6 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr
if((stat = nczmap_create(zinfo->features.mapimpl,nc->path,nc->mode,zinfo->features.flags,NULL,&zinfo->map)))
goto done;
/* Create super block (NCZMETAROOT) */
if((stat = ncz_create_superblock(zinfo))) goto done;
done:
ncurifree(uri);
NCJreclaim(json);
@ -386,14 +383,9 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
for(i=0;i<nclistlength(modelist);i++) {
const char* p = nclistget(modelist,i);
if(strcasecmp(p,PUREZARR)==0) zinfo->features.flags |= FLAG_PUREZARR;
else if(strcasecmp(p,"bytes")==0) zinfo->features.flags |= FLAG_BYTERANGE;
else if(strcasecmp(p,"zip")==0) zinfo->features.mapimpl = NCZM_ZIP;
else if(strcasecmp(p,"file")==0) zinfo->features.mapimpl = NCZM_FILE;
else if(strcasecmp(p,"s3")==0) zinfo->features.mapimpl = NCZM_S3;
else if(strcasecmp(p,"nz4")==0) zinfo->features.mapimpl = NCZM_NC4;
else if(strcasecmp(p,"nzf")==0) zinfo->features.mapimpl = NCZM_FILE;
else if(strcasecmp(p,"nzrf")==0)
{zinfo->features.mapimpl = NCZM_FILE; zinfo->features.flags |= FLAG_BYTERANGE;}
else if(strcasecmp(p,"nzr4")==0)
{zinfo->features.mapimpl = NCZM_NC4; zinfo->features.flags |= FLAG_BYTERANGE;}
}
/* Process other controls */
if((value = controllookup((const char**)zinfo->controls,"log")) != NULL) {

5
libnczarr/zarr.h
View File

@ -32,7 +32,7 @@ extern int NCZ_stringconvert1(nc_type typid, char* src, char** strp);
extern int NCZ_stringconvert(nc_type typid, size_t len, void* data0, NCjson** jdatap);
/* zsync.c */
extern int ncz_sync_file(NC_FILE_INFO_T* file);
extern int ncz_sync_file(NC_FILE_INFO_T* file, int isclose);
extern int ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
extern int ncz_sync_atts(NC_FILE_INFO_T*, NC_OBJ* container, NCindex* attlist);
extern int ncz_read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
@ -64,6 +64,9 @@ extern int NCZ_swapatomicdata(size_t datalen, void* data, int typesize);
extern char** NCZ_clonestringvec(size_t len, const char** vec);
extern void NCZ_freestringvec(size_t len, char** vec);
extern int NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fillchunkp);
extern int NCZ_s3clear(ZS3INFO* s3);
/* Export */
EXTERNL int NCZ_s3urlprocess(NCURI* url, ZS3INFO* s3);
/* zwalk.c */
EXTERNL int NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata);

5
libnczarr/zcache.c
View File

@ -454,7 +454,10 @@ put_chunk(NCZChunkCache* cache, const char* key, const NCZCacheEntry* entry)
case NC_NOERR: break;
case NC_EEMPTY:
/* Create the chunk */
if((stat = nczmap_defineobj(map,key))) goto done;
switch (stat = nczmap_defineobj(map,key)) {
case NC_NOERR: case NC_EFOUND: break;
default: goto done;
}
/* write again */
if((stat = nczmap_write(map,key,0,cache->chunksize,entry->data)))
goto done;

4
libnczarr/zcreate.c
View File

@ -49,8 +49,10 @@ ncz_create_file(const char *path, int cmode, size_t initialsz, const char** cont
h5->mem.persist = ((cmode & NC_PERSIST) == NC_PERSIST);
/* Do format specific setup */
/* Should check if file already exists, and if NC_NOCLOBBER is specified,
return an error */
return an error; but defer to the map */
if((retval = ncz_create_dataset(h5,h5->root_grp,controls)))
BAIL(retval);

9
libnczarr/zdebug.h
View File

@ -8,8 +8,8 @@
#undef ZDEBUG /* general debug */
#undef ZDEBUG1 /* detailed debug */
#define ZCATCH /* Warning: significant performance impact */
#define ZTRACING /* Warning: significant performance impact */
#undef ZCATCH /* Warning: significant performance impact */
#undef ZTRACING /* Warning: significant performance impact */
#include "ncexternl.h"
#include "nclog.h"
@ -32,8 +32,9 @@ EXTERNL int zthrow(int err, const char* fname, const char* fcn, int line);
#define ZUNTRACEX(e,fmt,...) ncuntrace(__func__,(e),fmt,##__VA_ARGS__)
#else
#define ZTRACE(level,fmt,...)
#define ZUNTRACE(e)
#define ZUNTRACEX(e,fmt,...)
#define ZTRACEMORE(level,fmt,...)
#define ZUNTRACE(e) (e)
#define ZUNTRACEX(e,fmt,...) (e)
#endif
/* printers */

16
libnczarr/zfile.c
View File

@ -16,7 +16,7 @@
/* Forward */
static int NCZ_enddef(int ncid);
static int ncz_sync_netcdf4_file(NC_FILE_INFO_T* file);
static int ncz_sync_netcdf4_file(NC_FILE_INFO_T* file, int isclose);
/**
* @internal Put the file back in redef mode. This is done
@ -137,8 +137,8 @@ done:
int
NCZ_sync(int ncid)
{
NC_FILE_INFO_T* file = NULL;
int stat = NC_NOERR;
NC_FILE_INFO_T* file = NULL;
ZTRACE(0,"ncid=%d",ncid);
@ -157,7 +157,9 @@ NCZ_sync(int ncid)
return stat;
}
return ncz_sync_netcdf4_file(file);
/* do not do this if file is writeonce */
stat = ncz_sync_netcdf4_file(file,!ZCLOSE);
return stat;
}
/**
@ -244,7 +246,7 @@ ncz_closeorabort(NC_FILE_INFO_T* h5, void* params, int abort)
if(h5->flags & NC_INDEF) h5->flags ^= NC_INDEF;
/* Sync the file unless this is a read-only file. */
if(!h5->no_write) {
if((stat = ncz_sync_netcdf4_file(h5)))
if((stat = ncz_sync_netcdf4_file(h5,ZCLOSE)))
goto done;
}
}
@ -356,7 +358,7 @@ NCZ_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp)
*/
static int
ncz_sync_netcdf4_file(NC_FILE_INFO_T* file)
ncz_sync_netcdf4_file(NC_FILE_INFO_T* file, int isclose)
{
int stat = NC_NOERR;
@ -392,7 +394,7 @@ ncz_sync_netcdf4_file(NC_FILE_INFO_T* file)
goto done;
/* Write all the metadata. */
if((stat = ncz_sync_file(file)))
if((stat = ncz_sync_file(file,isclose)))
goto done;
}
done:
@ -424,7 +426,7 @@ ncz_enddef_netcdf4_file(NC_FILE_INFO_T* file)
/* Redef mode needs to be tracked separately for nc_abort. */
file->redef = NC_FALSE;
return ncz_sync_netcdf4_file(file);
return ncz_sync_netcdf4_file(file,!ZCLOSE);
}
/**

4
libnczarr/zinternal.h
View File

@ -53,6 +53,9 @@
#define PUREZARR "zarr"
/* Mnemonics */
#define ZCLOSE 1 /* this is closeorabort as opposed to enddef */
/**************************************************/
/* Forward */
@ -90,7 +93,6 @@ typedef struct NCZ_FILE_INFO {
# define FLAG_PUREZARR 1
# define FLAG_SHOWFETCH 2
# define FLAG_LOGGING 4
# define FLAG_BYTERANGE 8
NCZM_IMPL mapimpl;
} features;
} NCZ_FILE_INFO_T;

2
libnczarr/zjson.c
View File

@ -295,7 +295,7 @@ NCJlex(NCJparser* parser)
start = parser->pos;
for(;;) {
c = *parser->pos++;
if(strchr(WORD,c) == NULL) break; /* end of word */
if(c == '\0' || strchr(WORD,c) == NULL) break; /* end of word */
}
/* Pushback c if not whitespace */
if(c > ' ' && c != '\177') parser->pos--;

186
libnczarr/zmap.c
View File

@ -4,18 +4,41 @@
*/
#include "zincludes.h"
#include <stdarg.h>
#include "ncpathmgr.h"
/**************************************************/
/* Import the current implementations */
extern NCZMAP_DS_API zmap_file;
#ifdef USE_HDF5
extern NCZMAP_DS_API zmap_nz4;
extern NCZMAP_DS_API zmap_nzf;
#endif
#ifdef ENABLE_NCZARR_ZIP
extern NCZMAP_DS_API zmap_zip;
#endif
#ifdef ENABLE_S3_SDK
extern NCZMAP_DS_API zmap_s3sdk;
#endif
/**************************************************/
NCZM_PROPERTIES
nczmap_properties(NCZM_IMPL impl)
{
switch (impl) {
case NCZM_FILE: return zmap_file.properties;
#ifdef ENABLE_NCZARR_ZIP
case NCZM_ZIP: return zmap_zip.properties;
#endif
#ifdef ENABLE_S3_SDK
case NCZM_S3: return zmap_s3sdk.properties;
#endif
default: break;
}
return NCZM_UNIMPLEMENTED;
}
int
nczmap_create(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp)
{
@ -29,14 +52,16 @@ nczmap_create(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void*
if(mapp) *mapp = NULL;
switch (impl) {
case NCZM_NC4:
stat = zmap_nz4.create(path, mode, flags, parameters, &map);
if(stat) goto done;
break;
case NCZM_FILE:
stat = zmap_nzf.create(path, mode, flags, parameters, &map);
stat = zmap_file.create(path, mode, flags, parameters, &map);
if(stat) goto done;
break;
#ifdef ENABLE_NCZARR_ZIP
case NCZM_ZIP:
stat = zmap_zip.create(path, mode, flags, parameters, &map);
if(stat) goto done;
break;
#endif
#ifdef ENABLE_S3_SDK
case NCZM_S3:
stat = zmap_s3sdk.create(path, mode, flags, parameters, &map);
@ -65,14 +90,16 @@ nczmap_open(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void* pa
if(mapp) *mapp = NULL;
switch (impl) {
case NCZM_NC4:
stat = zmap_nz4.open(path, mode, flags, parameters, &map);
if(stat) goto done;
break;
case NCZM_FILE:
stat = zmap_nzf.open(path, mode, flags, parameters, &map);
stat = zmap_file.open(path, mode, flags, parameters, &map);
if(stat) goto done;
break;
#ifdef ENABLE_NCZARR_ZIP
case NCZM_ZIP:
stat = zmap_zip.open(path, mode, flags, parameters, &map);
if(stat) goto done;
break;
#endif
#ifdef ENABLE_S3_SDK
case NCZM_S3:
stat = zmap_s3sdk.open(path, mode, flags, parameters, &map);
@ -115,12 +142,6 @@ nczmap_len(NCZMAP* map, const char* key, size64_t* lenp)
return map->api->len(map, key, lenp);
}
int
nczmap_defineobj(NCZMAP* map, const char* key)
{
return map->api->defineobj(map, key);
}
int
nczmap_read(NCZMAP* map, const char* key, size64_t start, size64_t count, void* content)
{
@ -230,6 +251,26 @@ nczm_concat(const char* prefix, const char* suffix, char** pathp)
return NC_NOERR;
}
/* Concat multiple strings, but with no intervening separators */
int
nczm_appendn(char** resultp, int n, ...)
{
va_list args;
NCbytes* buf = ncbytesnew();
int i;
va_start(args, n);
for(i=0;i<n;i++) {
char* s = va_arg(args,char*);
if(s != NULL) ncbytescat(buf,s);
}
ncbytesnull(buf);
va_end(args);
if(resultp) {*resultp = ncbytesextract(buf);}
ncbytesfree(buf);
return NC_NOERR;
}
/* A segment is defined as a '/' plus characters following up
to the end or upto the next '/'
*/
@ -340,8 +381,7 @@ nczm_localize(const char* path, char** localpathp, int localize)
/* Convert path0 to be:
1. absolute -- including drive letters
2. forward slashed -- we will convert back to back slash in
nczm_fixpath
2. forward slashed -- we will convert back to back slash in nczm_fixpath
*/
int
@ -366,3 +406,111 @@ done:
nullfree(cpath);
return THROW(ret);
}
/* extract the first segment of a path */
int
nczm_segment1(const char* path, char** seg1p)
{
int ret = NC_NOERR;
char* seg1 = NULL;
const char* p = NULL;
const char* q = NULL;
ptrdiff_t delta;
if(path == NULL)
{seg1 = NULL; goto done;}
p = path;
if(*p == '/') p++; /* skip any leading '/' */
q = strchr(p,'/');
if(q == NULL) q = p+strlen(p); /* point to stop character */
delta = (q-p);
if((seg1 = (char*)malloc(delta+1))==NULL)
{ret = NC_ENOMEM; goto done;}
memcpy(seg1,p,delta);
seg1[delta] = '\0';
if(seg1p) {*seg1p = seg1; seg1 = NULL;}
done:
nullfree(seg1);
return THROW(ret);
}
/*
Extract the basename from a path.
Basename is last segment minus one extension.
*/
int
nczm_basename(const char* path, char** basep)
{
int ret = NC_NOERR;
char* base = NULL;
const char* p = NULL;
const char* q = NULL;
ptrdiff_t delta;
if(path == NULL)
{base = NULL; goto done;}
p = strrchr(path,'/');
if(p == NULL) p = path; else p++;
q = strrchr(p,'.');
if(q == NULL) q = p + strlen(p);
delta = (q-p);
if((base = (char*)malloc(delta+1))==NULL)
{ret = NC_ENOMEM; goto done;}
memcpy(base,p,delta);
base[delta] = '\0';
if(basep) {*basep = base; base = NULL;}
done:
nullfree(base);
return THROW(ret);
}
/* bubble sort a list of strings */
void
nczm_sortlist(NClist* l)
{
nczm_sortenvv(nclistlength(l),(char**)nclistcontents(l));
}
/* bubble sort a list of strings */
void
nczm_sortenvv(int n, char** envv)
{
size_t i, switched;
if(n <= 1) return;
do {
switched = 0;
for(i=0;i<n-1;i++) {
char* ith = envv[i];
char* ith1 = envv[i+1];
if(strcmp(ith,ith1) > 0) {
envv[i] = ith1;
envv[i+1] = ith;
switched = 1;
}
}
} while(switched);
#if 0
for(i=0;i<n;i++)
fprintf(stderr,"sorted: [%d] %s\n",i,(const char*)envv[i]);
#endif
}
void
NCZ_freeenvv(int n, char** envv)
{
int i;
char** p;
if(envv == NULL) return;
if(n < 0)
{for(n=0, p = envv; *p; n++); /* count number of strings */}
for(i=0;i<n;i++)
if(envv[i]) free(envv[i]);
free(envv);
}

161
libnczarr/zmap.h
View File

@ -20,7 +20,8 @@ separator for the segments of the path.
As with Unix, all keys have this BNF syntax:
<pre>
key: '/' | key segment ;
key: '/' | keypath ;
keypath: '/' segment | keypath '/' segment ;
segment: <sequence of UTF-8 characters except control characters and '/'>
</pre>
@ -30,7 +31,8 @@ Thus one key is "contained" (possibly transitively)
by another if one key is a prefix (in the string sense) of the other.
So in this sense the key "/x/y/z" is contained by the key "/x/y".
As with S3, a key refers to an "object" that can contain content.
In this model all keys "exist" but only some keys refer to
objects containing content -- content bearing.
An important restriction is placed on the structure of the tree.
Namely, keys are only defined for content-bearing objects.
Further, all the leaves of the tree are these content-bearing objects.
@ -39,17 +41,11 @@ be a prefix of any other key.
There several other concepts of note.
1. Dataset - a dataset is the complete tree contained by the key defining
the root of the dataset.
the root of the dataset. Technically, the root of the tree is the key <dataset>/.nczarr,
where .nczarr can be considered the superblock of the dataset.
2. Object - equivalent of the S3 object; Each object has a unique key
and "contains" data in the form of an arbitrary sequence of 8-bit bytes.
Notes:
1. The search function is optional. It has two purposes:
a. Support reading of pure zarr datasets (because they do not explicitly
track their contents).
b. Debugging to allow raw examination of the storage. See zdump
for example.
The zmap API defined here isolates the key-value pair mapping code
from the Zarr-based implementation of NetCDF-4.
@ -57,6 +53,40 @@ It wraps an internal C dispatch table manager
for implementing an abstract data structure
implementing the key/object model.
Search:
The search function has two purposes:
a. Support reading of pure zarr datasets (because they do not explicitly
track their contents).
b. Debugging to allow raw examination of the storage. See zdump
for example.
v
The search function takes a prefix path which has a key syntax (see above).
The set of legal keys is the set of keys such that the key references
a content-bearing object -- e.g. /x/y/.zarray or /.zgroup. Essentially
this is the set of keys pointing to the leaf objects of the tree of keys
constituting a dataset. This set potentially limits the set of keys that need to be
examined during search.
Ideally the search function would return
the set of names that are immediate suffixes of a
given prefix path. That is, if <prefix> is the prefix path,
then search returns all <name> such that <prefix>/<name> is itself a prefix of a "legal" key.
This could be used to implement glob style searches such as "/x/y/ *" or "/x/y/ **"
This semantics was chosen because it appears to be the minimum required to implement
all other kinds of search using recursion. So for example
1. Avoid returning keys that are not a prefix of some legal key.
2. Avoid returning all the legal keys in the dataset because that set may be very large;
although the implementation may still have to examine all legal keys to get the desired subset.
3. Allow for use of partial read mechanisms such as iterators, if available.
This can support processing a limited set of keys for each iteration. This is a
straighforward tradeoff of space over time.
This is doable in S3 search using common prefixes with a delimiter of '/', although
the implementation is a bit tricky. For the file system zmap implementation, the legal search keys can be obtained
one level at a time, which directly implements the search semantics. For the zip file implementation,
this semantics is not possible, so the whole tree must be obtained and searched.
Issues:
1. S3 limits key lengths to 1024 bytes. Some deeply nested netcdf files
will almost certainly exceed this limit.
@ -81,12 +111,12 @@ this different than, say, a direvtory tree where a key will
always lead to something: a directory or a file.
In any case, the zmap API returns two distinguished error code:
1. NC_NOERR if a content bearing object is created or referenced.
1. NC_NOERR if a operation succeeded
2. NC_EEMPTY is returned when accessing a key that has no content.
This does not preclude other errors being returned such NC_EACCESS or NC_EPERM or NC_EINVAL
if there are permission errors or illegal function arguments, for example.
It also does not preclude the use of other error codes internal to the zmap
implementation. So zmap_nzf, for example, uses NC_ENOTFOUND internally
implementation. So zmap_file, for example, uses NC_ENOTFOUND internally
because it is possible to detect the existence of directories and files.
This does not propagate to the API.
@ -95,6 +125,10 @@ caller asked for non-content-bearing key.
The current set of operations defined for zmaps are define with the
generic nczm_xxx functions below.
Each zmap implementation has retrievable flags defining limitations
of the implementation.
*/
#ifndef ZMAP_H
@ -115,13 +149,20 @@ typedef struct NCZMAP_API NCZMAP_API;
/* Define the space of implemented (eventually) map implementations */
typedef enum NCZM_IMPL {
NCZM_UNDEF=0, /* In-memory implementation */
NCZM_S3=1, /* Amazon S3 implementation */
NCZM_NC4=2, /* Netcdf-4 file based implementation */
NCZM_FILE=3, /* File system directory-based implementation */
NCZM_FILE=1, /* File system directory-based implementation */
NCZM_ZIP=2, /* Zip-file based implementation */
NCZM_S3=3, /* Amazon S3 implementation */
} NCZM_IMPL;
/* Define the default map implementation */
#define NCZM_DEFAULT NCZM_NC4
#define NCZM_DEFAULT NCZM_ZIP
/* Define the per-implementation limitations flags */
typedef size64_t NCZM_PROPERTIES;
/* powers of 2 */
#define NCZM_UNIMPLEMENTED 1 /* Unknown/ unimplemented */
#define NCZM_WRITEONCE 2 /* Objects can only be written once */
#define NCZM_ZEROSTART 4 /* Objects can only be written using a start count of zero */
/*
For each dataset, we create what amounts to a class
@ -139,39 +180,62 @@ typedef struct NCZMAP {
NCZM_IMPL format;
char* url;
int mode;
size64_t flags;
size64_t flags; /* Passed in by caller */
struct NCZMAP_API* api;
} NCZMAP;
/* zmap_s3sdk related-types and constants */
#define AWSHOST ".amazonaws.com"
enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_OTHER=3};
typedef struct ZS3INFO {
enum URLFORMAT urlformat;
char* host; /* non-null if other*/
char* region; /* region */
char* bucket; /* bucket name */
char* rootkey;
} ZS3INFO;
/* Forward */
struct NClist;
/* Define the object-level API */
struct NCZMAP_API {
int version;
/* Map Operations */
int (*close)(NCZMAP* map, int deleteit);
/* Object Operations */
int (*exists)(NCZMAP* map, const char* key);
int (*len)(NCZMAP* map, const char* key, size64_t* sizep);
int (*defineobj)(NCZMAP* map, const char* key);
int (*read)(NCZMAP* map, const char* key, size64_t start, size64_t count, void* content);
int (*write)(NCZMAP* map, const char* key, size64_t start, size64_t count, const void* content);
int (*search)(NCZMAP* map, const char* prefix, NClist* matches);
int (*search)(NCZMAP* map, const char* prefix, struct NClist* matches);
};
/* Define the Dataset level API */
typedef struct NCZMAP_DS_API {
int version;
int (*create)(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp);
int (*open)(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp);
NCZM_PROPERTIES properties;
int (*create)(const char *path, int mode, size64_t constraints, void* parameters, NCZMAP** mapp);
int (*open)(const char *path, int mode, size64_t constraints, void* parameters, NCZMAP** mapp);
} NCZMAP_DS_API;
#ifdef __cplusplus
extern "C" {
#endif
/**
Get limitations of a particular implementation.
@param impl -- the map implemenation type
@param limitsp return limitation flags here
@return NC_NOERR if the operation succeeded
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL NCZM_PROPERTIES nczmap_properties(NCZM_IMPL);
/* Object API Wrappers; note that there are no group operations
because group keys do not map to directories.
*/
@ -181,7 +245,7 @@ Check if a specified content-bearing object exists or not.
@param map -- the containing map
@param key -- the key specifying the content-bearing object
@return NC_NOERR if the object exists
@return NC_ENOTFOUND if the object does not exist
@return NC_EEMPTY if the object is not content bearing.
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_exists(NCZMAP* map, const char* key);
@ -192,21 +256,11 @@ Return the current size of a specified content-bearing object exists or not.
@param key -- the key specifying the content-bearing object
@param sizep -- the object's size is returned thru this pointer.
@return NC_NOERR if the object exists
@return NC_ENOTFOUND if the object does not exist
@return NC_EEMPTY if the object is not content bearing
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_len(NCZMAP* map, const char* key, size64_t* sizep);
/**
Create a specified content-bearing object.
@param map -- the containing map
@param key -- the key specifying the content-bearing object
@return NC_NOERR if the object is created
@return NC_ENOTFOUND if the object does not exist
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_defineobj(NCZMAP* map, const char* key);
/**
Read the content of a specified content-bearing object.
@param map -- the containing map
@ -215,7 +269,7 @@ Read the content of a specified content-bearing object.
@param count -- number of bytes to read
@param content -- read the data into this memory
@return NC_NOERR if the operation succeeded
@return NC_ENOTFOUND if the object does not exist
@return NC_EEMPTY if the object is not content-bearing.
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_read(NCZMAP* map, const char* key, size64_t start, size64_t count, void* content);
@ -228,55 +282,57 @@ Write the content of a specified content-bearing object.
@param count -- number of bytes to write
@param content -- write the data from this memory
@return NC_NOERR if the operation succeeded
@return NC_ENOTFOUND if the object does not exist
@return NC_EXXX if the operation failed for one of several possible reasons
Note that this makes the key a content-bearing object.
*/
EXTERNL int nczmap_write(NCZMAP* map, const char* key, size64_t start, size64_t count, const void* content);
/**
Return a vector of keys representing the content-bearing
objects that are immediately contained by the prefix key.
Return a vector of names (not keys) representing the
next segment of legal objects that are immediately contained by the prefix key.
@param map -- the containing map
@param prefix -- the key into the tree where the search is to occur
@param matches -- return the set of keys in this list
@param matches -- return the set of names in this list; might be empty
@return NC_NOERR if the operation succeeded
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_search(NCZMAP* map, const char* prefix, NClist* matches);
EXTERNL int nczmap_search(NCZMAP* map, const char* prefix, struct NClist* matches);
/**
Close a map
@param map -- the map to close
@param deleteit-- if true, then delete the corresponding dataset
@return NC_NOERR if the operation succeeded
@return NC_ENOTFOUND if the object does not exist
@return NC_EXXX if the operation failed for one of several possible reasons
*/
EXTERNL int nczmap_close(NCZMAP* map, int deleteit);
/* Create/open and control a dataset using a specific implementation */
EXTERNL int nczmap_create(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp);
EXTERNL int nczmap_open(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp);
EXTERNL int nczmap_create(NCZM_IMPL impl, const char *path, int mode, size64_t constraints, void* parameters, NCZMAP** mapp);
EXTERNL int nczmap_open(NCZM_IMPL impl, const char *path, int mode, size64_t constraints, void* parameters, NCZMAP** mapp);
/* Utility functions */
/** Split a path into pieces along '/' character; elide any leading '/' */
EXTERNL int nczm_split(const char* path, NClist* segments);
EXTERNL int nczm_split(const char* path, struct NClist* segments);
/* Split a path into pieces along some character; elide any leading char */
EXTERNL int nczm_split_delim(const char* path, char delim, NClist* segments);
EXTERNL int nczm_split_delim(const char* path, char delim, struct NClist* segments);
/* Convenience: Join all segments into a path using '/' character */
EXTERNL int nczm_join(NClist* segments, char** pathp);
EXTERNL int nczm_join(struct NClist* segments, char** pathp);
/* Convenience: Join all segments into a path using '/' character
but taking possible lead windows drive letter into account
*/
EXTERNL int nczm_joinpath(NClist* segments, char** pathp);
EXTERNL int nczm_joinpath(struct NClist* segments, char** pathp);
/* Convenience: concat two strings; caller frees */
/* Convenience: concat two strings with '/' between; caller frees */
EXTERNL int nczm_concat(const char* prefix, const char* suffix, char** pathp);
/* Convenience: concat multiple strings with no intermediate separators; caller frees */
EXTERNL int nczm_appendn(char** resultp, int n, ...);
/* Break a key into prefix and suffix, where prefix is the first nsegs segments;
nsegs can be negative to specify that suffix is |nsegs| long
*/
@ -290,7 +346,16 @@ EXTERNL int nczm_isabsolutepath(const char* path);
/* Convert forward to back slash if needed */
EXTERNL int nczm_localize(const char* path, char** newpathp, int local);
EXTERNL int nczm_canonicalpath(const char* path, char** cpathp);
EXTERNL int nczm_basename(const char* path, char** basep);
EXTERNL int nczm_segment1(const char* path, char** seg1p);
/* bubble sorts (note arguments) */
EXTERNL void nczm_sortlist(struct NClist* l);
EXTERNL void nczm_sortenvv(int n, char** envv);
EXTERNL void NCZ_freeenvv(int n, char** envv);
#ifdef __cplusplus
}

118
libnczarr/zmap_nzf.c → libnczarr/zmap_file.c
View File

@ -122,7 +122,6 @@ static NCZMAP_API zapi;
static int zfileclose(NCZMAP* map, int delete);
static int zfcreategroup(ZFMAP*, const char* key, int nskip);
static int zflookupobj(ZFMAP*, const char* key, FD* fd);
static int zfcreateobj(ZFMAP*, const char* key,FD*);
static int zfparseurl(const char* path0, NCURI** urip);
static int zffullpath(ZFMAP* zfmap, const char* key, char**);
static void zfrelease(ZFMAP* zfmap, FD* fd);
@ -160,7 +159,7 @@ static void zfinitialize(void)
if(sscanf(env,"%d",&perms) == 1) NC_DEFAULT_DIR_PERMS = perms;
}
zfinitialized = 1;
ZUNTRACE(NC_NOERR);
(void)ZUNTRACE(NC_NOERR);
}
}
@ -189,9 +188,7 @@ zfilecreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP
if(!zfinitialized) zfinitialize();
/* Fixup mode flags */
mode = (NC_NETCDF4 | NC_WRITE | mode);
if(flags & FLAG_BYTERANGE)
mode &= ~(NC_CLOBBER | NC_WRITE);
mode |= (NC_NETCDF4 | NC_WRITE);
if(!(mode & NC_WRITE))
{stat = NC_EPERM; goto done;}
@ -267,8 +264,6 @@ zfileopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
/* Fixup mode flags */
mode = (NC_NETCDF4 | mode);
if(flags & FLAG_BYTERANGE)
mode &= ~(NC_CLOBBER | NC_WRITE);
/* path must be a url with file: protocol*/
if((stat=zfparseurl(path,&url)))
@ -292,12 +287,8 @@ zfileopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
truepath = NULL;
/* Verify root dir exists */
switch (stat = platformopendir(zfmap,zfmap->root)) {
case NC_NOERR: break;
case NC_ENOTFOUND: stat = NC_EEMPTY; /* fall thru */
default:
if((stat = platformopendir(zfmap,zfmap->root)))
goto done;
}
/* Dataset superblock will be read by higher layer */
@ -357,49 +348,13 @@ done:
return ZUNTRACEX(stat,"len=%llu",(lenp?*lenp:777777777777));
}
static int
zfiledefineobj(NCZMAP* map, const char* key)
{
int stat = NC_NOERR;
FD fd = FDNUL;
ZFMAP* zfmap = (ZFMAP*)map; /* cast to true type */
ZTRACE(5,"map=%s key=%s",map->url,key);
#ifdef VERIFY
if(!verify(key,!FLAG_ISDIR))
assert(!"expected file, have dir");
#endif
/* Create the intermediate groups as directories */
if((stat = zfcreategroup(zfmap,key,SKIPLAST)))
goto done;
stat = zflookupobj(zfmap,key,&fd);
zfrelease(zfmap,&fd);
switch (stat) {
case NC_NOERR: /* Already exists */
goto done;
case NC_ENOTFOUND: stat = NC_EEMPTY; /* file does not exist */
case NC_EEMPTY: /* empty */
if((stat = zfcreateobj(zfmap,key,&fd)))
goto done;
break;
default:
goto done;
}
done:
zfrelease(zfmap,&fd);
return ZUNTRACE(stat);
}
static int
zfileread(NCZMAP* map, const char* key, size64_t start, size64_t count, void* content)
{
int stat = NC_NOERR;
FD fd = FDNUL;
ZFMAP* zfmap = (ZFMAP*)map; /* cast to true type */
ZTRACE(5,"map=%s key=%s start=%llu count=%llu",map->url,key,start,count);
#ifdef VERIFY
@ -428,6 +383,7 @@ zfilewrite(NCZMAP* map, const char* key, size64_t start, size64_t count, const v
int stat = NC_NOERR;
FD fd = FDNUL;
ZFMAP* zfmap = (ZFMAP*)map; /* cast to true type */
char* truepath = NULL;
ZTRACE(5,"map=%s key=%s start=%llu count=%llu",map->url,key,start,count);
@ -437,16 +393,24 @@ zfilewrite(NCZMAP* map, const char* key, size64_t start, size64_t count, const v
#endif
switch (stat = zflookupobj(zfmap,key,&fd)) {
case NC_ENOTFOUND:
case NC_EEMPTY:
/* Create the directories leading to this */
if((stat = zfcreategroup(zfmap,key,SKIPLAST))) goto done;
/* Create truepath */
if((stat = zffullpath(zfmap,key,&truepath))) goto done;
/* Create file */
if((stat = platformcreatefile(zfmap,truepath,&fd))) goto done;
/* Fall thru to write the object */
case NC_NOERR:
if((stat = platformseek(zfmap, &fd, SEEK_SET, &start))) goto done;
if((stat = platformwrite(zfmap, &fd, count, content))) goto done;
break;
case NC_ENOTFOUND: stat = NC_EEMPTY;
case NC_EEMPTY: break;
default: break;
}
done:
nullfree(truepath);
zfrelease(zfmap,&fd);
return ZUNTRACE(stat);
}
@ -473,7 +437,7 @@ zfileclose(NCZMAP* map, int delete)
}
/*
Return a list of keys immediately "below" a specified prefix key.
Return a list of names immediately "below" a specified prefix key.
In theory, the returned list should be sorted in lexical order,
but it possible that it is not.
The prefix key is not included.
@ -482,21 +446,18 @@ int
zfilesearch(NCZMAP* map, const char* prefixkey, NClist* matches)
{
int stat = NC_NOERR;
int i;
ZFMAP* zfmap = (ZFMAP*)map;
char* truepath = NULL;
NClist* nextlevel = nclistnew();
NCbytes* buf = ncbytesnew();
int trailing;
ZTRACE(5,"map=%s prefixkey=%s",map->url,prefixkey);
/* Make the root path be true */
if(prefixkey == NULL || strlen(prefixkey)==0 || strcmp(prefixkey,"/")==0)
truepath = strdup(zfmap->root);
truepath = strdup(zfmap->root);
else if((stat = nczm_concat(zfmap->root,prefixkey,&truepath))) goto done;
trailing = (prefixkey[strlen(prefixkey)-1] == '/');
/* get names of the next level path entries */
switch (stat = platformdircontent(zfmap, truepath, nextlevel)) {
case NC_NOERR: /* ok */
@ -508,12 +469,9 @@ zfilesearch(NCZMAP* map, const char* prefixkey, NClist* matches)
default:
goto done;
}
for(i=0;i<nclistlength(nextlevel);i++) {
const char* segment = nclistget(nextlevel,i);
ncbytescat(buf,prefixkey);
if(!trailing) ncbytescat(buf,"/");
ncbytescat(buf,segment);
nclistpush(matches,ncbytesextract(buf));
while(nclistlength(nextlevel) > 0) {
char* segment = nclistremove(nextlevel,0);
nclistpush(matches,segment);
}
done:
@ -594,42 +552,15 @@ zfrelease(ZFMAP* zfmap, FD* fd)
{
ZTRACE(5,"map=%s fd=%d",zfmap->map.url,(fd?fd->fd:-1));
platformrelease(zfmap,fd);
ZUNTRACE(NC_NOERR);
}
/* Create an object file corresponding to a key; create any
necessary intermediate groups. Assumed that we actually
want to create this as a file.
*/
static int
zfcreateobj(ZFMAP* zfmap, const char* key, FD* fd)
{
int stat = NC_NOERR;
char* fullpath = NULL;
ZTRACE(5,"map=%s key=%s",zfmap->map.url,key);
#ifdef VERIFY
if(!verify(key,!FLAG_ISDIR))
assert(!"expected file, have dir");
#endif
/* Create all the prefix groups as directories */
if((stat = zfcreategroup(zfmap, key, SKIPLAST))) goto done;
/* Create the final object */
if((stat=zffullpath(zfmap,key,&fullpath))) goto done;
if((stat = platformcreatefile(zfmap,fullpath,fd)))
goto done;
done:
nullfree(fullpath);
return ZUNTRACE(stat);
(void)ZUNTRACE(NC_NOERR);
}
/**************************************************/
/* External API objects */
NCZMAP_DS_API zmap_nzf = {
NCZMAP_DS_API zmap_file = {
NCZM_FILE_V1,
0,
zfilecreate,
zfileopen,
};
@ -639,7 +570,6 @@ static NCZMAP_API zapi = {
zfileclose,
zfileexists,
zfilelen,
zfiledefineobj,
zfileread,
zfilewrite,
zfilesearch,
@ -1224,7 +1154,7 @@ platformrelease(ZFMAP* zfmap, FD* fd)
ZTRACE(6,"map=%s fd=%d",zfmap->map.url,(fd?fd->fd:-1));
if(fd->fd >=0) NCclose(fd->fd);
fd->fd = -1;
ZUNTRACE(NC_NOERR);
(void)ZUNTRACE(NC_NOERR);
}
#if 0

398
libnczarr/zmap_s3sdk.c
View File

@ -38,42 +38,25 @@ Notes:
#define NCZM_S3SDK_V1 1
#define AWSHOST ".amazonaws.com"
enum URLFORMAT {UF_NONE, UF_VIRTUAL, UF_PATH, UF_OTHER};
#define ZS3_PROPERTIES (0)
/* Define the "subclass" of NCZMAP */
typedef struct ZS3MAP {
NCZMAP map;
enum URLFORMAT urlformat;
char* host; /* non-null if other*/
char* region; /* region */
ZS3INFO s3;
void* s3config;
void* s3client;
char* errmsg;
char* bucket; /* bucket name */
char* rootkey;
} ZS3MAP;
/* Forward */
static NCZMAP_API nczs3sdkapi; // c++ will not allow static forward variables
static int zs3exists(NCZMAP* map, const char* key);
static int zs3len(NCZMAP* map, const char* key, size64_t* lenp);
static int zs3defineobj(NCZMAP* map, const char* key);
static int zs3read(NCZMAP* map, const char* key, size64_t start, size64_t count, void* content);
static int zs3write(NCZMAP* map, const char* key, size64_t start, size64_t count, const void* content);
static int zs3search(NCZMAP* map, const char* prefix, NClist* matches);
static int zs3close(NCZMAP* map, int deleteit);
static int z3createobj(ZS3MAP*, const char* key);
static int processurl(ZS3MAP* z3map, NCURI* url);
static int endswith(const char* s, const char* suffix);
static void freevector(size_t nkeys, char** list);
static void zs3initialize(void);
static int s3clear(ZS3MAP* z3map, const char* key);
static int isLegalBucketName(const char* bucket);
static int maketruekey(const char* rootpath, const char* key, char** truekeyp);
@ -109,9 +92,12 @@ static int zs3initialized = 0;
static void
zs3initialize(void)
{
if(!zs3initialized)
if(!zs3initialized) {
ZTRACE(7,NULL);
NCZ_s3sdkinitialize();
zs3initialized = 1;
zs3initialized = 1;
(void)ZUNTRACE(NC_NOERR);
}
}
#if 0
@ -136,7 +122,7 @@ zs3create(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
NC_UNUSED(flags);
NC_UNUSED(parameters);
ZTRACE(1,"%s:",__func__);
ZTRACE(6,"path=%s mode=%d flag=%llu",path,mode,flags);
if(!zs3initialized) zs3initialize();
@ -156,32 +142,35 @@ zs3create(const char *path, int mode, size64_t flags, void* parameters, NCZMAP**
{stat = NC_EURL; goto done;}
/* Convert to canonical path-style */
if((stat = processurl(z3map,url))) goto done;
if((stat = NCZ_s3urlprocess(url,&z3map->s3))) goto done;
/* Verify the root path */
if(z3map->s3.rootkey == NULL)
{stat = NC_EURL; goto done;}
if((stat=NCZ_s3sdkcreateconfig(z3map->host, z3map->region, &z3map->s3config))) goto done;
if((stat=NCZ_s3sdkcreateconfig(z3map->s3.host, z3map->s3.region, &z3map->s3config))) goto done;
if((stat = NCZ_s3sdkcreateclient(z3map->s3config,&z3map->s3client))) goto done;
{
int exists = 0;
/* Does bucket already exist */
if((stat = NCZ_s3sdkbucketexists(z3map->s3client,z3map->bucket,&exists, &z3map->errmsg))) goto done;
if((stat = NCZ_s3sdkbucketexists(z3map->s3client,z3map->s3.bucket,&exists, &z3map->errmsg))) goto done;
if(!exists) {
/* create it */
if((stat = NCZ_s3sdkbucketcreate(z3map->s3client,z3map->region,z3map->bucket,&z3map->errmsg)))
if((stat = NCZ_s3sdkbucketcreate(z3map->s3client,z3map->s3.region,z3map->s3.bucket,&z3map->errmsg)))
goto done;
}
/* The root object should not exist */
switch (stat = NCZ_s3sdkinfo(z3map->s3client,z3map->bucket,z3map->rootkey,NULL,&z3map->errmsg)) {
switch (stat = NCZ_s3sdkinfo(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,NULL,&z3map->errmsg)) {
case NC_EEMPTY: /* no such object */
stat = NC_NOERR; /* which is what we want */
errclear(z3map);
break;
case NC_NOERR: stat = NC_EEXIST; goto done; /* already exists */
case NC_NOERR: stat = NC_EFOUND; goto done; /* already exists */
default: reporterr(z3map); goto done;
}
if(!stat) {
/* Delete objects inside root object tree */
s3clear(z3map,z3map->rootkey);
s3clear(z3map,z3map->s3.rootkey);
}
}
@ -193,7 +182,7 @@ done:
nullfree(prefix);
nullfree(truekey);
if(stat) nczmap_close((NCZMAP*)z3map,1);
return (stat);
return ZUNTRACE(stat);
}
/* The problem with open is that there
@ -212,7 +201,7 @@ zs3open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m
NC_UNUSED(flags);
NC_UNUSED(parameters);
ZTRACE(1,"%s: ",__func__);
ZTRACE(6,"path=%s mode=%d flags=%llu",path,mode,flags);
if(!zs3initialized) zs3initialize();
@ -232,21 +221,23 @@ zs3open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m
{stat = NC_EURL; goto done;}
/* Convert to canonical path-style */
if((stat = processurl(z3map,url))) goto done;
if((stat = NCZ_s3urlprocess(url,&z3map->s3))) goto done;
/* Verify root path */
if(z3map->s3.rootkey == NULL)
{stat = NC_EURL; goto done;}
if((stat=NCZ_s3sdkcreateconfig(z3map->host,z3map->region,&z3map->s3config))) goto done;
if((stat=NCZ_s3sdkcreateconfig(z3map->s3.host,z3map->s3.region,&z3map->s3config))) goto done;
if((stat=NCZ_s3sdkcreateclient(z3map->s3config,&z3map->s3client))) goto done;
/* Search the root for content */
content = nclistnew();
if((stat = NCZ_s3sdkgetkeys(z3map->s3client,z3map->bucket,z3map->rootkey,&nkeys,NULL,&z3map->errmsg)))
if((stat = NCZ_s3sdkgetkeys(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,&nkeys,NULL,&z3map->errmsg)))
goto done;
if(nkeys == 0) {
/* dataset does not actually exist */
stat = NC_EEMPTY;
/* dataset does not actually exist; we choose to return ENOTFOUND instead of EEMPTY */
stat = NC_ENOTFOUND;
goto done;
}
if(mapp) *mapp = (NCZMAP*)z3map;
done:
@ -254,41 +245,12 @@ done:
nclistfreeall(content);
ncurifree(url);
if(stat) nczmap_close((NCZMAP*)z3map,0);
return (stat);
}
static int
isLegalBucketName(const char* bucket)
{
return 1; /*TBD*/
return ZUNTRACE(stat);
}
/**************************************************/
/* Object API */
static int
zs3close(NCZMAP* map, int deleteit)
{
int stat = NC_NOERR;
ZS3MAP* z3map = (ZS3MAP*)map;
ZTRACE(1,"%s: ",__func__);
if(deleteit)
s3clear(z3map,z3map->rootkey);
NCZ_s3sdkclose(z3map->s3client, z3map->s3config, z3map->bucket, z3map->rootkey, deleteit, &z3map->errmsg);
reporterr(z3map);
z3map->s3client = NULL;
z3map->s3config = NULL;
nullfree(z3map->bucket);
nullfree(z3map->region);
nullfree(z3map->host);
nullfree(z3map->errmsg);
nullfree(z3map->rootkey)
nczm_clear(map);
nullfree(map);
return (stat);
}
/*
@return NC_NOERR if key points to a content-bearing object.
@return NC_EEMPTY if object at key has no content.
@ -297,7 +259,10 @@ zs3close(NCZMAP* map, int deleteit)
static int
zs3exists(NCZMAP* map, const char* key)
{
return zs3len(map,key,NULL);
int stat = NC_NOERR;
ZTRACE(6,"map=%s key=%s",map->url,key);
stat = zs3len(map,key,NULL);
return ZUNTRACE(stat);
}
/*
@ -312,11 +277,11 @@ zs3len(NCZMAP* map, const char* key, size64_t* lenp)
ZS3MAP* z3map = (ZS3MAP*)map;
char* truekey = NULL;
ZTRACE(1,"%s: ",__func__);
ZTRACE(6,"map=%s key=%s",map->url,key);
if((stat = maketruekey(z3map->rootkey,key,&truekey))) goto done;
if((stat = maketruekey(z3map->s3.rootkey,key,&truekey))) goto done;
switch (stat = NCZ_s3sdkinfo(z3map->s3client,z3map->bucket,truekey,lenp,&z3map->errmsg)) {
switch (stat = NCZ_s3sdkinfo(z3map->s3client,z3map->s3.bucket,truekey,lenp,&z3map->errmsg)) {
case NC_NOERR: break;
case NC_EEMPTY:
if(lenp) *lenp = 0;
@ -327,36 +292,7 @@ zs3len(NCZMAP* map, const char* key, size64_t* lenp)
done:
nullfree(truekey);
reporterr(z3map);
return (stat);
}
/*
@return NC_NOERR if key points to a content-bearing object.
@return NC_EEMPTY if object at key has no content, if so, then create
a content-bearing object at that key.
@return NC_EXXX return true error
*/
static int
zs3defineobj(NCZMAP* map, const char* key)
{
int stat = NC_NOERR;
ZS3MAP* z3map = (ZS3MAP*)map; /* cast to true type */
ZTRACE(1,"%s: key=%s",__func__,key);
switch(stat = zs3exists(map,key)) {
case NC_NOERR: goto done; /* Already exists */
case NC_EEMPTY:
if((stat = z3createobj(z3map,key)))
goto done;
break;
default: /* Some other kind of failure */
goto done;
}
done:
reporterr(z3map);
return (stat);
return ZUNTRACE(stat);
}
/*
@ -371,12 +307,12 @@ zs3read(NCZMAP* map, const char* key, size64_t start, size64_t count, void* cont
ZS3MAP* z3map = (ZS3MAP*)map; /* cast to true type */
size64_t size = 0;
char* truekey = NULL;
ZTRACE(1,"%s: key=%s",__func__,key);
if((stat = maketruekey(z3map->rootkey,key,&truekey))) goto done;
switch (stat=NCZ_s3sdkinfo(z3map->s3client, z3map->bucket, truekey, &size, &z3map->errmsg)) {
ZTRACE(6,"map=%s key=%s start=%llu count=%llu",map->url,key,start,count);
if((stat = maketruekey(z3map->s3.rootkey,key,&truekey))) goto done;
switch (stat=NCZ_s3sdkinfo(z3map->s3client, z3map->s3.bucket, truekey, &size, &z3map->errmsg)) {
case NC_NOERR: break;
case NC_EEMPTY: goto done;
default: goto done;
@ -385,13 +321,13 @@ zs3read(NCZMAP* map, const char* key, size64_t start, size64_t count, void* cont
if(start >= size || start+count > size)
{stat = NC_EEDGE; goto done;}
if(count > 0) {
if((stat = NCZ_s3sdkread(z3map->s3client, z3map->bucket, truekey, start, count, content, &z3map->errmsg)))
if((stat = NCZ_s3sdkread(z3map->s3client, z3map->s3.bucket, truekey, start, count, content, &z3map->errmsg)))
goto done;
}
done:
nullfree(truekey);
reporterr(z3map);
return (stat);
return ZUNTRACE(stat);
}
/*
@ -404,54 +340,89 @@ zs3write(NCZMAP* map, const char* key, size64_t start, size64_t count, const voi
{
int stat = NC_NOERR;
ZS3MAP* z3map = (ZS3MAP*)map; /* cast to true type */
void* chunk = NULL;
char* chunk = NULL; /* use char* so we can do arithmetic with it */
size64_t objsize = 0;
size64_t newsize = start+count;
size64_t memsize = 0;
size64_t endwrite = start+count; /* first pos just above overwritten data */
char* truekey = NULL;
int isempty = 0;
ZTRACE(1,"%s: key=%s",__func__,key);
ZTRACE(6,"map=%s key=%s start=%llu count=%llu",map->url,key,start,count);
if(count == 0) {stat = NC_EEDGE; goto done;}
if((stat = maketruekey(z3map->rootkey,key,&truekey))) goto done;
if((stat = maketruekey(z3map->s3.rootkey,key,&truekey))) goto done;
/* Apparently S3 has no write byterange operation, so we need to read the whole object,
copy data, and then rewrite */
switch (stat=NCZ_s3sdkinfo(z3map->s3client, z3map->bucket, truekey, &objsize, &z3map->errmsg)) {
case NC_NOERR:
newsize = (newsize > objsize ? newsize : objsize);
switch (stat=NCZ_s3sdkinfo(z3map->s3client, z3map->s3.bucket, truekey, &objsize, &z3map->errmsg)) {
case NC_NOERR: /* Figure out the memory size of the object */
memsize = (endwrite > objsize ? endwrite : objsize);
break;
case NC_EEMPTY:
newsize = newsize;
memsize = endwrite;
isempty = 1;
break;
default: reporterr(z3map); goto done;
}
chunk = malloc(newsize);
if(isempty)
chunk = (char*)calloc(1,memsize); /* initialize it */
else
chunk = (char*)malloc(memsize);
if(chunk == NULL)
{stat = NC_ENOMEM; goto done;}
if(objsize > 0) {
if((stat = NCZ_s3sdkread(z3map->s3client, z3map->bucket, truekey, 0, objsize, chunk, &z3map->errmsg)))
if(start > 0 && objsize > 0) { /* must read to preserve data before start */
if((stat = NCZ_s3sdkread(z3map->s3client, z3map->s3.bucket, truekey, 0, objsize, (void*)chunk, &z3map->errmsg)))
goto done;
}
#if 0
if(newsize > objsize) {
/* Zeroize the part of the object added */
memset(((char*)chunk)+objsize,0,(newsize-objsize));
objsize = newsize;
}
/* overwrite with the contents */
memcpy(((char*)chunk)+start,content,count); /* remember there may be data above start+count */
if((stat = NCZ_s3sdkwriteobject(z3map->s3client, z3map->bucket, truekey, objsize, chunk, &z3map->errmsg)))
#endif
/* overwrite the relevant part of the memory with the contents */
if(count > 0)
memcpy(((char*)chunk)+start,content,count); /* there may be data above start+count */
/* (re-)write */
if((stat = NCZ_s3sdkwriteobject(z3map->s3client, z3map->s3.bucket, truekey, memsize, (void*)chunk, &z3map->errmsg)))
goto done;
done:
nullfree(truekey);
reporterr(z3map);
nullfree(chunk);
return (stat);
return ZUNTRACE(stat);
}
static int
zs3close(NCZMAP* map, int deleteit)
{
int stat = NC_NOERR;
ZS3MAP* z3map = (ZS3MAP*)map;
ZTRACE(6,"map=%s deleteit=%d",map->url, deleteit);
if(deleteit)
s3clear(z3map,z3map->s3.rootkey);
if(z3map->s3client && z3map->s3config && z3map->s3.bucket && z3map->s3.rootkey) {
NCZ_s3sdkclose(z3map->s3client, z3map->s3config, z3map->s3.bucket, z3map->s3.rootkey, deleteit, &z3map->errmsg);
}
reporterr(z3map);
z3map->s3client = NULL;
z3map->s3config = NULL;
nullfree(z3map->s3.bucket);
nullfree(z3map->s3.region);
nullfree(z3map->s3.host);
nullfree(z3map->errmsg);
nullfree(z3map->s3.rootkey)
nczm_clear(map);
nullfree(map);
return ZUNTRACE(stat);
}
/*
Return a list of keys immediately "below" a specified prefix,
Return a list of full keys immediately "below" a specified prefix,
but not including the prefix.
In theory, the returned list should be sorted in lexical order,
but it possible that it is not.
@ -467,29 +438,27 @@ zs3search(NCZMAP* map, const char* prefix, NClist* matches)
size_t nkeys;
NClist* tmp = NULL;
char* trueprefix = NULL;
char* newkey = NULL;
const char* p;
ZTRACE(1,"%s: prefix=%s",__func__,prefix);
ZTRACE(6,"map=%s prefix0=%s",map->url,prefix);
if((stat = maketruekey(z3map->rootkey,prefix,&trueprefix))) goto done;
if((stat = maketruekey(z3map->s3.rootkey,prefix,&trueprefix))) goto done;
if(*trueprefix != '/') return NC_EINTERNAL;
if((stat = NCZ_s3sdkgetkeys(z3map->s3client,z3map->bucket,trueprefix,&nkeys,&list,&z3map->errmsg)))
if((stat = NCZ_s3sdkgetkeys(z3map->s3client,z3map->s3.bucket,trueprefix,&nkeys,&list,&z3map->errmsg)))
goto done;
if(nkeys > 0) {
size_t tplen = strlen(trueprefix);
tmp = nclistnew();
/* Remove the trueprefix from the front of all the returned keys */
/* The returned keys may be of any depth, so capture and prune the keys */
for(i=0;i<nkeys;i++) {
char* newkey = NULL;
if(memcmp(trueprefix,list[i],tplen)==0) {
newkey = list[i];
newkey = newkey+tplen; /* Point to start of suffix */
const char* l = list[i];
if(memcmp(trueprefix,l,tplen)==0) {
p = l+tplen; /* Point to start of suffix */
/* If the key is same as trueprefix, ignore it */
if(*newkey == '\0') continue;
newkey--; /* Point to trailing '/' */
assert(newkey[0] == '/');
newkey = strdup(newkey);
if(*p == '\0') continue;
if(nczm_segment1(p,&newkey)) goto done;
nclistpush(tmp,newkey); newkey = NULL;
}
}
@ -516,146 +485,17 @@ zs3search(NCZMAP* map, const char* prefix, NClist* matches)
#endif
done:
nullfree(newkey);
nullfree(trueprefix);
reporterr(z3map);
nclistfreeall(tmp);
freevector(nkeys,list);
return THROW(stat);
}
/**************************************************/
/* Utilities */
static int
processurl(ZS3MAP* z3map, NCURI* url)
{
int stat = NC_NOERR;
NClist* segments = NULL;
NCbytes* buf = ncbytesnew();
if(url == NULL)
{stat = NC_EURL; goto done;}
/* do some verification */
if(strcmp(url->protocol,"https") != 0)
{stat = NC_EURL; goto done;}
/* Path better look absolute */
if(!nczm_isabsolutepath(url->path))
{stat = NC_EURL; goto done;}
/* Distinguish path-style from virtual-host style from other:
Virtual: https://bucket-name.s3.Region.amazonaws.com/<root>
Path: https://s3.Region.amazonaws.com/bucket-name/<root>
Other: https://<host>/bucketname/<root>
*/
if(url->host == NULL || strlen(url->host) == 0)
{stat = NC_EURL; goto done;}
if(endswith(url->host,AWSHOST)) { /* Virtual or path */
segments = nclistnew();
/* split the hostname by "." */
if((stat = nczm_split_delim(url->host,'.',segments))) goto done;
switch (nclistlength(segments)) {
default: stat = NC_EURL; goto done;
case 4:
if(strcasecmp(nclistget(segments,0),"s3")!=0)
{stat = NC_EURL; goto done;}
z3map->urlformat = UF_PATH;
z3map->region = strdup(nclistget(segments,1));
break;
case 5:
if(strcasecmp(nclistget(segments,1),"s3")!=0)
{stat = NC_EURL; goto done;}
z3map->urlformat = UF_VIRTUAL;
z3map->region = strdup(nclistget(segments,2));
z3map->bucket = strdup(nclistget(segments,0));
break;
}
/* Rebuild host to look like path-style */
ncbytescat(buf,"s3.");
ncbytescat(buf,z3map->region);
ncbytescat(buf,AWSHOST);
z3map->host = ncbytesextract(buf);
} else {
z3map->urlformat = UF_OTHER;
if((z3map->host = strdup(url->host))==NULL)
{stat = NC_ENOMEM; goto done;}
}
/* Do fixups to make everything look like it was path style */
switch (z3map->urlformat) {
case UF_PATH:
case UF_OTHER:
/* We have to process the path to get the bucket, and remove it in the path */
if(url->path != NULL && strlen(url->path) > 0) {
/* split the path by "/" */
nclistfreeall(segments);
segments = nclistnew();
if((stat = nczm_split_delim(url->path,'/',segments))) goto done;
if(nclistlength(segments) == 0)
{stat = NC_EURL; goto done;}
z3map->bucket = ((char*)nclistremove(segments,0));
if((stat = nczm_join(segments,&z3map->rootkey))) goto done;
nclistfreeall(segments); segments = NULL;
}
break;
case UF_VIRTUAL:
z3map->rootkey = strdup(url->path);
break;
default: stat = NC_EURL; goto done;
}
/* Verify bucket name */
if(z3map->bucket != NULL && !isLegalBucketName(z3map->bucket))
{stat = NC_EURL; goto done;}
/* Verify that this refers to an object that is below the bucket.
That is, there must be at least one segment past the bucket in the path */
nclistfreeall(segments);
segments = nclistnew();
if((stat = nczm_split_delim(z3map->rootkey,'/',segments))) goto done;
if(nclistlength(segments) == 0)
{stat = NC_EINTERNAL; goto done;} /* oops */
done:
ncbytesfree(buf);
nclistfreeall(segments);
return stat;
}
/* Create an object corresponding to a key
@return NC_NOERR if key points to a content-bearing object already
@return NC_EEMPTY if object at key has no content, if so, then create
a content-bearing object at that key.
@return NC_EXXX return true error
*/
static int
z3createobj(ZS3MAP* z3map, const char* key)
{
int stat = NC_NOERR;
char* truekey = NULL;
if((stat = maketruekey(z3map->rootkey,key,&truekey))) goto done;
stat = NCZ_s3sdkcreatekey(z3map->s3client, z3map->bucket, truekey, &z3map->errmsg);
done:
nullfree(truekey);
reporterr(z3map);
return THROW(stat);
return ZUNTRACEX(stat,"|matches|=%d",(int)nclistlength(matches));
}
/**************************************************/
/* S3 Utilities */
static int
endswith(const char* s, const char* suffix)
{
ssize_t ls, lsf, delta;
if(s == NULL || suffix == NULL) return 0;
ls = strlen(s);
lsf = strlen(suffix);
delta = (ls - lsf);
if(delta < 0) return 0;
if(memcmp(s+delta,suffix,lsf)!=0) return 0;
return 1;
}
/*
Remove all objects with keys which have
rootkey as prefix; rootkey is a truekey
@ -668,17 +508,19 @@ s3clear(ZS3MAP* z3map, const char* rootkey)
char** p;
size_t nkeys = 0;
if((stat = NCZ_s3sdkgetkeys(z3map->s3client, z3map->bucket, rootkey, &nkeys, &list, &z3map->errmsg)))
goto done;
if(list != NULL) {
for(p=list;*p;p++) {
/* If the key is the rootkey, skip it */
if(strcmp(rootkey,*p)==0) continue;
if(z3map->s3client && z3map->s3.bucket && rootkey) {
if((stat = NCZ_s3sdksearch(z3map->s3client, z3map->s3.bucket, rootkey, &nkeys, &list, &z3map->errmsg)))
goto done;
if(list != NULL) {
for(p=list;*p;p++) {
/* If the key is the rootkey, skip it */
if(strcmp(rootkey,*p)==0) continue;
#ifdef S3DEBUG
fprintf(stderr,"s3clear: %s\n",*p);
#endif
if((stat = NCZ_s3sdkdeletekey(z3map->s3client, z3map->bucket, *p, &z3map->errmsg)))
goto done;
if((stat = NCZ_s3sdkdeletekey(z3map->s3client, z3map->s3.bucket, *p, &z3map->errmsg)))
goto done;
}
}
}
@ -736,6 +578,7 @@ freevector(size_t nkeys, char** list)
NCZMAP_DS_API zmap_s3sdk;
NCZMAP_DS_API zmap_s3sdk = {
NCZM_S3SDK_V1,
ZS3_PROPERTIES,
zs3create,
zs3open,
};
@ -746,7 +589,6 @@ nczs3sdkapi = {
zs3close,
zs3exists,
zs3len,
zs3defineobj,
zs3read,
zs3write,
zs3search,

769
libnczarr/zmap_zip.c Executable file
View File

@ -0,0 +1,769 @@
/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#undef DEBUG
/* Not sure this has any effect */
#define _LARGEFILE_SOURCE 1
#define _LARGEFILE64_SOURCE 1
#include "zincludes.h"
#include <errno.h>
#include <zip.h>
#include "fbits.h"
#include "ncpathmgr.h"
#undef CACHESEARCH
#define VERIFY
/*Mnemonic*/
#define FLAG_ISDIR 1
#define FLAG_CREATE 1
#define SKIPLAST 1
#define WHOLEPATH 0
#define NCZM_ZIP_V1 1
#define ZIP_PROPERTIES (NCZM_WRITEONCE|NCZM_ZEROSTART)
/*
Do a simple mapping of our simplified map model
to a zip-file
Every dataset is assumed to be rooted at some directory in the
zip file tree. So, its location is defined by some path to a
zip file representing the dataset.
For the object API, the mapping is as follows:
1. Every content-bearing object (e.g. .zgroup or .zarray) is mapped to a zip entry.
This means that if a key points to a content bearing object then
no other key can have that content bearing key as a suffix.
2. The meta data containing files are assumed to contain
UTF-8 character data.
3. The chunk containing files are assumed to contain raw unsigned 8-bit byte data.
4. The objects may or may not be compressed; this implementation writes uncompressed objects.
*/
/* define the var name containing an objects content */
#define ZCONTENT "data"
/* Define the "subclass" of NCZMAP */
typedef struct ZZMAP {
NCZMAP map;
char* root;
char* dataset; /* prefix for all keys in zip file */
zip_t* archive;
char** searchcache;
} ZZMAP;
typedef zip_int64_t ZINDEX;;
/* Forward */
static NCZMAP_API zapi;
static int zipclose(NCZMAP* map, int delete);
static int zzcreategroup(ZZMAP*, const char* key, int nskip);
static int zzlookupobj(ZZMAP*, const char* key, ZINDEX* fd);
static int zzlen(ZZMAP* zzmap, ZINDEX zindex, size64_t* lenp);
static int zipmaperr(ZZMAP* zzmap);
static int ziperr(zip_error_t* zerror);
static int ziperrno(int zerror);
static void freesearchcache(char** cache);
static int zzinitialized = 0;
static void
zzinitialize(void)
{
if(!zzinitialized) {
ZTRACE(7,NULL);
zzinitialized = 1;
(void)ZUNTRACE(NC_NOERR);
}
}
/* Define the Dataset level API */
/*
@param datasetpath abs path in the file tree of the root of the dataset'
might be a relative path.
@param mode the netcdf-c mode flags
@param flags extra flags
@param flags extra parameters
@param mapp return the map object in this
*/
static int
zipcreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp)
{
int stat = NC_NOERR;
char* truepath = NULL;
char* dataset = NULL;
ZZMAP* zzmap = NULL;
NCURI* url = NULL;
zip_flags_t zipflags = 0;
int zerrno = ZIP_ER_OK;
ZINDEX zindex = -1;
NC_UNUSED(parameters);
ZTRACE(6,"path=%s mode=%d flag=%llu",path,mode,flags);
if(!zzinitialized) zzinitialize();
/* Fixup mode flags */
mode = (NC_NETCDF4 | NC_WRITE | mode);
/* path must be a url with file: protocol*/
ncuriparse(path,&url);
if(url == NULL)
{stat = NC_EURL; goto done;}
if(strcasecmp(url->protocol,"file") != 0)
{stat = NC_EURL; goto done;}
/* Canonicalize the root path */
if((stat = nczm_canonicalpath(url->path,&truepath))) goto done;
/* Extract the dataset name */
if((stat = nczm_basename(truepath,&dataset))) goto done;
/* Build the zz state */
if((zzmap = calloc(1,sizeof(ZZMAP))) == NULL)
{stat = NC_ENOMEM; goto done;}
zzmap->map.format = NCZM_ZIP;
zzmap->map.url = ncuribuild(url,NULL,NULL,NCURIALL);
zzmap->map.flags = flags;
/* create => NC_WRITE */
zzmap->map.mode = mode;
zzmap->map.api = &zapi;
zzmap->root = truepath;
truepath = NULL;
zzmap->dataset = dataset;
dataset = NULL;
/* Set zip openflags */
zipflags |= ZIP_CREATE;
if(fIsSet(mode,NC_NOCLOBBER))
zipflags |= ZIP_EXCL;
else
zipflags |= ZIP_TRUNCATE;
#ifdef VERIFY
zipflags |= ZIP_CHECKCONS;
#endif
if((zzmap->archive = zip_open(zzmap->root,zipflags,&zerrno))==NULL)
{stat = ziperrno(zerrno); goto done;}
/* Tell it about the dataset as a dir */
if((zindex = zip_dir_add(zzmap->archive, zzmap->dataset, ZIP_FL_ENC_UTF_8))<0)
{stat = zipmaperr(zzmap); goto done;}
/* Dataset superblock will be written by higher layer */
if(mapp) {*mapp = (NCZMAP*)zzmap; zzmap = NULL;}
done:
ncurifree(url);
nullfree(truepath);
nullfree(dataset);
if(zzmap) zipclose((NCZMAP*)zzmap,1);
return ZUNTRACE(stat);
}
/*
@param datasetpath abs path in the file tree of the root of the dataset'
might be a relative path.
@param mode the netcdf-c mode flags
@param flags extra flags
@param flags extra parameters
@param mapp return the map object in this
*/
static int
zipopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp)
{
int stat = NC_NOERR;
char* truepath = NULL;
char* dataset = NULL;
ZZMAP* zzmap = NULL;
NCURI*url = NULL;
zip_flags_t zipflags = 0;
int zerrno = ZIP_ER_OK;
NC_UNUSED(parameters);
ZTRACE(6,"path=%s mode=%d flags=%llu",path,mode,flags);
if(!zzinitialized) zzinitialize();
/* Fixup mode flags */
mode = (NC_NETCDF4 | mode);
/* path must be a url with file: protocol*/
ncuriparse(path,&url);
if(url == NULL)
{stat = NC_EURL; goto done;}
if(strcasecmp(url->protocol,"file") != 0)
{stat = NC_EURL; goto done;}
/* Canonicalize the root path */
if((stat = nczm_canonicalpath(url->path,&truepath))) goto done;
/* Build the zz state */
if((zzmap = calloc(1,sizeof(ZZMAP))) == NULL)
{stat = NC_ENOMEM; goto done;}
zzmap->map.format = NCZM_ZIP;
zzmap->map.url = ncuribuild(url,NULL,NULL,NCURIALL);
zzmap->map.flags = flags;
zzmap->map.mode = mode;
zzmap->map.api = (NCZMAP_API*)&zapi;
zzmap->root = truepath;
truepath = NULL;
/* Set zip open flags */
zipflags |= ZIP_CHECKCONS;
if(!fIsSet(mode,NC_WRITE))
zipflags |= ZIP_RDONLY;
#ifdef VERIFY
zipflags |= ZIP_CHECKCONS;
#endif
/* Open the file */
if((zzmap->archive = zip_open(zzmap->root,zipflags,&zerrno))==NULL)
{stat = ziperrno(zerrno); goto done;}
/* Use entry 0 to obtain the dataset name */
{
const char* name;
zip_int64_t num_entries;
num_entries = zip_get_num_entries(zzmap->archive, (zip_flags_t)0);
if(num_entries == 0) {stat = NC_EEMPTY; goto done;}
/* get 0'th entry name */
if((name = zip_get_name(zzmap->archive, 0, (zip_flags_t)0))==NULL)
{stat = zipmaperr(zzmap); goto done;}
if(name[0] == '\0' || name[0] == '/')
{stat = NC_EBADID; goto done;}
/* Extract the first segment as the dataset name */
if((nczm_segment1(name,&zzmap->dataset))) goto done;
}
/* Dataset superblock will be read by higher layer */
if(mapp) {*mapp = (NCZMAP*)zzmap; zzmap = NULL;}
done:
ncurifree(url);
nullfree(truepath);
nullfree(dataset);
if(zzmap) zipclose((NCZMAP*)zzmap,0);
return ZUNTRACE(stat);
}
/**************************************************/
/* Object API */
static int
zipexists(NCZMAP* map, const char* key)
{
int stat = NC_NOERR;
ZZMAP* zzmap = (ZZMAP*)map;
ZINDEX zindex = -1;
ZTRACE(6,"map=%s key=%s",map->url,key);
switch(stat=zzlookupobj(zzmap,key,&zindex)) {
case NC_NOERR: break;
case NC_ENOTFOUND: stat = NC_EEMPTY; break;
case NC_EEMPTY: break;
default: break;
}
return ZUNTRACE(stat);
}
static int
ziplen(NCZMAP* map, const char* key, size64_t* lenp)
{
int stat = NC_NOERR;
ZZMAP* zzmap = (ZZMAP*)map;
size64_t len = 0;
ZINDEX zindex = -1;
ZTRACE(6,"map=%s key=%s",map->url,key);
switch(stat = zzlookupobj(zzmap,key,&zindex)) {
case NC_NOERR:
if((stat = zzlen(zzmap,zindex,&len))) goto done;
break;
case NC_ENOTFOUND: stat = NC_EEMPTY; len = 0; break;
case NC_EEMPTY: len = 0; break; /* |dir|==0 */
default: goto done;
}
if(lenp) *lenp = len;
done:
return ZUNTRACEX(stat,"len=%llu",(lenp?*lenp:777777777777));
}
static int
zipread(NCZMAP* map, const char* key, size64_t start, size64_t count, void* content)
{
int stat = NC_NOERR;
ZZMAP* zzmap = (ZZMAP*)map; /* cast to true type */
zip_file_t* zfile = NULL;
ZINDEX zindex = -1;
zip_flags_t zipflags = 0;
int zerrno;
size64_t endpoint;
char* buffer = NULL;
char* truekey = NULL;
zip_int64_t red = 0;
ZTRACE(6,"map=%s key=%s start=%llu count=%llu",map->url,key,start,count);
switch(stat = zzlookupobj(zzmap,key,&zindex)) {
case NC_NOERR: break;
case NC_ENOTFOUND: stat = NC_EEMPTY; /* fall thru */
case NC_EEMPTY: /* its a dir; fall thru*/
default: goto done;
}
/* Note, assume key[0] == '/' */
if((stat = nczm_appendn(&truekey,2,zzmap->dataset,key)))
goto done;
zfile = zip_fopen(zzmap->archive, truekey, zipflags);
if(zfile == NULL)
{stat = (zipmaperr(zzmap)); goto done;}
/* Ideally, we would like to seek to the start point,
but that will fail if the file is compressed, so
we need to read whole thing and extract what we need
*/
/* read data starting at zero */
if(start == 0) { /*optimize to read directly into content */
if((red = zip_fread(zfile, content, (zip_uint64_t)count)) < 0)
{stat = (zipmaperr(zzmap)); goto done;}
if(red < count) {stat = NC_EINTERNAL; goto done;}
} else {
endpoint = start + count;
if((buffer = malloc(endpoint))==NULL) /* consider caching this */
{stat = NC_ENOMEM; goto done;}
if((red = zip_fread(zfile, buffer, (zip_uint64_t)endpoint)) < 0)
{stat = (zipmaperr(zzmap)); goto done;}
if(red < endpoint) {stat = NC_EINTERNAL; goto done;}
/* Extract what we need */
memcpy(content,buffer+start,count);
}
done:
nullfree(truekey);
nullfree(buffer);
if(zfile != NULL && (zerrno=zip_fclose(zfile)) != 0)
{stat = ziperrno(zerrno);}
return ZUNTRACE(stat);
}
static int
zipwrite(NCZMAP* map, const char* key, size64_t start, size64_t count, const void* content)
{
int stat = NC_NOERR;
ZZMAP* zzmap = (ZZMAP*)map; /* cast to true type */
char* truekey = NULL;
zip_flags_t zflags = 0;
zip_source_t* zs = NULL;
ZINDEX zindex = -1;
zip_int32_t compression = 0;
zip_error_t zerror;
void* localbuffer = NULL;
ZTRACE(6,"map=%s key=%s start=%llu count=%llu",map->url,key,start,count);
zip_error_init(&zerror);
if(start != 0 && (ZIP_PROPERTIES & NCZM_ZEROSTART))
{stat = NC_EEDGE; goto done;}
/* Create directories */
if((stat = zzcreategroup(zzmap,key,SKIPLAST))) goto done;
switch(stat = zzlookupobj(zzmap,key,&zindex)) {
case NC_NOERR:
stat = NC_EFOUND; //goto done; /* Zip files are write once */
zflags |= ZIP_FL_OVERWRITE;
break;
case NC_ENOTFOUND: stat = NC_NOERR; break;
case NC_EEMPTY: /* its a dir; fall thru */
default: goto done;
}
zflags |= ZIP_FL_ENC_UTF_8;
compression = ZIP_CM_STORE;
/* prepend the dataset to get truekey */
/* Note, assume key[0] == '/' */
if((stat = nczm_appendn(&truekey,2,zzmap->dataset,key)))
goto done;
if(count > 0) {
/* Apparently, the buffer to be written needs to be around at zip_close
so we need to make a local copy that will be freed by libzip after it is
no longer needed */
/* Duplicate the buffer */
if((localbuffer = malloc((size_t)count))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(localbuffer,content,count);
}
if((zs = zip_source_buffer(zzmap->archive, localbuffer, (zip_uint64_t)count, 1)) == NULL)
{stat = zipmaperr(zzmap); goto done;}
if((zindex = zip_file_add(zzmap->archive, truekey, zs, zflags))<0)
{stat = zipmaperr(zzmap); goto done;}
zs = NULL; localbuffer = NULL;
if(zip_set_file_compression(zzmap->archive, zindex, compression, 0) < 0)
{stat = zipmaperr(zzmap); goto done;}
freesearchcache(zzmap->searchcache); zzmap->searchcache = NULL;
done:
if(zs) zip_source_free(zs);
nullfree(localbuffer);
zip_error_fini(&zerror);
nullfree(truekey);
return ZUNTRACE(stat);
}
static int
zipclose(NCZMAP* map, int delete)
{
int stat = NC_NOERR;
int zerrno = 0;
ZZMAP* zzmap = (ZZMAP*)map;
if(zzmap == NULL) return NC_NOERR;
ZTRACE(6,"map=%s delete=%d",map->url,delete);
/* Close the zip */
if(delete)
zip_discard(zzmap->archive);
else {
if((zerrno=zip_close(zzmap->archive)))
stat = ziperrno(zerrno);
}
if(delete)
NCremove(zzmap->root);
zzmap->archive = NULL;
nczm_clear(map);
nullfree(zzmap->root);
nullfree(zzmap->dataset);
zzmap->root = NULL;
freesearchcache(zzmap->searchcache);
free(zzmap);
return ZUNTRACE(stat);
}
/*
Return a list of full keys immediately under a specified prefix key.
In theory, the returned list should be sorted in lexical order,
but it possible that it is not.
Note that for zip, it is not possible to get just the keys of length n+1,
so, this search must get all keys and process them one by one.
@return NC_NOERR if success, even if no keys returned.
@return NC_EXXX return true error
*/
int
zipsearch(NCZMAP* map, const char* prefix0, NClist* matches)
{
int stat = NC_NOERR;
ZZMAP* zzmap = (ZZMAP*)map;
char* trueprefix = NULL;
size_t truelen;
zip_int64_t num_entries, i;
char** cache = NULL;
size_t prefixlen;
NClist* tmp = NULL;
ZTRACE(6,"map=%s prefix0=%s",map->url,prefix0);
/* prefix constraints:
1. prefix is "/"
2. or prefix has leading '/' and no trailing '/'
*/
/* Fix up the prefix; including adding the dataset name to the front */
if(prefix0 == NULL || strlen(prefix0)==0)
prefix0 = "/";
/* make sure that prefix0 has leading '/' */
if(prefix0[0] != '/')
{stat = NC_EINVAL; goto done;}
prefixlen = strlen(prefix0);
truelen = prefixlen+strlen(zzmap->dataset)+1; /* possible trailing '/'*/
if((trueprefix = (char*)malloc(truelen+1+1))==NULL) /* nul term */
{stat = NC_ENOMEM; goto done;}
/* Build the true prefix */
trueprefix[0] = '\0';
strlcat(trueprefix,zzmap->dataset,truelen+1);
strlcat(trueprefix,prefix0,truelen+1); /* recall prefix starts with '/' */
/* If the prefix did not end in '/', then add it */
if(prefixlen > 1 && prefix0[prefixlen-1] != '/')
strlcat(trueprefix,"/",truelen+1);
truelen = strlen(trueprefix);
/* Get number of entries */
num_entries = zip_get_num_entries(zzmap->archive, (zip_flags_t)0);
#ifdef CACHESEARCH
if(num_entries > 0 && zzmap->searchcache == NULL) {
/* Release the current cache */
freesearchcache(zzmap->searchcache);
zzmap->searchcache = NULL;
/* Re-build the searchcache */
if((cache = calloc(sizeof(char*),num_entries+1))==NULL)
{stat = NC_ENOMEM; goto done;}
for(i=0;i < num_entries; i++) {
const char *name = NULL;
/* get ith entry */
name = zip_get_name(zzmap->archive, i, (zip_flags_t)0);
/* Add to cache */
if((cache[i] = strdup(name))==NULL)
{stat = NC_ENOMEM; goto done;}
}
cache[num_entries] = NULL;
zzmap->searchcache = cache; cache = NULL;
}
#endif
#ifdef CACHESEARCH
if(zzmap->searchcache != NULL)
#endif
{
const char *key = NULL;
size_t keylen = 0;
char* match = NULL;
const char* p;
tmp = nclistnew();
/* Walk cache looking for names with prefix plus exactly one other segment */
for(i=0;i < num_entries; i++) {
/* get ith entry */
#ifdef CACHESEARCH
key = zzmap->searchcache[i];
#else
key = zip_get_name(zzmap->archive, i, (zip_flags_t)0);
#endif
keylen = strlen(key);
/* Does this name begin with trueprefix? */
if(keylen > 0
&& (keylen <= truelen || strncmp(key,trueprefix,truelen) != 0))
continue; /* no match */
/* skip trueprefix and extract first segment */
p = (key+truelen);
if(*p == '\0') continue; /* key is all there is, so ignore it */
/* get seg 1 */
if((nczm_segment1(p,&match))) goto done;
nclistpush(tmp,match); match = NULL;
}
/* Now remove duplicates */
for(i=0;i<nclistlength(tmp);i++) {
int j;
int duplicate = 0;
const char* is = nclistget(tmp,i);
for(j=0;j<nclistlength(matches);j++) {
const char* js = nclistget(matches,j);
if(strcmp(js,is)==0) {duplicate = 1; break;} /* duplicate */
}
if(!duplicate)
nclistpush(matches,strdup(is));
}
}
done:
nclistfreeall(tmp);
if(cache != NULL) freesearchcache(cache);
nullfree(trueprefix);
return ZUNTRACEX(stat,"|matches|=%d",(int)nclistlength(matches));
}
/**************************************************/
/* Utilities */
/* Guarantee existence of a group */
static int
zzcreategroup(ZZMAP* zzmap, const char* key, int nskip)
{
int stat = NC_NOERR;
int i, len;
char* fullpath = NULL;
NCbytes* path = ncbytesnew();
NClist* segments = nclistnew();
ZINDEX zindex;
zip_flags_t zipflags = ZIP_FL_ENC_UTF_8;
ZTRACE(7,"map=%s key=%s nskip=%d",zzmap->map.url,key,nskip);
if((stat=nczm_split(key,segments)))
goto done;
len = nclistlength(segments);
len -= nskip; /* leave off last nskip segments */
/* Start with the dataset */
ncbytescat(path,zzmap->dataset);
for(i=0;i<len;i++) {
const char* seg = nclistget(segments,i);
ncbytescat(path,"/");
ncbytescat(path,seg);
/* open and/or create the directory */
if((zindex = zip_dir_add(zzmap->archive, ncbytescontents(path), zipflags))<0) {
switch(stat = zipmaperr(zzmap)) {
case NC_EFOUND: stat = NC_NOERR; break; /* ok */
default:
goto done;
}
}
}
done:
nullfree(fullpath);
ncbytesfree(path);
nclistfreeall(segments);
return ZUNTRACE(stat);
}
/* Lookup a key
@return NC_NOERR if found and is a content-bearing object
@return NC_ENOTFOUND if not found
@return NC_EEMPTY if a dir
*/
static int
zzlookupobj(ZZMAP* zzmap, const char* key, ZINDEX* zindex)
{
int stat = NC_NOERR;
char* zipfile = NULL;
char* zipdir = NULL;
ZTRACE(7,"map=%s key=%s",zzmap->map.url,key);
if(key == NULL)
{stat = NC_EINVAL; goto done;}
/* Note, assume key[0] == '/' */
if((stat = nczm_appendn(&zipfile,2,zzmap->dataset,key)))
goto done;
/* See if exists as a file */
if((*zindex = zip_name_locate(zzmap->archive, zipfile, 0))<0) {
/* do a second check to see if zippath as a dir */
if((stat = nczm_appendn(&zipdir,2,zipfile,"/")))
goto done;
if((*zindex = zip_name_locate(zzmap->archive, zipdir, 0))<0)
{stat = zipmaperr(zzmap); goto done;}
stat = NC_EEMPTY; /* signal a directory */
}
done:
nullfree(zipfile);
nullfree(zipdir);
return ZUNTRACE(stat);
}
/* Get length given the index */
static int
zzlen(ZZMAP* zzmap, ZINDEX zindex, size64_t* lenp)
{
int stat = NC_NOERR;
size64_t len = 0;
zip_stat_t statbuf;
zip_flags_t zipflags = 0;
assert(zindex >= 0);
ZTRACE(6,"zzmap=%s index=%llu",zzmap,zindex);
zip_stat_init(&statbuf);
if(zip_stat_index(zzmap->archive,zindex,zipflags,&statbuf) < 0)
{stat = (zipmaperr(zzmap)); goto done;}
assert(statbuf.valid & ZIP_STAT_SIZE);
len = statbuf.size; /* Always return uncompressed size */
if(lenp) *lenp = len;
done:
return ZUNTRACEX(stat,"len=%llu",(lenp?*lenp:777777777777));
}
static void
freesearchcache(char** cache)
{
char** p;
if(cache == NULL) return;
for(p=cache;*p;p++) {
free(*p);
}
free(cache);
}
/**************************************************/
/* External API objects */
NCZMAP_DS_API zmap_zip = {
NCZM_ZIP_V1,
ZIP_PROPERTIES,
zipcreate,
zipopen,
};
static NCZMAP_API zapi = {
NCZM_ZIP_V1,
zipclose,
zipexists,
ziplen,
zipread,
zipwrite,
zipsearch,
};
static int
zipmaperr(ZZMAP* zzmap)
{
zip_error_t* zerr = (zip_error_t*)zip_get_error(zzmap->archive);
return ziperr(zerr);
}
static int
ziperr(zip_error_t* zerror)
{
int zerrno = zip_error_code_zip(zerror);
return ziperrno(zerrno);
}
static int
ziperrno(int zerror)
{
int stat = NC_NOERR;
switch (zerror) {
case ZIP_ER_OK: stat = NC_NOERR; break;
case ZIP_ER_EXISTS: stat = NC_EFOUND; break;
case ZIP_ER_MEMORY: stat = NC_ENOMEM; break;
case ZIP_ER_SEEK:
case ZIP_ER_READ:
case ZIP_ER_WRITE:
case ZIP_ER_TMPOPEN:
case ZIP_ER_CRC: stat = NC_EIO; break;
case ZIP_ER_ZIPCLOSED: stat = NC_EBADID; break;
case ZIP_ER_NOENT: stat = NC_ENOTFOUND; break;
case ZIP_ER_OPEN: stat = NC_EACCESS; break;
case ZIP_ER_INVAL: stat = NC_EINVAL; break;
case ZIP_ER_INTERNAL: stat = NC_EINTERNAL; break;
case ZIP_ER_REMOVE: stat = NC_ECANTREMOVE; break;
case ZIP_ER_DELETED: stat = NC_ENOTFOUND; break;
case ZIP_ER_RDONLY: stat = NC_EPERM; break;
case ZIP_ER_CHANGED: stat = NC_EFOUND; break;
default: stat = NC_ENCZARR; break;
}
return stat;
}

360
libnczarr/zs3sdk.cpp
View File

@ -11,6 +11,7 @@
#include <iostream>
#include <streambuf>
#include "netcdf.h"
#include "zincludes.h"
#include "zs3sdk.h"
#ifdef __CYGWIN__
@ -25,24 +26,27 @@ static Aws::SDKOptions zs3options;
static Aws::S3::Model::BucketLocationConstraint s3findregion(const char* name);
static int s3objectinfo1(const Aws::S3::Model::Object& s3_object, char** fullkeyp, size64_t* lenp);
static int s3objectsinfo(Aws::Vector<Aws::S3::Model::Object> list, size_t*, char*** keysp, size64_t** lenp);
static int s3commonprefixes(Aws::Vector<Aws::S3::Model::CommonPrefix> list, char*** keysp);
static void freestringenvv(char** ss);
static int makes3key(const char* pathkey, const char** keyp);
static int makes3keydir(const char* prefix, char** prefixdirp);
static char** mergekeysets(size_t nkeys1, char** keys1, size_t nkeys2, char** keys2);
#ifndef nullfree
#define nullfree(x) {if(x) {free(x);}}
#endif
void
NCZ_s3sdkinitialize(void)
{
ZTRACE(11,NULL);
// zs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Error;
Aws::InitAPI(zs3options);
ZUNTRACE(NC_NOERR);
}
void
NCZ_s3sdkfinalize(void)
{
ZTRACE(11,NULL);
Aws::ShutdownAPI(zs3options);
ZUNTRACE(NC_NOERR);
}
static char*
@ -65,6 +69,8 @@ int
NCZ_s3sdkcreateconfig(const char* host, const char* region, void** configp)
{
int stat = NC_NOERR;
ZTRACE(11,"host=%s region=%s");
Aws::Client::ClientConfiguration *config = new Aws::Client::ClientConfiguration();
config->scheme = Aws::Http::Scheme::HTTPS;
config->connectTimeoutMs = 300000;
@ -74,12 +80,13 @@ NCZ_s3sdkcreateconfig(const char* host, const char* region, void** configp)
config->enableEndpointDiscovery = true;
config->followRedirects = true;
if(configp) * configp = config;
return stat;
return ZUNTRACE(stat);
}
int
NCZ_s3sdkcreateclient(void* config0, void** clientp)
{
ZTRACE(11,NULL);
Aws::Client::ClientConfiguration* config = (Aws::Client::ClientConfiguration*) config0;
Aws::S3::S3Client *s3client
= new Aws::S3::S3Client(*config,
@ -88,7 +95,7 @@ NCZ_s3sdkcreateclient(void* config0, void** clientp)
Aws::S3::US_EAST_1_REGIONAL_ENDPOINT_OPTION::NOT_SET
);
if(clientp) *clientp = (void*)s3client;
return NC_NOERR;
return ZUNTRACE(NC_NOERR);
}
int
@ -97,7 +104,7 @@ NCZ_s3sdkbucketexists(void* s3client0, const char* bucket, int* existsp, char**
int stat = NC_NOERR;
int exists = 0;
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
ZTRACE(11,"bucket=%s",bucket);
if(errmsgp) *errmsgp = NULL;
auto result = s3client->ListBuckets();
if(!result.IsSuccess()) {
@ -111,13 +118,15 @@ NCZ_s3sdkbucketexists(void* s3client0, const char* bucket, int* existsp, char**
}
}
if(existsp) *existsp = exists;
return stat;
return ZUNTRACEX(stat,"exists=%d",*existsp);
}
int
NCZ_s3sdkbucketcreate(void* s3client0, const char* region, const char* bucket, char** errmsgp)
{
int stat = NC_NOERR;
ZTRACE(11,"region=%s bucket=%s",region,bucket);
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
if(errmsgp) *errmsgp = NULL;
const Aws::S3::Model::BucketLocationConstraint &awsregion = s3findregion(region);
@ -143,13 +152,16 @@ NCZ_s3sdkbucketcreate(void* s3client0, const char* region, const char* bucket, c
fprintf(stderr,"create bucket: %s\n",bucket); fflush(stderr);
#endif
return stat;
return ZUNTRACE(stat);
}
int
NCZ_s3sdkbucketdelete(void* s3client0, void* config0, const char* region, const char* bucket, char** errmsgp)
{
int stat = NC_NOERR;
ZTRACE(11,"region=%s bucket=%s",region,bucket);
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::Client::ClientConfiguration *config = (Aws::Client::ClientConfiguration*)config0;
@ -174,7 +186,7 @@ NCZ_s3sdkbucketdelete(void* s3client0, void* config0, const char* region, const
fprintf(stderr,"delete bucket: %s\n",bucket); fflush(stderr);
#endif
return stat;
return ZUNTRACE(stat);
}
/**************************************************/
@ -189,13 +201,15 @@ int
NCZ_s3sdkinfo(void* s3client0, const char* bucket, const char* pathkey, size64_t* lenp, char** errmsgp)
{
int stat = NC_NOERR;
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::HeadObjectRequest head_request;
const char* key = NULL;
ZTRACE(11,"bucket=%s pathkey=%s",bucket,pathkey);
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::HeadObjectRequest head_request;
if(*pathkey != '/') return NC_EINTERNAL;
/* extract the true s3 key*/
if((stat = makes3key(pathkey,&key))) return stat;
if((stat = makes3key(pathkey,&key))) return ZUNTRACE(stat);
if(errmsgp) *errmsgp = NULL;
head_request.SetBucket(bucket);
@ -205,6 +219,7 @@ NCZ_s3sdkinfo(void* s3client0, const char* bucket, const char* pathkey, size64_t
long long l = head_outcome.GetResult().GetContentLength();
if(lenp) *lenp = (size64_t)l;
} else {
if(lenp) *lenp = 0;
/* Distinquish not-found from other errors */
switch (head_outcome.GetError().GetErrorType()) {
case Aws::S3::S3Errors::RESOURCE_NOT_FOUND:
@ -219,33 +234,7 @@ NCZ_s3sdkinfo(void* s3client0, const char* bucket, const char* pathkey, size64_t
break;
}
}
return (stat);
}
/* Define object key
@return NC_NOERR if success
@return NC_EXXX if fail
*/
int
NCZ_s3sdkcreatekey(void* s3client0, const char* bucket, const char* pathkey, char** errmsgp)
{
int stat = NC_NOERR;
const char* key = NULL;
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::PutObjectRequest put_request;
if(*pathkey != '/') return NC_EINTERNAL;
if((stat = makes3key(pathkey,&key))) return stat;
if(errmsgp) *errmsgp = NULL;
put_request.SetBucket(bucket);
put_request.SetKey(key);
auto put_result = s3client->PutObject(put_request);
if(!put_result.IsSuccess()) {
if(errmsgp) *errmsgp = makeerrmsg(put_result.GetError(),key);
stat = NC_ES3;
}
return (stat);
return ZUNTRACEX(stat,"len=%d",(int)(lenp?*lenp:-1));
}
/*
@ -256,17 +245,24 @@ int
NCZ_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t start, size64_t count, void* content, char** errmsgp)
{
int stat = NC_NOERR;
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::GetObjectRequest object_request;
char range[1024];
const char* key = NULL;
size64_t rangeend;
ZTRACE(11,"bucket=%s pathkey=%s start=%llu count=%llu content=%p",bucket,pathkey,start,count,content);
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::GetObjectRequest object_request;
if(count == 0) return ZUNTRACE(stat);
if(*pathkey != '/') return NC_EINTERNAL;
if((stat = makes3key(pathkey,&key))) return stat;
if((stat = makes3key(pathkey,&key))) return ZUNTRACE(stat);
object_request.SetBucket(bucket);
object_request.SetKey(key);
snprintf(range,sizeof(range),"bytes=%llu-%llu",start,(start+count)-1);
rangeend = (start+count)-1;
snprintf(range,sizeof(range),"bytes=%llu-%llu",start,rangeend);
object_request.SetRange(range);
auto get_object_result = s3client->GetObject(object_request);
if(!get_object_result.IsSuccess()) {
@ -283,7 +279,7 @@ NCZ_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t
if(content)
memcpy(content,s,slen);
}
return (stat);
return ZUNTRACE(stat);
}
/*
@ -295,11 +291,14 @@ NCZ_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey,
{
int stat = NC_NOERR;
const char* key = NULL;
ZTRACE(11,"bucket=%s pathkey=%s count=%lld content=%p",bucket,pathkey,count,content);
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::PutObjectRequest put_request;
if(*pathkey != '/') return NC_EINTERNAL;
if((stat = makes3key(pathkey,&key))) return stat;
if(*pathkey != '/') return ZUNTRACE(NC_EINTERNAL);
if((stat = makes3key(pathkey,&key))) return ZUNTRACE(stat);
if(errmsgp) *errmsgp = NULL;
put_request.SetBucket(bucket);
@ -314,70 +313,16 @@ NCZ_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey,
if(errmsgp) *errmsgp = makeerrmsg(put_result.GetError(),key);
stat = NC_ES3;
}
return (stat);
return ZUNTRACE(stat);
}
#if 0
/*
@return NC_NOERR if success
@return NC_EXXX if fail
*/
int
NCZ_s3sdkreadobject(void* s3client0, const char* bucket, const char* pathkey, size64_t* sizep, void** contentp, char** errmsgp)
{
int stat = NC_NOERR;
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::GetObjectRequest object_request;
size64_t size, red;
void* content = NULL;
const char* key = NULL;
if(*pathkey != '/') return NC_EINTERNAL;
if((stat = makes3key(pathkey,&key))) return stat;
if(errmsgp) *errmsgp = NULL;
object_request.SetBucket(bucket);
object_request.SetKey(key);
auto get_result = s3client->GetObject(object_request);
if(!get_result.IsSuccess()) {
if(errmsgp) *errmsgp = makeerrmsg(get_result.GetError(),key);
stat = NC_ES3;
} else {
/* Get the size */
size = (size64_t)get_result.GetResult().GetContentLength();
/* Get the whole result */
Aws::IOStream &result = get_result.GetResultWithOwnership().GetBody();
#if 0
std::string str((std::istreambuf_iterator<char>(result)),std::istreambuf_iterator<char>());
red = str.length();
if((content = malloc(red))==NULL)
{stat = NC_ENOMEM; goto done;}
memcpy(content,str.c_str(),red);
#else
red = result.rdbuf()->pubseekoff(0,std::ios::ios_base::end);
result.rdbuf()->pubseekoff(0,std::ios::ios_base::beg); /* reset for reading */
if((content = malloc(red))==NULL)
stat = NC_ENOMEM;
else
result.rdbuf()->sgetn((char*)content,red);
#endif
if(!stat) {
/* Verify actual result size */
if(red != size) {stat = NC_ES3; goto done;}
if(sizep) *sizep = red;
if(contentp) {*contentp = content; content = NULL;}
}
}
done:
nullfree(content);
return (stat);
}
#endif /*0*/
int
NCZ_s3sdkclose(void* s3client0, void* config0, const char* bucket, const char* rootkey, int deleteit, char** errmsgp)
{
int stat = NC_NOERR;
ZTRACE(11,"bucket=%s rootkey=%s deleteit=%d content=%p",bucket,rootkey,deleteit);
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::Client::ClientConfiguration *config = (Aws::Client::ClientConfiguration*)config0;
if(deleteit) {
@ -390,53 +335,151 @@ NCZ_s3sdkclose(void* s3client0, void* config0, const char* bucket, const char* r
}
delete s3client;
delete config;
return (stat);
return ZUNTRACE(stat);
}
/*
Return a list of keys for objects "below" a specified rootkey.
Return a list of names of legal objects immediately below a specified key.
In theory, the returned list should be sorted in lexical order,
but it possible that it is not.
*/
int
NCZ_s3sdkgetkeys(void* s3client0, const char* bucket, const char* rootkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
NCZ_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
{
int stat = NC_NOERR;
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
size_t nkeys = 0;
const char* rootkey = NULL;
const char* prefix = NULL;
char* prefixdir = NULL;
char** realkeys = NULL;
char** commonkeys = NULL;
char** allkeys = NULL;
ZTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::ListObjectsV2Request objects_request;
if(*rootkey0 != '/') return NC_EINTERNAL;
if((stat = makes3key(rootkey0,&rootkey))) return stat;
if(*prefixkey0 != '/') return ZUNTRACE(NC_EINTERNAL);
/* Make sure that the prefix ends with '/' */
if((stat = makes3keydir(prefixkey0,&prefixdir))) return ZUNTRACE(stat);
/* remove leading '/' */
if((stat = makes3key(prefixdir,&prefix))) return ZUNTRACE(stat);
if(errmsgp) *errmsgp = NULL;
objects_request.SetBucket(bucket);
objects_request.SetPrefix(rootkey);
objects_request.SetPrefix(prefix);
objects_request.SetDelimiter("/"); /* Force return of common prefixes */
auto objects_outcome = s3client->ListObjectsV2(objects_request);
if(objects_outcome.IsSuccess()) {
size_t nrealkeys = 0;
size_t ncommonkeys = 0;
Aws::Vector<Aws::S3::Model::Object> object_list =
objects_outcome.GetResult().GetContents();
nkeys = (size_t)object_list.size();
if(nkeysp) *nkeysp = nkeys;
stat = s3objectsinfo(object_list,NULL,keysp,NULL);
nrealkeys = (size_t)object_list.size();
stat = s3objectsinfo(object_list,NULL,&realkeys,NULL);
/* Add common prefixes */
Aws::Vector<Aws::S3::Model::CommonPrefix> common_list =
objects_outcome.GetResult().GetCommonPrefixes();
ncommonkeys = (size_t)common_list.size();
stat = s3commonprefixes(common_list,&commonkeys);
/* merge the two lists */
if((allkeys=mergekeysets(nrealkeys, realkeys, ncommonkeys, commonkeys))==NULL)
stat = NC_ENOMEM;
if(stat == NC_NOERR) {
if(nkeysp) {*nkeysp = nrealkeys + ncommonkeys;}
if(keysp) {*keysp = allkeys; allkeys = NULL;}
}
freestringenvv(allkeys);
freestringenvv(realkeys);
freestringenvv(commonkeys);
} else {
if(errmsgp) *errmsgp = makeerrmsg(objects_outcome.GetError());
stat = NC_ES3;
}
return stat;
if(prefixdir) free(prefixdir);
return ZUNTRACEX(stat,"nkeys=%u",(unsigned)*nkeysp);
}
/*
Return a list of full keys of legal objects immediately below a specified key.
Not necessarily sorted.
*/
int
NCZ_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
{
int stat = NC_NOERR;
size_t nkeys = 0;
const char* prefix = NULL;
char* prefixdir = NULL;
char** keys = NULL;
ZTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::ListObjectsV2Request objects_request;
if(*prefixkey0 != '/') return ZUNTRACE(NC_EINTERNAL);
/* Make sure that the prefix ends with '/' */
if((stat = makes3keydir(prefixkey0,&prefixdir))) return ZUNTRACE(stat);
/* remove leading '/' */
if((stat = makes3key(prefixdir,&prefix))) return ZUNTRACE(stat);
if(errmsgp) *errmsgp = NULL;
objects_request.SetBucket(bucket);
objects_request.SetPrefix(prefix);
/* Do not use delimiter so we get all full key paths */
auto objects_outcome = s3client->ListObjectsV2(objects_request);
if(objects_outcome.IsSuccess()) {
size_t nkeys = 0;
Aws::Vector<Aws::S3::Model::Object> object_list =
objects_outcome.GetResult().GetContents();
nkeys = (size_t)object_list.size();
if((keys=(char**)calloc(sizeof(char*),(nkeys+1)))==NULL) /* NULL terminate list */
stat = NC_ENOMEM;
if(!stat) {
int i;
i = 0;
for (auto const &s3obj : object_list) {
const char* s;
const Aws::String& name = s3obj.GetKey();
s = name.c_str();
if(s != NULL) {
char* p;
size_t slen = name.length();
p = (char*)malloc(slen+1+1); /* nul plus leading '/' */
if(s[0] != '/') {p[0] = '/'; memcpy(p+1,s,slen); slen++;} else {memcpy(p,s,slen);}
p[slen] = '\0';
keys[i++] = p;
}
}
}
if(stat == NC_NOERR) {
if(nkeysp) {*nkeysp = nkeys;}
if(keysp) {*keysp = keys; keys = NULL;}
}
freestringenvv(keys);
} else {
if(errmsgp) *errmsgp = makeerrmsg(objects_outcome.GetError());
stat = NC_ES3;
}
if(prefixdir) free(prefixdir);
return ZUNTRACEX(stat,"nkeys=%u",(unsigned)*nkeysp);
}
int
NCZ_s3sdkdeletekey(void* s3client0, const char* bucket, const char* pathkey, char** errmsgp)
{
int stat = NC_NOERR;
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::DeleteObjectRequest delete_request;
const char* key = NULL;
ZTRACE(11,"bucket=%s pathkey=%s",bucket,pathkey);
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
Aws::S3::Model::DeleteObjectRequest delete_request;
if(*pathkey != '/') return NC_EINTERNAL;
if((stat = makes3key(pathkey,&key))) return stat;
if((stat = makes3key(pathkey,&key))) return ZUNTRACE(stat);
/* Delete this key object */
delete_request.SetBucket(bucket);
delete_request.SetKey(key);
@ -445,7 +488,7 @@ NCZ_s3sdkdeletekey(void* s3client0, const char* bucket, const char* pathkey, cha
if(errmsgp) *errmsgp = makeerrmsg(delete_result.GetError(),key);
stat = NC_ES3;
}
return stat;
return ZUNTRACE(stat);
}
/*
@ -483,7 +526,7 @@ s3objectinfo1(const Aws::S3::Model::Object& s3_object, char** fullkeyp, size64_t
if(!stat) {
if(lenp) *lenp = (size64_t)s3_object.GetSize();
}
nullfree(cstr);
if(cstr) free(cstr);
return stat;
}
@ -524,6 +567,39 @@ s3objectsinfo(Aws::Vector<Aws::S3::Model::Object> list, size_t* nkeysp, char***
return stat;
}
static int
s3commonprefixes(Aws::Vector<Aws::S3::Model::CommonPrefix> list, char*** keysp)
{
int stat = NC_NOERR;
char** keys = NULL;
size_t nkeys;
int i;
nkeys = list.size();
if((keys=(char**)calloc(sizeof(char*),(nkeys+1)))==NULL)
stat = NC_ENOMEM;
if(!stat) {
i = 0;
for (auto const &s3prefix : list) {
char* p; char* p1;
size_t len;
const Aws::String& prefix = s3prefix.GetPrefix();
len = prefix.length();
if((p = (char*) malloc(len+1+1))==NULL) /* for nul + leading '/' */
stat = NC_ENOMEM;
if(stat == NC_NOERR) {
if(*p == '/') {p1 = p;} else {*p = '/'; p1 = p+1;}
memcpy(p1,prefix.c_str(),len);
p1[len] = '\0';
keys[i++] = p;
}
}
}
if(keysp) {keys[nkeys] = NULL; *keysp = keys; keys = NULL;}
if(keys != NULL) freestringenvv(keys);
return stat;
}
static Aws::S3::Model::BucketLocationConstraint
s3findregion(const char* name)
{
@ -543,7 +619,7 @@ freestringenvv(char** ss)
char** p;
if(ss != NULL) {
for(p=ss;*p;p++)
nullfree(*p);
if(*p) free(*p);
free(ss);
}
}
@ -555,3 +631,41 @@ makes3key(const char* pathkey, const char** keyp)
if(keyp) *keyp = pathkey+1;
return NC_NOERR;
}
static int
makes3keydir(const char* prefix, char** prefixdirp)
{
size_t plen = strlen(prefix);
char* prefixdir = (char*)malloc(plen+1+1); /* '/' + nul */
if(prefixdir == NULL) return NC_ENOMEM;
memcpy(prefixdir,prefix,plen);
if(prefixdir[plen-1] != '/') {
prefixdir[plen++] = '/';
}
prefixdir[plen] = '\0';
*prefixdirp = prefixdir; prefixdir = NULL;
return NC_NOERR;
}
static char**
mergekeysets(size_t nkeys1, char** keys1, size_t nkeys2, char** keys2)
{
char** merge = NULL;
char** p1;
char** p2;
merge = (char**)malloc(sizeof(char*)*(nkeys1+nkeys2+1));
if(merge == NULL) return NULL;
if(nkeys1 > 0) {
memcpy(merge,keys1,sizeof(char*)*nkeys1);
/* avoid double free */
memset(keys1,0,sizeof(char*)*nkeys1);
}
if(nkeys2 > 0) {
memcpy(merge+nkeys1,keys2,sizeof(char*)*nkeys2);
/* avoid double free */
memset(keys2,0,sizeof(char*)*nkeys2);
}
merge[nkeys1+nkeys2] = NULL;
return merge;
}

28
libnczarr/zs3sdk.h
View File

@ -10,20 +10,20 @@
extern "C" {
#endif
void NCZ_s3sdkinitialize(void);
void NCZ_s3sdkfinalize(void);
int NCZ_s3sdkcreateconfig(const char* host, const char* reqion, void** configp);
int NCZ_s3sdkcreateclient(void* config, void** clientp);
int NCZ_s3sdkbucketexists(void* s3client, const char* bucket, int* existsp, char** errmsgp);
int NCZ_s3sdkbucketcreate(void* s3client, const char* region, const char* bucket, char** errmsgp);
int NCZ_s3sdkbucketdelete(void* s3client, const char* region, const char* bucket, char** errmsgp);
int NCZ_s3sdkinfo(void* client0, const char* bucket, const char* pathkey, unsigned long long* lenp, char** errmsgp);
int NCZ_s3sdkread(void* client0, const char* bucket, const char* pathkey, unsigned long long start, unsigned long long count, void* content, char** errmsgp);
int NCZ_s3sdkwriteobject(void* client0, const char* bucket, const char* pathkey, unsigned long long count, const void* content, char** errmsgp);
int NCZ_s3sdkclose(void* s3client0, void* config0, const char* bucket, const char* rootkey, int deleteit, char** errmsgp);
int NCZ_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp);
int NCZ_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp);
int NCZ_s3sdkcreatekey(void* s3client0, const char* bucket, const char* pathkey, char** errmsgp);
EXTERNL void NCZ_s3sdkinitialize(void);
EXTERNL void NCZ_s3sdkfinalize(void);
EXTERNL int NCZ_s3sdkcreateconfig(const char* host, const char* reqion, void** configp);
EXTERNL int NCZ_s3sdkcreateclient(void* config, void** clientp);
EXTERNL int NCZ_s3sdkbucketexists(void* s3client, const char* bucket, int* existsp, char** errmsgp);
EXTERNL int NCZ_s3sdkbucketcreate(void* s3client, const char* region, const char* bucket, char** errmsgp);
EXTERNL int NCZ_s3sdkbucketdelete(void* s3client, const char* region, const char* bucket, char** errmsgp);
EXTERNL int NCZ_s3sdkinfo(void* client0, const char* bucket, const char* pathkey, unsigned long long* lenp, char** errmsgp);
EXTERNL int NCZ_s3sdkread(void* client0, const char* bucket, const char* pathkey, unsigned long long start, unsigned long long count, void* content, char** errmsgp);
EXTERNL int NCZ_s3sdkwriteobject(void* client0, const char* bucket, const char* pathkey, unsigned long long count, const void* content, char** errmsgp);
EXTERNL int NCZ_s3sdkclose(void* s3client0, void* config0, const char* bucket, const char* rootkey, int deleteit, char** errmsgp);
EXTERNL int NCZ_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp);
EXTERNL int NCZ_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp);
EXTERNL int NCZ_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp);
#ifdef __cplusplus
}

26
libnczarr/zsync.c
View File

@ -51,20 +51,31 @@ that the recursion occurs in the caller's code.
* @author Dennis Heimbigner
*/
int
ncz_sync_file(NC_FILE_INFO_T* file)
ncz_sync_file(NC_FILE_INFO_T* file, int isclose)
{
int stat = NC_NOERR;
NCjson* json = NULL;
NCZ_FILE_INFO_T* zinfo = NULL;
NC_UNUSED(isclose);
LOG((3, "%s: file: %s", __func__, file->controller->path));
ZTRACE(3,"file=%s isclose=%d",file->controller->path,isclose);
/* Write out root group recursively */
if((stat = ncz_sync_grp(file, file->root_grp)))
goto done;
zinfo = (NCZ_FILE_INFO_T*)file->format_file_info;
/* Create super block (NCZMETAROOT) */
{
if((stat = ncz_create_superblock(zinfo))) goto done;
/* Write out root group recursively */
if((stat = ncz_sync_grp(file, file->root_grp)))
goto done;
}
done:
NCJreclaim(json);
return THROW(stat);
return ZUNTRACE(stat);
}
/**
@ -485,7 +496,6 @@ ncz_write_var(NC_VAR_INFO_T* var)
default: goto done; /* some other error */
}
/* If we reach here, then chunk does not exist, create it with fill */
if((stat=nczmap_defineobj(map,key))) goto done;
/* ensure fillchunk exists */
if(zvar->cache->fillchunk == NULL) {
nc_type typecode;
@ -1846,6 +1856,8 @@ ncz_create_superblock(NCZ_FILE_INFO_T* zinfo)
NCZMAP* map = NULL;
char version[1024];
ZTRACE(4,"zinfo=%s",zinfo->common.file->controller->path);
map = zinfo->map;
/* create superblock json */
@ -1872,5 +1884,5 @@ ncz_create_superblock(NCZ_FILE_INFO_T* zinfo)
}
done:
NCJreclaim(json);
return THROW(stat);
return ZUNTRACE(stat);
}

167
libnczarr/zutil.c
View File

@ -65,6 +65,8 @@ NCJ_INT, /*NC_INT64*/
NCJ_INT, /*NC_UINT64*/
};
/* Forward */
static int endswith(const char* s, const char* suffix);
/**************************************************/
@ -213,24 +215,6 @@ done:
return stat;
}
/**
@internal Create a specified object exists; do nothing if already exists.
@param zmap - [in] controlling zarr map
@param key - [in] .z... object to create
@return NC_NOERR
@author Dennis Heimbigner
*/
int
NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size)
{
int stat = NC_NOERR;
/* create the target */
stat = nczmap_defineobj(zmap, key);
return stat;
}
/**
@internal Upload a modified json tree to a .z... structure.
@param zmap - [in] controlling zarr map
@ -244,24 +228,24 @@ NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json)
{
int stat = NC_NOERR;
char* content = NULL;
ZTRACE(4,"zmap=%p key=%s",zmap,key);
#ifdef DEBUG
fprintf(stderr,"uploadjson: %s\n",key); fflush(stderr);
#endif
/* Unparse the modified json tree */
if((stat = NCJunparse(json,0,&content)))
goto done;
/* create the target */
if((stat = nczmap_defineobj(zmap, key)))
goto done;
ZTRACEMORE(4,"\tjson=%s",content);
/* Write the metadata */
if((stat = nczmap_write(zmap, key, 0, strlen(content), content)))
goto done;
done:
nullfree(content);
return stat;
return ZUNTRACE(stat);
}
#if 0
@ -473,30 +457,30 @@ NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, NClist* objlist
NClist* matches = nclistnew();
NCbytes* path = ncbytesnew();
/* Get the list of object keys just below prefix */
/* Get the list of names just below prefix */
if((stat = nczmap_search(map,prefix,matches))) goto done;
for(i=0;i<nclistlength(matches);i++) {
const char* p;
const char* key = nclistget(matches,i);
size_t keylen = strlen(key);
const char* name = nclistget(matches,i);
size_t namelen= strlen(name);
/* Ignore keys that start with .z or .nc or a potential chunk name */
if(keylen >= 3 && key[0] == '.' && key[1] == 'n' && key[2] == 'c')
if(namelen >= 3 && name[0] == '.' && name[1] == 'n' && name[2] == 'c')
continue;
if(keylen >= 2 && key[0] == '.' && key[1] == 'z')
if(namelen >= 2 && name[0] == '.' && name[1] == 'z')
continue;
for(p=key;*p;p++) {
for(p=name;*p;p++) {
if(*p != '.' && strchr("0123456789",*p) == NULL) break;
}
if(*p == '\0') continue; /* looks like a chunk name */
/* Create <prefix>/<key>/<tag> and see if it exists */
/* Create <prefix>/<name>/<tag> and see if it exists */
ncbytesclear(path);
ncbytescat(path,prefix);
ncbytescat(path,"/");
ncbytescat(path,key);
ncbytescat(path,name);
ncbytescat(path,tag);
/* See if this object exists */
if((stat = nczmap_exists(map,ncbytescontents(path))) == NC_NOERR)
nclistpush(objlist,key);
nclistpush(objlist,name);
}
done:
@ -777,3 +761,120 @@ NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fi
return NC_NOERR;
}
/**************************************************/
/* S3 utilities */
EXTERNL int
NCZ_s3urlprocess(NCURI* url, ZS3INFO* s3)
{
int stat = NC_NOERR;
NClist* segments = NULL;
NCbytes* buf = ncbytesnew();
if(url == NULL)
{stat = NC_EURL; goto done;}
/* do some verification */
if(strcmp(url->protocol,"https") != 0)
{stat = NC_EURL; goto done;}
/* Path better look absolute */
if(!nczm_isabsolutepath(url->path))
{stat = NC_EURL; goto done;}
/* Distinguish path-style from virtual-host style from other:
Virtual: https://bucket-name.s3.Region.amazonaws.com/<root>
Path: https://s3.Region.amazonaws.com/bucket-name/<root>
Other: https://<host>/bucketname/<root>
*/
if(url->host == NULL || strlen(url->host) == 0)
{stat = NC_EURL; goto done;}
if(endswith(url->host,AWSHOST)) { /* Virtual or path */
segments = nclistnew();
/* split the hostname by "." */
if((stat = nczm_split_delim(url->host,'.',segments))) goto done;
switch (nclistlength(segments)) {
default: stat = NC_EURL; goto done;
case 4:
if(strcasecmp(nclistget(segments,0),"s3")!=0)
{stat = NC_EURL; goto done;}
s3->urlformat = UF_PATH;
s3->region = strdup(nclistget(segments,1));
break;
case 5:
if(strcasecmp(nclistget(segments,1),"s3")!=0)
{stat = NC_EURL; goto done;}
s3->urlformat = UF_VIRTUAL;
s3->region = strdup(nclistget(segments,2));
s3->bucket = strdup(nclistget(segments,0));
break;
}
/* Rebuild host to look like path-style */
ncbytescat(buf,"s3.");
ncbytescat(buf,s3->region);
ncbytescat(buf,AWSHOST);
s3->host = ncbytesextract(buf);
} else {
s3->urlformat = UF_OTHER;
if((s3->host = strdup(url->host))==NULL)
{stat = NC_ENOMEM; goto done;}
}
/* Do fixups to make everything look like it was path style */
switch (s3->urlformat) {
case UF_PATH:
case UF_OTHER:
/* We have to process the path to get the bucket, and remove it in the path */
if(url->path != NULL && strlen(url->path) > 0) {
/* split the path by "/" */
nclistfreeall(segments);
segments = nclistnew();
if((stat = nczm_split_delim(url->path,'/',segments))) goto done;
if(nclistlength(segments) == 0)
{stat = NC_EURL; goto done;}
s3->bucket = ((char*)nclistremove(segments,0));
if(nclistlength(segments) > 0) {
if((stat = nczm_join(segments,&s3->rootkey))) goto done;
} else
s3->rootkey = NULL;
nclistfreeall(segments); segments = NULL;
}
break;
case UF_VIRTUAL:
if(url->path == NULL || strlen(url->path) == 0)
s3->rootkey = NULL;
else
s3->rootkey = strdup(url->path);
break;
default: stat = NC_EURL; goto done;
}
done:
ncbytesfree(buf);
nclistfreeall(segments);
return stat;
}
int
NCZ_s3clear(ZS3INFO* s3)
{
if(s3) {
nullfree(s3->host);
nullfree(s3->region);
nullfree(s3->bucket);
nullfree(s3->rootkey);
}
return NC_NOERR;
}
static int
endswith(const char* s, const char* suffix)
{
ssize_t ls, lsf, delta;
if(s == NULL || suffix == NULL) return 0;
ls = strlen(s);
lsf = strlen(suffix);
delta = (ls - lsf);
if(delta < 0) return 0;
if(memcmp(s+delta,suffix,lsf)!=0) return 0;
return 1;
}

6
libnczarr/zwalk.c
View File

@ -225,7 +225,7 @@ NCZ_transfer(struct Common* common, NCZSlice* slices)
fprintf(stderr,"case: wholechunk: chunkindices: %s\n",nczprint_vector(common->rank,chunkindices));
/* Read the chunk */
switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) {
case NC_ENOTFOUND: /* cache created the chunk */
case NC_EEMPTY: /* cache created the chunk */
if((stat = NCZ_fillchunk(chunkdata,common))) goto done;
break;
case NC_NOERR: break;
@ -297,7 +297,7 @@ NCZ_transfer(struct Common* common, NCZSlice* slices)
/* Read from cache */
stat = common->reader.read(common->reader.source, chunkindices, &chunkdata);
switch (stat) {
case NC_ENOTFOUND: /* cache created the chunk */
case NC_EEMPTY: /* cache created the chunk */
if((stat = NCZ_fillchunk(chunkdata,common))) goto done;
break;
case NC_NOERR: break;
@ -728,7 +728,7 @@ NCZ_transferscalar(struct Common* common)
/* Read from single chunk from cache */
chunkindices[0] = 0;
switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) {
case NC_ENOTFOUND: /* cache created the chunk */
case NC_EEMPTY: /* cache created the chunk */
if((stat = NCZ_fillchunk(chunkdata,common))) goto done;
break;
case NC_NOERR: break;

55
libnczarr/zxcache.c
View File

@ -26,7 +26,6 @@
/* Forward */
static int get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry);
static int put_chunk(NCZChunkCache* cache, const NCZCacheEntry*);
static int create_chunk(NCZChunkCache* cache, NCZCacheEntry* entry);
static int makeroom(NCZChunkCache* cache);
/**************************************************/
@ -195,6 +194,9 @@ void
NCZ_free_chunk_cache(NCZChunkCache* cache)
{
if(cache == NULL) return;
ZTRACE(4,"cache.var=%s",cache->var->hdr.name);
/* Iterate over the entries */
while(nclistlength(cache->mru) > 0) {
void* ptr;
@ -211,6 +213,7 @@ fprintf(stderr,"|cache.free|=%ld\n",nclistlength(cache->mru));
cache->mru = NULL;
nullfree(cache->fillchunk);
nullfree(cache);
(void)ZUNTRACE(NC_NOERR);
}
size64_t
@ -271,12 +274,8 @@ NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap
switch (stat) {
case NC_NOERR: break;
case NC_EEMPTY:
case NC_ENOTFOUND: /*signals the chunk needs to be created */
/* If the file is read-only, then fake the chunk */
entry->modified = (!file->no_write);
if(!file->no_write) {
if((stat = create_chunk(cache,entry))) goto done;
}
entry->modified = (file->no_write?0:1);
#ifdef FILLONREAD
/* apply fill value */
memcpy(entry->data,cache->fillchunk,cache->chunksize);
@ -295,9 +294,9 @@ fprintf(stderr,"|cache.read.lru|=%ld\n",nclistlength(cache->mru));
#endif
if(datap) *datap = entry->data;
entry = NULL;
if(created) stat = NC_ENOTFOUND;
done:
if(created && stat == NC_NOERR) stat = NC_EEMPTY; /* tell upper layers */
if(entry) {nullfree(entry->data); nullfree(entry->key);}
nullfree(entry);
return THROW(stat);
@ -375,6 +374,8 @@ NCZ_flush_chunk_cache(NCZChunkCache* cache)
int stat = NC_NOERR;
size_t i;
ZTRACE(4,"cache.var=%s |cache|=%d",cache->var->hdr.name,(int)nclistlength(cache->mru));
if(NCZ_cache_size(cache) == 0) goto done;
/* Iterate over the entries in hashmap */
@ -389,7 +390,7 @@ NCZ_flush_chunk_cache(NCZChunkCache* cache)
}
done:
return THROW(stat);
return ZUNTRACE(stat);
}
#if 0
@ -478,6 +479,7 @@ put_chunk(NCZChunkCache* cache, const NCZCacheEntry* entry)
NCZ_FILE_INFO_T* zfile = NULL;
NCZMAP* map = NULL;
ZTRACE(5,"cache.var=%s entry.key=%s",cache->var->hdr.name,entry->key);
LOG((3, "%s: var: %p", __func__, cache->var));
zfile = ((cache->var->container)->nc4_info)->format_file_info;
@ -485,18 +487,13 @@ put_chunk(NCZChunkCache* cache, const NCZCacheEntry* entry)
stat = nczmap_write(map,entry->key,0,cache->chunksize,entry->data);
switch(stat) {
case NC_NOERR: break;
case NC_EEMPTY:
/* Create the chunk */
if((stat = nczmap_defineobj(map,entry->key))) goto done;
/* write again */
if((stat = nczmap_write(map,entry->key,0,cache->chunksize,entry->data)))
goto done;
case NC_NOERR:
break;
case NC_EEMPTY:
default: goto done;
}
done:
return THROW(stat);
return ZUNTRACE(stat);
}
/**
@ -517,6 +514,8 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
NC_FILE_INFO_T* file = NULL;
NCZ_FILE_INFO_T* zfile = NULL;
ZTRACE(5,"cache.var=%s entry.key=%s",cache->var->hdr.name,entry->key);
LOG((3, "%s: file: %p", __func__, file));
file = (cache->var->container)->nc4_info;
@ -526,29 +525,9 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
stat = nczmap_read(map,entry->key,0,cache->chunksize,(char*)entry->data);
return THROW(stat);
return ZUNTRACE(stat);
}
static int
create_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
{
int stat = NC_NOERR;
NC_FILE_INFO_T* file = NULL;
NCZ_FILE_INFO_T* zfile = NULL;
NCZMAP* map = NULL;
file = (cache->var->container)->nc4_info;
zfile = file->format_file_info;
map = zfile->map;
/* Create the chunk */
if((stat = nczmap_defineobj(map,entry->key))) goto done;
entry->modified = 1; /* mark as modified */
/* let higher function decide on fill */
done:
return THROW(stat);
}
int
NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, char** keyp)

12
nczarr_test/CMakeLists.txt
View File

@ -40,7 +40,7 @@ IF(ENABLE_TESTS)
BUILD_BIN_TEST(tst_zchunks ${COMMONSRC})
BUILD_BIN_TEST(tst_zchunks2 ${COMMONSRC})
BUILD_BIN_TEST(tst_zchunks3 ${COMMONSRC})
BUILD_BIN_TEST(tst_fillonlyz)
BUILD_BIN_TEST(tst_fillonlyz ${TSTCOMMONSRC})
TARGET_INCLUDE_DIRECTORIES(ut_map PUBLIC ../libnczarr)
TARGET_INCLUDE_DIRECTORIES(ut_mapapi PUBLIC ../libnczarr)
@ -50,6 +50,7 @@ IF(ENABLE_TESTS)
TARGET_INCLUDE_DIRECTORIES(tst_zchunks PUBLIC ../libnczarr)
TARGET_INCLUDE_DIRECTORIES(tst_zchunks2 PUBLIC ../libnczarr)
TARGET_INCLUDE_DIRECTORIES(tst_zchunks3 PUBLIC ../libnczarr)
TARGET_INCLUDE_DIRECTORIES(tst_fillonlyz PUBLIC ../libnczarr)
# Helper programs for testing
BUILD_BIN_TEST(zmapio ${COMMONSRC})
@ -59,6 +60,10 @@ IF(ENABLE_TESTS)
TARGET_INCLUDE_DIRECTORIES(zisjson PUBLIC ../libnczarr)
BUILD_BIN_TEST(zs3parse ${COMMONSRC})
TARGET_INCLUDE_DIRECTORIES(zs3parse PUBLIC ../libnczarr)
if(ENABLE_NCZARR_S3)
BUILD_BIN_TEST(s3util ${COMMONSRC})
TARGET_INCLUDE_DIRECTORIES(s3util PUBLIC ../libnczarr)
endif()
SET(ncdumpchunks_SOURCE ncdumpchunks.c)
IF(USE_X_GETOPT)
@ -78,12 +83,13 @@ IF(ENABLE_TESTS)
ENDIF()
add_sh_test(nczarr_test run_ncgen4)
IF(FALSE) # Suppress for now
BUILD_BIN_TEST(tst_chunkcases ${TSTCOMMONSRC})
TARGET_INCLUDE_DIRECTORIES(tst_chunkcases PUBLIC ../libnczarr)
add_sh_test(nczarr_test run_chunkcases)
ENDIF()
if(ENABLE_NCZARR_S3)
add_sh_test(nczarr_test run_s3_cleanup)
ENDIF()
ENDIF(BUILD_UTILITIES)
ENDIF(ENABLE_TESTS)

60
nczarr_test/Makefile.am
View File

@ -33,6 +33,7 @@ ut_mapapi_SOURCES = ut_mapapi.c ${commonsrc}
ut_json_SOURCES = ut_json.c ${commonsrc}
ut_projections_SOURCES = ut_projections.c ${commonsrc}
ut_chunking_SOURCES = ut_chunking.c ${commonsrc}
tst_fillonlyz_SOURCES = tst_fillonlyz.c ${tstcommonsrc}
check_PROGRAMS += tst_zchunks tst_zchunks2 tst_zchunks3 tst_fillonlyz
@ -50,19 +51,16 @@ endif
TESTS += run_ncgen4.sh
# Temporarily disable
if AX_IGNORE
check_PROGRAMS += tst_chunkcases
tst_chunkcases_SOURCES = tst_chunkcases.c ${tstcommonsrc}
TESTS += run_chunkcases.sh
endif
endif
if BUILD_BENCHMARKS
if BUILD_UTILITIES
UTILSRC = bm_utils.c timer_utils.c
UTILSRC = bm_utils.c timer_utils.c tst_utils.c
bm_chunks3_SOURCES = bm_chunks3.c ${UTILSRC}
@ -84,6 +82,12 @@ zisjson_SOURCES = zisjson.c
noinst_PROGRAMS += zs3parse
zs3parse_SOURCES = zs3parse.c
if ENABLE_NCZARR_S3
noinst_PROGRAMS += s3util
s3util_SOURCES = s3util.c
TESTS += run_s3_cleanup.sh
endif
# Given a netcdf4|NCZarr file, dump the actual chunk contents.
# Used to validate nczarr chunking code.
AM_CPPFLAGS += -I$(top_srcdir)/libnczarr
@ -92,38 +96,13 @@ ncdumpchunks_SOURCES = ncdumpchunks.c
EXTRA_DIST = CMakeLists.txt \
run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_ncgen4.sh \
run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh \
ref_ut_map_readmeta_nzf.txt \
ref_ut_map_writedata_nzf.cdl \
ref_ut_map_writemeta_nzf.cdl \
ref_ut_map_writemeta_nz4.cdl \
ref_ut_map_writedata_nz4.cdl \
ref_ut_map_write2meta_nzf.cdl \
ref_ut_map_write2meta_nz4.cdl \
ref_ut_map_search_nzf.txt \
ref_ut_map_search_nz4.txt \
ref_ut_map_readmeta_nz4.txt \
ref_ut_map_readdata_nz4.txt \
ref_ut_map_create_nz4.cdl \
ref_ut_json_parse.txt \
ref_ut_json_build.txt \
ref_t_meta_var1.cdl \
ref_t_meta_dim1.cdl \
ref_ut_testmap_create_nz4.cdl \
ref_ut_proj.txt \
ref_ut_map_create_nzf.cdl \
ref_ut_mapapi_create_nz4.cdl \
ref_ut_mapapi_create_nzf.cdl \
ref_ut_mapapi_create_s3.cdl \
ref_ut_mapapi_data_nz4.cdl \
ref_ut_mapapi_data_nzf.cdl \
ref_ut_mapapi_data_s3.cdl \
ref_ut_mapapi_meta_nz4.cdl \
ref_ut_mapapi_meta_nzf.cdl \
ref_ut_mapapi_meta_s3.cdl \
ref_ut_mapapi_search_nz4.txt \
ref_ut_mapapi_search_nzf.txt \
ref_ut_mapapi_search_s3.txt \
run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh run_s3_cleanup.sh \
ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \
ref_ut_map_readmeta.txt ref_ut_map_readmeta2.txt ref_ut_map_search.txt \
ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_mapapi_search.txt \
ref_t_meta_dim1.cdl ref_t_meta_var1.cdl \
ref_ut_json_build.txt ref_ut_json_parse.txt \
ref_ut_proj.txt ref_ut_testmap_create.cdl \
ref_perdimspecs.cdl ref_fillonly.cdl \
ref_whole.cdl ref_whole.txt \
ref_skip.cdl ref_skip.txt ref_skipw.cdl \
@ -131,13 +110,8 @@ ref_rem.cdl ref_rem.dmp ref_ndims.cdl ref_ndims.dmp \
ref_misc1.cdl ref_misc1.dmp \
ref_avail1.cdl ref_avail1.dmp ref_avail1.txt
CLEANFILES = test.nzf test.nz4 test.nz4 \
testmap.nzf testmap.nz4 testmapapi.nzf testmapapi.nz4 \
tst_chunks.nz4 tst_chunks2.nc.nz4 \
ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp test.nzf \
tst_chunks3.nc
CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc
# Remove directories
clean-local:
rm -fr tst_chunks.nzf tst_chunks2.nzf tst_perdimspecs.nzf testmap.nzf test.nzf tmp*.nzf \
test_mapapi.nzf test_mapapi.nz4 tmp*.nz4
rm -fr tmp*.file results.file results.s3 results.zip

38
nczarr_test/bm_utils.c
View File

@ -105,6 +105,7 @@ getoptions(int* argcp, char*** argvp, struct Options* opt)
int tag;
int argc = *argcp;
char** argv = *argvp;
const char* nczarr_s3_test_url = ncz_gets3testurl();
memset((void*)opt,0,sizeof(struct Options));
if(argc <= 1) return NC_NOERR;
@ -144,13 +145,15 @@ fprintf(stderr,"arg=%s value=%s\n",argv[optind-1],optarg);
opt->pathtemplate = strdup(optarg);
break;
case OPT_FORMAT:
if(strcasecmp(optarg,"nz4")==0)
opt->format = NC_FORMATX_NC4;
else if(strcasecmp(optarg,"nzf")==0)
if(strcasecmp(optarg,"file")==0)
opt->format = NC_FORMATX_NCZARR;
opt->impl = NCZM_FILE;
else if(strcasecmp(optarg,"zip")==0)
opt->format = NC_FORMATX_NCZARR;
opt->impl = NCZM_ZIP;
else if(strcasecmp(optarg,"s3")==0) {
opt->format = NC_FORMATX_NCZARR;
opt->iss3 = 1;
opt->impl = NCZM_S3;
} else
{fprintf(stderr,"illegal format\n"); return NC_EINVAL;}
break;
@ -190,10 +193,21 @@ fprintf(stderr,"arg=%s value=%s\n",argv[optind-1],optarg);
switch(opt->format) {
case NC_FORMATX_NCZARR:
if(opt->pathtemplate == NULL) {
if(opt->iss3)
opt->pathtemplate = strdup("https://stratus.ucar.edu/unidata-netcdf-zarr-testing/%s.%s#mode=nczarr,s3");
else
opt->pathtemplate = strdup("file://%s.%s#mode=nczarr,nzf");
size_t ptlen;
switch (opt->impl) {
case NCZM_S3:
ptlen = strlen(nczarr_s3_test_url) + strlen("/netcdf-c/%s.%s#mode=nczarr,s3");
opt->pathtemplate = (char*)calloc(1,ptlen+1);
if(opt->template == NULL) return NC_ENOMEM;
strlcat(opt->pathtemplate,nczarr_s3_test_url,ptlen+1);
strlcat(opt->pathtemplate,"/netcdf-c/%s.%s#mode=nczarr,s3",ptlen+1);
break;
case NCZM_FILE:
opt->pathtemplate = strdup("file://%s.%s#mode=nczarr,file");
case NCZM_ZIP:
opt->pathtemplate = strdup("file://%s.%s#mode=nczarr,zip");
default: abort();
}
}
break;
case NC_FORMATX_NC4:
@ -232,7 +246,13 @@ EXTERNL const char*
formatname(const struct Options* o)
{
switch (o->format) {
case NC_FORMATX_NCZARR: return (o->iss3?"s3":"nzf");
case NC_FORMATX_NCZARR:
switch (o->impl) {
case NCZM_S3: return "s3";
case NCZM_NC4: return "nz4";
case NCZM_FILE: return "file";
case NCZM_ZIP: return "zip";
break;
case NC_FORMATX_NC4: /* fall thru */
return "nz4";
default:

5
nczarr_test/ncdumpchunks.c
View File

@ -369,7 +369,7 @@ dump(Format* format)
for(r=0;r<format->rank;r++) zindices[r] = (size64_t)odom->index[r];
switch (stat=NCZ_read_chunk(ncid, varid, zindices, chunkdata)) {
case NC_NOERR: break;
case NC_ENOTFOUND: holechunk = 1; break;
case NC_EEMPTY: holechunk = 1; break;
default: usage(stat);
}
break;
@ -418,8 +418,7 @@ dump(Format* format)
return 0;
}
static const char* urlexts[] = {"nzf", "nz4", NULL};
static const char* urlexts[] = {"file", "zip", NULL};
static const char*
filenamefor(const char* f0)

1
nczarr_test/ref_ut_map_create.cdl Normal file
View File

@ -0,0 +1 @@
[0] /.nczarr : (0) ||

10
nczarr_test/ref_ut_map_create_nz4.cdl
View File

@ -1,10 +0,0 @@
netcdf testmap {
group: _nczarr {
dimensions:
data_dim = UNLIMITED ; // (0 currently)
variables:
ubyte data(data_dim) ;
data:
} // group _nczarr
}

2
nczarr_test/ref_ut_map_create_nzf.cdl
View File

@ -1,2 +0,0 @@
testmap.nzf
testmap.nzf/.nczarr : ||

4
nczarr_test/ref_ut_map_readmeta.txt Normal file
View File

@ -0,0 +1,4 @@
/meta1/.zarray: |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|

5
nczarr_test/ref_ut_map_readmeta2.txt Normal file
View File

@ -0,0 +1,5 @@
/meta2/.nczvar: |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4],
"extra": 137}|

8
nczarr_test/ref_ut_map_search.txt Normal file
View File

@ -0,0 +1,8 @@
[0] /
[1] /.nczarr
[2] /data1
[3] /data1/0
[4] /meta1
[5] /meta1/.zarray
[6] /meta2
[7] /meta2/.nczvar

42
nczarr_test/ref_ut_map_write2meta_nz4.cdl
View File

@ -1,42 +0,0 @@
netcdf testmap {
group: _nczarr {
dimensions:
data_dim = UNLIMITED ; // (0 currently)
variables:
ubyte data(data_dim) ;
data:
} // group _nczarr
group: meta1 {
group: _zarray {
dimensions:
data_dim = UNLIMITED ; // (50 currently)
variables:
ubyte data(data_dim) ;
data:
data = 123, 10, 34, 102, 111, 111, 34, 58, 32, 52, 50, 44, 10, 34, 98,
97, 114, 34, 58, 32, 34, 97, 112, 112, 108, 101, 115, 34, 44, 10, 34,
98, 97, 122, 34, 58, 32, 91, 49, 44, 32, 50, 44, 32, 51, 44, 32, 52,
93, 125 ;
} // group _zarray
} // group meta1
group: meta2 {
group: _nczvar {
dimensions:
data_dim = UNLIMITED ; // (64 currently)
variables:
ubyte data(data_dim) ;
data:
data = 123, 10, 34, 102, 111, 111, 34, 58, 32, 52, 50, 44, 10, 34, 98,
97, 114, 34, 58, 32, 34, 97, 112, 112, 108, 101, 115, 34, 44, 10, 34,
98, 97, 122, 34, 58, 32, 91, 49, 44, 32, 50, 44, 32, 51, 44, 32, 52,
93, 44, 10, 34, 101, 120, 116, 114, 97, 34, 58, 32, 49, 51, 55, 125 ;
} // group _nczvar
} // group meta2
}

6
nczarr_test/ref_ut_map_write2meta_nzf.cdl
View File

@ -1,6 +0,0 @@
testmap.nzf
testmap.nzf/.nczarr : ||
testmap.nzf/meta1
testmap.nzf/meta1/.zarray : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4]}|
testmap.nzf/meta2
testmap.nzf/meta2/.nczvar : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4], "extra": 137}|

11
nczarr_test/ref_ut_map_writedata.cdl Normal file
View File

@ -0,0 +1,11 @@
[0] /.nczarr : (0) ||
[2] /data1/0 : (25) (int) |0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24|
[4] /meta1/.zarray : (50) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[6] /meta2/.nczvar : (64) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4],
"extra": 137}|

59
nczarr_test/ref_ut_map_writedata_nz4.cdl
View File

@ -1,59 +0,0 @@
netcdf testmap {
group: _nczarr {
dimensions:
data_dim = UNLIMITED ; // (0 currently)
variables:
ubyte data(data_dim) ;
data:
} // group _nczarr
group: meta1 {
group: _zarray {
dimensions:
data_dim = UNLIMITED ; // (50 currently)
variables:
ubyte data(data_dim) ;
data:
data = 123, 10, 34, 102, 111, 111, 34, 58, 32, 52, 50, 44, 10, 34, 98,
97, 114, 34, 58, 32, 34, 97, 112, 112, 108, 101, 115, 34, 44, 10, 34,
98, 97, 122, 34, 58, 32, 91, 49, 44, 32, 50, 44, 32, 51, 44, 32, 52,
93, 125 ;
} // group _zarray
} // group meta1
group: meta2 {
group: _nczvar {
dimensions:
data_dim = UNLIMITED ; // (64 currently)
variables:
ubyte data(data_dim) ;
data:
data = 123, 10, 34, 102, 111, 111, 34, 58, 32, 52, 50, 44, 10, 34, 98,
97, 114, 34, 58, 32, 34, 97, 112, 112, 108, 101, 115, 34, 44, 10, 34,
98, 97, 122, 34, 58, 32, 91, 49, 44, 32, 50, 44, 32, 51, 44, 32, 52,
93, 44, 10, 34, 101, 120, 116, 114, 97, 34, 58, 32, 49, 51, 55, 125 ;
} // group _nczvar
} // group meta2
group: data1 {
group: \0 {
dimensions:
data_dim = UNLIMITED ; // (100 currently)
variables:
ubyte data(data_dim) ;
data:
data = 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0,
0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0,
11, 0, 0, 0, 12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0, 16,
0, 0, 0, 17, 0, 0, 0, 18, 0, 0, 0, 19, 0, 0, 0, 20, 0, 0, 0, 21, 0,
0, 0, 22, 0, 0, 0, 23, 0, 0, 0, 24, 0, 0, 0 ;
} // group \0
} // group data1
}

8
nczarr_test/ref_ut_map_writedata_nzf.cdl
View File

@ -1,8 +0,0 @@
testmap.nzf
testmap.nzf/.nczarr : ||
testmap.nzf/data1
testmap.nzf/data1/0 : |000000000100000002000000030000000400000005000000060000000700000008000000090000000a0000000b0000000c0000000d0000000e0000000f000000100000001100000012000000130000001400000015000000160000001700000018000000|
testmap.nzf/meta1
testmap.nzf/meta1/.zarray : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4]}|
testmap.nzf/meta2
testmap.nzf/meta2/.nczvar : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4], "extra": 137}|

5
nczarr_test/ref_ut_map_writemeta.cdl Normal file
View File

@ -0,0 +1,5 @@
[0] /.nczarr : (0) ||
[2] /meta1/.zarray : (50) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|

10
nczarr_test/ref_ut_map_writemeta2.cdl Normal file
View File

@ -0,0 +1,10 @@
[0] /.nczarr : (0) ||
[2] /meta1/.zarray : (50) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[4] /meta2/.nczvar : (64) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4],
"extra": 137}|

26
nczarr_test/ref_ut_map_writemeta_nz4.cdl
View File

@ -1,26 +0,0 @@
netcdf testmap {
group: _nczarr {
dimensions:
data_dim = UNLIMITED ; // (0 currently)
variables:
ubyte data(data_dim) ;
data:
} // group _nczarr
group: meta1 {
group: _zarray {
dimensions:
data_dim = UNLIMITED ; // (50 currently)
variables:
ubyte data(data_dim) ;
data:
data = 123, 10, 34, 102, 111, 111, 34, 58, 32, 52, 50, 44, 10, 34, 98,
97, 114, 34, 58, 32, 34, 97, 112, 112, 108, 101, 115, 34, 44, 10, 34,
98, 97, 122, 34, 58, 32, 91, 49, 44, 32, 50, 44, 32, 51, 44, 32, 52,
93, 125 ;
} // group _zarray
} // group meta1
}

4
nczarr_test/ref_ut_map_writemeta_nzf.cdl
View File

@ -1,4 +0,0 @@
testmap.nzf
testmap.nzf/.nczarr : ||
testmap.nzf/meta1
testmap.nzf/meta1/.zarray : |{ "foo": 42, "bar": "apples", "baz": [1, 2, 3, 4]}|

1
nczarr_test/ref_ut_mapapi_create.cdl Normal file
View File

@ -0,0 +1 @@
[0] /.nczarr : (0) ||

1
nczarr_test/ref_ut_mapapi_create_nz4.cdl
View File

@ -1 +0,0 @@
[1] /.nczarr : (0) ||

1
nczarr_test/ref_ut_mapapi_create_nzf.cdl
View File

@ -1 +0,0 @@
[1] /.nczarr : (0) ||

1
nczarr_test/ref_ut_mapapi_create_s3.cdl
View File

@ -1 +0,0 @@
[1] /.nczarr : (0) ||

4
nczarr_test/ref_ut_mapapi_data_s3.cdl → nczarr_test/ref_ut_mapapi_data.cdl
View File

@ -1,8 +1,8 @@
[1] /.nczarr : (50) |{
[0] /.nczarr : (50) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[2] /data1 : (100) (ubyte) |0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, 12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 18, 0, 0, 0, 19, 0, 0, 0, 20, 0, 0, 0, 21, 0, 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, 24, 0, 0, 0|
[1] /data1 : (100) (ubyte) |0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, 12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 18, 0, 0, 0, 19, 0, 0, 0, 20, 0, 0, 0, 21, 0, 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, 24, 0, 0, 0|
[3] /meta1/.zarray : (34) |{
"shape": [1,2,3],
"dtype": "<1"}|

8
nczarr_test/ref_ut_mapapi_data_nz4.cdl
View File

@ -1,8 +0,0 @@
[1] /.nczarr : (50) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[2] /data1 : (100) (ubyte) |0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, 12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 18, 0, 0, 0, 19, 0, 0, 0, 20, 0, 0, 0, 21, 0, 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, 24, 0, 0, 0|
[4] /meta1/.zarray : (34) |{
"shape": [1,2,3],
"dtype": "<1"}|

8
nczarr_test/ref_ut_mapapi_data_nzf.cdl
View File

@ -1,8 +0,0 @@
[1] /.nczarr : (50) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[2] /data1 : (100) (ubyte) |0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, 12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 18, 0, 0, 0, 19, 0, 0, 0, 20, 0, 0, 0, 21, 0, 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, 24, 0, 0, 0|
[4] /meta1/.zarray : (34) |{
"shape": [1,2,3],
"dtype": "<1"}|

2
nczarr_test/ref_ut_mapapi_meta_s3.cdl → nczarr_test/ref_ut_mapapi_meta.cdl
View File

@ -1,4 +1,4 @@
[1] /.nczarr : (50) |{
[0] /.nczarr : (50) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|

7
nczarr_test/ref_ut_mapapi_meta_nz4.cdl
View File

@ -1,7 +0,0 @@
[1] /.nczarr : (50) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[3] /meta1/.zarray : (34) |{
"shape": [1,2,3],
"dtype": "<1"}|

7
nczarr_test/ref_ut_mapapi_meta_nzf.cdl
View File

@ -1,7 +0,0 @@
[1] /.nczarr : (50) |{
"foo": 42,
"bar": "apples",
"baz": [1, 2, 3, 4]}|
[3] /meta1/.zarray : (34) |{
"shape": [1,2,3],
"dtype": "<1"}|

5
nczarr_test/ref_ut_mapapi_search.txt Normal file
View File

@ -0,0 +1,5 @@
[0] /
[1] /.nczarr
[2] /data1
[3] /meta1
[4] /meta1/.zarray

0
nczarr_test/ref_ut_testmap_create_nz4.cdl → nczarr_test/ref_ut_testmap_create.cdl
View File

29
nczarr_test/run_chunkcases.sh
View File

@ -28,8 +28,8 @@ ZM="${execdir}/zmapio -t int"
remfile() {
case "$zext" in
nc4) rm -fr $1 ;;
nz4) rm -fr $1 ;;
nzf) rm -fr $1 ;;
file) rm -fr $1 ;;
zip) rm -fr $1 ;;
s3) ;;
*) echo "no such extension: $zext" ; exit 1;;
esac
@ -38,8 +38,8 @@ remfile() {
remfile() {
case "$zext" in
nc4) rm -fr $1 ;;
nz4) rm -fr $1 ;;
nzf) rm -fr $1 ;;
file) rm -fr $1 ;;
zip) rm -fr $1 ;;
s3) ;;
*) echo "no such extension: $zext" ; exit 1;;
esac
@ -50,8 +50,8 @@ makefile() {
remfile $file
case "$zext" in
nc4) F=$file;;
nz4) F=$fileurl;;
nzf) F=$fileurl;;
file) F=$fileurl;;
zip) F=$fileurl;;
s3) F=$fileurl;;
*) echo "no such extension: $zext" ; exit 1;;
esac
@ -84,7 +84,7 @@ ${NCDUMP} $F > tmp_whole_${zext}.cdl
diff -b ${srcdir}/ref_whole.cdl tmp_whole_${zext}.cdl
# Test skipping whole chunks
necho "Test chunk skipping during read"
echo "Test chunk skipping during read"
makefile tmp_skip
rm -f tmp_skip_${zext}.txt tmp_skip_${zext}.cdl
$TC -d 6,6 -c 2,2 -Ow $F
@ -109,7 +109,7 @@ $TC -d 8,8 -c 3,3 -Ow $F
${NCDUMP} $F > tmp_rem_${zext}.cdl
diff -b ${srcdir}/ref_rem.cdl tmp_rem_${zext}.cdl
${execdir}/ncdumpchunks -v v $F > tmp_rem_${zext}.txt
diff -b ${srcdir}/ref_rem.txt tmp_rem_${zext}.txt
diff -b ${srcdir}/ref_rem.dmp tmp_rem_${zext}.txt
echo "Test rank > 2"
makefile tmp_ndims
@ -117,8 +117,8 @@ rm -f tmp_ndims_${zext}.txt tmp_ndims_${zext}.cdl
$TC -d 8,8,8,8 -c 3,3,4,4 -Ow $F
${NCDUMP} $F > tmp_ndims_${zext}.cdl
diff -b ${srcdir}/ref_ndims.cdl tmp_ndims_${zext}.cdl
${execdir}/ncdumpchunks -v v $F > tmp_ndims_${zext}.txt
diff -b ${srcdir}/ref_ndims.txt tmp_ndims_${zext}.txt
${execdir}/ncdumpchunks -v v $F > tmp_ndims_${zext}.dmp
diff -b ${srcdir}/ref_ndims.dmp tmp_ndims_${zext}.dmp
echo "Test miscellaneous 1"
makefile tmp_misc1
@ -126,11 +126,12 @@ rm -f tmp_misc1_${zext}.txt tmp_misc1_${zext}.cdl
$TC -d 6,12,4 -c 2,3,1 -f 0,0,0 -e 6,1,4 -Ow $F
${NCDUMP} $F > tmp_misc1_${zext}.cdl
diff -b ${srcdir}/ref_misc1.cdl tmp_misc1_${zext}.cdl
${execdir}/ncdumpchunks -v v $F > tmp_misc1_${zext}.txt
diff -b ${srcdir}/ref_misc1.txt tmp_misc1_${zext}.txt
${execdir}/ncdumpchunks -v v $F > tmp_misc1_${zext}.dmp
diff -b ${srcdir}/ref_misc1.dmp tmp_misc1_${zext}.dmp
} # testcases()
testcases nzf
if test "x$FEATURE_HDF5" = xyes ; then testcases nz4; fi
testcases file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcases zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testcases s3; fi

15
nczarr_test/run_fillonlyz.sh
View File

@ -3,6 +3,8 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. "$srcdir/test_nczarr.sh"
# This shell script tests bug reported in github issue
# https://github.com/Unidata/netcdf-c/issues/1826
@ -11,9 +13,16 @@ set -e
echo ""
echo "*** Testing data conversions when a variable has fill value but never written"
${NCGEN} -4 -b -o 'file://tmp_fillonly.nc#mode=nczarr,nzf' $srcdir/ref_fillonly.cdl
${execdir}/tst_fillonlyz${ext}
testcase() {
zext=$1
fileargs tmp_fillonly
deletemap $zext $file
${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_fillonly.cdl
${execdir}/tst_fillonlyz${ext} "$fileurl"
}
rm -fr ./tmp_fillonly.nc
testcase file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi
exit 0

9
nczarr_test/run_it_chunks1.sh
View File

@ -1,6 +1,6 @@
#!/bin/sh
# Run (tst_chunks,tst_chunks2) X (nzf,nc4,s3)
# Run (tst_chunks,tst_chunks2) X (file,zip,s3)
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
@ -19,9 +19,10 @@ ${execdir}/tst_chunks -e $1 $CLOUD
${execdir}/tst_chunks2 -e $1 $CLOUD
}
ittest nzf
if test "xFEATURE_HDF5" = xyes ; then ittest nz4; fi
if test "x$FEATURE_S3TESTS" = xyes ; then ittest s3 'https://stratus.ucar.edu/unidata-netcdf-zarr-testing'; fi
ittest file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then ittest zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then ittest s3 "${NCZARR_S3_TEST_URL}/netcdf-c'; fi
}

67
nczarr_test/run_nccopyz.sh
View File

@ -3,13 +3,15 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. "$srcdir/test_nczarr.sh"
set -e
echo ""
#chunkclean src dst
chunkclean() {
rm -f ./$2
cat ./$1 | sed -e '/:_Storage/d' | sed -e '/:_ChunkSizes/d' > ./$2
cat ./$1 | sed -e "/:_Storage/d" | sed -e "/:_ChunkSizes/d" > ./$2
}
#verifychunking cdl-file <chunkspec>...
@ -17,51 +19,64 @@ verifychunking() {
f=$1
shift
for t in "$@" ; do
x=`cat $f | tr -d '\t \r' | sed -e "/$t/p" -ed`
x=`cat $f | tr -d "\t \r" | sed -e "/$t/p" -ed`
if test "x$x" = x ; then echo "$f: $t not found"; exit 1; fi
done
}
./tst_zchunks3 -e nzf
testcase() {
zext=$1
fileargs tmp
./tst_zchunks3 -e ${zext}
echo "*** Test that nccopy -c can chunk files"
${NCCOPY} -M0 tst_chunks3.nc 'file://tmp.nzf#mode=nczarr,nzf'
${NCDUMP} -n tmp -sh 'file://tmp.nzf#mode=nczarr,nzf' > tmp.cdl
verifychunking tmp.cdl 'ivar:_ChunkSizes=7,4,2,3,5,6,9;' 'fvar:_ChunkSizes=9,6,5,3,2,4,7;'
${NCCOPY} -M0 tmp_chunks3.nc "$fileurl"
${NCDUMP} -n tmp -sh "$fileurl" > tmp.cdl
verifychunking tmp.cdl "ivar:_ChunkSizes=7,4,2,3,5,6,9;" "fvar:_ChunkSizes=9,6,5,3,2,4,7;"
${NCCOPY} -M0 -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ tst_chunks3.nc 'file://tmp_chunked.nzf#mode=nczarr,nzf'
${NCDUMP} -sh -n tmp 'file://tmp_chunked.nzf#mode=nczarr,nzf' > tmp_chunked.cdl
verifychunking tmp_chunked.cdl 'ivar:_ChunkSizes=7,1,2,1,5,1,9;' 'fvar:_ChunkSizes=9,1,5,1,2,1,7;'
fileargs tmp_chunked
./tst_zchunks3 -e ${zext}
${NCCOPY} -M0 -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ tmp_chunks3.nc "$fileurl"
${NCDUMP} -sh -n tmp "$fileurl" > tmp_chunked.cdl
verifychunking tmp_chunked.cdl "ivar:_ChunkSizes=7,1,2,1,5,1,9;" "fvar:_ChunkSizes=9,1,5,1,2,1,7;"
chunkclean tmp.cdl tmpx.cdl
chunkclean tmp_chunked.cdl tmp_chunkedx.cdl
diff tmpx.cdl tmp_chunkedx.cdl
# Note that unchunked means that there is only one chunk
${NCCOPY} -M0 -c dim0/,dim1/,dim2/,dim3/,dim4/,dim5/,dim6/ 'file://tmp_chunked.nzf#mode=nczarr,nzf' 'file://tmp_unchunked.nzf#mode=nczarr,nzf'
${NCDUMP} -sh -n tmp 'file://tmp_unchunked.nzf#mode=nczarr,nzf' > tmp_unchunked.cdl
verifychunking tmp_unchunked.cdl 'ivar:_ChunkSizes=7,4,2,3,5,6,9;' 'fvar:_ChunkSizes=9,6,5,3,2,4,7;'
SRC="$fileurl"
fileargs tmp_unchunked
${NCCOPY} -M0 -c dim0/,dim1/,dim2/,dim3/,dim4/,dim5/,dim6/ "$SRC" "$fileurl"
${NCDUMP} -sh -n tmp "$fileurl" > tmp_unchunked.cdl
verifychunking tmp_unchunked.cdl "ivar:_ChunkSizes=7,4,2,3,5,6,9;" "fvar:_ChunkSizes=9,6,5,3,2,4,7;"
chunkclean tmp_unchunked.cdl tmp_unchunkedx.cdl
diff tmpx.cdl tmp_unchunkedx.cdl
# Test -c /
${NCCOPY} -M0 -c '/' 'file://tmp_chunked.nzf#mode=nczarr,nzf' 'file://tmp_unchunked2.nzf#mode=nczarr,nzf'
${NCDUMP} -sh -n tmp 'file://tmp_unchunked2.nzf#mode=nczarr,nzf' > tmp_unchunked2.cdl
verifychunking tmp_unchunked.cdl 'ivar:_ChunkSizes=7,4,2,3,5,6,9;' 'fvar:_ChunkSizes=9,6,5,3,2,4,7;'
fileargs tmp_unchunked2
${NCCOPY} -M0 -c "/" "$SRC" "$fileurl"
${NCDUMP} -sh -n tmp "$fileurl" > tmp_unchunked2.cdl
verifychunking tmp_unchunked.cdl "ivar:_ChunkSizes=7,4,2,3,5,6,9;" "fvar:_ChunkSizes=9,6,5,3,2,4,7;"
chunkclean tmp_unchunked.cdl tmp_unchunkedx.cdl
diff tmpx.cdl tmp_unchunkedx.cdl
echo "*** Test that nccopy -c dim/n is used "
${NCGEN} -4 -b -o 'file://tst_perdimspecs.nzf#mode=nczarr,nzf' $srcdir/ref_perdimspecs.cdl
${NCDUMP} -n tmp_perdimspecs -hs 'file://tst_perdimspecs.nzf#mode=nczarr,nzf' > tmp_perdimspecs.cdl
${NCCOPY} -M0 -4 -c "time/10,lat/15,lon/20" 'file://tst_perdimspecs.nzf#mode=nczarr,nzf' 'file://tmppds.nzf#mode=nczarr,nzf'
${NCDUMP} -n tmppds -hs 'file://tmppds.nzf#mode=nczarr,nzf' > tmppds.cdl
STORAGE=`cat tmppds.cdl | sed -e '/tas:_Storage/p' -ed | tr -d '\t \r'`
test "x$STORAGE" = 'xtas:_Storage="chunked";'
CHUNKSIZES=`cat tmppds.cdl | sed -e '/tas:_ChunkSizes/p' -ed | tr -d '\t \r'`
test "x$CHUNKSIZES" = 'xtas:_ChunkSizes=10,15,20;'
fileargs tmp_perdimspecs
${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_perdimspecs.cdl
${NCDUMP} -n tmp_perdimspecs -hs "$fileurl" > tmp_perdimspecs.cdl
SRC=$fileurl
fileargs tmp_pds
# Cleanup
rm -fr tmp*.nc tmp*.cdl
rm -fr tmp*.nzf
${NCCOPY} -M0 -4 -c "time/10,lat/15,lon/20" "$SRC" "$fileurl"
${NCDUMP} -n tmp_pds -hs "$fileurl" > tmp_pds.cdl
STORAGE=`cat tmp_pds.cdl | sed -e "/tas:_Storage/p" -ed | tr '"' "'" | tr -d "\t \r"`
test "x$STORAGE" = "xtas:_Storage='chunked';"
CHUNKSIZES=`cat tmp_pds.cdl | sed -e "/tas:_ChunkSizes/p" -ed | tr -d "\t \r"`
test "x$CHUNKSIZES" = "xtas:_ChunkSizes=10,15,20;"
}
testcase file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi
echo "*** All nccopy nczarr tests passed!"
exit 0

6
nczarr_test/run_ncgen4.sh
View File

@ -54,10 +54,8 @@ cd ..
echo "*** PASSED: zext=${zext}"
}
runtestset nzf
if test "x$FEATURE_HDF5" = xyes ; then runtestset nz4; fi
runtestset file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then runtestset zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then runtestset s3; fi
rm -rf ${RESULTSDIR}.nzf ${RESULTSDIR}.nz4 ${RESULTSDIR}.s3
echo "*** PASSED ***"

4
nczarr_test/run_perf_chunks1.sh
View File

@ -31,8 +31,8 @@ echo '*** SUCCESS!!!'
}
testcases nzf
if test "x$FEATURE_HDF5" = xyes ; then testcases nz4; fi
testcases file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcases zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testcases s3; fi
exit 0

19
nczarr_test/run_s3_cleanup.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. "$srcdir/test_nczarr.sh"
set -e
echo ""
echo "*** Remove /netcdf-c from S3 repository"
fileargs netcdf-c
if test "x$FEATURE_S3TESTS" = xyes ; then
${execdir}/s3util -u "${NCZARR_S3_TEST_URL}" -k "/netcdf-c" clear
fi
exit 0

47
nczarr_test/run_ut_map.sh
View File

@ -9,7 +9,7 @@ set -e
# Test those map implementations where
# it is possible to look at the actual storage.
# .ncz and .nzf specifically. Note that we
# in some cases. Note that we
# cannot easily look inside S3 storage
# except using the aws-cli, if available
@ -21,15 +21,16 @@ testmapcreate() {
echo ""; echo "*** Test zmap create -k $1"
extfor "$1"
tag="map"
output="test$tag.$zext"
output="tmp_$tag.$zext"
deletemap $1 $output
# Create the test file
$CMD -k$1 -x create -o $output
cdl="ut_${tag}_create_${zext}.cdl"
ref="ref_ut_${tag}_create.cdl"
dumpmap $zext $output ./$cdl
diff -wb ${srcdir}/ref_$cdl ./$cdl
diff -wb ${srcdir}/$ref ./$cdl
# delete the test file
$CMD -k$1 -x delete -f $output
rm -f $cdl
@ -45,34 +46,42 @@ testmapmeta() {
echo ""; echo "*** Test zmap read/write meta -k $1"
extfor "$1"
tag="map"
file="test$tag.$zext"
file="tmp_$tag.$zext"
$CMD -k$1 -x writemeta -f $file
cdl="ut_${tag}_writemeta_${zext}.cdl"
ref="ref_ut_${tag}_writemeta.cdl"
dumpmap $zext $file ./$cdl
diff -wb ${srcdir}/ref_$cdl ./$cdl
diff -wb ${srcdir}/$ref ./$cdl
$CMD -k$1 -x writemeta2 -o ./$file
cdl="ut_${tag}_write2meta_${zext}.cdl"
cdl="ut_${tag}_writemeta2_${zext}.cdl"
ref="ref_ut_${tag}_writemeta2.cdl"
dumpmap $zext $file ./$cdl
diff -wb ${srcdir}/ref_$cdl ./$cdl
diff -wb ${srcdir}/$ref ./$cdl
output="ut_${tag}_readmeta_$zext.txt"
outref="ref_ut_${tag}_readmeta.txt"
$CMD -k$1 -x readmeta -f $file > ./$output
diff -wb ${srcdir}/ref_$output ./$output
diff -wb ${srcdir}/$outref ./$output
output="ut_${tag}_readmeta2_$zext.txt"
outref="ref_ut_${tag}_readmeta2.txt"
$CMD -k$1 -x readmeta2 -f $file > ./$output
diff -wb ${srcdir}/$outref ./$output
}
testmapdata() {
echo ""; echo "*** Test zmap read/write data -k $1"
extfor "$1"
tag="map"
file="test$tag.$zext"
file="tmp_$tag.$zext"
$CMD -k$1 -x "writedata" -f $file
cdl="ut_${tag}_writedata_${zext}.cdl"
ref="ref_ut_${tag}_writedata.cdl"
dumpmap $zext $file ./$cdl
diff -wb ${srcdir}/ref_$cdl ./$cdl
diff -wb ${srcdir}/$ref ./$cdl
# readata is verification only
$CMD -k$1 -x readdata -f $file
}
@ -81,24 +90,24 @@ testmapsearch() {
echo ""; echo "*** Test zmap search -k $1"
extfor "$1"
tag="map"
file="test$tag.$zext"
file="tmp_$tag.$zext"
txt=ut_${tag}_search_$zext.txt
ref=ref_ut_${tag}_search.txt
rm -f $txt
$CMD -k$1 -x "search" -f $file > $txt
diff -wb ${srcdir}/ref_$txt ./$txt
diff -wb ${srcdir}/$ref ./$txt
}
echo ""
echo "*** Map Unit Testing"
echo ""; echo "*** Test zmap_nzf"
testmapcreate nzf; testmapmeta nzf; testmapdata nzf; testmapsearch nzf
echo ""; echo "*** Test zmap_file"
testmapcreate file; testmapmeta file; testmapdata file; testmapsearch file
if test "x$FEATURE_HDF5" = xyes ; then
echo ""; echo "*** Test zmap_nz4"
testmapcreate nz4; testmapmeta nz4; testmapdata nz4; testmapsearch nz4
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then
echo ""; echo "*** Test zmap_zip"
testmapcreate zip; testmapmeta zip; testmapdata zip; testmapsearch zip
fi
exit 0

46
nczarr_test/run_ut_mapapi.sh
View File

@ -6,6 +6,8 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. "$srcdir/test_nczarr.sh"
TR="-T10"
set -e
# Test map implementations for consistency at the zmap API
@ -20,22 +22,23 @@ testmapcreate() {
echo ""; echo "*** Test zmap create -k $1"
extfor "$1"
tag=mapapi
base="test_$tag"
base="tmp_$tag"
fileargs $base
deletemap $1 $file
# Create the test file
$CMD -k$1 -x create -f $file
$CMD $TR -k$1 -x create -f $file
cdl="ut_${tag}_create_${zext}.cdl"
ref="ref_ut_${tag}_create.cdl"
${ZMD} $fileurl > ./$cdl
diff -wb ${srcdir}/ref_$cdl ./$cdl
diff -wb $srcdir/$ref ./$cdl
# delete the test file
$CMD -k$1 -x delete -f $file
$CMD $TR -k$1 -x delete -f $file
rm -f $cdl
mapexists $1
if test mapexists = 0 ; then exit 1; fi
mapstillexists $1
if test mapstillexists = 0 ; then exit 1; fi
# re-create the test file
$CMD -k$1 -x create -o $file
}
@ -44,46 +47,49 @@ testmapmeta() {
echo ""; echo "*** Test zmap read/write meta -k $1"
extfor "$1"
tag=mapapi
base="test_$tag"
base="tmp_$tag"
fileargs $base
$CMD -k$1 -x simplemeta -f $file
$CMD $TR -k$1 -x simplemeta -f $file
cdl="ut_${tag}_meta_${zext}.cdl"
ref="ref_ut_${tag}_meta.cdl"
${ZMD} $fileurl > ./$cdl
diff -wb ${srcdir}/ref_$cdl ./$cdl
diff -wb ${srcdir}/$ref ./$cdl
}
testmapdata() {
echo ""; echo "*** Test zmap read/write data -k $1"
extfor "$1"
tag=mapapi
base="test_$tag"
base="tmp_$tag"
fileargs $base
$CMD -k$1 -x "simpledata" -f $file
$CMD $TR -k$1 -x "simpledata" -f $file
cdl="ut_${tag}_data_${zext}.cdl"
ref="ref_ut_${tag}_data.cdl"
${ZMD} $fileurl > ./$cdl
diff -wb ${srcdir}/ref_$cdl ./$cdl
diff -wb ${srcdir}/$ref ./$cdl
}
testmapsearch() {
echo ""; echo "*** Test zmap search -k $1"
extfor "$1"
tag=mapapi
base="test_$tag"
base="tmp_$tag"
fileargs $base
txt=ut_${tag}_search_$zext.txt
ref=ref_ut_${tag}_search.txt
rm -f $txt
$CMD -k$1 -x "search" -f $file > $txt
diff -wb ${srcdir}/ref_$txt ./$txt
$CMD $TR -k$1 -x "search" -f $file > $txt
diff -wb ${srcdir}/$ref ./$txt
}
main() {
echo ""
echo "*** Map Unit Testing"
echo ""; echo "*** Test zmap_nzf"
testmapcreate nzf; testmapmeta nzf; testmapdata nzf; testmapsearch nzf
if test "x$FEATURE_HDF5" = xyes ; then
echo ""; echo "*** Test zmap_nz4"
testmapcreate nz4; testmapmeta nz4; testmapdata nz4; testmapsearch nz4
echo ""; echo "*** Test zmap_file"
testmapcreate file; testmapmeta file; testmapdata file; testmapsearch file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then
echo ""; echo "*** Test zmap_zip"
testmapcreate zip; testmapmeta zip; testmapdata zip; testmapsearch zip
fi
if test "x$FEATURE_S3TESTS" = xyes ; then
echo ""; echo "*** Test zmap_s3sdk"

445
nczarr_test/s3util.c Normal file
View File

@ -0,0 +1,445 @@
/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#ifdef _WIN32
#include "XGetopt.h"
#endif
#include "zincludes.h"
#include "zs3sdk.h"
#include "ncpathmgr.h"
#include "nclog.h"
#include "ncuri.h"
#include "netcdf_aux.h"
#undef NODELETE
#define DEBUG
#define DATANAME "data"
typedef enum S3op {
S3OP_NONE=0,
S3OP_LIST=1,
S3OP_CLEAR=2,
S3OP_PRINT=3,
S3OP_UPLOAD=4,
S3OP_DOWNLOAD=5,
} S3op;
static struct S3ops {
S3op s3op;
const char* opnames[3];
} s3ops[] = {
{S3OP_LIST,{"list","l",NULL}},
{S3OP_CLEAR,{"clear","c",NULL}},
{S3OP_PRINT,{"print","p",NULL}},
{S3OP_UPLOAD,{"upload","u",NULL}},
{S3OP_DOWNLOAD,{"download","d",NULL}},
{S3OP_NONE,{NULL,NULL,NULL}},
};
/* Command line options */
struct Dumpptions {
int debug;
S3op s3op;
NCURI* url;
char* key; /* via -k flag */
char* rootkey; /* from url | key */
nc_type nctype; /* for printing content */
char* filename;
} dumpoptions;
struct S3SDK {
ZS3INFO s3;
void* s3config;
void* s3client;
char* errmsg;
} s3sdk;
/* Forward */
static int s3list(void);
static int s3clear(void);
static int s3print(void);
static int s3upload(void);
static int s3download(void);
static nc_type typefor(const char* t);
static void printcontent(size64_t len, const char* content, nc_type nctype);
static void
usage(void)
{
fprintf(stderr,"usage: s3util list|print|upload|download|clear -u <url> [-k <key.] [-f <filename>]\n");
exit(1);
}
static S3op
decodeop(const char* name)
{
struct S3ops* s3op = s3ops;
const char** s = NULL;
for(;s3op->opnames[0] != NULL;s3op++) {
for(s=s3op->opnames;*s;s++) {
if(strcasecmp(*s,name)==0) return s3op->s3op;
}
}
return S3OP_NONE;
}
static void
s3initialize(void)
{
NCZ_s3sdkinitialize();
}
static void
s3finalize(void)
{
NCZ_s3sdkfinalize();
}
static int
s3setup(void)
{
int stat = NC_NOERR;
if((stat=NCZ_s3sdkcreateconfig(s3sdk.s3.host, s3sdk.s3.region, &s3sdk.s3config))) goto done;
if((stat = NCZ_s3sdkcreateclient(s3sdk.s3config,&s3sdk.s3client))) goto done;
done:
return stat;
}
static int
s3shutdown(int deleteit)
{
int stat = NC_NOERR;
stat = NCZ_s3sdkclose(s3sdk.s3client, s3sdk.s3config, s3sdk.s3.bucket, s3sdk.s3.rootkey, deleteit, &s3sdk.errmsg);
return stat;
}
int
main(int argc, char** argv)
{
int stat = NC_NOERR;
int c;
char* tmp = NULL;
memset((void*)&dumpoptions,0,sizeof(dumpoptions));
dumpoptions.nctype = NC_UBYTE; /* default */
while ((c = getopt(argc, argv, "df:k:u:vt:T:")) != EOF) {
switch(c) {
case 'd':
dumpoptions.debug = 1;
break;
case 'f':
dumpoptions.filename = strdup(optarg);
break;
case 'k': {
size_t len = strlen(optarg);
dumpoptions.key = (char*)malloc(len+1+1);
if(*optarg != '/') {
fprintf(stderr,"warning: -k option does not start with '/': %s",optarg);
dumpoptions.key[0] = '/';
memcpy(dumpoptions.key+1,optarg,len);
len++;
} else
memcpy(dumpoptions.key,optarg,strlen(optarg));
dumpoptions.key[len] = '\0';
} break;
case 't':
dumpoptions.nctype = typefor(optarg);
break;
case 'u': {
char* p = NCdeescape(optarg);
ncuriparse(p,&dumpoptions.url);
nullfree(p);
if(dumpoptions.url == NULL) {
fprintf(stderr,"malformed -f option: %s",optarg);
stat = NC_EINVAL;
goto done;
}
} break;
case 'v':
usage();
goto done;
case 'T':
nctracelevel(atoi(optarg));
break;
case '?':
fprintf(stderr,"unknown option\n");
stat = NC_EINVAL;
goto done;
}
}
/* get command argument */
argc -= optind;
argv += optind;
if (argc > 1) {
fprintf(stderr, "s3util: only one command argument permitted\n");
stat = NC_EINVAL;
goto done;
}
if (argc == 0) {
fprintf(stderr, "s3util: no command specified\n");
stat = NC_EINVAL;
goto done;
}
dumpoptions.s3op = decodeop(argv[0]);
memset(&s3sdk,0,sizeof(s3sdk));
if((stat = NCZ_s3urlprocess(dumpoptions.url, &s3sdk.s3))) goto done;
if(s3sdk.s3.rootkey != NULL && dumpoptions.key != NULL) {
size_t len = 0;
/* Make the root key be the concatenation of rootkey+dumpoptions.key */
len = strlen(s3sdk.s3.rootkey) + strlen(dumpoptions.key) + 1;
if((tmp = (char*)malloc(len+1))==NULL) {stat = NC_ENOMEM; goto done;}
tmp[0] = '\0';
strcat(tmp,s3sdk.s3.rootkey);
if(s3sdk.s3.rootkey[strlen(s3sdk.s3.rootkey)-1] != '/' && dumpoptions.key[0] != '/')
strcat(tmp,"/");
strcat(tmp,dumpoptions.key);
nullfree(s3sdk.s3.rootkey);
s3sdk.s3.rootkey = tmp; tmp = NULL;
} else if(dumpoptions.key != NULL) {
s3sdk.s3.rootkey = dumpoptions.key;
dumpoptions.key = NULL;
}
if(s3sdk.s3.rootkey == NULL || strlen(s3sdk.s3.rootkey)==0)
s3sdk.s3.rootkey = strdup("/");
s3initialize();
switch (dumpoptions.s3op) {
default:
fprintf(stderr,"Default action: list\n");
/* fall thru */
case S3OP_LIST:
if((stat = s3list())) goto done;
break;
case S3OP_CLEAR:
if((stat = s3clear())) goto done;
break;
case S3OP_PRINT:
if((stat = s3print())) goto done;
break;
case S3OP_UPLOAD:
if((stat = s3upload())) goto done;
break;
case S3OP_DOWNLOAD:
if((stat = s3download())) goto done;
break;
}
done:
s3finalize();
/* Reclaim dumpoptions */
ncurifree(dumpoptions.url);
nullfree(dumpoptions.rootkey);
nullfree(tmp);
if(stat)
fprintf(stderr,"fail: %s\n",nc_strerror(stat));
return (stat ? 1 : 0);
}
static int
s3list(void)
{
int stat = NC_NOERR;
size_t nkeys = 0;
char** keys = NULL;
if(s3setup()) goto done;
stat = NCZ_s3sdksearch(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg);
if(stat) goto done;
if(nkeys > 0) {
size_t i;
/* Sort the list -- shortest first */
nczm_sortenvv(nkeys,keys);
for(i=0;i<nkeys;i++) {
printf("[%u] %s\n",(unsigned)i,keys[i]);
}
} else
printf("<empty>\n");
done:
s3shutdown(0);
NCZ_freeenvv(nkeys,keys);
return stat;
}
static int
s3clear(void)
{
int stat = NC_NOERR;
size_t nkeys = 0;
char** keys = NULL;
if(s3setup()) goto done;
stat = NCZ_s3sdksearch(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg);
if(stat) goto done;
if(nkeys > 0) {
size_t i;
/* Sort the list -- shortest first */
nczm_sortenvv(nkeys,keys);
printf("deleted keys:\n");
for(i=0;i<nkeys;i++) {
printf("\t%s\n",keys[i]);
#ifndef NODELETE
if((stat = NCZ_s3sdkdeletekey(s3sdk.s3client, s3sdk.s3.bucket, keys[i], &s3sdk.errmsg)))
goto done;
#endif
}
}
done:
s3shutdown(0);
NCZ_freeenvv(nkeys,keys);
return stat;
}
static int
s3print(void)
{
int stat = NC_NOERR;
size_t nkeys = 0;
char** keys = NULL;
size64_t count;
char* content = NULL;
if(s3setup()) goto done;
if((stat = NCZ_s3sdkinfo(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &count,&s3sdk.errmsg)))
goto done;
if((content = (char*)calloc(1,count+1))==NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = NCZ_s3sdkread(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, 0, count, (void*) content, &s3sdk.errmsg)))
goto done;
printcontent(count,content,dumpoptions.nctype);
done:
if(content) free(content);
s3shutdown(0);
NCZ_freeenvv(nkeys,keys);
return stat;
}
static int
s3upload(void)
{
int stat = NC_NOERR;
size_t red = 0;
void* content = NULL;
if(s3setup()) goto done;
if((stat = ncaux_readfile(dumpoptions.filename,&red,&content)))
goto done;
if((stat = NCZ_s3sdkwriteobject(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, red, content, &s3sdk.errmsg)))
goto done;
done:
if(content) free(content);
s3shutdown(0);
return stat;
}
static int
s3download(void)
{
int stat = NC_NOERR;
size64_t count;
char* content = NULL;
if(s3setup()) goto done;
if((stat = NCZ_s3sdkinfo(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &count,&s3sdk.errmsg)))
goto done;
if((content = (char*)calloc(1,count))==NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = NCZ_s3sdkread(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, 0, count, (void*) content, &s3sdk.errmsg)))
goto done;
if((stat = ncaux_writefile(dumpoptions.filename,count,content)))
goto done;
done:
if(content) free(content);
s3shutdown(0);
return stat;
}
static void
printcontent(size64_t len, const char* content, nc_type nctype)
{
size64_t i;
if(len == 0) {
printf("<empty>\n");
return;
}
if(nctype == NC_STRING) printf("|");
for(i=0;i<len;i++) {
if(nctype != NC_STRING && i > 0) printf(", ");
switch(nctype) {
case NC_BYTE: printf("%d",((char*)content)[i]); break;
case NC_SHORT: printf("%d",((short*)content)[i]); break;
case NC_INT: printf("%d",((int*)content)[i]); break;
case NC_INT64: printf("%lld",((long long*)content)[i]); break;
case NC_UBYTE: printf("%u",((unsigned char*)content)[i]); break;
case NC_USHORT: printf("%u",((unsigned short*)content)[i]); break;
case NC_UINT: printf("%u",((unsigned int*)content)[i]); break;
case NC_UINT64: printf("%llu",((unsigned long long*)content)[i]); break;
case NC_FLOAT: printf("%f",((float*)content)[i]); break;
case NC_DOUBLE: printf("%lf",((double*)content)[i]); break;
case NC_STRING: putc(content[i],stdout); break;
default: abort();
}
}
if(nctype == NC_STRING) printf("|\n");
}
static nc_type
typefor(const char* t)
{
if(strcmp(t,"NC_BYTE")==0) return NC_BYTE;
else if(strcmp(t,"NC_SHORT")==0) return NC_SHORT;
else if(strcmp(t,"NC_INT")==0) return NC_INT;
else if(strcmp(t,"NC_INT64")==0) return NC_INT64;
else if(strcmp(t,"NC_UBYTE")==0) return NC_UBYTE;
else if(strcmp(t,"NC_USHORT")==0) return NC_USHORT;
else if(strcmp(t,"NC_UINT")==0) return NC_UINT;
else if(strcmp(t,"NC_UINT64")==0) return NC_UINT64;
else if(strcmp(t,"NC_FLOAT")==0) return NC_FLOAT;
else if(strcmp(t,"NC_DOUBLE")==0) return NC_DOUBLE;
else if(strcmp(t,"NC_STRING")==0) return NC_STRING;
return NC_NAT;
}

74
nczarr_test/test_nczarr.sh
View File

@ -2,10 +2,20 @@
if test "x$SETX" != x; then set -x; fi
# Figure out which cloud repo to use
if test "x$NCZARR_S3_TEST_HOST" = x ; then
export NCZARR_S3_TEST_HOST=stratus.ucar.edu
fi
if test "x$NCZARR_S3_TEST_BUCKET" = x ; then
export NCZARR_S3_TEST_BUCKET=unidata-netcdf-zarr-testing
fi
export NCZARR_S3_TEST_URL="https://${NCZARR_S3_TEST_HOST}/${NCZARR_S3_TEST_BUCKET}"
ZMD="${execdir}/zmapio"
awsdelete() {
aws s3api delete-object --endpoint-url=https://stratus.ucar.edu --bucket=unidata-netcdf-zarr-testing --key="$1"
${execdir}/s3util -u "${NCZARR_S3_TEST_URL}/" -k "$1" clear
#aws s3api delete-object --endpoint-url=https://${NCZARR_S3_TEST_HOST} --bucket=${NCZARR_S3_TEST_BUCKET} --key="netcdf-c/$1"
}
# Check settings
@ -39,8 +49,8 @@ checkprops() {
extfor() {
case "$1" in
nz4) zext="nz4" ;;
nzf) zext="nzf" ;;
file) zext="file" ;;
zip) zext="zip" ;;
s3) zext="s3" ;;
*) echo "unknown kind: $1" ; exit 1;;
esac
@ -48,25 +58,18 @@ extfor() {
deletemap() {
case "$1" in
nz4) rm -fr $2;;
nzf) rm -fr $2;;
file) rm -fr $2;;
zip) rm -f $2;;
s3) S3KEY=`${execdir}/zs3parse -k $2`; awsdelete $S3KEY;;
*) echo "unknown kind: $1" ; exit 1;;
esac
}
mapexists() {
mapexists=1
case "$1" in
nz4) if test -f $file; then mapexists=0; fi ;;
nzf) if test -f $file; then mapexists=0; fi ;;
s3)
if "./zmapio $fileurl" ; then mapexists=1; else mapexists=0; fi
;;
*) echo unknown format: $1 : abort ; exit 1 ;;
esac
if test $mapexists = 1 ; then
mapstillexists() {
mapstillexists=0
if "./zmapio $fileurl" ; then
echo "delete failed: $1"
mapstillexists=1
fi
}
@ -74,11 +77,7 @@ fileargs() {
f="$1"
case "$zext" in
s3)
if test "x$NCS3PATH" = x ; then
S3PATH="https://stratus.ucar.edu/unidata-netcdf-zarr-testing"
else
S3PATH="${NCS3PATH}"
fi
S3PATH="${NCZARR_S3_TEST_URL}/netcdf-c"
fileurl="${S3PATH}/${f}#mode=nczarr,$zext"
file=$fileurl
S3HOST=`${execdir}/zs3parse -h $S3PATH`
@ -92,36 +91,11 @@ fileargs() {
esac
}
dumpmap1() {
local ISJSON
tmp=
if test -f $1 ; then
ISJSON=`${execdir}/zisjson <$1`
if test "x$ISJSON" = x1 ; then
tmp=`tr '\r\n' ' ' <$1`
else
tmp=`${execdir}/zhex <$1`
fi
echo "$1 : |$tmp|" >> $2
else
echo "$1" >> $2
fi
}
dumpmap() {
case "$1" in
nz4) rm -f $3 ; ${NCDUMP} $2 > $3 ;;
nzf)
rm -f $3
export LC_ALL=C
lr=`find $2 | sort| tr '\r\n' ' '`
for f in $lr ; do dumpmap1 $f $3 ; done
;;
s3)
aws s3api list-objects --endpoint-url=$S3HOST --bucket=$S3BUCKET
;;
*) echo "dumpmap failed" ; exit 1;
esac
zext=$1
zbase=`basename $2 ".$zext"`
fileargs $zbase
${execdir}/zmapio -t int -x objdump $fileurl > $3
}
difftest() {

10
nczarr_test/test_nczarr_utils.h
View File

@ -66,8 +66,8 @@ static void
setimpl(const char* name)
{
if(strcasecmp(name,"s3")==0) itoptions.impl = NCZM_S3;
else if(strcasecmp(name,"nz4")==0) itoptions.impl = NCZM_NC4;
else if(strcasecmp(name,"nzf")==0) itoptions.impl = NCZM_FILE;
else if(strcasecmp(name,"file")==0) itoptions.impl = NCZM_FILE;
else if(strcasecmp(name,"zip")==0) itoptions.impl = NCZM_ZIP;
else test_usage();
}
@ -76,8 +76,8 @@ implname(void)
{
switch (itoptions.impl) {
case NCZM_S3: return "s3";
case NCZM_NC4: return "nz4";
case NCZM_FILE: return "nzf";
case NCZM_FILE: return "file";
case NCZM_ZIP: return "zip";
default: test_usage();
}
return NULL;
@ -88,7 +88,7 @@ buildpath(const char* target,NCZM_IMPL impl)
{
NCbytes* buf = ncbytesnew();
switch(itoptions.impl) {
case NCZM_NC4:
case NCZM_ZIP:
case NCZM_FILE:
ncbytescat(buf,"file://");
ncbytescat(buf,target);

9
nczarr_test/tst_chunkcases.c
View File

@ -2,19 +2,14 @@
#include "config.h"
#endif
#include <stdlib.h>
#ifdef HAVE_CONFIG_H
#include <stdio.h>
#endif
#include <string.h>
#include <netcdf.h>
#include <ncpathmgr.h>
#include <nclist.h>
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#ifdef _WIN32
#include "XGetopt.h"
#endif
#ifdef HAVE_HDF5_H
#include <hdf5.h>
#include <H5DSpublic.h>

22
nczarr_test/tst_fillonlyz.c
View File

@ -1,19 +1,27 @@
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "netcdf.h"
#include "nclist.h"
#include "zincludes.h"
#include "tst_utils.h"
#undef DEBUG
static void
nccheck(int ret)
nccheck(int ret, int lineno)
{
if(ret == NC_NOERR) return;
fprintf(stderr,"err=%s\n",nc_strerror(ret));
exit(1);
report(ret,lineno);
}
#define NCCHECK(err) nccheck(err)
#define NCCHECK(err) nccheck(err,__LINE__)
int
main(int argc, char *argv[] )
@ -22,11 +30,14 @@ main(int argc, char *argv[] )
size_t dimlen[1];
float *fdat;
int *idat;
const char* filename = "file://tmp_fillonly.nc#mode=nczarr,nzf";
const char* filename = NULL;
const char* varname = "f";
const char* dimname = "x";
size_t i;
NCCHECK(getoptions(&argc,&argv));
filename = options->file;
NCCHECK(err = nc_open(filename,NC_NETCDF4,&ncid));
NCCHECK(err = nc_inq_varid(ncid, varname, &varid));
NCCHECK(err = nc_inq_dimid(ncid, dimname, dimid));
@ -57,7 +68,6 @@ main(int argc, char *argv[] )
}
}
if(fdat) free(fdat);
if(idat) free(idat);
return 0;

24
nczarr_test/tst_utils.c
View File

@ -416,7 +416,7 @@ odom_printshort(Odometer* odom)
return odom_print1(odom,1);
}
static const char* urlexts[] = {"nzf", "nz4", NULL};
static const char* urlexts[] = {"nzf", "zip", "nz4", NULL};
const char*
filenamefor(const char* f0)
@ -443,3 +443,25 @@ filenamefor(const char* f0)
done:
return result;
}
static char s3testurl[8192];
static char* s3testurlp = NULL;
const char*
ncz_gets3testurl(void)
{
char* s;
if(s3testurlp == NULL) {
s3testurl[0] = '\0';
strcat(s3testurl,"https://");
s = getenv("NCZARR_S3_TEST_HOST");
if(s == NULL) s = "stratus.ucar.edu";
strcat(s3testurl,s);
strcat(s3testurl,"/");
s = getenv("NCZARR_S3_TEST_BUCKET");
if(s == NULL) s = "unidata-netcdf-zarr-testing";
strcat(s3testurl,s);
s3testurlp = s3testurl;
}
return s3testurlp;
}

2
nczarr_test/tst_utils.h
View File

@ -76,6 +76,8 @@ EXTERNL void cleanup(void);
EXTERNL int nc__testurl(const char*,char**);
EXTERNL const char* ncz_gets3testurl(void);
static void
report(int err, int lineno)
{

3
nczarr_test/tst_zchunks3.c
View File

@ -11,7 +11,7 @@
#define DEBUG
static int ret = NC_NOERR;
#define FILE_NAME "tst_chunks3.nc"
#define FILE_NAME "tmp_chunks3.nc"
#define VAR_RANK 7
#define IVAR_NAME "ivar"
@ -73,5 +73,6 @@ main(int argc, char** argv)
if ((ret=nc_close(ncid))) LERR;
SUMMARIZE_ERR;
FINAL_RESULTS;
}

2
nczarr_test/ut_includes.h
View File

@ -12,6 +12,8 @@
#include <unistd.h>
#endif
#include "zmap.h"
#include "netcdf.h"
#include "nclist.h"
#include "ncbytes.h"

123
nczarr_test/ut_map.c
View File

@ -30,6 +30,7 @@ static int simpledelete(void);
static int writemeta(void);
static int writemeta2(void);
static int readmeta(void);
static int readmeta2(void);
static int writedata(void);
static int readdata(void);
static int search(void);
@ -40,6 +41,7 @@ struct Test tests[] = {
{"writemeta", writemeta},
{"writemeta2", writemeta2},
{"readmeta", readmeta},
{"readmeta2", readmeta2},
{"writedata", writedata},
{"readdata", readdata},
{"search", search},
@ -77,13 +79,14 @@ simplecreate(void)
if((stat=nczm_concat(NULL,NCZMETAROOT,&path)))
goto done;
if((stat = nczmap_defineobj(map, path)))
/* Write empty metadata content */
if((stat = nczmap_write(map, path, 0, 0, (const void*)"")))
goto done;
/* Do not delete so we can look at it with ncdump */
if((stat = nczmap_close(map,0)))
goto done;
done:
/* Do not delete so we can look at it with ncdump */
stat = nczmap_close(map,0);
nullfree(path);
return THROW(stat);
}
@ -111,19 +114,11 @@ writemeta(void)
NCZMAP* map = NULL;
char* path = NULL;
if((stat = nczmap_create(impl,url,0,0,NULL,&map)))
if((stat = nczmap_open(impl,url,NC_WRITE,0,NULL,&map)))
goto done;
if((stat=nczm_concat(NULL,NCZMETAROOT,&path)))
goto done;
if((stat = nczmap_defineobj(map, path)))
goto done;
free(path); path = NULL;
if((stat=nczm_concat(META1,ZARRAY,&path)))
goto done;
if((stat = nczmap_defineobj(map, path)))
goto done;
if((stat = nczmap_write(map, path, 0, strlen(metadata1), metadata1)))
goto done;
free(path); path = NULL;
@ -147,8 +142,6 @@ writemeta2(void)
if((stat=nczm_concat(META2,NCZVAR,&path)))
goto done;
if((stat = nczmap_defineobj(map,path)))
goto done;
if((stat = nczmap_write(map, path, 0, strlen(metadata2), metadata2)))
goto done;
@ -160,18 +153,14 @@ done:
}
static int
readmeta(void)
readkey(NCZMAP* map, const char* prefix, const char* object)
{
int stat = NC_NOERR;
NCZMAP* map = NULL;
char* path = NULL;
size64_t olen;
char* content = NULL;
if((stat = nczmap_open(impl,url,0,0,NULL,&map)))
goto done;
if((stat=nczm_concat(META1,ZARRAY,&path)))
if((stat=nczm_concat(prefix,object,&path)))
goto done;
/* Get length */
@ -191,12 +180,44 @@ readmeta(void)
printf("%s: |%s|\n",path,content);
done:
(void)nczmap_close(map,0);
nullfree(content);
nullfree(path);
return THROW(stat);
}
static int
readmeta(void)
{
int stat = NC_NOERR;
NCZMAP* map = NULL;
if((stat = nczmap_open(impl,url,0,0,NULL,&map)))
goto done;
if((stat = readkey(map,META1,ZARRAY))) goto done;
done:
(void)nczmap_close(map,0);
return THROW(stat);
}
static int
readmeta2(void)
{
int stat = NC_NOERR;
NCZMAP* map = NULL;
if((stat = nczmap_open(impl,url,0,0,NULL,&map)))
goto done;
if((stat = readkey(map,META2,NCZVAR)))
goto done;
done:
(void)nczmap_close(map,0);
return THROW(stat);
}
static int
writedata(void)
{
@ -207,6 +228,7 @@ writedata(void)
int i;
size64_t totallen;
char* data1p = (char*)&data1[0]; /* byte level version of data1 */
NCZM_PROPERTIES props;
/* Create the data */
for(i=0;i<DATA1LEN;i++) data1[i] = i;
@ -219,21 +241,23 @@ writedata(void)
if((stat=nczm_concat(DATA1,"0",&path)))
goto done;
if((stat = nczmap_defineobj(map,path)))
goto done;
/* Write in 3 slices */
for(i=0;i<3;i++) {
size64_t start, count, third, last;
third = (totallen+2) / 3; /* round up */
start = i * third;
last = start + third;
if(last > totallen)
last = totallen;
count = last - start;
if((stat = nczmap_write(map, path, start, count, &data1p[start])))
goto done;
props = nczmap_properties(impl);
if((NCZM_ZEROSTART & props) || (NCZM_WRITEONCE & props)) {
if((stat = nczmap_write(map, path, 0, totallen, data1p)))
goto done;
} else {
/* Write in 3 slices */
for(i=0;i<3;i++) {
size64_t start, count, third, last;
third = (totallen+2) / 3; /* round up */
start = i * third;
last = start + third;
if(last > totallen)
last = totallen;
count = last - start;
if((stat = nczmap_write(map, path, start, count, &data1p[start])))
goto done;
}
}
done:
@ -299,28 +323,41 @@ done:
}
static int
searchR(NCZMAP* map, int depth, const char* prefix, NClist* objects)
searchR(NCZMAP* map, int depth, const char* prefix0, NClist* objects)
{
int i,stat = NC_NOERR;
NClist* matches = nclistnew();
char prefix[4096]; /* only ok because we know testdata */
size_t prefixlen;
/* add this prefix to object list */
nclistpush(objects,strdup(prefix));
/* get next level object keys **below** the prefix */
nclistpush(objects,strdup(prefix0));
prefix[0] = '\0';
strlcat(prefix,prefix0,sizeof(prefix));
prefixlen = strlen(prefix);
/* get next level object keys **below** the prefix: should have form: <name> */
switch (stat = nczmap_search(map, prefix, matches)) {
case NC_NOERR: break;
case NC_ENOTFOUND: stat = NC_NOERR; break;/* prefix is not a dir */
default: goto done;
}
/* recurse */
for(i=0;i<nclistlength(matches);i++) {
const char* key = nclistget(matches,i);
if((stat = searchR(map,depth+1,key,objects))) goto done;
/* ensure trailing '/' */
if(prefix[prefixlen-1] != '/')
strlcat(prefix,"/",sizeof(prefix));
strlcat(prefix,key,sizeof(prefix));
if((stat = searchR(map,depth+1,prefix,objects))) goto done;
/* restore prefix */
prefix[prefixlen] = '\0';
if(stat != NC_NOERR)
goto done;
}
done:
nclistfreeall(matches);
return stat;
return THROW(stat);
}
static int

100
nczarr_test/ut_mapapi.c
View File

@ -85,13 +85,14 @@ simplecreate(void)
printf("Pass: create: create: %s\n",url);
truekey = makekey(NCZMETAROOT);
if((stat = nczmap_defineobj(map, truekey)))
if((stat = nczmap_write(map, truekey, 0, 0, NULL)))
goto done;
printf("Pass: create: defineobj: %s\n",truekey);
/* Do not delete */
if((stat = nczmap_close(map,0)))
goto done;
map = NULL;
printf("Pass: create: close\n");
/* Reopen and see if exists */
@ -106,6 +107,7 @@ simplecreate(void)
/* close again */
if((stat = nczmap_close(map,0)))
goto done;
map = NULL;
printf("Pass: create: close\n");
done:
@ -126,24 +128,24 @@ simpledelete(void)
case NC_NOERR:
report(PASS,"open",map);
break;
case NC_EEMPTY:
{report(XFAIL,"open",map); stat = NC_NOERR; goto done;}
default:
{report(FAIL,"open",map); goto done;}
}
/* Delete dataset while closing */
if((stat = nczmap_close(map,1))) goto done;
map = NULL;
report(PASS,"close: delete",map);
switch ((stat = nczmap_open(impl,url,0,0,NULL,&map))) {
default:
case NC_NOERR:
report(FAIL,"open",map);
break;
case NC_EEMPTY:
case NC_ENOTFOUND:
report(XFAIL,"open",map);
stat = NC_NOERR;
break;
case NC_EEMPTY:
default: abort();
}
done:
@ -166,20 +168,21 @@ simplemeta(void)
goto done;
report(PASS,"open",map);
/* Make sure .nczarr exists (from simplecreate) */
truekey = makekey(NCZMETAROOT);
if((stat = nczmap_defineobj(map, truekey)))
if((stat = nczmap_exists(map,truekey)))
goto done;
report(PASS,".nczarr: def",map);
report(PASS,".nczarr: exists",map);
free(truekey); truekey = NULL;
if((stat=nczm_concat(META1,ZARRAY,&key)))
goto done;
truekey = makekey(key);
nullfree(key); key = NULL;
if((stat = nczmap_defineobj(map, truekey)))
if((stat = nczmap_write(map, truekey, 0, 0, NULL)))
goto done;
report(PASS,".zarray: def",map);
free(truekey); key = NULL;
free(truekey); truekey = NULL;
truekey = makekey(NCZMETAROOT);
if((stat = nczmap_write(map, truekey, 0, strlen(metadata1), metadata1)))
@ -199,6 +202,7 @@ simplemeta(void)
if((stat = nczmap_close(map,0)))
goto done;
map = NULL;
report(PASS,"close",map);
if((stat = nczmap_open(impl,url,0,0,NULL,&map)))
@ -251,9 +255,11 @@ simplemeta(void)
if((stat = nczmap_close(map,0)))
goto done;
map = NULL;
report(PASS,"close",map);
done:
if(map) nczmap_close(map,0);
nullfree(content);
nullfree(truekey);
nullfree(key);
@ -271,6 +277,7 @@ simpledata(void)
int i;
size64_t totallen, size;
char* data1p = (char*)&data1[0]; /* byte level version of data1 */
NCZM_PROPERTIES props;
title(__func__);
@ -283,26 +290,30 @@ simpledata(void)
report(PASS,"open",map);
truekey = makekey(DATA1);
if((stat = nczmap_defineobj(map, truekey)))
goto done;
report(PASS,DATA1": def",map);
/* Write in 3 slices */
for(i=0;i<3;i++) {
size64_t start, count, third, last;
third = (totallen+2) / 3; /* round up */
start = i * third;
last = start + third;
if(last > totallen)
last = totallen;
count = last - start;
if((stat = nczmap_write(map, truekey, start, count, &data1p[start])))
goto done;
props = nczmap_properties(impl);
if((NCZM_ZEROSTART & props) || (NCZM_WRITEONCE & props)) {
if((stat = nczmap_write(map, truekey, 0, totallen, data1p)))
goto done;
} else {
/* Write in 3 slices */
for(i=0;i<3;i++) {
size64_t start, count, third, last;
third = (totallen+2) / 3; /* round up */
start = i * third;
last = start + third;
if(last > totallen)
last = totallen;
count = last - start;
if((stat = nczmap_write(map, truekey, start, count, &data1p[start])))
goto done;
}
}
report(PASS,DATA1": write",map);
if((stat = nczmap_close(map,0)))
goto done;
map = NULL;
report(PASS,"close",map);
if((stat = nczmap_open(impl,url,0,0,NULL,&map)))
@ -328,32 +339,47 @@ simpledata(void)
done:
/* Do not delete so we can look at it with ncdump */
if((stat = nczmap_close(map,0)))
if(map && (stat = nczmap_close(map,0)))
goto done;
nullfree(truekey);
return THROW(stat);
}
static int
searchR(NCZMAP* map, int depth, const char* prefix, NClist* objects)
searchR(NCZMAP* map, int depth, const char* prefix0, NClist* objects)
{
int i,stat = NC_NOERR;
NClist* matches = nclistnew();
char prefix[4096]; /* only ok because we know testdata */
size_t prefixlen;
/* add this prefix to object list */
nclistpush(objects,strdup(prefix));
/* get next level object keys **below** the prefix */
if((stat = nczmap_search(map, prefix, matches)))
goto done;
nclistpush(objects,strdup(prefix0));
prefix[0] = '\0';
strlcat(prefix,prefix0,sizeof(prefix));
prefixlen = strlen(prefix);
/* get next level object keys **below** the prefix: should have form: <name> */
switch (stat = nczmap_search(map, prefix, matches)) {
case NC_NOERR: break;
case NC_ENOTFOUND: stat = NC_NOERR; break;/* prefix is not a dir */
default: goto done;
}
reportx(PASS,prefix,"search",map);
/* recurse */
for(i=0;i<nclistlength(matches);i++) {
const char* key = nclistget(matches,i);
if((stat = searchR(map,depth+1,key,objects))) goto done;
/* ensure trailing '/' */
if(prefix[prefixlen-1] != '/')
strlcat(prefix,"/",sizeof(prefix));
strlcat(prefix,key,sizeof(prefix));
if((stat = searchR(map,depth+1,prefix,objects))) goto done;
/* restore prefix */
prefix[prefixlen] = '\0';
if(stat != NC_NOERR)
goto done;
}
done:
nclistfreeall(matches);
return THROW(stat);
@ -373,7 +399,7 @@ search(void)
/* Do a recursive search on root to get all object keys */
if((stat=searchR(map,0,"/",objects)))
goto done;
/* sort list */
/* Sort */
ut_sortlist(objects);
/* Print out the list */
for(i=0;i<nclistlength(objects);i++) {
@ -383,8 +409,10 @@ search(void)
done:
/* Do not delete so later tests can use it */
(void)nczmap_close(map,0);
report(PASS,"close",map);
if(map) {
(void)nczmap_close(map,0);
report(PASS,"close",map);
}
nclistfreeall(objects);
return THROW(stat);
}

10
nczarr_test/ut_test.c
View File

@ -1,10 +1,11 @@
/*
* Copyright 2018, University Corporation for Atmospheric Research
x * Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#include "ut_includes.h"
#include "ncpathmgr.h"
#include "nclog.h"
#ifdef HAVE_GETOPT_H
#include <getopt.h>
@ -55,8 +56,11 @@ ut_init(int argc, char** argv, struct UTOptions * options)
if(options != NULL) {
options->dimdefs = nclistnew();
options->vardefs = nclistnew();
while ((c = getopt(argc, argv, "Dx:f:o:k:d:v:s:W:")) != EOF) {
while ((c = getopt(argc, argv, "T:Dx:f:o:k:d:v:s:W:")) != EOF) {
switch(c) {
case 'T':
nctracelevel(atoi(optarg));
break;
case 'D':
options->debug = 1;
break;
@ -188,8 +192,8 @@ makeurl(const char* file, NCZM_IMPL impl)
if(file && strlen(file) > 0) {
switch (impl) {
case NCZM_NC4: /* fall thru */
case NCZM_FILE:
case NCZM_ZIP:
/* Massage file to make it usable as URL path */
if((path = NCurlpath(file))==NULL) return NULL;
ncbytescat(buf,"file://");

8
nczarr_test/ut_util.c
View File

@ -361,8 +361,8 @@ NCZM_IMPL
kind2impl(const char* kind)
{
if(strcasecmp("s3",kind)==0) return NCZM_S3;
else if(strcasecmp("nz4",kind)==0) return NCZM_NC4;
else if(strcasecmp("nzf",kind)==0) return NCZM_FILE;
else if(strcasecmp("file",kind)==0) return NCZM_FILE;
else if(strcasecmp("zip",kind)==0) return NCZM_ZIP;
else return NCZM_UNDEF;
}
@ -371,8 +371,8 @@ impl2kind(NCZM_IMPL impl)
{
switch (impl) {
case NCZM_S3: return "s3";
case NCZM_NC4: return "nz4";
case NCZM_FILE: return "nzf";
case NCZM_FILE: return "file";
case NCZM_ZIP: return "zip";
case NCZM_UNDEF: break;
}
return NULL;

11
nczarr_test/zhex.c
View File

@ -21,9 +21,17 @@ main(int argc, char** argv)
{
unsigned char c;
unsigned int c0,c1;
FILE* f = NULL;
if(argc > 1) {
/* use argv[1] as input */
f = fopen(argv[1],"r");
if(f == NULL) {fprintf(stderr,"No such file: %s\n",argv[1]); exit(1);}
} else
f = stdin;
for(;;) {
int ret = fread(&c, 1, 1, stdin);
int ret = fread(&c, 1, 1, f);
if(ret != 1) break;
c1 = c;
c0 = c1 & 0xf;
@ -32,5 +40,6 @@ main(int argc, char** argv)
c1 = hex[c1];
printf("%c%c",(char)c1,(char)c0);
}
if(f != stdin) fclose(f);
return 0;
}

11
nczarr_test/zisjson.c
View File

@ -29,11 +29,19 @@ main(int argc, char** argv)
char text[MAXREAD+1];
NCjson* json = NULL;
int i, red;
FILE* f = NULL;
if(argc > 1) {
/* use argv[1] as input */
f = fopen(argv[1],"r");
if(f == NULL) {fprintf(stderr,"No such file: %s\n",argv[1]); exit(1);}
} else
f = stdin;
/* Read json from stdin */
for(i=0;;i++) {
unsigned char c;
red = fread(&c, 1, 1, stdin);
red = fread(&c, 1, 1, f);
if(red != 1) break;
if(i < MAXREAD) text[i] = (char)c;
}
@ -49,5 +57,6 @@ main(int argc, char** argv)
NCJreclaim(json);
}
printf("%d",(stat==NC_NOERR?1:0)); /* parse success|failure */
if(f != stdin) fclose(f);
return 0;
}

100
nczarr_test/zmapio.c
View File

@ -19,6 +19,8 @@
#include "zincludes.h"
#include "ncpathmgr.h"
#include "nclog.h"
#include "ncuri.h"
#define DEBUG
@ -33,7 +35,8 @@ MOP_CLEAR=2
typedef enum OBJKIND {
OK_NONE=0,
OK_META=1,
OK_CHUNK=2
OK_CHUNK=2,
OK_GROUP=3
} OBJKIND;
static struct Mops {
@ -78,7 +81,7 @@ struct Dumpptions {
static int objdump(void);
static NCZM_IMPL implfor(const char* path);
static void printcontent(size64_t len, const char* content, OBJKIND kind);
static int depthR(NCZMAP* map, char* key, NClist* stack);
static int breadthfirst(NCZMAP* map, const char*, NClist* stack);
static char* rootpathfor(const char* path);
static OBJKIND keykind(const char* key);
static void sortlist(NClist* l);
@ -105,7 +108,7 @@ static Mapop
decodeop(const char* name)
{
struct Mops* p = mapops;
while(p->opname != NULL) {
for(;p->opname != NULL;p++) {
if(strcasecmp(p->opname,name)==0) return p->mapop;
}
return MOP_NONE;
@ -129,7 +132,7 @@ main(int argc, char** argv)
memset((void*)&dumpoptions,0,sizeof(dumpoptions));
while ((c = getopt(argc, argv, "dvx:t:")) != EOF) {
while ((c = getopt(argc, argv, "dvx:t:T:")) != EOF) {
switch(c) {
case 'd':
dumpoptions.debug = 1;
@ -145,6 +148,9 @@ main(int argc, char** argv)
dumpoptions.mop = decodeop(optarg);
if(dumpoptions.mop == MOP_NONE) zmapusage();
break;
case 'T':
nctracelevel(atoi(optarg));
break;
case '?':
fprintf(stderr,"unknown option\n");
goto fail;
@ -219,8 +225,8 @@ implfor(const char* path)
NCCHECK(nczm_split_delim(mode,',',segments));
for(i=0;i<nclistlength(segments);i++) {
const char* value = nclistget(segments,i);
if(strcmp(value,"nz4")==0) {impl = NCZM_NC4; goto done;}
if(strcmp(value,"nzf")==0) {impl = NCZM_FILE; goto done;}
if(strcmp(value,"file")==0) {impl = NCZM_FILE; goto done;}
if(strcmp(value,"zip")==0) {impl = NCZM_ZIP; goto done;}
if(strcmp(value,"s3")==0) {impl = NCZM_S3; goto done;}
}
done:
@ -242,7 +248,7 @@ rootpathfor(const char* path)
if(uri == NULL) goto done;
switch (dumpoptions.impl) {
case NCZM_FILE:
case NCZM_NC4:
case NCZM_ZIP:
rootpath = strdup("/"); /*constant*/
break;
case NCZM_S3:
@ -272,7 +278,6 @@ objdump(void)
{
int stat = NC_NOERR;
NCZMAP* map = NULL;
NClist* matches = nclistnew();
NClist* stack = nclistnew();
char* obj = NULL;
char* content = NULL;
@ -281,11 +286,8 @@ objdump(void)
if((stat=nczmap_open(dumpoptions.impl, dumpoptions.infile, NC_NOCLOBBER, 0, NULL, &map)))
goto done;
/* Depth first walk all the groups to get all keys */
obj = strdup("/");
if((stat = depthR(map,obj,stack))) goto done;
obj = NULL; /* its now in the stack */
if((stat = breadthfirst(map,"/",stack))) goto done;
if(dumpoptions.debug) {
int i;
@ -298,6 +300,7 @@ objdump(void)
OBJKIND kind = 0;
int hascontent = 0;
obj = nclistremove(stack,0); /* zero pos is always top of stack */
kind = keykind(obj);
/* Now print info for this obj key */
switch (stat=nczmap_len(map,obj,&len)) {
case NC_NOERR: hascontent = 1; break;
@ -316,7 +319,6 @@ objdump(void)
if(hascontent) {
if(len > 0) {
assert(content != NULL);
kind = keykind(obj);
if(kind == OK_CHUNK) len /= dumpoptions.nctype->typesize;
printf("[%d] %s : (%llu)",depth,obj,len);
if(kind == OK_CHUNK) printf(" (%s)",dumpoptions.nctype->typename);
@ -337,32 +339,60 @@ done:
nullfree(obj);
nullfree(content);
nczmap_close(map,0);
nclistfreeall(matches);
nclistfreeall(stack);
return stat;
}
/* Depth first walk all the groups to get all keys */
static int
depthR(NCZMAP* map, char* key, NClist* stack)
breadthfirstR(NCZMAP* map, NCbytes* prefix, NClist* stack)
{
int stat = NC_NOERR;
NClist* nextlevel = nclistnew();
size_t mark;
const char* content;
int isroot = 0;
nclistpush(stack,key);
if((stat=nczmap_search(map,key,nextlevel))) goto done;
content = ncbytescontents(prefix);
if(content[0] == '/' && content[1] == '\0') isroot = 1;
if((stat=nczmap_search(map,content,nextlevel))) goto done;
/* Sort nextlevel */
sortlist(nextlevel);
/* Push new names onto the stack and recurse */
mark = ncbyteslength(prefix); /* save this position */
while(nclistlength(nextlevel) > 0) {
char* subkey = nclistremove(nextlevel,0);
if((stat = depthR(map,subkey,stack))) goto done;
if(!isroot) ncbytescat(prefix,"/");
ncbytescat(prefix,subkey);
nullfree(subkey);
nclistpush(stack,ncbytesdup(prefix));
if((stat = breadthfirstR(map,prefix,stack))) goto done;
ncbytessetlength(prefix,mark); ncbytesnull(prefix);
}
done:
nclistfreeall(nextlevel);
return stat;
}
/* Depth first walk all the groups to get all keys */
static int
breadthfirst(NCZMAP* map, const char* key, NClist* stack)
{
int stat = NC_NOERR;
NCbytes* prefix = ncbytesnew();
if(key == NULL || key[0] == '\0')
key = "/";
ncbytescat(prefix,key);
if(strlen(key) > 1 && key[strlen(key)-1]=='/') {
ncbytessetlength(prefix,ncbyteslength(prefix)-1); /* remove trailing '/' */
ncbytesnull(prefix);
}
stat = breadthfirstR(map,prefix,stack);
ncbytesfree(prefix);
return stat;
}
static char hex[16] = "0123456789abcdef";
static void
@ -417,6 +447,8 @@ keykind(const char* key)
kind = OK_NONE;
else if(suffix[1] == '.')
kind = OK_META;
else if(suffix[strlen(suffix)-1] == '/')
kind = OK_GROUP;
else {
char* p = suffix+1;
for(;*p;p++) {
@ -455,7 +487,7 @@ fprintf(stderr,"sorted: [%d] %s\n",i,(const char*)nclistget(l,i));
#endif
}
static const char* urlexts[] = {"nzf", "nz4", NULL};
static const char* urlexts[] = {"nzf", "zip", "nz4", NULL};
static const char*
filenamefor(const char* f0)
@ -463,22 +495,26 @@ filenamefor(const char* f0)
static char result[4096];
const char** extp;
char* p;
NCURI* uri = NULL;
strcpy(result,f0); /* default */
if(nc__testurl(f0,NULL)) goto done;
/* Not a URL */
p = strrchr(f0,'.'); /* look at the extension, if any */
if(p == NULL) goto done; /* No extension */
p++;
for(extp=urlexts;*extp;extp++) {
if(strcmp(p,*extp)==0) break;
ncuriparse(f0,&uri);
if(uri == NULL) {
/* Not a URL */
p = strrchr(f0,'.'); /* look at the extension, if any */
if(p == NULL) goto done; /* No extension */
p++;
for(extp=urlexts;*extp;extp++) {
if(strcmp(p,*extp)==0) break;
}
if(*extp == NULL) goto done; /* not found */
/* Assemble the url */
strcpy(result,"file://");
strcat(result,f0); /* core path */
strcat(result,"#mode=nczarr,");
strcat(result,*extp);
}
if(*extp == NULL) goto done; /* not found */
/* Assemble the url */
strcpy(result,"file://");
strcat(result,f0); /* core path */
strcat(result,"#mode=nczarr,");
strcat(result,*extp);
done:
ncurifree(uri);
return result;
}

1
test_common.in
View File

@ -12,6 +12,7 @@ TOPBUILDDIR='@abs_top_builddir@'
# Define selected features of the build
FEATURE_HDF5=@HAS_HDF5@
FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@
FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@
set -e