netcdf-c/libsrc/ncio.c
Dennis Heimbigner f6e25b695e Fix additional S3 support issues
re: https://github.com/Unidata/netcdf-c/issues/2117
re: https://github.com/Unidata/netcdf-c/issues/2119

* Modify libsrc to allow byte-range reading of netcdf-3 files in private S3 buckets; this required using the aws sdk. Also add a test case.
* The aws sdk can sometimes cause problems if the Awd::ShutdownAPI function is not called. So at optional atexit() support to ensure it is called. This is disabled for Windows.
* Add documentation to nczarr.md on how to build and use the aws sdk under windows. Currently it builds, but testing fails.
* Switch testing from stratus to the Unidata bucket on S3.
* Improve support for the s3: url protocol.
* Add a s3 specific utility code file: ds3util.c
* Modify NC_infermodel to attempt to read the magic number of byte-ranged files in S3.

## Misc.

* Move and rename the core S3 SDK wrapper code (libnczarr/zs3sdk.cpp) to libdispatch since it now used in libsrc as well as libnczarr.
* Add calls to nc_finalize in the utilities in case atexit is disabled.
* Add header only json parser to the distribution rather than as a built source.
2021-10-29 20:06:37 -06:00

165 lines
5.1 KiB
C

/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
#include "netcdf.h"
#include "ncio.h"
#include "fbits.h"
/* With the advent of diskless io, we need to provide
for multiple ncio packages at the same time,
so we have multiple versions of ncio_create.
*/
/* Define known ncio packages */
extern int posixio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int posixio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int stdio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int stdio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
#ifdef USE_FFIO
extern int ffio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int ffio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
#endif
# ifdef USE_MMAP
extern int mmapio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int mmapio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
# endif
#ifdef ENABLE_BYTERANGE
extern int httpio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
#endif
#ifdef ENABLE_S3_SDK
extern int s3io_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
#endif
extern int memio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int memio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
int
ncio_create(const char *path, int ioflags, size_t initialsz,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters,
ncio** iopp, void** const mempp)
{
if(fIsSet(ioflags,NC_DISKLESS)) {
return memio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
} else if(fIsSet(ioflags,NC_INMEMORY)) {
return memio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
# ifdef USE_MMAP
else if(fIsSet(ioflags,NC_MMAP)) {
return mmapio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
# endif /*USE_MMAP*/
#ifdef USE_STDIO
return stdio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#elif defined(USE_FFIO)
return ffio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#else
return posixio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#endif
}
int
ncio_open(const char *path, int ioflags,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters,
ncio** iopp, void** const mempp)
{
/* Diskless open has the following constraints:
1. file must be classic version 1 or 2 or 5
*/
if(fIsSet(ioflags,NC_DISKLESS)) {
return memio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
if(fIsSet(ioflags,NC_INMEMORY)) {
return memio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
# ifdef USE_MMAP
if(fIsSet(ioflags,NC_MMAP)) {
return mmapio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
# endif /*USE_MMAP*/
# ifdef ENABLE_BYTERANGE
/* The NC_HTTP flag is a big hack until we can reorganize the ncio interface */
if(fIsSet(ioflags,NC_HTTP)) {
return httpio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
# ifdef ENABLE_S3_SDK
if(fIsSet(ioflags,NC_S3SDK)) {
return s3io_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
# endif
# endif /*ENABLE_BYTERANGE*/
#ifdef USE_STDIO
return stdio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#elif defined(USE_FFIO)
return ffio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#else
return posixio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#endif
}
/**************************************************/
/* wrapper functions for the ncio dispatch table */
int
ncio_rel(ncio* const nciop, off_t offset, int rflags)
{
return nciop->rel(nciop,offset,rflags);
}
int
ncio_get(ncio* const nciop, off_t offset, size_t extent,
int rflags, void **const vpp)
{
return nciop->get(nciop,offset,extent,rflags,vpp);
}
int
ncio_move(ncio* const nciop, off_t to, off_t from, size_t nbytes, int rflags)
{
return nciop->move(nciop,to,from,nbytes,rflags);
}
int
ncio_sync(ncio* const nciop)
{
return nciop->sync(nciop);
}
int
ncio_filesize(ncio* const nciop, off_t *filesizep)
{
return nciop->filesize(nciop,filesizep);
}
int
ncio_pad_length(ncio* const nciop, off_t length)
{
return nciop->pad_length(nciop,length);
}
int
ncio_close(ncio* const nciop, int doUnlink)
{
/* close and release all resources associated
with nciop, including nciop
*/
int status = nciop->close(nciop,doUnlink);
return status;
}