mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-01-12 15:45:21 +08:00
f6e25b695e
re: https://github.com/Unidata/netcdf-c/issues/2117 re: https://github.com/Unidata/netcdf-c/issues/2119 * Modify libsrc to allow byte-range reading of netcdf-3 files in private S3 buckets; this required using the aws sdk. Also add a test case. * The aws sdk can sometimes cause problems if the Awd::ShutdownAPI function is not called. So at optional atexit() support to ensure it is called. This is disabled for Windows. * Add documentation to nczarr.md on how to build and use the aws sdk under windows. Currently it builds, but testing fails. * Switch testing from stratus to the Unidata bucket on S3. * Improve support for the s3: url protocol. * Add a s3 specific utility code file: ds3util.c * Modify NC_infermodel to attempt to read the magic number of byte-ranged files in S3. ## Misc. * Move and rename the core S3 SDK wrapper code (libnczarr/zs3sdk.cpp) to libdispatch since it now used in libsrc as well as libnczarr. * Add calls to nc_finalize in the utilities in case atexit is disabled. * Add header only json parser to the distribution rather than as a built source.
165 lines
5.1 KiB
C
165 lines
5.1 KiB
C
/*
|
|
* Copyright 2018, University Corporation for Atmospheric Research
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
*/
|
|
|
|
#ifdef HAVE_CONFIG_H
|
|
#include "config.h"
|
|
#endif
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include "netcdf.h"
|
|
#include "ncio.h"
|
|
#include "fbits.h"
|
|
|
|
/* With the advent of diskless io, we need to provide
|
|
for multiple ncio packages at the same time,
|
|
so we have multiple versions of ncio_create.
|
|
*/
|
|
|
|
/* Define known ncio packages */
|
|
extern int posixio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
extern int posixio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
|
|
extern int stdio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
extern int stdio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
|
|
#ifdef USE_FFIO
|
|
extern int ffio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
extern int ffio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
#endif
|
|
|
|
# ifdef USE_MMAP
|
|
extern int mmapio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
extern int mmapio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
# endif
|
|
|
|
#ifdef ENABLE_BYTERANGE
|
|
extern int httpio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
#endif
|
|
|
|
#ifdef ENABLE_S3_SDK
|
|
extern int s3io_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
#endif
|
|
|
|
extern int memio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
extern int memio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
|
|
|
|
int
|
|
ncio_create(const char *path, int ioflags, size_t initialsz,
|
|
off_t igeto, size_t igetsz, size_t *sizehintp,
|
|
void* parameters,
|
|
ncio** iopp, void** const mempp)
|
|
{
|
|
if(fIsSet(ioflags,NC_DISKLESS)) {
|
|
return memio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
} else if(fIsSet(ioflags,NC_INMEMORY)) {
|
|
return memio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
}
|
|
# ifdef USE_MMAP
|
|
else if(fIsSet(ioflags,NC_MMAP)) {
|
|
return mmapio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
}
|
|
# endif /*USE_MMAP*/
|
|
|
|
#ifdef USE_STDIO
|
|
return stdio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
#elif defined(USE_FFIO)
|
|
return ffio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
#else
|
|
return posixio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
#endif
|
|
}
|
|
|
|
int
|
|
ncio_open(const char *path, int ioflags,
|
|
off_t igeto, size_t igetsz, size_t *sizehintp,
|
|
void* parameters,
|
|
ncio** iopp, void** const mempp)
|
|
{
|
|
/* Diskless open has the following constraints:
|
|
1. file must be classic version 1 or 2 or 5
|
|
*/
|
|
if(fIsSet(ioflags,NC_DISKLESS)) {
|
|
return memio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
}
|
|
if(fIsSet(ioflags,NC_INMEMORY)) {
|
|
return memio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
}
|
|
# ifdef USE_MMAP
|
|
if(fIsSet(ioflags,NC_MMAP)) {
|
|
return mmapio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
}
|
|
# endif /*USE_MMAP*/
|
|
# ifdef ENABLE_BYTERANGE
|
|
/* The NC_HTTP flag is a big hack until we can reorganize the ncio interface */
|
|
if(fIsSet(ioflags,NC_HTTP)) {
|
|
return httpio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
}
|
|
# ifdef ENABLE_S3_SDK
|
|
if(fIsSet(ioflags,NC_S3SDK)) {
|
|
return s3io_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
}
|
|
# endif
|
|
# endif /*ENABLE_BYTERANGE*/
|
|
|
|
#ifdef USE_STDIO
|
|
return stdio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
#elif defined(USE_FFIO)
|
|
return ffio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
#else
|
|
return posixio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
|
|
#endif
|
|
}
|
|
|
|
/**************************************************/
|
|
/* wrapper functions for the ncio dispatch table */
|
|
|
|
int
|
|
ncio_rel(ncio* const nciop, off_t offset, int rflags)
|
|
{
|
|
return nciop->rel(nciop,offset,rflags);
|
|
}
|
|
|
|
int
|
|
ncio_get(ncio* const nciop, off_t offset, size_t extent,
|
|
int rflags, void **const vpp)
|
|
{
|
|
return nciop->get(nciop,offset,extent,rflags,vpp);
|
|
}
|
|
|
|
int
|
|
ncio_move(ncio* const nciop, off_t to, off_t from, size_t nbytes, int rflags)
|
|
{
|
|
return nciop->move(nciop,to,from,nbytes,rflags);
|
|
}
|
|
|
|
int
|
|
ncio_sync(ncio* const nciop)
|
|
{
|
|
return nciop->sync(nciop);
|
|
}
|
|
|
|
int
|
|
ncio_filesize(ncio* const nciop, off_t *filesizep)
|
|
{
|
|
return nciop->filesize(nciop,filesizep);
|
|
}
|
|
|
|
int
|
|
ncio_pad_length(ncio* const nciop, off_t length)
|
|
{
|
|
return nciop->pad_length(nciop,length);
|
|
}
|
|
|
|
int
|
|
ncio_close(ncio* const nciop, int doUnlink)
|
|
{
|
|
/* close and release all resources associated
|
|
with nciop, including nciop
|
|
*/
|
|
int status = nciop->close(nciop,doUnlink);
|
|
return status;
|
|
}
|