2010-06-03 21:24:43 +08:00
|
|
|
/*
|
2018-12-07 05:34:40 +08:00
|
|
|
* Copyright 2018, University Corporation for Atmospheric Research
|
2010-06-03 21:24:43 +08:00
|
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
|
|
*/
|
|
|
|
/* $Id: ncio.h,v 1.27 2006/01/03 04:56:28 russ Exp $ */
|
|
|
|
|
|
|
|
#ifndef _NCIO_H_
|
|
|
|
#define _NCIO_H_
|
|
|
|
|
|
|
|
#include <stddef.h> /* size_t */
|
|
|
|
#include <sys/types.h> /* off_t */
|
|
|
|
#include "netcdf.h"
|
|
|
|
|
Provide byte-range reading of remote datasets
re: issue https://github.com/Unidata/netcdf-c/issues/1251
Assume that you have the URL to a remote dataset
which is a normal netcdf-3 or netcdf-4 file.
This PR allows the netcdf-c to read that dataset's
contents as a netcdf file using HTTP byte ranges
if the remote server supports byte-range access.
Originally, this PR was set up to access Amazon S3 objects,
but it can also access other remote datasets such as those
provided by a Thredds server via the HTTPServer access protocol.
It may also work for other kinds of servers.
Note that this is not intended as a true production
capability because, as is known, this kind of access to
can be quite slow. In addition, the byte-range IO drivers
do not currently do any sort of optimization or caching.
An additional goal here is to gain some experience with
the Amazon S3 REST protocol.
This architecture and its use documented in
the file docs/byterange.dox.
There are currently two test cases:
1. nc_test/tst_s3raw.c - this does a simple open, check format, close cycle
for a remote netcdf-3 file and a remote netcdf-4 file.
2. nc_test/test_s3raw.sh - this uses ncdump to investigate some remote
datasets.
This PR also incorporates significantly changed model inference code
(see the superceded PR https://github.com/Unidata/netcdf-c/pull/1259).
1. It centralizes the code that infers the dispatcher.
2. It adds support for byte-range URLs
Other changes:
1. NC_HDF5_finalize was not being properly called by nc_finalize().
2. Fix minor bug in ncgen3.l
3. fix memory leak in nc4info.c
4. add code to walk the .daprc triples and to replace protocol=
fragment tag with a more general mode= tag.
Final Note:
Th inference code is still way too complicated. We need to move
to the validfile() model used by netcdf Java, where each
dispatcher is asked if it can process the file. This decentralizes
the inference code. This will be done after all the major new
dispatchers (PIO, Zarr, etc) have been implemented.
2019-01-02 09:27:36 +08:00
|
|
|
/* Define an internal use only mode flag to signal use of byte ranges.
|
|
|
|
This is temporary until we can re-organize the ncio open/create API.
|
|
|
|
*/
|
|
|
|
#define NC_HTTP 0x80000000
|
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
typedef struct ncio ncio; /* forward reference */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A value which is an invalid off_t
|
|
|
|
*/
|
|
|
|
#define OFF_NONE ((off_t)(-1))
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Flags used by the region layer,
|
|
|
|
* 'rflags' argument to ncio.rel() and ncio.get().
|
|
|
|
*/
|
|
|
|
#define RGN_NOLOCK 0x1 /* Don't lock region.
|
|
|
|
* Used when contention control handled
|
|
|
|
* elsewhere.
|
|
|
|
*/
|
|
|
|
#define RGN_NOWAIT 0x2 /* return immediate if can't lock, else wait */
|
|
|
|
|
|
|
|
#define RGN_WRITE 0x4 /* we intend to modify, else read only */
|
|
|
|
|
|
|
|
#define RGN_MODIFIED 0x8 /* we did modify, else, discard */
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The next four typedefs define the signatures
|
|
|
|
* of function pointers in struct ncio below.
|
|
|
|
* They are not used outside of this file and ncio.h,
|
|
|
|
* They just make some casts in the ncio.c more readable.
|
|
|
|
*/
|
|
|
|
/*
|
|
|
|
* Indicate that you are done with the region which begins
|
|
|
|
* at offset. Only reasonable flag value is RGN_MODIFIED.
|
|
|
|
*/
|
|
|
|
typedef int ncio_relfunc(ncio *const nciop,
|
|
|
|
off_t offset, int rflags);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Request that the region (offset, extent)
|
|
|
|
* be made available through *vpp.
|
|
|
|
*/
|
|
|
|
typedef int ncio_getfunc(ncio *const nciop,
|
|
|
|
off_t offset, size_t extent,
|
|
|
|
int rflags,
|
|
|
|
void **const vpp);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Like memmove(), safely move possibly overlapping data.
|
|
|
|
* Only reasonable flag value is RGN_NOLOCK.
|
|
|
|
*/
|
|
|
|
typedef int ncio_movefunc(ncio *const nciop, off_t to, off_t from,
|
|
|
|
size_t nbytes, int rflags);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Write out any dirty buffers to disk and
|
|
|
|
* ensure that next read will get data from disk.
|
|
|
|
*/
|
|
|
|
typedef int ncio_syncfunc(ncio *const nciop);
|
|
|
|
|
2012-03-26 09:34:32 +08:00
|
|
|
/*
|
|
|
|
* Sync any changes to disk, then truncate or extend file so its size
|
|
|
|
* is length. This is only intended to be called before close, if the
|
|
|
|
* file is open for writing and the actual size does not match the
|
|
|
|
* calculated size, perhaps as the result of having been previously
|
|
|
|
* written in NOFILL mode.
|
|
|
|
*/
|
|
|
|
typedef int ncio_pad_lengthfunc(ncio* nciop, off_t length);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get file size in bytes.
|
|
|
|
*/
|
|
|
|
typedef int ncio_filesizefunc(ncio *nciop, off_t *filesizep);
|
|
|
|
|
|
|
|
/* Write out any dirty buffers and
|
|
|
|
ensure that next read will not get cached data.
|
|
|
|
Sync any changes, then close the open file associated with the ncio
|
|
|
|
struct, and free its memory.
|
|
|
|
nciop - pointer to ncio to close.
|
|
|
|
doUnlink - if true, unlink file
|
|
|
|
*/
|
|
|
|
typedef int ncio_closefunc(ncio *nciop, int doUnlink);
|
2010-06-03 21:24:43 +08:00
|
|
|
|
|
|
|
/* Get around cplusplus "const xxx in class ncio without constructor" error */
|
|
|
|
#if defined(__cplusplus)
|
|
|
|
#define NCIO_CONST
|
|
|
|
#else
|
|
|
|
#define NCIO_CONST const
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* netcdf i/o abstraction
|
|
|
|
*/
|
|
|
|
struct ncio {
|
|
|
|
/*
|
|
|
|
* A copy of the ioflags argument passed in to ncio_open()
|
|
|
|
* or ncio_create().
|
|
|
|
*/
|
|
|
|
int ioflags;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The file descriptor of the netcdf file.
|
|
|
|
* This gets handed to the user as the netcdf id.
|
|
|
|
*/
|
|
|
|
NCIO_CONST int fd;
|
|
|
|
|
|
|
|
/* member functions do the work */
|
|
|
|
|
|
|
|
ncio_relfunc *NCIO_CONST rel;
|
|
|
|
|
|
|
|
ncio_getfunc *NCIO_CONST get;
|
|
|
|
|
|
|
|
ncio_movefunc *NCIO_CONST move;
|
|
|
|
|
|
|
|
ncio_syncfunc *NCIO_CONST sync;
|
|
|
|
|
2012-03-26 09:34:32 +08:00
|
|
|
ncio_pad_lengthfunc *NCIO_CONST pad_length;
|
|
|
|
|
|
|
|
ncio_filesizefunc *NCIO_CONST filesize;
|
2013-08-10 05:47:42 +08:00
|
|
|
|
2012-03-26 09:34:32 +08:00
|
|
|
ncio_closefunc *NCIO_CONST close;
|
2010-06-03 21:24:43 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* A copy of the 'path' argument passed in to ncio_open()
|
|
|
|
* or ncio_create(). Used by ncabort() to remove (unlink)
|
|
|
|
* the file and by error messages.
|
|
|
|
*/
|
|
|
|
const char *path;
|
|
|
|
|
|
|
|
/* implementation private stuff */
|
2012-01-16 10:41:07 +08:00
|
|
|
void *pvt;
|
2010-06-03 21:24:43 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
#undef NCIO_CONST
|
|
|
|
|
2012-03-26 09:34:32 +08:00
|
|
|
/* Define wrappers around the ncio dispatch table */
|
|
|
|
|
|
|
|
extern int ncio_rel(ncio* const, off_t, int);
|
|
|
|
extern int ncio_get(ncio* const, off_t, size_t, int, void** const);
|
|
|
|
extern int ncio_move(ncio* const, off_t, off_t, size_t, int);
|
|
|
|
extern int ncio_sync(ncio* const);
|
|
|
|
extern int ncio_filesize(ncio* const, off_t*);
|
|
|
|
extern int ncio_pad_length(ncio* const, off_t);
|
|
|
|
extern int ncio_close(ncio* const, int);
|
|
|
|
|
|
|
|
extern int ncio_create(const char *path, int ioflags, size_t initialsz,
|
|
|
|
off_t igeto, size_t igetsz, size_t *sizehintp,
|
2015-05-29 05:03:02 +08:00
|
|
|
void* parameters, /* new */
|
2012-03-26 09:34:32 +08:00
|
|
|
ncio** nciopp, void** const mempp);
|
2010-06-03 21:24:43 +08:00
|
|
|
|
2012-03-26 09:34:32 +08:00
|
|
|
extern int ncio_open(const char *path, int ioflags,
|
|
|
|
off_t igeto, size_t igetsz, size_t *sizehintp,
|
2015-05-29 05:03:02 +08:00
|
|
|
void* parameters, /* new */
|
2012-03-26 09:34:32 +08:00
|
|
|
ncio** nciopp, void** const mempp);
|
2010-06-03 21:24:43 +08:00
|
|
|
|
2012-03-26 09:34:32 +08:00
|
|
|
/* With the advent of diskless io, we need to provide
|
|
|
|
for multiple ncio packages at the same time,
|
|
|
|
so we have multiple versions of ncio_create.
|
|
|
|
If you create a new package, the you must do the following.
|
|
|
|
1. add an extern definition for it in ncio.c
|
|
|
|
2. modify ncio_create and ncio_open in ncio.c to invoke
|
|
|
|
the new package when appropriate.
|
|
|
|
*/
|
2010-06-03 21:24:43 +08:00
|
|
|
|
|
|
|
#endif /* _NCIO_H_ */
|