netcdf-c/libsrc/ncio.h

180 lines
4.9 KiB
C
Raw Normal View History

2010-06-03 21:24:43 +08:00
/*
* Copyright 2018, University Corporation for Atmospheric Research
2010-06-03 21:24:43 +08:00
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
/* $Id: ncio.h,v 1.27 2006/01/03 04:56:28 russ Exp $ */
#ifndef _NCIO_H_
#define _NCIO_H_
#include <stddef.h> /* size_t */
#include <sys/types.h> /* off_t */
#include "netcdf.h"
/* Define internal use only flags to signal use of byte ranges and S3. */
#define NC_HTTP 1
#define NC_S3SDK 2
Provide byte-range reading of remote datasets re: issue https://github.com/Unidata/netcdf-c/issues/1251 Assume that you have the URL to a remote dataset which is a normal netcdf-3 or netcdf-4 file. This PR allows the netcdf-c to read that dataset's contents as a netcdf file using HTTP byte ranges if the remote server supports byte-range access. Originally, this PR was set up to access Amazon S3 objects, but it can also access other remote datasets such as those provided by a Thredds server via the HTTPServer access protocol. It may also work for other kinds of servers. Note that this is not intended as a true production capability because, as is known, this kind of access to can be quite slow. In addition, the byte-range IO drivers do not currently do any sort of optimization or caching. An additional goal here is to gain some experience with the Amazon S3 REST protocol. This architecture and its use documented in the file docs/byterange.dox. There are currently two test cases: 1. nc_test/tst_s3raw.c - this does a simple open, check format, close cycle for a remote netcdf-3 file and a remote netcdf-4 file. 2. nc_test/test_s3raw.sh - this uses ncdump to investigate some remote datasets. This PR also incorporates significantly changed model inference code (see the superceded PR https://github.com/Unidata/netcdf-c/pull/1259). 1. It centralizes the code that infers the dispatcher. 2. It adds support for byte-range URLs Other changes: 1. NC_HDF5_finalize was not being properly called by nc_finalize(). 2. Fix minor bug in ncgen3.l 3. fix memory leak in nc4info.c 4. add code to walk the .daprc triples and to replace protocol= fragment tag with a more general mode= tag. Final Note: Th inference code is still way too complicated. We need to move to the validfile() model used by netcdf Java, where each dispatcher is asked if it can process the file. This decentralizes the inference code. This will be done after all the major new dispatchers (PIO, Zarr, etc) have been implemented.
2019-01-02 09:27:36 +08:00
2010-06-03 21:24:43 +08:00
typedef struct ncio ncio; /* forward reference */
/*
* A value which is an invalid off_t
*/
#define OFF_NONE ((off_t)(-1))
/*
* Flags used by the region layer,
* 'rflags' argument to ncio.rel() and ncio.get().
*/
#define RGN_NOLOCK 0x1 /* Don't lock region.
* Used when contention control handled
* elsewhere.
*/
#define RGN_NOWAIT 0x2 /* return immediate if can't lock, else wait */
#define RGN_WRITE 0x4 /* we intend to modify, else read only */
#define RGN_MODIFIED 0x8 /* we did modify, else, discard */
/*
* The next four typedefs define the signatures
* of function pointers in struct ncio below.
* They are not used outside of this file and ncio.h,
* They just make some casts in the ncio.c more readable.
*/
/*
* Indicate that you are done with the region which begins
* at offset. Only reasonable flag value is RGN_MODIFIED.
*/
typedef int ncio_relfunc(ncio *const nciop,
off_t offset, int rflags);
/*
* Request that the region (offset, extent)
* be made available through *vpp.
*/
typedef int ncio_getfunc(ncio *const nciop,
off_t offset, size_t extent,
int rflags,
void **const vpp);
/*
* Like memmove(), safely move possibly overlapping data.
* Only reasonable flag value is RGN_NOLOCK.
*/
typedef int ncio_movefunc(ncio *const nciop, off_t to, off_t from,
size_t nbytes, int rflags);
/*
* Write out any dirty buffers to disk and
* ensure that next read will get data from disk.
*/
typedef int ncio_syncfunc(ncio *const nciop);
/*
* Sync any changes to disk, then truncate or extend file so its size
* is length. This is only intended to be called before close, if the
* file is open for writing and the actual size does not match the
* calculated size, perhaps as the result of having been previously
* written in NOFILL mode.
*/
typedef int ncio_pad_lengthfunc(ncio* nciop, off_t length);
/*
* Get file size in bytes.
*/
typedef int ncio_filesizefunc(ncio *nciop, off_t *filesizep);
/* Write out any dirty buffers and
ensure that next read will not get cached data.
Sync any changes, then close the open file associated with the ncio
struct, and free its memory.
nciop - pointer to ncio to close.
doUnlink - if true, unlink file
*/
typedef int ncio_closefunc(ncio *nciop, int doUnlink);
2010-06-03 21:24:43 +08:00
/* Get around cplusplus "const xxx in class ncio without constructor" error */
#if defined(__cplusplus)
#define NCIO_CONST
#else
#define NCIO_CONST const
#endif
/*
* netcdf i/o abstraction
*/
struct ncio {
/*
* A copy of the ioflags argument passed in to ncio_open()
* or ncio_create().
*/
int ioflags;
/*
* The file descriptor of the netcdf file.
* This gets handed to the user as the netcdf id.
*/
NCIO_CONST int fd;
/* member functions do the work */
ncio_relfunc *NCIO_CONST rel;
ncio_getfunc *NCIO_CONST get;
ncio_movefunc *NCIO_CONST move;
ncio_syncfunc *NCIO_CONST sync;
ncio_pad_lengthfunc *NCIO_CONST pad_length;
ncio_filesizefunc *NCIO_CONST filesize;
ncio_closefunc *NCIO_CONST close;
2010-06-03 21:24:43 +08:00
/*
* A copy of the 'path' argument passed in to ncio_open()
* or ncio_create(). Used by ncabort() to remove (unlink)
* the file and by error messages.
*/
const char *path;
/* implementation private stuff */
2012-01-16 10:41:07 +08:00
void *pvt;
2010-06-03 21:24:43 +08:00
};
#undef NCIO_CONST
/* Define wrappers around the ncio dispatch table */
extern int ncio_rel(ncio* const, off_t, int);
extern int ncio_get(ncio* const, off_t, size_t, int, void** const);
extern int ncio_move(ncio* const, off_t, off_t, size_t, int);
extern int ncio_sync(ncio* const);
extern int ncio_filesize(ncio* const, off_t*);
extern int ncio_pad_length(ncio* const, off_t);
extern int ncio_close(ncio* const, int);
extern int ncio_create(const char *path, int ioflags, size_t initialsz,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters, /* new */
ncio** nciopp, void** const mempp);
2010-06-03 21:24:43 +08:00
extern int ncio_open(const char *path, int ioflags,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters, /* new */
ncio** nciopp, void** const mempp);
2010-06-03 21:24:43 +08:00
/* With the advent of diskless io, we need to provide
for multiple ncio packages at the same time,
so we have multiple versions of ncio_create.
If you create a new package, the you must do the following.
1. add an extern definition for it in ncio.c
2. modify ncio_create and ncio_open in ncio.c to invoke
the new package when appropriate.
*/
2010-06-03 21:24:43 +08:00
#endif /* _NCIO_H_ */