2017-08-31 07:44:57 +08:00
|
|
|
/*
|
2018-12-07 05:13:56 +08:00
|
|
|
Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
|
|
|
|
|
See COPYRIGHT for license information.
|
2017-08-31 07:44:57 +08:00
|
|
|
*/
|
2017-03-09 08:01:10 +08:00
|
|
|
|
2017-09-01 04:19:56 +08:00
|
|
|
/*
|
|
|
|
|
Common functionality for reading
|
|
|
|
|
and accessing rc files (e.g. .daprc).
|
|
|
|
|
*/
|
|
|
|
|
|
2017-03-09 08:01:10 +08:00
|
|
|
#ifndef NCRC_H
|
|
|
|
|
#define NCRC_H
|
|
|
|
|
|
2017-09-01 04:19:56 +08:00
|
|
|
/* Need these support includes */
|
2017-08-31 07:44:57 +08:00
|
|
|
#include "ncuri.h"
|
2017-09-01 04:19:56 +08:00
|
|
|
#include "nclist.h"
|
|
|
|
|
#include "ncbytes.h"
|
2017-03-09 08:01:10 +08:00
|
|
|
|
|
|
|
|
typedef struct NCTriple {
|
2017-08-31 07:44:57 +08:00
|
|
|
char* host; /* combined host:port */
|
2017-03-09 08:01:10 +08:00
|
|
|
char* key;
|
|
|
|
|
char* value;
|
|
|
|
|
} NCTriple;
|
|
|
|
|
|
2017-09-01 04:19:56 +08:00
|
|
|
/* collect all the relevant info around the rc file */
|
2017-08-31 07:44:57 +08:00
|
|
|
typedef struct NCRCinfo {
|
2017-09-01 04:19:56 +08:00
|
|
|
int ignore; /* if 1, then do not use any rc file */
|
|
|
|
|
int loaded; /* 1 => already loaded */
|
|
|
|
|
NClist* triples; /* the rc file triple store fields*/
|
|
|
|
|
char* rcfile; /* specified rcfile; overrides anything else */
|
2017-08-31 07:44:57 +08:00
|
|
|
} NCRCinfo;
|
2017-03-09 08:01:10 +08:00
|
|
|
|
2017-08-31 07:44:57 +08:00
|
|
|
/* Collect global state info in one place */
|
|
|
|
|
typedef struct NCRCglobalstate {
|
|
|
|
|
int initialized;
|
|
|
|
|
char* tempdir; /* track a usable temp dir */
|
|
|
|
|
char* home; /* track $HOME for use in creating $HOME/.oc dir */
|
2018-03-16 22:38:40 +08:00
|
|
|
NCRCinfo rcinfo; /* Currently only one rc file per session */
|
2017-08-31 07:44:57 +08:00
|
|
|
} NCRCglobalstate;
|
|
|
|
|
|
|
|
|
|
/* From drc.c */
|
2019-03-31 04:06:20 +08:00
|
|
|
extern NCRCglobalstate* ncrc_getglobalstate(void);
|
|
|
|
|
extern void ncrc_freeglobalstate(void);
|
2017-03-09 08:01:10 +08:00
|
|
|
/* read and compile the rc file, if any */
|
2017-08-31 07:44:57 +08:00
|
|
|
extern int NC_rcload(void);
|
|
|
|
|
extern char* NC_rclookup(const char* key, const char* hostport);
|
|
|
|
|
extern void NC_rcclear(NCRCinfo* info);
|
2017-09-01 04:19:56 +08:00
|
|
|
extern int NC_set_rcfile(const char* rcfile);
|
2018-08-27 07:04:46 +08:00
|
|
|
extern int NC_rcfile_insert(const char* key, const char* value, const char* hostport);
|
Provide byte-range reading of remote datasets
re: issue https://github.com/Unidata/netcdf-c/issues/1251
Assume that you have the URL to a remote dataset
which is a normal netcdf-3 or netcdf-4 file.
This PR allows the netcdf-c to read that dataset's
contents as a netcdf file using HTTP byte ranges
if the remote server supports byte-range access.
Originally, this PR was set up to access Amazon S3 objects,
but it can also access other remote datasets such as those
provided by a Thredds server via the HTTPServer access protocol.
It may also work for other kinds of servers.
Note that this is not intended as a true production
capability because, as is known, this kind of access to
can be quite slow. In addition, the byte-range IO drivers
do not currently do any sort of optimization or caching.
An additional goal here is to gain some experience with
the Amazon S3 REST protocol.
This architecture and its use documented in
the file docs/byterange.dox.
There are currently two test cases:
1. nc_test/tst_s3raw.c - this does a simple open, check format, close cycle
for a remote netcdf-3 file and a remote netcdf-4 file.
2. nc_test/test_s3raw.sh - this uses ncdump to investigate some remote
datasets.
This PR also incorporates significantly changed model inference code
(see the superceded PR https://github.com/Unidata/netcdf-c/pull/1259).
1. It centralizes the code that infers the dispatcher.
2. It adds support for byte-range URLs
Other changes:
1. NC_HDF5_finalize was not being properly called by nc_finalize().
2. Fix minor bug in ncgen3.l
3. fix memory leak in nc4info.c
4. add code to walk the .daprc triples and to replace protocol=
fragment tag with a more general mode= tag.
Final Note:
Th inference code is still way too complicated. We need to move
to the validfile() model used by netcdf Java, where each
dispatcher is asked if it can process the file. This decentralizes
the inference code. This will be done after all the major new
dispatchers (PIO, Zarr, etc) have been implemented.
2019-01-02 09:27:36 +08:00
|
|
|
/* Obtain the count of number of triples */
|
|
|
|
|
extern size_t NC_rcfile_length(NCRCinfo*);
|
|
|
|
|
/* Obtain the ith triple; return NULL if out of range */
|
|
|
|
|
extern NCTriple* NC_rcfile_ith(NCRCinfo*,size_t);
|
2017-08-31 07:44:57 +08:00
|
|
|
|
2017-09-01 04:19:56 +08:00
|
|
|
/* From dutil.c (Might later move to e.g. nc.h */
|
2017-08-31 07:44:57 +08:00
|
|
|
extern int NC__testurl(const char* path, char** basenamep);
|
|
|
|
|
extern int NC_isLittleEndian(void);
|
|
|
|
|
extern char* NC_entityescape(const char* s);
|
|
|
|
|
extern int NC_readfile(const char* filename, NCbytes* content);
|
2018-10-11 03:32:17 +08:00
|
|
|
extern int NC_writefile(const char* filename, size_t size, void* content);
|
2017-09-03 08:09:36 +08:00
|
|
|
extern char* NC_mktmp(const char* base);
|
2019-09-30 02:59:28 +08:00
|
|
|
extern int NC_getmodelist(const char* url, NClist** modelistp);
|
|
|
|
|
extern int NC_testmode(const char* path, const char* tag);
|
2017-03-09 08:01:10 +08:00
|
|
|
#endif /*NCRC_H*/
|
