mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-12-21 08:39:46 +08:00
49737888ca
## Improvements to S3 Documentation * Create a new document *quickstart_paths.md* that give a summary of the legal path formats used by netcdf-c. This includes both file paths and URL paths. * Modify *nczarr.md* to remove most of the S3 related text. * Move the S3 text from *nczarr.md* to a new document *cloud.md*. * Add some S3-related text to the *byterange.md* document. Hopefully, this will make it easier for users to find the information they want. ## Rebuild NCZarr Testing In order to avoid problems with running make check in parallel, two changes were made: 1. The *nczarr_test* test system was rebuilt. Now, for each test. any generated files are kept in a test-specific directory, isolated from all other test executions. 2. Similarly, since the S3 test bucket is shared, any generated S3 objects are isolated using a test-specific key path. ## Other S3 Related Changes * Add code to ensure that files created on S3 are reclaimed at end of testing. * Used the bash "trap" command to ensure S3 cleanup even if the test fails. * Cleanup the S3 related configure.ac flag set since S3 is used in several places. So now one should use the option *--enable-s3* instead of *--enable-nczarr-s3*, although the latter is still kept as a deprecated alias for the former. * Get some of the github actions yml to work with S3; required fixing various test scripts adding a secret to access the Unidata S3 bucket. * Cleanup S3 portion of libnetcdf.settings.in and netcdf_meta.h.in and test_common.in. * Merge partial S3 support into dhttp.c. * Create an experimental s3 access library especially for use with Windows. It is enabled by using the options *--enable-s3-internal* (automake) or *-DENABLE_S3_INTERNAL=ON* (CMake). Also add a unit-test for it. * Move some definitions from ncrc.h to ncs3sdk.h ## Other Changes * Provide a default implementation of strlcpy and move this and similar defaults into *dmissing.c*.
104 lines
3.3 KiB
C
104 lines
3.3 KiB
C
/*
|
|
Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
|
|
See COPYRIGHT for license information.
|
|
*/
|
|
|
|
/*
|
|
Common functionality for reading
|
|
and accessing rc files (e.g. .daprc).
|
|
*/
|
|
|
|
#ifndef NCRC_H
|
|
#define NCRC_H
|
|
|
|
/* Need these support includes */
|
|
#include "ncuri.h"
|
|
#include "nclist.h"
|
|
#include "ncbytes.h"
|
|
|
|
/* getenv() keys */
|
|
#define NCRCENVIGNORE "NCRCENV_IGNORE"
|
|
#define NCRCENVRC "NCRCENV_RC"
|
|
#define NCRCENVHOME "NCRCENV_HOME"
|
|
|
|
/* Known .aws profile keys */
|
|
#define AWS_ACCESS_KEY_ID "aws_access_key_id"
|
|
#define AWS_SECRET_ACCESS_KEY "aws_secret_access_key"
|
|
#define AWS_REGION "aws_region"
|
|
|
|
typedef struct NCRCentry {
|
|
char* host; /* combined host:port */
|
|
char* urlpath; /* prefix to match or NULL */
|
|
char* key;
|
|
char* value;
|
|
} NCRCentry;
|
|
|
|
struct AWSentry {
|
|
char* key;
|
|
char* value;
|
|
};
|
|
|
|
struct AWSprofile {
|
|
char* name;
|
|
NClist* entries; /* NClist<struct AWSentry*> */
|
|
};
|
|
|
|
/* collect all the relevant info around the rc file and AWS */
|
|
typedef struct NCRCinfo {
|
|
int ignore; /* if 1, then do not use any rc file */
|
|
int loaded; /* 1 => already loaded */
|
|
NClist* entries; /* the rc file entry store fields*/
|
|
char* rcfile; /* specified rcfile; overrides anything else */
|
|
char* rchome; /* Overrides $HOME when looking for .rc files */
|
|
NClist* s3profiles; /* NClist<struct AWSprofile*> */
|
|
} NCRCinfo;
|
|
|
|
#if defined(__cplusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
/* From drc.c */
|
|
EXTERNL void ncrc_initialize(void);
|
|
EXTERNL int NC_rcfile_insert(const char* key, const char* hostport, const char* path, const char* value);
|
|
EXTERNL char* NC_rclookup(const char* key, const char* hostport, const char* path);
|
|
EXTERNL char* NC_rclookupx(NCURI* uri, const char* key);
|
|
|
|
/* Following are primarily for debugging */
|
|
/* Obtain the count of number of entries */
|
|
EXTERNL size_t NC_rcfile_length(NCRCinfo*);
|
|
/* Obtain the ith entry; return NULL if out of range */
|
|
EXTERNL NCRCentry* NC_rcfile_ith(NCRCinfo*,size_t);
|
|
|
|
/* For internal use */
|
|
EXTERNL void NC_rcclear(NCRCinfo* info);
|
|
EXTERNL void NC_rcclear(NCRCinfo* info);
|
|
|
|
/* From dutil.c (Might later move to e.g. nc.h */
|
|
EXTERNL int NC__testurl(const char* path, char** basenamep);
|
|
EXTERNL int NC_isLittleEndian(void);
|
|
EXTERNL char* NC_entityescape(const char* s);
|
|
EXTERNL int NC_readfile(const char* filename, NCbytes* content);
|
|
EXTERNL int NC_readfilen(const char* filename, NCbytes* content, long long len);
|
|
EXTERNL int NC_readfileF(FILE* fp, NCbytes* content, long long len);
|
|
EXTERNL int NC_writefile(const char* filename, size_t size, void* content);
|
|
EXTERNL char* NC_mktmp(const char* base);
|
|
EXTERNL int NC_getmodelist(const char* modestr, NClist** modelistp);
|
|
EXTERNL int NC_testmode(NCURI* uri, const char* tag);
|
|
EXTERNL int NC_testpathmode(const char* path, const char* tag);
|
|
EXTERNL int NC_split_delim(const char* path, char delim, NClist* segments);
|
|
EXTERNL int NC_join(struct NClist* segments, char** pathp);
|
|
|
|
/* From ds3util.c */
|
|
/* S3 profiles */
|
|
EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep);
|
|
EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep);
|
|
EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep);
|
|
EXTERNL int NC_iss3(NCURI* uri);
|
|
EXTERNL int NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** newurlp);
|
|
|
|
#if defined(__cplusplus)
|
|
}
|
|
#endif
|
|
|
|
#endif /*NCRC_H*/
|