netcdf-c/include/ncrc.h
Dennis Heimbigner 49737888ca Improve S3 Documentation and Support
## Improvements to S3 Documentation
* Create a new document *quickstart_paths.md* that give a summary of the legal path formats used by netcdf-c. This includes both file paths and URL paths.
* Modify *nczarr.md* to remove most of the S3 related text.
* Move the S3 text from *nczarr.md* to a new document *cloud.md*.
* Add some S3-related text to the *byterange.md* document.

Hopefully, this will make it easier for users to find the information they want.

## Rebuild NCZarr Testing
In order to avoid problems with running make check in parallel, two changes were made:
1. The *nczarr_test* test system was rebuilt. Now, for each test.
any generated files are kept in a test-specific directory, isolated
from all other test executions.
2. Similarly, since the S3 test bucket is shared, any generated S3 objects
are isolated using a test-specific key path.

## Other S3 Related Changes
* Add code to ensure that files created on S3 are reclaimed at end of testing.
* Used the bash "trap" command to ensure S3 cleanup even if the test fails.
* Cleanup the S3 related configure.ac flag set since S3 is used in several places. So now one should use the option *--enable-s3* instead of *--enable-nczarr-s3*, although the latter is still kept as a deprecated alias for the former.
* Get some of the github actions yml to work with S3; required fixing various test scripts adding a secret to access the Unidata S3 bucket.
* Cleanup S3 portion of libnetcdf.settings.in and netcdf_meta.h.in and test_common.in.
* Merge partial S3 support into dhttp.c.
* Create an experimental s3 access library especially for use with Windows. It is enabled by using the options *--enable-s3-internal* (automake) or *-DENABLE_S3_INTERNAL=ON* (CMake). Also add a unit-test for it.
* Move some definitions from ncrc.h to ncs3sdk.h

## Other Changes
* Provide a default implementation of strlcpy and move this and similar defaults into *dmissing.c*.
2023-04-25 17:15:06 -06:00

104 lines
3.3 KiB
C

/*
Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT for license information.
*/
/*
Common functionality for reading
and accessing rc files (e.g. .daprc).
*/
#ifndef NCRC_H
#define NCRC_H
/* Need these support includes */
#include "ncuri.h"
#include "nclist.h"
#include "ncbytes.h"
/* getenv() keys */
#define NCRCENVIGNORE "NCRCENV_IGNORE"
#define NCRCENVRC "NCRCENV_RC"
#define NCRCENVHOME "NCRCENV_HOME"
/* Known .aws profile keys */
#define AWS_ACCESS_KEY_ID "aws_access_key_id"
#define AWS_SECRET_ACCESS_KEY "aws_secret_access_key"
#define AWS_REGION "aws_region"
typedef struct NCRCentry {
char* host; /* combined host:port */
char* urlpath; /* prefix to match or NULL */
char* key;
char* value;
} NCRCentry;
struct AWSentry {
char* key;
char* value;
};
struct AWSprofile {
char* name;
NClist* entries; /* NClist<struct AWSentry*> */
};
/* collect all the relevant info around the rc file and AWS */
typedef struct NCRCinfo {
int ignore; /* if 1, then do not use any rc file */
int loaded; /* 1 => already loaded */
NClist* entries; /* the rc file entry store fields*/
char* rcfile; /* specified rcfile; overrides anything else */
char* rchome; /* Overrides $HOME when looking for .rc files */
NClist* s3profiles; /* NClist<struct AWSprofile*> */
} NCRCinfo;
#if defined(__cplusplus)
extern "C" {
#endif
/* From drc.c */
EXTERNL void ncrc_initialize(void);
EXTERNL int NC_rcfile_insert(const char* key, const char* hostport, const char* path, const char* value);
EXTERNL char* NC_rclookup(const char* key, const char* hostport, const char* path);
EXTERNL char* NC_rclookupx(NCURI* uri, const char* key);
/* Following are primarily for debugging */
/* Obtain the count of number of entries */
EXTERNL size_t NC_rcfile_length(NCRCinfo*);
/* Obtain the ith entry; return NULL if out of range */
EXTERNL NCRCentry* NC_rcfile_ith(NCRCinfo*,size_t);
/* For internal use */
EXTERNL void NC_rcclear(NCRCinfo* info);
EXTERNL void NC_rcclear(NCRCinfo* info);
/* From dutil.c (Might later move to e.g. nc.h */
EXTERNL int NC__testurl(const char* path, char** basenamep);
EXTERNL int NC_isLittleEndian(void);
EXTERNL char* NC_entityescape(const char* s);
EXTERNL int NC_readfile(const char* filename, NCbytes* content);
EXTERNL int NC_readfilen(const char* filename, NCbytes* content, long long len);
EXTERNL int NC_readfileF(FILE* fp, NCbytes* content, long long len);
EXTERNL int NC_writefile(const char* filename, size_t size, void* content);
EXTERNL char* NC_mktmp(const char* base);
EXTERNL int NC_getmodelist(const char* modestr, NClist** modelistp);
EXTERNL int NC_testmode(NCURI* uri, const char* tag);
EXTERNL int NC_testpathmode(const char* path, const char* tag);
EXTERNL int NC_split_delim(const char* path, char delim, NClist* segments);
EXTERNL int NC_join(struct NClist* segments, char** pathp);
/* From ds3util.c */
/* S3 profiles */
EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep);
EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep);
EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep);
EXTERNL int NC_iss3(NCURI* uri);
EXTERNL int NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** newurlp);
#if defined(__cplusplus)
}
#endif
#endif /*NCRC_H*/