mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-01-18 15:55:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Properly handle missing regions in URLS

Browse Source 
NOTE: it is important that this fix gets into 4.9.3

re: Issue https://github.com/Unidata/netcdf-c/issues/2798

## Modifications
* This PR includes PR https://github.com/Unidata/netcdf-c/pull/2813
* Support the following AWS environment variables in the internal S3 library
  (they are already supported by aws-sdk-cpp).
  - AWS_REGION
  - AWS_DEFAULT_REGION
  - AWS_ACCESS_KEY_ID
  - AWS_CONFIG_FILE
  - AWS_PROFILE
  - AWS_SECRET_ACCESS_KEY
  - (source https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html).
* Support an empty region when specifying s3.amazonaws.com as the host.
* Move some S3/AWS related functions to ds3util.c
* Add a test case to test empty region and AWS_[DEFAULT]_REGION.
This commit is contained in:
Dennis Heimbigner 2023-12-02 21:03:59 -07:00
parent 0c6fd78251
commit 27f615bebc
31 changed files with 809 additions and 631 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/nc4internal.h
View File

@ -332,6 +332,13 @@ typedef struct NCglobalstate {
struct GlobalZarr { /* Zarr specific parameters */
char dimension_separator;
} zarr;
struct GlobalAWS { /* AWS S3 specific parameters/defaults */
char* default_region;
char* config_file;
char* profile;
char* access_key_id;
char* secret_access_key;
} aws;
struct Alignment { /* H5Pset_alignment parameters */
int defined; /* 1 => threshold and alignment explicitly set */
int threshold;

3
include/ncpathmgr.h
View File

@ -14,6 +14,9 @@
#ifdef HAVE_DIRENT_H
#include <dirent.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif

20
include/ncrc.h
View File

@ -33,16 +33,6 @@ typedef struct NCRCentry {
char* value;
} NCRCentry;
struct AWSentry {
char* key;
char* value;
};
struct AWSprofile {
char* name;
NClist* entries; /* NClist<struct AWSentry*> */
};
/* collect all the relevant info around the rc file and AWS */
typedef struct NCRCinfo {
int ignore; /* if 1, then do not use any rc file */
@ -88,16 +78,10 @@ EXTERNL char* NC_mktmp(const char* base);
EXTERNL int NC_getmodelist(const char* modestr, NClist** modelistp);
EXTERNL int NC_testmode(NCURI* uri, const char* tag);
EXTERNL int NC_testpathmode(const char* path, const char* tag);
EXTERNL int NC_addmodetag(NCURI* uri, const char* tag);
EXTERNL int NC_split_delim(const char* path, char delim, NClist* segments);
EXTERNL int NC_join(struct NClist* segments, char** pathp);
/* From ds3util.c */
/* S3 profiles */
EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep);
EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep);
EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep);
EXTERNL int NC_iss3(NCURI* uri);
EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp);
EXTERNL int NC_joinwith(NClist* segments, const char* sep, const char* prefix, const char* suffix, char** pathp);
#if defined(__cplusplus)
}

32
include/ncs3sdk.h
View File

@ -6,6 +6,12 @@
#ifndef NCS3SDK_H
#define NCS3SDK_H 1
#define AWSHOST ".amazonaws.com"
#define GOOGLEHOST "storage.googleapis.com"
/* Define the "global" default region to be used if no other region is specified */
#define AWS_GLOBAL_DEFAULT_REGION "us-east-1"
/* Track the server type, if known */
typedef enum NCS3SVC {NCS3UNK=0, /* unknown */
NCS3=1, /* s3.amazon.aws */
@ -21,6 +27,20 @@ typedef struct NCS3INFO {
NCS3SVC svc;
} NCS3INFO;
struct AWSentry {
char* key;
char* value;
};
struct AWSprofile {
char* name;
struct NClist* entries; /* NClist<struct AWSentry*> */
};
/* Opaque Types */
struct NClist;
struct NCglobalstate;
#ifdef __cplusplus
extern "C" {
#endif
@ -38,13 +58,23 @@ EXTERNL int NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char**
EXTERNL int NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp);
EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp);
EXTERNL int NC_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp);
EXTERNL const char* NC_s3dumps3info(NCS3INFO* info);
/* From ds3util.c */
EXTERNL int NC_s3sdkinitialize(void);
EXTERNL int NC_s3sdkfinalize(void);
EXTERNL int NC_getdefaults3region(NCURI* uri, const char** regionp);
EXTERNL int NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp);
EXTERNL int NC_s3clear(NCS3INFO* s3);
EXTERNL int NC_s3clone(NCS3INFO* s3, NCS3INFO** news3p);
EXTERNL const char* NC_s3dumps3info(NCS3INFO* info);
EXTERNL void NC_s3freeprofilelist(struct NClist* profiles);
EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep);
EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep);
EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep);
EXTERNL int NC_iss3(NCURI* uri, enum NCS3SVC*);
EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp);
EXTERNL int NC_aws_load_credentials(struct NCglobalstate* gstate);
#ifdef __cplusplus
}

3
libdispatch/dauth.c
View File

@ -17,9 +17,10 @@ See COPYRIGHT for license information.
#include "netcdf.h"
#include "ncbytes.h"
#include "ncuri.h"
#include "ncauth.h"
#include "nclog.h"
#include "ncpathmgr.h"
#include "ncs3sdk.h"
#include "ncauth.h"
#ifdef _MSC_VER
#include <windows.h>

4
libdispatch/dcopy.c
View File

@ -694,7 +694,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2)
int gid;
uintptr_t id;
id = grp;
id = (uintptr_t)grp;
nclistpush(queue,(void*)id); /* prime the queue */
while(nclistlength(queue) > 0) {
id = (uintptr_t)nclistremove(queue,0);
@ -712,7 +712,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2)
goto done;
/* push onto the end of the queue */
for(i=0;i<nids;i++) {
id = ids[i];
id = (uintptr_t)ids[i];
nclistpush(queue,(void*)id);
}
free(ids); ids = NULL;

2
libdispatch/ddispatch.c
View File

@ -134,12 +134,12 @@ int
NCDISPATCH_finalize(void)
{
int status = NC_NOERR;
NC_freeglobalstate();
#if defined(ENABLE_BYTERANGE) || defined(ENABLE_DAP) || defined(ENABLE_DAP4)
curl_global_cleanup();
#endif
#if defined(ENABLE_DAP4)
ncxml_finalize();
#endif
NC_freeglobalstate(); /* should be one of the last things done */
return status;
}

4
libdispatch/derror.c
View File

@ -267,9 +267,9 @@ const char *nc_strerror(int ncerr1)
case NC_EMPI: return "NetCDF: MPI operation failed.";
case NC_ERCFILE:
return "NetCDF: RC File Failure.";
case NC_ENULLPAD:
case NC_ENULLPAD:
return "NetCDF: File fails strict Null-Byte Header check.";
case NC_EINMEMORY:
case NC_EINMEMORY:
return "NetCDF: In-memory File operation failed.";
case NC_ENCZARR:
return "NetCDF: NCZarr error";

6
libdispatch/dhttp.c
View File

@ -99,7 +99,11 @@ nc_http_open_verbose(const char* path, int verbose, NC_HTTP_STATE** statep)
{stat = NCTHROW(NC_ENOMEM); goto done;}
state->path = strdup(path);
state->url = uri; uri = NULL;
state->format = (NC_iss3(state->url)?HTTPS3:HTTPCURL);
#ifdef ENABLE_S3
state->format = (NC_iss3(state->url,NULL)?HTTPS3:HTTPCURL);
#else
state->format = HTTPCURL;
#endif
switch (state->format) {
case HTTPCURL: {

7
libdispatch/dinfermodel.c
View File

@ -898,13 +898,16 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
ncurisetfragments(uri,sfrag);
nullfree(sfrag); sfrag = NULL;
#ifdef ENABLE_S3
/* If s3, then rebuild the url */
if(NC_iss3(uri)) {
if(NC_iss3(uri,NULL)) {
NCURI* newuri = NULL;
if((stat = NC_s3urlrebuild(uri,NULL,&newuri))) goto done;
ncurifree(uri);
uri = newuri;
} else if(strcmp(uri->protocol,"file")==0) {
} else
#endif
if(strcmp(uri->protocol,"file")==0) {
/* convert path to absolute */
char* canon = NULL;
abspath = NCpathabsolute(uri->path);

511
libdispatch/drc.c
View File

@ -19,11 +19,11 @@ See COPYRIGHT for license information.
#include "ncbytes.h"
#include "ncuri.h"
#include "ncrc.h"
#include "ncs3sdk.h"
#include "nclog.h"
#include "ncauth.h"
#include "ncpathmgr.h"
#include "nc4internal.h"
#include "ncs3sdk.h"
#include "ncdispatch.h"
#undef NOREAD
@ -31,7 +31,6 @@ See COPYRIGHT for license information.
#undef DRCDEBUG
#undef LEXDEBUG
#undef PARSEDEBUG
#undef AWSDEBUG
#define RTAG ']'
#define LTAG '['
@ -39,9 +38,6 @@ See COPYRIGHT for license information.
#undef MEMCHECK
#define MEMCHECK(x) if((x)==NULL) {goto nomem;} else {}
/* Alternate .aws directory location */
#define NC_TEST_AWS_DIR "NC_TEST_AWS_DIR"
/* Forward */
static int NC_rcload(void);
static char* rcreadline(char** nextlinep);
@ -57,16 +53,10 @@ static void rcfreeentry(NCRCentry* t);
#ifdef DRCDEBUG
static void storedump(char* msg, NClist* entrys);
#endif
static int aws_load_credentials(NCglobalstate*);
static void freeprofile(struct AWSprofile* profile);
static void freeprofilelist(NClist* profiles);
/* Define default rc files and aliases, also defines load order*/
static const char* rcfilenames[] = {".ncrc", ".daprc", ".dodsrc", NULL};
/* Read these files in order and later overriding earlier */
static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL};
static int NCRCinitialized = 0;
/**************************************************/
@ -158,7 +148,7 @@ ncrc_initialize(void)
nclog(NCLOGWARN,".rc loading failed");
}
/* Load .aws/config &/ credentials */
if((stat = aws_load_credentials(ncg))) {
if((stat = NC_aws_load_credentials(ncg))) {
nclog(NCLOGWARN,"AWS config file not loaded");
}
#endif
@ -187,7 +177,7 @@ NC_rcclear(NCRCinfo* info)
nullfree(info->rcfile);
nullfree(info->rchome);
rcfreeentries(info->entries);
freeprofilelist(info->s3profiles);
NC_s3freeprofilelist(info->s3profiles);
}
@ -483,7 +473,7 @@ rccompile(const char* filepath)
nclog(NCLOGERR, "Malformed [url] in %s entry: %s",filepath,line);
continue;
}
if(NC_iss3(uri)) {
if(NC_iss3(uri,NULL)) {
NCURI* newuri = NULL;
/* Rebuild the url to S3 "path" format */
NC_s3clear(&s3);
@ -755,496 +745,3 @@ storedump(char* msg, NClist* entries)
}
#endif
/**************************************************/
/*
Get the current active profile. The priority order is as follows:
1. aws.profile key in mode flags
2. aws.profile in .rc entries
4. "default"
5. "no" -- meaning do not use any profile => no secret key
@param uri uri with mode flags, may be NULL
@param profilep return profile name here or NULL if none found
@return NC_NOERR if no error.
@return NC_EINVAL if something else went wrong.
*/
int
NC_getactives3profile(NCURI* uri, const char** profilep)
{
int stat = NC_NOERR;
const char* profile = NULL;
struct AWSprofile* ap = NULL;
profile = ncurifragmentlookup(uri,"aws.profile");
if(profile == NULL)
profile = NC_rclookupx(uri,"AWS.PROFILE");
if(profile == NULL) {
if((stat=NC_authgets3profile("default",&ap))) goto done;
if(ap) profile = "default";
}
if(profile == NULL) {
if((stat=NC_authgets3profile("no",&ap))) goto done;
if(ap) profile = "no";
}
#ifdef AWSDEBUG
fprintf(stderr,">>> activeprofile = %s\n",(profile?profile:"null"));
#endif
if(profilep) *profilep = profile;
done:
return stat;
}
/*
Get the current default region. The search order is as follows:
1. aws.region key in mode flags
2. aws.region in .rc entries
3. aws_region key in current profile (only if profiles are being used)
4. "us-east-1"
@param uri uri with mode flags, may be NULL
@param regionp return region name here or NULL if none found
@return NC_NOERR if no error.
@return NC_EINVAL if something else went wrong.
*/
int
NC_getdefaults3region(NCURI* uri, const char** regionp)
{
int stat = NC_NOERR;
const char* region = NULL;
const char* profile = NULL;
region = ncurifragmentlookup(uri,"aws.region");
if(region == NULL)
region = NC_rclookupx(uri,"AWS.REGION");
if(region == NULL) {/* See if we can find a profile */
if(NC_getactives3profile(uri,&profile)==NC_NOERR) {
if(profile)
(void)NC_s3profilelookup(profile,"aws_region",&region);
}
}
if(region == NULL)
region = "us-east-1";
#ifdef AWSDEBUG
fprintf(stderr,">>> activeregion = %s\n",(region?region:"null"));
#endif
if(regionp) *regionp = region;
return stat;
}
/**
The .aws/config and .aws/credentials files
are in INI format (https://en.wikipedia.org/wiki/INI_file).
This format is not well defined, so the grammar used
here is restrictive. Here, the term "profile" is the same
as the INI term "section".
The grammar used is as follows:
Grammar:
inifile: profilelist ;
profilelist: profile | profilelist profile ;
profile: '[' profilename ']' EOL entries ;
entries: empty | entries entry ;
entry: WORD = WORD EOL ;
profilename: WORD ;
Lexical:
WORD sequence of printable characters - [ \[\]=]+
EOL '\n' | ';'
Note:
1. The semicolon at beginning of a line signals a comment.
2. # comments are not allowed
3. Duplicate profiles or keys are ignored.
4. Escape characters are not supported.
*/
#define AWS_EOF (-1)
#define AWS_ERR (0)
#define AWS_WORD (0x10001)
#define AWS_EOL (0x10002)
#ifdef LEXDEBUG
static const char*
tokenname(int token)
{
static char num[32];
switch(token) {
case AWS_EOF: return "EOF";
case AWS_ERR: return "ERR";
case AWS_WORD: return "WORD";
default: snprintf(num,sizeof(num),"%d",token); return num;
}
return "UNKNOWN";
}
#endif
typedef struct AWSparser {
char* text;
char* pos;
size_t yylen; /* |yytext| */
NCbytes* yytext;
int token; /* last token found */
int pushback; /* allow 1-token pushback */
} AWSparser;
static int
awslex(AWSparser* parser)
{
int c;
int token = 0;
char* start;
size_t count;
parser->token = AWS_ERR;
ncbytesclear(parser->yytext);
ncbytesnull(parser->yytext);
if(parser->pushback != AWS_ERR) {
token = parser->pushback;
parser->pushback = AWS_ERR;
goto done;
}
while(token == 0) { /* avoid need to goto when retrying */
c = *parser->pos;
if(c == '\0') {
token = AWS_EOF;
} else if(c == '\n') {
parser->pos++;
token = AWS_EOL;
} else if(c <= ' ' || c == '\177') {
parser->pos++;
continue; /* ignore whitespace */
} else if(c == ';') {
char* p = parser->pos - 1;
if(*p == '\n') {
/* Skip comment */
do {p++;} while(*p != '\n' && *p != '\0');
parser->pos = p;
token = (*p == '\n'?AWS_EOL:AWS_EOF);
} else {
token = ';';
ncbytesappend(parser->yytext,';');
parser->pos++;
}
} else if(c == '[' || c == ']' || c == '=') {
ncbytesappend(parser->yytext,c);
ncbytesnull(parser->yytext);
token = c;
parser->pos++;
} else { /*Assume a word*/
start = parser->pos;
for(;;) {
c = *parser->pos++;
if(c <= ' ' || c == '\177' || c == '[' || c == ']' || c == '=') break; /* end of word */
}
/* Pushback last char */
parser->pos--;
count = ((parser->pos) - start);
ncbytesappendn(parser->yytext,start,count);
ncbytesnull(parser->yytext);
token = AWS_WORD;
}
#ifdef LEXDEBUG
fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,ncbytescontents(parser->yytext));
#endif
} /*for(;;)*/
done:
parser->token = token;
return token;
}
/*
@param text of the aws credentials file
@param profiles list of form struct AWSprofile (see ncauth.h)
*/
#define LBR '['
#define RBR ']'
static int
awsparse(const char* text, NClist* profiles)
{
int i,stat = NC_NOERR;
size_t len;
AWSparser* parser = NULL;
struct AWSprofile* profile = NULL;
int token;
char* key = NULL;
char* value = NULL;
if(text == NULL) text = "";
parser = calloc(1,sizeof(AWSparser));
if(parser == NULL)
{stat = (NC_ENOMEM); goto done;}
len = strlen(text);
parser->text = (char*)malloc(len+1+1+1); /* double nul term plus leading EOL */
if(parser->text == NULL)
{stat = (NCTHROW(NC_EINVAL)); goto done;}
parser->pos = parser->text;
parser->pos[0] = '\n'; /* So we can test for comment unconditionally */
parser->pos++;
strcpy(parser->text+1,text);
parser->pos += len;
/* Double nul terminate */
parser->pos[0] = '\0';
parser->pos[1] = '\0';
parser->pos = &parser->text[0]; /* reset */
parser->yytext = ncbytesnew();
parser->pushback = AWS_ERR;
/* Do not need recursion, use simple loops */
for(;;) {
token = awslex(parser); /* make token always be defined */
if(token == AWS_EOF) break; /* finished */
if(token == AWS_EOL) {continue;} /* blank line */
if(token != LBR) {stat = NCTHROW(NC_EINVAL); goto done;}
/* parse [profile name] */
token = awslex(parser);
if(token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;}
assert(profile == NULL);
if((profile = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)))==NULL)
{stat = NC_ENOMEM; goto done;}
profile->name = ncbytesextract(parser->yytext);
profile->entries = nclistnew();
token = awslex(parser);
if(token != RBR) {stat = NCTHROW(NC_EINVAL); goto done;}
#ifdef PARSEDEBUG
fprintf(stderr,">>> parse: profile=%s\n",profile->name);
#endif
/* The fields can be in any order */
for(;;) {
struct AWSentry* entry = NULL;
token = awslex(parser);
if(token == AWS_EOL) {
continue; /* ignore empty lines */
} else if(token == AWS_EOF) {
break;
} else if(token == LBR) {/* start of next profile */
parser->pushback = token;
break;
} else if(token == AWS_WORD) {
key = ncbytesextract(parser->yytext);
token = awslex(parser);
if(token != '=') {stat = NCTHROW(NC_EINVAL); goto done;}
token = awslex(parser);
if(token != AWS_EOL && token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;}
value = ncbytesextract(parser->yytext);
if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL)
{stat = NC_ENOMEM; goto done;}
entry->key = key; key = NULL;
entry->value = value; value = NULL;
#ifdef PARSEDEBUG
fprintf(stderr,">>> parse: entry=(%s,%s)\n",entry->key,entry->value);
#endif
nclistpush(profile->entries,entry); entry = NULL;
if(token == AWS_WORD) token = awslex(parser); /* finish the line */
} else
{stat = NCTHROW(NC_EINVAL); goto done;}
}
/* If this profile already exists, then replace old one */
for(i=0;i<nclistlength(profiles);i++) {
struct AWSprofile* p = (struct AWSprofile*)nclistget(profiles,i);
if(strcasecmp(p->name,profile->name)==0) {
nclistset(profiles,i,profile);
profile = NULL;
/* reclaim old one */
freeprofile(p);
break;
}
}
if(profile) nclistpush(profiles,profile);
profile = NULL;
}
done:
if(profile) freeprofile(profile);
nullfree(key);
nullfree(value);
if(parser != NULL) {
nullfree(parser->text);
ncbytesfree(parser->yytext);
free(parser);
}
return (stat);
}
static void
freeentry(struct AWSentry* e)
{
if(e) {
#ifdef AWSDEBUG
fprintf(stderr,">>> freeentry: key=%p value=%p\n",e->key,e->value);
#endif
nullfree(e->key);
nullfree(e->value);
nullfree(e);
}
}
static void
freeprofile(struct AWSprofile* profile)
{
if(profile) {
int i;
#ifdef AWSDEBUG
fprintf(stderr,">>> freeprofile: %s\n",profile->name);
#endif
for(i=0;i<nclistlength(profile->entries);i++) {
struct AWSentry* e = (struct AWSentry*)nclistget(profile->entries,i);
freeentry(e);
}
nclistfree(profile->entries);
nullfree(profile->name);
nullfree(profile);
}
}
static void
freeprofilelist(NClist* profiles)
{
if(profiles) {
int i;
for(i=0;i<nclistlength(profiles);i++) {
struct AWSprofile* p = (struct AWSprofile*)nclistget(profiles,i);
freeprofile(p);
}
nclistfree(profiles);
}
}
/* Find, load, and parse the aws config &/or credentials file */
static int
aws_load_credentials(NCglobalstate* gstate)
{
int stat = NC_NOERR;
NClist* profiles = nclistnew();
const char** awscfg = awsconfigfiles;
const char* aws_root = getenv(NC_TEST_AWS_DIR);
NCbytes* buf = ncbytesnew();
char path[8192];
/* add a "no" credentials */
{
struct AWSprofile* noprof = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile));
noprof->name = strdup("no");
noprof->entries = nclistnew();
nclistpush(profiles,noprof); noprof = NULL;
}
for(;*awscfg;awscfg++) {
/* Construct the path ${HOME}/<file> or Windows equivalent. */
const char* cfg = *awscfg;
snprintf(path,sizeof(path),"%s%s%s",
(aws_root?aws_root:gstate->home),
(*cfg == '/'?"":"/"),
cfg);
ncbytesclear(buf);
if((stat=NC_readfile(path,buf))) {
nclog(NCLOGWARN, "Could not open file: %s",path);
} else {
/* Parse the credentials file */
const char* text = ncbytescontents(buf);
if((stat = awsparse(text,profiles))) goto done;
}
}
/* add a "none" credentials */
{
struct AWSprofile* noprof = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile));
if(noprof == NULL) {stat = NC_ENOMEM; goto done;}
noprof->name = strdup("none");
noprof->entries = nclistnew();
nclistpush(profiles,noprof); noprof = NULL;
}
if(gstate->rcinfo->s3profiles)
freeprofilelist(gstate->rcinfo->s3profiles);
gstate->rcinfo->s3profiles = profiles; profiles = NULL;
#ifdef AWSDEBUG
{int i,j;
fprintf(stderr,">>> profiles:\n");
for(i=0;i<nclistlength(gstate->rcinfo->s3profiles);i++) {
struct AWSprofile* p = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i);
fprintf(stderr," [%s]",p->name);
for(j=0;j<nclistlength(p->entries);j++) {
struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j);
if(strcmp(e->key,"aws_access_key_id")
fprintf(stderr," %s=%d",e->key,(int)strlen(e->value));
else if(strcmp(e->key,"aws_secret_access_key")
fprintf(stderr," %s=%d",e->key,(int)strlen(e->value));
else fprintf(stderr," %s=%s",e->key,e->value);
}
fprintf(stderr,"\n");
}
}
#endif
done:
ncbytesfree(buf);
freeprofilelist(profiles);
return stat;
}
/* Lookup a profile by name;
@param profilename to lookup
@param profilep return the matching profile; null if profile not found
@return NC_NOERR if no error
@return other error
*/
int
NC_authgets3profile(const char* profilename, struct AWSprofile** profilep)
{
int stat = NC_NOERR;
int i = -1;
NCglobalstate* gstate = NC_getglobalstate();
for(i=0;i<nclistlength(gstate->rcinfo->s3profiles);i++) {
struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i);
if(strcmp(profilename,profile->name)==0)
{if(profilep) {*profilep = profile; goto done;}}
}
if(profilep) *profilep = NULL; /* not found */
done:
return stat;
}
/**
@param profile name of profile
@param key key to search for in profile
@param value place to store the value if key is found; NULL if not found
@return NC_NOERR if key is found, Some other error otherwise.
*/
int
NC_s3profilelookup(const char* profile, const char* key, const char** valuep)
{
int i,stat = NC_NOERR;
struct AWSprofile* awsprof = NULL;
const char* value = NULL;
if(profile == NULL) return NC_ES3;
if((stat=NC_authgets3profile(profile,&awsprof))==NC_NOERR && awsprof != NULL) {
for(i=0;i<nclistlength(awsprof->entries);i++) {
struct AWSentry* entry = (struct AWSentry*)nclistget(awsprof->entries,i);
if(strcasecmp(entry->key,key)==0) {
value = entry->value;
break;
}
}
}
if(valuep) *valuep = value;
return stat;
}

620
libdispatch/ds3util.c
View File

@ -22,20 +22,73 @@
#endif
#include "netcdf.h"
#include "nc4internal.h"
#include "ncuri.h"
#include "nclist.h"
#include "ncrc.h"
#include "nclog.h"
#include "ncs3sdk.h"
#undef AWSDEBUG
/* Alternate .aws directory location */
#define NC_TEST_AWS_DIR "NC_TEST_AWS_DIR"
#define AWSHOST ".amazonaws.com"
#define GOOGLEHOST "storage.googleapis.com"
enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4};
/* Read these files in order and later overriding earlier */
static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL};
#define NCONFIGFILES (sizeof(awsconfigfiles)/sizeof(char*))
static int ncs3_initialized = 0;
static int ncs3_finalized = 0;
/**************************************************/
/* Forward */
static int endswith(const char* s, const char* suffix);
static void freeentry(struct AWSentry* e);
static int awsparse(const char* text, NClist* profiles);
/**************************************************/
/* Capture environmental Info */
EXTERNL int
NC_s3sdkinitialize(void)
{
if(!ncs3_initialized) {
ncs3_initialized = 1;
ncs3_finalized = 0;
}
{
/* Get various environment variables as defined by the AWS sdk */
NCglobalstate* gs = NC_getglobalstate();
if(getenv("AWS_REGION")!=NULL)
gs->aws.default_region = nulldup(getenv("AWS_REGION"));
else if(getenv("AWS_DEFAULT_REGION")!=NULL)
gs->aws.default_region = nulldup(getenv("AWS_DEFAULT_REGION"));
else if(gs->aws.default_region == NULL)
gs->aws.default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION);
gs->aws.access_key_id = nulldup(getenv("AWS_ACCESS_KEY_ID"));
gs->aws.config_file = nulldup(getenv("AWS_CONFIG_FILE"));
gs->aws.profile = nulldup(getenv("AWS_PROFILE"));
gs->aws.secret_access_key = nulldup(getenv("AWS_SECRET_ACCESS_KEY"));
}
return NC_NOERR;
}
EXTERNL int
NC_s3sdkfinalize(void)
{
if(!ncs3_finalized) {
ncs3_initialized = 0;
ncs3_finalized = 1;
}
return NC_NOERR;
}
/**************************************************/
/* Generic S3 Utilities */
@ -43,7 +96,7 @@ static int endswith(const char* s, const char* suffix);
/*
Rebuild an S3 url into a canonical path-style url.
If region is not in the host, then use specified region
if provided, otherwise us-east-1.
if provided, otherwise leave blank and let the S3 server deal with it.
@param url (in) the current url
@param s3 (in/out) NCS3INFO structure
@param pathurlp (out) the resulting pathified url string
@ -78,7 +131,7 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp)
/* Distinguish path-style from virtual-host style from s3: and from other.
Virtual: https://<bucket-name>.s3.<region>.amazonaws.com/<path> (1)
or: https://<bucket-name>.s3.amazonaws.com/<path> -- region defaults to us-east-1 (2)
or: https://<bucket-name>.s3.amazonaws.com/<path> -- region defaults (to us-east-1) (2)
Path: https://s3.<region>.amazonaws.com/<bucket-name>/<path> (3)
or: https://s3.amazonaws.com/<bucket-name>/<path> -- region defaults to us-east-1 (4)
S3: s3://<bucket-name>/<path> (5)
@ -148,7 +201,7 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp)
const char* region0 = NULL;
/* Get default region */
if((stat = NC_getdefaults3region(url,&region0))) goto done;
region = strdup(region0);
region = nulldup(region0);
}
if(region == NULL) {stat = NC_ES3; goto done;}
@ -163,8 +216,10 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp)
if(svc == NCS3) {
/* Construct the revised host */
ncbytesclear(buf);
ncbytescat(buf,"s3.");
ncbytescat(buf,region);
ncbytescat(buf,"s3");
assert(region != NULL);
ncbytescat(buf,".");
ncbytescat(buf,region);
ncbytescat(buf,AWSHOST);
nullfree(host);
host = ncbytesextract(buf);
@ -186,18 +241,24 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp)
}
path = ncbytesextract(buf);
/* complete the new url */
/* clone the url so we can modify it*/
if((newurl=ncuriclone(url))==NULL) {stat = NC_ENOMEM; goto done;}
/* Modify the URL to canonical form */
ncurisetprotocol(newurl,"https");
assert(host != NULL);
ncurisethost(newurl,host);
assert(path != NULL);
ncurisetpath(newurl,path);
/* Add "s3" to the mode list */
NC_addmodetag(newurl,"s3");
/* Rebuild the url->url */
ncurirebuild(newurl);
/* return various items */
#ifdef AWSDEBUG
fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=%s region=%s\n",uri->uri,bucket,region);
fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=|%s| region=|%s|\n",uri->uri,bucket,region);
#endif
if(newurlp) {*newurlp = newurl; newurl = NULL;}
if(s3 != NULL) {
@ -303,22 +364,24 @@ Check if a url has indicators that signal an S3 or Google S3 url.
*/
int
NC_iss3(NCURI* uri)
NC_iss3(NCURI* uri, enum NCS3SVC* svcp)
{
int iss3 = 0;
NCS3SVC svc = NCS3UNK;
if(uri == NULL) goto done; /* not a uri */
/* is the protocol "s3" or "gs3" ? */
if(strcasecmp(uri->protocol,"s3")==0) {iss3 = 1; goto done;}
if(strcasecmp(uri->protocol,"gs3")==0) {iss3 = 1; goto done;}
if(strcasecmp(uri->protocol,"s3")==0) {iss3 = 1; svc = NCS3; goto done;}
if(strcasecmp(uri->protocol,"gs3")==0) {iss3 = 1; svc = NCS3GS; goto done;}
/* Is "s3" or "gs3" in the mode list? */
if(NC_testmode(uri,"s3")) {iss3 = 1; goto done;}
if(NC_testmode(uri,"gs3")) {iss3 = 1; goto done;}
if(NC_testmode(uri,"s3")) {iss3 = 1; svc = NCS3; goto done;}
if(NC_testmode(uri,"gs3")) {iss3 = 1; svc = NCS3GS; goto done;}
/* Last chance; see if host looks s3'y */
if(uri->host != NULL) {
if(endswith(uri->host,AWSHOST)) {iss3 = 1; goto done;}
if(strcasecmp(uri->host,GOOGLEHOST)==0) {iss3 = 1; goto done;}
if(endswith(uri->host,AWSHOST)) {iss3 = 1; svc = NCS3; goto done;}
if(strcasecmp(uri->host,GOOGLEHOST)==0) {iss3 = 1; svc = NCS3GS; goto done;}
}
if(svcp) *svcp = svc;
done:
return iss3;
}
@ -336,3 +399,532 @@ NC_s3dumps3info(NCS3INFO* info)
return text;
}
static void
freeprofile(struct AWSprofile* profile)
{
if(profile) {
int i;
#ifdef AWSDEBUG
fprintf(stderr,">>> freeprofile: %s\n",profile->name);
#endif
for(i=0;i<nclistlength(profile->entries);i++) {
struct AWSentry* e = (struct AWSentry*)nclistget(profile->entries,i);
freeentry(e);
}
nclistfree(profile->entries);
nullfree(profile->name);
nullfree(profile);
}
}
void
NC_s3freeprofilelist(NClist* profiles)
{
if(profiles) {
int i;
for(i=0;i<nclistlength(profiles);i++) {
struct AWSprofile* p = (struct AWSprofile*)nclistget(profiles,i);
freeprofile(p);
}
nclistfree(profiles);
}
}
/* Find, load, and parse the aws config &/or credentials file */
int
NC_aws_load_credentials(NCglobalstate* gstate)
{
int stat = NC_NOERR;
NClist* profiles = nclistnew();
NCbytes* buf = ncbytesnew();
char path[8192];
const char* aws_root = getenv(NC_TEST_AWS_DIR);
const char* awscfg_local[NCONFIGFILES + 1]; /* +1 for the env variable */
const char** awscfg = NULL;
/* add a "no" credentials */
{
struct AWSprofile* noprof = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile));
noprof->name = strdup("no");
noprof->entries = nclistnew();
nclistpush(profiles,noprof); noprof = NULL;
}
awscfg = awsconfigfiles;
if((awscfg_local[0] = NC_getglobalstate()->aws.config_file)!=NULL) {
memcpy(&awscfg_local[1],awsconfigfiles,sizeof(char*)*NCONFIGFILES);
awscfg = awscfg_local;
}
for(;*awscfg;awscfg++) {
/* Construct the path ${HOME}/<file> or Windows equivalent. */
const char* cfg = *awscfg;
snprintf(path,sizeof(path),"%s%s%s",
(aws_root?aws_root:gstate->home),
(*cfg == '/'?"":"/"),
cfg);
ncbytesclear(buf);
if((stat=NC_readfile(path,buf))) {
nclog(NCLOGWARN, "Could not open file: %s",path);
} else {
/* Parse the credentials file */
const char* text = ncbytescontents(buf);
if((stat = awsparse(text,profiles))) goto done;
}
}
/* If there is no default credentials, then try to synthesize one
from various environment variables */
{
size_t i;
struct AWSprofile* dfalt = NULL;
struct AWSentry* entry = NULL;
NCglobalstate* gs = NC_getglobalstate();
/* Verify that we can build a default */
if(gs->aws.access_key_id != NULL && gs->aws.secret_access_key != NULL) {
/* Kill off any previous default profile */
for(i=nclistlength(profiles)-1;i>=0;i--) {/* walk backward because we are removing entries */
struct AWSprofile* prof = (struct AWSprofile*)nclistget(profiles,i);
if(strcasecmp(prof->name,"default")==0) {
nclistremove(profiles,i);
freeprofile(prof);
}
}
/* Build new default profile */
if((dfalt = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)))==NULL) {stat = NC_ENOMEM; goto done;}
dfalt->name = strdup("default");
dfalt->entries = nclistnew();
/* Save the new default profile */
nclistpush(profiles,dfalt); dfalt = NULL;
/* Create the entries for default */
if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL) {stat = NC_ENOMEM; goto done;}
entry->key = strdup("aws_access_key_id");
entry->value = strdup(gs->aws.access_key_id);
nclistpush(dfalt->entries,entry); entry = NULL;
if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL) {stat = NC_ENOMEM; goto done;}
entry->key = strdup("aws_secret_access_key");
entry->value = strdup(gs->aws.secret_access_key);
nclistpush(dfalt->entries,entry); entry = NULL;
}
}
if(gstate->rcinfo->s3profiles)
NC_s3freeprofilelist(gstate->rcinfo->s3profiles);
gstate->rcinfo->s3profiles = profiles; profiles = NULL;
#ifdef AWSDEBUG
{int i,j;
fprintf(stderr,">>> profiles:\n");
for(i=0;i<nclistlength(gstate->rcinfo->s3profiles);i++) {
struct AWSprofile* p = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i);
fprintf(stderr," [%s]",p->name);
for(j=0;j<nclistlength(p->entries);j++) {
struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j);
if(strcmp(e->key,"aws_access_key_id")
fprintf(stderr," %s=%d",e->key,(int)strlen(e->value));
else if(strcmp(e->key,"aws_secret_access_key")
fprintf(stderr," %s=%d",e->key,(int)strlen(e->value));
else fprintf(stderr," %s=%s",e->key,e->value);
}
fprintf(stderr,"\n");
}
}
#endif
done:
ncbytesfree(buf);
NC_s3freeprofilelist(profiles);
return stat;
}
/* Lookup a profile by name;
@param profilename to lookup
@param profilep return the matching profile; null if profile not found
@return NC_NOERR if no error
@return other error
*/
int
NC_authgets3profile(const char* profilename, struct AWSprofile** profilep)
{
int stat = NC_NOERR;
int i = -1;
NCglobalstate* gstate = NC_getglobalstate();
for(i=0;i<nclistlength(gstate->rcinfo->s3profiles);i++) {
struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i);
if(strcmp(profilename,profile->name)==0)
{if(profilep) {*profilep = profile; goto done;}}
}
if(profilep) *profilep = NULL; /* not found */
done:
return stat;
}
/**
@param profile name of profile
@param key key to search for in profile
@param value place to store the value if key is found; NULL if not found
@return NC_NOERR if key is found, Some other error otherwise.
*/
int
NC_s3profilelookup(const char* profile, const char* key, const char** valuep)
{
int i,stat = NC_NOERR;
struct AWSprofile* awsprof = NULL;
const char* value = NULL;
if(profile == NULL) return NC_ES3;
if((stat=NC_authgets3profile(profile,&awsprof))==NC_NOERR && awsprof != NULL) {
for(i=0;i<nclistlength(awsprof->entries);i++) {
struct AWSentry* entry = (struct AWSentry*)nclistget(awsprof->entries,i);
if(strcasecmp(entry->key,key)==0) {
value = entry->value;
break;
}
}
}
if(valuep) *valuep = value;
return stat;
}
/**************************************************/
/*
Get the current active profile. The priority order is as follows:
1. aws.profile key in mode flags
2. aws.profile in .rc entries
3. AWS_PROFILE env variable
4. "default"
5. "no" -- meaning do not use any profile => no secret key
@param uri uri with mode flags, may be NULL
@param profilep return profile name here or NULL if none found
@return NC_NOERR if no error.
@return NC_EINVAL if something else went wrong.
*/
int
NC_getactives3profile(NCURI* uri, const char** profilep)
{
int stat = NC_NOERR;
const char* profile = NULL;
struct AWSprofile* ap = NULL;
profile = ncurifragmentlookup(uri,"aws.profile");
if(profile == NULL)
profile = NC_rclookupx(uri,"AWS.PROFILE");
if(profile == NULL)
profile = NC_getglobalstate()->aws.profile;
if(profile == NULL) {
if((stat=NC_authgets3profile("default",&ap))) goto done;
if(ap) profile = "default";
}
if(profile == NULL) {
if((stat=NC_authgets3profile("no",&ap))) goto done;
if(ap) profile = "no";
}
#ifdef AWSDEBUG
fprintf(stderr,">>> activeprofile = %s\n",(profile?profile:"null"));
#endif
if(profilep) *profilep = profile;
done:
return stat;
}
/*
Get the current default region. The search order is as follows:
1. aws.region key in mode flags
2. aws.region in .rc entries
3. aws_region key in current profile (only if profiles are being used)
4. NCglobalstate.aws.default_region
@param uri uri with mode flags, may be NULL
@param regionp return region name here or NULL if none found
@return NC_NOERR if no error.
@return NC_EINVAL if something else went wrong.
*/
int
NC_getdefaults3region(NCURI* uri, const char** regionp)
{
int stat = NC_NOERR;
const char* region = NULL;
const char* profile = NULL;
region = ncurifragmentlookup(uri,"aws.region");
if(region == NULL)
region = NC_rclookupx(uri,"AWS.REGION");
if(region == NULL) {/* See if we can find a profile */
if(NC_getactives3profile(uri,&profile)==NC_NOERR) {
if(profile)
(void)NC_s3profilelookup(profile,"aws_region",&region);
}
}
if(region == NULL)
region = NC_getglobalstate()->aws.default_region; /* Force use of the Amazon default */
#ifdef AWSDEBUG
fprintf(stderr,">>> activeregion = |%s|\n",region));
#endif
if(regionp) *regionp = region;
return stat;
}
/**
The .aws/config and .aws/credentials files
are in INI format (https://en.wikipedia.org/wiki/INI_file).
This format is not well defined, so the grammar used
here is restrictive. Here, the term "profile" is the same
as the INI term "section".
The grammar used is as follows:
Grammar:
inifile: profilelist ;
profilelist: profile | profilelist profile ;
profile: '[' profilename ']' EOL entries ;
entries: empty | entries entry ;
entry: WORD = WORD EOL ;
profilename: WORD ;
Lexical:
WORD sequence of printable characters - [ \[\]=]+
EOL '\n' | ';'
Note:
1. The semicolon at beginning of a line signals a comment.
2. # comments are not allowed
3. Duplicate profiles or keys are ignored.
4. Escape characters are not supported.
*/
#define AWS_EOF (-1)
#define AWS_ERR (0)
#define AWS_WORD (0x10001)
#define AWS_EOL (0x10002)
#ifdef LEXDEBUG
static const char*
tokenname(int token)
{
static char num[32];
switch(token) {
case AWS_EOF: return "EOF";
case AWS_ERR: return "ERR";
case AWS_WORD: return "WORD";
default: snprintf(num,sizeof(num),"%d",token); return num;
}
return "UNKNOWN";
}
#endif
typedef struct AWSparser {
char* text;
char* pos;
size_t yylen; /* |yytext| */
NCbytes* yytext;
int token; /* last token found */
int pushback; /* allow 1-token pushback */
} AWSparser;
static int
awslex(AWSparser* parser)
{
int c;
int token = 0;
char* start;
size_t count;
parser->token = AWS_ERR;
ncbytesclear(parser->yytext);
ncbytesnull(parser->yytext);
if(parser->pushback != AWS_ERR) {
token = parser->pushback;
parser->pushback = AWS_ERR;
goto done;
}
while(token == 0) { /* avoid need to goto when retrying */
c = *parser->pos;
if(c == '\0') {
token = AWS_EOF;
} else if(c == '\n') {
parser->pos++;
token = AWS_EOL;
} else if(c <= ' ' || c == '\177') {
parser->pos++;
continue; /* ignore whitespace */
} else if(c == ';') {
char* p = parser->pos - 1;
if(*p == '\n') {
/* Skip comment */
do {p++;} while(*p != '\n' && *p != '\0');
parser->pos = p;
token = (*p == '\n'?AWS_EOL:AWS_EOF);
} else {
token = ';';
ncbytesappend(parser->yytext,';');
parser->pos++;
}
} else if(c == '[' || c == ']' || c == '=') {
ncbytesappend(parser->yytext,c);
ncbytesnull(parser->yytext);
token = c;
parser->pos++;
} else { /*Assume a word*/
start = parser->pos;
for(;;) {
c = *parser->pos++;
if(c <= ' ' || c == '\177' || c == '[' || c == ']' || c == '=') break; /* end of word */
}
/* Pushback last char */
parser->pos--;
count = ((parser->pos) - start);
ncbytesappendn(parser->yytext,start,count);
ncbytesnull(parser->yytext);
token = AWS_WORD;
}
#ifdef LEXDEBUG
fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,ncbytescontents(parser->yytext));
#endif
} /*for(;;)*/
done:
parser->token = token;
return token;
}
/*
@param text of the aws credentials file
@param profiles list of form struct AWSprofile (see ncauth.h)
*/
#define LBR '['
#define RBR ']'
static int
awsparse(const char* text, NClist* profiles)
{
int i,stat = NC_NOERR;
size_t len;
AWSparser* parser = NULL;
struct AWSprofile* profile = NULL;
int token;
char* key = NULL;
char* value = NULL;
if(text == NULL) text = "";
parser = calloc(1,sizeof(AWSparser));
if(parser == NULL)
{stat = (NC_ENOMEM); goto done;}
len = strlen(text);
parser->text = (char*)malloc(len+1+1+1); /* double nul term plus leading EOL */
if(parser->text == NULL)
{stat = (NCTHROW(NC_EINVAL)); goto done;}
parser->pos = parser->text;
parser->pos[0] = '\n'; /* So we can test for comment unconditionally */
parser->pos++;
strcpy(parser->text+1,text);
parser->pos += len;
/* Double nul terminate */
parser->pos[0] = '\0';
parser->pos[1] = '\0';
parser->pos = &parser->text[0]; /* reset */
parser->yytext = ncbytesnew();
parser->pushback = AWS_ERR;
/* Do not need recursion, use simple loops */
for(;;) {
token = awslex(parser); /* make token always be defined */
if(token == AWS_EOF) break; /* finished */
if(token == AWS_EOL) {continue;} /* blank line */
if(token != LBR) {stat = NCTHROW(NC_EINVAL); goto done;}
/* parse [profile name] */
token = awslex(parser);
if(token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;}
assert(profile == NULL);
if((profile = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)))==NULL)
{stat = NC_ENOMEM; goto done;}
profile->name = ncbytesextract(parser->yytext);
profile->entries = nclistnew();
token = awslex(parser);
if(token != RBR) {stat = NCTHROW(NC_EINVAL); goto done;}
#ifdef PARSEDEBUG
fprintf(stderr,">>> parse: profile=%s\n",profile->name);
#endif
/* The fields can be in any order */
for(;;) {
struct AWSentry* entry = NULL;
token = awslex(parser);
if(token == AWS_EOL) {
continue; /* ignore empty lines */
} else if(token == AWS_EOF) {
break;
} else if(token == LBR) {/* start of next profile */
parser->pushback = token;
break;
} else if(token == AWS_WORD) {
key = ncbytesextract(parser->yytext);
token = awslex(parser);
if(token != '=') {stat = NCTHROW(NC_EINVAL); goto done;}
token = awslex(parser);
if(token != AWS_EOL && token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;}
value = ncbytesextract(parser->yytext);
if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL)
{stat = NC_ENOMEM; goto done;}
entry->key = key; key = NULL;
entry->value = value; value = NULL;
#ifdef PARSEDEBUG
fprintf(stderr,">>> parse: entry=(%s,%s)\n",entry->key,entry->value);
#endif
nclistpush(profile->entries,entry); entry = NULL;
if(token == AWS_WORD) token = awslex(parser); /* finish the line */
} else
{stat = NCTHROW(NC_EINVAL); goto done;}
}
/* If this profile already exists, then replace old one */
for(i=0;i<nclistlength(profiles);i++) {
struct AWSprofile* p = (struct AWSprofile*)nclistget(profiles,i);
if(strcasecmp(p->name,profile->name)==0) {
nclistset(profiles,i,profile);
profile = NULL;
/* reclaim old one */
freeprofile(p);
break;
}
}
if(profile) nclistpush(profiles,profile);
profile = NULL;
}
done:
if(profile) freeprofile(profile);
nullfree(key);
nullfree(value);
if(parser != NULL) {
nullfree(parser->text);
ncbytesfree(parser->yytext);
free(parser);
}
return (stat);
}
static void
freeentry(struct AWSentry* e)
{
if(e) {
#ifdef AWSDEBUG
fprintf(stderr,">>> freeentry: key=%p value=%p\n",e->key,e->value);
#endif
nullfree(e->key);
nullfree(e->value);
nullfree(e);
}
}

67
libdispatch/dutil.c
View File

@ -396,6 +396,43 @@ done:
return found;
}
/** \internal
Add tag to fragment mode list unless already present.
*/
int
NC_addmodetag(NCURI* uri, const char* tag)
{
int stat = NC_NOERR;
int found = 0;
int i;
const char* modestr = NULL;
char* modevalue = NULL;
NClist* modelist = NULL;
modestr = ncurifragmentlookup(uri,"mode");
if(modestr != NULL) {
/* Parse mode str */
if((stat = NC_getmodelist(modestr,&modelist))) goto done;
} else
modelist = nclistnew();
/* Search for tag */
for(i=0;i<nclistlength(modelist);i++) {
const char* mode = (const char*)nclistget(modelist,i);
if(strcasecmp(mode,tag)==0) {found = 1; break;}
}
/* If not found, then add to modelist */
if(!found) nclistpush(modelist,strdup(tag));
/* Convert modelist back to string */
if((stat=NC_joinwith(modelist,",",NULL,NULL,&modevalue))) goto done;
/* modify the url */
if((stat=ncurisetfragmentkey(uri,"mode",modevalue))) goto done;
done:
nclistfreeall(modelist);
nullfree(modevalue);
return stat;
}
#if ! defined __INTEL_COMPILER
#if defined __APPLE__
/** \internal */
@ -465,28 +502,40 @@ done:
/** \internal concat the the segments with each segment preceded by '/' */
int
NC_join(NClist* segments, char** pathp)
{
return NC_joinwith(segments,"/","/",NULL,pathp);
}
/** \internal
Concat the the segments with separator.
@param segments to join
@param sep to use between segments
@param prefix put at front of joined string: NULL => no prefix
@param suffix put at end of joined string: NULL => no suffix
@param pathp return the join in this
*/
int
NC_joinwith(NClist* segments, const char* sep, const char* prefix, const char* suffix, char** pathp)
{
int stat = NC_NOERR;
int i;
NCbytes* buf = NULL;
size_t seplen = nulllen(sep);
if(segments == NULL)
{stat = NC_EINVAL; goto done;}
if((buf = ncbytesnew())==NULL)
{stat = NC_ENOMEM; goto done;}
if(nclistlength(segments) == 0)
ncbytescat(buf,"/");
else for(i=0;i<nclistlength(segments);i++) {
if(prefix) ncbytescat(buf,prefix);
for(i=0;i<nclistlength(segments);i++) {
const char* seg = nclistget(segments,i);
if(seg[0] != '/')
ncbytescat(buf,"/");
if(i>0 && strncmp(seg,sep,seplen)!=0)
ncbytescat(buf,sep);
ncbytescat(buf,seg);
}
if(suffix) ncbytescat(buf,suffix);
if(pathp) *pathp = ncbytesextract(buf);
done:
if(!stat) {
if(pathp) *pathp = ncbytesextract(buf);
}
ncbytesfree(buf);
return stat;
}

12
libdispatch/nch5s3comms.c
View File

@ -1073,6 +1073,7 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c
unsigned char *signing_key = NULL;
char iso8601now[ISO8601_SIZE];
struct tm *now = NULL;
const char* signingregion = AWS_GLOBAL_DEFAULT_REGION;
TRACE(0,"root=%s region=%s access_id=%s access_key=%s",root,region,access_id,access_key);
@ -1142,17 +1143,18 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c
/* Do optional authentication */
if(access_id != NULL && access_key != NULL) { /* We are authenticating */
/* Need several pieces of info for authentication */
if (nulllen(handle->region) == 0)
HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "region cannot be null.");
if (nulllen(handle->region) > 0)
signingregion = region;
// HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "region cannot be null.");
if (nulllen(handle->accessid)==0)
HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "access id cannot be null.");
if (nulllen(handle->accesskey)==0)
HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "signing key cannot be null.");
/* Compute the signing key */
if (SUCCEED != NCH5_s3comms_signing_key(&signing_key, access_key, region, iso8601now))
if (SUCCEED != NCH5_s3comms_signing_key(&signing_key, access_key, signingregion, iso8601now))
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "problem in NCH5_s3comms_s3comms_signing_key.");
if (nulllen(signing_key)==0)
if (signing_key == NULL)
HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "signing key cannot be null.");
handle->signing_key = signing_key;
signing_key = NULL;
@ -2033,7 +2035,7 @@ NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *re
if ((size_t)ret != (AWS4_secret_len - 1))
HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "problem writing AWS4+secret `%s`", secret);
if((md = (unsigned char*)malloc(SHA256_DIGEST_LENGTH))==NULL)
if((md = (unsigned char*)calloc(1,SHA256_DIGEST_LENGTH))==NULL)
HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for signing key .");
/* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest

7
libdispatch/nclist.c
View File

@ -31,7 +31,7 @@ NClist* nclistnew(void)
ncinitialized = 1;
}
*/
l = (NClist*)malloc(sizeof(NClist));
l = (NClist*)calloc(1,sizeof(NClist));
if(l) {
l->alloc=0;
l->length=0;
@ -286,10 +286,13 @@ done:
void*
nclistextract(NClist* l)
{
void* result = l->content;
void* result = NULL;
if(l) {
result = l->content;
l->alloc = 0;
l->length = 0;
l->content = NULL;
}
return result;
}

25
libdispatch/ncs3sdk_h5.c
View File

@ -82,9 +82,6 @@ struct LISTOBJECTSV2 {
char* startafter;
};
static int ncs3_initialized = 0;
static int ncs3_finalized = 0;
/* Forward */
static void s3client_destroy(NCS3CLIENT* s3client);
static char* makes3rooturl(NCS3INFO* info);
@ -141,28 +138,6 @@ dumps3client(void* s3client0, const char* tag)
/**************************************************/
EXTERNL int
NC_s3sdkinitialize(void)
{
NCTRACE(11,NULL);
if(!ncs3_initialized) {
ncs3_initialized = 1;
ncs3_finalized = 0;
}
return NCUNTRACE(NC_NOERR);
}
EXTERNL int
NC_s3sdkfinalize(void)
{
NCTRACE(11,NULL);
if(!ncs3_finalized) {
ncs3_initialized = 0;
ncs3_finalized = 1;
}
return NCUNTRACE(NC_NOERR);
}
EXTERNL void*
NC_s3sdkcreateclient(NCS3INFO* info)
{

6
libdispatch/ncuri.c
View File

@ -541,7 +541,7 @@ ncurirebuild(NCURI* duri)
/* Replace a specific fragment key*/
int
ncurisetfragmentkey(NCURI* duri,const char* key, const char* value)
ncurisetfragmentkey(NCURI* duri, const char* key, const char* value)
{
int ret = NC_NOERR;
int pos = -1;
@ -550,8 +550,8 @@ ncurisetfragmentkey(NCURI* duri,const char* key, const char* value)
pos = ncfind(duri->fraglist, key);
if(pos < 0) { /* does not exist */
if(duri->fraglist == NULL) duri->fraglist = nclistnew();
nclistpush(duri->fraglist,key);
nclistpush(duri->fraglist,value);
nclistpush(duri->fraglist,strdup(key));
nclistpush(duri->fraglist,strdup(value));
} else {
nullfree(nclistget(duri->fraglist,pos+1));
nclistset(duri->fraglist,pos+1,strdup(value));

2
libhdf5/H5FDhttp.h
View File

@ -27,8 +27,6 @@
#ifndef H5FDHTTP_H
#define H5FDHTTP_H
#define S3_REGION_DEFAULT "us-east-1"
#include "H5Ipublic.h"
#if H5_VERSION_GE(1,13,2)

4
libhdf5/hdf5open.c
View File

@ -889,7 +889,7 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
const char* awsaccessid0 = NULL;
const char* awssecretkey0 = NULL;
const char* profile0 = NULL;
int iss3 = NC_iss3(h5->uri);
int iss3 = NC_iss3(h5->uri,NULL);
fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION;
fa.authenticate = (hbool_t)0;
@ -914,7 +914,7 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
if((retval = NC_s3profilelookup(profile0,AWS_SECRET_ACCESS_KEY,&awssecretkey0)))
BAIL(retval);
if(s3.region == NULL)
s3.region = strdup(S3_REGION_DEFAULT);
s3.region = strdup(AWS_GLOBAL_DEFAULT_REGION);
if(awsaccessid0 == NULL || awssecretkey0 == NULL ) {
/* default, non-authenticating, "anonymous" fapl configuration */
fa.authenticate = (hbool_t)0;

2
libncpoco/cp_unix.c
View File

@ -46,7 +46,7 @@ static void
ncperr(const char* fcn, NCPSharedLib* lib)
{
const char* msg = dlerror();
lib->err.msg[0] = '\0';
memset(lib->err.msg,0,sizeof(lib->err.msg));
if(msg != NULL) {
strlcat(lib->err.msg,fcn,sizeof(lib->err.msg));
strlcat(lib->err.msg,": ",sizeof(lib->err.msg));

5
libncpoco/cp_win32.c
View File

@ -105,10 +105,9 @@ load(NCPSharedLib* lib , const char* path0, int flags)
char* msg = NULL;
FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, errcode, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &msg, 0, NULL);
if(msg) {
memset(lib->err.msg,0,sizeof(lib->err.msg));
if(msg)
strncpy(lib->err.msg,msg,sizeof(lib->err.msg));
} else
lib->err.msg[0] = '\0';
ret = NC_ENOTFOUND;
goto ldone;
}

11
libncpoco/ncpoco.c
View File

@ -68,6 +68,7 @@ EXTERNL int
ncpload(NCPSharedLib* lib, const char* path, int flags)
{
if(lib == NULL || path == NULL) return NC_EINVAL;
ncpclearerrmsg(lib);
return lib->api.load(lib,path,flags);
}
@ -75,6 +76,7 @@ EXTERNL int
ncpunload(NCPSharedLib* lib) /* Unloads a shared library. */
{
if(lib == NULL) return NC_EINVAL;
ncpclearerrmsg(lib);
return lib->api.unload(lib);
}
@ -93,6 +95,7 @@ EXTERNL void*
ncpgetsymbol(NCPSharedLib* lib,const char* name)
{
if(lib == NULL) return NULL;
ncpclearerrmsg(lib);
return lib->api.getsymbol(lib,name);
}
@ -113,3 +116,11 @@ ncpgeterrmsg(NCPSharedLib* lib)
if(lib == NULL) return NULL;
return (lib->err.msg[0] == '\0' ? NULL : lib->err.msg);
}
/* Clear the last err msg. */
EXTERNL void
ncpclearerrmsg(NCPSharedLib* lib)
{
if(lib == NULL) return;
memset(lib->err.msg,0,sizeof(lib->err.msg));
}

3
libncpoco/ncpoco.h
View File

@ -71,6 +71,9 @@ EXTERNL const char* ncpgetpath(NCPSharedLib*);
/* Return last err msg */
EXTERNL const char* ncpgeterrmsg(NCPSharedLib* lib);
/* Clear the last err msg. */
EXTERNL void ncpclearerrmsg(NCPSharedLib* lib);
EXTERNL const char* intstr(int err1);
#endif /*NCPOCO_H*/

2
libnczarr/zdebug.h
View File

@ -5,7 +5,7 @@
#ifndef ZDEBUG_H
#define ZDEBUG_H
#define ZCATCH /* Warning: significant performance impact */
#undef ZCATCH /* Warning: significant performance impact */
#undef ZTRACING /* Warning: significant performance impact */
#undef ZDEBUG /* general debug */

26
libsrc/httpio.c
View File

@ -46,7 +46,7 @@
typedef struct NCHTTP {
NC_HTTP_STATE* state;
long long size; /* of the object */
NCbytes* region;
NCbytes* interval;
} NCHTTP;
/* Forward */
@ -100,8 +100,8 @@ done:
fail:
if(http != NULL) {
if(http->region)
ncbytesfree(http->region);
if(http->interval)
ncbytesfree(http->interval);
free(http);
}
if(nciop != NULL) {
@ -237,7 +237,7 @@ httpio_close(ncio* nciop, int doUnlink)
/* do cleanup */
if(http != NULL) {
ncbytesfree(http->region);
ncbytesfree(http->interval);
free(http);
}
if(nciop->path != NULL) free((char*)nciop->path);
@ -246,7 +246,7 @@ httpio_close(ncio* nciop, int doUnlink)
}
/*
* Request that the region (offset, extent)
* Request that the interval (offset, extent)
* be made available through *vpp.
*/
static int
@ -258,13 +258,13 @@ httpio_get(ncio* const nciop, off_t offset, size_t extent, int rflags, void** co
if(nciop == NULL || nciop->pvt == NULL) {status = NC_EINVAL; goto done;}
http = (NCHTTP*)nciop->pvt;
assert(http->region == NULL);
http->region = ncbytesnew();
ncbytessetalloc(http->region,(unsigned long)extent);
if((status = nc_http_read(http->state,offset,extent,http->region)))
assert(http->interval == NULL);
http->interval = ncbytesnew();
ncbytessetalloc(http->interval,(unsigned long)extent);
if((status = nc_http_read(http->state,offset,extent,http->interval)))
goto done;
assert(ncbyteslength(http->region) == extent);
if(vpp) *vpp = ncbytescontents(http->region);
assert(ncbyteslength(http->interval) == extent);
if(vpp) *vpp = ncbytescontents(http->interval);
done:
return status;
}
@ -286,8 +286,8 @@ httpio_rel(ncio* const nciop, off_t offset, int rflags)
if(nciop == NULL || nciop->pvt == NULL) {status = NC_EINVAL; goto done;}
http = (NCHTTP*)nciop->pvt;
ncbytesfree(http->region);
http->region = NULL;
ncbytesfree(http->interval);
http->interval = NULL;
done:
return status;
}

14
libsrc/v1hpg.c
View File

@ -1254,7 +1254,7 @@ NC_computeshapes(NC3_INFO* ncp)
for( /*NADA*/; vpp < end; vpp++)
{
status = NC_var_shape(*vpp, &ncp->dims);
if(status != NC_NOERR)
if(status != NC_NOERR)
return(status);
if(IS_RECVAR(*vpp))
@ -1265,13 +1265,13 @@ NC_computeshapes(NC3_INFO* ncp)
}
else
{
if(first_var == NULL)
if(first_var == NULL)
first_var = *vpp;
/*
* Overwritten each time thru.
* Usually overwritten in first_rec != NULL clause below.
*/
ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len;
/*
* Overwritten each time thru.
* Usually overwritten in first_rec != NULL clause below.
*/
ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len;
}
}
}

11
libsrc4/nc4internal.c
View File

@ -2108,8 +2108,15 @@ NC_freeglobalstate(void)
nullfree(nc_globalstate->tempdir);
nullfree(nc_globalstate->home);
nullfree(nc_globalstate->cwd);
NC_rcclear(nc_globalstate->rcinfo);
free(nc_globalstate->rcinfo);
nullfree(nc_globalstate->aws.default_region);
nullfree(nc_globalstate->aws.config_file);
nullfree(nc_globalstate->aws.profile);
nullfree(nc_globalstate->aws.access_key_id);
nullfree(nc_globalstate->aws.secret_access_key);
if(nc_globalstate->rcinfo) {
NC_rcclear(nc_globalstate->rcinfo);
free(nc_globalstate->rcinfo);
}
free(nc_globalstate);
nc_globalstate = NULL;
}

6
nc_test/test_byterange.sh
View File

@ -14,9 +14,9 @@ URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files
fi
if test "x$FEATURE_S3TESTS" != xno ; then
URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
# Test alternate URL with no specified region
URL4e="http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3"
URL4c="https://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3"
# Test alternate URL with no specified region and using s3 protocol
URL4e="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
fi
if test "x$FEATURE_S3TESTS" = xyes ; then
# Requires auth

3
nczarr_test/Makefile.am
View File

@ -76,7 +76,10 @@ TESTS += run_strings.sh
TESTS += run_scalar.sh
TESTS += run_nulls.sh
TESTS += run_notzarr.sh
if ENABLE_EXTERNAL_SERVER_TESTS
TESTS += run_external.sh
endif
# Unlimited Dimension tests (at least in part)
TESTS += run_mud.sh

11
nczarr_test/run_external.sh
View File

@ -16,19 +16,24 @@ cd $ISOPATH
TESTCASES=
if test "x$FEATURE_BYTERANGE" = xyes && test "x$FEATURE_S3" = xyes && test "x$FP_ISCYGWIN" = x ; then
TESTCASES="${TESTCASES} OR_ABI;;http://s3.amazonaws.com/noaa-goes16/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3"
TESTCASES="${TESTCASES} cesmLE;blosc;http://s3.us-west-2.amazonaws.com/ncar-cesm-lens/atm/daily/cesmLE-20C-FLNS.zarr#mode=zarr,s3"
TESTCASES="${TESTCASES} OR_ABI;;;http://s3.amazonaws.com/noaa-goes16/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3"
TESTCASES="${TESTCASES} cesmLE;blosc;;http://s3.us-west-2.amazonaws.com/ncar-cesm-lens/atm/daily/cesmLE-20C-FLNS.zarr#mode=zarr,s3"
# Test TEST_REGION defaulting and s3 inferencing
TESTCASES="${TESTCASES} wtk;;us-west-2;https://nrel-pds-wtk.s3.amazonaws.com/wtk-techno-economic/pywtk-data/met_data/0/0.nc#mode=bytes"
fi
testcase() {
NM=`echo "$1" | cut -d';' -f1`
FILT=`echo "$1" | cut -d';' -f2`
URL=`echo "$1" | cut -d';' -f3`
REG=`echo "$1" | cut -d';' -f3`
URL=`echo "$1" | cut -d';' -f4`
echo "*** Test: $NM = $URL"
rm -f "tmp_external_$NM.cdl"
if test "x$FILT" != x ; then
if avail $FILT; then
if test "x$REG" != x ; then export TEST_REGION="$REG"; fi
${NCDUMP} -h -s -n $NM $URL > "tmp_external_${NM}.cdl"
unset TEST_REGION
fi
fi
}

4
s3gc.in
View File

@ -61,7 +61,9 @@ if ! aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTRE
fi
aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys
while read -r line; do
KEY=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'`
KEY0=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'`
# Strip off any leading '/'
KEY=`echo "$KEY0" | sed -e 's|^[/]*\(.*\)|\1|'`
# Ignore keys that do not start with ${S3TESTSUBTREE}
PREFIX=`echo "$KEY" | sed -e 's|\([^/]*\)/.*|\1|'`
if test "x$PREFIX" = "x$S3TESTSUBTREE" ; then