From 27f615bebcf6710df04fd1e7c6959e622226e076 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 2 Dec 2023 21:03:59 -0700 Subject: [PATCH] Properly handle missing regions in URLS NOTE: it is important that this fix gets into 4.9.3 re: Issue https://github.com/Unidata/netcdf-c/issues/2798 ## Modifications * This PR includes PR https://github.com/Unidata/netcdf-c/pull/2813 * Support the following AWS environment variables in the internal S3 library (they are already supported by aws-sdk-cpp). - AWS_REGION - AWS_DEFAULT_REGION - AWS_ACCESS_KEY_ID - AWS_CONFIG_FILE - AWS_PROFILE - AWS_SECRET_ACCESS_KEY - (source https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html). * Support an empty region when specifying s3.amazonaws.com as the host. * Move some S3/AWS related functions to ds3util.c * Add a test case to test empty region and AWS_[DEFAULT]_REGION. --- include/nc4internal.h | 7 + include/ncpathmgr.h | 3 + include/ncrc.h | 20 +- include/ncs3sdk.h | 32 +- libdispatch/dauth.c | 3 +- libdispatch/dcopy.c | 4 +- libdispatch/ddispatch.c | 2 +- libdispatch/derror.c | 4 +- libdispatch/dhttp.c | 6 +- libdispatch/dinfermodel.c | 7 +- libdispatch/drc.c | 511 +---------------------------- libdispatch/ds3util.c | 620 +++++++++++++++++++++++++++++++++++- libdispatch/dutil.c | 67 +++- libdispatch/nch5s3comms.c | 12 +- libdispatch/nclist.c | 7 +- libdispatch/ncs3sdk_h5.c | 25 -- libdispatch/ncuri.c | 6 +- libhdf5/H5FDhttp.h | 2 - libhdf5/hdf5open.c | 4 +- libncpoco/cp_unix.c | 2 +- libncpoco/cp_win32.c | 5 +- libncpoco/ncpoco.c | 11 + libncpoco/ncpoco.h | 3 + libnczarr/zdebug.h | 2 +- libsrc/httpio.c | 26 +- libsrc/v1hpg.c | 14 +- libsrc4/nc4internal.c | 11 +- nc_test/test_byterange.sh | 6 +- nczarr_test/Makefile.am | 3 + nczarr_test/run_external.sh | 11 +- s3gc.in | 4 +- 31 files changed, 809 insertions(+), 631 deletions(-) diff --git a/include/nc4internal.h b/include/nc4internal.h index f9925b1de..30ae821ec 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -332,6 +332,13 @@ typedef struct NCglobalstate { struct GlobalZarr { /* Zarr specific parameters */ char dimension_separator; } zarr; + struct GlobalAWS { /* AWS S3 specific parameters/defaults */ + char* default_region; + char* config_file; + char* profile; + char* access_key_id; + char* secret_access_key; + } aws; struct Alignment { /* H5Pset_alignment parameters */ int defined; /* 1 => threshold and alignment explicitly set */ int threshold; diff --git a/include/ncpathmgr.h b/include/ncpathmgr.h index 3f0bcd0ec..91dd38ff0 100644 --- a/include/ncpathmgr.h +++ b/include/ncpathmgr.h @@ -14,6 +14,9 @@ #ifdef HAVE_DIRENT_H #include #endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif #ifdef HAVE_SYS_STAT_H #include #endif diff --git a/include/ncrc.h b/include/ncrc.h index ae858532e..ad2af11ee 100644 --- a/include/ncrc.h +++ b/include/ncrc.h @@ -33,16 +33,6 @@ typedef struct NCRCentry { char* value; } NCRCentry; -struct AWSentry { - char* key; - char* value; -}; - -struct AWSprofile { - char* name; - NClist* entries; /* NClist */ -}; - /* collect all the relevant info around the rc file and AWS */ typedef struct NCRCinfo { int ignore; /* if 1, then do not use any rc file */ @@ -88,16 +78,10 @@ EXTERNL char* NC_mktmp(const char* base); EXTERNL int NC_getmodelist(const char* modestr, NClist** modelistp); EXTERNL int NC_testmode(NCURI* uri, const char* tag); EXTERNL int NC_testpathmode(const char* path, const char* tag); +EXTERNL int NC_addmodetag(NCURI* uri, const char* tag); EXTERNL int NC_split_delim(const char* path, char delim, NClist* segments); EXTERNL int NC_join(struct NClist* segments, char** pathp); - -/* From ds3util.c */ -/* S3 profiles */ -EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep); -EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep); -EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep); -EXTERNL int NC_iss3(NCURI* uri); -EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp); +EXTERNL int NC_joinwith(NClist* segments, const char* sep, const char* prefix, const char* suffix, char** pathp); #if defined(__cplusplus) } diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h index 771faa666..7be09da33 100644 --- a/include/ncs3sdk.h +++ b/include/ncs3sdk.h @@ -6,6 +6,12 @@ #ifndef NCS3SDK_H #define NCS3SDK_H 1 +#define AWSHOST ".amazonaws.com" +#define GOOGLEHOST "storage.googleapis.com" + +/* Define the "global" default region to be used if no other region is specified */ +#define AWS_GLOBAL_DEFAULT_REGION "us-east-1" + /* Track the server type, if known */ typedef enum NCS3SVC {NCS3UNK=0, /* unknown */ NCS3=1, /* s3.amazon.aws */ @@ -21,6 +27,20 @@ typedef struct NCS3INFO { NCS3SVC svc; } NCS3INFO; +struct AWSentry { + char* key; + char* value; +}; + +struct AWSprofile { + char* name; + struct NClist* entries; /* NClist */ +}; + +/* Opaque Types */ +struct NClist; +struct NCglobalstate; + #ifdef __cplusplus extern "C" { #endif @@ -38,13 +58,23 @@ EXTERNL int NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** EXTERNL int NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp); EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp); EXTERNL int NC_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp); -EXTERNL const char* NC_s3dumps3info(NCS3INFO* info); /* From ds3util.c */ +EXTERNL int NC_s3sdkinitialize(void); +EXTERNL int NC_s3sdkfinalize(void); + EXTERNL int NC_getdefaults3region(NCURI* uri, const char** regionp); EXTERNL int NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp); EXTERNL int NC_s3clear(NCS3INFO* s3); EXTERNL int NC_s3clone(NCS3INFO* s3, NCS3INFO** news3p); +EXTERNL const char* NC_s3dumps3info(NCS3INFO* info); +EXTERNL void NC_s3freeprofilelist(struct NClist* profiles); +EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep); +EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep); +EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep); +EXTERNL int NC_iss3(NCURI* uri, enum NCS3SVC*); +EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp); +EXTERNL int NC_aws_load_credentials(struct NCglobalstate* gstate); #ifdef __cplusplus } diff --git a/libdispatch/dauth.c b/libdispatch/dauth.c index 9ccef8593..9cd335eeb 100644 --- a/libdispatch/dauth.c +++ b/libdispatch/dauth.c @@ -17,9 +17,10 @@ See COPYRIGHT for license information. #include "netcdf.h" #include "ncbytes.h" #include "ncuri.h" -#include "ncauth.h" #include "nclog.h" #include "ncpathmgr.h" +#include "ncs3sdk.h" +#include "ncauth.h" #ifdef _MSC_VER #include diff --git a/libdispatch/dcopy.c b/libdispatch/dcopy.c index 2491a92b6..1f0ff034a 100644 --- a/libdispatch/dcopy.c +++ b/libdispatch/dcopy.c @@ -694,7 +694,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2) int gid; uintptr_t id; - id = grp; + id = (uintptr_t)grp; nclistpush(queue,(void*)id); /* prime the queue */ while(nclistlength(queue) > 0) { id = (uintptr_t)nclistremove(queue,0); @@ -712,7 +712,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2) goto done; /* push onto the end of the queue */ for(i=0;ipath = strdup(path); state->url = uri; uri = NULL; - state->format = (NC_iss3(state->url)?HTTPS3:HTTPCURL); +#ifdef ENABLE_S3 + state->format = (NC_iss3(state->url,NULL)?HTTPS3:HTTPCURL); +#else + state->format = HTTPCURL; +#endif switch (state->format) { case HTTPCURL: { diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 6f5e4a846..9bfdd7596 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -898,13 +898,16 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void ncurisetfragments(uri,sfrag); nullfree(sfrag); sfrag = NULL; +#ifdef ENABLE_S3 /* If s3, then rebuild the url */ - if(NC_iss3(uri)) { + if(NC_iss3(uri,NULL)) { NCURI* newuri = NULL; if((stat = NC_s3urlrebuild(uri,NULL,&newuri))) goto done; ncurifree(uri); uri = newuri; - } else if(strcmp(uri->protocol,"file")==0) { + } else +#endif + if(strcmp(uri->protocol,"file")==0) { /* convert path to absolute */ char* canon = NULL; abspath = NCpathabsolute(uri->path); diff --git a/libdispatch/drc.c b/libdispatch/drc.c index 77922b752..81b36e307 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -19,11 +19,11 @@ See COPYRIGHT for license information. #include "ncbytes.h" #include "ncuri.h" #include "ncrc.h" -#include "ncs3sdk.h" #include "nclog.h" #include "ncauth.h" #include "ncpathmgr.h" #include "nc4internal.h" +#include "ncs3sdk.h" #include "ncdispatch.h" #undef NOREAD @@ -31,7 +31,6 @@ See COPYRIGHT for license information. #undef DRCDEBUG #undef LEXDEBUG #undef PARSEDEBUG -#undef AWSDEBUG #define RTAG ']' #define LTAG '[' @@ -39,9 +38,6 @@ See COPYRIGHT for license information. #undef MEMCHECK #define MEMCHECK(x) if((x)==NULL) {goto nomem;} else {} -/* Alternate .aws directory location */ -#define NC_TEST_AWS_DIR "NC_TEST_AWS_DIR" - /* Forward */ static int NC_rcload(void); static char* rcreadline(char** nextlinep); @@ -57,16 +53,10 @@ static void rcfreeentry(NCRCentry* t); #ifdef DRCDEBUG static void storedump(char* msg, NClist* entrys); #endif -static int aws_load_credentials(NCglobalstate*); -static void freeprofile(struct AWSprofile* profile); -static void freeprofilelist(NClist* profiles); /* Define default rc files and aliases, also defines load order*/ static const char* rcfilenames[] = {".ncrc", ".daprc", ".dodsrc", NULL}; -/* Read these files in order and later overriding earlier */ -static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL}; - static int NCRCinitialized = 0; /**************************************************/ @@ -158,7 +148,7 @@ ncrc_initialize(void) nclog(NCLOGWARN,".rc loading failed"); } /* Load .aws/config &/ credentials */ - if((stat = aws_load_credentials(ncg))) { + if((stat = NC_aws_load_credentials(ncg))) { nclog(NCLOGWARN,"AWS config file not loaded"); } #endif @@ -187,7 +177,7 @@ NC_rcclear(NCRCinfo* info) nullfree(info->rcfile); nullfree(info->rchome); rcfreeentries(info->entries); - freeprofilelist(info->s3profiles); + NC_s3freeprofilelist(info->s3profiles); } @@ -483,7 +473,7 @@ rccompile(const char* filepath) nclog(NCLOGERR, "Malformed [url] in %s entry: %s",filepath,line); continue; } - if(NC_iss3(uri)) { + if(NC_iss3(uri,NULL)) { NCURI* newuri = NULL; /* Rebuild the url to S3 "path" format */ NC_s3clear(&s3); @@ -755,496 +745,3 @@ storedump(char* msg, NClist* entries) } #endif -/**************************************************/ -/* -Get the current active profile. The priority order is as follows: -1. aws.profile key in mode flags -2. aws.profile in .rc entries -4. "default" -5. "no" -- meaning do not use any profile => no secret key - -@param uri uri with mode flags, may be NULL -@param profilep return profile name here or NULL if none found -@return NC_NOERR if no error. -@return NC_EINVAL if something else went wrong. -*/ - -int -NC_getactives3profile(NCURI* uri, const char** profilep) -{ - int stat = NC_NOERR; - const char* profile = NULL; - struct AWSprofile* ap = NULL; - - profile = ncurifragmentlookup(uri,"aws.profile"); - if(profile == NULL) - profile = NC_rclookupx(uri,"AWS.PROFILE"); - - if(profile == NULL) { - if((stat=NC_authgets3profile("default",&ap))) goto done; - if(ap) profile = "default"; - } - - if(profile == NULL) { - if((stat=NC_authgets3profile("no",&ap))) goto done; - if(ap) profile = "no"; - } - -#ifdef AWSDEBUG - fprintf(stderr,">>> activeprofile = %s\n",(profile?profile:"null")); -#endif - if(profilep) *profilep = profile; -done: - return stat; -} - -/* -Get the current default region. The search order is as follows: -1. aws.region key in mode flags -2. aws.region in .rc entries -3. aws_region key in current profile (only if profiles are being used) -4. "us-east-1" - -@param uri uri with mode flags, may be NULL -@param regionp return region name here or NULL if none found -@return NC_NOERR if no error. -@return NC_EINVAL if something else went wrong. -*/ - -int -NC_getdefaults3region(NCURI* uri, const char** regionp) -{ - int stat = NC_NOERR; - const char* region = NULL; - const char* profile = NULL; - - region = ncurifragmentlookup(uri,"aws.region"); - if(region == NULL) - region = NC_rclookupx(uri,"AWS.REGION"); - if(region == NULL) {/* See if we can find a profile */ - if(NC_getactives3profile(uri,&profile)==NC_NOERR) { - if(profile) - (void)NC_s3profilelookup(profile,"aws_region",®ion); - } - } - if(region == NULL) - region = "us-east-1"; -#ifdef AWSDEBUG - fprintf(stderr,">>> activeregion = %s\n",(region?region:"null")); -#endif - if(regionp) *regionp = region; - return stat; -} - -/** -The .aws/config and .aws/credentials files -are in INI format (https://en.wikipedia.org/wiki/INI_file). -This format is not well defined, so the grammar used -here is restrictive. Here, the term "profile" is the same -as the INI term "section". - -The grammar used is as follows: - -Grammar: - -inifile: profilelist ; -profilelist: profile | profilelist profile ; -profile: '[' profilename ']' EOL entries ; -entries: empty | entries entry ; -entry: WORD = WORD EOL ; -profilename: WORD ; -Lexical: -WORD sequence of printable characters - [ \[\]=]+ -EOL '\n' | ';' - -Note: -1. The semicolon at beginning of a line signals a comment. -2. # comments are not allowed -3. Duplicate profiles or keys are ignored. -4. Escape characters are not supported. -*/ - -#define AWS_EOF (-1) -#define AWS_ERR (0) -#define AWS_WORD (0x10001) -#define AWS_EOL (0x10002) - -#ifdef LEXDEBUG -static const char* -tokenname(int token) -{ - static char num[32]; - switch(token) { - case AWS_EOF: return "EOF"; - case AWS_ERR: return "ERR"; - case AWS_WORD: return "WORD"; - default: snprintf(num,sizeof(num),"%d",token); return num; - } - return "UNKNOWN"; -} -#endif - -typedef struct AWSparser { - char* text; - char* pos; - size_t yylen; /* |yytext| */ - NCbytes* yytext; - int token; /* last token found */ - int pushback; /* allow 1-token pushback */ -} AWSparser; - -static int -awslex(AWSparser* parser) -{ - int c; - int token = 0; - char* start; - size_t count; - - parser->token = AWS_ERR; - ncbytesclear(parser->yytext); - ncbytesnull(parser->yytext); - - if(parser->pushback != AWS_ERR) { - token = parser->pushback; - parser->pushback = AWS_ERR; - goto done; - } - - while(token == 0) { /* avoid need to goto when retrying */ - c = *parser->pos; - if(c == '\0') { - token = AWS_EOF; - } else if(c == '\n') { - parser->pos++; - token = AWS_EOL; - } else if(c <= ' ' || c == '\177') { - parser->pos++; - continue; /* ignore whitespace */ - } else if(c == ';') { - char* p = parser->pos - 1; - if(*p == '\n') { - /* Skip comment */ - do {p++;} while(*p != '\n' && *p != '\0'); - parser->pos = p; - token = (*p == '\n'?AWS_EOL:AWS_EOF); - } else { - token = ';'; - ncbytesappend(parser->yytext,';'); - parser->pos++; - } - } else if(c == '[' || c == ']' || c == '=') { - ncbytesappend(parser->yytext,c); - ncbytesnull(parser->yytext); - token = c; - parser->pos++; - } else { /*Assume a word*/ - start = parser->pos; - for(;;) { - c = *parser->pos++; - if(c <= ' ' || c == '\177' || c == '[' || c == ']' || c == '=') break; /* end of word */ - } - /* Pushback last char */ - parser->pos--; - count = ((parser->pos) - start); - ncbytesappendn(parser->yytext,start,count); - ncbytesnull(parser->yytext); - token = AWS_WORD; - } -#ifdef LEXDEBUG -fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,ncbytescontents(parser->yytext)); -#endif - } /*for(;;)*/ - -done: - parser->token = token; - return token; -} - -/* -@param text of the aws credentials file -@param profiles list of form struct AWSprofile (see ncauth.h) -*/ - -#define LBR '[' -#define RBR ']' - -static int -awsparse(const char* text, NClist* profiles) -{ - int i,stat = NC_NOERR; - size_t len; - AWSparser* parser = NULL; - struct AWSprofile* profile = NULL; - int token; - char* key = NULL; - char* value = NULL; - - if(text == NULL) text = ""; - - parser = calloc(1,sizeof(AWSparser)); - if(parser == NULL) - {stat = (NC_ENOMEM); goto done;} - len = strlen(text); - parser->text = (char*)malloc(len+1+1+1); /* double nul term plus leading EOL */ - if(parser->text == NULL) - {stat = (NCTHROW(NC_EINVAL)); goto done;} - parser->pos = parser->text; - parser->pos[0] = '\n'; /* So we can test for comment unconditionally */ - parser->pos++; - strcpy(parser->text+1,text); - parser->pos += len; - /* Double nul terminate */ - parser->pos[0] = '\0'; - parser->pos[1] = '\0'; - parser->pos = &parser->text[0]; /* reset */ - parser->yytext = ncbytesnew(); - parser->pushback = AWS_ERR; - - /* Do not need recursion, use simple loops */ - for(;;) { - token = awslex(parser); /* make token always be defined */ - if(token == AWS_EOF) break; /* finished */ - if(token == AWS_EOL) {continue;} /* blank line */ - if(token != LBR) {stat = NCTHROW(NC_EINVAL); goto done;} - /* parse [profile name] */ - token = awslex(parser); - if(token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;} - assert(profile == NULL); - if((profile = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)))==NULL) - {stat = NC_ENOMEM; goto done;} - profile->name = ncbytesextract(parser->yytext); - profile->entries = nclistnew(); - token = awslex(parser); - if(token != RBR) {stat = NCTHROW(NC_EINVAL); goto done;} -#ifdef PARSEDEBUG -fprintf(stderr,">>> parse: profile=%s\n",profile->name); -#endif - /* The fields can be in any order */ - for(;;) { - struct AWSentry* entry = NULL; - token = awslex(parser); - if(token == AWS_EOL) { - continue; /* ignore empty lines */ - } else if(token == AWS_EOF) { - break; - } else if(token == LBR) {/* start of next profile */ - parser->pushback = token; - break; - } else if(token == AWS_WORD) { - key = ncbytesextract(parser->yytext); - token = awslex(parser); - if(token != '=') {stat = NCTHROW(NC_EINVAL); goto done;} - token = awslex(parser); - if(token != AWS_EOL && token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;} - value = ncbytesextract(parser->yytext); - if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL) - {stat = NC_ENOMEM; goto done;} - entry->key = key; key = NULL; - entry->value = value; value = NULL; -#ifdef PARSEDEBUG -fprintf(stderr,">>> parse: entry=(%s,%s)\n",entry->key,entry->value); -#endif - nclistpush(profile->entries,entry); entry = NULL; - if(token == AWS_WORD) token = awslex(parser); /* finish the line */ - } else - {stat = NCTHROW(NC_EINVAL); goto done;} - } - - /* If this profile already exists, then replace old one */ - for(i=0;iname,profile->name)==0) { - nclistset(profiles,i,profile); - profile = NULL; - /* reclaim old one */ - freeprofile(p); - break; - } - } - if(profile) nclistpush(profiles,profile); - profile = NULL; - } - -done: - if(profile) freeprofile(profile); - nullfree(key); - nullfree(value); - if(parser != NULL) { - nullfree(parser->text); - ncbytesfree(parser->yytext); - free(parser); - } - return (stat); -} - -static void -freeentry(struct AWSentry* e) -{ - if(e) { -#ifdef AWSDEBUG -fprintf(stderr,">>> freeentry: key=%p value=%p\n",e->key,e->value); -#endif - nullfree(e->key); - nullfree(e->value); - nullfree(e); - } -} - -static void -freeprofile(struct AWSprofile* profile) -{ - if(profile) { - int i; -#ifdef AWSDEBUG -fprintf(stderr,">>> freeprofile: %s\n",profile->name); -#endif - for(i=0;ientries);i++) { - struct AWSentry* e = (struct AWSentry*)nclistget(profile->entries,i); - freeentry(e); - } - nclistfree(profile->entries); - nullfree(profile->name); - nullfree(profile); - } -} - -static void -freeprofilelist(NClist* profiles) -{ - if(profiles) { - int i; - for(i=0;iname = strdup("no"); - noprof->entries = nclistnew(); - nclistpush(profiles,noprof); noprof = NULL; - } - - for(;*awscfg;awscfg++) { - /* Construct the path ${HOME}/ or Windows equivalent. */ - const char* cfg = *awscfg; - - snprintf(path,sizeof(path),"%s%s%s", - (aws_root?aws_root:gstate->home), - (*cfg == '/'?"":"/"), - cfg); - ncbytesclear(buf); - if((stat=NC_readfile(path,buf))) { - nclog(NCLOGWARN, "Could not open file: %s",path); - } else { - /* Parse the credentials file */ - const char* text = ncbytescontents(buf); - if((stat = awsparse(text,profiles))) goto done; - } - } - - /* add a "none" credentials */ - { - struct AWSprofile* noprof = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)); - if(noprof == NULL) {stat = NC_ENOMEM; goto done;} - noprof->name = strdup("none"); - noprof->entries = nclistnew(); - nclistpush(profiles,noprof); noprof = NULL; - } - - if(gstate->rcinfo->s3profiles) - freeprofilelist(gstate->rcinfo->s3profiles); - gstate->rcinfo->s3profiles = profiles; profiles = NULL; - -#ifdef AWSDEBUG - {int i,j; - fprintf(stderr,">>> profiles:\n"); - for(i=0;ircinfo->s3profiles);i++) { - struct AWSprofile* p = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i); - fprintf(stderr," [%s]",p->name); - for(j=0;jentries);j++) { - struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j); - if(strcmp(e->key,"aws_access_key_id") - fprintf(stderr," %s=%d",e->key,(int)strlen(e->value)); - else if(strcmp(e->key,"aws_secret_access_key") - fprintf(stderr," %s=%d",e->key,(int)strlen(e->value)); - else fprintf(stderr," %s=%s",e->key,e->value); - } - fprintf(stderr,"\n"); - } - } -#endif - -done: - ncbytesfree(buf); - freeprofilelist(profiles); - return stat; -} - -/* Lookup a profile by name; -@param profilename to lookup -@param profilep return the matching profile; null if profile not found -@return NC_NOERR if no error -@return other error -*/ - -int -NC_authgets3profile(const char* profilename, struct AWSprofile** profilep) -{ - int stat = NC_NOERR; - int i = -1; - NCglobalstate* gstate = NC_getglobalstate(); - - for(i=0;ircinfo->s3profiles);i++) { - struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i); - if(strcmp(profilename,profile->name)==0) - {if(profilep) {*profilep = profile; goto done;}} - } - if(profilep) *profilep = NULL; /* not found */ -done: - return stat; -} - -/** -@param profile name of profile -@param key key to search for in profile -@param value place to store the value if key is found; NULL if not found -@return NC_NOERR if key is found, Some other error otherwise. -*/ - -int -NC_s3profilelookup(const char* profile, const char* key, const char** valuep) -{ - int i,stat = NC_NOERR; - struct AWSprofile* awsprof = NULL; - const char* value = NULL; - - if(profile == NULL) return NC_ES3; - if((stat=NC_authgets3profile(profile,&awsprof))==NC_NOERR && awsprof != NULL) { - for(i=0;ientries);i++) { - struct AWSentry* entry = (struct AWSentry*)nclistget(awsprof->entries,i); - if(strcasecmp(entry->key,key)==0) { - value = entry->value; - break; - } - } - } - if(valuep) *valuep = value; - return stat; -} diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index 5091b1a24..3ecd97e5d 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -22,20 +22,73 @@ #endif #include "netcdf.h" +#include "nc4internal.h" #include "ncuri.h" #include "nclist.h" #include "ncrc.h" +#include "nclog.h" #include "ncs3sdk.h" #undef AWSDEBUG +/* Alternate .aws directory location */ +#define NC_TEST_AWS_DIR "NC_TEST_AWS_DIR" + #define AWSHOST ".amazonaws.com" #define GOOGLEHOST "storage.googleapis.com" enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4}; +/* Read these files in order and later overriding earlier */ +static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL}; +#define NCONFIGFILES (sizeof(awsconfigfiles)/sizeof(char*)) + +static int ncs3_initialized = 0; +static int ncs3_finalized = 0; + +/**************************************************/ /* Forward */ + static int endswith(const char* s, const char* suffix); +static void freeentry(struct AWSentry* e); +static int awsparse(const char* text, NClist* profiles); + +/**************************************************/ +/* Capture environmental Info */ + +EXTERNL int +NC_s3sdkinitialize(void) +{ + if(!ncs3_initialized) { + ncs3_initialized = 1; + ncs3_finalized = 0; + } + { + /* Get various environment variables as defined by the AWS sdk */ + NCglobalstate* gs = NC_getglobalstate(); + if(getenv("AWS_REGION")!=NULL) + gs->aws.default_region = nulldup(getenv("AWS_REGION")); + else if(getenv("AWS_DEFAULT_REGION")!=NULL) + gs->aws.default_region = nulldup(getenv("AWS_DEFAULT_REGION")); + else if(gs->aws.default_region == NULL) + gs->aws.default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION); + gs->aws.access_key_id = nulldup(getenv("AWS_ACCESS_KEY_ID")); + gs->aws.config_file = nulldup(getenv("AWS_CONFIG_FILE")); + gs->aws.profile = nulldup(getenv("AWS_PROFILE")); + gs->aws.secret_access_key = nulldup(getenv("AWS_SECRET_ACCESS_KEY")); + } + return NC_NOERR; +} + +EXTERNL int +NC_s3sdkfinalize(void) +{ + if(!ncs3_finalized) { + ncs3_initialized = 0; + ncs3_finalized = 1; + } + return NC_NOERR; +} /**************************************************/ /* Generic S3 Utilities */ @@ -43,7 +96,7 @@ static int endswith(const char* s, const char* suffix); /* Rebuild an S3 url into a canonical path-style url. If region is not in the host, then use specified region -if provided, otherwise us-east-1. +if provided, otherwise leave blank and let the S3 server deal with it. @param url (in) the current url @param s3 (in/out) NCS3INFO structure @param pathurlp (out) the resulting pathified url string @@ -78,7 +131,7 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp) /* Distinguish path-style from virtual-host style from s3: and from other. Virtual: https://.s3..amazonaws.com/ (1) - or: https://.s3.amazonaws.com/ -- region defaults to us-east-1 (2) + or: https://.s3.amazonaws.com/ -- region defaults (to us-east-1) (2) Path: https://s3..amazonaws.com// (3) or: https://s3.amazonaws.com// -- region defaults to us-east-1 (4) S3: s3:/// (5) @@ -148,7 +201,7 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp) const char* region0 = NULL; /* Get default region */ if((stat = NC_getdefaults3region(url,®ion0))) goto done; - region = strdup(region0); + region = nulldup(region0); } if(region == NULL) {stat = NC_ES3; goto done;} @@ -163,8 +216,10 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp) if(svc == NCS3) { /* Construct the revised host */ ncbytesclear(buf); - ncbytescat(buf,"s3."); - ncbytescat(buf,region); + ncbytescat(buf,"s3"); + assert(region != NULL); + ncbytescat(buf,"."); + ncbytescat(buf,region); ncbytescat(buf,AWSHOST); nullfree(host); host = ncbytesextract(buf); @@ -186,18 +241,24 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp) } path = ncbytesextract(buf); - /* complete the new url */ + /* clone the url so we can modify it*/ if((newurl=ncuriclone(url))==NULL) {stat = NC_ENOMEM; goto done;} + + /* Modify the URL to canonical form */ ncurisetprotocol(newurl,"https"); assert(host != NULL); ncurisethost(newurl,host); assert(path != NULL); ncurisetpath(newurl,path); + + /* Add "s3" to the mode list */ + NC_addmodetag(newurl,"s3"); + /* Rebuild the url->url */ ncurirebuild(newurl); /* return various items */ #ifdef AWSDEBUG - fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=%s region=%s\n",uri->uri,bucket,region); + fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=|%s| region=|%s|\n",uri->uri,bucket,region); #endif if(newurlp) {*newurlp = newurl; newurl = NULL;} if(s3 != NULL) { @@ -303,22 +364,24 @@ Check if a url has indicators that signal an S3 or Google S3 url. */ int -NC_iss3(NCURI* uri) +NC_iss3(NCURI* uri, enum NCS3SVC* svcp) { int iss3 = 0; + NCS3SVC svc = NCS3UNK; if(uri == NULL) goto done; /* not a uri */ /* is the protocol "s3" or "gs3" ? */ - if(strcasecmp(uri->protocol,"s3")==0) {iss3 = 1; goto done;} - if(strcasecmp(uri->protocol,"gs3")==0) {iss3 = 1; goto done;} + if(strcasecmp(uri->protocol,"s3")==0) {iss3 = 1; svc = NCS3; goto done;} + if(strcasecmp(uri->protocol,"gs3")==0) {iss3 = 1; svc = NCS3GS; goto done;} /* Is "s3" or "gs3" in the mode list? */ - if(NC_testmode(uri,"s3")) {iss3 = 1; goto done;} - if(NC_testmode(uri,"gs3")) {iss3 = 1; goto done;} + if(NC_testmode(uri,"s3")) {iss3 = 1; svc = NCS3; goto done;} + if(NC_testmode(uri,"gs3")) {iss3 = 1; svc = NCS3GS; goto done;} /* Last chance; see if host looks s3'y */ if(uri->host != NULL) { - if(endswith(uri->host,AWSHOST)) {iss3 = 1; goto done;} - if(strcasecmp(uri->host,GOOGLEHOST)==0) {iss3 = 1; goto done;} + if(endswith(uri->host,AWSHOST)) {iss3 = 1; svc = NCS3; goto done;} + if(strcasecmp(uri->host,GOOGLEHOST)==0) {iss3 = 1; svc = NCS3GS; goto done;} } + if(svcp) *svcp = svc; done: return iss3; } @@ -336,3 +399,532 @@ NC_s3dumps3info(NCS3INFO* info) return text; } +static void +freeprofile(struct AWSprofile* profile) +{ + if(profile) { + int i; +#ifdef AWSDEBUG +fprintf(stderr,">>> freeprofile: %s\n",profile->name); +#endif + for(i=0;ientries);i++) { + struct AWSentry* e = (struct AWSentry*)nclistget(profile->entries,i); + freeentry(e); + } + nclistfree(profile->entries); + nullfree(profile->name); + nullfree(profile); + } +} + +void +NC_s3freeprofilelist(NClist* profiles) +{ + if(profiles) { + int i; + for(i=0;iname = strdup("no"); + noprof->entries = nclistnew(); + nclistpush(profiles,noprof); noprof = NULL; + } + + awscfg = awsconfigfiles; + if((awscfg_local[0] = NC_getglobalstate()->aws.config_file)!=NULL) { + memcpy(&awscfg_local[1],awsconfigfiles,sizeof(char*)*NCONFIGFILES); + awscfg = awscfg_local; + } + for(;*awscfg;awscfg++) { + /* Construct the path ${HOME}/ or Windows equivalent. */ + const char* cfg = *awscfg; + + snprintf(path,sizeof(path),"%s%s%s", + (aws_root?aws_root:gstate->home), + (*cfg == '/'?"":"/"), + cfg); + ncbytesclear(buf); + if((stat=NC_readfile(path,buf))) { + nclog(NCLOGWARN, "Could not open file: %s",path); + } else { + /* Parse the credentials file */ + const char* text = ncbytescontents(buf); + if((stat = awsparse(text,profiles))) goto done; + } + } + + /* If there is no default credentials, then try to synthesize one + from various environment variables */ + { + size_t i; + struct AWSprofile* dfalt = NULL; + struct AWSentry* entry = NULL; + NCglobalstate* gs = NC_getglobalstate(); + /* Verify that we can build a default */ + if(gs->aws.access_key_id != NULL && gs->aws.secret_access_key != NULL) { + /* Kill off any previous default profile */ + for(i=nclistlength(profiles)-1;i>=0;i--) {/* walk backward because we are removing entries */ + struct AWSprofile* prof = (struct AWSprofile*)nclistget(profiles,i); + if(strcasecmp(prof->name,"default")==0) { + nclistremove(profiles,i); + freeprofile(prof); + } + } + /* Build new default profile */ + if((dfalt = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)))==NULL) {stat = NC_ENOMEM; goto done;} + dfalt->name = strdup("default"); + dfalt->entries = nclistnew(); + /* Save the new default profile */ + nclistpush(profiles,dfalt); dfalt = NULL; + /* Create the entries for default */ + if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL) {stat = NC_ENOMEM; goto done;} + entry->key = strdup("aws_access_key_id"); + entry->value = strdup(gs->aws.access_key_id); + nclistpush(dfalt->entries,entry); entry = NULL; + if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL) {stat = NC_ENOMEM; goto done;} + entry->key = strdup("aws_secret_access_key"); + entry->value = strdup(gs->aws.secret_access_key); + nclistpush(dfalt->entries,entry); entry = NULL; + } + } + + if(gstate->rcinfo->s3profiles) + NC_s3freeprofilelist(gstate->rcinfo->s3profiles); + gstate->rcinfo->s3profiles = profiles; profiles = NULL; + +#ifdef AWSDEBUG + {int i,j; + fprintf(stderr,">>> profiles:\n"); + for(i=0;ircinfo->s3profiles);i++) { + struct AWSprofile* p = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i); + fprintf(stderr," [%s]",p->name); + for(j=0;jentries);j++) { + struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j); + if(strcmp(e->key,"aws_access_key_id") + fprintf(stderr," %s=%d",e->key,(int)strlen(e->value)); + else if(strcmp(e->key,"aws_secret_access_key") + fprintf(stderr," %s=%d",e->key,(int)strlen(e->value)); + else fprintf(stderr," %s=%s",e->key,e->value); + } + fprintf(stderr,"\n"); + } + } +#endif + +done: + ncbytesfree(buf); + NC_s3freeprofilelist(profiles); + return stat; +} + +/* Lookup a profile by name; +@param profilename to lookup +@param profilep return the matching profile; null if profile not found +@return NC_NOERR if no error +@return other error +*/ + +int +NC_authgets3profile(const char* profilename, struct AWSprofile** profilep) +{ + int stat = NC_NOERR; + int i = -1; + NCglobalstate* gstate = NC_getglobalstate(); + + for(i=0;ircinfo->s3profiles);i++) { + struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i); + if(strcmp(profilename,profile->name)==0) + {if(profilep) {*profilep = profile; goto done;}} + } + if(profilep) *profilep = NULL; /* not found */ +done: + return stat; +} + +/** +@param profile name of profile +@param key key to search for in profile +@param value place to store the value if key is found; NULL if not found +@return NC_NOERR if key is found, Some other error otherwise. +*/ + +int +NC_s3profilelookup(const char* profile, const char* key, const char** valuep) +{ + int i,stat = NC_NOERR; + struct AWSprofile* awsprof = NULL; + const char* value = NULL; + + if(profile == NULL) return NC_ES3; + if((stat=NC_authgets3profile(profile,&awsprof))==NC_NOERR && awsprof != NULL) { + for(i=0;ientries);i++) { + struct AWSentry* entry = (struct AWSentry*)nclistget(awsprof->entries,i); + if(strcasecmp(entry->key,key)==0) { + value = entry->value; + break; + } + } + } + if(valuep) *valuep = value; + return stat; +} + +/**************************************************/ +/* +Get the current active profile. The priority order is as follows: +1. aws.profile key in mode flags +2. aws.profile in .rc entries +3. AWS_PROFILE env variable +4. "default" +5. "no" -- meaning do not use any profile => no secret key + +@param uri uri with mode flags, may be NULL +@param profilep return profile name here or NULL if none found +@return NC_NOERR if no error. +@return NC_EINVAL if something else went wrong. +*/ + +int +NC_getactives3profile(NCURI* uri, const char** profilep) +{ + int stat = NC_NOERR; + const char* profile = NULL; + struct AWSprofile* ap = NULL; + + profile = ncurifragmentlookup(uri,"aws.profile"); + if(profile == NULL) + profile = NC_rclookupx(uri,"AWS.PROFILE"); + + if(profile == NULL) + profile = NC_getglobalstate()->aws.profile; + + if(profile == NULL) { + if((stat=NC_authgets3profile("default",&ap))) goto done; + if(ap) profile = "default"; + } + + if(profile == NULL) { + if((stat=NC_authgets3profile("no",&ap))) goto done; + if(ap) profile = "no"; + } + +#ifdef AWSDEBUG + fprintf(stderr,">>> activeprofile = %s\n",(profile?profile:"null")); +#endif + if(profilep) *profilep = profile; +done: + return stat; +} + +/* +Get the current default region. The search order is as follows: +1. aws.region key in mode flags +2. aws.region in .rc entries +3. aws_region key in current profile (only if profiles are being used) +4. NCglobalstate.aws.default_region + +@param uri uri with mode flags, may be NULL +@param regionp return region name here or NULL if none found +@return NC_NOERR if no error. +@return NC_EINVAL if something else went wrong. +*/ + +int +NC_getdefaults3region(NCURI* uri, const char** regionp) +{ + int stat = NC_NOERR; + const char* region = NULL; + const char* profile = NULL; + + region = ncurifragmentlookup(uri,"aws.region"); + if(region == NULL) + region = NC_rclookupx(uri,"AWS.REGION"); + if(region == NULL) {/* See if we can find a profile */ + if(NC_getactives3profile(uri,&profile)==NC_NOERR) { + if(profile) + (void)NC_s3profilelookup(profile,"aws_region",®ion); + } + } + if(region == NULL) + region = NC_getglobalstate()->aws.default_region; /* Force use of the Amazon default */ +#ifdef AWSDEBUG + fprintf(stderr,">>> activeregion = |%s|\n",region)); +#endif + if(regionp) *regionp = region; + return stat; +} + +/** +The .aws/config and .aws/credentials files +are in INI format (https://en.wikipedia.org/wiki/INI_file). +This format is not well defined, so the grammar used +here is restrictive. Here, the term "profile" is the same +as the INI term "section". + +The grammar used is as follows: + +Grammar: + +inifile: profilelist ; +profilelist: profile | profilelist profile ; +profile: '[' profilename ']' EOL entries ; +entries: empty | entries entry ; +entry: WORD = WORD EOL ; +profilename: WORD ; +Lexical: +WORD sequence of printable characters - [ \[\]=]+ +EOL '\n' | ';' + +Note: +1. The semicolon at beginning of a line signals a comment. +2. # comments are not allowed +3. Duplicate profiles or keys are ignored. +4. Escape characters are not supported. +*/ + +#define AWS_EOF (-1) +#define AWS_ERR (0) +#define AWS_WORD (0x10001) +#define AWS_EOL (0x10002) + +#ifdef LEXDEBUG +static const char* +tokenname(int token) +{ + static char num[32]; + switch(token) { + case AWS_EOF: return "EOF"; + case AWS_ERR: return "ERR"; + case AWS_WORD: return "WORD"; + default: snprintf(num,sizeof(num),"%d",token); return num; + } + return "UNKNOWN"; +} +#endif + +typedef struct AWSparser { + char* text; + char* pos; + size_t yylen; /* |yytext| */ + NCbytes* yytext; + int token; /* last token found */ + int pushback; /* allow 1-token pushback */ +} AWSparser; + +static int +awslex(AWSparser* parser) +{ + int c; + int token = 0; + char* start; + size_t count; + + parser->token = AWS_ERR; + ncbytesclear(parser->yytext); + ncbytesnull(parser->yytext); + + if(parser->pushback != AWS_ERR) { + token = parser->pushback; + parser->pushback = AWS_ERR; + goto done; + } + + while(token == 0) { /* avoid need to goto when retrying */ + c = *parser->pos; + if(c == '\0') { + token = AWS_EOF; + } else if(c == '\n') { + parser->pos++; + token = AWS_EOL; + } else if(c <= ' ' || c == '\177') { + parser->pos++; + continue; /* ignore whitespace */ + } else if(c == ';') { + char* p = parser->pos - 1; + if(*p == '\n') { + /* Skip comment */ + do {p++;} while(*p != '\n' && *p != '\0'); + parser->pos = p; + token = (*p == '\n'?AWS_EOL:AWS_EOF); + } else { + token = ';'; + ncbytesappend(parser->yytext,';'); + parser->pos++; + } + } else if(c == '[' || c == ']' || c == '=') { + ncbytesappend(parser->yytext,c); + ncbytesnull(parser->yytext); + token = c; + parser->pos++; + } else { /*Assume a word*/ + start = parser->pos; + for(;;) { + c = *parser->pos++; + if(c <= ' ' || c == '\177' || c == '[' || c == ']' || c == '=') break; /* end of word */ + } + /* Pushback last char */ + parser->pos--; + count = ((parser->pos) - start); + ncbytesappendn(parser->yytext,start,count); + ncbytesnull(parser->yytext); + token = AWS_WORD; + } +#ifdef LEXDEBUG +fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,ncbytescontents(parser->yytext)); +#endif + } /*for(;;)*/ + +done: + parser->token = token; + return token; +} + +/* +@param text of the aws credentials file +@param profiles list of form struct AWSprofile (see ncauth.h) +*/ + +#define LBR '[' +#define RBR ']' + +static int +awsparse(const char* text, NClist* profiles) +{ + int i,stat = NC_NOERR; + size_t len; + AWSparser* parser = NULL; + struct AWSprofile* profile = NULL; + int token; + char* key = NULL; + char* value = NULL; + + if(text == NULL) text = ""; + + parser = calloc(1,sizeof(AWSparser)); + if(parser == NULL) + {stat = (NC_ENOMEM); goto done;} + len = strlen(text); + parser->text = (char*)malloc(len+1+1+1); /* double nul term plus leading EOL */ + if(parser->text == NULL) + {stat = (NCTHROW(NC_EINVAL)); goto done;} + parser->pos = parser->text; + parser->pos[0] = '\n'; /* So we can test for comment unconditionally */ + parser->pos++; + strcpy(parser->text+1,text); + parser->pos += len; + /* Double nul terminate */ + parser->pos[0] = '\0'; + parser->pos[1] = '\0'; + parser->pos = &parser->text[0]; /* reset */ + parser->yytext = ncbytesnew(); + parser->pushback = AWS_ERR; + + /* Do not need recursion, use simple loops */ + for(;;) { + token = awslex(parser); /* make token always be defined */ + if(token == AWS_EOF) break; /* finished */ + if(token == AWS_EOL) {continue;} /* blank line */ + if(token != LBR) {stat = NCTHROW(NC_EINVAL); goto done;} + /* parse [profile name] */ + token = awslex(parser); + if(token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;} + assert(profile == NULL); + if((profile = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)))==NULL) + {stat = NC_ENOMEM; goto done;} + profile->name = ncbytesextract(parser->yytext); + profile->entries = nclistnew(); + token = awslex(parser); + if(token != RBR) {stat = NCTHROW(NC_EINVAL); goto done;} +#ifdef PARSEDEBUG +fprintf(stderr,">>> parse: profile=%s\n",profile->name); +#endif + /* The fields can be in any order */ + for(;;) { + struct AWSentry* entry = NULL; + token = awslex(parser); + if(token == AWS_EOL) { + continue; /* ignore empty lines */ + } else if(token == AWS_EOF) { + break; + } else if(token == LBR) {/* start of next profile */ + parser->pushback = token; + break; + } else if(token == AWS_WORD) { + key = ncbytesextract(parser->yytext); + token = awslex(parser); + if(token != '=') {stat = NCTHROW(NC_EINVAL); goto done;} + token = awslex(parser); + if(token != AWS_EOL && token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;} + value = ncbytesextract(parser->yytext); + if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL) + {stat = NC_ENOMEM; goto done;} + entry->key = key; key = NULL; + entry->value = value; value = NULL; +#ifdef PARSEDEBUG +fprintf(stderr,">>> parse: entry=(%s,%s)\n",entry->key,entry->value); +#endif + nclistpush(profile->entries,entry); entry = NULL; + if(token == AWS_WORD) token = awslex(parser); /* finish the line */ + } else + {stat = NCTHROW(NC_EINVAL); goto done;} + } + + /* If this profile already exists, then replace old one */ + for(i=0;iname,profile->name)==0) { + nclistset(profiles,i,profile); + profile = NULL; + /* reclaim old one */ + freeprofile(p); + break; + } + } + if(profile) nclistpush(profiles,profile); + profile = NULL; + } + +done: + if(profile) freeprofile(profile); + nullfree(key); + nullfree(value); + if(parser != NULL) { + nullfree(parser->text); + ncbytesfree(parser->yytext); + free(parser); + } + return (stat); +} + +static void +freeentry(struct AWSentry* e) +{ + if(e) { +#ifdef AWSDEBUG +fprintf(stderr,">>> freeentry: key=%p value=%p\n",e->key,e->value); +#endif + nullfree(e->key); + nullfree(e->value); + nullfree(e); + } +} diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c index 83b20dfb6..1f37e5516 100644 --- a/libdispatch/dutil.c +++ b/libdispatch/dutil.c @@ -396,6 +396,43 @@ done: return found; } +/** \internal +Add tag to fragment mode list unless already present. +*/ +int +NC_addmodetag(NCURI* uri, const char* tag) +{ + int stat = NC_NOERR; + int found = 0; + int i; + const char* modestr = NULL; + char* modevalue = NULL; + NClist* modelist = NULL; + + modestr = ncurifragmentlookup(uri,"mode"); + if(modestr != NULL) { + /* Parse mode str */ + if((stat = NC_getmodelist(modestr,&modelist))) goto done; + } else + modelist = nclistnew(); + /* Search for tag */ + for(i=0;i no prefix +@param suffix put at end of joined string: NULL => no suffix +@param pathp return the join in this +*/ +int +NC_joinwith(NClist* segments, const char* sep, const char* prefix, const char* suffix, char** pathp) { int stat = NC_NOERR; int i; NCbytes* buf = NULL; + size_t seplen = nulllen(sep); if(segments == NULL) {stat = NC_EINVAL; goto done;} if((buf = ncbytesnew())==NULL) {stat = NC_ENOMEM; goto done;} - if(nclistlength(segments) == 0) - ncbytescat(buf,"/"); - else for(i=0;i0 && strncmp(seg,sep,seplen)!=0) + ncbytescat(buf,sep); ncbytescat(buf,seg); } - + if(suffix) ncbytescat(buf,suffix); + if(pathp) *pathp = ncbytesextract(buf); done: - if(!stat) { - if(pathp) *pathp = ncbytesextract(buf); - } ncbytesfree(buf); return stat; } diff --git a/libdispatch/nch5s3comms.c b/libdispatch/nch5s3comms.c index ba09b931b..29aedea32 100644 --- a/libdispatch/nch5s3comms.c +++ b/libdispatch/nch5s3comms.c @@ -1073,6 +1073,7 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c unsigned char *signing_key = NULL; char iso8601now[ISO8601_SIZE]; struct tm *now = NULL; + const char* signingregion = AWS_GLOBAL_DEFAULT_REGION; TRACE(0,"root=%s region=%s access_id=%s access_key=%s",root,region,access_id,access_key); @@ -1142,17 +1143,18 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c /* Do optional authentication */ if(access_id != NULL && access_key != NULL) { /* We are authenticating */ /* Need several pieces of info for authentication */ - if (nulllen(handle->region) == 0) - HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "region cannot be null."); + if (nulllen(handle->region) > 0) + signingregion = region; +// HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "region cannot be null."); if (nulllen(handle->accessid)==0) HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "access id cannot be null."); if (nulllen(handle->accesskey)==0) HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "signing key cannot be null."); /* Compute the signing key */ - if (SUCCEED != NCH5_s3comms_signing_key(&signing_key, access_key, region, iso8601now)) + if (SUCCEED != NCH5_s3comms_signing_key(&signing_key, access_key, signingregion, iso8601now)) HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "problem in NCH5_s3comms_s3comms_signing_key."); - if (nulllen(signing_key)==0) + if (signing_key == NULL) HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "signing key cannot be null."); handle->signing_key = signing_key; signing_key = NULL; @@ -2033,7 +2035,7 @@ NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *re if ((size_t)ret != (AWS4_secret_len - 1)) HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "problem writing AWS4+secret `%s`", secret); - if((md = (unsigned char*)malloc(SHA256_DIGEST_LENGTH))==NULL) + if((md = (unsigned char*)calloc(1,SHA256_DIGEST_LENGTH))==NULL) HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for signing key ."); /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest diff --git a/libdispatch/nclist.c b/libdispatch/nclist.c index f2c3f4d47..5ed21753e 100644 --- a/libdispatch/nclist.c +++ b/libdispatch/nclist.c @@ -31,7 +31,7 @@ NClist* nclistnew(void) ncinitialized = 1; } */ - l = (NClist*)malloc(sizeof(NClist)); + l = (NClist*)calloc(1,sizeof(NClist)); if(l) { l->alloc=0; l->length=0; @@ -286,10 +286,13 @@ done: void* nclistextract(NClist* l) { - void* result = l->content; + void* result = NULL; + if(l) { + result = l->content; l->alloc = 0; l->length = 0; l->content = NULL; + } return result; } diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c index e51f8d811..5f7f223db 100644 --- a/libdispatch/ncs3sdk_h5.c +++ b/libdispatch/ncs3sdk_h5.c @@ -82,9 +82,6 @@ struct LISTOBJECTSV2 { char* startafter; }; -static int ncs3_initialized = 0; -static int ncs3_finalized = 0; - /* Forward */ static void s3client_destroy(NCS3CLIENT* s3client); static char* makes3rooturl(NCS3INFO* info); @@ -141,28 +138,6 @@ dumps3client(void* s3client0, const char* tag) /**************************************************/ -EXTERNL int -NC_s3sdkinitialize(void) -{ - NCTRACE(11,NULL); - if(!ncs3_initialized) { - ncs3_initialized = 1; - ncs3_finalized = 0; - } - return NCUNTRACE(NC_NOERR); -} - -EXTERNL int -NC_s3sdkfinalize(void) -{ - NCTRACE(11,NULL); - if(!ncs3_finalized) { - ncs3_initialized = 0; - ncs3_finalized = 1; - } - return NCUNTRACE(NC_NOERR); -} - EXTERNL void* NC_s3sdkcreateclient(NCS3INFO* info) { diff --git a/libdispatch/ncuri.c b/libdispatch/ncuri.c index 6f699a859..69a7212ff 100644 --- a/libdispatch/ncuri.c +++ b/libdispatch/ncuri.c @@ -541,7 +541,7 @@ ncurirebuild(NCURI* duri) /* Replace a specific fragment key*/ int -ncurisetfragmentkey(NCURI* duri,const char* key, const char* value) +ncurisetfragmentkey(NCURI* duri, const char* key, const char* value) { int ret = NC_NOERR; int pos = -1; @@ -550,8 +550,8 @@ ncurisetfragmentkey(NCURI* duri,const char* key, const char* value) pos = ncfind(duri->fraglist, key); if(pos < 0) { /* does not exist */ if(duri->fraglist == NULL) duri->fraglist = nclistnew(); - nclistpush(duri->fraglist,key); - nclistpush(duri->fraglist,value); + nclistpush(duri->fraglist,strdup(key)); + nclistpush(duri->fraglist,strdup(value)); } else { nullfree(nclistget(duri->fraglist,pos+1)); nclistset(duri->fraglist,pos+1,strdup(value)); diff --git a/libhdf5/H5FDhttp.h b/libhdf5/H5FDhttp.h index f3c46dca5..10bcd4222 100644 --- a/libhdf5/H5FDhttp.h +++ b/libhdf5/H5FDhttp.h @@ -27,8 +27,6 @@ #ifndef H5FDHTTP_H #define H5FDHTTP_H -#define S3_REGION_DEFAULT "us-east-1" - #include "H5Ipublic.h" #if H5_VERSION_GE(1,13,2) diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 0c2f5b523..5ef506fb3 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -889,7 +889,7 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) const char* awsaccessid0 = NULL; const char* awssecretkey0 = NULL; const char* profile0 = NULL; - int iss3 = NC_iss3(h5->uri); + int iss3 = NC_iss3(h5->uri,NULL); fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION; fa.authenticate = (hbool_t)0; @@ -914,7 +914,7 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) if((retval = NC_s3profilelookup(profile0,AWS_SECRET_ACCESS_KEY,&awssecretkey0))) BAIL(retval); if(s3.region == NULL) - s3.region = strdup(S3_REGION_DEFAULT); + s3.region = strdup(AWS_GLOBAL_DEFAULT_REGION); if(awsaccessid0 == NULL || awssecretkey0 == NULL ) { /* default, non-authenticating, "anonymous" fapl configuration */ fa.authenticate = (hbool_t)0; diff --git a/libncpoco/cp_unix.c b/libncpoco/cp_unix.c index 3ccea2ed0..7ff01f79e 100755 --- a/libncpoco/cp_unix.c +++ b/libncpoco/cp_unix.c @@ -46,7 +46,7 @@ static void ncperr(const char* fcn, NCPSharedLib* lib) { const char* msg = dlerror(); - lib->err.msg[0] = '\0'; + memset(lib->err.msg,0,sizeof(lib->err.msg)); if(msg != NULL) { strlcat(lib->err.msg,fcn,sizeof(lib->err.msg)); strlcat(lib->err.msg,": ",sizeof(lib->err.msg)); diff --git a/libncpoco/cp_win32.c b/libncpoco/cp_win32.c index b4284ac10..896344f63 100755 --- a/libncpoco/cp_win32.c +++ b/libncpoco/cp_win32.c @@ -105,10 +105,9 @@ load(NCPSharedLib* lib , const char* path0, int flags) char* msg = NULL; FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, errcode, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &msg, 0, NULL); - if(msg) { + memset(lib->err.msg,0,sizeof(lib->err.msg)); + if(msg) strncpy(lib->err.msg,msg,sizeof(lib->err.msg)); - } else - lib->err.msg[0] = '\0'; ret = NC_ENOTFOUND; goto ldone; } diff --git a/libncpoco/ncpoco.c b/libncpoco/ncpoco.c index 12d4b41ee..b12b2e0a9 100755 --- a/libncpoco/ncpoco.c +++ b/libncpoco/ncpoco.c @@ -68,6 +68,7 @@ EXTERNL int ncpload(NCPSharedLib* lib, const char* path, int flags) { if(lib == NULL || path == NULL) return NC_EINVAL; + ncpclearerrmsg(lib); return lib->api.load(lib,path,flags); } @@ -75,6 +76,7 @@ EXTERNL int ncpunload(NCPSharedLib* lib) /* Unloads a shared library. */ { if(lib == NULL) return NC_EINVAL; + ncpclearerrmsg(lib); return lib->api.unload(lib); } @@ -93,6 +95,7 @@ EXTERNL void* ncpgetsymbol(NCPSharedLib* lib,const char* name) { if(lib == NULL) return NULL; + ncpclearerrmsg(lib); return lib->api.getsymbol(lib,name); } @@ -113,3 +116,11 @@ ncpgeterrmsg(NCPSharedLib* lib) if(lib == NULL) return NULL; return (lib->err.msg[0] == '\0' ? NULL : lib->err.msg); } + +/* Clear the last err msg. */ +EXTERNL void +ncpclearerrmsg(NCPSharedLib* lib) +{ + if(lib == NULL) return; + memset(lib->err.msg,0,sizeof(lib->err.msg)); +} diff --git a/libncpoco/ncpoco.h b/libncpoco/ncpoco.h index 43fb077ba..fc1e993b9 100755 --- a/libncpoco/ncpoco.h +++ b/libncpoco/ncpoco.h @@ -71,6 +71,9 @@ EXTERNL const char* ncpgetpath(NCPSharedLib*); /* Return last err msg */ EXTERNL const char* ncpgeterrmsg(NCPSharedLib* lib); +/* Clear the last err msg. */ +EXTERNL void ncpclearerrmsg(NCPSharedLib* lib); + EXTERNL const char* intstr(int err1); #endif /*NCPOCO_H*/ diff --git a/libnczarr/zdebug.h b/libnczarr/zdebug.h index 8c19d9f47..83f45cdef 100644 --- a/libnczarr/zdebug.h +++ b/libnczarr/zdebug.h @@ -5,7 +5,7 @@ #ifndef ZDEBUG_H #define ZDEBUG_H -#define ZCATCH /* Warning: significant performance impact */ +#undef ZCATCH /* Warning: significant performance impact */ #undef ZTRACING /* Warning: significant performance impact */ #undef ZDEBUG /* general debug */ diff --git a/libsrc/httpio.c b/libsrc/httpio.c index d9b376d2e..17766f111 100644 --- a/libsrc/httpio.c +++ b/libsrc/httpio.c @@ -46,7 +46,7 @@ typedef struct NCHTTP { NC_HTTP_STATE* state; long long size; /* of the object */ - NCbytes* region; + NCbytes* interval; } NCHTTP; /* Forward */ @@ -100,8 +100,8 @@ done: fail: if(http != NULL) { - if(http->region) - ncbytesfree(http->region); + if(http->interval) + ncbytesfree(http->interval); free(http); } if(nciop != NULL) { @@ -237,7 +237,7 @@ httpio_close(ncio* nciop, int doUnlink) /* do cleanup */ if(http != NULL) { - ncbytesfree(http->region); + ncbytesfree(http->interval); free(http); } if(nciop->path != NULL) free((char*)nciop->path); @@ -246,7 +246,7 @@ httpio_close(ncio* nciop, int doUnlink) } /* - * Request that the region (offset, extent) + * Request that the interval (offset, extent) * be made available through *vpp. */ static int @@ -258,13 +258,13 @@ httpio_get(ncio* const nciop, off_t offset, size_t extent, int rflags, void** co if(nciop == NULL || nciop->pvt == NULL) {status = NC_EINVAL; goto done;} http = (NCHTTP*)nciop->pvt; - assert(http->region == NULL); - http->region = ncbytesnew(); - ncbytessetalloc(http->region,(unsigned long)extent); - if((status = nc_http_read(http->state,offset,extent,http->region))) + assert(http->interval == NULL); + http->interval = ncbytesnew(); + ncbytessetalloc(http->interval,(unsigned long)extent); + if((status = nc_http_read(http->state,offset,extent,http->interval))) goto done; - assert(ncbyteslength(http->region) == extent); - if(vpp) *vpp = ncbytescontents(http->region); + assert(ncbyteslength(http->interval) == extent); + if(vpp) *vpp = ncbytescontents(http->interval); done: return status; } @@ -286,8 +286,8 @@ httpio_rel(ncio* const nciop, off_t offset, int rflags) if(nciop == NULL || nciop->pvt == NULL) {status = NC_EINVAL; goto done;} http = (NCHTTP*)nciop->pvt; - ncbytesfree(http->region); - http->region = NULL; + ncbytesfree(http->interval); + http->interval = NULL; done: return status; } diff --git a/libsrc/v1hpg.c b/libsrc/v1hpg.c index f93b219bd..d1c668446 100644 --- a/libsrc/v1hpg.c +++ b/libsrc/v1hpg.c @@ -1254,7 +1254,7 @@ NC_computeshapes(NC3_INFO* ncp) for( /*NADA*/; vpp < end; vpp++) { status = NC_var_shape(*vpp, &ncp->dims); - if(status != NC_NOERR) + if(status != NC_NOERR) return(status); if(IS_RECVAR(*vpp)) @@ -1265,13 +1265,13 @@ NC_computeshapes(NC3_INFO* ncp) } else { - if(first_var == NULL) + if(first_var == NULL) first_var = *vpp; - /* - * Overwritten each time thru. - * Usually overwritten in first_rec != NULL clause below. - */ - ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len; + /* + * Overwritten each time thru. + * Usually overwritten in first_rec != NULL clause below. + */ + ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len; } } } diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 2c6e64d1b..28d3ac5f6 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -2108,8 +2108,15 @@ NC_freeglobalstate(void) nullfree(nc_globalstate->tempdir); nullfree(nc_globalstate->home); nullfree(nc_globalstate->cwd); - NC_rcclear(nc_globalstate->rcinfo); - free(nc_globalstate->rcinfo); + nullfree(nc_globalstate->aws.default_region); + nullfree(nc_globalstate->aws.config_file); + nullfree(nc_globalstate->aws.profile); + nullfree(nc_globalstate->aws.access_key_id); + nullfree(nc_globalstate->aws.secret_access_key); + if(nc_globalstate->rcinfo) { + NC_rcclear(nc_globalstate->rcinfo); + free(nc_globalstate->rcinfo); + } free(nc_globalstate); nc_globalstate = NULL; } diff --git a/nc_test/test_byterange.sh b/nc_test/test_byterange.sh index 5fb9f7706..7ec0f4655 100755 --- a/nc_test/test_byterange.sh +++ b/nc_test/test_byterange.sh @@ -14,9 +14,9 @@ URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files fi if test "x$FEATURE_S3TESTS" != xno ; then URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes" -URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes" -# Test alternate URL with no specified region -URL4e="http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" +URL4c="https://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" +# Test alternate URL with no specified region and using s3 protocol +URL4e="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes" fi if test "x$FEATURE_S3TESTS" = xyes ; then # Requires auth diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index d8cb5d4a5..3df945179 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -76,7 +76,10 @@ TESTS += run_strings.sh TESTS += run_scalar.sh TESTS += run_nulls.sh TESTS += run_notzarr.sh + +if ENABLE_EXTERNAL_SERVER_TESTS TESTS += run_external.sh +endif # Unlimited Dimension tests (at least in part) TESTS += run_mud.sh diff --git a/nczarr_test/run_external.sh b/nczarr_test/run_external.sh index bc019bc6d..bd4285f98 100755 --- a/nczarr_test/run_external.sh +++ b/nczarr_test/run_external.sh @@ -16,19 +16,24 @@ cd $ISOPATH TESTCASES= if test "x$FEATURE_BYTERANGE" = xyes && test "x$FEATURE_S3" = xyes && test "x$FP_ISCYGWIN" = x ; then -TESTCASES="${TESTCASES} OR_ABI;;http://s3.amazonaws.com/noaa-goes16/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" -TESTCASES="${TESTCASES} cesmLE;blosc;http://s3.us-west-2.amazonaws.com/ncar-cesm-lens/atm/daily/cesmLE-20C-FLNS.zarr#mode=zarr,s3" +TESTCASES="${TESTCASES} OR_ABI;;;http://s3.amazonaws.com/noaa-goes16/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" +TESTCASES="${TESTCASES} cesmLE;blosc;;http://s3.us-west-2.amazonaws.com/ncar-cesm-lens/atm/daily/cesmLE-20C-FLNS.zarr#mode=zarr,s3" +# Test TEST_REGION defaulting and s3 inferencing +TESTCASES="${TESTCASES} wtk;;us-west-2;https://nrel-pds-wtk.s3.amazonaws.com/wtk-techno-economic/pywtk-data/met_data/0/0.nc#mode=bytes" fi testcase() { NM=`echo "$1" | cut -d';' -f1` FILT=`echo "$1" | cut -d';' -f2` -URL=`echo "$1" | cut -d';' -f3` +REG=`echo "$1" | cut -d';' -f3` +URL=`echo "$1" | cut -d';' -f4` echo "*** Test: $NM = $URL" rm -f "tmp_external_$NM.cdl" if test "x$FILT" != x ; then if avail $FILT; then + if test "x$REG" != x ; then export TEST_REGION="$REG"; fi ${NCDUMP} -h -s -n $NM $URL > "tmp_external_${NM}.cdl" + unset TEST_REGION fi fi } diff --git a/s3gc.in b/s3gc.in index 612e371d7..a63345c6c 100755 --- a/s3gc.in +++ b/s3gc.in @@ -61,7 +61,9 @@ if ! aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTRE fi aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys while read -r line; do - KEY=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'` + KEY0=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'` + # Strip off any leading '/' + KEY=`echo "$KEY0" | sed -e 's|^[/]*\(.*\)|\1|'` # Ignore keys that do not start with ${S3TESTSUBTREE} PREFIX=`echo "$KEY" | sed -e 's|\([^/]*\)/.*|\1|'` if test "x$PREFIX" = "x$S3TESTSUBTREE" ; then