mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-11-21 03:13:42 +08:00
Merge pull request #2969 from mannreis/awsconfig
[S3] Parse AWS configuration with support for profile section
This commit is contained in:
Ward Fisher
GitHub
commit
62583f1371
@ -603,13 +603,18 @@ NC_getactives3profile(NCURI* uri, const char** profilep)
|
||||
int stat = NC_NOERR;
|
||||
const char* profile = NULL;
|
||||
struct AWSprofile* ap = NULL;
|
||||
struct NCglobalstate* gs = NC_getglobalstate();
|
||||
|
||||
profile = ncurifragmentlookup(uri,"aws.profile");
|
||||
if(profile == NULL)
|
||||
profile = NC_rclookupx(uri,"AWS.PROFILE");
|
||||
if (uri != NULL) {
|
||||
profile = ncurifragmentlookup(uri,"aws.profile");
|
||||
if(profile == NULL)
|
||||
profile = NC_rclookupx(uri,"AWS.PROFILE");
|
||||
}
|
||||
|
||||
if(profile == NULL)
|
||||
profile = NC_getglobalstate()->aws.profile;
|
||||
if(profile == NULL && gs->aws.profile != NULL) {
|
||||
if((stat=NC_authgets3profile(gs->aws.profile,&ap))) goto done;
|
||||
if(ap) profile = nulldup(gs->aws.profile);
|
||||
}
|
||||
|
||||
if(profile == NULL) {
|
||||
if((stat=NC_authgets3profile("default",&ap))) goto done;
|
||||
@ -837,13 +842,19 @@ awsparse(const char* text, NClist* profiles)
|
||||
if(token == AWS_EOF) break; /* finished */
|
||||
if(token == AWS_EOL) {continue;} /* blank line */
|
||||
if(token != LBR) {stat = NCTHROW(NC_EINVAL); goto done;}
|
||||
/* parse [profile name] */
|
||||
/* parse [profile name] or [name] */
|
||||
token = awslex(parser);
|
||||
if(token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;}
|
||||
assert(profile == NULL);
|
||||
if((profile = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)))==NULL)
|
||||
{stat = NC_ENOMEM; goto done;}
|
||||
profile->name = ncbytesextract(parser->yytext);
|
||||
if(strncmp("profile", profile->name, sizeof("profile")) == 0 ) {
|
||||
token = awslex(parser);
|
||||
if(token != AWS_WORD) {stat = NCTHROW(NC_EINVAL); goto done;}
|
||||
nullfree(profile->name);
|
||||
profile->name = ncbytesextract(parser->yytext);
|
||||
}
|
||||
profile->entries = nclistnew();
|
||||
token = awslex(parser);
|
||||
if(token != RBR) {stat = NCTHROW(NC_EINVAL); goto done;}
|
||||
@ -881,10 +892,22 @@ fprintf(stderr,">>> parse: entry=(%s,%s)\n",entry->key,entry->value);
|
||||
{stat = NCTHROW(NC_EINVAL); goto done;}
|
||||
}
|
||||
|
||||
/* If this profile already exists, then replace old one */
|
||||
/* If this profile already exists, then overwrite old one */
|
||||
for(size_t i=0;i<nclistlength(profiles);i++) {
|
||||
struct AWSprofile* p = (struct AWSprofile*)nclistget(profiles,i);
|
||||
if(strcasecmp(p->name,profile->name)==0) {
|
||||
// Keep unique parameters from previous (incomplete!?) profile
|
||||
for (size_t j=0;j<nclistlength(p->entries);j++){
|
||||
struct AWSentry* old = (struct AWSentry*)nclistget(p->entries,j);
|
||||
int add = 1;
|
||||
for (size_t z=0;z<nclistlength(profile->entries);z++){
|
||||
struct AWSentry* new = (struct AWSentry*)nclistget(profile->entries,z);
|
||||
add &= (strcasecmp(old->key,new->key)!=0);
|
||||
}
|
||||
if(add){
|
||||
nclistpush(profile->entries, nclistremove(p->entries,j--));
|
||||
}
|
||||
}
|
||||
nclistset(profiles,i,profile);
|
||||
profile = NULL;
|
||||
/* reclaim old one */
|
||||
|
||||
@ -37,6 +37,9 @@ IF(NETCDF_BUILD_UTILITIES)
|
||||
# SDK Test
|
||||
build_bin_test(test_s3sdk ${XGETOPTSRC})
|
||||
add_sh_test(unit_test run_s3sdk)
|
||||
#AWS Configuration test
|
||||
build_bin_test(aws_config)
|
||||
add_sh_test(unit_test run_aws_config)
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
|
||||
|
||||
@ -43,6 +43,8 @@ if NETCDF_ENABLE_S3_TESTALL
|
||||
check_PROGRAMS += test_s3sdk
|
||||
TESTS += run_s3sdk.sh
|
||||
endif
|
||||
check_PROGRAMS += aws_config
|
||||
TESTS += run_aws_config.sh
|
||||
endif
|
||||
|
||||
EXTRA_DIST = CMakeLists.txt run_s3sdk.sh run_reclaim_tests.sh
|
||||
|
||||
90
unit_test/aws_config.c
Normal file
90
unit_test/aws_config.c
Normal file
@ -0,0 +1,90 @@
|
||||
/*********************************************************************
|
||||
* Copyright 2018, UCAR/Unidata
|
||||
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
||||
*********************************************************************/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include "netcdf.h"
|
||||
#include "ncrc.h"
|
||||
#include "ncpathmgr.h"
|
||||
#include "ncs3sdk.h"
|
||||
#include "ncuri.h"
|
||||
#include "nc4internal.h"
|
||||
|
||||
NCS3INFO s3info;
|
||||
void* s3client = NULL;
|
||||
|
||||
/* Forward */
|
||||
static void cleanup(void);
|
||||
|
||||
#define STR(p) p?p:"NULL"
|
||||
#define CHECK(code) do {stat = check(code,__func__,__LINE__); if(stat) {goto done;}} while(0)
|
||||
|
||||
static int
|
||||
check(int code, const char* fcn, int line)
|
||||
{
|
||||
if(code == NC_NOERR) return code;
|
||||
fprintf(stderr,"***FAIL: (%d) %s @ %s:%d\n",code,nc_strerror(code),fcn,line);
|
||||
abort();
|
||||
}
|
||||
|
||||
static void
|
||||
cleanup(void)
|
||||
{
|
||||
if(s3client)
|
||||
NC_s3sdkclose(s3client, &s3info, 0/*deleteit*/, NULL);
|
||||
s3client = NULL;
|
||||
NC_s3clear(&s3info);
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char** argv)
|
||||
{
|
||||
int c = 0,stat = NC_NOERR;
|
||||
|
||||
/* Load RC and .aws/config */
|
||||
CHECK(nc_initialize());
|
||||
CHECK(NC_s3sdkinitialize());
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
//Not checking, aborts if ~/.aws/config doesn't exist
|
||||
CHECK(NC_aws_load_credentials(gs));
|
||||
|
||||
// Lets ensure the active profile is loaded
|
||||
// from the configurtion files instead of an URL
|
||||
const char* activeprofile = NULL;
|
||||
CHECK(NC_getactives3profile(NULL, &activeprofile));
|
||||
|
||||
fprintf(stderr, "Active profile:%s\n", STR(activeprofile));
|
||||
|
||||
// ARGV contains should contain "key[=value]" to verify
|
||||
// if key was parsed when loading the aws config and if it's
|
||||
// value is updated in case it's redefined on the .aws/credentials
|
||||
for(int i = 1; i < argc; i++) {
|
||||
const char *argkey = strtok(argv[i],"=");
|
||||
const char *argvalue = strtok(NULL,"=");
|
||||
const char* value = NULL;
|
||||
|
||||
NC_s3profilelookup(activeprofile,argkey,&value);
|
||||
fprintf(stderr, "%s\t%s -> %s\n",value?"":"*** FAIL:", argv[i],value?value:"NOT DEFINED!");
|
||||
if ( value == NULL
|
||||
|| (argvalue != NULL
|
||||
&& strncmp(value, argvalue, strlen(value)))
|
||||
){
|
||||
c++;
|
||||
stat |= NC_ERCFILE;
|
||||
}
|
||||
}
|
||||
|
||||
done:
|
||||
cleanup();
|
||||
if(stat)
|
||||
printf("*** FAIL: a total of %d keys were not found on the profile %s\n", c, STR(activeprofile));
|
||||
else
|
||||
printf("***PASS\n");
|
||||
(void)NC_s3sdkfinalize();
|
||||
(void)nc_finalize();
|
||||
exit(stat?:0);
|
||||
}
|
||||
69
unit_test/run_aws_config.sh
Executable file
69
unit_test/run_aws_config.sh
Executable file
@ -0,0 +1,69 @@
|
||||
#!/bin/sh
|
||||
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
set -e
|
||||
|
||||
#CMD="valgrind --leak-check=full"
|
||||
|
||||
isolate "testdir_ut_aws_config"
|
||||
|
||||
THISDIR=`pwd`
|
||||
cd $ISOPATH
|
||||
|
||||
mkdir -p $THISDIR/.aws/
|
||||
|
||||
test_cleanup() {
|
||||
rm -rfv $THISDIR/.aws/
|
||||
}
|
||||
trap test_cleanup EXIT
|
||||
|
||||
cat << 'EOF' > $THISDIR/.aws/config
|
||||
[uni]
|
||||
region = somewhere-1
|
||||
endpoint_url = https://example.com/bucket/prefix/1
|
||||
key = value
|
||||
extrakey = willbepropagated
|
||||
|
||||
[profile unidata]
|
||||
region = us-east-1
|
||||
endpoint_url = https://s3.example.domain/
|
||||
dummy_key = dummy_value
|
||||
|
||||
[profile play]
|
||||
region = us-east-1
|
||||
endpoint_url = https://endpoint.example.com/
|
||||
EOF
|
||||
|
||||
cat << 'EOF' > $THISDIR/.aws/credentials
|
||||
[play]
|
||||
aws_access_key_id = DummyKeys
|
||||
aws_secret_access_key = DummySecret
|
||||
|
||||
[uni]
|
||||
region = somewhere-2
|
||||
endpoint_url = https://example.com/bucket/prefix/2
|
||||
key = value-overwritten
|
||||
EOF
|
||||
|
||||
echo -e "Testing loading AWS configuration in ${THISDIR}/.aws/config"
|
||||
NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=unidata ${CMD} ${execdir}/aws_config endpoint_url region dummy_key
|
||||
echo "Status: $?"
|
||||
|
||||
NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=play ${CMD} ${execdir}/aws_config endpoint_url region
|
||||
echo "Status: $?"
|
||||
|
||||
NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=uni ${CMD} ${execdir}/aws_config endpoint_url region key
|
||||
echo "Status: $?"
|
||||
|
||||
NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=uni ${CMD} ${execdir}/aws_config key=value-overwritten region=somewhere-2 endpoint_url=https://example.com/bucket/prefix/2 extrakey=willbepropagated
|
||||
echo "Status: $?"
|
||||
|
||||
# Will use profile=no
|
||||
NC_TEST_AWS_DIR=${THISDIR} ${CMD} ${execdir}/aws_config 2>&1 | grep -q 'Active profile:no'
|
||||
echo "Status: $?"
|
||||
echo -e "Finished"
|
||||
|
||||
exit
|
||||
|
||||
Loading…
Reference in New Issue
Block a user