mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-21 03:13:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix duplicate definition when using aws-sdk-cpp.

Browse Source 
re: Issue https://github.com/Unidata/netcdf-c/issues/2927

The NC_s3sdkinitialize NC_s3sdkfinalize functions were
misplaced. They should have been moved from ds3util.c to
ncs3sdk_h5.c.  When using ncs3sdl_aws.cpp, this resulted in a
duplicate definition.

Also, found and fixed a memory leak in the NCZarr S3 code.
This commit is contained in:
Dennis Heimbigner 2024-05-20 19:15:19 -06:00
parent 517d0b45c4
commit 9a478edb06
6 changed files with 62 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/ncs3sdk.h
View File

@ -45,6 +45,7 @@ struct NCglobalstate;
extern "C" {
#endif
/* API for ncs3sdk_XXX.[c|cpp] */
EXTERNL int NC_s3sdkinitialize(void);
EXTERNL int NC_s3sdkfinalize(void);
EXTERNL void* NC_s3sdkcreateclient(NCS3INFO* context);
@ -60,8 +61,7 @@ EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* pref
EXTERNL int NC_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp);
/* From ds3util.c */
EXTERNL int NC_s3sdkinitialize(void);
EXTERNL int NC_s3sdkfinalize(void);
EXTERNL void NC_s3sdkenvironment(void);
EXTERNL int NC_getdefaults3region(NCURI* uri, const char** regionp);
EXTERNL int NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp);

48
libdispatch/ds3util.c
View File

@ -43,9 +43,6 @@ enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4};
static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL};
#define NCONFIGFILES (sizeof(awsconfigfiles)/sizeof(char*))
static int ncs3_initialized = 0;
static int ncs3_finalized = 0;
/**************************************************/
/* Forward */
@ -56,38 +53,21 @@ static int awsparse(const char* text, NClist* profiles);
/**************************************************/
/* Capture environmental Info */
EXTERNL int
NC_s3sdkinitialize(void)
EXTERNL void
NC_s3sdkenvironment(void)
{
if(!ncs3_initialized) {
ncs3_initialized = 1;
ncs3_finalized = 0;
}
{
/* Get various environment variables as defined by the AWS sdk */
NCglobalstate* gs = NC_getglobalstate();
if(getenv("AWS_REGION")!=NULL)
gs->aws.default_region = nulldup(getenv("AWS_REGION"));
else if(getenv("AWS_DEFAULT_REGION")!=NULL)
gs->aws.default_region = nulldup(getenv("AWS_DEFAULT_REGION"));
else if(gs->aws.default_region == NULL)
gs->aws.default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION);
gs->aws.access_key_id = nulldup(getenv("AWS_ACCESS_KEY_ID"));
gs->aws.config_file = nulldup(getenv("AWS_CONFIG_FILE"));
gs->aws.profile = nulldup(getenv("AWS_PROFILE"));
gs->aws.secret_access_key = nulldup(getenv("AWS_SECRET_ACCESS_KEY"));
}
return NC_NOERR;
}
EXTERNL int
NC_s3sdkfinalize(void)
{
if(!ncs3_finalized) {
ncs3_initialized = 0;
ncs3_finalized = 1;
}
return NC_NOERR;
/* Get various environment variables as defined by the AWS sdk */
NCglobalstate* gs = NC_getglobalstate();
if(getenv("AWS_REGION")!=NULL)
gs->aws.default_region = nulldup(getenv("AWS_REGION"));
else if(getenv("AWS_DEFAULT_REGION")!=NULL)
gs->aws.default_region = nulldup(getenv("AWS_DEFAULT_REGION"));
else if(gs->aws.default_region == NULL)
gs->aws.default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION);
gs->aws.access_key_id = nulldup(getenv("AWS_ACCESS_KEY_ID"));
gs->aws.config_file = nulldup(getenv("AWS_CONFIG_FILE"));
gs->aws.profile = nulldup(getenv("AWS_PROFILE"));
gs->aws.secret_access_key = nulldup(getenv("AWS_SECRET_ACCESS_KEY"));
}
/**************************************************/

9
libdispatch/ncs3sdk_aws.cpp
View File

@ -133,10 +133,9 @@ NC_s3sdkinitialize(void)
if(!ncs3_initialized) {
ncs3_initialized = 1;
ncs3_finalized = 0;
#ifdef DEBUG
//ncs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
//ncs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
ncs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Trace;
ncs3options.httpOptions.installSigPipeHandler = true;
ncs3options.loggingOptions.logger_create_fn = [] { return std::make_shared<Aws::Utils::Logging::ConsoleLogSystem>(Aws::Utils::Logging::LogLevel::Trace); };
@ -144,6 +143,9 @@ NC_s3sdkinitialize(void)
#endif
Aws::InitAPI(ncs3options);
/* Get environment information */
NC_s3sdkenvironment();
}
return NCUNTRACE(NC_NOERR);
}
@ -500,7 +502,6 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey, s
int stat = NC_NOERR;
const char* key = NULL;
const char* mcontent = (char*)content;
NCTRACE(11,"bucket=%s pathkey=%s count=%lld content=%p",bucket,pathkey,count,content);
AWSS3CLIENT s3client = (AWSS3CLIENT)s3client0;
@ -535,7 +536,7 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey, s
put_request.SetContentLength((long long)count);
std::shared_ptr<Aws::IOStream> data = std::shared_ptr<Aws::IOStream>(new Aws::StringStream());
data->rdbuf()->pubsetbuf((char*)content,count);
data->rdbuf()->pubsetbuf((char*)content,(std::streamsize)count);
put_request.SetBody(data);
auto put_result = AWSS3GET(s3client)->PutObject(put_request);
if(!put_result.IsSuccess()) {

31
libdispatch/ncs3sdk_h5.c
View File

@ -108,6 +108,37 @@ static int queryinsert(NClist* list, char* ekey, char* evalue);
#define NT(x) ((x)==NULL?"null":x)
/**************************************************/
static int ncs3_initialized = 0;
static int ncs3_finalized = 0;
EXTERNL int
NC_s3sdkinitialize(void)
{
if(!ncs3_initialized) {
ncs3_initialized = 1;
ncs3_finalized = 0;
}
/* Get environment information */
NC_s3sdkenvironment(void);
return NC_NOERR;
}
EXTERNL int
NC_s3sdkfinalize(void)
{
if(!ncs3_finalized) {
ncs3_initialized = 0;
ncs3_finalized = 1;
}
return NC_NOERR;
}
/**************************************************/
#if 0
static void
dumps3info(NCS3INFO* s3info, const char* tag)

11
libnczarr/zmap_s3sdk.c
View File

@ -499,20 +499,21 @@ s3clear(void* s3client, const char* bucket, const char* rootkey)
{
int stat = NC_NOERR;
char** list = NULL;
char** p;
size_t nkeys = 0;
if(s3client && bucket && rootkey) {
if((stat = NC_s3sdksearch(s3client, bucket, rootkey, &nkeys, &list, NULL)))
goto done;
if(list != NULL) {
for(p=list;*p;p++) {
size_t i;
for(i=0;i<nkeys;i++) {
char* p = list[i];
/* If the key is the rootkey, skip it */
if(strcmp(rootkey,*p)==0) continue;
if(strcmp(rootkey,p)==0) continue;
#ifdef S3DEBUG
fprintf(stderr,"s3clear: %s\n",*p);
fprintf(stderr,"s3clear: %s\n",p);
#endif
if((stat = NC_s3sdkdeletekey(s3client, bucket, *p, NULL)))
if((stat = NC_s3sdkdeletekey(s3client, bucket, p, NULL)))
goto done;
}
}

5
nczarr_test/run_corrupt.sh
View File

@ -32,4 +32,7 @@ testnoshape2() {
}
testnoshape1
if test "x$FEATURE_S3TESTS" = xyes ; then testnoshape2; fi
if test "x$FEATURE_S3TESTS" = xyes && test "x$FEATURE_S3_INTERNAL" = xyes ; then
# The aws-sdk-cpp driver does not support google storage
testnoshape2
fi