mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-11-21 03:13:42 +08:00
f6e25b695e
re: https://github.com/Unidata/netcdf-c/issues/2117 re: https://github.com/Unidata/netcdf-c/issues/2119 * Modify libsrc to allow byte-range reading of netcdf-3 files in private S3 buckets; this required using the aws sdk. Also add a test case. * The aws sdk can sometimes cause problems if the Awd::ShutdownAPI function is not called. So at optional atexit() support to ensure it is called. This is disabled for Windows. * Add documentation to nczarr.md on how to build and use the aws sdk under windows. Currently it builds, but testing fails. * Switch testing from stratus to the Unidata bucket on S3. * Improve support for the s3: url protocol. * Add a s3 specific utility code file: ds3util.c * Modify NC_infermodel to attempt to read the magic number of byte-ranged files in S3. ## Misc. * Move and rename the core S3 SDK wrapper code (libnczarr/zs3sdk.cpp) to libdispatch since it now used in libsrc as well as libnczarr. * Add calls to nc_finalize in the utilities in case atexit is disabled. * Add header only json parser to the distribution rather than as a built source.
726 lines
22 KiB
C++
726 lines
22 KiB
C++
/*
|
|
* Copyright 2018, University Corporation for Atmospheric Research
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
*/
|
|
|
|
/* WARNING: changes to this file may need to be propagated to libsrc/s3sdk.cpp */
|
|
|
|
#define NOOP
|
|
#define DEBUG
|
|
|
|
#include "awsincludes.h"
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <iostream>
|
|
#include <streambuf>
|
|
#include "netcdf.h"
|
|
#include "ncrc.h"
|
|
|
|
#include "ncs3sdk.h"
|
|
|
|
#undef NCTRACING
|
|
#ifdef NCTRACING
|
|
#define NCTRACE(level,fmt,...) nctrace((level),__func__,fmt,##__VA_ARGS__)
|
|
#define NCTRACEMORE(level,fmt,...) nctracemore((level),fmt,##__VA_ARGS__)
|
|
#define NCUNTRACE(e) ncuntrace(__func__,THROW(e),NULL)
|
|
#define NCUNTRACEX(e,fmt,...) ncuntrace(__func__,THROW(e),fmt,##__VA_ARGS__)
|
|
#else
|
|
#define NCTRACE(level,fmt,...)
|
|
#define NCTRACEMORE(level,fmt,...)
|
|
#define NCUNTRACE(e) (e)
|
|
#define NCUNTRACEX(e,fmt,...) (e)
|
|
#endif
|
|
|
|
#ifdef __CYGWIN__
|
|
extern char* strdup(const char*);
|
|
#endif
|
|
|
|
#define size64_t unsigned long long
|
|
|
|
static int ncs3_initialized = 0;
|
|
static int ncs3_finalized = 0;
|
|
|
|
static Aws::SDKOptions ncs3options;
|
|
|
|
/* Forward */
|
|
static Aws::S3::Model::BucketLocationConstraint s3findregion(const char* name);
|
|
static int s3objectinfo1(const Aws::S3::Model::Object& s3_object, char** fullkeyp, size64_t* lenp);
|
|
static int s3objectsinfo(Aws::Vector<Aws::S3::Model::Object> list, size_t*, char*** keysp, size64_t** lenp);
|
|
static int s3commonprefixes(Aws::Vector<Aws::S3::Model::CommonPrefix> list, char*** keysp);
|
|
static void freestringenvv(char** ss);
|
|
static int makes3key(const char* pathkey, const char** keyp);
|
|
static int makes3keydir(const char* prefix, char** prefixdirp);
|
|
static char** mergekeysets(size_t nkeys1, char** keys1, size_t nkeys2, char** keys2);
|
|
|
|
static const char*
|
|
dumps3info(NCS3INFO* info)
|
|
{
|
|
static char text[8192];
|
|
snprintf(text,sizeof(text),"host=%s region=%s bucket=%s rootkey=%s profile=%s",
|
|
(info->host?info->host:"null"),
|
|
(info->region?info->region:"null"),
|
|
(info->bucket?info->bucket:"null"),
|
|
(info->rootkey?info->rootkey:"null"),
|
|
(info->profile?info->profile:"null"));
|
|
return text;
|
|
}
|
|
|
|
EXTERNL void
|
|
NC_s3sdkinitialize(void)
|
|
{
|
|
if(!ncs3_initialized) {
|
|
ncs3_initialized = 1;
|
|
ncs3_finalized = 0;
|
|
NCTRACE(11,NULL);
|
|
Aws::InitAPI(ncs3options);
|
|
NCUNTRACE(NC_NOERR);
|
|
}
|
|
}
|
|
|
|
EXTERNL void
|
|
NC_s3sdkfinalize(void)
|
|
{
|
|
if(!ncs3_finalized) {
|
|
ncs3_initialized = 0;
|
|
ncs3_finalized = 1;
|
|
NCTRACE(11,NULL);
|
|
Aws::ShutdownAPI(ncs3options);
|
|
NCUNTRACE(NC_NOERR);
|
|
}
|
|
}
|
|
|
|
static char*
|
|
makeerrmsg(const Aws::Client::AWSError<Aws::S3::S3Errors> err, const char* key="")
|
|
{
|
|
char* errmsg;
|
|
size_t len;
|
|
len = strlen(err.GetExceptionName().c_str()) + strlen(err.GetMessage().c_str()) + strlen(key) + 10;
|
|
if((errmsg = (char*)malloc(len+1))==NULL)
|
|
return NULL;
|
|
snprintf(errmsg,len,"%s %s key=%s",
|
|
err.GetExceptionName().c_str(),
|
|
err.GetMessage().c_str(),
|
|
key);
|
|
return errmsg;
|
|
}
|
|
|
|
|
|
static Aws::Client::ClientConfiguration
|
|
s3sdkcreateconfig(NCS3INFO* info)
|
|
{
|
|
int stat = NC_NOERR;
|
|
NCTRACE(11,"info=%s", dumps3info(info));
|
|
|
|
Aws::Client::ClientConfiguration config;
|
|
|
|
if(info->profile)
|
|
#if 0
|
|
config = new Aws::Client::ClientConfiguration(info->profile);
|
|
else
|
|
config = new Aws::Client::ClientConfiguration();
|
|
#else
|
|
config.profileName = info->profile;
|
|
#endif
|
|
config.scheme = Aws::Http::Scheme::HTTPS;
|
|
config.connectTimeoutMs = 300000;
|
|
config.requestTimeoutMs = 600000;
|
|
if(info->region) config.region = info->region;
|
|
if(info->host) config.endpointOverride = info->host;
|
|
config.enableEndpointDiscovery = true;
|
|
config.followRedirects = Aws::Client::FollowRedirectsPolicy::ALWAYS;
|
|
NCUNTRACE(NC_NOERR);
|
|
return config;
|
|
}
|
|
|
|
EXTERNL void*
|
|
NC_s3sdkcreateclient(NCS3INFO* info)
|
|
{
|
|
NCTRACE(11,NULL);
|
|
|
|
Aws::Client::ClientConfiguration config = s3sdkcreateconfig(info);
|
|
Aws::S3::S3Client *s3client;
|
|
|
|
if(info->profile == NULL || strcmp(info->profile,"none")==0) {
|
|
Aws::Auth::AWSCredentials creds;
|
|
creds.SetAWSAccessKeyId(Aws::String(""));
|
|
creds.SetAWSSecretKey(Aws::String(""));
|
|
s3client = new Aws::S3::S3Client(creds,config,
|
|
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent,
|
|
false);
|
|
} else {
|
|
s3client = new Aws::S3::S3Client(config,
|
|
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent,
|
|
false);
|
|
}
|
|
// delete config;
|
|
NCUNTRACE(NC_NOERR);
|
|
return (void*)s3client;
|
|
}
|
|
|
|
EXTERNL int
|
|
NC_s3sdkbucketexists(void* s3client0, const char* bucket, int* existsp, char** errmsgp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
int exists = 0;
|
|
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
|
|
|
|
NCTRACE(11,"bucket=%s",bucket);
|
|
if(errmsgp) *errmsgp = NULL;
|
|
auto result = s3client->ListBuckets();
|
|
if(!result.IsSuccess()) {
|
|
if(errmsgp) *errmsgp = makeerrmsg(result.GetError());
|
|
stat = NC_ES3;
|
|
} else {
|
|
Aws::Vector<Aws::S3::Model::Bucket> bucket_list = result.GetResult().GetBuckets();
|
|
for(auto const &awsbucket : bucket_list) {
|
|
auto name = awsbucket.GetName();
|
|
if(name == bucket) {exists = 1; break;}
|
|
}
|
|
}
|
|
if(existsp) *existsp = exists;
|
|
return NCUNTRACEX(stat,"exists=%d",*existsp);
|
|
}
|
|
|
|
EXTERNL int
|
|
NC_s3sdkbucketcreate(void* s3client0, const char* region, const char* bucket, char** errmsgp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
|
|
NCTRACE(11,"region=%s bucket=%s",region,bucket);
|
|
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
|
|
if(errmsgp) *errmsgp = NULL;
|
|
const Aws::S3::Model::BucketLocationConstraint &awsregion = s3findregion(region);
|
|
if(awsregion == Aws::S3::Model::BucketLocationConstraint::NOT_SET)
|
|
return NC_EURL;
|
|
/* Set up the request */
|
|
Aws::S3::Model::CreateBucketRequest create_request;
|
|
create_request.SetBucket(bucket);
|
|
if(region) {
|
|
/* Specify the region as a location constraint */
|
|
Aws::S3::Model::CreateBucketConfiguration bucket_config;
|
|
bucket_config.SetLocationConstraint(awsregion);
|
|
create_request.SetCreateBucketConfiguration(bucket_config);
|
|
}
|
|
#ifdef NOOP
|
|
/* Create the bucket */
|
|
auto create_result = s3client->CreateBucket(create_request);
|
|
if(!create_result.IsSuccess()) {
|
|
if(errmsgp) *errmsgp = makeerrmsg(create_result.GetError());
|
|
stat = NC_ES3;;
|
|
}
|
|
#else
|
|
fprintf(stderr,"create bucket: %s\n",bucket); fflush(stderr);
|
|
#endif
|
|
|
|
return NCUNTRACE(stat);
|
|
}
|
|
|
|
EXTERNL int
|
|
NC_s3sdkbucketdelete(void* s3client0, NCS3INFO* info, char** errmsgp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
|
|
NCTRACE(11,"info=%s%s",dumps3info(info));
|
|
|
|
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
|
|
|
|
if(errmsgp) *errmsgp = NULL;
|
|
const Aws::S3::Model::BucketLocationConstraint &awsregion = s3findregion(info->region);
|
|
if(awsregion == Aws::S3::Model::BucketLocationConstraint::NOT_SET)
|
|
return NC_EURL;
|
|
/* Set up the request */
|
|
Aws::S3::Model::DeleteBucketRequest request;
|
|
request.SetBucket(info->bucket);
|
|
|
|
#ifdef NOOP
|
|
/* Delete the bucket */
|
|
auto result = s3client->DeleteBucket(request);
|
|
if(!result.IsSuccess()) {
|
|
if(errmsgp) *errmsgp = makeerrmsg(result.GetError());
|
|
stat = NC_ES3;;
|
|
}
|
|
#else
|
|
fprintf(stderr,"delete bucket: %s\n",bucket); fflush(stderr);
|
|
#endif
|
|
|
|
return NCUNTRACE(stat);
|
|
}
|
|
|
|
/**************************************************/
|
|
/* Object API */
|
|
|
|
/*
|
|
@return NC_NOERR if key points to a content-bearing object.
|
|
@return NC_EEMPTY if object at key has no content.
|
|
@return NC_EXXX return true error
|
|
*/
|
|
EXTERNL int
|
|
NC_s3sdkinfo(void* s3client0, const char* bucket, const char* pathkey, size64_t* lenp, char** errmsgp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
const char* key = NULL;
|
|
|
|
NCTRACE(11,"bucket=%s pathkey=%s",bucket,pathkey);
|
|
|
|
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
|
|
Aws::S3::Model::HeadObjectRequest head_request;
|
|
if(*pathkey != '/') return NC_EINTERNAL;
|
|
/* extract the true s3 key*/
|
|
if((stat = makes3key(pathkey,&key))) return NCUNTRACE(stat);
|
|
|
|
if(errmsgp) *errmsgp = NULL;
|
|
head_request.SetBucket(bucket);
|
|
head_request.SetKey(key);
|
|
auto head_outcome = s3client->HeadObject(head_request);
|
|
if(head_outcome.IsSuccess()) {
|
|
long long l = head_outcome.GetResult().GetContentLength();
|
|
if(lenp) *lenp = (size64_t)l;
|
|
} else {
|
|
if(lenp) *lenp = 0;
|
|
/* Distinquish not-found from other errors */
|
|
switch (head_outcome.GetError().GetErrorType()) {
|
|
case Aws::S3::S3Errors::RESOURCE_NOT_FOUND:
|
|
stat = NC_EEMPTY;
|
|
break;
|
|
case Aws::S3::S3Errors::ACCESS_DENIED:
|
|
stat = NC_EACCESS;
|
|
/* fall thru */
|
|
default:
|
|
if(!stat) stat = NC_ES3;
|
|
if(errmsgp) *errmsgp = makeerrmsg(head_outcome.GetError(),key);
|
|
break;
|
|
}
|
|
}
|
|
return NCUNTRACEX(stat,"len=%d",(int)(lenp?*lenp:-1));
|
|
}
|
|
|
|
/*
|
|
@return NC_NOERR if success
|
|
@return NC_EXXX if fail
|
|
*/
|
|
EXTERNL int
|
|
NC_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t start, size64_t count, void* content, char** errmsgp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
char range[1024];
|
|
const char* key = NULL;
|
|
size64_t rangeend;
|
|
|
|
NCTRACE(11,"bucket=%s pathkey=%s start=%llu count=%llu content=%p",bucket,pathkey,start,count,content);
|
|
|
|
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
|
|
Aws::S3::Model::GetObjectRequest object_request;
|
|
|
|
if(count == 0) return NCUNTRACE(stat);
|
|
|
|
if(*pathkey != '/') return NC_EINTERNAL;
|
|
if((stat = makes3key(pathkey,&key))) return NCUNTRACE(stat);
|
|
|
|
object_request.SetBucket(bucket);
|
|
object_request.SetKey(key);
|
|
rangeend = (start+count)-1;
|
|
snprintf(range,sizeof(range),"bytes=%llu-%llu",start,rangeend);
|
|
object_request.SetRange(range);
|
|
auto get_object_result = s3client->GetObject(object_request);
|
|
if(!get_object_result.IsSuccess()) {
|
|
if(errmsgp) *errmsgp = makeerrmsg(get_object_result.GetError(),key);
|
|
stat = NC_ES3;
|
|
} else {
|
|
/* Get the whole result */
|
|
Aws::IOStream &result = get_object_result.GetResultWithOwnership().GetBody();
|
|
std::string str((std::istreambuf_iterator<char>(result)),std::istreambuf_iterator<char>());
|
|
/* Verify actual result size */
|
|
size_t slen = str.size();
|
|
if(slen > count) return NC_ES3;
|
|
const char* s = str.c_str();
|
|
if(content)
|
|
memcpy(content,s,slen);
|
|
}
|
|
return NCUNTRACE(stat);
|
|
}
|
|
|
|
/*
|
|
For S3, I can see no way to do a byterange write;
|
|
so we are effectively writing the whole object
|
|
*/
|
|
EXTERNL int
|
|
NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey, size64_t count, const void* content, char** errmsgp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
const char* key = NULL;
|
|
|
|
NCTRACE(11,"bucket=%s pathkey=%s count=%lld content=%p",bucket,pathkey,count,content);
|
|
|
|
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
|
|
Aws::S3::Model::PutObjectRequest put_request;
|
|
|
|
if(*pathkey != '/') return NCUNTRACE(NC_EINTERNAL);
|
|
if((stat = makes3key(pathkey,&key))) return NCUNTRACE(stat);
|
|
|
|
if(errmsgp) *errmsgp = NULL;
|
|
put_request.SetBucket(bucket);
|
|
put_request.SetKey(key);
|
|
put_request.SetContentLength((long long)count);
|
|
|
|
std::shared_ptr<Aws::IOStream> data = std::shared_ptr<Aws::IOStream>(new Aws::StringStream());
|
|
data->rdbuf()->pubsetbuf((char*)content,count);
|
|
put_request.SetBody(data);
|
|
auto put_result = s3client->PutObject(put_request);
|
|
if(!put_result.IsSuccess()) {
|
|
if(errmsgp) *errmsgp = makeerrmsg(put_result.GetError(),key);
|
|
stat = NC_ES3;
|
|
}
|
|
return NCUNTRACE(stat);
|
|
}
|
|
|
|
EXTERNL int
|
|
NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** errmsgp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
|
|
NCTRACE(11,"info=%s rootkey=%s deleteit=%d",dumps3info(info),deleteit);
|
|
|
|
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
|
|
if(deleteit) {
|
|
/* Delete the root key; ok it if does not exist */
|
|
switch (stat = NC_s3sdkdeletekey(s3client0,info->bucket,info->rootkey,errmsgp)) {
|
|
case NC_NOERR: break;
|
|
case NC_EEMPTY: case NC_ENOTFOUND: stat = NC_NOERR; break;
|
|
default: break;
|
|
}
|
|
}
|
|
delete s3client;
|
|
return NCUNTRACE(stat);
|
|
}
|
|
|
|
/*
|
|
Return a list of names of legal objects immediately below a specified key.
|
|
In theory, the returned list should be sorted in lexical order,
|
|
but it possible that it is not.
|
|
*/
|
|
EXTERNL int
|
|
NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
size_t nkeys = 0;
|
|
const char* prefix = NULL;
|
|
char* prefixdir = NULL;
|
|
char** realkeys = NULL;
|
|
char** commonkeys = NULL;
|
|
char** allkeys = NULL;
|
|
|
|
NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
|
|
|
|
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
|
|
Aws::S3::Model::ListObjectsV2Request objects_request;
|
|
|
|
if(*prefixkey0 != '/') return NCUNTRACE(NC_EINTERNAL);
|
|
/* Make sure that the prefix ends with '/' */
|
|
if((stat = makes3keydir(prefixkey0,&prefixdir))) return NCUNTRACE(stat);
|
|
/* remove leading '/' */
|
|
if((stat = makes3key(prefixdir,&prefix))) return NCUNTRACE(stat);
|
|
|
|
if(errmsgp) *errmsgp = NULL;
|
|
objects_request.SetBucket(bucket);
|
|
objects_request.SetPrefix(prefix);
|
|
objects_request.SetDelimiter("/"); /* Force return of common prefixes */
|
|
auto objects_outcome = s3client->ListObjectsV2(objects_request);
|
|
if(objects_outcome.IsSuccess()) {
|
|
size_t nrealkeys = 0;
|
|
size_t ncommonkeys = 0;
|
|
Aws::Vector<Aws::S3::Model::Object> object_list =
|
|
objects_outcome.GetResult().GetContents();
|
|
nrealkeys = (size_t)object_list.size();
|
|
stat = s3objectsinfo(object_list,NULL,&realkeys,NULL);
|
|
/* Add common prefixes */
|
|
Aws::Vector<Aws::S3::Model::CommonPrefix> common_list =
|
|
objects_outcome.GetResult().GetCommonPrefixes();
|
|
ncommonkeys = (size_t)common_list.size();
|
|
stat = s3commonprefixes(common_list,&commonkeys);
|
|
/* merge the two lists */
|
|
if((allkeys=mergekeysets(nrealkeys, realkeys, ncommonkeys, commonkeys))==NULL)
|
|
stat = NC_ENOMEM;
|
|
if(stat == NC_NOERR) {
|
|
if(nkeysp) {*nkeysp = nrealkeys + ncommonkeys;}
|
|
if(keysp) {*keysp = allkeys; allkeys = NULL;}
|
|
}
|
|
freestringenvv(allkeys);
|
|
freestringenvv(realkeys);
|
|
freestringenvv(commonkeys);
|
|
} else {
|
|
if(errmsgp) *errmsgp = makeerrmsg(objects_outcome.GetError());
|
|
stat = NC_ES3;
|
|
}
|
|
if(prefixdir) free(prefixdir);
|
|
return NCUNTRACEX(stat,"nkeys=%u",(unsigned)*nkeysp);
|
|
}
|
|
|
|
/*
|
|
Return a list of full keys of legal objects immediately below a specified key.
|
|
Not necessarily sorted.
|
|
*/
|
|
EXTERNL int
|
|
NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
size_t nkeys = 0;
|
|
const char* prefix = NULL;
|
|
char* prefixdir = NULL;
|
|
char** keys = NULL;
|
|
|
|
NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
|
|
|
|
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
|
|
Aws::S3::Model::ListObjectsV2Request objects_request;
|
|
|
|
if(*prefixkey0 != '/') return NCUNTRACE(NC_EINTERNAL);
|
|
/* Make sure that the prefix ends with '/' */
|
|
if((stat = makes3keydir(prefixkey0,&prefixdir))) return NCUNTRACE(stat);
|
|
/* remove leading '/' */
|
|
if((stat = makes3key(prefixdir,&prefix))) return NCUNTRACE(stat);
|
|
|
|
if(errmsgp) *errmsgp = NULL;
|
|
objects_request.SetBucket(bucket);
|
|
objects_request.SetPrefix(prefix);
|
|
/* Do not use delimiter so we get all full key paths */
|
|
auto objects_outcome = s3client->ListObjectsV2(objects_request);
|
|
if(objects_outcome.IsSuccess()) {
|
|
size_t nkeys = 0;
|
|
Aws::Vector<Aws::S3::Model::Object> object_list =
|
|
objects_outcome.GetResult().GetContents();
|
|
nkeys = (size_t)object_list.size();
|
|
if((keys=(char**)calloc(sizeof(char*),(nkeys+1)))==NULL) /* NULL terminate list */
|
|
stat = NC_ENOMEM;
|
|
if(!stat) {
|
|
int i;
|
|
i = 0;
|
|
for (auto const &s3obj : object_list) {
|
|
const char* s;
|
|
const Aws::String& name = s3obj.GetKey();
|
|
s = name.c_str();
|
|
if(s != NULL) {
|
|
char* p;
|
|
size_t slen = name.length();
|
|
p = (char*)malloc(slen+1+1); /* nul plus leading '/' */
|
|
if(s[0] != '/') {p[0] = '/'; memcpy(p+1,s,slen); slen++;} else {memcpy(p,s,slen);}
|
|
p[slen] = '\0';
|
|
keys[i++] = p;
|
|
}
|
|
}
|
|
}
|
|
if(stat == NC_NOERR) {
|
|
if(nkeysp) {*nkeysp = nkeys;}
|
|
if(keysp) {*keysp = keys; keys = NULL;}
|
|
}
|
|
freestringenvv(keys);
|
|
} else {
|
|
if(errmsgp) *errmsgp = makeerrmsg(objects_outcome.GetError());
|
|
stat = NC_ES3;
|
|
}
|
|
if(prefixdir) free(prefixdir);
|
|
return NCUNTRACEX(stat,"nkeys=%u",(unsigned)*nkeysp);
|
|
}
|
|
|
|
EXTERNL int
|
|
NC_s3sdkdeletekey(void* s3client0, const char* bucket, const char* pathkey, char** errmsgp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
const char* key = NULL;
|
|
|
|
NCTRACE(11,"bucket=%s pathkey=%s",bucket,pathkey);
|
|
|
|
Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0;
|
|
Aws::S3::Model::DeleteObjectRequest delete_request;
|
|
|
|
if(*pathkey != '/') return NC_EINTERNAL;
|
|
if((stat = makes3key(pathkey,&key))) return NCUNTRACE(stat);
|
|
/* Delete this key object */
|
|
delete_request.SetBucket(bucket);
|
|
delete_request.SetKey(key);
|
|
auto delete_result = s3client->DeleteObject(delete_request);
|
|
if(!delete_result.IsSuccess()) {
|
|
if(errmsgp) *errmsgp = makeerrmsg(delete_result.GetError(),key);
|
|
stat = NC_ES3;
|
|
}
|
|
return NCUNTRACE(stat);
|
|
}
|
|
|
|
/*
|
|
Get Info about a single object from a vector
|
|
*/
|
|
static int
|
|
s3objectinfo1(const Aws::S3::Model::Object& s3_object, char** fullkeyp, size64_t* lenp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
char* cstr = NULL;
|
|
|
|
if(fullkeyp) {
|
|
const char* key = NULL;
|
|
char* cstr = NULL;
|
|
auto s = s3_object.GetKey();
|
|
key = s.c_str();
|
|
if(key[0] == '/') {
|
|
cstr = strdup(key);
|
|
} else {
|
|
size_t len = strlen(key);
|
|
cstr = (char*)malloc(len+1+1);
|
|
if(cstr != NULL) {
|
|
cstr[0] = '/';
|
|
memcpy((void*)(cstr+1),(void*)key,len);
|
|
cstr[len+1] = '\0';
|
|
}
|
|
}
|
|
if(cstr == NULL) {
|
|
stat = NC_ENOMEM;
|
|
} else if(fullkeyp) {
|
|
*fullkeyp = cstr;
|
|
cstr = NULL;
|
|
}
|
|
}
|
|
if(!stat) {
|
|
if(lenp) *lenp = (size64_t)s3_object.GetSize();
|
|
}
|
|
if(cstr) free(cstr);
|
|
return stat;
|
|
}
|
|
|
|
/*
|
|
Get Info about a vector of objects; Keys are fixed up to start with a '/'
|
|
*/
|
|
static int
|
|
s3objectsinfo(Aws::Vector<Aws::S3::Model::Object> list, size_t* nkeysp, char*** keysp, size64_t** lenp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
char** keys = NULL;
|
|
size_t nkeys;
|
|
size64_t *lengths = NULL;
|
|
int i;
|
|
|
|
nkeys = list.size();
|
|
if(nkeysp) *nkeysp = nkeys;
|
|
if((keys=(char**)calloc(sizeof(char*),(nkeys+1)))==NULL)
|
|
stat = NC_ENOMEM;
|
|
if(!stat) {
|
|
if((lengths=(size64_t*)calloc(sizeof(size64_t),(nkeys)))==NULL)
|
|
stat = NC_ENOMEM;
|
|
}
|
|
if(!stat) {
|
|
i = 0;
|
|
for (auto const &s3_object : list) {
|
|
stat = s3objectinfo1(s3_object,&keys[i],&lengths[i]);
|
|
i++;
|
|
if(stat) break;
|
|
}
|
|
}
|
|
if(!stat) {
|
|
if(keysp) {keys[nkeys] = NULL; *keysp = keys; keys = NULL;}
|
|
if(lenp) {*lenp = lengths; lengths = NULL;}
|
|
}
|
|
if(keys != NULL) freestringenvv(keys);
|
|
if(lengths != NULL) free(lengths);
|
|
return stat;
|
|
}
|
|
|
|
static int
|
|
s3commonprefixes(Aws::Vector<Aws::S3::Model::CommonPrefix> list, char*** keysp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
char** keys = NULL;
|
|
size_t nkeys;
|
|
int i;
|
|
|
|
nkeys = list.size();
|
|
if((keys=(char**)calloc(sizeof(char*),(nkeys+1)))==NULL)
|
|
stat = NC_ENOMEM;
|
|
if(!stat) {
|
|
i = 0;
|
|
for (auto const &s3prefix : list) {
|
|
char* p; char* p1;
|
|
size_t len;
|
|
const Aws::String& prefix = s3prefix.GetPrefix();
|
|
len = prefix.length();
|
|
if((p = (char*) malloc(len+1+1))==NULL) /* for nul + leading '/' */
|
|
stat = NC_ENOMEM;
|
|
if(stat == NC_NOERR) {
|
|
if(*p == '/') {p1 = p;} else {*p = '/'; p1 = p+1;}
|
|
memcpy(p1,prefix.c_str(),len);
|
|
p1[len] = '\0';
|
|
keys[i++] = p;
|
|
}
|
|
}
|
|
}
|
|
if(keysp) {keys[nkeys] = NULL; *keysp = keys; keys = NULL;}
|
|
if(keys != NULL) freestringenvv(keys);
|
|
return stat;
|
|
}
|
|
|
|
static Aws::S3::Model::BucketLocationConstraint
|
|
s3findregion(const char* name)
|
|
{
|
|
return Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(name);
|
|
}
|
|
|
|
struct MemBuf: std::streambuf {
|
|
MemBuf(char* base, size_t size) {
|
|
char* p(base);
|
|
this->setg(p, p, p + size);
|
|
}
|
|
};
|
|
|
|
static void
|
|
freestringenvv(char** ss)
|
|
{
|
|
char** p;
|
|
if(ss != NULL) {
|
|
for(p=ss;*p;p++)
|
|
if(*p) free(*p);
|
|
free(ss);
|
|
}
|
|
}
|
|
|
|
static int
|
|
makes3key(const char* pathkey, const char** keyp)
|
|
{
|
|
assert(pathkey != NULL && pathkey[0] == '/');
|
|
if(keyp) *keyp = pathkey+1;
|
|
return NC_NOERR;
|
|
}
|
|
|
|
static int
|
|
makes3keydir(const char* prefix, char** prefixdirp)
|
|
{
|
|
size_t plen = strlen(prefix);
|
|
char* prefixdir = (char*)malloc(plen+1+1); /* '/' + nul */
|
|
if(prefixdir == NULL) return NC_ENOMEM;
|
|
memcpy(prefixdir,prefix,plen);
|
|
if(prefixdir[plen-1] != '/') {
|
|
prefixdir[plen++] = '/';
|
|
}
|
|
prefixdir[plen] = '\0';
|
|
*prefixdirp = prefixdir; prefixdir = NULL;
|
|
return NC_NOERR;
|
|
}
|
|
|
|
static char**
|
|
mergekeysets(size_t nkeys1, char** keys1, size_t nkeys2, char** keys2)
|
|
{
|
|
char** merge = NULL;
|
|
char** p1;
|
|
char** p2;
|
|
|
|
merge = (char**)malloc(sizeof(char*)*(nkeys1+nkeys2+1));
|
|
if(merge == NULL) return NULL;
|
|
if(nkeys1 > 0) {
|
|
memcpy(merge,keys1,sizeof(char*)*nkeys1);
|
|
/* avoid double free */
|
|
memset(keys1,0,sizeof(char*)*nkeys1);
|
|
}
|
|
if(nkeys2 > 0) {
|
|
memcpy(merge+nkeys1,keys2,sizeof(char*)*nkeys2);
|
|
/* avoid double free */
|
|
memset(keys2,0,sizeof(char*)*nkeys2);
|
|
}
|
|
merge[nkeys1+nkeys2] = NULL;
|
|
return merge;
|
|
}
|