mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-11-27 02:10:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

ROS3: (feature) Temporary security credentials (#3030)

Browse Source 
- Implemented support for AWS temporary security credentials.
  For this kind of credentials also a session/security token
  should be included in the request by adding the
  x-amz-security-token header.

Co-authored-by: Larry Knox <lrknox@hdfgroup.org>
Co-authored-by: Jordan Henderson <jhenderson@hdfgroup.org>
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Jan-Willem Blokland 2023-07-14 22:15:11 +02:00 committed by GitHub
parent f21cd485a7
commit 305ac88865
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 769 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
release_docs/RELEASE.txt
View File

@ -169,6 +169,15 @@ New Features
Library:
--------
- Implemented support for temporary security credentials for the Read-Only
S3 (ROS3) file driver.
When using temporary security credentials, one also needs to specify a
session/security token next to the access key id and secret access key.
This token can be specified by the new API function H5Pset_fapl_ros3_token().
The API function H5Pget_fapl_ros3_token() can be used to retrieve
the currently set token.
- Added a Subfiling VFD configuration file prefix environment variable
The Subfiling VFD now checks for values set in a new environment

283
src/H5FDros3.c
View File

@ -47,6 +47,10 @@
*/
static hid_t H5FD_ROS3_g = 0;
/* Session/security token property name
*/
#define ROS3_TOKEN_PROP_NAME "ros3_token_prop"
#if ROS3_STATS
/* arbitrarily large value, such that any reasonable size read will be "less"
@ -225,6 +229,11 @@ static herr_t H5FD__ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing
static herr_t H5FD__ros3_validate_config(const H5FD_ros3_fapl_t *fa);
static herr_t H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value);
static int H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size);
static herr_t H5FD__ros3_str_token_close(const char *name, size_t size, void *_value);
static herr_t H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *_value);
static const H5FD_class_t H5FD_ros3_g = {
H5FD_CLASS_VERSION, /* struct version */
H5FD_ROS3_VALUE, /* value */
@ -566,6 +575,257 @@ H5FD__ros3_fapl_free(void *_fa)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD__ros3_fapl_free() */
/*-------------------------------------------------------------------------
* Function: H5Pget_fapl_ros3_token
*
* Purpose: Returns session/security token of the ros3 file access
* property list though the function arguments.
*
* Return: Success: Non-negative
*
* Failure: Negative
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/)
{
H5P_genplist_t *plist = NULL;
char *token_src;
htri_t token_exists;
size_t tokenlen = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_API(FAIL)
H5TRACE3("e", "izx", fapl_id, size, token_dst);
#if ROS3_DEBUG
HDfprintf(stdout, "H5Pget_fapl_ros3_token() called.\n");
#endif
if (size == 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "size cannot be zero.")
if (token_dst == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "token_dst is NULL")
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list")
if (H5FD_ROS3 != H5P_peek_driver(plist))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver")
if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist")
if (token_exists) {
if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token_src) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value")
}
/* Copy the token data out */
tokenlen = HDstrlen(token_src);
if (size <= tokenlen) {
tokenlen = size - 1;
}
H5MM_memcpy(token_dst, token_src, sizeof(char) * tokenlen);
token_dst[tokenlen] = '\0';
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_fapl_ros3_token() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_token_copy()
*
* Purpose: Create a copy of the token string.
*
* Return: SUCCEED/FAIL
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_str_token_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *_value)
{
char **value = (char **)_value;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
#if ROS3_DEBUG
HDfprintf(stdout, "H5FD__ros3_str_token_copy() called.\n");
#endif
if (*value)
if (NULL == (*value = HDstrdup(*value)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't copy string property token")
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* H5FD__ros3_str_token_copy() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_token_cmp()
*
* Purpose: Compares two token strings with each other.
*
* Return:
* - Equivalent: 0
* - Not Equivalent: non-zero value
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
static int
H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t H5_ATTR_UNUSED size)
{
char *const *value1 = (char *const *)_value1;
char *const *value2 = (char *const *)_value2;
int ret_value = SUCCEED;
FUNC_ENTER_PACKAGE_NOERR
if (*value1) {
if (*value2)
ret_value = HDstrcmp(*value1, *value2);
else
ret_value = 1;
}
else {
if (*value2)
ret_value = -1;
else
ret_value = 0;
}
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_str_token_cmp */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_token_close()
*
* Purpose: Closes/frees the memory associated to the token string.
* Currently, it is an empty implementation since there no
* additional treatment needed for this property.
*
* Return: SUCCEED/FAIL
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_str_token_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *_value)
{
char **value = (char **)_value;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE_NOERR
if (*value)
free(*value);
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_str_token_close */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_token_delete()
*
* Purpose: Deletes the property token from the property list and frees
* the memory associated to the token string.
* Currently, it is an empty implementation since there no
* additional treatment needed for this property.
*
* Return: SUCCEED/FAIL
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_str_token_delete(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name,
size_t H5_ATTR_UNUSED size, void *_value)
{
char **value = (char **)_value;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE_NOERR
if (*value)
free(*value);
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_str_token_delete */
/*-------------------------------------------------------------------------
* Function: H5Pset_fapl_ros3_token()
*
* Purpose: Modify the file access property list to use the H5FD_ROS3
* driver defined in this source file by adding or
* modifying the session/security token property.
*
* Return: SUCCEED/FAIL
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token)
{
H5P_genplist_t *plist = NULL;
char *token_src;
htri_t token_exists;
herr_t ret_value = SUCCEED;
FUNC_ENTER_API(FAIL)
H5TRACE2("e", "i*s", fapl_id, token);
#if ROS3_DEBUG
HDfprintf(stdout, "H5Pset_fapl_ros3_token() called.\n");
#endif
if (fapl_id == H5P_DEFAULT)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list")
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list")
if (H5FD_ROS3 != H5P_peek_driver(plist))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver")
if (HDstrlen(token) > H5FD_ROS3_MAX_SECRET_TOK_LEN)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL,
"specified token exceeds the internally specified maximum string length")
if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist")
if (token_exists) {
if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token_src) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value")
memcpy(token_src, token, HDstrlen(token) + 1);
}
else {
token_src = (char *)malloc(sizeof(char) * (H5FD_ROS3_MAX_SECRET_TOK_LEN + 1));
if (token_src == NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for token_src variable.");
memcpy(token_src, token, HDstrlen(token) + 1);
if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char *), &token_src, NULL, NULL, NULL, NULL,
H5FD__ros3_str_token_delete, H5FD__ros3_str_token_copy, H5FD__ros3_str_token_cmp,
H5FD__ros3_str_token_close) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "unable to register property in plist")
}
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pset_fapl_ros3_token() */
#if ROS3_STATS
/*----------------------------------------------------------------------------
*
@ -654,6 +914,9 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
unsigned char signing_key[SHA256_DIGEST_LENGTH];
s3r_t *handle = NULL;
H5FD_ros3_fapl_t fa;
H5P_genplist_t *plist = NULL;
htri_t token_exists;
char *token;
H5FD_t *ret_value = NULL;
FUNC_ENTER_PACKAGE
@ -681,6 +944,16 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
if (CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to initialize curl global (placeholder flags)")
/* Session/security token */
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, NULL, "not a file access property list")
if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "failed to check if property token exists in plist")
if (token_exists) {
if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to get token value")
}
/* open file; procedure depends on whether or not the fapl instructs to
* authenticate requests or not.
*/
@ -697,11 +970,15 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
(const char *)fa.aws_region, (const char *)iso8601now))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while computing signing key")
handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id,
(const unsigned char *)signing_key);
if (token_exists)
handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id,
(const unsigned char *)signing_key, (const char *)token);
else
handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id,
(const unsigned char *)signing_key, "");
}
else
handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL);
handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL, NULL);
if (handle == NULL)
/* If we want to check CURL's say on the matter in a controlled

109
src/H5FDros3.h
View File

@ -70,12 +70,63 @@
*
****************************************************************************/
/**
* \def H5FD_CURR_ROS3_FAPL_T_VERSION
* The version number of the H5FD_ros3_fapl_t configuration
* structure for the $H5FD_ROS3 driver.
*/
#define H5FD_CURR_ROS3_FAPL_T_VERSION 1
#define H5FD_ROS3_MAX_REGION_LEN 32
#define H5FD_ROS3_MAX_SECRET_ID_LEN 128
/**
* \def H5FD_ROS3_MAX_REGION_LEN
* Maximum string length for specifying the region of the S3 bucket.
*/
#define H5FD_ROS3_MAX_REGION_LEN 32
/**
* \def H5FD_ROS3_MAX_SECRET_ID_LEN
* Maximum string length for specifying the security ID.
*/
#define H5FD_ROS3_MAX_SECRET_ID_LEN 128
/**
* \def H5FD_ROS3_MAX_SECRET_KEY_LEN
* Maximum string length for specifying the security key.
*/
#define H5FD_ROS3_MAX_SECRET_KEY_LEN 128
/**
* \def H5FD_ROS3_MAX_SECRET_TOK_LEN
* Maximum string length for specifying the session/security token.
*/
#define H5FD_ROS3_MAX_SECRET_TOK_LEN 1024
/**
*\struct H5FD_ros3_fapl_t
* \brief Configuration structure for H5Pset_fapl_ros3() / H5Pget_fapl_ros3().
*
* \details H5FD_ros_fapl_t is a public structure that is used to pass
* configuration data to the #H5FD_ROS3 driver via a File Access
* Property List. A pointer to an instance of this structure is
* a parameter to H5Pset_fapl_ros3() and H5Pget_fapl_ros3().
*
* \var int32_t H5FD_ros3_fapl_t::version
* Version number of the H5FD_ros3_fapl_t structure. Any instance passed
* to H5Pset_fapl_ros3() / H5Pget_fapl_ros3() must have a recognized version
* number or an error will be raised. Currently, this field should be set
* to #H5FD_CURR_ROS3_FAPL_T_VERSION.
*
* \var hbool_t H5FD_ros3_fapl_t::authenticate
* A boolean which specifies if security credentials should be used for
* accessing a S3 bucket.
*
* \var char H5FD_ros3_fapl_t::aws_region[H5FD_ROS3_MAX_REGION_LEN + 1]
* A string which specifies the AWS region of the S3 bucket.
*
* \var char H5FD_ros3_fapl_t::secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1]
* A string which specifies the security ID.
*
* \var char H5FD_ros3_fapl_t::secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1]
* A string which specifies the security key.
*
*/
typedef struct H5FD_ros3_fapl_t {
int32_t version;
hbool_t authenticate;
@ -88,22 +139,72 @@ typedef struct H5FD_ros3_fapl_t {
extern "C" {
#endif
/**
* \brief Internal routine to initialize #H5FD_ROS3 driver. Not meant to be
* called directly by an HDF5 application.
*/
H5_DLL hid_t H5FD_ros3_init(void);
/**
* \ingroup FAPL
*
* \todo Add missing documentation
* \brief Queries a File Access Property List for #H5FD_ROS3 file driver properties.
*
* \fapl_id
* \param[out] fa_out Pointer to #H5FD_ROS3 driver configuration structure.
* \returns \herr_t
*/
H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out);
/**
* \ingroup FAPL
*
* \todo Add missing documentation
* \brief Modifies the specified File Access Property List to use the #H5FD_ROS3 driver.
*
* \fapl_id
* \param[in] fa Pointer to #H5FD_ROS3 driver configuration structure.
* \returns \herr_t
*/
H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, const H5FD_ros3_fapl_t *fa);
/**
* \ingroup FAPL
*
* \brief Queries a File Access Property List for #H5FD_ROS3 file driver session/security
* token.
*
* \fapl_id
* \param[in] size Size of the provided char array for storing the session/security token.
* \param[out] token Session/security token.
* \returns \herr_t
*
* \since 1.14.2
*/
H5_DLL herr_t H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token);
/**
* \ingroup FAPL
*
* \brief Modifies the specified File Access Property List to use the #H5FD_ROS3 driver
* by adding the specified session/security token.
*
* \fapl_id
* \param[in] token Session/security token.
* \returns \herr_t
*
* \details H5Pset_fapl_ros3_token() modifies an existing File Access Property List which
* is used by #H5FD_ROS3 driver by adding or updating the session/security token
* of the property list. Be aware, to set the token first you need to create
* a proper File Access Property List using H5Pset_fapl_ros() and use this list
* as input argument of the function H5Pset_fapl_ros3_token().
*
* Note, the session token is only needed when you want to access a S3 bucket
* using temporary security credentials.
*
* \since 1.14.2
*/
H5_DLL herr_t H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token);
#ifdef __cplusplus
}
#endif

109
src/H5FDs3comms.c
View File

@ -39,6 +39,7 @@
#include "H5Eprivate.h" /* error handling */
#include "H5MMprivate.h" /* memory management */
#include "H5FDs3comms.h" /* S3 Communications */
#include "H5FDros3.h" /* ros3 file driver */
/****************/
/* Local Macros */
@ -775,6 +776,7 @@ H5FD_s3comms_s3r_close(s3r_t *handle)
H5MM_xfree(handle->secret_id);
H5MM_xfree(handle->region);
H5MM_xfree(handle->signing_key);
H5MM_xfree(handle->token);
assert(handle->httpverb != NULL);
H5MM_xfree(handle->httpverb);
@ -1001,7 +1003,8 @@ done:
*----------------------------------------------------------------------------
*/
s3r_t *
H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key)
H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key,
const char *token)
{
size_t tmplen = 0;
CURL *curlh = NULL;
@ -1035,13 +1038,15 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const
handle->region = NULL;
handle->secret_id = NULL;
handle->signing_key = NULL;
handle->token = NULL;
handle->httpverb = NULL;
/*************************************
* RECORD AUTHENTICATION INFORMATION *
*************************************/
if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL)) {
if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL) ||
(token != NULL)) {
/* if one exists, all three must exist */
if (region == NULL || region[0] == '\0')
@ -1050,6 +1055,8 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "secret id cannot be null.");
if (signing_key == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "signing key cannot be null.");
if (token == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "token cannot be null.");
/* copy strings */
tmplen = HDstrlen(region) + 1;
@ -1069,6 +1076,12 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const
if (handle->signing_key == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle key copy.");
H5MM_memcpy(handle->signing_key, signing_key, tmplen);
tmplen = HDstrlen(token) + 1;
handle->token = (char *)H5MM_malloc(sizeof(char) * tmplen);
if (handle->token == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle token copy.");
H5MM_memcpy(handle->token, token, tmplen);
} /* if authentication information provided */
/************************
@ -1130,6 +1143,7 @@ done:
H5MM_xfree(handle->region);
H5MM_xfree(handle->secret_id);
H5MM_xfree(handle->signing_key);
H5MM_xfree(handle->token);
if (handle->httpverb != NULL)
H5MM_xfree(handle->httpverb);
H5MM_xfree(handle);
@ -1190,8 +1204,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
hrb_t *request = NULL;
int ret = 0; /* working variable to check */
/* return value of HDsnprintf */
struct s3r_datastruct *sds = NULL;
herr_t ret_value = SUCCEED;
char *authorization = NULL;
char *buffer1 = NULL;
char *signed_headers = NULL;
struct s3r_datastruct *sds = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
@ -1284,23 +1301,31 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
else {
/* authenticate request
*/
char authorization[512 + 1];
/* 512 := approximate max length...
* 67 <len("AWS4-HMAC-SHA256 Credential=///s3/aws4_request,"
* "SignedHeaders=,Signature=")>
* + 8 <yyyyMMDD>
* + 64 <hex(sha256())>
* + 128 <max? len(secret_id)>
* + 20 <max? len(region)>
* + 128 <max? len(signed_headers)>
authorization = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + 1);
if (authorization == NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for authorization variable.");
/* 2048 := approximate max length...
* 67 <len("AWS4-HMAC-SHA256 Credential=///s3/aws4_request,"
* "SignedHeaders=,Signature=")>
* + 8 <yyyyMMDD>
* + 64 <hex(sha256())>
* + 128 <max? len(secret_id)>
* + 20 <max? len(region)>
* + 128 <max? len(signed_headers)>
* + 1024 <max? len(session_token)>
*/
char buffer1[512 + 1]; /* -> Canonical Request -> Signature */
char buffer2[256 + 1]; /* -> String To Sign -> Credential */
char iso8601now[ISO8601_SIZE];
char signed_headers[48 + 1];
buffer1 = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN +
1); /* -> Canonical Request -> Signature */
if (buffer1 == NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for buffer1 variable.");
signed_headers = (char *)H5MM_malloc(48 + H5FD_ROS3_MAX_SECRET_KEY_LEN + 1);
if (signed_headers == NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for signed_headers variable.");
/* should be large enough for nominal listing:
* "host;range;x-amz-content-sha256;x-amz-date"
* + '\0', with "range;" possibly absent
* "host;range;x-amz-content-sha256;x-amz-date;x-amz-security-token"
* + '\0', with "range;" and/or "x-amz-security-token" possibly absent
*/
/* zero start of strings */
@ -1318,6 +1343,8 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null secret_id.");
if (handle->signing_key == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null signing_key.");
if (handle->token == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null token.");
if (handle->httpverb == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null httpverb.");
if (handle->purl->host == NULL)
@ -1349,6 +1376,15 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list.");
assert(headers->magic == S3COMMS_HRB_NODE_MAGIC);
if (HDstrlen((const char *)handle->token) > 0) {
if (FAIL ==
H5FD_s3comms_hrb_node_set(&headers, "x-amz-security-token", (const char *)handle->token))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set x-amz-security-token header")
if (headers == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list.");
assert(headers->magic == S3COMMS_HRB_NODE_MAGIC);
}
if (rangebytesstr != NULL) {
if (FAIL == H5FD_s3comms_hrb_node_set(&headers, "Range", rangebytesstr))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set range header")
@ -1368,8 +1404,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
/**** COMPUTE AUTHORIZATION ****/
/* buffer1 -> canonical request */
if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512, signed_headers, 48, request))
if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
signed_headers, 48 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
request)) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad canonical request");
}
/* buffer2->string-to-sign */
if (FAIL == H5FD_s3comms_tostringtosign(buffer2, buffer1, iso8601now, handle->region))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad string-to-sign");
@ -1383,9 +1422,10 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 credential string");
ret = HDsnprintf(authorization, 512, "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s",
buffer2, signed_headers, buffer1);
if (ret <= 0 || ret >= 512)
ret = HDsnprintf(authorization, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
"AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", buffer2,
signed_headers, buffer1);
if (ret <= 0 || ret >= 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 authorization string");
/* append authorization header to http request buffer */
@ -1440,8 +1480,9 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
if (p_status != CURLE_OK) {
if (CURLE_OK != curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem getting response code")
fprintf(stdout, "CURL ERROR CODE: %d\nHTTP CODE: %d\n", p_status, httpcode);
fprintf(stdout, "CURL ERROR CODE: %d\nHTTP CODE: %ld\n", p_status, httpcode);
fprintf(stdout, "%s\n", curl_easy_strerror(p_status));
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "problem while performing request.");
}
if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL))
@ -1474,6 +1515,18 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
done:
/* clean any malloc'd resources
*/
if (authorization != NULL) {
H5MM_xfree(authorization);
authorization = NULL;
}
if (buffer1 != NULL) {
H5MM_xfree(buffer1);
buffer1 = NULL;
}
if (signed_headers != NULL) {
H5MM_xfree(signed_headers);
signed_headers = NULL;
}
if (curlheaders != NULL) {
curl_slist_free_all(curlheaders);
curlheaders = NULL;
@ -1592,8 +1645,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
size_t sh_size = (size_t)_sh_size;
size_t cr_len = 0; /* working length of canonical request str */
size_t sh_len = 0; /* working length of signed headers str */
char tmpstr[256 + 1];
tmpstr[256] = 0; /* terminating NULL */
char tmpstr[1024];
/* "query params" refers to the optional element in the URL, e.g.
* http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J
@ -1626,6 +1678,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
(size_t)3); /* three newline chars */
if (cr_len >= cr_size)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request");
/* TODO: compiler warning */
ret = HDsnprintf(canonical_request_dest, (cr_size - 1), "%s\n%s\n%s\n", http_request->verb,
http_request->resource, query_params);
@ -1638,8 +1691,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
assert(node->magic == S3COMMS_HRB_NODE_MAGIC);
ret = HDsnprintf(tmpstr, 256, "%s:%s\n", node->lowername, node->value);
if (ret < 0 || ret >= 256)
ret = HDsnprintf(tmpstr, 1024, "%s:%s\n", node->lowername, node->value);
if (ret < 0 || ret >= 1024)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to concatenate HTTP header %s:%s",
node->lowername, node->value);
cr_len += HDstrlen(tmpstr);
@ -1647,8 +1700,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request");
HDstrcat(canonical_request_dest, tmpstr);
ret = HDsnprintf(tmpstr, 256, "%s;", node->lowername);
if (ret < 0 || ret >= 256)
ret = HDsnprintf(tmpstr, 1024, "%s;", node->lowername);
if (ret < 0 || ret >= 1024)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to append semicolon to lowername %s",
node->lowername);
sh_len += HDstrlen(tmpstr);

3
src/H5FDs3comms.h
View File

@ -478,6 +478,7 @@ typedef struct {
char *region;
char *secret_id;
unsigned char *signing_key;
char *token;
} s3r_t;
#define S3COMMS_S3R_MAGIC 0x44d8d79
@ -509,7 +510,7 @@ H5_DLL herr_t H5FD_s3comms_s3r_close(s3r_t *handle);
H5_DLL size_t H5FD_s3comms_s3r_get_filesize(s3r_t *handle);
H5_DLL s3r_t *H5FD_s3comms_s3r_open(const char url[], const char region[], const char id[],
const unsigned char signing_key[]);
const unsigned char signing_key[], const char token[]);
H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest);

42
test/s3comms.c
View File

@ -348,6 +348,7 @@ static int s3_test_credentials_loaded = 0;
static char s3_test_aws_region[16] = "";
static char s3_test_aws_access_key_id[64] = "";
static char s3_test_aws_secret_access_key[128] = "";
static char s3_test_aws_security_token[1024] = "";
static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = "";
static hbool_t s3_test_bucket_defined = FALSE;
@ -1722,7 +1723,7 @@ test_s3r_get_filesize(void)
JSVERIFY(0, H5FD_s3comms_s3r_get_filesize(NULL), "filesize of the null handle should be 0")
handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL);
handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL);
FAIL_IF(handle == NULL)
JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL)
@ -1830,14 +1831,14 @@ test_s3r_open(void)
/* attempt anonymously
*/
handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL);
handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL, NULL);
FAIL_IF(handle != NULL);
/* attempt with authentication
*/
handle =
H5FD_s3comms_s3r_open(url_missing, (const char *)s3_test_aws_region,
(const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key);
handle = H5FD_s3comms_s3r_open(
url_missing, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
(const unsigned char *)signing_key, (const char *)s3_test_aws_security_token);
FAIL_IF(handle != NULL);
/*************************
@ -1846,7 +1847,7 @@ test_s3r_open(void)
#if S3_TEST_RUN_TIMEOUT
printf("Opening on inactive port may hang for a minute; waiting for timeout\n");
handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL);
handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL, NULL);
FAIL_IF(handle != NULL);
#endif
@ -1856,20 +1857,21 @@ test_s3r_open(void)
/* anonymous access on restricted file
*/
handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL);
handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL, NULL);
FAIL_IF(handle != NULL);
/* passed in a bad ID
*/
handle = H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID",
(const unsigned char *)signing_key);
handle =
H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID",
(const unsigned char *)signing_key, (const char *)s3_test_aws_security_token);
FAIL_IF(handle != NULL);
/* using an invalid signing key
*/
handle =
H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region,
(const char *)s3_test_aws_access_key_id, (const unsigned char *)EMPTY_SHA256);
handle = H5FD_s3comms_s3r_open(
url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
(const unsigned char *)EMPTY_SHA256, (const char *)s3_test_aws_security_token);
FAIL_IF(handle != NULL);
/*******************************
@ -1878,7 +1880,7 @@ test_s3r_open(void)
/* anonymous
*/
handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL);
handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL);
FAIL_IF(handle == NULL);
JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), "did not get expected filesize")
JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file")
@ -1886,9 +1888,9 @@ test_s3r_open(void)
/* using authentication on anonymously-accessible file?
*/
handle =
H5FD_s3comms_s3r_open(url_raven, (const char *)s3_test_aws_region,
(const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key);
handle = H5FD_s3comms_s3r_open(
url_raven, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
(const unsigned char *)signing_key, (const char *)s3_test_aws_security_token);
FAIL_IF(handle == NULL);
JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL)
JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file")
@ -1896,9 +1898,9 @@ test_s3r_open(void)
/* authenticating
*/
handle =
H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region,
(const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key);
handle = H5FD_s3comms_s3r_open(
url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
(const unsigned char *)signing_key, (const char *)s3_test_aws_security_token);
FAIL_IF(handle == NULL);
JSVERIFY(5458199, H5FD_s3comms_s3r_get_filesize(handle), NULL)
JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file")
@ -1984,7 +1986,7 @@ test_s3r_read(void)
/* open file
*/
handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL);
handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL);
FAIL_IF(handle == NULL)
JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL)

6
tools/lib/h5tools.c
View File

@ -565,8 +565,12 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info)
#ifdef H5_HAVE_ROS3_VFD
if (!vfd_info->info)
H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD info is invalid");
if (H5Pset_fapl_ros3(fapl_id, (const H5FD_ros3_fapl_t *)vfd_info->info) < 0)
if (H5Pset_fapl_ros3(fapl_id, &((const H5FD_ros3_fapl_ext_t *)vfd_info->info)->fa) < 0)
H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3() failed");
if (H5Pset_fapl_ros3_token(fapl_id, ((const H5FD_ros3_fapl_ext_t *)vfd_info->info)->token) <
0)
H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3_token() failed");
#else
H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD is not enabled");
#endif

85
tools/lib/h5tools_utils.c
View File

@ -1025,9 +1025,9 @@ done:
*----------------------------------------------------------------------------
*/
herr_t
h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t *fapl_config_out)
h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_ext_t *fapl_config_out)
{
const char *ccred[3];
const char *ccred[4];
unsigned nelems = 0;
char *s3cred_src = NULL;
char **s3cred = NULL;
@ -1038,12 +1038,18 @@ h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t
H5TOOLS_GOTO_ERROR(FAIL, "failed to parse S3 VFD info tuple");
/* Sanity-check tuple count */
if (nelems != 3)
if (nelems != 3 && nelems != 4)
H5TOOLS_GOTO_ERROR(FAIL, "invalid S3 VFD credentials");
ccred[0] = (const char *)s3cred[0];
ccred[1] = (const char *)s3cred[1];
ccred[2] = (const char *)s3cred[2];
if (nelems == 3) {
ccred[3] = "";
}
else {
ccred[3] = (const char *)s3cred[3];
}
if (0 == h5tools_populate_ros3_fapl(fapl_config_out, ccred))
H5TOOLS_GOTO_ERROR(FAIL, "failed to populate S3 VFD FAPL config");
@ -1090,34 +1096,35 @@ done:
* * NULL fapl pointer: (NULL, {...} )
* * Warning: In all cases below, fapl will be set as "default"
* before error occurs.
* * NULL value strings: (&fa, {NULL?, NULL? NULL?, ...})
* * NULL value strings: (&fa, {NULL?, NULL? NULL?, NULL?, ...})
* * Incomplete fapl info:
* * empty region, non-empty id, key either way
* * (&fa, {"", "...", "?"})
* * empty id, non-empty region, key either way
* * (&fa, {"...", "", "?"})
* * "non-empty key and either id or region empty
* * (&fa, {"", "", "...")
* * (&fa, {"", "...", "...")
* * (&fa, {"...", "", "...")
* * empty region, non-empty id, key either way, token either way
* * (&fa, token, {"", "...", "?", "?"})
* * empty id, non-empty region, key either way, token either way
* * (&fa, token, {"...", "", "?", "?"})
* * "non-empty key, token either way and either id or region empty
* * (&fa, token, {"", "", "...", "?")
* * (&fa, token, {"", "...", "...", "?")
* * (&fa, token, {"...", "", "...", "?")
* * Any string would overflow allowed space in fapl definition.
* or
* 1 (success)
* * Sets components in fapl_t pointer, copying strings as appropriate.
* * "Default" fapl (valid version, authenticate->False, empty strings)
* * `values` pointer is NULL
* * (&fa, NULL)
* * first three strings in `values` are empty ("")
* * (&fa, {"", "", "", ...}
* * (&fa, token, NULL)
* * first four strings in `values` are empty ("")
* * (&fa, token, {"", "", "", "", ...})
* * Authenticating fapl
* * region, id, and optional key provided
* * (&fa, {"...", "...", ""})
* * (&fa, {"...", "...", "..."})
* * region, id, optional key and option session token provided
* * (&fa, token, {"...", "...", "", ""})
* * (&fa, token, {"...", "...", "...", ""})
* * (&fa, token, {"...", "...", "...", "..."})
*
*----------------------------------------------------------------------------
*/
int
h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values)
h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values)
{
int show_progress = 0; /* set to 1 for debugging */
int ret_value = 1; /* 1 for success, 0 for failure */
@ -1138,11 +1145,12 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values)
if (show_progress) {
printf(" preset fapl with default values\n");
}
fa->version = H5FD_CURR_ROS3_FAPL_T_VERSION;
fa->authenticate = FALSE;
*(fa->aws_region) = '\0';
*(fa->secret_id) = '\0';
*(fa->secret_key) = '\0';
fa->fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION;
fa->fa.authenticate = FALSE;
*(fa->fa.aws_region) = '\0';
*(fa->fa.secret_id) = '\0';
*(fa->fa.secret_key) = '\0';
*(fa->token) = '\0';
/* sanity-check supplied values
*/
@ -1168,6 +1176,13 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values)
ret_value = 0;
goto done;
}
if (values[3] == NULL) {
if (show_progress) {
printf(" ERROR: token value cannot be NULL\n");
}
ret_value = 0;
goto done;
}
/* if region and ID are supplied (key optional), write to fapl...
* fail if value would overflow
@ -1180,7 +1195,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values)
ret_value = 0;
goto done;
}
memcpy(fa->aws_region, values[0], (HDstrlen(values[0]) + 1));
memcpy(fa->fa.aws_region, values[0], (HDstrlen(values[0]) + 1));
if (show_progress) {
printf(" aws_region set\n");
}
@ -1192,7 +1207,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values)
ret_value = 0;
goto done;
}
memcpy(fa->secret_id, values[1], (HDstrlen(values[1]) + 1));
memcpy(fa->fa.secret_id, values[1], (HDstrlen(values[1]) + 1));
if (show_progress) {
printf(" secret_id set\n");
}
@ -1204,17 +1219,29 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values)
ret_value = 0;
goto done;
}
memcpy(fa->secret_key, values[2], (HDstrlen(values[2]) + 1));
memcpy(fa->fa.secret_key, values[2], (HDstrlen(values[2]) + 1));
if (show_progress) {
printf(" secret_key set\n");
}
fa->authenticate = TRUE;
if (HDstrlen(values[3]) > H5FD_ROS3_MAX_SECRET_TOK_LEN) {
if (show_progress) {
printf(" ERROR: token value too long\n");
}
ret_value = 0;
goto done;
}
memcpy(fa->token, values[3], (HDstrlen(values[3]) + 1));
if (show_progress) {
printf(" token set\n");
}
fa->fa.authenticate = TRUE;
if (show_progress) {
printf(" set to authenticate\n");
}
}
else if (*values[0] != '\0' || *values[1] != '\0' || *values[2] != '\0') {
else if (*values[0] != '\0' || *values[1] != '\0' || *values[2] != '\0' || *values[3] != '\0') {
if (show_progress) {
printf(" ERROR: invalid assortment of empty/non-empty values\n");
}

12
tools/lib/h5tools_utils.h
View File

@ -61,6 +61,14 @@ typedef struct find_objs_t {
table_t *dset_table;
} find_objs_t;
#ifdef H5_HAVE_ROS3_VFD
/*extended configuration struct for holding the configuration data to the #H5FD_ROS3 driver */
typedef struct H5FD_ros3_fapl_ext_t {
H5FD_ros3_fapl_t fa; /* ROS3 configuration struct*/
char token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1]; /* Session/security token*/
} H5FD_ros3_fapl_ext_t;
#endif /* H5_HAVE_ROS3_VFD */
H5TOOLS_DLLVAR unsigned h5tools_nCols; /*max number of columns for outputting */
/* Definitions of useful routines */
@ -126,8 +134,8 @@ H5TOOLS_DLL void h5tools_setstatus(int d_status);
H5TOOLS_DLL int h5tools_getenv_update_hyperslab_bufsize(void);
#ifdef H5_HAVE_ROS3_VFD
H5TOOLS_DLL herr_t h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim,
H5FD_ros3_fapl_t *fapl_config_out);
H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values);
H5FD_ros3_fapl_ext_t *fapl_config_out);
H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values);
#endif /* H5_HAVE_ROS3_VFD */
#ifdef H5_HAVE_LIBHDFS
H5TOOLS_DLL herr_t h5tools_parse_hdfs_fapl_tuple(const char *tuple_str, int delim,

338
tools/libtest/h5tools_test_utils.c
View File

@ -22,10 +22,6 @@
#define UTIL_TEST_DEBUG 0
#ifndef __js_test__
#define __js_test__ 1L
/*****************************************************************************
*
* FILE-LOCAL TESTING MACROS
@ -59,7 +55,6 @@
*
* JSVERIFY_EXP_ACT - ifdef flag, configures comparison order
* FAIL_IF() - check condition
* FAIL_UNLESS() - check _not_ condition
* JSVERIFY() - long-int equality check; prints reason/comparison
* JSVERIFY_NOT() - long-int inequality check; prints
* JSVERIFY_STR() - string equality check; prints
@ -123,31 +118,6 @@ H5_GCC_CLANG_DIAG_OFF("format")
goto error; \
}
/*----------------------------------------------------------------------------
*
* Macro: FAIL_UNLESS()
*
* Purpose:
*
* TEST_ERROR wrapper to reduce cognitive overhead from "negative tests",
* e.g., "a != b".
*
* Opposite of FAIL_IF; fails if the given condition is _not_ true.
*
* `FAIL_IF( 5 != my_op() )`
* is equivalent to
* `FAIL_UNLESS( 5 == my_op() )`
* However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer.
* (see JSVERIFY)
*
*----------------------------------------------------------------------------
*/
#define FAIL_UNLESS(condition) \
if (!(condition)) { \
JSFAILED_AT() \
goto error; \
}
/*----------------------------------------------------------------------------
*
* Macro: JSERR_LONG()
@ -314,8 +284,6 @@ H5_GCC_CLANG_DIAG_OFF("format")
#endif /* ifdef/else JSVERIFY_EXP_ACT */
#endif /* __js_test__ */
/* if > 0, be very verbose when performing tests */
#define H5TOOLS_UTILS_TEST_DEBUG 0
@ -508,11 +476,11 @@ test_parse_tuple(void)
success = parse_tuple(tc.in_str, tc.sep, &cpy, &count, &parsed);
JSVERIFY(tc.exp_ret, success, "function returned incorrect value")
JSVERIFY(tc.exp_nelems, count, NULL)
JSVERIFY(tc.exp_nelems, count, (char *)NULL)
if (success == SUCCEED) {
FAIL_IF(parsed == NULL)
for (elem_i = 0; elem_i < count; elem_i++) {
JSVERIFY_STR(tc.exp_elems[elem_i], parsed[elem_i], NULL)
JSVERIFY_STR(tc.exp_elems[elem_i], parsed[elem_i], (char *)NULL)
}
/* TEARDOWN */
assert(parsed != NULL);
@ -605,104 +573,109 @@ test_populate_ros3_fa(void)
/* NULL values pointer yields default fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"};
H5FD_ros3_fapl_ext_t fa = {{bad_version, TRUE, "u", "v", "w"}, "x"};
if (show_progress) {
printf("NULL values pointer\n");
}
JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, NULL), "NULL values pointer yields \"default\" fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* all-empty values
* yields default fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"};
const char *values[] = {"", "", ""};
H5FD_ros3_fapl_ext_t fa = {{bad_version, TRUE, "u", "v", "w"}, "x"};
const char *values[] = {"", "", "", ""};
if (show_progress) {
printf("all empty values\n");
}
JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "empty values yields \"default\" fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* successfully set fapl with values
* excess value is ignored
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"x", "y", "z", "a"};
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"x", "y", "z", "a", "b"};
if (show_progress) {
printf("successful full set\n");
}
JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "four values")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(TRUE, fa.authenticate, NULL)
JSVERIFY_STR("x", fa.aws_region, NULL)
JSVERIFY_STR("y", fa.secret_id, NULL)
JSVERIFY_STR("z", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(TRUE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("z", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("a", fa.token, (char *)NULL)
}
/* NULL region
* yields default fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {NULL, "y", "z", NULL};
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {NULL, "y", "z", ""};
if (show_progress) {
printf("NULL region\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* empty region
* yields default fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"", "y", "z", NULL};
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"", "y", "z", ""};
if (show_progress) {
printf("empty region; non-empty id, key\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* region overflow
* yields default fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"somewhere over the rainbow not too high "
"there is another rainbow bounding some darkened sky",
"y", "z"};
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"somewhere over the rainbow not too high "
"there is another rainbow bounding some darkened sky",
"y", "z", ""};
if (show_progress) {
printf("region overflow\n");
@ -711,68 +684,71 @@ test_populate_ros3_fa(void)
assert(HDstrlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN);
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* NULL id
* yields default fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"x", NULL, "z", NULL};
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"x", NULL, "z", ""};
if (show_progress) {
printf("NULL id\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* empty id (non-empty region, key)
* yields default fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"x", "", "z", NULL};
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"x", "", "z", ""};
if (show_progress) {
printf("empty id; non-empty region and key\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* id overflow
* partial set: region
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"x",
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"x",
"Why is it necessary to solve the problem? "
"What benefits will you receive by solving the problem? "
"What is the unknown? "
"What is it you don't yet understand? "
"What is the information you have? "
"What isn't the problem? "
"Is the information insufficient, redundant, or contradictory? "
"Should you draw a diagram or figure of the problem? "
"What are the boundaries of the problem? "
"Can you separate the various parts of the problem?",
"z"};
"What benefits will you receive by solving the problem? "
"What is the unknown? "
"What is it you don't yet understand? "
"What is the information you have? "
"What isn't the problem? "
"Is the information insufficient, redundant, or contradictory? "
"Should you draw a diagram or figure of the problem? "
"What are the boundaries of the problem? "
"Can you separate the various parts of the problem?",
"z", ""};
if (show_progress) {
printf("id overflow\n");
@ -781,105 +757,131 @@ test_populate_ros3_fa(void)
assert(HDstrlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN);
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("x", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* NULL key
* yields default fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"x", "y", NULL, NULL};
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"x", "y", NULL, ""};
if (show_progress) {
printf("NULL key\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* NULL token
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"x", "y", "z", NULL};
if (show_progress) {
printf("NULL key\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill token")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* empty key (non-empty region, id)
* yields authenticating fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"x", "y", "", NULL};
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"x", "y", "", ""};
if (show_progress) {
printf("empty key; non-empty region and id\n");
}
JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(TRUE, fa.authenticate, NULL)
JSVERIFY_STR("x", fa.aws_region, NULL)
JSVERIFY_STR("y", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(TRUE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* empty key, region (non-empty id)
* yields default fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"", "y", "", NULL};
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"", "y", "", ""};
if (show_progress) {
printf("empty key and region; non-empty id\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* empty key, id (non-empty region)
* yields default fapl
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"x", "", "", NULL};
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"x", "", "", ""};
if (show_progress) {
printf("empty key and id; non-empty region\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("", fa.aws_region, NULL)
JSVERIFY_STR("", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* key overflow
* partial set: region, id
*/
{
H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
const char *values[] = {"x", "y",
H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"};
const char *values[] = {"x", "y",
"Why is it necessary to solve the problem? "
"What benefits will you receive by solving the problem? "
"What is the unknown? "
"What is it you don't yet understand? "
"What is the information you have? "
"What isn't the problem? "
"Is the information insufficient, redundant, or contradictory? "
"Should you draw a diagram or figure of the problem? "
"What are the boundaries of the problem? "
"Can you separate the various parts of the problem?"};
"What benefits will you receive by solving the problem? "
"What is the unknown? "
"What is it you don't yet understand? "
"What is the information you have? "
"What isn't the problem? "
"Is the information insufficient, redundant, or contradictory? "
"Should you draw a diagram or figure of the problem? "
"What are the boundaries of the problem? "
"Can you separate the various parts of the problem?",
""};
if (show_progress) {
printf("key overflow\n");
@ -888,22 +890,23 @@ test_populate_ros3_fa(void)
assert(HDstrlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN);
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL)
JSVERIFY(FALSE, fa.authenticate, NULL)
JSVERIFY_STR("x", fa.aws_region, NULL)
JSVERIFY_STR("y", fa.secret_id, NULL)
JSVERIFY_STR("", fa.secret_key, NULL)
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
}
/* use case
*/
{
H5FD_ros3_fapl_t fa = {0, 0, "", "", ""};
const char *values[] = {"us-east-2", "AKIAIMC3D3XLYXLN5COA",
"ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+"};
H5FD_ros3_fapl_ext_t fa = {{0, 0, "", "", ""}, ""};
const char *values[] = {"us-east-2", "AKIAIMC3D3XLYXLN5COA",
"ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+", ""};
JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "unable to set use case")
JSVERIFY(1, fa.version, "version check")
JSVERIFY(1, fa.authenticate, "should authenticate")
JSVERIFY(1, fa.fa.version, "version check")
JSVERIFY(1, fa.fa.authenticate, "should authenticate")
}
PASSED();
@ -962,14 +965,7 @@ test_set_configured_fapl(void)
hid_t fapl_id = H5I_INVALID_HID;
other_fa_t wrong_fa = {0x432, 0xf82, 0x9093};
#ifdef H5_HAVE_ROS3_VFD
H5FD_ros3_fapl_t ros3_anon_fa = {1, FALSE, "", "", ""};
H5FD_ros3_fapl_t ros3_auth_fa = {
1, /* fapl version */
TRUE, /* authenticate */
"us-east-1", /* aws region */
"12345677890abcdef", /* simulate access key ID */
"oiwnerwe9u0234nJw0-aoj+dsf", /* simulate secret key */
};
H5FD_ros3_fapl_ext_t ros3_anon_fa = {{1, FALSE, "", "", ""}, ""};
#endif /* H5_HAVE_ROS3_VFD */
#ifdef H5_HAVE_LIBHDFS
H5FD_hdfs_fapl_t hdfs_fa = {
@ -1114,7 +1110,17 @@ test_set_configured_fapl(void)
vfd_info.type = VFD_BY_NAME;
vfd_info.info = C.conf_fa;
vfd_info.u.name = C.vfdname;
result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info);
if (C.expected == 1)
result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info);
else {
H5E_BEGIN_TRY
{
result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info);
}
H5E_END_TRY;
}
if (C.expected == 0) {
JSVERIFY(result, H5I_INVALID_HID, C.message)
}

15
tools/src/h5dump/h5dump.c
View File

@ -32,12 +32,15 @@ static hbool_t get_onion_revision_count = FALSE;
#ifdef H5_HAVE_ROS3_VFD
/* Default "anonymous" S3 configuration */
static H5FD_ros3_fapl_t ros3_fa_g = {
1, /* Structure Version */
FALSE, /* Authenticate? */
"", /* AWS Region */
"", /* Access Key ID */
"", /* Secret Access Key */
static H5FD_ros3_fapl_ext_t ros3_fa_g = {
{
1, /* Structure Version */
FALSE, /* Authenticate? */
"", /* AWS Region */
"", /* Access Key ID */
"", /* Secret Access Key */
},
"", /* Session/security token */
};
#endif /* H5_HAVE_ROS3_VFD */

15
tools/src/h5ls/h5ls.c
View File

@ -2656,12 +2656,15 @@ main(int argc, char *argv[])
#ifdef H5_HAVE_ROS3_VFD
/* Default "anonymous" S3 configuration */
H5FD_ros3_fapl_t ros3_fa = {
1, /* Structure Version */
FALSE, /* Authenticate? */
"", /* AWS Region */
"", /* Access Key ID */
"", /* Secret Access Key */
H5FD_ros3_fapl_ext_t ros3_fa = {
{
1, /* Structure Version */
FALSE, /* Authenticate? */
"", /* AWS Region */
"", /* Access Key ID */
"", /* Secret Access Key */
},
"", /* Session/security token */
};
#endif /* H5_HAVE_ROS3_VFD */

15
tools/src/h5stat/h5stat.c
View File

@ -119,12 +119,15 @@ static const char *drivername = NULL;
#ifdef H5_HAVE_ROS3_VFD
/* Default "anonymous" S3 configuration */
static H5FD_ros3_fapl_t ros3_fa = {
1, /* Structure Version */
FALSE, /* Authenticate? */
"", /* AWS Region */
"", /* Access Key ID */
"", /* Secret Access Key */
static H5FD_ros3_fapl_ext_t ros3_fa = {
{
1, /* Structure Version */
FALSE, /* Authenticate? */
"", /* AWS Region */
"", /* Access Key ID */
"", /* Secret Access Key */
},
"", /* Session/security token */
};
#endif /* H5_HAVE_ROS3_VFD */