mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-12-21 08:39:46 +08:00
49737888ca
## Improvements to S3 Documentation * Create a new document *quickstart_paths.md* that give a summary of the legal path formats used by netcdf-c. This includes both file paths and URL paths. * Modify *nczarr.md* to remove most of the S3 related text. * Move the S3 text from *nczarr.md* to a new document *cloud.md*. * Add some S3-related text to the *byterange.md* document. Hopefully, this will make it easier for users to find the information they want. ## Rebuild NCZarr Testing In order to avoid problems with running make check in parallel, two changes were made: 1. The *nczarr_test* test system was rebuilt. Now, for each test. any generated files are kept in a test-specific directory, isolated from all other test executions. 2. Similarly, since the S3 test bucket is shared, any generated S3 objects are isolated using a test-specific key path. ## Other S3 Related Changes * Add code to ensure that files created on S3 are reclaimed at end of testing. * Used the bash "trap" command to ensure S3 cleanup even if the test fails. * Cleanup the S3 related configure.ac flag set since S3 is used in several places. So now one should use the option *--enable-s3* instead of *--enable-nczarr-s3*, although the latter is still kept as a deprecated alias for the former. * Get some of the github actions yml to work with S3; required fixing various test scripts adding a secret to access the Unidata S3 bucket. * Cleanup S3 portion of libnetcdf.settings.in and netcdf_meta.h.in and test_common.in. * Merge partial S3 support into dhttp.c. * Create an experimental s3 access library especially for use with Windows. It is enabled by using the options *--enable-s3-internal* (automake) or *-DENABLE_S3_INTERNAL=ON* (CMake). Also add a unit-test for it. * Move some definitions from ncrc.h to ncs3sdk.h ## Other Changes * Provide a default implementation of strlcpy and move this and similar defaults into *dmissing.c*.
184 lines
5.2 KiB
C
184 lines
5.2 KiB
C
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
|
|
* Copyright by The HDF Group. *
|
|
* Copyright by the Board of Trustees of the University of Illinois. *
|
|
* All rights reserved. *
|
|
* *
|
|
* This file is part of HDF5. The full HDF5 copyright notice, including *
|
|
* terms governing use, modification, and redistribution, is contained in *
|
|
* the COPYING file, which can be found at the root of the source code *
|
|
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
|
|
* If you do not have access to either file, you may request a copy from *
|
|
* help@hdfgroup.org. *
|
|
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
|
|
|
|
/*
|
|
* Programmer: Robb Matzke
|
|
* Friday, August 27, 1999
|
|
*/
|
|
|
|
/* Converted to NCZarr support by Dennis Heimbigner 5/1/2021 */
|
|
|
|
#include <stdlib.h>
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
#include <math.h>
|
|
#include <assert.h>
|
|
|
|
#include "netcdf_json.h"
|
|
|
|
#include "netcdf.h"
|
|
#include "netcdf_filter.h"
|
|
#include "netcdf_filter_build.h"
|
|
|
|
#include "h5misc.h"
|
|
|
|
/**************************************************/
|
|
/* NCZarr Filter Objects */
|
|
/* Codec Format
|
|
{
|
|
"id": "test",
|
|
"testcase": "n",
|
|
"byte": "<unsigned int>",
|
|
"ubyte": "<unsigned int>",
|
|
"short": "<unsigned int>",
|
|
"ushort": "<unsigned int>",
|
|
"int": "<unsigned int>",
|
|
"uint": "<unsigned int>",
|
|
"float": "<unsigned int>",
|
|
"double0": "<unsigned int>",
|
|
"double1": "<unsigned int>",
|
|
"int640": "<unsigned int>",
|
|
"int641": "<unsigned int>",
|
|
"uint640": "<unsigned int>",
|
|
"uint641": "<unsigned int>",
|
|
}
|
|
*/
|
|
|
|
/* Give unique dict key names for parameters */
|
|
static const char* fields[14] = {
|
|
"testcase",
|
|
"byte",
|
|
"ubyte",
|
|
"short",
|
|
"ushort",
|
|
"int",
|
|
"uint",
|
|
"float",
|
|
"double_0",
|
|
"double_1",
|
|
"int64_0",
|
|
"int64_1",
|
|
"uint64_0",
|
|
"uint64_1"
|
|
};
|
|
|
|
/* Forward */
|
|
static int NCZ_misc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp);
|
|
static int NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp);
|
|
|
|
/* Structure for NCZ_PLUGIN_CODEC */
|
|
static NCZ_codec_t NCZ_misc_codec = {/* NCZ_codec_t codec fields */
|
|
NCZ_CODEC_CLASS_VER, /* Struct version number */
|
|
NCZ_CODEC_HDF5, /* Struct sort */
|
|
"test", /* Standard name/id of the codec */
|
|
H5Z_FILTER_TEST, /* HDF5 alias for misc */
|
|
NULL, /*NCZ_misc_codec_initialize*/
|
|
NULL, /*NCZ_misc_codec_finalize*/
|
|
NCZ_misc_codec_to_hdf5,
|
|
NCZ_misc_hdf5_to_codec,
|
|
NULL, /*NCZ_misc_modify_parameters*/
|
|
};
|
|
|
|
/* External Export API */
|
|
DLLEXPORT
|
|
const void*
|
|
NCZ_get_codec_info(void)
|
|
{
|
|
return (void*)&NCZ_misc_codec;
|
|
}
|
|
|
|
/* NCZarr Interface Functions */
|
|
|
|
static int
|
|
NCZ_misc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
NCjson* jcodec = NULL;
|
|
NCjson* jtmp = NULL;
|
|
size_t i,nparams = 0;
|
|
unsigned* params = NULL;
|
|
|
|
/* parse the JSON */
|
|
if(NCJparse(codec_json,0,&jcodec))
|
|
{stat = NC_EFILTER; goto done;}
|
|
if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;}
|
|
|
|
/* Verify the codec ID */
|
|
if(NCJdictget(jcodec,"id",&jtmp))
|
|
{stat = NC_EFILTER; goto done;}
|
|
if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;}
|
|
if(strcmp(NCJstring(jtmp),NCZ_misc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;}
|
|
|
|
/* The codec will have (2*14 + 1) +1 = 29 dict entries + id*/
|
|
nparams = (2*14 + 1) + 1;
|
|
if(NCJlength(jcodec) != nparams) {
|
|
fprintf(stderr,"Incorrect no. of codec parameters: need=29 sent=%ld\n",(unsigned long)(nparams-1));
|
|
stat = NC_EINVAL;
|
|
goto done;
|
|
}
|
|
|
|
/* Actual # of parameters is 14 (ignoring the testcase number) */
|
|
nparams = 14;
|
|
if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL)
|
|
{stat = NC_ENOMEM; goto done;}
|
|
|
|
for(i=0;i<nparams;i++) {
|
|
struct NCJconst jc;
|
|
if(NCJdictget(jcodec,fields[i],&jtmp))
|
|
{stat = NC_EFILTER; goto done;}
|
|
if(NCJcvt(jtmp,NCJ_INT,&jc))
|
|
{stat = NC_EFILTER; goto done;}
|
|
if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;}
|
|
params[i] = (unsigned)jc.ival;
|
|
}
|
|
if(nparamsp) *nparamsp = nparams;
|
|
if(paramsp) {*paramsp = params; params = NULL;}
|
|
|
|
done:
|
|
if(params) free(params);
|
|
NCJreclaim(jcodec);
|
|
return stat;
|
|
}
|
|
|
|
static int
|
|
NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp)
|
|
{
|
|
int i,stat = NC_NOERR;
|
|
char json[4096];
|
|
char value[1024];
|
|
size_t count, jlen;
|
|
|
|
if(nparams == 0 || params == NULL)
|
|
{stat = NC_EINVAL; goto done;}
|
|
if(nparams != 14) {
|
|
fprintf(stderr,"Incorrect no. of parameters: need=14 sent=%ld\n",(unsigned long)nparams);
|
|
stat = NC_EINVAL;
|
|
goto done;
|
|
}
|
|
jlen = sizeof(json);
|
|
count = snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_misc_codec.codecid);
|
|
for(i=0;i<14;i++) {
|
|
size_t len = snprintf(value,sizeof(value),", \"%s\": \"%u\"",fields[i],params[i]);
|
|
count += len; assert(jlen > count);
|
|
strcat(json,value);
|
|
}
|
|
count += 1; assert(jlen > count);
|
|
strcat(json,"}");
|
|
if(codecp) {
|
|
if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;}
|
|
}
|
|
|
|
done:
|
|
return stat;
|
|
}
|