mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-12-21 08:39:46 +08:00
f6e25b695e
re: https://github.com/Unidata/netcdf-c/issues/2117 re: https://github.com/Unidata/netcdf-c/issues/2119 * Modify libsrc to allow byte-range reading of netcdf-3 files in private S3 buckets; this required using the aws sdk. Also add a test case. * The aws sdk can sometimes cause problems if the Awd::ShutdownAPI function is not called. So at optional atexit() support to ensure it is called. This is disabled for Windows. * Add documentation to nczarr.md on how to build and use the aws sdk under windows. Currently it builds, but testing fails. * Switch testing from stratus to the Unidata bucket on S3. * Improve support for the s3: url protocol. * Add a s3 specific utility code file: ds3util.c * Modify NC_infermodel to attempt to read the magic number of byte-ranged files in S3. ## Misc. * Move and rename the core S3 SDK wrapper code (libnczarr/zs3sdk.cpp) to libdispatch since it now used in libsrc as well as libnczarr. * Add calls to nc_finalize in the utilities in case atexit is disabled. * Add header only json parser to the distribution rather than as a built source.
138 lines
3.4 KiB
C
138 lines
3.4 KiB
C
/*
|
|
* Copyright 2018, University Corporation for Atmospheric Research
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
*/
|
|
|
|
#include "ut_includes.h"
|
|
|
|
void ut_chunk_test(int sort, ...);
|
|
|
|
/**
|
|
Test computation of applying a slice to a sequence of chunks
|
|
*/
|
|
|
|
int
|
|
main(int argc, char** argv)
|
|
{
|
|
int i,r,stat = NC_NOERR;
|
|
Vardef* var = NULL;
|
|
struct Common common;
|
|
NCZSliceProjections slpv[NC_MAX_VAR_DIMS];
|
|
NCZChunkRange ncrv[NC_MAX_VAR_DIMS];
|
|
|
|
/* Initialize */
|
|
memset(&slpv,0,sizeof(slpv));
|
|
memset(&common,0,sizeof(common));
|
|
|
|
if((stat = ut_init(argc, argv, &utoptions))) goto done;
|
|
|
|
/* printer off for these tests */
|
|
zutester.tests = 0;
|
|
zutester.print = NULL;
|
|
zutest = &zutester;
|
|
|
|
printoptions(&utoptions);
|
|
|
|
var = nclistget(utoptions.vardefs,0);
|
|
|
|
fillcommon(&common,var);
|
|
|
|
/* Compute chunk ranges */
|
|
if((stat = NCZ_compute_chunk_ranges(var->rank,utoptions.slices,var->chunksizes,ncrv)))
|
|
goto done;
|
|
|
|
if((stat=NCZ_compute_all_slice_projections(
|
|
&common,
|
|
utoptions.slices,
|
|
ncrv,
|
|
slpv))) goto done;
|
|
|
|
/* Dump Results */
|
|
for(r=0;r<var->rank;r++) {
|
|
NCZSliceProjections* slp = &slpv[r];
|
|
if(r != slp->r) usage(NC_EINTERNAL);
|
|
printf("[r=%d] %s %s\n",r,nczprint_chunkrange(slp->range),nczprint_slice(utoptions.slices[r]));
|
|
for(i=0;i<slp->count;i++) {
|
|
NCZProjection* proj = &slp->projections[i];
|
|
printf("[%d] %s\n",i,nczprint_projection(*proj));
|
|
}
|
|
}
|
|
|
|
/* Cleanup */
|
|
NCZ_clearsliceprojections(var->rank,slpv);
|
|
|
|
#if 0
|
|
/* Compute corresponding slice projections */
|
|
for(r=0;r<var->rank;r++) {
|
|
if((stat = NCZ_compute_per_slice_projections(
|
|
r,
|
|
&utoptions.slices[r],
|
|
&ncrv[r],
|
|
var->dimsizes[r],
|
|
var->chunksizes[r],
|
|
&slpv[r]))) goto done;
|
|
}
|
|
|
|
/* Dump Results */
|
|
for(r=0;r<var->rank;r++) {
|
|
NCZSliceProjections* slp = &slpv[r];
|
|
char *sr, *sl;
|
|
if(r != slp->r) usage(NC_EINTERNAL);
|
|
sr = nczprint_chunkrange(slp->range);
|
|
sl = nczprint_slice(utoptions.slices[r]);
|
|
printf("[r=%d] %s %s\n",r,sr,sl);
|
|
nullfree(sr); nullfree(sl);
|
|
for(i=0;i<slp->count;i++) {
|
|
NCZProjection* proj = &slp->projections[i];
|
|
printf("[%d] %s\n",i,nczprint_projection(*proj));
|
|
}
|
|
}
|
|
/* Cleanup */
|
|
NCZ_clearsliceprojections(var->rank,slpv);
|
|
#endif
|
|
|
|
done:
|
|
fflush(stdout);
|
|
nczprint_reclaim();
|
|
ut_final();
|
|
if(stat) usage(stat);
|
|
return 0;
|
|
}
|
|
|
|
void
|
|
ut_chunk_test(int sort,...)
|
|
{
|
|
int i;
|
|
va_list ap;
|
|
#if 0
|
|
struct Common* common = NULL;
|
|
#endif
|
|
int rank; /* variable rank */
|
|
NCZSlice* slices = NULL; /* the complete set of slices |slices| == R*/
|
|
size64_t* chunksizes = NULL; /* the chunk length corresponding to the dimensions */
|
|
NCZChunkRange* ranges = NULL; /* computed chunk ranges */
|
|
|
|
va_start(ap,sort);
|
|
|
|
switch (sort) {
|
|
default: break; /* ignore */
|
|
case UTEST_RANGE: /* () */
|
|
rank = va_arg(ap,size64_t);
|
|
slices = va_arg(ap,NCZSlice*);
|
|
chunksizes = va_arg(ap,size64_t*);
|
|
ranges = va_arg(ap,NCZChunkRange*);
|
|
printf("[r=%d] Chunksizes: %s\n",rank,nczprint_vector(rank,chunksizes));
|
|
printf("Slices: ");
|
|
for(i=0;i<rank;i++)
|
|
printf(" %s",nczprint_slicesx(rank,slices,1));
|
|
printf("\n");
|
|
printf("Ranges: ");
|
|
for(i=0;i<rank;i++)
|
|
printf(" %s",nczprint_chunkrange(ranges[i]));
|
|
printf("\n");
|
|
break;
|
|
}
|
|
va_end(ap);
|
|
}
|
|
|