mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-12-21 08:39:46 +08:00
e7d5f24078
The primary change is to support the use of a zip file as a storage format. Simultaneously the .nz4 support is made obsolete Use of zip requires the libzip support library, so a number of changes to the build files (Makefile.am, CMakeLists.txt) are necessary to locate and incorporate libzip. The nczarr_tests tests are also changed to add zip testing. Other changes: * Make sure distcheck leaves no files around. * Add some functions to netcdf_aux to export some functions of libnetcdf. * Add a new error NC_EFOUND as the complement of NC_EEMPTY. * Add tracing support to nclog and use it in libnczarr. * Modify the zmap interface to support the writeonce semantics of zip. * Create a new s3util.c to support a variety of S3 auxilliary functions. * EXTERNL'ize a number of functions so they can be used in s3util. * Add support for the S3 ListObjects CommonPrefixes mechanism to improve search. * Add experimental support for running nczarr X s3 tests against the actual Amazon S3 cloud.
228 lines
5.3 KiB
C
228 lines
5.3 KiB
C
#ifdef HAVE_CONFIG_H
|
|
#include "config.h"
|
|
#endif
|
|
#include <stdlib.h>
|
|
#ifdef HAVE_CONFIG_H
|
|
#include <stdio.h>
|
|
#endif
|
|
#include <string.h>
|
|
#include <netcdf.h>
|
|
#include <ncpathmgr.h>
|
|
#include <nclist.h>
|
|
|
|
#ifdef HAVE_HDF5_H
|
|
#include <hdf5.h>
|
|
#include <H5DSpublic.h>
|
|
#endif
|
|
|
|
#ifdef ENABLE_NCZARR
|
|
#include "zincludes.h"
|
|
#endif
|
|
|
|
|
|
#include "tst_utils.h"
|
|
|
|
static unsigned chunkprod;
|
|
static unsigned dimprod;
|
|
static int* data = NULL;
|
|
static size_t datasize = 0;
|
|
|
|
static int setupwholechunk(void);
|
|
static int reportwholechunk(void);
|
|
static void zutest_print(int sort, ...);
|
|
|
|
static int
|
|
writedata(void)
|
|
{
|
|
int ret = NC_NOERR;
|
|
int i;
|
|
|
|
for(i=0;i<dimprod;i++) data[i] = i;
|
|
|
|
if(options->wholechunk)
|
|
setupwholechunk();
|
|
|
|
if(options->debug >= 1) {
|
|
fprintf(stderr,"write: dimlens=%s chunklens=%s\n",
|
|
printvector(options->rank,options->dimlens),printvector(options->rank,options->chunks));
|
|
}
|
|
if(options->wholechunk) {
|
|
fprintf(stderr,"write var: wholechunk\n");
|
|
if((ret = nc_put_vars(meta->ncid,meta->varid,options->start,options->count,(ptrdiff_t*)options->stride,data)))
|
|
ERR(ret);
|
|
} else {
|
|
fprintf(stderr,"write vars: start=%s count=%s stride=%s\n",
|
|
printvector(options->rank,options->start),printvector(options->rank,options->count),printvector(options->rank,options->stride));
|
|
if((ret = nc_put_vars(meta->ncid,meta->varid,options->start,options->count,(ptrdiff_t*)options->stride,data)))
|
|
ERR(ret);
|
|
}
|
|
|
|
if(options->wholechunk) {
|
|
if((ret=reportwholechunk()))
|
|
ERR(ret);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
readdata(void)
|
|
{
|
|
int ret = NC_NOERR;
|
|
int i;
|
|
|
|
memset(data,0,datasize);
|
|
|
|
if(options->wholechunk) {
|
|
setupwholechunk();
|
|
}
|
|
|
|
if(options->debug >= 1)
|
|
fprintf(stderr,"read: dimlens=%s chunklens=%s\n",
|
|
printvector(options->rank,options->dimlens),printvector(options->rank,options->chunks));
|
|
fprintf(stderr,"read vars: start=%s count=%s stride=%s",
|
|
printvector(options->rank,options->start),
|
|
printvector(options->rank,options->count),
|
|
printvector(options->rank,options->stride));
|
|
if(options->wholechunk)
|
|
fprintf(stderr," wholechunk");
|
|
fprintf(stderr,"\n");
|
|
if((ret = nc_get_vars(meta->ncid,meta->varid,options->start,options->count,(ptrdiff_t*)options->stride,data)))
|
|
ERR(ret);
|
|
|
|
for(i=0;i<dimprod;i++) {
|
|
printf("[%d] %d\n",i,data[i]);
|
|
}
|
|
if(options->wholechunk)
|
|
reportwholechunk();
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
genodom(void)
|
|
{
|
|
int i,ret = NC_NOERR;
|
|
Odometer* odom = odom_new(options->rank, options->start, options->stop, options->stride, options->max);
|
|
if(odom == NULL) {ret = NC_ENOMEM; goto done;}
|
|
if(options->debug > 1)
|
|
fprintf(stderr,"genodom: odom = %s\n",odom_print(odom));
|
|
/* Iterate the odometer */
|
|
for(i=0;odom_more(odom);odom_next(odom),i++) {
|
|
printf("[%02d] %s\n",i,(i==0?odom_print(odom):odom_printshort(odom)));
|
|
}
|
|
done:
|
|
odom_free(odom);
|
|
return ret;
|
|
}
|
|
|
|
static int wholechunkcalls = 0;
|
|
static struct ZUTEST zutester;
|
|
|
|
static int
|
|
setupwholechunk(void)
|
|
{
|
|
int ret = NC_NOERR;
|
|
|
|
wholechunkcalls = 0;
|
|
|
|
#ifdef ENABLE_NCZARR
|
|
/* Set the printer */
|
|
zutester.tests = UTEST_WHOLECHUNK;
|
|
zutester.print = zutest_print;
|
|
zutest = &zutester; /* See zdebug.h */
|
|
#endif
|
|
return ret;
|
|
}
|
|
|
|
|
|
static void
|
|
zutest_print(int sort, ...)
|
|
{
|
|
va_list ap;
|
|
struct Common* common = NULL;
|
|
size64_t* chunkindices;
|
|
|
|
NC_UNUSED(common);
|
|
|
|
va_start(ap,sort);
|
|
|
|
switch (sort) {
|
|
default: break; /* ignore */
|
|
case UTEST_WHOLECHUNK:
|
|
common = va_arg(ap,struct Common*);
|
|
chunkindices = va_arg(ap,size64_t*);
|
|
if(options->debug >= 1)
|
|
fprintf(stderr,"wholechunk: indices=%s\n",printvector64(common->rank,chunkindices));
|
|
wholechunkcalls++;
|
|
break;
|
|
}
|
|
va_end(ap);
|
|
}
|
|
|
|
static int
|
|
reportwholechunk(void)
|
|
{
|
|
int ret = NC_NOERR;
|
|
#ifdef ENABLE_NCZARR
|
|
if(options->debug > 0)
|
|
fprintf(stderr,"wholechunkcalls=%d\n",wholechunkcalls);
|
|
if(wholechunkcalls != 1)
|
|
return NC_EINVAL;
|
|
#endif
|
|
return ret;
|
|
}
|
|
|
|
|
|
|
|
|
|
int
|
|
main(int argc, char** argv)
|
|
{
|
|
int stat = NC_NOERR;
|
|
int i;
|
|
|
|
if((stat=getoptions(&argc,&argv))) goto done;
|
|
|
|
if(options->formatx != NC_FORMATX_NCZARR && options->wholechunk) {
|
|
fprintf(stderr,"Format is not NCZarr; -OW_ ignored\n");
|
|
options->wholechunk = 0;
|
|
}
|
|
|
|
switch (options->op) {
|
|
case Read:
|
|
if((stat = getmetadata(0)))
|
|
ERR(stat);
|
|
if (argc == 0) {fprintf(stderr, "no input file specified\n");exit(1);}
|
|
break;
|
|
case Write:
|
|
if((stat = getmetadata(1)))
|
|
ERR(stat);
|
|
if (argc == 0) {fprintf(stderr, "no output file specified\n");exit(1);}
|
|
break;
|
|
default:
|
|
break; /* do not need a file */
|
|
}
|
|
|
|
dimprod = 1;
|
|
chunkprod = 1;
|
|
for(i=0;i<options->rank;i++) {dimprod *= options->dimlens[i]; chunkprod *= options->chunks[i];}
|
|
|
|
datasize = dimprod*sizeof(int);
|
|
if((data = calloc(1,datasize)) == NULL)
|
|
{fprintf(stderr,"out of memory\n"); exit(1);}
|
|
|
|
switch (options->op) {
|
|
case Read: readdata(); break;
|
|
case Write: writedata(); break;
|
|
case Odom: genodom(); break;
|
|
default:
|
|
fprintf(stderr,"Unknown operation\n");
|
|
exit(1);
|
|
}
|
|
done:
|
|
if(data) free(data);
|
|
cleanup();
|
|
return 0;
|
|
}
|