mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-11-21 03:13:42 +08:00
Merge pull request #2238 from WardF/gh2206.wif
Conflict resolution for #2206
This commit is contained in:
Ward Fisher
GitHub
commit
c30801f7b0
@ -9,6 +9,7 @@ This file contains a high-level description of this package's evolution. Release
|
||||
|
||||
* [Bug Fix] Require that the type of the variable in nc_def_var_filter is not variable length. See [Github #/2231](https://github.com/Unidata/netcdf-c/pull/2231).
|
||||
* [File Change] Apply HDF5 v1.8 format compatibility when writing to previous files, as well as when creating new files. The superblock version remains at 2 for newly created files. Full backward read/write compatibility for netCDF-4 is maintained in all cases. See [Github #2176](https://github.com/Unidata/netcdf-c/issues/2176).
|
||||
* [Enhancement] Add ability to set dataset alignment for netcdf-4/HDF5 files. See [Github #2206](https://github.com/Unidata/netcdf-c/pull/2206).
|
||||
* [Enhancement] Add complete bitgroom support to NCZarr. See [Github #2197](https://github.com/Unidata/netcdf-c/pull/2197).
|
||||
* [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179](https://github.com/Unidata/netcdf-c/pull/2179).
|
||||
* [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183).
|
||||
|
||||
@ -384,4 +384,6 @@ MOSTLYCLEANFILES += $(valgrind_log_files)
|
||||
AC_SUBST([VALGRIND_CHECK_RULES])
|
||||
m4_ifdef([_AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE([VALGRIND_CHECK_RULES])])
|
||||
])
|
||||
|
||||
AC_DEFUN([_AC_FINALIZE],[])
|
||||
|
||||
|
||||
@ -781,7 +781,6 @@ INPUT = \
|
||||
@abs_top_srcdir@/libdispatch/derror.c \
|
||||
@abs_top_srcdir@/libdispatch/dv2i.c \
|
||||
@abs_top_srcdir@/libdispatch/dcopy.c \
|
||||
@abs_top_srcdir@/libdispatch/daux.c \
|
||||
@abs_top_srcdir@/libsrc4/nc4var.c \
|
||||
@abs_top_srcdir@/libhdf5/nc4hdf.c \
|
||||
@abs_top_srcdir@/libsrc4/nc4internal.c \
|
||||
|
||||
@ -118,6 +118,7 @@ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t;
|
||||
/* Forward declarations. */
|
||||
struct NC_GRP_INFO;
|
||||
struct NC_TYPE_INFO;
|
||||
struct NCRCinfo;
|
||||
|
||||
/**
|
||||
* This struct provides indexed Access to Meta-data objects. See the
|
||||
@ -205,9 +206,11 @@ typedef struct NC_VAR_INFO
|
||||
int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */
|
||||
nc_bool_t shuffle; /**< True if var has shuffle filter applied. */
|
||||
nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */
|
||||
size_t chunk_cache_size; /**< Size in bytes of the var chunk cache. */
|
||||
size_t chunk_cache_nelems; /**< Number of slots in var chunk cache. */
|
||||
float chunk_cache_preemption; /**< Chunk cache preemtion policy. */
|
||||
struct ChunkCache {
|
||||
size_t size; /**< Size in bytes of the var chunk cache. */
|
||||
size_t nelems; /**< Number of slots in var chunk cache. */
|
||||
float preemption; /**< Chunk cache preemtion policy. */
|
||||
} chunkcache;
|
||||
int quantize_mode; /**< Quantize mode. NC_NOQUANTIZE is 0, and means no quantization. */
|
||||
int nsd; /**< Number of significant digits if quantization is used, 0 if not. */
|
||||
void *format_var_info; /**< Pointer to any binary format info. */
|
||||
@ -327,6 +330,24 @@ typedef struct NC_FILE_INFO
|
||||
} mem;
|
||||
} NC_FILE_INFO_T;
|
||||
|
||||
/* Collect global state info in one place */
|
||||
typedef struct NCglobalstate {
|
||||
int initialized;
|
||||
char* tempdir; /* track a usable temp dir */
|
||||
char* home; /* track $HOME */
|
||||
char* cwd; /* track getcwd */
|
||||
struct NCRCinfo* rcinfo; /* Currently only one rc file per session */
|
||||
struct GlobalZarr { /* Zarr specific parameters */
|
||||
char dimension_separator;
|
||||
} zarr;
|
||||
struct Alignment { /* H5Pset_alignment parameters */
|
||||
int defined; /* 1 => threshold and alignment explicitly set */
|
||||
int threshold;
|
||||
int alignment;
|
||||
} alignment;
|
||||
struct ChunkCache chunkcache;
|
||||
} NCglobalstate;
|
||||
|
||||
/** Variable Length Datatype struct in memory. Must be identical to
|
||||
* HDF5 hvl_t. (This is only used for VL sequences, not VL strings,
|
||||
* which are stored in char *'s) */
|
||||
@ -460,6 +481,10 @@ extern const char* nc4_atomic_name[NUM_ATOMIC_TYPES];
|
||||
/* Binary searcher for reserved attributes */
|
||||
extern const NC_reservedatt* NC_findreserved(const char* name);
|
||||
|
||||
/* Global State Management */
|
||||
extern NCglobalstate* NC_getglobalstate(void);
|
||||
extern void NC_freeglobalstate(void);
|
||||
|
||||
/* Generic reserved Attributes */
|
||||
#define NC_ATT_REFERENCE_LIST "REFERENCE_LIST"
|
||||
#define NC_ATT_CLASS "CLASS"
|
||||
|
||||
@ -33,39 +33,25 @@ typedef struct NCRCentry {
|
||||
char* value;
|
||||
} NCRCentry;
|
||||
|
||||
/* collect all the relevant info around the rc file */
|
||||
typedef struct NCRCinfo {
|
||||
int ignore; /* if 1, then do not use any rc file */
|
||||
int loaded; /* 1 => already loaded */
|
||||
NClist* entries; /* the rc file entry store fields*/
|
||||
char* rcfile; /* specified rcfile; overrides anything else */
|
||||
char* rchome; /* Overrides $HOME when looking for .rc files */
|
||||
} NCRCinfo;
|
||||
|
||||
/* Collect global state info in one place */
|
||||
typedef struct NCRCglobalstate {
|
||||
int initialized;
|
||||
char* tempdir; /* track a usable temp dir */
|
||||
char* home; /* track $HOME */
|
||||
char* cwd; /* track getcwd */
|
||||
NCRCinfo rcinfo; /* Currently only one rc file per session */
|
||||
struct GlobalZarr { /* Zarr specific parameters */
|
||||
char dimension_separator;
|
||||
} zarr;
|
||||
struct S3credentials {
|
||||
NClist* profiles; /* NClist<struct AWSprofile*> */
|
||||
} s3creds;
|
||||
} NCRCglobalstate;
|
||||
struct AWSentry {
|
||||
char* key;
|
||||
char* value;
|
||||
};
|
||||
|
||||
struct AWSprofile {
|
||||
char* name;
|
||||
NClist* entries; /* NClist<struct AWSentry*> */
|
||||
};
|
||||
|
||||
struct AWSentry {
|
||||
char* key;
|
||||
char* value;
|
||||
};
|
||||
/* collect all the relevant info around the rc file and AWS */
|
||||
typedef struct NCRCinfo {
|
||||
int ignore; /* if 1, then do not use any rc file */
|
||||
int loaded; /* 1 => already loaded */
|
||||
NClist* entries; /* the rc file entry store fields*/
|
||||
char* rcfile; /* specified rcfile; overrides anything else */
|
||||
char* rchome; /* Overrides $HOME when looking for .rc files */
|
||||
NClist* s3profiles; /* NClist<struct AWSprofile*> */
|
||||
} NCRCinfo;
|
||||
|
||||
typedef struct NCS3INFO {
|
||||
char* host; /* non-null if other*/
|
||||
@ -80,9 +66,7 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
/* From drc.c */
|
||||
EXTERNL int ncrc_createglobalstate(void);
|
||||
EXTERNL void ncrc_initialize(void);
|
||||
EXTERNL void ncrc_freeglobalstate(void);
|
||||
EXTERNL int NC_rcfile_insert(const char* key, const char* value, const char* hostport, const char* path);
|
||||
EXTERNL char* NC_rclookup(const char* key, const char* hostport, const char* path);
|
||||
EXTERNL char* NC_rclookupx(NCURI* uri, const char* key);
|
||||
@ -94,7 +78,6 @@ EXTERNL size_t NC_rcfile_length(NCRCinfo*);
|
||||
EXTERNL NCRCentry* NC_rcfile_ith(NCRCinfo*,size_t);
|
||||
|
||||
/* For internal use */
|
||||
EXTERNL NCRCglobalstate* ncrc_getglobalstate(void);
|
||||
EXTERNL void NC_rcclear(NCRCinfo* info);
|
||||
EXTERNL void NC_rcclear(NCRCinfo* info);
|
||||
|
||||
|
||||
@ -560,6 +560,14 @@ nc_def_user_format(int mode_flag, NC_Dispatch *dispatch_table, char *magic_numbe
|
||||
EXTERNL int
|
||||
nc_inq_user_format(int mode_flag, NC_Dispatch **dispatch_table, char *magic_number);
|
||||
|
||||
/* Set the global alignment property */
|
||||
EXTERNL int
|
||||
nc_set_alignment(int threshold, int alignment);
|
||||
|
||||
/* Get the global alignment property */
|
||||
EXTERNL int
|
||||
nc_get_alignment(int* thresholdp, int* alignmentp);
|
||||
|
||||
EXTERNL int
|
||||
nc__create(const char *path, int cmode, size_t initialsz,
|
||||
size_t *chunksizehintp, int *ncidp);
|
||||
|
||||
@ -367,7 +367,7 @@ set_curl_properties(NCD4INFO* d4info)
|
||||
char* newpath = NULL;
|
||||
int len;
|
||||
errno = 0;
|
||||
NCRCglobalstate* globalstate = ncrc_getglobalstate();
|
||||
NCglobalstate* globalstate = NC_getglobalstate();
|
||||
|
||||
/* Create the unique cookie file name */
|
||||
len =
|
||||
|
||||
@ -12,6 +12,7 @@ See LICENSE.txt for license information.
|
||||
#include "ncoffsets.h"
|
||||
#include "ncpathmgr.h"
|
||||
#include "ncxml.h"
|
||||
#include "nc4internal.h"
|
||||
|
||||
/* Required for getcwd, other functions. */
|
||||
#ifdef HAVE_UNISTD_H
|
||||
@ -48,7 +49,7 @@ NCDISPATCH_initialize(void)
|
||||
{
|
||||
int status = NC_NOERR;
|
||||
int i;
|
||||
NCRCglobalstate* globalstate = NULL;
|
||||
NCglobalstate* globalstate = NULL;
|
||||
|
||||
for(i=0;i<NC_MAX_VAR_DIMS;i++) {
|
||||
NC_coord_zero[i] = 0;
|
||||
@ -56,8 +57,7 @@ NCDISPATCH_initialize(void)
|
||||
NC_stride_one[i] = 1;
|
||||
}
|
||||
|
||||
status = ncrc_createglobalstate(); /* will allocate and clear */
|
||||
globalstate = ncrc_getglobalstate(); /* will allocate and clear */
|
||||
globalstate = NC_getglobalstate(); /* will allocate and clear */
|
||||
|
||||
/* Capture temp dir*/
|
||||
{
|
||||
@ -123,7 +123,7 @@ int
|
||||
NCDISPATCH_finalize(void)
|
||||
{
|
||||
int status = NC_NOERR;
|
||||
ncrc_freeglobalstate();
|
||||
NC_freeglobalstate();
|
||||
#if defined(ENABLE_BYTERANGE) || defined(ENABLE_DAP) || defined(ENABLE_DAP4)
|
||||
curl_global_cleanup();
|
||||
#endif
|
||||
|
||||
@ -22,6 +22,7 @@ See COPYRIGHT for license information.
|
||||
#include "nclog.h"
|
||||
#include "ncauth.h"
|
||||
#include "ncpathmgr.h"
|
||||
#include "nc4internal.h"
|
||||
|
||||
#ifndef nulldup
|
||||
#define nulldup(x) ((x)?strdup(x):(x))
|
||||
@ -54,11 +55,10 @@ static int rccompile(const char* path);
|
||||
static struct NCRCentry* rclocate(const char* key, const char* hostport, const char* path);
|
||||
static int rcsearch(const char* prefix, const char* rcname, char** pathp);
|
||||
static void rcfreeentries(NClist* rc);
|
||||
static void clearS3credentials(struct S3credentials* creds);
|
||||
#ifdef DRCDEBUG
|
||||
static void storedump(char* msg, NClist* entrys);
|
||||
#endif
|
||||
static int aws_load_credentials(NCRCglobalstate*);
|
||||
static int aws_load_credentials(NCglobalstate*);
|
||||
static void freeprofile(struct AWSprofile* profile);
|
||||
static void freeprofilelist(NClist* profiles);
|
||||
|
||||
@ -71,28 +71,8 @@ static const char* awsconfigfiles[] = {".aws/credentials",".aws/config",NULL};
|
||||
/**************************************************/
|
||||
/* External Entry Points */
|
||||
|
||||
static NCRCglobalstate* ncrc_globalstate = NULL;
|
||||
|
||||
static int NCRCinitialized = 0;
|
||||
|
||||
int
|
||||
ncrc_createglobalstate(void)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
const char* tmp = NULL;
|
||||
|
||||
if(ncrc_globalstate == NULL) {
|
||||
ncrc_globalstate = calloc(1,sizeof(NCRCglobalstate));
|
||||
}
|
||||
/* Get environment variables */
|
||||
if(getenv(NCRCENVIGNORE) != NULL)
|
||||
ncrc_globalstate->rcinfo.ignore = 1;
|
||||
tmp = getenv(NCRCENVRC);
|
||||
if(tmp != NULL && strlen(tmp) > 0)
|
||||
ncrc_globalstate->rcinfo.rcfile = strdup(tmp);
|
||||
return stat;
|
||||
}
|
||||
|
||||
/*
|
||||
Initialize defaults and load:
|
||||
* .ncrc
|
||||
@ -111,6 +91,7 @@ void
|
||||
ncrc_initialize(void)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCglobalstate* ncg = NULL;
|
||||
|
||||
if(NCRCinitialized) return;
|
||||
NCRCinitialized = 1; /* prevent recursion */
|
||||
@ -121,7 +102,8 @@ ncrc_initialize(void)
|
||||
nclog(NCLOGWARN,".rc loading failed");
|
||||
}
|
||||
/* Load .aws/config */
|
||||
if((stat = aws_load_credentials(ncrc_globalstate))) {
|
||||
ncg = NC_getglobalstate();
|
||||
if((stat = aws_load_credentials(ncg))) {
|
||||
nclog(NCLOGWARN,"AWS config file not loaded");
|
||||
}
|
||||
#endif
|
||||
@ -131,35 +113,13 @@ static void
|
||||
ncrc_setrchome(void)
|
||||
{
|
||||
const char* tmp = NULL;
|
||||
if(ncrc_globalstate->rcinfo.rchome) return;
|
||||
assert(ncrc_globalstate && ncrc_globalstate->home);
|
||||
NCglobalstate* ncg = NC_getglobalstate();
|
||||
assert(ncg && ncg->home);
|
||||
if(ncg->rcinfo->rchome) return;
|
||||
tmp = getenv(NCRCENVHOME);
|
||||
if(tmp == NULL || strlen(tmp) == 0)
|
||||
tmp = ncrc_globalstate->home;
|
||||
ncrc_globalstate->rcinfo.rchome = strdup(tmp);
|
||||
}
|
||||
|
||||
/* Get global state */
|
||||
NCRCglobalstate*
|
||||
ncrc_getglobalstate(void)
|
||||
{
|
||||
if(ncrc_globalstate == NULL)
|
||||
ncrc_createglobalstate();
|
||||
return ncrc_globalstate;
|
||||
}
|
||||
|
||||
void
|
||||
ncrc_freeglobalstate(void)
|
||||
{
|
||||
if(ncrc_globalstate != NULL) {
|
||||
nullfree(ncrc_globalstate->tempdir);
|
||||
nullfree(ncrc_globalstate->home);
|
||||
nullfree(ncrc_globalstate->cwd);
|
||||
NC_rcclear(&ncrc_globalstate->rcinfo);
|
||||
clearS3credentials(&ncrc_globalstate->s3creds);
|
||||
free(ncrc_globalstate);
|
||||
ncrc_globalstate = NULL;
|
||||
}
|
||||
tmp = ncg->home;
|
||||
ncg->rcinfo->rchome = strdup(tmp);
|
||||
}
|
||||
|
||||
void
|
||||
@ -169,9 +129,10 @@ NC_rcclear(NCRCinfo* info)
|
||||
nullfree(info->rcfile);
|
||||
nullfree(info->rchome);
|
||||
rcfreeentries(info->entries);
|
||||
freeprofilelist(info->s3profiles);
|
||||
}
|
||||
|
||||
void
|
||||
static void
|
||||
rcfreeentries(NClist* rc)
|
||||
{
|
||||
int i;
|
||||
@ -191,18 +152,17 @@ NC_rcload(void)
|
||||
{
|
||||
int i,ret = NC_NOERR;
|
||||
char* path = NULL;
|
||||
NCRCglobalstate* globalstate = NULL;
|
||||
NCglobalstate* globalstate = NULL;
|
||||
NClist* rcfileorder = nclistnew();
|
||||
|
||||
if(!NCRCinitialized) ncrc_initialize();
|
||||
globalstate = ncrc_getglobalstate();
|
||||
globalstate = NC_getglobalstate();
|
||||
|
||||
|
||||
if(globalstate->rcinfo.ignore) {
|
||||
if(globalstate->rcinfo->ignore) {
|
||||
nclog(NCLOGDBG,".rc file loading suppressed");
|
||||
goto done;
|
||||
}
|
||||
if(globalstate->rcinfo.loaded) goto done;
|
||||
if(globalstate->rcinfo->loaded) goto done;
|
||||
|
||||
/* locate the configuration files in order of use:
|
||||
1. Specified by NCRCENV_RC environment variable.
|
||||
@ -215,8 +175,8 @@ NC_rcload(void)
|
||||
6. $CWD/.docsrc
|
||||
Entry in later files override any of the earlier files
|
||||
*/
|
||||
if(globalstate->rcinfo.rcfile != NULL) { /* always use this */
|
||||
nclistpush(rcfileorder,strdup(globalstate->rcinfo.rcfile));
|
||||
if(globalstate->rcinfo->rcfile != NULL) { /* always use this */
|
||||
nclistpush(rcfileorder,strdup(globalstate->rcinfo->rcfile));
|
||||
} else {
|
||||
const char** rcname;
|
||||
const char* dirnames[3];
|
||||
@ -225,8 +185,7 @@ NC_rcload(void)
|
||||
|
||||
/* Make sure rcinfo.rchome is defined */
|
||||
ncrc_setrchome();
|
||||
|
||||
dirnames[0] = globalstate->rcinfo.rchome;
|
||||
dirnames[0] = globalstate->rcinfo->rchome;
|
||||
dirnames[1] = globalstate->cwd;
|
||||
dirnames[2] = NULL;
|
||||
|
||||
@ -249,7 +208,7 @@ NC_rcload(void)
|
||||
}
|
||||
|
||||
done:
|
||||
globalstate->rcinfo.loaded = 1; /* even if not exists */
|
||||
globalstate->rcinfo->loaded = 1; /* even if not exists */
|
||||
nclistfreeall(rcfileorder);
|
||||
return (ret);
|
||||
}
|
||||
@ -301,7 +260,7 @@ NC_set_rcfile(const char* rcfile)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
FILE* f = NULL;
|
||||
NCRCglobalstate* globalstate = ncrc_getglobalstate();
|
||||
NCglobalstate* globalstate = NC_getglobalstate();
|
||||
|
||||
if(rcfile != NULL && strlen(rcfile) == 0)
|
||||
rcfile = NULL;
|
||||
@ -311,8 +270,8 @@ NC_set_rcfile(const char* rcfile)
|
||||
goto done;
|
||||
}
|
||||
fclose(f);
|
||||
nullfree(globalstate->rcinfo.rcfile);
|
||||
globalstate->rcinfo.rcfile = strdup(rcfile);
|
||||
nullfree(globalstate->rcinfo->rcfile);
|
||||
globalstate->rcinfo->rcfile = strdup(rcfile);
|
||||
/* Clear globalstate->rcinfo */
|
||||
NC_rcclear(&globalstate->rcinfo);
|
||||
/* (re) load the rcfile and esp the entriestore*/
|
||||
@ -414,7 +373,7 @@ rccompile(const char* path)
|
||||
NCbytes* tmp = ncbytesnew();
|
||||
NCURI* uri = NULL;
|
||||
char* nextline = NULL;
|
||||
NCRCglobalstate* globalstate = ncrc_getglobalstate();
|
||||
NCglobalstate* globalstate = NC_getglobalstate();
|
||||
char* bucket = NULL;
|
||||
|
||||
if((ret=NC_readfile(path,tmp))) {
|
||||
@ -424,10 +383,10 @@ rccompile(const char* path)
|
||||
contents = ncbytesextract(tmp);
|
||||
if(contents == NULL) contents = strdup("");
|
||||
/* Either reuse or create new */
|
||||
rc = globalstate->rcinfo.entries;
|
||||
rc = globalstate->rcinfo->entries;
|
||||
if(rc == NULL) {
|
||||
rc = nclistnew();
|
||||
globalstate->rcinfo.entries = rc;
|
||||
globalstate->rcinfo->entries = rc;
|
||||
}
|
||||
nextline = contents;
|
||||
for(;;) {
|
||||
@ -529,11 +488,11 @@ static struct NCRCentry*
|
||||
rclocate(const char* key, const char* hostport, const char* path)
|
||||
{
|
||||
int i,found;
|
||||
NCRCglobalstate* globalstate = ncrc_getglobalstate();
|
||||
NClist* rc = globalstate->rcinfo.entries;
|
||||
NCglobalstate* globalstate = NC_getglobalstate();
|
||||
NClist* rc = globalstate->rcinfo->entries;
|
||||
NCRCentry* entry = NULL;
|
||||
|
||||
if(globalstate->rcinfo.ignore)
|
||||
if(globalstate->rcinfo->ignore)
|
||||
return NULL;
|
||||
|
||||
if(key == NULL || rc == NULL) return NULL;
|
||||
@ -610,12 +569,12 @@ NC_rcfile_insert(const char* key, const char* value, const char* hostport, const
|
||||
int ret = NC_NOERR;
|
||||
/* See if this key already defined */
|
||||
struct NCRCentry* entry = NULL;
|
||||
NCRCglobalstate* globalstate = NULL;
|
||||
NCglobalstate* globalstate = NULL;
|
||||
NClist* rc = NULL;
|
||||
|
||||
if(!NCRCinitialized) ncrc_initialize();
|
||||
globalstate = ncrc_getglobalstate();
|
||||
rc = globalstate->rcinfo.entries;
|
||||
globalstate = NC_getglobalstate();
|
||||
rc = globalstate->rcinfo->entries;
|
||||
|
||||
if(rc == NULL) {
|
||||
rc = nclistnew();
|
||||
@ -744,13 +703,6 @@ NC_getdefaults3region(NCURI* uri, const char** regionp)
|
||||
return stat;
|
||||
}
|
||||
|
||||
static void
|
||||
clearS3credentials(struct S3credentials* creds)
|
||||
{
|
||||
if(creds)
|
||||
freeprofilelist(creds->profiles);
|
||||
}
|
||||
|
||||
/**
|
||||
The .aws/config and .aws/credentials files
|
||||
are in INI format (https://en.wikipedia.org/wiki/INI_file).
|
||||
@ -1038,10 +990,9 @@ freeprofilelist(NClist* profiles)
|
||||
|
||||
/* Find, load, and parse the aws credentials file */
|
||||
static int
|
||||
aws_load_credentials(NCRCglobalstate* gstate)
|
||||
aws_load_credentials(NCglobalstate* gstate)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
struct S3credentials* creds = &gstate->s3creds;
|
||||
NClist* profiles = nclistnew();
|
||||
const char** awscfg = awsconfigfiles;
|
||||
const char* aws_root = getenv(NC_TEST_AWS_DIR);
|
||||
@ -1074,7 +1025,7 @@ aws_load_credentials(NCRCglobalstate* gstate)
|
||||
nclistpush(profiles,noprof); noprof = NULL;
|
||||
}
|
||||
|
||||
creds->profiles = profiles; profiles = NULL;
|
||||
gstate->rcinfo->s3profiles = profiles; profiles = NULL;
|
||||
|
||||
#ifdef AWSDEBUG
|
||||
{int i,j;
|
||||
@ -1102,10 +1053,10 @@ NC_authgets3profile(const char* profilename, struct AWSprofile** profilep)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
int i = -1;
|
||||
NCRCglobalstate* gstate = ncrc_getglobalstate();
|
||||
|
||||
for(i=0;i<nclistlength(gstate->s3creds.profiles);i++) {
|
||||
struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->s3creds.profiles,i);
|
||||
NCglobalstate* gstate = NC_getglobalstate();
|
||||
|
||||
for(i=0;i<nclistlength(gstate->rcinfo->s3profiles);i++) {
|
||||
struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i);
|
||||
if(strcmp(profilename,profile->name)==0)
|
||||
{if(profilep) {*profilep = profile; goto done;}}
|
||||
}
|
||||
|
||||
@ -33,7 +33,7 @@
|
||||
#define nulldup(x) ((x)?strdup(x):(x))
|
||||
#endif
|
||||
/**************************************************/
|
||||
/**
|
||||
/** \internal
|
||||
* Provide a hidden interface to allow utilities
|
||||
* to check if a given path name is really an ncdap4 url.
|
||||
* If no, return null, else return basename of the url
|
||||
@ -66,7 +66,7 @@ NC__testurl(const char* path, char** basenamep)
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Return 1 if this machine is little endian */
|
||||
/** \internal Return 1 if this machine is little endian */
|
||||
int
|
||||
NC_isLittleEndian(void)
|
||||
{
|
||||
@ -78,6 +78,7 @@ NC_isLittleEndian(void)
|
||||
return (u.bytes[0] == 1 ? 1 : 0);
|
||||
}
|
||||
|
||||
/** \internal */
|
||||
char*
|
||||
NC_backslashEscape(const char* s)
|
||||
{
|
||||
@ -105,6 +106,7 @@ NC_backslashEscape(const char* s)
|
||||
return escaped;
|
||||
}
|
||||
|
||||
/** \internal */
|
||||
char*
|
||||
NC_backslashUnescape(const char* esc)
|
||||
{
|
||||
@ -129,6 +131,7 @@ NC_backslashUnescape(const char* esc)
|
||||
return s;
|
||||
}
|
||||
|
||||
/** \internal */
|
||||
char*
|
||||
NC_entityescape(const char* s)
|
||||
{
|
||||
@ -163,13 +166,13 @@ NC_entityescape(const char* s)
|
||||
return escaped;
|
||||
}
|
||||
|
||||
char*
|
||||
/*
|
||||
/** \internal
|
||||
Depending on the platform, the shell will sometimes
|
||||
pass an escaped octotherpe character without removing
|
||||
the backslash. So this function is appropriate to be called
|
||||
on possible url paths to unescape such cases. See e.g. ncgen.
|
||||
*/
|
||||
char*
|
||||
NC_shellUnescape(const char* esc)
|
||||
{
|
||||
size_t len;
|
||||
@ -194,7 +197,7 @@ NC_shellUnescape(const char* esc)
|
||||
return s;
|
||||
}
|
||||
|
||||
/**
|
||||
/** \internal
|
||||
Wrap mktmp and return the generated path,
|
||||
or null if failed.
|
||||
Base is the base file path. XXXXX is appended
|
||||
@ -224,6 +227,7 @@ done:
|
||||
return tmp;
|
||||
}
|
||||
|
||||
/** \internal */
|
||||
int
|
||||
NC_readfile(const char* filename, NCbytes* content)
|
||||
{
|
||||
@ -246,6 +250,7 @@ done:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/** \internal */
|
||||
int
|
||||
NC_writefile(const char* filename, size_t size, void* content)
|
||||
{
|
||||
@ -271,7 +276,7 @@ done:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
/** \internal
|
||||
Parse a path as a url and extract the modelist.
|
||||
If the path is not a URL, then return a NULL list.
|
||||
If a URL, but modelist is empty or does not exist,
|
||||
@ -297,7 +302,7 @@ done:
|
||||
return stat;
|
||||
}
|
||||
|
||||
/*
|
||||
/** \internal
|
||||
Check "mode=" list for a path and return 1 if present, 0 otherwise.
|
||||
*/
|
||||
int
|
||||
@ -313,7 +318,7 @@ NC_testpathmode(const char* path, const char* tag)
|
||||
return found;
|
||||
}
|
||||
|
||||
/*
|
||||
/** \internal
|
||||
Check "mode=" list for a url and return 1 if present, 0 otherwise.
|
||||
*/
|
||||
int
|
||||
@ -339,8 +344,10 @@ done:
|
||||
return found;
|
||||
}
|
||||
|
||||
#if ! defined __INTEL_COMPILER
|
||||
#if defined __APPLE__
|
||||
#if ! defined __INTEL_COMPILER
|
||||
#if defined __APPLE__
|
||||
/** \internal */
|
||||
|
||||
int isinf(double x)
|
||||
{
|
||||
union { unsigned long long u; double f; } ieee754;
|
||||
@ -349,6 +356,7 @@ int isinf(double x)
|
||||
( (unsigned)ieee754.u == 0 );
|
||||
}
|
||||
|
||||
/** \internal */
|
||||
int isnan(double x)
|
||||
{
|
||||
union { unsigned long long u; double f; } ieee754;
|
||||
@ -360,7 +368,7 @@ int isnan(double x)
|
||||
#endif /*APPLE*/
|
||||
#endif /*!_INTEL_COMPILER*/
|
||||
|
||||
|
||||
/** \internal */
|
||||
int
|
||||
NC_split_delim(const char* arg, char delim, NClist* segments)
|
||||
{
|
||||
@ -395,7 +403,7 @@ done:
|
||||
return stat;
|
||||
}
|
||||
|
||||
/* concat the the segments with each segment preceded by '/' */
|
||||
/** \internal concat the the segments with each segment preceded by '/' */
|
||||
int
|
||||
NC_join(NClist* segments, char** pathp)
|
||||
{
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
* @file
|
||||
* @internal The netCDF-4 file functions relating to file creation.
|
||||
*
|
||||
* @author Ed Hartnett
|
||||
* @author Ed Hartnett, Mark Harfouche, Dennis Heimbigner
|
||||
*/
|
||||
|
||||
#include "config.h"
|
||||
@ -14,11 +14,6 @@
|
||||
#include "ncpathmgr.h"
|
||||
#include "hdf5internal.h"
|
||||
|
||||
/* From hdf5file.c. */
|
||||
extern size_t nc4_chunk_cache_size;
|
||||
extern size_t nc4_chunk_cache_nelems;
|
||||
extern float nc4_chunk_cache_preemption;
|
||||
|
||||
/** @internal These flags may not be set for create. */
|
||||
static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_CDF5);
|
||||
|
||||
@ -157,12 +152,22 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
|
||||
/* Only set cache for non-parallel creates. */
|
||||
if (!nc4_info->parallel)
|
||||
{
|
||||
if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
|
||||
nc4_chunk_cache_preemption) < 0)
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
if (H5Pset_cache(fapl_id, 0, gs->chunkcache.nelems, gs->chunkcache.size,
|
||||
gs->chunkcache.preemption) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f",
|
||||
__func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems,
|
||||
nc4_chunk_cache_preemption));
|
||||
__func__, gs->chunkcache.size, gs->chunkcache.nelems,
|
||||
gs->chunkcache.preemption));
|
||||
}
|
||||
|
||||
{
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
if(gs->alignment.defined) {
|
||||
if (H5Pset_alignment(fapl_id, gs->alignment.threshold, gs->alignment.alignment) < 0) {
|
||||
BAIL(NC_EHDFERR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Set HDF5 format compatibility in the FILE ACCESS property list.
|
||||
|
||||
@ -39,12 +39,6 @@ h5catch(void* ignored)
|
||||
}
|
||||
#endif
|
||||
|
||||
/* These are the default chunk cache sizes for HDF5 files created or
|
||||
* opened with netCDF-4. */
|
||||
extern size_t nc4_chunk_cache_size;
|
||||
extern size_t nc4_chunk_cache_nelems;
|
||||
extern float nc4_chunk_cache_preemption;
|
||||
|
||||
#ifdef LOGGING
|
||||
/* This is the severity level of messages which will be logged. Use
|
||||
severity 0 for errors, 1 for important log messages, 2 for less
|
||||
|
||||
@ -59,11 +59,6 @@ static const int nc_type_size_g[NUM_TYPES] = {sizeof(char), sizeof(char), sizeof
|
||||
/** @internal These flags may not be set for open mode. */
|
||||
static const int ILLEGAL_OPEN_FLAGS = (NC_MMAP);
|
||||
|
||||
/* From libsrc4, these are the netcdf-4 cache sizes. */
|
||||
extern size_t nc4_chunk_cache_size;
|
||||
extern size_t nc4_chunk_cache_nelems;
|
||||
extern float nc4_chunk_cache_preemption;
|
||||
|
||||
/* From nc4mem.c */
|
||||
extern int NC4_open_image_file(NC_FILE_INFO_T* h5);
|
||||
|
||||
@ -813,12 +808,22 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
|
||||
/* Only set cache for non-parallel opens. */
|
||||
if (!nc4_info->parallel)
|
||||
{
|
||||
if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
|
||||
nc4_chunk_cache_preemption) < 0)
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
if (H5Pset_cache(fapl_id, 0, gs->chunkcache.nelems, gs->chunkcache.size,
|
||||
gs->chunkcache.preemption) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f",
|
||||
__func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems,
|
||||
nc4_chunk_cache_preemption));
|
||||
__func__, gs->chunkcache.size, gs->chunkcache.nelems,
|
||||
gs->chunkcache.preemption));
|
||||
}
|
||||
|
||||
{
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
if(gs->alignment.defined) {
|
||||
if (H5Pset_alignment(fapl_id, gs->alignment.threshold, gs->alignment.alignment) < 0) {
|
||||
BAIL(NC_EHDFERR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Set HDF5 format compatibility in the FILE ACCESS property list.
|
||||
@ -1453,10 +1458,10 @@ nc4_get_var_meta(NC_VAR_INFO_T *var)
|
||||
BAIL(NC_EVARMETA);
|
||||
|
||||
/* Learn about current chunk cache settings. */
|
||||
if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems),
|
||||
&(var->chunk_cache_size), &rdcc_w0)) < 0)
|
||||
if ((H5Pget_chunk_cache(access_pid, &(var->chunkcache.nelems),
|
||||
&(var->chunkcache.size), &rdcc_w0)) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
var->chunk_cache_preemption = rdcc_w0;
|
||||
var->chunkcache.preemption = rdcc_w0;
|
||||
|
||||
/* Get the dataset creation properties. */
|
||||
if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0)
|
||||
|
||||
@ -85,9 +85,9 @@ nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
|
||||
|
||||
if ((access_pid = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
|
||||
return NC_EHDFERR;
|
||||
if (H5Pset_chunk_cache(access_pid, var->chunk_cache_nelems,
|
||||
var->chunk_cache_size,
|
||||
var->chunk_cache_preemption) < 0)
|
||||
if (H5Pset_chunk_cache(access_pid, var->chunkcache.nelems,
|
||||
var->chunkcache.size,
|
||||
var->chunkcache.preemption) < 0)
|
||||
return NC_EHDFERR;
|
||||
if (H5Dclose(hdf5_var->hdf_datasetid) < 0)
|
||||
return NC_EHDFERR;
|
||||
@ -2314,9 +2314,9 @@ NC4_HDF5_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
|
||||
assert(var && var->hdr.id == varid);
|
||||
|
||||
/* Set the values. */
|
||||
var->chunk_cache_size = size;
|
||||
var->chunk_cache_nelems = nelems;
|
||||
var->chunk_cache_preemption = preemption;
|
||||
var->chunkcache.size = size;
|
||||
var->chunkcache.nelems = nelems;
|
||||
var->chunkcache.preemption = preemption;
|
||||
|
||||
/* Reopen the dataset to bring new settings into effect. */
|
||||
if ((retval = nc4_reopen_dataset(grp, var)))
|
||||
|
||||
@ -977,9 +977,9 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
|
||||
}
|
||||
|
||||
/* Set per-var chunk cache, for chunked datasets. */
|
||||
if (var->storage == NC_CHUNKED && var->chunk_cache_size)
|
||||
if (H5Pset_chunk_cache(access_plistid, var->chunk_cache_nelems,
|
||||
var->chunk_cache_size, var->chunk_cache_preemption) < 0)
|
||||
if (var->storage == NC_CHUNKED && var->chunkcache.size)
|
||||
if (H5Pset_chunk_cache(access_plistid, var->chunkcache.nelems,
|
||||
var->chunkcache.size, var->chunkcache.preemption) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
|
||||
/* At long last, create the dataset. */
|
||||
@ -1101,12 +1101,12 @@ nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
|
||||
/* If the chunk cache is too small, and the user has not changed
|
||||
* the default value of the chunk cache size, then increase the
|
||||
* size of the cache. */
|
||||
if (var->chunk_cache_size == CHUNK_CACHE_SIZE)
|
||||
if (chunk_size_bytes > var->chunk_cache_size)
|
||||
if (var->chunkcache.size == CHUNK_CACHE_SIZE)
|
||||
if (chunk_size_bytes > var->chunkcache.size)
|
||||
{
|
||||
var->chunk_cache_size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
|
||||
if (var->chunk_cache_size > MAX_DEFAULT_CACHE_SIZE)
|
||||
var->chunk_cache_size = MAX_DEFAULT_CACHE_SIZE;
|
||||
var->chunkcache.size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
|
||||
if (var->chunkcache.size > MAX_DEFAULT_CACHE_SIZE)
|
||||
var->chunkcache.size = MAX_DEFAULT_CACHE_SIZE;
|
||||
if ((retval = nc4_reopen_dataset(grp, var)))
|
||||
return retval;
|
||||
}
|
||||
|
||||
@ -18,12 +18,6 @@
|
||||
#include "zincludes.h"
|
||||
#include "zfilter.h"
|
||||
|
||||
/* These are the default chunk cache sizes for ZARR files created or
|
||||
* opened with netCDF-4. */
|
||||
extern size_t ncz_chunk_cache_size;
|
||||
extern size_t ncz_chunk_cache_nelems;
|
||||
extern float ncz_chunk_cache_preemption;
|
||||
|
||||
/* Forward */
|
||||
|
||||
#ifdef LOGGING
|
||||
@ -61,10 +55,10 @@ NCZ_initialize_internal(void)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
char* dimsep = NULL;
|
||||
NCRCglobalstate* ngs = NULL;
|
||||
NCglobalstate* ngs = NULL;
|
||||
|
||||
ncz_initialized = 1;
|
||||
ngs = ncrc_getglobalstate();
|
||||
ngs = NC_getglobalstate();
|
||||
if(ngs != NULL) {
|
||||
/* Defaults */
|
||||
ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR;
|
||||
|
||||
@ -1534,7 +1534,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
|
||||
}
|
||||
/* Capture dimension_separator (must precede chunk cache creation) */
|
||||
{
|
||||
NCRCglobalstate* ngs = ncrc_getglobalstate();
|
||||
NCglobalstate* ngs = NC_getglobalstate();
|
||||
assert(ngs != NULL);
|
||||
zvar->dimension_separator = 0;
|
||||
if((stat = NCJdictget(jvar,"dimension_separator",&jvalue))) goto done;
|
||||
|
||||
@ -381,7 +381,7 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims,
|
||||
zvar->common.file = h5;
|
||||
zvar->scalar = (ndims == 0 ? 1 : 0);
|
||||
|
||||
zvar->dimension_separator = ncrc_getglobalstate()->zarr.dimension_separator;
|
||||
zvar->dimension_separator = NC_getglobalstate()->zarr.dimension_separator;
|
||||
assert(zvar->dimension_separator != 0);
|
||||
|
||||
/* Set these state flags for the var. */
|
||||
@ -456,9 +456,9 @@ var->type_info->rc++;
|
||||
zvar->chunksize = zvar->chunkproduct * var->type_info->size;
|
||||
|
||||
/* Override the cache setting to use NCZarr defaults */
|
||||
var->chunk_cache_size = CHUNK_CACHE_SIZE_NCZARR;
|
||||
var->chunk_cache_nelems = ceildiv(var->chunk_cache_size,zvar->chunksize);
|
||||
var->chunk_cache_preemption = 1; /* not used */
|
||||
var->chunkcache.size = CHUNK_CACHE_SIZE_NCZARR;
|
||||
var->chunkcache.nelems = ceildiv(var->chunkcache.size,zvar->chunksize);
|
||||
var->chunkcache.preemption = 1; /* not used */
|
||||
|
||||
/* Create the cache */
|
||||
if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,zvar->dimension_separator,&zvar->cache)))
|
||||
|
||||
@ -77,9 +77,9 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl
|
||||
assert(zvar != NULL && zvar->cache != NULL);
|
||||
|
||||
/* Set the values. */
|
||||
var->chunk_cache_size = cachesize;
|
||||
var->chunk_cache_nelems = nelems;
|
||||
var->chunk_cache_preemption = preemption;
|
||||
var->chunkcache.size = cachesize;
|
||||
var->chunkcache.nelems = nelems;
|
||||
var->chunkcache.preemption = preemption;
|
||||
|
||||
/* Fix up cache */
|
||||
if((retval = NCZ_adjust_var_cache(var))) goto done;
|
||||
@ -120,8 +120,8 @@ fprintf(stderr,"xxx: adjusting cache for: %s\n",var->hdr.name);
|
||||
/* Reclaim any existing fill_chunk */
|
||||
if((stat = NCZ_reclaim_fill_chunk(zcache))) goto done;
|
||||
/* Reset the parameters */
|
||||
zvar->cache->maxsize = var->chunk_cache_size;
|
||||
zvar->cache->maxentries = var->chunk_cache_nelems;
|
||||
zvar->cache->maxsize = var->chunkcache.size;
|
||||
zvar->cache->maxentries = var->chunkcache.nelems;
|
||||
#ifdef DEBUG
|
||||
fprintf(stderr,"%s.cache.adjust: size=%ld nelems=%ld\n",
|
||||
var->hdr.name,(unsigned long)zvar->cache->maxsize,(unsigned long)zvar->cache->maxentries);
|
||||
|
||||
@ -13,12 +13,6 @@
|
||||
#include "config.h"
|
||||
#include "nc4internal.h"
|
||||
|
||||
/* These are the default chunk cache sizes for HDF5 files created or
|
||||
* opened with netCDF-4. */
|
||||
extern size_t nc4_chunk_cache_size;
|
||||
extern size_t nc4_chunk_cache_nelems;
|
||||
extern float nc4_chunk_cache_preemption;
|
||||
|
||||
/**
|
||||
* Set chunk cache size. Only affects netCDF-4/HDF5 files
|
||||
* opened/created *after* it is called.
|
||||
@ -71,11 +65,12 @@ extern float nc4_chunk_cache_preemption;
|
||||
int
|
||||
nc_set_chunk_cache(size_t size, size_t nelems, float preemption)
|
||||
{
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
if (preemption < 0 || preemption > 1)
|
||||
return NC_EINVAL;
|
||||
nc4_chunk_cache_size = size;
|
||||
nc4_chunk_cache_nelems = nelems;
|
||||
nc4_chunk_cache_preemption = preemption;
|
||||
gs->chunkcache.size = size;
|
||||
gs->chunkcache.nelems = nelems;
|
||||
gs->chunkcache.preemption = preemption;
|
||||
return NC_NOERR;
|
||||
}
|
||||
|
||||
@ -99,14 +94,15 @@ nc_set_chunk_cache(size_t size, size_t nelems, float preemption)
|
||||
int
|
||||
nc_get_chunk_cache(size_t *sizep, size_t *nelemsp, float *preemptionp)
|
||||
{
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
if (sizep)
|
||||
*sizep = nc4_chunk_cache_size;
|
||||
*sizep = gs->chunkcache.size;
|
||||
|
||||
if (nelemsp)
|
||||
*nelemsp = nc4_chunk_cache_nelems;
|
||||
*nelemsp = gs->chunkcache.nelems;
|
||||
|
||||
if (preemptionp)
|
||||
*preemptionp = nc4_chunk_cache_preemption;
|
||||
*preemptionp = gs->chunkcache.preemption;
|
||||
return NC_NOERR;
|
||||
}
|
||||
|
||||
@ -128,11 +124,12 @@ nc_get_chunk_cache(size_t *sizep, size_t *nelemsp, float *preemptionp)
|
||||
int
|
||||
nc_set_chunk_cache_ints(int size, int nelems, int preemption)
|
||||
{
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
if (size <= 0 || nelems <= 0 || preemption < 0 || preemption > 100)
|
||||
return NC_EINVAL;
|
||||
nc4_chunk_cache_size = size;
|
||||
nc4_chunk_cache_nelems = nelems;
|
||||
nc4_chunk_cache_preemption = (float)preemption / 100;
|
||||
gs->chunkcache.size = size;
|
||||
gs->chunkcache.nelems = nelems;
|
||||
gs->chunkcache.preemption = (float)preemption / 100;
|
||||
return NC_NOERR;
|
||||
}
|
||||
|
||||
@ -154,12 +151,13 @@ nc_set_chunk_cache_ints(int size, int nelems, int preemption)
|
||||
int
|
||||
nc_get_chunk_cache_ints(int *sizep, int *nelemsp, int *preemptionp)
|
||||
{
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
if (sizep)
|
||||
*sizep = (int)nc4_chunk_cache_size;
|
||||
*sizep = (int)gs->chunkcache.size;
|
||||
if (nelemsp)
|
||||
*nelemsp = (int)nc4_chunk_cache_nelems;
|
||||
*nelemsp = (int)gs->chunkcache.nelems;
|
||||
if (preemptionp)
|
||||
*preemptionp = (int)(nc4_chunk_cache_preemption * 100);
|
||||
*preemptionp = (int)(gs->chunkcache.preemption * 100);
|
||||
|
||||
return NC_NOERR;
|
||||
}
|
||||
|
||||
@ -24,6 +24,7 @@
|
||||
#include "ncdispatch.h" /* from libdispatch */
|
||||
#include "ncutf8.h"
|
||||
#include <stdarg.h>
|
||||
#include "ncrc.h"
|
||||
|
||||
/** @internal Number of reserved attributes. These attributes are
|
||||
* hidden from the netcdf user, but exist in the implementation
|
||||
@ -53,11 +54,6 @@ static const NC_reservedatt NC_reserved[] = {
|
||||
};
|
||||
#define NRESERVED (sizeof(NC_reserved) / sizeof(NC_reservedatt)) /*|NC_reservedatt|*/
|
||||
|
||||
/* These hold the file caching settings for the library. */
|
||||
size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE; /**< Default chunk cache size. */
|
||||
size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */
|
||||
float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */
|
||||
|
||||
static int NC4_move_in_NCList(NC* nc, int new_id);
|
||||
|
||||
#if NC_HAS_LOGGING
|
||||
@ -737,6 +733,7 @@ int
|
||||
nc4_var_list_add2(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var)
|
||||
{
|
||||
NC_VAR_INFO_T *new_var = NULL;
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
|
||||
/* Allocate storage for new variable. */
|
||||
if (!(new_var = calloc(1, sizeof(NC_VAR_INFO_T))))
|
||||
@ -745,9 +742,9 @@ nc4_var_list_add2(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var)
|
||||
new_var->container = grp;
|
||||
|
||||
/* These are the HDF5-1.8.4 defaults. */
|
||||
new_var->chunk_cache_size = nc4_chunk_cache_size;
|
||||
new_var->chunk_cache_nelems = nc4_chunk_cache_nelems;
|
||||
new_var->chunk_cache_preemption = nc4_chunk_cache_preemption;
|
||||
new_var->chunkcache.size = gs->chunkcache.size;
|
||||
new_var->chunkcache.nelems = gs->chunkcache.nelems;
|
||||
new_var->chunkcache.preemption = gs->chunkcache.preemption;
|
||||
|
||||
/* Now fill in the values in the var info structure. */
|
||||
new_var->hdr.id = ncindexsize(grp->vars);
|
||||
@ -2093,3 +2090,141 @@ NC4_move_in_NCList(NC* nc, int new_id)
|
||||
return stat;
|
||||
}
|
||||
|
||||
/**************************************************/
|
||||
/* NCglobal state management */
|
||||
|
||||
static NCglobalstate* nc_globalstate = NULL;
|
||||
|
||||
static int
|
||||
NC_createglobalstate(void)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
const char* tmp = NULL;
|
||||
|
||||
if(nc_globalstate == NULL) {
|
||||
nc_globalstate = calloc(1,sizeof(NCglobalstate));
|
||||
}
|
||||
/* Initialize struct pointers */
|
||||
nc_globalstate->rcinfo = (struct NCRCinfo*)calloc(1,sizeof(struct NCRCinfo));
|
||||
if(nc_globalstate == NULL) return NC_ENOMEM;
|
||||
|
||||
/* Get environment variables */
|
||||
if(getenv(NCRCENVIGNORE) != NULL)
|
||||
nc_globalstate->rcinfo->ignore = 1;
|
||||
tmp = getenv(NCRCENVRC);
|
||||
if(tmp != NULL && strlen(tmp) > 0)
|
||||
nc_globalstate->rcinfo->rcfile = strdup(tmp);
|
||||
/* Initialize chunk cache defaults */
|
||||
nc_globalstate->chunkcache.size = CHUNK_CACHE_SIZE; /**< Default chunk cache size. */
|
||||
nc_globalstate->chunkcache.nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */
|
||||
nc_globalstate->chunkcache.preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */
|
||||
|
||||
return stat;
|
||||
}
|
||||
|
||||
/* Get global state */
|
||||
NCglobalstate*
|
||||
NC_getglobalstate(void)
|
||||
{
|
||||
if(nc_globalstate == NULL)
|
||||
NC_createglobalstate();
|
||||
return nc_globalstate;
|
||||
}
|
||||
|
||||
void
|
||||
NC_freeglobalstate(void)
|
||||
{
|
||||
if(nc_globalstate != NULL) {
|
||||
nullfree(nc_globalstate->tempdir);
|
||||
nullfree(nc_globalstate->home);
|
||||
nullfree(nc_globalstate->cwd);
|
||||
NC_rcclear(nc_globalstate->rcinfo);
|
||||
free(nc_globalstate->rcinfo);
|
||||
free(nc_globalstate);
|
||||
nc_globalstate = NULL;
|
||||
}
|
||||
}
|
||||
/**************************************************/
|
||||
/* Specific property functions */
|
||||
|
||||
/**
|
||||
Provide a function to store global data alignment
|
||||
information.
|
||||
Repeated calls to nc_set_alignment will overwrite any existing values.
|
||||
|
||||
If defined, then for every file created or opened after the call to
|
||||
nc_set_alignment, and for every new variable added to the file, the
|
||||
most recently set threshold and alignment values will be applied
|
||||
to that variable.
|
||||
|
||||
The nc_set_alignment function causes new data written to a
|
||||
netCDF-4 file to be aligned on disk to a specified block
|
||||
size. To be effective, alignment should be the system disk block
|
||||
size, or a multiple of it. This setting is effective with MPI
|
||||
I/O and other parallel systems.
|
||||
|
||||
This is a trade-off of write speed versus file size. Alignment
|
||||
leaves holes between file objects. The default of no alignment
|
||||
writes file objects contiguously, without holes. Alignment has
|
||||
no impact on file readability.
|
||||
|
||||
Alignment settings apply only indirectly, through the file open
|
||||
functions. Call nc_set_alignment first, then nc_create or
|
||||
nc_open for one or more files. Current alignment settings are
|
||||
locked in when each file is opened, then forgotten when the same
|
||||
file is closed. For illustration, it is possible to write
|
||||
different files at the same time with different alignments, by
|
||||
interleaving nc_set_alignment and nc_open calls.
|
||||
|
||||
Alignment applies to all newly written low-level file objects at
|
||||
or above the threshold size, including chunks of variables,
|
||||
attributes, and internal infrastructure. Alignment is not locked
|
||||
in to a data variable. It can change between data chunks of the
|
||||
same variable, based on a file's history.
|
||||
|
||||
Refer to H5Pset_alignment in HDF5 documentation for more
|
||||
specific details, interactions, and additional rules.
|
||||
|
||||
@param threshold The minimum size to which alignment is applied.
|
||||
@param alignment The alignment value.
|
||||
|
||||
@return ::NC_NOERR No error.
|
||||
@return ::NC_EINVAL Invalid input.
|
||||
@author Dennis Heimbigner
|
||||
@ingroup datasets
|
||||
*/
|
||||
int
|
||||
nc_set_alignment(int threshold, int alignment)
|
||||
{
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
gs->alignment.threshold = threshold;
|
||||
gs->alignment.alignment = alignment;
|
||||
gs->alignment.defined = 1;
|
||||
return NC_NOERR;
|
||||
}
|
||||
|
||||
/**
|
||||
Provide get function to retrieve global data alignment
|
||||
information.
|
||||
|
||||
The nc_get_alignment function return the last values set by
|
||||
nc_set_alignment. If nc_set_alignment has not been called, then
|
||||
it returns the value 0 for both threshold and alignment.
|
||||
|
||||
@param thresholdp Return the current minimum size to which alignment is applied or zero.
|
||||
@param alignmentp Return the current alignment value or zero.
|
||||
|
||||
@return ::NC_NOERR No error.
|
||||
@return ::NC_EINVAL Invalid input.
|
||||
@author Dennis Heimbigner
|
||||
@ingroup datasets
|
||||
*/
|
||||
|
||||
int
|
||||
nc_get_alignment(int* thresholdp, int* alignmentp)
|
||||
{
|
||||
NCglobalstate* gs = NC_getglobalstate();
|
||||
if(thresholdp) *thresholdp = gs->alignment.threshold;
|
||||
if(alignmentp) *alignmentp = gs->alignment.alignment;
|
||||
return NC_NOERR;
|
||||
}
|
||||
|
||||
@ -97,11 +97,11 @@ NC4_get_var_chunk_cache(int ncid, int varid, size_t *sizep,
|
||||
|
||||
/* Give the user what they want. */
|
||||
if (sizep)
|
||||
*sizep = var->chunk_cache_size;
|
||||
*sizep = var->chunkcache.size;
|
||||
if (nelemsp)
|
||||
*nelemsp = var->chunk_cache_nelems;
|
||||
*nelemsp = var->chunkcache.nelems;
|
||||
if (preemptionp)
|
||||
*preemptionp = var->chunk_cache_preemption;
|
||||
*preemptionp = var->chunkcache.preemption;
|
||||
|
||||
return NC_NOERR;
|
||||
}
|
||||
|
||||
@ -24,6 +24,10 @@ SET(NC4_TESTS tst_dims tst_dims2 tst_dims3 tst_files tst_files4
|
||||
tst_hdf5_file_compat tst_fill_attr_vanish tst_rehash tst_types tst_bug324
|
||||
tst_atts3 tst_put_vars tst_elatefill tst_udf tst_bug1442 tst_quantize)
|
||||
|
||||
IF(HAS_PAR_FILTERS)
|
||||
SET(NC4_tests $NC4_TESTS tst_alignment)
|
||||
ENDIF()
|
||||
|
||||
# Note, renamegroup needs to be compiled before run_grp_rename
|
||||
|
||||
IF(BUILD_UTILITIES)
|
||||
|
||||
@ -37,6 +37,10 @@ tst_rehash tst_filterparser tst_bug324 tst_types tst_atts3 \
|
||||
tst_put_vars tst_elatefill tst_udf tst_put_vars_two_unlim_dim \
|
||||
tst_bug1442 tst_quantize
|
||||
|
||||
if HAS_PAR_FILTERS
|
||||
NC4_TESTS += tst_alignment
|
||||
endif
|
||||
|
||||
# Temporary I hoped, but hoped in vain.
|
||||
if !ISCYGWIN
|
||||
NC4_TESTS += tst_h_strbug tst_h_refs
|
||||
|
||||
96
nc_test4/tst_alignment.c
Normal file
96
nc_test4/tst_alignment.c
Normal file
@ -0,0 +1,96 @@
|
||||
/* This is part of the netCDF package.
|
||||
Copyright 2018 University Corporation for Atmospheric Research/Unidata
|
||||
See COPYRIGHT file for conditions of use.
|
||||
|
||||
Test HDF5 alignment
|
||||
Dennis Heimbigner
|
||||
*/
|
||||
|
||||
#include <nc_tests.h>
|
||||
#include "err_macros.h"
|
||||
|
||||
#include <hdf5.h>
|
||||
#include <H5DSpublic.h>
|
||||
|
||||
#undef DEBUG
|
||||
|
||||
#define THRESHOLD 512
|
||||
#define ALIGNMENT 4096
|
||||
#define CHUNKSIZE 513
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
int i,ncid, varid, dimids[1];
|
||||
size_t chunks[1];
|
||||
unsigned char data[CHUNKSIZE];
|
||||
hid_t fileid, grpid, datasetid;
|
||||
hid_t dxpl_id = H5P_DEFAULT; /*data transfer property list */
|
||||
unsigned int filter_mask = 0;
|
||||
hsize_t hoffset[1];
|
||||
haddr_t addr;
|
||||
hsize_t size ;
|
||||
hid_t fspace;
|
||||
|
||||
H5Eset_auto2(H5E_DEFAULT,(H5E_auto2_t)H5Eprint1,stderr);
|
||||
|
||||
printf("\n*** Testing HDF5 alignment.\n");
|
||||
|
||||
printf("chunksize=%d threshold=%d alignment=%d\n",CHUNKSIZE,THRESHOLD,ALIGNMENT);
|
||||
|
||||
if(nc_set_alignment(THRESHOLD,ALIGNMENT)) ERR;
|
||||
if (nc_create("tst_alignment.nc", NC_NETCDF4, &ncid)) ERR;
|
||||
if (nc_def_dim(ncid, "d0", CHUNKSIZE, &dimids[0])) ERR;
|
||||
if (nc_def_var(ncid, "var", NC_UBYTE, 1, dimids, &varid)) ERR;
|
||||
chunks[0] = CHUNKSIZE;
|
||||
if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunks)) ERR;
|
||||
if (nc_enddef(ncid)) ERR;
|
||||
|
||||
for(i=0;i<CHUNKSIZE;i++) data[i] = (i)%2;
|
||||
if(nc_put_var(ncid,varid,data)) ERR;
|
||||
|
||||
if (nc_close(ncid)) ERR;
|
||||
|
||||
/* Use HDF5 API to verify */
|
||||
|
||||
if ((fileid = H5Fopen("tst_alignment.nc", H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) ERR;
|
||||
if ((grpid = H5Gopen1(fileid, "/")) < 0) ERR;
|
||||
if ((datasetid = H5Dopen1(grpid, "var")) < 0) ERR;
|
||||
|
||||
/* Test the offset */
|
||||
if((fspace = H5Dget_space(datasetid)) < 0) ERR;
|
||||
if(H5Dget_chunk_info(datasetid, fspace, 0, hoffset, &filter_mask, &addr, &size) < 0) ERR;
|
||||
#ifdef DEBUG
|
||||
fprintf(stderr,"H5Dget_chunk_info: offset=%lu addr=%lu size=%lu\n",(unsigned long)hoffset[0], (unsigned long)addr, (unsigned long)size);
|
||||
fprintf(stderr,"\t%% alignment: offset=%lu, addr=%lu\n",
|
||||
(((unsigned long)hoffset[0]) % ALIGNMENT),
|
||||
(((unsigned long)addr) % ALIGNMENT)
|
||||
);
|
||||
#endif
|
||||
|
||||
printf("H5Dget_chunk_info: addr=%lu (addr %% alignment)=%lu\n", (unsigned long)addr, (((unsigned long)hoffset[0]) % ALIGNMENT));
|
||||
if((addr % ALIGNMENT) != 0) ERR;
|
||||
|
||||
/* Test chunk content */
|
||||
memset(data,0,sizeof(data));
|
||||
hoffset[0] = 0;
|
||||
|
||||
if(H5Dread_chunk(datasetid, dxpl_id, hoffset, &filter_mask, data) < 0) ERR;
|
||||
|
||||
#ifdef DEBUG
|
||||
fprintf(stderr,"H5Dread_chunk: offset=%lu offset %% alignment=%lu\n",(unsigned long)hoffset[0], (((unsigned long)hoffset) % ALIGNMENT));
|
||||
#endif
|
||||
for(i=0;i<CHUNKSIZE;i++) {
|
||||
if(data[i] != (i)%2) {
|
||||
fprintf(stderr,"data[%d] mismatch\n",i);
|
||||
ERR;
|
||||
}
|
||||
}
|
||||
|
||||
if (H5Dclose(datasetid) < 0) ERR;
|
||||
if (H5Gclose(grpid) < 0) ERR;
|
||||
if (H5Fclose(fileid) < 0) ERR;
|
||||
|
||||
SUMMARIZE_ERR;
|
||||
FINAL_RESULTS;
|
||||
}
|
||||
@ -8,6 +8,8 @@
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
export SETX=1
|
||||
|
||||
|
||||
set -e
|
||||
echo ""
|
||||
|
||||
@ -3,20 +3,21 @@
|
||||
#include <string.h>
|
||||
#include "netcdf.h"
|
||||
#include "ncrc.h"
|
||||
#include "nc4internal.h"
|
||||
|
||||
int
|
||||
main(int argc, char** argv)
|
||||
{
|
||||
size_t i,nentries = 0;
|
||||
NCRCglobalstate* ngs = ncrc_getglobalstate();
|
||||
NCglobalstate* ngs = NC_getglobalstate();
|
||||
NCRCinfo* info = NULL;
|
||||
NCRCentry* entry = NULL;
|
||||
|
||||
/* Cause the .rc files to be read and merged */
|
||||
nc_initialize();
|
||||
|
||||
if((ngs = ncrc_getglobalstate())==NULL) abort();
|
||||
info = &ngs->rcinfo;
|
||||
if((ngs = NC_getglobalstate())==NULL) abort();
|
||||
info = ngs->rcinfo;
|
||||
|
||||
if(info->ignore) {
|
||||
fprintf(stderr,".rc ignored\n");
|
||||
|
||||
@ -167,7 +167,7 @@ main(int argc, char *argv[])
|
||||
/* variable shapes */
|
||||
int var_dims[RANK];
|
||||
|
||||
NCCHECK(getoptions(&argc,&argv,&bmoptions));
|
||||
NCCHECK(bm_getoptions(&argc,&argv,&bmoptions));
|
||||
NCCHECK(nc4_buildpath(&bmoptions,&path));
|
||||
|
||||
if(bmoptions.debug) {
|
||||
|
||||
@ -89,6 +89,10 @@ EXTERNL void bm_reportmetaoptions(struct BMMeta* o);
|
||||
EXTERNL const char* bm_printvector(int rank, const size_t* vec);
|
||||
EXTERNL const char* bm_printvectorp(int rank, const ptrdiff_t* vec);
|
||||
EXTERNL const char* bm_varasprint(int rank, const size_t* start, const size_t* edges, const ptrdiff_t* stride);
|
||||
EXTERNL int nc4_buildpath(struct BMOptions* o, char** pathp);
|
||||
EXTERNL void reportoptions(struct BMOptions* o);
|
||||
EXTERNL void reportmetaoptions(struct BMMeta* o);
|
||||
EXTERNL void clearoptions(struct BMOptions*);
|
||||
|
||||
#include "ut_test.h"
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#define DEBUG
|
||||
#undef DEBUG
|
||||
|
||||
static char hex[16] = "0123456789abcdef";
|
||||
|
||||
|
||||
@ -23,7 +23,7 @@
|
||||
#include "nclog.h"
|
||||
#include "ncuri.h"
|
||||
|
||||
#define DEBUG
|
||||
#undef DEBUG
|
||||
|
||||
#define DATANAME "data"
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@
|
||||
#include "zincludes.h"
|
||||
#include "ncpathmgr.h"
|
||||
|
||||
#define DEBUG
|
||||
#undef DEBUG
|
||||
|
||||
#define AWSHOST ".amazonaws.com"
|
||||
|
||||
|
||||
@ -25,7 +25,7 @@
|
||||
|
||||
#include <errno.h>
|
||||
|
||||
#include "ncrc.h"
|
||||
#include "nc4internal.h"
|
||||
#include "ocinternal.h"
|
||||
#include "ocdebug.h"
|
||||
#include "occurlfunctions.h"
|
||||
@ -321,7 +321,7 @@ createtempfile(OCstate* state, OCtree* tree)
|
||||
char* path = NULL;
|
||||
char* tmppath = NULL;
|
||||
int len;
|
||||
NCRCglobalstate* globalstate = ncrc_getglobalstate();
|
||||
NCglobalstate* globalstate = NC_getglobalstate();
|
||||
|
||||
len =
|
||||
strlen(globalstate->tempdir)
|
||||
@ -526,7 +526,7 @@ static OCerror
|
||||
ocset_curlproperties(OCstate* state)
|
||||
{
|
||||
OCerror stat = OC_NOERR;
|
||||
NCRCglobalstate* globalstate = ncrc_getglobalstate();
|
||||
NCglobalstate* globalstate = NC_getglobalstate();
|
||||
|
||||
if(state->auth->curlflags.useragent == NULL) {
|
||||
size_t len = strlen(DFALTUSERAGENT) + strlen(VERSION) + 1;
|
||||
|
||||
@ -15,7 +15,7 @@ Test the handling of aws profiles and regions.
|
||||
#include "ncrc.h"
|
||||
#include "ncpathmgr.h"
|
||||
|
||||
#define DEBUG
|
||||
#undef DEBUG
|
||||
|
||||
typedef struct ProfileTest {
|
||||
const char* profile;
|
||||
|
||||
@ -13,7 +13,7 @@ Test the NCpathcvt
|
||||
#include "netcdf.h"
|
||||
#include "ncpathmgr.h"
|
||||
|
||||
#define DEBUG
|
||||
#undef DEBUG
|
||||
|
||||
#define NKINDS 4
|
||||
static const int kinds[NKINDS] = {NCPD_NIX,NCPD_MSYS,NCPD_CYGWIN,NCPD_WIN};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user