mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-19 17:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add support for setting HDF5 alignment property when creating a file

Browse Source 
re: https://github.com/Unidata/netcdf-c/issues/2177
re: https://github.com/Unidata/netcdf-c/pull/2178

Provide get/set functions to store global data alignment
information and apply it when a file is created.

The api is as follows:
````
int nc_set_alignment(int threshold, int alignment);
int nc_get_alignment(int* thresholdp, int* alignmentp);
````

If defined, then for every file created opened after the call to
nc_set_alignment, for every new variable added to the file, the
most recently set threshold and alignment values will be applied
to that variable.

The nc_get_alignment function return the last values set by
nc_set_alignment.  If nc_set_alignment has not been called, then
it returns the value 0 for both threshold and alignment.

The alignment parameters are stored in the NCglobalstate object
(see below) for use as needed. Repeated calls to nc_set_alignment
will overwrite any existing values in NCglobalstate.

The alignment parameters are applied in libhdf5/hdf5create.c
and libhdf5/hdf5open.c

The set/get alignment functions are defined in libsrc4/nc4internal.c.

A test program was added as nc_test4/tst_alignment.c.

## Misc. Changes Unrelated to Alignment

* The NCRCglobalstate type was renamed to NCglobalstate to
  indicate that it represented more general global state than
  just .rc data.  It was also moved to nc4internal.h.  This led
  to a large number of small changes: mostly renaming. The
  global state management functions were moved to nc4internal.c.

* The global chunk cache variables have been moved into
  NCglobalstate.  As warranted, other global state will be moved
  as well.

* Some misc. problems with the nczarr performance tests were corrected.
This commit is contained in:
Dennis Heimbigner 2022-01-29 15:27:52 -07:00
parent 8b328d1207
commit f3e711e2b8
46 changed files with 409 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
RELEASE_NOTES.md
View File

@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.2 - TBD
* [Enhancement] Add ability to set dataset alignment for netcdf-4/HDF5 files. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????).
* [Enhancement] Add complete bitgroom support to NCZarr. See [Github #2197](https://github.com/Unidata/netcdf-c/pull/2197).
* [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179(https://github.com/Unidata/netcdf-c/pull/2179).
* [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183).

2
acinclude.m4
View File

@ -384,4 +384,6 @@ MOSTLYCLEANFILES += $(valgrind_log_files)
AC_SUBST([VALGRIND_CHECK_RULES])
m4_ifdef([_AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE([VALGRIND_CHECK_RULES])])
])
AC_DEFUN([_AC_FINALIZE],[])

2
dap4_test/test_data.c
View File

@ -12,7 +12,7 @@ Test the netcdf-4 data building process.
#include <stdio.h>
#include "netcdf.h"
#define DEBUG
#undef DEBUG
static void
fail(int code)

1
docs/Doxyfile.in
View File

@ -781,7 +781,6 @@ INPUT = \
@abs_top_srcdir@/libdispatch/derror.c \
@abs_top_srcdir@/libdispatch/dv2i.c \
@abs_top_srcdir@/libdispatch/dcopy.c \
@abs_top_srcdir@/libdispatch/daux.c \
@abs_top_srcdir@/libsrc4/nc4var.c \
@abs_top_srcdir@/libhdf5/nc4hdf.c \
@abs_top_srcdir@/libsrc4/nc4internal.c \

31
include/nc4internal.h
View File

@ -118,6 +118,7 @@ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t;
/* Forward declarations. */
struct NC_GRP_INFO;
struct NC_TYPE_INFO;
struct NCRCinfo;
/**
* This struct provides indexed Access to Meta-data objects. See the
@ -205,9 +206,11 @@ typedef struct NC_VAR_INFO
int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */
nc_bool_t shuffle; /**< True if var has shuffle filter applied. */
nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */
size_t chunk_cache_size; /**< Size in bytes of the var chunk cache. */
size_t chunk_cache_nelems; /**< Number of slots in var chunk cache. */
float chunk_cache_preemption; /**< Chunk cache preemtion policy. */
struct ChunkCache {
size_t size; /**< Size in bytes of the var chunk cache. */
size_t nelems; /**< Number of slots in var chunk cache. */
float preemption; /**< Chunk cache preemtion policy. */
} chunkcache;
int quantize_mode; /**< Quantize mode. NC_NOQUANTIZE is 0, and means no quantization. */
int nsd; /**< Number of significant digits if quantization is used, 0 if not. */
void *format_var_info; /**< Pointer to any binary format info. */
@ -327,6 +330,24 @@ typedef struct NC_FILE_INFO
} mem;
} NC_FILE_INFO_T;
/* Collect global state info in one place */
typedef struct NCglobalstate {
int initialized;
char* tempdir; /* track a usable temp dir */
char* home; /* track $HOME */
char* cwd; /* track getcwd */
struct NCRCinfo* rcinfo; /* Currently only one rc file per session */
struct GlobalZarr { /* Zarr specific parameters */
char dimension_separator;
} zarr;
struct Alignment { /* H5Pset_alignment parameters */
int defined; /* 1 => threshold and alignment explicitly set */
int threshold;
int alignment;
} alignment;
struct ChunkCache chunkcache;
} NCglobalstate;
/** Variable Length Datatype struct in memory. Must be identical to
* HDF5 hvl_t. (This is only used for VL sequences, not VL strings,
* which are stored in char *'s) */
@ -460,6 +481,10 @@ extern const char* nc4_atomic_name[NUM_ATOMIC_TYPES];
/* Binary searcher for reserved attributes */
extern const NC_reservedatt* NC_findreserved(const char* name);
/* Global State Management */
extern NCglobalstate* NC_getglobalstate(void);
extern void NC_freeglobalstate(void);
/* Generic reserved Attributes */
#define NC_ATT_REFERENCE_LIST "REFERENCE_LIST"
#define NC_ATT_CLASS "CLASS"

43
include/ncrc.h
View File

@ -33,39 +33,25 @@ typedef struct NCRCentry {
char* value;
} NCRCentry;
/* collect all the relevant info around the rc file */
typedef struct NCRCinfo {
int ignore; /* if 1, then do not use any rc file */
int loaded; /* 1 => already loaded */
NClist* entries; /* the rc file entry store fields*/
char* rcfile; /* specified rcfile; overrides anything else */
char* rchome; /* Overrides $HOME when looking for .rc files */
} NCRCinfo;
/* Collect global state info in one place */
typedef struct NCRCglobalstate {
int initialized;
char* tempdir; /* track a usable temp dir */
char* home; /* track $HOME */
char* cwd; /* track getcwd */
NCRCinfo rcinfo; /* Currently only one rc file per session */
struct GlobalZarr { /* Zarr specific parameters */
char dimension_separator;
} zarr;
struct S3credentials {
NClist* profiles; /* NClist<struct AWSprofile*> */
} s3creds;
} NCRCglobalstate;
struct AWSentry {
char* key;
char* value;
};
struct AWSprofile {
char* name;
NClist* entries; /* NClist<struct AWSentry*> */
};
struct AWSentry {
char* key;
char* value;
};
/* collect all the relevant info around the rc file and AWS */
typedef struct NCRCinfo {
int ignore; /* if 1, then do not use any rc file */
int loaded; /* 1 => already loaded */
NClist* entries; /* the rc file entry store fields*/
char* rcfile; /* specified rcfile; overrides anything else */
char* rchome; /* Overrides $HOME when looking for .rc files */
NClist* s3profiles; /* NClist<struct AWSprofile*> */
} NCRCinfo;
typedef struct NCS3INFO {
char* host; /* non-null if other*/
@ -80,9 +66,7 @@ extern "C" {
#endif
/* From drc.c */
EXTERNL int ncrc_createglobalstate(void);
EXTERNL void ncrc_initialize(void);
EXTERNL void ncrc_freeglobalstate(void);
EXTERNL int NC_rcfile_insert(const char* key, const char* value, const char* hostport, const char* path);
EXTERNL char* NC_rclookup(const char* key, const char* hostport, const char* path);
EXTERNL char* NC_rclookupx(NCURI* uri, const char* key);
@ -94,7 +78,6 @@ EXTERNL size_t NC_rcfile_length(NCRCinfo*);
EXTERNL NCRCentry* NC_rcfile_ith(NCRCinfo*,size_t);
/* For internal use */
EXTERNL NCRCglobalstate* ncrc_getglobalstate(void);
EXTERNL void NC_rcclear(NCRCinfo* info);
EXTERNL void NC_rcclear(NCRCinfo* info);

8
include/netcdf.h
View File

@ -560,6 +560,14 @@ nc_def_user_format(int mode_flag, NC_Dispatch *dispatch_table, char *magic_numbe
EXTERNL int
nc_inq_user_format(int mode_flag, NC_Dispatch **dispatch_table, char *magic_number);
/* Set the global alignment property */
EXTERNL int
nc_set_alignment(int threshold, int alignment);
/* Get the global alignment property */
EXTERNL int
nc_get_alignment(int* thresholdp, int* alignmentp);
EXTERNL int
nc__create(const char *path, int cmode, size_t initialsz,
size_t *chunksizehintp, int *ncidp);

2
libdap2/dceconstraints.c
View File

@ -13,7 +13,7 @@
#include "dapincludes.h"
#include "dceparselex.h"
#define DEBUG
#undef DEBUG
#define LBRACE "{"
#define RBRACE "}"

2
libdap4/d4file.c
View File

@ -367,7 +367,7 @@ set_curl_properties(NCD4INFO* d4info)
char* newpath = NULL;
int len;
errno = 0;
NCRCglobalstate* globalstate = ncrc_getglobalstate();
NCglobalstate* globalstate = NC_getglobalstate();
/* Create the unique cookie file name */
len =

8
libdispatch/ddispatch.c
View File

@ -12,6 +12,7 @@ See LICENSE.txt for license information.
#include "ncoffsets.h"
#include "ncpathmgr.h"
#include "ncxml.h"
#include "nc4internal.h"
/* Required for getcwd, other functions. */
#ifdef HAVE_UNISTD_H
@ -48,7 +49,7 @@ NCDISPATCH_initialize(void)
{
int status = NC_NOERR;
int i;
NCRCglobalstate* globalstate = NULL;
NCglobalstate* globalstate = NULL;
for(i=0;i<NC_MAX_VAR_DIMS;i++) {
NC_coord_zero[i] = 0;
@ -56,8 +57,7 @@ NCDISPATCH_initialize(void)
NC_stride_one[i] = 1;
}
status = ncrc_createglobalstate(); /* will allocate and clear */
globalstate = ncrc_getglobalstate(); /* will allocate and clear */
globalstate = NC_getglobalstate(); /* will allocate and clear */
/* Capture temp dir*/
{
@ -123,7 +123,7 @@ int
NCDISPATCH_finalize(void)
{
int status = NC_NOERR;
ncrc_freeglobalstate();
NC_freeglobalstate();
#if defined(ENABLE_BYTERANGE) || defined(ENABLE_DAP) || defined(ENABLE_DAP4)
curl_global_cleanup();
#endif

2
libdispatch/dpathmgr.c
View File

@ -42,7 +42,7 @@
#undef DEBUGPATH
static int pathdebug = -1;
#define DEBUG
#undef DEBUG
#ifdef DEBUG
#define REPORT(e,msg) report((e),(msg),__LINE__)

125
libdispatch/drc.c
View File

@ -22,6 +22,7 @@ See COPYRIGHT for license information.
#include "nclog.h"
#include "ncauth.h"
#include "ncpathmgr.h"
#include "nc4internal.h"
#ifndef nulldup
#define nulldup(x) ((x)?strdup(x):(x))
@ -54,11 +55,10 @@ static int rccompile(const char* path);
static struct NCRCentry* rclocate(const char* key, const char* hostport, const char* path);
static int rcsearch(const char* prefix, const char* rcname, char** pathp);
static void rcfreeentries(NClist* rc);
static void clearS3credentials(struct S3credentials* creds);
#ifdef DRCDEBUG
static void storedump(char* msg, NClist* entrys);
#endif
static int aws_load_credentials(NCRCglobalstate*);
static int aws_load_credentials(NCglobalstate*);
static void freeprofile(struct AWSprofile* profile);
static void freeprofilelist(NClist* profiles);
@ -71,28 +71,8 @@ static const char* awsconfigfiles[] = {".aws/credentials",".aws/config",NULL};
/**************************************************/
/* External Entry Points */
static NCRCglobalstate* ncrc_globalstate = NULL;
static int NCRCinitialized = 0;
int
ncrc_createglobalstate(void)
{
int stat = NC_NOERR;
const char* tmp = NULL;
if(ncrc_globalstate == NULL) {
ncrc_globalstate = calloc(1,sizeof(NCRCglobalstate));
}
/* Get environment variables */
if(getenv(NCRCENVIGNORE) != NULL)
ncrc_globalstate->rcinfo.ignore = 1;
tmp = getenv(NCRCENVRC);
if(tmp != NULL && strlen(tmp) > 0)
ncrc_globalstate->rcinfo.rcfile = strdup(tmp);
return stat;
}
/*
Initialize defaults and load:
* .ncrc
@ -111,6 +91,7 @@ void
ncrc_initialize(void)
{
int stat = NC_NOERR;
NCglobalstate* ncg = NULL;
if(NCRCinitialized) return;
NCRCinitialized = 1; /* prevent recursion */
@ -121,7 +102,8 @@ ncrc_initialize(void)
nclog(NCLOGWARN,".rc loading failed");
}
/* Load .aws/config */
if((stat = aws_load_credentials(ncrc_globalstate))) {
ncg = NC_getglobalstate();
if((stat = aws_load_credentials(ncg))) {
nclog(NCLOGWARN,"AWS config file not loaded");
}
#endif
@ -131,35 +113,13 @@ static void
ncrc_setrchome(void)
{
const char* tmp = NULL;
if(ncrc_globalstate->rcinfo.rchome) return;
assert(ncrc_globalstate && ncrc_globalstate->home);
NCglobalstate* ncg = NC_getglobalstate();
assert(ncg && ncg->home);
if(ncg->rcinfo->rchome) return;
tmp = getenv(NCRCENVHOME);
if(tmp == NULL || strlen(tmp) == 0)
tmp = ncrc_globalstate->home;
ncrc_globalstate->rcinfo.rchome = strdup(tmp);
}
/* Get global state */
NCRCglobalstate*
ncrc_getglobalstate(void)
{
if(ncrc_globalstate == NULL)
ncrc_createglobalstate();
return ncrc_globalstate;
}
void
ncrc_freeglobalstate(void)
{
if(ncrc_globalstate != NULL) {
nullfree(ncrc_globalstate->tempdir);
nullfree(ncrc_globalstate->home);
nullfree(ncrc_globalstate->cwd);
NC_rcclear(&ncrc_globalstate->rcinfo);
clearS3credentials(&ncrc_globalstate->s3creds);
free(ncrc_globalstate);
ncrc_globalstate = NULL;
}
tmp = ncg->home;
ncg->rcinfo->rchome = strdup(tmp);
}
void
@ -169,9 +129,10 @@ NC_rcclear(NCRCinfo* info)
nullfree(info->rcfile);
nullfree(info->rchome);
rcfreeentries(info->entries);
freeprofilelist(info->s3profiles);
}
void
static void
rcfreeentries(NClist* rc)
{
int i;
@ -191,18 +152,17 @@ NC_rcload(void)
{
int i,ret = NC_NOERR;
char* path = NULL;
NCRCglobalstate* globalstate = NULL;
NCglobalstate* globalstate = NULL;
NClist* rcfileorder = nclistnew();
if(!NCRCinitialized) ncrc_initialize();
globalstate = ncrc_getglobalstate();
globalstate = NC_getglobalstate();
if(globalstate->rcinfo.ignore) {
if(globalstate->rcinfo->ignore) {
nclog(NCLOGDBG,".rc file loading suppressed");
goto done;
}
if(globalstate->rcinfo.loaded) goto done;
if(globalstate->rcinfo->loaded) goto done;
/* locate the configuration files in order of use:
1. Specified by NCRCENV_RC environment variable.
@ -215,8 +175,8 @@ NC_rcload(void)
6. $CWD/.docsrc
Entry in later files override any of the earlier files
*/
if(globalstate->rcinfo.rcfile != NULL) { /* always use this */
nclistpush(rcfileorder,strdup(globalstate->rcinfo.rcfile));
if(globalstate->rcinfo->rcfile != NULL) { /* always use this */
nclistpush(rcfileorder,strdup(globalstate->rcinfo->rcfile));
} else {
const char** rcname;
const char* dirnames[3];
@ -225,8 +185,7 @@ NC_rcload(void)
/* Make sure rcinfo.rchome is defined */
ncrc_setrchome();
dirnames[0] = globalstate->rcinfo.rchome;
dirnames[0] = globalstate->rcinfo->rchome;
dirnames[1] = globalstate->cwd;
dirnames[2] = NULL;
@ -249,7 +208,7 @@ NC_rcload(void)
}
done:
globalstate->rcinfo.loaded = 1; /* even if not exists */
globalstate->rcinfo->loaded = 1; /* even if not exists */
nclistfreeall(rcfileorder);
return (ret);
}
@ -301,7 +260,7 @@ NC_set_rcfile(const char* rcfile)
{
int stat = NC_NOERR;
FILE* f = NULL;
NCRCglobalstate* globalstate = ncrc_getglobalstate();
NCglobalstate* globalstate = NC_getglobalstate();
if(rcfile != NULL && strlen(rcfile) == 0)
rcfile = NULL;
@ -311,8 +270,8 @@ NC_set_rcfile(const char* rcfile)
goto done;
}
fclose(f);
nullfree(globalstate->rcinfo.rcfile);
globalstate->rcinfo.rcfile = strdup(rcfile);
nullfree(globalstate->rcinfo->rcfile);
globalstate->rcinfo->rcfile = strdup(rcfile);
/* Clear globalstate->rcinfo */
NC_rcclear(&globalstate->rcinfo);
/* (re) load the rcfile and esp the entriestore*/
@ -414,7 +373,7 @@ rccompile(const char* path)
NCbytes* tmp = ncbytesnew();
NCURI* uri = NULL;
char* nextline = NULL;
NCRCglobalstate* globalstate = ncrc_getglobalstate();
NCglobalstate* globalstate = NC_getglobalstate();
char* bucket = NULL;
if((ret=NC_readfile(path,tmp))) {
@ -424,10 +383,10 @@ rccompile(const char* path)
contents = ncbytesextract(tmp);
if(contents == NULL) contents = strdup("");
/* Either reuse or create new */
rc = globalstate->rcinfo.entries;
rc = globalstate->rcinfo->entries;
if(rc == NULL) {
rc = nclistnew();
globalstate->rcinfo.entries = rc;
globalstate->rcinfo->entries = rc;
}
nextline = contents;
for(;;) {
@ -529,11 +488,11 @@ static struct NCRCentry*
rclocate(const char* key, const char* hostport, const char* path)
{
int i,found;
NCRCglobalstate* globalstate = ncrc_getglobalstate();
NClist* rc = globalstate->rcinfo.entries;
NCglobalstate* globalstate = NC_getglobalstate();
NClist* rc = globalstate->rcinfo->entries;
NCRCentry* entry = NULL;
if(globalstate->rcinfo.ignore)
if(globalstate->rcinfo->ignore)
return NULL;
if(key == NULL || rc == NULL) return NULL;
@ -610,12 +569,12 @@ NC_rcfile_insert(const char* key, const char* value, const char* hostport, const
int ret = NC_NOERR;
/* See if this key already defined */
struct NCRCentry* entry = NULL;
NCRCglobalstate* globalstate = NULL;
NCglobalstate* globalstate = NULL;
NClist* rc = NULL;
if(!NCRCinitialized) ncrc_initialize();
globalstate = ncrc_getglobalstate();
rc = globalstate->rcinfo.entries;
globalstate = NC_getglobalstate();
rc = globalstate->rcinfo->entries;
if(rc == NULL) {
rc = nclistnew();
@ -744,13 +703,6 @@ NC_getdefaults3region(NCURI* uri, const char** regionp)
return stat;
}
static void
clearS3credentials(struct S3credentials* creds)
{
if(creds)
freeprofilelist(creds->profiles);
}
/**
The .aws/config and .aws/credentials files
are in INI format (https://en.wikipedia.org/wiki/INI_file).
@ -1038,10 +990,9 @@ freeprofilelist(NClist* profiles)
/* Find, load, and parse the aws credentials file */
static int
aws_load_credentials(NCRCglobalstate* gstate)
aws_load_credentials(NCglobalstate* gstate)
{
int stat = NC_NOERR;
struct S3credentials* creds = &gstate->s3creds;
NClist* profiles = nclistnew();
const char** awscfg = awsconfigfiles;
const char* aws_root = getenv(NC_TEST_AWS_DIR);
@ -1074,7 +1025,7 @@ aws_load_credentials(NCRCglobalstate* gstate)
nclistpush(profiles,noprof); noprof = NULL;
}
creds->profiles = profiles; profiles = NULL;
gstate->rcinfo->s3profiles = profiles; profiles = NULL;
#ifdef AWSDEBUG
{int i,j;
@ -1102,10 +1053,10 @@ NC_authgets3profile(const char* profilename, struct AWSprofile** profilep)
{
int stat = NC_NOERR;
int i = -1;
NCRCglobalstate* gstate = ncrc_getglobalstate();
for(i=0;i<nclistlength(gstate->s3creds.profiles);i++) {
struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->s3creds.profiles,i);
NCglobalstate* gstate = NC_getglobalstate();
for(i=0;i<nclistlength(gstate->rcinfo->s3profiles);i++) {
struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i);
if(strcmp(profilename,profile->name)==0)
{if(profilep) {*profilep = profile; goto done;}}
}

32
libdispatch/dutil.c
View File

@ -33,7 +33,7 @@
#define nulldup(x) ((x)?strdup(x):(x))
#endif
/**************************************************/
/**
/** \internal
* Provide a hidden interface to allow utilities
* to check if a given path name is really an ncdap4 url.
* If no, return null, else return basename of the url
@ -66,7 +66,7 @@ NC__testurl(const char* path, char** basenamep)
return ok;
}
/* Return 1 if this machine is little endian */
/** \internal Return 1 if this machine is little endian */
int
NC_isLittleEndian(void)
{
@ -78,6 +78,7 @@ NC_isLittleEndian(void)
return (u.bytes[0] == 1 ? 1 : 0);
}
/** \internal */
char*
NC_backslashEscape(const char* s)
{
@ -105,6 +106,7 @@ NC_backslashEscape(const char* s)
return escaped;
}
/** \internal */
char*
NC_backslashUnescape(const char* esc)
{
@ -129,6 +131,7 @@ NC_backslashUnescape(const char* esc)
return s;
}
/** \internal */
char*
NC_entityescape(const char* s)
{
@ -163,13 +166,13 @@ NC_entityescape(const char* s)
return escaped;
}
char*
/*
/** \internal
Depending on the platform, the shell will sometimes
pass an escaped octotherpe character without removing
the backslash. So this function is appropriate to be called
on possible url paths to unescape such cases. See e.g. ncgen.
*/
char*
NC_shellUnescape(const char* esc)
{
size_t len;
@ -194,7 +197,7 @@ NC_shellUnescape(const char* esc)
return s;
}
/**
/** \internal
Wrap mktmp and return the generated path,
or null if failed.
Base is the base file path. XXXXX is appended
@ -224,6 +227,7 @@ done:
return tmp;
}
/** \internal */
int
NC_readfile(const char* filename, NCbytes* content)
{
@ -246,6 +250,7 @@ done:
return ret;
}
/** \internal */
int
NC_writefile(const char* filename, size_t size, void* content)
{
@ -271,7 +276,7 @@ done:
return ret;
}
/*
/** \internal
Parse a path as a url and extract the modelist.
If the path is not a URL, then return a NULL list.
If a URL, but modelist is empty or does not exist,
@ -297,7 +302,7 @@ done:
return stat;
}
/*
/** \internal
Check "mode=" list for a path and return 1 if present, 0 otherwise.
*/
int
@ -313,7 +318,7 @@ NC_testpathmode(const char* path, const char* tag)
return found;
}
/*
/** \internal
Check "mode=" list for a url and return 1 if present, 0 otherwise.
*/
int
@ -339,8 +344,10 @@ done:
return found;
}
#if ! defined __INTEL_COMPILER
#if defined __APPLE__
#if ! defined __INTEL_COMPILER
#if defined __APPLE__
/** \internal */
int isinf(double x)
{
union { unsigned long long u; double f; } ieee754;
@ -349,6 +356,7 @@ int isinf(double x)
( (unsigned)ieee754.u == 0 );
}
/** \internal */
int isnan(double x)
{
union { unsigned long long u; double f; } ieee754;
@ -360,7 +368,7 @@ int isnan(double x)
#endif /*APPLE*/
#endif /*!_INTEL_COMPILER*/
/** \internal */
int
NC_split_delim(const char* arg, char delim, NClist* segments)
{
@ -395,7 +403,7 @@ done:
return stat;
}
/* concat the the segments with each segment preceded by '/' */
/** \internal concat the the segments with each segment preceded by '/' */
int
NC_join(NClist* segments, char** pathp)
{

7
libdispatch/ncs3sdk.cpp
View File

@ -73,9 +73,8 @@ NC_s3sdkinitialize(void)
ncs3_finalized = 0;
NCTRACE(11,NULL);
Aws::InitAPI(ncs3options);
NCUNTRACE(NC_NOERR);
}
return 1;
return NCUNTRACE(NC_NOERR);
}
EXTERNL int
@ -86,9 +85,9 @@ NC_s3sdkfinalize(void)
ncs3_finalized = 1;
NCTRACE(11,NULL);
Aws::ShutdownAPI(ncs3options);
NCUNTRACE(NC_NOERR);
}
return 1;
return NCUNTRACE(NC_NOERR);
}
static char*

25
libhdf5/hdf5create.c
View File

@ -5,7 +5,7 @@
* @file
* @internal The netCDF-4 file functions relating to file creation.
*
* @author Ed Hartnett
* @author Ed Hartnett, Mark Harfouche, Dennis Heimbigner
*/
#include "config.h"
@ -14,11 +14,6 @@
#include "ncpathmgr.h"
#include "hdf5internal.h"
/* From hdf5file.c. */
extern size_t nc4_chunk_cache_size;
extern size_t nc4_chunk_cache_nelems;
extern float nc4_chunk_cache_preemption;
/** @internal These flags may not be set for create. */
static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_CDF5);
@ -157,12 +152,22 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
/* Only set cache for non-parallel creates. */
if (!nc4_info->parallel)
{
if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
nc4_chunk_cache_preemption) < 0)
NCglobalstate* gs = NC_getglobalstate();
if (H5Pset_cache(fapl_id, 0, gs->chunkcache.nelems, gs->chunkcache.size,
gs->chunkcache.preemption) < 0)
BAIL(NC_EHDFERR);
LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f",
__func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems,
nc4_chunk_cache_preemption));
__func__, gs->chunkcache.size, gs->chunkcache.nelems,
gs->chunkcache.preemption));
}
{
NCglobalstate* gs = NC_getglobalstate();
if(gs->alignment.defined) {
if (H5Pset_alignment(fapl_id, gs->alignment.threshold, gs->alignment.alignment) < 0) {
BAIL(NC_EHDFERR);
}
}
}
/* Set HDF5 format compatibility in the FILE ACCESS property list.

6
libhdf5/hdf5internal.c
View File

@ -39,12 +39,6 @@ h5catch(void* ignored)
}
#endif
/* These are the default chunk cache sizes for HDF5 files created or
* opened with netCDF-4. */
extern size_t nc4_chunk_cache_size;
extern size_t nc4_chunk_cache_nelems;
extern float nc4_chunk_cache_preemption;
#ifdef LOGGING
/* This is the severity level of messages which will be logged. Use
severity 0 for errors, 1 for important log messages, 2 for less

29
libhdf5/hdf5open.c
View File

@ -59,11 +59,6 @@ static const int nc_type_size_g[NUM_TYPES] = {sizeof(char), sizeof(char), sizeof
/** @internal These flags may not be set for open mode. */
static const int ILLEGAL_OPEN_FLAGS = (NC_MMAP);
/* From libsrc4, these are the netcdf-4 cache sizes. */
extern size_t nc4_chunk_cache_size;
extern size_t nc4_chunk_cache_nelems;
extern float nc4_chunk_cache_preemption;
/* From nc4mem.c */
extern int NC4_open_image_file(NC_FILE_INFO_T* h5);
@ -813,12 +808,22 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
/* Only set cache for non-parallel opens. */
if (!nc4_info->parallel)
{
if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
nc4_chunk_cache_preemption) < 0)
NCglobalstate* gs = NC_getglobalstate();
if (H5Pset_cache(fapl_id, 0, gs->chunkcache.nelems, gs->chunkcache.size,
gs->chunkcache.preemption) < 0)
BAIL(NC_EHDFERR);
LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f",
__func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems,
nc4_chunk_cache_preemption));
__func__, gs->chunkcache.size, gs->chunkcache.nelems,
gs->chunkcache.preemption));
}
{
NCglobalstate* gs = NC_getglobalstate();
if(gs->alignment.defined) {
if (H5Pset_alignment(fapl_id, gs->alignment.threshold, gs->alignment.alignment) < 0) {
BAIL(NC_EHDFERR);
}
}
}
/* Set HDF5 format compatibility in the FILE ACCESS property list.
@ -1453,10 +1458,10 @@ nc4_get_var_meta(NC_VAR_INFO_T *var)
BAIL(NC_EVARMETA);
/* Learn about current chunk cache settings. */
if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems),
&(var->chunk_cache_size), &rdcc_w0)) < 0)
if ((H5Pget_chunk_cache(access_pid, &(var->chunkcache.nelems),
&(var->chunkcache.size), &rdcc_w0)) < 0)
BAIL(NC_EHDFERR);
var->chunk_cache_preemption = rdcc_w0;
var->chunkcache.preemption = rdcc_w0;
/* Get the dataset creation properties. */
if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0)

12
libhdf5/hdf5var.c
View File

@ -85,9 +85,9 @@ nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
if ((access_pid = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
return NC_EHDFERR;
if (H5Pset_chunk_cache(access_pid, var->chunk_cache_nelems,
var->chunk_cache_size,
var->chunk_cache_preemption) < 0)
if (H5Pset_chunk_cache(access_pid, var->chunkcache.nelems,
var->chunkcache.size,
var->chunkcache.preemption) < 0)
return NC_EHDFERR;
if (H5Dclose(hdf5_var->hdf_datasetid) < 0)
return NC_EHDFERR;
@ -2314,9 +2314,9 @@ NC4_HDF5_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
assert(var && var->hdr.id == varid);
/* Set the values. */
var->chunk_cache_size = size;
var->chunk_cache_nelems = nelems;
var->chunk_cache_preemption = preemption;
var->chunkcache.size = size;
var->chunkcache.nelems = nelems;
var->chunkcache.preemption = preemption;
/* Reopen the dataset to bring new settings into effect. */
if ((retval = nc4_reopen_dataset(grp, var)))

16
libhdf5/nc4hdf.c
View File

@ -977,9 +977,9 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
}
/* Set per-var chunk cache, for chunked datasets. */
if (var->storage == NC_CHUNKED && var->chunk_cache_size)
if (H5Pset_chunk_cache(access_plistid, var->chunk_cache_nelems,
var->chunk_cache_size, var->chunk_cache_preemption) < 0)
if (var->storage == NC_CHUNKED && var->chunkcache.size)
if (H5Pset_chunk_cache(access_plistid, var->chunkcache.nelems,
var->chunkcache.size, var->chunkcache.preemption) < 0)
BAIL(NC_EHDFERR);
/* At long last, create the dataset. */
@ -1101,12 +1101,12 @@ nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
/* If the chunk cache is too small, and the user has not changed
* the default value of the chunk cache size, then increase the
* size of the cache. */
if (var->chunk_cache_size == CHUNK_CACHE_SIZE)
if (chunk_size_bytes > var->chunk_cache_size)
if (var->chunkcache.size == CHUNK_CACHE_SIZE)
if (chunk_size_bytes > var->chunkcache.size)
{
var->chunk_cache_size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
if (var->chunk_cache_size > MAX_DEFAULT_CACHE_SIZE)
var->chunk_cache_size = MAX_DEFAULT_CACHE_SIZE;
var->chunkcache.size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
if (var->chunkcache.size > MAX_DEFAULT_CACHE_SIZE)
var->chunkcache.size = MAX_DEFAULT_CACHE_SIZE;
if ((retval = nc4_reopen_dataset(grp, var)))
return retval;
}

10
libnczarr/zinternal.c
View File

@ -18,12 +18,6 @@
#include "zincludes.h"
#include "zfilter.h"
/* These are the default chunk cache sizes for ZARR files created or
* opened with netCDF-4. */
extern size_t ncz_chunk_cache_size;
extern size_t ncz_chunk_cache_nelems;
extern float ncz_chunk_cache_preemption;
/* Forward */
#ifdef LOGGING
@ -61,10 +55,10 @@ NCZ_initialize_internal(void)
{
int stat = NC_NOERR;
char* dimsep = NULL;
NCRCglobalstate* ngs = NULL;
NCglobalstate* ngs = NULL;
ncz_initialized = 1;
ngs = ncrc_getglobalstate();
ngs = NC_getglobalstate();
if(ngs != NULL) {
/* Defaults */
ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR;

2
libnczarr/zsync.c
View File

@ -1534,7 +1534,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
}
/* Capture dimension_separator (must precede chunk cache creation) */
{
NCRCglobalstate* ngs = ncrc_getglobalstate();
NCglobalstate* ngs = NC_getglobalstate();
assert(ngs != NULL);
zvar->dimension_separator = 0;
if((stat = NCJdictget(jvar,"dimension_separator",&jvalue))) goto done;

8
libnczarr/zvar.c
View File

@ -381,7 +381,7 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims,
zvar->common.file = h5;
zvar->scalar = (ndims == 0 ? 1 : 0);
zvar->dimension_separator = ncrc_getglobalstate()->zarr.dimension_separator;
zvar->dimension_separator = NC_getglobalstate()->zarr.dimension_separator;
assert(zvar->dimension_separator != 0);
/* Set these state flags for the var. */
@ -456,9 +456,9 @@ var->type_info->rc++;
zvar->chunksize = zvar->chunkproduct * var->type_info->size;
/* Override the cache setting to use NCZarr defaults */
var->chunk_cache_size = CHUNK_CACHE_SIZE_NCZARR;
var->chunk_cache_nelems = ceildiv(var->chunk_cache_size,zvar->chunksize);
var->chunk_cache_preemption = 1; /* not used */
var->chunkcache.size = CHUNK_CACHE_SIZE_NCZARR;
var->chunkcache.nelems = ceildiv(var->chunkcache.size,zvar->chunksize);
var->chunkcache.preemption = 1; /* not used */
/* Create the cache */
if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,zvar->dimension_separator,&zvar->cache)))

10
libnczarr/zxcache.c
View File

@ -77,9 +77,9 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl
assert(zvar != NULL && zvar->cache != NULL);
/* Set the values. */
var->chunk_cache_size = cachesize;
var->chunk_cache_nelems = nelems;
var->chunk_cache_preemption = preemption;
var->chunkcache.size = cachesize;
var->chunkcache.nelems = nelems;
var->chunkcache.preemption = preemption;
/* Fix up cache */
if((retval = NCZ_adjust_var_cache(var))) goto done;
@ -120,8 +120,8 @@ fprintf(stderr,"xxx: adjusting cache for: %s\n",var->hdr.name);
/* Reclaim any existing fill_chunk */
if((stat = NCZ_reclaim_fill_chunk(zcache))) goto done;
/* Reset the parameters */
zvar->cache->maxsize = var->chunk_cache_size;
zvar->cache->maxentries = var->chunk_cache_nelems;
zvar->cache->maxsize = var->chunkcache.size;
zvar->cache->maxentries = var->chunkcache.nelems;
#ifdef DEBUG
fprintf(stderr,"%s.cache.adjust: size=%ld nelems=%ld\n",
var->hdr.name,(unsigned long)zvar->cache->maxsize,(unsigned long)zvar->cache->maxentries);

34
libsrc4/nc4cache.c
View File

@ -13,12 +13,6 @@
#include "config.h"
#include "nc4internal.h"
/* These are the default chunk cache sizes for HDF5 files created or
* opened with netCDF-4. */
extern size_t nc4_chunk_cache_size;
extern size_t nc4_chunk_cache_nelems;
extern float nc4_chunk_cache_preemption;
/**
* Set chunk cache size. Only affects netCDF-4/HDF5 files
* opened/created *after* it is called.
@ -71,11 +65,12 @@ extern float nc4_chunk_cache_preemption;
int
nc_set_chunk_cache(size_t size, size_t nelems, float preemption)
{
NCglobalstate* gs = NC_getglobalstate();
if (preemption < 0 || preemption > 1)
return NC_EINVAL;
nc4_chunk_cache_size = size;
nc4_chunk_cache_nelems = nelems;
nc4_chunk_cache_preemption = preemption;
gs->chunkcache.size = size;
gs->chunkcache.nelems = nelems;
gs->chunkcache.preemption = preemption;
return NC_NOERR;
}
@ -99,14 +94,15 @@ nc_set_chunk_cache(size_t size, size_t nelems, float preemption)
int
nc_get_chunk_cache(size_t *sizep, size_t *nelemsp, float *preemptionp)
{
NCglobalstate* gs = NC_getglobalstate();
if (sizep)
*sizep = nc4_chunk_cache_size;
*sizep = gs->chunkcache.size;
if (nelemsp)
*nelemsp = nc4_chunk_cache_nelems;
*nelemsp = gs->chunkcache.nelems;
if (preemptionp)
*preemptionp = nc4_chunk_cache_preemption;
*preemptionp = gs->chunkcache.preemption;
return NC_NOERR;
}
@ -128,11 +124,12 @@ nc_get_chunk_cache(size_t *sizep, size_t *nelemsp, float *preemptionp)
int
nc_set_chunk_cache_ints(int size, int nelems, int preemption)
{
NCglobalstate* gs = NC_getglobalstate();
if (size <= 0 || nelems <= 0 || preemption < 0 || preemption > 100)
return NC_EINVAL;
nc4_chunk_cache_size = size;
nc4_chunk_cache_nelems = nelems;
nc4_chunk_cache_preemption = (float)preemption / 100;
gs->chunkcache.size = size;
gs->chunkcache.nelems = nelems;
gs->chunkcache.preemption = (float)preemption / 100;
return NC_NOERR;
}
@ -154,12 +151,13 @@ nc_set_chunk_cache_ints(int size, int nelems, int preemption)
int
nc_get_chunk_cache_ints(int *sizep, int *nelemsp, int *preemptionp)
{
NCglobalstate* gs = NC_getglobalstate();
if (sizep)
*sizep = (int)nc4_chunk_cache_size;
*sizep = (int)gs->chunkcache.size;
if (nelemsp)
*nelemsp = (int)nc4_chunk_cache_nelems;
*nelemsp = (int)gs->chunkcache.nelems;
if (preemptionp)
*preemptionp = (int)(nc4_chunk_cache_preemption * 100);
*preemptionp = (int)(gs->chunkcache.preemption * 100);
return NC_NOERR;
}

89
libsrc4/nc4internal.c
View File

@ -22,6 +22,7 @@
#include "nc.h" /* from libsrc */
#include "ncdispatch.h" /* from libdispatch */
#include "ncutf8.h"
#include "ncrc.h"
/** @internal Number of reserved attributes. These attributes are
* hidden from the netcdf user, but exist in the implementation
@ -51,11 +52,6 @@ static const NC_reservedatt NC_reserved[] = {
};
#define NRESERVED (sizeof(NC_reserved) / sizeof(NC_reservedatt)) /*|NC_reservedatt|*/
/* These hold the file caching settings for the library. */
size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE; /**< Default chunk cache size. */
size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */
float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */
static int NC4_move_in_NCList(NC* nc, int new_id);
#ifdef LOGGING
@ -670,6 +666,7 @@ int
nc4_var_list_add2(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var)
{
NC_VAR_INFO_T *new_var = NULL;
NCglobalstate* gs = NC_getglobalstate();
/* Allocate storage for new variable. */
if (!(new_var = calloc(1, sizeof(NC_VAR_INFO_T))))
@ -678,9 +675,9 @@ nc4_var_list_add2(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var)
new_var->container = grp;
/* These are the HDF5-1.8.4 defaults. */
new_var->chunk_cache_size = nc4_chunk_cache_size;
new_var->chunk_cache_nelems = nc4_chunk_cache_nelems;
new_var->chunk_cache_preemption = nc4_chunk_cache_preemption;
new_var->chunkcache.size = gs->chunkcache.size;
new_var->chunkcache.nelems = gs->chunkcache.nelems;
new_var->chunkcache.preemption = gs->chunkcache.preemption;
/* Now fill in the values in the var info structure. */
new_var->hdr.id = ncindexsize(grp->vars);
@ -1951,3 +1948,79 @@ NC4_move_in_NCList(NC* nc, int new_id)
return stat;
}
/**************************************************/
/* NCglobal state management */
static NCglobalstate* nc_globalstate = NULL;
static int
NC_createglobalstate(void)
{
int stat = NC_NOERR;
const char* tmp = NULL;
if(nc_globalstate == NULL) {
nc_globalstate = calloc(1,sizeof(NCglobalstate));
}
/* Initialize struct pointers */
nc_globalstate->rcinfo = (struct NCRCinfo*)calloc(1,sizeof(struct NCRCinfo));
if(nc_globalstate == NULL) return NC_ENOMEM;
/* Get environment variables */
if(getenv(NCRCENVIGNORE) != NULL)
nc_globalstate->rcinfo->ignore = 1;
tmp = getenv(NCRCENVRC);
if(tmp != NULL && strlen(tmp) > 0)
nc_globalstate->rcinfo->rcfile = strdup(tmp);
/* Initialize chunk cache defaults */
nc_globalstate->chunkcache.size = CHUNK_CACHE_SIZE; /**< Default chunk cache size. */
nc_globalstate->chunkcache.nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */
nc_globalstate->chunkcache.preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */
return stat;
}
/* Get global state */
NCglobalstate*
NC_getglobalstate(void)
{
if(nc_globalstate == NULL)
NC_createglobalstate();
return nc_globalstate;
}
void
NC_freeglobalstate(void)
{
if(nc_globalstate != NULL) {
nullfree(nc_globalstate->tempdir);
nullfree(nc_globalstate->home);
nullfree(nc_globalstate->cwd);
NC_rcclear(nc_globalstate->rcinfo);
free(nc_globalstate->rcinfo);
free(nc_globalstate);
nc_globalstate = NULL;
}
}
/**************************************************/
/* Specific property functions */
/** \internal Set global parameters for H5Pset_alignment */
int
nc_set_alignment(int threshold, int alignment)
{
NCglobalstate* gs = NC_getglobalstate();
gs->alignment.threshold = threshold;
gs->alignment.alignment = alignment;
gs->alignment.defined = 1;
return NC_NOERR;
}
int
nc_get_alignment(int* thresholdp, int* alignmentp)
{
NCglobalstate* gs = NC_getglobalstate();
if(thresholdp) *thresholdp = gs->alignment.threshold;
if(alignmentp) *alignmentp = gs->alignment.alignment;
return NC_NOERR;
}

6
libsrc4/nc4var.c
View File

@ -97,11 +97,11 @@ NC4_get_var_chunk_cache(int ncid, int varid, size_t *sizep,
/* Give the user what they want. */
if (sizep)
*sizep = var->chunk_cache_size;
*sizep = var->chunkcache.size;
if (nelemsp)
*nelemsp = var->chunk_cache_nelems;
*nelemsp = var->chunkcache.nelems;
if (preemptionp)
*preemptionp = var->chunk_cache_preemption;
*preemptionp = var->chunkcache.preemption;
return NC_NOERR;
}

4
nc_test4/CMakeLists.txt
View File

@ -24,6 +24,10 @@ SET(NC4_TESTS tst_dims tst_dims2 tst_dims3 tst_files tst_files4
tst_hdf5_file_compat tst_fill_attr_vanish tst_rehash tst_types tst_bug324
tst_atts3 tst_put_vars tst_elatefill tst_udf tst_bug1442 tst_quantize)
IF(HAS_PAR_FILTERS)
SET(NC4_tests $NC4_TESTS tst_alignment)
ENDIF()
# Note, renamegroup needs to be compiled before run_grp_rename
IF(BUILD_UTILITIES)

4
nc_test4/Makefile.am
View File

@ -37,6 +37,10 @@ tst_rehash tst_filterparser tst_bug324 tst_types tst_atts3 \
tst_put_vars tst_elatefill tst_udf tst_put_vars_two_unlim_dim \
tst_bug1442 tst_quantize
if HAS_PAR_FILTERS
NC4_TESTS += tst_alignment
endif
# Temporary I hoped, but hoped in vain.
if !ISCYGWIN
NC4_TESTS += tst_h_strbug tst_h_refs

96
nc_test4/tst_alignment.c Normal file
View File

@ -0,0 +1,96 @@
/* This is part of the netCDF package.
Copyright 2018 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT file for conditions of use.
Test HDF5 alignment
Dennis Heimbigner
*/
#include <nc_tests.h>
#include "err_macros.h"
#include <hdf5.h>
#include <H5DSpublic.h>
#undef DEBUG
#define THRESHOLD 512
#define ALIGNMENT 4096
#define CHUNKSIZE 513
int
main(int argc, char **argv)
{
int i,ncid, varid, dimids[1];
size_t chunks[1];
unsigned char data[CHUNKSIZE];
hid_t fileid, grpid, datasetid;
hid_t dxpl_id = H5P_DEFAULT; /*data transfer property list */
unsigned int filter_mask = 0;
hsize_t hoffset[1];
haddr_t addr;
hsize_t size ;
hid_t fspace;
H5Eset_auto2(H5E_DEFAULT,(H5E_auto2_t)H5Eprint1,stderr);
printf("\n*** Testing HDF5 alignment.\n");
printf("chunksize=%d threshold=%d alignment=%d\n",CHUNKSIZE,THRESHOLD,ALIGNMENT);
if(nc_set_alignment(THRESHOLD,ALIGNMENT)) ERR;
if (nc_create("tst_alignment.nc", NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, "d0", CHUNKSIZE, &dimids[0])) ERR;
if (nc_def_var(ncid, "var", NC_UBYTE, 1, dimids, &varid)) ERR;
chunks[0] = CHUNKSIZE;
if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunks)) ERR;
if (nc_enddef(ncid)) ERR;
for(i=0;i<CHUNKSIZE;i++) data[i] = (i)%2;
if(nc_put_var(ncid,varid,data)) ERR;
if (nc_close(ncid)) ERR;
/* Use HDF5 API to verify */
if ((fileid = H5Fopen("tst_alignment.nc", H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) ERR;
if ((grpid = H5Gopen1(fileid, "/")) < 0) ERR;
if ((datasetid = H5Dopen1(grpid, "var")) < 0) ERR;
/* Test the offset */
if((fspace = H5Dget_space(datasetid)) < 0) ERR;
if(H5Dget_chunk_info(datasetid, fspace, 0, hoffset, &filter_mask, &addr, &size) < 0) ERR;
#ifdef DEBUG
fprintf(stderr,"H5Dget_chunk_info: offset=%lu addr=%lu size=%lu\n",(unsigned long)hoffset[0], (unsigned long)addr, (unsigned long)size);
fprintf(stderr,"\t%% alignment: offset=%lu, addr=%lu\n",
(((unsigned long)hoffset[0]) % ALIGNMENT),
(((unsigned long)addr) % ALIGNMENT)
);
#endif
printf("H5Dget_chunk_info: addr=%lu (addr %% alignment)=%lu\n", (unsigned long)addr, (((unsigned long)hoffset[0]) % ALIGNMENT));
if((addr % ALIGNMENT) != 0) ERR;
/* Test chunk content */
memset(data,0,sizeof(data));
hoffset[0] = 0;
if(H5Dread_chunk(datasetid, dxpl_id, hoffset, &filter_mask, data) < 0) ERR;
#ifdef DEBUG
fprintf(stderr,"H5Dread_chunk: offset=%lu offset %% alignment=%lu\n",(unsigned long)hoffset[0], (((unsigned long)hoffset) % ALIGNMENT));
#endif
for(i=0;i<CHUNKSIZE;i++) {
if(data[i] != (i)%2) {
fprintf(stderr,"data[%d] mismatch\n",i);
ERR;
}
}
if (H5Dclose(datasetid) < 0) ERR;
if (H5Gclose(grpid) < 0) ERR;
if (H5Fclose(fileid) < 0) ERR;
SUMMARIZE_ERR;
FINAL_RESULTS;
}

2
ncdap_test/t_auth.c
View File

@ -16,7 +16,7 @@ See \ref copyright file for more info.
#include <unistd.h>
#endif
#define DEBUG
#undef DEBUG
#include "netcdf.h"
#include "nctestserver.h"

2
ncdump/tst_chunking.c
View File

@ -9,7 +9,7 @@
#include <nc_tests.h>
#include "err_macros.h"
#define DEBUG
#undef DEBUG
static int ret = NC_NOERR;

2
ncdump/tst_nccopy3.sh
View File

@ -8,6 +8,8 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
export SETX=1
set -e
echo ""

7
ncdump/tst_rcmerge.c
View File

@ -3,20 +3,21 @@
#include <string.h>
#include "netcdf.h"
#include "ncrc.h"
#include "nc4internal.h"
int
main(int argc, char** argv)
{
size_t i,nentries = 0;
NCRCglobalstate* ngs = ncrc_getglobalstate();
NCglobalstate* ngs = NC_getglobalstate();
NCRCinfo* info = NULL;
NCRCentry* entry = NULL;
/* Cause the .rc files to be read and merged */
nc_initialize();
if((ngs = ncrc_getglobalstate())==NULL) abort();
info = &ngs->rcinfo;
if((ngs = NC_getglobalstate())==NULL) abort();
info = ngs->rcinfo;
if(info->ignore) {
fprintf(stderr,".rc ignored\n");

2
ncdump/tst_unicode.c
View File

@ -23,7 +23,7 @@
#include <locale.h>
#endif
#define DEBUG
#undef DEBUG
/* The data file we will create. */
static const unsigned char prefix[] = {

2
ncgen/dump.c
View File

@ -8,7 +8,7 @@
#include "includes.h"
#include "dump.h"
#define DEBUGSRC
#undef DEBUGSRC
#define MAXELEM 8
#define MAXDEPTH 4

2
nczarr_test/bm_chunks3.c
View File

@ -167,7 +167,7 @@ main(int argc, char *argv[])
/* variable shapes */
int var_dims[RANK];
NCCHECK(getoptions(&argc,&argv,&bmoptions));
NCCHECK(bm_getoptions(&argc,&argv,&bmoptions));
NCCHECK(nc4_buildpath(&bmoptions,&path));
if(bmoptions.debug) {

4
nczarr_test/bm_utils.h
View File

@ -89,6 +89,10 @@ EXTERNL void bm_reportmetaoptions(struct BMMeta* o);
EXTERNL const char* bm_printvector(int rank, const size_t* vec);
EXTERNL const char* bm_printvectorp(int rank, const ptrdiff_t* vec);
EXTERNL const char* bm_varasprint(int rank, const size_t* start, const size_t* edges, const ptrdiff_t* stride);
EXTERNL int nc4_buildpath(struct BMOptions* o, char** pathp);
EXTERNL void reportoptions(struct BMOptions* o);
EXTERNL void reportmetaoptions(struct BMMeta* o);
EXTERNL void clearoptions(struct BMOptions*);
#include "ut_test.h"

2
nczarr_test/s3util.c
View File

@ -27,7 +27,7 @@
#undef NODELETE
#define DEBUG
#undef DEBUG
#define DATANAME "data"

2
nczarr_test/tst_zchunks3.c
View File

@ -8,7 +8,7 @@
#include "ut_includes.h"
#include "test_nczarr_utils.h"
#define DEBUG
#undef DEBUG
static int ret = NC_NOERR;
#define FILE_NAME "tmp_chunks3.nc"

2
nczarr_test/ut_json.c
View File

@ -5,7 +5,7 @@
#include "ut_includes.h"
#define DEBUG
#undef DEBUG
typedef enum Cmds {
cmd_none = 0,

2
nczarr_test/zhex.c
View File

@ -12,7 +12,7 @@
#include <unistd.h>
#endif
#define DEBUG
#undef DEBUG
static char hex[16] = "0123456789abcdef";

2
nczarr_test/zmapio.c
View File

@ -23,7 +23,7 @@
#include "nclog.h"
#include "ncuri.h"
#define DEBUG
#undef DEBUG
#define DATANAME "data"

2
nczarr_test/zs3parse.c
View File

@ -21,7 +21,7 @@
#include "zincludes.h"
#include "ncpathmgr.h"
#define DEBUG
#undef DEBUG
#define AWSHOST ".amazonaws.com"

6
oc2/ocinternal.c
View File

@ -25,7 +25,7 @@
#include <errno.h>
#include "ncrc.h"
#include "nc4internal.h"
#include "ocinternal.h"
#include "ocdebug.h"
#include "occurlfunctions.h"
@ -321,7 +321,7 @@ createtempfile(OCstate* state, OCtree* tree)
char* path = NULL;
char* tmppath = NULL;
int len;
NCRCglobalstate* globalstate = ncrc_getglobalstate();
NCglobalstate* globalstate = NC_getglobalstate();
len =
strlen(globalstate->tempdir)
@ -526,7 +526,7 @@ static OCerror
ocset_curlproperties(OCstate* state)
{
OCerror stat = OC_NOERR;
NCRCglobalstate* globalstate = ncrc_getglobalstate();
NCglobalstate* globalstate = NC_getglobalstate();
if(state->auth->curlflags.useragent == NULL) {
size_t len = strlen(DFALTUSERAGENT) + strlen(VERSION) + 1;

2
unit_test/test_aws.c
View File

@ -15,7 +15,7 @@ Test the handling of aws profiles and regions.
#include "ncrc.h"
#include "ncpathmgr.h"
#define DEBUG
#undef DEBUG
typedef struct ProfileTest {
const char* profile;

2
unit_test/test_pathcvt.c
View File

@ -13,7 +13,7 @@ Test the NCpathcvt
#include "netcdf.h"
#include "ncpathmgr.h"
#define DEBUG
#undef DEBUG
#define NKINDS 4
static const int kinds[NKINDS] = {NCPD_NIX,NCPD_MSYS,NCPD_CYGWIN,NCPD_WIN};