From f3e711e2b8b2330fa7e77a8a898529d001bdb467 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 Jan 2022 15:27:52 -0700 Subject: [PATCH] Add support for setting HDF5 alignment property when creating a file re: https://github.com/Unidata/netcdf-c/issues/2177 re: https://github.com/Unidata/netcdf-c/pull/2178 Provide get/set functions to store global data alignment information and apply it when a file is created. The api is as follows: ```` int nc_set_alignment(int threshold, int alignment); int nc_get_alignment(int* thresholdp, int* alignmentp); ```` If defined, then for every file created opened after the call to nc_set_alignment, for every new variable added to the file, the most recently set threshold and alignment values will be applied to that variable. The nc_get_alignment function return the last values set by nc_set_alignment. If nc_set_alignment has not been called, then it returns the value 0 for both threshold and alignment. The alignment parameters are stored in the NCglobalstate object (see below) for use as needed. Repeated calls to nc_set_alignment will overwrite any existing values in NCglobalstate. The alignment parameters are applied in libhdf5/hdf5create.c and libhdf5/hdf5open.c The set/get alignment functions are defined in libsrc4/nc4internal.c. A test program was added as nc_test4/tst_alignment.c. ## Misc. Changes Unrelated to Alignment * The NCRCglobalstate type was renamed to NCglobalstate to indicate that it represented more general global state than just .rc data. It was also moved to nc4internal.h. This led to a large number of small changes: mostly renaming. The global state management functions were moved to nc4internal.c. * The global chunk cache variables have been moved into NCglobalstate. As warranted, other global state will be moved as well. * Some misc. problems with the nczarr performance tests were corrected. --- RELEASE_NOTES.md | 1 + acinclude.m4 | 2 + dap4_test/test_data.c | 2 +- docs/Doxyfile.in | 1 - include/nc4internal.h | 31 ++++++++- include/ncrc.h | 43 ++++--------- include/netcdf.h | 8 +++ libdap2/dceconstraints.c | 2 +- libdap4/d4file.c | 2 +- libdispatch/ddispatch.c | 8 +-- libdispatch/dpathmgr.c | 2 +- libdispatch/drc.c | 125 +++++++++++-------------------------- libdispatch/dutil.c | 32 ++++++---- libdispatch/ncs3sdk.cpp | 7 +-- libhdf5/hdf5create.c | 25 +++++--- libhdf5/hdf5internal.c | 6 -- libhdf5/hdf5open.c | 29 +++++---- libhdf5/hdf5var.c | 12 ++-- libhdf5/nc4hdf.c | 16 ++--- libnczarr/zinternal.c | 10 +-- libnczarr/zsync.c | 2 +- libnczarr/zvar.c | 8 +-- libnczarr/zxcache.c | 10 +-- libsrc4/nc4cache.c | 34 +++++----- libsrc4/nc4internal.c | 89 +++++++++++++++++++++++--- libsrc4/nc4var.c | 6 +- nc_test4/CMakeLists.txt | 4 ++ nc_test4/Makefile.am | 4 ++ nc_test4/tst_alignment.c | 96 ++++++++++++++++++++++++++++ ncdap_test/t_auth.c | 2 +- ncdump/tst_chunking.c | 2 +- ncdump/tst_nccopy3.sh | 2 + ncdump/tst_rcmerge.c | 7 ++- ncdump/tst_unicode.c | 2 +- ncgen/dump.c | 2 +- nczarr_test/bm_chunks3.c | 2 +- nczarr_test/bm_utils.h | 4 ++ nczarr_test/s3util.c | 2 +- nczarr_test/tst_zchunks3.c | 2 +- nczarr_test/ut_json.c | 2 +- nczarr_test/zhex.c | 2 +- nczarr_test/zmapio.c | 2 +- nczarr_test/zs3parse.c | 2 +- oc2/ocinternal.c | 6 +- unit_test/test_aws.c | 2 +- unit_test/test_pathcvt.c | 2 +- 46 files changed, 409 insertions(+), 253 deletions(-) create mode 100644 nc_test4/tst_alignment.c diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e64d4f451..23b86110d 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD +* [Enhancement] Add ability to set dataset alignment for netcdf-4/HDF5 files. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). * [Enhancement] Add complete bitgroom support to NCZarr. See [Github #2197](https://github.com/Unidata/netcdf-c/pull/2197). * [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179(https://github.com/Unidata/netcdf-c/pull/2179). * [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183). diff --git a/acinclude.m4 b/acinclude.m4 index a3fdc712a..b6420954a 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -384,4 +384,6 @@ MOSTLYCLEANFILES += $(valgrind_log_files) AC_SUBST([VALGRIND_CHECK_RULES]) m4_ifdef([_AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE([VALGRIND_CHECK_RULES])]) ]) + AC_DEFUN([_AC_FINALIZE],[]) + diff --git a/dap4_test/test_data.c b/dap4_test/test_data.c index 946c37b2a..ab9f8f3a8 100644 --- a/dap4_test/test_data.c +++ b/dap4_test/test_data.c @@ -12,7 +12,7 @@ Test the netcdf-4 data building process. #include #include "netcdf.h" -#define DEBUG +#undef DEBUG static void fail(int code) diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index f525ac33a..fcb2b3368 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -781,7 +781,6 @@ INPUT = \ @abs_top_srcdir@/libdispatch/derror.c \ @abs_top_srcdir@/libdispatch/dv2i.c \ @abs_top_srcdir@/libdispatch/dcopy.c \ - @abs_top_srcdir@/libdispatch/daux.c \ @abs_top_srcdir@/libsrc4/nc4var.c \ @abs_top_srcdir@/libhdf5/nc4hdf.c \ @abs_top_srcdir@/libsrc4/nc4internal.c \ diff --git a/include/nc4internal.h b/include/nc4internal.h index 3be336439..8d8906032 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -118,6 +118,7 @@ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t; /* Forward declarations. */ struct NC_GRP_INFO; struct NC_TYPE_INFO; +struct NCRCinfo; /** * This struct provides indexed Access to Meta-data objects. See the @@ -205,9 +206,11 @@ typedef struct NC_VAR_INFO int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */ nc_bool_t shuffle; /**< True if var has shuffle filter applied. */ nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */ - size_t chunk_cache_size; /**< Size in bytes of the var chunk cache. */ - size_t chunk_cache_nelems; /**< Number of slots in var chunk cache. */ - float chunk_cache_preemption; /**< Chunk cache preemtion policy. */ + struct ChunkCache { + size_t size; /**< Size in bytes of the var chunk cache. */ + size_t nelems; /**< Number of slots in var chunk cache. */ + float preemption; /**< Chunk cache preemtion policy. */ + } chunkcache; int quantize_mode; /**< Quantize mode. NC_NOQUANTIZE is 0, and means no quantization. */ int nsd; /**< Number of significant digits if quantization is used, 0 if not. */ void *format_var_info; /**< Pointer to any binary format info. */ @@ -327,6 +330,24 @@ typedef struct NC_FILE_INFO } mem; } NC_FILE_INFO_T; +/* Collect global state info in one place */ +typedef struct NCglobalstate { + int initialized; + char* tempdir; /* track a usable temp dir */ + char* home; /* track $HOME */ + char* cwd; /* track getcwd */ + struct NCRCinfo* rcinfo; /* Currently only one rc file per session */ + struct GlobalZarr { /* Zarr specific parameters */ + char dimension_separator; + } zarr; + struct Alignment { /* H5Pset_alignment parameters */ + int defined; /* 1 => threshold and alignment explicitly set */ + int threshold; + int alignment; + } alignment; + struct ChunkCache chunkcache; +} NCglobalstate; + /** Variable Length Datatype struct in memory. Must be identical to * HDF5 hvl_t. (This is only used for VL sequences, not VL strings, * which are stored in char *'s) */ @@ -460,6 +481,10 @@ extern const char* nc4_atomic_name[NUM_ATOMIC_TYPES]; /* Binary searcher for reserved attributes */ extern const NC_reservedatt* NC_findreserved(const char* name); +/* Global State Management */ +extern NCglobalstate* NC_getglobalstate(void); +extern void NC_freeglobalstate(void); + /* Generic reserved Attributes */ #define NC_ATT_REFERENCE_LIST "REFERENCE_LIST" #define NC_ATT_CLASS "CLASS" diff --git a/include/ncrc.h b/include/ncrc.h index 1b3228674..14c1c219a 100644 --- a/include/ncrc.h +++ b/include/ncrc.h @@ -33,39 +33,25 @@ typedef struct NCRCentry { char* value; } NCRCentry; -/* collect all the relevant info around the rc file */ -typedef struct NCRCinfo { - int ignore; /* if 1, then do not use any rc file */ - int loaded; /* 1 => already loaded */ - NClist* entries; /* the rc file entry store fields*/ - char* rcfile; /* specified rcfile; overrides anything else */ - char* rchome; /* Overrides $HOME when looking for .rc files */ -} NCRCinfo; - -/* Collect global state info in one place */ -typedef struct NCRCglobalstate { - int initialized; - char* tempdir; /* track a usable temp dir */ - char* home; /* track $HOME */ - char* cwd; /* track getcwd */ - NCRCinfo rcinfo; /* Currently only one rc file per session */ - struct GlobalZarr { /* Zarr specific parameters */ - char dimension_separator; - } zarr; - struct S3credentials { - NClist* profiles; /* NClist */ - } s3creds; -} NCRCglobalstate; +struct AWSentry { + char* key; + char* value; +}; struct AWSprofile { char* name; NClist* entries; /* NClist */ }; -struct AWSentry { - char* key; - char* value; -}; +/* collect all the relevant info around the rc file and AWS */ +typedef struct NCRCinfo { + int ignore; /* if 1, then do not use any rc file */ + int loaded; /* 1 => already loaded */ + NClist* entries; /* the rc file entry store fields*/ + char* rcfile; /* specified rcfile; overrides anything else */ + char* rchome; /* Overrides $HOME when looking for .rc files */ + NClist* s3profiles; /* NClist */ +} NCRCinfo; typedef struct NCS3INFO { char* host; /* non-null if other*/ @@ -80,9 +66,7 @@ extern "C" { #endif /* From drc.c */ -EXTERNL int ncrc_createglobalstate(void); EXTERNL void ncrc_initialize(void); -EXTERNL void ncrc_freeglobalstate(void); EXTERNL int NC_rcfile_insert(const char* key, const char* value, const char* hostport, const char* path); EXTERNL char* NC_rclookup(const char* key, const char* hostport, const char* path); EXTERNL char* NC_rclookupx(NCURI* uri, const char* key); @@ -94,7 +78,6 @@ EXTERNL size_t NC_rcfile_length(NCRCinfo*); EXTERNL NCRCentry* NC_rcfile_ith(NCRCinfo*,size_t); /* For internal use */ -EXTERNL NCRCglobalstate* ncrc_getglobalstate(void); EXTERNL void NC_rcclear(NCRCinfo* info); EXTERNL void NC_rcclear(NCRCinfo* info); diff --git a/include/netcdf.h b/include/netcdf.h index b8540c6ab..0816b12bb 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -560,6 +560,14 @@ nc_def_user_format(int mode_flag, NC_Dispatch *dispatch_table, char *magic_numbe EXTERNL int nc_inq_user_format(int mode_flag, NC_Dispatch **dispatch_table, char *magic_number); +/* Set the global alignment property */ +EXTERNL int +nc_set_alignment(int threshold, int alignment); + +/* Get the global alignment property */ +EXTERNL int +nc_get_alignment(int* thresholdp, int* alignmentp); + EXTERNL int nc__create(const char *path, int cmode, size_t initialsz, size_t *chunksizehintp, int *ncidp); diff --git a/libdap2/dceconstraints.c b/libdap2/dceconstraints.c index 800e5eb08..b4895768f 100644 --- a/libdap2/dceconstraints.c +++ b/libdap2/dceconstraints.c @@ -13,7 +13,7 @@ #include "dapincludes.h" #include "dceparselex.h" -#define DEBUG +#undef DEBUG #define LBRACE "{" #define RBRACE "}" diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 1c9404010..5ad408824 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -367,7 +367,7 @@ set_curl_properties(NCD4INFO* d4info) char* newpath = NULL; int len; errno = 0; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCglobalstate* globalstate = NC_getglobalstate(); /* Create the unique cookie file name */ len = diff --git a/libdispatch/ddispatch.c b/libdispatch/ddispatch.c index 12ba32995..dc41b45fa 100644 --- a/libdispatch/ddispatch.c +++ b/libdispatch/ddispatch.c @@ -12,6 +12,7 @@ See LICENSE.txt for license information. #include "ncoffsets.h" #include "ncpathmgr.h" #include "ncxml.h" +#include "nc4internal.h" /* Required for getcwd, other functions. */ #ifdef HAVE_UNISTD_H @@ -48,7 +49,7 @@ NCDISPATCH_initialize(void) { int status = NC_NOERR; int i; - NCRCglobalstate* globalstate = NULL; + NCglobalstate* globalstate = NULL; for(i=0;ircinfo.ignore = 1; - tmp = getenv(NCRCENVRC); - if(tmp != NULL && strlen(tmp) > 0) - ncrc_globalstate->rcinfo.rcfile = strdup(tmp); - return stat; -} - /* Initialize defaults and load: * .ncrc @@ -111,6 +91,7 @@ void ncrc_initialize(void) { int stat = NC_NOERR; + NCglobalstate* ncg = NULL; if(NCRCinitialized) return; NCRCinitialized = 1; /* prevent recursion */ @@ -121,7 +102,8 @@ ncrc_initialize(void) nclog(NCLOGWARN,".rc loading failed"); } /* Load .aws/config */ - if((stat = aws_load_credentials(ncrc_globalstate))) { + ncg = NC_getglobalstate(); + if((stat = aws_load_credentials(ncg))) { nclog(NCLOGWARN,"AWS config file not loaded"); } #endif @@ -131,35 +113,13 @@ static void ncrc_setrchome(void) { const char* tmp = NULL; - if(ncrc_globalstate->rcinfo.rchome) return; - assert(ncrc_globalstate && ncrc_globalstate->home); + NCglobalstate* ncg = NC_getglobalstate(); + assert(ncg && ncg->home); + if(ncg->rcinfo->rchome) return; tmp = getenv(NCRCENVHOME); if(tmp == NULL || strlen(tmp) == 0) - tmp = ncrc_globalstate->home; - ncrc_globalstate->rcinfo.rchome = strdup(tmp); -} - -/* Get global state */ -NCRCglobalstate* -ncrc_getglobalstate(void) -{ - if(ncrc_globalstate == NULL) - ncrc_createglobalstate(); - return ncrc_globalstate; -} - -void -ncrc_freeglobalstate(void) -{ - if(ncrc_globalstate != NULL) { - nullfree(ncrc_globalstate->tempdir); - nullfree(ncrc_globalstate->home); - nullfree(ncrc_globalstate->cwd); - NC_rcclear(&ncrc_globalstate->rcinfo); - clearS3credentials(&ncrc_globalstate->s3creds); - free(ncrc_globalstate); - ncrc_globalstate = NULL; - } + tmp = ncg->home; + ncg->rcinfo->rchome = strdup(tmp); } void @@ -169,9 +129,10 @@ NC_rcclear(NCRCinfo* info) nullfree(info->rcfile); nullfree(info->rchome); rcfreeentries(info->entries); + freeprofilelist(info->s3profiles); } -void +static void rcfreeentries(NClist* rc) { int i; @@ -191,18 +152,17 @@ NC_rcload(void) { int i,ret = NC_NOERR; char* path = NULL; - NCRCglobalstate* globalstate = NULL; + NCglobalstate* globalstate = NULL; NClist* rcfileorder = nclistnew(); if(!NCRCinitialized) ncrc_initialize(); - globalstate = ncrc_getglobalstate(); + globalstate = NC_getglobalstate(); - - if(globalstate->rcinfo.ignore) { + if(globalstate->rcinfo->ignore) { nclog(NCLOGDBG,".rc file loading suppressed"); goto done; } - if(globalstate->rcinfo.loaded) goto done; + if(globalstate->rcinfo->loaded) goto done; /* locate the configuration files in order of use: 1. Specified by NCRCENV_RC environment variable. @@ -215,8 +175,8 @@ NC_rcload(void) 6. $CWD/.docsrc Entry in later files override any of the earlier files */ - if(globalstate->rcinfo.rcfile != NULL) { /* always use this */ - nclistpush(rcfileorder,strdup(globalstate->rcinfo.rcfile)); + if(globalstate->rcinfo->rcfile != NULL) { /* always use this */ + nclistpush(rcfileorder,strdup(globalstate->rcinfo->rcfile)); } else { const char** rcname; const char* dirnames[3]; @@ -225,8 +185,7 @@ NC_rcload(void) /* Make sure rcinfo.rchome is defined */ ncrc_setrchome(); - - dirnames[0] = globalstate->rcinfo.rchome; + dirnames[0] = globalstate->rcinfo->rchome; dirnames[1] = globalstate->cwd; dirnames[2] = NULL; @@ -249,7 +208,7 @@ NC_rcload(void) } done: - globalstate->rcinfo.loaded = 1; /* even if not exists */ + globalstate->rcinfo->loaded = 1; /* even if not exists */ nclistfreeall(rcfileorder); return (ret); } @@ -301,7 +260,7 @@ NC_set_rcfile(const char* rcfile) { int stat = NC_NOERR; FILE* f = NULL; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCglobalstate* globalstate = NC_getglobalstate(); if(rcfile != NULL && strlen(rcfile) == 0) rcfile = NULL; @@ -311,8 +270,8 @@ NC_set_rcfile(const char* rcfile) goto done; } fclose(f); - nullfree(globalstate->rcinfo.rcfile); - globalstate->rcinfo.rcfile = strdup(rcfile); + nullfree(globalstate->rcinfo->rcfile); + globalstate->rcinfo->rcfile = strdup(rcfile); /* Clear globalstate->rcinfo */ NC_rcclear(&globalstate->rcinfo); /* (re) load the rcfile and esp the entriestore*/ @@ -414,7 +373,7 @@ rccompile(const char* path) NCbytes* tmp = ncbytesnew(); NCURI* uri = NULL; char* nextline = NULL; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCglobalstate* globalstate = NC_getglobalstate(); char* bucket = NULL; if((ret=NC_readfile(path,tmp))) { @@ -424,10 +383,10 @@ rccompile(const char* path) contents = ncbytesextract(tmp); if(contents == NULL) contents = strdup(""); /* Either reuse or create new */ - rc = globalstate->rcinfo.entries; + rc = globalstate->rcinfo->entries; if(rc == NULL) { rc = nclistnew(); - globalstate->rcinfo.entries = rc; + globalstate->rcinfo->entries = rc; } nextline = contents; for(;;) { @@ -529,11 +488,11 @@ static struct NCRCentry* rclocate(const char* key, const char* hostport, const char* path) { int i,found; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); - NClist* rc = globalstate->rcinfo.entries; + NCglobalstate* globalstate = NC_getglobalstate(); + NClist* rc = globalstate->rcinfo->entries; NCRCentry* entry = NULL; - if(globalstate->rcinfo.ignore) + if(globalstate->rcinfo->ignore) return NULL; if(key == NULL || rc == NULL) return NULL; @@ -610,12 +569,12 @@ NC_rcfile_insert(const char* key, const char* value, const char* hostport, const int ret = NC_NOERR; /* See if this key already defined */ struct NCRCentry* entry = NULL; - NCRCglobalstate* globalstate = NULL; + NCglobalstate* globalstate = NULL; NClist* rc = NULL; if(!NCRCinitialized) ncrc_initialize(); - globalstate = ncrc_getglobalstate(); - rc = globalstate->rcinfo.entries; + globalstate = NC_getglobalstate(); + rc = globalstate->rcinfo->entries; if(rc == NULL) { rc = nclistnew(); @@ -744,13 +703,6 @@ NC_getdefaults3region(NCURI* uri, const char** regionp) return stat; } -static void -clearS3credentials(struct S3credentials* creds) -{ - if(creds) - freeprofilelist(creds->profiles); -} - /** The .aws/config and .aws/credentials files are in INI format (https://en.wikipedia.org/wiki/INI_file). @@ -1038,10 +990,9 @@ freeprofilelist(NClist* profiles) /* Find, load, and parse the aws credentials file */ static int -aws_load_credentials(NCRCglobalstate* gstate) +aws_load_credentials(NCglobalstate* gstate) { int stat = NC_NOERR; - struct S3credentials* creds = &gstate->s3creds; NClist* profiles = nclistnew(); const char** awscfg = awsconfigfiles; const char* aws_root = getenv(NC_TEST_AWS_DIR); @@ -1074,7 +1025,7 @@ aws_load_credentials(NCRCglobalstate* gstate) nclistpush(profiles,noprof); noprof = NULL; } - creds->profiles = profiles; profiles = NULL; + gstate->rcinfo->s3profiles = profiles; profiles = NULL; #ifdef AWSDEBUG {int i,j; @@ -1102,10 +1053,10 @@ NC_authgets3profile(const char* profilename, struct AWSprofile** profilep) { int stat = NC_NOERR; int i = -1; - NCRCglobalstate* gstate = ncrc_getglobalstate(); - - for(i=0;is3creds.profiles);i++) { - struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->s3creds.profiles,i); + NCglobalstate* gstate = NC_getglobalstate(); + + for(i=0;ircinfo->s3profiles);i++) { + struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i); if(strcmp(profilename,profile->name)==0) {if(profilep) {*profilep = profile; goto done;}} } diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c index fc8a018d1..41d5e1a45 100644 --- a/libdispatch/dutil.c +++ b/libdispatch/dutil.c @@ -33,7 +33,7 @@ #define nulldup(x) ((x)?strdup(x):(x)) #endif /**************************************************/ -/** +/** \internal * Provide a hidden interface to allow utilities * to check if a given path name is really an ncdap4 url. * If no, return null, else return basename of the url @@ -66,7 +66,7 @@ NC__testurl(const char* path, char** basenamep) return ok; } -/* Return 1 if this machine is little endian */ +/** \internal Return 1 if this machine is little endian */ int NC_isLittleEndian(void) { @@ -78,6 +78,7 @@ NC_isLittleEndian(void) return (u.bytes[0] == 1 ? 1 : 0); } +/** \internal */ char* NC_backslashEscape(const char* s) { @@ -105,6 +106,7 @@ NC_backslashEscape(const char* s) return escaped; } +/** \internal */ char* NC_backslashUnescape(const char* esc) { @@ -129,6 +131,7 @@ NC_backslashUnescape(const char* esc) return s; } +/** \internal */ char* NC_entityescape(const char* s) { @@ -163,13 +166,13 @@ NC_entityescape(const char* s) return escaped; } -char* -/* +/** \internal Depending on the platform, the shell will sometimes pass an escaped octotherpe character without removing the backslash. So this function is appropriate to be called on possible url paths to unescape such cases. See e.g. ncgen. */ +char* NC_shellUnescape(const char* esc) { size_t len; @@ -194,7 +197,7 @@ NC_shellUnescape(const char* esc) return s; } -/** +/** \internal Wrap mktmp and return the generated path, or null if failed. Base is the base file path. XXXXX is appended @@ -224,6 +227,7 @@ done: return tmp; } +/** \internal */ int NC_readfile(const char* filename, NCbytes* content) { @@ -246,6 +250,7 @@ done: return ret; } +/** \internal */ int NC_writefile(const char* filename, size_t size, void* content) { @@ -271,7 +276,7 @@ done: return ret; } -/* +/** \internal Parse a path as a url and extract the modelist. If the path is not a URL, then return a NULL list. If a URL, but modelist is empty or does not exist, @@ -297,7 +302,7 @@ done: return stat; } -/* +/** \internal Check "mode=" list for a path and return 1 if present, 0 otherwise. */ int @@ -313,7 +318,7 @@ NC_testpathmode(const char* path, const char* tag) return found; } -/* +/** \internal Check "mode=" list for a url and return 1 if present, 0 otherwise. */ int @@ -339,8 +344,10 @@ done: return found; } -#if ! defined __INTEL_COMPILER -#if defined __APPLE__ +#if ! defined __INTEL_COMPILER +#if defined __APPLE__ +/** \internal */ + int isinf(double x) { union { unsigned long long u; double f; } ieee754; @@ -349,6 +356,7 @@ int isinf(double x) ( (unsigned)ieee754.u == 0 ); } +/** \internal */ int isnan(double x) { union { unsigned long long u; double f; } ieee754; @@ -360,7 +368,7 @@ int isnan(double x) #endif /*APPLE*/ #endif /*!_INTEL_COMPILER*/ - +/** \internal */ int NC_split_delim(const char* arg, char delim, NClist* segments) { @@ -395,7 +403,7 @@ done: return stat; } -/* concat the the segments with each segment preceded by '/' */ +/** \internal concat the the segments with each segment preceded by '/' */ int NC_join(NClist* segments, char** pathp) { diff --git a/libdispatch/ncs3sdk.cpp b/libdispatch/ncs3sdk.cpp index 2c51b3e76..0c524c793 100644 --- a/libdispatch/ncs3sdk.cpp +++ b/libdispatch/ncs3sdk.cpp @@ -73,9 +73,8 @@ NC_s3sdkinitialize(void) ncs3_finalized = 0; NCTRACE(11,NULL); Aws::InitAPI(ncs3options); - NCUNTRACE(NC_NOERR); } - return 1; + return NCUNTRACE(NC_NOERR); } EXTERNL int @@ -86,9 +85,9 @@ NC_s3sdkfinalize(void) ncs3_finalized = 1; NCTRACE(11,NULL); Aws::ShutdownAPI(ncs3options); - NCUNTRACE(NC_NOERR); + } - return 1; + return NCUNTRACE(NC_NOERR); } static char* diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index b40447faa..2c2af85fb 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -5,7 +5,7 @@ * @file * @internal The netCDF-4 file functions relating to file creation. * - * @author Ed Hartnett + * @author Ed Hartnett, Mark Harfouche, Dennis Heimbigner */ #include "config.h" @@ -14,11 +14,6 @@ #include "ncpathmgr.h" #include "hdf5internal.h" -/* From hdf5file.c. */ -extern size_t nc4_chunk_cache_size; -extern size_t nc4_chunk_cache_nelems; -extern float nc4_chunk_cache_preemption; - /** @internal These flags may not be set for create. */ static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_CDF5); @@ -157,12 +152,22 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, /* Only set cache for non-parallel creates. */ if (!nc4_info->parallel) { - if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, - nc4_chunk_cache_preemption) < 0) + NCglobalstate* gs = NC_getglobalstate(); + if (H5Pset_cache(fapl_id, 0, gs->chunkcache.nelems, gs->chunkcache.size, + gs->chunkcache.preemption) < 0) BAIL(NC_EHDFERR); LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f", - __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems, - nc4_chunk_cache_preemption)); + __func__, gs->chunkcache.size, gs->chunkcache.nelems, + gs->chunkcache.preemption)); + } + + { + NCglobalstate* gs = NC_getglobalstate(); + if(gs->alignment.defined) { + if (H5Pset_alignment(fapl_id, gs->alignment.threshold, gs->alignment.alignment) < 0) { + BAIL(NC_EHDFERR); + } + } } /* Set HDF5 format compatibility in the FILE ACCESS property list. diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index dc597eb2b..972508dcd 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -39,12 +39,6 @@ h5catch(void* ignored) } #endif -/* These are the default chunk cache sizes for HDF5 files created or - * opened with netCDF-4. */ -extern size_t nc4_chunk_cache_size; -extern size_t nc4_chunk_cache_nelems; -extern float nc4_chunk_cache_preemption; - #ifdef LOGGING /* This is the severity level of messages which will be logged. Use severity 0 for errors, 1 for important log messages, 2 for less diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index f301d63e3..1fe4bed9d 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -59,11 +59,6 @@ static const int nc_type_size_g[NUM_TYPES] = {sizeof(char), sizeof(char), sizeof /** @internal These flags may not be set for open mode. */ static const int ILLEGAL_OPEN_FLAGS = (NC_MMAP); -/* From libsrc4, these are the netcdf-4 cache sizes. */ -extern size_t nc4_chunk_cache_size; -extern size_t nc4_chunk_cache_nelems; -extern float nc4_chunk_cache_preemption; - /* From nc4mem.c */ extern int NC4_open_image_file(NC_FILE_INFO_T* h5); @@ -813,12 +808,22 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) /* Only set cache for non-parallel opens. */ if (!nc4_info->parallel) { - if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, - nc4_chunk_cache_preemption) < 0) + NCglobalstate* gs = NC_getglobalstate(); + if (H5Pset_cache(fapl_id, 0, gs->chunkcache.nelems, gs->chunkcache.size, + gs->chunkcache.preemption) < 0) BAIL(NC_EHDFERR); LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f", - __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems, - nc4_chunk_cache_preemption)); + __func__, gs->chunkcache.size, gs->chunkcache.nelems, + gs->chunkcache.preemption)); + } + + { + NCglobalstate* gs = NC_getglobalstate(); + if(gs->alignment.defined) { + if (H5Pset_alignment(fapl_id, gs->alignment.threshold, gs->alignment.alignment) < 0) { + BAIL(NC_EHDFERR); + } + } } /* Set HDF5 format compatibility in the FILE ACCESS property list. @@ -1453,10 +1458,10 @@ nc4_get_var_meta(NC_VAR_INFO_T *var) BAIL(NC_EVARMETA); /* Learn about current chunk cache settings. */ - if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems), - &(var->chunk_cache_size), &rdcc_w0)) < 0) + if ((H5Pget_chunk_cache(access_pid, &(var->chunkcache.nelems), + &(var->chunkcache.size), &rdcc_w0)) < 0) BAIL(NC_EHDFERR); - var->chunk_cache_preemption = rdcc_w0; + var->chunkcache.preemption = rdcc_w0; /* Get the dataset creation properties. */ if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index d1eedcc73..fd3d06deb 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -85,9 +85,9 @@ nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) if ((access_pid = H5Pcreate(H5P_DATASET_ACCESS)) < 0) return NC_EHDFERR; - if (H5Pset_chunk_cache(access_pid, var->chunk_cache_nelems, - var->chunk_cache_size, - var->chunk_cache_preemption) < 0) + if (H5Pset_chunk_cache(access_pid, var->chunkcache.nelems, + var->chunkcache.size, + var->chunkcache.preemption) < 0) return NC_EHDFERR; if (H5Dclose(hdf5_var->hdf_datasetid) < 0) return NC_EHDFERR; @@ -2314,9 +2314,9 @@ NC4_HDF5_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, assert(var && var->hdr.id == varid); /* Set the values. */ - var->chunk_cache_size = size; - var->chunk_cache_nelems = nelems; - var->chunk_cache_preemption = preemption; + var->chunkcache.size = size; + var->chunkcache.nelems = nelems; + var->chunkcache.preemption = preemption; /* Reopen the dataset to bring new settings into effect. */ if ((retval = nc4_reopen_dataset(grp, var))) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 819c056e4..9c93bf861 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -977,9 +977,9 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid } /* Set per-var chunk cache, for chunked datasets. */ - if (var->storage == NC_CHUNKED && var->chunk_cache_size) - if (H5Pset_chunk_cache(access_plistid, var->chunk_cache_nelems, - var->chunk_cache_size, var->chunk_cache_preemption) < 0) + if (var->storage == NC_CHUNKED && var->chunkcache.size) + if (H5Pset_chunk_cache(access_plistid, var->chunkcache.nelems, + var->chunkcache.size, var->chunkcache.preemption) < 0) BAIL(NC_EHDFERR); /* At long last, create the dataset. */ @@ -1101,12 +1101,12 @@ nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) /* If the chunk cache is too small, and the user has not changed * the default value of the chunk cache size, then increase the * size of the cache. */ - if (var->chunk_cache_size == CHUNK_CACHE_SIZE) - if (chunk_size_bytes > var->chunk_cache_size) + if (var->chunkcache.size == CHUNK_CACHE_SIZE) + if (chunk_size_bytes > var->chunkcache.size) { - var->chunk_cache_size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE; - if (var->chunk_cache_size > MAX_DEFAULT_CACHE_SIZE) - var->chunk_cache_size = MAX_DEFAULT_CACHE_SIZE; + var->chunkcache.size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE; + if (var->chunkcache.size > MAX_DEFAULT_CACHE_SIZE) + var->chunkcache.size = MAX_DEFAULT_CACHE_SIZE; if ((retval = nc4_reopen_dataset(grp, var))) return retval; } diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 6980c1785..018842d09 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -18,12 +18,6 @@ #include "zincludes.h" #include "zfilter.h" -/* These are the default chunk cache sizes for ZARR files created or - * opened with netCDF-4. */ -extern size_t ncz_chunk_cache_size; -extern size_t ncz_chunk_cache_nelems; -extern float ncz_chunk_cache_preemption; - /* Forward */ #ifdef LOGGING @@ -61,10 +55,10 @@ NCZ_initialize_internal(void) { int stat = NC_NOERR; char* dimsep = NULL; - NCRCglobalstate* ngs = NULL; + NCglobalstate* ngs = NULL; ncz_initialized = 1; - ngs = ncrc_getglobalstate(); + ngs = NC_getglobalstate(); if(ngs != NULL) { /* Defaults */ ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR; diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index fa864091b..4ca278168 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -1534,7 +1534,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) } /* Capture dimension_separator (must precede chunk cache creation) */ { - NCRCglobalstate* ngs = ncrc_getglobalstate(); + NCglobalstate* ngs = NC_getglobalstate(); assert(ngs != NULL); zvar->dimension_separator = 0; if((stat = NCJdictget(jvar,"dimension_separator",&jvalue))) goto done; diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index d4325be8f..452684057 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -381,7 +381,7 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims, zvar->common.file = h5; zvar->scalar = (ndims == 0 ? 1 : 0); - zvar->dimension_separator = ncrc_getglobalstate()->zarr.dimension_separator; + zvar->dimension_separator = NC_getglobalstate()->zarr.dimension_separator; assert(zvar->dimension_separator != 0); /* Set these state flags for the var. */ @@ -456,9 +456,9 @@ var->type_info->rc++; zvar->chunksize = zvar->chunkproduct * var->type_info->size; /* Override the cache setting to use NCZarr defaults */ - var->chunk_cache_size = CHUNK_CACHE_SIZE_NCZARR; - var->chunk_cache_nelems = ceildiv(var->chunk_cache_size,zvar->chunksize); - var->chunk_cache_preemption = 1; /* not used */ + var->chunkcache.size = CHUNK_CACHE_SIZE_NCZARR; + var->chunkcache.nelems = ceildiv(var->chunkcache.size,zvar->chunksize); + var->chunkcache.preemption = 1; /* not used */ /* Create the cache */ if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,zvar->dimension_separator,&zvar->cache))) diff --git a/libnczarr/zxcache.c b/libnczarr/zxcache.c index 52a8c7d35..52f8abf2e 100644 --- a/libnczarr/zxcache.c +++ b/libnczarr/zxcache.c @@ -77,9 +77,9 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl assert(zvar != NULL && zvar->cache != NULL); /* Set the values. */ - var->chunk_cache_size = cachesize; - var->chunk_cache_nelems = nelems; - var->chunk_cache_preemption = preemption; + var->chunkcache.size = cachesize; + var->chunkcache.nelems = nelems; + var->chunkcache.preemption = preemption; /* Fix up cache */ if((retval = NCZ_adjust_var_cache(var))) goto done; @@ -120,8 +120,8 @@ fprintf(stderr,"xxx: adjusting cache for: %s\n",var->hdr.name); /* Reclaim any existing fill_chunk */ if((stat = NCZ_reclaim_fill_chunk(zcache))) goto done; /* Reset the parameters */ - zvar->cache->maxsize = var->chunk_cache_size; - zvar->cache->maxentries = var->chunk_cache_nelems; + zvar->cache->maxsize = var->chunkcache.size; + zvar->cache->maxentries = var->chunkcache.nelems; #ifdef DEBUG fprintf(stderr,"%s.cache.adjust: size=%ld nelems=%ld\n", var->hdr.name,(unsigned long)zvar->cache->maxsize,(unsigned long)zvar->cache->maxentries); diff --git a/libsrc4/nc4cache.c b/libsrc4/nc4cache.c index 9a7edec48..f41b327a3 100644 --- a/libsrc4/nc4cache.c +++ b/libsrc4/nc4cache.c @@ -13,12 +13,6 @@ #include "config.h" #include "nc4internal.h" -/* These are the default chunk cache sizes for HDF5 files created or - * opened with netCDF-4. */ -extern size_t nc4_chunk_cache_size; -extern size_t nc4_chunk_cache_nelems; -extern float nc4_chunk_cache_preemption; - /** * Set chunk cache size. Only affects netCDF-4/HDF5 files * opened/created *after* it is called. @@ -71,11 +65,12 @@ extern float nc4_chunk_cache_preemption; int nc_set_chunk_cache(size_t size, size_t nelems, float preemption) { + NCglobalstate* gs = NC_getglobalstate(); if (preemption < 0 || preemption > 1) return NC_EINVAL; - nc4_chunk_cache_size = size; - nc4_chunk_cache_nelems = nelems; - nc4_chunk_cache_preemption = preemption; + gs->chunkcache.size = size; + gs->chunkcache.nelems = nelems; + gs->chunkcache.preemption = preemption; return NC_NOERR; } @@ -99,14 +94,15 @@ nc_set_chunk_cache(size_t size, size_t nelems, float preemption) int nc_get_chunk_cache(size_t *sizep, size_t *nelemsp, float *preemptionp) { + NCglobalstate* gs = NC_getglobalstate(); if (sizep) - *sizep = nc4_chunk_cache_size; + *sizep = gs->chunkcache.size; if (nelemsp) - *nelemsp = nc4_chunk_cache_nelems; + *nelemsp = gs->chunkcache.nelems; if (preemptionp) - *preemptionp = nc4_chunk_cache_preemption; + *preemptionp = gs->chunkcache.preemption; return NC_NOERR; } @@ -128,11 +124,12 @@ nc_get_chunk_cache(size_t *sizep, size_t *nelemsp, float *preemptionp) int nc_set_chunk_cache_ints(int size, int nelems, int preemption) { + NCglobalstate* gs = NC_getglobalstate(); if (size <= 0 || nelems <= 0 || preemption < 0 || preemption > 100) return NC_EINVAL; - nc4_chunk_cache_size = size; - nc4_chunk_cache_nelems = nelems; - nc4_chunk_cache_preemption = (float)preemption / 100; + gs->chunkcache.size = size; + gs->chunkcache.nelems = nelems; + gs->chunkcache.preemption = (float)preemption / 100; return NC_NOERR; } @@ -154,12 +151,13 @@ nc_set_chunk_cache_ints(int size, int nelems, int preemption) int nc_get_chunk_cache_ints(int *sizep, int *nelemsp, int *preemptionp) { + NCglobalstate* gs = NC_getglobalstate(); if (sizep) - *sizep = (int)nc4_chunk_cache_size; + *sizep = (int)gs->chunkcache.size; if (nelemsp) - *nelemsp = (int)nc4_chunk_cache_nelems; + *nelemsp = (int)gs->chunkcache.nelems; if (preemptionp) - *preemptionp = (int)(nc4_chunk_cache_preemption * 100); + *preemptionp = (int)(gs->chunkcache.preemption * 100); return NC_NOERR; } diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index c4377c3aa..d90c51da9 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -22,6 +22,7 @@ #include "nc.h" /* from libsrc */ #include "ncdispatch.h" /* from libdispatch */ #include "ncutf8.h" +#include "ncrc.h" /** @internal Number of reserved attributes. These attributes are * hidden from the netcdf user, but exist in the implementation @@ -51,11 +52,6 @@ static const NC_reservedatt NC_reserved[] = { }; #define NRESERVED (sizeof(NC_reserved) / sizeof(NC_reservedatt)) /*|NC_reservedatt|*/ -/* These hold the file caching settings for the library. */ -size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE; /**< Default chunk cache size. */ -size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */ -float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */ - static int NC4_move_in_NCList(NC* nc, int new_id); #ifdef LOGGING @@ -670,6 +666,7 @@ int nc4_var_list_add2(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var) { NC_VAR_INFO_T *new_var = NULL; + NCglobalstate* gs = NC_getglobalstate(); /* Allocate storage for new variable. */ if (!(new_var = calloc(1, sizeof(NC_VAR_INFO_T)))) @@ -678,9 +675,9 @@ nc4_var_list_add2(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var) new_var->container = grp; /* These are the HDF5-1.8.4 defaults. */ - new_var->chunk_cache_size = nc4_chunk_cache_size; - new_var->chunk_cache_nelems = nc4_chunk_cache_nelems; - new_var->chunk_cache_preemption = nc4_chunk_cache_preemption; + new_var->chunkcache.size = gs->chunkcache.size; + new_var->chunkcache.nelems = gs->chunkcache.nelems; + new_var->chunkcache.preemption = gs->chunkcache.preemption; /* Now fill in the values in the var info structure. */ new_var->hdr.id = ncindexsize(grp->vars); @@ -1951,3 +1948,79 @@ NC4_move_in_NCList(NC* nc, int new_id) return stat; } +/**************************************************/ +/* NCglobal state management */ + +static NCglobalstate* nc_globalstate = NULL; + +static int +NC_createglobalstate(void) +{ + int stat = NC_NOERR; + const char* tmp = NULL; + + if(nc_globalstate == NULL) { + nc_globalstate = calloc(1,sizeof(NCglobalstate)); + } + /* Initialize struct pointers */ + nc_globalstate->rcinfo = (struct NCRCinfo*)calloc(1,sizeof(struct NCRCinfo)); + if(nc_globalstate == NULL) return NC_ENOMEM; + + /* Get environment variables */ + if(getenv(NCRCENVIGNORE) != NULL) + nc_globalstate->rcinfo->ignore = 1; + tmp = getenv(NCRCENVRC); + if(tmp != NULL && strlen(tmp) > 0) + nc_globalstate->rcinfo->rcfile = strdup(tmp); + /* Initialize chunk cache defaults */ + nc_globalstate->chunkcache.size = CHUNK_CACHE_SIZE; /**< Default chunk cache size. */ + nc_globalstate->chunkcache.nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */ + nc_globalstate->chunkcache.preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */ + + return stat; +} + +/* Get global state */ +NCglobalstate* +NC_getglobalstate(void) +{ + if(nc_globalstate == NULL) + NC_createglobalstate(); + return nc_globalstate; +} + +void +NC_freeglobalstate(void) +{ + if(nc_globalstate != NULL) { + nullfree(nc_globalstate->tempdir); + nullfree(nc_globalstate->home); + nullfree(nc_globalstate->cwd); + NC_rcclear(nc_globalstate->rcinfo); + free(nc_globalstate->rcinfo); + free(nc_globalstate); + nc_globalstate = NULL; + } +} +/**************************************************/ +/* Specific property functions */ + +/** \internal Set global parameters for H5Pset_alignment */ +int +nc_set_alignment(int threshold, int alignment) +{ + NCglobalstate* gs = NC_getglobalstate(); + gs->alignment.threshold = threshold; + gs->alignment.alignment = alignment; + gs->alignment.defined = 1; + return NC_NOERR; +} + +int +nc_get_alignment(int* thresholdp, int* alignmentp) +{ + NCglobalstate* gs = NC_getglobalstate(); + if(thresholdp) *thresholdp = gs->alignment.threshold; + if(alignmentp) *alignmentp = gs->alignment.alignment; + return NC_NOERR; +} diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index c6c241059..fac6520bf 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -97,11 +97,11 @@ NC4_get_var_chunk_cache(int ncid, int varid, size_t *sizep, /* Give the user what they want. */ if (sizep) - *sizep = var->chunk_cache_size; + *sizep = var->chunkcache.size; if (nelemsp) - *nelemsp = var->chunk_cache_nelems; + *nelemsp = var->chunkcache.nelems; if (preemptionp) - *preemptionp = var->chunk_cache_preemption; + *preemptionp = var->chunkcache.preemption; return NC_NOERR; } diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index f63e83938..e14b6e29f 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -24,6 +24,10 @@ SET(NC4_TESTS tst_dims tst_dims2 tst_dims3 tst_files tst_files4 tst_hdf5_file_compat tst_fill_attr_vanish tst_rehash tst_types tst_bug324 tst_atts3 tst_put_vars tst_elatefill tst_udf tst_bug1442 tst_quantize) +IF(HAS_PAR_FILTERS) +SET(NC4_tests $NC4_TESTS tst_alignment) +ENDIF() + # Note, renamegroup needs to be compiled before run_grp_rename IF(BUILD_UTILITIES) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index c8c3a0c61..65abfad58 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -37,6 +37,10 @@ tst_rehash tst_filterparser tst_bug324 tst_types tst_atts3 \ tst_put_vars tst_elatefill tst_udf tst_put_vars_two_unlim_dim \ tst_bug1442 tst_quantize +if HAS_PAR_FILTERS +NC4_TESTS += tst_alignment +endif + # Temporary I hoped, but hoped in vain. if !ISCYGWIN NC4_TESTS += tst_h_strbug tst_h_refs diff --git a/nc_test4/tst_alignment.c b/nc_test4/tst_alignment.c new file mode 100644 index 000000000..385df0229 --- /dev/null +++ b/nc_test4/tst_alignment.c @@ -0,0 +1,96 @@ +/* This is part of the netCDF package. + Copyright 2018 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Test HDF5 alignment + Dennis Heimbigner +*/ + +#include +#include "err_macros.h" + +#include +#include + +#undef DEBUG + +#define THRESHOLD 512 +#define ALIGNMENT 4096 +#define CHUNKSIZE 513 + +int +main(int argc, char **argv) +{ + int i,ncid, varid, dimids[1]; + size_t chunks[1]; + unsigned char data[CHUNKSIZE]; + hid_t fileid, grpid, datasetid; + hid_t dxpl_id = H5P_DEFAULT; /*data transfer property list */ + unsigned int filter_mask = 0; + hsize_t hoffset[1]; + haddr_t addr; + hsize_t size ; + hid_t fspace; + + H5Eset_auto2(H5E_DEFAULT,(H5E_auto2_t)H5Eprint1,stderr); + + printf("\n*** Testing HDF5 alignment.\n"); + + printf("chunksize=%d threshold=%d alignment=%d\n",CHUNKSIZE,THRESHOLD,ALIGNMENT); + + if(nc_set_alignment(THRESHOLD,ALIGNMENT)) ERR; + if (nc_create("tst_alignment.nc", NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, "d0", CHUNKSIZE, &dimids[0])) ERR; + if (nc_def_var(ncid, "var", NC_UBYTE, 1, dimids, &varid)) ERR; + chunks[0] = CHUNKSIZE; + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunks)) ERR; + if (nc_enddef(ncid)) ERR; + + for(i=0;i #endif -#define DEBUG +#undef DEBUG #include "netcdf.h" #include "nctestserver.h" diff --git a/ncdump/tst_chunking.c b/ncdump/tst_chunking.c index 95cd92010..46c9ddc5b 100644 --- a/ncdump/tst_chunking.c +++ b/ncdump/tst_chunking.c @@ -9,7 +9,7 @@ #include #include "err_macros.h" -#define DEBUG +#undef DEBUG static int ret = NC_NOERR; diff --git a/ncdump/tst_nccopy3.sh b/ncdump/tst_nccopy3.sh index 0436392b4..88bf94928 100755 --- a/ncdump/tst_nccopy3.sh +++ b/ncdump/tst_nccopy3.sh @@ -8,6 +8,8 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh +export SETX=1 + set -e echo "" diff --git a/ncdump/tst_rcmerge.c b/ncdump/tst_rcmerge.c index f04a42320..daa3c7bbb 100644 --- a/ncdump/tst_rcmerge.c +++ b/ncdump/tst_rcmerge.c @@ -3,20 +3,21 @@ #include #include "netcdf.h" #include "ncrc.h" +#include "nc4internal.h" int main(int argc, char** argv) { size_t i,nentries = 0; - NCRCglobalstate* ngs = ncrc_getglobalstate(); + NCglobalstate* ngs = NC_getglobalstate(); NCRCinfo* info = NULL; NCRCentry* entry = NULL; /* Cause the .rc files to be read and merged */ nc_initialize(); - if((ngs = ncrc_getglobalstate())==NULL) abort(); - info = &ngs->rcinfo; + if((ngs = NC_getglobalstate())==NULL) abort(); + info = ngs->rcinfo; if(info->ignore) { fprintf(stderr,".rc ignored\n"); diff --git a/ncdump/tst_unicode.c b/ncdump/tst_unicode.c index e742260db..a3369559b 100644 --- a/ncdump/tst_unicode.c +++ b/ncdump/tst_unicode.c @@ -23,7 +23,7 @@ #include #endif -#define DEBUG +#undef DEBUG /* The data file we will create. */ static const unsigned char prefix[] = { diff --git a/ncgen/dump.c b/ncgen/dump.c index c4e493dc4..bd4d775b8 100644 --- a/ncgen/dump.c +++ b/ncgen/dump.c @@ -8,7 +8,7 @@ #include "includes.h" #include "dump.h" -#define DEBUGSRC +#undef DEBUGSRC #define MAXELEM 8 #define MAXDEPTH 4 diff --git a/nczarr_test/bm_chunks3.c b/nczarr_test/bm_chunks3.c index 1e777bcbf..9a34bda33 100644 --- a/nczarr_test/bm_chunks3.c +++ b/nczarr_test/bm_chunks3.c @@ -167,7 +167,7 @@ main(int argc, char *argv[]) /* variable shapes */ int var_dims[RANK]; - NCCHECK(getoptions(&argc,&argv,&bmoptions)); + NCCHECK(bm_getoptions(&argc,&argv,&bmoptions)); NCCHECK(nc4_buildpath(&bmoptions,&path)); if(bmoptions.debug) { diff --git a/nczarr_test/bm_utils.h b/nczarr_test/bm_utils.h index 52103225a..1537116ed 100644 --- a/nczarr_test/bm_utils.h +++ b/nczarr_test/bm_utils.h @@ -89,6 +89,10 @@ EXTERNL void bm_reportmetaoptions(struct BMMeta* o); EXTERNL const char* bm_printvector(int rank, const size_t* vec); EXTERNL const char* bm_printvectorp(int rank, const ptrdiff_t* vec); EXTERNL const char* bm_varasprint(int rank, const size_t* start, const size_t* edges, const ptrdiff_t* stride); +EXTERNL int nc4_buildpath(struct BMOptions* o, char** pathp); +EXTERNL void reportoptions(struct BMOptions* o); +EXTERNL void reportmetaoptions(struct BMMeta* o); +EXTERNL void clearoptions(struct BMOptions*); #include "ut_test.h" diff --git a/nczarr_test/s3util.c b/nczarr_test/s3util.c index 03ec172a2..07e03ef1c 100644 --- a/nczarr_test/s3util.c +++ b/nczarr_test/s3util.c @@ -27,7 +27,7 @@ #undef NODELETE -#define DEBUG +#undef DEBUG #define DATANAME "data" diff --git a/nczarr_test/tst_zchunks3.c b/nczarr_test/tst_zchunks3.c index b3d238d88..fa0c57519 100644 --- a/nczarr_test/tst_zchunks3.c +++ b/nczarr_test/tst_zchunks3.c @@ -8,7 +8,7 @@ #include "ut_includes.h" #include "test_nczarr_utils.h" -#define DEBUG +#undef DEBUG static int ret = NC_NOERR; #define FILE_NAME "tmp_chunks3.nc" diff --git a/nczarr_test/ut_json.c b/nczarr_test/ut_json.c index 22600fcb5..37ab65d23 100644 --- a/nczarr_test/ut_json.c +++ b/nczarr_test/ut_json.c @@ -5,7 +5,7 @@ #include "ut_includes.h" -#define DEBUG +#undef DEBUG typedef enum Cmds { cmd_none = 0, diff --git a/nczarr_test/zhex.c b/nczarr_test/zhex.c index 3a49ff649..972407cb4 100644 --- a/nczarr_test/zhex.c +++ b/nczarr_test/zhex.c @@ -12,7 +12,7 @@ #include #endif -#define DEBUG +#undef DEBUG static char hex[16] = "0123456789abcdef"; diff --git a/nczarr_test/zmapio.c b/nczarr_test/zmapio.c index b1e06295e..c9c982c88 100644 --- a/nczarr_test/zmapio.c +++ b/nczarr_test/zmapio.c @@ -23,7 +23,7 @@ #include "nclog.h" #include "ncuri.h" -#define DEBUG +#undef DEBUG #define DATANAME "data" diff --git a/nczarr_test/zs3parse.c b/nczarr_test/zs3parse.c index e1dba30ca..7286dd75a 100644 --- a/nczarr_test/zs3parse.c +++ b/nczarr_test/zs3parse.c @@ -21,7 +21,7 @@ #include "zincludes.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG #define AWSHOST ".amazonaws.com" diff --git a/oc2/ocinternal.c b/oc2/ocinternal.c index eba1288fa..fbaf49fba 100644 --- a/oc2/ocinternal.c +++ b/oc2/ocinternal.c @@ -25,7 +25,7 @@ #include -#include "ncrc.h" +#include "nc4internal.h" #include "ocinternal.h" #include "ocdebug.h" #include "occurlfunctions.h" @@ -321,7 +321,7 @@ createtempfile(OCstate* state, OCtree* tree) char* path = NULL; char* tmppath = NULL; int len; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCglobalstate* globalstate = NC_getglobalstate(); len = strlen(globalstate->tempdir) @@ -526,7 +526,7 @@ static OCerror ocset_curlproperties(OCstate* state) { OCerror stat = OC_NOERR; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCglobalstate* globalstate = NC_getglobalstate(); if(state->auth->curlflags.useragent == NULL) { size_t len = strlen(DFALTUSERAGENT) + strlen(VERSION) + 1; diff --git a/unit_test/test_aws.c b/unit_test/test_aws.c index 826d3f82c..8dd9e9276 100644 --- a/unit_test/test_aws.c +++ b/unit_test/test_aws.c @@ -15,7 +15,7 @@ Test the handling of aws profiles and regions. #include "ncrc.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG typedef struct ProfileTest { const char* profile; diff --git a/unit_test/test_pathcvt.c b/unit_test/test_pathcvt.c index 7f1d67b6b..7426e5495 100644 --- a/unit_test/test_pathcvt.c +++ b/unit_test/test_pathcvt.c @@ -13,7 +13,7 @@ Test the NCpathcvt #include "netcdf.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG #define NKINDS 4 static const int kinds[NKINDS] = {NCPD_NIX,NCPD_MSYS,NCPD_CYGWIN,NCPD_WIN};