diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 49451a274..70bf7c3fe 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -9,6 +9,7 @@ This file contains a high-level description of this package's evolution. Release * [Bug Fix] Require that the type of the variable in nc_def_var_filter is not variable length. See [Github #/2231](https://github.com/Unidata/netcdf-c/pull/2231). * [File Change] Apply HDF5 v1.8 format compatibility when writing to previous files, as well as when creating new files. The superblock version remains at 2 for newly created files. Full backward read/write compatibility for netCDF-4 is maintained in all cases. See [Github #2176](https://github.com/Unidata/netcdf-c/issues/2176). +* [Enhancement] Add ability to set dataset alignment for netcdf-4/HDF5 files. See [Github #2206](https://github.com/Unidata/netcdf-c/pull/2206). * [Enhancement] Add complete bitgroom support to NCZarr. See [Github #2197](https://github.com/Unidata/netcdf-c/pull/2197). * [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179](https://github.com/Unidata/netcdf-c/pull/2179). * [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183). diff --git a/acinclude.m4 b/acinclude.m4 index a3fdc712a..b6420954a 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -384,4 +384,6 @@ MOSTLYCLEANFILES += $(valgrind_log_files) AC_SUBST([VALGRIND_CHECK_RULES]) m4_ifdef([_AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE([VALGRIND_CHECK_RULES])]) ]) + AC_DEFUN([_AC_FINALIZE],[]) + diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index f525ac33a..fcb2b3368 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -781,7 +781,6 @@ INPUT = \ @abs_top_srcdir@/libdispatch/derror.c \ @abs_top_srcdir@/libdispatch/dv2i.c \ @abs_top_srcdir@/libdispatch/dcopy.c \ - @abs_top_srcdir@/libdispatch/daux.c \ @abs_top_srcdir@/libsrc4/nc4var.c \ @abs_top_srcdir@/libhdf5/nc4hdf.c \ @abs_top_srcdir@/libsrc4/nc4internal.c \ diff --git a/include/nc4internal.h b/include/nc4internal.h index 3be336439..8d8906032 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -118,6 +118,7 @@ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t; /* Forward declarations. */ struct NC_GRP_INFO; struct NC_TYPE_INFO; +struct NCRCinfo; /** * This struct provides indexed Access to Meta-data objects. See the @@ -205,9 +206,11 @@ typedef struct NC_VAR_INFO int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */ nc_bool_t shuffle; /**< True if var has shuffle filter applied. */ nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */ - size_t chunk_cache_size; /**< Size in bytes of the var chunk cache. */ - size_t chunk_cache_nelems; /**< Number of slots in var chunk cache. */ - float chunk_cache_preemption; /**< Chunk cache preemtion policy. */ + struct ChunkCache { + size_t size; /**< Size in bytes of the var chunk cache. */ + size_t nelems; /**< Number of slots in var chunk cache. */ + float preemption; /**< Chunk cache preemtion policy. */ + } chunkcache; int quantize_mode; /**< Quantize mode. NC_NOQUANTIZE is 0, and means no quantization. */ int nsd; /**< Number of significant digits if quantization is used, 0 if not. */ void *format_var_info; /**< Pointer to any binary format info. */ @@ -327,6 +330,24 @@ typedef struct NC_FILE_INFO } mem; } NC_FILE_INFO_T; +/* Collect global state info in one place */ +typedef struct NCglobalstate { + int initialized; + char* tempdir; /* track a usable temp dir */ + char* home; /* track $HOME */ + char* cwd; /* track getcwd */ + struct NCRCinfo* rcinfo; /* Currently only one rc file per session */ + struct GlobalZarr { /* Zarr specific parameters */ + char dimension_separator; + } zarr; + struct Alignment { /* H5Pset_alignment parameters */ + int defined; /* 1 => threshold and alignment explicitly set */ + int threshold; + int alignment; + } alignment; + struct ChunkCache chunkcache; +} NCglobalstate; + /** Variable Length Datatype struct in memory. Must be identical to * HDF5 hvl_t. (This is only used for VL sequences, not VL strings, * which are stored in char *'s) */ @@ -460,6 +481,10 @@ extern const char* nc4_atomic_name[NUM_ATOMIC_TYPES]; /* Binary searcher for reserved attributes */ extern const NC_reservedatt* NC_findreserved(const char* name); +/* Global State Management */ +extern NCglobalstate* NC_getglobalstate(void); +extern void NC_freeglobalstate(void); + /* Generic reserved Attributes */ #define NC_ATT_REFERENCE_LIST "REFERENCE_LIST" #define NC_ATT_CLASS "CLASS" diff --git a/include/ncrc.h b/include/ncrc.h index 1b3228674..14c1c219a 100644 --- a/include/ncrc.h +++ b/include/ncrc.h @@ -33,39 +33,25 @@ typedef struct NCRCentry { char* value; } NCRCentry; -/* collect all the relevant info around the rc file */ -typedef struct NCRCinfo { - int ignore; /* if 1, then do not use any rc file */ - int loaded; /* 1 => already loaded */ - NClist* entries; /* the rc file entry store fields*/ - char* rcfile; /* specified rcfile; overrides anything else */ - char* rchome; /* Overrides $HOME when looking for .rc files */ -} NCRCinfo; - -/* Collect global state info in one place */ -typedef struct NCRCglobalstate { - int initialized; - char* tempdir; /* track a usable temp dir */ - char* home; /* track $HOME */ - char* cwd; /* track getcwd */ - NCRCinfo rcinfo; /* Currently only one rc file per session */ - struct GlobalZarr { /* Zarr specific parameters */ - char dimension_separator; - } zarr; - struct S3credentials { - NClist* profiles; /* NClist */ - } s3creds; -} NCRCglobalstate; +struct AWSentry { + char* key; + char* value; +}; struct AWSprofile { char* name; NClist* entries; /* NClist */ }; -struct AWSentry { - char* key; - char* value; -}; +/* collect all the relevant info around the rc file and AWS */ +typedef struct NCRCinfo { + int ignore; /* if 1, then do not use any rc file */ + int loaded; /* 1 => already loaded */ + NClist* entries; /* the rc file entry store fields*/ + char* rcfile; /* specified rcfile; overrides anything else */ + char* rchome; /* Overrides $HOME when looking for .rc files */ + NClist* s3profiles; /* NClist */ +} NCRCinfo; typedef struct NCS3INFO { char* host; /* non-null if other*/ @@ -80,9 +66,7 @@ extern "C" { #endif /* From drc.c */ -EXTERNL int ncrc_createglobalstate(void); EXTERNL void ncrc_initialize(void); -EXTERNL void ncrc_freeglobalstate(void); EXTERNL int NC_rcfile_insert(const char* key, const char* value, const char* hostport, const char* path); EXTERNL char* NC_rclookup(const char* key, const char* hostport, const char* path); EXTERNL char* NC_rclookupx(NCURI* uri, const char* key); @@ -94,7 +78,6 @@ EXTERNL size_t NC_rcfile_length(NCRCinfo*); EXTERNL NCRCentry* NC_rcfile_ith(NCRCinfo*,size_t); /* For internal use */ -EXTERNL NCRCglobalstate* ncrc_getglobalstate(void); EXTERNL void NC_rcclear(NCRCinfo* info); EXTERNL void NC_rcclear(NCRCinfo* info); diff --git a/include/netcdf.h b/include/netcdf.h index b8540c6ab..0816b12bb 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -560,6 +560,14 @@ nc_def_user_format(int mode_flag, NC_Dispatch *dispatch_table, char *magic_numbe EXTERNL int nc_inq_user_format(int mode_flag, NC_Dispatch **dispatch_table, char *magic_number); +/* Set the global alignment property */ +EXTERNL int +nc_set_alignment(int threshold, int alignment); + +/* Get the global alignment property */ +EXTERNL int +nc_get_alignment(int* thresholdp, int* alignmentp); + EXTERNL int nc__create(const char *path, int cmode, size_t initialsz, size_t *chunksizehintp, int *ncidp); diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 1c9404010..5ad408824 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -367,7 +367,7 @@ set_curl_properties(NCD4INFO* d4info) char* newpath = NULL; int len; errno = 0; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCglobalstate* globalstate = NC_getglobalstate(); /* Create the unique cookie file name */ len = diff --git a/libdispatch/ddispatch.c b/libdispatch/ddispatch.c index 12ba32995..dc41b45fa 100644 --- a/libdispatch/ddispatch.c +++ b/libdispatch/ddispatch.c @@ -12,6 +12,7 @@ See LICENSE.txt for license information. #include "ncoffsets.h" #include "ncpathmgr.h" #include "ncxml.h" +#include "nc4internal.h" /* Required for getcwd, other functions. */ #ifdef HAVE_UNISTD_H @@ -48,7 +49,7 @@ NCDISPATCH_initialize(void) { int status = NC_NOERR; int i; - NCRCglobalstate* globalstate = NULL; + NCglobalstate* globalstate = NULL; for(i=0;ircinfo.ignore = 1; - tmp = getenv(NCRCENVRC); - if(tmp != NULL && strlen(tmp) > 0) - ncrc_globalstate->rcinfo.rcfile = strdup(tmp); - return stat; -} - /* Initialize defaults and load: * .ncrc @@ -111,6 +91,7 @@ void ncrc_initialize(void) { int stat = NC_NOERR; + NCglobalstate* ncg = NULL; if(NCRCinitialized) return; NCRCinitialized = 1; /* prevent recursion */ @@ -121,7 +102,8 @@ ncrc_initialize(void) nclog(NCLOGWARN,".rc loading failed"); } /* Load .aws/config */ - if((stat = aws_load_credentials(ncrc_globalstate))) { + ncg = NC_getglobalstate(); + if((stat = aws_load_credentials(ncg))) { nclog(NCLOGWARN,"AWS config file not loaded"); } #endif @@ -131,35 +113,13 @@ static void ncrc_setrchome(void) { const char* tmp = NULL; - if(ncrc_globalstate->rcinfo.rchome) return; - assert(ncrc_globalstate && ncrc_globalstate->home); + NCglobalstate* ncg = NC_getglobalstate(); + assert(ncg && ncg->home); + if(ncg->rcinfo->rchome) return; tmp = getenv(NCRCENVHOME); if(tmp == NULL || strlen(tmp) == 0) - tmp = ncrc_globalstate->home; - ncrc_globalstate->rcinfo.rchome = strdup(tmp); -} - -/* Get global state */ -NCRCglobalstate* -ncrc_getglobalstate(void) -{ - if(ncrc_globalstate == NULL) - ncrc_createglobalstate(); - return ncrc_globalstate; -} - -void -ncrc_freeglobalstate(void) -{ - if(ncrc_globalstate != NULL) { - nullfree(ncrc_globalstate->tempdir); - nullfree(ncrc_globalstate->home); - nullfree(ncrc_globalstate->cwd); - NC_rcclear(&ncrc_globalstate->rcinfo); - clearS3credentials(&ncrc_globalstate->s3creds); - free(ncrc_globalstate); - ncrc_globalstate = NULL; - } + tmp = ncg->home; + ncg->rcinfo->rchome = strdup(tmp); } void @@ -169,9 +129,10 @@ NC_rcclear(NCRCinfo* info) nullfree(info->rcfile); nullfree(info->rchome); rcfreeentries(info->entries); + freeprofilelist(info->s3profiles); } -void +static void rcfreeentries(NClist* rc) { int i; @@ -191,18 +152,17 @@ NC_rcload(void) { int i,ret = NC_NOERR; char* path = NULL; - NCRCglobalstate* globalstate = NULL; + NCglobalstate* globalstate = NULL; NClist* rcfileorder = nclistnew(); if(!NCRCinitialized) ncrc_initialize(); - globalstate = ncrc_getglobalstate(); + globalstate = NC_getglobalstate(); - - if(globalstate->rcinfo.ignore) { + if(globalstate->rcinfo->ignore) { nclog(NCLOGDBG,".rc file loading suppressed"); goto done; } - if(globalstate->rcinfo.loaded) goto done; + if(globalstate->rcinfo->loaded) goto done; /* locate the configuration files in order of use: 1. Specified by NCRCENV_RC environment variable. @@ -215,8 +175,8 @@ NC_rcload(void) 6. $CWD/.docsrc Entry in later files override any of the earlier files */ - if(globalstate->rcinfo.rcfile != NULL) { /* always use this */ - nclistpush(rcfileorder,strdup(globalstate->rcinfo.rcfile)); + if(globalstate->rcinfo->rcfile != NULL) { /* always use this */ + nclistpush(rcfileorder,strdup(globalstate->rcinfo->rcfile)); } else { const char** rcname; const char* dirnames[3]; @@ -225,8 +185,7 @@ NC_rcload(void) /* Make sure rcinfo.rchome is defined */ ncrc_setrchome(); - - dirnames[0] = globalstate->rcinfo.rchome; + dirnames[0] = globalstate->rcinfo->rchome; dirnames[1] = globalstate->cwd; dirnames[2] = NULL; @@ -249,7 +208,7 @@ NC_rcload(void) } done: - globalstate->rcinfo.loaded = 1; /* even if not exists */ + globalstate->rcinfo->loaded = 1; /* even if not exists */ nclistfreeall(rcfileorder); return (ret); } @@ -301,7 +260,7 @@ NC_set_rcfile(const char* rcfile) { int stat = NC_NOERR; FILE* f = NULL; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCglobalstate* globalstate = NC_getglobalstate(); if(rcfile != NULL && strlen(rcfile) == 0) rcfile = NULL; @@ -311,8 +270,8 @@ NC_set_rcfile(const char* rcfile) goto done; } fclose(f); - nullfree(globalstate->rcinfo.rcfile); - globalstate->rcinfo.rcfile = strdup(rcfile); + nullfree(globalstate->rcinfo->rcfile); + globalstate->rcinfo->rcfile = strdup(rcfile); /* Clear globalstate->rcinfo */ NC_rcclear(&globalstate->rcinfo); /* (re) load the rcfile and esp the entriestore*/ @@ -414,7 +373,7 @@ rccompile(const char* path) NCbytes* tmp = ncbytesnew(); NCURI* uri = NULL; char* nextline = NULL; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCglobalstate* globalstate = NC_getglobalstate(); char* bucket = NULL; if((ret=NC_readfile(path,tmp))) { @@ -424,10 +383,10 @@ rccompile(const char* path) contents = ncbytesextract(tmp); if(contents == NULL) contents = strdup(""); /* Either reuse or create new */ - rc = globalstate->rcinfo.entries; + rc = globalstate->rcinfo->entries; if(rc == NULL) { rc = nclistnew(); - globalstate->rcinfo.entries = rc; + globalstate->rcinfo->entries = rc; } nextline = contents; for(;;) { @@ -529,11 +488,11 @@ static struct NCRCentry* rclocate(const char* key, const char* hostport, const char* path) { int i,found; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); - NClist* rc = globalstate->rcinfo.entries; + NCglobalstate* globalstate = NC_getglobalstate(); + NClist* rc = globalstate->rcinfo->entries; NCRCentry* entry = NULL; - if(globalstate->rcinfo.ignore) + if(globalstate->rcinfo->ignore) return NULL; if(key == NULL || rc == NULL) return NULL; @@ -610,12 +569,12 @@ NC_rcfile_insert(const char* key, const char* value, const char* hostport, const int ret = NC_NOERR; /* See if this key already defined */ struct NCRCentry* entry = NULL; - NCRCglobalstate* globalstate = NULL; + NCglobalstate* globalstate = NULL; NClist* rc = NULL; if(!NCRCinitialized) ncrc_initialize(); - globalstate = ncrc_getglobalstate(); - rc = globalstate->rcinfo.entries; + globalstate = NC_getglobalstate(); + rc = globalstate->rcinfo->entries; if(rc == NULL) { rc = nclistnew(); @@ -744,13 +703,6 @@ NC_getdefaults3region(NCURI* uri, const char** regionp) return stat; } -static void -clearS3credentials(struct S3credentials* creds) -{ - if(creds) - freeprofilelist(creds->profiles); -} - /** The .aws/config and .aws/credentials files are in INI format (https://en.wikipedia.org/wiki/INI_file). @@ -1038,10 +990,9 @@ freeprofilelist(NClist* profiles) /* Find, load, and parse the aws credentials file */ static int -aws_load_credentials(NCRCglobalstate* gstate) +aws_load_credentials(NCglobalstate* gstate) { int stat = NC_NOERR; - struct S3credentials* creds = &gstate->s3creds; NClist* profiles = nclistnew(); const char** awscfg = awsconfigfiles; const char* aws_root = getenv(NC_TEST_AWS_DIR); @@ -1074,7 +1025,7 @@ aws_load_credentials(NCRCglobalstate* gstate) nclistpush(profiles,noprof); noprof = NULL; } - creds->profiles = profiles; profiles = NULL; + gstate->rcinfo->s3profiles = profiles; profiles = NULL; #ifdef AWSDEBUG {int i,j; @@ -1102,10 +1053,10 @@ NC_authgets3profile(const char* profilename, struct AWSprofile** profilep) { int stat = NC_NOERR; int i = -1; - NCRCglobalstate* gstate = ncrc_getglobalstate(); - - for(i=0;is3creds.profiles);i++) { - struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->s3creds.profiles,i); + NCglobalstate* gstate = NC_getglobalstate(); + + for(i=0;ircinfo->s3profiles);i++) { + struct AWSprofile* profile = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i); if(strcmp(profilename,profile->name)==0) {if(profilep) {*profilep = profile; goto done;}} } diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c index fc8a018d1..41d5e1a45 100644 --- a/libdispatch/dutil.c +++ b/libdispatch/dutil.c @@ -33,7 +33,7 @@ #define nulldup(x) ((x)?strdup(x):(x)) #endif /**************************************************/ -/** +/** \internal * Provide a hidden interface to allow utilities * to check if a given path name is really an ncdap4 url. * If no, return null, else return basename of the url @@ -66,7 +66,7 @@ NC__testurl(const char* path, char** basenamep) return ok; } -/* Return 1 if this machine is little endian */ +/** \internal Return 1 if this machine is little endian */ int NC_isLittleEndian(void) { @@ -78,6 +78,7 @@ NC_isLittleEndian(void) return (u.bytes[0] == 1 ? 1 : 0); } +/** \internal */ char* NC_backslashEscape(const char* s) { @@ -105,6 +106,7 @@ NC_backslashEscape(const char* s) return escaped; } +/** \internal */ char* NC_backslashUnescape(const char* esc) { @@ -129,6 +131,7 @@ NC_backslashUnescape(const char* esc) return s; } +/** \internal */ char* NC_entityescape(const char* s) { @@ -163,13 +166,13 @@ NC_entityescape(const char* s) return escaped; } -char* -/* +/** \internal Depending on the platform, the shell will sometimes pass an escaped octotherpe character without removing the backslash. So this function is appropriate to be called on possible url paths to unescape such cases. See e.g. ncgen. */ +char* NC_shellUnescape(const char* esc) { size_t len; @@ -194,7 +197,7 @@ NC_shellUnescape(const char* esc) return s; } -/** +/** \internal Wrap mktmp and return the generated path, or null if failed. Base is the base file path. XXXXX is appended @@ -224,6 +227,7 @@ done: return tmp; } +/** \internal */ int NC_readfile(const char* filename, NCbytes* content) { @@ -246,6 +250,7 @@ done: return ret; } +/** \internal */ int NC_writefile(const char* filename, size_t size, void* content) { @@ -271,7 +276,7 @@ done: return ret; } -/* +/** \internal Parse a path as a url and extract the modelist. If the path is not a URL, then return a NULL list. If a URL, but modelist is empty or does not exist, @@ -297,7 +302,7 @@ done: return stat; } -/* +/** \internal Check "mode=" list for a path and return 1 if present, 0 otherwise. */ int @@ -313,7 +318,7 @@ NC_testpathmode(const char* path, const char* tag) return found; } -/* +/** \internal Check "mode=" list for a url and return 1 if present, 0 otherwise. */ int @@ -339,8 +344,10 @@ done: return found; } -#if ! defined __INTEL_COMPILER -#if defined __APPLE__ +#if ! defined __INTEL_COMPILER +#if defined __APPLE__ +/** \internal */ + int isinf(double x) { union { unsigned long long u; double f; } ieee754; @@ -349,6 +356,7 @@ int isinf(double x) ( (unsigned)ieee754.u == 0 ); } +/** \internal */ int isnan(double x) { union { unsigned long long u; double f; } ieee754; @@ -360,7 +368,7 @@ int isnan(double x) #endif /*APPLE*/ #endif /*!_INTEL_COMPILER*/ - +/** \internal */ int NC_split_delim(const char* arg, char delim, NClist* segments) { @@ -395,7 +403,7 @@ done: return stat; } -/* concat the the segments with each segment preceded by '/' */ +/** \internal concat the the segments with each segment preceded by '/' */ int NC_join(NClist* segments, char** pathp) { diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index b40447faa..2c2af85fb 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -5,7 +5,7 @@ * @file * @internal The netCDF-4 file functions relating to file creation. * - * @author Ed Hartnett + * @author Ed Hartnett, Mark Harfouche, Dennis Heimbigner */ #include "config.h" @@ -14,11 +14,6 @@ #include "ncpathmgr.h" #include "hdf5internal.h" -/* From hdf5file.c. */ -extern size_t nc4_chunk_cache_size; -extern size_t nc4_chunk_cache_nelems; -extern float nc4_chunk_cache_preemption; - /** @internal These flags may not be set for create. */ static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_CDF5); @@ -157,12 +152,22 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, /* Only set cache for non-parallel creates. */ if (!nc4_info->parallel) { - if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, - nc4_chunk_cache_preemption) < 0) + NCglobalstate* gs = NC_getglobalstate(); + if (H5Pset_cache(fapl_id, 0, gs->chunkcache.nelems, gs->chunkcache.size, + gs->chunkcache.preemption) < 0) BAIL(NC_EHDFERR); LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f", - __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems, - nc4_chunk_cache_preemption)); + __func__, gs->chunkcache.size, gs->chunkcache.nelems, + gs->chunkcache.preemption)); + } + + { + NCglobalstate* gs = NC_getglobalstate(); + if(gs->alignment.defined) { + if (H5Pset_alignment(fapl_id, gs->alignment.threshold, gs->alignment.alignment) < 0) { + BAIL(NC_EHDFERR); + } + } } /* Set HDF5 format compatibility in the FILE ACCESS property list. diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index dc597eb2b..972508dcd 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -39,12 +39,6 @@ h5catch(void* ignored) } #endif -/* These are the default chunk cache sizes for HDF5 files created or - * opened with netCDF-4. */ -extern size_t nc4_chunk_cache_size; -extern size_t nc4_chunk_cache_nelems; -extern float nc4_chunk_cache_preemption; - #ifdef LOGGING /* This is the severity level of messages which will be logged. Use severity 0 for errors, 1 for important log messages, 2 for less diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index f301d63e3..1fe4bed9d 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -59,11 +59,6 @@ static const int nc_type_size_g[NUM_TYPES] = {sizeof(char), sizeof(char), sizeof /** @internal These flags may not be set for open mode. */ static const int ILLEGAL_OPEN_FLAGS = (NC_MMAP); -/* From libsrc4, these are the netcdf-4 cache sizes. */ -extern size_t nc4_chunk_cache_size; -extern size_t nc4_chunk_cache_nelems; -extern float nc4_chunk_cache_preemption; - /* From nc4mem.c */ extern int NC4_open_image_file(NC_FILE_INFO_T* h5); @@ -813,12 +808,22 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) /* Only set cache for non-parallel opens. */ if (!nc4_info->parallel) { - if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, - nc4_chunk_cache_preemption) < 0) + NCglobalstate* gs = NC_getglobalstate(); + if (H5Pset_cache(fapl_id, 0, gs->chunkcache.nelems, gs->chunkcache.size, + gs->chunkcache.preemption) < 0) BAIL(NC_EHDFERR); LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f", - __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems, - nc4_chunk_cache_preemption)); + __func__, gs->chunkcache.size, gs->chunkcache.nelems, + gs->chunkcache.preemption)); + } + + { + NCglobalstate* gs = NC_getglobalstate(); + if(gs->alignment.defined) { + if (H5Pset_alignment(fapl_id, gs->alignment.threshold, gs->alignment.alignment) < 0) { + BAIL(NC_EHDFERR); + } + } } /* Set HDF5 format compatibility in the FILE ACCESS property list. @@ -1453,10 +1458,10 @@ nc4_get_var_meta(NC_VAR_INFO_T *var) BAIL(NC_EVARMETA); /* Learn about current chunk cache settings. */ - if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems), - &(var->chunk_cache_size), &rdcc_w0)) < 0) + if ((H5Pget_chunk_cache(access_pid, &(var->chunkcache.nelems), + &(var->chunkcache.size), &rdcc_w0)) < 0) BAIL(NC_EHDFERR); - var->chunk_cache_preemption = rdcc_w0; + var->chunkcache.preemption = rdcc_w0; /* Get the dataset creation properties. */ if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index d1eedcc73..fd3d06deb 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -85,9 +85,9 @@ nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) if ((access_pid = H5Pcreate(H5P_DATASET_ACCESS)) < 0) return NC_EHDFERR; - if (H5Pset_chunk_cache(access_pid, var->chunk_cache_nelems, - var->chunk_cache_size, - var->chunk_cache_preemption) < 0) + if (H5Pset_chunk_cache(access_pid, var->chunkcache.nelems, + var->chunkcache.size, + var->chunkcache.preemption) < 0) return NC_EHDFERR; if (H5Dclose(hdf5_var->hdf_datasetid) < 0) return NC_EHDFERR; @@ -2314,9 +2314,9 @@ NC4_HDF5_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, assert(var && var->hdr.id == varid); /* Set the values. */ - var->chunk_cache_size = size; - var->chunk_cache_nelems = nelems; - var->chunk_cache_preemption = preemption; + var->chunkcache.size = size; + var->chunkcache.nelems = nelems; + var->chunkcache.preemption = preemption; /* Reopen the dataset to bring new settings into effect. */ if ((retval = nc4_reopen_dataset(grp, var))) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 819c056e4..9c93bf861 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -977,9 +977,9 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid } /* Set per-var chunk cache, for chunked datasets. */ - if (var->storage == NC_CHUNKED && var->chunk_cache_size) - if (H5Pset_chunk_cache(access_plistid, var->chunk_cache_nelems, - var->chunk_cache_size, var->chunk_cache_preemption) < 0) + if (var->storage == NC_CHUNKED && var->chunkcache.size) + if (H5Pset_chunk_cache(access_plistid, var->chunkcache.nelems, + var->chunkcache.size, var->chunkcache.preemption) < 0) BAIL(NC_EHDFERR); /* At long last, create the dataset. */ @@ -1101,12 +1101,12 @@ nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) /* If the chunk cache is too small, and the user has not changed * the default value of the chunk cache size, then increase the * size of the cache. */ - if (var->chunk_cache_size == CHUNK_CACHE_SIZE) - if (chunk_size_bytes > var->chunk_cache_size) + if (var->chunkcache.size == CHUNK_CACHE_SIZE) + if (chunk_size_bytes > var->chunkcache.size) { - var->chunk_cache_size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE; - if (var->chunk_cache_size > MAX_DEFAULT_CACHE_SIZE) - var->chunk_cache_size = MAX_DEFAULT_CACHE_SIZE; + var->chunkcache.size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE; + if (var->chunkcache.size > MAX_DEFAULT_CACHE_SIZE) + var->chunkcache.size = MAX_DEFAULT_CACHE_SIZE; if ((retval = nc4_reopen_dataset(grp, var))) return retval; } diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 6980c1785..018842d09 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -18,12 +18,6 @@ #include "zincludes.h" #include "zfilter.h" -/* These are the default chunk cache sizes for ZARR files created or - * opened with netCDF-4. */ -extern size_t ncz_chunk_cache_size; -extern size_t ncz_chunk_cache_nelems; -extern float ncz_chunk_cache_preemption; - /* Forward */ #ifdef LOGGING @@ -61,10 +55,10 @@ NCZ_initialize_internal(void) { int stat = NC_NOERR; char* dimsep = NULL; - NCRCglobalstate* ngs = NULL; + NCglobalstate* ngs = NULL; ncz_initialized = 1; - ngs = ncrc_getglobalstate(); + ngs = NC_getglobalstate(); if(ngs != NULL) { /* Defaults */ ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR; diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index fa864091b..4ca278168 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -1534,7 +1534,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) } /* Capture dimension_separator (must precede chunk cache creation) */ { - NCRCglobalstate* ngs = ncrc_getglobalstate(); + NCglobalstate* ngs = NC_getglobalstate(); assert(ngs != NULL); zvar->dimension_separator = 0; if((stat = NCJdictget(jvar,"dimension_separator",&jvalue))) goto done; diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index d4325be8f..452684057 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -381,7 +381,7 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims, zvar->common.file = h5; zvar->scalar = (ndims == 0 ? 1 : 0); - zvar->dimension_separator = ncrc_getglobalstate()->zarr.dimension_separator; + zvar->dimension_separator = NC_getglobalstate()->zarr.dimension_separator; assert(zvar->dimension_separator != 0); /* Set these state flags for the var. */ @@ -456,9 +456,9 @@ var->type_info->rc++; zvar->chunksize = zvar->chunkproduct * var->type_info->size; /* Override the cache setting to use NCZarr defaults */ - var->chunk_cache_size = CHUNK_CACHE_SIZE_NCZARR; - var->chunk_cache_nelems = ceildiv(var->chunk_cache_size,zvar->chunksize); - var->chunk_cache_preemption = 1; /* not used */ + var->chunkcache.size = CHUNK_CACHE_SIZE_NCZARR; + var->chunkcache.nelems = ceildiv(var->chunkcache.size,zvar->chunksize); + var->chunkcache.preemption = 1; /* not used */ /* Create the cache */ if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,zvar->dimension_separator,&zvar->cache))) diff --git a/libnczarr/zxcache.c b/libnczarr/zxcache.c index 52a8c7d35..52f8abf2e 100644 --- a/libnczarr/zxcache.c +++ b/libnczarr/zxcache.c @@ -77,9 +77,9 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl assert(zvar != NULL && zvar->cache != NULL); /* Set the values. */ - var->chunk_cache_size = cachesize; - var->chunk_cache_nelems = nelems; - var->chunk_cache_preemption = preemption; + var->chunkcache.size = cachesize; + var->chunkcache.nelems = nelems; + var->chunkcache.preemption = preemption; /* Fix up cache */ if((retval = NCZ_adjust_var_cache(var))) goto done; @@ -120,8 +120,8 @@ fprintf(stderr,"xxx: adjusting cache for: %s\n",var->hdr.name); /* Reclaim any existing fill_chunk */ if((stat = NCZ_reclaim_fill_chunk(zcache))) goto done; /* Reset the parameters */ - zvar->cache->maxsize = var->chunk_cache_size; - zvar->cache->maxentries = var->chunk_cache_nelems; + zvar->cache->maxsize = var->chunkcache.size; + zvar->cache->maxentries = var->chunkcache.nelems; #ifdef DEBUG fprintf(stderr,"%s.cache.adjust: size=%ld nelems=%ld\n", var->hdr.name,(unsigned long)zvar->cache->maxsize,(unsigned long)zvar->cache->maxentries); diff --git a/libsrc4/nc4cache.c b/libsrc4/nc4cache.c index 9a7edec48..f41b327a3 100644 --- a/libsrc4/nc4cache.c +++ b/libsrc4/nc4cache.c @@ -13,12 +13,6 @@ #include "config.h" #include "nc4internal.h" -/* These are the default chunk cache sizes for HDF5 files created or - * opened with netCDF-4. */ -extern size_t nc4_chunk_cache_size; -extern size_t nc4_chunk_cache_nelems; -extern float nc4_chunk_cache_preemption; - /** * Set chunk cache size. Only affects netCDF-4/HDF5 files * opened/created *after* it is called. @@ -71,11 +65,12 @@ extern float nc4_chunk_cache_preemption; int nc_set_chunk_cache(size_t size, size_t nelems, float preemption) { + NCglobalstate* gs = NC_getglobalstate(); if (preemption < 0 || preemption > 1) return NC_EINVAL; - nc4_chunk_cache_size = size; - nc4_chunk_cache_nelems = nelems; - nc4_chunk_cache_preemption = preemption; + gs->chunkcache.size = size; + gs->chunkcache.nelems = nelems; + gs->chunkcache.preemption = preemption; return NC_NOERR; } @@ -99,14 +94,15 @@ nc_set_chunk_cache(size_t size, size_t nelems, float preemption) int nc_get_chunk_cache(size_t *sizep, size_t *nelemsp, float *preemptionp) { + NCglobalstate* gs = NC_getglobalstate(); if (sizep) - *sizep = nc4_chunk_cache_size; + *sizep = gs->chunkcache.size; if (nelemsp) - *nelemsp = nc4_chunk_cache_nelems; + *nelemsp = gs->chunkcache.nelems; if (preemptionp) - *preemptionp = nc4_chunk_cache_preemption; + *preemptionp = gs->chunkcache.preemption; return NC_NOERR; } @@ -128,11 +124,12 @@ nc_get_chunk_cache(size_t *sizep, size_t *nelemsp, float *preemptionp) int nc_set_chunk_cache_ints(int size, int nelems, int preemption) { + NCglobalstate* gs = NC_getglobalstate(); if (size <= 0 || nelems <= 0 || preemption < 0 || preemption > 100) return NC_EINVAL; - nc4_chunk_cache_size = size; - nc4_chunk_cache_nelems = nelems; - nc4_chunk_cache_preemption = (float)preemption / 100; + gs->chunkcache.size = size; + gs->chunkcache.nelems = nelems; + gs->chunkcache.preemption = (float)preemption / 100; return NC_NOERR; } @@ -154,12 +151,13 @@ nc_set_chunk_cache_ints(int size, int nelems, int preemption) int nc_get_chunk_cache_ints(int *sizep, int *nelemsp, int *preemptionp) { + NCglobalstate* gs = NC_getglobalstate(); if (sizep) - *sizep = (int)nc4_chunk_cache_size; + *sizep = (int)gs->chunkcache.size; if (nelemsp) - *nelemsp = (int)nc4_chunk_cache_nelems; + *nelemsp = (int)gs->chunkcache.nelems; if (preemptionp) - *preemptionp = (int)(nc4_chunk_cache_preemption * 100); + *preemptionp = (int)(gs->chunkcache.preemption * 100); return NC_NOERR; } diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 0312d8fbe..878be8c76 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -24,6 +24,7 @@ #include "ncdispatch.h" /* from libdispatch */ #include "ncutf8.h" #include +#include "ncrc.h" /** @internal Number of reserved attributes. These attributes are * hidden from the netcdf user, but exist in the implementation @@ -53,11 +54,6 @@ static const NC_reservedatt NC_reserved[] = { }; #define NRESERVED (sizeof(NC_reserved) / sizeof(NC_reservedatt)) /*|NC_reservedatt|*/ -/* These hold the file caching settings for the library. */ -size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE; /**< Default chunk cache size. */ -size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */ -float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */ - static int NC4_move_in_NCList(NC* nc, int new_id); #if NC_HAS_LOGGING @@ -737,6 +733,7 @@ int nc4_var_list_add2(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var) { NC_VAR_INFO_T *new_var = NULL; + NCglobalstate* gs = NC_getglobalstate(); /* Allocate storage for new variable. */ if (!(new_var = calloc(1, sizeof(NC_VAR_INFO_T)))) @@ -745,9 +742,9 @@ nc4_var_list_add2(NC_GRP_INFO_T *grp, const char *name, NC_VAR_INFO_T **var) new_var->container = grp; /* These are the HDF5-1.8.4 defaults. */ - new_var->chunk_cache_size = nc4_chunk_cache_size; - new_var->chunk_cache_nelems = nc4_chunk_cache_nelems; - new_var->chunk_cache_preemption = nc4_chunk_cache_preemption; + new_var->chunkcache.size = gs->chunkcache.size; + new_var->chunkcache.nelems = gs->chunkcache.nelems; + new_var->chunkcache.preemption = gs->chunkcache.preemption; /* Now fill in the values in the var info structure. */ new_var->hdr.id = ncindexsize(grp->vars); @@ -2093,3 +2090,141 @@ NC4_move_in_NCList(NC* nc, int new_id) return stat; } +/**************************************************/ +/* NCglobal state management */ + +static NCglobalstate* nc_globalstate = NULL; + +static int +NC_createglobalstate(void) +{ + int stat = NC_NOERR; + const char* tmp = NULL; + + if(nc_globalstate == NULL) { + nc_globalstate = calloc(1,sizeof(NCglobalstate)); + } + /* Initialize struct pointers */ + nc_globalstate->rcinfo = (struct NCRCinfo*)calloc(1,sizeof(struct NCRCinfo)); + if(nc_globalstate == NULL) return NC_ENOMEM; + + /* Get environment variables */ + if(getenv(NCRCENVIGNORE) != NULL) + nc_globalstate->rcinfo->ignore = 1; + tmp = getenv(NCRCENVRC); + if(tmp != NULL && strlen(tmp) > 0) + nc_globalstate->rcinfo->rcfile = strdup(tmp); + /* Initialize chunk cache defaults */ + nc_globalstate->chunkcache.size = CHUNK_CACHE_SIZE; /**< Default chunk cache size. */ + nc_globalstate->chunkcache.nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */ + nc_globalstate->chunkcache.preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */ + + return stat; +} + +/* Get global state */ +NCglobalstate* +NC_getglobalstate(void) +{ + if(nc_globalstate == NULL) + NC_createglobalstate(); + return nc_globalstate; +} + +void +NC_freeglobalstate(void) +{ + if(nc_globalstate != NULL) { + nullfree(nc_globalstate->tempdir); + nullfree(nc_globalstate->home); + nullfree(nc_globalstate->cwd); + NC_rcclear(nc_globalstate->rcinfo); + free(nc_globalstate->rcinfo); + free(nc_globalstate); + nc_globalstate = NULL; + } +} +/**************************************************/ +/* Specific property functions */ + +/** +Provide a function to store global data alignment +information. +Repeated calls to nc_set_alignment will overwrite any existing values. + +If defined, then for every file created or opened after the call to +nc_set_alignment, and for every new variable added to the file, the +most recently set threshold and alignment values will be applied +to that variable. + +The nc_set_alignment function causes new data written to a +netCDF-4 file to be aligned on disk to a specified block +size. To be effective, alignment should be the system disk block +size, or a multiple of it. This setting is effective with MPI +I/O and other parallel systems. + +This is a trade-off of write speed versus file size. Alignment +leaves holes between file objects. The default of no alignment +writes file objects contiguously, without holes. Alignment has +no impact on file readability. + +Alignment settings apply only indirectly, through the file open +functions. Call nc_set_alignment first, then nc_create or +nc_open for one or more files. Current alignment settings are +locked in when each file is opened, then forgotten when the same +file is closed. For illustration, it is possible to write +different files at the same time with different alignments, by +interleaving nc_set_alignment and nc_open calls. + +Alignment applies to all newly written low-level file objects at +or above the threshold size, including chunks of variables, +attributes, and internal infrastructure. Alignment is not locked +in to a data variable. It can change between data chunks of the +same variable, based on a file's history. + +Refer to H5Pset_alignment in HDF5 documentation for more +specific details, interactions, and additional rules. + +@param threshold The minimum size to which alignment is applied. +@param alignment The alignment value. + +@return ::NC_NOERR No error. +@return ::NC_EINVAL Invalid input. +@author Dennis Heimbigner +@ingroup datasets +*/ +int +nc_set_alignment(int threshold, int alignment) +{ + NCglobalstate* gs = NC_getglobalstate(); + gs->alignment.threshold = threshold; + gs->alignment.alignment = alignment; + gs->alignment.defined = 1; + return NC_NOERR; +} + +/** +Provide get function to retrieve global data alignment +information. + +The nc_get_alignment function return the last values set by +nc_set_alignment. If nc_set_alignment has not been called, then +it returns the value 0 for both threshold and alignment. + +@param thresholdp Return the current minimum size to which alignment is applied or zero. +@param alignmentp Return the current alignment value or zero. + +@return ::NC_NOERR No error. +@return ::NC_EINVAL Invalid input. +@author Dennis Heimbigner +@ingroup datasets +*/ + +int +nc_get_alignment(int* thresholdp, int* alignmentp) +{ + NCglobalstate* gs = NC_getglobalstate(); + if(thresholdp) *thresholdp = gs->alignment.threshold; + if(alignmentp) *alignmentp = gs->alignment.alignment; + return NC_NOERR; +} diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index c6c241059..fac6520bf 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -97,11 +97,11 @@ NC4_get_var_chunk_cache(int ncid, int varid, size_t *sizep, /* Give the user what they want. */ if (sizep) - *sizep = var->chunk_cache_size; + *sizep = var->chunkcache.size; if (nelemsp) - *nelemsp = var->chunk_cache_nelems; + *nelemsp = var->chunkcache.nelems; if (preemptionp) - *preemptionp = var->chunk_cache_preemption; + *preemptionp = var->chunkcache.preemption; return NC_NOERR; } diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index c2eb3be05..478f38093 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -24,6 +24,10 @@ SET(NC4_TESTS tst_dims tst_dims2 tst_dims3 tst_files tst_files4 tst_hdf5_file_compat tst_fill_attr_vanish tst_rehash tst_types tst_bug324 tst_atts3 tst_put_vars tst_elatefill tst_udf tst_bug1442 tst_quantize) +IF(HAS_PAR_FILTERS) +SET(NC4_tests $NC4_TESTS tst_alignment) +ENDIF() + # Note, renamegroup needs to be compiled before run_grp_rename IF(BUILD_UTILITIES) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 2eb2545b8..5305d8c8b 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -37,6 +37,10 @@ tst_rehash tst_filterparser tst_bug324 tst_types tst_atts3 \ tst_put_vars tst_elatefill tst_udf tst_put_vars_two_unlim_dim \ tst_bug1442 tst_quantize +if HAS_PAR_FILTERS +NC4_TESTS += tst_alignment +endif + # Temporary I hoped, but hoped in vain. if !ISCYGWIN NC4_TESTS += tst_h_strbug tst_h_refs diff --git a/nc_test4/tst_alignment.c b/nc_test4/tst_alignment.c new file mode 100644 index 000000000..385df0229 --- /dev/null +++ b/nc_test4/tst_alignment.c @@ -0,0 +1,96 @@ +/* This is part of the netCDF package. + Copyright 2018 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Test HDF5 alignment + Dennis Heimbigner +*/ + +#include +#include "err_macros.h" + +#include +#include + +#undef DEBUG + +#define THRESHOLD 512 +#define ALIGNMENT 4096 +#define CHUNKSIZE 513 + +int +main(int argc, char **argv) +{ + int i,ncid, varid, dimids[1]; + size_t chunks[1]; + unsigned char data[CHUNKSIZE]; + hid_t fileid, grpid, datasetid; + hid_t dxpl_id = H5P_DEFAULT; /*data transfer property list */ + unsigned int filter_mask = 0; + hsize_t hoffset[1]; + haddr_t addr; + hsize_t size ; + hid_t fspace; + + H5Eset_auto2(H5E_DEFAULT,(H5E_auto2_t)H5Eprint1,stderr); + + printf("\n*** Testing HDF5 alignment.\n"); + + printf("chunksize=%d threshold=%d alignment=%d\n",CHUNKSIZE,THRESHOLD,ALIGNMENT); + + if(nc_set_alignment(THRESHOLD,ALIGNMENT)) ERR; + if (nc_create("tst_alignment.nc", NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, "d0", CHUNKSIZE, &dimids[0])) ERR; + if (nc_def_var(ncid, "var", NC_UBYTE, 1, dimids, &varid)) ERR; + chunks[0] = CHUNKSIZE; + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunks)) ERR; + if (nc_enddef(ncid)) ERR; + + for(i=0;i #include "netcdf.h" #include "ncrc.h" +#include "nc4internal.h" int main(int argc, char** argv) { size_t i,nentries = 0; - NCRCglobalstate* ngs = ncrc_getglobalstate(); + NCglobalstate* ngs = NC_getglobalstate(); NCRCinfo* info = NULL; NCRCentry* entry = NULL; /* Cause the .rc files to be read and merged */ nc_initialize(); - if((ngs = ncrc_getglobalstate())==NULL) abort(); - info = &ngs->rcinfo; + if((ngs = NC_getglobalstate())==NULL) abort(); + info = ngs->rcinfo; if(info->ignore) { fprintf(stderr,".rc ignored\n"); diff --git a/nczarr_test/bm_chunks3.c b/nczarr_test/bm_chunks3.c index 1e777bcbf..9a34bda33 100644 --- a/nczarr_test/bm_chunks3.c +++ b/nczarr_test/bm_chunks3.c @@ -167,7 +167,7 @@ main(int argc, char *argv[]) /* variable shapes */ int var_dims[RANK]; - NCCHECK(getoptions(&argc,&argv,&bmoptions)); + NCCHECK(bm_getoptions(&argc,&argv,&bmoptions)); NCCHECK(nc4_buildpath(&bmoptions,&path)); if(bmoptions.debug) { diff --git a/nczarr_test/bm_utils.h b/nczarr_test/bm_utils.h index 52103225a..1537116ed 100644 --- a/nczarr_test/bm_utils.h +++ b/nczarr_test/bm_utils.h @@ -89,6 +89,10 @@ EXTERNL void bm_reportmetaoptions(struct BMMeta* o); EXTERNL const char* bm_printvector(int rank, const size_t* vec); EXTERNL const char* bm_printvectorp(int rank, const ptrdiff_t* vec); EXTERNL const char* bm_varasprint(int rank, const size_t* start, const size_t* edges, const ptrdiff_t* stride); +EXTERNL int nc4_buildpath(struct BMOptions* o, char** pathp); +EXTERNL void reportoptions(struct BMOptions* o); +EXTERNL void reportmetaoptions(struct BMMeta* o); +EXTERNL void clearoptions(struct BMOptions*); #include "ut_test.h" diff --git a/nczarr_test/zhex.c b/nczarr_test/zhex.c index 3a49ff649..972407cb4 100644 --- a/nczarr_test/zhex.c +++ b/nczarr_test/zhex.c @@ -12,7 +12,7 @@ #include #endif -#define DEBUG +#undef DEBUG static char hex[16] = "0123456789abcdef"; diff --git a/nczarr_test/zmapio.c b/nczarr_test/zmapio.c index b1e06295e..c9c982c88 100644 --- a/nczarr_test/zmapio.c +++ b/nczarr_test/zmapio.c @@ -23,7 +23,7 @@ #include "nclog.h" #include "ncuri.h" -#define DEBUG +#undef DEBUG #define DATANAME "data" diff --git a/nczarr_test/zs3parse.c b/nczarr_test/zs3parse.c index e1dba30ca..7286dd75a 100644 --- a/nczarr_test/zs3parse.c +++ b/nczarr_test/zs3parse.c @@ -21,7 +21,7 @@ #include "zincludes.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG #define AWSHOST ".amazonaws.com" diff --git a/oc2/ocinternal.c b/oc2/ocinternal.c index eba1288fa..fbaf49fba 100644 --- a/oc2/ocinternal.c +++ b/oc2/ocinternal.c @@ -25,7 +25,7 @@ #include -#include "ncrc.h" +#include "nc4internal.h" #include "ocinternal.h" #include "ocdebug.h" #include "occurlfunctions.h" @@ -321,7 +321,7 @@ createtempfile(OCstate* state, OCtree* tree) char* path = NULL; char* tmppath = NULL; int len; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCglobalstate* globalstate = NC_getglobalstate(); len = strlen(globalstate->tempdir) @@ -526,7 +526,7 @@ static OCerror ocset_curlproperties(OCstate* state) { OCerror stat = OC_NOERR; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCglobalstate* globalstate = NC_getglobalstate(); if(state->auth->curlflags.useragent == NULL) { size_t len = strlen(DFALTUSERAGENT) + strlen(VERSION) + 1; diff --git a/unit_test/test_aws.c b/unit_test/test_aws.c index 826d3f82c..8dd9e9276 100644 --- a/unit_test/test_aws.c +++ b/unit_test/test_aws.c @@ -15,7 +15,7 @@ Test the handling of aws profiles and regions. #include "ncrc.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG typedef struct ProfileTest { const char* profile; diff --git a/unit_test/test_pathcvt.c b/unit_test/test_pathcvt.c index 7f1d67b6b..7426e5495 100644 --- a/unit_test/test_pathcvt.c +++ b/unit_test/test_pathcvt.c @@ -13,7 +13,7 @@ Test the NCpathcvt #include "netcdf.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG #define NKINDS 4 static const int kinds[NKINDS] = {NCPD_NIX,NCPD_MSYS,NCPD_CYGWIN,NCPD_WIN};