Merge branch 'ncgenchunks.dmh' of https://github.com/DennisHeimbigner/netcdf-c into master

This commit is contained in:
Ward Fisher 2020-09-09 10:24:33 -06:00
commit a89e1f73b8
15 changed files with 1832 additions and 1930 deletions

View File

@ -767,8 +767,15 @@ IF(USE_HDF5)
SET(HDF5_CC h5cc)
ENDIF()
# Check to see if H5Dread_chunk is available
CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5Dread_chunk "" HAS_READCHUNKS)
# Check to see if this is hdf5-1.10.3 or later.
CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5Dread_chunk "" HDF5_SUPPORTS_PAR_FILTERS)
IF(HAS_READCHUNKS)
SET(HDF5_SUPPORTS_PAR_FILTERS ON)
SET(ENABLE_NCDUMPCHUNKS ON)
ENDIF()
IF (HDF5_SUPPORTS_PAR_FILTERS)
SET(HDF5_HAS_PAR_FILTERS TRUE CACHE BOOL "")
SET(HAS_PAR_FILTERS yes CACHE STRING "")

View File

@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.0 - TBD
* [Bug Fix] Fixed byte-range support with cURL > 7.69. See [https://github.com/Unidata/netcdf-c/pull/1798].
* [Enhancement] Added new test for using compression with parallel I/O: nc_test4/tst_h_par_compress.c. See [https://github.com/Unidata/netcdf-c/pull/1784].
* [Bug Fix] Don't return error for extra calls to nc_redef() for netCDF/HDF5 files, unless classic model is in use. See [https://github.com/Unidata/netcdf-c/issues/1779].
@ -14,7 +15,16 @@ This file contains a high-level description of this package's evolution. Release
* [Bug Fix] Now allow szip to be used on variables with unlimited dimension [https://github.com/Unidata/netcdf-c/issues/1774].
* [Enhancement] Add support for cloud storage using a variant of the Zarr storage format. Warning: this feature is highly experimental and is subject to rapid evolution [https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in].
* [Bug Fix] Fix nccopy to properly set default chunking parameters when not otherwise specified. This can significantly improve performance in selected cases. Note that if seeing slow performance with nccopy, then, as a work-around, specifically set the chunking parameters. [https://github.com/Unidata/netcdf-c/issues/1763].
* [Bug Fix] Fix some protocol bugs/differences between the netcdf-c library and the OPeNDAP Hyrax server. Also cleanup checksum handling [https://github.com/Unidata/netcdf-c/issues/1712].
* [Bug Fix] Fix some protocol bugs/differences between the netcdf-c library and the OPeNDAP Hyrax server. Also cleanup checksum handling [https://github.com/Unidata/netcdf-c/issues/1712].* [Bug Fix] IMPORTANT: Ncgen was not properly handling large
data sections. The problem manifests as incorrect ordering of
data in the created file. Aside from examining the file with
ncdump, the error can be detected by running ncgen with the -lc
flag (to produce a C file). Examine the file to see if any
variable is written in pieces as opposed to a single call to
nc_put_vara. If multiple calls to nc_put_vara are used to write
a variable, then it is probable that the data order is
incorrect. Such multiple writes can occur for large variables
and especially when one of the dimensions is unlimited.
* [Bug Fix] Add necessary __declspec declarations to allow compilation
of netcdf library without causing errors or (_declspec related)
warnings [https://github.com/Unidata/netcdf-c/issues/1725].

View File

@ -672,7 +672,7 @@ else
AC_DEFINE_UNQUOTED(USE_NETCDF_2, 1, [build the netCDF version 2 API])
fi
# Does the user want to disable ncgen/ncdump/nccopy?
# Does the user want to disable ncgen/ncdump/nccopy/...?
AC_MSG_CHECKING([whether the ncgen/ncdump/nccopy should be built])
AC_ARG_ENABLE([utilities],
[AS_HELP_STRING([--disable-utilities],
@ -1144,7 +1144,7 @@ if test "x$enable_hdf5" = xyes; then
# H5Pset_fapl_mpiposix and H5Pget_fapl_mpiposix have been removed since HDF5 1.8.12.
# Use H5Pset_fapl_mpio and H5Pget_fapl_mpio, instead.
AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5free_memory H5resize_memory H5allocate_memory H5Pset_libver_bounds H5Pset_all_coll_metadata_ops H5Dread_chunk])
AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5free_memory H5resize_memory H5allocate_memory H5Pset_libver_bounds H5Pset_all_coll_metadata_ops])
# Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0)
if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then
@ -1155,19 +1155,21 @@ if test "x$enable_hdf5" = xyes; then
if test "x$ac_cv_func_H5Pget_fapl_mpio" = xyes -o "x$ac_cv_func_H5Pget_fapl_mpiposix" = xyes; then
hdf5_parallel=yes
fi
AC_MSG_CHECKING([whether parallel io is enabled in hdf5])
AC_MSG_RESULT([$hdf5_parallel])
# See if H5Dread_chunk is available
AC_SEARCH_LIBS([H5Dread_chunk],[hdf5_hldll hdf5_hl], [has_readchunks=yes], [has_readdhunks=no])
# Check to see if HDF5 library is 1.10.3 or greater. If so, allows
# parallel I/O with filters. This allows zlib/szip compression to
# be used with parallel I/O, which is very helpful to HPC users.
if test "x$ac_cv_func_H5Dread_chunk" = xyes; then
if test "x$has_readchunks" = xyes; then
AC_DEFINE([HDF5_SUPPORTS_PAR_FILTERS], [1], [if true, HDF5 is at least version 1.10.3 and allows parallel I/O with zip])
hdf5_supports_par_filters=yes
fi
AC_MSG_CHECKING([whether HDF5 allows parallel filters])
AC_MSG_RESULT([$ac_cv_func_H5Dread_chunk])
AC_MSG_RESULT([$has_readchunks])
# Check to see if user asked for parallel build, but HDF5 does not support it.
if test "x$hdf5_parallel" = "xno"; then
@ -1184,7 +1186,9 @@ if test "x$enable_hdf5" = xyes; then
AC_DEFINE([USE_SZIP], [1], [if true, compile in szip compression in netCDF-4 variables])
fi
AC_MSG_RESULT([$enable_szlib])
fi
AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ])
# If the user wants hdf4 built in, check it out.
if test "x$enable_hdf4" = xyes; then

View File

@ -212,7 +212,7 @@ NCD4_set_flags_perlink(NCD4INFO* state)
if(ret == NC_NOERR && state->curl->keepalive.active != 0)
ret = set_curlflag(state, CURLOPT_TCP_KEEPALIVE);
#endif
#if 0
/* Set the CURL. options */
if(ret == NC_NOERR) ret = set_curl_options(state);
@ -303,7 +303,7 @@ NCD4_curl_protocols(NCD4INFO* state)
for(proto=curldata->protocols;*proto;proto++) {
if(strcmp("http",*proto)==0) {state->auth.curlflags.proto_https=1;}
}
#ifdef D4DEBUG
#ifdef D4DEBUG
nclog(NCLOGNOTE,"Curl https:// support = %d",state->auth.curlflags.proto_https);
#endif
}
@ -320,7 +320,7 @@ NCD4_get_rcproperties(NCD4INFO* state)
option = NC_rclookup(D4BUFFERSIZE,state->uri->uri);
if(option != NULL && strlen(option) != 0) {
long bufsize;
if(strcasecmp(option,"max")==0)
if(strcasecmp(option,"max")==0)
bufsize = CURL_MAX_READ_SIZE;
else if(sscanf(option,"%ld",&bufsize) != 1 || bufsize <= 0)
fprintf(stderr,"Illegal %s size\n",D4BUFFERSIZE);

View File

@ -74,6 +74,7 @@ THIS SOFTWARE.
#endif
#ifdef _MSC_VER
#include <io.h>
#include "XGetopt.h"
#define snprintf _snprintf
#endif

View File

@ -35,10 +35,10 @@ TARGET_LINK_LIBRARIES(ncgen netcdf ${ALL_TLL_LIBS})
# Given a netcdf4 file, dump the actual chunk contents.
# Used to validate nczarr chunking code.
IF(USE_HDF5)
IF(NOT MSVC)
SET(ncdumpchunks_FILES ncdumpchunks.c)
ADD_EXECUTABLE(ncdumpchunks ${ncdumpchunks_FILES})
TARGET_LINK_LIBRARIES(ncdumpchunks netcdf ${ALL_TLL_LIBS})
IF(ENABLE_NCDUMPCHUNKS)
SET(ncdumpchunks_FILES ncdumpchunks.c)
ADD_EXECUTABLE(ncdumpchunks ${ncdumpchunks_FILES})
TARGET_LINK_LIBRARIES(ncdumpchunks netcdf ${ALL_TLL_LIBS})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG

View File

@ -71,7 +71,7 @@ makeparser::
flex -Pncg -8 ncgen.l
sed -e s/lex.ncg.c/ncgenl.c/g <lex.ncg.c >ncgenl.c
bison -pncg -t -d ncgen.y
rm -f ncgeny.c ncgeny.h
rm -f ncgeny.c ncgeny.h
sed -e s/ncgen.tab.c/ncgeny.c/g -e s/ncgen.tab.h/ncgeny.h/g <ncgen.tab.c >ncgeny.c
sed -e s/ncgen.tab.c/ncgeny.c/g -e s/ncgen.tab.h/ncgeny.h/g <ncgen.tab.h >ncgeny.h
rm -f lex.ncg.c ncgen.tab.h ncgen.tab.c

View File

@ -46,6 +46,9 @@ extern int ncgdebug;
extern void fdebug(const char *fmt, ...);
#define CHECK_ERR(x) check_err((x),__LINE__,__FILE__,__func__)
#define CHECK_ERR2(x,cl) check_err2((x),(cl),__LINE__,__FILE__,__func__)
#define PANIC(msg) assert(panic(msg))
#define PANIC1(msg,arg) assert(panic(msg,arg))
#define PANIC2(msg,arg1,arg2) assert(panic(msg,arg1,arg2))

View File

@ -125,7 +125,7 @@ gen_leafchararray(Dimset* dimset, int dimindex, Datalist* data,
concatenated with any trailing or leading string (with double quotes).
*/
rebuildsingletons(data);
/* Compute crossproduct from dimindex up to (but not including) the last dimension */
xproduct = crossproduct(dimset,dimindex,rank-1);
@ -162,7 +162,7 @@ gen_leafchararray(Dimset* dimset, int dimindex, Datalist* data,
}
}
freedatalist(flat);
/* If |databuf| > expectedsize, complain: exception is zero length */
if(bbLength(charbuf) == 0 && expectedsize == 1) {
/* this is okay */
@ -260,7 +260,7 @@ getfillchar(Datalist* fillsrc)
NCConstant* ccon = fillsrc->data[0];
if(ccon->nctype == NC_CHAR) {
fillchar = ccon->value.charv;
} else if(ccon->nctype == NC_STRING) {
} else if(ccon->nctype == NC_STRING) {
if(ccon->value.stringv.len > 0) {
fillchar = ccon->value.stringv.stringv[0];
}

View File

@ -386,7 +386,7 @@ generate_arrayR(struct Args* args, int dimindex, size_t* index, Datalist* data)
if(dimindex > 0 && dim->dim.isunlimited) {
/* Get the unlimited list */
NCConstant* con = datalistith(data,0);
actual = compoundfor(con);
actual = compoundfor(con);
} else
actual = data;
/* For last index, dump all of its elements */
@ -401,7 +401,7 @@ generate_arrayR(struct Args* args, int dimindex, size_t* index, Datalist* data)
count[dimindex] = stop;
args->writer(args->generator,args->vsym,args->code,args->rank,index,count);
bbClear(args->code);
} else {
} else {
actual = data;
/* Iterate over this dimension */
for(counter = 0;counter < stop; counter++) {
@ -441,7 +441,7 @@ generate_array(Symbol* vsym, Bytebuffer* code, Datalist* filler, Generator* gene
args.typecode = vsym->typ.basetype->typ.typecode;
assert(args.rank > 0);
totalsize = 1; /* total # elements in the array */
for(i=0;i<args.rank;i++) {
args.dimsizes[i] = args.dimset->dimsyms[i]->dim.declsize;
@ -474,7 +474,7 @@ generate_array(Symbol* vsym, Bytebuffer* code, Datalist* filler, Generator* gene
if(totalsize <= wholevarsize && nunlimited == 0) {
Symbol* basetype = args.vsym->typ.basetype;
size_t counter;
int uid;
int uid;
Datalist* flat = flatten(vsym->data,args.rank);
args.generator->listbegin(args.generator,basetype,NULL,LISTDATA,totalsize,args.code,&uid);
for(counter=0;counter<totalsize;counter++) {
@ -488,5 +488,5 @@ generate_array(Symbol* vsym, Bytebuffer* code, Datalist* filler, Generator* gene
args.writer(args.generator,args.vsym,args.code,args.rank,zerosvector,args.dimsizes);
freedatalist(flat);
} else
generate_arrayR(&args, 0, index, vsym->data);
generate_arrayR(&args, 0, index, vsym->data);
}

View File

@ -67,7 +67,7 @@ main(int argc, char** argv)
size_t chunkprod;
Odometer* odom = NULL;
hsize_t offset[NC_MAX_VAR_DIMS];
if(argc < 3)
usage(0);
file_name = argv[1];
@ -80,15 +80,15 @@ main(int argc, char** argv)
if(rank == 0) usage(NC_EDIMSIZE);
if((stat=nc_inq_var_chunking(ncid,varid,&storage,chunklens))) usage(stat);
if(storage != NC_CHUNKED) usage(NC_EBADCHUNK);
chunkprod = 1;
for(i=0;i<rank;i++) {
if((stat=nc_inq_dimlen(ncid,dimids[i],&dimlens[i]))) usage(stat);
chunkcounts[i] = ceildiv(dimlens[i],chunklens[i]);
chunkprod *= chunklens[i];
chunkprod *= chunklens[i];
}
if((stat=nc_close(ncid))) usage(stat);
if ((fileid = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) usage(NC_EHDFERR);
if ((grpid = H5Gopen(fileid, "/", H5P_DEFAULT)) < 0) usage(NC_EHDFERR);
if ((datasetid = H5Dopen1(grpid, var_name)) < 0) usage(NC_EHDFERR);
@ -98,7 +98,7 @@ main(int argc, char** argv)
if((chunkdata = calloc(sizeof(int),chunkprod))==NULL) usage(NC_ENOMEM);
while(odom_more(odom)) {
setoffset(odom,chunklens,offset);
setoffset(odom,chunklens,offset);
#ifdef DEBUG
fprintf(stderr,"(");
for(i=0;i<rank;i++)
@ -139,7 +139,7 @@ odom_new(size_t rank, const size_t* stop, const size_t* max)
if((odom = calloc(1,sizeof(Odometer))) == NULL)
return NULL;
odom->rank = rank;
for(i=0;i<rank;i++) {
for(i=0;i<rank;i++) {
odom->stop[i] = stop[i];
odom->max[i] = max[i];
odom->index[i] = 0;
@ -171,7 +171,7 @@ odom_next(Odometer* odom)
}
return 1;
}
/* Get the value of the odometer */
size_t*
odom_indices(Odometer* odom)
@ -189,7 +189,7 @@ odom_offset(Odometer* odom)
for(i=0;i<odom->rank;i++) {
offset *= odom->max[i];
offset += odom->index[i];
}
}
return offset;
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,13 +1,14 @@
/* A Bison parser, made by GNU Bison 3.0.4. */
/* A Bison parser, made by GNU Bison 2.3. */
/* Bison interface for Yacc-like parsers in C
/* Skeleton interface for Bison's Yacc-like parsers in C
Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc.
Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
the Free Software Foundation; either version 2, or (at your option)
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -15,7 +16,9 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. */
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor,
Boston, MA 02110-1301, USA. */
/* As a special exception, you may create a larger work that contains
part or all of the Bison parser skeleton and distribute that work
@ -30,102 +33,143 @@
This special exception was added by the Free Software Foundation in
version 2.2 of Bison. */
#ifndef YY_NCG_NCGEN_TAB_H_INCLUDED
# define YY_NCG_NCGEN_TAB_H_INCLUDED
/* Debug traces. */
#ifndef YYDEBUG
# define YYDEBUG 1
#endif
#if YYDEBUG
extern int ncgdebug;
#endif
/* Token type. */
/* Tokens. */
#ifndef YYTOKENTYPE
# define YYTOKENTYPE
enum yytokentype
{
NC_UNLIMITED_K = 258,
CHAR_K = 259,
BYTE_K = 260,
SHORT_K = 261,
INT_K = 262,
FLOAT_K = 263,
DOUBLE_K = 264,
UBYTE_K = 265,
USHORT_K = 266,
UINT_K = 267,
INT64_K = 268,
UINT64_K = 269,
STRING_K = 270,
IDENT = 271,
TERMSTRING = 272,
CHAR_CONST = 273,
BYTE_CONST = 274,
SHORT_CONST = 275,
INT_CONST = 276,
INT64_CONST = 277,
UBYTE_CONST = 278,
USHORT_CONST = 279,
UINT_CONST = 280,
UINT64_CONST = 281,
FLOAT_CONST = 282,
DOUBLE_CONST = 283,
DIMENSIONS = 284,
VARIABLES = 285,
NETCDF = 286,
DATA = 287,
TYPES = 288,
COMPOUND = 289,
ENUM = 290,
OPAQUE_ = 291,
OPAQUESTRING = 292,
GROUP = 293,
PATH = 294,
FILLMARKER = 295,
NIL = 296,
_FILLVALUE = 297,
_FORMAT = 298,
_STORAGE = 299,
_CHUNKSIZES = 300,
_DEFLATELEVEL = 301,
_SHUFFLE = 302,
_ENDIANNESS = 303,
_NOFILL = 304,
_FLETCHER32 = 305,
_NCPROPS = 306,
_ISNETCDF4 = 307,
_SUPERBLOCK = 308,
_FILTER = 309,
DATASETID = 310
};
/* Put the tokens into the symbol table, so that GDB and other debuggers
know about them. */
enum yytokentype {
NC_UNLIMITED_K = 258,
CHAR_K = 259,
BYTE_K = 260,
SHORT_K = 261,
INT_K = 262,
FLOAT_K = 263,
DOUBLE_K = 264,
UBYTE_K = 265,
USHORT_K = 266,
UINT_K = 267,
INT64_K = 268,
UINT64_K = 269,
STRING_K = 270,
IDENT = 271,
TERMSTRING = 272,
CHAR_CONST = 273,
BYTE_CONST = 274,
SHORT_CONST = 275,
INT_CONST = 276,
INT64_CONST = 277,
UBYTE_CONST = 278,
USHORT_CONST = 279,
UINT_CONST = 280,
UINT64_CONST = 281,
FLOAT_CONST = 282,
DOUBLE_CONST = 283,
DIMENSIONS = 284,
VARIABLES = 285,
NETCDF = 286,
DATA = 287,
TYPES = 288,
COMPOUND = 289,
ENUM = 290,
OPAQUE_ = 291,
OPAQUESTRING = 292,
GROUP = 293,
PATH = 294,
FILLMARKER = 295,
NIL = 296,
_FILLVALUE = 297,
_FORMAT = 298,
_STORAGE = 299,
_CHUNKSIZES = 300,
_DEFLATELEVEL = 301,
_SHUFFLE = 302,
_ENDIANNESS = 303,
_NOFILL = 304,
_FLETCHER32 = 305,
_NCPROPS = 306,
_ISNETCDF4 = 307,
_SUPERBLOCK = 308,
_FILTER = 309,
DATASETID = 310
};
#endif
/* Tokens. */
#define NC_UNLIMITED_K 258
#define CHAR_K 259
#define BYTE_K 260
#define SHORT_K 261
#define INT_K 262
#define FLOAT_K 263
#define DOUBLE_K 264
#define UBYTE_K 265
#define USHORT_K 266
#define UINT_K 267
#define INT64_K 268
#define UINT64_K 269
#define STRING_K 270
#define IDENT 271
#define TERMSTRING 272
#define CHAR_CONST 273
#define BYTE_CONST 274
#define SHORT_CONST 275
#define INT_CONST 276
#define INT64_CONST 277
#define UBYTE_CONST 278
#define USHORT_CONST 279
#define UINT_CONST 280
#define UINT64_CONST 281
#define FLOAT_CONST 282
#define DOUBLE_CONST 283
#define DIMENSIONS 284
#define VARIABLES 285
#define NETCDF 286
#define DATA 287
#define TYPES 288
#define COMPOUND 289
#define ENUM 290
#define OPAQUE_ 291
#define OPAQUESTRING 292
#define GROUP 293
#define PATH 294
#define FILLMARKER 295
#define NIL 296
#define _FILLVALUE 297
#define _FORMAT 298
#define _STORAGE 299
#define _CHUNKSIZES 300
#define _DEFLATELEVEL 301
#define _SHUFFLE 302
#define _ENDIANNESS 303
#define _NOFILL 304
#define _FLETCHER32 305
#define _NCPROPS 306
#define _ISNETCDF4 307
#define _SUPERBLOCK 308
#define _FILTER 309
#define DATASETID 310
/* Value type. */
#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
union YYSTYPE
typedef union YYSTYPE
#line 152 "ncgen.y"
{
#line 152 "ncgen.y" /* yacc.c:1909 */
Symbol* sym;
unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/
long mark; /* track indices into the sequence*/
int nctype; /* for tracking attribute list type*/
Datalist* datalist;
NCConstant* constant;
#line 119 "ncgeny.h" /* yacc.c:1909 */
};
typedef union YYSTYPE YYSTYPE;
# define YYSTYPE_IS_TRIVIAL 1
}
/* Line 1529 of yacc.c. */
#line 168 "ncgen.tab.h"
YYSTYPE;
# define yystype YYSTYPE /* obsolescent; will be withdrawn */
# define YYSTYPE_IS_DECLARED 1
# define YYSTYPE_IS_TRIVIAL 1
#endif
extern YYSTYPE ncglval;
int ncgparse (void);
#endif /* !YY_NCG_NCGEN_TAB_H_INCLUDED */

View File

@ -35,6 +35,9 @@ ocset_curlopt(OCstate* state, int flag, void* value)
return stat;
}
/* Check return value */
#define CHECK(state,flag,value) {if(ocset_curlopt(state,flag,(void*)value) != OC_NOERR) {goto done;}}
/*
Update a specific flag from state
*/