2010-06-03 21:24:43 +08:00
|
|
|
/*********************************************************************
|
2018-12-07 06:40:43 +08:00
|
|
|
* Copyright 2018, UCAR/Unidata
|
2010-06-03 21:24:43 +08:00
|
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
|
|
* $Id: ncgen.y,v 1.42 2010/05/18 21:32:46 dmh Exp $
|
|
|
|
*********************************************************************/
|
|
|
|
|
|
|
|
/* yacc source for "ncgen", a netCDL parser and netCDF generator */
|
|
|
|
|
2021-09-03 07:04:26 +08:00
|
|
|
%define parse.error verbose
|
2011-07-28 04:48:58 +08:00
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
%{
|
|
|
|
/*
|
|
|
|
static char SccsId[] = "$Id: ncgen.y,v 1.42 2010/05/18 21:32:46 dmh Exp $";
|
|
|
|
*/
|
|
|
|
#include "includes.h"
|
2018-11-16 01:00:38 +08:00
|
|
|
#include "netcdf_aux.h"
|
2016-05-12 05:31:17 +08:00
|
|
|
#include "ncgeny.h"
|
|
|
|
#include "ncgen.h"
|
2018-02-09 10:53:40 +08:00
|
|
|
#ifdef USE_NETCDF4
|
2019-02-01 12:13:06 +08:00
|
|
|
#include "netcdf_filter.h"
|
2018-02-09 10:53:40 +08:00
|
|
|
#endif
|
2011-09-16 00:57:16 +08:00
|
|
|
|
2011-09-17 02:36:08 +08:00
|
|
|
/* Following are in ncdump (for now)*/
|
|
|
|
/* Need some (unused) definitions to get it to compile */
|
|
|
|
#define ncatt_t void*
|
|
|
|
#define ncvar_t void
|
2013-06-27 02:55:30 +08:00
|
|
|
#include "nctime.h"
|
2010-06-03 21:24:43 +08:00
|
|
|
|
2021-04-14 06:56:43 +08:00
|
|
|
#undef GENLIB1
|
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
/* parser controls */
|
|
|
|
#define YY_NO_INPUT 1
|
|
|
|
|
|
|
|
/* True if string a equals string b*/
|
2018-05-12 22:55:51 +08:00
|
|
|
#ifndef NCSTREQ
|
|
|
|
#define NCSTREQ(a, b) (*(a) == *(b) && strcmp((a), (b)) == 0)
|
2017-03-09 08:01:10 +08:00
|
|
|
#endif
|
2010-06-03 21:24:43 +08:00
|
|
|
#define VLENSIZE (sizeof(nc_vlen_t))
|
|
|
|
#define MAXFLOATDIM 4294967295.0
|
|
|
|
|
2018-11-16 01:00:38 +08:00
|
|
|
/* mnemonics */
|
2010-06-03 21:24:43 +08:00
|
|
|
typedef enum Attrkind {ATTRVAR, ATTRGLOBAL, DONTKNOW} Attrkind;
|
|
|
|
|
2018-11-16 01:00:38 +08:00
|
|
|
#define ISCONST 1
|
|
|
|
#define ISLIST 0
|
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
typedef nc_vlen_t vlen_t;
|
|
|
|
|
|
|
|
/* We retain the old representation of the symbol list
|
|
|
|
as a linked list.
|
|
|
|
*/
|
2018-11-16 01:00:38 +08:00
|
|
|
List* symlist;
|
2010-06-03 21:24:43 +08:00
|
|
|
|
|
|
|
/* Track rootgroup separately*/
|
|
|
|
Symbol* rootgroup;
|
|
|
|
|
|
|
|
/* Track the group sequence */
|
|
|
|
static List* groupstack;
|
|
|
|
|
|
|
|
/* Provide a separate sequence for accumulating values
|
|
|
|
during the parse.
|
|
|
|
*/
|
|
|
|
static List* stack;
|
|
|
|
|
|
|
|
/* track homogeneity of types for data lists*/
|
|
|
|
static nc_type consttype;
|
|
|
|
|
|
|
|
/* Misc. */
|
|
|
|
static int stackbase;
|
|
|
|
static int stacklen;
|
|
|
|
static int count;
|
|
|
|
static int opaqueid; /* counter for opaque constants*/
|
|
|
|
static int arrayuid; /* counter for pseudo-array types*/
|
|
|
|
|
|
|
|
char* primtypenames[PRIMNO] = {
|
|
|
|
"nat",
|
|
|
|
"byte", "char", "short",
|
|
|
|
"int", "float", "double",
|
|
|
|
"ubyte", "ushort", "uint",
|
|
|
|
"int64", "uint64",
|
|
|
|
"string"
|
|
|
|
};
|
|
|
|
|
2017-04-28 03:01:59 +08:00
|
|
|
static int GLOBAL_SPECIAL = _NCPROPS_FLAG
|
|
|
|
| _ISNETCDF4_FLAG
|
|
|
|
| _SUPERBLOCK_FLAG
|
|
|
|
| _FORMAT_FLAG ;
|
2016-05-04 11:17:06 +08:00
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
/*Defined in ncgen.l*/
|
|
|
|
extern int lineno; /* line number for error messages */
|
2012-02-14 08:25:32 +08:00
|
|
|
extern Bytebuffer* lextext; /* name or string with escapes removed */
|
2010-06-03 21:24:43 +08:00
|
|
|
|
|
|
|
extern double double_val; /* last double value read */
|
|
|
|
extern float float_val; /* last float value read */
|
|
|
|
extern long long int64_val; /* last int64 value read */
|
|
|
|
extern int int32_val; /* last int32 value read */
|
|
|
|
extern short int16_val; /* last short value read */
|
|
|
|
extern unsigned long long uint64_val; /* last int64 value read */
|
|
|
|
extern unsigned int uint32_val; /* last int32 value read */
|
|
|
|
extern unsigned short uint16_val; /* last short value read */
|
|
|
|
extern char char_val; /* last char value read */
|
|
|
|
extern signed char byte_val; /* last byte value read */
|
|
|
|
extern unsigned char ubyte_val; /* last byte value read */
|
|
|
|
|
|
|
|
/* Track definitions of dims, types, attributes, and vars*/
|
|
|
|
List* grpdefs;
|
|
|
|
List* dimdefs;
|
|
|
|
List* attdefs; /* variable-specific attributes*/
|
|
|
|
List* gattdefs; /* global attributes only*/
|
|
|
|
List* xattdefs; /* unknown attributes*/
|
|
|
|
List* typdefs;
|
|
|
|
List* vardefs;
|
|
|
|
List* tmp;
|
|
|
|
|
|
|
|
/* Forward */
|
2018-11-16 01:00:38 +08:00
|
|
|
static NCConstant* makeconstdata(nc_type);
|
|
|
|
static NCConstant* evaluate(Symbol* fcn, Datalist* arglist);
|
|
|
|
static NCConstant* makeenumconstref(Symbol*);
|
2010-06-03 21:24:43 +08:00
|
|
|
static void addtogroup(Symbol*);
|
|
|
|
static Symbol* currentgroup(void);
|
2013-09-21 10:31:21 +08:00
|
|
|
static Symbol* createrootgroup(const char*);
|
2010-06-03 21:24:43 +08:00
|
|
|
static Symbol* creategroup(Symbol*);
|
|
|
|
static int dupobjectcheck(nc_class,Symbol*);
|
|
|
|
static void setpathcurrent(Symbol* sym);
|
|
|
|
static Symbol* makeattribute(Symbol*,Symbol*,Symbol*,Datalist*,Attrkind);
|
|
|
|
static Symbol* makeprimitivetype(nc_type i);
|
|
|
|
static Symbol* makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst);
|
|
|
|
static int containsfills(Datalist* list);
|
2011-05-13 01:51:32 +08:00
|
|
|
static void vercheck(int ncid);
|
2018-11-16 01:00:38 +08:00
|
|
|
static long long extractint(NCConstant* con);
|
2018-02-09 10:53:40 +08:00
|
|
|
#ifdef USE_NETCDF4
|
2017-05-15 08:10:02 +08:00
|
|
|
static int parsefilterflag(const char* sdata0, Specialdata* special);
|
2021-09-03 07:04:26 +08:00
|
|
|
static int parsecodecsflag(const char* sdata0, Specialdata* special);
|
2021-12-24 13:18:56 +08:00
|
|
|
static Symbol* identkeyword(const Symbol*);
|
|
|
|
|
2020-02-17 03:59:33 +08:00
|
|
|
#ifdef GENDEBUG1
|
2020-09-28 02:43:46 +08:00
|
|
|
static void printfilters(int nfilters, NC_ParsedFilterSpec** filters);
|
2020-02-17 03:59:33 +08:00
|
|
|
#endif
|
2018-02-09 10:53:40 +08:00
|
|
|
#endif
|
2010-06-03 21:24:43 +08:00
|
|
|
|
|
|
|
int yylex(void);
|
|
|
|
|
|
|
|
#ifndef NO_STDARG
|
|
|
|
static void yyerror(const char *fmt, ...);
|
|
|
|
#else
|
|
|
|
static void yyerror(fmt,va_alist) const char* fmt; va_dcl;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* Extern */
|
|
|
|
extern int lex_init(void);
|
|
|
|
|
|
|
|
%}
|
|
|
|
|
|
|
|
/* DECLARATIONS */
|
|
|
|
|
|
|
|
%union {
|
|
|
|
Symbol* sym;
|
|
|
|
unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/
|
|
|
|
long mark; /* track indices into the sequence*/
|
|
|
|
int nctype; /* for tracking attribute list type*/
|
|
|
|
Datalist* datalist;
|
2018-11-16 01:00:38 +08:00
|
|
|
NCConstant* constant;
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
%token <sym>
|
|
|
|
NC_UNLIMITED_K /* keyword for unbounded record dimension */
|
|
|
|
CHAR_K /* keyword for char datatype */
|
|
|
|
BYTE_K /* keyword for byte datatype */
|
|
|
|
SHORT_K /* keyword for short datatype */
|
|
|
|
INT_K /* keyword for int datatype */
|
|
|
|
FLOAT_K /* keyword for float datatype */
|
|
|
|
DOUBLE_K /* keyword for double datatype */
|
2017-04-28 03:01:59 +08:00
|
|
|
UBYTE_K /* keyword for unsigned byte datatype */
|
|
|
|
USHORT_K /* keyword for unsigned short datatype */
|
|
|
|
UINT_K /* keyword for unsigned int datatype */
|
|
|
|
INT64_K /* keyword for long long datatype */
|
|
|
|
UINT64_K /* keyword for unsigned long long datatype */
|
|
|
|
STRING_K /* keyword for string datatype */
|
2010-06-03 21:24:43 +08:00
|
|
|
IDENT /* name for a dimension, variable, or attribute */
|
|
|
|
TERMSTRING /* terminal string */
|
|
|
|
CHAR_CONST /* char constant (not ever generated by ncgen.l) */
|
|
|
|
BYTE_CONST /* byte constant */
|
|
|
|
SHORT_CONST /* short constant */
|
|
|
|
INT_CONST /* int constant */
|
|
|
|
INT64_CONST /* long long constant */
|
|
|
|
UBYTE_CONST /* unsigned byte constant */
|
|
|
|
USHORT_CONST /* unsigned short constant */
|
2016-01-06 12:26:25 +08:00
|
|
|
UINT_CONST /* unsigned int constant */
|
|
|
|
UINT64_CONST /* unsigned long long constant */
|
2010-06-03 21:24:43 +08:00
|
|
|
FLOAT_CONST /* float constant */
|
2013-07-11 04:00:48 +08:00
|
|
|
DOUBLE_CONST/* double constant */
|
2010-06-03 21:24:43 +08:00
|
|
|
DIMENSIONS /* keyword starting dimensions section, if any */
|
|
|
|
VARIABLES /* keyword starting variables section, if any */
|
|
|
|
NETCDF /* keyword declaring netcdf name */
|
|
|
|
DATA /* keyword starting data section, if any */
|
|
|
|
TYPES
|
|
|
|
COMPOUND
|
|
|
|
ENUM
|
2017-02-24 13:51:32 +08:00
|
|
|
OPAQUE_ /* 'OPAQUE' apparently conflicts with HDF4 code */
|
2010-06-03 21:24:43 +08:00
|
|
|
OPAQUESTRING /* 0x<even number of hexdigits> */
|
|
|
|
GROUP
|
2013-09-21 10:31:21 +08:00
|
|
|
PATH /* / or (/IDENT)+(.IDENT)? */
|
2010-06-03 21:24:43 +08:00
|
|
|
FILLMARKER /* "_" as opposed to the attribute */
|
2013-07-11 04:00:48 +08:00
|
|
|
NIL /* NIL */
|
2010-06-03 21:24:43 +08:00
|
|
|
_FILLVALUE
|
|
|
|
_FORMAT
|
|
|
|
_STORAGE
|
|
|
|
_CHUNKSIZES
|
|
|
|
_DEFLATELEVEL
|
|
|
|
_SHUFFLE
|
|
|
|
_ENDIANNESS
|
|
|
|
_NOFILL
|
|
|
|
_FLETCHER32
|
2016-05-04 11:17:06 +08:00
|
|
|
_NCPROPS
|
|
|
|
_ISNETCDF4
|
|
|
|
_SUPERBLOCK
|
2017-05-15 08:10:02 +08:00
|
|
|
_FILTER
|
2021-09-03 07:04:26 +08:00
|
|
|
_CODECS
|
2022-01-29 04:04:16 +08:00
|
|
|
_QUANTIZEBG
|
2022-02-20 07:55:36 +08:00
|
|
|
_QUANTIZEGBR
|
2015-06-19 04:31:10 +08:00
|
|
|
DATASETID
|
2010-06-03 21:24:43 +08:00
|
|
|
|
2012-01-16 09:29:19 +08:00
|
|
|
%type <sym> ident typename primtype dimd varspec
|
2010-06-03 21:24:43 +08:00
|
|
|
attrdecl enumid path dimref fielddim fieldspec
|
2021-12-24 13:18:56 +08:00
|
|
|
varident
|
2010-06-03 21:24:43 +08:00
|
|
|
%type <sym> typeref
|
|
|
|
%type <sym> varref
|
2020-06-06 07:03:29 +08:00
|
|
|
%type <sym> ambiguous_ref
|
2010-06-03 21:24:43 +08:00
|
|
|
%type <mark> enumidlist fieldlist fields varlist dimspec dimlist field
|
|
|
|
fielddimspec fielddimlist
|
|
|
|
%type <constant> dataitem constdata constint conststring constbool
|
2013-09-21 10:31:21 +08:00
|
|
|
%type <constant> simpleconstant function econstref
|
2011-09-16 00:57:16 +08:00
|
|
|
%type <datalist> datalist intlist datalist1 datalist0 arglist
|
2010-06-03 21:24:43 +08:00
|
|
|
|
|
|
|
|
|
|
|
%start ncdesc /* start symbol for grammar */
|
|
|
|
|
|
|
|
%%
|
|
|
|
|
|
|
|
/* RULES */
|
|
|
|
|
|
|
|
ncdesc: NETCDF
|
2013-09-21 10:31:21 +08:00
|
|
|
datasetid
|
2010-06-03 21:24:43 +08:00
|
|
|
rootgroup
|
2012-03-15 07:26:48 +08:00
|
|
|
{if (error_count > 0) YYABORT;}
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
2013-09-21 10:31:21 +08:00
|
|
|
datasetid: DATASETID {createrootgroup(datasetname);};
|
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
rootgroup: '{'
|
|
|
|
groupbody
|
|
|
|
subgrouplist
|
|
|
|
'}';
|
|
|
|
|
|
|
|
/* 2/3/08 - Allow group body with only attributes. (H/T John Storrs). */
|
|
|
|
groupbody:
|
|
|
|
attrdecllist
|
|
|
|
typesection /* Type definitions */
|
|
|
|
dimsection /* dimension declarations */
|
|
|
|
vasection /* variable and attribute declarations */
|
|
|
|
datasection /* data for variables within the group */
|
|
|
|
;
|
|
|
|
|
|
|
|
subgrouplist: /*empty*/ | subgrouplist namedgroup;
|
|
|
|
|
2012-01-16 09:29:19 +08:00
|
|
|
namedgroup: GROUP ident '{'
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
2010-08-26 03:01:07 +08:00
|
|
|
Symbol* id = $2;
|
2011-05-13 01:51:32 +08:00
|
|
|
markcdf4("Group specification");
|
2015-06-19 04:31:10 +08:00
|
|
|
if(creategroup(id) == NULL)
|
2010-06-03 21:24:43 +08:00
|
|
|
yyerror("duplicate group declaration within parent group for %s",
|
2010-08-26 03:01:07 +08:00
|
|
|
id->name);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
groupbody
|
|
|
|
subgrouplist
|
|
|
|
{listpop(groupstack);}
|
|
|
|
'}'
|
|
|
|
attrdecllist
|
|
|
|
;
|
2015-06-19 04:31:10 +08:00
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
typesection: /* empty */
|
|
|
|
| TYPES {}
|
2011-05-13 01:51:32 +08:00
|
|
|
| TYPES typedecls
|
|
|
|
{markcdf4("Type specification");}
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
typedecls: type_or_attr_decl | typedecls type_or_attr_decl ;
|
|
|
|
|
2012-01-16 09:29:19 +08:00
|
|
|
typename: ident
|
2010-06-03 21:24:43 +08:00
|
|
|
{ /* Use when defining a type */
|
|
|
|
$1->objectclass = NC_TYPE;
|
|
|
|
if(dupobjectcheck(NC_TYPE,$1))
|
|
|
|
yyerror("duplicate type declaration for %s",
|
|
|
|
$1->name);
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush(typdefs,(void*)$1);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
type_or_attr_decl: typedecl {} | attrdecl ';' {} ;
|
|
|
|
|
2010-11-10 06:53:03 +08:00
|
|
|
typedecl:
|
|
|
|
enumdecl optsemicolon
|
|
|
|
| compounddecl optsemicolon
|
|
|
|
| vlendecl optsemicolon
|
|
|
|
| opaquedecl optsemicolon
|
|
|
|
;
|
|
|
|
|
|
|
|
optsemicolon: /*empty*/ | ';' ;
|
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
|
|
|
|
enumdecl: primtype ENUM typename
|
|
|
|
'{' enumidlist '}'
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
addtogroup($3); /* sets prefix*/
|
|
|
|
$3->objectclass=NC_TYPE;
|
|
|
|
$3->subclass=NC_ENUM;
|
|
|
|
$3->typ.basetype=$1;
|
|
|
|
$3->typ.size = $1->typ.size;
|
|
|
|
$3->typ.alignment = $1->typ.alignment;
|
|
|
|
stackbase=$5;
|
|
|
|
stacklen=listlength(stack);
|
|
|
|
$3->subnodes = listnew();
|
|
|
|
/* Variety of field fixups*/
|
|
|
|
/* 1. add in the enum values*/
|
|
|
|
/* 2. make this type be their container*/
|
|
|
|
/* 3. make constant names visible in the group*/
|
|
|
|
/* 4. set field basetype to be same as enum basetype*/
|
|
|
|
for(i=stackbase;i<stacklen;i++) {
|
|
|
|
Symbol* eid = (Symbol*)listget(stack,i);
|
|
|
|
assert(eid->subclass == NC_ECONST);
|
|
|
|
addtogroup(eid);
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush($3->subnodes,(void*)eid);
|
2010-06-03 21:24:43 +08:00
|
|
|
eid->container = $3;
|
|
|
|
eid->typ.basetype = $3->typ.basetype;
|
2015-06-19 04:31:10 +08:00
|
|
|
}
|
2010-06-03 21:24:43 +08:00
|
|
|
listsetlength(stack,stackbase);/* remove stack nodes*/
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
enumidlist: enumid
|
2012-08-20 05:54:30 +08:00
|
|
|
{$$=listlength(stack); listpush(stack,(void*)$1);}
|
2010-06-03 21:24:43 +08:00
|
|
|
| enumidlist ',' enumid
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
$$=$1;
|
|
|
|
/* check for duplicates*/
|
|
|
|
stackbase=$1;
|
|
|
|
stacklen=listlength(stack);
|
|
|
|
for(i=stackbase;i<stacklen;i++) {
|
|
|
|
Symbol* elem = (Symbol*)listget(stack,i);
|
|
|
|
if(strcmp($3->name,elem->name)==0)
|
|
|
|
yyerror("duplicate enum declaration for %s",
|
|
|
|
elem->name);
|
2015-06-19 04:31:10 +08:00
|
|
|
}
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush(stack,(void*)$3);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
;
|
|
|
|
|
2013-09-21 10:31:21 +08:00
|
|
|
enumid: ident '=' constint
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
|
|
|
$1->objectclass=NC_TYPE;
|
|
|
|
$1->subclass=NC_ECONST;
|
|
|
|
$1->typ.econst=$3;
|
|
|
|
$$=$1;
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
2017-02-24 13:34:11 +08:00
|
|
|
opaquedecl: OPAQUE_ '(' INT_CONST ')' typename
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
2011-05-13 01:51:32 +08:00
|
|
|
vercheck(NC_OPAQUE);
|
2010-06-03 21:24:43 +08:00
|
|
|
addtogroup($5); /*sets prefix*/
|
|
|
|
$5->objectclass=NC_TYPE;
|
|
|
|
$5->subclass=NC_OPAQUE;
|
|
|
|
$5->typ.typecode=NC_OPAQUE;
|
|
|
|
$5->typ.size=int32_val;
|
Fix various problem around VLEN's
re: https://github.com/Unidata/netcdf-c/issues/541
re: https://github.com/Unidata/netcdf-c/issues/1208
re: https://github.com/Unidata/netcdf-c/issues/2078
re: https://github.com/Unidata/netcdf-c/issues/2041
re: https://github.com/Unidata/netcdf-c/issues/2143
For a long time, there have been known problems with the
management of complex types containing VLENs. This also
involves the string type because it is stored as a VLEN of
chars.
This PR (mostly) fixes this problem. But note that it adds new
functions to netcdf.h (see below) and this may require bumping
the .so number. These new functions can be removed, if desired,
in favor of functions in netcdf_aux.h, but netcdf.h seems the
better place for them because they are intended as alternatives
to the nc_free_vlen and nc_free_string functions already in
netcdf.h.
The term complex type refers to any type that directly or
transitively references a VLEN type. So an array of VLENS, a
compound with a VLEN field, and so on.
In order to properly handle instances of these complex types, it
is necessary to have function that can recursively walk
instances of such types to perform various actions on them. The
term "deep" is also used to mean recursive.
At the moment, the two operations needed by the netcdf library are:
* free'ing an instance of the complex type
* copying an instance of the complex type.
The current library does only shallow free and shallow copy of
complex types. This means that only the top level is properly
free'd or copied, but deep internal blocks in the instance are
not touched.
Note that the term "vector" will be used to mean a contiguous (in
memory) sequence of instances of some type. Given an array with,
say, dimensions 2 X 3 X 4, this will be stored in memory as a
vector of length 2*3*4=24 instances.
The use cases are primarily these.
## nc_get_vars
Suppose one is reading a vector of instances using nc_get_vars
(or nc_get_vara or nc_get_var, etc.). These functions will
return the vector in the top-level memory provided. All
interior blocks (form nested VLEN or strings) will have been
dynamically allocated.
After using this vector of instances, it is necessary to free
(aka reclaim) the dynamically allocated memory, otherwise a
memory leak occurs. So, the recursive reclaim function is used
to walk the returned instance vector and do a deep reclaim of
the data.
Currently functions are defined in netcdf.h that are supposed to
handle this: nc_free_vlen(), nc_free_vlens(), and
nc_free_string(). Unfortunately, these functions only do a
shallow free, so deeply nested instances are not properly
handled by them.
Note that internally, the provided data is immediately written so
there is no need to copy it. But the caller may need to reclaim the
data it passed into the function.
## nc_put_att
Suppose one is writing a vector of instances as the data of an attribute
using, say, nc_put_att.
Internally, the incoming attribute data must be copied and stored
so that changes/reclamation of the input data will not affect
the attribute.
Again, the code inside the netcdf library does only shallow copying
rather than deep copy. As a result, one sees effects such as described
in Github Issue https://github.com/Unidata/netcdf-c/issues/2143.
Also, after defining the attribute, it may be necessary for the user
to free the data that was provided as input to nc_put_att().
## nc_get_att
Suppose one is reading a vector of instances as the data of an attribute
using, say, nc_get_att.
Internally, the existing attribute data must be copied and returned
to the caller, and the caller is responsible for reclaiming
the returned data.
Again, the code inside the netcdf library does only shallow copying
rather than deep copy. So this can lead to memory leaks and errors
because the deep data is shared between the library and the user.
# Solution
The solution is to build properly recursive reclaim and copy
functions and use those as needed.
These recursive functions are defined in libdispatch/dinstance.c
and their signatures are defined in include/netcdf.h.
For back compatibility, corresponding "ncaux_XXX" functions
are defined in include/netcdf_aux.h.
````
int nc_reclaim_data(int ncid, nc_type xtypeid, void* memory, size_t count);
int nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t count);
int nc_copy_data(int ncid, nc_type xtypeid, const void* memory, size_t count, void* copy);
int nc_copy_data_all(int ncid, nc_type xtypeid, const void* memory, size_t count, void** copyp);
````
There are two variants. The first two, nc_reclaim_data() and
nc_copy_data(), assume the top-level vector is managed by the
caller. For reclaim, this is so the user can use, for example, a
statically allocated vector. For copy, it assumes the user
provides the space into which the copy is stored.
The second two, nc_reclaim_data_all() and
nc_copy_data_all(), allows the functions to manage the
top-level. So for nc_reclaim_data_all, the top level is
assumed to be dynamically allocated and will be free'd by
nc_reclaim_data_all(). The nc_copy_data_all() function
will allocate the top level and return a pointer to it to the
user. The user can later pass that pointer to
nc_reclaim_data_all() to reclaim the instance(s).
# Internal Changes
The netcdf-c library internals are changed to use the proper
reclaim and copy functions. It turns out that the places where
these functions are needed is quite pervasive in the netcdf-c
library code. Using these functions also allows some
simplification of the code since the stdata and vldata fields of
NC_ATT_INFO are no longer needed. Currently this is commented
out using the SEPDATA \#define macro. When any bugs are largely
fixed, all this code will be removed.
# Known Bugs
1. There is still one known failure that has not been solved.
All the failures revolve around some variant of this .cdl file.
The proximate cause of failure is the use of a VLEN FillValue.
````
netcdf x {
types:
float(*) row_of_floats ;
dimensions:
m = 5 ;
variables:
row_of_floats ragged_array(m) ;
row_of_floats ragged_array:_FillValue = {-999} ;
data:
ragged_array = {10, 11, 12, 13, 14}, {20, 21, 22, 23}, {30, 31, 32},
{40, 41}, _ ;
}
````
When a solution is found, I will either add it to this PR or post a new PR.
# Related Changes
* Mark nc_free_vlen(s) as deprecated in favor of ncaux_reclaim_data.
* Remove the --enable-unfixed-memory-leaks option.
* Remove the NC_VLENS_NOTEST code that suppresses some vlen tests.
* Document this change in docs/internal.md
* Disable the tst_vlen_data test in ncdump/tst_nccopy4.sh.
* Mark types as fixed size or not (transitively) to optimize the reclaim
and copy functions.
# Misc. Changes
* Make Doxygen process libdispatch/daux.c
* Make sure the NC_ATT_INFO_T.container field is set.
2022-01-09 09:30:00 +08:00
|
|
|
(void)ncaux_class_alignment(NC_OPAQUE,&$5->typ.alignment);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
vlendecl: typeref '(' '*' ')' typename
|
|
|
|
{
|
|
|
|
Symbol* basetype = $1;
|
2011-05-13 01:51:32 +08:00
|
|
|
vercheck(NC_VLEN);
|
2010-06-03 21:24:43 +08:00
|
|
|
addtogroup($5); /*sets prefix*/
|
|
|
|
$5->objectclass=NC_TYPE;
|
|
|
|
$5->subclass=NC_VLEN;
|
|
|
|
$5->typ.basetype=basetype;
|
|
|
|
$5->typ.typecode=NC_VLEN;
|
|
|
|
$5->typ.size=VLENSIZE;
|
Fix various problem around VLEN's
re: https://github.com/Unidata/netcdf-c/issues/541
re: https://github.com/Unidata/netcdf-c/issues/1208
re: https://github.com/Unidata/netcdf-c/issues/2078
re: https://github.com/Unidata/netcdf-c/issues/2041
re: https://github.com/Unidata/netcdf-c/issues/2143
For a long time, there have been known problems with the
management of complex types containing VLENs. This also
involves the string type because it is stored as a VLEN of
chars.
This PR (mostly) fixes this problem. But note that it adds new
functions to netcdf.h (see below) and this may require bumping
the .so number. These new functions can be removed, if desired,
in favor of functions in netcdf_aux.h, but netcdf.h seems the
better place for them because they are intended as alternatives
to the nc_free_vlen and nc_free_string functions already in
netcdf.h.
The term complex type refers to any type that directly or
transitively references a VLEN type. So an array of VLENS, a
compound with a VLEN field, and so on.
In order to properly handle instances of these complex types, it
is necessary to have function that can recursively walk
instances of such types to perform various actions on them. The
term "deep" is also used to mean recursive.
At the moment, the two operations needed by the netcdf library are:
* free'ing an instance of the complex type
* copying an instance of the complex type.
The current library does only shallow free and shallow copy of
complex types. This means that only the top level is properly
free'd or copied, but deep internal blocks in the instance are
not touched.
Note that the term "vector" will be used to mean a contiguous (in
memory) sequence of instances of some type. Given an array with,
say, dimensions 2 X 3 X 4, this will be stored in memory as a
vector of length 2*3*4=24 instances.
The use cases are primarily these.
## nc_get_vars
Suppose one is reading a vector of instances using nc_get_vars
(or nc_get_vara or nc_get_var, etc.). These functions will
return the vector in the top-level memory provided. All
interior blocks (form nested VLEN or strings) will have been
dynamically allocated.
After using this vector of instances, it is necessary to free
(aka reclaim) the dynamically allocated memory, otherwise a
memory leak occurs. So, the recursive reclaim function is used
to walk the returned instance vector and do a deep reclaim of
the data.
Currently functions are defined in netcdf.h that are supposed to
handle this: nc_free_vlen(), nc_free_vlens(), and
nc_free_string(). Unfortunately, these functions only do a
shallow free, so deeply nested instances are not properly
handled by them.
Note that internally, the provided data is immediately written so
there is no need to copy it. But the caller may need to reclaim the
data it passed into the function.
## nc_put_att
Suppose one is writing a vector of instances as the data of an attribute
using, say, nc_put_att.
Internally, the incoming attribute data must be copied and stored
so that changes/reclamation of the input data will not affect
the attribute.
Again, the code inside the netcdf library does only shallow copying
rather than deep copy. As a result, one sees effects such as described
in Github Issue https://github.com/Unidata/netcdf-c/issues/2143.
Also, after defining the attribute, it may be necessary for the user
to free the data that was provided as input to nc_put_att().
## nc_get_att
Suppose one is reading a vector of instances as the data of an attribute
using, say, nc_get_att.
Internally, the existing attribute data must be copied and returned
to the caller, and the caller is responsible for reclaiming
the returned data.
Again, the code inside the netcdf library does only shallow copying
rather than deep copy. So this can lead to memory leaks and errors
because the deep data is shared between the library and the user.
# Solution
The solution is to build properly recursive reclaim and copy
functions and use those as needed.
These recursive functions are defined in libdispatch/dinstance.c
and their signatures are defined in include/netcdf.h.
For back compatibility, corresponding "ncaux_XXX" functions
are defined in include/netcdf_aux.h.
````
int nc_reclaim_data(int ncid, nc_type xtypeid, void* memory, size_t count);
int nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t count);
int nc_copy_data(int ncid, nc_type xtypeid, const void* memory, size_t count, void* copy);
int nc_copy_data_all(int ncid, nc_type xtypeid, const void* memory, size_t count, void** copyp);
````
There are two variants. The first two, nc_reclaim_data() and
nc_copy_data(), assume the top-level vector is managed by the
caller. For reclaim, this is so the user can use, for example, a
statically allocated vector. For copy, it assumes the user
provides the space into which the copy is stored.
The second two, nc_reclaim_data_all() and
nc_copy_data_all(), allows the functions to manage the
top-level. So for nc_reclaim_data_all, the top level is
assumed to be dynamically allocated and will be free'd by
nc_reclaim_data_all(). The nc_copy_data_all() function
will allocate the top level and return a pointer to it to the
user. The user can later pass that pointer to
nc_reclaim_data_all() to reclaim the instance(s).
# Internal Changes
The netcdf-c library internals are changed to use the proper
reclaim and copy functions. It turns out that the places where
these functions are needed is quite pervasive in the netcdf-c
library code. Using these functions also allows some
simplification of the code since the stdata and vldata fields of
NC_ATT_INFO are no longer needed. Currently this is commented
out using the SEPDATA \#define macro. When any bugs are largely
fixed, all this code will be removed.
# Known Bugs
1. There is still one known failure that has not been solved.
All the failures revolve around some variant of this .cdl file.
The proximate cause of failure is the use of a VLEN FillValue.
````
netcdf x {
types:
float(*) row_of_floats ;
dimensions:
m = 5 ;
variables:
row_of_floats ragged_array(m) ;
row_of_floats ragged_array:_FillValue = {-999} ;
data:
ragged_array = {10, 11, 12, 13, 14}, {20, 21, 22, 23}, {30, 31, 32},
{40, 41}, _ ;
}
````
When a solution is found, I will either add it to this PR or post a new PR.
# Related Changes
* Mark nc_free_vlen(s) as deprecated in favor of ncaux_reclaim_data.
* Remove the --enable-unfixed-memory-leaks option.
* Remove the NC_VLENS_NOTEST code that suppresses some vlen tests.
* Document this change in docs/internal.md
* Disable the tst_vlen_data test in ncdump/tst_nccopy4.sh.
* Mark types as fixed size or not (transitively) to optimize the reclaim
and copy functions.
# Misc. Changes
* Make Doxygen process libdispatch/daux.c
* Make sure the NC_ATT_INFO_T.container field is set.
2022-01-09 09:30:00 +08:00
|
|
|
(void)ncaux_class_alignment(NC_VLEN,&$5->typ.alignment);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
compounddecl: COMPOUND typename '{' fields '}'
|
|
|
|
{
|
|
|
|
int i,j;
|
2011-05-13 01:51:32 +08:00
|
|
|
vercheck(NC_COMPOUND);
|
2010-06-03 21:24:43 +08:00
|
|
|
addtogroup($2);
|
|
|
|
/* check for duplicate field names*/
|
|
|
|
stackbase=$4;
|
|
|
|
stacklen=listlength(stack);
|
|
|
|
for(i=stackbase;i<stacklen;i++) {
|
|
|
|
Symbol* elem1 = (Symbol*)listget(stack,i);
|
|
|
|
for(j=i+1;j<stacklen;j++) {
|
|
|
|
Symbol* elem2 = (Symbol*)listget(stack,j);
|
|
|
|
if(strcmp(elem1->name,elem2->name)==0) {
|
|
|
|
yyerror("duplicate field declaration for %s",elem1->name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
$2->objectclass=NC_TYPE;
|
|
|
|
$2->subclass=NC_COMPOUND;
|
|
|
|
$2->typ.basetype=NULL;
|
|
|
|
$2->typ.typecode=NC_COMPOUND;
|
|
|
|
$2->subnodes = listnew();
|
|
|
|
/* Add in the fields*/
|
|
|
|
for(i=stackbase;i<stacklen;i++) {
|
|
|
|
Symbol* fsym = (Symbol*)listget(stack,i);
|
|
|
|
fsym->container = $2;
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush($2->subnodes,(void*)fsym);
|
2015-06-19 04:31:10 +08:00
|
|
|
}
|
2010-06-03 21:24:43 +08:00
|
|
|
listsetlength(stack,stackbase);/* remove stack nodes*/
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
|
|
|
|
fields: field ';' {$$=$1;}
|
|
|
|
| fields field ';' {$$=$1;}
|
|
|
|
;
|
|
|
|
|
|
|
|
field: typeref fieldlist
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
$$=$2;
|
|
|
|
stackbase=$2;
|
|
|
|
stacklen=listlength(stack);
|
|
|
|
/* process each field in the fieldlist*/
|
|
|
|
for(i=stackbase;i<stacklen;i++) {
|
|
|
|
Symbol* f = (Symbol*)listget(stack,i);
|
|
|
|
f->typ.basetype = $1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
primtype: CHAR_K { $$ = primsymbols[NC_CHAR]; }
|
|
|
|
| BYTE_K { $$ = primsymbols[NC_BYTE]; }
|
|
|
|
| SHORT_K { $$ = primsymbols[NC_SHORT]; }
|
|
|
|
| INT_K { $$ = primsymbols[NC_INT]; }
|
|
|
|
| FLOAT_K { $$ = primsymbols[NC_FLOAT]; }
|
|
|
|
| DOUBLE_K{ $$ = primsymbols[NC_DOUBLE]; }
|
2011-05-13 01:51:32 +08:00
|
|
|
| UBYTE_K { vercheck(NC_UBYTE); $$ = primsymbols[NC_UBYTE]; }
|
|
|
|
| USHORT_K { vercheck(NC_USHORT); $$ = primsymbols[NC_USHORT]; }
|
|
|
|
| UINT_K { vercheck(NC_UINT); $$ = primsymbols[NC_UINT]; }
|
|
|
|
| INT64_K { vercheck(NC_INT64); $$ = primsymbols[NC_INT64]; }
|
|
|
|
| UINT64_K { vercheck(NC_UINT64); $$ = primsymbols[NC_UINT64]; }
|
2017-04-28 03:01:59 +08:00
|
|
|
| STRING_K { vercheck(NC_STRING); $$ = primsymbols[NC_STRING]; }
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
dimsection: /* empty */
|
|
|
|
| DIMENSIONS {}
|
|
|
|
| DIMENSIONS dimdecls {}
|
|
|
|
;
|
|
|
|
|
|
|
|
dimdecls: dim_or_attr_decl ';'
|
|
|
|
| dimdecls dim_or_attr_decl ';'
|
|
|
|
;
|
|
|
|
|
|
|
|
dim_or_attr_decl: dimdeclist {} | attrdecl {} ;
|
|
|
|
|
|
|
|
dimdeclist: dimdecl
|
|
|
|
| dimdeclist ',' dimdecl
|
|
|
|
;
|
|
|
|
|
|
|
|
dimdecl:
|
2016-05-04 11:17:06 +08:00
|
|
|
dimd '=' constint
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
2016-05-04 11:17:06 +08:00
|
|
|
$1->dim.declsize = (size_t)extractint($3);
|
2013-11-15 06:13:20 +08:00
|
|
|
#ifdef GENDEBUG1
|
2016-01-06 12:26:25 +08:00
|
|
|
fprintf(stderr,"dimension: %s = %llu\n",$1->name,(unsigned long long)$1->dim.declsize);
|
2012-02-14 08:25:32 +08:00
|
|
|
#endif
|
2018-11-16 01:00:38 +08:00
|
|
|
reclaimconstant($3);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
| dimd '=' NC_UNLIMITED_K
|
|
|
|
{
|
2012-02-14 08:25:32 +08:00
|
|
|
$1->dim.declsize = NC_UNLIMITED;
|
|
|
|
$1->dim.isunlimited = 1;
|
2013-11-15 06:13:20 +08:00
|
|
|
#ifdef GENDEBUG1
|
2012-02-14 08:25:32 +08:00
|
|
|
fprintf(stderr,"dimension: %s = UNLIMITED\n",$1->name);
|
|
|
|
#endif
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
;
|
|
|
|
|
2012-01-16 09:29:19 +08:00
|
|
|
dimd: ident
|
2015-06-19 04:31:10 +08:00
|
|
|
{
|
2010-06-03 21:24:43 +08:00
|
|
|
$1->objectclass=NC_DIM;
|
|
|
|
if(dupobjectcheck(NC_DIM,$1))
|
|
|
|
yyerror( "Duplicate dimension declaration for %s",
|
|
|
|
$1->name);
|
|
|
|
addtogroup($1);
|
|
|
|
$$=$1;
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush(dimdefs,(void*)$1);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
vasection: /* empty */
|
|
|
|
| VARIABLES {}
|
|
|
|
| VARIABLES vadecls {}
|
|
|
|
;
|
|
|
|
|
|
|
|
vadecls: vadecl_or_attr ';'
|
|
|
|
| vadecls vadecl_or_attr ';'
|
|
|
|
;
|
|
|
|
|
|
|
|
vadecl_or_attr: vardecl {} | attrdecl {} ;
|
|
|
|
|
|
|
|
vardecl: typeref varlist
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
stackbase=$2;
|
|
|
|
stacklen=listlength(stack);
|
|
|
|
/* process each variable in the varlist*/
|
|
|
|
for(i=stackbase;i<stacklen;i++) {
|
|
|
|
Symbol* sym = (Symbol*)listget(stack,i);
|
|
|
|
sym->objectclass = NC_VAR;
|
|
|
|
if(dupobjectcheck(NC_VAR,sym)) {
|
|
|
|
yyerror("Duplicate variable declaration for %s",
|
|
|
|
sym->name);
|
|
|
|
} else {
|
|
|
|
sym->typ.basetype = $1;
|
|
|
|
addtogroup(sym);
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush(vardefs,(void*)sym);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
listsetlength(stack,stackbase);/* remove stack nodes*/
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
varlist: varspec
|
|
|
|
{$$=listlength(stack);
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush(stack,(void*)$1);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
| varlist ',' varspec
|
2012-08-20 05:54:30 +08:00
|
|
|
{$$=$1; listpush(stack,(void*)$3);}
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
2021-12-24 13:18:56 +08:00
|
|
|
varspec: varident dimspec
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
Dimset dimset;
|
2018-11-16 01:00:38 +08:00
|
|
|
Symbol* var = $1; /* for debugging */
|
2010-06-03 21:24:43 +08:00
|
|
|
stacklen=listlength(stack);
|
|
|
|
stackbase=$2;
|
|
|
|
count = stacklen - stackbase;
|
|
|
|
if(count >= NC_MAX_VAR_DIMS) {
|
|
|
|
yyerror("%s has too many dimensions",$1->name);
|
|
|
|
count = NC_MAX_VAR_DIMS - 1;
|
|
|
|
stacklen = stackbase + count;
|
|
|
|
}
|
|
|
|
dimset.ndims = count;
|
|
|
|
/* extract the actual dimensions*/
|
|
|
|
if(dimset.ndims > 0) {
|
|
|
|
for(i=0;i<count;i++) {
|
|
|
|
Symbol* dsym = (Symbol*)listget(stack,stackbase+i);
|
|
|
|
dimset.dimsyms[i] = dsym;
|
|
|
|
}
|
2018-11-16 01:00:38 +08:00
|
|
|
var->typ.dimset = dimset;
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
2018-11-16 01:00:38 +08:00
|
|
|
var->typ.basetype = NULL; /* not yet known*/
|
|
|
|
var->objectclass=NC_VAR;
|
2010-06-03 21:24:43 +08:00
|
|
|
listsetlength(stack,stackbase);/* remove stack nodes*/
|
2018-11-16 01:00:38 +08:00
|
|
|
$$ = var;
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
dimspec: /* empty */ {$$=listlength(stack);}
|
|
|
|
| '(' dimlist ')' {$$=$2;}
|
|
|
|
;
|
|
|
|
|
2012-08-20 05:54:30 +08:00
|
|
|
dimlist: dimref {$$=listlength(stack); listpush(stack,(void*)$1);}
|
2010-06-03 21:24:43 +08:00
|
|
|
| dimlist ',' dimref
|
2012-08-20 05:54:30 +08:00
|
|
|
{$$=$1; listpush(stack,(void*)$3);}
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
dimref: path
|
|
|
|
{Symbol* dimsym = $1;
|
|
|
|
dimsym->objectclass = NC_DIM;
|
|
|
|
/* Find the actual dimension*/
|
|
|
|
dimsym = locate(dimsym);
|
|
|
|
if(dimsym == NULL) {
|
|
|
|
derror("Undefined or forward referenced dimension: %s",$1->name);
|
|
|
|
YYABORT;
|
|
|
|
}
|
|
|
|
$$=dimsym;
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
fieldlist:
|
|
|
|
fieldspec
|
|
|
|
{$$=listlength(stack);
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush(stack,(void*)$1);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
| fieldlist ',' fieldspec
|
2012-08-20 05:54:30 +08:00
|
|
|
{$$=$1; listpush(stack,(void*)$3);}
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
fieldspec:
|
2012-01-16 09:29:19 +08:00
|
|
|
ident fielddimspec
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
Dimset dimset;
|
|
|
|
stackbase=$2;
|
|
|
|
stacklen=listlength(stack);
|
|
|
|
count = stacklen - stackbase;
|
|
|
|
if(count >= NC_MAX_VAR_DIMS) {
|
|
|
|
yyerror("%s has too many dimensions",$1->name);
|
|
|
|
count = NC_MAX_VAR_DIMS - 1;
|
|
|
|
stacklen = stackbase + count;
|
|
|
|
}
|
|
|
|
dimset.ndims = count;
|
|
|
|
if(count > 0) {
|
|
|
|
/* extract the actual dimensions*/
|
|
|
|
for(i=0;i<count;i++) {
|
|
|
|
Symbol* dsym = (Symbol*)listget(stack,stackbase+i);
|
|
|
|
dimset.dimsyms[i] = dsym;
|
|
|
|
}
|
|
|
|
$1->typ.dimset = dimset;
|
|
|
|
}
|
|
|
|
$1->typ.basetype = NULL; /* not yet known*/
|
|
|
|
$1->objectclass=NC_TYPE;
|
|
|
|
$1->subclass=NC_FIELD;
|
|
|
|
listsetlength(stack,stackbase);/* remove stack nodes*/
|
|
|
|
$$ = $1;
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
fielddimspec: /* empty */ {$$=listlength(stack);}
|
|
|
|
| '(' fielddimlist ')' {$$=$2;}
|
|
|
|
;
|
|
|
|
|
|
|
|
fielddimlist:
|
2012-08-20 05:54:30 +08:00
|
|
|
fielddim {$$=listlength(stack); listpush(stack,(void*)$1);}
|
2010-06-03 21:24:43 +08:00
|
|
|
| fielddimlist ',' fielddim
|
2012-08-20 05:54:30 +08:00
|
|
|
{$$=$1; listpush(stack,(void*)$3);}
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
fielddim:
|
|
|
|
UINT_CONST
|
|
|
|
{ /* Anonymous integer dimension.
|
|
|
|
Can only occur in type definitions*/
|
|
|
|
char anon[32];
|
|
|
|
sprintf(anon,"const%u",uint32_val);
|
|
|
|
$$ = install(anon);
|
|
|
|
$$->objectclass = NC_DIM;
|
|
|
|
$$->dim.isconstant = 1;
|
|
|
|
$$->dim.declsize = uint32_val;
|
|
|
|
}
|
|
|
|
| INT_CONST
|
|
|
|
{ /* Anonymous integer dimension.
|
|
|
|
Can only occur in type definitions*/
|
|
|
|
char anon[32];
|
|
|
|
if(int32_val <= 0) {
|
|
|
|
derror("field dimension must be positive");
|
|
|
|
YYABORT;
|
|
|
|
}
|
|
|
|
sprintf(anon,"const%d",int32_val);
|
|
|
|
$$ = install(anon);
|
|
|
|
$$->objectclass = NC_DIM;
|
|
|
|
$$->dim.isconstant = 1;
|
|
|
|
$$->dim.declsize = int32_val;
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
|
|
|
|
/* Use this when referencing defined objects */
|
|
|
|
|
|
|
|
varref:
|
2020-06-06 07:03:29 +08:00
|
|
|
ambiguous_ref
|
2010-06-03 21:24:43 +08:00
|
|
|
{Symbol* vsym = $1;
|
|
|
|
if(vsym->objectclass != NC_VAR) {
|
|
|
|
derror("Undefined or forward referenced variable: %s",vsym->name);
|
|
|
|
YYABORT;
|
|
|
|
}
|
|
|
|
$$=vsym;
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
typeref:
|
2020-06-06 07:03:29 +08:00
|
|
|
ambiguous_ref
|
2010-06-03 21:24:43 +08:00
|
|
|
{Symbol* tsym = $1;
|
|
|
|
if(tsym->objectclass != NC_TYPE) {
|
|
|
|
derror("Undefined or forward referenced type: %s",tsym->name);
|
|
|
|
YYABORT;
|
|
|
|
}
|
|
|
|
$$=tsym;
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
2020-06-06 07:03:29 +08:00
|
|
|
ambiguous_ref:
|
2015-06-19 04:31:10 +08:00
|
|
|
path
|
2010-06-03 21:24:43 +08:00
|
|
|
{Symbol* tvsym = $1; Symbol* sym;
|
|
|
|
/* disambiguate*/
|
|
|
|
tvsym->objectclass = NC_VAR;
|
|
|
|
sym = locate(tvsym);
|
|
|
|
if(sym == NULL) {
|
|
|
|
tvsym->objectclass = NC_TYPE;
|
|
|
|
sym = locate(tvsym);
|
|
|
|
if(tvsym == NULL) {
|
|
|
|
derror("Undefined or forward referenced name: %s",$1->name);
|
|
|
|
YYABORT;
|
|
|
|
} else tvsym = sym;
|
|
|
|
} else tvsym = sym;
|
|
|
|
if(tvsym == NULL) {
|
2013-11-18 05:26:14 +08:00
|
|
|
derror("Undefined name (line %d): %s",$1->lineno,$1->name);
|
2010-06-03 21:24:43 +08:00
|
|
|
YYABORT;
|
|
|
|
}
|
|
|
|
$$=tvsym;
|
|
|
|
}
|
|
|
|
| primtype {$$=$1;}
|
|
|
|
;
|
|
|
|
|
|
|
|
/* Use this for all attribute decls */
|
|
|
|
/* Global vs var-specific will be separated in makeattribute */
|
|
|
|
|
|
|
|
/* Watch out; this is left recursive */
|
|
|
|
attrdecllist: /*empty*/ {} | attrdecl ';' attrdecllist {} ;
|
|
|
|
|
|
|
|
attrdecl:
|
2016-05-06 05:59:54 +08:00
|
|
|
':' _NCPROPS '=' conststring
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_NCPROPS_FLAG,NULL,NULL,(void*)$4,ISCONST);}
|
2016-05-04 11:17:06 +08:00
|
|
|
| ':' _ISNETCDF4 '=' constbool
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_ISNETCDF4_FLAG,NULL,NULL,(void*)$4,ISCONST);}
|
2016-05-04 11:17:06 +08:00
|
|
|
| ':' _SUPERBLOCK '=' constint
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_SUPERBLOCK_FLAG,NULL,NULL,(void*)$4,ISCONST);}
|
2016-05-06 05:59:54 +08:00
|
|
|
| ':' ident '=' datalist
|
|
|
|
{ $$=makeattribute($2,NULL,NULL,$4,ATTRGLOBAL);}
|
2020-06-06 07:03:29 +08:00
|
|
|
| typeref ambiguous_ref ':' ident '=' datalist
|
2010-06-03 21:24:43 +08:00
|
|
|
{Symbol* tsym = $1; Symbol* vsym = $2; Symbol* asym = $4;
|
|
|
|
if(vsym->objectclass == NC_VAR) {
|
|
|
|
$$=makeattribute(asym,vsym,tsym,$6,ATTRVAR);
|
|
|
|
} else {
|
|
|
|
derror("Doubly typed attribute: %s",asym->name);
|
|
|
|
YYABORT;
|
|
|
|
}
|
|
|
|
}
|
2020-06-06 07:03:29 +08:00
|
|
|
| ambiguous_ref ':' ident '=' datalist
|
2010-06-03 21:24:43 +08:00
|
|
|
{Symbol* sym = $1; Symbol* asym = $3;
|
|
|
|
if(sym->objectclass == NC_VAR) {
|
|
|
|
$$=makeattribute(asym,sym,NULL,$5,ATTRVAR);
|
|
|
|
} else if(sym->objectclass == NC_TYPE) {
|
|
|
|
$$=makeattribute(asym,NULL,sym,$5,ATTRGLOBAL);
|
|
|
|
} else {
|
|
|
|
derror("Attribute prefix not a variable or type: %s",asym->name);
|
|
|
|
YYABORT;
|
|
|
|
}
|
|
|
|
}
|
2020-06-06 07:03:29 +08:00
|
|
|
| ambiguous_ref ':' _FILLVALUE '=' datalist
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_FILLVALUE_FLAG,$1,NULL,(void*)$5,ISLIST);}
|
2020-06-06 07:03:29 +08:00
|
|
|
| typeref ambiguous_ref ':' _FILLVALUE '=' datalist
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_FILLVALUE_FLAG,$2,$1,(void*)$6,ISLIST);}
|
2020-06-06 07:03:29 +08:00
|
|
|
| ambiguous_ref ':' _STORAGE '=' conststring
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_STORAGE_FLAG,$1,NULL,(void*)$5,ISCONST);}
|
2020-06-06 07:03:29 +08:00
|
|
|
| ambiguous_ref ':' _CHUNKSIZES '=' intlist
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_CHUNKSIZES_FLAG,$1,NULL,(void*)$5,ISLIST);}
|
2020-06-06 07:03:29 +08:00
|
|
|
| ambiguous_ref ':' _FLETCHER32 '=' constbool
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_FLETCHER32_FLAG,$1,NULL,(void*)$5,ISCONST);}
|
2020-06-06 07:03:29 +08:00
|
|
|
| ambiguous_ref ':' _DEFLATELEVEL '=' constint
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_DEFLATE_FLAG,$1,NULL,(void*)$5,ISCONST);}
|
2020-06-06 07:03:29 +08:00
|
|
|
| ambiguous_ref ':' _SHUFFLE '=' constbool
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_SHUFFLE_FLAG,$1,NULL,(void*)$5,ISCONST);}
|
2020-06-06 07:03:29 +08:00
|
|
|
| ambiguous_ref ':' _ENDIANNESS '=' conststring
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_ENDIAN_FLAG,$1,NULL,(void*)$5,ISCONST);}
|
2020-06-06 07:03:29 +08:00
|
|
|
| ambiguous_ref ':' _FILTER '=' conststring
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_FILTER_FLAG,$1,NULL,(void*)$5,ISCONST);}
|
2021-09-03 07:04:26 +08:00
|
|
|
| ambiguous_ref ':' _CODECS '=' conststring
|
|
|
|
{$$ = makespecial(_CODECS_FLAG,$1,NULL,(void*)$5,ISCONST);}
|
2022-01-29 04:04:16 +08:00
|
|
|
| ambiguous_ref ':' _QUANTIZEBG '=' constint
|
|
|
|
{$$ = makespecial(_QUANTIZEBG_FLAG,$1,NULL,(void*)$5,ISCONST);}
|
2022-02-20 07:55:36 +08:00
|
|
|
| ambiguous_ref ':' _QUANTIZEGBR '=' constint
|
|
|
|
{$$ = makespecial(_QUANTIZEGBR_FLAG,$1,NULL,(void*)$5,ISCONST);}
|
2020-06-06 07:03:29 +08:00
|
|
|
| ambiguous_ref ':' _NOFILL '=' constbool
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_NOFILL_FLAG,$1,NULL,(void*)$5,ISCONST);}
|
2010-06-03 21:24:43 +08:00
|
|
|
| ':' _FORMAT '=' conststring
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = makespecial(_FORMAT_FLAG,NULL,NULL,(void*)$4,ISCONST);}
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
path:
|
2012-01-16 09:29:19 +08:00
|
|
|
ident
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
|
|
|
$$=$1;
|
2013-09-21 10:31:21 +08:00
|
|
|
$1->ref.is_ref=1;
|
|
|
|
$1->is_prefixed=0;
|
2010-06-03 21:24:43 +08:00
|
|
|
setpathcurrent($1);
|
|
|
|
}
|
|
|
|
| PATH
|
|
|
|
{
|
|
|
|
$$=$1;
|
2013-09-21 10:31:21 +08:00
|
|
|
$1->ref.is_ref=1;
|
2010-06-03 21:24:43 +08:00
|
|
|
$1->is_prefixed=1;
|
|
|
|
/* path is set in ncgen.l*/
|
|
|
|
}
|
|
|
|
;
|
|
|
|
|
|
|
|
datasection: /* empty */
|
|
|
|
| DATA {}
|
|
|
|
| DATA datadecls {}
|
|
|
|
;
|
|
|
|
|
|
|
|
datadecls: datadecl ';'
|
|
|
|
| datadecls datadecl ';'
|
|
|
|
;
|
|
|
|
|
|
|
|
datadecl: varref '=' datalist
|
|
|
|
{$1->data = $3;}
|
|
|
|
;
|
|
|
|
datalist:
|
|
|
|
datalist0 {$$ = $1;}
|
|
|
|
| datalist1 {$$ = $1;}
|
|
|
|
;
|
|
|
|
|
|
|
|
datalist0:
|
2012-03-08 07:38:51 +08:00
|
|
|
/*empty*/ {$$ = builddatalist(0);}
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
datalist1: /* Must have at least 1 element */
|
2018-11-16 01:00:38 +08:00
|
|
|
dataitem {$$ = const2list($1);}
|
2010-06-03 21:24:43 +08:00
|
|
|
| datalist ',' dataitem
|
2018-11-16 01:00:38 +08:00
|
|
|
{dlappend($1,($3)); $$=$1; }
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
dataitem:
|
|
|
|
constdata {$$=$1;}
|
|
|
|
| '{' datalist '}' {$$=builddatasublist($2);}
|
|
|
|
;
|
|
|
|
|
|
|
|
constdata:
|
2011-09-16 00:57:16 +08:00
|
|
|
simpleconstant {$$=$1;}
|
|
|
|
| OPAQUESTRING {$$=makeconstdata(NC_OPAQUE);}
|
|
|
|
| FILLMARKER {$$=makeconstdata(NC_FILLVALUE);}
|
2013-07-11 04:00:48 +08:00
|
|
|
| NIL {$$=makeconstdata(NC_NIL);}
|
2013-09-21 10:31:21 +08:00
|
|
|
| econstref {$$=$1;}
|
2011-09-16 00:57:16 +08:00
|
|
|
| function
|
|
|
|
;
|
|
|
|
|
2013-09-21 10:31:21 +08:00
|
|
|
econstref:
|
|
|
|
path {$$ = makeenumconstref($1);}
|
|
|
|
;
|
|
|
|
|
2011-09-16 00:57:16 +08:00
|
|
|
function:
|
2012-01-16 09:29:19 +08:00
|
|
|
ident '(' arglist ')' {$$=evaluate($1,$3);}
|
2011-09-16 00:57:16 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
arglist:
|
|
|
|
simpleconstant
|
2018-11-16 01:00:38 +08:00
|
|
|
{$$ = const2list($1);}
|
2011-09-16 00:57:16 +08:00
|
|
|
| arglist ',' simpleconstant
|
2018-11-16 01:00:38 +08:00
|
|
|
{dlappend($1,($3)); $$=$1;}
|
2011-09-16 00:57:16 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
simpleconstant:
|
2010-06-03 21:24:43 +08:00
|
|
|
CHAR_CONST {$$=makeconstdata(NC_CHAR);} /* never used apparently*/
|
|
|
|
| BYTE_CONST {$$=makeconstdata(NC_BYTE);}
|
|
|
|
| SHORT_CONST {$$=makeconstdata(NC_SHORT);}
|
|
|
|
| INT_CONST {$$=makeconstdata(NC_INT);}
|
|
|
|
| INT64_CONST {$$=makeconstdata(NC_INT64);}
|
|
|
|
| UBYTE_CONST {$$=makeconstdata(NC_UBYTE);}
|
|
|
|
| USHORT_CONST {$$=makeconstdata(NC_USHORT);}
|
|
|
|
| UINT_CONST {$$=makeconstdata(NC_UINT);}
|
|
|
|
| UINT64_CONST {$$=makeconstdata(NC_UINT64);}
|
|
|
|
| FLOAT_CONST {$$=makeconstdata(NC_FLOAT);}
|
|
|
|
| DOUBLE_CONST {$$=makeconstdata(NC_DOUBLE);}
|
|
|
|
| TERMSTRING {$$=makeconstdata(NC_STRING);}
|
|
|
|
;
|
|
|
|
|
|
|
|
intlist:
|
2018-11-16 01:00:38 +08:00
|
|
|
constint {$$ = const2list($1);}
|
|
|
|
| intlist ',' constint {$$=$1; dlappend($1,($3));}
|
2010-06-03 21:24:43 +08:00
|
|
|
;
|
|
|
|
|
|
|
|
constint:
|
|
|
|
INT_CONST
|
|
|
|
{$$=makeconstdata(NC_INT);}
|
|
|
|
| UINT_CONST
|
|
|
|
{$$=makeconstdata(NC_UINT);}
|
|
|
|
| INT64_CONST
|
|
|
|
{$$=makeconstdata(NC_INT64);}
|
|
|
|
| UINT64_CONST
|
|
|
|
{$$=makeconstdata(NC_UINT64);}
|
|
|
|
;
|
|
|
|
|
|
|
|
conststring:
|
|
|
|
TERMSTRING {$$=makeconstdata(NC_STRING);}
|
|
|
|
;
|
|
|
|
|
|
|
|
constbool:
|
|
|
|
conststring {$$=$1;}
|
|
|
|
| constint {$$=$1;}
|
|
|
|
|
|
|
|
/* End OF RULES */
|
|
|
|
|
2021-12-24 13:18:56 +08:00
|
|
|
|
|
|
|
/* Push all idents thru these*/
|
|
|
|
|
|
|
|
varident:
|
|
|
|
IDENT {$$=$1;}
|
|
|
|
| DATA {$$=identkeyword($1);}
|
|
|
|
;
|
|
|
|
|
2012-01-16 09:29:19 +08:00
|
|
|
ident:
|
|
|
|
IDENT {$$=$1;}
|
|
|
|
;
|
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
%%
|
|
|
|
|
|
|
|
#ifndef NO_STDARG
|
|
|
|
static void
|
|
|
|
yyerror(const char *fmt, ...)
|
|
|
|
#else
|
|
|
|
static void
|
|
|
|
yyerror(fmt,va_alist) const char* fmt; va_dcl
|
|
|
|
#endif
|
|
|
|
{
|
|
|
|
va_list argv;
|
2018-11-16 01:00:38 +08:00
|
|
|
va_start(argv,fmt);
|
2010-06-03 21:24:43 +08:00
|
|
|
(void)fprintf(stderr,"%s: %s line %d: ", progname, cdlname, lineno);
|
|
|
|
vderror(fmt,argv);
|
2018-11-16 01:00:38 +08:00
|
|
|
va_end(argv);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* undefine yywrap macro, in case we are using bison instead of yacc */
|
|
|
|
#ifdef yywrap
|
|
|
|
#undef yywrap
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static int
|
|
|
|
ncgwrap(void) /* returns 1 on EOF if no more input */
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* get lexical input routine generated by lex */
|
2015-11-20 04:44:07 +08:00
|
|
|
#include "ncgenl.c"
|
2010-06-03 21:24:43 +08:00
|
|
|
|
|
|
|
/* Really should init our data within this file */
|
|
|
|
void
|
|
|
|
parse_init(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
opaqueid = 0;
|
|
|
|
arrayuid = 0;
|
2018-11-16 01:00:38 +08:00
|
|
|
symlist = listnew();
|
2010-06-03 21:24:43 +08:00
|
|
|
stack = listnew();
|
|
|
|
groupstack = listnew();
|
|
|
|
consttype = NC_NAT;
|
|
|
|
grpdefs = listnew();
|
|
|
|
dimdefs = listnew();
|
|
|
|
attdefs = listnew();
|
|
|
|
gattdefs = listnew();
|
|
|
|
xattdefs = listnew();
|
|
|
|
typdefs = listnew();
|
|
|
|
vardefs = listnew();
|
|
|
|
tmp = listnew();
|
|
|
|
/* Create the primitive types */
|
|
|
|
for(i=NC_NAT+1;i<=NC_STRING;i++) {
|
|
|
|
primsymbols[i] = makeprimitivetype(i);
|
|
|
|
}
|
|
|
|
lex_init();
|
|
|
|
}
|
|
|
|
|
|
|
|
static Symbol*
|
|
|
|
makeprimitivetype(nc_type nctype)
|
|
|
|
{
|
|
|
|
Symbol* sym = install(primtypenames[nctype]);
|
|
|
|
sym->objectclass=NC_TYPE;
|
|
|
|
sym->subclass=NC_PRIM;
|
2018-11-16 01:00:38 +08:00
|
|
|
sym->nc_id = nctype;
|
2010-06-03 21:24:43 +08:00
|
|
|
sym->typ.typecode = nctype;
|
|
|
|
sym->typ.size = ncsize(nctype);
|
|
|
|
sym->typ.nelems = 1;
|
Fix various problem around VLEN's
re: https://github.com/Unidata/netcdf-c/issues/541
re: https://github.com/Unidata/netcdf-c/issues/1208
re: https://github.com/Unidata/netcdf-c/issues/2078
re: https://github.com/Unidata/netcdf-c/issues/2041
re: https://github.com/Unidata/netcdf-c/issues/2143
For a long time, there have been known problems with the
management of complex types containing VLENs. This also
involves the string type because it is stored as a VLEN of
chars.
This PR (mostly) fixes this problem. But note that it adds new
functions to netcdf.h (see below) and this may require bumping
the .so number. These new functions can be removed, if desired,
in favor of functions in netcdf_aux.h, but netcdf.h seems the
better place for them because they are intended as alternatives
to the nc_free_vlen and nc_free_string functions already in
netcdf.h.
The term complex type refers to any type that directly or
transitively references a VLEN type. So an array of VLENS, a
compound with a VLEN field, and so on.
In order to properly handle instances of these complex types, it
is necessary to have function that can recursively walk
instances of such types to perform various actions on them. The
term "deep" is also used to mean recursive.
At the moment, the two operations needed by the netcdf library are:
* free'ing an instance of the complex type
* copying an instance of the complex type.
The current library does only shallow free and shallow copy of
complex types. This means that only the top level is properly
free'd or copied, but deep internal blocks in the instance are
not touched.
Note that the term "vector" will be used to mean a contiguous (in
memory) sequence of instances of some type. Given an array with,
say, dimensions 2 X 3 X 4, this will be stored in memory as a
vector of length 2*3*4=24 instances.
The use cases are primarily these.
## nc_get_vars
Suppose one is reading a vector of instances using nc_get_vars
(or nc_get_vara or nc_get_var, etc.). These functions will
return the vector in the top-level memory provided. All
interior blocks (form nested VLEN or strings) will have been
dynamically allocated.
After using this vector of instances, it is necessary to free
(aka reclaim) the dynamically allocated memory, otherwise a
memory leak occurs. So, the recursive reclaim function is used
to walk the returned instance vector and do a deep reclaim of
the data.
Currently functions are defined in netcdf.h that are supposed to
handle this: nc_free_vlen(), nc_free_vlens(), and
nc_free_string(). Unfortunately, these functions only do a
shallow free, so deeply nested instances are not properly
handled by them.
Note that internally, the provided data is immediately written so
there is no need to copy it. But the caller may need to reclaim the
data it passed into the function.
## nc_put_att
Suppose one is writing a vector of instances as the data of an attribute
using, say, nc_put_att.
Internally, the incoming attribute data must be copied and stored
so that changes/reclamation of the input data will not affect
the attribute.
Again, the code inside the netcdf library does only shallow copying
rather than deep copy. As a result, one sees effects such as described
in Github Issue https://github.com/Unidata/netcdf-c/issues/2143.
Also, after defining the attribute, it may be necessary for the user
to free the data that was provided as input to nc_put_att().
## nc_get_att
Suppose one is reading a vector of instances as the data of an attribute
using, say, nc_get_att.
Internally, the existing attribute data must be copied and returned
to the caller, and the caller is responsible for reclaiming
the returned data.
Again, the code inside the netcdf library does only shallow copying
rather than deep copy. So this can lead to memory leaks and errors
because the deep data is shared between the library and the user.
# Solution
The solution is to build properly recursive reclaim and copy
functions and use those as needed.
These recursive functions are defined in libdispatch/dinstance.c
and their signatures are defined in include/netcdf.h.
For back compatibility, corresponding "ncaux_XXX" functions
are defined in include/netcdf_aux.h.
````
int nc_reclaim_data(int ncid, nc_type xtypeid, void* memory, size_t count);
int nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t count);
int nc_copy_data(int ncid, nc_type xtypeid, const void* memory, size_t count, void* copy);
int nc_copy_data_all(int ncid, nc_type xtypeid, const void* memory, size_t count, void** copyp);
````
There are two variants. The first two, nc_reclaim_data() and
nc_copy_data(), assume the top-level vector is managed by the
caller. For reclaim, this is so the user can use, for example, a
statically allocated vector. For copy, it assumes the user
provides the space into which the copy is stored.
The second two, nc_reclaim_data_all() and
nc_copy_data_all(), allows the functions to manage the
top-level. So for nc_reclaim_data_all, the top level is
assumed to be dynamically allocated and will be free'd by
nc_reclaim_data_all(). The nc_copy_data_all() function
will allocate the top level and return a pointer to it to the
user. The user can later pass that pointer to
nc_reclaim_data_all() to reclaim the instance(s).
# Internal Changes
The netcdf-c library internals are changed to use the proper
reclaim and copy functions. It turns out that the places where
these functions are needed is quite pervasive in the netcdf-c
library code. Using these functions also allows some
simplification of the code since the stdata and vldata fields of
NC_ATT_INFO are no longer needed. Currently this is commented
out using the SEPDATA \#define macro. When any bugs are largely
fixed, all this code will be removed.
# Known Bugs
1. There is still one known failure that has not been solved.
All the failures revolve around some variant of this .cdl file.
The proximate cause of failure is the use of a VLEN FillValue.
````
netcdf x {
types:
float(*) row_of_floats ;
dimensions:
m = 5 ;
variables:
row_of_floats ragged_array(m) ;
row_of_floats ragged_array:_FillValue = {-999} ;
data:
ragged_array = {10, 11, 12, 13, 14}, {20, 21, 22, 23}, {30, 31, 32},
{40, 41}, _ ;
}
````
When a solution is found, I will either add it to this PR or post a new PR.
# Related Changes
* Mark nc_free_vlen(s) as deprecated in favor of ncaux_reclaim_data.
* Remove the --enable-unfixed-memory-leaks option.
* Remove the NC_VLENS_NOTEST code that suppresses some vlen tests.
* Document this change in docs/internal.md
* Disable the tst_vlen_data test in ncdump/tst_nccopy4.sh.
* Mark types as fixed size or not (transitively) to optimize the reclaim
and copy functions.
# Misc. Changes
* Make Doxygen process libdispatch/daux.c
* Make sure the NC_ATT_INFO_T.container field is set.
2022-01-09 09:30:00 +08:00
|
|
|
(void)ncaux_class_alignment(nctype,&sym->typ.alignment);
|
2012-05-05 03:22:30 +08:00
|
|
|
/* Make the basetype circular so we can always ask for it */
|
|
|
|
sym->typ.basetype = sym;
|
2010-06-03 21:24:43 +08:00
|
|
|
sym->prefix = listnew();
|
|
|
|
return sym;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Symbol table operations for ncgen tool */
|
|
|
|
/* install sname in symbol table even if it is already there */
|
|
|
|
Symbol*
|
|
|
|
install(const char *sname)
|
2021-04-14 06:56:43 +08:00
|
|
|
{
|
|
|
|
return installin(sname,currentgroup());
|
|
|
|
}
|
|
|
|
|
|
|
|
Symbol*
|
|
|
|
installin(const char *sname, Symbol* grp)
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
|
|
|
Symbol* sp;
|
2017-10-31 05:52:08 +08:00
|
|
|
sp = (Symbol*) ecalloc (sizeof (struct Symbol));
|
2010-06-03 21:24:43 +08:00
|
|
|
sp->name = nulldup(sname);
|
|
|
|
sp->lineno = lineno;
|
2021-04-14 06:56:43 +08:00
|
|
|
sp->location = grp;
|
|
|
|
sp->container = grp;
|
2018-11-16 01:00:38 +08:00
|
|
|
listpush(symlist,sp);
|
2010-06-03 21:24:43 +08:00
|
|
|
return sp;
|
|
|
|
}
|
|
|
|
|
|
|
|
static Symbol*
|
|
|
|
currentgroup(void)
|
|
|
|
{
|
|
|
|
if(listlength(groupstack) == 0) return rootgroup;
|
|
|
|
return (Symbol*)listtop(groupstack);
|
|
|
|
}
|
|
|
|
|
|
|
|
static Symbol*
|
2013-09-21 10:31:21 +08:00
|
|
|
createrootgroup(const char* dataset)
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
2013-09-21 10:31:21 +08:00
|
|
|
Symbol* gsym = install(dataset);
|
2010-06-03 21:24:43 +08:00
|
|
|
gsym->objectclass = NC_GRP;
|
|
|
|
gsym->container = NULL;
|
|
|
|
gsym->subnodes = listnew();
|
|
|
|
gsym->grp.is_root = 1;
|
|
|
|
gsym->prefix = listnew();
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush(grpdefs,(void*)gsym);
|
2010-06-03 21:24:43 +08:00
|
|
|
rootgroup = gsym;
|
|
|
|
return gsym;
|
|
|
|
}
|
|
|
|
|
|
|
|
static Symbol*
|
|
|
|
creategroup(Symbol * gsym)
|
|
|
|
{
|
|
|
|
/* See if this group already exists in currentgroup */
|
|
|
|
gsym->objectclass = NC_GRP;
|
|
|
|
if(dupobjectcheck(NC_GRP,gsym)) {
|
|
|
|
derror("Duplicate group name in same scope: %s",gsym->name);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
addtogroup(gsym);
|
|
|
|
gsym->subnodes = listnew();
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush(groupstack,(void*)gsym);
|
|
|
|
listpush(grpdefs,(void*)gsym);
|
2010-06-03 21:24:43 +08:00
|
|
|
return gsym;
|
|
|
|
}
|
|
|
|
|
2018-11-16 01:00:38 +08:00
|
|
|
static NCConstant*
|
2010-06-03 21:24:43 +08:00
|
|
|
makeconstdata(nc_type nctype)
|
|
|
|
{
|
2018-11-16 01:00:38 +08:00
|
|
|
NCConstant* con = nullconst();
|
2010-06-03 21:24:43 +08:00
|
|
|
consttype = nctype;
|
2018-11-16 01:00:38 +08:00
|
|
|
con->nctype = nctype;
|
|
|
|
con->lineno = lineno;
|
|
|
|
con->filled = 0;
|
2010-06-03 21:24:43 +08:00
|
|
|
switch (nctype) {
|
2018-11-16 01:00:38 +08:00
|
|
|
case NC_CHAR: con->value.charv = char_val; break;
|
|
|
|
case NC_BYTE: con->value.int8v = byte_val; break;
|
|
|
|
case NC_SHORT: con->value.int16v = int16_val; break;
|
|
|
|
case NC_INT: con->value.int32v = int32_val; break;
|
2010-11-10 06:53:03 +08:00
|
|
|
case NC_FLOAT:
|
2018-11-16 01:00:38 +08:00
|
|
|
con->value.floatv = float_val;
|
2010-11-10 06:53:03 +08:00
|
|
|
break;
|
|
|
|
case NC_DOUBLE:
|
2018-11-16 01:00:38 +08:00
|
|
|
con->value.doublev = double_val;
|
2010-11-10 06:53:03 +08:00
|
|
|
break;
|
2010-06-03 21:24:43 +08:00
|
|
|
case NC_STRING: { /* convert to a set of chars*/
|
2012-03-08 07:38:51 +08:00
|
|
|
size_t len;
|
2012-02-14 08:25:32 +08:00
|
|
|
len = bbLength(lextext);
|
2018-11-16 01:00:38 +08:00
|
|
|
con->value.stringv.len = len;
|
|
|
|
con->value.stringv.stringv = bbExtract(lextext);
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
/* Allow these constants even in netcdf-3 */
|
2018-11-16 01:00:38 +08:00
|
|
|
case NC_UBYTE: con->value.uint8v = ubyte_val; break;
|
|
|
|
case NC_USHORT: con->value.uint16v = uint16_val; break;
|
|
|
|
case NC_UINT: con->value.uint32v = uint32_val; break;
|
|
|
|
case NC_INT64: con->value.int64v = int64_val; break;
|
|
|
|
case NC_UINT64: con->value.uint64v = uint64_val; break;
|
2010-06-03 21:24:43 +08:00
|
|
|
|
|
|
|
#ifdef USE_NETCDF4
|
|
|
|
case NC_OPAQUE: {
|
|
|
|
char* s;
|
2012-05-06 06:31:24 +08:00
|
|
|
int len;
|
2012-02-14 08:25:32 +08:00
|
|
|
len = bbLength(lextext);
|
2017-10-31 05:52:08 +08:00
|
|
|
s = (char*)ecalloc(len+1);
|
2012-02-14 08:25:32 +08:00
|
|
|
strncpy(s,bbContents(lextext),len);
|
2012-05-06 06:31:24 +08:00
|
|
|
s[len] = '\0';
|
2018-11-16 01:00:38 +08:00
|
|
|
con->value.opaquev.stringv = s;
|
|
|
|
con->value.opaquev.len = len;
|
2010-06-03 21:24:43 +08:00
|
|
|
} break;
|
2013-07-11 04:00:48 +08:00
|
|
|
|
|
|
|
case NC_NIL:
|
|
|
|
break; /* no associated value*/
|
2010-06-03 21:24:43 +08:00
|
|
|
#endif
|
|
|
|
|
2017-11-01 04:03:57 +08:00
|
|
|
case NC_FILLVALUE:
|
2010-06-03 21:24:43 +08:00
|
|
|
break; /* no associated value*/
|
2013-07-11 04:00:48 +08:00
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
default:
|
|
|
|
yyerror("Data constant: unexpected NC type: %s",
|
|
|
|
nctypename(nctype));
|
2018-11-16 01:00:38 +08:00
|
|
|
con->value.stringv.stringv = NULL;
|
|
|
|
con->value.stringv.len = 0;
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
return con;
|
|
|
|
}
|
|
|
|
|
2018-11-16 01:00:38 +08:00
|
|
|
static NCConstant*
|
2013-09-21 10:31:21 +08:00
|
|
|
makeenumconstref(Symbol* refsym)
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
2018-11-16 01:00:38 +08:00
|
|
|
NCConstant* con = nullconst();
|
2013-09-21 10:31:21 +08:00
|
|
|
|
2012-02-14 08:25:32 +08:00
|
|
|
markcdf4("Enum type");
|
2010-06-03 21:24:43 +08:00
|
|
|
consttype = NC_ENUM;
|
2018-11-16 01:00:38 +08:00
|
|
|
con->nctype = NC_ECONST;
|
|
|
|
con->lineno = lineno;
|
|
|
|
con->filled = 0;
|
2013-09-21 10:31:21 +08:00
|
|
|
refsym->objectclass = NC_TYPE;
|
|
|
|
refsym->subclass = NC_ECONST;
|
2018-11-16 01:00:38 +08:00
|
|
|
con->value.enumv = refsym;
|
2010-06-03 21:24:43 +08:00
|
|
|
return con;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
addtogroup(Symbol* sym)
|
|
|
|
{
|
|
|
|
Symbol* grp = currentgroup();
|
|
|
|
sym->container = grp;
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush(grp->subnodes,(void*)sym);
|
2010-06-03 21:24:43 +08:00
|
|
|
setpathcurrent(sym);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Check for duplicate name of given type within current group*/
|
|
|
|
static int
|
|
|
|
dupobjectcheck(nc_class objectclass, Symbol* pattern)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
Symbol* grp;
|
|
|
|
if(pattern == NULL) return 0;
|
|
|
|
grp = pattern->container;
|
|
|
|
if(grp == NULL || grp->subnodes == NULL) return 0;
|
|
|
|
for(i=0;i<listlength(grp->subnodes);i++) {
|
|
|
|
Symbol* sym = (Symbol*)listget(grp->subnodes,i);
|
2013-09-21 10:31:21 +08:00
|
|
|
if(!sym->ref.is_ref && sym->objectclass == objectclass
|
2010-06-03 21:24:43 +08:00
|
|
|
&& strcmp(sym->name,pattern->name)==0) return 1;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
setpathcurrent(Symbol* sym)
|
|
|
|
{
|
|
|
|
sym->is_prefixed = 0;
|
|
|
|
sym->prefix = prefixdup(groupstack);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Convert an nc_type code to the corresponding Symbol*/
|
|
|
|
Symbol*
|
|
|
|
basetypefor(nc_type nctype)
|
|
|
|
{
|
|
|
|
return primsymbols[nctype];
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2013-08-15 00:12:49 +08:00
|
|
|
truefalse(NCConstant* con, int tag)
|
2010-06-03 21:24:43 +08:00
|
|
|
{
|
|
|
|
if(con->nctype == NC_STRING) {
|
|
|
|
char* sdata = con->value.stringv.stringv;
|
|
|
|
if(strncmp(sdata,"false",NC_MAX_NAME) == 0
|
|
|
|
|| strncmp(sdata,"0",NC_MAX_NAME) == 0)
|
|
|
|
return 0;
|
|
|
|
else if(strncmp(sdata,"true",NC_MAX_NAME) == 0
|
|
|
|
|| strncmp(sdata,"1",NC_MAX_NAME) == 0)
|
|
|
|
return 1;
|
|
|
|
else goto fail;
|
|
|
|
} else if(con->value.int32v < 0 || con->value.int32v > 1)
|
|
|
|
goto fail;
|
|
|
|
return con->value.int32v;
|
|
|
|
|
|
|
|
fail:
|
|
|
|
derror("%s: illegal value",specialname(tag));
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Since this may be affected by the _Format attribute, which
|
|
|
|
may come last, capture all the special info and sort it out
|
|
|
|
in semantics.
|
|
|
|
*/
|
|
|
|
static Symbol*
|
|
|
|
makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
|
|
|
|
{
|
|
|
|
Symbol* attr = NULL;
|
2018-11-16 01:00:38 +08:00
|
|
|
Datalist* list = NULL;
|
|
|
|
NCConstant* con = NULL;
|
|
|
|
NCConstant* tmp = NULL;
|
2010-06-03 21:24:43 +08:00
|
|
|
int tf = 0;
|
|
|
|
char* sdata = NULL;
|
|
|
|
int idata = -1;
|
2013-08-15 00:12:49 +08:00
|
|
|
|
2016-05-04 11:17:06 +08:00
|
|
|
if((GLOBAL_SPECIAL & tag) != 0) {
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
if(vsym != NULL) {
|
|
|
|
derror("_Format: must be global attribute");
|
|
|
|
vsym = NULL;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if(vsym == NULL) {
|
|
|
|
derror("%s: must have non-NULL vsym", specialname(tag));
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
2013-08-15 00:12:49 +08:00
|
|
|
|
2014-03-09 11:41:30 +08:00
|
|
|
if(tag != _FILLVALUE_FLAG && tag != _FORMAT_FLAG)
|
|
|
|
/*Main.*/specials_flag++;
|
2012-02-14 08:25:32 +08:00
|
|
|
|
2018-11-16 01:00:38 +08:00
|
|
|
if(isconst)
|
2013-08-15 00:12:49 +08:00
|
|
|
con = (NCConstant*)data;
|
2018-11-16 01:00:38 +08:00
|
|
|
else
|
2010-06-03 21:24:43 +08:00
|
|
|
list = (Datalist*)data;
|
|
|
|
|
|
|
|
switch (tag) {
|
|
|
|
case _FLETCHER32_FLAG:
|
|
|
|
case _SHUFFLE_FLAG:
|
2016-05-04 11:17:06 +08:00
|
|
|
case _ISNETCDF4_FLAG:
|
2010-06-03 21:24:43 +08:00
|
|
|
case _NOFILL_FLAG:
|
2018-11-16 01:00:38 +08:00
|
|
|
tmp = nullconst();
|
|
|
|
tmp->nctype = (con->nctype == NC_STRING?NC_STRING:NC_INT);
|
|
|
|
convert1(con,tmp);
|
|
|
|
tf = truefalse(tmp,tag);
|
|
|
|
reclaimconstant(tmp);
|
2010-06-03 21:24:43 +08:00
|
|
|
break;
|
|
|
|
case _FORMAT_FLAG:
|
|
|
|
case _STORAGE_FLAG:
|
2016-05-04 11:17:06 +08:00
|
|
|
case _NCPROPS_FLAG:
|
2010-06-03 21:24:43 +08:00
|
|
|
case _ENDIAN_FLAG:
|
2017-05-15 08:10:02 +08:00
|
|
|
case _FILTER_FLAG:
|
2021-09-03 07:04:26 +08:00
|
|
|
case _CODECS_FLAG:
|
2018-11-16 01:00:38 +08:00
|
|
|
tmp = nullconst();
|
|
|
|
tmp->nctype = NC_STRING;
|
|
|
|
convert1(con,tmp);
|
|
|
|
if(tmp->nctype == NC_STRING) {
|
|
|
|
sdata = tmp->value.stringv.stringv;
|
|
|
|
tmp->value.stringv.stringv = NULL;
|
|
|
|
tmp->value.stringv.len = 0;
|
|
|
|
} else
|
2010-06-03 21:24:43 +08:00
|
|
|
derror("%s: illegal value",specialname(tag));
|
2018-11-16 01:00:38 +08:00
|
|
|
reclaimconstant(tmp);
|
2010-06-03 21:24:43 +08:00
|
|
|
break;
|
2016-05-04 11:17:06 +08:00
|
|
|
case _SUPERBLOCK_FLAG:
|
2010-06-03 21:24:43 +08:00
|
|
|
case _DEFLATE_FLAG:
|
2022-01-29 04:04:16 +08:00
|
|
|
case _QUANTIZEBG_FLAG:
|
2022-02-20 07:55:36 +08:00
|
|
|
case _QUANTIZEGBR_FLAG:
|
2018-11-16 01:00:38 +08:00
|
|
|
tmp = nullconst();
|
|
|
|
tmp->nctype = NC_INT;
|
|
|
|
convert1(con,tmp);
|
|
|
|
if(tmp->nctype == NC_INT)
|
|
|
|
idata = tmp->value.int32v;
|
2010-06-03 21:24:43 +08:00
|
|
|
else
|
|
|
|
derror("%s: illegal value",specialname(tag));
|
2018-11-16 01:00:38 +08:00
|
|
|
reclaimconstant(tmp);
|
2010-06-03 21:24:43 +08:00
|
|
|
break;
|
|
|
|
case _CHUNKSIZES_FLAG:
|
|
|
|
case _FILLVALUE_FLAG:
|
|
|
|
/* Handle below */
|
|
|
|
break;
|
|
|
|
default: PANIC1("unexpected special tag: %d",tag);
|
|
|
|
}
|
2015-06-19 04:31:10 +08:00
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
if(tag == _FORMAT_FLAG) {
|
2014-03-09 11:41:30 +08:00
|
|
|
/* Watch out: this is a global attribute */
|
2010-06-03 21:24:43 +08:00
|
|
|
struct Kvalues* kvalue;
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
int found = 0;
|
2011-04-09 01:47:16 +08:00
|
|
|
/* Use the table in main.c */
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
for(kvalue = legalkinds; kvalue->name; kvalue++) {
|
2015-06-19 04:31:10 +08:00
|
|
|
if(sdata) {
|
|
|
|
if(strcmp(sdata, kvalue->name) == 0) {
|
2016-05-04 11:17:06 +08:00
|
|
|
globalspecials._Format = kvalue->k_flag;
|
|
|
|
/*Main.*/format_attribute = 1;
|
2015-06-19 04:31:10 +08:00
|
|
|
found = 1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
if(!found)
|
|
|
|
derror("_Format: illegal value: %s",sdata);
|
2016-05-04 11:17:06 +08:00
|
|
|
} else if((GLOBAL_SPECIAL & tag) != 0) {
|
|
|
|
if(tag == _ISNETCDF4_FLAG)
|
|
|
|
globalspecials._IsNetcdf4 = tf;
|
|
|
|
else if(tag == _SUPERBLOCK_FLAG)
|
|
|
|
globalspecials._Superblock = idata;
|
2018-11-16 01:00:38 +08:00
|
|
|
else if(tag == _NCPROPS_FLAG) {
|
|
|
|
globalspecials._NCProperties = sdata;
|
|
|
|
sdata = NULL;
|
2018-12-07 06:40:43 +08:00
|
|
|
}
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
} else {
|
|
|
|
Specialdata* special;
|
|
|
|
/* Set up special info */
|
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
|
|
|
special = &vsym->var.special;
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
if(tag == _FILLVALUE_FLAG) {
|
|
|
|
/* fillvalue must be a single value*/
|
2018-11-16 01:00:38 +08:00
|
|
|
if(!isconst && datalistlen(list) != 1)
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
derror("_FillValue: must be a single (possibly compound) value",
|
|
|
|
vsym->name);
|
2018-11-16 01:00:38 +08:00
|
|
|
if(isconst) {
|
|
|
|
list = const2list(con);
|
|
|
|
con = NULL;
|
|
|
|
}
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
/* check that the attribute value contains no fill values*/
|
|
|
|
if(containsfills(list)) {
|
|
|
|
derror("Attribute data may not contain fill values (i.e. _ )");
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
/* _FillValue is also a real attribute*/
|
|
|
|
if(vsym->objectclass != NC_VAR) {
|
|
|
|
derror("_FillValue attribute not associated with variable: %s",vsym->name);
|
|
|
|
}
|
|
|
|
if(tsym == NULL) tsym = vsym->typ.basetype;
|
2021-12-24 13:18:56 +08:00
|
|
|
#if 0 /* No longer require matching types */
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
else if(vsym->typ.basetype != tsym) {
|
|
|
|
derror("_FillValue attribute type does not match variable type: %s",vsym->name);
|
|
|
|
}
|
2021-12-24 13:18:56 +08:00
|
|
|
#endif
|
2018-11-16 01:00:38 +08:00
|
|
|
special->_Fillvalue = clonedatalist(list);
|
|
|
|
/* Create the corresponding attribute */
|
2014-03-09 11:41:30 +08:00
|
|
|
attr = makeattribute(install("_FillValue"),vsym,tsym,list,ATTRVAR);
|
2018-11-16 01:00:38 +08:00
|
|
|
list = NULL;
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
} else switch (tag) {
|
2015-08-16 06:26:35 +08:00
|
|
|
/* These will be output as attributes later */
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
case _STORAGE_FLAG:
|
2015-06-19 04:31:10 +08:00
|
|
|
if(!sdata)
|
|
|
|
derror("_Storage: illegal NULL value");
|
|
|
|
else if(strcmp(sdata,"contiguous") == 0)
|
|
|
|
special->_Storage = NC_CONTIGUOUS;
|
2020-03-01 03:06:21 +08:00
|
|
|
else if(strcmp(sdata,"compact") == 0)
|
|
|
|
special->_Storage = NC_COMPACT;
|
2015-06-19 04:31:10 +08:00
|
|
|
else if(strcmp(sdata,"chunked") == 0)
|
|
|
|
special->_Storage = NC_CHUNKED;
|
|
|
|
else
|
|
|
|
derror("_Storage: illegal value: %s",sdata);
|
|
|
|
special->flags |= _STORAGE_FLAG;
|
|
|
|
break;
|
|
|
|
case _FLETCHER32_FLAG:
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
special->_Fletcher32 = tf;
|
|
|
|
special->flags |= _FLETCHER32_FLAG;
|
|
|
|
break;
|
|
|
|
case _DEFLATE_FLAG:
|
|
|
|
special->_DeflateLevel = idata;
|
|
|
|
special->flags |= _DEFLATE_FLAG;
|
|
|
|
break;
|
2022-01-29 04:04:16 +08:00
|
|
|
case _QUANTIZEBG_FLAG:
|
2022-01-25 06:22:24 +08:00
|
|
|
special->_Quantizer = NC_QUANTIZE_BITGROOM;
|
|
|
|
special->_NSD = idata;
|
2022-01-29 04:04:16 +08:00
|
|
|
special->flags |= _QUANTIZEBG_FLAG;
|
|
|
|
break;
|
2022-02-20 07:55:36 +08:00
|
|
|
case _QUANTIZEGBR_FLAG:
|
2022-01-29 04:04:16 +08:00
|
|
|
special->_Quantizer = NC_QUANTIZE_GRANULARBR;
|
|
|
|
special->_NSD = idata;
|
2022-02-20 07:55:36 +08:00
|
|
|
special->flags |= _QUANTIZEGBR_FLAG;
|
2022-01-25 06:22:24 +08:00
|
|
|
break;
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
case _SHUFFLE_FLAG:
|
|
|
|
special->_Shuffle = tf;
|
|
|
|
special->flags |= _SHUFFLE_FLAG;
|
|
|
|
break;
|
|
|
|
case _ENDIAN_FLAG:
|
2015-06-19 04:31:10 +08:00
|
|
|
if(!sdata)
|
|
|
|
derror("_Endianness: illegal NULL value");
|
|
|
|
else if(strcmp(sdata,"little") == 0)
|
|
|
|
special->_Endianness = 1;
|
|
|
|
else if(strcmp(sdata,"big") == 0)
|
|
|
|
special->_Endianness = 2;
|
|
|
|
else
|
|
|
|
derror("_Endianness: illegal value: %s",sdata);
|
|
|
|
special->flags |= _ENDIAN_FLAG;
|
|
|
|
break;
|
|
|
|
case _NOFILL_FLAG:
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
special->_Fill = (1 - tf); /* negate */
|
|
|
|
special->flags |= _NOFILL_FLAG;
|
|
|
|
break;
|
2016-05-04 11:17:06 +08:00
|
|
|
case _CHUNKSIZES_FLAG: {
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
int i;
|
2018-12-07 06:40:43 +08:00
|
|
|
list = (isconst ? const2list(con) : list);
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
special->nchunks = list->length;
|
2017-10-31 05:52:08 +08:00
|
|
|
special->_ChunkSizes = (size_t*)ecalloc(sizeof(size_t)*special->nchunks);
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
for(i=0;i<special->nchunks;i++) {
|
2018-11-16 01:00:38 +08:00
|
|
|
tmp = nullconst();
|
|
|
|
tmp->nctype = NC_INT;
|
|
|
|
convert1(list->data[i],tmp);
|
|
|
|
if(tmp->nctype == NC_INT) {
|
|
|
|
special->_ChunkSizes[i] = (size_t)tmp->value.int32v;
|
Corrected "BAIL" macros to avoid infinite loop when logging is disabled and an
error occurs after an "exit:" label.
Corrected a dozen Coverity errors (mainly allocation issues, along with a few
other things):
711711, 711802, 711803, 711905, 970825, 996123, 996124, 1025787,
1047274, 1130013, 1130014, 1139538
Refactored internal fill-value code to correctly handle string types, and
especially to allow NULL pointers and null strings (ie. "") to be
distinguished. The code now avoids partially aliasing the two together
(which only happened on the 'write' side of things and wasn't reflected on
the 'read' side, adding to the previous confusion).
Probably still weak on handling fill-values of variable-length and compound
datatypes.
Refactored the recursive metadata reads a bit more, to process HDF5 named
datatypes and datasets immediately, avoiding chewing up memory for those
types of objects, etc.
Finished uncommenting and updating the nc_test4/tst_fills2.c code (as I'm
proceeding alphabetically through the nc_test4 code files).
2013-12-29 15:12:43 +08:00
|
|
|
} else {
|
|
|
|
efree(special->_ChunkSizes);
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
derror("%s: illegal value",specialname(tag));
|
Corrected "BAIL" macros to avoid infinite loop when logging is disabled and an
error occurs after an "exit:" label.
Corrected a dozen Coverity errors (mainly allocation issues, along with a few
other things):
711711, 711802, 711803, 711905, 970825, 996123, 996124, 1025787,
1047274, 1130013, 1130014, 1139538
Refactored internal fill-value code to correctly handle string types, and
especially to allow NULL pointers and null strings (ie. "") to be
distinguished. The code now avoids partially aliasing the two together
(which only happened on the 'write' side of things and wasn't reflected on
the 'read' side, adding to the previous confusion).
Probably still weak on handling fill-values of variable-length and compound
datatypes.
Refactored the recursive metadata reads a bit more, to process HDF5 named
datatypes and datasets immediately, avoiding chewing up memory for those
types of objects, etc.
Finished uncommenting and updating the nc_test4/tst_fills2.c code (as I'm
proceeding alphabetically through the nc_test4 code files).
2013-12-29 15:12:43 +08:00
|
|
|
}
|
2018-11-16 01:00:38 +08:00
|
|
|
reclaimconstant(tmp);
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
}
|
|
|
|
special->flags |= _CHUNKSIZES_FLAG;
|
|
|
|
/* Chunksizes => storage == chunked */
|
|
|
|
special->flags |= _STORAGE_FLAG;
|
|
|
|
special->_Storage = NC_CHUNKED;
|
|
|
|
} break;
|
2017-05-15 08:10:02 +08:00
|
|
|
case _FILTER_FLAG:
|
2018-02-09 10:53:40 +08:00
|
|
|
#ifdef USE_NETCDF4
|
2017-05-15 08:10:02 +08:00
|
|
|
/* Parse the filter spec */
|
2018-01-24 07:00:11 +08:00
|
|
|
if(parsefilterflag(sdata,special) == NC_NOERR)
|
2017-05-15 08:10:02 +08:00
|
|
|
special->flags |= _FILTER_FLAG;
|
2018-01-24 07:00:11 +08:00
|
|
|
else {
|
2019-09-18 10:27:43 +08:00
|
|
|
derror("_Filter: unparsable filter spec: %s",sdata);
|
2018-01-24 07:00:11 +08:00
|
|
|
}
|
2018-02-09 10:53:40 +08:00
|
|
|
#else
|
2018-11-16 01:00:38 +08:00
|
|
|
derror("%s: the filter attribute requires netcdf-4 to be enabled",specialname(tag));
|
2021-09-03 07:04:26 +08:00
|
|
|
#endif
|
|
|
|
break;
|
|
|
|
case _CODECS_FLAG:
|
|
|
|
#ifdef USE_NETCDF4
|
|
|
|
/* Parse the codec spec */
|
|
|
|
if(parsecodecsflag(sdata,special) == NC_NOERR)
|
|
|
|
special->flags |= _CODECS_FLAG;
|
|
|
|
else {
|
|
|
|
derror("_Codecs: unparsable codec spec: %s",sdata);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
derror("%s: the _Codecs attribute requires netcdf-4 to be enabled",specialname(tag));
|
2018-02-09 10:53:40 +08:00
|
|
|
#endif
|
2017-04-28 03:01:59 +08:00
|
|
|
break;
|
Refactored read_scale(), memio_new(), var_create_dataset() and makespecial()
to clean up resources properly on failure.
Refactored doubly-linked list code for objects in the libsrc4 directory,
cleaning up the add/del routines, breaking out the common next/prev
pointers into a struct and extracting the add/del operations on them,
changed the list of dims to add new dims in the same order as the other
types, made all add routines able to optionally return a pointer to the
newly created object.
Removed some dead code (pg_var(), nc4_pg_var1(), nc4_pg_varm(), misc. small
routines, etc)
Fixed fill value handling for string types in nc4_get_vara().
Changed many malloc()+strcpy() pairs into calls to strdup().
Cleaned up misc. other minor Coverity issues.
2013-12-08 17:29:26 +08:00
|
|
|
default: PANIC1("makespecial: illegal token: %d",tag);
|
|
|
|
}
|
2014-03-09 11:41:30 +08:00
|
|
|
}
|
2018-11-16 01:00:38 +08:00
|
|
|
if(sdata) efree(sdata);
|
|
|
|
if(con) reclaimconstant(con);
|
|
|
|
if(list) reclaimdatalist(list);
|
2010-06-03 21:24:43 +08:00
|
|
|
return attr;
|
|
|
|
}
|
|
|
|
|
|
|
|
static Symbol*
|
|
|
|
makeattribute(Symbol* asym,
|
|
|
|
Symbol* vsym,
|
|
|
|
Symbol* tsym,
|
|
|
|
Datalist* data,
|
|
|
|
Attrkind kind) /* global var or unknown*/
|
|
|
|
{
|
|
|
|
asym->objectclass = NC_ATT;
|
|
|
|
asym->data = data;
|
|
|
|
switch (kind) {
|
|
|
|
case ATTRVAR:
|
|
|
|
asym->att.var = vsym;
|
|
|
|
asym->typ.basetype = tsym;
|
2012-08-20 05:54:30 +08:00
|
|
|
listpush(attdefs,(void*)asym);
|
2016-05-04 11:17:06 +08:00
|
|
|
addtogroup(asym);
|
2010-06-03 21:24:43 +08:00
|
|
|
break;
|
|
|
|
case ATTRGLOBAL:
|
|
|
|
asym->att.var = NULL; /* NULL => NC_GLOBAL*/
|
|
|
|
asym->typ.basetype = tsym;
|
2016-05-06 05:59:54 +08:00
|
|
|
listpush(gattdefs,(void*)asym);
|
|
|
|
addtogroup(asym);
|
2010-06-03 21:24:43 +08:00
|
|
|
break;
|
|
|
|
default: PANIC1("unexpected attribute type: %d",kind);
|
|
|
|
}
|
|
|
|
/* finally; check that the attribute value contains no fill values*/
|
|
|
|
if(containsfills(data)) {
|
|
|
|
derror("Attribute data may not contain fill values (i.e. _ ): %s",asym->name);
|
|
|
|
}
|
|
|
|
return asym;
|
|
|
|
}
|
|
|
|
|
2016-05-04 11:17:06 +08:00
|
|
|
static long long
|
2018-11-16 01:00:38 +08:00
|
|
|
extractint(NCConstant* con)
|
2016-05-04 11:17:06 +08:00
|
|
|
{
|
2018-11-16 01:00:38 +08:00
|
|
|
switch (con->nctype) {
|
|
|
|
case NC_BYTE: return (long long)(con->value.int8v);
|
|
|
|
case NC_SHORT: return (long long)(con->value.int16v);
|
|
|
|
case NC_INT: return (long long)(con->value.int32v);
|
|
|
|
case NC_UBYTE: return (long long)(con->value.uint8v);
|
|
|
|
case NC_USHORT: return (long long)(con->value.uint16v);
|
|
|
|
case NC_UINT: return (long long)(con->value.uint32v);
|
|
|
|
case NC_INT64: return (long long)(con->value.int64v);
|
2016-05-04 11:17:06 +08:00
|
|
|
default:
|
2018-11-16 01:00:38 +08:00
|
|
|
derror("Not a signed integer type: %d",con->nctype);
|
2016-05-04 11:17:06 +08:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-06-03 21:24:43 +08:00
|
|
|
static int
|
|
|
|
containsfills(Datalist* list)
|
|
|
|
{
|
2012-02-14 08:25:32 +08:00
|
|
|
if(list != NULL) {
|
|
|
|
int i;
|
2018-11-16 01:00:38 +08:00
|
|
|
NCConstant** cons = list->data;
|
|
|
|
for(i=0;i<list->length;i++) {
|
|
|
|
if(cons[i]->nctype == NC_COMPOUND) {
|
|
|
|
if(containsfills(cons[i]->value.compoundv)) return 1;
|
|
|
|
} else if(cons[i]->nctype == NC_FILLVALUE)
|
2017-11-01 04:03:57 +08:00
|
|
|
return 1;
|
2012-02-14 08:25:32 +08:00
|
|
|
}
|
2010-06-03 21:24:43 +08:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-09-16 11:03:05 +08:00
|
|
|
/*
|
|
|
|
Try to infer the file type from the
|
|
|
|
kinds of constructs used in the cdl file.
|
|
|
|
*/
|
2011-05-13 01:51:32 +08:00
|
|
|
static void
|
2015-11-07 08:03:28 +08:00
|
|
|
vercheck(int tid)
|
2011-05-13 01:51:32 +08:00
|
|
|
{
|
2015-11-07 08:03:28 +08:00
|
|
|
switch (tid) {
|
2017-09-16 11:03:05 +08:00
|
|
|
case NC_UBYTE: markcdf4("netCDF4/5 type: UBYTE"); break;
|
|
|
|
case NC_USHORT: markcdf4("netCDF4/5 type: USHORT"); break;
|
|
|
|
case NC_UINT: markcdf4("netCDF4/5 type: UINT"); break;
|
|
|
|
case NC_INT64: markcdf4("netCDF4/5 type: INT64"); break;
|
|
|
|
case NC_UINT64: markcdf4("netCDF4/5 type: UINT64"); break;
|
2015-11-07 08:03:28 +08:00
|
|
|
case NC_STRING: markcdf4("netCDF4 type: STRING"); break;
|
|
|
|
case NC_VLEN: markcdf4("netCDF4 type: VLEN"); break;
|
|
|
|
case NC_OPAQUE: markcdf4("netCDF4 type: OPAQUE"); break;
|
|
|
|
case NC_ENUM: markcdf4("netCDF4 type: ENUM"); break;
|
|
|
|
case NC_COMPOUND: markcdf4("netCDF4 type: COMPOUND"); break;
|
2011-05-13 01:51:32 +08:00
|
|
|
default: break;
|
|
|
|
}
|
|
|
|
}
|
2011-09-16 00:57:16 +08:00
|
|
|
|
2016-05-04 11:17:06 +08:00
|
|
|
const char*
|
|
|
|
specialname(int tag)
|
|
|
|
{
|
|
|
|
struct Specialtoken* spp = specials;
|
|
|
|
for(;spp->name;spp++) {
|
|
|
|
if(spp->tag == tag)
|
|
|
|
return spp->name;
|
|
|
|
}
|
|
|
|
return "<unknown>";
|
|
|
|
}
|
|
|
|
|
2018-02-09 10:53:40 +08:00
|
|
|
#ifdef USE_NETCDF4
|
2017-05-15 08:10:02 +08:00
|
|
|
/*
|
|
|
|
Parse a filter spec string and store it in special
|
|
|
|
*/
|
|
|
|
static int
|
2018-02-09 10:53:40 +08:00
|
|
|
parsefilterflag(const char* sdata, Specialdata* special)
|
2017-05-15 08:10:02 +08:00
|
|
|
{
|
2018-01-24 07:00:11 +08:00
|
|
|
int stat = NC_NOERR;
|
2017-11-14 04:19:46 +08:00
|
|
|
|
2018-02-09 10:53:40 +08:00
|
|
|
if(sdata == NULL || strlen(sdata) == 0) return NC_EINVAL;
|
2017-11-14 04:19:46 +08:00
|
|
|
|
2020-09-28 02:43:46 +08:00
|
|
|
stat = ncaux_h5filterspec_parselist(sdata, NULL, &special->nfilters, &special->_Filters);
|
2018-01-24 07:00:11 +08:00
|
|
|
if(stat)
|
|
|
|
derror("Malformed filter spec: %s",sdata);
|
2020-02-17 03:59:33 +08:00
|
|
|
#ifdef GENDEBUG1
|
|
|
|
printfilters(special->nfilters,special->_Filters);
|
|
|
|
#endif
|
2018-01-24 07:00:11 +08:00
|
|
|
return stat;
|
2017-05-15 08:10:02 +08:00
|
|
|
}
|
2021-09-03 07:04:26 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
Store a Codecs spec string in special
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
parsecodecsflag(const char* sdata, Specialdata* special)
|
|
|
|
{
|
|
|
|
int stat = NC_NOERR;
|
|
|
|
|
|
|
|
if(sdata == NULL || strlen(sdata) == 0) return NC_EINVAL;
|
|
|
|
|
|
|
|
if((special->_Codecs = strdup(sdata))==NULL)
|
|
|
|
return NC_ENOMEM;
|
|
|
|
return stat;
|
|
|
|
}
|
2018-02-09 10:53:40 +08:00
|
|
|
#endif
|
2017-05-15 08:10:02 +08:00
|
|
|
|
2011-09-16 00:57:16 +08:00
|
|
|
/*
|
2011-09-17 02:36:08 +08:00
|
|
|
Since the arguments are all simple constants,
|
2011-09-16 00:57:16 +08:00
|
|
|
we can evaluate the function immediately
|
|
|
|
and return its value.
|
2011-09-17 02:36:08 +08:00
|
|
|
Note that currently, only a single value can
|
|
|
|
be returned.
|
2011-09-16 00:57:16 +08:00
|
|
|
*/
|
|
|
|
|
2018-11-16 01:00:38 +08:00
|
|
|
static NCConstant*
|
2011-09-16 00:57:16 +08:00
|
|
|
evaluate(Symbol* fcn, Datalist* arglist)
|
|
|
|
{
|
2018-11-16 01:00:38 +08:00
|
|
|
NCConstant* result = nullconst();
|
2011-09-16 00:57:16 +08:00
|
|
|
|
|
|
|
/* prepare the result */
|
2018-11-16 01:00:38 +08:00
|
|
|
result->lineno = fcn->lineno;
|
2011-09-16 00:57:16 +08:00
|
|
|
|
|
|
|
if(strcasecmp(fcn->name,"time") == 0) {
|
2011-09-17 02:36:08 +08:00
|
|
|
char* timekind = NULL;
|
|
|
|
char* timevalue = NULL;
|
2018-11-16 01:00:38 +08:00
|
|
|
result->nctype = NC_DOUBLE;
|
|
|
|
result->value.doublev = 0;
|
2011-09-17 02:36:08 +08:00
|
|
|
/* int time([string],string) */
|
|
|
|
switch (arglist->length) {
|
|
|
|
case 2:
|
2018-11-16 01:00:38 +08:00
|
|
|
if(arglist->data[1]->nctype != NC_STRING) {
|
2011-09-17 02:36:08 +08:00
|
|
|
derror("Expected function signature: time([string,]string)");
|
2011-09-16 00:57:16 +08:00
|
|
|
goto done;
|
|
|
|
}
|
2011-09-17 02:36:08 +08:00
|
|
|
/* fall thru */
|
|
|
|
case 1:
|
2018-11-16 01:00:38 +08:00
|
|
|
if(arglist->data[0]->nctype != NC_STRING) {
|
2011-09-17 02:36:08 +08:00
|
|
|
derror("Expected function signature: time([string,]string)");
|
2011-09-16 00:57:16 +08:00
|
|
|
goto done;
|
|
|
|
}
|
2011-09-17 02:36:08 +08:00
|
|
|
break;
|
|
|
|
case 0:
|
2015-06-19 04:31:10 +08:00
|
|
|
default:
|
2011-09-17 02:36:08 +08:00
|
|
|
derror("Expected function signature: time([string,]string)");
|
|
|
|
goto done;
|
2011-09-16 00:57:16 +08:00
|
|
|
}
|
2011-09-17 02:36:08 +08:00
|
|
|
if(arglist->length == 2) {
|
2018-11-16 01:00:38 +08:00
|
|
|
timekind = arglist->data[0]->value.stringv.stringv;
|
|
|
|
timevalue = arglist->data[1]->value.stringv.stringv;
|
2011-09-17 02:36:08 +08:00
|
|
|
} else
|
2018-11-16 01:00:38 +08:00
|
|
|
timevalue = arglist->data[0]->value.stringv.stringv;
|
2011-09-17 02:36:08 +08:00
|
|
|
if(timekind == NULL) { /* use cd time as the default */
|
|
|
|
cdCompTime comptime;
|
|
|
|
CdTime cdtime;
|
|
|
|
cdCalenType timetype = cdStandard;
|
|
|
|
cdChar2Comp(timetype,timevalue,&comptime);
|
|
|
|
/* convert comptime to cdTime */
|
2015-06-19 04:31:10 +08:00
|
|
|
cdtime.year = comptime.year;
|
2011-09-17 02:36:08 +08:00
|
|
|
cdtime.month = comptime.month;
|
2015-06-19 04:31:10 +08:00
|
|
|
cdtime.day = comptime.day;
|
2011-09-17 02:36:08 +08:00
|
|
|
cdtime.hour = comptime.hour;
|
|
|
|
cdtime.baseYear = 1970;
|
|
|
|
cdtime.timeType = CdChron;
|
|
|
|
/* convert to double value */
|
2018-11-16 01:00:38 +08:00
|
|
|
Cdh2e(&cdtime,&result->value.doublev);
|
2011-09-17 02:36:08 +08:00
|
|
|
} else {
|
|
|
|
derror("Time conversion '%s' not supported",timekind);
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
} else { /* Unknown function */
|
2011-09-16 00:57:16 +08:00
|
|
|
derror("Unknown function name: %s",fcn->name);
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
done:
|
|
|
|
return result;
|
|
|
|
}
|
2020-02-17 03:59:33 +08:00
|
|
|
|
|
|
|
#ifdef GENDEBUG1
|
|
|
|
static void
|
2021-04-14 06:56:43 +08:00
|
|
|
printfilters(int nfilters, NC_FilterSpec** filters)
|
2020-02-17 03:59:33 +08:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
fprintf(stderr,"xxx: nfilters=%lu: ",(unsigned long)nfilters);
|
|
|
|
for(i=0;i<nfilters;i++) {
|
|
|
|
int k;
|
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
|
|
|
NC_Filterspec* sp = filters[i];
|
2020-02-17 03:59:33 +08:00
|
|
|
fprintf(stderr,"{");
|
2021-04-14 06:56:43 +08:00
|
|
|
fprintf(stderr,"filterid=%llu format=%d nparams=%lu params=%p",
|
2020-02-17 03:59:33 +08:00
|
|
|
sp->filterid,sp->format,(unsigned long)sp->nparams,sp->params);
|
This PR adds EXPERIMENTAL support for accessing data in the
cloud using a variant of the Zarr protocol and storage
format. This enhancement is generically referred to as "NCZarr".
The data model supported by NCZarr is netcdf-4 minus the user-defined
types and the String type. In this sense it is similar to the CDF-5
data model.
More detailed information about enabling and using NCZarr is
described in the document NUG/nczarr.md and in a
[Unidata Developer's blog entry](https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in).
WARNING: this code has had limited testing, so do use this version
for production work. Also, performance improvements are ongoing.
Note especially the following platform matrix of successful tests:
Platform | Build System | S3 support
------------------------------------
Linux+gcc | Automake | yes
Linux+gcc | CMake | yes
Visual Studio | CMake | no
Additionally, and as a consequence of the addition of NCZarr,
major changes have been made to the Filter API. NOTE: NCZarr
does not yet support filters, but these changes are enablers for
that support in the future. Note that it is possible
(probable?) that there will be some accidental reversions if the
changes here did not correctly mimic the existing filter testing.
In any case, previously filter ids and parameters were of type
unsigned int. In order to support the more general zarr filter
model, this was all converted to char*. The old HDF5-specific,
unsigned int operations are still supported but they are
wrappers around the new, char* based nc_filterx_XXX functions.
This entailed at least the following changes:
1. Added the files libdispatch/dfilterx.c and include/ncfilter.h
2. Some filterx utilities have been moved to libdispatch/daux.c
3. A new entry, "filter_actions" was added to the NCDispatch table
and the version bumped.
4. An overly complex set of structs was created to support funnelling
all of the filterx operations thru a single dispatch
"filter_actions" entry.
5. Move common code to from libhdf5 to libsrc4 so that it is accessible
to nczarr.
Changes directly related to Zarr:
1. Modified CMakeList.txt and configure.ac to support both C and C++
-- this is in support of S3 support via the awd-sdk libraries.
2. Define a size64_t type to support nczarr.
3. More reworking of libdispatch/dinfermodel.c to
support zarr and to regularize the structure of the fragments
section of a URL.
Changes not directly related to Zarr:
1. Make client-side filter registration be conditional, with default off.
2. Hack include/nc4internal.h to make some flags added by Ed be unique:
e.g. NC_CREAT, NC_INDEF, etc.
3. cleanup include/nchttp.h and libdispatch/dhttp.c.
4. Misc. changes to support compiling under Visual Studio including:
* Better testing under windows for dirent.h and opendir and closedir.
5. Misc. changes to the oc2 code to support various libcurl CURLOPT flags
and to centralize error reporting.
6. By default, suppress the vlen tests that have unfixed memory leaks; add option to enable them.
7. Make part of the nc_test/test_byterange.sh test be contingent on remotetest.unidata.ucar.edu being accessible.
Changes Left TO-DO:
1. fix provenance code, it is too HDF5 specific.
2020-06-29 08:02:47 +08:00
|
|
|
if(sp->nparams > 0 && sp->params != NULL) {
|
2020-02-17 03:59:33 +08:00
|
|
|
fprintf(stderr," params={");
|
|
|
|
for(k=0;k<sp->nparams;k++) {
|
|
|
|
if(i==0) fprintf(stderr,",");
|
2021-04-14 06:56:43 +08:00
|
|
|
fprintf(stderr,"%u",sp->params[i]);
|
2020-02-17 03:59:33 +08:00
|
|
|
}
|
|
|
|
fprintf(stderr,"}");
|
|
|
|
} else
|
|
|
|
fprintf(stderr,"params=NULL");
|
|
|
|
fprintf(stderr,"}");
|
|
|
|
}
|
|
|
|
fprintf(stderr,"\n");
|
|
|
|
fflush(stderr);
|
|
|
|
}
|
|
|
|
#endif
|