mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-11-21 03:13:42 +08:00
75759ca957
re: pull request https://github.com/Unidata/netcdf-c/pull/1242 This pr should be applied before https://github.com/Unidata/netcdf-c/pull/1242. It fixes only the -ansi '//' comment problems. There may be some slight conflicts with that other pr when it is applied, since in some cases I converted #if 0...#endif to /*...*/
1150 lines
32 KiB
C
1150 lines
32 KiB
C
/*********************************************************************
|
|
* Copyright 2018, UCAR/Unidata
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
*********************************************************************/
|
|
|
|
#include "d4includes.h"
|
|
#include <stdarg.h>
|
|
#include "nc4internal.h"
|
|
#include "ncoffsets.h"
|
|
#include "ezxml.h"
|
|
|
|
/**
|
|
* Build the netcdf-4 metadata from the NCD4node nodes.
|
|
*/
|
|
|
|
/***************************************************/
|
|
/* Forwards */
|
|
|
|
static char* backslashEscape(const char* s);
|
|
static char* getFieldFQN(NCD4node* field, const char* tail);
|
|
static int build(NCD4meta* builder, NCD4node* root);
|
|
static int buildAtomicVar(NCD4meta* builder, NCD4node* var);
|
|
static int buildAttributes(NCD4meta* builder, NCD4node* varorgroup);
|
|
static int buildCompound(NCD4meta* builder, NCD4node* cmpdtype, NCD4node* group, char* name);
|
|
static int buildDimension(NCD4meta* builder, NCD4node* dim);
|
|
static int buildEnumeration(NCD4meta* builder, NCD4node* en);
|
|
static int buildGroups(NCD4meta*, NCD4node* parent);
|
|
static int buildMaps(NCD4meta* builder, NCD4node* var);
|
|
static int buildMetaData(NCD4meta* builder, NCD4node* var);
|
|
static int buildOpaque(NCD4meta* builder, NCD4node* op);
|
|
static int buildSequence(NCD4meta* builder, NCD4node* seq);
|
|
static int buildStructure(NCD4meta* builder, NCD4node* structvar);
|
|
static int buildStructureType(NCD4meta* builder, NCD4node* structtype);
|
|
static int buildVariable(NCD4meta* builder, NCD4node* var);
|
|
static int buildVlenType(NCD4meta* builder, NCD4node* seqtype);
|
|
static int compileAttrValues(NCD4meta* builder, NCD4node* attr, void** memoryp, NClist* blobs);
|
|
static void computeOffsets(NCD4meta* builder, NCD4node* cmpd);
|
|
static int convertString(union ATOMICS* converter, NCD4node* type, const char* s);
|
|
static void* copyAtomic(union ATOMICS* converter, nc_type type, size_t len, void* dst, NClist* blobs);
|
|
static int decodeEconst(NCD4meta* builder, NCD4node* enumtype, const char* nameorval, union ATOMICS* converter);
|
|
static int downConvert(union ATOMICS* converter, NCD4node* type);
|
|
static void freeStringMemory(char** mem, int count);
|
|
static size_t getDimrefs(NCD4node* var, int* dimids);
|
|
static size_t getDimsizes(NCD4node* var, int* dimsizes);
|
|
static d4size_t getpadding(d4size_t offset, size_t alignment);
|
|
static int markdapsize(NCD4meta* meta);
|
|
static int markfixedsize(NCD4meta* meta);
|
|
static void savegroupbyid(NCD4meta*,NCD4node* group);
|
|
static void savevarbyid(NCD4node* group, NCD4node* var);
|
|
#ifndef FIXEDOPAQUE
|
|
static int buildBytestringType(NCD4meta* builder);
|
|
#endif
|
|
|
|
/***************************************************/
|
|
/* API */
|
|
|
|
int
|
|
NCD4_metabuild(NCD4meta* metadata, int ncid)
|
|
{
|
|
int ret = NC_NOERR;
|
|
int i;
|
|
|
|
metadata->ncid = ncid;
|
|
metadata->root->meta.id = ncid;
|
|
|
|
/* Fix up the atomic types */
|
|
for(i=0;i<nclistlength(metadata->allnodes);i++) {
|
|
NCD4node* n = (NCD4node*)nclistget(metadata->allnodes,i);
|
|
if(n->sort != NCD4_TYPE) continue;
|
|
if(n->subsort > NC_MAX_ATOMIC_TYPE) continue;
|
|
n->meta.id = n->subsort;
|
|
n->meta.isfixedsize = (n->subsort == NC_STRING ? 0 : 1);
|
|
if(n->subsort <= NC_STRING)
|
|
n->meta.dapsize = NCD4_typesize(n->subsort);
|
|
}
|
|
|
|
/* Topo sort the set of all nodes */
|
|
NCD4_toposort(metadata);
|
|
markfixedsize(metadata);
|
|
markdapsize(metadata);
|
|
/* Process the metadata state */
|
|
if((ret = build(metadata,metadata->root))) goto done;
|
|
/* Done with the metadata*/
|
|
if((ret=nc_enddef(metadata->ncid)))
|
|
goto done;
|
|
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
|
|
/* Create an empty NCD4meta object for
|
|
use in subsequent calls
|
|
(is the the right src file to hold this?)
|
|
*/
|
|
|
|
NCD4meta*
|
|
NCD4_newmeta(size_t rawsize, void* rawdata)
|
|
{
|
|
NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta));
|
|
if(meta == NULL) return NULL;
|
|
meta->allnodes = nclistnew();
|
|
meta->serial.rawsize = rawsize;
|
|
meta->serial.rawdata = rawdata;
|
|
#ifdef D4DEBUG
|
|
meta->debuglevel = 1;
|
|
#endif
|
|
return meta;
|
|
}
|
|
|
|
void
|
|
NCD4_setdebuglevel(NCD4meta* meta, int debuglevel)
|
|
{
|
|
meta->debuglevel = debuglevel;
|
|
}
|
|
|
|
void
|
|
NCD4_reclaimMeta(NCD4meta* dataset)
|
|
{
|
|
int i;
|
|
if(dataset == NULL) return;
|
|
for(i=0;i<nclistlength(dataset->allnodes);i++) {
|
|
NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i);
|
|
reclaimNode(node);
|
|
}
|
|
nullfree(dataset->error.parseerror);
|
|
nullfree(dataset->error.message);
|
|
nullfree(dataset->error.context);
|
|
nullfree(dataset->error.otherinfo);
|
|
nullfree(dataset->serial.errdata);
|
|
nclistfree(dataset->groupbyid);
|
|
#if 0
|
|
for(i=0;i<nclistlength(dataset->blobs);i++) {
|
|
void* p = nclistget(dataset->blobs,i);
|
|
nullfree(p);
|
|
}
|
|
nclistfree(dataset->blobs);
|
|
#endif
|
|
nclistfree(dataset->allnodes);
|
|
nullfree(dataset->serial.dmr);
|
|
free(dataset);
|
|
}
|
|
|
|
void
|
|
reclaimNode(NCD4node* node)
|
|
{
|
|
if(node == NULL) return;
|
|
nullfree(node->name);
|
|
nclistfree(node->groups);
|
|
nclistfree(node->vars);
|
|
nclistfree(node->types);
|
|
nclistfree(node->dims);
|
|
nclistfree(node->attributes);
|
|
nclistfree(node->maps);
|
|
nclistfreeall(node->xmlattributes);
|
|
nclistfreeall(node->attr.values);
|
|
nclistfree(node->en.econsts);
|
|
nclistfree(node->group.elements);
|
|
nullfree(node->group.dapversion);
|
|
nullfree(node->group.dmrversion);
|
|
nullfree(node->group.datasetname);
|
|
nclistfree(node->group.varbyid);
|
|
nullfree(node->nc4.orig.name);
|
|
nullfree(node);
|
|
}
|
|
|
|
/**************************************************/
|
|
|
|
/* Recursively walk the tree to create the metadata */
|
|
static int
|
|
build(NCD4meta* builder, NCD4node* root)
|
|
{
|
|
int i,ret = NC_NOERR;
|
|
size_t len = nclistlength(builder->allnodes);
|
|
|
|
/* Tag the root group */
|
|
savegroupbyid(builder,root);
|
|
|
|
/* Compute the sizes for all type objects. Will of necessity
|
|
compute the offsets for compound types as well
|
|
*/
|
|
for(i=0;i<len;i++) {/* Walk in postfix order */
|
|
NCD4node* x = (NCD4node*)nclistget(builder->allnodes,i);
|
|
if(x->sort != NCD4_TYPE) continue;
|
|
switch (x->subsort) {
|
|
case NC_OPAQUE:
|
|
case NC_ENUM:
|
|
case NC_SEQ:
|
|
default: /* Atomic */
|
|
x->meta.memsize = NCD4_computeTypeSize(builder,x);
|
|
x->meta.alignment = x->meta.memsize; /* Same for these cases */
|
|
break;
|
|
case NC_STRUCT:
|
|
/* We need to compute the field offsets in order to compute the struct size */
|
|
computeOffsets(builder,x);
|
|
break;
|
|
}
|
|
}
|
|
|
|
/* Start by defining group tree separately so we can maintain
|
|
order */
|
|
if((ret=buildGroups(builder,root))) goto done;
|
|
|
|
/* Now, walks selected other nodes to define the netcdf-4 substrate metadata */
|
|
|
|
/* Walk and define the dimensions */
|
|
for(i=0;i<len;i++) {/* Walk in postfix order */
|
|
NCD4node* x = (NCD4node*)nclistget(builder->allnodes,i);
|
|
if(x->sort != NCD4_DIM) continue;
|
|
if((ret=buildDimension(builder,x)))
|
|
goto done;
|
|
}
|
|
|
|
/* Walk and define the enums */
|
|
for(i=0;i<len;i++) {/* Walk in postfix order */
|
|
NCD4node* x = (NCD4node*)nclistget(builder->allnodes,i);
|
|
if(x->sort != NCD4_TYPE) continue;
|
|
if(x->subsort != NC_ENUM) continue;
|
|
if((ret=buildEnumeration(builder,x)))
|
|
goto done;
|
|
}
|
|
|
|
/* Walk and define the opaques */
|
|
#ifndef FIXEDOPAQUE
|
|
/* If _bytestring was required by parser, then create it */
|
|
if(builder->_bytestring != NULL && (ret = buildBytestringType(builder)))
|
|
goto done;
|
|
#endif
|
|
/* Create other opaque types */
|
|
for(i=0;i<len;i++) {/* Walk in postfix order */
|
|
NCD4node* x = (NCD4node*)nclistget(builder->allnodes,i);
|
|
if(x->sort != NCD4_TYPE) continue;
|
|
if(x->subsort != NC_OPAQUE) continue;
|
|
if(x->opaque.size > 0 && (ret=buildOpaque(builder,x)))
|
|
goto done;
|
|
}
|
|
|
|
/* Walk and define the compounds and sequences */
|
|
for(i=0;i<len;i++) {/* Walk in postfix order */
|
|
NCD4node* x = (NCD4node*)nclistget(builder->allnodes,i);
|
|
if(x->sort != NCD4_TYPE) continue;
|
|
switch(x->subsort) {
|
|
case NC_STRUCT:
|
|
if((ret=buildStructureType(builder,x)))
|
|
goto done;
|
|
break;
|
|
case NC_SEQ:
|
|
if((ret=buildVlenType(builder,x)))
|
|
goto done;
|
|
break;
|
|
default: /* ignore */ break;
|
|
}
|
|
}
|
|
|
|
/* Compute the type size for all type */
|
|
for(i=0;i<len;i++) {/* Walk in postfix order */
|
|
NCD4node* x = (NCD4node*)nclistget(builder->allnodes,i);
|
|
if(x->sort != NCD4_TYPE) continue;;
|
|
NCD4_computeTypeSize(builder,x);
|
|
}
|
|
|
|
/* Finally, define the top-level variables */
|
|
for(i=0;i<len;i++) {
|
|
NCD4node* x = (NCD4node*)nclistget(builder->allnodes,i);
|
|
if(ISVAR(x->sort) && ISTOPLEVEL(x)) {
|
|
if((ret=buildVariable(builder,x))) goto done;
|
|
}
|
|
}
|
|
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildGroups(NCD4meta* builder, NCD4node* parent)
|
|
{
|
|
int i,ret=NC_NOERR;
|
|
#ifdef D4DEBUG
|
|
fprintf(stderr,"build group: %s\n",parent->name);
|
|
#endif
|
|
/* Define any group level attributes */
|
|
if((ret = buildAttributes(builder,parent))) goto done;
|
|
|
|
for(i=0;i<nclistlength(parent->groups);i++) {
|
|
NCD4node* g = (NCD4node*)nclistget(parent->groups,i);
|
|
if(g->group.isdataset) {
|
|
g->meta.id = builder->ncid;
|
|
} else {
|
|
NCCHECK((nc_def_grp(parent->meta.id,g->name,&g->meta.id)));
|
|
savegroupbyid(builder,g);
|
|
}
|
|
if((ret=buildGroups(builder,g))) goto done; /* recurse */
|
|
}
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildDimension(NCD4meta* builder, NCD4node* dim)
|
|
{
|
|
int ret = NC_NOERR;
|
|
NCD4node* group = NCD4_groupFor(dim);
|
|
if(dim->dim.isunlimited) {
|
|
NCCHECK((nc_def_dim(group->meta.id,dim->name,NC_UNLIMITED,&dim->meta.id)));
|
|
} else {
|
|
NCCHECK((nc_def_dim(group->meta.id,dim->name,(size_t)dim->dim.size,&dim->meta.id)));
|
|
}
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildEnumeration(NCD4meta* builder, NCD4node* en)
|
|
{
|
|
int i,ret = NC_NOERR;
|
|
NCD4node* group = NCD4_groupFor(en);
|
|
NCCHECK((nc_def_enum(group->meta.id,en->basetype->meta.id,en->name,&en->meta.id)));
|
|
for(i=0;i<nclistlength(en->en.econsts);i++) {
|
|
NCD4node* ec = (NCD4node*)nclistget(en->en.econsts,i);
|
|
NCCHECK((nc_insert_enum(group->meta.id, en->meta.id, ec->name, ec->en.ecvalue.i8)));
|
|
}
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildOpaque(NCD4meta* builder, NCD4node* op)
|
|
{
|
|
int ret = NC_NOERR;
|
|
NCD4node* group = NCD4_groupFor(op);
|
|
char* name = op->name;
|
|
|
|
assert(op->opaque.size > 0);
|
|
/* Two cases, with and without UCARTAGORIGTYPE */
|
|
if(op->nc4.orig.name != NULL) {
|
|
name = op->nc4.orig.name;
|
|
group = op->nc4.orig.group;
|
|
}
|
|
NCCHECK((nc_def_opaque(group->meta.id,op->opaque.size,name,&op->meta.id)));
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
#ifndef FIXEDOPAQUE
|
|
static int
|
|
buildBytestringType(NCD4meta* builder)
|
|
{
|
|
int ret = NC_NOERR;
|
|
NCD4node* bstring = builder->_bytestring;
|
|
|
|
assert(bstring != NULL); /* Will fail if we need bytestring and it was not created in d4parse*/
|
|
|
|
/* Define once */
|
|
if(bstring->meta.id > 0) goto done;
|
|
/* create in root as ubyte(*) vlen named "_bytestring" */
|
|
NCCHECK((nc_def_vlen(builder->root->meta.id,bstring->name,NC_UBYTE,&bstring->meta.id)));
|
|
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
#endif
|
|
|
|
static int
|
|
buildVariable(NCD4meta* builder, NCD4node* var)
|
|
{
|
|
int ret = NC_NOERR;
|
|
|
|
switch (var->subsort) {
|
|
default:
|
|
if((ret = buildAtomicVar(builder,var))) goto done;
|
|
break;
|
|
case NC_STRUCT:
|
|
if((ret = buildStructure(builder,var))) goto done;
|
|
break;
|
|
case NC_SEQ:
|
|
if((ret = buildSequence(builder,var))) goto done;
|
|
break;
|
|
}
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildMetaData(NCD4meta* builder, NCD4node* var)
|
|
{
|
|
int ret = NC_NOERR;
|
|
if((ret = buildAttributes(builder,var))) goto done;
|
|
if((ret = buildMaps(builder,var))) goto done;
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildMaps(NCD4meta* builder, NCD4node* var)
|
|
{
|
|
int i,ret = NC_NOERR;
|
|
size_t count = nclistlength(var->maps);
|
|
char** memory = NULL;
|
|
char** p;
|
|
NCD4node* group;
|
|
|
|
if(count == 0) goto done;
|
|
|
|
/* Add an attribute to the parent variable
|
|
listing fqn's of all specified variables in map order*/
|
|
memory = (char**)d4alloc(count*sizeof(char*));
|
|
if(memory == NULL) {ret=NC_ENOMEM; goto done;}
|
|
p = memory;
|
|
for(i=0;i<count;i++) {
|
|
NCD4node* mapref = (NCD4node*)nclistget(var->maps,i);
|
|
char* fqn = NCD4_makeFQN(mapref);
|
|
*p++ = fqn;
|
|
}
|
|
/* Make map info visible in the netcdf-4 file */
|
|
group = NCD4_groupFor(var);
|
|
NCCHECK((nc_put_att(group->meta.id,var->meta.id,NC4TAGMAPS,NC_STRING,count,memory)));
|
|
done:
|
|
if(memory != NULL)
|
|
freeStringMemory(memory,count);
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildAttributes(NCD4meta* builder, NCD4node* varorgroup)
|
|
{
|
|
int i,ret = NC_NOERR;
|
|
NClist* blobs = NULL;
|
|
|
|
for(i=0;i<nclistlength(varorgroup->attributes);i++) {
|
|
NCD4node* attr = nclistget(varorgroup->attributes,i);
|
|
void* memory = NULL;
|
|
size_t count = nclistlength(attr->attr.values);
|
|
NCD4node* group;
|
|
int varid;
|
|
|
|
/* Suppress all UCARTAG attributes */
|
|
if(strncmp(attr->name,UCARTAG,strlen(UCARTAG)) == 0)
|
|
continue;
|
|
|
|
if(ISGROUP(varorgroup->sort))
|
|
varid = NC_GLOBAL;
|
|
else
|
|
varid = varorgroup->meta.id;
|
|
blobs = nclistnew();
|
|
if((ret=compileAttrValues(builder,attr,&memory,blobs))) {
|
|
nullfree(memory);
|
|
FAIL(ret,"Malformed attribute value(s) for: %s",attr->name);
|
|
}
|
|
group = NCD4_groupFor(varorgroup);
|
|
NCCHECK((nc_put_att(group->meta.id,varid,attr->name,attr->basetype->meta.id,count,memory)));
|
|
nclistfreeall(blobs); blobs = NULL;
|
|
nullfree(memory);
|
|
}
|
|
done:
|
|
nclistfreeall(blobs);
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildStructureType(NCD4meta* builder, NCD4node* structtype)
|
|
{
|
|
int tid,ret = NC_NOERR;
|
|
NCD4node* group = NULL;
|
|
char* name = NULL;
|
|
|
|
group = NCD4_groupFor(structtype); /* default */
|
|
|
|
/* Figure out the type name and containing group */
|
|
if(structtype->nc4.orig.name != NULL) {
|
|
name = strdup(structtype->nc4.orig.name);
|
|
group = structtype->nc4.orig.group;
|
|
} else {
|
|
name = getFieldFQN(structtype,"_t");
|
|
}
|
|
|
|
/* Step 2: See if already defined */
|
|
if(nc_inq_typeid(group->meta.id,name,&tid) == NC_NOERR) {/* Already exists */
|
|
FAIL(NC_ENAMEINUSE,"Inferred type name conflict",name);
|
|
}
|
|
|
|
/* Since netcdf does not support forward references,
|
|
we presume all field types are defined */
|
|
if((ret=buildCompound(builder,structtype,group,name))) goto done;
|
|
|
|
done:
|
|
nullfree(name);
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildVlenType(NCD4meta* builder, NCD4node* vlentype)
|
|
{
|
|
int ret = NC_NOERR;
|
|
NCD4node* group;
|
|
NCD4node* basetype;
|
|
nc_type tid = NC_NAT;
|
|
char* name = NULL;
|
|
|
|
group = NCD4_groupFor(vlentype);
|
|
|
|
/* Figure out the type name and containing group */
|
|
if(vlentype->nc4.orig.name != NULL) {
|
|
name = strdup(vlentype->nc4.orig.name);
|
|
group = vlentype->nc4.orig.group;
|
|
} else {
|
|
name = getFieldFQN(vlentype,NULL);
|
|
}
|
|
|
|
/* See if already defined */
|
|
if(nc_inq_typeid(group->meta.id,name,&tid) == NC_NOERR) {/* Already exists */
|
|
FAIL(NC_ENAMEINUSE,"Inferred type name conflict",name);
|
|
}
|
|
|
|
/* Get the baseline type */
|
|
basetype = vlentype->basetype;
|
|
/* build the vlen type */
|
|
NCCHECK(nc_def_vlen(group->meta.id, name, basetype->meta.id, &vlentype->meta.id));
|
|
|
|
done:
|
|
nullfree(name);
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildCompound(NCD4meta* builder, NCD4node* cmpdtype, NCD4node* group, char* name)
|
|
{
|
|
int i,ret = NC_NOERR;
|
|
|
|
/* Step 1: compute field offsets */
|
|
computeOffsets(builder,cmpdtype);
|
|
|
|
/* Step 2: define this node's compound type */
|
|
NCCHECK((nc_def_compound(group->meta.id,(size_t)cmpdtype->meta.memsize,name,&cmpdtype->meta.id)));
|
|
|
|
/* Step 3: add the fields to type */
|
|
for(i=0;i<nclistlength(cmpdtype->vars);i++) {
|
|
int rank;
|
|
int dimsizes[NC_MAX_VAR_DIMS];
|
|
NCD4node* field = (NCD4node*)nclistget(cmpdtype->vars,i);
|
|
rank = nclistlength(field->dims);
|
|
if(rank == 0) { /* scalar */
|
|
NCCHECK((nc_insert_compound(group->meta.id, cmpdtype->meta.id,
|
|
field->name, field->meta.offset,
|
|
field->basetype->meta.id)));
|
|
} else if(rank > 0) { /* array */
|
|
int idimsizes[NC_MAX_VAR_DIMS];
|
|
int j;
|
|
getDimsizes(field,dimsizes);
|
|
/* nc_insert_array_compound: dimsizes arg is not size_t */
|
|
for(j=0;j<rank;j++) idimsizes[j] = (int)dimsizes[j];
|
|
NCCHECK((nc_insert_array_compound(group->meta.id, cmpdtype->meta.id,
|
|
field->name, field->meta.offset,
|
|
field->basetype->meta.id,
|
|
rank, idimsizes)));
|
|
}
|
|
}
|
|
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildAtomicVar(NCD4meta* builder, NCD4node* var)
|
|
{
|
|
int ret = NC_NOERR;
|
|
size_t rank;
|
|
int dimids[NC_MAX_VAR_DIMS];
|
|
NCD4node* group;
|
|
|
|
group = NCD4_groupFor(var);
|
|
|
|
#ifdef D4DEBUG
|
|
fprintf(stderr,"build var: %s.%s\n",group->name,var->name); fflush(stderr);
|
|
#endif
|
|
|
|
rank = getDimrefs(var,dimids);
|
|
NCCHECK((nc_def_var(group->meta.id,var->name,var->basetype->meta.id,rank,dimids,&var->meta.id)));
|
|
/* Tag the var */
|
|
savevarbyid(group,var);
|
|
|
|
/* Build attributes and map attributes */
|
|
if((ret = buildMetaData(builder,var))) goto done;
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildStructure(NCD4meta* builder, NCD4node* structvar)
|
|
{
|
|
int ret = NC_NOERR;
|
|
NCD4node* group;
|
|
int rank;
|
|
int dimids[NC_MAX_VAR_DIMS];
|
|
|
|
/* Step 1: define the variable */
|
|
rank = nclistlength(structvar->dims);
|
|
getDimrefs(structvar,dimids);
|
|
group = NCD4_groupFor(structvar);
|
|
NCCHECK((nc_def_var(group->meta.id,structvar->name,structvar->basetype->meta.id,rank,dimids,&structvar->meta.id)));
|
|
/* Tag the var */
|
|
savevarbyid(group,structvar);
|
|
|
|
/* Build attributes and map attributes WRT the variable */
|
|
if((ret = buildMetaData(builder,structvar))) goto done;
|
|
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
buildSequence(NCD4meta* builder, NCD4node* seq)
|
|
{
|
|
|
|
int ret = NC_NOERR;
|
|
NCD4node* group;
|
|
int rank;
|
|
int dimids[NC_MAX_VAR_DIMS];
|
|
|
|
rank = nclistlength(seq->dims);
|
|
getDimrefs(seq,dimids);
|
|
group = NCD4_groupFor(seq);
|
|
NCCHECK((nc_def_var(group->meta.id,seq->name,seq->basetype->meta.id,rank,dimids,&seq->meta.id)));
|
|
savevarbyid(group,seq);
|
|
|
|
/* Build attributes and map attributes WRT the variable */
|
|
if((ret = buildMetaData(builder,seq))) goto done;
|
|
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
/***************************************************/
|
|
/* Utilities */
|
|
|
|
/* Insert a group into the groupbyid for a group */
|
|
static void
|
|
savegroupbyid(NCD4meta* meta, NCD4node* group)
|
|
{
|
|
if(meta->groupbyid == NULL)
|
|
meta->groupbyid = nclistnew();
|
|
nclistsetalloc(meta->groupbyid,GROUPIDPART(group->meta.id));
|
|
nclistinsert(meta->groupbyid,GROUPIDPART(group->meta.id),group);
|
|
}
|
|
|
|
/* Insert a var into the varbyid for a group */
|
|
static void
|
|
savevarbyid(NCD4node* group, NCD4node* var)
|
|
{
|
|
if(group->group.varbyid == NULL)
|
|
group->group.varbyid = nclistnew();
|
|
nclistsetalloc(group->group.varbyid,var->meta.id);
|
|
nclistinsert(group->group.varbyid,var->meta.id,var);
|
|
}
|
|
|
|
/* Collect FQN path from node upto (but not including)
|
|
the first enclosing group and create an name from it
|
|
*/
|
|
static char*
|
|
getFieldFQN(NCD4node* field, const char* tail)
|
|
{
|
|
int i;
|
|
NCD4node* x = NULL;
|
|
NClist* path = NULL;
|
|
NCbytes* fqn = NULL;
|
|
char* result;
|
|
|
|
path = nclistnew();
|
|
for(x=field;!ISGROUP(x->sort);x=x->container) {
|
|
nclistinsert(path,0,x);
|
|
}
|
|
fqn = ncbytesnew();
|
|
for(i=0;i<nclistlength(path);i++) {
|
|
NCD4node* elem = (NCD4node*)nclistget(path,i);
|
|
char* escaped = backslashEscape(elem->name);
|
|
if(escaped == NULL) return NULL;
|
|
if(i > 0) ncbytesappend(fqn,'.');
|
|
ncbytescat(fqn,escaped);
|
|
free(escaped);
|
|
}
|
|
nclistfree(path);
|
|
if(tail != NULL)
|
|
ncbytescat(fqn,tail);
|
|
result = ncbytesextract(fqn);
|
|
ncbytesfree(fqn);
|
|
return result;
|
|
}
|
|
|
|
static size_t
|
|
getDimrefs(NCD4node* var, int* dimids)
|
|
{
|
|
int i;
|
|
int rank = nclistlength(var->dims);
|
|
for(i=0;i<rank;i++) {
|
|
NCD4node* dim = (NCD4node*)nclistget(var->dims,i);
|
|
dimids[i] = dim->meta.id;
|
|
}
|
|
return rank;
|
|
}
|
|
|
|
static size_t
|
|
getDimsizes(NCD4node* var, int* dimsizes)
|
|
{
|
|
int i;
|
|
int rank = nclistlength(var->dims);
|
|
for(i=0;i<rank;i++) {
|
|
NCD4node* dim = (NCD4node*)nclistget(var->dims,i);
|
|
dimsizes[i] = (int)dim->dim.size;
|
|
}
|
|
return rank;
|
|
}
|
|
|
|
/**************************************************/
|
|
/* Utilities */
|
|
|
|
static void
|
|
freeStringMemory(char** mem, int count)
|
|
{
|
|
int i;
|
|
if(mem == NULL) return;
|
|
for(i=0;i<count;i++) {
|
|
char* p = mem[i];
|
|
if(p) free(p);
|
|
}
|
|
free(mem);
|
|
}
|
|
|
|
/**
|
|
Convert a list of attribute value strings
|
|
into a memory chunk capable of being passed
|
|
to nc_put_att().
|
|
*/
|
|
static int
|
|
compileAttrValues(NCD4meta* builder, NCD4node* attr, void** memoryp, NClist* blobs)
|
|
{
|
|
int i,ret = NC_NOERR;
|
|
unsigned char* memory = NULL;
|
|
unsigned char* p;
|
|
size_t size;
|
|
NCD4node* truebase = NULL;
|
|
union ATOMICS converter;
|
|
int isenum = 0;
|
|
NCD4node* container = attr->container;
|
|
NCD4node* basetype = attr->basetype;
|
|
NClist* values = attr->attr.values;
|
|
int count = nclistlength(values);
|
|
|
|
memset((void*)&converter,0,sizeof(converter));
|
|
|
|
/* Deal with _FillValue */
|
|
if(container->sort == NCD4_VAR && strcmp(attr->name,"_FillValue")==0) {
|
|
/* Verify or fix or fail on type match */
|
|
if(container->basetype != basetype) {
|
|
/* _FillValue/Variable type mismatch */
|
|
if(FLAGSET(builder->controller->controls.flags,NCF_FILLMISMATCH)) {
|
|
/* Force type match */
|
|
basetype = (attr->basetype = container->basetype);
|
|
} else {/* Fail */
|
|
FAIL(NC_EBADTYPE,"_FillValue/Variable type mismatch: %s:%s",container->name,attr->name);
|
|
}
|
|
}
|
|
}
|
|
isenum = (basetype->subsort == NC_ENUM);
|
|
truebase = (isenum ? basetype->basetype : basetype);
|
|
if(!ISTYPE(truebase->sort) || (truebase->meta.id > NC_MAX_ATOMIC_TYPE))
|
|
FAIL(NC_EBADTYPE,"Illegal attribute type: %s",basetype->name);
|
|
size = NCD4_typesize(truebase->meta.id);
|
|
if((memory = (unsigned char*)d4alloc(count*size))==NULL)
|
|
return THROW(NC_ENOMEM);
|
|
p = memory;
|
|
for(i=0;i<count;i++) {
|
|
char* s = (char*)nclistget(values,i);
|
|
if(isenum) {
|
|
if((ret=decodeEconst(builder,basetype,s,&converter)))
|
|
FAIL(ret,"Illegal enum const: ",s);
|
|
} else {
|
|
if((ret = convertString(&converter,basetype,s)))
|
|
FAIL(NC_EBADTYPE,"Illegal attribute type: ",basetype->name);
|
|
}
|
|
ret = downConvert(&converter,truebase);
|
|
p = copyAtomic(&converter,truebase->meta.id,NCD4_typesize(truebase->meta.id),p,blobs);
|
|
}
|
|
if(memoryp) *memoryp = memory;
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static void*
|
|
copyAtomic(union ATOMICS* converter, nc_type type, size_t len, void* dst, NClist* blobs)
|
|
{
|
|
switch (type) {
|
|
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
|
|
memcpy(dst,&converter->u8[0],len); break;
|
|
case NC_SHORT: case NC_USHORT:
|
|
memcpy(dst,&converter->u16[0],len); break;
|
|
case NC_INT: case NC_UINT:
|
|
memcpy(dst,&converter->u32[0],len); break;
|
|
case NC_INT64: case NC_UINT64:
|
|
memcpy(dst,&converter->u64[0],len); break;
|
|
case NC_FLOAT:
|
|
memcpy(dst,&converter->f32[0],len); break;
|
|
case NC_DOUBLE:
|
|
memcpy(dst,&converter->f64[0],len); break;
|
|
case NC_STRING:
|
|
memcpy(dst,&converter->s[0],len);
|
|
nclistpush(blobs,converter->s[0]);
|
|
converter->s[0] = NULL; /* avoid duplicate free */
|
|
break;
|
|
}/*switch*/
|
|
return (((char*)dst)+len);
|
|
}
|
|
|
|
static int
|
|
convertString(union ATOMICS* converter, NCD4node* type, const char* s)
|
|
{
|
|
switch (type->subsort) {
|
|
case NC_BYTE:
|
|
case NC_SHORT:
|
|
case NC_INT:
|
|
case NC_INT64:
|
|
if(sscanf(s,"%lld",&converter->i64[0]) != 1) return THROW(NC_ERANGE);
|
|
break;
|
|
case NC_UBYTE:
|
|
case NC_USHORT:
|
|
case NC_UINT:
|
|
case NC_UINT64:
|
|
if(sscanf(s,"%llu",&converter->u64[0]) != 1) return THROW(NC_ERANGE);
|
|
break;
|
|
case NC_FLOAT:
|
|
case NC_DOUBLE:
|
|
if(sscanf(s,"%lf",&converter->f64[0]) != 1) return THROW(NC_ERANGE);
|
|
break;
|
|
case NC_CHAR:
|
|
converter->i8[0] = s[0];
|
|
break;
|
|
case NC_STRING:
|
|
converter->s[0]= strdup(s);
|
|
break;
|
|
}/*switch*/
|
|
return downConvert(converter,type);
|
|
}
|
|
|
|
static int
|
|
downConvert(union ATOMICS* converter, NCD4node* type)
|
|
{
|
|
d4size_t u64 = converter->u64[0];
|
|
long long i64 = converter->i64[0];
|
|
double f64 = converter->f64[0];
|
|
char* s = converter->s[0];
|
|
switch (type->subsort) {
|
|
case NC_CHAR:
|
|
case NC_BYTE:
|
|
converter->i8[0] = (char)i64;
|
|
break;
|
|
case NC_UBYTE:
|
|
converter->u8[0] = (unsigned char)u64;
|
|
break;
|
|
case NC_SHORT:
|
|
converter->i16[0] = (short)i64;
|
|
break;
|
|
case NC_USHORT:
|
|
converter->u16[0] = (unsigned short)u64;
|
|
break;
|
|
case NC_INT:
|
|
converter->i32[0] = (int)i64;
|
|
break;
|
|
case NC_UINT:
|
|
converter->u32[0] = (unsigned int)u64;
|
|
break;
|
|
case NC_INT64:
|
|
converter->i64[0] = i64;
|
|
break;
|
|
case NC_UINT64:
|
|
converter->u64[0]= u64;
|
|
break;
|
|
case NC_FLOAT:
|
|
converter->f32[0] = (float)f64;
|
|
break;
|
|
case NC_DOUBLE:
|
|
converter->f64[0] = f64;
|
|
break;
|
|
case NC_STRING:
|
|
converter->s[0]= s;
|
|
break;
|
|
}/*switch*/
|
|
return THROW(NC_NOERR);
|
|
}
|
|
|
|
/*
|
|
Given an enum type, and a string representing an econst,
|
|
convert to integer.
|
|
Note: this will work if the econst string is a number or a econst name
|
|
*/
|
|
static int
|
|
decodeEconst(NCD4meta* builder, NCD4node* enumtype, const char* nameorval, union ATOMICS* converter)
|
|
{
|
|
int i,ret=NC_NOERR;
|
|
union ATOMICS number;
|
|
NCD4node* match = NULL;
|
|
|
|
/* First, see if the value is an econst name */
|
|
for(i=0;i<nclistlength(enumtype->en.econsts);i++) {
|
|
NCD4node* ec = (NCD4node*)nclistget(enumtype->en.econsts,i);
|
|
if(strcmp(ec->name,nameorval)==0) {match = ec; break;}
|
|
}
|
|
/* If no match, try to invert as a number to see if there is a matching econst */
|
|
if(!match) {
|
|
/* get the incoming value as number */
|
|
if((ret=convertString(&number,enumtype->basetype,nameorval)))
|
|
goto done;
|
|
for(i=0;i<nclistlength(enumtype->en.econsts);i++) {
|
|
NCD4node* ec = (NCD4node*)nclistget(enumtype->en.econsts,i);
|
|
if(ec->en.ecvalue.u64[0] == number.u64[0]) {match = ec; break;}
|
|
}
|
|
}
|
|
if(match == NULL)
|
|
FAIL(NC_EINVAL,"No enum const matching value: %s",nameorval);
|
|
if(converter) *converter = match->en.ecvalue;
|
|
done:
|
|
return THROW(ret);
|
|
}
|
|
|
|
static char*
|
|
backslashEscape(const char* s)
|
|
{
|
|
const char* p;
|
|
char* q;
|
|
size_t len;
|
|
char* escaped = NULL;
|
|
|
|
len = strlen(s);
|
|
escaped = (char*)d4alloc(1+(2*len)); /* max is everychar is escaped */
|
|
if(escaped == NULL) return NULL;
|
|
for(p=s,q=escaped;*p;p++) {
|
|
char c = *p;
|
|
switch (c) {
|
|
case '\\':
|
|
case '/':
|
|
case '.':
|
|
case '@':
|
|
*q++ = '\\'; *q++ = '\\';
|
|
break;
|
|
default: *q++ = c; break;
|
|
}
|
|
}
|
|
*q = '\0';
|
|
return escaped;
|
|
}
|
|
|
|
/* Tag each compound type as fixed size or not
|
|
Assumes:
|
|
- atomic types defined and marked
|
|
- topo sorted
|
|
*/
|
|
|
|
static int
|
|
markfixedsize(NCD4meta* meta)
|
|
{
|
|
int i,j;
|
|
for(i=0;i<nclistlength(meta->allnodes);i++) {
|
|
int fixed = 1;
|
|
NCD4node* n = (NCD4node*)nclistget(meta->allnodes,i);
|
|
if(n->sort != NCD4_TYPE) continue;
|
|
switch (n->subsort) {
|
|
case NC_STRUCT:
|
|
for(j=0;j<nclistlength(n->vars);j++) {
|
|
NCD4node* field = (NCD4node*)nclistget(n->vars,j);
|
|
if(!field->basetype->meta.isfixedsize) {
|
|
fixed = 0;
|
|
break;
|
|
}
|
|
}
|
|
n->meta.isfixedsize = fixed;
|
|
break;
|
|
case NC_ENUM:
|
|
n->meta.isfixedsize = 1;
|
|
break;
|
|
default: /* leave as is */
|
|
break;
|
|
}
|
|
}
|
|
return NC_NOERR;
|
|
}
|
|
|
|
/* Compute compound type field offsets and compound type total size */
|
|
static void
|
|
computeOffsets(NCD4meta* builder, NCD4node* cmpd)
|
|
{
|
|
int i;
|
|
d4size_t offset = 0;
|
|
d4size_t largestalign = 1;
|
|
d4size_t size = 0;
|
|
|
|
for(i=0;i<nclistlength(cmpd->vars);i++) {
|
|
NCD4node* field = (NCD4node*)nclistget(cmpd->vars,i);
|
|
NCD4node* ftype = field->basetype;
|
|
size_t alignment;
|
|
if(ftype->subsort == NC_STRUCT) {
|
|
/* Recurse */
|
|
computeOffsets(builder, ftype);
|
|
assert(ftype->meta.memsize > 0);
|
|
size=ftype->meta.memsize;
|
|
alignment = ftype->meta.alignment;
|
|
} else {/* Size and alignment will already have been set */
|
|
assert(ftype->meta.memsize > 0);
|
|
alignment = ftype->meta.alignment;
|
|
size=ftype->meta.memsize;
|
|
}
|
|
#if 0
|
|
} else if(ftype->subsort == NC_SEQ) { /* VLEN */
|
|
alignment = nctypealignment(NC_VLEN);
|
|
assert(ftype->meta.memsize > 0); size=ftype->meta.memsize;
|
|
//size = NCD4_computeTypeSize(builder,ftype);
|
|
} else if(ftype->subsort == NC_OPAQUE) {
|
|
/* Either fixed or a vlen */
|
|
assert(ftype->meta.memsize > 0); size=ftype->meta.memsize;
|
|
if(ftype->opaque.size == 0) {/* treat like vlen */
|
|
alignment = nctypealignment(NC_VLEN);
|
|
//size = NCD4_computeTypeSize(builder,ftype);
|
|
} else { /* fixed size */
|
|
alignment = nctypealignment(NC_OPAQUE);
|
|
//size = NCD4_computeTypeSize(builder,ftype);
|
|
}
|
|
} else if(ftype->subsort == NC_ENUM) {
|
|
NCD4node* truetype = ftype->basetype;
|
|
alignment = nctypealignment(truetype->meta.id);
|
|
assert(ftype->meta.memsize > 0); size=ftype->meta.memsize;
|
|
//size = NCD4_computeTypeSize(builder,truetype);
|
|
} else { /* Basically a primitive */
|
|
alignment = nctypealignment(ftype->meta.id);
|
|
assert(ftype->meta.memsize > 0); size=ftype->meta.memsize;
|
|
//size = NCD4_computeTypeSize(builder,ftype);
|
|
}
|
|
#endif
|
|
if(alignment > largestalign)
|
|
largestalign = alignment;
|
|
/* Add possible padding wrt to previous field */
|
|
offset += getpadding(offset,alignment);
|
|
field->meta.offset = offset;
|
|
assert(ftype->meta.memsize > 0);
|
|
size = ftype->meta.memsize;
|
|
/*field->meta.memsize = size;*/
|
|
/* Now ultiply by the field dimproduct*/
|
|
if(nclistlength(field->dims) > 0) {
|
|
d4size_t count = NCD4_dimproduct(field);
|
|
size = (size * count);
|
|
}
|
|
offset += size;
|
|
}
|
|
/* Compute compound-level info */
|
|
/* A struct alignment is the same as largestalign */
|
|
cmpd->meta.alignment = largestalign;
|
|
offset += (offset % largestalign); /* round up compound size */
|
|
cmpd->meta.memsize = offset;
|
|
}
|
|
|
|
/*
|
|
Compute the in-memory size of an instance of a type.
|
|
Note that nc_inq_type is used, so that C struct field
|
|
alignment is taken into account for compound types.
|
|
The variables total size will be this * dimproduct.
|
|
*/
|
|
size_t
|
|
NCD4_computeTypeSize(NCD4meta* builder, NCD4node* type)
|
|
{
|
|
size_t size = 0;
|
|
|
|
switch (type->sort) {
|
|
case NCD4_TYPE:
|
|
switch (type->subsort) {
|
|
default: size = NCD4_typesize(type->meta.id); break;
|
|
case NC_OPAQUE:
|
|
size = (type->opaque.size == 0 ? sizeof(nc_vlen_t) : type->opaque.size);
|
|
break;
|
|
case NC_ENUM:
|
|
size = NCD4_computeTypeSize(builder,type->basetype);
|
|
break;
|
|
case NC_SEQ:
|
|
size = sizeof(nc_vlen_t);
|
|
break;
|
|
case NC_STRUCT: {
|
|
int ret;
|
|
NCD4node* group = NCD4_groupFor(type);
|
|
if((ret = nc_inq_type(group->meta.id,type->meta.id,/*name*/NULL,&size)))
|
|
return 0;
|
|
}; break;
|
|
}
|
|
break;
|
|
default: break; /* ignore */
|
|
}
|
|
type->meta.memsize = size;
|
|
return size;
|
|
}
|
|
|
|
static d4size_t
|
|
getpadding(d4size_t offset, size_t alignment)
|
|
{
|
|
d4size_t rem = (alignment==0?0:(offset % alignment));
|
|
d4size_t pad = (rem==0?0:(alignment - rem));
|
|
return pad;
|
|
}
|
|
|
|
/* Compute the dap data size for each type; note that this
|
|
is unlikely to be the same as the meta.memsize unless
|
|
the type is atomic and is <= NC_UINT64.
|
|
*/
|
|
|
|
static int
|
|
markdapsize(NCD4meta* meta)
|
|
{
|
|
int i,j;
|
|
for(i=0;i<nclistlength(meta->allnodes);i++) {
|
|
NCD4node* type = (NCD4node*)nclistget(meta->allnodes,i);
|
|
size_t totalsize;
|
|
if(type->sort != NCD4_TYPE) continue;
|
|
switch (type->subsort) {
|
|
case NC_STRUCT:
|
|
totalsize = 0;
|
|
for(j=0;j<nclistlength(type->vars);j++) {
|
|
NCD4node* field = (NCD4node*)nclistget(type->vars,j);
|
|
size_t size = field->basetype->meta.dapsize;
|
|
if(size == 0) {
|
|
totalsize = 0;
|
|
break;
|
|
} else
|
|
totalsize += size;
|
|
}
|
|
type->meta.dapsize = totalsize;
|
|
break;
|
|
case NC_SEQ:
|
|
type->meta.dapsize = 0; /* has no fixed size */
|
|
break;
|
|
case NC_OPAQUE:
|
|
type->meta.dapsize = type->opaque.size;
|
|
break;
|
|
case NC_ENUM:
|
|
type->meta.dapsize = type->basetype->meta.dapsize;
|
|
break;
|
|
case NC_STRING:
|
|
type->meta.dapsize = 0; /* has no fixed size */
|
|
break;
|
|
default:
|
|
assert(type->subsort <= NC_UINT64);
|
|
/* Already assigned */
|
|
break;
|
|
}
|
|
}
|
|
return NC_NOERR;
|
|
}
|
|
|