netcdf-c/ncgen/genbin.c
Dennis Heimbigner 751300ec59 Fix more memory leaks in netcdf-c library
This is a follow up to PR https://github.com/Unidata/netcdf-c/pull/1173

Sorry that it is so big, but leak suppression can be complex.

This PR fixes all remaining memory leaks -- as determined by
-fsanitize=address, and with the exceptions noted below.

Unfortunately. there remains a significant leak that I cannot
solve. It involves vlens, and it is unclear if the leak is
occurring in the netcdf-c library or the HDF5 library.

I have added a check_PROGRAM to the ncdump directory to show the
problem.  The program is called tst_vlen_demo.c To exercise it,
build the netcdf library with -fsanitize=address enabled. Then
go into ncdump and do a "make clean check".  This should build
tst_vlen_demo without actually executing it.  Then do the
command "./tst_vlen_demo" to see the output of the memory
checker.  Note the the lost malloc is deep in the HDF5 library
(in H5Tvlen.c).

I am temporarily working around this error in the following way.
1. I modified several test scripts to not execute known vlen tests
   that fail as described above.
2. Added an environment variable called NC_VLEN_NOTEST.
   If set, then those specific tests are suppressed.

This should mean that the --disable-utilities option to
./configure should not need to be set to get a memory leak clean
build.  This should allow for detection of any new leaks.

Note: I used an environment variable rather than a ./configure
option to control the vlen tests. This is because it is
temporary (I hope) and because it is a bit tricky for shell
scripts to access ./configure options.

Finally, as before, this only been tested with netcdf-4 and hdf5 support.
2018-11-15 10:00:38 -07:00

571 lines
19 KiB
C

/*********************************************************************
* Copyright 1993, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* $Header: /upc/share/CVS/netcdf-3/ncgen/genbin.c,v 1.4 2010/05/27 21:34:17 dmh Exp $
*********************************************************************/
#include "includes.h"
#include <ctype.h> /* for isprint() */
#include "netcdf_aux.h"
#ifdef ENABLE_BINARY
#undef TRACE
/* Forward*/
static int genbin_defineattr(Symbol* asym);
static int genbin_definevardata(Symbol* vsym);
static int genbin_write(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
static int genbin_writevar(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
static int genbin_writeattr(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
#ifdef USE_NETCDF4
static int genbin_deftype(Symbol* tsym);
static int genbin_definespecialattributes(Symbol* var);
#endif
/*
* Generate C code for creating netCDF from in-memory structure.
*/
void
genbin_netcdf(void)
{
int stat, ncid;
int idim, ivar, iatt;
int ndims, nvars, natts, ngatts;
const char* filename = rootgroup->file.filename;
#ifdef USE_NETCDF4
int ntyps, ngrps, igrp;
#endif
ndims = listlength(dimdefs);
nvars = listlength(vardefs);
natts = listlength(attdefs);
ngatts = listlength(gattdefs);
#ifdef USE_NETCDF4
ntyps = listlength(typdefs);
ngrps = listlength(grpdefs);
#endif /*USE_NETCDF4*/
/* Turn on logging */
#ifdef LOGGING
nc_set_log_level(ncloglevel);
#endif
/* create netCDF file, uses NC_CLOBBER mode */
cmode_modifier |= NC_CLOBBER;
#ifdef USE_NETCDF4
if(!usingclassic)
cmode_modifier |= NC_NETCDF4;
#endif
stat = nc_create(filename, cmode_modifier, &ncid);
check_err(stat,__LINE__,__FILE__);
/* ncid created above is also root group*/
rootgroup->nc_id = ncid;
#ifdef USE_NETCDF4
/* Define the group structure */
/* walking grdefs list will do a preorder walk of all defined groups*/
for(igrp=0;igrp<ngrps;igrp++) {
Symbol* gsym = (Symbol*)listget(grpdefs,igrp);
if(gsym == rootgroup) continue; /* ignore root group*/
stat = nc_def_grp(gsym->container->nc_id,gsym->name,&gsym->nc_id);
check_err(stat,__LINE__,__FILE__);
}
#endif
#ifdef USE_NETCDF4
/* Define the types*/
if (ntyps > 0) {
int ityp;
for(ityp = 0; ityp < ntyps; ityp++) {
Symbol* tsym = (Symbol*)listget(typdefs,ityp);
genbin_deftype(tsym);
}
}
#endif
/* define dimensions from info in dims array */
if (ndims > 0) {
for(idim = 0; idim < ndims; idim++) {
Symbol* dsym = (Symbol*)listget(dimdefs,idim);
stat = nc_def_dim(dsym->container->nc_id,
dsym->name,
(dsym->dim.isunlimited?NC_UNLIMITED:dsym->dim.declsize),
&dsym->nc_id);
check_err(stat,__LINE__,__FILE__);
}
}
/* define variables from info in vars array */
if (nvars > 0) {
for(ivar = 0; ivar < nvars; ivar++) {
Symbol* vsym = (Symbol*)listget(vardefs,ivar);
if (vsym->typ.dimset.ndims > 0) { /* a dimensioned variable */
/* construct a vector of dimension ids*/
int dimids[NC_MAX_VAR_DIMS];
for(idim=0;idim<vsym->typ.dimset.ndims;idim++)
dimids[idim] = vsym->typ.dimset.dimsyms[idim]->nc_id;
stat = nc_def_var(vsym->container->nc_id,
vsym->name,
vsym->typ.basetype->nc_id,
vsym->typ.dimset.ndims,
dimids,
&vsym->nc_id);
} else { /* a scalar */
stat = nc_def_var(vsym->container->nc_id,
vsym->name,
vsym->typ.basetype->nc_id,
vsym->typ.dimset.ndims,
NULL,
&vsym->nc_id);
}
check_err(stat,__LINE__,__FILE__);
}
}
#ifdef USE_NETCDF4
/* define special variable properties */
if(nvars > 0) {
for(ivar = 0; ivar < nvars; ivar++) {
Symbol* var = (Symbol*)listget(vardefs,ivar);
genbin_definespecialattributes(var);
}
}
#endif /*USE_NETCDF4*/
/* define global attributes */
if(ngatts > 0) {
for(iatt = 0; iatt < ngatts; iatt++) {
Symbol* gasym = (Symbol*)listget(gattdefs,iatt);
genbin_defineattr(gasym);
}
}
/* define per-variable attributes */
if(natts > 0) {
for(iatt = 0; iatt < natts; iatt++) {
Symbol* asym = (Symbol*)listget(attdefs,iatt);
genbin_defineattr(asym);
}
}
if (nofill_flag) {
stat = nc_set_fill(rootgroup->nc_id, NC_NOFILL, 0);
check_err(stat,__LINE__,__FILE__);
}
/* leave define mode */
stat = nc_enddef(rootgroup->nc_id);
check_err(stat,__LINE__,__FILE__);
if(!header_only) {
/* Load values into those variables with defined data */
if(nvars > 0) {
for(ivar = 0; ivar < nvars; ivar++) {
Symbol* vsym = (Symbol*)listget(vardefs,ivar);
if(vsym->data != NULL) {
genbin_definevardata(vsym);
}
}
}
}
}
#ifdef USE_NETCDF4
#if 0
Turn off for now.
static void
genbin_defineglobalspecials(void)
{
int stat = NC_NOERR;
const char* format = NULL;
if(usingclassic) return;
if(!/*Main.*/format_attribute) return;
/* Watch out, this is a global Attribute */
format = kind_string(/*Main.*/format_flag);
stat = nc_put_att_text(rootgroup->nc_id,NC_GLOBAL,"_Format",strlen(format),format);
check_err(stat,__LINE__,__FILE__);
}
#endif /*0*/
static int
genbin_definespecialattributes(Symbol* var)
{
int stat = NC_NOERR;
Specialdata* special = var->var.special;
if(special->flags & _STORAGE_FLAG) {
if(special->_Storage == NC_CONTIGUOUS) {
stat = nc_def_var_chunking(var->container->nc_id, var->nc_id, NC_CONTIGUOUS, NULL);
} else { /* chunked */
if(special->nchunks == 0 || special->_ChunkSizes == NULL)
derror("NC_CHUNKED requested, but no chunksizes specified");
stat = nc_def_var_chunking(var->container->nc_id, var->nc_id, NC_CHUNKED, special->_ChunkSizes);
}
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & _FLETCHER32_FLAG) {
stat = nc_def_var_fletcher32(var->container->nc_id,
var->nc_id,
special->_Fletcher32);
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & (_DEFLATE_FLAG | _SHUFFLE_FLAG)) {
stat = nc_def_var_deflate(var->container->nc_id,
var->nc_id,
(special->_Shuffle == 1?1:0),
(special->_DeflateLevel >= 0?1:0),
(special->_DeflateLevel >= 0?special->_DeflateLevel
:0));
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & _ENDIAN_FLAG) {
stat = nc_def_var_endian(var->container->nc_id,
var->nc_id,
(special->_Endianness == NC_ENDIAN_LITTLE?
NC_ENDIAN_LITTLE
:NC_ENDIAN_BIG));
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & _NOFILL_FLAG) {
stat = nc_def_var_fill(var->container->nc_id,
var->nc_id,
(special->_Fill?NC_FILL:NC_NOFILL),
NULL);
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & _FILTER_FLAG) {
/* Special check for alternate way to specify _Deflate */
if(special->_FilterID == ZIP_ID) {
unsigned int level;
if(special->nparams == 0 || special->_FilterParams == NULL)
level = 9; /* default */
else
level = special->_FilterParams[0];
if(level > 9)
derror("Illegal deflate level");
else {
stat = nc_def_var_deflate(var->container->nc_id,
var->nc_id,
(special->_Shuffle == 1?1:0),
(level > 0?1:0),
level);
}
} else {
stat = nc_def_var_filter(var->container->nc_id,
var->nc_id,
special->_FilterID,
special->nparams,
special->_FilterParams
);
}
check_err(stat,__LINE__,__FILE__);
}
return stat;
}
#endif /*USE_NETCDF4*/
void
genbin_close(void)
{
int stat;
stat = nc_close(rootgroup->nc_id);
check_err(stat,__LINE__,__FILE__);
}
#ifdef USE_NETCDF4
/*
Generate type definitions
*/
static int
genbin_deftype(Symbol* tsym)
{
unsigned long i;
int stat;
ASSERT(tsym->objectclass == NC_TYPE);
switch (tsym->subclass) {
case NC_PRIM: break; /* these are already taken care of*/
case NC_OPAQUE:
stat = nc_def_opaque(tsym->container->nc_id,
tsym->typ.size,
tsym->name,
&tsym->nc_id);
check_err(stat,__LINE__,__FILE__);
break;
case NC_ENUM:
{
Bytebuffer* datum;
stat = nc_def_enum(tsym->container->nc_id,
tsym->typ.basetype->nc_id,
tsym->name,
&tsym->nc_id);
check_err(stat,__LINE__,__FILE__);
datum = bbNew();
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* econst = (Symbol*)listget(tsym->subnodes,i);
ASSERT(econst->subclass == NC_ECONST);
generator_reset(bin_generator,NULL);
bbClear(datum);
generate_basetype(econst->typ.basetype,econst->typ.econst,datum,NULL,bin_generator);
stat = nc_insert_enum(tsym->container->nc_id,
tsym->nc_id,
econst->name,
bbContents(datum));
check_err(stat,__LINE__,__FILE__);
}
bbFree(datum);
}
break;
case NC_VLEN:
stat = nc_def_vlen(tsym->container->nc_id,
tsym->name,
tsym->typ.basetype->nc_id,
&tsym->nc_id);
check_err(stat,__LINE__,__FILE__);
break;
case NC_COMPOUND:
stat = nc_def_compound(tsym->container->nc_id,
tsym->typ.size,
tsym->name,
&tsym->nc_id);
check_err(stat,__LINE__,__FILE__);
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* efield = (Symbol*)listget(tsym->subnodes,i);
ASSERT(efield->subclass == NC_FIELD);
if(efield->typ.dimset.ndims == 0){
stat = nc_insert_compound(
tsym->container->nc_id,
tsym->nc_id,
efield->name,
efield->typ.offset,
efield->typ.basetype->nc_id);
} else {
int j;
int dimsizes[NC_MAX_VAR_DIMS]; /* int because inside compound */
/* Generate the field dimension constants*/
for(j=0;j<efield->typ.dimset.ndims;j++) {
unsigned int size = efield->typ.dimset.dimsyms[j]->dim.declsize;
dimsizes[j] = size;
}
stat = nc_insert_array_compound(
tsym->container->nc_id,
tsym->nc_id,
efield->name,
efield->typ.offset,
efield->typ.basetype->nc_id,
efield->typ.dimset.ndims,
dimsizes);
}
check_err(stat,__LINE__,__FILE__);
}
break;
default: panic("definectype: unexpected type subclass");
}
return stat;
}
#endif /*USE_NETCDF4*/
static int
genbin_defineattr(Symbol* asym)
{
int stat = NC_NOERR;
Bytebuffer* databuf = bbNew();
generator_reset(bin_generator,NULL);
generate_attrdata(asym,bin_generator,(Writer)genbin_write,databuf);
if((stat = ncaux_reclaim_data(asym->container->nc_id,asym->typ.basetype->nc_id,bbContents(databuf),datalistlen(asym->data))))
goto done;
done:
bbFree(databuf);
return stat;
}
/* Following is patterned after the walk functions in semantics.c */
static int
genbin_definevardata(Symbol* vsym)
{
int stat = NC_NOERR;
Bytebuffer* databuf = NULL;
if(vsym->data == NULL) goto done;
databuf = bbNew();
generator_reset(bin_generator,NULL);
generate_vardata(vsym,bin_generator,(Writer)genbin_write,databuf);
ncaux_reclaim_data(vsym->container->nc_id,vsym->typ.basetype->nc_id,bbContents(databuf),datalistlen(vsym->data));
done:
bbFree(databuf);
return stat;
}
static int
genbin_write(Generator* generator, Symbol* sym, Bytebuffer* memory,
int rank, size_t* start, size_t* count)
{
if(sym->objectclass == NC_ATT)
return genbin_writeattr(generator,sym,memory,rank,start,count);
else if(sym->objectclass == NC_VAR)
return genbin_writevar(generator,sym,memory,rank,start,count);
else
PANIC("illegal symbol for genbin_write");
return NC_EINVAL;
}
static int
genbin_writevar(Generator* generator, Symbol* vsym, Bytebuffer* memory,
int rank, size_t* start, size_t* count)
{
int stat = NC_NOERR;
int i;
char* data = bbContents(memory);
size_t nelems;
/* Compute total number of elements */
for(nelems=1,i=0;i<rank;i++) nelems *= count[i];
#ifdef GENDEBUG
{
int i;
fprintf(stderr,"startset = [");
for(i=0;i<rank;i++)
fprintf(stderr,"%s%lu",(i>0?", ":""),(unsigned long)start[i]);
fprintf(stderr,"] ");
fprintf(stderr,"countset = [");
for(i=0;i<rank;i++)
fprintf(stderr,"%s%lu",(i>0?", ":""),(unsigned long)count[i]);
fprintf(stderr,"]\n");
fflush(stderr);
}
#endif
if(rank == 0) {
size_t count[1] = {1};
stat = nc_put_var1(vsym->container->nc_id, vsym->nc_id, count, data);
} else {
stat = nc_put_vara(vsym->container->nc_id, vsym->nc_id, start, count, data);
}
check_err(stat,__LINE__,__FILE__);
// /* Reclaim the data */
// stat = ncaux_reclaim_data(vsym->container->nc_id, vsym->typ.basetype->nc_id, data, nelems);
// check_err(stat,__LINE__,__FILE__);
// bbClear(memory); /* reclaim top-level memory */
return stat;
}
static int
genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
int rank, size_t* start, size_t* count)
{
int stat = NC_NOERR;
size_t len;
Datalist* list;
int varid, grpid, typid;
Symbol* basetype = asym->typ.basetype;
grpid = asym->container->nc_id;
varid = (asym->att.var == NULL?NC_GLOBAL : asym->att.var->nc_id);
typid = basetype->nc_id;
list = asym->data;
len = list->length;
/* Use the specialized put_att_XX routines if possible*/
if(isprim(basetype->typ.typecode)) {
switch (basetype->typ.typecode) {
case NC_BYTE: {
signed char* data = (signed char*)bbContents(databuf);
stat = nc_put_att_schar(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_CHAR: {
char* data = (char*)bbContents(databuf);
size_t slen = bbLength(databuf);
/* Revise length if slen == 0 */
if(slen == 0) {
bbAppend(databuf,'\0');
/* bbAppend frees the memory pointed to by char* data,
so re-assign. See Coverity issue: 1265731.*/
data = (char*)bbContents(databuf);
slen++;
}
stat = nc_put_att_text(grpid,varid,asym->name,slen,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_SHORT: {
short* data = (short*)bbContents(databuf);
stat = nc_put_att_short(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_INT: {
int* data = (int*)bbContents(databuf);
stat = nc_put_att_int(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_FLOAT: {
float* data = (float*)bbContents(databuf);
stat = nc_put_att_float(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_DOUBLE: {
double* data = (double*)bbContents(databuf);
stat = nc_put_att_double(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_STRING: {
const char** data;
data = (const char**)bbContents(databuf);
stat = nc_put_att_string(grpid,varid,asym->name,
bbLength(databuf)/sizeof(char*),
data);
} break;
case NC_UBYTE: {
unsigned char* data = (unsigned char*)bbContents(databuf);
stat = nc_put_att_uchar(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_USHORT: {
unsigned short* data = (unsigned short*)bbContents(databuf);
stat = nc_put_att_ushort(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_UINT: {
unsigned int* data = (unsigned int*)bbContents(databuf);
stat = nc_put_att_uint(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_INT64: {
long long* data = (long long*)bbContents(databuf);
stat = nc_put_att_longlong(grpid,varid,asym->name,typid,len,data);
check_err2(stat,asym->lineno,__LINE__,__FILE__);
} break;
case NC_UINT64: {
unsigned long long* data = (unsigned long long*)bbContents(databuf);
stat = nc_put_att_ulonglong(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
default: PANIC1("genbin_defineattr: unexpected basetype: %d",basetype->typ.typecode);
}
} else { /* use the generic put_attribute for user defined types*/
const char* data;
data = (const char*)bbContents(databuf);
stat = nc_put_att(grpid,varid,asym->name,typid,
len,(void*)data);
check_err(stat,__LINE__,__FILE__);
#ifdef GENDEBUG
{
char out[4096];
memset(out,0x77,sizeof(out));
stat = nc_get_att(grpid,varid,asym->name,&out);
check_err(stat,__LINE__,__FILE__);
}
#endif
}
return stat;
}
#endif /*ENABLE_BINARY*/