netcdf-c/ncgen/jdatajni.c

531 lines
15 KiB
C
Raw Normal View History

2018-12-07 06:40:43 +08:00
/*! \file
Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
2015, 2016, 2017, 2018
University Corporation for Atmospheric Research/Unidata.
See \ref copyright file for more info.
*/
2010-06-03 21:24:43 +08:00
#include "includes.h"
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
#include "ncoffsets.h"
2010-06-03 21:24:43 +08:00
#ifdef ENABLE_JAVA
extern List* vlenconstants; /* List<Constant*>;*/
/* Forward*/
static void genjjni_primdata(Symbol*, Datasrc*, Datalist*, char*, Bytebuffer*);
static void genjjni_data(Symbol* sym, Datasrc*, Datalist*, char*, Bytebuffer*);
static void genjjni_fieldarray(Symbol*, Datasrc*, Dimset*, int, char*, Bytebuffer*);
static void genjjni_arraydatar(Symbol* basetype,
Datasrc* src,
Odometer* odom,
Putvar* closure,
int index,
int checkpoint,
Bytebuffer* code);
static void jopaquestring(Symbol* tsym, Constant* prim, Constant* target);
static int genjjni_putmemory(struct Symbol*, char*, Bytebuffer*, Bytebuffer*);
static int jalignment(Symbol* tsym);
static int jvlenalignment(void);
/**************************************************/
/* Code for generating Java language data lists*/
/**************************************************/
/* Datalist rules: see the rules on the man page */
/* Specialty wrappers for genjjni_data */
void
genjjni_attrdata(Symbol* asym, Bytebuffer* databuf)
{
Datasrc* src;
int typecode = asym->typ.basetype->typ.typecode;
char* memname = "memory";
if(asym->data == NULL) return;
if(typecode == NC_CHAR) {
gen_charattr(asym,databuf);
} else {
src = datalist2src(asym->data);
while(srcmore(src)) {
genjjni_data(asym->typ.basetype,src,NULL,memname,databuf);
}
}
}
void
genjjni_scalardata(Symbol* vsym, Bytebuffer* databuf)
{
Datasrc* src;
char* memname = "memory";
if(vsym->data == NULL) return;
src = datalist2src(vsym->data);
genjjni_data(vsym->typ.basetype,src,
vsym->var.special._Fillvalue,memname,databuf);
if(srcmore(src)) {
semerror(srcline(src),"Extra data at end of datalist");
}
}
void
genjjni_fillvalue(Symbol* tsym, Datalist* fillsrc, Datasrc* src,
char* memname, Bytebuffer* databuf)
{
Datalist* list = NULL;
ASSERT(tsym->objectclass == NC_TYPE);
list = fillsrc;
if(list == NULL) list = getfiller(tsym);
srcpushlist(src,list);
genjjni_data(tsym,src,NULL,memname,databuf);
srcpop(src);
}
void
genjjni_arraydata(Symbol* vsym, Putvar* closure, Bytebuffer* databuf)
{
Datasrc* src;
Datalist* list;
int i,checkpoint;
nc_type typecode = vsym->typ.basetype->typ.typecode;
if(vsym->data == NULL) return;
list = vsym->data;
ASSERT(list->dimdata != NULL);
src = datalist2src(list);
/* Locate checkpoint */
checkpoint=0;
for(i=0;i<vsym->typ.dimset.ndims;i++) {
if(vsym->typ.dimset.dimsyms[i]->dim.size == NC_UNLIMITED) {
checkpoint = i;
break;
}
}
if(typecode == NC_CHAR)
gen_chararray(vsym,closure,databuf);
else {
genjjni_arraydatar(vsym,src,list->dimdata,closure,0,checkpoint,databuf);
}
commify(databuf);
}
static void
genjjni_arraydatar(Symbol* vsym,
Datasrc* src,
Odometer* odom,
Putvar* closure,
int index,
int checkpoint,
Bytebuffer* databuf)
{
int i;
int rank = odom->rank;
int lastdim = (index == (rank - 1)); /* last dimension*/
int firstdim = (index == 0);
int docheckpoint = (closure != NULL && index == checkpoint);
int declsize = odom->dims[index].declsize;
int isunlimited = (declsize == 0);
Symbol* basetype = vsym->typ.basetype;
Datalist* fillsrc = vsym->var.special._Fillvalue;
Constant* con;
nc_type typecode = basetype->typ.typecode;
char* memname = "memory";
ASSERT(index >= 0 && index < rank);
odom->dims[index].index = 0; /* reset*/
if(isunlimited) {
Constant* con;
if(!firstdim) {
if(!issublist(src)) {
semerror(srcline(src),"Unlimited data must be enclosed in {..}");
return;
}
srcpush(src); /* enter the unlimited data */
}
while((con=srcpeek(src))!=NULL) {
if(lastdim) {
if(isprimplus(typecode)) {
bbCat(databuf," ");
genjjni_primdata(basetype,src,fillsrc,memname,databuf);
} else
genjjni_data(basetype,src,fillsrc,memname,databuf);
} else {
genjjni_arraydatar(vsym,src,odom,closure,index+1,
checkpoint,databuf);
}
odom->dims[index].index++;
if(docheckpoint) {
closure->putvar(closure,odom,databuf);
}
}
odom->dims[index].datasize = odom->dims[index].index;
if(!firstdim) srcpop(src);
} else { /* !isunlimited*/
for(i=0;i<declsize;i++) {
con = srcpeek(src);
if(lastdim) {
if(isprimplus(typecode)) {
bbCat(databuf," ");
genjjni_primdata(basetype,src,fillsrc,memname,databuf);
} else
genjjni_data(basetype,src,fillsrc,memname,databuf);
} else { /* ! lastdim*/
(void)genjjni_arraydatar(vsym,src,odom,closure,
index+1,checkpoint,
databuf);
}
odom->dims[index].index++;
if(docheckpoint) {
closure->putvar(closure,odom,databuf);
}
}
}
}
/**************************************************/
/* This is the general datalist processing procedure */
static void
genjjni_data(Symbol* tsym, Datasrc* datasrc, Datalist* fillsrc,
char* memname, Bytebuffer* databuf)
{
int usecmpd;
Constant* con = srcpeek(datasrc);
Bytebuffer* constdata = NULL;
nc_type typecode = tsym->typ.typecode;
if(con == NULL || con->nctype == NC_FILLVALUE) {
srcnext(datasrc);
genjjni_fillvalue(tsym,fillsrc,datasrc,memname,databuf);
return;
}
switch (tsym->subclass) {
case NC_ENUM:
case NC_OPAQUE:
case NC_PRIM:
if(issublist(datasrc)) {
semerror(srcline(datasrc),"Expected primitive found {..}");
}
bbCat(databuf," ");
genjjni_primdata(tsym,datasrc,NULL,memname,databuf);
break;
case NC_COMPOUND: {
int i;
if(!issublist(datasrc)) {
semerror(srcline(datasrc),
"Compound data must be enclosed in {..}");
}
srcpush(datasrc);
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* field = (Symbol*)listget(tsym->subnodes,i);
genjjni_data(field,datasrc,fillsrc,memname,databuf);
}
srcpop(datasrc);
} break;
case NC_VLEN: {
char stmt[C_MAX_STMT];
Constant* cp;
char vlenname[1024];
if(!issublist(datasrc)) {
semerror(srcline(datasrc),"Vlen data must be enclosed in {..}");
}
cp = srcnext(datasrc);
sprintf(vlenname,"vlen_%u",cp->value.compoundv->vlen.uid);
/* Use special alignment */
nprintf(stmt,sizeof(stmt),"Memory.align(%s,%d);\n",
vlenname,jvlenalignment());
bbCat(databuf,stmt);
/* generate and store the nc_vlen_t instance*/
sprintf(stmt,"Memory.put_vlen(%s,new long[]{%lld, %s_addr});\n",
memname,
(long long)cp->value.compoundv->vlen.count,
vlenname);
bbCat(databuf,stmt);
} break;
case NC_FIELD:
/* enclose in braces if and only if field is an array */
usecmpd = (issublist(datasrc) && tsym->typ.dimset.ndims > 0);
if(usecmpd) srcpush(datasrc);
constdata = bbNew();
if(tsym->typ.dimset.ndims > 0) {
genjjni_fieldarray(tsym->typ.basetype,datasrc,&tsym->typ.dimset,0,
memname,constdata);
} else {
genjjni_data(tsym->typ.basetype,datasrc,NULL,memname,constdata);
}
/* If this was a primitive valued field, then generate a
Memory.put_XXX call
*/
if(isprimplus(typecode)) {
genjjni_putmemory(tsym,memname,databuf,constdata);
} else /* It is already one or more Memory.put_XX calls */
bbAppendn(databuf,bbContents(constdata),bbLength(constdata));
if(usecmpd) srcpop(datasrc);
bbFree(constdata);
bbNull(databuf);
break;
default: PANIC1("genjjni_data: unexpected subclass %d",tsym->subclass);
}
}
/* Used only for structure field arrays*/
static void
genjjni_fieldarray(Symbol* basetype, Datasrc* src, Dimset* dimset, int index,
char* memname, Bytebuffer* databuf)
{
int i;
Symbol* dim = dimset->dimsyms[index];
unsigned int size = dim->dim.size;
int lastdim = (index == (dimset->ndims - 1)); /* last dimension*/
int typecode = basetype->typ.typecode;
if(typecode == NC_CHAR) {
/* Collect the char field in a separate buffer */
Bytebuffer* fieldbuf = bbNew();
gen_charfield(src,dimset,index,fieldbuf);
/* Add to the existing data buf using Memory */
jquotestring(fieldbuf,'"');
bbCat(databuf," ");
bbCat(databuf,bbContents(fieldbuf));
bbFree(fieldbuf);
} else {
ASSERT(size != 0);
for(i=0;i<size;i++) {
bbCat(databuf," ");
if(lastdim) {
if(isprimplus(typecode)) {
bbCat(databuf," ");
genjjni_primdata(basetype,src,NULL,memname,databuf);
} else
genjjni_data(basetype,src,NULL,memname,databuf);
} else { /* !lastdim*/
genjjni_fieldarray(basetype,src,dimset,index+1,memname,databuf);
}
}
}
}
void
genjjni_vlendata(List* vlenconstants, Bytebuffer* databuf)
{
int i,count;
Datasrc* vlensrc;
char tmp[C_MAX_STMT];
char memname[1024];
Bytebuffer* vlenbuf = bbNew();
for(i=0;i<listlength(vlenconstants);i++) {
Constant* cmpd = (Constant*)listget(vlenconstants,i);
Symbol* tsym = cmpd->value.compoundv->vlen.schema;
Symbol* basetype = tsym->typ.basetype;
int typecode = basetype->typ.typecode;
sprintf(memname,"vlen_%u",cmpd->value.compoundv->vlen.uid);
count = 0;
vlensrc = datalist2src(cmpd->value.compoundv);
bbClear(vlenbuf);
if(typecode == NC_CHAR) {
Fix more memory leaks in netcdf-c library This is a follow up to PR https://github.com/Unidata/netcdf-c/pull/1173 Sorry that it is so big, but leak suppression can be complex. This PR fixes all remaining memory leaks -- as determined by -fsanitize=address, and with the exceptions noted below. Unfortunately. there remains a significant leak that I cannot solve. It involves vlens, and it is unclear if the leak is occurring in the netcdf-c library or the HDF5 library. I have added a check_PROGRAM to the ncdump directory to show the problem. The program is called tst_vlen_demo.c To exercise it, build the netcdf library with -fsanitize=address enabled. Then go into ncdump and do a "make clean check". This should build tst_vlen_demo without actually executing it. Then do the command "./tst_vlen_demo" to see the output of the memory checker. Note the the lost malloc is deep in the HDF5 library (in H5Tvlen.c). I am temporarily working around this error in the following way. 1. I modified several test scripts to not execute known vlen tests that fail as described above. 2. Added an environment variable called NC_VLEN_NOTEST. If set, then those specific tests are suppressed. This should mean that the --disable-utilities option to ./configure should not need to be set to get a memory leak clean build. This should allow for detection of any new leaks. Note: I used an environment variable rather than a ./configure option to control the vlen tests. This is because it is temporary (I hope) and because it is a bit tricky for shell scripts to access ./configure options. Finally, as before, this only been tested with netcdf-4 and hdf5 support.
2018-11-16 01:00:38 +08:00
/* Collect the char sequence in a separate buffer */
gen_charseq(vlensrc,vlenbuf);
2010-06-03 21:24:43 +08:00
count = bbLength(vlenbuf);
/* Add to the existing data buf as a single constant */
jquotestring(vlenbuf,'"');
/* Insert into the Memory buffer */
genjjni_putmemory(basetype,memname,databuf,vlenbuf);
} else {
while(srcmore(vlensrc)) {
genjjni_data(tsym->typ.basetype,vlensrc,NULL,memname,vlenbuf);
count++;
}
ASSERT(count == cmpd->value.compoundv->vlen.count);
/* Hack to handle user-defined vs primitive basetypes */
if(isprimplus(typecode)) {
genjjni_putmemory(basetype,memname,databuf,vlenbuf);
} else { /* constdata should have the necessary Memory.put ops */
bbCat(databuf,bbContents(vlenbuf));
}
/* Now store the address */
sprintf(tmp,"%s_addr = Memory.address(%s);\n",
memname,memname);
bbCat(databuf,tmp);
}
}
bbFree(vlenbuf);
}
void
genjjni_vlenconstants(List* vlenconstants, Bytebuffer* databuf)
{
int i;
char tmp[C_MAX_STMT];
char memname[1024];
for(i=0;i<listlength(vlenconstants);i++) {
Constant* cmpd = (Constant*)listget(vlenconstants,i);
sprintf(memname,"vlen_%u",cmpd->value.compoundv->vlen.uid);
/* Define the memory buffer and a place to stick its address */
sprintf(tmp,"static long %s = Memory.create();\n",memname);
bbCat(databuf,tmp);
sprintf(tmp,"static long %s_addr = 0L;\n",memname);
bbCat(databuf,tmp);
}
}
/**************************************************/
static void
genjjni_primdata(Symbol* tsym, Datasrc* src, Datalist* filler,
char* memname, Bytebuffer* databuf)
{
Constant target, *prim;
prim = srcnext(src);
if(prim == NULL || prim->nctype == NC_FILLVALUE) {
genjjni_fillvalue(tsym,filler,src,memname,databuf);
return;
}
target.nctype = tsym->typ.typecode;
if(prim == NULL) {
#ifdef GENFILL
/* generate a fill value*/
nc_getfill(&target);
/* fall thru*/
#else
return;
#endif
}
ASSERT(prim->nctype != NC_COMPOUND);
switch (target.nctype) {
case NC_ECONST:
if(tsym->subclass != NC_ENUM) {
semerror(prim->lineno,"Conversion to enum not supported (yet)");
nc_getfill(&target);
} else {
target.nctype = NC_ECONST;
convert1(prim,&target);
}
break;
case NC_OPAQUE:
target.nctype = NC_STRING;
jopaquestring(tsym,prim,&target);
break;
case NC_CHAR:
/* Hack to handle the char case as a string */
target = cloneconstant(prim);
break;
default:
convert1(prim,&target);
break;
}
/* add hack for java bug in converting floats */
if(target.nctype == NC_FLOAT) bbCat(databuf," (float)");
bbCat(databuf,jconst(&target));
}
/* Convert an opaque to a java string */
static void
jopaquestring(Symbol* tsym, Constant* prim, Constant* target)
{
int i;
static char hexdigits[] = "0123456789ABCDEF";
char* op = prim->value.opaquev.stringv;
size_t oplen = prim->value.opaquev.len;
unsigned char* p;
char* opstring;
ASSERT((oplen%2) == 0);
2017-10-31 05:52:08 +08:00
opstring = (char*)ecalloc(oplen*(2+4));
2018-12-07 06:40:43 +08:00
opstring[0]='\0';
2010-06-03 21:24:43 +08:00
p=(unsigned char*)op;
for(i=0;i<oplen;i++,p++) {
char tmp[8];
unsigned int b = *p;
char digit1 = hexdigits[(b & 0x0f)];
char digit2 = hexdigits[((b>>4) & 0x0f)];
strcpy(tmp,"\\u00");
tmp[5] = digit2;
tmp[6] = digit1;
tmp[7] = '\0';
strcat(opstring,tmp);
}
target->value.stringv.stringv = opstring;
target->value.stringv.len = strlen(opstring);
}
static int
jalignment(Symbol* tsym)
{
nc_type nctype = tsym->typ.typecode;
int alignment = 0;
if(nctype == NC_ECONST)
alignment = nctypealignment(tsym->typ.typecode);
else if(usingclassic && nctype == NC_STRING)
alignment = nctypealignment(NC_CHAR);
else if(nctype == NC_CHAR)
alignment = nctypealignment(NC_CHAR);
else
alignment = nctypealignment(nctype);
return alignment;
}
static int
genjjni_putmemory(Symbol* tsym, char* memname, Bytebuffer* databuf, Bytebuffer* constbuf)
{
char stmt[1024];
nc_type typecode = tsym->typ.typecode;
commify(constbuf);
nprintf(stmt,sizeof(stmt),"Memory.align(%s,%d);\n",
memname,jalignment(tsym));
bbCat(databuf,stmt);
if(typecode == NC_CHAR) {
nprintf(stmt,sizeof(stmt),"Memory.put_%sarray(%s,",
jstype(typecode),memname);
bbCat(databuf,stmt);
bbAppendn(databuf,bbContents(constbuf),bbLength(constbuf));
bbCat(databuf,");\n");
} else {
nprintf(stmt,sizeof(stmt),"Memory.put_%sarray(%s,new %s[]{",
jstype(typecode),
memname,
jarraytype(typecode));
bbCat(databuf,stmt);
bbAppendn(databuf,bbContents(constbuf),bbLength(constbuf));
bbCat(databuf,"});\n");
}
return NC_NOERR;
}
static int
jvlenalignment(void)
{
return nctypealignment(NC_VLEN);
}
#endif /*ENABLE_JAVA*/