netcdf-c/libdap2/nccommon.h

316 lines
11 KiB
C
Raw Normal View History

2011-04-18 02:56:10 +08:00
/*********************************************************************
2018-12-07 05:21:03 +08:00
* Copyright 2018, UCAR/Unidata
2011-04-18 02:56:10 +08:00
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
2011-04-18 02:56:10 +08:00
#ifndef NCCOMMON_H
#define NCCOMMON_H 1
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
#include "dapincludes.h"
2011-04-18 02:56:10 +08:00
/* Mnemonics */
#ifndef TRUE
#define TRUE 1
#define FALSE 0
#endif
/* Define an alias for int to indicate an error return */
typedef int NCerror;
2011-04-18 02:56:10 +08:00
#ifndef nullfree
#define nullfree(m) {if((m)!=NULL) {free(m);} else {}}
2011-04-18 02:56:10 +08:00
#endif
2013-11-16 06:41:17 +08:00
/* Misc. code controls */
2011-04-18 02:56:10 +08:00
#define FILLCONSTRAINT TRUE
2013-11-16 06:41:17 +08:00
/* Hopefully, this flag will someday not be needed */
#define COLUMBIA_HACK "columbia.edu"
2011-04-18 02:56:10 +08:00
/* Use an extended version of the netCDF-4 type system */
#define NC_URL 50
#define NC_SET 51
/* Merge relevant translations of OC types */
#define NC_Dataset 52
#define NC_Sequence 53
#define NC_Structure 54
#define NC_Grid 55
#define NC_Dimension 56
#define NC_Atomic 57
2011-04-18 02:56:10 +08:00
#undef OCCOMPILEBYDEFAULT
#define DEFAULTSTRINGLENGTH 64
/* The sequence limit default is zero because
most servers do not implement projections
on sequences.
*/
#define DEFAULTSEQLIMIT 0
/**************************************************/
/* sigh, do the forwards */
struct NCDAPCOMMON;
struct NCprojection;
struct NCselection;
struct Getvara;
struct NCcachenode;
struct NCcache;
struct NCslice;
struct NCsegment;
struct OClist;
2011-04-18 02:56:10 +08:00
struct NCTMODEL {
int translation;
char* model;
unsigned int flags;
};
/* Define the cache control flags */
2011-04-18 02:56:10 +08:00
/* Detail information about each cache item */
typedef struct NCcachenode {
int wholevariable; /* does this cache node only have wholevariables? */
int isprefetch;
off_t xdrsize;
2011-04-18 02:56:10 +08:00
DCEconstraint* constraint; /* as used to create this node */
NClist* vars; /* vars potentially covered by this cache node */
struct CDFnode* datadds;
OCddsnode ocroot;
OCdatanode content;
2011-04-18 02:56:10 +08:00
} NCcachenode;
2011-04-18 02:56:10 +08:00
/* All cache info */
typedef struct NCcache {
size_t cachelimit; /* max total size for all cached entries */
size_t cachesize; /* current size */
size_t cachecount; /* max # nodes in cache */
NCcachenode* prefetch;
NClist* nodes; /* cache nodes other than prefetch */
} NCcache;
/**************************************************/
/* The DAP packet info from OC */
typedef struct NCOC {
OClink conn;
2011-11-14 12:20:19 +08:00
char* rawurltext; /* as given to nc3d_open */
char* urltext; /* as modified by nc3d_open */
NCURI* url; /* parse of rawuritext */
OCdasnode ocdasroot;
2011-04-18 02:56:10 +08:00
DCEconstraint* dapconstraint; /* from url */
int inmemory; /* store fetched data in memory? */
2011-04-18 02:56:10 +08:00
} NCOC;
typedef struct NCCDF {
2011-09-16 00:57:16 +08:00
struct CDFnode* ddsroot; /* constrained dds */
struct CDFnode* fullddsroot; /* unconstrained dds */
NClist* projectedvars; /* vars appearing in nc_open url projections */
2011-04-18 02:56:10 +08:00
unsigned int defaultstringlength;
unsigned int defaultsequencelimit; /* global sequence limit;0=>no limit */
struct NCcache* cache;
size_t fetchlimit;
size_t smallsizelimit; /* what constitutes a small object? */
size_t totalestimatedsize;
const char* separator; /* constant; do not free */
/* Following fields should be set from the unconstrained dds only */
2011-11-14 12:20:19 +08:00
/* global string dimension */
struct CDFnode* globalstringdim;
char* recorddimname; /* From DODS_EXTRA */
struct CDFnode* recorddim;
2011-04-18 02:56:10 +08:00
} NCCDF;
/* Define a structure holding common info for NCDAP */
2011-04-18 02:56:10 +08:00
typedef struct NCDAPCOMMON {
2011-09-17 02:36:08 +08:00
NC* controller; /* Parent instance of NCDAPCOMMON */
2011-04-18 02:56:10 +08:00
NCCDF cdf;
2011-11-14 12:20:19 +08:00
NCOC oc;
2011-04-18 02:56:10 +08:00
NCCONTROLS controls; /* Control flags and parameters */
struct {
int realfile; /* 1 => we created actual temp file */
char* filename; /* of the substrate file */
int nc3id; /* substrate nc4 file ncid used to hold metadata; not same as external id */
} substrate;
2011-04-18 02:56:10 +08:00
} NCDAPCOMMON;
/**************************************************/
/* Create our own node tree to mimic ocnode trees*/
/* Each root CDFnode contains info about the whole tree */
typedef struct CDFtree {
OCddsnode ocroot;
2011-04-18 02:56:10 +08:00
OCdxd occlass;
NClist* nodes; /* all nodes in tree*/
struct CDFnode* root; /* cross link */
struct NCDAPCOMMON* owner;
/* Collected sets of useful nodes */
NClist* varnodes; /* nodes which can represent netcdf variables */
NClist* seqnodes; /* sequence nodes; */
NClist* gridnodes; /* grid nodes */
NClist* dimnodes; /* (base) dimension nodes */
2011-04-18 02:56:10 +08:00
/* Classification flags */
int restructed; /* Was this tree passed thru restruct? */
2011-04-18 02:56:10 +08:00
} CDFtree;
/* Track the kinds of dimensions */
typedef int CDFdimflags;
#define CDFDIMNORMAL 0x0
#define CDFDIMSEQ 0x1
#define CDFDIMSTRING 0x2
#define CDFDIMCLONE 0x4
#define CDFDIMRECORD 0x20
#define DIMFLAG(d,flag) ((d)->dim.dimflags & (flag))
#define DIMFLAGSET(d,flag) ((d)->dim.dimflags |= (flag))
#define DIMFLAGCLR(d,flag) ((d)->dim.dimflags &= ~(flag))
typedef struct CDFdim {
CDFdimflags dimflags;
struct CDFnode* basedim; /* for duplicate dimensions*/
struct CDFnode* array; /* parent array node */
size_t declsize; /* from constrained DDS*/
size_t declsize0; /* from unconstrained DDS*/
2011-11-14 12:20:19 +08:00
int index1; /* dimension name index +1; 0=>no index */
2011-04-18 02:56:10 +08:00
} CDFdim;
typedef struct CDFarray {
NClist* dimsetall; /* dimsetplus + inherited */
NClist* dimsettrans; /* dimset0 plus inherited(transitive closure) */
NClist* dimsetplus; /* dimset0 + pseudo */
NClist* dimset0; /* original dims from the dds */
2011-04-18 02:56:10 +08:00
struct CDFnode* stringdim;
2011-11-14 12:20:19 +08:00
/* Track sequence related information */
struct CDFnode* seqdim; /* if this node is a sequence */
2011-04-18 02:56:10 +08:00
/* note: unlike string dim; seqdim is also stored in dimensions vector */
struct CDFnode* sequence; /* containing usable sequence, if any */
struct CDFnode* basevar; /* for duplicate grid variables*/
} CDFarray;
typedef struct NCattribute {
char* name;
nc_type etype; /* dap type of the attribute */
NClist* values; /* strings come from the oc values */
int invisible; /* Do not materialize to the user */
2011-04-18 02:56:10 +08:00
} NCattribute;
/* Extend as additional DODS attribute values are defined */
typedef struct NCDODS {
size_t maxstrlen;
char* dimname;
} NCDODS;
Fix more memory leaks in netcdf-c library This is a follow up to PR https://github.com/Unidata/netcdf-c/pull/1173 Sorry that it is so big, but leak suppression can be complex. This PR fixes all remaining memory leaks -- as determined by -fsanitize=address, and with the exceptions noted below. Unfortunately. there remains a significant leak that I cannot solve. It involves vlens, and it is unclear if the leak is occurring in the netcdf-c library or the HDF5 library. I have added a check_PROGRAM to the ncdump directory to show the problem. The program is called tst_vlen_demo.c To exercise it, build the netcdf library with -fsanitize=address enabled. Then go into ncdump and do a "make clean check". This should build tst_vlen_demo without actually executing it. Then do the command "./tst_vlen_demo" to see the output of the memory checker. Note the the lost malloc is deep in the HDF5 library (in H5Tvlen.c). I am temporarily working around this error in the following way. 1. I modified several test scripts to not execute known vlen tests that fail as described above. 2. Added an environment variable called NC_VLEN_NOTEST. If set, then those specific tests are suppressed. This should mean that the --disable-utilities option to ./configure should not need to be set to get a memory leak clean build. This should allow for detection of any new leaks. Note: I used an environment variable rather than a ./configure option to control the vlen tests. This is because it is temporary (I hope) and because it is a bit tricky for shell scripts to access ./configure options. Finally, as before, this only been tested with netcdf-4 and hdf5 support.
2018-11-16 01:00:38 +08:00
typedef struct NCD2alignment {
2011-04-18 02:56:10 +08:00
unsigned long size; /* size of single instance of this type*/
unsigned long alignment; /* alignment of this field */
unsigned long offset; /* offset of this field in parent */
Fix more memory leaks in netcdf-c library This is a follow up to PR https://github.com/Unidata/netcdf-c/pull/1173 Sorry that it is so big, but leak suppression can be complex. This PR fixes all remaining memory leaks -- as determined by -fsanitize=address, and with the exceptions noted below. Unfortunately. there remains a significant leak that I cannot solve. It involves vlens, and it is unclear if the leak is occurring in the netcdf-c library or the HDF5 library. I have added a check_PROGRAM to the ncdump directory to show the problem. The program is called tst_vlen_demo.c To exercise it, build the netcdf library with -fsanitize=address enabled. Then go into ncdump and do a "make clean check". This should build tst_vlen_demo without actually executing it. Then do the command "./tst_vlen_demo" to see the output of the memory checker. Note the the lost malloc is deep in the HDF5 library (in H5Tvlen.c). I am temporarily working around this error in the following way. 1. I modified several test scripts to not execute known vlen tests that fail as described above. 2. Added an environment variable called NC_VLEN_NOTEST. If set, then those specific tests are suppressed. This should mean that the --disable-utilities option to ./configure should not need to be set to get a memory leak clean build. This should allow for detection of any new leaks. Note: I used an environment variable rather than a ./configure option to control the vlen tests. This is because it is temporary (I hope) and because it is a bit tricky for shell scripts to access ./configure options. Finally, as before, this only been tested with netcdf-4 and hdf5 support.
2018-11-16 01:00:38 +08:00
} NCD2alignment;
2011-04-18 02:56:10 +08:00
typedef struct NCtypesize {
2021-02-03 13:48:22 +08:00
int aligned; /* have instance and field been defined? */
Fix more memory leaks in netcdf-c library This is a follow up to PR https://github.com/Unidata/netcdf-c/pull/1173 Sorry that it is so big, but leak suppression can be complex. This PR fixes all remaining memory leaks -- as determined by -fsanitize=address, and with the exceptions noted below. Unfortunately. there remains a significant leak that I cannot solve. It involves vlens, and it is unclear if the leak is occurring in the netcdf-c library or the HDF5 library. I have added a check_PROGRAM to the ncdump directory to show the problem. The program is called tst_vlen_demo.c To exercise it, build the netcdf library with -fsanitize=address enabled. Then go into ncdump and do a "make clean check". This should build tst_vlen_demo without actually executing it. Then do the command "./tst_vlen_demo" to see the output of the memory checker. Note the the lost malloc is deep in the HDF5 library (in H5Tvlen.c). I am temporarily working around this error in the following way. 1. I modified several test scripts to not execute known vlen tests that fail as described above. 2. Added an environment variable called NC_VLEN_NOTEST. If set, then those specific tests are suppressed. This should mean that the --disable-utilities option to ./configure should not need to be set to get a memory leak clean build. This should allow for detection of any new leaks. Note: I used an environment variable rather than a ./configure option to control the vlen tests. This is because it is temporary (I hope) and because it is a bit tricky for shell scripts to access ./configure options. Finally, as before, this only been tested with netcdf-4 and hdf5 support.
2018-11-16 01:00:38 +08:00
NCD2alignment instance; /* Alignment, etc for instance data */
NCD2alignment field; /* Alignment, etc WRT to parent */
2011-04-18 02:56:10 +08:00
} NCtypesize;
/* Closely mimics struct OCnode*/
typedef struct CDFnode {
2011-11-14 12:20:19 +08:00
nc_type nctype; /* e.g. var, dimension */
nc_type etype; /* e.g. NC_INT, NC_FLOAT if applicable,*/
char* ocname; /* oc base name */
char* ncbasename; /* generally cdflegalname(ocname) */
char* ncfullname; /* complete path name from root to this node*/
OCddsnode ocnode; /* oc mirror node*/
2011-12-06 04:53:43 +08:00
struct CDFnode* group; /* null => in root group */
struct CDFnode* container; /* e.g. struct or sequence, but not group */
2011-04-18 02:56:10 +08:00
struct CDFnode* root;
CDFtree* tree; /* root level metadata;only defined if root*/
CDFdim dim; /* nctype == dimension */
CDFarray array; /* nctype == grid,var,etc. with dimensions */
2011-11-14 12:20:19 +08:00
NClist* subnodes; /* if nctype == grid, sequence, etc. */
NClist* attributes; /*NClist<NCattribute*>*/
NCDODS dodsspecial; /* special attributes like maxStrlen */
nc_type externaltype; /* the type as represented to nc_inq*/
int ncid; /* relevant NC id for this object*/
2011-04-18 02:56:10 +08:00
unsigned long maxstringlength;
unsigned long sequencelimit; /* 0=>unlimited */
2021-02-03 13:48:22 +08:00
int usesequence; /* If this sequence is usable */
int elided; /* 1 => node does not partipate in naming*/
struct CDFnode* basenode; /* derived tree map to pattern tree */
2021-02-03 13:48:22 +08:00
int invisible; /* 1 => do not show to user */
int zerodim; /* 1 => node has a zero dimension */
2011-04-18 02:56:10 +08:00
/* These two flags track the effects on grids of constraints */
2021-02-03 13:48:22 +08:00
int nc_virtual; /* node added by regrid */
2011-11-14 12:20:19 +08:00
struct CDFnode* attachment; /* DDS<->DATADDS cross link*/
struct CDFnode* pattern; /* temporary field for regridding */
2011-04-18 02:56:10 +08:00
/* Fields for use by libncdap4 */
NCtypesize typesize;
2011-11-14 12:20:19 +08:00
int typeid; /* when treating field as type */
int basetypeid; /* when typeid is vlen */
2011-04-18 02:56:10 +08:00
char* typename;
2011-11-14 12:20:19 +08:00
char* vlenname; /* for sequence types */
int singleton; /* for singleton sequences */
2011-04-18 02:56:10 +08:00
unsigned long estimatedsize; /* > 0 Only for var nodes */
2021-02-03 13:48:22 +08:00
int whole; /* projected as whole node */
int prefetchable; /* eligible to be prefetched (if whole) */
2011-04-18 02:56:10 +08:00
} CDFnode;
/**************************************************/
/* Shared procedures */
/* From error.c*/
extern NCerror ocerrtoncerr(OCerror);
extern NCerror attach(CDFnode* xroot, CDFnode* ddstarget);
extern void unattach(CDFnode*);
extern int nodematch(CDFnode* node1, CDFnode* node2);
extern int simplenodematch(CDFnode* node1, CDFnode* node2);
extern CDFnode* findxnode(CDFnode* target, CDFnode* xroot);
extern int constrainable(NCURI*);
extern char* makeconstraintstring(DCEconstraint*);
extern size_t estimatedataddssize(CDFnode* datadds);
extern void canonicalprojection(NClist*, NClist*);
extern NClist* getalldims(NCDAPCOMMON* nccomm, int visibleonly);
/* From cdf.c */
extern NCerror fixgrid(NCDAPCOMMON* drno, CDFnode* grid);
extern NCerror computecdfinfo(NCDAPCOMMON*, NClist*);
extern char* cdfname(char* basename);
extern NCerror augmentddstree(NCDAPCOMMON*, NClist*);
extern NCerror computecdfdimnames(NCDAPCOMMON*);
extern NCerror buildcdftree(NCDAPCOMMON*, OCddsnode, OCdxd, CDFnode**);
extern CDFnode* makecdfnode(NCDAPCOMMON*, char* nm, OCtype,
/*optional*/ OCddsnode ocnode, CDFnode* container);
extern void freecdfroot(CDFnode*);
extern NCerror dimimprint(NCDAPCOMMON*);
extern NCerror definedimsets(NCDAPCOMMON*,CDFtree*);
extern NCerror definedimsettrans(NCDAPCOMMON*,CDFtree*);
2011-11-14 12:20:19 +08:00
2011-04-18 02:56:10 +08:00
/* From cache.c */
extern int iscached(NCDAPCOMMON*, CDFnode* target, NCcachenode** cachenodep);
extern NCerror prefetchdata(NCDAPCOMMON*);
extern NCerror markprefetch(NCDAPCOMMON*);
extern NCerror buildcachenode(NCDAPCOMMON*,
2011-04-18 02:56:10 +08:00
DCEconstraint* constraint,
NClist* varlist,
NCcachenode** cachep,
NCFLAGS flags);
2011-04-18 02:56:10 +08:00
extern NCcachenode* createnccachenode(void);
extern void freenccachenode(NCDAPCOMMON*, NCcachenode* node);
extern NCcache* createnccache(void);
extern void freenccache(NCDAPCOMMON*, NCcache* cache);
/* Add an extra function whose sole purpose is to allow
configure(.ac) to test for the presence of thiscode.
*/
extern int nc__opendap(void);
/* Define accessors for the dispatchdata */
#define NCD2_DATA(nc) ((NCDAPCOMMON*)(nc)->dispatchdata)
#define NCD2_DATA_SET(nc,data) ((nc)->dispatchdata = (void*)(data))
2016-04-07 04:05:58 +08:00
#define getncid(drno) (((NC*)drno)->ext_ncid)
2016-04-07 09:51:40 +08:00
#define getdap(drno) ((NCDAPCOMMON*)((NC*)drno)->dispatchdata)
#define getnc3id(drno) (getdap(drno)->substrate.nc3id)
2011-04-18 02:56:10 +08:00
#endif /*NCCOMMON_H*/