mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-19 17:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main'

Browse Source
This commit is contained in:
Ward Fisher 2023-11-17 15:16:35 -07:00
commit e8be174475
43 changed files with 1076 additions and 1227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
RELEASE_NOTES.md
View File

@ -7,6 +7,10 @@ This file contains a high-level description of this package's evolution. Release
## 4.9.3 - TBD
* Improve the speed and data quantity for DAP4 queries. See [Github #2765](https://github.com/Unidata/netcdf-c/pull/2765).
* Remove the use of execinfo to programmatically dump the stack; it never worked. See [Github #2789](https://github.com/Unidata/netcdf-c/pull/2789).
* Update the internal copy of tinyxml2 to latest code. See [Github #2771](https://github.com/Unidata/netcdf-c/pull/2771).
* Mitigate the problem of remote/nczarr-related test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).
* Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764).
* Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763).
* Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).

3
config.h.cmake.in
View File

@ -468,9 +468,6 @@ with zip */
/* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */
#cmakedefine HDF5_UTF8_PATHS 1
/* if true, backtrace support will be used. */
#cmakedefine HAVE_EXECINFO_H 1
/* if true, include JNA bug fix */
#cmakedefine JNA 1

3
configure.ac
View File

@ -1312,9 +1312,6 @@ AC_CHECK_HEADERS([sys/resource.h])
# See if we have ftw.h to walk directory trees
AC_CHECK_HEADERS([ftw.h])
# See if we can do stack tracing programmatically
AC_CHECK_HEADERS([execinfo.h])
# Check for these functions...
AC_CHECK_FUNCS([strlcat snprintf strcasecmp fileno \
strdup strtoll strtoull \

18
dap4_test/test_common.h
View File

@ -21,6 +21,7 @@ typedef int TDMR;
static NCbytes* input = NULL;
static NCbytes* output = NULL;
static NCD4meta* metadata = NULL;
static NCD4response* resp = NULL;
static char* infile = NULL;
static char* outfile = NULL;
static int ncid = 0;
@ -85,16 +86,21 @@ setup(int tdmr, int argc, char** argv)
if(translatenc4)
controller->controls.translation = NCD4_TRANSNC4;
NCD4_applyclientfragmentcontrols(controller);
if((metadata=NCD4_newmeta(controller))==NULL)
fail(NC_ENOMEM);
metadata->mode = mode;
NCD4_attachraw(metadata, ncbyteslength(input),ncbytescontents(input));
if((ret=NCD4_dechunk(metadata))) /* ok for mode == DMR or mode == DAP */
if((ret=NCD4_newMeta(controller,&metadata)))
fail(ret);
if((ret=NCD4_newResponse(controller,&resp)))
fail(ret);
resp->raw.size = ncbyteslength(input);
resp->raw.memory = ncbytescontents(input);
resp->mode = mode;
if((ret=NCD4_dechunk(resp))) /* ok for mode == DMR or mode == DAP */
fail(ret);
#ifdef DEBUG
{
int swap = (metadata->serial.hostbigendian != metadata->serial.remotebigendian);
int swap = (controller->platform.hostlittleendian != resp->remotelittleendian);
void* d = metadata->serial.dap;
size_t sz = metadata->serial.dapsize;
fprintf(stderr,"====================\n");

2
dap4_test/test_meta.c
View File

@ -20,7 +20,7 @@ main(int argc, char** argv)
fprintf(stderr,"t_dmrmeta %s -> %s\n",infile,outfile);
#endif
if((ret = NCD4_parse(metadata))) goto done;
if((ret = NCD4_parse(metadata,resp,0))) goto done;
if((ret = NCD4_metabuild(metadata,ncid))) goto done;
done:

2
dap4_test/test_parse.c
View File

@ -17,7 +17,7 @@ main(int argc, char** argv)
setup(TDMR_PARSE,argc,argv);
if((ret = NCD4_parse(metadata))) goto done;
if((ret = NCD4_parse(metadata,resp,0))) goto done;
ret = NCD4_print(metadata,output);
ncbytesnull(output);
if(ret == NC_NOERR) {

2
include/ncconfigure.h
View File

@ -61,7 +61,7 @@ extern "C" {
#endif
/* WARNING: in some systems, these functions may be defined as macros, so check */
#ifndef HAVE_STRDUP
#if ! defined(HAVE_STRDUP) || defined(__CYGWIN__)
#ifndef strdup
char* strdup(const char*);
#endif

26
include/ncuri.h
View File

@ -35,12 +35,8 @@ typedef struct NCURI {
char* path; /*!< path */
char* query; /*!< query */
char* fragment; /*!< fragment */
char** fraglist; /* envv style list of decomposed fragment*/
char** querylist; /* envv style list of decomposed query*/
#if 0
char* projection; /*!< without leading '?'*/
char* selection; /*!< with leading '&'*/
#endif
void* fraglist; /* some representation of the decomposed fragment string */
void* querylist; /* some representation of the decomposed query string */
} NCURI;
#if 0
@ -90,6 +86,18 @@ EXTERNL int ncurisetfragmentkey(NCURI* duri,const char* key, const char* value);
/* append a specific &key=...& in uri fragment */
EXTERNL int ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value);
/* Replace a specific &key=...& in uri query */
EXTERNL int ncurisetquerykey(NCURI* duri,const char* key, const char* value);
/* append a specific &key=...& in uri query */
EXTERNL int ncuriappendquerykey(NCURI* duri,const char* key, const char* value);
/* Get the actual list of queryies */
EXTERNL void* ncuriqueryparams(NCURI* uri);
/* Get the actual list of frags */
EXTERNL void* ncurifragmentparams(NCURI* uri);
/* Construct a complete NC URI; caller frees returned string */
EXTERNL char* ncuribuild(NCURI*,const char* prefix, const char* suffix, int flags);
@ -105,12 +113,6 @@ EXTERNL const char* ncurifragmentlookup(NCURI*, const char* param);
*/
EXTERNL const char* ncuriquerylookup(NCURI*, const char* param);
/* Obtain the complete list of fragment pairs in envv format */
EXTERNL const char** ncurifragmentparams(NCURI*);
/* Obtain the complete list of query pairs in envv format */
EXTERNL const char** ncuriqueryparams(NCURI*);
/* URL Encode/Decode */
EXTERNL char* ncuridecode(const char* s);
/* Partial decode */

135
libdap4/d4chunk.c
View File

@ -22,133 +22,128 @@ Notes:
*/
/* Forward */
static int processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count);
static int processerrchunk(NCD4response*, void* errchunk, unsigned int count);
/**************************************************/
void
NCD4_resetSerial(NCD4serial* serial, size_t rawsize, void* rawdata)
{
nullfree(serial->errdata);
nullfree(serial->dmr);
nullfree(serial->dap);
nullfree(serial->rawdata);
/* clear all fields */
memset(serial,0,sizeof(NCD4serial));
/* Reset fields */
serial->hostlittleendian = NCD4_isLittleEndian();
serial->rawsize = rawsize;
serial->rawdata = rawdata;
}
int
NCD4_dechunk(NCD4meta* metadata)
NCD4_dechunk(NCD4response* resp)
{
unsigned char *praw, *phdr, *pdap;
unsigned char *praw, *pdmr, *phdr, *pdap, *pappend, *pchunk;
NCD4HDR hdr;
int firstchunk;
#ifdef D4DUMPRAW
NCD4_tagdump(metadata->serial.rawsize,metadata->serial.rawdata,0,"RAW");
NCD4_tagdump(resp->serial.raw.size,resp->serial.raw.data,0,"RAW");
#endif
/* Access the returned raw data */
praw = metadata->serial.rawdata;
praw = (unsigned char*)resp->raw.memory;
if(metadata->mode == NCD4_DSR) {
if(resp->mode == NCD4_DSR) {
return THROW(NC_EDMR);
} else if(metadata->mode == NCD4_DMR) {
} else if(resp->mode == NCD4_DMR) {
/* Verify the mode; assume that the <?xml...?> is optional */
if(memcmp(praw,"<?xml",strlen("<?xml"))==0
|| memcmp(praw,"<Dataset",strlen("<Dataset"))==0) {
size_t len = 0;
/* setup as dmr only */
/* Avoid strdup since rawdata might contain nul chars */
len = metadata->serial.rawsize;
if((metadata->serial.dmr = malloc(len+1)) == NULL)
len = resp->raw.size;
if((resp->serial.dmr = malloc(len+1)) == NULL)
return THROW(NC_ENOMEM);
memcpy(metadata->serial.dmr,praw,len);
metadata->serial.dmr[len] = '\0';
memcpy(resp->serial.dmr,praw,len);
resp->serial.dmr[len] = '\0';
/* Suppress nuls */
(void)NCD4_elidenuls(metadata->serial.dmr,len);
(void)NCD4_elidenuls(resp->serial.dmr,len);
return THROW(NC_NOERR);
}
} else if(metadata->mode != NCD4_DAP)
} else if(resp->mode != NCD4_DAP)
return THROW(NC_EDAP);
/* We must be processing a DAP mode packet */
praw = (metadata->serial.dap = metadata->serial.rawdata);
metadata->serial.rawdata = NULL;
praw = resp->raw.memory;
/* If the raw data looks like xml, then we almost certainly have an error */
if(memcmp(praw,"<?xml",strlen("<?xml"))==0
|| memcmp(praw,"<!doctype",strlen("<!doctype"))==0) {
/* Set up to report the error */
int stat = NCD4_seterrormessage(metadata, metadata->serial.rawsize, metadata->serial.rawdata);
int stat = NCD4_seterrormessage(resp, resp->raw.size, resp->raw.memory);
return THROW(stat); /* slight lie */
}
/* Get the DMR chunk header*/
phdr = NCD4_getheader(praw,&hdr,metadata->serial.hostlittleendian);
/* Get the first header to get dmr content and endian flags*/
pdmr = NCD4_getheader(praw,&hdr,resp->controller->platform.hostlittleendian);
if(hdr.count == 0)
return THROW(NC_EDMR);
if(hdr.flags & NCD4_ERR_CHUNK) {
return processerrchunk(metadata, (void*)phdr, hdr.count);
}
if(hdr.flags & NCD4_ERR_CHUNK)
return processerrchunk(resp, (void*)pdmr, hdr.count);
resp->remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0);
metadata->serial.remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0);
/* Again, avoid strxxx operations on dmr */
if((metadata->serial.dmr = malloc(hdr.count+1)) == NULL)
/* avoid strxxx operations on dmr */
if((resp->serial.dmr = malloc(hdr.count+1)) == NULL)
return THROW(NC_ENOMEM);
memcpy(metadata->serial.dmr,phdr,hdr.count);
metadata->serial.dmr[hdr.count-1] = '\0';
memcpy(resp->serial.dmr,pdmr,hdr.count);
resp->serial.dmr[hdr.count-1] = '\0';
/* Suppress nuls */
(void)NCD4_elidenuls(metadata->serial.dmr,hdr.count);
(void)NCD4_elidenuls(resp->serial.dmr,hdr.count);
/* See if there is any data after the DMR */
if(hdr.flags & NCD4_LAST_CHUNK)
return THROW(NC_ENODATA);
/* Read and concat together the data chunks */
phdr = phdr + hdr.count; /* point to data chunk header */
phdr = pdmr + hdr.count; /* point to data chunk header */
/* Do a sanity check in case the server has shorted us with no data */
if((hdr.count + CHUNKHDRSIZE) >= metadata->serial.rawsize) {
if((hdr.count + CHUNKHDRSIZE) >= resp->raw.size) {
/* Server only sent the DMR part */
metadata->serial.dapsize = 0;
resp->serial.dapsize = 0;
return THROW(NC_EDATADDS);
}
pdap = metadata->serial.dap;
for(;;) {
phdr = NCD4_getheader(phdr,&hdr,metadata->serial.hostlittleendian);
if(hdr.flags & NCD4_ERR_CHUNK) {
return processerrchunk(metadata, (void*)phdr, hdr.count);
/* walk all the data chunks */
/* invariants:
praw -- beginning of the raw response
pdmr -- beginning of the dmr in the raw data
pdap -- beginning of the dechunked dap data
phdr -- pointer to the hdr of the current chunk
pchunk -- pointer to the data part of the current chunk
pappend -- where to append next chunk to the growing dechunked data
*/
for(firstchunk=1;;firstchunk=0) {
pchunk = NCD4_getheader(phdr,&hdr,resp->controller->platform.hostlittleendian); /* Process first data chunk header */
if(firstchunk) {
pdap = phdr; /* remember start point of the dechunked data */
pappend = phdr; /* start appending here */
}
if(hdr.flags & NCD4_ERR_CHUNK)
return processerrchunk(resp, (void*)pchunk, hdr.count);
/* data chunk; possibly last; possibly empty */
if(hdr.count > 0) {
d4memmove(pdap,phdr,hdr.count); /* will overwrite the header */
phdr += hdr.count;
pdap += hdr.count;
}
if(hdr.count > 0)
d4memmove(pappend,pchunk,hdr.count); /* overwrite the header; this the heart of dechunking */
pappend += hdr.count; /* next append point */
phdr = pchunk + hdr.count; /* point to header of next chunk */
if(hdr.flags & NCD4_LAST_CHUNK) break;
}
metadata->serial.dapsize = (size_t)DELTA(pdap,metadata->serial.dap);
resp->serial.dap = pdap; /* start of dechunked data */
resp->serial.dapsize = (size_t)DELTA(pappend,pdap);
#ifdef D4DUMPDMR
fprintf(stderr,"%s\n",metadata->serial.dmr);
fprintf(stderr,"%s\n",resp->serial.dmr);
fflush(stderr);
#endif
#ifdef D4DUMPDAP
NCD4_tagdump(metadata->serial.dapsize,metadata->serial.dap,0,"DAP");
NCD4_tagdump(resp->serial.dapsize,resp->serial.dap,0,"DAP");
#endif
return THROW(NC_NOERR);
}
static int
processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count)
processerrchunk(NCD4response* resp, void* errchunk, unsigned int count)
{
metadata->serial.errdata = (char*)d4alloc(count+1);
if(metadata->serial.errdata == NULL)
resp->serial.errdata = (char*)d4alloc(count+1);
if(resp->serial.errdata == NULL)
return THROW(NC_ENOMEM);
memcpy(metadata->serial.errdata,errchunk,count);
metadata->serial.errdata[count] = '\0';
memcpy(resp->serial.errdata,errchunk,count);
resp->serial.errdata[count] = '\0';
return THROW(NC_ENODATA); /* slight lie */
}
@ -157,26 +152,26 @@ Given a raw response, attempt to infer the mode: DMR, DAP, DSR.
Since DSR is not standardizes, it becomes the default.
*/
int
NCD4_infermode(NCD4meta* meta)
NCD4_infermode(NCD4response* resp)
{
d4size_t size = meta->serial.rawsize;
char* raw = meta->serial.rawdata;
d4size_t size = resp->raw.size;
char* raw = resp->raw.memory;
if(size < 16)
return THROW(NC_EDAP); /* must have at least this to hold a hdr + partial dmr*/
if(memcmp(raw,"<?xml",strlen("<?xml"))==0
|| memcmp(raw,"<Dataset",strlen("<Dataset"))==0) {
meta->mode = NCD4_DMR;
resp->mode = NCD4_DMR;
goto done;
}
raw += 4; /* Pretend we have a DAP hdr */
if(memcmp(raw,"<?xml",strlen("<?xml"))==0
|| memcmp(raw,"<Dataset",strlen("<Dataset"))==0) {
meta->mode = NCD4_DAP;
resp->mode = NCD4_DAP;
goto done;
}
/* Default to DSR */
meta->mode = NCD4_DSR;
resp->mode = NCD4_DSR;
done:
return NC_NOERR;

4
libdap4/d4curlfunctions.c
View File

@ -340,7 +340,7 @@ NCD4_get_rcproperties(NCD4INFO* state)
ncerror err = NC_NOERR;
char* option = NULL;
#ifdef HAVE_CURLOPT_BUFFERSIZE
option = NC_rclookup(D4BUFFERSIZE,state->uri->uri,NULL);
option = NC_rclookup(D4BUFFERSIZE,state->dmruri->uri,NULL);
if(option != NULL && strlen(option) != 0) {
long bufsize;
if(strcasecmp(option,"max")==0)
@ -351,7 +351,7 @@ NCD4_get_rcproperties(NCD4INFO* state)
}
#endif
#ifdef HAVE_CURLOPT_KEEPALIVE
option = NC_rclookup(D4KEEPALIVE,state->uri->uri,NULL);
option = NC_rclookup(D4KEEPALIVE,state->dmruri->uri,NULL);
if(option != NULL && strlen(option) != 0) {
/* The keepalive value is of the form 0 or n/m,
where n is the idle time and m is the interval time;

101
libdap4/d4data.c
View File

@ -16,7 +16,7 @@ This code serves two purposes
(NCD4_processdata)
2. Walk a specified variable instance to convert to netcdf4
memory representation.
(NCD4_fillinstance)
(NCD4_movetoinstance)
*/
@ -29,7 +29,6 @@ static int fillopfixed(NCD4meta*, d4size_t opaquesize, NCD4offset* offset, void*
static int fillopvar(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs);
static int fillstruct(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs);
static int fillseq(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs);
static int NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel);
static unsigned NCD4_computeChecksum(NCD4meta* meta, NCD4node* topvar);
/***************************************************/
@ -54,8 +53,9 @@ static unsigned int debugcrc32(unsigned int crc, const void *buf, size_t size)
/***************************************************/
/* API */
/* Parcel out the dechunked data to the corresponding vars */
int
NCD4_processdata(NCD4meta* meta)
NCD4_parcelvars(NCD4meta* meta, NCD4response* resp)
{
int ret = NC_NOERR;
int i;
@ -68,35 +68,57 @@ NCD4_processdata(NCD4meta* meta)
toplevel = nclistnew();
NCD4_getToplevelVars(meta,root,toplevel);
/* Otherwise */
NCD4_inferChecksums(meta,toplevel);
/* If necessary, byte swap the serialized data */
/* Do we need to swap the dap4 data? */
meta->swap = (meta->serial.hostlittleendian != meta->serial.remotelittleendian);
/* Compute the offset and size of the toplevel vars in the raw dap data. */
/* Also extract remote checksums */
offset = BUILDOFFSET(meta->serial.dap,meta->serial.dapsize);
offset = BUILDOFFSET(resp->serial.dap,resp->serial.dapsize);
for(i=0;i<nclistlength(toplevel);i++) {
NCD4node* var = (NCD4node*)nclistget(toplevel,i);
if((ret=NCD4_delimit(meta,var,offset)))
if((ret=NCD4_delimit(meta,var,offset,resp->inferredchecksumming))) {
FAIL(ret,"delimit failure");
if(meta->controller->data.inferredchecksumming) {
/* Compute remote checksum: must occur before any byte swapping */
}
var->data.response = resp; /* cross link */
}
done:
nclistfree(toplevel);
nullfree(offset);
return THROW(ret);
}
/* Process top level vars wrt checksums and swapping */
int
NCD4_processdata(NCD4meta* meta, NCD4response* resp)
{
int ret = NC_NOERR;
int i;
NClist* toplevel = NULL;
NCD4node* root = meta->root;
NCD4offset* offset = NULL;
/* Do we need to swap the dap4 data? */
meta->swap = (meta->controller->platform.hostlittleendian != resp->remotelittleendian);
/* Recursively walk the tree in prefix order
to get the top-level variables; also mark as unvisited */
toplevel = nclistnew();
NCD4_getToplevelVars(meta,root,toplevel);
/* Extract remote checksums */
for(i=0;i<nclistlength(toplevel);i++) {
NCD4node* var = (NCD4node*)nclistget(toplevel,i);
if(resp->inferredchecksumming) {
/* Compute checksum of response data: must occur before any byte swapping and after delimiting */
var->data.localchecksum = NCD4_computeChecksum(meta,var);
#ifdef DUMPCHECKSUM
fprintf(stderr,"var %s: remote-checksum = 0x%x\n",var->name,var->data.remotechecksum);
#endif
/* verify checksums */
if(!meta->controller->data.checksumignore) {
if(!resp->checksumignore) {
if(var->data.localchecksum != var->data.remotechecksum) {
nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name);
ret = NC_EDAP;
goto done;
}
/* Also verify checksum attribute */
if(meta->controller->data.attrchecksumming) {
if(resp->attrchecksumming) {
if(var->data.attrchecksum != var->data.remotechecksum) {
nclog(NCLOGERR,"Attribute Checksum mismatch: %s\n",var->name);
ret = NC_EDAP;
@ -105,13 +127,11 @@ NCD4_processdata(NCD4meta* meta)
}
}
}
}
/* Swap the data for each top level variable,
*/
if(meta->swap) {
if((ret=NCD4_swapdata(meta,toplevel)))
FAIL(ret,"byte swapping failed");
if(meta->swap) {
if((ret=NCD4_swapdata(resp,var,meta->swap)))
FAIL(ret,"byte swapping failed");
}
var->data.valid = 1; /* Everything should be in place */
}
done:
@ -133,7 +153,7 @@ Assumes that NCD4_processdata has been called.
*/
int
NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs)
NCD4_movetoinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs)
{
int ret = NC_NOERR;
void* dst = *dstp;
@ -149,30 +169,30 @@ NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dst
} else switch(type->subsort) {
case NC_STRING: /* oob strings */
if((ret=fillstring(meta,offset,&dst,blobs)))
FAIL(ret,"fillinstance");
FAIL(ret,"movetoinstance");
break;
case NC_OPAQUE:
if(type->opaque.size > 0) {
/* We know the size and its the same for all instances */
if((ret=fillopfixed(meta,type->opaque.size,offset,&dst)))
FAIL(ret,"fillinstance");
FAIL(ret,"movetoinstance");
} else {
/* Size differs per instance, so we need to convert each opaque to a vlen */
if((ret=fillopvar(meta,type,offset,&dst,blobs)))
FAIL(ret,"fillinstance");
FAIL(ret,"movetoinstance");
}
break;
case NC_STRUCT:
if((ret=fillstruct(meta,type,offset,&dst,blobs)))
FAIL(ret,"fillinstance");
FAIL(ret,"movetoinstance");
break;
case NC_SEQ:
if((ret=fillseq(meta,type,offset,&dst,blobs)))
FAIL(ret,"fillinstance");
FAIL(ret,"movetoinstance");
break;
default:
ret = NC_EINVAL;
FAIL(ret,"fillinstance");
FAIL(ret,"movetoinstance");
}
*dstp = dst;
@ -196,7 +216,7 @@ fillstruct(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NCli
NCD4node* field = nclistget(type->vars,i);
NCD4node* ftype = field->basetype;
void* fdst = (((char*)dst) + field->meta.offset);
if((ret=NCD4_fillinstance(meta,ftype,offset,&fdst,blobs)))
if((ret=NCD4_movetoinstance(meta,ftype,offset,&fdst,blobs)))
FAIL(ret,"fillstruct");
}
dst = ((char*)dst) + type->meta.memsize;
@ -231,7 +251,7 @@ fillseq(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist*
for(i=0;i<recordcount;i++) {
/* Read each record instance */
void* recdst = ((char*)(dst->p))+(recordsize * i);
if((ret=NCD4_fillinstance(meta,vlentype,offset,&recdst,blobs)))
if((ret=NCD4_movetoinstance(meta,vlentype,offset,&recdst,blobs)))
FAIL(ret,"fillseq");
}
dst++;
@ -373,12 +393,16 @@ done:
return THROW(ret);
}
static int
NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel)
int
NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp)
{
int ret = NC_NOERR;
int i, attrfound;
NCD4INFO* info = meta->controller;
NClist* toplevel = NULL;
/* Get the toplevel vars */
toplevel = nclistnew();
NCD4_getToplevelVars(meta,meta->root,toplevel);
/* First, look thru the DMR to see if there is a checksum attribute */
attrfound = 0;
@ -399,9 +423,10 @@ NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel)
}
}
}
info->data.attrchecksumming = (attrfound ? 1 : 0);
nclistfree(toplevel);
resp->attrchecksumming = (attrfound ? 1 : 0);
/* Infer checksums */
info->data.inferredchecksumming = ((info->data.attrchecksumming || info->data.querychecksumming) ? 1 : 0);
resp->inferredchecksumming = ((resp->attrchecksumming || resp->querychecksumming) ? 1 : 0);
return THROW(ret);
}

2
libdap4/d4debug.c
View File

@ -97,7 +97,7 @@ int
NCD4_debugcopy(NCD4INFO* info)
{
int i,ret=NC_NOERR;
NCD4meta* meta = info->substrate.metadata;
NCD4meta* meta = info->dmrmetadata;
NClist* topvars = nclistnew();
NC* ncp = info->controller;
void* memory = NULL;

347
libdap4/d4file.c
View File

@ -6,7 +6,6 @@
#include "ncdispatch.h"
#include "ncd4dispatch.h"
#include "d4includes.h"
#include "d4read.h"
#include "d4curlfunctions.h"
#ifdef _MSC_VER
@ -23,19 +22,20 @@
static int constrainable(NCURI*);
static void freeCurl(NCD4curl*);
static void freeInfo(NCD4INFO*);
static int fragmentcheck(NCD4INFO*, const char* key, const char* subkey);
static const char* getfragment(NCD4INFO* info, const char* key);
static const char* getquery(NCD4INFO* info, const char* key);
static int set_curl_properties(NCD4INFO*);
static int makesubstrate(NCD4INFO* d4info);
static void resetInfoforRead(NCD4INFO* d4info);
/**************************************************/
/* Constants */
static const char* checkseps = "+,:;";
/*Define the set of protocols known to be constrainable */
static const char* constrainableprotocols[] = {"http", "https",NULL};
/**************************************************/
int
NCD4_open(const char * path, int mode,
@ -46,10 +46,10 @@ NCD4_open(const char * path, int mode,
NCD4INFO* d4info = NULL;
const char* value;
NC* nc;
NCD4meta* meta = NULL;
size_t len = 0;
void* contents = NULL;
NCD4response* dmrresp = NULL;
if(path == NULL)
return THROW(NC_EDAPURL);
@ -61,29 +61,27 @@ NCD4_open(const char * path, int mode,
/* Setup our NC and NCDAPCOMMON state*/
d4info = (NCD4INFO*)calloc(1,sizeof(NCD4INFO));
if(d4info == NULL) {ret = NC_ENOMEM; goto done;}
if((ret=NCD4_newInfo(&d4info))) goto done;
nc->dispatchdata = d4info;
nc->int_ncid = nc__pseudofd(); /* create a unique id */
d4info->controller = (NC*)nc;
/* Parse url and params */
if(ncuriparse(nc->path,&d4info->uri))
if(ncuriparse(nc->path,&d4info->dmruri))
{ret = NC_EDAPURL; goto done;}
/* Load auth info from rc file */
if((ret = NC_authsetup(&d4info->auth, d4info->uri)))
if((ret = NC_authsetup(&d4info->auth, d4info->dmruri)))
goto done;
NCD4_curl_protocols(d4info);
if(!constrainable(d4info->uri))
if(!constrainable(d4info->dmruri))
SETFLAG(d4info->controls.flags,NCF_UNCONSTRAINABLE);
/* fail if we are unconstrainable but have constraints */
if(FLAGSET(d4info->controls.flags,NCF_UNCONSTRAINABLE)) {
if(d4info->uri != NULL) {
const char* ce = ncuriquerylookup(d4info->uri,DAP4CE); /* Look for dap4.ce */
if(d4info->dmruri != NULL) {
const char* ce = ncuriquerylookup(d4info->dmruri,DAP4CE); /* Look for dap4.ce */
if(ce != NULL) {
nclog(NCLOGWARN,"Attempt to constrain an unconstrainable data source: %s=%s",
DAP4CE,ce);
@ -115,7 +113,7 @@ NCD4_open(const char * path, int mode,
}
/* Turn on logging; only do this after oc_open*/
if((value = ncurifragmentlookup(d4info->uri,"log")) != NULL) {
if((value = ncurifragmentlookup(d4info->dmruri,"log")) != NULL) {
ncloginit();
ncsetloglevel(NCLOGNOTE);
}
@ -150,30 +148,34 @@ NCD4_open(const char * path, int mode,
/* Reset the substrate */
if((ret=makesubstrate(d4info))) goto done;
/* Always start by reading the DMR only */
/* Always start by reading the whole DMR only */
/* reclaim substrate.metadata */
resetInfoforRead(d4info);
NCD4_resetInfoForRead(d4info);
/* Rebuild metadata */
if((d4info->substrate.metadata=NCD4_newmeta(d4info))==NULL)
{ret = NC_ENOMEM; goto done;}
if((ret = NCD4_newMeta(d4info,&d4info->dmrmetadata))) goto done;
if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags))) goto done;
/* Capture response */
if((dmrresp = (NCD4response*)calloc(1,sizeof(NCD4response)))==NULL)
{ret = NC_ENOMEM; goto done;}
dmrresp->controller = d4info;
if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags, d4info->dmruri, dmrresp))) goto done;
/* set serial.rawdata */
len = ncbyteslength(d4info->curl->packet);
contents = ncbytesextract(d4info->curl->packet);
NCD4_attachraw(d4info->substrate.metadata, len, contents);
assert(dmrresp != NULL);
dmrresp->raw.size = len;
dmrresp->raw.memory = contents;
/* process query parameters */
NCD4_applyclientquerycontrols(d4info);
meta = d4info->substrate.metadata;
/* process checksum parameters */
NCD4_applychecksumcontrols(d4info,dmrresp);
/* Infer the mode */
if((ret=NCD4_infermode(meta))) goto done;
if((ret=NCD4_infermode(dmrresp))) goto done;
/* Process the dmr part */
if((ret=NCD4_dechunk(meta))) goto done;
if((ret=NCD4_dechunk(dmrresp))) goto done;
#ifdef D4DUMPDMR
{
@ -184,13 +186,14 @@ NCD4_open(const char * path, int mode,
}
#endif
if((ret = NCD4_parse(d4info->substrate.metadata))) goto done;
if((ret = NCD4_parse(d4info->dmrmetadata,dmrresp,0))) goto done;
#ifdef D4DEBUGMETA
{
meta = d4info->dmrmetadata;
fprintf(stderr,"\n/////////////\n");
NCbytes* buf = ncbytesnew();
NCD4_print(d4info->substrate.metadata,buf);
NCD4_print(meta,buf);
ncbytesnull(buf);
fputs(ncbytescontents(buf),stderr);
ncbytesfree(buf);
@ -200,12 +203,20 @@ NCD4_open(const char * path, int mode,
#endif
/* Build the substrate metadata */
ret = NCD4_metabuild(d4info->substrate.metadata,d4info->substrate.metadata->ncid);
ret = NCD4_metabuild(d4info->dmrmetadata,d4info->dmrmetadata->ncid);
if(ret != NC_NOERR && ret != NC_EVARSIZE) goto done;
/* Remember the response */
nclistpush(d4info->responses,dmrresp);
/* Avoid duplicate reclaims */
dmrresp = NULL;
d4info = NULL;
done:
NCD4_reclaimResponse(dmrresp);
NCD4_reclaimInfo(d4info);
if(ret) {
freeInfo(d4info);
nc->dispatchdata = NULL;
}
return THROW(ret);
@ -236,7 +247,7 @@ NCD4_close(int ncid, void* ignore)
ret = nc_abort(substrateid);
}
freeInfo(d4info);
NCD4_reclaimInfo(d4info);
done:
return THROW(ret);
@ -248,82 +259,6 @@ NCD4_abort(int ncid)
return NCD4_close(ncid,NULL);
}
/**************************************************/
/* Reclaim an NCD4INFO instance */
static void
freeInfo(NCD4INFO* d4info)
{
if(d4info == NULL) return;
d4info->controller = NULL; /* break link */
nullfree(d4info->rawurltext);
nullfree(d4info->urltext);
ncurifree(d4info->uri);
freeCurl(d4info->curl);
nullfree(d4info->data.memory);
nullfree(d4info->data.ondiskfilename);
if(d4info->data.ondiskfile != NULL)
fclose(d4info->data.ondiskfile);
nullfree(d4info->fileproto.filename);
if(d4info->substrate.realfile
&& !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) {
/* We used real file, so we need to delete the temp file
unless we are debugging.
Assume caller has done nc_close|nc_abort on the ncid.
Note that in theory, this should not be necessary since
AFAIK the substrate file is still in def mode, and
when aborted, it should be deleted. But that is not working
for some reason, so we delete it ourselves.
*/
if(d4info->substrate.filename != NULL) {
unlink(d4info->substrate.filename);
}
}
nullfree(d4info->substrate.filename); /* always reclaim */
NCD4_reclaimMeta(d4info->substrate.metadata);
NC_authfree(d4info->auth);
nclistfree(d4info->blobs);
free(d4info);
}
/* Reset NCD4INFO instance for new read request */
static void
resetInfoforRead(NCD4INFO* d4info)
{
if(d4info == NULL) return;
if(d4info->substrate.realfile
&& !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) {
/* We used real file, so we need to delete the temp file
unless we are debugging.
Assume caller has done nc_close|nc_abort on the ncid.
Note that in theory, this should not be necessary since
AFAIK the substrate file is still in def mode, and
when aborted, it should be deleted. But that is not working
for some reason, so we delete it ourselves.
*/
if(d4info->substrate.filename != NULL) {
unlink(d4info->substrate.filename);
}
}
NCD4_resetMeta(d4info->substrate.metadata);
nullfree(d4info->substrate.metadata);
d4info->substrate.metadata = NULL;
}
static void
freeCurl(NCD4curl* curl)
{
if(curl == NULL) return;
NCD4_curlclose(curl->curl);
ncbytesfree(curl->packet);
nullfree(curl->errdata.code);
nullfree(curl->errdata.message);
free(curl);
}
/* Define the set of protocols known to be constrainable */
static const char* constrainableprotocols[] = {"http", "https",NULL};
static int
constrainable(NCURI* durl)
{
@ -449,11 +384,6 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info)
if(value != NULL)
strncpy(info->controls.substratename,value,(NC_MAX_NAME-1));
value = getfragment(info,"hyrax");
if(value != NULL) {
info->data.checksumignore = 1; /* Assume checksum, but ignore */
}
info->controls.opaquesize = DFALTOPAQUESIZE;
value = getfragment(info,"opaquesize");
if(value != NULL) {
@ -476,22 +406,29 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info)
}
}
/* Checksum controls are found both in the query and fragment
parts of a URL.
*/
void
NCD4_applyclientquerycontrols(NCD4INFO* info)
NCD4_applychecksumcontrols(NCD4INFO* info, NCD4response* resp)
{
const char* value = getquery(info,DAP4CSUM);
if(value == NULL) {
info->data.querychecksumming = DEFAULT_CHECKSUM_STATE;
resp->querychecksumming = DEFAULT_CHECKSUM_STATE;
} else {
if(strcasecmp(value,"false")==0) {
info->data.querychecksumming = 0;
resp->querychecksumming = 0;
} else if(strcasecmp(value,"true")==0) {
info->data.querychecksumming = 1;
resp->querychecksumming = 1;
} else {
nclog(NCLOGWARN,"Unknown checksum mode: %s ; using default",value);
info->data.querychecksumming = DEFAULT_CHECKSUM_STATE;
resp->querychecksumming = DEFAULT_CHECKSUM_STATE;
}
}
value = getfragment(info,"hyrax");
if(value != NULL) {
resp->checksumignore = 1; /* Assume checksum, but ignore */
}
}
/* Search for substring in value of param. If substring == NULL; then just
@ -523,7 +460,7 @@ getfragment(NCD4INFO* info, const char* key)
const char* value;
if(info == NULL || key == NULL) return NULL;
if((value=ncurifragmentlookup(info->uri,key)) == NULL)
if((value=ncurifragmentlookup(info->dmruri,key)) == NULL)
return NULL;
return value;
}
@ -537,7 +474,7 @@ getquery(NCD4INFO* info, const char* key)
const char* value;
if(info == NULL || key == NULL) return NULL;
if((value=ncuriquerylookup(info->uri,key)) == NULL)
if((value=ncuriquerylookup(info->dmruri,key)) == NULL)
return NULL;
return value;
}
@ -596,3 +533,175 @@ NCD4_get_substrate(NC* nc)
} else subnc = nc;
return subnc;
}
/**************************************************/
/* Allocate/Free for various structures */
int
NCD4_newInfo(NCD4INFO** d4infop)
{
int ret = NC_NOERR;
NCD4INFO* info = NULL;
if((info = calloc(1,sizeof(NCD4INFO)))==NULL)
{ret = NC_ENOMEM; goto done;}
info->platform.hostlittleendian = NCD4_isLittleEndian();
info->responses = nclistnew();
if(d4infop) {*d4infop = info; info = NULL;}
done:
if(info) NCD4_reclaimInfo(info);
return THROW(ret);
}
/* Reclaim an NCD4INFO instance */
void
NCD4_reclaimInfo(NCD4INFO* d4info)
{
size_t i;
if(d4info == NULL) return;
d4info->controller = NULL; /* break link */
nullfree(d4info->rawdmrurltext);
nullfree(d4info->dmrurltext);
ncurifree(d4info->dmruri);
freeCurl(d4info->curl);
nullfree(d4info->fileproto.filename);
NCD4_resetInfoForRead(d4info);
nullfree(d4info->substrate.filename); /* always reclaim */
NC_authfree(d4info->auth);
nclistfree(d4info->blobs);
/* Reclaim dmr node tree */
NCD4_reclaimMeta(d4info->dmrmetadata);
/* Reclaim all responses */
for(i=0;i<nclistlength(d4info->responses);i++) {
NCD4response* resp = nclistget(d4info->responses,i);
NCD4_reclaimResponse(resp);
}
nclistfree(d4info->responses);
free(d4info);
}
/* Reset NCD4INFO instance for new read request */
void
NCD4_resetInfoForRead(NCD4INFO* d4info)
{
if(d4info == NULL) return;
if(d4info->substrate.realfile
&& !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) {
/* We used real file, so we need to delete the temp file
unless we are debugging.
Assume caller has done nc_close|nc_abort on the ncid.
Note that in theory, this should not be necessary since
AFAIK the substrate file is still in def mode, and
when aborted, it should be deleted. But that is not working
for some reason, so we delete it ourselves.
*/
if(d4info->substrate.filename != NULL) {
unlink(d4info->substrate.filename);
}
}
NCD4_reclaimMeta(d4info->dmrmetadata);
d4info->dmrmetadata = NULL;
}
static void
freeCurl(NCD4curl* curl)
{
if(curl == NULL) return;
NCD4_curlclose(curl->curl);
ncbytesfree(curl->packet);
nullfree(curl->errdata.code);
nullfree(curl->errdata.message);
free(curl);
}
int
NCD4_newResponse(NCD4INFO* info, NCD4response** respp)
{
int ret = NC_NOERR;
NCD4response* resp = NULL;
NC_UNUSED(info);
if((resp = calloc(1,sizeof(NCD4response)))==NULL)
{ret = NC_ENOMEM; goto done;}
resp->controller = info;
if(respp) {*respp = resp; resp = NULL;}
done:
if(resp) NCD4_reclaimResponse(resp);
return THROW(ret);
}
/* Reclaim an NCD4response instance */
void
NCD4_reclaimResponse(NCD4response* d4resp)
{
struct NCD4serial* serial = NULL;
if(d4resp == NULL) return;
serial = &d4resp->serial;
d4resp->controller = NULL; /* break link */
nullfree(d4resp->raw.memory);
nullfree(serial->dmr);
nullfree(serial->errdata);
/* clear all fields */
memset(serial,0,sizeof(struct NCD4serial));
nullfree(d4resp->error.parseerror);
nullfree(d4resp->error.message);
nullfree(d4resp->error.context);
nullfree(d4resp->error.otherinfo);
memset(&d4resp->error,0,sizeof(d4resp->error));
free(d4resp);
}
/* Create an empty NCD4meta object for
use in subsequent calls
(is the the right src file to hold this?)
*/
int
NCD4_newMeta(NCD4INFO* info, NCD4meta** metap)
{
int ret = NC_NOERR;
NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta));
if(meta == NULL) return NC_ENOMEM;
meta->allnodes = nclistnew();
#ifdef D4DEBUG
meta->debuglevel = 1;
#endif
meta->controller = info;
meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */
if(metap) {*metap = meta; meta = NULL;}
return THROW(ret);
}
void
NCD4_reclaimMeta(NCD4meta* dataset)
{
int i;
if(dataset == NULL) return;
for(i=0;i<nclistlength(dataset->allnodes);i++) {
NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i);
reclaimNode(node);
}
nclistfree(dataset->allnodes);
nclistfree(dataset->groupbyid);
nclistfree(dataset->atomictypes);
free(dataset);
}
#if 0
void
NCD4_resetMeta(NCD4meta* dataset)
{
if(dataset == NULL) return;
#if 0
for(i=0;i<nclistlength(dataset->blobs);i++) {
void* p = nclistget(dataset->blobs,i);
nullfree(p);
}
nclistfree(dataset->blobs);
#endif
}
#endif

4
libdap4/d4fix.c
View File

@ -190,7 +190,7 @@ walk(NCD4node* node, NClist* sorted)
*/
int
NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset)
NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset, int inferredchecksumming)
{
int ret = NC_NOERR;
NCD4mark mark = 0;
@ -214,7 +214,7 @@ NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset)
topvar->data.dap4data.memory = mark;
topvar->data.dap4data.size = OFFSETSIZE(offset,mark);
/* extract the dap4 data checksum, if present */
if(compiler->controller->data.inferredchecksumming) {
if(inferredchecksumming) {
union ATOMICS csum;
TRANSFER(csum.u8,offset,CHECKSUMSIZE);
topvar->data.remotechecksum = csum.u32[0];

75
libdap4/d4http.c
View File

@ -6,7 +6,6 @@
#include "d4includes.h"
#include "d4curlfunctions.h"
static size_t WriteFileCallback(void*, size_t, size_t, void*);
static size_t WriteMemoryCallback(void*, size_t, size_t, void*);
static int curlerrtoncerr(CURLcode cstat);
@ -33,59 +32,6 @@ NCD4_fetchhttpcode(CURL* curl)
return httpcode;
}
int
NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream,
d4size_t* sizep, long* filetime)
{
int ret = NC_NOERR;
CURLcode cstat = CURLE_OK;
struct Fetchdata fetchdata;
/* Set the URL */
cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url);
if (cstat != CURLE_OK) goto fail;
/* send all data to this function */
cstat = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteFileCallback);
if (cstat != CURLE_OK) goto fail;
/* we pass our file to the callback function */
cstat = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fetchdata);
if(cstat != CURLE_OK) goto fail;
/* One last thing; always try to get the last modified time */
cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1);
if (cstat != CURLE_OK) goto fail;
fetchdata.stream = stream;
fetchdata.size = 0;
cstat = curl_easy_perform(curl);
if (cstat != CURLE_OK)
{ret = NC_EDAPSVC; goto fail;}
if (ret == NC_NOERR) {
/* return the file size*/
#ifdef D4DEBUG
nclog(NCLOGNOTE,"filesize: %lu bytes",fetchdata.size);
#endif
if (sizep != NULL)
*sizep = fetchdata.size;
/* Get the last modified time */
if(filetime != NULL)
cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime);
if(cstat != CURLE_OK)
{ret = NC_ECURL; goto fail;}
}
return THROW(ret);
fail:
if(cstat != CURLE_OK) {
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
ret = curlerrtoncerr(cstat);
}
return THROW(ret);
}
int
NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcodep)
{
@ -155,27 +101,6 @@ done:
return THROW(ret);
}
static size_t
WriteFileCallback(void* ptr, size_t size, size_t nmemb, void* data)
{
size_t realsize = size * nmemb;
size_t count;
struct Fetchdata* fetchdata;
fetchdata = (struct Fetchdata*) data;
if(realsize == 0)
nclog(NCLOGWARN,"WriteFileCallback: zero sized chunk");
count = fwrite(ptr, size, nmemb, fetchdata->stream);
if (count > 0) {
fetchdata->size += (count * size);
} else {
nclog(NCLOGWARN,"WriteFileCallback: zero sized write");
}
#ifdef PROGRESS
nclog(NCLOGNOTE,"callback: %lu bytes",(d4size_t)realsize);
#endif
return count;
}
static size_t
WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
{

110
libdap4/d4meta.c
View File

@ -89,75 +89,6 @@ done:
return THROW(ret);
}
/* Create an empty NCD4meta object for
use in subsequent calls
(is the the right src file to hold this?)
*/
NCD4meta*
NCD4_newmeta(NCD4INFO* info)
{
NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta));
if(meta == NULL) return NULL;
meta->allnodes = nclistnew();
#ifdef D4DEBUG
meta->debuglevel = 1;
#endif
meta->controller = info;
meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */
return meta;
}
/* Attach raw data to metadata */
void
NCD4_attachraw(NCD4meta* meta, size_t rawsize, void* rawdata)
{
assert(meta != NULL);
NCD4_resetSerial(&meta->serial,rawsize,rawdata);
}
void
NCD4_setdebuglevel(NCD4meta* meta, int debuglevel)
{
meta->debuglevel = debuglevel;
}
void
NCD4_reclaimMeta(NCD4meta* dataset)
{
int i;
if(dataset == NULL) return;
NCD4_resetMeta(dataset);
for(i=0;i<nclistlength(dataset->allnodes);i++) {
NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i);
reclaimNode(node);
}
nclistfree(dataset->allnodes);
nclistfree(dataset->groupbyid);
nclistfree(dataset->atomictypes);
free(dataset);
}
void
NCD4_resetMeta(NCD4meta* dataset)
{
if(dataset == NULL) return;
nullfree(dataset->error.parseerror); dataset->error.parseerror = NULL;
nullfree(dataset->error.message); dataset->error.message = NULL;
nullfree(dataset->error.context); dataset->error.context = NULL;
nullfree(dataset->error.otherinfo); dataset->error.otherinfo = NULL;
NCD4_resetSerial(&dataset->serial,0,NULL);
#if 0
for(i=0;i<nclistlength(dataset->blobs);i++) {
void* p = nclistget(dataset->blobs,i);
nullfree(p);
}
nclistfree(dataset->blobs);
#endif
}
void
reclaimNode(NCD4node* node)
{
@ -676,6 +607,39 @@ savevarbyid(NCD4node* group, NCD4node* var)
nclistinsert(group->group.varbyid,var->meta.id,var);
}
/* Collect FQN path from var node up to and including
the root group and create an name from it
*/
char*
NCD4_getVarFQN(NCD4node* var, const char* tail)
{
int i;
NCD4node* x = NULL;
NClist* path = NULL;
NCbytes* fqn = NULL;
char* result;
path = nclistnew();
for(x=var->container;ISGROUP(x->sort);x=x->container) {
nclistinsert(path,0,x);
}
fqn = ncbytesnew();
for(i=0;i<nclistlength(path);i++) {
NCD4node* grp = (NCD4node*)nclistget(path,i);
char* escaped = backslashEscape(grp->name);
if(escaped == NULL) return NULL;
if(i > 0) ncbytesappend(fqn,'/');
ncbytescat(fqn,escaped);
free(escaped);
}
nclistfree(path);
if(tail != NULL)
ncbytescat(fqn,tail);
result = ncbytesextract(fqn);
ncbytesfree(fqn);
return result;
}
/* Collect FQN path from node up to (but not including)
the first enclosing group and create an name from it
*/
@ -1180,7 +1144,7 @@ markdapsize(NCD4meta* meta)
}
int
NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp)
NCD4_findvar(NC* ncp, int gid, int varid, NCD4node** varp, NCD4node** grpp)
{
int ret = NC_NOERR;
NCD4INFO* info = NULL;
@ -1192,11 +1156,11 @@ NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp)
info = getdap(ncp);
if(info == NULL)
return THROW(NC_EBADID);
meta = info->substrate.metadata;
meta = info->dmrmetadata;
if(meta == NULL)
return THROW(NC_EBADID);
/* Locate var node via (grpid,varid) */
grp_id = GROUPIDPART(ncid);
grp_id = GROUPIDPART(gid);
group = nclistget(meta->groupbyid,grp_id);
if(group == NULL)
return THROW(NC_EBADID);
@ -1205,7 +1169,7 @@ NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp)
return THROW(NC_EBADID);
if(varp) *varp = var;
if(grpp) *grpp = group;
return ret;
return THROW(ret);
}
static int

37
libdap4/d4parser.c
View File

@ -154,7 +154,7 @@ static int defineBytestringType(NCD4parser*);
/* API */
int
NCD4_parse(NCD4meta* metadata)
NCD4_parse(NCD4meta* metadata, NCD4response* resp, int dapparse)
{
int ret = NC_NOERR;
NCD4parser* parser = NULL;
@ -168,8 +168,10 @@ NCD4_parse(NCD4meta* metadata)
/* Create and fill in the parser state */
parser = (NCD4parser*)calloc(1,sizeof(NCD4parser));
if(parser == NULL) {ret=NC_ENOMEM; goto done;}
parser->controller = metadata->controller;
parser->metadata = metadata;
doc = ncxml_parse(parser->metadata->serial.dmr,strlen(parser->metadata->serial.dmr));
parser->response = resp;
doc = ncxml_parse(parser->response->serial.dmr,strlen(parser->response->serial.dmr));
if(doc == NULL) {ret=NC_ENOMEM; goto done;}
dom = ncxml_root(doc);
parser->types = nclistnew();
@ -178,6 +180,7 @@ NCD4_parse(NCD4meta* metadata)
#ifdef D4DEBUG
parser->debuglevel = 1;
#endif
parser->dapparse = dapparse;
/*Walk the DOM tree to build the DAP4 node tree*/
ret = traverse(parser,dom);
@ -214,9 +217,9 @@ traverse(NCD4parser* parser, ncxml_t dom)
ret=parseError(parser,dom);
/* Report the error */
fprintf(stderr,"DAP4 Error: http-code=%d message=\"%s\" context=\"%s\"\n",
parser->metadata->error.httpcode,
parser->metadata->error.message,
parser->metadata->error.context);
parser->response->error.httpcode,
parser->response->error.message,
parser->response->error.context);
fflush(stderr);
ret=NC_EDMR;
goto done;
@ -225,7 +228,8 @@ traverse(NCD4parser* parser, ncxml_t dom)
if((ret=makeNode(parser,NULL,NULL,NCD4_GROUP,NC_NULL,&parser->metadata->root))) goto done;
parser->metadata->root->group.isdataset = 1;
parser->metadata->root->meta.id = parser->metadata->ncid;
parser->metadata->groupbyid = nclistnew();
if(parser->metadata->groupbyid == NULL)
parser->metadata->groupbyid = nclistnew();
SETNAME(parser->metadata->root,"/");
xattr = ncxml_attr(dom,"name");
if(xattr != NULL) parser->metadata->root->group.datasetname = xattr;
@ -847,23 +851,23 @@ parseError(NCD4parser* parser, ncxml_t errxml)
char* shttpcode = ncxml_attr(errxml,"httpcode");
ncxml_t x;
if(shttpcode == NULL) shttpcode = strdup("400");
if(sscanf(shttpcode,"%d",&parser->metadata->error.httpcode) != 1)
if(sscanf(shttpcode,"%d",&parser->response->error.httpcode) != 1)
nclog(NCLOGERR,"Malformed <ERROR> response");
nullfree(shttpcode);
x=ncxml_child(errxml, "Message");
if(x != NULL) {
char* txt = ncxml_text(x);
parser->metadata->error.message = (txt == NULL ? NULL : txt);
parser->response->error.message = (txt == NULL ? NULL : txt);
}
x = ncxml_child(errxml, "Context");
if(x != NULL) {
const char* txt = ncxml_text(x);
parser->metadata->error.context = (txt == NULL ? NULL : strdup(txt));
parser->response->error.context = (txt == NULL ? NULL : strdup(txt));
}
x=ncxml_child(errxml, "OtherInformation");
if(x != NULL) {
const char* txt = ncxml_text(x);
parser->metadata->error.otherinfo = (txt == NULL ? NULL : strdup(txt));
parser->response->error.otherinfo = (txt == NULL ? NULL : strdup(txt));
}
return THROW(NC_NOERR);
}
@ -1321,7 +1325,7 @@ makeNode(NCD4parser* parser, NCD4node* parent, ncxml_t xml, NCD4sort sort, nc_ty
record(parser,node);
if(nodep) *nodep = node;
done:
return ret;
return THROW(ret);
}
static int
@ -1652,12 +1656,19 @@ parseForwards(NCD4parser* parser, NCD4node* root)
const char* mapname = (const char*)nclistget(var->mapnames,j);
/* Find the corresponding variable */
NCD4node* mapref = lookupFQN(parser,mapname,NCD4_VAR);
if(mapref == NULL)
if(mapref != NULL)
PUSH(var->maps,mapref);
else if(!parser->dapparse)
FAIL(NC_ENOTVAR,"<Map> name does not refer to a variable: %s",mapname);
PUSH(var->maps,mapref);
}
}
done:
return THROW(ret);
}
void
NCD4_setdebuglevel(NCD4parser* parser, int debuglevel)
{
parser->debuglevel = debuglevel;
}

121
libdap4/d4read.c
View File

@ -19,9 +19,8 @@ See \ref copyright file for more info.
/* Do conversion if this code was compiled via Vis. Studio or Mingw */
/*Forward*/
static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, long*);
static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, int*, long*);
static int readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet);
static int readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep);
static int readfileDAPDMR(NCD4INFO* state, const NCURI* uri, NCbytes* packet);
#ifdef HAVE_GETTIMEOFDAY
@ -38,82 +37,27 @@ deltatime(struct timeval time0,struct timeval time1)
#endif
int
NCD4_readDMR(NCD4INFO* state, int flags)
NCD4_readDMR(NCD4INFO* state, int flags, NCURI* url, NCD4response* resp)
{
int stat = NC_NOERR;
long lastmod = -1;
if((flags & NCF_ONDISK) == 0) {
ncbytesclear(state->curl->packet);
stat = readpacket(state,state->uri,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&lastmod);
if(stat == NC_NOERR)
state->data.dmrlastmodified = lastmod;
} else { /*((flags & NCF_ONDISK) != 0) */
NCURI* url = state->uri;
int fileprotocol = (strcmp(url->protocol,"file")==0);
if(fileprotocol) {
stat = readfiletofile(state, url, NCD4_DMR, NCD4_FORMAT_XML, state->data.ondiskfile, &state->data.datasize);
} else {
char* readurl = NULL;
int flags = 0;
if(!fileprotocol) flags |= NCURIQUERY;
flags |= NCURIENCODE;
flags |= NCURIPWD;
#ifdef FIX
ncurisetconstraints(url,state->constraint);
#endif
readurl = ncuribuild(url,NULL,".dmr.xml",NCURISVC);
if(readurl == NULL)
return THROW(NC_ENOMEM);
stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile,
&state->data.datasize, &lastmod);
nullfree(readurl);
if(stat == NC_NOERR)
state->data.dmrlastmodified = lastmod;
}
}
ncbytesclear(state->curl->packet);
stat = readpacket(state,url,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&resp->serial.httpcode,NULL);
return THROW(stat);
}
int
NCD4_readDAP(NCD4INFO* state, int flags)
NCD4_readDAP(NCD4INFO* state, int flags, NCURI* url, NCD4response* resp)
{
int stat = NC_NOERR;
long lastmod = -1;
if((flags & NCF_ONDISK) == 0) {
ncbytesclear(state->curl->packet);
stat = readpacket(state,state->uri,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&lastmod);
if(stat) {
NCD4_seterrormessage(state->substrate.metadata, nclistlength(state->curl->packet), nclistcontents(state->curl->packet));
goto done;
} else
state->data.daplastmodified = lastmod;
} else { /*((flags & NCF_ONDISK) != 0) */
NCURI* url = state->uri;
int fileprotocol = (strcmp(url->protocol,"file")==0);
if(fileprotocol) {
stat = readfiletofile(state, url, NCD4_DAP, NCD4_FORMAT_NONE, state->data.ondiskfile, &state->data.datasize);
} else {
char* readurl = NULL;
int flags = 0;
if(!fileprotocol) flags |= NCURIQUERY;
flags |= NCURIENCODE;
flags |= NCURIPWD;
#ifdef FIX
ncurisetconstraints(url,state->constraint);
#endif
readurl = ncuribuild(url,NULL,".dap",NCURISVC);
if(readurl == NULL)
return THROW(NC_ENOMEM);
stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile,
&state->data.datasize, &lastmod);
nullfree(readurl);
if(stat == NC_NOERR)
state->data.daplastmodified = lastmod;
}
ncbytesclear(state->curl->packet);
stat = readpacket(state,url,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&resp->serial.httpcode,NULL);
if(stat) {
NCD4_seterrormessage(resp, nclistlength(state->curl->packet), nclistcontents(state->curl->packet));
} else {
resp->raw.size = ncbyteslength(state->curl->packet);
resp->raw.memory = ncbytesextract(state->curl->packet);
}
done:
return THROW(stat);
}
@ -150,7 +94,7 @@ dxxformat(int fxx, int dxx)
}
static int
readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, long* lastmodified)
readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, int* httpcodep, long* lastmodified)
{
int stat = NC_NOERR;
int fileprotocol = 0;
@ -185,7 +129,7 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4forma
gettimeofday(&time0,NULL);
#endif
}
stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified,&state->substrate.metadata->error.httpcode);
stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified,httpcodep);
nullfree(fetchurl);
if(stat) goto fail;
if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) {
@ -207,37 +151,26 @@ fail:
return THROW(stat);
}
#if 0
static int
readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep)
readfromfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, d4size_t* sizep)
{
int stat = NC_NOERR;
NCbytes* packet = ncbytesnew();
size_t len;
stat = readfile(state, uri, dxx, fxx, packet);
ncbytesclear(state->curl->packet);
stat = readfile(state, uri, dxx, fxx, state->curl->packet);
#ifdef D4DEBUG
fprintf(stderr,"readfiletofile: packet.size=%lu\n",
(unsigned long)ncbyteslength(packet));
(unsigned long)ncbyteslength(state->curl->packet));
#endif
if(stat != NC_NOERR) goto unwind;
len = nclistlength(packet);
if(stat == NC_NOERR) {
size_t written;
fseek(stream,0,SEEK_SET);
written = fwrite(ncbytescontents(packet),1,len,stream);
if(written != len) {
#ifdef D4DEBUG
fprintf(stderr,"readfiletofile: written!=length: %lu :: %lu\n",
(unsigned long)written,(unsigned long)len);
#endif
stat = NC_EIO;
}
}
len = nclistlength(state->curl->packet);
if(sizep != NULL) *sizep = len;
unwind:
ncbytesfree(packet);
return THROW(stat);
}
#endif
static int
readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet)
@ -365,12 +298,12 @@ done:
/* Extract packet as error message; assume httpcode set */
int
NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg)
NCD4_seterrormessage(NCD4response* resp, size_t len, char* msg)
{
metadata->error.message = (char*)d4alloc(len+1);
if(metadata->error.message == NULL)
resp->error.message = (char*)d4alloc(len+1);
if(resp->error.message == NULL)
return THROW(NC_ENOMEM);
memcpy(metadata->error.message,msg,len);
metadata->error.message[len] = '\0';
memcpy(resp->error.message,msg,len);
resp->error.message[len] = '\0';
return THROW(NC_ENODATA); /* slight lie */
}

2
libdap4/d4read.h
View File

@ -7,6 +7,6 @@
#define D4READ_H
extern int NCD4_readDMR(NCD4INFO*, int flags);
extern int NCD4_readDAP(NCD4INFO*, int flags);
extern int NCD4_readDAP(NCD4INFO*, int flags, NCURI* uri);
#endif /*READ_H*/

70
libdap4/d4swap.c
View File

@ -14,12 +14,12 @@ the incoming data to get the endianness correct.
/* Forward */
static int walkAtomicVar(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset);
static int walkOpaqueVar(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset);
static int walkStructArray(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset);
static int walkStruct(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset);
static int walkSeqArray(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset);
static int walkSeq(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset);
static int walkAtomicVar(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap);
static int walkOpaqueVar(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap);
static int walkStructArray(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap);
static int walkStruct(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap);
static int walkSeqArray(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap);
static int walkSeq(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap);
/**************************************************/
@ -28,43 +28,39 @@ Assumes that compiler->swap is true; does necessary
byte swapping.
*/
int
NCD4_swapdata(NCD4meta* compiler, NClist* topvars)
NCD4_swapdata(NCD4response* resp, NCD4node* var, int doswap)
{
int ret = NC_NOERR;
int i;
NCD4offset* offset = NULL;
offset = BUILDOFFSET(compiler->serial.dap,compiler->serial.dapsize);
for(i=0;i<nclistlength(topvars);i++) {
NCD4node* var = (NCD4node*)nclistget(topvars,i);
offset = BUILDOFFSET(resp->serial.dap,resp->serial.dapsize);
OFFSET2BLOB(var->data.dap4data,offset);
switch (var->subsort) {
default:
if((ret=walkAtomicVar(compiler,var,var,offset))) goto done;
if((ret=walkAtomicVar(resp,var,var,offset,doswap))) goto done;
break;
case NC_OPAQUE:
/* The only thing we need to do is swap the counts */
if((ret=walkOpaqueVar(compiler,var,var,offset))) goto done;
if((ret=walkOpaqueVar(resp,var,var,offset,doswap))) goto done;
break;
case NC_STRUCT:
if((ret=walkStructArray(compiler,var,var,offset))) goto done;
if((ret=walkStructArray(resp,var,var,offset,doswap))) goto done;
break;
case NC_SEQ:
if((ret=walkSeqArray(compiler,var,var,offset))) goto done;
if((ret=walkSeqArray(resp,var,var,offset,doswap))) goto done;
break;
}
var->data.dap4data.size = DELTA(offset,var->data.dap4data.memory);
/* skip checksum, if there is one */
if(compiler->controller->data.inferredchecksumming)
if(resp->inferredchecksumming)
INCR(offset,CHECKSUMSIZE);
}
done:
if(offset) free(offset);
return THROW(ret);
}
static int
walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset)
walkAtomicVar(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap)
{
int ret = NC_NOERR;
d4size_t i;
@ -87,7 +83,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o
} else { /*(typesize > 1)*/
for(i=0;i<dimproduct;i++) {
char* sp = offset->offset;
if(compiler->swap) {
if(doswap) {
switch (typesize) {
case 2: swapinline16(sp); break;
case 4: swapinline32(sp); break;
@ -102,7 +98,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o
COUNTERTYPE count;
for(i=0;i<dimproduct;i++) {
/* Get string count */
if(compiler->swap)
if(doswap)
swapinline64(offset);
count = GETCOUNTER(offset);
SKIPCOUNTER(offset);
@ -114,7 +110,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o
}
static int
walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset)
walkOpaqueVar(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap)
{
int ret = NC_NOERR;
d4size_t i;
@ -125,7 +121,7 @@ walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o
for(i=0;i<dimproduct;i++) {
/* Get and swap opaque count */
if(compiler->swap)
if(doswap)
swapinline64(offset);
count = GETCOUNTER(offset);
SKIPCOUNTER(offset);
@ -135,7 +131,7 @@ walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o
}
static int
walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset)
walkStructArray(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap)
{
int ret = NC_NOERR;
d4size_t i;
@ -144,7 +140,7 @@ walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset
for(i=0;i<dimproduct;i++) {
/* Swap, possibly recursively, the single struct pointed to by offset*/
if((ret=walkStruct(compiler,topvar,basetype,offset))) goto done;
if((ret=walkStruct(resp,topvar,basetype,offset,doswap))) goto done;
}
done:
@ -152,7 +148,7 @@ done:
}
static int
walkStruct(NCD4meta* compiler, NCD4node* topvar, NCD4node* structtype, NCD4offset* offset)
walkStruct(NCD4response* resp, NCD4node* topvar, NCD4node* structtype, NCD4offset* offset, int doswap)
{
int ret = NC_NOERR;
int i;
@ -162,17 +158,17 @@ walkStruct(NCD4meta* compiler, NCD4node* topvar, NCD4node* structtype, NCD4offse
NCD4node* fieldbase = field->basetype;
switch (fieldbase->subsort) {
default:
if((ret=walkAtomicVar(compiler,topvar,field,offset))) goto done;
if((ret=walkAtomicVar(resp,topvar,field,offset,doswap))) goto done;
break;
case NC_OPAQUE:
/* The only thing we need to do is swap the counts */
if((ret=walkOpaqueVar(compiler,topvar,field,offset))) goto done;
if((ret=walkOpaqueVar(resp,topvar,field,offset,doswap))) goto done;
break;
case NC_STRUCT:
if((ret=walkStructArray(compiler,topvar,field,offset))) goto done;
if((ret=walkStructArray(resp,topvar,field,offset,doswap))) goto done;
break;
case NC_SEQ:
if((ret=walkSeqArray(compiler,topvar,field,offset))) goto done;
if((ret=walkSeqArray(resp,topvar,field,offset,doswap))) goto done;
break;
}
}
@ -182,7 +178,7 @@ done:
}
static int
walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset)
walkSeqArray(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap)
{
int ret = NC_NOERR;
d4size_t i;
@ -195,7 +191,7 @@ walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* of
for(i=0;i<dimproduct;i++) {
/* Swap, possibly recursively, the single seq pointed to by offset*/
if((ret=walkSeq(compiler,topvar,seqtype,offset))) goto done;
if((ret=walkSeq(resp,topvar,seqtype,offset,doswap))) goto done;
}
done:
@ -206,7 +202,7 @@ done:
Remember that the base type of var is a vlen.
*/
static int
walkSeq(NCD4meta* compiler, NCD4node* topvar, NCD4node* vlentype, NCD4offset* offset)
walkSeq(NCD4response* resp, NCD4node* topvar, NCD4node* vlentype, NCD4offset* offset, int doswap)
{
int ret = NC_NOERR;
int i;
@ -214,7 +210,7 @@ walkSeq(NCD4meta* compiler, NCD4node* topvar, NCD4node* vlentype, NCD4offset* of
NCD4node* basetype;
/* process the record count */
if(compiler->swap)
if(doswap)
swapinline64(offset);
recordcount = GETCOUNTER(offset);
SKIPCOUNTER(offset);
@ -225,17 +221,17 @@ walkSeq(NCD4meta* compiler, NCD4node* topvar, NCD4node* vlentype, NCD4offset* of
for(i=0;i<recordcount;i++) {
switch(basetype->subsort) {
default: /* atomic basetype */
if((ret=walkAtomicVar(compiler,topvar,basetype,offset))) goto done;
if((ret=walkAtomicVar(resp,topvar,basetype,offset,doswap))) goto done;
break;
case NC_OPAQUE:
if((ret=walkOpaqueVar(compiler,topvar,basetype,offset))) goto done;
if((ret=walkOpaqueVar(resp,topvar,basetype,offset,doswap))) goto done;
break;
case NC_STRUCT:
/* We can treat each record like a structure instance */
if((ret=walkStruct(compiler,topvar,basetype,offset))) goto done;
if((ret=walkStruct(resp,topvar,basetype,offset,doswap))) goto done;
break;
case NC_SEQ:
if((ret=walkSeq(compiler,topvar,basetype,offset))) goto done;
if((ret=walkSeq(resp,topvar,basetype,offset,doswap))) goto done;
break;
}
}

67
libdap4/d4util.c
View File

@ -107,44 +107,45 @@ NCD4_makeFQN(NCD4node* node)
{
char* fqn = NULL;
char* escaped;
int i;
NCD4node* g = node;
NClist* path = nclistnew();
size_t estimate;
NCbytes* buf = ncbytesnew();
NClist* grps = nclistnew();
NClist* parts = nclistnew();
NCD4node* n;
size_t i;
for(estimate=0;g != NULL;g=g->container) {
estimate += strlen(g->name);
nclistinsert(path,0,g);
/* collect all the non-groups */
for(n=node;n;n=n->container) {
if(ISGROUP(n->sort))
nclistinsert(grps,0,n); /* keep the correct order of groups */
else
nclistinsert(parts,0,n);
}
estimate = (estimate*2) + 2*nclistlength(path);
estimate++; /*strlcat nul*/
fqn = (char*)malloc(estimate+1);
if(fqn == NULL) goto done;
fqn[0] = '\0';
/* Create the group-based fqn prefix */
/* start at 1 to avoid dataset */
for(i=1;i<nclistlength(path);i++) {
NCD4node* elem = (NCD4node*)nclistget(path,i);
if(elem->sort != NCD4_GROUP) break;
/* Build grp prefix of the fqn */
for(i=1;i<nclistlength(grps);i++) {
n = (NCD4node*)nclistget(grps,i);
/* Add in the group name */
escaped = backslashEscape(elem->name);
if(escaped == NULL) {free(fqn); fqn = NULL; goto done;}
strlcat(fqn,"/",estimate);
strlcat(fqn,escaped,estimate);
escaped = backslashEscape(n->name);
if(escaped == NULL) goto done;
ncbytescat(buf,"/");
ncbytescat(buf,escaped);
free(escaped);
}
/* Add in the final name part (if not group) */
if(i < nclistlength(path)) {
int last = nclistlength(path)-1;
NCD4node* n = (NCD4node*)nclistget(path,last);
char* name = NCD4_makeName(n,".");
strlcat(fqn,"/",estimate);
strlcat(fqn,name,estimate);
nullfree(name);
for(i=0;i<nclistlength(parts);i++) {
n = (NCD4node*)nclistget(parts,i);
escaped = backslashEscape(n->name);
if(escaped == NULL) goto done;
ncbytescat(buf,(i==0?"/":"."));
ncbytescat(buf,escaped);
free(escaped);
}
fqn = ncbytesextract(buf);
done:
nclistfree(path);
ncbytesfree(buf);
nclistfree(grps);
nclistfree(parts);
return fqn;
}
@ -446,11 +447,9 @@ NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian)
}
void
NCD4_reporterror(NCD4INFO* state)
NCD4_reporterror(NCD4response* resp, NCURI* uri)
{
NCD4meta* meta = state->substrate.metadata;
char* u = NULL;
if(meta == NULL) return;
u = ncuribuild(state->uri,NULL,NULL,NCURIALL);
fprintf(stderr,"***FAIL: url=%s httpcode=%d errmsg->\n%s\n",u,meta->error.httpcode,meta->error.message);
u = ncuribuild(uri,NULL,NULL,NCURIALL);
fprintf(stderr,"***FAIL: url=%s httpcode=%d errmsg->\n%s\n",u,resp->serial.httpcode,resp->error.message);
}

5
libdap4/d4util.h
View File

@ -17,9 +17,12 @@
*/
typedef unsigned long long d4size_t;
/* Define a counted memory marker */
/* Define a (size, memory) pair */
typedef struct D4blob {d4size_t size; void* memory;} D4blob;
/* Empty blob constant */
#define NULLBLOB(blob) {blob.size = 0; blob.memory = NULL;}
#define OFFSET2BLOB(blob,offset) do{(blob).size = ((offset)->limit - (offset)->base); (blob).memory = (offset)->base; }while(0)
#define BLOB2OFFSET(offset,blob) do{\
(offset)->base = (blob).memory; \

211
libdap4/d4varx.c
View File

@ -11,7 +11,8 @@
#include "d4odom.h"
/* Forward */
static int getvarx(int ncid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*);
static int getvarx(int gid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*);
static int mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming);
int
NCD4_get_vara(int ncid, int varid,
@ -22,11 +23,11 @@ NCD4_get_vara(int ncid, int varid,
int ret;
/* TODO: optimize since we know stride is 1 */
ret = NCD4_get_vars(ncid,varid,start,edges,NC_stride_one,value,memtype);
return ret;
return THROW(ret);
}
int
NCD4_get_vars(int ncid, int varid,
NCD4_get_vars(int gid, int varid,
const size_t *start, const size_t *edges, const ptrdiff_t* stride,
void *memoryin, nc_type xtype)
{
@ -43,14 +44,14 @@ NCD4_get_vars(int ncid, int varid,
int rank;
size_t dimsizes[NC_MAX_VAR_DIMS];
d4size_t dimproduct;
size_t dstcount;
size_t dstpos;
NCD4offset* offset = NULL;
/* Get netcdf type info */
if((ret=getvarx(ncid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size)))
/* Get netcdf var metadata and data */
if((ret=getvarx(gid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size)))
{goto done;}
meta = info->substrate.metadata;
meta = info->dmrmetadata;
nctype = ncvar->basetype;
rank = nclistlength(ncvar->dims);
blobs = nclistnew();
@ -74,18 +75,18 @@ NCD4_get_vars(int ncid, int varid,
odom = d4odom_new(rank,start,edges,stride,dimsizes);
else
odom = d4scalarodom_new();
dstcount = 0; /* We always write into dst starting at position 0*/
for(;d4odom_more(odom);dstcount++) {
dstpos = 0; /* We always write into dst starting at position 0*/
for(;d4odom_more(odom);dstpos++) {
void* xpos;
void* dst;
d4size_t count;
d4size_t pos;
count = d4odom_next(odom);
if(count >= dimproduct) {
pos = d4odom_next(odom);
if(pos >= dimproduct) {
ret = THROW(NC_EINVALCOORDS);
goto done;
}
xpos = ((char*)memoryin)+(xsize * dstcount); /* ultimate destination */
xpos = ((char*)memoryin)+(xsize * dstpos); /* ultimate destination */
/* We need to compute the offset in the dap4 data of this instance;
for fixed size types, this is easy, otherwise we have to walk
the variable size type
@ -95,16 +96,16 @@ NCD4_get_vars(int ncid, int varid,
offset = NULL;
offset = BUILDOFFSET(NULL,0);
BLOB2OFFSET(offset,ncvar->data.dap4data);
/* Move offset to the count'th element of the array */
/* Move offset to the pos'th element of the array */
if(nctype->meta.isfixedsize) {
INCR(offset,(dapsize*count));
INCR(offset,(dapsize*pos));
} else {
/* We have to walk to the count'th location in the data */
if((ret=NCD4_moveto(meta,ncvar,count,offset)))
/* We have to walk to the pos'th location in the data */
if((ret=NCD4_moveto(meta,ncvar,pos,offset)))
{goto done;}
}
dst = instance;
if((ret=NCD4_fillinstance(meta,nctype,offset,&dst,blobs)))
if((ret=NCD4_movetoinstance(meta,nctype,offset,&dst,blobs)))
{goto done;}
if(xtype == nc4type) {
/* We can just copy out the data */
@ -132,45 +133,29 @@ done:
}
static int
getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp,
getvarx(int gid, int varid, NCD4INFO** infop, NCD4node** varp,
nc_type* xtypep, size_t* xsizep, nc_type* nc4typep, size_t* nc4sizep)
{
int ret = NC_NOERR;
NC* ncp;
NCD4INFO* info;
NCD4meta* meta;
NCD4node* group;
NCD4node* var;
NCD4node* type;
NC* ncp = NULL;
NCD4INFO* info = NULL;
NCD4meta* dmrmeta = NULL;
NCD4node* group = NULL;
NCD4node* var = NULL;
NCD4node* type = NULL;
nc_type xtype, actualtype;
size_t instancesize, xsize;
NCURI* ceuri = NULL; /* Constrained uri */
NCD4meta* dapmeta = NULL;
NCD4response* dapresp = NULL;
if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR)
if((ret = NC_check_id(gid, (NC**)&ncp)) != NC_NOERR)
goto done;
info = getdap(ncp);
meta = info->substrate.metadata;
/* If the data has not already been read and processed, then do so. */
if(meta->serial.dap == NULL) {
size_t len = 0;
void* content = NULL;
/* (Re)Build the meta data; sets serial.rawdata */
NCD4_resetMeta(info->substrate.metadata);
meta->controller = info;
meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */
if((ret=NCD4_readDAP(info, info->controls.flags.flags))) goto done;
len = ncbyteslength(info->curl->packet);
content = ncbytesextract(info->curl->packet);
NCD4_resetSerial(&meta->serial, len, content);
/* Process the data part */
if((ret=NCD4_dechunk(meta))) goto done;
if((ret = NCD4_processdata(info->substrate.metadata))) goto done;
}
if((ret = NCD4_findvar(ncp,ncid,varid,&var,&group))) goto done;
dmrmeta = info->dmrmetadata;
if((ret = NCD4_findvar(ncp,gid,varid,&var,&group))) goto done;
type = var->basetype;
actualtype = type->meta.id;
instancesize = type->meta.memsize;
@ -189,6 +174,46 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp,
else
xsize = instancesize;
/* If we already have valid data, then just return */
if(var->data.valid) goto validated;
/* Ok, we need to read from the server */
/* Add the variable to the URI, unless the URI is already constrained or is unconstrainable */
ceuri = ncuriclone(info->dmruri);
/* append the request for a specific variable */
if(ncuriquerylookup(ceuri,DAP4CE) == NULL && !FLAGSET(info->controls.flags,NCF_UNCONSTRAINABLE)) {
ncurisetquerykey(ceuri,strdup("dap4.ce"),NCD4_makeFQN(var));
}
/* Read and process the data */
/* Setup the meta-data for the DAP */
if((ret=NCD4_newMeta(info,&dapmeta))) goto done;
if((ret=NCD4_newResponse(info,&dapresp))) goto done;
dapresp->mode = NCD4_DAP;
nclistpush(info->responses,dapresp);
if((ret=NCD4_readDAP(info, info->controls.flags.flags, ceuri, dapresp))) goto done;
/* Extract DMR and dechunk the data part */
if((ret=NCD4_dechunk(dapresp))) goto done;
/* Process the dmr part */
if((ret=NCD4_parse(dapmeta,dapresp,1))) goto done;
/* See if we are checksumming */
if((ret=NCD4_inferChecksums(dapmeta,dapresp))) goto done;
/* connect variables and corresponding dap data */
if((ret = NCD4_parcelvars(dapmeta,dapresp))) goto done;
/* Process checksums and byte-order swapping */
if((ret = NCD4_processdata(dapmeta,dapresp))) goto done;
/* Transfer and process the data */
if((ret = mapvars(dapmeta,dmrmeta,dapresp->inferredchecksumming))) goto done;
validated:
/* Return relevant info */
if(infop) *infop = info;
if(xtypep) *xtypep = xtype;
@ -197,8 +222,96 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp,
if(nc4sizep) *nc4sizep = instancesize;
if(varp) *varp = var;
done:
if(meta->error.message != NULL)
NCD4_reporterror(info); /* Make sure the user sees this */
if(dapmeta) NCD4_reclaimMeta(dapmeta);
ncurifree(ceuri);
if(dapresp != NULL && dapresp->error.message != NULL)
NCD4_reporterror(dapresp,ceuri); /* Make sure the user sees this */
return THROW(ret);
}
#if 0
static NCD4node*
findbyname(const char* name, NClist* nodes)
{
int i;
for(i=0;i<nclistlength(nodes);i++) {
NCD4node* node = (NCD4node*)nclistget(nodes,i);
if(strcmp(name,node->name)==0)
return node;
}
return NULL;
}
#endif
static int
matchvar(NCD4meta* dmrmeta, NCD4node* dapvar, NCD4node** dmrvarp)
{
int i,ret = NC_NOERR;
NCD4node* x = NULL;
NClist* dappath = nclistnew();
NClist* dmrpath = nclistnew(); /* compute path for this dmr var */
int found = 0;
NCD4node* match = NULL;
/* Capture the dap path starting at root and ending at the dapvar (assumed to be topvar) */
for(x=dapvar;x != NULL;x=x->container) nclistinsert(dappath,0,x);
/* Iterate over all variable nodes to find matching one */
for(i=0;i<nclistlength(dmrmeta->allnodes);i++) {
NCD4node* node = (NCD4node*)nclistget(dmrmeta->allnodes,i);
if(ISVAR(node->sort) && strcmp(node->name,dapvar->name)==0) { /* possible candidate */
int j;
found = 0;
nclistclear(dmrpath);
for(x=node;x != NULL;x=x->container) nclistinsert(dmrpath,0,x);
if(nclistlength(dmrpath) == nclistlength(dappath)) { /* same length paths */
/* compare paths: name and sort */
for(found=1,j=0;j<nclistlength(dmrpath);j++) {
NCD4node* pdmr = (NCD4node*)nclistget(dmrpath,j);
NCD4node* pdap = (NCD4node*)nclistget(dappath,j);
if(pdmr->sort != pdap->sort || strcmp(pdmr->name,pdap->name) != 0)
{found = 0; break;}
}
if(found) {match = node; break;}
}
}
}
if(!found) {ret = NC_EINVAL; goto done;}
if(dmrvarp) *dmrvarp = match;
done:
nclistfree(dappath);
nclistfree(dmrpath);
return THROW(ret);
}
/*
Map each toplevel dap var to the corresponding
toplevel dmr var and transfer necessary info;
*/
static int
mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming)
{
int i, ret = NC_NOERR;
NCD4node* daproot = dapmeta->root;
NClist* daptop = NULL; /* top variables in dap tree */
/* Get top level variables from the dap node tree */
daptop = nclistnew();
NCD4_getToplevelVars(dapmeta,daproot,daptop);
/* Match up the dap top variables with the dmr top variables */
for(i=0;i<nclistlength(daptop);i++) {
NCD4node* dapvar = (NCD4node*)nclistget(daptop,i);
NCD4node* dmrvar = NULL;
if((ret=matchvar(dmrmeta,dapvar,&dmrvar))) goto done;
/* Transfer info from dap var to dmr var */
dmrvar->data = dapvar->data;
memset(&dapvar->data,0,sizeof(NCD4vardata));
dmrvar->data.valid = 1;
}
done:
nclistfree(daptop);
return THROW(ret);
}

40
libdap4/ncd4.h
View File

@ -77,7 +77,6 @@ EXTERNL int dsp_open(const char* path, ND4dsp** dspp);
/* From d4http.c */
EXTERNL long NCD4_fetchhttpcode(CURL* curl);
EXTERNL int NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, d4size_t* sizep, long* filetime);
EXTERNL int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcode);
EXTERNL int NCD4_curlopen(CURL** curlp);
EXTERNL void NCD4_curlclose(CURL* curl);
@ -85,15 +84,16 @@ EXTERNL int NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime);
EXTERNL int NCD4_ping(const char* url);
/* From d4read.c */
EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags);
EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags);
EXTERNL int NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg);
EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags, NCURI* url, NCD4response*);
EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags, NCURI* ceuri, NCD4response*);
EXTERNL int NCD4_seterrormessage(NCD4response*, size_t len, char* msg);
/* From d4parser.c */
EXTERNL int NCD4_parse(NCD4meta*);
EXTERNL int NCD4_parse(NCD4meta*, NCD4response*, int dapparse);
EXTERNL NCD4node* NCD4_findAttr(NCD4node* container, const char* attrname);
EXTERNL NCD4node* NCD4_groupFor(NCD4node* node);
EXTERNL int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char* typename, NCD4node** attrp);
EXTERNL void NCD4_setdebuglevel(NCD4parser*,int);
/* From d4printer.c */
EXTERNL int NCD4_print(NCD4meta*, NCbytes* output);
@ -104,29 +104,33 @@ EXTERNL void NCD4_attachraw(NCD4meta*, size_t size, void* rawdata);
EXTERNL void NCD4_reclaimMeta(NCD4meta*);
EXTERNL void NCD4_resetMeta(NCD4meta*);
EXTERNL void reclaimNode(NCD4node* node);
EXTERNL void NCD4_setdebuglevel(NCD4meta*,int);
EXTERNL int NCD4_metabuild(NCD4meta*, int ncid);
EXTERNL size_t NCD4_computeTypeSize(NCD4meta*, NCD4node* type);
EXTERNL int NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp);
EXTERNL char* NCD4_getVarFQN(NCD4node* var, const char* tail);
/* From d4chunk.c */
EXTERNL int NCD4_dechunk(NCD4meta*);
EXTERNL int NCD4_infermode(NCD4meta* meta);
EXTERNL int NCD4_dechunk(NCD4response*);
EXTERNL int NCD4_infermode(NCD4response*);
struct NCD4serial;
EXTERNL void NCD4_resetSerial(struct NCD4serial* serial, size_t rawsize, void* rawdata);
EXTERNL void NCD4_moveSerial(struct NCD4serial* serial, struct NCD4serial* dst);
/* From d4swap.c */
EXTERNL int NCD4_swapdata(NCD4meta*, NClist* topvars);
EXTERNL int NCD4_swapdata(NCD4response*, NCD4node* topvar, int doswap);
/* From d4fix.c */
EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, NCD4offset* offset);
EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, NCD4offset* offset, int inferredchecksumming);
EXTERNL int NCD4_moveto(NCD4meta*, NCD4node* var, d4size_t count, NCD4offset* offset);
EXTERNL int NCD4_toposort(NCD4meta*);
/* From d4data.c */
EXTERNL int NCD4_processdata(NCD4meta*);
EXTERNL int NCD4_fillinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs);
EXTERNL int NCD4_parcelvars(NCD4meta* meta, NCD4response* resp);
EXTERNL int NCD4_processdata(NCD4meta*,NCD4response*);
EXTERNL int NCD4_movetoinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs);
EXTERNL int NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel);
EXTERNL int NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp);
/* From d4util.c */
EXTERNL d4size_t NCD4_dimproduct(NCD4node* node);
@ -141,7 +145,7 @@ EXTERNL char* NCD4_deescape(const char* esc);
EXTERNL char* NCD4_entityescape(const char* s);
EXTERNL size_t NCD4_elidenuls(char* s, size_t slen);
EXTERNL void* NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian);
EXTERNL void NCD4_reporterror(NCD4INFO* state);
EXTERNL void NCD4_reporterror(NCD4response*, NCURI* uri);
/* From d4dump.c */
EXTERNL void NCD4_dumpbytes(size_t size, const void* data0, int swap);
@ -163,7 +167,15 @@ EXTERNL int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char*
/* d4file.c */
EXTERNL int NCD4_readDMRorDAP(NCD4INFO* d4info, NCD4mode mode);
EXTERNL void NCD4_applyclientfragmentcontrols(NCD4INFO* d4info);
EXTERNL void NCD4_applyclientquerycontrols(NCD4INFO* d4info);
EXTERNL void NCD4_applychecksumcontrols(NCD4INFO* d4info, NCD4response*);
EXTERNL int NCD4_newInfo(NCD4INFO** d4infop);
EXTERNL void NCD4_reclaimInfo(NCD4INFO* d4info);
EXTERNL void NCD4_resetInfoforRead(NCD4INFO* d4info);
EXTERNL int NCD4_newResponse(NCD4INFO*,NCD4response** respp);
EXTERNL void NCD4_reclaimResponse(NCD4response* d4resp);
EXTERNL void NCD4_resetInfoForRead(NCD4INFO* d4info);
EXTERNL int NCD4_newMeta(NCD4INFO*,NCD4meta**);
EXTERNL void NCD4_reclaimMeta(NCD4meta*);
/* ncd4dispatch.c */
struct NC_reservedatt; /*forward*/

15
libdap4/ncd4dispatch.c
View File

@ -388,11 +388,11 @@ NCD4_inq_attname(int ncid, int varid, int attnum, char* name)
const NC_reservedatt* rsvp = NULL;
if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) return (ret);
substrateid = makenc4id(ncp,ncid);
ret = nc_inq_attname(substrateid, varid, attnum, name);
/* Is this a reserved attribute name? */
if(name && (rsvp = NCD4_lookupreserved(name)))
return NC_EATTMETA;
substrateid = makenc4id(ncp,ncid);
ret = nc_inq_attname(substrateid, varid, attnum, name);
return (ret);
}
@ -847,7 +847,7 @@ NCD4_inq_dim(int ncid, int dimid, char* name, size_t* lenp)
if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR)
goto done;
info = (NCD4INFO*)ncp->dispatchdata;
meta = info->substrate.metadata;
meta = info->dmrmetadata;
/* Locate the dimension specified by dimid */
for(i=0;i<nclistlength(meta->allnodes);i++) {
@ -871,16 +871,15 @@ static int
ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* value, nc_type t, const NC_reservedatt* rsvp)
{
int ret = NC_NOERR;
NCD4INFO* info = (NCD4INFO*)(ncp->dispatchdata);
NCD4meta* meta = info->substrate.metadata;
NCD4node* var = NULL;
if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done;
if(strcmp(rsvp->name,D4CHECKSUMATTR)==0) {
unsigned int* ip = (unsigned int*)value;
if(varid == NC_GLOBAL)
{ret = NC_EBADID; goto done;}
if(t != NC_UINT) {ret = NC_EBADTYPE; goto done;}
if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done;
if(var->data.checksumattr == 0)
{ret = NC_ENOTATT; goto done;}
*ip = (var->data.remotechecksum);
@ -889,7 +888,7 @@ ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* valu
if(varid != NC_GLOBAL)
{ret = NC_EBADID; goto done;}
if(t != NC_INT) {ret = NC_EBADTYPE; goto done;}
*ip = (meta->serial.remotelittleendian?1:0);
*ip = (var->data.response->remotelittleendian?1:0);
}
done:
return THROW(ret);
@ -925,7 +924,7 @@ static int
globalinit(void)
{
int stat = NC_NOERR;
return stat;
return THROW(stat);
}
/**************************************************/

96
libdap4/ncd4types.h
View File

@ -50,6 +50,8 @@ typedef struct NCD4node NCD4node;
typedef struct NCD4params NCD4params;
typedef struct NCD4HDR NCD4HDR;
typedef struct NCD4offset NCD4offset;
typedef struct NCD4vardata NCD4vardata;
typedef struct NCD4response NCD4response;
/* Define the NCD4HDR flags */
/* Header flags */
@ -228,19 +230,20 @@ struct NCD4node {
int isfixedsize; /* sort == NCD4_TYPE; Is this a fixed size (recursively) type? */
d4size_t dapsize; /* size of the type as stored in the dap data; will, as a rule,
be same as memsize only for types <= NC_UINT64 */
nc_type cmpdid; /*netcdf id for the compound type created for seq type */
nc_type cmpdid; /* netcdf id for the compound type created for seq type */
size_t memsize; /* size of a memory instance without taking dimproduct into account,
but taking compound alignment into account */
d4size_t offset; /* computed structure field offset in memory */
size_t alignment; /* computed structure field alignment in memory */
} meta;
struct { /* Data compilation info */
int flags; /* See d4data for actual flags */
struct NCD4vardata { /* Data compilation info */
int valid; /* 1 => this contains valid data */
D4blob dap4data; /* offset and start pos for this var's data in serialization */
unsigned int remotechecksum; /* checksum from data as sent by server */
unsigned int localchecksum; /* toplevel variable checksum as computed by client */
int checksumattr; /* 1=> _DAP4_Checksum_CRC32 is defined */
int attrchecksum; /* _DAP4_Checksum_CRC32 value */
unsigned remotechecksum; /* toplevel per-variable checksum contained in the data */
unsigned localchecksum; /* toplevel variable checksum as computed by client */
int checksumattr; /* 1 => _DAP4_Checksum_CRC32 is defined */
unsigned attrchecksum; /* _DAP4_Checksum_CRC32 value; this is the checksum computed by server */
NCD4response* response; /* Response from which this data is taken */
} data;
struct { /* Track netcdf-4 conversion info */
int isvlen; /* _edu.ucar.isvlen */
@ -253,36 +256,13 @@ struct NCD4node {
} nc4;
};
/* Tracking info about the serialized input before and after de-chunking */
typedef struct NCD4serial {
size_t rawsize; /* |rawdata| */
void* rawdata;
size_t dapsize; /* |dap|; this is transient */
void* dap; /* pointer into rawdata where dap data starts */
char* dmr;/* copy of dmr */
char* errdata; /* null || error chunk (null terminated) */
int httpcode; /* returned from last request */
int hostlittleendian; /* 1 if the host is little endian */
int remotelittleendian; /* 1 if the packet says data is little endian */
} NCD4serial;
/* This will be passed out of the parse */
/* DMR information from a response; this will be passed out of the parse */
struct NCD4meta {
NCD4INFO* controller;
int ncid; /* root ncid of the substrate netcdf-4 file;
warning: copy of NCD4Info.substrate.nc4id */
NCD4node* root;
NCD4mode mode; /* Are we reading DMR (only) or DAP (includes DMR) */
NClist* allnodes; /*list<NCD4node>*/
struct Error { /* Content of any error response */
char* parseerror;
int httpcode;
char* message;
char* context;
char* otherinfo;
} error;
int debuglevel;
NCD4serial serial;
int swap; /* 1 => swap data */
/* Define some "global" (to a DMR) data */
NClist* groupbyid; /* NClist<NCD4node*> indexed by groupid >> 16; this is global */
@ -292,9 +272,12 @@ struct NCD4meta {
};
typedef struct NCD4parser {
NCD4INFO* controller;
char* input;
int debuglevel;
int dapparse; /* 1 => we are parsing the DAP DMR */
NCD4meta* metadata;
NCD4response* response;
/* Capture useful subsets of dataset->allnodes */
NClist* types; /*list<NCD4node>; user-defined types only*/
NClist* dims; /*list<NCD4node>*/
@ -303,6 +286,32 @@ typedef struct NCD4parser {
NCD4node* dapopaque; /* Single non-fixed-size opaque type */
} NCD4parser;
/* Capture all the relevant info about the response to a server request */
struct NCD4response { /* possibly processed response from a query */
NCD4INFO* controller; /* controlling connection */
D4blob raw; /* complete response in memory */
int querychecksumming; /* 1 => user specified dap4.ce value */
int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */
int inferredchecksumming; /* 1 => either query checksum || att checksum */
int checksumignore; /* 1 => assume checksum, but do not validate */
int remotelittleendian; /* 1 if the packet says data is little endian */
NCD4mode mode; /* Are we reading DMR (only) or DAP (includes DMR) */
struct NCD4serial {
size_t dapsize; /* |dap|; this is transient */
void* dap; /* pointer into raw where dap data starts */
char* dmr;/* copy of dmr */
char* errdata; /* null || error chunk (null terminated) */
int httpcode; /* returned from last request */
} serial; /* Dechunked and processed DAP part of the response */
struct Error { /* Content of any error response */
char* parseerror;
int httpcode;
char* message;
char* context;
char* otherinfo;
} error;
};
/**************************************************/
/* Curl info */
@ -328,28 +337,20 @@ struct NCD4curl {
struct NCD4INFO {
NC* controller; /* Parent instance of NCD4INFO */
char* rawurltext; /* as given to ncd4_open */
char* urltext; /* as modified by ncd4_open */
NCURI* uri; /* parse of rawuritext */
char* rawdmrurltext; /* as given to ncd4_open */
char* dmrurltext; /* as modified by ncd4_open */
NCURI* dmruri; /* parse of rawuritext */
NCD4curl* curl;
int inmemory; /* store fetched data in memory? */
struct {
char* memory; /* allocated memory if ONDISK is not set */
char* ondiskfilename; /* If ONDISK is set */
FILE* ondiskfile; /* ditto */
d4size_t datasize; /* size on disk or in memory */
long dmrlastmodified;
long daplastmodified;
int querychecksumming; /* 1 => user specified dap4.ce value */
int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */
int inferredchecksumming; /* 1 => either query checksum || att checksum */
int checksumignore; /* 1 => assume checksum, but do not validate */
} data;
NCD4meta* dmrmetadata; /* Independent of responses */
NClist* responses; /* NClist<NCD4response> all responses from this curl handle */
struct { /* Properties that are per-platform */
int hostlittleendian; /* 1 if the host is little endian */
} platform;
struct {
int realfile; /* 1 => we created actual temp file */
char* filename; /* of the substrate file */
int nc4id; /* substrate nc4 file ncid used to hold metadata; not same as external id */
NCD4meta* metadata;
} substrate;
struct {
NCCONTROLS flags;
@ -362,6 +363,7 @@ struct NCD4INFO {
struct {
char* filename;
} fileproto;
int debuglevel;
NClist* blobs;
};

97
libdispatch/dinfermodel.c
View File

@ -205,7 +205,7 @@ static struct NCPROTOCOLLIST {
static int NC_omodeinfer(int useparallel, int omode, NCmodel*);
static int check_file_type(const char *path, int omode, int use_parallel, void *parameters, NCmodel* model, NCURI* uri);
static int processuri(const char* path, NCURI** urip, NClist* fraglist);
static int processmacros(NClist** fraglistp);
static int processmacros(NClist* fraglistp, NClist* expanded);
static char* envvlist2string(NClist* pairs, const char*);
static void set_default_mode(int* cmodep);
static int parseonchar(const char* s, int ch, NClist* segments);
@ -242,8 +242,8 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv)
NCURI* uri = NULL;
size_t pathlen = strlen(path);
char* str = NULL;
const char** ufrags;
const char** p;
const NClist* ufrags;
size_t i;
if(path == NULL || pathlen == 0) {stat = NC_EURL; goto done;}
@ -265,7 +265,6 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv)
/* process the corresponding fragments for that protocol */
if(protolist->fragments != NULL) {
int i;
tmp = nclistnew();
if((stat = parseonchar(protolist->fragments,'&',tmp))) goto done;
for(i=0;i<nclistlength(tmp);i++) {
@ -283,15 +282,13 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv)
if(protolist->substitute) ncurisetprotocol(uri,protolist->substitute);
/* capture the fragments of the url */
ufrags = ncurifragmentparams(uri);
if(ufrags != NULL) {
for(p=ufrags;*p;p+=2) {
const char* key = p[0];
const char* value = p[1];
nclistpush(fraglenv,nulldup(key));
value = (value==NULL?"":value);
nclistpush(fraglenv,strdup(value));
}
ufrags = (const NClist*)ncurifragmentparams(uri);
for(i=0;i<nclistlength(ufrags);i+=2) {
const char* key = nclistget(ufrags,i);
const char* value = nclistget(ufrags,i+1);
nclistpush(fraglenv,nulldup(key));
value = (value==NULL?"":value);
nclistpush(fraglenv,strdup(value));
}
if(urip) {
*urip = uri;
@ -442,22 +439,15 @@ processmodearg(const char* arg, NCmodel* model)
/* Given an envv fragment list, do macro replacement */
static int
processmacros(NClist** fraglenvp)
processmacros(NClist* fraglenv, NClist* expanded)
{
int stat = NC_NOERR;
int i, stat = NC_NOERR;
const struct MACRODEF* macros = NULL;
NClist* fraglenv = NULL;
NClist* expanded = NULL;
if(fraglenvp == NULL || nclistlength(*fraglenvp) == 0) goto done;
fraglenv = *fraglenvp;
expanded = nclistnew();
while(nclistlength(fraglenv) > 0) {
for(i=0;i<nclistlength(fraglenv);i+=2) {
int found = 0;
char* key = NULL;
char* value = NULL;
key = nclistremove(fraglenv,0); /* remove from changing front */
value = nclistremove(fraglenv,0); /* remove from changing front */
char* key = nclistget(fraglenv,i);
char* value = nclistget(fraglenv,i+1);
if(strlen(value) == 0) { /* must be a singleton */
for(macros=macrodefs;macros->name;macros++) {
if(strcmp(macros->name,key)==0) {
@ -474,14 +464,8 @@ processmacros(NClist** fraglenvp)
nclistpush(expanded,strdup(key));
nclistpush(expanded,strdup(value));
}
nullfree(key);
nullfree(value);
}
*fraglenvp = expanded; expanded = NULL;
done:
nclistfreeall(expanded);
nclistfreeall(fraglenv);
return check(stat);
}
@ -497,8 +481,6 @@ processinferences(NClist* fraglenv)
int i;
char* newmodeval = NULL;
if(fraglenv == NULL || nclistlength(fraglenv) == 0) goto done;
/* Get "mode" entry */
if((modeval = getmodekey(fraglenv))==NULL) goto done;
@ -536,7 +518,7 @@ printlist(newmodes,"processinferences: new mode list");
nextmodes = tmp;
tmp = NULL;
}
/* cleanup any unused elements in currenmodes */
/* cleanup any unused elements in currentmodes */
nclistclearall(currentmodes);
/* Ensure no duplicates */
@ -691,21 +673,15 @@ collectallkeys(NClist* fraglenv, NClist* allkeys)
/* Given a fragment envv list, coalesce duplicate keys and remove duplicate values*/
static int
cleanfragments(NClist** fraglenvp)
cleanfragments(NClist* fraglenv, NClist* newlist)
{
int i,stat = NC_NOERR;
NClist* fraglenv = NULL;
NClist* tmp = NULL;
NClist* allkeys = NULL;
NClist* newlist = NULL;
NCbytes* buf = NULL;
char* key = NULL;
char* value = NULL;
if(fraglenvp == NULL || nclistlength(*fraglenvp) == 0) return NC_NOERR;
fraglenv = *fraglenvp; /* take control of this list */
*fraglenvp = NULL;
newlist = nclistnew();
buf = ncbytesnew();
allkeys = nclistnew();
tmp = nclistnew();
@ -725,13 +701,10 @@ cleanfragments(NClist** fraglenvp)
nclistpush(newlist,value);
nclistclear(tmp);
}
*fraglenvp = newlist; newlist = NULL;
done:
nclistfree(allkeys);
nclistfree(tmp);
ncbytesfree(buf);
nclistfreeall(fraglenv);
nclistfreeall(newlist);
return check(stat);
}
@ -871,7 +844,8 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
char* sfrag = NULL;
const char* modeval = NULL;
char* abspath = NULL;
NClist* tmp = NULL;
/* Phase 1:
1. convert special protocols to http|https
2. begin collecting fragments
@ -884,13 +858,20 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
#endif
/* Phase 2: Expand macros and add to fraglenv */
if((stat = processmacros(&fraglenv))) goto done;
nclistfreeall(tmp);
tmp = nclistnew();
if((stat = processmacros(fraglenv,tmp))) goto done;
nclistfreeall(fraglenv);
fraglenv = tmp; tmp = NULL;
#ifdef DEBUG
printlist(fraglenv,"processmacros");
#endif
/* Cleanup the fragment list */
if((stat = cleanfragments(&fraglenv))) goto done;
nclistfreeall(tmp);
tmp = nclistnew();
if((stat = cleanfragments(fraglenv,tmp))) goto done;
nclistfreeall(fraglenv);
fraglenv = tmp; tmp = NULL;
/* Phase 2a: Expand mode inferences and add to fraglenv */
if((stat = processinferences(fraglenv))) goto done;
@ -899,7 +880,11 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
#endif
/* Phase 3: coalesce duplicate fragment keys and remove duplicate values */
if((stat = cleanfragments(&fraglenv))) goto done;
nclistfreeall(tmp);
tmp = nclistnew();
if((stat = cleanfragments(fraglenv,tmp))) goto done;
nclistfreeall(fraglenv);
fraglenv = tmp; tmp = NULL;
#ifdef DEBUG
printlist(fraglenv,"cleanfragments");
#endif
@ -949,13 +934,12 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
/* Phase 6: Process the non-mode keys to see if we can tell the formatx */
if(!modelcomplete(model)) {
const char** p = ncurifragmentparams(uri); /* envv format */
if(p != NULL) {
for(;*p;p+=2) {
const char* key = p[0];
const char* value = p[1];;
if((stat=processfragmentkeys(key,value,model))) goto done;
}
size_t i;
NClist* p = (NClist*)ncurifragmentparams(uri); /* envv format */
for(i=0;i<nclistlength(p);i+=2) {
const char* key = nclistget(p,0);
const char* value = nclistget(p,1);
if((stat=processfragmentkeys(key,value,model))) goto done;
}
}
@ -1036,6 +1020,7 @@ done:
ncurifree(uri);
nclistfreeall(modeargs);
nclistfreeall(fraglenv);
nclistfreeall(tmp);
*omodep = omode; /* in/out */
return check(stat);
}

4
libdispatch/ncexhash.c
View File

@ -111,7 +111,7 @@ ncexinit(void)
int i;
bitmasks[0] = 0;
for(i=1;i<NCEXHASHKEYBITS;i++)
bitmasks[i] = (1 << i) - 1;
bitmasks[i] = (1ULL << i) - 1;
ncexinitialized = 1;
}
@ -855,7 +855,7 @@ ncexhashprintstats(NCexhashmap* map)
fprintf(stderr," |leaf|=%d nactive/nleaves=%g", map->leaflen, leafavg);
fprintf(stderr," load=%g",leafload);
fprintf(stderr,"]\n");
dirsize = (1<<(map->depth))*((unsigned long long)sizeof(void*));
dirsize = (1ULL<<(map->depth))*((unsigned long long)sizeof(void*));
leafsize = (nleaves)*((unsigned long long)sizeof(NCexleaf));
total = dirsize + leafsize;
fprintf(stderr,"\tsizeof(directory)=%llu sizeof(leaves)=%lld total=%lld\n",

6
libdispatch/nclistmgr.c
View File

@ -81,8 +81,8 @@ free_NCList(void)
int
add_to_NCList(NC* ncp)
{
int i;
int new_id;
unsigned int i;
unsigned int new_id;
if(nc_filelist == NULL) {
if (!(nc_filelist = calloc(1, sizeof(NC*)*NCFILELISTLENGTH)))
return NC_ENOMEM;
@ -96,7 +96,7 @@ add_to_NCList(NC* ncp)
if(new_id == 0) return NC_ENOMEM; /* no more slots */
nc_filelist[new_id] = ncp;
numfiles++;
ncp->ext_ncid = (new_id << ID_SHIFT);
ncp->ext_ncid = (int)(new_id << ID_SHIFT);
return NC_NOERR;
}

32
libdispatch/nclog.c
View File

@ -21,10 +21,6 @@
#include <unistd.h>
#endif
#ifdef HAVE_EXECINFO_H
#include <execinfo.h>
#endif
#include "netcdf.h"
#include "nclog.h"
@ -278,10 +274,6 @@ ncuntrace(const char* fcn, int err, const char* fmt, ...)
vfprintf(nclog_global.nclogstream, fmt, args);
fprintf(nclog_global.nclogstream, "\n" );
fflush(nclog_global.nclogstream);
#ifdef HAVE_EXECINFO_H
if(err != 0)
ncbacktrace();
#endif
}
done:
va_end(args);
@ -304,28 +296,4 @@ ncbreakpoint(int err)
return err;
}
#ifdef HAVE_EXECINFO_H
#define MAXSTACKDEPTH 100
void
ncbacktrace(void)
{
int j, nptrs;
void* buffer[MAXSTACKDEPTH];
char **strings;
if(getenv("NCBACKTRACE") == NULL) return;
nptrs = backtrace(buffer, MAXSTACKDEPTH);
strings = backtrace_symbols(buffer, nptrs);
if (strings == NULL) {
perror("backtrace_symbols");
errno = 0;
return;
}
fprintf(stderr,"Backtrace:\n");
for(j = 0; j < nptrs; j++)
fprintf(stderr,"%s\n", strings[j]);
free(strings);
}
#endif
/**@}*/

285
libdispatch/ncuri.c
View File

@ -95,16 +95,13 @@ ncstrndup(const char* s, size_t len)
/* Forward */
static int collectprefixparams(char* text, char** nextp);
static void freestringlist(NClist* list);
static void freestringvec(char** list);
static int ncfind(char** params, const char* key);
static int ncfind(NClist* params, const char* key);
static char* nclocate(char* p, const char* charlist);
static int parselist(const char* ptext, NClist* list);
static int unparselist(const char** vec, const char* prefix, int encode, char** svecp);
static int unparselist(const NClist* vec, const char* prefix, int encode, NCbytes*);
static int ensurefraglist(NCURI* uri);
static int ensurequerylist(NCURI* uri);
static void buildlist(const char** list, int encode, NCbytes* buf);
static void removedups(NClist* list);
static int extendenvv(char*** envvp, int amount, int* oldlen);
/**************************************************/
/*
@ -147,9 +144,6 @@ ncuriparse(const char* uri0, NCURI** durip)
size_t len0;
int pathchar;
tmp.fraglist = NULL;
tmp.querylist = NULL;
if(uri0 == NULL)
{THROW(NC_EURL);}
@ -227,19 +221,15 @@ ncuriparse(const char* uri0, NCURI** durip)
}
/* Remove duplicates */
removedups(params);
if(nclistlength(params) > 0) {
nclistpush(params,NULL);
tmp.fraglist = nclistextract(params);
} else
tmp.fraglist = NULL;
tmp.fraglist = params;
params = NULL;
/* Parse the query */
if(tmp.query != NULL) {
if(parselist(tmp.query,querylist) != NC_NOERR)
{THROW(NC_EURL);}
if(nclistlength(querylist) > 0) {
nclistpush(querylist,NULL);
tmp.querylist = nclistextract(querylist);
}
tmp.querylist = querylist;
querylist = NULL;
}
/* Now parse the core of the url */
@ -312,7 +302,7 @@ ncuriparse(const char* uri0, NCURI** durip)
pathchar = EOFCHAR;
} else { /* assume there should be a host section */
/* We already extracted the query and/or fragment sections above,
splocate the end of the host section and therefore the start
so locate the end of the host section and therefore the start
of the path.
*/
tmp.host = p;
@ -430,9 +420,9 @@ done:
freestringlist(params);
freestringlist(querylist);
if(tmp.fraglist)
freestringvec(tmp.fraglist);
nclistfreeall(tmp.fraglist);
if(tmp.querylist)
freestringvec(tmp.querylist);
nclistfreeall(tmp.querylist);
return ret;
}
@ -450,6 +440,7 @@ freestringlist(NClist* list)
}
}
#if 0
static void
freestringvec(char** list)
{
@ -459,6 +450,7 @@ freestringvec(char** list)
nullfree(list);
}
}
#endif
void
ncurifree(NCURI* duri)
@ -473,8 +465,8 @@ ncurifree(NCURI* duri)
nullfree(duri->path);
nullfree(duri->query);
nullfree(duri->fragment);
freestringvec(duri->querylist);
freestringvec(duri->fraglist);
nclistfreeall(duri->querylist);
nclistfreeall(duri->fraglist);
free(duri);
}
@ -510,21 +502,14 @@ int
ncurisetquery(NCURI* duri,const char* query)
{
int ret = NC_NOERR;
freestringvec(duri->querylist);
nclistfreeall((NClist*)duri->querylist);
nullfree(duri->query);
duri->query = NULL;
duri->querylist = NULL;
if(query != NULL && strlen(query) > 0) {
NClist* params = nclistnew();
duri->query = strdup(query);
ret = parselist(duri->query,params);
if(ret != NC_NOERR)
{THROW(NC_EURL);}
nclistpush(params,NULL);
duri->querylist = nclistextract(params);
nclistfree(params);
duri->query = strdup(query);
ensurequerylist(duri);
}
done:
return ret;
}
@ -533,12 +518,13 @@ int
ncurisetfragments(NCURI* duri,const char* fragments)
{
int ret = NC_NOERR;
freestringvec(duri->fraglist);
nclistfreeall((NClist*)duri->fraglist);
nullfree(duri->fragment);
duri->fragment = NULL;
duri->fraglist = NULL;
if(fragments != NULL && strlen(fragments) > 0) {
duri->fragment = strdup(fragments);
ensurefraglist(duri);
}
return ret;
}
@ -559,17 +545,20 @@ ncurisetfragmentkey(NCURI* duri,const char* key, const char* value)
{
int ret = NC_NOERR;
int pos = -1;
char* newlist = NULL;
ensurefraglist(duri);
pos = ncfind(duri->fraglist, key);
if(pos < 0) return NC_EINVAL; /* does not exist */
nullfree(duri->fraglist[pos+1]);
duri->fraglist[pos+1] = strdup(value);
if(pos < 0) { /* does not exist */
if(duri->fraglist == NULL) duri->fraglist = nclistnew();
nclistpush(duri->fraglist,key);
nclistpush(duri->fraglist,value);
} else {
nullfree(nclistget(duri->fraglist,pos+1));
nclistset(duri->fraglist,pos+1,strdup(value));
}
/* Rebuild the fragment */
if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done;
nullfree(duri->fragment);
duri->fragment = newlist; newlist = NULL;
nullfree(duri->fragment); duri->fragment = NULL;
if((ret = ensurefraglist(duri))) goto done;
done:
return ret;
}
@ -579,25 +568,67 @@ int
ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value)
{
int ret = NC_NOERR;
int len;
int pos = -1;
char* newlist = NULL;
ensurefraglist(duri);
pos = ncfind(duri->fraglist, key);
if(pos < 0) { /* does not exist */
if((ret = extendenvv(&duri->fraglist,2,&len))) goto done;
duri->fraglist[len] = strdup(key);
duri->fraglist[len+1] = nulldup(value);
duri->fraglist[len+2] = NULL;
nclistpush((NClist*)duri->fraglist,strdup(key));
nclistpush((NClist*)duri->fraglist,nulldup(value));
} else {
nullfree(duri->fraglist[pos+1]);
duri->fraglist[pos+1] = strdup(value);
nullfree(nclistget(duri->fraglist,pos+1));
nclistset(duri->fraglist,pos+1,nulldup(value));
}
/* Rebuild the fragment */
if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done;
nullfree(duri->fragment);
duri->fragment = newlist; newlist = NULL;
nullfree(duri->fraglist); duri->fraglist = NULL;
if((ret=ensurefraglist(duri))) goto done;
done:
return ret;
}
/* Replace a specific query key*/
int
ncurisetquerykey(NCURI* duri,const char* key, const char* value)
{
int ret = NC_NOERR;
int pos = -1;
ensurequerylist(duri);
pos = ncfind(duri->querylist, key);
if(pos < 0) { /* does not exist */
if(duri->querylist == NULL) duri->querylist = nclistnew();
nclistpush(duri->querylist,key);
nclistpush(duri->querylist,value);
} else {
nullfree(nclistget(duri->querylist,pos+1));
nclistset(duri->querylist,pos+1,strdup(value));
}
/* Rebuild the query */
nullfree(duri->query); duri->query = NULL;
if((ret = ensurequerylist(duri))) goto done;
done:
return ret;
}
/* Replace or add a specific query key*/
int
ncuriappendquerykey(NCURI* duri,const char* key, const char* value)
{
int ret = NC_NOERR;
int pos = -1;
ensurequerylist(duri);
pos = ncfind(duri->querylist, key);
if(pos < 0) { /* does not exist */
nclistpush((NClist*)duri->querylist,strdup(key));
nclistpush((NClist*)duri->querylist,nulldup(value));
} else {
nullfree(nclistget(duri->querylist,pos+1));
nclistset(duri->querylist,pos+1,nulldup(value));
}
/* Rebuild the query */
nullfree(duri->querylist); duri->querylist = NULL;
if((ret=ensurequerylist(duri))) goto done;
done:
return ret;
}
@ -734,11 +765,10 @@ ncurifragmentlookup(NCURI* uri, const char* key)
int i;
char* value = NULL;
if(uri == NULL || key == NULL) return NULL;
ensurefraglist(uri);
if(ensurefraglist(uri)) return NULL;
i = ncfind(uri->fraglist,key);
if(i < 0)
return NULL;
value = uri->fraglist[(2*i)+1];
if(i < 0) return NULL;
value = nclistget(uri->fraglist,i+1);
return value;
}
@ -747,20 +777,21 @@ ncuriquerylookup(NCURI* uri, const char* key)
{
int i;
char* value = NULL;
if(uri == NULL || key == NULL || uri->querylist == NULL) return NULL;
if(uri == NULL || key == NULL) return NULL;
if(ensurequerylist(uri)) return NULL;
i = ncfind(uri->querylist,key);
if(i < 0)
return NULL;
value = uri->querylist[(2*i)+1];
if(i < 0) return NULL;
value = nclistget(uri->querylist,i+1);
return value;
}
#if 0
/* Obtain the complete list of fragment pairs in envv format */
const char**
ncurifragmentparams(NCURI* uri)
{
ensurefraglist(uri);
return (const char**)uri->fraglist;
return (const char**)nclistcontents(uri->fraglist;
}
/* Obtain the complete list of query pairs in envv format */
@ -771,7 +802,6 @@ ncuriqueryparams(NCURI* uri)
return (const char**)uri->querylist;
}
#if 0
int
ncuriremoveparam(NCURI* uri, const char* key)
{
@ -796,19 +826,18 @@ ncuriremoveparam(NCURI* uri, const char* key)
case insensitive
*/
static int
ncfind(char** params, const char* key)
ncfind(NClist* params, const char* key)
{
int i;
char** p;
if(key == NULL) return -1;
if(params == NULL) return -1;
for(i=0,p=params;*p;p+=2,i++) {
if(strcasecmp(key,*p)==0) return i;
for(i=0;i<nclistlength(params);i+=2) {
char* p=nclistget(params,i);
if(strcasecmp(key,p)==0) return i;
}
return -1;
}
#if 0
static void
ncparamfree(char** params)
@ -1134,36 +1163,35 @@ parselist(const char* text, NClist* list)
}
static int
unparselist(const char** vec, const char* prefix, int encode, char** svecp)
unparselist(const NClist* vec, const char* prefix, int encode, NCbytes* buf)
{
int stat = NC_NOERR;
NCbytes* buf = ncbytesnew();
const char** p;
size_t i;
int first = 1;
if(vec == NULL || vec[0] == NULL) goto done;
if(nclistlength(vec) == 0) goto done;
if(prefix != NULL) ncbytescat(buf,prefix);
for(p=vec;*p;p+=2,first=0) {
for(i=0;i<nclistlength(vec);i+=2,first=0) {
const char* p0 = (const char*)nclistget(vec,i);
const char* p1 = (const char*)nclistget(vec,i+1);
if(!first) ncbytescat(buf,"&");
if(encode) {
char* encoded = ncuriencodeonly(p[0],queryallow);
char* encoded = ncuriencodeonly(p0,queryallow);
ncbytescat(buf,encoded);
nullfree(encoded);
} else
ncbytescat(buf,p[0]);
if(p[1] != NULL && strlen(p[1]) > 0) {
ncbytescat(buf,p0);
if(p1 != NULL && strlen(p1) > 0) {
ncbytescat(buf,"=");
if(encode) {
char* encoded = ncuriencodeonly(p[1],queryallow);
char* encoded = ncuriencodeonly(p1,queryallow);
ncbytescat(buf,encoded);
nullfree(encoded);
} else
ncbytescat(buf,p[1]);
ncbytescat(buf,p1);
}
}
if(svecp) {*svecp = ncbytesextract(buf);}
done:
ncbytesfree(buf);
return stat;
}
@ -1171,24 +1199,32 @@ static int
ensurefraglist(NCURI* uri)
{
int stat = NC_NOERR;
int nofrag = 0;
int nolist = 0;
NClist* fraglist = NULL;
int hastext = 0;
int haslist = 0;
NClist* fraglist = nclistnew();
NCbytes* frag = NULL;
if(uri->fragment == NULL || strlen(uri->fragment) == 0)
{nullfree(uri->fragment); uri->fragment = NULL; nofrag=1;}
if(uri->fraglist == NULL) nolist = 1;
if(nolist && !nofrag) {
fraglist = nclistnew();
if(nulllen(uri->fragment) == 0)
{nullfree(uri->fragment); uri->fragment = NULL; hastext=0;}
else hastext = 1;
if(nclistlength((NClist*)uri->fraglist) == 0)
{nclistfree((NClist*)uri->fraglist); uri->fraglist = NULL; haslist=0;}
else haslist = 1;
/* Four cases: */
if(!haslist && !hastext) {
/* do nothing */
} else if(!haslist && hastext) {
if((stat = parselist(uri->fragment,fraglist))) goto done;
removedups(fraglist);
uri->fraglist = nclistextract(fraglist);
} else if(!nolist && nofrag) {
uri->fraglist = fraglist; fraglist = NULL;
} else if(haslist && !hastext) {
/* Create the fragment string from fraglist */
frag = ncbytesnew();
buildlist((const char**)uri->fraglist,0,frag); /* do not encode */
if((stat=unparselist((const NClist*)uri->fraglist,NULL,0,frag))) goto done; /* do not encode */
uri->fragment = ncbytesextract(frag);
} else if(haslist && hastext) {
/* assume already consistent */
}
done:
@ -1201,24 +1237,32 @@ static int
ensurequerylist(NCURI* uri)
{
int stat = NC_NOERR;
int noquery = 0;
int nolist = 0;
NClist* querylist = NULL;
int hastext = 0;
int haslist = 0;
NClist* querylist = nclistnew();
NCbytes* query = NULL;
if(uri->query == NULL || strlen(uri->query) == 0)
{nullfree(uri->query); uri->query = NULL; noquery=1;}
if(uri->querylist == NULL) nolist = 1;
if(nolist && !noquery) {
querylist = nclistnew();
if(nulllen(uri->query) == 0)
{nullfree(uri->query); uri->query = NULL; hastext=0;}
else hastext = 1;
if(nclistlength((NClist*)uri->querylist) == 0)
{nclistfree((NClist*)uri->querylist); uri->querylist = NULL; haslist=0;}
else haslist = 1;
/* Four cases: */
if(!haslist && !hastext) {
/* do nothing */
} else if(!haslist && hastext) {
if((stat = parselist(uri->query,querylist))) goto done;
removedups(querylist);
uri->querylist = nclistextract(querylist);
} else if(!nolist && noquery) {
uri->querylist = querylist; querylist = NULL;
} else if(haslist && !hastext) {
/* Create the query string from querylist */
query = ncbytesnew();
buildlist((const char**)uri->querylist,0,query); /* do not encode */
if((stat=unparselist((const NClist*)uri->querylist,NULL,0,query))) goto done; /* do not encode */
uri->query = ncbytesextract(query);
} else if(haslist && hastext) {
/* assume consistent */
}
done:
@ -1242,30 +1286,9 @@ removedups(NClist* list)
}
}
}
/* NULL terminate the list */
nclistpush(list,NULL);
}
static void
buildlist(const char** list, int encode, NCbytes* buf)
{
const char** p;
int first = 1;
for(p=list;*p;p+=2,first=0) {
if(!first) ncbytescat(buf,"&");
ncbytescat(buf,p[0]);
if(p[1] != NULL && strlen(p[1]) > 0) {
ncbytescat(buf,"=");
if(encode) {
char* encoded = ncuriencodeonly(p[1],queryallow);
ncbytescat(buf,encoded);
nullfree(encoded);
} else
ncbytescat(buf,p[1]);
}
}
}
#if 0
static int
extendenvv(char*** envvp, int amount, int* oldlenp)
{
@ -1281,6 +1304,7 @@ extendenvv(char*** envvp, int amount, int* oldlenp)
*envvp = envv; envv = NULL;
return NC_NOERR;
}
#endif
/* Use for gdb debug */
char*
@ -1288,3 +1312,20 @@ ncuriunescape(const char* s)
{
return ncuridecodepartial(s,ascii);
}
/* Get the actual list of queryies */
void*
ncuriqueryparams(NCURI* uri)
{
ensurequerylist(uri);
return uri->querylist;
}
/* Get the actual list of frags */
void*
ncurifragmentparams(NCURI* uri)
{
ensurefraglist(uri);
return uri->fraglist;
}

7
libdispatch/utf8proc.c
View File

@ -355,7 +355,8 @@ static nc_utf8proc_ssize_t nc_seqindex_write_char_decomposed(nc_utf8proc_uint16_
for (; len >= 0; entry++, len--) {
nc_utf8proc_int32_t entry_cp = nc_seqindex_decode_entry(&entry);
written += nc_utf8proc_decompose_char(entry_cp, dst+written,
nc_utf8proc_int32_t *dest = dst ? (dst+written) : NULL;
written += nc_utf8proc_decompose_char(entry_cp, dest,
(bufsize > written) ? (bufsize - written) : 0, options,
last_boundclass);
if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
@ -525,8 +526,10 @@ static nc_utf8proc_ssize_t nc_seqindex_write_char_decomposed(nc_utf8proc_uint16_
if (custom_func != NULL) {
uc = custom_func(uc, custom_data); /* user-specified custom mapping */
}
nc_utf8proc_int32_t *dest = NULL;
if (buffer) dest = buffer + wpos;
decomp_result = nc_utf8proc_decompose_char(
uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
uc, dest, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
&boundclass
);
if (decomp_result < 0) return decomp_result;

28
libhdf5/hdf5debug.c
View File

@ -14,37 +14,9 @@
#ifdef H5CATCH
#define STSIZE 1000
#ifdef HAVE_EXECINFO_H
#ifdef H5BACKTRACE
# if !defined _WIN32 && !defined __CYGWIN__
static void* stacktrace[STSIZE];
# endif
#endif
#endif
int
nch5breakpoint(int err)
{
#ifdef HAVE_EXECINFO_H
#ifdef H5BACKTRACE
# if !defined _WIN32 && !defined __CYGWIN__
int count = 0;
char** trace = NULL;
int i;
count = backtrace(stacktrace,STSIZE);
trace = backtrace_symbols(stacktrace, STSIZE);
fprintf(stderr,"backtrace:\n");
for(i=0;i<count;i++)
fprintf(stderr,"[%03d] %s\n",i,trace[i]);
# if 0
if(trace != NULL) free(trace);
# endif
# endif
#endif
#endif
return ncbreakpoint(err);
}

19
libncxml/Makefile.am
View File

@ -35,7 +35,20 @@ tinyxml2::
rm -fr ./tinyxml2 ./license.txt
git clone --depth=1 ${REPO}
cat tinyxml2/LICENSE.txt > ./license.txt
cat tinyxml2/tinyxml2.h > ./tinyxml2.h
sed -e 's/__BORLANDC__/__APPLE__/' < tinyxml2/tinyxml2.cpp \
| sed -e 's/ptrdiff_t/long/g' > ./tinyxml2.cpp
tr -d '\r' < tinyxml2/tinyxml2.h > tinyxml2.h
cat tinyxml2/tinyxml2.cpp \
| sed -e 's/__BORLANDC__/__APPLE__/' \
| sed -e 's/ptrdiff_t/long/g' \
| sed -e '/^static[ ]*FILE[*][ ]*callfopen(/i\
\#if 0' \
| sed -e '/^void[ ]*XMLDocument::DeleteNode(/i\
\#endif /*0*/\
' \
| sed -e '/^XMLError[ ]*XMLDocument::LoadFile([ ]*const[ ]*char[*]/i\
\#if 0' \
| sed -e '/^XMLError[ ]*XMLDocument::Parse(/i\
\#endif /*0*/\
' \
| tr -d '\r' \
| cat > ./tinyxml2.cpp
rm -fr tinyxml2

6
libncxml/tinyxml2.cpp
View File

@ -2283,6 +2283,7 @@ XMLUnknown* XMLDocument::NewUnknown( const char* str )
return unk;
}
#if 0
static FILE* callfopen( const char* filepath, const char* mode )
{
TIXMLASSERT( filepath );
@ -2299,6 +2300,8 @@ static FILE* callfopen( const char* filepath, const char* mode )
return fp;
}
#endif /*0*/
void XMLDocument::DeleteNode( XMLNode* node ) {
TIXMLASSERT( node );
TIXMLASSERT(node->_document == this );
@ -2317,6 +2320,7 @@ void XMLDocument::DeleteNode( XMLNode* node ) {
}
#if 0
XMLError XMLDocument::LoadFile( const char* filename )
{
if ( !filename ) {
@ -2420,6 +2424,8 @@ XMLError XMLDocument::SaveFile( FILE* fp, bool compact )
}
#endif /*0*/
XMLError XMLDocument::Parse( const char* xml, size_t nBytes )
{
Clear();

27
libnczarr/zarr.c
View File

@ -22,7 +22,7 @@ static int applycontrols(NCZ_FILE_INFO_T* zinfo);
*/
int
ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** controls)
ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* controls)
{
int stat = NC_NOERR;
NCZ_FILE_INFO_T* zinfo = NULL;
@ -32,7 +32,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr
NCjson* json = NULL;
char* key = NULL;
ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_envv(controls):"null"));
ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_env(controls):"null"));
nc = (NC*)file->controller;
@ -52,7 +52,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr
zinfo->creating = 1;
zinfo->common.file = file;
zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG);
if((zinfo->envv_controls=NCZ_clonestringvec(0,controls)) == NULL)
if((zinfo->controllist=nclistclone(controls,1)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* fill in some of the zinfo and zroot fields */
@ -94,7 +94,7 @@ done:
*/
int
ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls)
ncz_open_dataset(NC_FILE_INFO_T* file, NClist* controls)
{
int stat = NC_NOERR;
NC* nc = NULL;
@ -126,7 +126,7 @@ ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls)
zinfo->creating = 0;
zinfo->common.file = file;
zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG);
if((zinfo->envv_controls = NCZ_clonestringvec(0,controls))==NULL) /*0=>envv style*/
if((zinfo->controllist=nclistclone(controls,1)) == NULL)
{stat = NC_ENOMEM; goto done;}
zinfo->default_maxstrlen = NCZ_MAXSTR_DEFAULT;
@ -294,12 +294,13 @@ done:
static const char*
controllookup(const char** envv_controls, const char* key)
controllookup(NClist* controls, const char* key)
{
const char** p;
for(p=envv_controls;*p;p+=2) {
if(strcasecmp(key,*p)==0) {
return p[1];
int i;
for(i=0;i<nclistlength(controls);i+=2) {
const char* p = (char*)nclistget(controls,i);
if(strcasecmp(key,p)==0) {
return (const char*)nclistget(controls,i+1);
}
}
return NULL;
@ -314,7 +315,7 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
NClist* modelist = nclistnew();
int noflags = 0; /* track non-default negative flags */
if((value = controllookup((const char**)zinfo->envv_controls,"mode")) != NULL) {
if((value = controllookup(zinfo->controllist,"mode")) != NULL) {
if((stat = NCZ_comma_parse(value,modelist))) goto done;
}
/* Process the modelist first */
@ -337,11 +338,11 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
zinfo->controls.flags &= (~noflags);
/* Process other controls */
if((value = controllookup((const char**)zinfo->envv_controls,"log")) != NULL) {
if((value = controllookup(zinfo->controllist,"log")) != NULL) {
zinfo->controls.flags |= FLAG_LOGGING;
ncsetloglevel(NCLOGNOTE);
}
if((value = controllookup((const char**)zinfo->envv_controls,"show")) != NULL) {
if((value = controllookup(zinfo->controllist,"show")) != NULL) {
if(strcasecmp(value,"fetch")==0)
zinfo->controls.flags |= FLAG_SHOWFETCH;
}

4
libnczarr/zarr.h
View File

@ -26,8 +26,8 @@ struct ZCVT {
#define zcvt_empty {0,0,0.0,NULL}
/* zarr.c */
EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, const char** controls);
EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, const char** controls);
EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, NClist* controls);
EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, NClist* controls);
EXTERNL int ncz_del_attr(NC_FILE_INFO_T* file, NC_OBJ* container, const char* name);
/* HDF5 Mimics */

2
libnczarr/zclose.c
View File

@ -49,7 +49,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort)
if((stat = nczmap_close(zinfo->map,(abort && zinfo->creating)?1:0)))
goto done;
NCZ_freestringvec(0,zinfo->envv_controls);
nclistfreeall(zinfo->controllist);
NC_authfree(zinfo->auth);
nullfree(zinfo);

2
libnczarr/zcreate.c
View File

@ -30,7 +30,7 @@ static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_DISKLESS|NC_64BIT
* @author Dennis Heimbigner, Ed Hartnett
*/
static int
ncz_create_file(const char *path, int cmode, size_t initialsz, const char** controls, int ncid)
ncz_create_file(const char *path, int cmode, size_t initialsz, NClist* controls, int ncid)
{
int retval = NC_NOERR;
NC_FILE_INFO_T* h5 = NULL;

2
libnczarr/zinternal.h
View File

@ -144,7 +144,7 @@ typedef struct NCZ_FILE_INFO {
} zarr;
int creating; /* 1=> created 0=>open */
int native_endianness; /* NC_ENDIAN_LITTLE | NC_ENDIAN_BIG */
char** envv_controls; /* Envv format */
NClist* controllist; /* Envv format */
struct Controls {
size64_t flags;
# define FLAG_PUREZARR 1

2
libnczarr/zopen.c
View File

@ -64,7 +64,7 @@ check_for_classic_model(NC_GRP_INFO_T *root_grp, int *is_classic)
* @author Dennis Heimbigner, Ed Hartnett
*/
static int
ncz_open_file(const char *path, int mode, const char** controls, int ncid)
ncz_open_file(const char *path, int mode, NClist* controls, int ncid)
{
int stat = NC_NOERR;
NC_FILE_INFO_T *h5 = NULL;

268
libsrc/ncx.m4
View File

@ -307,88 +307,24 @@ swapn2b(void *dst, const void *src, IntType nn)
IntType i;
uint16_t *op = (uint16_t*) dst;
uint16_t *ip = (uint16_t*) src;
uint16_t tmp;
for (i=0; i<nn; i++) {
op[i] = ip[i];
op[i] = (uint16_t)SWAP2(op[i]);
/* memcpy is used to handle the case of unaligned memory */
memcpy(&tmp, &ip[i], sizeof(tmp));
tmp = SWAP2(tmp);
memcpy(&op[i], &tmp, sizeof(tmp));
}
#if 0
char *op = dst;
const char *ip = src;
/* unroll the following to reduce loop overhead
*
* while (nn-- > 0)
* {
* *op++ = *(++ip);
* *op++ = *(ip++ -1);
* }
*/
while (nn > 3)
{
*op++ = *(++ip);
*op++ = *(ip++ -1);
*op++ = *(++ip);
*op++ = *(ip++ -1);
*op++ = *(++ip);
*op++ = *(ip++ -1);
*op++ = *(++ip);
*op++ = *(ip++ -1);
nn -= 4;
}
while (nn-- > 0)
{
*op++ = *(++ip);
*op++ = *(ip++ -1);
}
#endif
}
# ifndef vax
inline static void
swap4b(void *dst, const void *src)
{
/* copy over, make the below swap in-place */
uint32_t tmp;
/* use memcpy to avoid type punning */
/* memcpy is used to handle the case of unaligned memory */
memcpy(&tmp, src, sizeof(tmp));
tmp = SWAP4(tmp);
memcpy(dst, &tmp, 4);
/* Codes below will cause "break strict-aliasing rules" in gcc
uint32_t *op = (uint32_t*)dst;
*op = *(uint32_t*)src;
*op = SWAP4(*op);
*/
/* Below are copied from netCDF-4.
* See https://bugtracking.unidata.ucar.edu/browse/NCF-338
* Quote "One issue we are wrestling with is how compilers optimize this
* code. For some reason, we are actually needing to add an artificial
* move to a 4 byte space to get it to work. I think what is happening is
* that the optimizer is bit shifting within a double, which is incorrect.
* The following code actually does work correctly.
* This is in Linux land, gcc.
*
* However, the above in-place byte-swap does not appear affected by this.
*/
#if 0
uint32_t *ip = (uint32_t*)src;
uint32_t tempOut; /* cannot use pointer when gcc O2 optimizer is used */
tempOut = SWAP4(*ip);
*(float *)dst = *(float *)(&tempOut);
#endif
/* OLD implementation that results in four load and four store CPU
instructions
char *op = dst;
const char *ip = src;
op[0] = ip[3];
op[1] = ip[2];
op[2] = ip[1];
op[3] = ip[0];
*/
}
# endif /* !vax */
@ -398,110 +334,24 @@ swapn4b(void *dst, const void *src, IntType nn)
IntType i;
uint32_t *op = (uint32_t*) dst;
uint32_t *ip = (uint32_t*) src;
uint32_t tmp;
for (i=0; i<nn; i++) {
/* copy over, make the below swap in-place */
op[i] = ip[i];
op[i] = SWAP4(op[i]);
/* memcpy is used to handle the case of unaligned memory */
memcpy(&tmp, &ip[i], sizeof(tmp));
tmp = SWAP4(tmp);
memcpy(&op[i], &tmp, sizeof(tmp));
}
#if 0
char *op = dst;
const char *ip = src;
/* unroll the following to reduce loop overhead
* while (nn-- > 0)
* {
* op[0] = ip[3];
* op[1] = ip[2];
* op[2] = ip[1];
* op[3] = ip[0];
* op += 4;
* ip += 4;
* }
*/
while (nn > 3)
{
op[0] = ip[3];
op[1] = ip[2];
op[2] = ip[1];
op[3] = ip[0];
op[4] = ip[7];
op[5] = ip[6];
op[6] = ip[5];
op[7] = ip[4];
op[8] = ip[11];
op[9] = ip[10];
op[10] = ip[9];
op[11] = ip[8];
op[12] = ip[15];
op[13] = ip[14];
op[14] = ip[13];
op[15] = ip[12];
op += 16;
ip += 16;
nn -= 4;
}
while (nn-- > 0)
{
op[0] = ip[3];
op[1] = ip[2];
op[2] = ip[1];
op[3] = ip[0];
op += 4;
ip += 4;
}
#endif
}
# ifndef vax
inline static void
swap8b(void *dst, const void *src)
{
#ifdef FLOAT_WORDS_BIGENDIAN
/* copy over, make the below swap in-place */
*(uint64_t*)dst = *(uint64_t*)src;
uint32_t *op = (uint32_t*)dst;
*op = SWAP4(*op);
op = (uint32_t*)((char*)dst+4);
*op = SWAP4(*op);
#else
uint64_t tmp;
/* use memcpy to avoid type punning */
/* memcpy is used to handle the case of unaligned memory */
memcpy(&tmp, src, sizeof(tmp));
tmp = SWAP8(tmp);
memcpy(dst, &tmp, 8);
/* Codes below will cause "break strict-aliasing rules" in gcc
uint64_t *op = (uint64_t*)dst;
*op = *(uint64_t*)src;
*op = SWAP8(*op);
*/
#endif
#if 0
char *op = dst;
const char *ip = src;
# ifndef FLOAT_WORDS_BIGENDIAN
op[0] = ip[7];
op[1] = ip[6];
op[2] = ip[5];
op[3] = ip[4];
op[4] = ip[3];
op[5] = ip[2];
op[6] = ip[1];
op[7] = ip[0];
# else
op[0] = ip[3];
op[1] = ip[2];
op[2] = ip[1];
op[3] = ip[0];
op[4] = ip[7];
op[5] = ip[6];
op[6] = ip[5];
op[7] = ip[4];
#endif
#endif
}
# endif /* !vax */
@ -509,100 +359,16 @@ swap8b(void *dst, const void *src)
inline static void
swapn8b(void *dst, const void *src, IntType nn)
{
#ifdef FLOAT_WORDS_BIGENDIAN
IntType i;
uint64_t *dst_p = (uint64_t*) dst;
uint64_t *src_p = (uint64_t*) src;
for (i=0; i<nn; i++) {
/* copy over, make the below swap in-place */
dst_p[i] = src_p[i];
uint32_t *op = (uint32_t*)(&dst_p[i]);
*op = SWAP4(*op);
op = (uint32_t*)((char*)op+4);
*op = SWAP4(*op);
}
#else
IntType i;
uint64_t *op = (uint64_t*) dst;
uint64_t *ip = (uint64_t*) src;
uint64_t tmp;
for (i=0; i<nn; i++) {
/* copy over, make the below swap in-place */
op[i] = ip[i];
op[i] = SWAP8(op[i]);
/* memcpy is used to handle the case of unaligned memory */
memcpy(&tmp, &ip[i], sizeof(tmp));
tmp = SWAP8(tmp);
memcpy(&op[i], &tmp, sizeof(tmp));
}
#endif
#if 0
char *op = dst;
const char *ip = src;
/* unroll the following to reduce loop overhead
* while (nn-- > 0)
* {
* op[0] = ip[7];
* op[1] = ip[6];
* op[2] = ip[5];
* op[3] = ip[4];
* op[4] = ip[3];
* op[5] = ip[2];
* op[6] = ip[1];
* op[7] = ip[0];
* op += 8;
* ip += 8;
* }
*/
# ifndef FLOAT_WORDS_BIGENDIAN
while (nn > 1)
{
op[0] = ip[7];
op[1] = ip[6];
op[2] = ip[5];
op[3] = ip[4];
op[4] = ip[3];
op[5] = ip[2];
op[6] = ip[1];
op[7] = ip[0];
op[8] = ip[15];
op[9] = ip[14];
op[10] = ip[13];
op[11] = ip[12];
op[12] = ip[11];
op[13] = ip[10];
op[14] = ip[9];
op[15] = ip[8];
op += 16;
ip += 16;
nn -= 2;
}
while (nn-- > 0)
{
op[0] = ip[7];
op[1] = ip[6];
op[2] = ip[5];
op[3] = ip[4];
op[4] = ip[3];
op[5] = ip[2];
op[6] = ip[1];
op[7] = ip[0];
op += 8;
ip += 8;
}
# else
while (nn-- > 0)
{
op[0] = ip[3];
op[1] = ip[2];
op[2] = ip[1];
op[3] = ip[0];
op[4] = ip[7];
op[5] = ip[6];
op[6] = ip[5];
op[7] = ip[4];
op += 8;
ip += 8;
}
#endif
#endif
}
# endif /* !vax */

10
nctest/CMakeLists.txt
View File

@ -18,7 +18,9 @@ TARGET_LINK_LIBRARIES(nctest netcdf)
ADD_TEST(nctest ${EXECUTABLE_OUTPUT_PATH}/nctest)
add_bin_test_no_prefix(tst_rename)
add_sh_test(nctest compare_test_files)
IF(HAVE_BASH)
SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest)
ENDIF(HAVE_BASH)
IF(BUILD_UTILITIES)
add_sh_test(nctest compare_test_files)
IF(HAVE_BASH)
SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest)
ENDIF(HAVE_BASH)
ENDIF(BUILD_UTILITIES)