From 5fa2defc7e67c5b47a3e4ee3fbb883790ed15593 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 8 Oct 2023 19:59:28 -0600 Subject: [PATCH 01/13] Improve fetch performance of DAP4 Prior to this PR, DAP4 always fetched the whole (constrained) dataset This PR changes the query processing so 1. It reads data on a per-variable request (equivalent to calling nc_get_var()). 2. It tracks a response for every query. Most of the changes reflect having to do per-variable requests. In any case, doing all this significantly reduces the amount of data transmitted and hence speeds up DAP4 requests. --- RELEASE_NOTES.md | 1 + dap4_test/test_common.h | 18 +- dap4_test/test_meta.c | 2 +- dap4_test/test_parse.c | 2 +- include/ncuri.h | 26 +-- libdap4/d4chunk.c | 135 +++++++-------- libdap4/d4curlfunctions.c | 4 +- libdap4/d4data.c | 98 ++++++----- libdap4/d4debug.c | 2 +- libdap4/d4file.c | 348 +++++++++++++++++++++++++------------- libdap4/d4fix.c | 4 +- libdap4/d4http.c | 75 -------- libdap4/d4meta.c | 110 ++++-------- libdap4/d4parser.c | 34 ++-- libdap4/d4read.c | 121 +++---------- libdap4/d4read.h | 2 +- libdap4/d4swap.c | 70 ++++---- libdap4/d4util.c | 67 ++++---- libdap4/d4util.h | 5 +- libdap4/d4varx.c | 209 +++++++++++++++++------ libdap4/ncd4.h | 40 +++-- libdap4/ncd4dispatch.c | 11 +- libdap4/ncd4types.h | 96 ++++++----- libdispatch/dinfermodel.c | 97 +++++------ libdispatch/ncuri.c | 285 ++++++++++++++++++------------- libnczarr/zarr.c | 27 +-- libnczarr/zarr.h | 4 +- libnczarr/zclose.c | 2 +- libnczarr/zcreate.c | 2 +- libnczarr/zinternal.h | 2 +- libnczarr/zopen.c | 2 +- 31 files changed, 1010 insertions(+), 891 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c92394242..ed19b87f7 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Improve the speed and data quantity for DAP4 queries. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737). diff --git a/dap4_test/test_common.h b/dap4_test/test_common.h index a005d9c9c..d7e096e2d 100644 --- a/dap4_test/test_common.h +++ b/dap4_test/test_common.h @@ -21,6 +21,7 @@ typedef int TDMR; static NCbytes* input = NULL; static NCbytes* output = NULL; static NCD4meta* metadata = NULL; +static NCD4response* resp = NULL; static char* infile = NULL; static char* outfile = NULL; static int ncid = 0; @@ -85,16 +86,21 @@ setup(int tdmr, int argc, char** argv) if(translatenc4) controller->controls.translation = NCD4_TRANSNC4; NCD4_applyclientfragmentcontrols(controller); - if((metadata=NCD4_newmeta(controller))==NULL) - fail(NC_ENOMEM); - metadata->mode = mode; - NCD4_attachraw(metadata, ncbyteslength(input),ncbytescontents(input)); - if((ret=NCD4_dechunk(metadata))) /* ok for mode == DMR or mode == DAP */ + if((ret=NCD4_newMeta(controller,&metadata))) + fail(ret); + + if((ret=NCD4_newResponse(controller,&resp))) + fail(ret); + resp->raw.size = ncbyteslength(input); + resp->raw.memory = ncbytescontents(input); + resp->mode = mode; + + if((ret=NCD4_dechunk(resp))) /* ok for mode == DMR or mode == DAP */ fail(ret); #ifdef DEBUG { - int swap = (metadata->serial.hostbigendian != metadata->serial.remotebigendian); + int swap = (controller->platform.hostlittleendian != resp->remotelittleendian); void* d = metadata->serial.dap; size_t sz = metadata->serial.dapsize; fprintf(stderr,"====================\n"); diff --git a/dap4_test/test_meta.c b/dap4_test/test_meta.c index f0bf2273b..478fcfb62 100644 --- a/dap4_test/test_meta.c +++ b/dap4_test/test_meta.c @@ -20,7 +20,7 @@ main(int argc, char** argv) fprintf(stderr,"t_dmrmeta %s -> %s\n",infile,outfile); #endif - if((ret = NCD4_parse(metadata))) goto done; + if((ret = NCD4_parse(metadata,resp,0))) goto done; if((ret = NCD4_metabuild(metadata,ncid))) goto done; done: diff --git a/dap4_test/test_parse.c b/dap4_test/test_parse.c index 44075ffaa..59afb74d5 100644 --- a/dap4_test/test_parse.c +++ b/dap4_test/test_parse.c @@ -17,7 +17,7 @@ main(int argc, char** argv) setup(TDMR_PARSE,argc,argv); - if((ret = NCD4_parse(metadata))) goto done; + if((ret = NCD4_parse(metadata,resp,0))) goto done; ret = NCD4_print(metadata,output); ncbytesnull(output); if(ret == NC_NOERR) { diff --git a/include/ncuri.h b/include/ncuri.h index eafb9be97..bd1b64101 100644 --- a/include/ncuri.h +++ b/include/ncuri.h @@ -35,12 +35,8 @@ typedef struct NCURI { char* path; /*!< path */ char* query; /*!< query */ char* fragment; /*!< fragment */ - char** fraglist; /* envv style list of decomposed fragment*/ - char** querylist; /* envv style list of decomposed query*/ -#if 0 - char* projection; /*!< without leading '?'*/ - char* selection; /*!< with leading '&'*/ -#endif + void* fraglist; /* some representation of the decomposed fragment string */ + void* querylist; /* some representation of the decomposed query string */ } NCURI; #if 0 @@ -90,6 +86,18 @@ EXTERNL int ncurisetfragmentkey(NCURI* duri,const char* key, const char* value); /* append a specific &key=...& in uri fragment */ EXTERNL int ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value); +/* Replace a specific &key=...& in uri query */ +EXTERNL int ncurisetquerykey(NCURI* duri,const char* key, const char* value); + +/* append a specific &key=...& in uri query */ +EXTERNL int ncuriappendquerykey(NCURI* duri,const char* key, const char* value); + +/* Get the actual list of queryies */ +EXTERNL void* ncuriqueryparams(NCURI* uri); +/* Get the actual list of frags */ +EXTERNL void* ncurifragmentparams(NCURI* uri); + + /* Construct a complete NC URI; caller frees returned string */ EXTERNL char* ncuribuild(NCURI*,const char* prefix, const char* suffix, int flags); @@ -105,12 +113,6 @@ EXTERNL const char* ncurifragmentlookup(NCURI*, const char* param); */ EXTERNL const char* ncuriquerylookup(NCURI*, const char* param); -/* Obtain the complete list of fragment pairs in envv format */ -EXTERNL const char** ncurifragmentparams(NCURI*); - -/* Obtain the complete list of query pairs in envv format */ -EXTERNL const char** ncuriqueryparams(NCURI*); - /* URL Encode/Decode */ EXTERNL char* ncuridecode(const char* s); /* Partial decode */ diff --git a/libdap4/d4chunk.c b/libdap4/d4chunk.c index e95f34b9c..3f2fe44d7 100644 --- a/libdap4/d4chunk.c +++ b/libdap4/d4chunk.c @@ -22,133 +22,128 @@ Notes: */ /* Forward */ -static int processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count); +static int processerrchunk(NCD4response*, void* errchunk, unsigned int count); /**************************************************/ - -void -NCD4_resetSerial(NCD4serial* serial, size_t rawsize, void* rawdata) -{ - nullfree(serial->errdata); - nullfree(serial->dmr); - nullfree(serial->dap); - nullfree(serial->rawdata); - /* clear all fields */ - memset(serial,0,sizeof(NCD4serial)); - /* Reset fields */ - serial->hostlittleendian = NCD4_isLittleEndian(); - serial->rawsize = rawsize; - serial->rawdata = rawdata; -} - int -NCD4_dechunk(NCD4meta* metadata) +NCD4_dechunk(NCD4response* resp) { - unsigned char *praw, *phdr, *pdap; + unsigned char *praw, *pdmr, *phdr, *pdap, *pappend, *pchunk; NCD4HDR hdr; + int firstchunk; #ifdef D4DUMPRAW - NCD4_tagdump(metadata->serial.rawsize,metadata->serial.rawdata,0,"RAW"); + NCD4_tagdump(resp->serial.raw.size,resp->serial.raw.data,0,"RAW"); #endif /* Access the returned raw data */ - praw = metadata->serial.rawdata; + praw = (unsigned char*)resp->raw.memory; - if(metadata->mode == NCD4_DSR) { + if(resp->mode == NCD4_DSR) { return THROW(NC_EDMR); - } else if(metadata->mode == NCD4_DMR) { + } else if(resp->mode == NCD4_DMR) { /* Verify the mode; assume that the is optional */ if(memcmp(praw,"serial.rawsize; - if((metadata->serial.dmr = malloc(len+1)) == NULL) + len = resp->raw.size; + if((resp->serial.dmr = malloc(len+1)) == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.dmr,praw,len); - metadata->serial.dmr[len] = '\0'; + memcpy(resp->serial.dmr,praw,len); + resp->serial.dmr[len] = '\0'; /* Suppress nuls */ - (void)NCD4_elidenuls(metadata->serial.dmr,len); + (void)NCD4_elidenuls(resp->serial.dmr,len); return THROW(NC_NOERR); } - } else if(metadata->mode != NCD4_DAP) + } else if(resp->mode != NCD4_DAP) return THROW(NC_EDAP); /* We must be processing a DAP mode packet */ - praw = (metadata->serial.dap = metadata->serial.rawdata); - metadata->serial.rawdata = NULL; + praw = resp->raw.memory; /* If the raw data looks like xml, then we almost certainly have an error */ if(memcmp(praw,"serial.rawsize, metadata->serial.rawdata); + int stat = NCD4_seterrormessage(resp, resp->raw.size, resp->raw.memory); return THROW(stat); /* slight lie */ } - /* Get the DMR chunk header*/ - phdr = NCD4_getheader(praw,&hdr,metadata->serial.hostlittleendian); + /* Get the first header to get dmr content and endian flags*/ + pdmr = NCD4_getheader(praw,&hdr,resp->controller->platform.hostlittleendian); if(hdr.count == 0) return THROW(NC_EDMR); - if(hdr.flags & NCD4_ERR_CHUNK) { - return processerrchunk(metadata, (void*)phdr, hdr.count); - } + if(hdr.flags & NCD4_ERR_CHUNK) + return processerrchunk(resp, (void*)pdmr, hdr.count); + resp->remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0); - metadata->serial.remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0); - /* Again, avoid strxxx operations on dmr */ - if((metadata->serial.dmr = malloc(hdr.count+1)) == NULL) + /* avoid strxxx operations on dmr */ + if((resp->serial.dmr = malloc(hdr.count+1)) == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.dmr,phdr,hdr.count); - metadata->serial.dmr[hdr.count-1] = '\0'; + memcpy(resp->serial.dmr,pdmr,hdr.count); + resp->serial.dmr[hdr.count-1] = '\0'; /* Suppress nuls */ - (void)NCD4_elidenuls(metadata->serial.dmr,hdr.count); + (void)NCD4_elidenuls(resp->serial.dmr,hdr.count); + /* See if there is any data after the DMR */ if(hdr.flags & NCD4_LAST_CHUNK) return THROW(NC_ENODATA); /* Read and concat together the data chunks */ - phdr = phdr + hdr.count; /* point to data chunk header */ + phdr = pdmr + hdr.count; /* point to data chunk header */ /* Do a sanity check in case the server has shorted us with no data */ - if((hdr.count + CHUNKHDRSIZE) >= metadata->serial.rawsize) { + if((hdr.count + CHUNKHDRSIZE) >= resp->raw.size) { /* Server only sent the DMR part */ - metadata->serial.dapsize = 0; + resp->serial.dapsize = 0; return THROW(NC_EDATADDS); } - pdap = metadata->serial.dap; - for(;;) { - phdr = NCD4_getheader(phdr,&hdr,metadata->serial.hostlittleendian); - if(hdr.flags & NCD4_ERR_CHUNK) { - return processerrchunk(metadata, (void*)phdr, hdr.count); + /* walk all the data chunks */ + /* invariants: + praw -- beginning of the raw response + pdmr -- beginning of the dmr in the raw data + pdap -- beginning of the dechunked dap data + phdr -- pointer to the hdr of the current chunk + pchunk -- pointer to the data part of the current chunk + pappend -- where to append next chunk to the growing dechunked data + */ + for(firstchunk=1;;firstchunk=0) { + pchunk = NCD4_getheader(phdr,&hdr,resp->controller->platform.hostlittleendian); /* Process first data chunk header */ + if(firstchunk) { + pdap = phdr; /* remember start point of the dechunked data */ + pappend = phdr; /* start appending here */ } + if(hdr.flags & NCD4_ERR_CHUNK) + return processerrchunk(resp, (void*)pchunk, hdr.count); /* data chunk; possibly last; possibly empty */ - if(hdr.count > 0) { - d4memmove(pdap,phdr,hdr.count); /* will overwrite the header */ - phdr += hdr.count; - pdap += hdr.count; - } + if(hdr.count > 0) + d4memmove(pappend,pchunk,hdr.count); /* overwrite the header; this the heart of dechunking */ + pappend += hdr.count; /* next append point */ + phdr = pchunk + hdr.count; /* point to header of next chunk */ if(hdr.flags & NCD4_LAST_CHUNK) break; } - metadata->serial.dapsize = (size_t)DELTA(pdap,metadata->serial.dap); + resp->serial.dap = pdap; /* start of dechunked data */ + resp->serial.dapsize = (size_t)DELTA(pappend,pdap); #ifdef D4DUMPDMR - fprintf(stderr,"%s\n",metadata->serial.dmr); + fprintf(stderr,"%s\n",resp->serial.dmr); fflush(stderr); #endif #ifdef D4DUMPDAP - NCD4_tagdump(metadata->serial.dapsize,metadata->serial.dap,0,"DAP"); + NCD4_tagdump(resp->serial.dapsize,resp->serial.dap,0,"DAP"); #endif return THROW(NC_NOERR); } static int -processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count) +processerrchunk(NCD4response* resp, void* errchunk, unsigned int count) { - metadata->serial.errdata = (char*)d4alloc(count+1); - if(metadata->serial.errdata == NULL) + resp->serial.errdata = (char*)d4alloc(count+1); + if(resp->serial.errdata == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.errdata,errchunk,count); - metadata->serial.errdata[count] = '\0'; + memcpy(resp->serial.errdata,errchunk,count); + resp->serial.errdata[count] = '\0'; return THROW(NC_ENODATA); /* slight lie */ } @@ -157,26 +152,26 @@ Given a raw response, attempt to infer the mode: DMR, DAP, DSR. Since DSR is not standardizes, it becomes the default. */ int -NCD4_infermode(NCD4meta* meta) +NCD4_infermode(NCD4response* resp) { - d4size_t size = meta->serial.rawsize; - char* raw = meta->serial.rawdata; + d4size_t size = resp->raw.size; + char* raw = resp->raw.memory; if(size < 16) return THROW(NC_EDAP); /* must have at least this to hold a hdr + partial dmr*/ if(memcmp(raw,"mode = NCD4_DMR; + resp->mode = NCD4_DMR; goto done; } raw += 4; /* Pretend we have a DAP hdr */ if(memcmp(raw,"mode = NCD4_DAP; + resp->mode = NCD4_DAP; goto done; } /* Default to DSR */ - meta->mode = NCD4_DSR; + resp->mode = NCD4_DSR; done: return NC_NOERR; diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index eb1fe9fc6..e60a05673 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -319,7 +319,7 @@ NCD4_get_rcproperties(NCD4INFO* state) ncerror err = NC_NOERR; char* option = NULL; #ifdef HAVE_CURLOPT_BUFFERSIZE - option = NC_rclookup(D4BUFFERSIZE,state->uri->uri,NULL); + option = NC_rclookup(D4BUFFERSIZE,state->dmruri->uri,NULL); if(option != NULL && strlen(option) != 0) { long bufsize; if(strcasecmp(option,"max")==0) @@ -330,7 +330,7 @@ NCD4_get_rcproperties(NCD4INFO* state) } #endif #ifdef HAVE_CURLOPT_KEEPALIVE - option = NC_rclookup(D4KEEPALIVE,state->uri->uri,NULL); + option = NC_rclookup(D4KEEPALIVE,state->dmruri->uri,NULL); if(option != NULL && strlen(option) != 0) { /* The keepalive value is of the form 0 or n/m, where n is the idle time and m is the interval time; diff --git a/libdap4/d4data.c b/libdap4/d4data.c index 5d61b81dc..234f3e377 100644 --- a/libdap4/d4data.c +++ b/libdap4/d4data.c @@ -16,7 +16,7 @@ This code serves two purposes (NCD4_processdata) 2. Walk a specified variable instance to convert to netcdf4 memory representation. - (NCD4_fillinstance) + (NCD4_movetoinstance) */ @@ -29,7 +29,6 @@ static int fillopfixed(NCD4meta*, d4size_t opaquesize, NCD4offset* offset, void* static int fillopvar(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); static int fillstruct(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); static int fillseq(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); -static int NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel); static unsigned NCD4_computeChecksum(NCD4meta* meta, NCD4node* topvar); /***************************************************/ @@ -54,8 +53,9 @@ static unsigned int debugcrc32(unsigned int crc, const void *buf, size_t size) /***************************************************/ /* API */ +/* Parcel out the dechunked data to the corresponding vars */ int -NCD4_processdata(NCD4meta* meta) +NCD4_parcelvars(NCD4meta* meta, NCD4response* resp) { int ret = NC_NOERR; int i; @@ -68,35 +68,55 @@ NCD4_processdata(NCD4meta* meta) toplevel = nclistnew(); NCD4_getToplevelVars(meta,root,toplevel); - /* Otherwise */ - NCD4_inferChecksums(meta,toplevel); - - /* If necessary, byte swap the serialized data */ - /* Do we need to swap the dap4 data? */ - meta->swap = (meta->serial.hostlittleendian != meta->serial.remotelittleendian); - /* Compute the offset and size of the toplevel vars in the raw dap data. */ - /* Also extract remote checksums */ - offset = BUILDOFFSET(meta->serial.dap,meta->serial.dapsize); + offset = BUILDOFFSET(resp->serial.dap,resp->serial.dapsize); for(i=0;iinferredchecksumming))) { FAIL(ret,"delimit failure"); - if(meta->controller->data.inferredchecksumming) { - /* Compute remote checksum: must occur before any byte swapping */ + } + var->data.response = resp; /* cross link */ + } +done: + return THROW(ret); +} + +/* Process top level vars wrt checksums and swapping */ +int +NCD4_processdata(NCD4meta* meta, NCD4response* resp) +{ + int ret = NC_NOERR; + int i; + NClist* toplevel = NULL; + NCD4node* root = meta->root; + NCD4offset* offset = NULL; + + /* Do we need to swap the dap4 data? */ + meta->swap = (meta->controller->platform.hostlittleendian != resp->remotelittleendian); + + /* Recursively walk the tree in prefix order + to get the top-level variables; also mark as unvisited */ + toplevel = nclistnew(); + NCD4_getToplevelVars(meta,root,toplevel); + + /* Extract remote checksums */ + for(i=0;iinferredchecksumming) { + /* Compute checksum of response data: must occur before any byte swapping and after delimiting */ var->data.localchecksum = NCD4_computeChecksum(meta,var); #ifdef DUMPCHECKSUM fprintf(stderr,"var %s: remote-checksum = 0x%x\n",var->name,var->data.remotechecksum); #endif /* verify checksums */ - if(!meta->controller->data.checksumignore) { + if(!resp->checksumignore) { if(var->data.localchecksum != var->data.remotechecksum) { nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name); ret = NC_EDAP; goto done; } /* Also verify checksum attribute */ - if(meta->controller->data.attrchecksumming) { + if(resp->attrchecksumming) { if(var->data.attrchecksum != var->data.remotechecksum) { nclog(NCLOGERR,"Attribute Checksum mismatch: %s\n",var->name); ret = NC_EDAP; @@ -105,13 +125,11 @@ NCD4_processdata(NCD4meta* meta) } } } - } - - /* Swap the data for each top level variable, - */ - if(meta->swap) { - if((ret=NCD4_swapdata(meta,toplevel))) - FAIL(ret,"byte swapping failed"); + if(meta->swap) { + if((ret=NCD4_swapdata(resp,var,meta->swap))) + FAIL(ret,"byte swapping failed"); + } + var->data.valid = 1; /* Everything should be in place */ } done: @@ -133,7 +151,7 @@ Assumes that NCD4_processdata has been called. */ int -NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs) +NCD4_movetoinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs) { int ret = NC_NOERR; void* dst = *dstp; @@ -149,30 +167,30 @@ NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dst } else switch(type->subsort) { case NC_STRING: /* oob strings */ if((ret=fillstring(meta,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; case NC_OPAQUE: if(type->opaque.size > 0) { /* We know the size and its the same for all instances */ if((ret=fillopfixed(meta,type->opaque.size,offset,&dst))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } else { /* Size differs per instance, so we need to convert each opaque to a vlen */ if((ret=fillopvar(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } break; case NC_STRUCT: if((ret=fillstruct(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; case NC_SEQ: if((ret=fillseq(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; default: ret = NC_EINVAL; - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } *dstp = dst; @@ -196,7 +214,7 @@ fillstruct(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NCli NCD4node* field = nclistget(type->vars,i); NCD4node* ftype = field->basetype; void* fdst = (((char*)dst) + field->meta.offset); - if((ret=NCD4_fillinstance(meta,ftype,offset,&fdst,blobs))) + if((ret=NCD4_movetoinstance(meta,ftype,offset,&fdst,blobs))) FAIL(ret,"fillstruct"); } dst = ((char*)dst) + type->meta.memsize; @@ -231,7 +249,7 @@ fillseq(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* for(i=0;ip))+(recordsize * i); - if((ret=NCD4_fillinstance(meta,vlentype,offset,&recdst,blobs))) + if((ret=NCD4_movetoinstance(meta,vlentype,offset,&recdst,blobs))) FAIL(ret,"fillseq"); } dst++; @@ -373,12 +391,16 @@ done: return THROW(ret); } -static int -NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel) +int +NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp) { int ret = NC_NOERR; int i, attrfound; - NCD4INFO* info = meta->controller; + NClist* toplevel = NULL; + + /* Get the toplevel vars */ + toplevel = nclistnew(); + NCD4_getToplevelVars(meta,meta->root,toplevel); /* First, look thru the DMR to see if there is a checksum attribute */ attrfound = 0; @@ -399,9 +421,9 @@ NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel) } } } - info->data.attrchecksumming = (attrfound ? 1 : 0); + resp->attrchecksumming = (attrfound ? 1 : 0); /* Infer checksums */ - info->data.inferredchecksumming = ((info->data.attrchecksumming || info->data.querychecksumming) ? 1 : 0); + resp->inferredchecksumming = ((resp->attrchecksumming || resp->querychecksumming) ? 1 : 0); return THROW(ret); } diff --git a/libdap4/d4debug.c b/libdap4/d4debug.c index d2d40b44a..caed80027 100644 --- a/libdap4/d4debug.c +++ b/libdap4/d4debug.c @@ -97,7 +97,7 @@ int NCD4_debugcopy(NCD4INFO* info) { int i,ret=NC_NOERR; - NCD4meta* meta = info->substrate.metadata; + NCD4meta* meta = info->dmrmetadata; NClist* topvars = nclistnew(); NC* ncp = info->controller; void* memory = NULL; diff --git a/libdap4/d4file.c b/libdap4/d4file.c index aeccc7423..471afa15d 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -6,7 +6,6 @@ #include "ncdispatch.h" #include "ncd4dispatch.h" #include "d4includes.h" -#include "d4read.h" #include "d4curlfunctions.h" #ifdef _MSC_VER @@ -23,19 +22,20 @@ static int constrainable(NCURI*); static void freeCurl(NCD4curl*); -static void freeInfo(NCD4INFO*); static int fragmentcheck(NCD4INFO*, const char* key, const char* subkey); static const char* getfragment(NCD4INFO* info, const char* key); static const char* getquery(NCD4INFO* info, const char* key); static int set_curl_properties(NCD4INFO*); static int makesubstrate(NCD4INFO* d4info); -static void resetInfoforRead(NCD4INFO* d4info); /**************************************************/ /* Constants */ static const char* checkseps = "+,:;"; +/*Define the set of protocols known to be constrainable */ +static const char* constrainableprotocols[] = {"http", "https",NULL}; + /**************************************************/ int NCD4_open(const char * path, int mode, @@ -49,7 +49,8 @@ NCD4_open(const char * path, int mode, NCD4meta* meta = NULL; size_t len = 0; void* contents = NULL; - + NCD4response* dmrresp = NULL; + if(path == NULL) return THROW(NC_EDAPURL); @@ -61,29 +62,27 @@ NCD4_open(const char * path, int mode, /* Setup our NC and NCDAPCOMMON state*/ - d4info = (NCD4INFO*)calloc(1,sizeof(NCD4INFO)); - if(d4info == NULL) {ret = NC_ENOMEM; goto done;} - + if((ret=NCD4_newInfo(&d4info))) goto done; nc->dispatchdata = d4info; nc->int_ncid = nc__pseudofd(); /* create a unique id */ d4info->controller = (NC*)nc; /* Parse url and params */ - if(ncuriparse(nc->path,&d4info->uri)) + if(ncuriparse(nc->path,&d4info->dmruri)) {ret = NC_EDAPURL; goto done;} /* Load auth info from rc file */ - if((ret = NC_authsetup(&d4info->auth, d4info->uri))) + if((ret = NC_authsetup(&d4info->auth, d4info->dmruri))) goto done; NCD4_curl_protocols(d4info); - if(!constrainable(d4info->uri)) + if(!constrainable(d4info->dmruri)) SETFLAG(d4info->controls.flags,NCF_UNCONSTRAINABLE); /* fail if we are unconstrainable but have constraints */ if(FLAGSET(d4info->controls.flags,NCF_UNCONSTRAINABLE)) { - if(d4info->uri != NULL) { - const char* ce = ncuriquerylookup(d4info->uri,DAP4CE); /* Look for dap4.ce */ + if(d4info->dmruri != NULL) { + const char* ce = ncuriquerylookup(d4info->dmruri,DAP4CE); /* Look for dap4.ce */ if(ce != NULL) { nclog(NCLOGWARN,"Attempt to constrain an unconstrainable data source: %s=%s", DAP4CE,ce); @@ -115,7 +114,7 @@ NCD4_open(const char * path, int mode, } /* Turn on logging; only do this after oc_open*/ - if((value = ncurifragmentlookup(d4info->uri,"log")) != NULL) { + if((value = ncurifragmentlookup(d4info->dmruri,"log")) != NULL) { ncloginit(); ncsetloglevel(NCLOGNOTE); } @@ -150,30 +149,35 @@ NCD4_open(const char * path, int mode, /* Reset the substrate */ if((ret=makesubstrate(d4info))) goto done; - /* Always start by reading the DMR only */ + /* Always start by reading the whole DMR only */ /* reclaim substrate.metadata */ - resetInfoforRead(d4info); + NCD4_resetInfoForRead(d4info); /* Rebuild metadata */ - if((d4info->substrate.metadata=NCD4_newmeta(d4info))==NULL) - {ret = NC_ENOMEM; goto done;} + if((ret = NCD4_newMeta(d4info,&d4info->dmrmetadata))) goto done; + meta = d4info->dmrmetadata; - if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags))) goto done; + /* Capture response */ + if((dmrresp = (NCD4response*)calloc(1,sizeof(NCD4response)))==NULL) + {ret = NC_ENOMEM; goto done;} + dmrresp->controller = d4info; + + if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags, d4info->dmruri, dmrresp))) goto done; /* set serial.rawdata */ len = ncbyteslength(d4info->curl->packet); contents = ncbytesextract(d4info->curl->packet); - NCD4_attachraw(d4info->substrate.metadata, len, contents); + assert(dmrresp != NULL); + dmrresp->raw.size = len; + dmrresp->raw.memory = contents; - /* process query parameters */ - NCD4_applyclientquerycontrols(d4info); - - meta = d4info->substrate.metadata; + /* process checksum parameters */ + NCD4_applychecksumcontrols(d4info,dmrresp); /* Infer the mode */ - if((ret=NCD4_infermode(meta))) goto done; + if((ret=NCD4_infermode(dmrresp))) goto done; /* Process the dmr part */ - if((ret=NCD4_dechunk(meta))) goto done; + if((ret=NCD4_dechunk(dmrresp))) goto done; #ifdef D4DUMPDMR { @@ -184,13 +188,13 @@ NCD4_open(const char * path, int mode, } #endif - if((ret = NCD4_parse(d4info->substrate.metadata))) goto done; + if((ret = NCD4_parse(meta,dmrresp,0))) goto done; #ifdef D4DEBUGMETA { fprintf(stderr,"\n/////////////\n"); NCbytes* buf = ncbytesnew(); - NCD4_print(d4info->substrate.metadata,buf); + NCD4_print(meta,buf); ncbytesnull(buf); fputs(ncbytescontents(buf),stderr); ncbytesfree(buf); @@ -200,12 +204,20 @@ NCD4_open(const char * path, int mode, #endif /* Build the substrate metadata */ - ret = NCD4_metabuild(d4info->substrate.metadata,d4info->substrate.metadata->ncid); + ret = NCD4_metabuild(meta,meta->ncid); if(ret != NC_NOERR && ret != NC_EVARSIZE) goto done; + /* Remember the response */ + nclistpush(d4info->responses,dmrresp); + + /* Avoid duplicate reclaims */ + dmrresp = NULL; + d4info = NULL; + done: + NCD4_reclaimResponse(dmrresp); + NCD4_reclaimInfo(d4info); if(ret) { - freeInfo(d4info); nc->dispatchdata = NULL; } return THROW(ret); @@ -236,7 +248,7 @@ NCD4_close(int ncid, void* ignore) ret = nc_abort(substrateid); } - freeInfo(d4info); + NCD4_reclaimInfo(d4info); done: return THROW(ret); @@ -248,82 +260,6 @@ NCD4_abort(int ncid) return NCD4_close(ncid,NULL); } -/**************************************************/ - -/* Reclaim an NCD4INFO instance */ -static void -freeInfo(NCD4INFO* d4info) -{ - if(d4info == NULL) return; - d4info->controller = NULL; /* break link */ - nullfree(d4info->rawurltext); - nullfree(d4info->urltext); - ncurifree(d4info->uri); - freeCurl(d4info->curl); - nullfree(d4info->data.memory); - nullfree(d4info->data.ondiskfilename); - if(d4info->data.ondiskfile != NULL) - fclose(d4info->data.ondiskfile); - nullfree(d4info->fileproto.filename); - if(d4info->substrate.realfile - && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { - /* We used real file, so we need to delete the temp file - unless we are debugging. - Assume caller has done nc_close|nc_abort on the ncid. - Note that in theory, this should not be necessary since - AFAIK the substrate file is still in def mode, and - when aborted, it should be deleted. But that is not working - for some reason, so we delete it ourselves. - */ - if(d4info->substrate.filename != NULL) { - unlink(d4info->substrate.filename); - } - } - nullfree(d4info->substrate.filename); /* always reclaim */ - NCD4_reclaimMeta(d4info->substrate.metadata); - NC_authfree(d4info->auth); - nclistfree(d4info->blobs); - free(d4info); -} - -/* Reset NCD4INFO instance for new read request */ -static void -resetInfoforRead(NCD4INFO* d4info) -{ - if(d4info == NULL) return; - if(d4info->substrate.realfile - && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { - /* We used real file, so we need to delete the temp file - unless we are debugging. - Assume caller has done nc_close|nc_abort on the ncid. - Note that in theory, this should not be necessary since - AFAIK the substrate file is still in def mode, and - when aborted, it should be deleted. But that is not working - for some reason, so we delete it ourselves. - */ - if(d4info->substrate.filename != NULL) { - unlink(d4info->substrate.filename); - } - } - NCD4_resetMeta(d4info->substrate.metadata); - nullfree(d4info->substrate.metadata); - d4info->substrate.metadata = NULL; -} - -static void -freeCurl(NCD4curl* curl) -{ - if(curl == NULL) return; - NCD4_curlclose(curl->curl); - ncbytesfree(curl->packet); - nullfree(curl->errdata.code); - nullfree(curl->errdata.message); - free(curl); -} - -/* Define the set of protocols known to be constrainable */ -static const char* constrainableprotocols[] = {"http", "https",NULL}; - static int constrainable(NCURI* durl) { @@ -449,11 +385,6 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info) if(value != NULL) strncpy(info->controls.substratename,value,(NC_MAX_NAME-1)); - value = getfragment(info,"hyrax"); - if(value != NULL) { - info->data.checksumignore = 1; /* Assume checksum, but ignore */ - } - info->controls.opaquesize = DFALTOPAQUESIZE; value = getfragment(info,"opaquesize"); if(value != NULL) { @@ -476,22 +407,29 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info) } } +/* Checksum controls are found both in the query and fragment + parts of a URL. +*/ void -NCD4_applyclientquerycontrols(NCD4INFO* info) +NCD4_applychecksumcontrols(NCD4INFO* info, NCD4response* resp) { const char* value = getquery(info,DAP4CSUM); if(value == NULL) { - info->data.querychecksumming = DEFAULT_CHECKSUM_STATE; + resp->querychecksumming = DEFAULT_CHECKSUM_STATE; } else { if(strcasecmp(value,"false")==0) { - info->data.querychecksumming = 0; + resp->querychecksumming = 0; } else if(strcasecmp(value,"true")==0) { - info->data.querychecksumming = 1; + resp->querychecksumming = 1; } else { nclog(NCLOGWARN,"Unknown checksum mode: %s ; using default",value); - info->data.querychecksumming = DEFAULT_CHECKSUM_STATE; + resp->querychecksumming = DEFAULT_CHECKSUM_STATE; } } + value = getfragment(info,"hyrax"); + if(value != NULL) { + resp->checksumignore = 1; /* Assume checksum, but ignore */ + } } /* Search for substring in value of param. If substring == NULL; then just @@ -523,7 +461,7 @@ getfragment(NCD4INFO* info, const char* key) const char* value; if(info == NULL || key == NULL) return NULL; - if((value=ncurifragmentlookup(info->uri,key)) == NULL) + if((value=ncurifragmentlookup(info->dmruri,key)) == NULL) return NULL; return value; } @@ -537,7 +475,7 @@ getquery(NCD4INFO* info, const char* key) const char* value; if(info == NULL || key == NULL) return NULL; - if((value=ncuriquerylookup(info->uri,key)) == NULL) + if((value=ncuriquerylookup(info->dmruri,key)) == NULL) return NULL; return value; } @@ -596,3 +534,177 @@ NCD4_get_substrate(NC* nc) } else subnc = nc; return subnc; } + +/**************************************************/ +/* Allocate/Free for various structures */ + +int +NCD4_newInfo(NCD4INFO** d4infop) +{ + int ret = NC_NOERR; + NCD4INFO* info = NULL; + if((info = calloc(1,sizeof(NCD4INFO)))==NULL) + {ret = NC_ENOMEM; goto done;} + info->platform.hostlittleendian = NCD4_isLittleEndian(); + if(d4infop) {*d4infop = info; info = NULL;} +done: + if(info) NCD4_reclaimInfo(info); + return THROW(ret); +} + +/* Reclaim an NCD4INFO instance */ +void +NCD4_reclaimInfo(NCD4INFO* d4info) +{ + size_t i; + if(d4info == NULL) return; + d4info->controller = NULL; /* break link */ + nullfree(d4info->rawdmrurltext); + nullfree(d4info->dmrurltext); + ncurifree(d4info->dmruri); + freeCurl(d4info->curl); + nullfree(d4info->fileproto.filename); + NCD4_resetInfoForRead(d4info); + nullfree(d4info->substrate.filename); /* always reclaim */ + NC_authfree(d4info->auth); + nclistfree(d4info->blobs); + /* Reclaim dmr node tree */ + NCD4_reclaimMeta(d4info->dmrmetadata); + for(i=0;iresponses);i++) { + NCD4response* resp = nclistget(d4info->responses,i); + NCD4_reclaimResponse(resp); + } + nclistfree(d4info->responses); + /* Reclaim all responses */ + NCD4_resetMeta(d4info->dmrmetadata); + free(d4info); +} + +/* Reset NCD4INFO instance for new read request */ +void +NCD4_resetInfoForRead(NCD4INFO* d4info) +{ + if(d4info == NULL) return; + if(d4info->substrate.realfile + && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { + /* We used real file, so we need to delete the temp file + unless we are debugging. + Assume caller has done nc_close|nc_abort on the ncid. + Note that in theory, this should not be necessary since + AFAIK the substrate file is still in def mode, and + when aborted, it should be deleted. But that is not working + for some reason, so we delete it ourselves. + */ + if(d4info->substrate.filename != NULL) { + unlink(d4info->substrate.filename); + } + } + NCD4_resetMeta(d4info->dmrmetadata); + nullfree(d4info->dmrmetadata); + d4info->dmrmetadata = NULL; +} + +static void +freeCurl(NCD4curl* curl) +{ + if(curl == NULL) return; + NCD4_curlclose(curl->curl); + ncbytesfree(curl->packet); + nullfree(curl->errdata.code); + nullfree(curl->errdata.message); + free(curl); +} + +int +NCD4_newResponse(NCD4INFO* info, NCD4response** respp) +{ + int ret = NC_NOERR; + NCD4response* resp = NULL; + NC_UNUSED(info); + if((resp = calloc(1,sizeof(NCD4response)))==NULL) + {ret = NC_ENOMEM; goto done;} + resp->controller = info; + if(respp) {*respp = resp; resp = NULL;} +done: + if(resp) NCD4_reclaimResponse(resp); + return THROW(ret); +} + + +/* Reclaim an NCD4response instance */ +void +NCD4_reclaimResponse(NCD4response* d4resp) +{ + struct NCD4serial* serial = NULL; + if(d4resp == NULL) return; + serial = &d4resp->serial; + d4resp->controller = NULL; /* break link */ + + nullfree(d4resp->raw.memory); + + nullfree(serial->errdata); + nullfree(serial->dmr); + nullfree(serial->dap); + /* clear all fields */ + memset(serial,0,sizeof(struct NCD4serial)); + + nullfree(d4resp->error.parseerror); + nullfree(d4resp->error.message); + nullfree(d4resp->error.context); + nullfree(d4resp->error.otherinfo); + memset(&d4resp->error,0,sizeof(d4resp->error)); + + free(d4resp); +} + + +/* Create an empty NCD4meta object for + use in subsequent calls + (is the the right src file to hold this?) +*/ + +int +NCD4_newMeta(NCD4INFO* info, NCD4meta** metap) +{ + int ret = NC_NOERR; + NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); + if(meta == NULL) return NC_ENOMEM; + meta->allnodes = nclistnew(); +#ifdef D4DEBUG + meta->debuglevel = 1; +#endif + meta->controller = info; + meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ + if(metap) {*metap = meta; meta = NULL;} + return THROW(ret); +} + +void +NCD4_reclaimMeta(NCD4meta* dataset) +{ + int i; + if(dataset == NULL) return; + NCD4_resetMeta(dataset); + + for(i=0;iallnodes);i++) { + NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); + reclaimNode(node); + } + nclistfree(dataset->allnodes); + nclistfree(dataset->groupbyid); + nclistfree(dataset->atomictypes); + free(dataset); +} + +void +NCD4_resetMeta(NCD4meta* dataset) +{ + if(dataset == NULL) return; +#if 0 + for(i=0;iblobs);i++) { + void* p = nclistget(dataset->blobs,i); + nullfree(p); + } + nclistfree(dataset->blobs); +#endif +} diff --git a/libdap4/d4fix.c b/libdap4/d4fix.c index 82049dd90..b76a52d28 100644 --- a/libdap4/d4fix.c +++ b/libdap4/d4fix.c @@ -190,7 +190,7 @@ walk(NCD4node* node, NClist* sorted) */ int -NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset) +NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset, int inferredchecksumming) { int ret = NC_NOERR; NCD4mark mark = 0; @@ -214,7 +214,7 @@ NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset) topvar->data.dap4data.memory = mark; topvar->data.dap4data.size = OFFSETSIZE(offset,mark); /* extract the dap4 data checksum, if present */ - if(compiler->controller->data.inferredchecksumming) { + if(inferredchecksumming) { union ATOMICS csum; TRANSFER(csum.u8,offset,CHECKSUMSIZE); topvar->data.remotechecksum = csum.u32[0]; diff --git a/libdap4/d4http.c b/libdap4/d4http.c index 8e370d879..13cb82a19 100644 --- a/libdap4/d4http.c +++ b/libdap4/d4http.c @@ -6,7 +6,6 @@ #include "d4includes.h" #include "d4curlfunctions.h" -static size_t WriteFileCallback(void*, size_t, size_t, void*); static size_t WriteMemoryCallback(void*, size_t, size_t, void*); static int curlerrtoncerr(CURLcode cstat); @@ -33,59 +32,6 @@ NCD4_fetchhttpcode(CURL* curl) return httpcode; } -int -NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, - d4size_t* sizep, long* filetime) -{ - int ret = NC_NOERR; - CURLcode cstat = CURLE_OK; - struct Fetchdata fetchdata; - - /* Set the URL */ - cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url); - if (cstat != CURLE_OK) goto fail; - - /* send all data to this function */ - cstat = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteFileCallback); - if (cstat != CURLE_OK) goto fail; - - /* we pass our file to the callback function */ - cstat = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fetchdata); - if(cstat != CURLE_OK) goto fail; - - /* One last thing; always try to get the last modified time */ - cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1); - if (cstat != CURLE_OK) goto fail; - - fetchdata.stream = stream; - fetchdata.size = 0; - cstat = curl_easy_perform(curl); - if (cstat != CURLE_OK) - {ret = NC_EDAPSVC; goto fail;} - - if (ret == NC_NOERR) { - /* return the file size*/ -#ifdef D4DEBUG - nclog(NCLOGNOTE,"filesize: %lu bytes",fetchdata.size); -#endif - if (sizep != NULL) - *sizep = fetchdata.size; - /* Get the last modified time */ - if(filetime != NULL) - cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime); - if(cstat != CURLE_OK) - {ret = NC_ECURL; goto fail;} - } - return THROW(ret); - -fail: - if(cstat != CURLE_OK) { - nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat)); - ret = curlerrtoncerr(cstat); - } - return THROW(ret); -} - int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcodep) { @@ -155,27 +101,6 @@ done: return THROW(ret); } -static size_t -WriteFileCallback(void* ptr, size_t size, size_t nmemb, void* data) -{ - size_t realsize = size * nmemb; - size_t count; - struct Fetchdata* fetchdata; - fetchdata = (struct Fetchdata*) data; - if(realsize == 0) - nclog(NCLOGWARN,"WriteFileCallback: zero sized chunk"); - count = fwrite(ptr, size, nmemb, fetchdata->stream); - if (count > 0) { - fetchdata->size += (count * size); - } else { - nclog(NCLOGWARN,"WriteFileCallback: zero sized write"); - } -#ifdef PROGRESS - nclog(NCLOGNOTE,"callback: %lu bytes",(d4size_t)realsize); -#endif - return count; -} - static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) { diff --git a/libdap4/d4meta.c b/libdap4/d4meta.c index 4293b3727..1cea62f61 100644 --- a/libdap4/d4meta.c +++ b/libdap4/d4meta.c @@ -89,75 +89,6 @@ done: return THROW(ret); } - -/* Create an empty NCD4meta object for - use in subsequent calls - (is the the right src file to hold this?) -*/ - -NCD4meta* -NCD4_newmeta(NCD4INFO* info) -{ - NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); - if(meta == NULL) return NULL; - meta->allnodes = nclistnew(); -#ifdef D4DEBUG - meta->debuglevel = 1; -#endif - meta->controller = info; - meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ - return meta; -} - -/* Attach raw data to metadata */ -void -NCD4_attachraw(NCD4meta* meta, size_t rawsize, void* rawdata) -{ - assert(meta != NULL); - NCD4_resetSerial(&meta->serial,rawsize,rawdata); -} - -void -NCD4_setdebuglevel(NCD4meta* meta, int debuglevel) -{ - meta->debuglevel = debuglevel; -} - -void -NCD4_reclaimMeta(NCD4meta* dataset) -{ - int i; - if(dataset == NULL) return; - NCD4_resetMeta(dataset); - - for(i=0;iallnodes);i++) { - NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); - reclaimNode(node); - } - nclistfree(dataset->allnodes); - nclistfree(dataset->groupbyid); - nclistfree(dataset->atomictypes); - free(dataset); -} - -void -NCD4_resetMeta(NCD4meta* dataset) -{ - if(dataset == NULL) return; - nullfree(dataset->error.parseerror); dataset->error.parseerror = NULL; - nullfree(dataset->error.message); dataset->error.message = NULL; - nullfree(dataset->error.context); dataset->error.context = NULL; - nullfree(dataset->error.otherinfo); dataset->error.otherinfo = NULL; - NCD4_resetSerial(&dataset->serial,0,NULL); -#if 0 - for(i=0;iblobs);i++) { - void* p = nclistget(dataset->blobs,i); - nullfree(p); - } - nclistfree(dataset->blobs); -#endif -} - void reclaimNode(NCD4node* node) { @@ -676,6 +607,39 @@ savevarbyid(NCD4node* group, NCD4node* var) nclistinsert(group->group.varbyid,var->meta.id,var); } +/* Collect FQN path from var node up to and including + the root group and create an name from it +*/ +char* +NCD4_getVarFQN(NCD4node* var, const char* tail) +{ + int i; + NCD4node* x = NULL; + NClist* path = NULL; + NCbytes* fqn = NULL; + char* result; + + path = nclistnew(); + for(x=var->container;ISGROUP(x->sort);x=x->container) { + nclistinsert(path,0,x); + } + fqn = ncbytesnew(); + for(i=0;iname); + if(escaped == NULL) return NULL; + if(i > 0) ncbytesappend(fqn,'/'); + ncbytescat(fqn,escaped); + free(escaped); + } + nclistfree(path); + if(tail != NULL) + ncbytescat(fqn,tail); + result = ncbytesextract(fqn); + ncbytesfree(fqn); + return result; +} + /* Collect FQN path from node up to (but not including) the first enclosing group and create an name from it */ @@ -1180,7 +1144,7 @@ markdapsize(NCD4meta* meta) } int -NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) +NCD4_findvar(NC* ncp, int gid, int varid, NCD4node** varp, NCD4node** grpp) { int ret = NC_NOERR; NCD4INFO* info = NULL; @@ -1192,11 +1156,11 @@ NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) info = getdap(ncp); if(info == NULL) return THROW(NC_EBADID); - meta = info->substrate.metadata; + meta = info->dmrmetadata; if(meta == NULL) return THROW(NC_EBADID); /* Locate var node via (grpid,varid) */ - grp_id = GROUPIDPART(ncid); + grp_id = GROUPIDPART(gid); group = nclistget(meta->groupbyid,grp_id); if(group == NULL) return THROW(NC_EBADID); @@ -1205,7 +1169,7 @@ NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) return THROW(NC_EBADID); if(varp) *varp = var; if(grpp) *grpp = group; - return ret; + return THROW(ret); } static int diff --git a/libdap4/d4parser.c b/libdap4/d4parser.c index 013e32092..776163a01 100644 --- a/libdap4/d4parser.c +++ b/libdap4/d4parser.c @@ -154,7 +154,7 @@ static int defineBytestringType(NCD4parser*); /* API */ int -NCD4_parse(NCD4meta* metadata) +NCD4_parse(NCD4meta* metadata, NCD4response* resp, int dapparse) { int ret = NC_NOERR; NCD4parser* parser = NULL; @@ -168,8 +168,10 @@ NCD4_parse(NCD4meta* metadata) /* Create and fill in the parser state */ parser = (NCD4parser*)calloc(1,sizeof(NCD4parser)); if(parser == NULL) {ret=NC_ENOMEM; goto done;} + parser->controller = metadata->controller; parser->metadata = metadata; - doc = ncxml_parse(parser->metadata->serial.dmr,strlen(parser->metadata->serial.dmr)); + parser->response = resp; + doc = ncxml_parse(parser->response->serial.dmr,strlen(parser->response->serial.dmr)); if(doc == NULL) {ret=NC_ENOMEM; goto done;} dom = ncxml_root(doc); parser->types = nclistnew(); @@ -178,6 +180,7 @@ NCD4_parse(NCD4meta* metadata) #ifdef D4DEBUG parser->debuglevel = 1; #endif + parser->dapparse = dapparse; /*Walk the DOM tree to build the DAP4 node tree*/ ret = traverse(parser,dom); @@ -214,9 +217,9 @@ traverse(NCD4parser* parser, ncxml_t dom) ret=parseError(parser,dom); /* Report the error */ fprintf(stderr,"DAP4 Error: http-code=%d message=\"%s\" context=\"%s\"\n", - parser->metadata->error.httpcode, - parser->metadata->error.message, - parser->metadata->error.context); + parser->response->error.httpcode, + parser->response->error.message, + parser->response->error.context); fflush(stderr); ret=NC_EDMR; goto done; @@ -847,23 +850,23 @@ parseError(NCD4parser* parser, ncxml_t errxml) char* shttpcode = ncxml_attr(errxml,"httpcode"); ncxml_t x; if(shttpcode == NULL) shttpcode = strdup("400"); - if(sscanf(shttpcode,"%d",&parser->metadata->error.httpcode) != 1) + if(sscanf(shttpcode,"%d",&parser->response->error.httpcode) != 1) nclog(NCLOGERR,"Malformed response"); nullfree(shttpcode); x=ncxml_child(errxml, "Message"); if(x != NULL) { char* txt = ncxml_text(x); - parser->metadata->error.message = (txt == NULL ? NULL : txt); + parser->response->error.message = (txt == NULL ? NULL : txt); } x = ncxml_child(errxml, "Context"); if(x != NULL) { const char* txt = ncxml_text(x); - parser->metadata->error.context = (txt == NULL ? NULL : strdup(txt)); + parser->response->error.context = (txt == NULL ? NULL : strdup(txt)); } x=ncxml_child(errxml, "OtherInformation"); if(x != NULL) { const char* txt = ncxml_text(x); - parser->metadata->error.otherinfo = (txt == NULL ? NULL : strdup(txt)); + parser->response->error.otherinfo = (txt == NULL ? NULL : strdup(txt)); } return THROW(NC_NOERR); } @@ -1321,7 +1324,7 @@ makeNode(NCD4parser* parser, NCD4node* parent, ncxml_t xml, NCD4sort sort, nc_ty record(parser,node); if(nodep) *nodep = node; done: - return ret; + return THROW(ret); } static int @@ -1652,12 +1655,19 @@ parseForwards(NCD4parser* parser, NCD4node* root) const char* mapname = (const char*)nclistget(var->mapnames,j); /* Find the corresponding variable */ NCD4node* mapref = lookupFQN(parser,mapname,NCD4_VAR); - if(mapref == NULL) + if(mapref != NULL) + PUSH(var->maps,mapref); + else if(!parser->dapparse) FAIL(NC_ENOTVAR," name does not refer to a variable: %s",mapname); - PUSH(var->maps,mapref); } } done: return THROW(ret); } + +void +NCD4_setdebuglevel(NCD4parser* parser, int debuglevel) +{ + parser->debuglevel = debuglevel; +} diff --git a/libdap4/d4read.c b/libdap4/d4read.c index 3a0e95a1d..3a24b08ff 100644 --- a/libdap4/d4read.c +++ b/libdap4/d4read.c @@ -19,9 +19,8 @@ See \ref copyright file for more info. /* Do conversion if this code was compiled via Vis. Studio or Mingw */ /*Forward*/ -static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, long*); +static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, int*, long*); static int readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet); -static int readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep); static int readfileDAPDMR(NCD4INFO* state, const NCURI* uri, NCbytes* packet); #ifdef HAVE_GETTIMEOFDAY @@ -38,82 +37,27 @@ deltatime(struct timeval time0,struct timeval time1) #endif int -NCD4_readDMR(NCD4INFO* state, int flags) +NCD4_readDMR(NCD4INFO* state, int flags, NCURI* url, NCD4response* resp) { int stat = NC_NOERR; - long lastmod = -1; - - if((flags & NCF_ONDISK) == 0) { - ncbytesclear(state->curl->packet); - stat = readpacket(state,state->uri,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&lastmod); - if(stat == NC_NOERR) - state->data.dmrlastmodified = lastmod; - } else { /*((flags & NCF_ONDISK) != 0) */ - NCURI* url = state->uri; - int fileprotocol = (strcmp(url->protocol,"file")==0); - if(fileprotocol) { - stat = readfiletofile(state, url, NCD4_DMR, NCD4_FORMAT_XML, state->data.ondiskfile, &state->data.datasize); - } else { - char* readurl = NULL; - int flags = 0; - if(!fileprotocol) flags |= NCURIQUERY; - flags |= NCURIENCODE; - flags |= NCURIPWD; -#ifdef FIX - ncurisetconstraints(url,state->constraint); -#endif - readurl = ncuribuild(url,NULL,".dmr.xml",NCURISVC); - if(readurl == NULL) - return THROW(NC_ENOMEM); - stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile, - &state->data.datasize, &lastmod); - nullfree(readurl); - if(stat == NC_NOERR) - state->data.dmrlastmodified = lastmod; - } - } + ncbytesclear(state->curl->packet); + stat = readpacket(state,url,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&resp->serial.httpcode,NULL); return THROW(stat); } int -NCD4_readDAP(NCD4INFO* state, int flags) +NCD4_readDAP(NCD4INFO* state, int flags, NCURI* url, NCD4response* resp) { int stat = NC_NOERR; - long lastmod = -1; - - if((flags & NCF_ONDISK) == 0) { - ncbytesclear(state->curl->packet); - stat = readpacket(state,state->uri,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&lastmod); - if(stat) { - NCD4_seterrormessage(state->substrate.metadata, nclistlength(state->curl->packet), nclistcontents(state->curl->packet)); - goto done; - } else - state->data.daplastmodified = lastmod; - } else { /*((flags & NCF_ONDISK) != 0) */ - NCURI* url = state->uri; - int fileprotocol = (strcmp(url->protocol,"file")==0); - if(fileprotocol) { - stat = readfiletofile(state, url, NCD4_DAP, NCD4_FORMAT_NONE, state->data.ondiskfile, &state->data.datasize); - } else { - char* readurl = NULL; - int flags = 0; - if(!fileprotocol) flags |= NCURIQUERY; - flags |= NCURIENCODE; - flags |= NCURIPWD; -#ifdef FIX - ncurisetconstraints(url,state->constraint); -#endif - readurl = ncuribuild(url,NULL,".dap",NCURISVC); - if(readurl == NULL) - return THROW(NC_ENOMEM); - stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile, - &state->data.datasize, &lastmod); - nullfree(readurl); - if(stat == NC_NOERR) - state->data.daplastmodified = lastmod; - } + + ncbytesclear(state->curl->packet); + stat = readpacket(state,url,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&resp->serial.httpcode,NULL); + if(stat) { + NCD4_seterrormessage(resp, nclistlength(state->curl->packet), nclistcontents(state->curl->packet)); + } else { + resp->raw.size = ncbyteslength(state->curl->packet); + resp->raw.memory = ncbytesextract(state->curl->packet); } -done: return THROW(stat); } @@ -150,7 +94,7 @@ dxxformat(int fxx, int dxx) } static int -readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, long* lastmodified) +readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, int* httpcodep, long* lastmodified) { int stat = NC_NOERR; int fileprotocol = 0; @@ -185,7 +129,7 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4forma gettimeofday(&time0,NULL); #endif } - stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified,&state->substrate.metadata->error.httpcode); + stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified,httpcodep); nullfree(fetchurl); if(stat) goto fail; if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { @@ -207,37 +151,26 @@ fail: return THROW(stat); } +#if 0 static int -readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep) +readfromfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, d4size_t* sizep) { int stat = NC_NOERR; - NCbytes* packet = ncbytesnew(); size_t len; - stat = readfile(state, uri, dxx, fxx, packet); + ncbytesclear(state->curl->packet); + stat = readfile(state, uri, dxx, fxx, state->curl->packet); #ifdef D4DEBUG fprintf(stderr,"readfiletofile: packet.size=%lu\n", - (unsigned long)ncbyteslength(packet)); + (unsigned long)ncbyteslength(state->curl->packet)); #endif if(stat != NC_NOERR) goto unwind; - len = nclistlength(packet); - if(stat == NC_NOERR) { - size_t written; - fseek(stream,0,SEEK_SET); - written = fwrite(ncbytescontents(packet),1,len,stream); - if(written != len) { -#ifdef D4DEBUG -fprintf(stderr,"readfiletofile: written!=length: %lu :: %lu\n", - (unsigned long)written,(unsigned long)len); -#endif - stat = NC_EIO; - } - } + len = nclistlength(state->curl->packet); if(sizep != NULL) *sizep = len; unwind: - ncbytesfree(packet); return THROW(stat); } +#endif static int readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet) @@ -365,12 +298,12 @@ done: /* Extract packet as error message; assume httpcode set */ int -NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg) +NCD4_seterrormessage(NCD4response* resp, size_t len, char* msg) { - metadata->error.message = (char*)d4alloc(len+1); - if(metadata->error.message == NULL) + resp->error.message = (char*)d4alloc(len+1); + if(resp->error.message == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->error.message,msg,len); - metadata->error.message[len] = '\0'; + memcpy(resp->error.message,msg,len); + resp->error.message[len] = '\0'; return THROW(NC_ENODATA); /* slight lie */ } diff --git a/libdap4/d4read.h b/libdap4/d4read.h index 826a2217c..39ad8d97c 100644 --- a/libdap4/d4read.h +++ b/libdap4/d4read.h @@ -7,6 +7,6 @@ #define D4READ_H extern int NCD4_readDMR(NCD4INFO*, int flags); -extern int NCD4_readDAP(NCD4INFO*, int flags); +extern int NCD4_readDAP(NCD4INFO*, int flags, NCURI* uri); #endif /*READ_H*/ diff --git a/libdap4/d4swap.c b/libdap4/d4swap.c index ed4c8dc1d..8568fd328 100644 --- a/libdap4/d4swap.c +++ b/libdap4/d4swap.c @@ -14,12 +14,12 @@ the incoming data to get the endianness correct. /* Forward */ -static int walkAtomicVar(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkOpaqueVar(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); -static int walkStructArray(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); -static int walkStruct(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkSeqArray(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkSeq(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); +static int walkAtomicVar(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkOpaqueVar(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkStructArray(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkStruct(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkSeqArray(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkSeq(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); /**************************************************/ @@ -28,43 +28,39 @@ Assumes that compiler->swap is true; does necessary byte swapping. */ int -NCD4_swapdata(NCD4meta* compiler, NClist* topvars) +NCD4_swapdata(NCD4response* resp, NCD4node* var, int doswap) { int ret = NC_NOERR; - int i; NCD4offset* offset = NULL; - offset = BUILDOFFSET(compiler->serial.dap,compiler->serial.dapsize); - for(i=0;iserial.dap,resp->serial.dapsize); OFFSET2BLOB(var->data.dap4data,offset); switch (var->subsort) { default: - if((ret=walkAtomicVar(compiler,var,var,offset))) goto done; + if((ret=walkAtomicVar(resp,var,var,offset,doswap))) goto done; break; case NC_OPAQUE: /* The only thing we need to do is swap the counts */ - if((ret=walkOpaqueVar(compiler,var,var,offset))) goto done; + if((ret=walkOpaqueVar(resp,var,var,offset,doswap))) goto done; break; case NC_STRUCT: - if((ret=walkStructArray(compiler,var,var,offset))) goto done; + if((ret=walkStructArray(resp,var,var,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeqArray(compiler,var,var,offset))) goto done; + if((ret=walkSeqArray(resp,var,var,offset,doswap))) goto done; break; } var->data.dap4data.size = DELTA(offset,var->data.dap4data.memory); /* skip checksum, if there is one */ - if(compiler->controller->data.inferredchecksumming) + if(resp->inferredchecksumming) INCR(offset,CHECKSUMSIZE); - } done: if(offset) free(offset); return THROW(ret); } static int -walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkAtomicVar(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -87,7 +83,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } else { /*(typesize > 1)*/ for(i=0;ioffset; - if(compiler->swap) { + if(doswap) { switch (typesize) { case 2: swapinline16(sp); break; case 4: swapinline32(sp); break; @@ -102,7 +98,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o COUNTERTYPE count; for(i=0;iswap) + if(doswap) swapinline64(offset); count = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -114,7 +110,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } static int -walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkOpaqueVar(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -125,7 +121,7 @@ walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o for(i=0;iswap) + if(doswap) swapinline64(offset); count = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -135,7 +131,7 @@ walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } static int -walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkStructArray(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -144,7 +140,7 @@ walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset for(i=0;ibasetype; switch (fieldbase->subsort) { default: - if((ret=walkAtomicVar(compiler,topvar,field,offset))) goto done; + if((ret=walkAtomicVar(resp,topvar,field,offset,doswap))) goto done; break; case NC_OPAQUE: /* The only thing we need to do is swap the counts */ - if((ret=walkOpaqueVar(compiler,topvar,field,offset))) goto done; + if((ret=walkOpaqueVar(resp,topvar,field,offset,doswap))) goto done; break; case NC_STRUCT: - if((ret=walkStructArray(compiler,topvar,field,offset))) goto done; + if((ret=walkStructArray(resp,topvar,field,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeqArray(compiler,topvar,field,offset))) goto done; + if((ret=walkSeqArray(resp,topvar,field,offset,doswap))) goto done; break; } } @@ -182,7 +178,7 @@ done: } static int -walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkSeqArray(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -195,7 +191,7 @@ walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* of for(i=0;iswap) + if(doswap) swapinline64(offset); recordcount = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -225,17 +221,17 @@ walkSeq(NCD4meta* compiler, NCD4node* topvar, NCD4node* vlentype, NCD4offset* of for(i=0;isubsort) { default: /* atomic basetype */ - if((ret=walkAtomicVar(compiler,topvar,basetype,offset))) goto done; + if((ret=walkAtomicVar(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_OPAQUE: - if((ret=walkOpaqueVar(compiler,topvar,basetype,offset))) goto done; + if((ret=walkOpaqueVar(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_STRUCT: /* We can treat each record like a structure instance */ - if((ret=walkStruct(compiler,topvar,basetype,offset))) goto done; + if((ret=walkStruct(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeq(compiler,topvar,basetype,offset))) goto done; + if((ret=walkSeq(resp,topvar,basetype,offset,doswap))) goto done; break; } } diff --git a/libdap4/d4util.c b/libdap4/d4util.c index 415651930..d1ba7f582 100644 --- a/libdap4/d4util.c +++ b/libdap4/d4util.c @@ -107,44 +107,45 @@ NCD4_makeFQN(NCD4node* node) { char* fqn = NULL; char* escaped; - int i; - NCD4node* g = node; - NClist* path = nclistnew(); - size_t estimate; + NCbytes* buf = ncbytesnew(); + NClist* grps = nclistnew(); + NClist* parts = nclistnew(); + NCD4node* n; + size_t i; - for(estimate=0;g != NULL;g=g->container) { - estimate += strlen(g->name); - nclistinsert(path,0,g); + /* collect all the non-groups */ + for(n=node;n;n=n->container) { + if(ISGROUP(n->sort)) + nclistinsert(grps,0,n); /* keep the correct order of groups */ + else + nclistinsert(parts,0,n); } - estimate = (estimate*2) + 2*nclistlength(path); - estimate++; /*strlcat nul*/ - fqn = (char*)malloc(estimate+1); - if(fqn == NULL) goto done; - fqn[0] = '\0'; - /* Create the group-based fqn prefix */ - /* start at 1 to avoid dataset */ - for(i=1;isort != NCD4_GROUP) break; + + /* Build grp prefix of the fqn */ + for(i=1;iname); - if(escaped == NULL) {free(fqn); fqn = NULL; goto done;} - strlcat(fqn,"/",estimate); - strlcat(fqn,escaped,estimate); + escaped = backslashEscape(n->name); + if(escaped == NULL) goto done; + ncbytescat(buf,"/"); + ncbytescat(buf,escaped); free(escaped); } /* Add in the final name part (if not group) */ - if(i < nclistlength(path)) { - int last = nclistlength(path)-1; - NCD4node* n = (NCD4node*)nclistget(path,last); - char* name = NCD4_makeName(n,"."); - strlcat(fqn,"/",estimate); - strlcat(fqn,name,estimate); - nullfree(name); + for(i=0;iname); + if(escaped == NULL) goto done; + ncbytescat(buf,(i==0?"/":".")); + ncbytescat(buf,escaped); + free(escaped); } + fqn = ncbytesextract(buf); done: - nclistfree(path); + ncbytesfree(buf); + nclistfree(grps); + nclistfree(parts); return fqn; } @@ -446,11 +447,9 @@ NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian) } void -NCD4_reporterror(NCD4INFO* state) +NCD4_reporterror(NCD4response* resp, NCURI* uri) { - NCD4meta* meta = state->substrate.metadata; char* u = NULL; - if(meta == NULL) return; - u = ncuribuild(state->uri,NULL,NULL,NCURIALL); - fprintf(stderr,"***FAIL: url=%s httpcode=%d errmsg->\n%s\n",u,meta->error.httpcode,meta->error.message); + u = ncuribuild(uri,NULL,NULL,NCURIALL); + fprintf(stderr,"***FAIL: url=%s httpcode=%d errmsg->\n%s\n",u,resp->serial.httpcode,resp->error.message); } diff --git a/libdap4/d4util.h b/libdap4/d4util.h index b6cbde8f0..38cda6a17 100644 --- a/libdap4/d4util.h +++ b/libdap4/d4util.h @@ -17,9 +17,12 @@ */ typedef unsigned long long d4size_t; -/* Define a counted memory marker */ +/* Define a (size, memory) pair */ typedef struct D4blob {d4size_t size; void* memory;} D4blob; +/* Empty blob constant */ +#define NULLBLOB(blob) {blob.size = 0; blob.memory = NULL;} + #define OFFSET2BLOB(blob,offset) do{(blob).size = ((offset)->limit - (offset)->base); (blob).memory = (offset)->base; }while(0) #define BLOB2OFFSET(offset,blob) do{\ (offset)->base = (blob).memory; \ diff --git a/libdap4/d4varx.c b/libdap4/d4varx.c index 9a708a5d5..170171800 100644 --- a/libdap4/d4varx.c +++ b/libdap4/d4varx.c @@ -11,7 +11,8 @@ #include "d4odom.h" /* Forward */ -static int getvarx(int ncid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*); +static int getvarx(int gid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*); +static int mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming); int NCD4_get_vara(int ncid, int varid, @@ -22,11 +23,11 @@ NCD4_get_vara(int ncid, int varid, int ret; /* TODO: optimize since we know stride is 1 */ ret = NCD4_get_vars(ncid,varid,start,edges,NC_stride_one,value,memtype); - return ret; + return THROW(ret); } int -NCD4_get_vars(int ncid, int varid, +NCD4_get_vars(int gid, int varid, const size_t *start, const size_t *edges, const ptrdiff_t* stride, void *memoryin, nc_type xtype) { @@ -43,14 +44,14 @@ NCD4_get_vars(int ncid, int varid, int rank; size_t dimsizes[NC_MAX_VAR_DIMS]; d4size_t dimproduct; - size_t dstcount; + size_t dstpos; NCD4offset* offset = NULL; - /* Get netcdf type info */ - if((ret=getvarx(ncid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size))) + /* Get netcdf var metadata and data */ + if((ret=getvarx(gid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size))) {goto done;} - meta = info->substrate.metadata; + meta = info->dmrmetadata; nctype = ncvar->basetype; rank = nclistlength(ncvar->dims); blobs = nclistnew(); @@ -74,18 +75,18 @@ NCD4_get_vars(int ncid, int varid, odom = d4odom_new(rank,start,edges,stride,dimsizes); else odom = d4scalarodom_new(); - dstcount = 0; /* We always write into dst starting at position 0*/ - for(;d4odom_more(odom);dstcount++) { + dstpos = 0; /* We always write into dst starting at position 0*/ + for(;d4odom_more(odom);dstpos++) { void* xpos; void* dst; - d4size_t count; + d4size_t pos; - count = d4odom_next(odom); - if(count >= dimproduct) { + pos = d4odom_next(odom); + if(pos >= dimproduct) { ret = THROW(NC_EINVALCOORDS); goto done; } - xpos = ((char*)memoryin)+(xsize * dstcount); /* ultimate destination */ + xpos = ((char*)memoryin)+(xsize * dstpos); /* ultimate destination */ /* We need to compute the offset in the dap4 data of this instance; for fixed size types, this is easy, otherwise we have to walk the variable size type @@ -95,16 +96,16 @@ NCD4_get_vars(int ncid, int varid, offset = NULL; offset = BUILDOFFSET(NULL,0); BLOB2OFFSET(offset,ncvar->data.dap4data); - /* Move offset to the count'th element of the array */ + /* Move offset to the pos'th element of the array */ if(nctype->meta.isfixedsize) { - INCR(offset,(dapsize*count)); + INCR(offset,(dapsize*pos)); } else { - /* We have to walk to the count'th location in the data */ - if((ret=NCD4_moveto(meta,ncvar,count,offset))) + /* We have to walk to the pos'th location in the data */ + if((ret=NCD4_moveto(meta,ncvar,pos,offset))) {goto done;} } dst = instance; - if((ret=NCD4_fillinstance(meta,nctype,offset,&dst,blobs))) + if((ret=NCD4_movetoinstance(meta,nctype,offset,&dst,blobs))) {goto done;} if(xtype == nc4type) { /* We can just copy out the data */ @@ -132,45 +133,29 @@ done: } static int -getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, +getvarx(int gid, int varid, NCD4INFO** infop, NCD4node** varp, nc_type* xtypep, size_t* xsizep, nc_type* nc4typep, size_t* nc4sizep) { int ret = NC_NOERR; - NC* ncp; - NCD4INFO* info; - NCD4meta* meta; - NCD4node* group; - NCD4node* var; - NCD4node* type; + NC* ncp = NULL; + NCD4INFO* info = NULL; + NCD4meta* dmrmeta = NULL; + NCD4node* group = NULL; + NCD4node* var = NULL; + NCD4node* type = NULL; nc_type xtype, actualtype; size_t instancesize, xsize; + NCURI* ceuri = NULL; /* Constrained uri */ + NCD4meta* dapmeta = NULL; + NCD4response* dapresp = NULL; - if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) + if((ret = NC_check_id(gid, (NC**)&ncp)) != NC_NOERR) goto done; info = getdap(ncp); - meta = info->substrate.metadata; - - /* If the data has not already been read and processed, then do so. */ - if(meta->serial.dap == NULL) { - size_t len = 0; - void* content = NULL; - /* (Re)Build the meta data; sets serial.rawdata */ - NCD4_resetMeta(info->substrate.metadata); - meta->controller = info; - meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ - - if((ret=NCD4_readDAP(info, info->controls.flags.flags))) goto done; - len = ncbyteslength(info->curl->packet); - content = ncbytesextract(info->curl->packet); - NCD4_resetSerial(&meta->serial, len, content); - /* Process the data part */ - if((ret=NCD4_dechunk(meta))) goto done; - if((ret = NCD4_processdata(info->substrate.metadata))) goto done; - } - - if((ret = NCD4_findvar(ncp,ncid,varid,&var,&group))) goto done; + dmrmeta = info->dmrmetadata; + if((ret = NCD4_findvar(ncp,gid,varid,&var,&group))) goto done; type = var->basetype; actualtype = type->meta.id; instancesize = type->meta.memsize; @@ -189,6 +174,39 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, else xsize = instancesize; + /* If we already have valid data, then just return */ + if(var->data.valid) goto validated; + + /* Ok, we need to read from the server */ + + /* Add the variable to the URI, unless the URI is already constrained or is unconstrainable */ + ceuri = ncuriclone(info->dmruri); + /* append the request for a specific variable */ + if(ncuriquerylookup(ceuri,DAP4CE) == NULL && !FLAGSET(info->controls.flags,NCF_UNCONSTRAINABLE)) { + ncurisetquerykey(ceuri,strdup("dap4.ce"),NCD4_makeFQN(var)); + } + + /* Read and process the data */ + /* Setup the meta-data for the DAP */ + if((ret=NCD4_newMeta(info,&dapmeta))) goto done; + if((ret=NCD4_newResponse(info,&dapresp))) goto done; + dapresp->mode = NCD4_DAP; + nclistpush(info->responses,dapresp); + if((ret=NCD4_readDAP(info, info->controls.flags.flags, ceuri, dapresp))) goto done; + /* Extract DMR and dechunk the data part */ + if((ret=NCD4_dechunk(dapresp))) goto done; + /* Process the dmr part */ + if((ret=NCD4_parse(dapmeta,dapresp,1))) goto done; + /* See if we are checksumming */ + if((ret=NCD4_inferChecksums(dapmeta,dapresp))) goto done; + /* connect variables and corresponding dap data */ + if((ret = NCD4_parcelvars(dapmeta,dapresp))) goto done; + /* Process checksums and byte-order swapping */ + if((ret = NCD4_processdata(dapmeta,dapresp))) goto done; + /* Transfer and process the data */ + if((ret = mapvars(dapmeta,dmrmeta,dapresp->inferredchecksumming))) goto done; + +validated: /* Return relevant info */ if(infop) *infop = info; if(xtypep) *xtypep = xtype; @@ -197,8 +215,101 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, if(nc4sizep) *nc4sizep = instancesize; if(varp) *varp = var; done: - if(meta->error.message != NULL) - NCD4_reporterror(info); /* Make sure the user sees this */ + if(dapmeta) NCD4_reclaimMeta(dapmeta); + ncurifree(ceuri); + if(dapresp != NULL && dapresp->error.message != NULL) + NCD4_reporterror(dapresp,ceuri); /* Make sure the user sees this */ return THROW(ret); } +#if 0 +static NCD4node* +findbyname(const char* name, NClist* nodes) +{ + int i; + for(i=0;iname)==0) + return node; + } + return NULL; +} +#endif + +static int +matchvar(NCD4meta* dmrmeta, NCD4node* dapvar, NCD4node** dmrvarp) +{ + int i,ret = NC_NOERR; + NCD4node* x = NULL; + NClist* dappath = nclistnew(); + NClist* dmrpath = nclistnew(); /* compute path for this dmr var */ + int found = 0; + NCD4node* match = NULL; + + /* Capture the dap path starting at root and ending at the dapvar (assumed to be topvar) */ + for(x=dapvar;x != NULL;x=x->container) nclistinsert(dappath,0,x); + /* Iterate over all variable nodes to find matching one */ + for(i=0;iallnodes);i++) { + NCD4node* node = (NCD4node*)nclistget(dmrmeta->allnodes,i); + if(ISVAR(node->sort) && strcmp(node->name,dapvar->name)==0) { /* possible candidate */ + int j; + found = 0; + nclistclear(dmrpath); + for(x=node;x != NULL;x=x->container) nclistinsert(dmrpath,0,x); + if(nclistlength(dmrpath) == nclistlength(dappath)) { /* same length paths */ + /* compare paths: name and sort */ + for(found=1,j=0;jsort != pdap->sort || strcmp(pdmr->name,pdap->name) != 0) + {found = 0; break;} + } + if(found) {match = node; break;} + } + } + } + if(!found) {ret = NC_EINVAL; goto done;} + if(dmrvarp) *dmrvarp = match; + +done: + nclistfree(dappath); + nclistfree(dmrpath); + return THROW(ret); +} + +/* +Map each toplevel dap var to the corresponding +toplevel dmr var and transfer necessary info; +*/ + +static int +mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming) +{ + int i, ret = NC_NOERR; + NCD4node* dmrroot = dmrmeta->root; + NClist* dmrtop = NULL; /* top variables in dmr tree */ + NCD4node* daproot = dapmeta->root; + NClist* daptop = NULL; /* top variables in dap tree */ + + /* Get top level variables from the dmr node tree */ + dmrtop = nclistnew(); + NCD4_getToplevelVars(dmrmeta,dmrroot,dmrtop); + + /* Get top level variables from the dap node tree */ + daptop = nclistnew(); + NCD4_getToplevelVars(dapmeta,daproot,daptop); + + /* Match up the dap top variables with the dmr top variables */ + for(i=0;idata = dapvar->data; + memset(&dapvar->data,0,sizeof(NCD4vardata)); + dmrvar->data.valid = 1; + } + +done: + return THROW(ret); +} diff --git a/libdap4/ncd4.h b/libdap4/ncd4.h index 058e40ff7..286406d77 100644 --- a/libdap4/ncd4.h +++ b/libdap4/ncd4.h @@ -77,7 +77,6 @@ EXTERNL int dsp_open(const char* path, ND4dsp** dspp); /* From d4http.c */ EXTERNL long NCD4_fetchhttpcode(CURL* curl); -EXTERNL int NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, d4size_t* sizep, long* filetime); EXTERNL int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcode); EXTERNL int NCD4_curlopen(CURL** curlp); EXTERNL void NCD4_curlclose(CURL* curl); @@ -85,15 +84,16 @@ EXTERNL int NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime); EXTERNL int NCD4_ping(const char* url); /* From d4read.c */ -EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags); -EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags); -EXTERNL int NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg); +EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags, NCURI* url, NCD4response*); +EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags, NCURI* ceuri, NCD4response*); +EXTERNL int NCD4_seterrormessage(NCD4response*, size_t len, char* msg); /* From d4parser.c */ -EXTERNL int NCD4_parse(NCD4meta*); +EXTERNL int NCD4_parse(NCD4meta*, NCD4response*, int dapparse); EXTERNL NCD4node* NCD4_findAttr(NCD4node* container, const char* attrname); EXTERNL NCD4node* NCD4_groupFor(NCD4node* node); EXTERNL int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char* typename, NCD4node** attrp); +EXTERNL void NCD4_setdebuglevel(NCD4parser*,int); /* From d4printer.c */ EXTERNL int NCD4_print(NCD4meta*, NCbytes* output); @@ -104,29 +104,33 @@ EXTERNL void NCD4_attachraw(NCD4meta*, size_t size, void* rawdata); EXTERNL void NCD4_reclaimMeta(NCD4meta*); EXTERNL void NCD4_resetMeta(NCD4meta*); EXTERNL void reclaimNode(NCD4node* node); -EXTERNL void NCD4_setdebuglevel(NCD4meta*,int); EXTERNL int NCD4_metabuild(NCD4meta*, int ncid); EXTERNL size_t NCD4_computeTypeSize(NCD4meta*, NCD4node* type); EXTERNL int NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp); +EXTERNL char* NCD4_getVarFQN(NCD4node* var, const char* tail); + /* From d4chunk.c */ -EXTERNL int NCD4_dechunk(NCD4meta*); -EXTERNL int NCD4_infermode(NCD4meta* meta); +EXTERNL int NCD4_dechunk(NCD4response*); +EXTERNL int NCD4_infermode(NCD4response*); struct NCD4serial; EXTERNL void NCD4_resetSerial(struct NCD4serial* serial, size_t rawsize, void* rawdata); +EXTERNL void NCD4_moveSerial(struct NCD4serial* serial, struct NCD4serial* dst); /* From d4swap.c */ -EXTERNL int NCD4_swapdata(NCD4meta*, NClist* topvars); +EXTERNL int NCD4_swapdata(NCD4response*, NCD4node* topvar, int doswap); /* From d4fix.c */ -EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, NCD4offset* offset); +EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, NCD4offset* offset, int inferredchecksumming); EXTERNL int NCD4_moveto(NCD4meta*, NCD4node* var, d4size_t count, NCD4offset* offset); EXTERNL int NCD4_toposort(NCD4meta*); /* From d4data.c */ -EXTERNL int NCD4_processdata(NCD4meta*); -EXTERNL int NCD4_fillinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); +EXTERNL int NCD4_parcelvars(NCD4meta* meta, NCD4response* resp); +EXTERNL int NCD4_processdata(NCD4meta*,NCD4response*); +EXTERNL int NCD4_movetoinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); EXTERNL int NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel); +EXTERNL int NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp); /* From d4util.c */ EXTERNL d4size_t NCD4_dimproduct(NCD4node* node); @@ -141,7 +145,7 @@ EXTERNL char* NCD4_deescape(const char* esc); EXTERNL char* NCD4_entityescape(const char* s); EXTERNL size_t NCD4_elidenuls(char* s, size_t slen); EXTERNL void* NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian); -EXTERNL void NCD4_reporterror(NCD4INFO* state); +EXTERNL void NCD4_reporterror(NCD4response*, NCURI* uri); /* From d4dump.c */ EXTERNL void NCD4_dumpbytes(size_t size, const void* data0, int swap); @@ -163,7 +167,15 @@ EXTERNL int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char* /* d4file.c */ EXTERNL int NCD4_readDMRorDAP(NCD4INFO* d4info, NCD4mode mode); EXTERNL void NCD4_applyclientfragmentcontrols(NCD4INFO* d4info); -EXTERNL void NCD4_applyclientquerycontrols(NCD4INFO* d4info); +EXTERNL void NCD4_applychecksumcontrols(NCD4INFO* d4info, NCD4response*); +EXTERNL int NCD4_newInfo(NCD4INFO** d4infop); +EXTERNL void NCD4_reclaimInfo(NCD4INFO* d4info); +EXTERNL void NCD4_resetInfoforRead(NCD4INFO* d4info); +EXTERNL int NCD4_newResponse(NCD4INFO*,NCD4response** respp); +EXTERNL void NCD4_reclaimResponse(NCD4response* d4resp); +EXTERNL void NCD4_resetInfoForRead(NCD4INFO* d4info); +EXTERNL int NCD4_newMeta(NCD4INFO*,NCD4meta**); +EXTERNL void NCD4_reclaimMeta(NCD4meta*); /* ncd4dispatch.c */ struct NC_reservedatt; /*forward*/ diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c index 9d9cf08a1..5f5ebbfec 100644 --- a/libdap4/ncd4dispatch.c +++ b/libdap4/ncd4dispatch.c @@ -847,7 +847,7 @@ NCD4_inq_dim(int ncid, int dimid, char* name, size_t* lenp) if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) goto done; info = (NCD4INFO*)ncp->dispatchdata; - meta = info->substrate.metadata; + meta = info->dmrmetadata; /* Locate the dimension specified by dimid */ for(i=0;iallnodes);i++) { @@ -871,16 +871,15 @@ static int ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* value, nc_type t, const NC_reservedatt* rsvp) { int ret = NC_NOERR; - NCD4INFO* info = (NCD4INFO*)(ncp->dispatchdata); - NCD4meta* meta = info->substrate.metadata; NCD4node* var = NULL; + if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done; + if(strcmp(rsvp->name,D4CHECKSUMATTR)==0) { unsigned int* ip = (unsigned int*)value; if(varid == NC_GLOBAL) {ret = NC_EBADID; goto done;} if(t != NC_UINT) {ret = NC_EBADTYPE; goto done;} - if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done; if(var->data.checksumattr == 0) {ret = NC_ENOTATT; goto done;} *ip = (var->data.remotechecksum); @@ -889,7 +888,7 @@ ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* valu if(varid != NC_GLOBAL) {ret = NC_EBADID; goto done;} if(t != NC_INT) {ret = NC_EBADTYPE; goto done;} - *ip = (meta->serial.remotelittleendian?1:0); + *ip = (var->data.response->remotelittleendian?1:0); } done: return THROW(ret); @@ -925,7 +924,7 @@ static int globalinit(void) { int stat = NC_NOERR; - return stat; + return THROW(stat); } /**************************************************/ diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h index 1f84a0f56..ca0a5118a 100644 --- a/libdap4/ncd4types.h +++ b/libdap4/ncd4types.h @@ -50,6 +50,8 @@ typedef struct NCD4node NCD4node; typedef struct NCD4params NCD4params; typedef struct NCD4HDR NCD4HDR; typedef struct NCD4offset NCD4offset; +typedef struct NCD4vardata NCD4vardata; +typedef struct NCD4response NCD4response; /* Define the NCD4HDR flags */ /* Header flags */ @@ -228,19 +230,20 @@ struct NCD4node { int isfixedsize; /* sort == NCD4_TYPE; Is this a fixed size (recursively) type? */ d4size_t dapsize; /* size of the type as stored in the dap data; will, as a rule, be same as memsize only for types <= NC_UINT64 */ - nc_type cmpdid; /*netcdf id for the compound type created for seq type */ + nc_type cmpdid; /* netcdf id for the compound type created for seq type */ size_t memsize; /* size of a memory instance without taking dimproduct into account, but taking compound alignment into account */ d4size_t offset; /* computed structure field offset in memory */ size_t alignment; /* computed structure field alignment in memory */ } meta; - struct { /* Data compilation info */ - int flags; /* See d4data for actual flags */ + struct NCD4vardata { /* Data compilation info */ + int valid; /* 1 => this contains valid data */ D4blob dap4data; /* offset and start pos for this var's data in serialization */ - unsigned int remotechecksum; /* checksum from data as sent by server */ - unsigned int localchecksum; /* toplevel variable checksum as computed by client */ - int checksumattr; /* 1=> _DAP4_Checksum_CRC32 is defined */ - int attrchecksum; /* _DAP4_Checksum_CRC32 value */ + unsigned remotechecksum; /* toplevel per-variable checksum contained in the data */ + unsigned localchecksum; /* toplevel variable checksum as computed by client */ + int checksumattr; /* 1 => _DAP4_Checksum_CRC32 is defined */ + unsigned attrchecksum; /* _DAP4_Checksum_CRC32 value; this is the checksum computed by server */ + NCD4response* response; /* Response from which this data is taken */ } data; struct { /* Track netcdf-4 conversion info */ int isvlen; /* _edu.ucar.isvlen */ @@ -253,36 +256,13 @@ struct NCD4node { } nc4; }; -/* Tracking info about the serialized input before and after de-chunking */ -typedef struct NCD4serial { - size_t rawsize; /* |rawdata| */ - void* rawdata; - size_t dapsize; /* |dap|; this is transient */ - void* dap; /* pointer into rawdata where dap data starts */ - char* dmr;/* copy of dmr */ - char* errdata; /* null || error chunk (null terminated) */ - int httpcode; /* returned from last request */ - int hostlittleendian; /* 1 if the host is little endian */ - int remotelittleendian; /* 1 if the packet says data is little endian */ -} NCD4serial; - -/* This will be passed out of the parse */ +/* DMR information from a response; this will be passed out of the parse */ struct NCD4meta { NCD4INFO* controller; int ncid; /* root ncid of the substrate netcdf-4 file; warning: copy of NCD4Info.substrate.nc4id */ NCD4node* root; - NCD4mode mode; /* Are we reading DMR (only) or DAP (includes DMR) */ NClist* allnodes; /*list*/ - struct Error { /* Content of any error response */ - char* parseerror; - int httpcode; - char* message; - char* context; - char* otherinfo; - } error; - int debuglevel; - NCD4serial serial; int swap; /* 1 => swap data */ /* Define some "global" (to a DMR) data */ NClist* groupbyid; /* NClist indexed by groupid >> 16; this is global */ @@ -292,9 +272,12 @@ struct NCD4meta { }; typedef struct NCD4parser { + NCD4INFO* controller; char* input; int debuglevel; + int dapparse; /* 1 => we are parsing the DAP DMR */ NCD4meta* metadata; + NCD4response* response; /* Capture useful subsets of dataset->allnodes */ NClist* types; /*list; user-defined types only*/ NClist* dims; /*list*/ @@ -303,6 +286,32 @@ typedef struct NCD4parser { NCD4node* dapopaque; /* Single non-fixed-size opaque type */ } NCD4parser; +/* Capture all the relevant info about the response to a server request */ +struct NCD4response { /* possibly processed response from a query */ + NCD4INFO* controller; /* controlling connection */ + D4blob raw; /* complete response in memory */ + int querychecksumming; /* 1 => user specified dap4.ce value */ + int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */ + int inferredchecksumming; /* 1 => either query checksum || att checksum */ + int checksumignore; /* 1 => assume checksum, but do not validate */ + int remotelittleendian; /* 1 if the packet says data is little endian */ + NCD4mode mode; /* Are we reading DMR (only) or DAP (includes DMR) */ + struct NCD4serial { + size_t dapsize; /* |dap|; this is transient */ + void* dap; /* pointer into raw where dap data starts */ + char* dmr;/* copy of dmr */ + char* errdata; /* null || error chunk (null terminated) */ + int httpcode; /* returned from last request */ + } serial; /* Dechunked and processed DAP part of the response */ + struct Error { /* Content of any error response */ + char* parseerror; + int httpcode; + char* message; + char* context; + char* otherinfo; + } error; +}; + /**************************************************/ /* Curl info */ @@ -328,28 +337,20 @@ struct NCD4curl { struct NCD4INFO { NC* controller; /* Parent instance of NCD4INFO */ - char* rawurltext; /* as given to ncd4_open */ - char* urltext; /* as modified by ncd4_open */ - NCURI* uri; /* parse of rawuritext */ + char* rawdmrurltext; /* as given to ncd4_open */ + char* dmrurltext; /* as modified by ncd4_open */ + NCURI* dmruri; /* parse of rawuritext */ NCD4curl* curl; int inmemory; /* store fetched data in memory? */ - struct { - char* memory; /* allocated memory if ONDISK is not set */ - char* ondiskfilename; /* If ONDISK is set */ - FILE* ondiskfile; /* ditto */ - d4size_t datasize; /* size on disk or in memory */ - long dmrlastmodified; - long daplastmodified; - int querychecksumming; /* 1 => user specified dap4.ce value */ - int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */ - int inferredchecksumming; /* 1 => either query checksum || att checksum */ - int checksumignore; /* 1 => assume checksum, but do not validate */ - } data; + NCD4meta* dmrmetadata; + NClist* responses; /* NClist all responses from this curl handle */ + struct { /* Properties that are per-platform */ + int hostlittleendian; /* 1 if the host is little endian */ + } platform; struct { int realfile; /* 1 => we created actual temp file */ char* filename; /* of the substrate file */ int nc4id; /* substrate nc4 file ncid used to hold metadata; not same as external id */ - NCD4meta* metadata; } substrate; struct { NCCONTROLS flags; @@ -362,6 +363,7 @@ struct NCD4INFO { struct { char* filename; } fileproto; + int debuglevel; NClist* blobs; }; diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 937fd8bd0..3657afeda 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -203,7 +203,7 @@ static struct NCPROTOCOLLIST { static int NC_omodeinfer(int useparallel, int omode, NCmodel*); static int check_file_type(const char *path, int omode, int use_parallel, void *parameters, NCmodel* model, NCURI* uri); static int processuri(const char* path, NCURI** urip, NClist* fraglist); -static int processmacros(NClist** fraglistp); +static int processmacros(NClist* fraglistp, NClist* expanded); static char* envvlist2string(NClist* pairs, const char*); static void set_default_mode(int* cmodep); static int parseonchar(const char* s, int ch, NClist* segments); @@ -240,8 +240,8 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv) NCURI* uri = NULL; size_t pathlen = strlen(path); char* str = NULL; - const char** ufrags; - const char** p; + const NClist* ufrags; + size_t i; if(path == NULL || pathlen == 0) {stat = NC_EURL; goto done;} @@ -263,7 +263,6 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv) /* process the corresponding fragments for that protocol */ if(protolist->fragments != NULL) { - int i; tmp = nclistnew(); if((stat = parseonchar(protolist->fragments,'&',tmp))) goto done; for(i=0;isubstitute) ncurisetprotocol(uri,protolist->substitute); /* capture the fragments of the url */ - ufrags = ncurifragmentparams(uri); - if(ufrags != NULL) { - for(p=ufrags;*p;p+=2) { - const char* key = p[0]; - const char* value = p[1]; - nclistpush(fraglenv,nulldup(key)); - value = (value==NULL?"":value); - nclistpush(fraglenv,strdup(value)); - } + ufrags = (const NClist*)ncurifragmentparams(uri); + for(i=0;i 0) { + for(i=0;iname;macros++) { if(strcmp(macros->name,key)==0) { @@ -472,14 +462,8 @@ processmacros(NClist** fraglenvp) nclistpush(expanded,strdup(key)); nclistpush(expanded,strdup(value)); } - nullfree(key); - nullfree(value); } - *fraglenvp = expanded; expanded = NULL; -done: - nclistfreeall(expanded); - nclistfreeall(fraglenv); return check(stat); } @@ -495,8 +479,6 @@ processinferences(NClist* fraglenv) int i; char* newmodeval = NULL; - if(fraglenv == NULL || nclistlength(fraglenv) == 0) goto done; - /* Get "mode" entry */ if((modeval = getmodekey(fraglenv))==NULL) goto done; @@ -534,7 +516,7 @@ printlist(newmodes,"processinferences: new mode list"); nextmodes = tmp; tmp = NULL; } - /* cleanup any unused elements in currenmodes */ + /* cleanup any unused elements in currentmodes */ nclistclearall(currentmodes); /* Ensure no duplicates */ @@ -689,21 +671,15 @@ collectallkeys(NClist* fraglenv, NClist* allkeys) /* Given a fragment envv list, coalesce duplicate keys and remove duplicate values*/ static int -cleanfragments(NClist** fraglenvp) +cleanfragments(NClist* fraglenv, NClist* newlist) { int i,stat = NC_NOERR; - NClist* fraglenv = NULL; NClist* tmp = NULL; NClist* allkeys = NULL; - NClist* newlist = NULL; NCbytes* buf = NULL; char* key = NULL; char* value = NULL; - if(fraglenvp == NULL || nclistlength(*fraglenvp) == 0) return NC_NOERR; - fraglenv = *fraglenvp; /* take control of this list */ - *fraglenvp = NULL; - newlist = nclistnew(); buf = ncbytesnew(); allkeys = nclistnew(); tmp = nclistnew(); @@ -723,13 +699,10 @@ cleanfragments(NClist** fraglenvp) nclistpush(newlist,value); nclistclear(tmp); } - *fraglenvp = newlist; newlist = NULL; done: nclistfree(allkeys); nclistfree(tmp); ncbytesfree(buf); - nclistfreeall(fraglenv); - nclistfreeall(newlist); return check(stat); } @@ -869,7 +842,8 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void char* sfrag = NULL; const char* modeval = NULL; char* abspath = NULL; - + NClist* tmp = NULL; + /* Phase 1: 1. convert special protocols to http|https 2. begin collecting fragments @@ -882,13 +856,20 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void #endif /* Phase 2: Expand macros and add to fraglenv */ - if((stat = processmacros(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = processmacros(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; #ifdef DEBUG printlist(fraglenv,"processmacros"); #endif - /* Cleanup the fragment list */ - if((stat = cleanfragments(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = cleanfragments(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; /* Phase 2a: Expand mode inferences and add to fraglenv */ if((stat = processinferences(fraglenv))) goto done; @@ -897,7 +878,11 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void #endif /* Phase 3: coalesce duplicate fragment keys and remove duplicate values */ - if((stat = cleanfragments(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = cleanfragments(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; #ifdef DEBUG printlist(fraglenv,"cleanfragments"); #endif @@ -947,13 +932,12 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void /* Phase 6: Process the non-mode keys to see if we can tell the formatx */ if(!modelcomplete(model)) { - const char** p = ncurifragmentparams(uri); /* envv format */ - if(p != NULL) { - for(;*p;p+=2) { - const char* key = p[0]; - const char* value = p[1];; - if((stat=processfragmentkeys(key,value,model))) goto done; - } + size_t i; + NClist* p = (NClist*)ncurifragmentparams(uri); /* envv format */ + for(i=0;i 0) { - nclistpush(params,NULL); - tmp.fraglist = nclistextract(params); - } else - tmp.fraglist = NULL; + tmp.fraglist = params; + params = NULL; + /* Parse the query */ if(tmp.query != NULL) { if(parselist(tmp.query,querylist) != NC_NOERR) {THROW(NC_EURL);} - if(nclistlength(querylist) > 0) { - nclistpush(querylist,NULL); - tmp.querylist = nclistextract(querylist); - } + tmp.querylist = querylist; + querylist = NULL; } /* Now parse the core of the url */ @@ -312,7 +302,7 @@ ncuriparse(const char* uri0, NCURI** durip) pathchar = EOFCHAR; } else { /* assume there should be a host section */ /* We already extracted the query and/or fragment sections above, - splocate the end of the host section and therefore the start + so locate the end of the host section and therefore the start of the path. */ tmp.host = p; @@ -430,9 +420,9 @@ done: freestringlist(params); freestringlist(querylist); if(tmp.fraglist) - freestringvec(tmp.fraglist); + nclistfreeall(tmp.fraglist); if(tmp.querylist) - freestringvec(tmp.querylist); + nclistfreeall(tmp.querylist); return ret; } @@ -450,6 +440,7 @@ freestringlist(NClist* list) } } +#if 0 static void freestringvec(char** list) { @@ -459,6 +450,7 @@ freestringvec(char** list) nullfree(list); } } +#endif void ncurifree(NCURI* duri) @@ -473,8 +465,8 @@ ncurifree(NCURI* duri) nullfree(duri->path); nullfree(duri->query); nullfree(duri->fragment); - freestringvec(duri->querylist); - freestringvec(duri->fraglist); + nclistfreeall(duri->querylist); + nclistfreeall(duri->fraglist); free(duri); } @@ -510,21 +502,14 @@ int ncurisetquery(NCURI* duri,const char* query) { int ret = NC_NOERR; - freestringvec(duri->querylist); + nclistfreeall((NClist*)duri->querylist); nullfree(duri->query); duri->query = NULL; duri->querylist = NULL; if(query != NULL && strlen(query) > 0) { - NClist* params = nclistnew(); - duri->query = strdup(query); - ret = parselist(duri->query,params); - if(ret != NC_NOERR) - {THROW(NC_EURL);} - nclistpush(params,NULL); - duri->querylist = nclistextract(params); - nclistfree(params); + duri->query = strdup(query); + ensurequerylist(duri); } -done: return ret; } @@ -533,12 +518,13 @@ int ncurisetfragments(NCURI* duri,const char* fragments) { int ret = NC_NOERR; - freestringvec(duri->fraglist); + nclistfreeall((NClist*)duri->fraglist); nullfree(duri->fragment); duri->fragment = NULL; duri->fraglist = NULL; if(fragments != NULL && strlen(fragments) > 0) { duri->fragment = strdup(fragments); + ensurefraglist(duri); } return ret; } @@ -559,17 +545,20 @@ ncurisetfragmentkey(NCURI* duri,const char* key, const char* value) { int ret = NC_NOERR; int pos = -1; - char* newlist = NULL; ensurefraglist(duri); pos = ncfind(duri->fraglist, key); - if(pos < 0) return NC_EINVAL; /* does not exist */ - nullfree(duri->fraglist[pos+1]); - duri->fraglist[pos+1] = strdup(value); + if(pos < 0) { /* does not exist */ + if(duri->fraglist == NULL) duri->fraglist = nclistnew(); + nclistpush(duri->fraglist,key); + nclistpush(duri->fraglist,value); + } else { + nullfree(nclistget(duri->fraglist,pos+1)); + nclistset(duri->fraglist,pos+1,strdup(value)); + } /* Rebuild the fragment */ - if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done; - nullfree(duri->fragment); - duri->fragment = newlist; newlist = NULL; + nullfree(duri->fragment); duri->fragment = NULL; + if((ret = ensurefraglist(duri))) goto done; done: return ret; } @@ -579,25 +568,67 @@ int ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value) { int ret = NC_NOERR; - int len; int pos = -1; - char* newlist = NULL; ensurefraglist(duri); pos = ncfind(duri->fraglist, key); if(pos < 0) { /* does not exist */ - if((ret = extendenvv(&duri->fraglist,2,&len))) goto done; - duri->fraglist[len] = strdup(key); - duri->fraglist[len+1] = nulldup(value); - duri->fraglist[len+2] = NULL; + nclistpush((NClist*)duri->fraglist,strdup(key)); + nclistpush((NClist*)duri->fraglist,nulldup(value)); } else { - nullfree(duri->fraglist[pos+1]); - duri->fraglist[pos+1] = strdup(value); + nullfree(nclistget(duri->fraglist,pos+1)); + nclistset(duri->fraglist,pos+1,nulldup(value)); } /* Rebuild the fragment */ - if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done; - nullfree(duri->fragment); - duri->fragment = newlist; newlist = NULL; + nullfree(duri->fraglist); duri->fraglist = NULL; + if((ret=ensurefraglist(duri))) goto done; +done: + return ret; +} + +/* Replace a specific query key*/ +int +ncurisetquerykey(NCURI* duri,const char* key, const char* value) +{ + int ret = NC_NOERR; + int pos = -1; + + ensurequerylist(duri); + pos = ncfind(duri->querylist, key); + if(pos < 0) { /* does not exist */ + if(duri->querylist == NULL) duri->querylist = nclistnew(); + nclistpush(duri->querylist,key); + nclistpush(duri->querylist,value); + } else { + nullfree(nclistget(duri->querylist,pos+1)); + nclistset(duri->querylist,pos+1,strdup(value)); + } + /* Rebuild the query */ + nullfree(duri->query); duri->query = NULL; + if((ret = ensurequerylist(duri))) goto done; +done: + return ret; +} + +/* Replace or add a specific query key*/ +int +ncuriappendquerykey(NCURI* duri,const char* key, const char* value) +{ + int ret = NC_NOERR; + int pos = -1; + + ensurequerylist(duri); + pos = ncfind(duri->querylist, key); + if(pos < 0) { /* does not exist */ + nclistpush((NClist*)duri->querylist,strdup(key)); + nclistpush((NClist*)duri->querylist,nulldup(value)); + } else { + nullfree(nclistget(duri->querylist,pos+1)); + nclistset(duri->querylist,pos+1,nulldup(value)); + } + /* Rebuild the query */ + nullfree(duri->querylist); duri->querylist = NULL; + if((ret=ensurequerylist(duri))) goto done; done: return ret; } @@ -734,11 +765,10 @@ ncurifragmentlookup(NCURI* uri, const char* key) int i; char* value = NULL; if(uri == NULL || key == NULL) return NULL; - ensurefraglist(uri); + if(ensurefraglist(uri)) return NULL; i = ncfind(uri->fraglist,key); - if(i < 0) - return NULL; - value = uri->fraglist[(2*i)+1]; + if(i < 0) return NULL; + value = nclistget(uri->fraglist,i+1); return value; } @@ -747,20 +777,21 @@ ncuriquerylookup(NCURI* uri, const char* key) { int i; char* value = NULL; - if(uri == NULL || key == NULL || uri->querylist == NULL) return NULL; + if(uri == NULL || key == NULL) return NULL; + if(ensurequerylist(uri)) return NULL; i = ncfind(uri->querylist,key); - if(i < 0) - return NULL; - value = uri->querylist[(2*i)+1]; + if(i < 0) return NULL; + value = nclistget(uri->querylist,i+1); return value; } +#if 0 /* Obtain the complete list of fragment pairs in envv format */ const char** ncurifragmentparams(NCURI* uri) { ensurefraglist(uri); - return (const char**)uri->fraglist; + return (const char**)nclistcontents(uri->fraglist; } /* Obtain the complete list of query pairs in envv format */ @@ -771,7 +802,6 @@ ncuriqueryparams(NCURI* uri) return (const char**)uri->querylist; } -#if 0 int ncuriremoveparam(NCURI* uri, const char* key) { @@ -796,19 +826,18 @@ ncuriremoveparam(NCURI* uri, const char* key) case insensitive */ static int -ncfind(char** params, const char* key) +ncfind(NClist* params, const char* key) { int i; - char** p; if(key == NULL) return -1; if(params == NULL) return -1; - for(i=0,p=params;*p;p+=2,i++) { - if(strcasecmp(key,*p)==0) return i; + for(i=0;i 0) { + ncbytescat(buf,p0); + if(p1 != NULL && strlen(p1) > 0) { ncbytescat(buf,"="); if(encode) { - char* encoded = ncuriencodeonly(p[1],queryallow); + char* encoded = ncuriencodeonly(p1,queryallow); ncbytescat(buf,encoded); nullfree(encoded); } else - ncbytescat(buf,p[1]); + ncbytescat(buf,p1); } } - if(svecp) {*svecp = ncbytesextract(buf);} done: - ncbytesfree(buf); return stat; } @@ -1171,24 +1199,32 @@ static int ensurefraglist(NCURI* uri) { int stat = NC_NOERR; - int nofrag = 0; - int nolist = 0; - NClist* fraglist = NULL; + int hastext = 0; + int haslist = 0; + NClist* fraglist = nclistnew(); NCbytes* frag = NULL; - if(uri->fragment == NULL || strlen(uri->fragment) == 0) - {nullfree(uri->fragment); uri->fragment = NULL; nofrag=1;} - if(uri->fraglist == NULL) nolist = 1; - if(nolist && !nofrag) { - fraglist = nclistnew(); + if(nulllen(uri->fragment) == 0) + {nullfree(uri->fragment); uri->fragment = NULL; hastext=0;} + else hastext = 1; + if(nclistlength((NClist*)uri->fraglist) == 0) + {nclistfree((NClist*)uri->fraglist); uri->fraglist = NULL; haslist=0;} + else haslist = 1; + + /* Four cases: */ + if(!haslist && !hastext) { + /* do nothing */ + } else if(!haslist && hastext) { if((stat = parselist(uri->fragment,fraglist))) goto done; removedups(fraglist); - uri->fraglist = nclistextract(fraglist); - } else if(!nolist && nofrag) { + uri->fraglist = fraglist; fraglist = NULL; + } else if(haslist && !hastext) { /* Create the fragment string from fraglist */ frag = ncbytesnew(); - buildlist((const char**)uri->fraglist,0,frag); /* do not encode */ + if((stat=unparselist((const NClist*)uri->fraglist,NULL,0,frag))) goto done; /* do not encode */ uri->fragment = ncbytesextract(frag); + } else if(haslist && hastext) { + /* assume already consistent */ } done: @@ -1201,24 +1237,32 @@ static int ensurequerylist(NCURI* uri) { int stat = NC_NOERR; - int noquery = 0; - int nolist = 0; - NClist* querylist = NULL; + int hastext = 0; + int haslist = 0; + NClist* querylist = nclistnew(); NCbytes* query = NULL; - if(uri->query == NULL || strlen(uri->query) == 0) - {nullfree(uri->query); uri->query = NULL; noquery=1;} - if(uri->querylist == NULL) nolist = 1; - if(nolist && !noquery) { - querylist = nclistnew(); + if(nulllen(uri->query) == 0) + {nullfree(uri->query); uri->query = NULL; hastext=0;} + else hastext = 1; + if(nclistlength((NClist*)uri->querylist) == 0) + {nclistfree((NClist*)uri->querylist); uri->querylist = NULL; haslist=0;} + else haslist = 1; + + /* Four cases: */ + if(!haslist && !hastext) { + /* do nothing */ + } else if(!haslist && hastext) { if((stat = parselist(uri->query,querylist))) goto done; removedups(querylist); - uri->querylist = nclistextract(querylist); - } else if(!nolist && noquery) { + uri->querylist = querylist; querylist = NULL; + } else if(haslist && !hastext) { /* Create the query string from querylist */ query = ncbytesnew(); - buildlist((const char**)uri->querylist,0,query); /* do not encode */ + if((stat=unparselist((const NClist*)uri->querylist,NULL,0,query))) goto done; /* do not encode */ uri->query = ncbytesextract(query); + } else if(haslist && hastext) { + /* assume consistent */ } done: @@ -1242,30 +1286,9 @@ removedups(NClist* list) } } } - /* NULL terminate the list */ - nclistpush(list,NULL); -} - -static void -buildlist(const char** list, int encode, NCbytes* buf) -{ - const char** p; - int first = 1; - for(p=list;*p;p+=2,first=0) { - if(!first) ncbytescat(buf,"&"); - ncbytescat(buf,p[0]); - if(p[1] != NULL && strlen(p[1]) > 0) { - ncbytescat(buf,"="); - if(encode) { - char* encoded = ncuriencodeonly(p[1],queryallow); - ncbytescat(buf,encoded); - nullfree(encoded); - } else - ncbytescat(buf,p[1]); - } - } } +#if 0 static int extendenvv(char*** envvp, int amount, int* oldlenp) { @@ -1281,6 +1304,7 @@ extendenvv(char*** envvp, int amount, int* oldlenp) *envvp = envv; envv = NULL; return NC_NOERR; } +#endif /* Use for gdb debug */ char* @@ -1288,3 +1312,20 @@ ncuriunescape(const char* s) { return ncuridecodepartial(s,ascii); } + +/* Get the actual list of queryies */ +void* +ncuriqueryparams(NCURI* uri) +{ + ensurequerylist(uri); + return uri->querylist; +} + +/* Get the actual list of frags */ +void* +ncurifragmentparams(NCURI* uri) +{ + ensurefraglist(uri); + return uri->fraglist; +} + diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c index 966adfbfb..9e22f73d4 100644 --- a/libnczarr/zarr.c +++ b/libnczarr/zarr.c @@ -22,7 +22,7 @@ static int applycontrols(NCZ_FILE_INFO_T* zinfo); */ int -ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** controls) +ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* controls) { int stat = NC_NOERR; NCZ_FILE_INFO_T* zinfo = NULL; @@ -32,7 +32,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr NCjson* json = NULL; char* key = NULL; - ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_envv(controls):"null")); + ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_env(controls):"null")); nc = (NC*)file->controller; @@ -52,7 +52,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr zinfo->creating = 1; zinfo->common.file = file; zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); - if((zinfo->envv_controls=NCZ_clonestringvec(0,controls)) == NULL) + if((zinfo->controllist=nclistclone(controls,1)) == NULL) {stat = NC_ENOMEM; goto done;} /* fill in some of the zinfo and zroot fields */ @@ -94,7 +94,7 @@ done: */ int -ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) +ncz_open_dataset(NC_FILE_INFO_T* file, NClist* controls) { int stat = NC_NOERR; NC* nc = NULL; @@ -126,7 +126,7 @@ ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) zinfo->creating = 0; zinfo->common.file = file; zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); - if((zinfo->envv_controls = NCZ_clonestringvec(0,controls))==NULL) /*0=>envv style*/ + if((zinfo->controllist=nclistclone(controls,1)) == NULL) {stat = NC_ENOMEM; goto done;} zinfo->default_maxstrlen = NCZ_MAXSTR_DEFAULT; @@ -294,12 +294,13 @@ done: static const char* -controllookup(const char** envv_controls, const char* key) +controllookup(NClist* controls, const char* key) { - const char** p; - for(p=envv_controls;*p;p+=2) { - if(strcasecmp(key,*p)==0) { - return p[1]; + int i; + for(i=0;ienvv_controls,"mode")) != NULL) { + if((value = controllookup(zinfo->controllist,"mode")) != NULL) { if((stat = NCZ_comma_parse(value,modelist))) goto done; } /* Process the modelist first */ @@ -337,11 +338,11 @@ applycontrols(NCZ_FILE_INFO_T* zinfo) zinfo->controls.flags &= (~noflags); /* Process other controls */ - if((value = controllookup((const char**)zinfo->envv_controls,"log")) != NULL) { + if((value = controllookup(zinfo->controllist,"log")) != NULL) { zinfo->controls.flags |= FLAG_LOGGING; ncsetloglevel(NCLOGNOTE); } - if((value = controllookup((const char**)zinfo->envv_controls,"show")) != NULL) { + if((value = controllookup(zinfo->controllist,"show")) != NULL) { if(strcasecmp(value,"fetch")==0) zinfo->controls.flags |= FLAG_SHOWFETCH; } diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index 8b3b37ca0..22dd2d1cf 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -26,8 +26,8 @@ struct ZCVT { #define zcvt_empty {0,0,0.0,NULL} /* zarr.c */ -EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, const char** controls); -EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, const char** controls); +EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, NClist* controls); +EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, NClist* controls); EXTERNL int ncz_del_attr(NC_FILE_INFO_T* file, NC_OBJ* container, const char* name); /* HDF5 Mimics */ diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index b70e86f79..9baa0fd63 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -49,7 +49,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort) if((stat = nczmap_close(zinfo->map,(abort && zinfo->creating)?1:0))) goto done; - NCZ_freestringvec(0,zinfo->envv_controls); + nclistfreeall(zinfo->controllist); NC_authfree(zinfo->auth); nullfree(zinfo); diff --git a/libnczarr/zcreate.c b/libnczarr/zcreate.c index 3243b6f8c..bf7c1db3e 100644 --- a/libnczarr/zcreate.c +++ b/libnczarr/zcreate.c @@ -30,7 +30,7 @@ static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_DISKLESS|NC_64BIT * @author Dennis Heimbigner, Ed Hartnett */ static int -ncz_create_file(const char *path, int cmode, size_t initialsz, const char** controls, int ncid) +ncz_create_file(const char *path, int cmode, size_t initialsz, NClist* controls, int ncid) { int retval = NC_NOERR; NC_FILE_INFO_T* h5 = NULL; diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 50ff91e44..14c8079f0 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -144,7 +144,7 @@ typedef struct NCZ_FILE_INFO { } zarr; int creating; /* 1=> created 0=>open */ int native_endianness; /* NC_ENDIAN_LITTLE | NC_ENDIAN_BIG */ - char** envv_controls; /* Envv format */ + NClist* controllist; /* Envv format */ struct Controls { size64_t flags; # define FLAG_PUREZARR 1 diff --git a/libnczarr/zopen.c b/libnczarr/zopen.c index 8d8644e0f..2171f7b23 100644 --- a/libnczarr/zopen.c +++ b/libnczarr/zopen.c @@ -64,7 +64,7 @@ check_for_classic_model(NC_GRP_INFO_T *root_grp, int *is_classic) * @author Dennis Heimbigner, Ed Hartnett */ static int -ncz_open_file(const char *path, int mode, const char** controls, int ncid) +ncz_open_file(const char *path, int mode, NClist* controls, int ncid) { int stat = NC_NOERR; NC_FILE_INFO_T *h5 = NULL; From fe3ee028596c32debf22360e63126af4fa542365 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 8 Oct 2023 20:40:24 -0600 Subject: [PATCH 02/13] Update release notes --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ed19b87f7..3a82aa9a1 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD -* Improve the speed and data quantity for DAP4 queries. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* Improve the speed and data quantity for DAP4 queries. See [Github #2765](https://github.com/Unidata/netcdf-c/pull/2765). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737). From 24e49b818e60f43083147cdf50551a816882c0d7 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 12 Oct 2023 16:22:47 -0600 Subject: [PATCH 03/13] Update internal tinyxml2 code to the latest version Periodic update of the internal tinyxml2 code for processing XML. This also required some changes to the "tinyxml2" target in libncxml/Makefile.am to modify the code to compile in the netcdf-c environment. --- include/ncconfigure.h | 2 +- libncxml/Makefile.am | 19 ++++++++++++++++--- libncxml/tinyxml2.cpp | 6 ++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/include/ncconfigure.h b/include/ncconfigure.h index cad59c9e2..496ec59e0 100644 --- a/include/ncconfigure.h +++ b/include/ncconfigure.h @@ -61,7 +61,7 @@ extern "C" { #endif /* WARNING: in some systems, these functions may be defined as macros, so check */ -#ifndef HAVE_STRDUP +#if ! defined(HAVE_STRDUP) || defined(__CYGWIN__) #ifndef strdup char* strdup(const char*); #endif diff --git a/libncxml/Makefile.am b/libncxml/Makefile.am index 3866eae5b..f5fab3b10 100644 --- a/libncxml/Makefile.am +++ b/libncxml/Makefile.am @@ -35,7 +35,20 @@ tinyxml2:: rm -fr ./tinyxml2 ./license.txt git clone --depth=1 ${REPO} cat tinyxml2/LICENSE.txt > ./license.txt - cat tinyxml2/tinyxml2.h > ./tinyxml2.h - sed -e 's/__BORLANDC__/__APPLE__/' < tinyxml2/tinyxml2.cpp \ - | sed -e 's/ptrdiff_t/long/g' > ./tinyxml2.cpp + tr -d '\r' < tinyxml2/tinyxml2.h > tinyxml2.h + cat tinyxml2/tinyxml2.cpp \ + | sed -e 's/__BORLANDC__/__APPLE__/' \ + | sed -e 's/ptrdiff_t/long/g' \ + | sed -e '/^static[ ]*FILE[*][ ]*callfopen(/i\ +\#if 0' \ + | sed -e '/^void[ ]*XMLDocument::DeleteNode(/i\ +\#endif /*0*/\ +' \ + | sed -e '/^XMLError[ ]*XMLDocument::LoadFile([ ]*const[ ]*char[*]/i\ +\#if 0' \ + | sed -e '/^XMLError[ ]*XMLDocument::Parse(/i\ +\#endif /*0*/\ +' \ + | tr -d '\r' \ + | cat > ./tinyxml2.cpp rm -fr tinyxml2 diff --git a/libncxml/tinyxml2.cpp b/libncxml/tinyxml2.cpp index 4f761ad31..c0d16eda0 100644 --- a/libncxml/tinyxml2.cpp +++ b/libncxml/tinyxml2.cpp @@ -2283,6 +2283,7 @@ XMLUnknown* XMLDocument::NewUnknown( const char* str ) return unk; } +#if 0 static FILE* callfopen( const char* filepath, const char* mode ) { TIXMLASSERT( filepath ); @@ -2299,6 +2300,8 @@ static FILE* callfopen( const char* filepath, const char* mode ) return fp; } +#endif /*0*/ + void XMLDocument::DeleteNode( XMLNode* node ) { TIXMLASSERT( node ); TIXMLASSERT(node->_document == this ); @@ -2317,6 +2320,7 @@ void XMLDocument::DeleteNode( XMLNode* node ) { } +#if 0 XMLError XMLDocument::LoadFile( const char* filename ) { if ( !filename ) { @@ -2420,6 +2424,8 @@ XMLError XMLDocument::SaveFile( FILE* fp, bool compact ) } +#endif /*0*/ + XMLError XMLDocument::Parse( const char* xml, size_t nBytes ) { Clear(); From 0701250bbcc4aefc7a7a1e5a3a4faae3ba698e0d Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 12 Oct 2023 16:25:18 -0600 Subject: [PATCH 04/13] Update release notes --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c92394242..245bc2834 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Update the internal copy of tinyxml2 to latest code. See [Github #2771](https://github.com/Unidata/netcdf-c/pull/2771). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737). From 1162f64d2a71c73527dcae84f744598e17cd86d5 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Mon, 1 May 2023 13:25:53 -0400 Subject: [PATCH 05/13] Fixed various UBSan warnings about invalid bit shifting Just made sure to use unsigned, so that a bit does not get shifted into a sign bit. --- libdispatch/ncexhash.c | 4 ++-- libdispatch/nclistmgr.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libdispatch/ncexhash.c b/libdispatch/ncexhash.c index ea3957a05..26375e8d8 100644 --- a/libdispatch/ncexhash.c +++ b/libdispatch/ncexhash.c @@ -111,7 +111,7 @@ ncexinit(void) int i; bitmasks[0] = 0; for(i=1;ileaflen, leafavg); fprintf(stderr," load=%g",leafload); fprintf(stderr,"]\n"); - dirsize = (1<<(map->depth))*((unsigned long long)sizeof(void*)); + dirsize = (1ULL<<(map->depth))*((unsigned long long)sizeof(void*)); leafsize = (nleaves)*((unsigned long long)sizeof(NCexleaf)); total = dirsize + leafsize; fprintf(stderr,"\tsizeof(directory)=%llu sizeof(leaves)=%lld total=%lld\n", diff --git a/libdispatch/nclistmgr.c b/libdispatch/nclistmgr.c index 48b4eeb58..4069e8769 100644 --- a/libdispatch/nclistmgr.c +++ b/libdispatch/nclistmgr.c @@ -81,8 +81,8 @@ free_NCList(void) int add_to_NCList(NC* ncp) { - int i; - int new_id; + unsigned int i; + unsigned int new_id; if(nc_filelist == NULL) { if (!(nc_filelist = calloc(1, sizeof(NC*)*NCFILELISTLENGTH))) return NC_ENOMEM; @@ -96,7 +96,7 @@ add_to_NCList(NC* ncp) if(new_id == 0) return NC_ENOMEM; /* no more slots */ nc_filelist[new_id] = ncp; numfiles++; - ncp->ext_ncid = (new_id << ID_SHIFT); + ncp->ext_ncid = (int)(new_id << ID_SHIFT); return NC_NOERR; } From 64111c7f5ec9b6d4bae27bfab29e47dc457f02e1 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 4 Nov 2023 16:17:09 -0600 Subject: [PATCH 06/13] remove conflicts --- libdap4/d4data.c | 3 +++ libdap4/d4file.c | 55 ++++++------------------------------------ libdap4/d4parser.c | 3 ++- libdap4/d4varx.c | 7 +----- libdap4/ncd4dispatch.c | 4 +-- 5 files changed, 15 insertions(+), 57 deletions(-) diff --git a/libdap4/d4data.c b/libdap4/d4data.c index 234f3e377..eae5f32ed 100644 --- a/libdap4/d4data.c +++ b/libdap4/d4data.c @@ -78,6 +78,8 @@ NCD4_parcelvars(NCD4meta* meta, NCD4response* resp) var->data.response = resp; /* cross link */ } done: + nclistfree(toplevel); + nullfree(offset); return THROW(ret); } @@ -421,6 +423,7 @@ NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp) } } } + nclistfree(toplevel); resp->attrchecksumming = (attrfound ? 1 : 0); /* Infer checksums */ resp->inferredchecksumming = ((resp->attrchecksumming || resp->querychecksumming) ? 1 : 0); diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 471afa15d..45ee9a406 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -546,6 +546,7 @@ NCD4_newInfo(NCD4INFO** d4infop) if((info = calloc(1,sizeof(NCD4INFO)))==NULL) {ret = NC_ENOMEM; goto done;} info->platform.hostlittleendian = NCD4_isLittleEndian(); + info->responses = nclistnew(); if(d4infop) {*d4infop = info; info = NULL;} done: if(info) NCD4_reclaimInfo(info); @@ -570,13 +571,12 @@ NCD4_reclaimInfo(NCD4INFO* d4info) nclistfree(d4info->blobs); /* Reclaim dmr node tree */ NCD4_reclaimMeta(d4info->dmrmetadata); + /* Reclaim all responses */ for(i=0;iresponses);i++) { NCD4response* resp = nclistget(d4info->responses,i); NCD4_reclaimResponse(resp); } nclistfree(d4info->responses); - /* Reclaim all responses */ - NCD4_resetMeta(d4info->dmrmetadata); free(d4info); } @@ -597,10 +597,9 @@ NCD4_resetInfoForRead(NCD4INFO* d4info) */ if(d4info->substrate.filename != NULL) { unlink(d4info->substrate.filename); - } + } } - NCD4_resetMeta(d4info->dmrmetadata); - nullfree(d4info->dmrmetadata); + NCD4_reclaimMeta(d4info->dmrmetadata); d4info->dmrmetadata = NULL; } @@ -641,50 +640,8 @@ NCD4_reclaimResponse(NCD4response* d4resp) d4resp->controller = NULL; /* break link */ nullfree(d4resp->raw.memory); - - nullfree(serial->errdata); nullfree(serial->dmr); - nullfree(serial->dap); - /* clear all fields */ - memset(serial,0,sizeof(struct NCD4serial)); - - nullfree(d4resp->error.parseerror); - nullfree(d4resp->error.message); - nullfree(d4resp->error.context); - nullfree(d4resp->error.otherinfo); - memset(&d4resp->error,0,sizeof(d4resp->error)); - - free(d4resp); -} - - -/* Create an empty NCD4meta object for - use in subsequent calls - (is the the right src file to hold this?) -*/ - -int -NCD4_newMeta(NCD4INFO* info, NCD4meta** metap) -{ - int ret = NC_NOERR; - NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); - if(meta == NULL) return NC_ENOMEM; - meta->allnodes = nclistnew(); -#ifdef D4DEBUG - meta->debuglevel = 1; -#endif - meta->controller = info; - meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ - if(metap) {*metap = meta; meta = NULL;} - return THROW(ret); -} - -void -NCD4_reclaimMeta(NCD4meta* dataset) -{ - int i; - if(dataset == NULL) return; - NCD4_resetMeta(dataset); + nullfree(serial->errdata); for(i=0;iallnodes);i++) { NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); @@ -696,6 +653,7 @@ NCD4_reclaimMeta(NCD4meta* dataset) free(dataset); } +#if 0 void NCD4_resetMeta(NCD4meta* dataset) { @@ -708,3 +666,4 @@ NCD4_resetMeta(NCD4meta* dataset) nclistfree(dataset->blobs); #endif } +#endif diff --git a/libdap4/d4parser.c b/libdap4/d4parser.c index 776163a01..2948f8067 100644 --- a/libdap4/d4parser.c +++ b/libdap4/d4parser.c @@ -228,7 +228,8 @@ traverse(NCD4parser* parser, ncxml_t dom) if((ret=makeNode(parser,NULL,NULL,NCD4_GROUP,NC_NULL,&parser->metadata->root))) goto done; parser->metadata->root->group.isdataset = 1; parser->metadata->root->meta.id = parser->metadata->ncid; - parser->metadata->groupbyid = nclistnew(); + if(parser->metadata->groupbyid == NULL) + parser->metadata->groupbyid = nclistnew(); SETNAME(parser->metadata->root,"/"); xattr = ncxml_attr(dom,"name"); if(xattr != NULL) parser->metadata->root->group.datasetname = xattr; diff --git a/libdap4/d4varx.c b/libdap4/d4varx.c index 170171800..8956486e7 100644 --- a/libdap4/d4varx.c +++ b/libdap4/d4varx.c @@ -286,15 +286,9 @@ static int mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming) { int i, ret = NC_NOERR; - NCD4node* dmrroot = dmrmeta->root; - NClist* dmrtop = NULL; /* top variables in dmr tree */ NCD4node* daproot = dapmeta->root; NClist* daptop = NULL; /* top variables in dap tree */ - /* Get top level variables from the dmr node tree */ - dmrtop = nclistnew(); - NCD4_getToplevelVars(dmrmeta,dmrroot,dmrtop); - /* Get top level variables from the dap node tree */ daptop = nclistnew(); NCD4_getToplevelVars(dapmeta,daproot,daptop); @@ -311,5 +305,6 @@ mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming) } done: + nclistfree(daptop); return THROW(ret); } diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c index 5f5ebbfec..e7eefb40a 100644 --- a/libdap4/ncd4dispatch.c +++ b/libdap4/ncd4dispatch.c @@ -388,11 +388,11 @@ NCD4_inq_attname(int ncid, int varid, int attnum, char* name) const NC_reservedatt* rsvp = NULL; if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) return (ret); + substrateid = makenc4id(ncp,ncid); + ret = nc_inq_attname(substrateid, varid, attnum, name); /* Is this a reserved attribute name? */ if(name && (rsvp = NCD4_lookupreserved(name))) return NC_EATTMETA; - substrateid = makenc4id(ncp,ncid); - ret = nc_inq_attname(substrateid, varid, attnum, name); return (ret); } From f8cb89e679566b375797f511a595a9b2ee4f2962 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 4 Nov 2023 16:42:36 -0600 Subject: [PATCH 07/13] ckp --- libdap4/d4file.c | 7 ++++--- libdap4/ncd4types.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 45ee9a406..64e1abc6e 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -154,7 +154,6 @@ NCD4_open(const char * path, int mode, NCD4_resetInfoForRead(d4info); /* Rebuild metadata */ if((ret = NCD4_newMeta(d4info,&d4info->dmrmetadata))) goto done; - meta = d4info->dmrmetadata; /* Capture response */ if((dmrresp = (NCD4response*)calloc(1,sizeof(NCD4response)))==NULL) @@ -188,10 +187,11 @@ NCD4_open(const char * path, int mode, } #endif - if((ret = NCD4_parse(meta,dmrresp,0))) goto done; + if((ret = NCD4_parse(d4info->dmrmetadata,dmrresp,0))) goto done; #ifdef D4DEBUGMETA { + meta = d4info->dmrmetadata; fprintf(stderr,"\n/////////////\n"); NCbytes* buf = ncbytesnew(); NCD4_print(meta,buf); @@ -204,7 +204,7 @@ NCD4_open(const char * path, int mode, #endif /* Build the substrate metadata */ - ret = NCD4_metabuild(meta,meta->ncid); + ret = NCD4_metabuild(d4info->dmrmetadata,meta->ncid); if(ret != NC_NOERR && ret != NC_EVARSIZE) goto done; /* Remember the response */ @@ -634,6 +634,7 @@ done: void NCD4_reclaimResponse(NCD4response* d4resp) { + int i; struct NCD4serial* serial = NULL; if(d4resp == NULL) return; serial = &d4resp->serial; diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h index ca0a5118a..7bed38a84 100644 --- a/libdap4/ncd4types.h +++ b/libdap4/ncd4types.h @@ -342,7 +342,7 @@ struct NCD4INFO { NCURI* dmruri; /* parse of rawuritext */ NCD4curl* curl; int inmemory; /* store fetched data in memory? */ - NCD4meta* dmrmetadata; + NCD4meta* dmrmetadata; /* Independent of responses */ NClist* responses; /* NClist all responses from this curl handle */ struct { /* Properties that are per-platform */ int hostlittleendian; /* 1 if the host is little endian */ From f878acf99eb7f645ea39c6d8ce1e82b0af2186f5 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 4 Nov 2023 20:23:55 -0600 Subject: [PATCH 08/13] fix memory leak --- libdap4/d4file.c | 43 ++++++++++++++++++++++++++++++++++++++++--- libdap4/d4varx.c | 43 +++++++++++++++++++++++++------------------ 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 64e1abc6e..46c1a0fdb 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -46,7 +46,6 @@ NCD4_open(const char * path, int mode, NCD4INFO* d4info = NULL; const char* value; NC* nc; - NCD4meta* meta = NULL; size_t len = 0; void* contents = NULL; NCD4response* dmrresp = NULL; @@ -204,7 +203,7 @@ NCD4_open(const char * path, int mode, #endif /* Build the substrate metadata */ - ret = NCD4_metabuild(d4info->dmrmetadata,meta->ncid); + ret = NCD4_metabuild(d4info->dmrmetadata,d4info->dmrmetadata->ncid); if(ret != NC_NOERR && ret != NC_EVARSIZE) goto done; /* Remember the response */ @@ -634,7 +633,6 @@ done: void NCD4_reclaimResponse(NCD4response* d4resp) { - int i; struct NCD4serial* serial = NULL; if(d4resp == NULL) return; serial = &d4resp->serial; @@ -644,6 +642,45 @@ NCD4_reclaimResponse(NCD4response* d4resp) nullfree(serial->dmr); nullfree(serial->errdata); + /* clear all fields */ + memset(serial,0,sizeof(struct NCD4serial)); + + nullfree(d4resp->error.parseerror); + nullfree(d4resp->error.message); + nullfree(d4resp->error.context); + nullfree(d4resp->error.otherinfo); + memset(&d4resp->error,0,sizeof(d4resp->error)); + + free(d4resp); +} + +/* Create an empty NCD4meta object for + use in subsequent calls + (is the the right src file to hold this?) +*/ + +int +NCD4_newMeta(NCD4INFO* info, NCD4meta** metap) +{ + int ret = NC_NOERR; + NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); + if(meta == NULL) return NC_ENOMEM; + meta->allnodes = nclistnew(); +#ifdef D4DEBUG + meta->debuglevel = 1; +#endif + meta->controller = info; + meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ + if(metap) {*metap = meta; meta = NULL;} + return THROW(ret); +} + +void +NCD4_reclaimMeta(NCD4meta* dataset) +{ + int i; + if(dataset == NULL) return; + for(i=0;iallnodes);i++) { NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); reclaimNode(node); diff --git a/libdap4/d4varx.c b/libdap4/d4varx.c index 8956486e7..b098855f8 100644 --- a/libdap4/d4varx.c +++ b/libdap4/d4varx.c @@ -187,24 +187,31 @@ getvarx(int gid, int varid, NCD4INFO** infop, NCD4node** varp, } /* Read and process the data */ - /* Setup the meta-data for the DAP */ - if((ret=NCD4_newMeta(info,&dapmeta))) goto done; - if((ret=NCD4_newResponse(info,&dapresp))) goto done; - dapresp->mode = NCD4_DAP; - nclistpush(info->responses,dapresp); - if((ret=NCD4_readDAP(info, info->controls.flags.flags, ceuri, dapresp))) goto done; - /* Extract DMR and dechunk the data part */ - if((ret=NCD4_dechunk(dapresp))) goto done; - /* Process the dmr part */ - if((ret=NCD4_parse(dapmeta,dapresp,1))) goto done; - /* See if we are checksumming */ - if((ret=NCD4_inferChecksums(dapmeta,dapresp))) goto done; - /* connect variables and corresponding dap data */ - if((ret = NCD4_parcelvars(dapmeta,dapresp))) goto done; - /* Process checksums and byte-order swapping */ - if((ret = NCD4_processdata(dapmeta,dapresp))) goto done; - /* Transfer and process the data */ - if((ret = mapvars(dapmeta,dmrmeta,dapresp->inferredchecksumming))) goto done; + + /* Setup the meta-data for the DAP */ + if((ret=NCD4_newMeta(info,&dapmeta))) goto done; + if((ret=NCD4_newResponse(info,&dapresp))) goto done; + dapresp->mode = NCD4_DAP; + nclistpush(info->responses,dapresp); + if((ret=NCD4_readDAP(info, info->controls.flags.flags, ceuri, dapresp))) goto done; + + /* Extract DMR and dechunk the data part */ + if((ret=NCD4_dechunk(dapresp))) goto done; + + /* Process the dmr part */ + if((ret=NCD4_parse(dapmeta,dapresp,1))) goto done; + + /* See if we are checksumming */ + if((ret=NCD4_inferChecksums(dapmeta,dapresp))) goto done; + + /* connect variables and corresponding dap data */ + if((ret = NCD4_parcelvars(dapmeta,dapresp))) goto done; + + /* Process checksums and byte-order swapping */ + if((ret = NCD4_processdata(dapmeta,dapresp))) goto done; + + /* Transfer and process the data */ + if((ret = mapvars(dapmeta,dmrmeta,dapresp->inferredchecksumming))) goto done; validated: /* Return relevant info */ From adea80f3763ef59c1e2471dd3d5e045758d6e6b8 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 4 Nov 2023 21:32:33 -0600 Subject: [PATCH 09/13] Remove the execinfo capability re: Issue https://github.com/Unidata/netcdf-c/issues/2766 This attempt to programmatically dump stack never worked, so it is time to kill it off. --- RELEASE_NOTES.md | 1 + config.h.cmake.in | 3 --- configure.ac | 3 --- libdispatch/nclog.c | 32 -------------------------------- libhdf5/hdf5debug.c | 28 ---------------------------- 5 files changed, 1 insertion(+), 66 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 8d5eca680..ee0d03cc9 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Remove the use of execinfo to programmatically dump the stack; it never worked. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). * Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764). * Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). diff --git a/config.h.cmake.in b/config.h.cmake.in index 04d876ff1..233c2f1e2 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -462,9 +462,6 @@ with zip */ /* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */ #cmakedefine HDF5_UTF8_PATHS 1 -/* if true, backtrace support will be used. */ -#cmakedefine HAVE_EXECINFO_H 1 - /* if true, include JNA bug fix */ #cmakedefine JNA 1 diff --git a/configure.ac b/configure.ac index 19386fe14..b9c12a1ff 100644 --- a/configure.ac +++ b/configure.ac @@ -1305,9 +1305,6 @@ AC_CHECK_HEADERS([sys/resource.h]) # See if we have ftw.h to walk directory trees AC_CHECK_HEADERS([ftw.h]) -# See if we can do stack tracing programmatically -AC_CHECK_HEADERS([execinfo.h]) - # Check for these functions... AC_CHECK_FUNCS([strlcat snprintf strcasecmp fileno \ strdup strtoll strtoull \ diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c index 5f6b8be7d..c9664a23c 100644 --- a/libdispatch/nclog.c +++ b/libdispatch/nclog.c @@ -21,10 +21,6 @@ #include #endif -#ifdef HAVE_EXECINFO_H -#include -#endif - #include "netcdf.h" #include "nclog.h" @@ -278,10 +274,6 @@ ncuntrace(const char* fcn, int err, const char* fmt, ...) vfprintf(nclog_global.nclogstream, fmt, args); fprintf(nclog_global.nclogstream, "\n" ); fflush(nclog_global.nclogstream); -#ifdef HAVE_EXECINFO_H - if(err != 0) - ncbacktrace(); -#endif } done: va_end(args); @@ -304,28 +296,4 @@ ncbreakpoint(int err) return err; } -#ifdef HAVE_EXECINFO_H -#define MAXSTACKDEPTH 100 -void -ncbacktrace(void) -{ - int j, nptrs; - void* buffer[MAXSTACKDEPTH]; - char **strings; - - if(getenv("NCBACKTRACE") == NULL) return; - nptrs = backtrace(buffer, MAXSTACKDEPTH); - strings = backtrace_symbols(buffer, nptrs); - if (strings == NULL) { - perror("backtrace_symbols"); - errno = 0; - return; - } - fprintf(stderr,"Backtrace:\n"); - for(j = 0; j < nptrs; j++) - fprintf(stderr,"%s\n", strings[j]); - free(strings); -} -#endif - /**@}*/ diff --git a/libhdf5/hdf5debug.c b/libhdf5/hdf5debug.c index 40871c148..b4dace8d5 100644 --- a/libhdf5/hdf5debug.c +++ b/libhdf5/hdf5debug.c @@ -14,37 +14,9 @@ #ifdef H5CATCH -#define STSIZE 1000 - -#ifdef HAVE_EXECINFO_H -#ifdef H5BACKTRACE -# if !defined _WIN32 && !defined __CYGWIN__ -static void* stacktrace[STSIZE]; -# endif -#endif -#endif - int nch5breakpoint(int err) { -#ifdef HAVE_EXECINFO_H -#ifdef H5BACKTRACE -# if !defined _WIN32 && !defined __CYGWIN__ - int count = 0; - char** trace = NULL; - int i; - - count = backtrace(stacktrace,STSIZE); - trace = backtrace_symbols(stacktrace, STSIZE); - fprintf(stderr,"backtrace:\n"); - for(i=0;i Date: Sat, 4 Nov 2023 21:34:50 -0600 Subject: [PATCH 10/13] Update RELEASENOTES --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ee0d03cc9..75e335445 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD -* Remove the use of execinfo to programmatically dump the stack; it never worked. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* Remove the use of execinfo to programmatically dump the stack; it never worked. See [Github #2789](https://github.com/Unidata/netcdf-c/pull/2789). * Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764). * Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). From 99d0760c0defab221f89659ab43563a3a7da36c2 Mon Sep 17 00:00:00 2001 From: Tobias Bussmann Date: Thu, 9 Nov 2023 00:39:16 +0100 Subject: [PATCH 11/13] disable test that depend on ncpathcvt in cmake build w/o utilities fixes #2794 --- nctest/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nctest/CMakeLists.txt b/nctest/CMakeLists.txt index d7de84a6c..ada220f2a 100644 --- a/nctest/CMakeLists.txt +++ b/nctest/CMakeLists.txt @@ -18,7 +18,9 @@ TARGET_LINK_LIBRARIES(nctest netcdf) ADD_TEST(nctest ${EXECUTABLE_OUTPUT_PATH}/nctest) add_bin_test_no_prefix(tst_rename) -add_sh_test(nctest compare_test_files) -IF(HAVE_BASH) - SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest) -ENDIF(HAVE_BASH) +IF(BUILD_UTILITIES) + add_sh_test(nctest compare_test_files) + IF(HAVE_BASH) + SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest) + ENDIF(HAVE_BASH) +ENDIF(BUILD_UTILITIES) From 374d65464452284e6e88736d12523cb6cae38895 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Sun, 30 Apr 2023 18:00:35 -0400 Subject: [PATCH 12/13] Fixed misaligned memory access flagged by UBSan Use memcpy to copy correctly even for unaligned memory. This was already done for some functions here, but not all. Also took the oppurtunity to remove a bunch of seemingly obsolete/commented code. --- libsrc/ncx.m4 | 268 ++++---------------------------------------------- 1 file changed, 17 insertions(+), 251 deletions(-) diff --git a/libsrc/ncx.m4 b/libsrc/ncx.m4 index c8ea6df41..a9fb11f41 100644 --- a/libsrc/ncx.m4 +++ b/libsrc/ncx.m4 @@ -307,88 +307,24 @@ swapn2b(void *dst, const void *src, IntType nn) IntType i; uint16_t *op = (uint16_t*) dst; uint16_t *ip = (uint16_t*) src; + uint16_t tmp; for (i=0; i 0) - * { - * *op++ = *(++ip); - * *op++ = *(ip++ -1); - * } - */ - while (nn > 3) - { - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - nn -= 4; - } - while (nn-- > 0) - { - *op++ = *(++ip); - *op++ = *(ip++ -1); - } -#endif } # ifndef vax inline static void swap4b(void *dst, const void *src) { - /* copy over, make the below swap in-place */ uint32_t tmp; - /* use memcpy to avoid type punning */ + /* memcpy is used to handle the case of unaligned memory */ memcpy(&tmp, src, sizeof(tmp)); tmp = SWAP4(tmp); memcpy(dst, &tmp, 4); - - /* Codes below will cause "break strict-aliasing rules" in gcc - uint32_t *op = (uint32_t*)dst; - *op = *(uint32_t*)src; - *op = SWAP4(*op); - */ - - /* Below are copied from netCDF-4. - * See https://bugtracking.unidata.ucar.edu/browse/NCF-338 - * Quote "One issue we are wrestling with is how compilers optimize this - * code. For some reason, we are actually needing to add an artificial - * move to a 4 byte space to get it to work. I think what is happening is - * that the optimizer is bit shifting within a double, which is incorrect. - * The following code actually does work correctly. - * This is in Linux land, gcc. - * - * However, the above in-place byte-swap does not appear affected by this. - */ -#if 0 - uint32_t *ip = (uint32_t*)src; - uint32_t tempOut; /* cannot use pointer when gcc O2 optimizer is used */ - tempOut = SWAP4(*ip); - - *(float *)dst = *(float *)(&tempOut); -#endif - - /* OLD implementation that results in four load and four store CPU - instructions - char *op = dst; - const char *ip = src; - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - */ - } # endif /* !vax */ @@ -398,110 +334,24 @@ swapn4b(void *dst, const void *src, IntType nn) IntType i; uint32_t *op = (uint32_t*) dst; uint32_t *ip = (uint32_t*) src; + uint32_t tmp; for (i=0; i 0) - * { - * op[0] = ip[3]; - * op[1] = ip[2]; - * op[2] = ip[1]; - * op[3] = ip[0]; - * op += 4; - * ip += 4; - * } - */ - while (nn > 3) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; - op[8] = ip[11]; - op[9] = ip[10]; - op[10] = ip[9]; - op[11] = ip[8]; - op[12] = ip[15]; - op[13] = ip[14]; - op[14] = ip[13]; - op[15] = ip[12]; - op += 16; - ip += 16; - nn -= 4; - } - while (nn-- > 0) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op += 4; - ip += 4; - } -#endif } # ifndef vax inline static void swap8b(void *dst, const void *src) { -#ifdef FLOAT_WORDS_BIGENDIAN - /* copy over, make the below swap in-place */ - *(uint64_t*)dst = *(uint64_t*)src; - - uint32_t *op = (uint32_t*)dst; - *op = SWAP4(*op); - op = (uint32_t*)((char*)dst+4); - *op = SWAP4(*op); -#else uint64_t tmp; - /* use memcpy to avoid type punning */ + /* memcpy is used to handle the case of unaligned memory */ memcpy(&tmp, src, sizeof(tmp)); tmp = SWAP8(tmp); memcpy(dst, &tmp, 8); - - /* Codes below will cause "break strict-aliasing rules" in gcc - uint64_t *op = (uint64_t*)dst; - *op = *(uint64_t*)src; - *op = SWAP8(*op); - */ -#endif - -#if 0 - char *op = dst; - const char *ip = src; -# ifndef FLOAT_WORDS_BIGENDIAN - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; -# else - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; -#endif -#endif } # endif /* !vax */ @@ -509,100 +359,16 @@ swap8b(void *dst, const void *src) inline static void swapn8b(void *dst, const void *src, IntType nn) { -#ifdef FLOAT_WORDS_BIGENDIAN - IntType i; - uint64_t *dst_p = (uint64_t*) dst; - uint64_t *src_p = (uint64_t*) src; - for (i=0; i 0) - * { - * op[0] = ip[7]; - * op[1] = ip[6]; - * op[2] = ip[5]; - * op[3] = ip[4]; - * op[4] = ip[3]; - * op[5] = ip[2]; - * op[6] = ip[1]; - * op[7] = ip[0]; - * op += 8; - * ip += 8; - * } - */ -# ifndef FLOAT_WORDS_BIGENDIAN - while (nn > 1) - { - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; - op[8] = ip[15]; - op[9] = ip[14]; - op[10] = ip[13]; - op[11] = ip[12]; - op[12] = ip[11]; - op[13] = ip[10]; - op[14] = ip[9]; - op[15] = ip[8]; - op += 16; - ip += 16; - nn -= 2; - } - while (nn-- > 0) - { - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; - op += 8; - ip += 8; - } -# else - while (nn-- > 0) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; - op += 8; - ip += 8; - } -#endif -#endif } # endif /* !vax */ From 23aa46fcf51437de23a83862427acb605a8a8740 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 15 Nov 2023 15:11:56 -0500 Subject: [PATCH 13/13] Fixed various UBSan warnings about working with NULL pointers Any pointer arithmetic with NULL pointers is technically UB, even if you don't end up dereferencing the pointer. --- libdispatch/utf8proc.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libdispatch/utf8proc.c b/libdispatch/utf8proc.c index 3a05dd3b3..68d8dd4a2 100644 --- a/libdispatch/utf8proc.c +++ b/libdispatch/utf8proc.c @@ -355,7 +355,8 @@ static nc_utf8proc_ssize_t nc_seqindex_write_char_decomposed(nc_utf8proc_uint16_ for (; len >= 0; entry++, len--) { nc_utf8proc_int32_t entry_cp = nc_seqindex_decode_entry(&entry); - written += nc_utf8proc_decompose_char(entry_cp, dst+written, + nc_utf8proc_int32_t *dest = dst ? (dst+written) : NULL; + written += nc_utf8proc_decompose_char(entry_cp, dest, (bufsize > written) ? (bufsize - written) : 0, options, last_boundclass); if (written < 0) return UTF8PROC_ERROR_OVERFLOW; @@ -525,8 +526,10 @@ static nc_utf8proc_ssize_t nc_seqindex_write_char_decomposed(nc_utf8proc_uint16_ if (custom_func != NULL) { uc = custom_func(uc, custom_data); /* user-specified custom mapping */ } + nc_utf8proc_int32_t *dest = NULL; + if (buffer) dest = buffer + wpos; decomp_result = nc_utf8proc_decompose_char( - uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options, + uc, dest, (bufsize > wpos) ? (bufsize - wpos) : 0, options, &boundclass ); if (decomp_result < 0) return decomp_result;