diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 8d5eca680..8ec6c93b7 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,10 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Improve the speed and data quantity for DAP4 queries. See [Github #2765](https://github.com/Unidata/netcdf-c/pull/2765). +* Remove the use of execinfo to programmatically dump the stack; it never worked. See [Github #2789](https://github.com/Unidata/netcdf-c/pull/2789). +* Update the internal copy of tinyxml2 to latest code. See [Github #2771](https://github.com/Unidata/netcdf-c/pull/2771). +* Mitigate the problem of remote/nczarr-related test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764). * Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). diff --git a/config.h.cmake.in b/config.h.cmake.in index 941240390..f10a3a91a 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -468,9 +468,6 @@ with zip */ /* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */ #cmakedefine HDF5_UTF8_PATHS 1 -/* if true, backtrace support will be used. */ -#cmakedefine HAVE_EXECINFO_H 1 - /* if true, include JNA bug fix */ #cmakedefine JNA 1 diff --git a/configure.ac b/configure.ac index a35fd764d..3c5b7e0a9 100644 --- a/configure.ac +++ b/configure.ac @@ -1312,9 +1312,6 @@ AC_CHECK_HEADERS([sys/resource.h]) # See if we have ftw.h to walk directory trees AC_CHECK_HEADERS([ftw.h]) -# See if we can do stack tracing programmatically -AC_CHECK_HEADERS([execinfo.h]) - # Check for these functions... AC_CHECK_FUNCS([strlcat snprintf strcasecmp fileno \ strdup strtoll strtoull \ diff --git a/dap4_test/test_common.h b/dap4_test/test_common.h index a005d9c9c..d7e096e2d 100644 --- a/dap4_test/test_common.h +++ b/dap4_test/test_common.h @@ -21,6 +21,7 @@ typedef int TDMR; static NCbytes* input = NULL; static NCbytes* output = NULL; static NCD4meta* metadata = NULL; +static NCD4response* resp = NULL; static char* infile = NULL; static char* outfile = NULL; static int ncid = 0; @@ -85,16 +86,21 @@ setup(int tdmr, int argc, char** argv) if(translatenc4) controller->controls.translation = NCD4_TRANSNC4; NCD4_applyclientfragmentcontrols(controller); - if((metadata=NCD4_newmeta(controller))==NULL) - fail(NC_ENOMEM); - metadata->mode = mode; - NCD4_attachraw(metadata, ncbyteslength(input),ncbytescontents(input)); - if((ret=NCD4_dechunk(metadata))) /* ok for mode == DMR or mode == DAP */ + if((ret=NCD4_newMeta(controller,&metadata))) + fail(ret); + + if((ret=NCD4_newResponse(controller,&resp))) + fail(ret); + resp->raw.size = ncbyteslength(input); + resp->raw.memory = ncbytescontents(input); + resp->mode = mode; + + if((ret=NCD4_dechunk(resp))) /* ok for mode == DMR or mode == DAP */ fail(ret); #ifdef DEBUG { - int swap = (metadata->serial.hostbigendian != metadata->serial.remotebigendian); + int swap = (controller->platform.hostlittleendian != resp->remotelittleendian); void* d = metadata->serial.dap; size_t sz = metadata->serial.dapsize; fprintf(stderr,"====================\n"); diff --git a/dap4_test/test_meta.c b/dap4_test/test_meta.c index f0bf2273b..478fcfb62 100644 --- a/dap4_test/test_meta.c +++ b/dap4_test/test_meta.c @@ -20,7 +20,7 @@ main(int argc, char** argv) fprintf(stderr,"t_dmrmeta %s -> %s\n",infile,outfile); #endif - if((ret = NCD4_parse(metadata))) goto done; + if((ret = NCD4_parse(metadata,resp,0))) goto done; if((ret = NCD4_metabuild(metadata,ncid))) goto done; done: diff --git a/dap4_test/test_parse.c b/dap4_test/test_parse.c index 44075ffaa..59afb74d5 100644 --- a/dap4_test/test_parse.c +++ b/dap4_test/test_parse.c @@ -17,7 +17,7 @@ main(int argc, char** argv) setup(TDMR_PARSE,argc,argv); - if((ret = NCD4_parse(metadata))) goto done; + if((ret = NCD4_parse(metadata,resp,0))) goto done; ret = NCD4_print(metadata,output); ncbytesnull(output); if(ret == NC_NOERR) { diff --git a/include/ncconfigure.h b/include/ncconfigure.h index cad59c9e2..496ec59e0 100644 --- a/include/ncconfigure.h +++ b/include/ncconfigure.h @@ -61,7 +61,7 @@ extern "C" { #endif /* WARNING: in some systems, these functions may be defined as macros, so check */ -#ifndef HAVE_STRDUP +#if ! defined(HAVE_STRDUP) || defined(__CYGWIN__) #ifndef strdup char* strdup(const char*); #endif diff --git a/include/ncuri.h b/include/ncuri.h index eafb9be97..bd1b64101 100644 --- a/include/ncuri.h +++ b/include/ncuri.h @@ -35,12 +35,8 @@ typedef struct NCURI { char* path; /*!< path */ char* query; /*!< query */ char* fragment; /*!< fragment */ - char** fraglist; /* envv style list of decomposed fragment*/ - char** querylist; /* envv style list of decomposed query*/ -#if 0 - char* projection; /*!< without leading '?'*/ - char* selection; /*!< with leading '&'*/ -#endif + void* fraglist; /* some representation of the decomposed fragment string */ + void* querylist; /* some representation of the decomposed query string */ } NCURI; #if 0 @@ -90,6 +86,18 @@ EXTERNL int ncurisetfragmentkey(NCURI* duri,const char* key, const char* value); /* append a specific &key=...& in uri fragment */ EXTERNL int ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value); +/* Replace a specific &key=...& in uri query */ +EXTERNL int ncurisetquerykey(NCURI* duri,const char* key, const char* value); + +/* append a specific &key=...& in uri query */ +EXTERNL int ncuriappendquerykey(NCURI* duri,const char* key, const char* value); + +/* Get the actual list of queryies */ +EXTERNL void* ncuriqueryparams(NCURI* uri); +/* Get the actual list of frags */ +EXTERNL void* ncurifragmentparams(NCURI* uri); + + /* Construct a complete NC URI; caller frees returned string */ EXTERNL char* ncuribuild(NCURI*,const char* prefix, const char* suffix, int flags); @@ -105,12 +113,6 @@ EXTERNL const char* ncurifragmentlookup(NCURI*, const char* param); */ EXTERNL const char* ncuriquerylookup(NCURI*, const char* param); -/* Obtain the complete list of fragment pairs in envv format */ -EXTERNL const char** ncurifragmentparams(NCURI*); - -/* Obtain the complete list of query pairs in envv format */ -EXTERNL const char** ncuriqueryparams(NCURI*); - /* URL Encode/Decode */ EXTERNL char* ncuridecode(const char* s); /* Partial decode */ diff --git a/libdap4/d4chunk.c b/libdap4/d4chunk.c index e95f34b9c..3f2fe44d7 100644 --- a/libdap4/d4chunk.c +++ b/libdap4/d4chunk.c @@ -22,133 +22,128 @@ Notes: */ /* Forward */ -static int processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count); +static int processerrchunk(NCD4response*, void* errchunk, unsigned int count); /**************************************************/ - -void -NCD4_resetSerial(NCD4serial* serial, size_t rawsize, void* rawdata) -{ - nullfree(serial->errdata); - nullfree(serial->dmr); - nullfree(serial->dap); - nullfree(serial->rawdata); - /* clear all fields */ - memset(serial,0,sizeof(NCD4serial)); - /* Reset fields */ - serial->hostlittleendian = NCD4_isLittleEndian(); - serial->rawsize = rawsize; - serial->rawdata = rawdata; -} - int -NCD4_dechunk(NCD4meta* metadata) +NCD4_dechunk(NCD4response* resp) { - unsigned char *praw, *phdr, *pdap; + unsigned char *praw, *pdmr, *phdr, *pdap, *pappend, *pchunk; NCD4HDR hdr; + int firstchunk; #ifdef D4DUMPRAW - NCD4_tagdump(metadata->serial.rawsize,metadata->serial.rawdata,0,"RAW"); + NCD4_tagdump(resp->serial.raw.size,resp->serial.raw.data,0,"RAW"); #endif /* Access the returned raw data */ - praw = metadata->serial.rawdata; + praw = (unsigned char*)resp->raw.memory; - if(metadata->mode == NCD4_DSR) { + if(resp->mode == NCD4_DSR) { return THROW(NC_EDMR); - } else if(metadata->mode == NCD4_DMR) { + } else if(resp->mode == NCD4_DMR) { /* Verify the mode; assume that the is optional */ if(memcmp(praw,"serial.rawsize; - if((metadata->serial.dmr = malloc(len+1)) == NULL) + len = resp->raw.size; + if((resp->serial.dmr = malloc(len+1)) == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.dmr,praw,len); - metadata->serial.dmr[len] = '\0'; + memcpy(resp->serial.dmr,praw,len); + resp->serial.dmr[len] = '\0'; /* Suppress nuls */ - (void)NCD4_elidenuls(metadata->serial.dmr,len); + (void)NCD4_elidenuls(resp->serial.dmr,len); return THROW(NC_NOERR); } - } else if(metadata->mode != NCD4_DAP) + } else if(resp->mode != NCD4_DAP) return THROW(NC_EDAP); /* We must be processing a DAP mode packet */ - praw = (metadata->serial.dap = metadata->serial.rawdata); - metadata->serial.rawdata = NULL; + praw = resp->raw.memory; /* If the raw data looks like xml, then we almost certainly have an error */ if(memcmp(praw,"serial.rawsize, metadata->serial.rawdata); + int stat = NCD4_seterrormessage(resp, resp->raw.size, resp->raw.memory); return THROW(stat); /* slight lie */ } - /* Get the DMR chunk header*/ - phdr = NCD4_getheader(praw,&hdr,metadata->serial.hostlittleendian); + /* Get the first header to get dmr content and endian flags*/ + pdmr = NCD4_getheader(praw,&hdr,resp->controller->platform.hostlittleendian); if(hdr.count == 0) return THROW(NC_EDMR); - if(hdr.flags & NCD4_ERR_CHUNK) { - return processerrchunk(metadata, (void*)phdr, hdr.count); - } + if(hdr.flags & NCD4_ERR_CHUNK) + return processerrchunk(resp, (void*)pdmr, hdr.count); + resp->remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0); - metadata->serial.remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0); - /* Again, avoid strxxx operations on dmr */ - if((metadata->serial.dmr = malloc(hdr.count+1)) == NULL) + /* avoid strxxx operations on dmr */ + if((resp->serial.dmr = malloc(hdr.count+1)) == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.dmr,phdr,hdr.count); - metadata->serial.dmr[hdr.count-1] = '\0'; + memcpy(resp->serial.dmr,pdmr,hdr.count); + resp->serial.dmr[hdr.count-1] = '\0'; /* Suppress nuls */ - (void)NCD4_elidenuls(metadata->serial.dmr,hdr.count); + (void)NCD4_elidenuls(resp->serial.dmr,hdr.count); + /* See if there is any data after the DMR */ if(hdr.flags & NCD4_LAST_CHUNK) return THROW(NC_ENODATA); /* Read and concat together the data chunks */ - phdr = phdr + hdr.count; /* point to data chunk header */ + phdr = pdmr + hdr.count; /* point to data chunk header */ /* Do a sanity check in case the server has shorted us with no data */ - if((hdr.count + CHUNKHDRSIZE) >= metadata->serial.rawsize) { + if((hdr.count + CHUNKHDRSIZE) >= resp->raw.size) { /* Server only sent the DMR part */ - metadata->serial.dapsize = 0; + resp->serial.dapsize = 0; return THROW(NC_EDATADDS); } - pdap = metadata->serial.dap; - for(;;) { - phdr = NCD4_getheader(phdr,&hdr,metadata->serial.hostlittleendian); - if(hdr.flags & NCD4_ERR_CHUNK) { - return processerrchunk(metadata, (void*)phdr, hdr.count); + /* walk all the data chunks */ + /* invariants: + praw -- beginning of the raw response + pdmr -- beginning of the dmr in the raw data + pdap -- beginning of the dechunked dap data + phdr -- pointer to the hdr of the current chunk + pchunk -- pointer to the data part of the current chunk + pappend -- where to append next chunk to the growing dechunked data + */ + for(firstchunk=1;;firstchunk=0) { + pchunk = NCD4_getheader(phdr,&hdr,resp->controller->platform.hostlittleendian); /* Process first data chunk header */ + if(firstchunk) { + pdap = phdr; /* remember start point of the dechunked data */ + pappend = phdr; /* start appending here */ } + if(hdr.flags & NCD4_ERR_CHUNK) + return processerrchunk(resp, (void*)pchunk, hdr.count); /* data chunk; possibly last; possibly empty */ - if(hdr.count > 0) { - d4memmove(pdap,phdr,hdr.count); /* will overwrite the header */ - phdr += hdr.count; - pdap += hdr.count; - } + if(hdr.count > 0) + d4memmove(pappend,pchunk,hdr.count); /* overwrite the header; this the heart of dechunking */ + pappend += hdr.count; /* next append point */ + phdr = pchunk + hdr.count; /* point to header of next chunk */ if(hdr.flags & NCD4_LAST_CHUNK) break; } - metadata->serial.dapsize = (size_t)DELTA(pdap,metadata->serial.dap); + resp->serial.dap = pdap; /* start of dechunked data */ + resp->serial.dapsize = (size_t)DELTA(pappend,pdap); #ifdef D4DUMPDMR - fprintf(stderr,"%s\n",metadata->serial.dmr); + fprintf(stderr,"%s\n",resp->serial.dmr); fflush(stderr); #endif #ifdef D4DUMPDAP - NCD4_tagdump(metadata->serial.dapsize,metadata->serial.dap,0,"DAP"); + NCD4_tagdump(resp->serial.dapsize,resp->serial.dap,0,"DAP"); #endif return THROW(NC_NOERR); } static int -processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count) +processerrchunk(NCD4response* resp, void* errchunk, unsigned int count) { - metadata->serial.errdata = (char*)d4alloc(count+1); - if(metadata->serial.errdata == NULL) + resp->serial.errdata = (char*)d4alloc(count+1); + if(resp->serial.errdata == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.errdata,errchunk,count); - metadata->serial.errdata[count] = '\0'; + memcpy(resp->serial.errdata,errchunk,count); + resp->serial.errdata[count] = '\0'; return THROW(NC_ENODATA); /* slight lie */ } @@ -157,26 +152,26 @@ Given a raw response, attempt to infer the mode: DMR, DAP, DSR. Since DSR is not standardizes, it becomes the default. */ int -NCD4_infermode(NCD4meta* meta) +NCD4_infermode(NCD4response* resp) { - d4size_t size = meta->serial.rawsize; - char* raw = meta->serial.rawdata; + d4size_t size = resp->raw.size; + char* raw = resp->raw.memory; if(size < 16) return THROW(NC_EDAP); /* must have at least this to hold a hdr + partial dmr*/ if(memcmp(raw,"mode = NCD4_DMR; + resp->mode = NCD4_DMR; goto done; } raw += 4; /* Pretend we have a DAP hdr */ if(memcmp(raw,"mode = NCD4_DAP; + resp->mode = NCD4_DAP; goto done; } /* Default to DSR */ - meta->mode = NCD4_DSR; + resp->mode = NCD4_DSR; done: return NC_NOERR; diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index ee06e4cac..b1f949b9f 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -340,7 +340,7 @@ NCD4_get_rcproperties(NCD4INFO* state) ncerror err = NC_NOERR; char* option = NULL; #ifdef HAVE_CURLOPT_BUFFERSIZE - option = NC_rclookup(D4BUFFERSIZE,state->uri->uri,NULL); + option = NC_rclookup(D4BUFFERSIZE,state->dmruri->uri,NULL); if(option != NULL && strlen(option) != 0) { long bufsize; if(strcasecmp(option,"max")==0) @@ -351,7 +351,7 @@ NCD4_get_rcproperties(NCD4INFO* state) } #endif #ifdef HAVE_CURLOPT_KEEPALIVE - option = NC_rclookup(D4KEEPALIVE,state->uri->uri,NULL); + option = NC_rclookup(D4KEEPALIVE,state->dmruri->uri,NULL); if(option != NULL && strlen(option) != 0) { /* The keepalive value is of the form 0 or n/m, where n is the idle time and m is the interval time; diff --git a/libdap4/d4data.c b/libdap4/d4data.c index 5d61b81dc..eae5f32ed 100644 --- a/libdap4/d4data.c +++ b/libdap4/d4data.c @@ -16,7 +16,7 @@ This code serves two purposes (NCD4_processdata) 2. Walk a specified variable instance to convert to netcdf4 memory representation. - (NCD4_fillinstance) + (NCD4_movetoinstance) */ @@ -29,7 +29,6 @@ static int fillopfixed(NCD4meta*, d4size_t opaquesize, NCD4offset* offset, void* static int fillopvar(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); static int fillstruct(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); static int fillseq(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); -static int NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel); static unsigned NCD4_computeChecksum(NCD4meta* meta, NCD4node* topvar); /***************************************************/ @@ -54,8 +53,9 @@ static unsigned int debugcrc32(unsigned int crc, const void *buf, size_t size) /***************************************************/ /* API */ +/* Parcel out the dechunked data to the corresponding vars */ int -NCD4_processdata(NCD4meta* meta) +NCD4_parcelvars(NCD4meta* meta, NCD4response* resp) { int ret = NC_NOERR; int i; @@ -68,35 +68,57 @@ NCD4_processdata(NCD4meta* meta) toplevel = nclistnew(); NCD4_getToplevelVars(meta,root,toplevel); - /* Otherwise */ - NCD4_inferChecksums(meta,toplevel); - - /* If necessary, byte swap the serialized data */ - /* Do we need to swap the dap4 data? */ - meta->swap = (meta->serial.hostlittleendian != meta->serial.remotelittleendian); - /* Compute the offset and size of the toplevel vars in the raw dap data. */ - /* Also extract remote checksums */ - offset = BUILDOFFSET(meta->serial.dap,meta->serial.dapsize); + offset = BUILDOFFSET(resp->serial.dap,resp->serial.dapsize); for(i=0;iinferredchecksumming))) { FAIL(ret,"delimit failure"); - if(meta->controller->data.inferredchecksumming) { - /* Compute remote checksum: must occur before any byte swapping */ + } + var->data.response = resp; /* cross link */ + } +done: + nclistfree(toplevel); + nullfree(offset); + return THROW(ret); +} + +/* Process top level vars wrt checksums and swapping */ +int +NCD4_processdata(NCD4meta* meta, NCD4response* resp) +{ + int ret = NC_NOERR; + int i; + NClist* toplevel = NULL; + NCD4node* root = meta->root; + NCD4offset* offset = NULL; + + /* Do we need to swap the dap4 data? */ + meta->swap = (meta->controller->platform.hostlittleendian != resp->remotelittleendian); + + /* Recursively walk the tree in prefix order + to get the top-level variables; also mark as unvisited */ + toplevel = nclistnew(); + NCD4_getToplevelVars(meta,root,toplevel); + + /* Extract remote checksums */ + for(i=0;iinferredchecksumming) { + /* Compute checksum of response data: must occur before any byte swapping and after delimiting */ var->data.localchecksum = NCD4_computeChecksum(meta,var); #ifdef DUMPCHECKSUM fprintf(stderr,"var %s: remote-checksum = 0x%x\n",var->name,var->data.remotechecksum); #endif /* verify checksums */ - if(!meta->controller->data.checksumignore) { + if(!resp->checksumignore) { if(var->data.localchecksum != var->data.remotechecksum) { nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name); ret = NC_EDAP; goto done; } /* Also verify checksum attribute */ - if(meta->controller->data.attrchecksumming) { + if(resp->attrchecksumming) { if(var->data.attrchecksum != var->data.remotechecksum) { nclog(NCLOGERR,"Attribute Checksum mismatch: %s\n",var->name); ret = NC_EDAP; @@ -105,13 +127,11 @@ NCD4_processdata(NCD4meta* meta) } } } - } - - /* Swap the data for each top level variable, - */ - if(meta->swap) { - if((ret=NCD4_swapdata(meta,toplevel))) - FAIL(ret,"byte swapping failed"); + if(meta->swap) { + if((ret=NCD4_swapdata(resp,var,meta->swap))) + FAIL(ret,"byte swapping failed"); + } + var->data.valid = 1; /* Everything should be in place */ } done: @@ -133,7 +153,7 @@ Assumes that NCD4_processdata has been called. */ int -NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs) +NCD4_movetoinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs) { int ret = NC_NOERR; void* dst = *dstp; @@ -149,30 +169,30 @@ NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dst } else switch(type->subsort) { case NC_STRING: /* oob strings */ if((ret=fillstring(meta,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; case NC_OPAQUE: if(type->opaque.size > 0) { /* We know the size and its the same for all instances */ if((ret=fillopfixed(meta,type->opaque.size,offset,&dst))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } else { /* Size differs per instance, so we need to convert each opaque to a vlen */ if((ret=fillopvar(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } break; case NC_STRUCT: if((ret=fillstruct(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; case NC_SEQ: if((ret=fillseq(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; default: ret = NC_EINVAL; - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } *dstp = dst; @@ -196,7 +216,7 @@ fillstruct(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NCli NCD4node* field = nclistget(type->vars,i); NCD4node* ftype = field->basetype; void* fdst = (((char*)dst) + field->meta.offset); - if((ret=NCD4_fillinstance(meta,ftype,offset,&fdst,blobs))) + if((ret=NCD4_movetoinstance(meta,ftype,offset,&fdst,blobs))) FAIL(ret,"fillstruct"); } dst = ((char*)dst) + type->meta.memsize; @@ -231,7 +251,7 @@ fillseq(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* for(i=0;ip))+(recordsize * i); - if((ret=NCD4_fillinstance(meta,vlentype,offset,&recdst,blobs))) + if((ret=NCD4_movetoinstance(meta,vlentype,offset,&recdst,blobs))) FAIL(ret,"fillseq"); } dst++; @@ -373,12 +393,16 @@ done: return THROW(ret); } -static int -NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel) +int +NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp) { int ret = NC_NOERR; int i, attrfound; - NCD4INFO* info = meta->controller; + NClist* toplevel = NULL; + + /* Get the toplevel vars */ + toplevel = nclistnew(); + NCD4_getToplevelVars(meta,meta->root,toplevel); /* First, look thru the DMR to see if there is a checksum attribute */ attrfound = 0; @@ -399,9 +423,10 @@ NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel) } } } - info->data.attrchecksumming = (attrfound ? 1 : 0); + nclistfree(toplevel); + resp->attrchecksumming = (attrfound ? 1 : 0); /* Infer checksums */ - info->data.inferredchecksumming = ((info->data.attrchecksumming || info->data.querychecksumming) ? 1 : 0); + resp->inferredchecksumming = ((resp->attrchecksumming || resp->querychecksumming) ? 1 : 0); return THROW(ret); } diff --git a/libdap4/d4debug.c b/libdap4/d4debug.c index d2d40b44a..caed80027 100644 --- a/libdap4/d4debug.c +++ b/libdap4/d4debug.c @@ -97,7 +97,7 @@ int NCD4_debugcopy(NCD4INFO* info) { int i,ret=NC_NOERR; - NCD4meta* meta = info->substrate.metadata; + NCD4meta* meta = info->dmrmetadata; NClist* topvars = nclistnew(); NC* ncp = info->controller; void* memory = NULL; diff --git a/libdap4/d4file.c b/libdap4/d4file.c index aeccc7423..46c1a0fdb 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -6,7 +6,6 @@ #include "ncdispatch.h" #include "ncd4dispatch.h" #include "d4includes.h" -#include "d4read.h" #include "d4curlfunctions.h" #ifdef _MSC_VER @@ -23,19 +22,20 @@ static int constrainable(NCURI*); static void freeCurl(NCD4curl*); -static void freeInfo(NCD4INFO*); static int fragmentcheck(NCD4INFO*, const char* key, const char* subkey); static const char* getfragment(NCD4INFO* info, const char* key); static const char* getquery(NCD4INFO* info, const char* key); static int set_curl_properties(NCD4INFO*); static int makesubstrate(NCD4INFO* d4info); -static void resetInfoforRead(NCD4INFO* d4info); /**************************************************/ /* Constants */ static const char* checkseps = "+,:;"; +/*Define the set of protocols known to be constrainable */ +static const char* constrainableprotocols[] = {"http", "https",NULL}; + /**************************************************/ int NCD4_open(const char * path, int mode, @@ -46,10 +46,10 @@ NCD4_open(const char * path, int mode, NCD4INFO* d4info = NULL; const char* value; NC* nc; - NCD4meta* meta = NULL; size_t len = 0; void* contents = NULL; - + NCD4response* dmrresp = NULL; + if(path == NULL) return THROW(NC_EDAPURL); @@ -61,29 +61,27 @@ NCD4_open(const char * path, int mode, /* Setup our NC and NCDAPCOMMON state*/ - d4info = (NCD4INFO*)calloc(1,sizeof(NCD4INFO)); - if(d4info == NULL) {ret = NC_ENOMEM; goto done;} - + if((ret=NCD4_newInfo(&d4info))) goto done; nc->dispatchdata = d4info; nc->int_ncid = nc__pseudofd(); /* create a unique id */ d4info->controller = (NC*)nc; /* Parse url and params */ - if(ncuriparse(nc->path,&d4info->uri)) + if(ncuriparse(nc->path,&d4info->dmruri)) {ret = NC_EDAPURL; goto done;} /* Load auth info from rc file */ - if((ret = NC_authsetup(&d4info->auth, d4info->uri))) + if((ret = NC_authsetup(&d4info->auth, d4info->dmruri))) goto done; NCD4_curl_protocols(d4info); - if(!constrainable(d4info->uri)) + if(!constrainable(d4info->dmruri)) SETFLAG(d4info->controls.flags,NCF_UNCONSTRAINABLE); /* fail if we are unconstrainable but have constraints */ if(FLAGSET(d4info->controls.flags,NCF_UNCONSTRAINABLE)) { - if(d4info->uri != NULL) { - const char* ce = ncuriquerylookup(d4info->uri,DAP4CE); /* Look for dap4.ce */ + if(d4info->dmruri != NULL) { + const char* ce = ncuriquerylookup(d4info->dmruri,DAP4CE); /* Look for dap4.ce */ if(ce != NULL) { nclog(NCLOGWARN,"Attempt to constrain an unconstrainable data source: %s=%s", DAP4CE,ce); @@ -115,7 +113,7 @@ NCD4_open(const char * path, int mode, } /* Turn on logging; only do this after oc_open*/ - if((value = ncurifragmentlookup(d4info->uri,"log")) != NULL) { + if((value = ncurifragmentlookup(d4info->dmruri,"log")) != NULL) { ncloginit(); ncsetloglevel(NCLOGNOTE); } @@ -150,30 +148,34 @@ NCD4_open(const char * path, int mode, /* Reset the substrate */ if((ret=makesubstrate(d4info))) goto done; - /* Always start by reading the DMR only */ + /* Always start by reading the whole DMR only */ /* reclaim substrate.metadata */ - resetInfoforRead(d4info); + NCD4_resetInfoForRead(d4info); /* Rebuild metadata */ - if((d4info->substrate.metadata=NCD4_newmeta(d4info))==NULL) - {ret = NC_ENOMEM; goto done;} + if((ret = NCD4_newMeta(d4info,&d4info->dmrmetadata))) goto done; - if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags))) goto done; + /* Capture response */ + if((dmrresp = (NCD4response*)calloc(1,sizeof(NCD4response)))==NULL) + {ret = NC_ENOMEM; goto done;} + dmrresp->controller = d4info; + + if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags, d4info->dmruri, dmrresp))) goto done; /* set serial.rawdata */ len = ncbyteslength(d4info->curl->packet); contents = ncbytesextract(d4info->curl->packet); - NCD4_attachraw(d4info->substrate.metadata, len, contents); + assert(dmrresp != NULL); + dmrresp->raw.size = len; + dmrresp->raw.memory = contents; - /* process query parameters */ - NCD4_applyclientquerycontrols(d4info); - - meta = d4info->substrate.metadata; + /* process checksum parameters */ + NCD4_applychecksumcontrols(d4info,dmrresp); /* Infer the mode */ - if((ret=NCD4_infermode(meta))) goto done; + if((ret=NCD4_infermode(dmrresp))) goto done; /* Process the dmr part */ - if((ret=NCD4_dechunk(meta))) goto done; + if((ret=NCD4_dechunk(dmrresp))) goto done; #ifdef D4DUMPDMR { @@ -184,13 +186,14 @@ NCD4_open(const char * path, int mode, } #endif - if((ret = NCD4_parse(d4info->substrate.metadata))) goto done; + if((ret = NCD4_parse(d4info->dmrmetadata,dmrresp,0))) goto done; #ifdef D4DEBUGMETA { + meta = d4info->dmrmetadata; fprintf(stderr,"\n/////////////\n"); NCbytes* buf = ncbytesnew(); - NCD4_print(d4info->substrate.metadata,buf); + NCD4_print(meta,buf); ncbytesnull(buf); fputs(ncbytescontents(buf),stderr); ncbytesfree(buf); @@ -200,12 +203,20 @@ NCD4_open(const char * path, int mode, #endif /* Build the substrate metadata */ - ret = NCD4_metabuild(d4info->substrate.metadata,d4info->substrate.metadata->ncid); + ret = NCD4_metabuild(d4info->dmrmetadata,d4info->dmrmetadata->ncid); if(ret != NC_NOERR && ret != NC_EVARSIZE) goto done; + /* Remember the response */ + nclistpush(d4info->responses,dmrresp); + + /* Avoid duplicate reclaims */ + dmrresp = NULL; + d4info = NULL; + done: + NCD4_reclaimResponse(dmrresp); + NCD4_reclaimInfo(d4info); if(ret) { - freeInfo(d4info); nc->dispatchdata = NULL; } return THROW(ret); @@ -236,7 +247,7 @@ NCD4_close(int ncid, void* ignore) ret = nc_abort(substrateid); } - freeInfo(d4info); + NCD4_reclaimInfo(d4info); done: return THROW(ret); @@ -248,82 +259,6 @@ NCD4_abort(int ncid) return NCD4_close(ncid,NULL); } -/**************************************************/ - -/* Reclaim an NCD4INFO instance */ -static void -freeInfo(NCD4INFO* d4info) -{ - if(d4info == NULL) return; - d4info->controller = NULL; /* break link */ - nullfree(d4info->rawurltext); - nullfree(d4info->urltext); - ncurifree(d4info->uri); - freeCurl(d4info->curl); - nullfree(d4info->data.memory); - nullfree(d4info->data.ondiskfilename); - if(d4info->data.ondiskfile != NULL) - fclose(d4info->data.ondiskfile); - nullfree(d4info->fileproto.filename); - if(d4info->substrate.realfile - && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { - /* We used real file, so we need to delete the temp file - unless we are debugging. - Assume caller has done nc_close|nc_abort on the ncid. - Note that in theory, this should not be necessary since - AFAIK the substrate file is still in def mode, and - when aborted, it should be deleted. But that is not working - for some reason, so we delete it ourselves. - */ - if(d4info->substrate.filename != NULL) { - unlink(d4info->substrate.filename); - } - } - nullfree(d4info->substrate.filename); /* always reclaim */ - NCD4_reclaimMeta(d4info->substrate.metadata); - NC_authfree(d4info->auth); - nclistfree(d4info->blobs); - free(d4info); -} - -/* Reset NCD4INFO instance for new read request */ -static void -resetInfoforRead(NCD4INFO* d4info) -{ - if(d4info == NULL) return; - if(d4info->substrate.realfile - && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { - /* We used real file, so we need to delete the temp file - unless we are debugging. - Assume caller has done nc_close|nc_abort on the ncid. - Note that in theory, this should not be necessary since - AFAIK the substrate file is still in def mode, and - when aborted, it should be deleted. But that is not working - for some reason, so we delete it ourselves. - */ - if(d4info->substrate.filename != NULL) { - unlink(d4info->substrate.filename); - } - } - NCD4_resetMeta(d4info->substrate.metadata); - nullfree(d4info->substrate.metadata); - d4info->substrate.metadata = NULL; -} - -static void -freeCurl(NCD4curl* curl) -{ - if(curl == NULL) return; - NCD4_curlclose(curl->curl); - ncbytesfree(curl->packet); - nullfree(curl->errdata.code); - nullfree(curl->errdata.message); - free(curl); -} - -/* Define the set of protocols known to be constrainable */ -static const char* constrainableprotocols[] = {"http", "https",NULL}; - static int constrainable(NCURI* durl) { @@ -449,11 +384,6 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info) if(value != NULL) strncpy(info->controls.substratename,value,(NC_MAX_NAME-1)); - value = getfragment(info,"hyrax"); - if(value != NULL) { - info->data.checksumignore = 1; /* Assume checksum, but ignore */ - } - info->controls.opaquesize = DFALTOPAQUESIZE; value = getfragment(info,"opaquesize"); if(value != NULL) { @@ -476,22 +406,29 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info) } } +/* Checksum controls are found both in the query and fragment + parts of a URL. +*/ void -NCD4_applyclientquerycontrols(NCD4INFO* info) +NCD4_applychecksumcontrols(NCD4INFO* info, NCD4response* resp) { const char* value = getquery(info,DAP4CSUM); if(value == NULL) { - info->data.querychecksumming = DEFAULT_CHECKSUM_STATE; + resp->querychecksumming = DEFAULT_CHECKSUM_STATE; } else { if(strcasecmp(value,"false")==0) { - info->data.querychecksumming = 0; + resp->querychecksumming = 0; } else if(strcasecmp(value,"true")==0) { - info->data.querychecksumming = 1; + resp->querychecksumming = 1; } else { nclog(NCLOGWARN,"Unknown checksum mode: %s ; using default",value); - info->data.querychecksumming = DEFAULT_CHECKSUM_STATE; + resp->querychecksumming = DEFAULT_CHECKSUM_STATE; } } + value = getfragment(info,"hyrax"); + if(value != NULL) { + resp->checksumignore = 1; /* Assume checksum, but ignore */ + } } /* Search for substring in value of param. If substring == NULL; then just @@ -523,7 +460,7 @@ getfragment(NCD4INFO* info, const char* key) const char* value; if(info == NULL || key == NULL) return NULL; - if((value=ncurifragmentlookup(info->uri,key)) == NULL) + if((value=ncurifragmentlookup(info->dmruri,key)) == NULL) return NULL; return value; } @@ -537,7 +474,7 @@ getquery(NCD4INFO* info, const char* key) const char* value; if(info == NULL || key == NULL) return NULL; - if((value=ncuriquerylookup(info->uri,key)) == NULL) + if((value=ncuriquerylookup(info->dmruri,key)) == NULL) return NULL; return value; } @@ -596,3 +533,175 @@ NCD4_get_substrate(NC* nc) } else subnc = nc; return subnc; } + +/**************************************************/ +/* Allocate/Free for various structures */ + +int +NCD4_newInfo(NCD4INFO** d4infop) +{ + int ret = NC_NOERR; + NCD4INFO* info = NULL; + if((info = calloc(1,sizeof(NCD4INFO)))==NULL) + {ret = NC_ENOMEM; goto done;} + info->platform.hostlittleendian = NCD4_isLittleEndian(); + info->responses = nclistnew(); + if(d4infop) {*d4infop = info; info = NULL;} +done: + if(info) NCD4_reclaimInfo(info); + return THROW(ret); +} + +/* Reclaim an NCD4INFO instance */ +void +NCD4_reclaimInfo(NCD4INFO* d4info) +{ + size_t i; + if(d4info == NULL) return; + d4info->controller = NULL; /* break link */ + nullfree(d4info->rawdmrurltext); + nullfree(d4info->dmrurltext); + ncurifree(d4info->dmruri); + freeCurl(d4info->curl); + nullfree(d4info->fileproto.filename); + NCD4_resetInfoForRead(d4info); + nullfree(d4info->substrate.filename); /* always reclaim */ + NC_authfree(d4info->auth); + nclistfree(d4info->blobs); + /* Reclaim dmr node tree */ + NCD4_reclaimMeta(d4info->dmrmetadata); + /* Reclaim all responses */ + for(i=0;iresponses);i++) { + NCD4response* resp = nclistget(d4info->responses,i); + NCD4_reclaimResponse(resp); + } + nclistfree(d4info->responses); + free(d4info); +} + +/* Reset NCD4INFO instance for new read request */ +void +NCD4_resetInfoForRead(NCD4INFO* d4info) +{ + if(d4info == NULL) return; + if(d4info->substrate.realfile + && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { + /* We used real file, so we need to delete the temp file + unless we are debugging. + Assume caller has done nc_close|nc_abort on the ncid. + Note that in theory, this should not be necessary since + AFAIK the substrate file is still in def mode, and + when aborted, it should be deleted. But that is not working + for some reason, so we delete it ourselves. + */ + if(d4info->substrate.filename != NULL) { + unlink(d4info->substrate.filename); + } + } + NCD4_reclaimMeta(d4info->dmrmetadata); + d4info->dmrmetadata = NULL; +} + +static void +freeCurl(NCD4curl* curl) +{ + if(curl == NULL) return; + NCD4_curlclose(curl->curl); + ncbytesfree(curl->packet); + nullfree(curl->errdata.code); + nullfree(curl->errdata.message); + free(curl); +} + +int +NCD4_newResponse(NCD4INFO* info, NCD4response** respp) +{ + int ret = NC_NOERR; + NCD4response* resp = NULL; + NC_UNUSED(info); + if((resp = calloc(1,sizeof(NCD4response)))==NULL) + {ret = NC_ENOMEM; goto done;} + resp->controller = info; + if(respp) {*respp = resp; resp = NULL;} +done: + if(resp) NCD4_reclaimResponse(resp); + return THROW(ret); +} + + +/* Reclaim an NCD4response instance */ +void +NCD4_reclaimResponse(NCD4response* d4resp) +{ + struct NCD4serial* serial = NULL; + if(d4resp == NULL) return; + serial = &d4resp->serial; + d4resp->controller = NULL; /* break link */ + + nullfree(d4resp->raw.memory); + nullfree(serial->dmr); + nullfree(serial->errdata); + + /* clear all fields */ + memset(serial,0,sizeof(struct NCD4serial)); + + nullfree(d4resp->error.parseerror); + nullfree(d4resp->error.message); + nullfree(d4resp->error.context); + nullfree(d4resp->error.otherinfo); + memset(&d4resp->error,0,sizeof(d4resp->error)); + + free(d4resp); +} + +/* Create an empty NCD4meta object for + use in subsequent calls + (is the the right src file to hold this?) +*/ + +int +NCD4_newMeta(NCD4INFO* info, NCD4meta** metap) +{ + int ret = NC_NOERR; + NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); + if(meta == NULL) return NC_ENOMEM; + meta->allnodes = nclistnew(); +#ifdef D4DEBUG + meta->debuglevel = 1; +#endif + meta->controller = info; + meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ + if(metap) {*metap = meta; meta = NULL;} + return THROW(ret); +} + +void +NCD4_reclaimMeta(NCD4meta* dataset) +{ + int i; + if(dataset == NULL) return; + + for(i=0;iallnodes);i++) { + NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); + reclaimNode(node); + } + nclistfree(dataset->allnodes); + nclistfree(dataset->groupbyid); + nclistfree(dataset->atomictypes); + free(dataset); +} + +#if 0 +void +NCD4_resetMeta(NCD4meta* dataset) +{ + if(dataset == NULL) return; +#if 0 + for(i=0;iblobs);i++) { + void* p = nclistget(dataset->blobs,i); + nullfree(p); + } + nclistfree(dataset->blobs); +#endif +} +#endif diff --git a/libdap4/d4fix.c b/libdap4/d4fix.c index 82049dd90..b76a52d28 100644 --- a/libdap4/d4fix.c +++ b/libdap4/d4fix.c @@ -190,7 +190,7 @@ walk(NCD4node* node, NClist* sorted) */ int -NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset) +NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset, int inferredchecksumming) { int ret = NC_NOERR; NCD4mark mark = 0; @@ -214,7 +214,7 @@ NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset) topvar->data.dap4data.memory = mark; topvar->data.dap4data.size = OFFSETSIZE(offset,mark); /* extract the dap4 data checksum, if present */ - if(compiler->controller->data.inferredchecksumming) { + if(inferredchecksumming) { union ATOMICS csum; TRANSFER(csum.u8,offset,CHECKSUMSIZE); topvar->data.remotechecksum = csum.u32[0]; diff --git a/libdap4/d4http.c b/libdap4/d4http.c index 8e370d879..13cb82a19 100644 --- a/libdap4/d4http.c +++ b/libdap4/d4http.c @@ -6,7 +6,6 @@ #include "d4includes.h" #include "d4curlfunctions.h" -static size_t WriteFileCallback(void*, size_t, size_t, void*); static size_t WriteMemoryCallback(void*, size_t, size_t, void*); static int curlerrtoncerr(CURLcode cstat); @@ -33,59 +32,6 @@ NCD4_fetchhttpcode(CURL* curl) return httpcode; } -int -NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, - d4size_t* sizep, long* filetime) -{ - int ret = NC_NOERR; - CURLcode cstat = CURLE_OK; - struct Fetchdata fetchdata; - - /* Set the URL */ - cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url); - if (cstat != CURLE_OK) goto fail; - - /* send all data to this function */ - cstat = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteFileCallback); - if (cstat != CURLE_OK) goto fail; - - /* we pass our file to the callback function */ - cstat = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fetchdata); - if(cstat != CURLE_OK) goto fail; - - /* One last thing; always try to get the last modified time */ - cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1); - if (cstat != CURLE_OK) goto fail; - - fetchdata.stream = stream; - fetchdata.size = 0; - cstat = curl_easy_perform(curl); - if (cstat != CURLE_OK) - {ret = NC_EDAPSVC; goto fail;} - - if (ret == NC_NOERR) { - /* return the file size*/ -#ifdef D4DEBUG - nclog(NCLOGNOTE,"filesize: %lu bytes",fetchdata.size); -#endif - if (sizep != NULL) - *sizep = fetchdata.size; - /* Get the last modified time */ - if(filetime != NULL) - cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime); - if(cstat != CURLE_OK) - {ret = NC_ECURL; goto fail;} - } - return THROW(ret); - -fail: - if(cstat != CURLE_OK) { - nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat)); - ret = curlerrtoncerr(cstat); - } - return THROW(ret); -} - int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcodep) { @@ -155,27 +101,6 @@ done: return THROW(ret); } -static size_t -WriteFileCallback(void* ptr, size_t size, size_t nmemb, void* data) -{ - size_t realsize = size * nmemb; - size_t count; - struct Fetchdata* fetchdata; - fetchdata = (struct Fetchdata*) data; - if(realsize == 0) - nclog(NCLOGWARN,"WriteFileCallback: zero sized chunk"); - count = fwrite(ptr, size, nmemb, fetchdata->stream); - if (count > 0) { - fetchdata->size += (count * size); - } else { - nclog(NCLOGWARN,"WriteFileCallback: zero sized write"); - } -#ifdef PROGRESS - nclog(NCLOGNOTE,"callback: %lu bytes",(d4size_t)realsize); -#endif - return count; -} - static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) { diff --git a/libdap4/d4meta.c b/libdap4/d4meta.c index 4293b3727..1cea62f61 100644 --- a/libdap4/d4meta.c +++ b/libdap4/d4meta.c @@ -89,75 +89,6 @@ done: return THROW(ret); } - -/* Create an empty NCD4meta object for - use in subsequent calls - (is the the right src file to hold this?) -*/ - -NCD4meta* -NCD4_newmeta(NCD4INFO* info) -{ - NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); - if(meta == NULL) return NULL; - meta->allnodes = nclistnew(); -#ifdef D4DEBUG - meta->debuglevel = 1; -#endif - meta->controller = info; - meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ - return meta; -} - -/* Attach raw data to metadata */ -void -NCD4_attachraw(NCD4meta* meta, size_t rawsize, void* rawdata) -{ - assert(meta != NULL); - NCD4_resetSerial(&meta->serial,rawsize,rawdata); -} - -void -NCD4_setdebuglevel(NCD4meta* meta, int debuglevel) -{ - meta->debuglevel = debuglevel; -} - -void -NCD4_reclaimMeta(NCD4meta* dataset) -{ - int i; - if(dataset == NULL) return; - NCD4_resetMeta(dataset); - - for(i=0;iallnodes);i++) { - NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); - reclaimNode(node); - } - nclistfree(dataset->allnodes); - nclistfree(dataset->groupbyid); - nclistfree(dataset->atomictypes); - free(dataset); -} - -void -NCD4_resetMeta(NCD4meta* dataset) -{ - if(dataset == NULL) return; - nullfree(dataset->error.parseerror); dataset->error.parseerror = NULL; - nullfree(dataset->error.message); dataset->error.message = NULL; - nullfree(dataset->error.context); dataset->error.context = NULL; - nullfree(dataset->error.otherinfo); dataset->error.otherinfo = NULL; - NCD4_resetSerial(&dataset->serial,0,NULL); -#if 0 - for(i=0;iblobs);i++) { - void* p = nclistget(dataset->blobs,i); - nullfree(p); - } - nclistfree(dataset->blobs); -#endif -} - void reclaimNode(NCD4node* node) { @@ -676,6 +607,39 @@ savevarbyid(NCD4node* group, NCD4node* var) nclistinsert(group->group.varbyid,var->meta.id,var); } +/* Collect FQN path from var node up to and including + the root group and create an name from it +*/ +char* +NCD4_getVarFQN(NCD4node* var, const char* tail) +{ + int i; + NCD4node* x = NULL; + NClist* path = NULL; + NCbytes* fqn = NULL; + char* result; + + path = nclistnew(); + for(x=var->container;ISGROUP(x->sort);x=x->container) { + nclistinsert(path,0,x); + } + fqn = ncbytesnew(); + for(i=0;iname); + if(escaped == NULL) return NULL; + if(i > 0) ncbytesappend(fqn,'/'); + ncbytescat(fqn,escaped); + free(escaped); + } + nclistfree(path); + if(tail != NULL) + ncbytescat(fqn,tail); + result = ncbytesextract(fqn); + ncbytesfree(fqn); + return result; +} + /* Collect FQN path from node up to (but not including) the first enclosing group and create an name from it */ @@ -1180,7 +1144,7 @@ markdapsize(NCD4meta* meta) } int -NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) +NCD4_findvar(NC* ncp, int gid, int varid, NCD4node** varp, NCD4node** grpp) { int ret = NC_NOERR; NCD4INFO* info = NULL; @@ -1192,11 +1156,11 @@ NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) info = getdap(ncp); if(info == NULL) return THROW(NC_EBADID); - meta = info->substrate.metadata; + meta = info->dmrmetadata; if(meta == NULL) return THROW(NC_EBADID); /* Locate var node via (grpid,varid) */ - grp_id = GROUPIDPART(ncid); + grp_id = GROUPIDPART(gid); group = nclistget(meta->groupbyid,grp_id); if(group == NULL) return THROW(NC_EBADID); @@ -1205,7 +1169,7 @@ NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) return THROW(NC_EBADID); if(varp) *varp = var; if(grpp) *grpp = group; - return ret; + return THROW(ret); } static int diff --git a/libdap4/d4parser.c b/libdap4/d4parser.c index 013e32092..2948f8067 100644 --- a/libdap4/d4parser.c +++ b/libdap4/d4parser.c @@ -154,7 +154,7 @@ static int defineBytestringType(NCD4parser*); /* API */ int -NCD4_parse(NCD4meta* metadata) +NCD4_parse(NCD4meta* metadata, NCD4response* resp, int dapparse) { int ret = NC_NOERR; NCD4parser* parser = NULL; @@ -168,8 +168,10 @@ NCD4_parse(NCD4meta* metadata) /* Create and fill in the parser state */ parser = (NCD4parser*)calloc(1,sizeof(NCD4parser)); if(parser == NULL) {ret=NC_ENOMEM; goto done;} + parser->controller = metadata->controller; parser->metadata = metadata; - doc = ncxml_parse(parser->metadata->serial.dmr,strlen(parser->metadata->serial.dmr)); + parser->response = resp; + doc = ncxml_parse(parser->response->serial.dmr,strlen(parser->response->serial.dmr)); if(doc == NULL) {ret=NC_ENOMEM; goto done;} dom = ncxml_root(doc); parser->types = nclistnew(); @@ -178,6 +180,7 @@ NCD4_parse(NCD4meta* metadata) #ifdef D4DEBUG parser->debuglevel = 1; #endif + parser->dapparse = dapparse; /*Walk the DOM tree to build the DAP4 node tree*/ ret = traverse(parser,dom); @@ -214,9 +217,9 @@ traverse(NCD4parser* parser, ncxml_t dom) ret=parseError(parser,dom); /* Report the error */ fprintf(stderr,"DAP4 Error: http-code=%d message=\"%s\" context=\"%s\"\n", - parser->metadata->error.httpcode, - parser->metadata->error.message, - parser->metadata->error.context); + parser->response->error.httpcode, + parser->response->error.message, + parser->response->error.context); fflush(stderr); ret=NC_EDMR; goto done; @@ -225,7 +228,8 @@ traverse(NCD4parser* parser, ncxml_t dom) if((ret=makeNode(parser,NULL,NULL,NCD4_GROUP,NC_NULL,&parser->metadata->root))) goto done; parser->metadata->root->group.isdataset = 1; parser->metadata->root->meta.id = parser->metadata->ncid; - parser->metadata->groupbyid = nclistnew(); + if(parser->metadata->groupbyid == NULL) + parser->metadata->groupbyid = nclistnew(); SETNAME(parser->metadata->root,"/"); xattr = ncxml_attr(dom,"name"); if(xattr != NULL) parser->metadata->root->group.datasetname = xattr; @@ -847,23 +851,23 @@ parseError(NCD4parser* parser, ncxml_t errxml) char* shttpcode = ncxml_attr(errxml,"httpcode"); ncxml_t x; if(shttpcode == NULL) shttpcode = strdup("400"); - if(sscanf(shttpcode,"%d",&parser->metadata->error.httpcode) != 1) + if(sscanf(shttpcode,"%d",&parser->response->error.httpcode) != 1) nclog(NCLOGERR,"Malformed response"); nullfree(shttpcode); x=ncxml_child(errxml, "Message"); if(x != NULL) { char* txt = ncxml_text(x); - parser->metadata->error.message = (txt == NULL ? NULL : txt); + parser->response->error.message = (txt == NULL ? NULL : txt); } x = ncxml_child(errxml, "Context"); if(x != NULL) { const char* txt = ncxml_text(x); - parser->metadata->error.context = (txt == NULL ? NULL : strdup(txt)); + parser->response->error.context = (txt == NULL ? NULL : strdup(txt)); } x=ncxml_child(errxml, "OtherInformation"); if(x != NULL) { const char* txt = ncxml_text(x); - parser->metadata->error.otherinfo = (txt == NULL ? NULL : strdup(txt)); + parser->response->error.otherinfo = (txt == NULL ? NULL : strdup(txt)); } return THROW(NC_NOERR); } @@ -1321,7 +1325,7 @@ makeNode(NCD4parser* parser, NCD4node* parent, ncxml_t xml, NCD4sort sort, nc_ty record(parser,node); if(nodep) *nodep = node; done: - return ret; + return THROW(ret); } static int @@ -1652,12 +1656,19 @@ parseForwards(NCD4parser* parser, NCD4node* root) const char* mapname = (const char*)nclistget(var->mapnames,j); /* Find the corresponding variable */ NCD4node* mapref = lookupFQN(parser,mapname,NCD4_VAR); - if(mapref == NULL) + if(mapref != NULL) + PUSH(var->maps,mapref); + else if(!parser->dapparse) FAIL(NC_ENOTVAR," name does not refer to a variable: %s",mapname); - PUSH(var->maps,mapref); } } done: return THROW(ret); } + +void +NCD4_setdebuglevel(NCD4parser* parser, int debuglevel) +{ + parser->debuglevel = debuglevel; +} diff --git a/libdap4/d4read.c b/libdap4/d4read.c index 3a0e95a1d..3a24b08ff 100644 --- a/libdap4/d4read.c +++ b/libdap4/d4read.c @@ -19,9 +19,8 @@ See \ref copyright file for more info. /* Do conversion if this code was compiled via Vis. Studio or Mingw */ /*Forward*/ -static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, long*); +static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, int*, long*); static int readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet); -static int readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep); static int readfileDAPDMR(NCD4INFO* state, const NCURI* uri, NCbytes* packet); #ifdef HAVE_GETTIMEOFDAY @@ -38,82 +37,27 @@ deltatime(struct timeval time0,struct timeval time1) #endif int -NCD4_readDMR(NCD4INFO* state, int flags) +NCD4_readDMR(NCD4INFO* state, int flags, NCURI* url, NCD4response* resp) { int stat = NC_NOERR; - long lastmod = -1; - - if((flags & NCF_ONDISK) == 0) { - ncbytesclear(state->curl->packet); - stat = readpacket(state,state->uri,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&lastmod); - if(stat == NC_NOERR) - state->data.dmrlastmodified = lastmod; - } else { /*((flags & NCF_ONDISK) != 0) */ - NCURI* url = state->uri; - int fileprotocol = (strcmp(url->protocol,"file")==0); - if(fileprotocol) { - stat = readfiletofile(state, url, NCD4_DMR, NCD4_FORMAT_XML, state->data.ondiskfile, &state->data.datasize); - } else { - char* readurl = NULL; - int flags = 0; - if(!fileprotocol) flags |= NCURIQUERY; - flags |= NCURIENCODE; - flags |= NCURIPWD; -#ifdef FIX - ncurisetconstraints(url,state->constraint); -#endif - readurl = ncuribuild(url,NULL,".dmr.xml",NCURISVC); - if(readurl == NULL) - return THROW(NC_ENOMEM); - stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile, - &state->data.datasize, &lastmod); - nullfree(readurl); - if(stat == NC_NOERR) - state->data.dmrlastmodified = lastmod; - } - } + ncbytesclear(state->curl->packet); + stat = readpacket(state,url,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&resp->serial.httpcode,NULL); return THROW(stat); } int -NCD4_readDAP(NCD4INFO* state, int flags) +NCD4_readDAP(NCD4INFO* state, int flags, NCURI* url, NCD4response* resp) { int stat = NC_NOERR; - long lastmod = -1; - - if((flags & NCF_ONDISK) == 0) { - ncbytesclear(state->curl->packet); - stat = readpacket(state,state->uri,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&lastmod); - if(stat) { - NCD4_seterrormessage(state->substrate.metadata, nclistlength(state->curl->packet), nclistcontents(state->curl->packet)); - goto done; - } else - state->data.daplastmodified = lastmod; - } else { /*((flags & NCF_ONDISK) != 0) */ - NCURI* url = state->uri; - int fileprotocol = (strcmp(url->protocol,"file")==0); - if(fileprotocol) { - stat = readfiletofile(state, url, NCD4_DAP, NCD4_FORMAT_NONE, state->data.ondiskfile, &state->data.datasize); - } else { - char* readurl = NULL; - int flags = 0; - if(!fileprotocol) flags |= NCURIQUERY; - flags |= NCURIENCODE; - flags |= NCURIPWD; -#ifdef FIX - ncurisetconstraints(url,state->constraint); -#endif - readurl = ncuribuild(url,NULL,".dap",NCURISVC); - if(readurl == NULL) - return THROW(NC_ENOMEM); - stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile, - &state->data.datasize, &lastmod); - nullfree(readurl); - if(stat == NC_NOERR) - state->data.daplastmodified = lastmod; - } + + ncbytesclear(state->curl->packet); + stat = readpacket(state,url,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&resp->serial.httpcode,NULL); + if(stat) { + NCD4_seterrormessage(resp, nclistlength(state->curl->packet), nclistcontents(state->curl->packet)); + } else { + resp->raw.size = ncbyteslength(state->curl->packet); + resp->raw.memory = ncbytesextract(state->curl->packet); } -done: return THROW(stat); } @@ -150,7 +94,7 @@ dxxformat(int fxx, int dxx) } static int -readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, long* lastmodified) +readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, int* httpcodep, long* lastmodified) { int stat = NC_NOERR; int fileprotocol = 0; @@ -185,7 +129,7 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4forma gettimeofday(&time0,NULL); #endif } - stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified,&state->substrate.metadata->error.httpcode); + stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified,httpcodep); nullfree(fetchurl); if(stat) goto fail; if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { @@ -207,37 +151,26 @@ fail: return THROW(stat); } +#if 0 static int -readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep) +readfromfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, d4size_t* sizep) { int stat = NC_NOERR; - NCbytes* packet = ncbytesnew(); size_t len; - stat = readfile(state, uri, dxx, fxx, packet); + ncbytesclear(state->curl->packet); + stat = readfile(state, uri, dxx, fxx, state->curl->packet); #ifdef D4DEBUG fprintf(stderr,"readfiletofile: packet.size=%lu\n", - (unsigned long)ncbyteslength(packet)); + (unsigned long)ncbyteslength(state->curl->packet)); #endif if(stat != NC_NOERR) goto unwind; - len = nclistlength(packet); - if(stat == NC_NOERR) { - size_t written; - fseek(stream,0,SEEK_SET); - written = fwrite(ncbytescontents(packet),1,len,stream); - if(written != len) { -#ifdef D4DEBUG -fprintf(stderr,"readfiletofile: written!=length: %lu :: %lu\n", - (unsigned long)written,(unsigned long)len); -#endif - stat = NC_EIO; - } - } + len = nclistlength(state->curl->packet); if(sizep != NULL) *sizep = len; unwind: - ncbytesfree(packet); return THROW(stat); } +#endif static int readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet) @@ -365,12 +298,12 @@ done: /* Extract packet as error message; assume httpcode set */ int -NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg) +NCD4_seterrormessage(NCD4response* resp, size_t len, char* msg) { - metadata->error.message = (char*)d4alloc(len+1); - if(metadata->error.message == NULL) + resp->error.message = (char*)d4alloc(len+1); + if(resp->error.message == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->error.message,msg,len); - metadata->error.message[len] = '\0'; + memcpy(resp->error.message,msg,len); + resp->error.message[len] = '\0'; return THROW(NC_ENODATA); /* slight lie */ } diff --git a/libdap4/d4read.h b/libdap4/d4read.h index 826a2217c..39ad8d97c 100644 --- a/libdap4/d4read.h +++ b/libdap4/d4read.h @@ -7,6 +7,6 @@ #define D4READ_H extern int NCD4_readDMR(NCD4INFO*, int flags); -extern int NCD4_readDAP(NCD4INFO*, int flags); +extern int NCD4_readDAP(NCD4INFO*, int flags, NCURI* uri); #endif /*READ_H*/ diff --git a/libdap4/d4swap.c b/libdap4/d4swap.c index ed4c8dc1d..8568fd328 100644 --- a/libdap4/d4swap.c +++ b/libdap4/d4swap.c @@ -14,12 +14,12 @@ the incoming data to get the endianness correct. /* Forward */ -static int walkAtomicVar(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkOpaqueVar(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); -static int walkStructArray(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); -static int walkStruct(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkSeqArray(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkSeq(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); +static int walkAtomicVar(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkOpaqueVar(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkStructArray(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkStruct(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkSeqArray(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkSeq(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); /**************************************************/ @@ -28,43 +28,39 @@ Assumes that compiler->swap is true; does necessary byte swapping. */ int -NCD4_swapdata(NCD4meta* compiler, NClist* topvars) +NCD4_swapdata(NCD4response* resp, NCD4node* var, int doswap) { int ret = NC_NOERR; - int i; NCD4offset* offset = NULL; - offset = BUILDOFFSET(compiler->serial.dap,compiler->serial.dapsize); - for(i=0;iserial.dap,resp->serial.dapsize); OFFSET2BLOB(var->data.dap4data,offset); switch (var->subsort) { default: - if((ret=walkAtomicVar(compiler,var,var,offset))) goto done; + if((ret=walkAtomicVar(resp,var,var,offset,doswap))) goto done; break; case NC_OPAQUE: /* The only thing we need to do is swap the counts */ - if((ret=walkOpaqueVar(compiler,var,var,offset))) goto done; + if((ret=walkOpaqueVar(resp,var,var,offset,doswap))) goto done; break; case NC_STRUCT: - if((ret=walkStructArray(compiler,var,var,offset))) goto done; + if((ret=walkStructArray(resp,var,var,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeqArray(compiler,var,var,offset))) goto done; + if((ret=walkSeqArray(resp,var,var,offset,doswap))) goto done; break; } var->data.dap4data.size = DELTA(offset,var->data.dap4data.memory); /* skip checksum, if there is one */ - if(compiler->controller->data.inferredchecksumming) + if(resp->inferredchecksumming) INCR(offset,CHECKSUMSIZE); - } done: if(offset) free(offset); return THROW(ret); } static int -walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkAtomicVar(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -87,7 +83,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } else { /*(typesize > 1)*/ for(i=0;ioffset; - if(compiler->swap) { + if(doswap) { switch (typesize) { case 2: swapinline16(sp); break; case 4: swapinline32(sp); break; @@ -102,7 +98,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o COUNTERTYPE count; for(i=0;iswap) + if(doswap) swapinline64(offset); count = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -114,7 +110,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } static int -walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkOpaqueVar(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -125,7 +121,7 @@ walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o for(i=0;iswap) + if(doswap) swapinline64(offset); count = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -135,7 +131,7 @@ walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } static int -walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkStructArray(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -144,7 +140,7 @@ walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset for(i=0;ibasetype; switch (fieldbase->subsort) { default: - if((ret=walkAtomicVar(compiler,topvar,field,offset))) goto done; + if((ret=walkAtomicVar(resp,topvar,field,offset,doswap))) goto done; break; case NC_OPAQUE: /* The only thing we need to do is swap the counts */ - if((ret=walkOpaqueVar(compiler,topvar,field,offset))) goto done; + if((ret=walkOpaqueVar(resp,topvar,field,offset,doswap))) goto done; break; case NC_STRUCT: - if((ret=walkStructArray(compiler,topvar,field,offset))) goto done; + if((ret=walkStructArray(resp,topvar,field,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeqArray(compiler,topvar,field,offset))) goto done; + if((ret=walkSeqArray(resp,topvar,field,offset,doswap))) goto done; break; } } @@ -182,7 +178,7 @@ done: } static int -walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkSeqArray(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -195,7 +191,7 @@ walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* of for(i=0;iswap) + if(doswap) swapinline64(offset); recordcount = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -225,17 +221,17 @@ walkSeq(NCD4meta* compiler, NCD4node* topvar, NCD4node* vlentype, NCD4offset* of for(i=0;isubsort) { default: /* atomic basetype */ - if((ret=walkAtomicVar(compiler,topvar,basetype,offset))) goto done; + if((ret=walkAtomicVar(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_OPAQUE: - if((ret=walkOpaqueVar(compiler,topvar,basetype,offset))) goto done; + if((ret=walkOpaqueVar(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_STRUCT: /* We can treat each record like a structure instance */ - if((ret=walkStruct(compiler,topvar,basetype,offset))) goto done; + if((ret=walkStruct(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeq(compiler,topvar,basetype,offset))) goto done; + if((ret=walkSeq(resp,topvar,basetype,offset,doswap))) goto done; break; } } diff --git a/libdap4/d4util.c b/libdap4/d4util.c index 415651930..d1ba7f582 100644 --- a/libdap4/d4util.c +++ b/libdap4/d4util.c @@ -107,44 +107,45 @@ NCD4_makeFQN(NCD4node* node) { char* fqn = NULL; char* escaped; - int i; - NCD4node* g = node; - NClist* path = nclistnew(); - size_t estimate; + NCbytes* buf = ncbytesnew(); + NClist* grps = nclistnew(); + NClist* parts = nclistnew(); + NCD4node* n; + size_t i; - for(estimate=0;g != NULL;g=g->container) { - estimate += strlen(g->name); - nclistinsert(path,0,g); + /* collect all the non-groups */ + for(n=node;n;n=n->container) { + if(ISGROUP(n->sort)) + nclistinsert(grps,0,n); /* keep the correct order of groups */ + else + nclistinsert(parts,0,n); } - estimate = (estimate*2) + 2*nclistlength(path); - estimate++; /*strlcat nul*/ - fqn = (char*)malloc(estimate+1); - if(fqn == NULL) goto done; - fqn[0] = '\0'; - /* Create the group-based fqn prefix */ - /* start at 1 to avoid dataset */ - for(i=1;isort != NCD4_GROUP) break; + + /* Build grp prefix of the fqn */ + for(i=1;iname); - if(escaped == NULL) {free(fqn); fqn = NULL; goto done;} - strlcat(fqn,"/",estimate); - strlcat(fqn,escaped,estimate); + escaped = backslashEscape(n->name); + if(escaped == NULL) goto done; + ncbytescat(buf,"/"); + ncbytescat(buf,escaped); free(escaped); } /* Add in the final name part (if not group) */ - if(i < nclistlength(path)) { - int last = nclistlength(path)-1; - NCD4node* n = (NCD4node*)nclistget(path,last); - char* name = NCD4_makeName(n,"."); - strlcat(fqn,"/",estimate); - strlcat(fqn,name,estimate); - nullfree(name); + for(i=0;iname); + if(escaped == NULL) goto done; + ncbytescat(buf,(i==0?"/":".")); + ncbytescat(buf,escaped); + free(escaped); } + fqn = ncbytesextract(buf); done: - nclistfree(path); + ncbytesfree(buf); + nclistfree(grps); + nclistfree(parts); return fqn; } @@ -446,11 +447,9 @@ NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian) } void -NCD4_reporterror(NCD4INFO* state) +NCD4_reporterror(NCD4response* resp, NCURI* uri) { - NCD4meta* meta = state->substrate.metadata; char* u = NULL; - if(meta == NULL) return; - u = ncuribuild(state->uri,NULL,NULL,NCURIALL); - fprintf(stderr,"***FAIL: url=%s httpcode=%d errmsg->\n%s\n",u,meta->error.httpcode,meta->error.message); + u = ncuribuild(uri,NULL,NULL,NCURIALL); + fprintf(stderr,"***FAIL: url=%s httpcode=%d errmsg->\n%s\n",u,resp->serial.httpcode,resp->error.message); } diff --git a/libdap4/d4util.h b/libdap4/d4util.h index b6cbde8f0..38cda6a17 100644 --- a/libdap4/d4util.h +++ b/libdap4/d4util.h @@ -17,9 +17,12 @@ */ typedef unsigned long long d4size_t; -/* Define a counted memory marker */ +/* Define a (size, memory) pair */ typedef struct D4blob {d4size_t size; void* memory;} D4blob; +/* Empty blob constant */ +#define NULLBLOB(blob) {blob.size = 0; blob.memory = NULL;} + #define OFFSET2BLOB(blob,offset) do{(blob).size = ((offset)->limit - (offset)->base); (blob).memory = (offset)->base; }while(0) #define BLOB2OFFSET(offset,blob) do{\ (offset)->base = (blob).memory; \ diff --git a/libdap4/d4varx.c b/libdap4/d4varx.c index 9a708a5d5..b098855f8 100644 --- a/libdap4/d4varx.c +++ b/libdap4/d4varx.c @@ -11,7 +11,8 @@ #include "d4odom.h" /* Forward */ -static int getvarx(int ncid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*); +static int getvarx(int gid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*); +static int mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming); int NCD4_get_vara(int ncid, int varid, @@ -22,11 +23,11 @@ NCD4_get_vara(int ncid, int varid, int ret; /* TODO: optimize since we know stride is 1 */ ret = NCD4_get_vars(ncid,varid,start,edges,NC_stride_one,value,memtype); - return ret; + return THROW(ret); } int -NCD4_get_vars(int ncid, int varid, +NCD4_get_vars(int gid, int varid, const size_t *start, const size_t *edges, const ptrdiff_t* stride, void *memoryin, nc_type xtype) { @@ -43,14 +44,14 @@ NCD4_get_vars(int ncid, int varid, int rank; size_t dimsizes[NC_MAX_VAR_DIMS]; d4size_t dimproduct; - size_t dstcount; + size_t dstpos; NCD4offset* offset = NULL; - /* Get netcdf type info */ - if((ret=getvarx(ncid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size))) + /* Get netcdf var metadata and data */ + if((ret=getvarx(gid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size))) {goto done;} - meta = info->substrate.metadata; + meta = info->dmrmetadata; nctype = ncvar->basetype; rank = nclistlength(ncvar->dims); blobs = nclistnew(); @@ -74,18 +75,18 @@ NCD4_get_vars(int ncid, int varid, odom = d4odom_new(rank,start,edges,stride,dimsizes); else odom = d4scalarodom_new(); - dstcount = 0; /* We always write into dst starting at position 0*/ - for(;d4odom_more(odom);dstcount++) { + dstpos = 0; /* We always write into dst starting at position 0*/ + for(;d4odom_more(odom);dstpos++) { void* xpos; void* dst; - d4size_t count; + d4size_t pos; - count = d4odom_next(odom); - if(count >= dimproduct) { + pos = d4odom_next(odom); + if(pos >= dimproduct) { ret = THROW(NC_EINVALCOORDS); goto done; } - xpos = ((char*)memoryin)+(xsize * dstcount); /* ultimate destination */ + xpos = ((char*)memoryin)+(xsize * dstpos); /* ultimate destination */ /* We need to compute the offset in the dap4 data of this instance; for fixed size types, this is easy, otherwise we have to walk the variable size type @@ -95,16 +96,16 @@ NCD4_get_vars(int ncid, int varid, offset = NULL; offset = BUILDOFFSET(NULL,0); BLOB2OFFSET(offset,ncvar->data.dap4data); - /* Move offset to the count'th element of the array */ + /* Move offset to the pos'th element of the array */ if(nctype->meta.isfixedsize) { - INCR(offset,(dapsize*count)); + INCR(offset,(dapsize*pos)); } else { - /* We have to walk to the count'th location in the data */ - if((ret=NCD4_moveto(meta,ncvar,count,offset))) + /* We have to walk to the pos'th location in the data */ + if((ret=NCD4_moveto(meta,ncvar,pos,offset))) {goto done;} } dst = instance; - if((ret=NCD4_fillinstance(meta,nctype,offset,&dst,blobs))) + if((ret=NCD4_movetoinstance(meta,nctype,offset,&dst,blobs))) {goto done;} if(xtype == nc4type) { /* We can just copy out the data */ @@ -132,45 +133,29 @@ done: } static int -getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, +getvarx(int gid, int varid, NCD4INFO** infop, NCD4node** varp, nc_type* xtypep, size_t* xsizep, nc_type* nc4typep, size_t* nc4sizep) { int ret = NC_NOERR; - NC* ncp; - NCD4INFO* info; - NCD4meta* meta; - NCD4node* group; - NCD4node* var; - NCD4node* type; + NC* ncp = NULL; + NCD4INFO* info = NULL; + NCD4meta* dmrmeta = NULL; + NCD4node* group = NULL; + NCD4node* var = NULL; + NCD4node* type = NULL; nc_type xtype, actualtype; size_t instancesize, xsize; + NCURI* ceuri = NULL; /* Constrained uri */ + NCD4meta* dapmeta = NULL; + NCD4response* dapresp = NULL; - if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) + if((ret = NC_check_id(gid, (NC**)&ncp)) != NC_NOERR) goto done; info = getdap(ncp); - meta = info->substrate.metadata; - - /* If the data has not already been read and processed, then do so. */ - if(meta->serial.dap == NULL) { - size_t len = 0; - void* content = NULL; - /* (Re)Build the meta data; sets serial.rawdata */ - NCD4_resetMeta(info->substrate.metadata); - meta->controller = info; - meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ - - if((ret=NCD4_readDAP(info, info->controls.flags.flags))) goto done; - len = ncbyteslength(info->curl->packet); - content = ncbytesextract(info->curl->packet); - NCD4_resetSerial(&meta->serial, len, content); - /* Process the data part */ - if((ret=NCD4_dechunk(meta))) goto done; - if((ret = NCD4_processdata(info->substrate.metadata))) goto done; - } - - if((ret = NCD4_findvar(ncp,ncid,varid,&var,&group))) goto done; + dmrmeta = info->dmrmetadata; + if((ret = NCD4_findvar(ncp,gid,varid,&var,&group))) goto done; type = var->basetype; actualtype = type->meta.id; instancesize = type->meta.memsize; @@ -189,6 +174,46 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, else xsize = instancesize; + /* If we already have valid data, then just return */ + if(var->data.valid) goto validated; + + /* Ok, we need to read from the server */ + + /* Add the variable to the URI, unless the URI is already constrained or is unconstrainable */ + ceuri = ncuriclone(info->dmruri); + /* append the request for a specific variable */ + if(ncuriquerylookup(ceuri,DAP4CE) == NULL && !FLAGSET(info->controls.flags,NCF_UNCONSTRAINABLE)) { + ncurisetquerykey(ceuri,strdup("dap4.ce"),NCD4_makeFQN(var)); + } + + /* Read and process the data */ + + /* Setup the meta-data for the DAP */ + if((ret=NCD4_newMeta(info,&dapmeta))) goto done; + if((ret=NCD4_newResponse(info,&dapresp))) goto done; + dapresp->mode = NCD4_DAP; + nclistpush(info->responses,dapresp); + if((ret=NCD4_readDAP(info, info->controls.flags.flags, ceuri, dapresp))) goto done; + + /* Extract DMR and dechunk the data part */ + if((ret=NCD4_dechunk(dapresp))) goto done; + + /* Process the dmr part */ + if((ret=NCD4_parse(dapmeta,dapresp,1))) goto done; + + /* See if we are checksumming */ + if((ret=NCD4_inferChecksums(dapmeta,dapresp))) goto done; + + /* connect variables and corresponding dap data */ + if((ret = NCD4_parcelvars(dapmeta,dapresp))) goto done; + + /* Process checksums and byte-order swapping */ + if((ret = NCD4_processdata(dapmeta,dapresp))) goto done; + + /* Transfer and process the data */ + if((ret = mapvars(dapmeta,dmrmeta,dapresp->inferredchecksumming))) goto done; + +validated: /* Return relevant info */ if(infop) *infop = info; if(xtypep) *xtypep = xtype; @@ -197,8 +222,96 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, if(nc4sizep) *nc4sizep = instancesize; if(varp) *varp = var; done: - if(meta->error.message != NULL) - NCD4_reporterror(info); /* Make sure the user sees this */ + if(dapmeta) NCD4_reclaimMeta(dapmeta); + ncurifree(ceuri); + if(dapresp != NULL && dapresp->error.message != NULL) + NCD4_reporterror(dapresp,ceuri); /* Make sure the user sees this */ return THROW(ret); } +#if 0 +static NCD4node* +findbyname(const char* name, NClist* nodes) +{ + int i; + for(i=0;iname)==0) + return node; + } + return NULL; +} +#endif + +static int +matchvar(NCD4meta* dmrmeta, NCD4node* dapvar, NCD4node** dmrvarp) +{ + int i,ret = NC_NOERR; + NCD4node* x = NULL; + NClist* dappath = nclistnew(); + NClist* dmrpath = nclistnew(); /* compute path for this dmr var */ + int found = 0; + NCD4node* match = NULL; + + /* Capture the dap path starting at root and ending at the dapvar (assumed to be topvar) */ + for(x=dapvar;x != NULL;x=x->container) nclistinsert(dappath,0,x); + /* Iterate over all variable nodes to find matching one */ + for(i=0;iallnodes);i++) { + NCD4node* node = (NCD4node*)nclistget(dmrmeta->allnodes,i); + if(ISVAR(node->sort) && strcmp(node->name,dapvar->name)==0) { /* possible candidate */ + int j; + found = 0; + nclistclear(dmrpath); + for(x=node;x != NULL;x=x->container) nclistinsert(dmrpath,0,x); + if(nclistlength(dmrpath) == nclistlength(dappath)) { /* same length paths */ + /* compare paths: name and sort */ + for(found=1,j=0;jsort != pdap->sort || strcmp(pdmr->name,pdap->name) != 0) + {found = 0; break;} + } + if(found) {match = node; break;} + } + } + } + if(!found) {ret = NC_EINVAL; goto done;} + if(dmrvarp) *dmrvarp = match; + +done: + nclistfree(dappath); + nclistfree(dmrpath); + return THROW(ret); +} + +/* +Map each toplevel dap var to the corresponding +toplevel dmr var and transfer necessary info; +*/ + +static int +mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming) +{ + int i, ret = NC_NOERR; + NCD4node* daproot = dapmeta->root; + NClist* daptop = NULL; /* top variables in dap tree */ + + /* Get top level variables from the dap node tree */ + daptop = nclistnew(); + NCD4_getToplevelVars(dapmeta,daproot,daptop); + + /* Match up the dap top variables with the dmr top variables */ + for(i=0;idata = dapvar->data; + memset(&dapvar->data,0,sizeof(NCD4vardata)); + dmrvar->data.valid = 1; + } + +done: + nclistfree(daptop); + return THROW(ret); +} diff --git a/libdap4/ncd4.h b/libdap4/ncd4.h index 058e40ff7..286406d77 100644 --- a/libdap4/ncd4.h +++ b/libdap4/ncd4.h @@ -77,7 +77,6 @@ EXTERNL int dsp_open(const char* path, ND4dsp** dspp); /* From d4http.c */ EXTERNL long NCD4_fetchhttpcode(CURL* curl); -EXTERNL int NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, d4size_t* sizep, long* filetime); EXTERNL int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcode); EXTERNL int NCD4_curlopen(CURL** curlp); EXTERNL void NCD4_curlclose(CURL* curl); @@ -85,15 +84,16 @@ EXTERNL int NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime); EXTERNL int NCD4_ping(const char* url); /* From d4read.c */ -EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags); -EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags); -EXTERNL int NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg); +EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags, NCURI* url, NCD4response*); +EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags, NCURI* ceuri, NCD4response*); +EXTERNL int NCD4_seterrormessage(NCD4response*, size_t len, char* msg); /* From d4parser.c */ -EXTERNL int NCD4_parse(NCD4meta*); +EXTERNL int NCD4_parse(NCD4meta*, NCD4response*, int dapparse); EXTERNL NCD4node* NCD4_findAttr(NCD4node* container, const char* attrname); EXTERNL NCD4node* NCD4_groupFor(NCD4node* node); EXTERNL int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char* typename, NCD4node** attrp); +EXTERNL void NCD4_setdebuglevel(NCD4parser*,int); /* From d4printer.c */ EXTERNL int NCD4_print(NCD4meta*, NCbytes* output); @@ -104,29 +104,33 @@ EXTERNL void NCD4_attachraw(NCD4meta*, size_t size, void* rawdata); EXTERNL void NCD4_reclaimMeta(NCD4meta*); EXTERNL void NCD4_resetMeta(NCD4meta*); EXTERNL void reclaimNode(NCD4node* node); -EXTERNL void NCD4_setdebuglevel(NCD4meta*,int); EXTERNL int NCD4_metabuild(NCD4meta*, int ncid); EXTERNL size_t NCD4_computeTypeSize(NCD4meta*, NCD4node* type); EXTERNL int NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp); +EXTERNL char* NCD4_getVarFQN(NCD4node* var, const char* tail); + /* From d4chunk.c */ -EXTERNL int NCD4_dechunk(NCD4meta*); -EXTERNL int NCD4_infermode(NCD4meta* meta); +EXTERNL int NCD4_dechunk(NCD4response*); +EXTERNL int NCD4_infermode(NCD4response*); struct NCD4serial; EXTERNL void NCD4_resetSerial(struct NCD4serial* serial, size_t rawsize, void* rawdata); +EXTERNL void NCD4_moveSerial(struct NCD4serial* serial, struct NCD4serial* dst); /* From d4swap.c */ -EXTERNL int NCD4_swapdata(NCD4meta*, NClist* topvars); +EXTERNL int NCD4_swapdata(NCD4response*, NCD4node* topvar, int doswap); /* From d4fix.c */ -EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, NCD4offset* offset); +EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, NCD4offset* offset, int inferredchecksumming); EXTERNL int NCD4_moveto(NCD4meta*, NCD4node* var, d4size_t count, NCD4offset* offset); EXTERNL int NCD4_toposort(NCD4meta*); /* From d4data.c */ -EXTERNL int NCD4_processdata(NCD4meta*); -EXTERNL int NCD4_fillinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); +EXTERNL int NCD4_parcelvars(NCD4meta* meta, NCD4response* resp); +EXTERNL int NCD4_processdata(NCD4meta*,NCD4response*); +EXTERNL int NCD4_movetoinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); EXTERNL int NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel); +EXTERNL int NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp); /* From d4util.c */ EXTERNL d4size_t NCD4_dimproduct(NCD4node* node); @@ -141,7 +145,7 @@ EXTERNL char* NCD4_deescape(const char* esc); EXTERNL char* NCD4_entityescape(const char* s); EXTERNL size_t NCD4_elidenuls(char* s, size_t slen); EXTERNL void* NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian); -EXTERNL void NCD4_reporterror(NCD4INFO* state); +EXTERNL void NCD4_reporterror(NCD4response*, NCURI* uri); /* From d4dump.c */ EXTERNL void NCD4_dumpbytes(size_t size, const void* data0, int swap); @@ -163,7 +167,15 @@ EXTERNL int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char* /* d4file.c */ EXTERNL int NCD4_readDMRorDAP(NCD4INFO* d4info, NCD4mode mode); EXTERNL void NCD4_applyclientfragmentcontrols(NCD4INFO* d4info); -EXTERNL void NCD4_applyclientquerycontrols(NCD4INFO* d4info); +EXTERNL void NCD4_applychecksumcontrols(NCD4INFO* d4info, NCD4response*); +EXTERNL int NCD4_newInfo(NCD4INFO** d4infop); +EXTERNL void NCD4_reclaimInfo(NCD4INFO* d4info); +EXTERNL void NCD4_resetInfoforRead(NCD4INFO* d4info); +EXTERNL int NCD4_newResponse(NCD4INFO*,NCD4response** respp); +EXTERNL void NCD4_reclaimResponse(NCD4response* d4resp); +EXTERNL void NCD4_resetInfoForRead(NCD4INFO* d4info); +EXTERNL int NCD4_newMeta(NCD4INFO*,NCD4meta**); +EXTERNL void NCD4_reclaimMeta(NCD4meta*); /* ncd4dispatch.c */ struct NC_reservedatt; /*forward*/ diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c index 9d9cf08a1..e7eefb40a 100644 --- a/libdap4/ncd4dispatch.c +++ b/libdap4/ncd4dispatch.c @@ -388,11 +388,11 @@ NCD4_inq_attname(int ncid, int varid, int attnum, char* name) const NC_reservedatt* rsvp = NULL; if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) return (ret); + substrateid = makenc4id(ncp,ncid); + ret = nc_inq_attname(substrateid, varid, attnum, name); /* Is this a reserved attribute name? */ if(name && (rsvp = NCD4_lookupreserved(name))) return NC_EATTMETA; - substrateid = makenc4id(ncp,ncid); - ret = nc_inq_attname(substrateid, varid, attnum, name); return (ret); } @@ -847,7 +847,7 @@ NCD4_inq_dim(int ncid, int dimid, char* name, size_t* lenp) if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) goto done; info = (NCD4INFO*)ncp->dispatchdata; - meta = info->substrate.metadata; + meta = info->dmrmetadata; /* Locate the dimension specified by dimid */ for(i=0;iallnodes);i++) { @@ -871,16 +871,15 @@ static int ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* value, nc_type t, const NC_reservedatt* rsvp) { int ret = NC_NOERR; - NCD4INFO* info = (NCD4INFO*)(ncp->dispatchdata); - NCD4meta* meta = info->substrate.metadata; NCD4node* var = NULL; + if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done; + if(strcmp(rsvp->name,D4CHECKSUMATTR)==0) { unsigned int* ip = (unsigned int*)value; if(varid == NC_GLOBAL) {ret = NC_EBADID; goto done;} if(t != NC_UINT) {ret = NC_EBADTYPE; goto done;} - if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done; if(var->data.checksumattr == 0) {ret = NC_ENOTATT; goto done;} *ip = (var->data.remotechecksum); @@ -889,7 +888,7 @@ ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* valu if(varid != NC_GLOBAL) {ret = NC_EBADID; goto done;} if(t != NC_INT) {ret = NC_EBADTYPE; goto done;} - *ip = (meta->serial.remotelittleendian?1:0); + *ip = (var->data.response->remotelittleendian?1:0); } done: return THROW(ret); @@ -925,7 +924,7 @@ static int globalinit(void) { int stat = NC_NOERR; - return stat; + return THROW(stat); } /**************************************************/ diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h index 1f84a0f56..7bed38a84 100644 --- a/libdap4/ncd4types.h +++ b/libdap4/ncd4types.h @@ -50,6 +50,8 @@ typedef struct NCD4node NCD4node; typedef struct NCD4params NCD4params; typedef struct NCD4HDR NCD4HDR; typedef struct NCD4offset NCD4offset; +typedef struct NCD4vardata NCD4vardata; +typedef struct NCD4response NCD4response; /* Define the NCD4HDR flags */ /* Header flags */ @@ -228,19 +230,20 @@ struct NCD4node { int isfixedsize; /* sort == NCD4_TYPE; Is this a fixed size (recursively) type? */ d4size_t dapsize; /* size of the type as stored in the dap data; will, as a rule, be same as memsize only for types <= NC_UINT64 */ - nc_type cmpdid; /*netcdf id for the compound type created for seq type */ + nc_type cmpdid; /* netcdf id for the compound type created for seq type */ size_t memsize; /* size of a memory instance without taking dimproduct into account, but taking compound alignment into account */ d4size_t offset; /* computed structure field offset in memory */ size_t alignment; /* computed structure field alignment in memory */ } meta; - struct { /* Data compilation info */ - int flags; /* See d4data for actual flags */ + struct NCD4vardata { /* Data compilation info */ + int valid; /* 1 => this contains valid data */ D4blob dap4data; /* offset and start pos for this var's data in serialization */ - unsigned int remotechecksum; /* checksum from data as sent by server */ - unsigned int localchecksum; /* toplevel variable checksum as computed by client */ - int checksumattr; /* 1=> _DAP4_Checksum_CRC32 is defined */ - int attrchecksum; /* _DAP4_Checksum_CRC32 value */ + unsigned remotechecksum; /* toplevel per-variable checksum contained in the data */ + unsigned localchecksum; /* toplevel variable checksum as computed by client */ + int checksumattr; /* 1 => _DAP4_Checksum_CRC32 is defined */ + unsigned attrchecksum; /* _DAP4_Checksum_CRC32 value; this is the checksum computed by server */ + NCD4response* response; /* Response from which this data is taken */ } data; struct { /* Track netcdf-4 conversion info */ int isvlen; /* _edu.ucar.isvlen */ @@ -253,36 +256,13 @@ struct NCD4node { } nc4; }; -/* Tracking info about the serialized input before and after de-chunking */ -typedef struct NCD4serial { - size_t rawsize; /* |rawdata| */ - void* rawdata; - size_t dapsize; /* |dap|; this is transient */ - void* dap; /* pointer into rawdata where dap data starts */ - char* dmr;/* copy of dmr */ - char* errdata; /* null || error chunk (null terminated) */ - int httpcode; /* returned from last request */ - int hostlittleendian; /* 1 if the host is little endian */ - int remotelittleendian; /* 1 if the packet says data is little endian */ -} NCD4serial; - -/* This will be passed out of the parse */ +/* DMR information from a response; this will be passed out of the parse */ struct NCD4meta { NCD4INFO* controller; int ncid; /* root ncid of the substrate netcdf-4 file; warning: copy of NCD4Info.substrate.nc4id */ NCD4node* root; - NCD4mode mode; /* Are we reading DMR (only) or DAP (includes DMR) */ NClist* allnodes; /*list*/ - struct Error { /* Content of any error response */ - char* parseerror; - int httpcode; - char* message; - char* context; - char* otherinfo; - } error; - int debuglevel; - NCD4serial serial; int swap; /* 1 => swap data */ /* Define some "global" (to a DMR) data */ NClist* groupbyid; /* NClist indexed by groupid >> 16; this is global */ @@ -292,9 +272,12 @@ struct NCD4meta { }; typedef struct NCD4parser { + NCD4INFO* controller; char* input; int debuglevel; + int dapparse; /* 1 => we are parsing the DAP DMR */ NCD4meta* metadata; + NCD4response* response; /* Capture useful subsets of dataset->allnodes */ NClist* types; /*list; user-defined types only*/ NClist* dims; /*list*/ @@ -303,6 +286,32 @@ typedef struct NCD4parser { NCD4node* dapopaque; /* Single non-fixed-size opaque type */ } NCD4parser; +/* Capture all the relevant info about the response to a server request */ +struct NCD4response { /* possibly processed response from a query */ + NCD4INFO* controller; /* controlling connection */ + D4blob raw; /* complete response in memory */ + int querychecksumming; /* 1 => user specified dap4.ce value */ + int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */ + int inferredchecksumming; /* 1 => either query checksum || att checksum */ + int checksumignore; /* 1 => assume checksum, but do not validate */ + int remotelittleendian; /* 1 if the packet says data is little endian */ + NCD4mode mode; /* Are we reading DMR (only) or DAP (includes DMR) */ + struct NCD4serial { + size_t dapsize; /* |dap|; this is transient */ + void* dap; /* pointer into raw where dap data starts */ + char* dmr;/* copy of dmr */ + char* errdata; /* null || error chunk (null terminated) */ + int httpcode; /* returned from last request */ + } serial; /* Dechunked and processed DAP part of the response */ + struct Error { /* Content of any error response */ + char* parseerror; + int httpcode; + char* message; + char* context; + char* otherinfo; + } error; +}; + /**************************************************/ /* Curl info */ @@ -328,28 +337,20 @@ struct NCD4curl { struct NCD4INFO { NC* controller; /* Parent instance of NCD4INFO */ - char* rawurltext; /* as given to ncd4_open */ - char* urltext; /* as modified by ncd4_open */ - NCURI* uri; /* parse of rawuritext */ + char* rawdmrurltext; /* as given to ncd4_open */ + char* dmrurltext; /* as modified by ncd4_open */ + NCURI* dmruri; /* parse of rawuritext */ NCD4curl* curl; int inmemory; /* store fetched data in memory? */ - struct { - char* memory; /* allocated memory if ONDISK is not set */ - char* ondiskfilename; /* If ONDISK is set */ - FILE* ondiskfile; /* ditto */ - d4size_t datasize; /* size on disk or in memory */ - long dmrlastmodified; - long daplastmodified; - int querychecksumming; /* 1 => user specified dap4.ce value */ - int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */ - int inferredchecksumming; /* 1 => either query checksum || att checksum */ - int checksumignore; /* 1 => assume checksum, but do not validate */ - } data; + NCD4meta* dmrmetadata; /* Independent of responses */ + NClist* responses; /* NClist all responses from this curl handle */ + struct { /* Properties that are per-platform */ + int hostlittleendian; /* 1 if the host is little endian */ + } platform; struct { int realfile; /* 1 => we created actual temp file */ char* filename; /* of the substrate file */ int nc4id; /* substrate nc4 file ncid used to hold metadata; not same as external id */ - NCD4meta* metadata; } substrate; struct { NCCONTROLS flags; @@ -362,6 +363,7 @@ struct NCD4INFO { struct { char* filename; } fileproto; + int debuglevel; NClist* blobs; }; diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 61941d1d7..6f5e4a846 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -205,7 +205,7 @@ static struct NCPROTOCOLLIST { static int NC_omodeinfer(int useparallel, int omode, NCmodel*); static int check_file_type(const char *path, int omode, int use_parallel, void *parameters, NCmodel* model, NCURI* uri); static int processuri(const char* path, NCURI** urip, NClist* fraglist); -static int processmacros(NClist** fraglistp); +static int processmacros(NClist* fraglistp, NClist* expanded); static char* envvlist2string(NClist* pairs, const char*); static void set_default_mode(int* cmodep); static int parseonchar(const char* s, int ch, NClist* segments); @@ -242,8 +242,8 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv) NCURI* uri = NULL; size_t pathlen = strlen(path); char* str = NULL; - const char** ufrags; - const char** p; + const NClist* ufrags; + size_t i; if(path == NULL || pathlen == 0) {stat = NC_EURL; goto done;} @@ -265,7 +265,6 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv) /* process the corresponding fragments for that protocol */ if(protolist->fragments != NULL) { - int i; tmp = nclistnew(); if((stat = parseonchar(protolist->fragments,'&',tmp))) goto done; for(i=0;isubstitute) ncurisetprotocol(uri,protolist->substitute); /* capture the fragments of the url */ - ufrags = ncurifragmentparams(uri); - if(ufrags != NULL) { - for(p=ufrags;*p;p+=2) { - const char* key = p[0]; - const char* value = p[1]; - nclistpush(fraglenv,nulldup(key)); - value = (value==NULL?"":value); - nclistpush(fraglenv,strdup(value)); - } + ufrags = (const NClist*)ncurifragmentparams(uri); + for(i=0;i 0) { + for(i=0;iname;macros++) { if(strcmp(macros->name,key)==0) { @@ -474,14 +464,8 @@ processmacros(NClist** fraglenvp) nclistpush(expanded,strdup(key)); nclistpush(expanded,strdup(value)); } - nullfree(key); - nullfree(value); } - *fraglenvp = expanded; expanded = NULL; -done: - nclistfreeall(expanded); - nclistfreeall(fraglenv); return check(stat); } @@ -497,8 +481,6 @@ processinferences(NClist* fraglenv) int i; char* newmodeval = NULL; - if(fraglenv == NULL || nclistlength(fraglenv) == 0) goto done; - /* Get "mode" entry */ if((modeval = getmodekey(fraglenv))==NULL) goto done; @@ -536,7 +518,7 @@ printlist(newmodes,"processinferences: new mode list"); nextmodes = tmp; tmp = NULL; } - /* cleanup any unused elements in currenmodes */ + /* cleanup any unused elements in currentmodes */ nclistclearall(currentmodes); /* Ensure no duplicates */ @@ -691,21 +673,15 @@ collectallkeys(NClist* fraglenv, NClist* allkeys) /* Given a fragment envv list, coalesce duplicate keys and remove duplicate values*/ static int -cleanfragments(NClist** fraglenvp) +cleanfragments(NClist* fraglenv, NClist* newlist) { int i,stat = NC_NOERR; - NClist* fraglenv = NULL; NClist* tmp = NULL; NClist* allkeys = NULL; - NClist* newlist = NULL; NCbytes* buf = NULL; char* key = NULL; char* value = NULL; - if(fraglenvp == NULL || nclistlength(*fraglenvp) == 0) return NC_NOERR; - fraglenv = *fraglenvp; /* take control of this list */ - *fraglenvp = NULL; - newlist = nclistnew(); buf = ncbytesnew(); allkeys = nclistnew(); tmp = nclistnew(); @@ -725,13 +701,10 @@ cleanfragments(NClist** fraglenvp) nclistpush(newlist,value); nclistclear(tmp); } - *fraglenvp = newlist; newlist = NULL; done: nclistfree(allkeys); nclistfree(tmp); ncbytesfree(buf); - nclistfreeall(fraglenv); - nclistfreeall(newlist); return check(stat); } @@ -871,7 +844,8 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void char* sfrag = NULL; const char* modeval = NULL; char* abspath = NULL; - + NClist* tmp = NULL; + /* Phase 1: 1. convert special protocols to http|https 2. begin collecting fragments @@ -884,13 +858,20 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void #endif /* Phase 2: Expand macros and add to fraglenv */ - if((stat = processmacros(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = processmacros(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; #ifdef DEBUG printlist(fraglenv,"processmacros"); #endif - /* Cleanup the fragment list */ - if((stat = cleanfragments(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = cleanfragments(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; /* Phase 2a: Expand mode inferences and add to fraglenv */ if((stat = processinferences(fraglenv))) goto done; @@ -899,7 +880,11 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void #endif /* Phase 3: coalesce duplicate fragment keys and remove duplicate values */ - if((stat = cleanfragments(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = cleanfragments(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; #ifdef DEBUG printlist(fraglenv,"cleanfragments"); #endif @@ -949,13 +934,12 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void /* Phase 6: Process the non-mode keys to see if we can tell the formatx */ if(!modelcomplete(model)) { - const char** p = ncurifragmentparams(uri); /* envv format */ - if(p != NULL) { - for(;*p;p+=2) { - const char* key = p[0]; - const char* value = p[1];; - if((stat=processfragmentkeys(key,value,model))) goto done; - } + size_t i; + NClist* p = (NClist*)ncurifragmentparams(uri); /* envv format */ + for(i=0;ileaflen, leafavg); fprintf(stderr," load=%g",leafload); fprintf(stderr,"]\n"); - dirsize = (1<<(map->depth))*((unsigned long long)sizeof(void*)); + dirsize = (1ULL<<(map->depth))*((unsigned long long)sizeof(void*)); leafsize = (nleaves)*((unsigned long long)sizeof(NCexleaf)); total = dirsize + leafsize; fprintf(stderr,"\tsizeof(directory)=%llu sizeof(leaves)=%lld total=%lld\n", diff --git a/libdispatch/nclistmgr.c b/libdispatch/nclistmgr.c index 48b4eeb58..4069e8769 100644 --- a/libdispatch/nclistmgr.c +++ b/libdispatch/nclistmgr.c @@ -81,8 +81,8 @@ free_NCList(void) int add_to_NCList(NC* ncp) { - int i; - int new_id; + unsigned int i; + unsigned int new_id; if(nc_filelist == NULL) { if (!(nc_filelist = calloc(1, sizeof(NC*)*NCFILELISTLENGTH))) return NC_ENOMEM; @@ -96,7 +96,7 @@ add_to_NCList(NC* ncp) if(new_id == 0) return NC_ENOMEM; /* no more slots */ nc_filelist[new_id] = ncp; numfiles++; - ncp->ext_ncid = (new_id << ID_SHIFT); + ncp->ext_ncid = (int)(new_id << ID_SHIFT); return NC_NOERR; } diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c index 5f6b8be7d..c9664a23c 100644 --- a/libdispatch/nclog.c +++ b/libdispatch/nclog.c @@ -21,10 +21,6 @@ #include #endif -#ifdef HAVE_EXECINFO_H -#include -#endif - #include "netcdf.h" #include "nclog.h" @@ -278,10 +274,6 @@ ncuntrace(const char* fcn, int err, const char* fmt, ...) vfprintf(nclog_global.nclogstream, fmt, args); fprintf(nclog_global.nclogstream, "\n" ); fflush(nclog_global.nclogstream); -#ifdef HAVE_EXECINFO_H - if(err != 0) - ncbacktrace(); -#endif } done: va_end(args); @@ -304,28 +296,4 @@ ncbreakpoint(int err) return err; } -#ifdef HAVE_EXECINFO_H -#define MAXSTACKDEPTH 100 -void -ncbacktrace(void) -{ - int j, nptrs; - void* buffer[MAXSTACKDEPTH]; - char **strings; - - if(getenv("NCBACKTRACE") == NULL) return; - nptrs = backtrace(buffer, MAXSTACKDEPTH); - strings = backtrace_symbols(buffer, nptrs); - if (strings == NULL) { - perror("backtrace_symbols"); - errno = 0; - return; - } - fprintf(stderr,"Backtrace:\n"); - for(j = 0; j < nptrs; j++) - fprintf(stderr,"%s\n", strings[j]); - free(strings); -} -#endif - /**@}*/ diff --git a/libdispatch/ncuri.c b/libdispatch/ncuri.c index efd6fa449..6f699a859 100644 --- a/libdispatch/ncuri.c +++ b/libdispatch/ncuri.c @@ -95,16 +95,13 @@ ncstrndup(const char* s, size_t len) /* Forward */ static int collectprefixparams(char* text, char** nextp); static void freestringlist(NClist* list); -static void freestringvec(char** list); -static int ncfind(char** params, const char* key); +static int ncfind(NClist* params, const char* key); static char* nclocate(char* p, const char* charlist); static int parselist(const char* ptext, NClist* list); -static int unparselist(const char** vec, const char* prefix, int encode, char** svecp); +static int unparselist(const NClist* vec, const char* prefix, int encode, NCbytes*); static int ensurefraglist(NCURI* uri); static int ensurequerylist(NCURI* uri); -static void buildlist(const char** list, int encode, NCbytes* buf); static void removedups(NClist* list); -static int extendenvv(char*** envvp, int amount, int* oldlen); /**************************************************/ /* @@ -147,9 +144,6 @@ ncuriparse(const char* uri0, NCURI** durip) size_t len0; int pathchar; - tmp.fraglist = NULL; - tmp.querylist = NULL; - if(uri0 == NULL) {THROW(NC_EURL);} @@ -227,19 +221,15 @@ ncuriparse(const char* uri0, NCURI** durip) } /* Remove duplicates */ removedups(params); - if(nclistlength(params) > 0) { - nclistpush(params,NULL); - tmp.fraglist = nclistextract(params); - } else - tmp.fraglist = NULL; + tmp.fraglist = params; + params = NULL; + /* Parse the query */ if(tmp.query != NULL) { if(parselist(tmp.query,querylist) != NC_NOERR) {THROW(NC_EURL);} - if(nclistlength(querylist) > 0) { - nclistpush(querylist,NULL); - tmp.querylist = nclistextract(querylist); - } + tmp.querylist = querylist; + querylist = NULL; } /* Now parse the core of the url */ @@ -312,7 +302,7 @@ ncuriparse(const char* uri0, NCURI** durip) pathchar = EOFCHAR; } else { /* assume there should be a host section */ /* We already extracted the query and/or fragment sections above, - splocate the end of the host section and therefore the start + so locate the end of the host section and therefore the start of the path. */ tmp.host = p; @@ -430,9 +420,9 @@ done: freestringlist(params); freestringlist(querylist); if(tmp.fraglist) - freestringvec(tmp.fraglist); + nclistfreeall(tmp.fraglist); if(tmp.querylist) - freestringvec(tmp.querylist); + nclistfreeall(tmp.querylist); return ret; } @@ -450,6 +440,7 @@ freestringlist(NClist* list) } } +#if 0 static void freestringvec(char** list) { @@ -459,6 +450,7 @@ freestringvec(char** list) nullfree(list); } } +#endif void ncurifree(NCURI* duri) @@ -473,8 +465,8 @@ ncurifree(NCURI* duri) nullfree(duri->path); nullfree(duri->query); nullfree(duri->fragment); - freestringvec(duri->querylist); - freestringvec(duri->fraglist); + nclistfreeall(duri->querylist); + nclistfreeall(duri->fraglist); free(duri); } @@ -510,21 +502,14 @@ int ncurisetquery(NCURI* duri,const char* query) { int ret = NC_NOERR; - freestringvec(duri->querylist); + nclistfreeall((NClist*)duri->querylist); nullfree(duri->query); duri->query = NULL; duri->querylist = NULL; if(query != NULL && strlen(query) > 0) { - NClist* params = nclistnew(); - duri->query = strdup(query); - ret = parselist(duri->query,params); - if(ret != NC_NOERR) - {THROW(NC_EURL);} - nclistpush(params,NULL); - duri->querylist = nclistextract(params); - nclistfree(params); + duri->query = strdup(query); + ensurequerylist(duri); } -done: return ret; } @@ -533,12 +518,13 @@ int ncurisetfragments(NCURI* duri,const char* fragments) { int ret = NC_NOERR; - freestringvec(duri->fraglist); + nclistfreeall((NClist*)duri->fraglist); nullfree(duri->fragment); duri->fragment = NULL; duri->fraglist = NULL; if(fragments != NULL && strlen(fragments) > 0) { duri->fragment = strdup(fragments); + ensurefraglist(duri); } return ret; } @@ -559,17 +545,20 @@ ncurisetfragmentkey(NCURI* duri,const char* key, const char* value) { int ret = NC_NOERR; int pos = -1; - char* newlist = NULL; ensurefraglist(duri); pos = ncfind(duri->fraglist, key); - if(pos < 0) return NC_EINVAL; /* does not exist */ - nullfree(duri->fraglist[pos+1]); - duri->fraglist[pos+1] = strdup(value); + if(pos < 0) { /* does not exist */ + if(duri->fraglist == NULL) duri->fraglist = nclistnew(); + nclistpush(duri->fraglist,key); + nclistpush(duri->fraglist,value); + } else { + nullfree(nclistget(duri->fraglist,pos+1)); + nclistset(duri->fraglist,pos+1,strdup(value)); + } /* Rebuild the fragment */ - if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done; - nullfree(duri->fragment); - duri->fragment = newlist; newlist = NULL; + nullfree(duri->fragment); duri->fragment = NULL; + if((ret = ensurefraglist(duri))) goto done; done: return ret; } @@ -579,25 +568,67 @@ int ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value) { int ret = NC_NOERR; - int len; int pos = -1; - char* newlist = NULL; ensurefraglist(duri); pos = ncfind(duri->fraglist, key); if(pos < 0) { /* does not exist */ - if((ret = extendenvv(&duri->fraglist,2,&len))) goto done; - duri->fraglist[len] = strdup(key); - duri->fraglist[len+1] = nulldup(value); - duri->fraglist[len+2] = NULL; + nclistpush((NClist*)duri->fraglist,strdup(key)); + nclistpush((NClist*)duri->fraglist,nulldup(value)); } else { - nullfree(duri->fraglist[pos+1]); - duri->fraglist[pos+1] = strdup(value); + nullfree(nclistget(duri->fraglist,pos+1)); + nclistset(duri->fraglist,pos+1,nulldup(value)); } /* Rebuild the fragment */ - if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done; - nullfree(duri->fragment); - duri->fragment = newlist; newlist = NULL; + nullfree(duri->fraglist); duri->fraglist = NULL; + if((ret=ensurefraglist(duri))) goto done; +done: + return ret; +} + +/* Replace a specific query key*/ +int +ncurisetquerykey(NCURI* duri,const char* key, const char* value) +{ + int ret = NC_NOERR; + int pos = -1; + + ensurequerylist(duri); + pos = ncfind(duri->querylist, key); + if(pos < 0) { /* does not exist */ + if(duri->querylist == NULL) duri->querylist = nclistnew(); + nclistpush(duri->querylist,key); + nclistpush(duri->querylist,value); + } else { + nullfree(nclistget(duri->querylist,pos+1)); + nclistset(duri->querylist,pos+1,strdup(value)); + } + /* Rebuild the query */ + nullfree(duri->query); duri->query = NULL; + if((ret = ensurequerylist(duri))) goto done; +done: + return ret; +} + +/* Replace or add a specific query key*/ +int +ncuriappendquerykey(NCURI* duri,const char* key, const char* value) +{ + int ret = NC_NOERR; + int pos = -1; + + ensurequerylist(duri); + pos = ncfind(duri->querylist, key); + if(pos < 0) { /* does not exist */ + nclistpush((NClist*)duri->querylist,strdup(key)); + nclistpush((NClist*)duri->querylist,nulldup(value)); + } else { + nullfree(nclistget(duri->querylist,pos+1)); + nclistset(duri->querylist,pos+1,nulldup(value)); + } + /* Rebuild the query */ + nullfree(duri->querylist); duri->querylist = NULL; + if((ret=ensurequerylist(duri))) goto done; done: return ret; } @@ -734,11 +765,10 @@ ncurifragmentlookup(NCURI* uri, const char* key) int i; char* value = NULL; if(uri == NULL || key == NULL) return NULL; - ensurefraglist(uri); + if(ensurefraglist(uri)) return NULL; i = ncfind(uri->fraglist,key); - if(i < 0) - return NULL; - value = uri->fraglist[(2*i)+1]; + if(i < 0) return NULL; + value = nclistget(uri->fraglist,i+1); return value; } @@ -747,20 +777,21 @@ ncuriquerylookup(NCURI* uri, const char* key) { int i; char* value = NULL; - if(uri == NULL || key == NULL || uri->querylist == NULL) return NULL; + if(uri == NULL || key == NULL) return NULL; + if(ensurequerylist(uri)) return NULL; i = ncfind(uri->querylist,key); - if(i < 0) - return NULL; - value = uri->querylist[(2*i)+1]; + if(i < 0) return NULL; + value = nclistget(uri->querylist,i+1); return value; } +#if 0 /* Obtain the complete list of fragment pairs in envv format */ const char** ncurifragmentparams(NCURI* uri) { ensurefraglist(uri); - return (const char**)uri->fraglist; + return (const char**)nclistcontents(uri->fraglist; } /* Obtain the complete list of query pairs in envv format */ @@ -771,7 +802,6 @@ ncuriqueryparams(NCURI* uri) return (const char**)uri->querylist; } -#if 0 int ncuriremoveparam(NCURI* uri, const char* key) { @@ -796,19 +826,18 @@ ncuriremoveparam(NCURI* uri, const char* key) case insensitive */ static int -ncfind(char** params, const char* key) +ncfind(NClist* params, const char* key) { int i; - char** p; if(key == NULL) return -1; if(params == NULL) return -1; - for(i=0,p=params;*p;p+=2,i++) { - if(strcasecmp(key,*p)==0) return i; + for(i=0;i 0) { + ncbytescat(buf,p0); + if(p1 != NULL && strlen(p1) > 0) { ncbytescat(buf,"="); if(encode) { - char* encoded = ncuriencodeonly(p[1],queryallow); + char* encoded = ncuriencodeonly(p1,queryallow); ncbytescat(buf,encoded); nullfree(encoded); } else - ncbytescat(buf,p[1]); + ncbytescat(buf,p1); } } - if(svecp) {*svecp = ncbytesextract(buf);} done: - ncbytesfree(buf); return stat; } @@ -1171,24 +1199,32 @@ static int ensurefraglist(NCURI* uri) { int stat = NC_NOERR; - int nofrag = 0; - int nolist = 0; - NClist* fraglist = NULL; + int hastext = 0; + int haslist = 0; + NClist* fraglist = nclistnew(); NCbytes* frag = NULL; - if(uri->fragment == NULL || strlen(uri->fragment) == 0) - {nullfree(uri->fragment); uri->fragment = NULL; nofrag=1;} - if(uri->fraglist == NULL) nolist = 1; - if(nolist && !nofrag) { - fraglist = nclistnew(); + if(nulllen(uri->fragment) == 0) + {nullfree(uri->fragment); uri->fragment = NULL; hastext=0;} + else hastext = 1; + if(nclistlength((NClist*)uri->fraglist) == 0) + {nclistfree((NClist*)uri->fraglist); uri->fraglist = NULL; haslist=0;} + else haslist = 1; + + /* Four cases: */ + if(!haslist && !hastext) { + /* do nothing */ + } else if(!haslist && hastext) { if((stat = parselist(uri->fragment,fraglist))) goto done; removedups(fraglist); - uri->fraglist = nclistextract(fraglist); - } else if(!nolist && nofrag) { + uri->fraglist = fraglist; fraglist = NULL; + } else if(haslist && !hastext) { /* Create the fragment string from fraglist */ frag = ncbytesnew(); - buildlist((const char**)uri->fraglist,0,frag); /* do not encode */ + if((stat=unparselist((const NClist*)uri->fraglist,NULL,0,frag))) goto done; /* do not encode */ uri->fragment = ncbytesextract(frag); + } else if(haslist && hastext) { + /* assume already consistent */ } done: @@ -1201,24 +1237,32 @@ static int ensurequerylist(NCURI* uri) { int stat = NC_NOERR; - int noquery = 0; - int nolist = 0; - NClist* querylist = NULL; + int hastext = 0; + int haslist = 0; + NClist* querylist = nclistnew(); NCbytes* query = NULL; - if(uri->query == NULL || strlen(uri->query) == 0) - {nullfree(uri->query); uri->query = NULL; noquery=1;} - if(uri->querylist == NULL) nolist = 1; - if(nolist && !noquery) { - querylist = nclistnew(); + if(nulllen(uri->query) == 0) + {nullfree(uri->query); uri->query = NULL; hastext=0;} + else hastext = 1; + if(nclistlength((NClist*)uri->querylist) == 0) + {nclistfree((NClist*)uri->querylist); uri->querylist = NULL; haslist=0;} + else haslist = 1; + + /* Four cases: */ + if(!haslist && !hastext) { + /* do nothing */ + } else if(!haslist && hastext) { if((stat = parselist(uri->query,querylist))) goto done; removedups(querylist); - uri->querylist = nclistextract(querylist); - } else if(!nolist && noquery) { + uri->querylist = querylist; querylist = NULL; + } else if(haslist && !hastext) { /* Create the query string from querylist */ query = ncbytesnew(); - buildlist((const char**)uri->querylist,0,query); /* do not encode */ + if((stat=unparselist((const NClist*)uri->querylist,NULL,0,query))) goto done; /* do not encode */ uri->query = ncbytesextract(query); + } else if(haslist && hastext) { + /* assume consistent */ } done: @@ -1242,30 +1286,9 @@ removedups(NClist* list) } } } - /* NULL terminate the list */ - nclistpush(list,NULL); -} - -static void -buildlist(const char** list, int encode, NCbytes* buf) -{ - const char** p; - int first = 1; - for(p=list;*p;p+=2,first=0) { - if(!first) ncbytescat(buf,"&"); - ncbytescat(buf,p[0]); - if(p[1] != NULL && strlen(p[1]) > 0) { - ncbytescat(buf,"="); - if(encode) { - char* encoded = ncuriencodeonly(p[1],queryallow); - ncbytescat(buf,encoded); - nullfree(encoded); - } else - ncbytescat(buf,p[1]); - } - } } +#if 0 static int extendenvv(char*** envvp, int amount, int* oldlenp) { @@ -1281,6 +1304,7 @@ extendenvv(char*** envvp, int amount, int* oldlenp) *envvp = envv; envv = NULL; return NC_NOERR; } +#endif /* Use for gdb debug */ char* @@ -1288,3 +1312,20 @@ ncuriunescape(const char* s) { return ncuridecodepartial(s,ascii); } + +/* Get the actual list of queryies */ +void* +ncuriqueryparams(NCURI* uri) +{ + ensurequerylist(uri); + return uri->querylist; +} + +/* Get the actual list of frags */ +void* +ncurifragmentparams(NCURI* uri) +{ + ensurefraglist(uri); + return uri->fraglist; +} + diff --git a/libdispatch/utf8proc.c b/libdispatch/utf8proc.c index 3a05dd3b3..68d8dd4a2 100644 --- a/libdispatch/utf8proc.c +++ b/libdispatch/utf8proc.c @@ -355,7 +355,8 @@ static nc_utf8proc_ssize_t nc_seqindex_write_char_decomposed(nc_utf8proc_uint16_ for (; len >= 0; entry++, len--) { nc_utf8proc_int32_t entry_cp = nc_seqindex_decode_entry(&entry); - written += nc_utf8proc_decompose_char(entry_cp, dst+written, + nc_utf8proc_int32_t *dest = dst ? (dst+written) : NULL; + written += nc_utf8proc_decompose_char(entry_cp, dest, (bufsize > written) ? (bufsize - written) : 0, options, last_boundclass); if (written < 0) return UTF8PROC_ERROR_OVERFLOW; @@ -525,8 +526,10 @@ static nc_utf8proc_ssize_t nc_seqindex_write_char_decomposed(nc_utf8proc_uint16_ if (custom_func != NULL) { uc = custom_func(uc, custom_data); /* user-specified custom mapping */ } + nc_utf8proc_int32_t *dest = NULL; + if (buffer) dest = buffer + wpos; decomp_result = nc_utf8proc_decompose_char( - uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options, + uc, dest, (bufsize > wpos) ? (bufsize - wpos) : 0, options, &boundclass ); if (decomp_result < 0) return decomp_result; diff --git a/libhdf5/hdf5debug.c b/libhdf5/hdf5debug.c index 40871c148..b4dace8d5 100644 --- a/libhdf5/hdf5debug.c +++ b/libhdf5/hdf5debug.c @@ -14,37 +14,9 @@ #ifdef H5CATCH -#define STSIZE 1000 - -#ifdef HAVE_EXECINFO_H -#ifdef H5BACKTRACE -# if !defined _WIN32 && !defined __CYGWIN__ -static void* stacktrace[STSIZE]; -# endif -#endif -#endif - int nch5breakpoint(int err) { -#ifdef HAVE_EXECINFO_H -#ifdef H5BACKTRACE -# if !defined _WIN32 && !defined __CYGWIN__ - int count = 0; - char** trace = NULL; - int i; - - count = backtrace(stacktrace,STSIZE); - trace = backtrace_symbols(stacktrace, STSIZE); - fprintf(stderr,"backtrace:\n"); - for(i=0;i ./license.txt - cat tinyxml2/tinyxml2.h > ./tinyxml2.h - sed -e 's/__BORLANDC__/__APPLE__/' < tinyxml2/tinyxml2.cpp \ - | sed -e 's/ptrdiff_t/long/g' > ./tinyxml2.cpp + tr -d '\r' < tinyxml2/tinyxml2.h > tinyxml2.h + cat tinyxml2/tinyxml2.cpp \ + | sed -e 's/__BORLANDC__/__APPLE__/' \ + | sed -e 's/ptrdiff_t/long/g' \ + | sed -e '/^static[ ]*FILE[*][ ]*callfopen(/i\ +\#if 0' \ + | sed -e '/^void[ ]*XMLDocument::DeleteNode(/i\ +\#endif /*0*/\ +' \ + | sed -e '/^XMLError[ ]*XMLDocument::LoadFile([ ]*const[ ]*char[*]/i\ +\#if 0' \ + | sed -e '/^XMLError[ ]*XMLDocument::Parse(/i\ +\#endif /*0*/\ +' \ + | tr -d '\r' \ + | cat > ./tinyxml2.cpp rm -fr tinyxml2 diff --git a/libncxml/tinyxml2.cpp b/libncxml/tinyxml2.cpp index 4f761ad31..c0d16eda0 100644 --- a/libncxml/tinyxml2.cpp +++ b/libncxml/tinyxml2.cpp @@ -2283,6 +2283,7 @@ XMLUnknown* XMLDocument::NewUnknown( const char* str ) return unk; } +#if 0 static FILE* callfopen( const char* filepath, const char* mode ) { TIXMLASSERT( filepath ); @@ -2299,6 +2300,8 @@ static FILE* callfopen( const char* filepath, const char* mode ) return fp; } +#endif /*0*/ + void XMLDocument::DeleteNode( XMLNode* node ) { TIXMLASSERT( node ); TIXMLASSERT(node->_document == this ); @@ -2317,6 +2320,7 @@ void XMLDocument::DeleteNode( XMLNode* node ) { } +#if 0 XMLError XMLDocument::LoadFile( const char* filename ) { if ( !filename ) { @@ -2420,6 +2424,8 @@ XMLError XMLDocument::SaveFile( FILE* fp, bool compact ) } +#endif /*0*/ + XMLError XMLDocument::Parse( const char* xml, size_t nBytes ) { Clear(); diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c index 966adfbfb..9e22f73d4 100644 --- a/libnczarr/zarr.c +++ b/libnczarr/zarr.c @@ -22,7 +22,7 @@ static int applycontrols(NCZ_FILE_INFO_T* zinfo); */ int -ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** controls) +ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* controls) { int stat = NC_NOERR; NCZ_FILE_INFO_T* zinfo = NULL; @@ -32,7 +32,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr NCjson* json = NULL; char* key = NULL; - ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_envv(controls):"null")); + ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_env(controls):"null")); nc = (NC*)file->controller; @@ -52,7 +52,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr zinfo->creating = 1; zinfo->common.file = file; zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); - if((zinfo->envv_controls=NCZ_clonestringvec(0,controls)) == NULL) + if((zinfo->controllist=nclistclone(controls,1)) == NULL) {stat = NC_ENOMEM; goto done;} /* fill in some of the zinfo and zroot fields */ @@ -94,7 +94,7 @@ done: */ int -ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) +ncz_open_dataset(NC_FILE_INFO_T* file, NClist* controls) { int stat = NC_NOERR; NC* nc = NULL; @@ -126,7 +126,7 @@ ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) zinfo->creating = 0; zinfo->common.file = file; zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); - if((zinfo->envv_controls = NCZ_clonestringvec(0,controls))==NULL) /*0=>envv style*/ + if((zinfo->controllist=nclistclone(controls,1)) == NULL) {stat = NC_ENOMEM; goto done;} zinfo->default_maxstrlen = NCZ_MAXSTR_DEFAULT; @@ -294,12 +294,13 @@ done: static const char* -controllookup(const char** envv_controls, const char* key) +controllookup(NClist* controls, const char* key) { - const char** p; - for(p=envv_controls;*p;p+=2) { - if(strcasecmp(key,*p)==0) { - return p[1]; + int i; + for(i=0;ienvv_controls,"mode")) != NULL) { + if((value = controllookup(zinfo->controllist,"mode")) != NULL) { if((stat = NCZ_comma_parse(value,modelist))) goto done; } /* Process the modelist first */ @@ -337,11 +338,11 @@ applycontrols(NCZ_FILE_INFO_T* zinfo) zinfo->controls.flags &= (~noflags); /* Process other controls */ - if((value = controllookup((const char**)zinfo->envv_controls,"log")) != NULL) { + if((value = controllookup(zinfo->controllist,"log")) != NULL) { zinfo->controls.flags |= FLAG_LOGGING; ncsetloglevel(NCLOGNOTE); } - if((value = controllookup((const char**)zinfo->envv_controls,"show")) != NULL) { + if((value = controllookup(zinfo->controllist,"show")) != NULL) { if(strcasecmp(value,"fetch")==0) zinfo->controls.flags |= FLAG_SHOWFETCH; } diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index 8b3b37ca0..22dd2d1cf 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -26,8 +26,8 @@ struct ZCVT { #define zcvt_empty {0,0,0.0,NULL} /* zarr.c */ -EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, const char** controls); -EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, const char** controls); +EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, NClist* controls); +EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, NClist* controls); EXTERNL int ncz_del_attr(NC_FILE_INFO_T* file, NC_OBJ* container, const char* name); /* HDF5 Mimics */ diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index fdef3f5c1..a84f08048 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -49,7 +49,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort) if((stat = nczmap_close(zinfo->map,(abort && zinfo->creating)?1:0))) goto done; - NCZ_freestringvec(0,zinfo->envv_controls); + nclistfreeall(zinfo->controllist); NC_authfree(zinfo->auth); nullfree(zinfo); diff --git a/libnczarr/zcreate.c b/libnczarr/zcreate.c index 3243b6f8c..bf7c1db3e 100644 --- a/libnczarr/zcreate.c +++ b/libnczarr/zcreate.c @@ -30,7 +30,7 @@ static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_DISKLESS|NC_64BIT * @author Dennis Heimbigner, Ed Hartnett */ static int -ncz_create_file(const char *path, int cmode, size_t initialsz, const char** controls, int ncid) +ncz_create_file(const char *path, int cmode, size_t initialsz, NClist* controls, int ncid) { int retval = NC_NOERR; NC_FILE_INFO_T* h5 = NULL; diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 50ff91e44..14c8079f0 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -144,7 +144,7 @@ typedef struct NCZ_FILE_INFO { } zarr; int creating; /* 1=> created 0=>open */ int native_endianness; /* NC_ENDIAN_LITTLE | NC_ENDIAN_BIG */ - char** envv_controls; /* Envv format */ + NClist* controllist; /* Envv format */ struct Controls { size64_t flags; # define FLAG_PUREZARR 1 diff --git a/libnczarr/zopen.c b/libnczarr/zopen.c index 8d8644e0f..2171f7b23 100644 --- a/libnczarr/zopen.c +++ b/libnczarr/zopen.c @@ -64,7 +64,7 @@ check_for_classic_model(NC_GRP_INFO_T *root_grp, int *is_classic) * @author Dennis Heimbigner, Ed Hartnett */ static int -ncz_open_file(const char *path, int mode, const char** controls, int ncid) +ncz_open_file(const char *path, int mode, NClist* controls, int ncid) { int stat = NC_NOERR; NC_FILE_INFO_T *h5 = NULL; diff --git a/libsrc/ncx.m4 b/libsrc/ncx.m4 index 48b8cf4ad..176ca27cc 100644 --- a/libsrc/ncx.m4 +++ b/libsrc/ncx.m4 @@ -307,88 +307,24 @@ swapn2b(void *dst, const void *src, IntType nn) IntType i; uint16_t *op = (uint16_t*) dst; uint16_t *ip = (uint16_t*) src; + uint16_t tmp; for (i=0; i 0) - * { - * *op++ = *(++ip); - * *op++ = *(ip++ -1); - * } - */ - while (nn > 3) - { - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - nn -= 4; - } - while (nn-- > 0) - { - *op++ = *(++ip); - *op++ = *(ip++ -1); - } -#endif } # ifndef vax inline static void swap4b(void *dst, const void *src) { - /* copy over, make the below swap in-place */ uint32_t tmp; - /* use memcpy to avoid type punning */ + /* memcpy is used to handle the case of unaligned memory */ memcpy(&tmp, src, sizeof(tmp)); tmp = SWAP4(tmp); memcpy(dst, &tmp, 4); - - /* Codes below will cause "break strict-aliasing rules" in gcc - uint32_t *op = (uint32_t*)dst; - *op = *(uint32_t*)src; - *op = SWAP4(*op); - */ - - /* Below are copied from netCDF-4. - * See https://bugtracking.unidata.ucar.edu/browse/NCF-338 - * Quote "One issue we are wrestling with is how compilers optimize this - * code. For some reason, we are actually needing to add an artificial - * move to a 4 byte space to get it to work. I think what is happening is - * that the optimizer is bit shifting within a double, which is incorrect. - * The following code actually does work correctly. - * This is in Linux land, gcc. - * - * However, the above in-place byte-swap does not appear affected by this. - */ -#if 0 - uint32_t *ip = (uint32_t*)src; - uint32_t tempOut; /* cannot use pointer when gcc O2 optimizer is used */ - tempOut = SWAP4(*ip); - - *(float *)dst = *(float *)(&tempOut); -#endif - - /* OLD implementation that results in four load and four store CPU - instructions - char *op = dst; - const char *ip = src; - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - */ - } # endif /* !vax */ @@ -398,110 +334,24 @@ swapn4b(void *dst, const void *src, IntType nn) IntType i; uint32_t *op = (uint32_t*) dst; uint32_t *ip = (uint32_t*) src; + uint32_t tmp; for (i=0; i 0) - * { - * op[0] = ip[3]; - * op[1] = ip[2]; - * op[2] = ip[1]; - * op[3] = ip[0]; - * op += 4; - * ip += 4; - * } - */ - while (nn > 3) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; - op[8] = ip[11]; - op[9] = ip[10]; - op[10] = ip[9]; - op[11] = ip[8]; - op[12] = ip[15]; - op[13] = ip[14]; - op[14] = ip[13]; - op[15] = ip[12]; - op += 16; - ip += 16; - nn -= 4; - } - while (nn-- > 0) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op += 4; - ip += 4; - } -#endif } # ifndef vax inline static void swap8b(void *dst, const void *src) { -#ifdef FLOAT_WORDS_BIGENDIAN - /* copy over, make the below swap in-place */ - *(uint64_t*)dst = *(uint64_t*)src; - - uint32_t *op = (uint32_t*)dst; - *op = SWAP4(*op); - op = (uint32_t*)((char*)dst+4); - *op = SWAP4(*op); -#else uint64_t tmp; - /* use memcpy to avoid type punning */ + /* memcpy is used to handle the case of unaligned memory */ memcpy(&tmp, src, sizeof(tmp)); tmp = SWAP8(tmp); memcpy(dst, &tmp, 8); - - /* Codes below will cause "break strict-aliasing rules" in gcc - uint64_t *op = (uint64_t*)dst; - *op = *(uint64_t*)src; - *op = SWAP8(*op); - */ -#endif - -#if 0 - char *op = dst; - const char *ip = src; -# ifndef FLOAT_WORDS_BIGENDIAN - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; -# else - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; -#endif -#endif } # endif /* !vax */ @@ -509,100 +359,16 @@ swap8b(void *dst, const void *src) inline static void swapn8b(void *dst, const void *src, IntType nn) { -#ifdef FLOAT_WORDS_BIGENDIAN - IntType i; - uint64_t *dst_p = (uint64_t*) dst; - uint64_t *src_p = (uint64_t*) src; - for (i=0; i 0) - * { - * op[0] = ip[7]; - * op[1] = ip[6]; - * op[2] = ip[5]; - * op[3] = ip[4]; - * op[4] = ip[3]; - * op[5] = ip[2]; - * op[6] = ip[1]; - * op[7] = ip[0]; - * op += 8; - * ip += 8; - * } - */ -# ifndef FLOAT_WORDS_BIGENDIAN - while (nn > 1) - { - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; - op[8] = ip[15]; - op[9] = ip[14]; - op[10] = ip[13]; - op[11] = ip[12]; - op[12] = ip[11]; - op[13] = ip[10]; - op[14] = ip[9]; - op[15] = ip[8]; - op += 16; - ip += 16; - nn -= 2; - } - while (nn-- > 0) - { - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; - op += 8; - ip += 8; - } -# else - while (nn-- > 0) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; - op += 8; - ip += 8; - } -#endif -#endif } # endif /* !vax */ diff --git a/nctest/CMakeLists.txt b/nctest/CMakeLists.txt index d7de84a6c..ada220f2a 100644 --- a/nctest/CMakeLists.txt +++ b/nctest/CMakeLists.txt @@ -18,7 +18,9 @@ TARGET_LINK_LIBRARIES(nctest netcdf) ADD_TEST(nctest ${EXECUTABLE_OUTPUT_PATH}/nctest) add_bin_test_no_prefix(tst_rename) -add_sh_test(nctest compare_test_files) -IF(HAVE_BASH) - SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest) -ENDIF(HAVE_BASH) +IF(BUILD_UTILITIES) + add_sh_test(nctest compare_test_files) + IF(HAVE_BASH) + SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest) + ENDIF(HAVE_BASH) +ENDIF(BUILD_UTILITIES)