mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-01-12 15:45:21 +08:00
3db4f013bf
Specific changes:
1. Add dap4 code: libdap4 and dap4_test.
Note that until the d4ts server problem is solved, dap4 is turned off.
2. Modify various files to support dap4 flags:
configure.ac, Makefile.am, CMakeLists.txt, etc.
3. Add nc_test/test_common.sh. This centralizes
the handling of the locations of various
things in the build tree: e.g. where is
ncgen.exe located. See nc_test/test_common.sh
for details.
4. Modify .sh files to use test_common.sh
5. Obsolete separate oc2 by moving it to be part of
netcdf-c. This means replacing code with netcdf-c
equivalents.
5. Add --with-testserver to configure.ac to allow
override of the servers to be used for --enable-dap-remote-tests.
6. There were multiple versions of nctypealignment code. Try to
centralize in libdispatch/doffset.c and include/ncoffsets.h
7. Add a unit test for the ncuri code because of its complexity.
8. Move the findserver code out of libdispatch and into
a separate, self contained program in ncdap_test and dap4_test.
9. Move the dispatch header files (nc{3,4}dispatch.h) to
.../include because they are now shared by modules.
10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts.
11. Make use of MREMAP if available
12. Misc. minor changes e.g.
- #include <config.h> -> #include "config.h"
- Add some no-install headers to /include
- extern -> EXTERNL and vice versa as needed
- misc header cleanup
- clean up checking for misc. unix vs microsoft functions
13. Change copyright decls in some files to point to LICENSE file.
14. Add notes to RELEASENOTES.md
374 lines
9.6 KiB
C
374 lines
9.6 KiB
C
/* Copyright 2009, UCAR/Unidata and OPeNDAP, Inc.
|
|
See the COPYRIGHT file for more information. */
|
|
|
|
#include "config.h"
|
|
#ifdef HAVE_UNISTD_H
|
|
#include <unistd.h>
|
|
#endif
|
|
#include "ocinternal.h"
|
|
#include "ocdebug.h"
|
|
#include "ochttp.h"
|
|
|
|
static size_t WriteFileCallback(void*, size_t, size_t, void*);
|
|
static size_t WriteMemoryCallback(void*, size_t, size_t, void*);
|
|
|
|
struct Fetchdata {
|
|
FILE* stream;
|
|
size_t size;
|
|
};
|
|
|
|
long
|
|
ocfetchhttpcode(CURL* curl)
|
|
{
|
|
long httpcode = 200;
|
|
CURLcode cstat = CURLE_OK;
|
|
/* Extract the http code */
|
|
#ifdef HAVE_CURLINFO_RESPONSE_CODE
|
|
cstat = CURLERR(curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE,&httpcode));
|
|
#else
|
|
cstat = curl_easy_getinfo(curl,CURLINFO_HTTP_CODE,&httpcode);
|
|
#endif
|
|
if(cstat != CURLE_OK) httpcode = 0;
|
|
return httpcode;
|
|
}
|
|
|
|
OCerror
|
|
ocfetchurl_file(CURL* curl, const char* url, FILE* stream,
|
|
off_t* sizep, long* filetime)
|
|
{
|
|
int stat = OC_NOERR;
|
|
CURLcode cstat = CURLE_OK;
|
|
struct Fetchdata fetchdata;
|
|
|
|
/* Set the URL */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_URL, (void*)url));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
|
|
/* send all data to this function */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteFileCallback));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
|
|
/* we pass our file to the callback function */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fetchdata));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
|
|
/* One last thing; always try to get the last modified time */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
|
|
fetchdata.stream = stream;
|
|
fetchdata.size = 0;
|
|
cstat = CURLERR(curl_easy_perform(curl));
|
|
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
|
|
if (stat == OC_NOERR) {
|
|
/* return the file size*/
|
|
#ifdef OCDEBUG
|
|
oclog(OCLOGNOTE,"filesize: %lu bytes",fetchdata.size);
|
|
#endif
|
|
if (sizep != NULL)
|
|
*sizep = fetchdata.size;
|
|
/* Get the last modified time */
|
|
if(filetime != NULL)
|
|
cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime);
|
|
if(cstat != CURLE_OK) goto fail;
|
|
}
|
|
return OCTHROW(stat);
|
|
|
|
fail:
|
|
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
|
|
return OCTHROW(OC_ECURL);
|
|
}
|
|
|
|
OCerror
|
|
ocfetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime,
|
|
struct OCcredentials* creds)
|
|
{
|
|
OCerror stat = OC_NOERR;
|
|
CURLcode cstat = CURLE_OK;
|
|
size_t len;
|
|
long httpcode = 0;
|
|
|
|
/* Set the URL */
|
|
cstat = CURLERR(CURLERR(curl_easy_setopt(curl, CURLOPT_URL, (void*)url)));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
|
|
#if 0
|
|
if(creds != NULL && creds->password != NULL && creds->username != NULL) {
|
|
/* Set user and password */
|
|
#if defined (HAVE_CURLOPT_USERNAME) && defined (HAVE_CURLOPT_PASSWORD)
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_USERNAME, creds->username));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_PASSWORD, creds->password));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
#else
|
|
snprintf(tbuf,1023,"%s:%s",creds->username,creds->password);
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_USERPWD, tbuf));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
#endif
|
|
}
|
|
#endif
|
|
|
|
/* send all data to this function */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
|
|
/* we pass our file to the callback function */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)buf));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
|
|
/* One last thing; always try to get the last modified time */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1));
|
|
|
|
cstat = CURLERR(curl_easy_perform(curl));
|
|
|
|
if(cstat == CURLE_PARTIAL_FILE) {
|
|
/* Log it but otherwise ignore */
|
|
nclog(NCLOGWARN, "curl error: %s; ignored",
|
|
curl_easy_strerror(cstat));
|
|
cstat = CURLE_OK;
|
|
}
|
|
httpcode = ocfetchhttpcode(curl);
|
|
|
|
if(cstat != CURLE_OK) goto fail;
|
|
|
|
/* Get the last modified time */
|
|
if(filetime != NULL)
|
|
cstat = CURLERR(curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime));
|
|
if(cstat != CURLE_OK) goto fail;
|
|
|
|
/* Null terminate the buffer*/
|
|
len = ncbyteslength(buf);
|
|
ncbytesappend(buf, '\0');
|
|
ncbytessetlength(buf, len); /* dont count null in buffer size*/
|
|
#ifdef OCDEBUG
|
|
nclog(NCLOGNOTE,"buffersize: %lu bytes",(off_t)ncbyteslength(buf));
|
|
#endif
|
|
|
|
return OCTHROW(stat);
|
|
|
|
fail:
|
|
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
|
|
switch (httpcode) {
|
|
case 401: stat = OC_EAUTH; break;
|
|
case 404: stat = OC_ENOFILE; break;
|
|
case 500: stat = OC_EDAPSVC; break;
|
|
case 200: break;
|
|
default: stat = OC_ECURL; break;
|
|
}
|
|
return OCTHROW(stat);
|
|
}
|
|
|
|
static size_t
|
|
WriteFileCallback(void* ptr, size_t size, size_t nmemb, void* data)
|
|
{
|
|
size_t realsize = size * nmemb;
|
|
size_t count;
|
|
struct Fetchdata* fetchdata;
|
|
fetchdata = (struct Fetchdata*) data;
|
|
if(realsize == 0)
|
|
nclog(NCLOGWARN,"WriteFileCallback: zero sized chunk");
|
|
count = fwrite(ptr, size, nmemb, fetchdata->stream);
|
|
if (count > 0) {
|
|
fetchdata->size += (count * size);
|
|
} else {
|
|
nclog(NCLOGWARN,"WriteFileCallback: zero sized write");
|
|
}
|
|
#ifdef OCPROGRESS
|
|
nclog(NCLOGNOTE,"callback: %lu bytes",(off_t)realsize);
|
|
#endif
|
|
return count;
|
|
}
|
|
|
|
static size_t
|
|
WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
|
|
{
|
|
size_t realsize = size * nmemb;
|
|
NCbytes* buf = (NCbytes*) data;
|
|
if(realsize == 0)
|
|
nclog(NCLOGWARN,"WriteMemoryCallback: zero sized chunk");
|
|
/* Optimize for reading potentially large dods datasets */
|
|
if(!ncbytesavail(buf,realsize)) {
|
|
/* double the size of the packet */
|
|
ncbytessetalloc(buf,2*ncbytesalloc(buf));
|
|
}
|
|
ncbytesappendn(buf, ptr, realsize);
|
|
#ifdef OCPROGRESS
|
|
nclog(NCLOGNOTE,"callback: %lu bytes",(off_t)realsize);
|
|
#endif
|
|
return realsize;
|
|
}
|
|
|
|
#if 0
|
|
static void
|
|
assembleurl(DAPURL* durl, NCbytes* buf, int what)
|
|
{
|
|
encodeurltext(durl->url,buf);
|
|
if(what & WITHPROJ) {
|
|
ncbytescat(buf,"?");
|
|
encodeurltext(durl->projection,buf);
|
|
}
|
|
if(what & WITHSEL) encodeurltext(durl->selection,buf);
|
|
|
|
}
|
|
|
|
static char mustencode="";
|
|
static char hexchars[16] = {
|
|
'0', '1', '2', '3',
|
|
'4', '5', '6', '7',
|
|
'8', '9', 'a', 'b',
|
|
'c', 'd', 'e', 'f',
|
|
};
|
|
|
|
static void
|
|
encodeurltext(char* text, NCbytes* buf)
|
|
{
|
|
/* Encode the URL to handle illegal characters */
|
|
len = strlen(url);
|
|
encoded = ocmalloc(len*4+1); /* should never be larger than this*/
|
|
if(encoded==NULL) return;
|
|
p = url; q = encoded;
|
|
while((c=*p++)) {
|
|
if(strchr(mustencode,c) != NULL) {
|
|
char tmp[8];
|
|
int hex1, hex2;
|
|
hex1 = (c & 0x0F);
|
|
hex2 = (c & 0xF0) >> 4;
|
|
tmp[0] = '0'; tmp[1] = 'x';
|
|
tmp[2] = hexchars[hex2]; tmp[3] = hexchars[hex1];
|
|
tmp[4] = '\0';
|
|
ncbytescat(buf,tmp);
|
|
} else *q++ = (char)c;
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
OCerror
|
|
occurlopen(CURL** curlp)
|
|
{
|
|
int stat = OC_NOERR;
|
|
CURLcode cstat = CURLE_OK;
|
|
CURL* curl;
|
|
/* initialize curl*/
|
|
curl = curl_easy_init();
|
|
if (curl == NULL)
|
|
stat = OC_ECURL;
|
|
else {
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1));
|
|
if (cstat != CURLE_OK)
|
|
stat = OC_ECURL;
|
|
}
|
|
if (curlp)
|
|
*curlp = curl;
|
|
return OCTHROW(stat);
|
|
}
|
|
|
|
void
|
|
occurlclose(CURL* curl)
|
|
{
|
|
if (curl != NULL)
|
|
curl_easy_cleanup(curl);
|
|
}
|
|
|
|
OCerror
|
|
ocfetchlastmodified(CURL* curl, char* url, long* filetime)
|
|
{
|
|
int stat = OC_NOERR;
|
|
CURLcode cstat = CURLE_OK;
|
|
|
|
/* Set the URL */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_URL, (void*)url));
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
|
|
/* Ask for head */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30)); /* 30sec timeout*/
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 2));
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_HEADER, 1));
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_NOBODY, 1));
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1));
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1));
|
|
|
|
cstat = CURLERR(curl_easy_perform(curl));
|
|
if(cstat != CURLE_OK) goto fail;
|
|
if(filetime != NULL)
|
|
cstat = CURLERR(curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime));
|
|
if(cstat != CURLE_OK) goto fail;
|
|
|
|
return OCTHROW(stat);
|
|
|
|
fail:
|
|
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
|
|
return OCTHROW(OC_ECURL);
|
|
}
|
|
|
|
OCerror
|
|
ocping(const char* url)
|
|
{
|
|
int stat = OC_NOERR;
|
|
CURLcode cstat = CURLE_OK;
|
|
CURL* curl = NULL;
|
|
NCbytes* buf = NULL;
|
|
|
|
/* Create a CURL instance */
|
|
stat = occurlopen(&curl);
|
|
if(stat != OC_NOERR) return stat;
|
|
|
|
/* Use redirects */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10L));
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L));
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
|
|
/* use a very short timeout: 10 seconds */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_TIMEOUT, (long)5));
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
|
|
/* fail on HTTP 400 code errors */
|
|
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_FAILONERROR, (long)1));
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
|
|
/* Try to get the file */
|
|
buf = ncbytesnew();
|
|
stat = ocfetchurl(curl,url,buf,NULL,NULL);
|
|
if(stat == OC_NOERR) {
|
|
/* Don't trust curl to return an error when request gets 404 */
|
|
long http_code = 0;
|
|
cstat = CURLERR(curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &http_code));
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
if(http_code >= 400) {
|
|
cstat = CURLE_HTTP_RETURNED_ERROR;
|
|
goto done;
|
|
}
|
|
} else
|
|
goto done;
|
|
|
|
done:
|
|
ncbytesfree(buf);
|
|
occurlclose(curl);
|
|
if(cstat != CURLE_OK) {
|
|
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
|
|
stat = OC_EDAPSVC;
|
|
}
|
|
return OCTHROW(stat);
|
|
}
|