mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-12-09 08:11:38 +08:00
835b81a285
NOTE: This PR should not be included in 4.9.1 since additional
DAP4 related PRs will be forthcoming.
This PR makes major changes to libdap4 and dap4_test driven by changes to TDS.
* Enable DAP4
* Clean up the test input files and the test baseline comparison files. This entails:
* Remove a multitude of unused test input and baseline data files; among them are dap4_test/: daptestfiles, dmrtestfiles, nctestfiles, and misctestfiles.
* Define a canonical set of test input files and record in dap4_test/cdltestfiles.
* Use the cdltestfiles to generate the .nc test inputs. This set of .nc files is then moved to the d4ts (DAP4 test server) war file in the tds repository. This set then becomes the canonical set of DAP4 test sources.
* Scrape d4ts to obtain copies of the raw streams of DAP4 encoded data. The .dmr and .dap streams are then stored in dap4_test/rawtestfiles.
* Disable some remote server tests until those servers are fixed.
* Add an option to ncdump (-XF) that forces the type of the _FillValue attribute; this is primarily to simplify testing of fill mismatch.
* Minor bug fixes to ncgen.
* Changes to libdap4:
* Replace old checksum hack with the dap4.checksum flag.
* Support the dap4.XXX controls.
* Cleanup _FillValue handling, especially var-attribute type mismatches.
* Fix enum handling based on changes to netcdf-java.
* Changes to dap4_test:
* Add getopt support to various test support programs.
* Remove unneeded shell scripts.
* Add new scripts: test_curlopt.sh
344 lines
9.6 KiB
C
344 lines
9.6 KiB
C
/*********************************************************************
|
|
* Copyright 2018, UCAR/Unidata
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
*********************************************************************/
|
|
|
|
#include "d4includes.h"
|
|
#include "d4curlfunctions.h"
|
|
|
|
static size_t WriteFileCallback(void*, size_t, size_t, void*);
|
|
static size_t WriteMemoryCallback(void*, size_t, size_t, void*);
|
|
static int curlerrtoncerr(CURLcode cstat);
|
|
|
|
struct Fetchdata {
|
|
FILE* stream;
|
|
size_t size;
|
|
};
|
|
|
|
long
|
|
NCD4_fetchhttpcode(CURL* curl)
|
|
{
|
|
long httpcode = 200;
|
|
CURLcode cstat = CURLE_OK;
|
|
/* Extract the http code */
|
|
#ifdef HAVE_CURLINFO_RESPONSE_CODE
|
|
cstat = curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE,&httpcode);
|
|
#else
|
|
cstat = curl_easy_getinfo(curl,CURLINFO_HTTP_CONNECTCODE,&httpcode);
|
|
#endif
|
|
if(cstat != CURLE_OK) {
|
|
httpcode = 0;
|
|
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
|
|
}
|
|
return httpcode;
|
|
}
|
|
|
|
int
|
|
NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream,
|
|
d4size_t* sizep, long* filetime)
|
|
{
|
|
int ret = NC_NOERR;
|
|
CURLcode cstat = CURLE_OK;
|
|
struct Fetchdata fetchdata;
|
|
|
|
/* Set the URL */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url);
|
|
if (cstat != CURLE_OK) goto fail;
|
|
|
|
/* send all data to this function */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteFileCallback);
|
|
if (cstat != CURLE_OK) goto fail;
|
|
|
|
/* we pass our file to the callback function */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fetchdata);
|
|
if(cstat != CURLE_OK) goto fail;
|
|
|
|
/* One last thing; always try to get the last modified time */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1);
|
|
if (cstat != CURLE_OK) goto fail;
|
|
|
|
fetchdata.stream = stream;
|
|
fetchdata.size = 0;
|
|
cstat = curl_easy_perform(curl);
|
|
if (cstat != CURLE_OK)
|
|
{ret = NC_EDAPSVC; goto fail;}
|
|
|
|
if (ret == NC_NOERR) {
|
|
/* return the file size*/
|
|
#ifdef D4DEBUG
|
|
nclog(NCLOGNOTE,"filesize: %lu bytes",fetchdata.size);
|
|
#endif
|
|
if (sizep != NULL)
|
|
*sizep = fetchdata.size;
|
|
/* Get the last modified time */
|
|
if(filetime != NULL)
|
|
cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime);
|
|
if(cstat != CURLE_OK)
|
|
{ret = NC_ECURL; goto fail;}
|
|
}
|
|
return THROW(ret);
|
|
|
|
fail:
|
|
if(cstat != CURLE_OK) {
|
|
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
|
|
ret = curlerrtoncerr(cstat);
|
|
}
|
|
return THROW(ret);
|
|
}
|
|
|
|
int
|
|
NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcodep)
|
|
{
|
|
int ret = NC_NOERR;
|
|
CURLcode cstat = CURLE_OK;
|
|
size_t len;
|
|
long httpcode = 0;
|
|
|
|
/* send all data to this function */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
|
|
/* we pass our file to the callback function */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)buf);
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
|
|
/* One last thing; always try to get the last modified time */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1);
|
|
|
|
/* Set the URL */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)"");
|
|
cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url);
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
|
|
cstat = curl_easy_perform(curl);
|
|
|
|
if(cstat == CURLE_PARTIAL_FILE) {
|
|
/* Log it but otherwise ignore */
|
|
nclog(NCLOGWARN, "curl error: %s; ignored",
|
|
curl_easy_strerror(cstat));
|
|
cstat = CURLE_OK;
|
|
}
|
|
if(cstat != CURLE_OK) goto done;
|
|
|
|
httpcode = NCD4_fetchhttpcode(curl);
|
|
if(httpcodep) *httpcodep = httpcode;
|
|
|
|
/* Get the last modified time */
|
|
if(filetime != NULL)
|
|
cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime);
|
|
if(cstat != CURLE_OK) goto done;
|
|
|
|
/* Null terminate the buffer*/
|
|
len = ncbyteslength(buf);
|
|
ncbytesappend(buf, '\0');
|
|
ncbytessetlength(buf, len); /* don't count null in buffer size*/
|
|
#ifdef D4DEBUG
|
|
nclog(NCLOGNOTE,"buffersize: %lu bytes",(d4size_t)ncbyteslength(buf));
|
|
#endif
|
|
|
|
done:
|
|
if(cstat != CURLE_OK) {
|
|
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
|
|
ret = curlerrtoncerr(cstat);
|
|
} else switch (httpcode) {
|
|
case 400: ret = NC_EDATADAP; break;
|
|
case 401: ret = NC_EACCESS; break;
|
|
case 403: ret = NC_EAUTH; break;
|
|
case 404: ret = ENOENT; break;
|
|
case 500: ret = NC_EDAPSVC; break;
|
|
case 200: break;
|
|
default: ret = NC_ECURL; break;
|
|
}
|
|
return THROW(ret);
|
|
}
|
|
|
|
static size_t
|
|
WriteFileCallback(void* ptr, size_t size, size_t nmemb, void* data)
|
|
{
|
|
size_t realsize = size * nmemb;
|
|
size_t count;
|
|
struct Fetchdata* fetchdata;
|
|
fetchdata = (struct Fetchdata*) data;
|
|
if(realsize == 0)
|
|
nclog(NCLOGWARN,"WriteFileCallback: zero sized chunk");
|
|
count = fwrite(ptr, size, nmemb, fetchdata->stream);
|
|
if (count > 0) {
|
|
fetchdata->size += (count * size);
|
|
} else {
|
|
nclog(NCLOGWARN,"WriteFileCallback: zero sized write");
|
|
}
|
|
#ifdef PROGRESS
|
|
nclog(NCLOGNOTE,"callback: %lu bytes",(d4size_t)realsize);
|
|
#endif
|
|
return count;
|
|
}
|
|
|
|
static size_t
|
|
WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
|
|
{
|
|
size_t realsize = size * nmemb;
|
|
NCbytes* buf = (NCbytes*) data;
|
|
if(realsize == 0)
|
|
nclog(NCLOGWARN,"WriteMemoryCallback: zero sized chunk");
|
|
/* Optimize for reading potentially large dods datasets */
|
|
while(!ncbytesavail(buf,realsize)) {
|
|
/* double the size of the packet */
|
|
ncbytessetalloc(buf,2*ncbytesalloc(buf));
|
|
}
|
|
ncbytesappendn(buf, ptr, realsize);
|
|
#ifdef PROGRESS
|
|
nclog(NCLOGNOTE,"callback: %lu bytes",(d4size_t)realsize);
|
|
#endif
|
|
return realsize;
|
|
}
|
|
|
|
int
|
|
NCD4_curlopen(CURL** curlp)
|
|
{
|
|
int ret = NC_NOERR;
|
|
CURLcode cstat = CURLE_OK;
|
|
CURL* curl;
|
|
/* initialize curl*/
|
|
curl = curl_easy_init();
|
|
if (curl == NULL)
|
|
ret = NC_ECURL;
|
|
else {
|
|
cstat = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
|
|
if (cstat != CURLE_OK)
|
|
ret = NC_ECURL;
|
|
}
|
|
if (curlp)
|
|
*curlp = curl;
|
|
if(cstat != CURLE_OK) {
|
|
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
|
|
ret = curlerrtoncerr(cstat);
|
|
}
|
|
return THROW(ret);
|
|
}
|
|
|
|
void
|
|
NCD4_curlclose(CURL* curl)
|
|
{
|
|
if (curl != NULL)
|
|
curl_easy_cleanup(curl);
|
|
}
|
|
|
|
int
|
|
NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime)
|
|
{
|
|
int ret = NC_NOERR;
|
|
CURLcode cstat = CURLE_OK;
|
|
|
|
/* Set the URL */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url);
|
|
if (cstat != CURLE_OK)
|
|
goto fail;
|
|
|
|
/* Ask for head */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30); /* 30sec timeout*/
|
|
cstat = curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 2);
|
|
cstat = curl_easy_setopt(curl, CURLOPT_HEADER, 1);
|
|
cstat = curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
|
|
cstat = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
|
|
cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1);
|
|
|
|
cstat = curl_easy_perform(curl);
|
|
if(cstat != CURLE_OK) goto fail;
|
|
if(filetime != NULL)
|
|
cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime);
|
|
if(cstat != CURLE_OK) goto fail;
|
|
return THROW(ret);
|
|
|
|
fail:
|
|
if(cstat != CURLE_OK) {
|
|
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
|
|
ret = curlerrtoncerr(cstat);
|
|
}
|
|
return THROW(ret);
|
|
}
|
|
|
|
int
|
|
NCD4_ping(const char* url)
|
|
{
|
|
int ret = NC_NOERR;
|
|
CURLcode cstat = CURLE_OK;
|
|
CURL* curl = NULL;
|
|
NCbytes* buf = NULL;
|
|
|
|
/* Create a CURL instance */
|
|
ret = NCD4_curlopen(&curl);
|
|
if(ret != NC_NOERR) return THROW(ret);
|
|
|
|
/* Use redirects */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10L);
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
cstat = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
|
|
/* use a very short timeout: 10 seconds */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_TIMEOUT, (long)5);
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
|
|
/* fail on HTTP 400 code errors */
|
|
cstat = curl_easy_setopt(curl, CURLOPT_FAILONERROR, (long)1);
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
|
|
/* Try to get the file */
|
|
buf = ncbytesnew();
|
|
ret = NCD4_fetchurl(curl,url,buf,NULL,NULL);
|
|
if(ret == NC_NOERR) {
|
|
/* Don't trust curl to return an error when request gets 404 */
|
|
long http_code = 0;
|
|
cstat = curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &http_code);
|
|
if (cstat != CURLE_OK)
|
|
goto done;
|
|
if(http_code >= 400) {
|
|
cstat = CURLE_HTTP_RETURNED_ERROR;
|
|
goto done;
|
|
}
|
|
} else
|
|
goto done;
|
|
|
|
done:
|
|
ncbytesfree(buf);
|
|
NCD4_curlclose(curl);
|
|
if(cstat != CURLE_OK) {
|
|
nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat));
|
|
ret = curlerrtoncerr(cstat);
|
|
}
|
|
return THROW(ret);
|
|
}
|
|
|
|
static int
|
|
curlerrtoncerr(CURLcode cstat)
|
|
{
|
|
switch (cstat) {
|
|
case CURLE_OK: return THROW(NC_NOERR);
|
|
case CURLE_URL_MALFORMAT:
|
|
return THROW(NC_EURL);
|
|
case CURLE_COULDNT_RESOLVE_HOST:
|
|
case CURLE_COULDNT_CONNECT:
|
|
case CURLE_REMOTE_ACCESS_DENIED:
|
|
case CURLE_TOO_MANY_REDIRECTS:
|
|
return THROW(NC_EDAPSVC);
|
|
case CURLE_OUT_OF_MEMORY:
|
|
return THROW(NC_ENOMEM);
|
|
/* Eventually would be nice to convert these */
|
|
case CURLE_SSL_CONNECT_ERROR:
|
|
case CURLE_READ_ERROR:
|
|
case CURLE_OPERATION_TIMEDOUT:
|
|
case CURLE_SSL_CERTPROBLEM:
|
|
case CURLE_FILESIZE_EXCEEDED:
|
|
case CURLE_SSL_CACERT_BADFILE:
|
|
default: break;
|
|
}
|
|
return THROW(NC_ECURL);
|
|
}
|