mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-21 03:13:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1884 from DennisHeimbigner/encodeparam.dmh

Browse Source 
Provide ability to control URL encoding for DAP2
This commit is contained in:
Ward Fisher 2020-11-12 17:17:10 -07:00 committed by GitHub
commit 2409ab4e5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 142 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
RELEASE_NOTES.md
View File

@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.0 - TBD
* [Enhancement] Give the client control over what parts of a DAP2 URL are URL encoded (i.e. %xx). This is to support the different decoding rules that servers apply to incoming URLS.
* [Bug Fix] Fix incorrect time offsets from `ncdump -t`, in some cases when the time `units` attribute contains both a **non-zero** time-of-day, and a time zone suffix containing the letter "T", such as "UTC". See [Github #1866](https://github.com/Unidata/netcdf-c/pull/1866) for more information.
* [Bug Fix] Cleanup the NCZarr S3 build options. See [Github #1869](https://github.com/Unidata/netcdf-c/pull/1869) for more information.
* [Bug Fix] Support aligned access for selected ARM processors. See [Github #1871](https://github.com/Unidata/netcdf-c/pull/1871) for more information.

8
configure.ac
View File

@ -498,11 +498,11 @@ CFLAGS="$SAVECFLAGS"
# --enable-dap => enable-dap4
enable_dap4=$enable_dap
# Default is to do the short remote tests.
# Temporary: Change default to npt do these tests
AC_MSG_CHECKING([whether dap remote testing should be enabled (default on)])
# Temporary: Change default to not do these tests
AC_MSG_CHECKING([whether dap remote testing should be enabled (default off)])
AC_ARG_ENABLE([dap-remote-tests],
[AS_HELP_STRING([--disable-dap-remote-tests],
[disable dap remote tests])])
[AS_HELP_STRING([--enable-dap-remote-tests],
[enable dap remote tests])])
test "x$enable_dap_remote_tests" = xyes || enable_dap_remote_tests=no
if test "x$enable_dap" = "xno" ; then
enable_dap_remote_tests=no

3
include/ncdap.h
View File

@ -36,6 +36,9 @@ typedef unsigned int NCFLAGS;
#define NCF_PREFETCH_ALL (0x0800) /* Prefetch all variables */
/* Allow _FillValue/Variable type mismatch */
#define NCF_FILLMISMATCH (0x1000)
/* Hack to control URL encoding */
#define NCF_ENCODE_PATH (0x2000)
#define NCF_ENCODE_QUERY (0x4000)
/*COLUMBIA_HACK*/
#define NCF_COLUMBIA (0x80000000) /* Hack for columbia server */

19
include/ncuri.h
View File

@ -9,15 +9,16 @@
#include "ncexternl.h"
/* Define flags to control what is included by ncuribuild*/
#define NCURIPATH 1
#define NCURIPWD 2
#define NCURIQUERY 4
#define NCURIFRAG 8
#define NCURIENCODE 16 /* If output should be encoded */
#define NCURIBASE (NCURIPWD|NCURIPATH)
#define NCURISVC (NCURIQUERY|NCURIBASE) /* for sending to server */
#define NCURIALL (NCURIPATH|NCURIPWD|NCURIQUERY|NCURIFRAG) /* for rebuilding after changes */
#define NCURIPATH 1
#define NCURIPWD 2
#define NCURIQUERY 4
#define NCURIFRAG 8
#define NCURIENCODEPATH 16 /* If output url path should be encoded */
#define NCURIENCODEQUERY 32 /* If output url query should be encoded */
#define NCURIENCODE (NCURIENCODEPATH|NCURIENCODEQUERY)
#define NCURIBASE (NCURIPWD|NCURIPATH)
#define NCURISVC (NCURIQUERY|NCURIBASE) /* for sending to server */
#define NCURIALL (NCURIPATH|NCURIPWD|NCURIQUERY|NCURIFRAG) /* for rebuilding after changes */
/*! This is an open structure meaning
it is ok to directly access its fields

33
libdap2/daputil.c
View File

@ -654,8 +654,8 @@ dap_fetch(NCDAPCOMMON* nccomm, OClink conn, const char* ce,
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;
char* ext = NULL;
OCflags flags = 0;
int httpcode = 0;
OCflags ocflags = 0;
#ifdef HAVE_GETTIMEOFDAY
struct timeval time0;
struct timeval time1;
@ -668,13 +668,14 @@ dap_fetch(NCDAPCOMMON* nccomm, OClink conn, const char* ce,
if(ce != NULL && strlen(ce) == 0)
ce = NULL;
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE))
ce = NULL;
}
if(FLAGSET(nccomm->controls,NCF_ONDISK)) {
flags |= OCONDISK;
}
if(FLAGSET(nccomm->controls,NCF_ONDISK))
ocflags |= OCONDISK;
if(FLAGSET(nccomm->controls,NCF_ENCODE_PATH))
ocflags |= OCENCODEPATH;
if(FLAGSET(nccomm->controls,NCF_ENCODE_QUERY))
ocflags |= OCENCODEQUERY;
if(SHOWFETCH) {
/* Build uri string minus the constraint and #tag */
@ -688,7 +689,7 @@ dap_fetch(NCDAPCOMMON* nccomm, OClink conn, const char* ce,
gettimeofday(&time0,NULL);
#endif
}
ocstat = oc_fetch(conn,ce,dxd,flags,rootp);
ocstat = oc_fetch(conn,ce,dxd,ocflags,rootp);
if(FLAGSET(nccomm->controls,NCF_SHOWFETCH)) {
#ifdef HAVE_GETTIMEOFDAY
double secs;
@ -810,3 +811,19 @@ nccpadding(unsigned long offset, int alignment)
return pad;
}
int
dapparamparselist(const char* s0, int delim, NClist* list)
{
int stat = NC_NOERR;
char* s = strdup(s0);
char* p;
int i,count = 1;
if(s0 == NULL || strlen(s) == 0) goto done;
for(p=s;*p;p++) {if(*p == delim) {*p = '\0'; count++;}}
for(i=0,p=s;i<count;i++,p+=(strlen(p)+1)) {
if(strlen(p)>0)
nclistpush(list,strdup(p));
}
done:
return stat;
}

2
libdap2/daputil.h
View File

@ -72,4 +72,6 @@ extern int dap_badname(char* name);
extern char* dap_repairname(char* name);
extern char* dap_getselection(NCURI* uri);
extern int dapparamparselist(const char* s0, int delim, NClist* list);
#endif /*DAPUTIL_H*/

28
libdap2/ncd2dispatch.c
View File

@ -2207,6 +2207,8 @@ fixzerodims(NCDAPCOMMON* dapcomm)
static void
applyclientparamcontrols(NCDAPCOMMON* dapcomm)
{
const char* value = NULL;
/* clear the flags */
CLRFLAG(dapcomm->controls,NCF_CACHE);
CLRFLAG(dapcomm->controls,NCF_SHOWFETCH);
@ -2214,6 +2216,8 @@ applyclientparamcontrols(NCDAPCOMMON* dapcomm)
CLRFLAG(dapcomm->controls,NCF_NCDAP);
CLRFLAG(dapcomm->controls,NCF_PREFETCH);
CLRFLAG(dapcomm->controls,NCF_PREFETCH_EAGER);
CLRFLAG(dapcomm->controls,NCF_ENCODE_PATH);
CLRFLAG(dapcomm->controls,NCF_ENCODE_QUERY);
/* Turn on any default on flags */
SETFLAG(dapcomm->controls,DFALT_ON_FLAGS);
@ -2248,6 +2252,30 @@ applyclientparamcontrols(NCDAPCOMMON* dapcomm)
else if(dapparamcheck(dapcomm,"nofillmismatch",NULL))
CLRFLAG(dapcomm->controls,NCF_FILLMISMATCH);
if((value=dapparamvalue(dapcomm,"encode")) != NULL) {
int i;
NClist* encode = nclistnew();
if(dapparamparselist(value,',',encode))
nclog(NCLOGERR,"Malformed encode parameter: %s",value);
else {
/* First, turn off all the encode flags */
CLRFLAG(dapcomm->controls,NCF_ENCODE_PATH|NCF_ENCODE_QUERY);
for(i=0;i<nclistlength(encode);i++) {
char* s = nclistremove(encode,i);
if(strcmp(s,"path")==0)
SETFLAG(dapcomm->controls,NCF_ENCODE_PATH);
else if(strcmp(s,"query")==0)
SETFLAG(dapcomm->controls,NCF_ENCODE_QUERY);
else if(strcmp(s,"all")==0)
SETFLAG(dapcomm->controls,NCF_ENCODE_PATH|NCF_ENCODE_QUERY);
else if(strcmp(s,"none")==0)
CLRFLAG(dapcomm->controls,NCF_ENCODE_PATH|NCF_ENCODE_QUERY);
}
}
nclistfree(encode);
} else { /* Set defaults */
SETFLAG(dapcomm->controls,NCF_ENCODE_QUERY);
}
nclog(NCLOGNOTE,"Caching=%d",FLAGSET(dapcomm->controls,NCF_CACHE));
}

14
libdispatch/ncuri.c
View File

@ -629,7 +629,8 @@ ncuribuild(NCURI* duri, const char* prefix, const char* suffix, int flags)
{
char* newuri = NULL;
NCbytes* buf = ncbytesnew();
const int encode = (flags&NCURIENCODE ? 1 : 0);
const int encodepath = (flags&NCURIENCODEPATH ? 1 : 0);
const int encodequery = (flags&NCURIENCODEQUERY ? 1 : 0);
if(prefix != NULL)
ncbytescat(buf,prefix);
@ -656,7 +657,7 @@ ncuribuild(NCURI* duri, const char* prefix, const char* suffix, int flags)
if((flags & NCURIPATH)) {
if(duri->path == NULL)
ncbytescat(buf,"/");
else if(encode) {
else if(encodepath) {
char* encoded = ncuriencodeonly(duri->path,pathallow);
ncbytescat(buf,encoded);
nullfree(encoded);
@ -675,8 +676,13 @@ ncuribuild(NCURI* duri, const char* prefix, const char* suffix, int flags)
ensurequerylist(duri);
if(duri->query != NULL) {
ncbytescat(buf,"?");
ncbytescat(buf,duri->query);
}
if(encodequery) {
char* encoded = ncuriencodeonly(duri->query,queryallow);
ncbytescat(buf,encoded);
nullfree(encoded);
} else
ncbytescat(buf,duri->query);
}
}
if(flags & NCURIFRAG) {
ensurefraglist(duri);

4
libsrc/putget.m4
View File

@ -644,6 +644,8 @@ putNCvx_$1_$2(NC3_INFO* ncp, const NC_var *varp,
void *xp;
void *fillp=NULL;
NC_UNUSED(fillp);
if(nelems == 0)
return NC_NOERR;
@ -847,7 +849,9 @@ getNCvx_$1_$2(const NC3_INFO* ncp, const NC_var *varp,
}
')dnl
#if 0 /*unused*/
GETNCVX(char, char)
#endif
GETNCVX(schar, schar)
GETNCVX(schar, short)

1
ncdap_test/CMakeLists.txt
View File

@ -40,6 +40,7 @@ IF(ENABLE_TESTS)
add_sh_test(ncdap tst_ber)
add_sh_test(ncdap tst_remote3)
add_sh_test(ncdap tst_zero_len_var)
add_sh_test(ncdap tst_encode)
# not yet add_sh_test(ncdap tst_hyrax)
add_sh_test(ncdap tst_fillmismatch)
IF(ENABLE_DAP_LONG_TESTS)

2
ncdap_test/Makefile.am
View File

@ -45,7 +45,7 @@ endif
if ENABLE_DAP_REMOTE_TESTS
if BUILD_UTILITIES
TESTS += tst_ber.sh tst_remote3.sh tst_formatx.sh testurl.sh tst_fillmismatch.sh tst_zero_len_var.sh
TESTS += tst_ber.sh tst_remote3.sh tst_formatx.sh testurl.sh tst_fillmismatch.sh tst_zero_len_var.sh tst_encode.sh
endif
TESTS += test_partvar

14
ncdap_test/tst_encode.sh Normal file
View File

@ -0,0 +1,14 @@
#!/bin/sh
# Dennis Heimbigner
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
echo ""
echo "*** Testing #encode=" mechanism
${NCDUMP} -h 'http://opendap2.oceanbrowser.net/thredds/dodsC/data/emodnet1-domains/tmp%20test.nc?lon[0:8]#encode=none'
${NCDUMP} -h 'http://iridl.ldeo.columbia.edu/SOURCES/.Indices/.soi/.c8110/.anomaly/T/%28Jan%201979%29/VALUE/dods?anomaly[0]'

4
oc2/oc.c
View File

@ -1804,12 +1804,12 @@ oc_raw_xdrsize(OCobject link, OCobject ddsroot, off_t* xdrsizep)
/* Resend a url as a head request to check the Last-Modified time */
OCerror
oc_update_lastmodified_data(OCobject link)
oc_update_lastmodified_data(OCobject link, OCflags flags)
{
OCstate* state;
OCVERIFY(OC_State,link);
OCDEREF(OCstate*,state,link);
return OCTHROW(ocupdatelastmodifieddata(state));
return OCTHROW(ocupdatelastmodifieddata(state,flags));
}
long

13
oc2/oc.h
View File

@ -51,9 +51,18 @@ typedef int OCflags;
/*!\def OCONDISK
Cause oc_fetch to store the retrieved data on disk.
*/
#define OCONDISK 1
/*!\def OCENCODEPATH
Cause oc_fetch to encode path part of a URL
*/
#define OCENCODEPATH 2
/*!\def OCENCODEQUERY
Cause oc_fetch to encode query part of a URL
*/
#define OCENCODEQUERY 4
/**************************************************/
/* OCtype */
@ -583,7 +592,7 @@ EXTERNL OCerror oc_set_curlopt(OClink link, const char* option, void* value);
EXTERNL OCerror oc_get_connection(OCobject ocnode, OCobject* linkp);
/* Resend a url as a head request to check the Last-Modified time */
EXTERNL OCerror oc_update_lastmodified_data(OClink);
EXTERNL OCerror oc_update_lastmodified_data(OClink,OCflags);
/* Get last known modification time; -1 => data unknown */
EXTERNL long oc_get_lastmodified_data(OClink);

11
oc2/ocinternal.c
View File

@ -180,14 +180,14 @@ ocfetch(OCstate* state, const char* constraint, OCdxd kind, OCflags flags,
switch (kind) {
case OCDAS:
stat = readDAS(state,tree);
stat = readDAS(state,tree,flags);
if(stat == OC_NOERR) {
tree->text = ncbytesdup(state->packet);
if(tree->text == NULL) stat = OC_EDAS;
}
break;
case OCDDS:
stat = readDDS(state,tree);
stat = readDDS(state,tree,flags);
if(stat == OC_NOERR) {
tree->text = ncbytesdup(state->packet);
if(tree->text == NULL) stat = OC_EDDS;
@ -460,12 +460,15 @@ fprintf(stderr,"missing bod: ddslen=%lu bod=%lu\n",
}
OCerror
ocupdatelastmodifieddata(OCstate* state)
ocupdatelastmodifieddata(OCstate* state, OCflags ocflags)
{
OCerror status = OC_NOERR;
long lastmodified;
char* base = NULL;
base = ncuribuild(state->uri,NULL,NULL,NCURIENCODE);
int flags = 0;
if(ocflags & OCENCODEPATH) flags |= NCURIENCODEPATH;
if(ocflags & OCENCODEQUERY) flags |= NCURIENCODEQUERY;
base = ncuribuild(state->uri,NULL,NULL,flags);
status = ocfetchlastmodified(state->curl, base, &lastmodified);
free(base);
if(status == OC_NOERR) {

2
oc2/ocinternal.h
View File

@ -229,7 +229,7 @@ extern int oc_network_order;
extern int oc_invert_xdr_double;
extern OCerror ocinternalinitialize(void);
extern OCerror ocupdatelastmodifieddata(OCstate* state);
extern OCerror ocupdatelastmodifieddata(OCstate* state, OCflags);
extern OCerror ocset_useragent(OCstate* state, const char* agent);
extern OCerror ocset_netrc(OCstate* state, const char* path);

26
oc2/ocread.c
View File

@ -29,12 +29,12 @@
#include "ncpathmgr.h"
/*Forward*/
static int readpacket(OCstate* state, NCURI*, NCbytes*, OCdxd, long*);
static int readpacket(OCstate* state, NCURI*, NCbytes*, OCdxd, OCflags, long*);
static int readfile(const char* path, const char* suffix, NCbytes* packet);
static int readfiletofile(const char* path, const char* suffix, FILE* stream, off_t*);
int
readDDS(OCstate* state, OCtree* tree)
readDDS(OCstate* state, OCtree* tree, OCflags flags)
{
int stat = OC_NOERR;
long lastmodified = -1;
@ -44,7 +44,7 @@ readDDS(OCstate* state, OCtree* tree)
#ifdef OCDEBUG
fprintf(stderr,"readDDS:\n");
#endif
stat = readpacket(state,state->uri,state->packet,OCDDS,
stat = readpacket(state,state->uri,state->packet,OCDDS, flags,
&lastmodified);
if(stat == OC_NOERR) state->ddslastmodified = lastmodified;
@ -52,7 +52,7 @@ fprintf(stderr,"readDDS:\n");
}
int
readDAS(OCstate* state, OCtree* tree)
readDAS(OCstate* state, OCtree* tree, OCflags flags)
{
int stat = OC_NOERR;
@ -60,7 +60,7 @@ readDAS(OCstate* state, OCtree* tree)
#ifdef OCDEBUG
fprintf(stderr,"readDAS:\n");
#endif
stat = readpacket(state,state->uri,state->packet,OCDAS,NULL);
stat = readpacket(state,state->uri,state->packet,OCDAS,flags,NULL);
return stat;
}
@ -86,7 +86,7 @@ ocdxdextension(OCdxd dxd)
}
static int
readpacket(OCstate* state, NCURI* url,NCbytes* packet,OCdxd dxd,long* lastmodified)
readpacket(OCstate* state, NCURI* url, NCbytes* packet, OCdxd dxd, OCflags ocflags, long* lastmodified)
{
int stat = OC_NOERR;
int fileprotocol = 0;
@ -102,8 +102,9 @@ readpacket(OCstate* state, NCURI* url,NCbytes* packet,OCdxd dxd,long* lastmodifi
stat = readfile(fetchurl,suffix,packet);
} else {
int flags = NCURIBASE;
if(ocflags & OCENCODEPATH)flags |= NCURIENCODEPATH;
if(ocflags & OCENCODEQUERY) flags |= NCURIENCODEQUERY;
if(!fileprotocol) flags |= NCURIQUERY;
flags |= NCURIENCODE;
fetchurl = ncuribuild(url,NULL,suffix,flags);
MEMCHECK(fetchurl,OC_ENOMEM);
if(ocdebug > 0)
@ -125,7 +126,7 @@ fprintf(stderr,"readpacket: packet.size=%lu\n",
}
int
readDATADDS(OCstate* state, OCtree* tree, OCflags flags)
readDATADDS(OCstate* state, OCtree* tree, OCflags ocflags)
{
int stat = OC_NOERR;
long lastmod = -1;
@ -133,9 +134,9 @@ readDATADDS(OCstate* state, OCtree* tree, OCflags flags)
#ifdef OCDEBUG
fprintf(stderr,"readDATADDS:\n");
#endif
if((flags & OCONDISK) == 0) {
if((ocflags & OCONDISK) == 0) {
ncurisetquery(state->uri,tree->constraint);
stat = readpacket(state,state->uri,state->packet,OCDATADDS,&lastmod);
stat = readpacket(state,state->uri,state->packet,OCDATADDS,ocflags,&lastmod);
if(stat == OC_NOERR)
state->datalastmodified = lastmod;
tree->data.datasize = ncbyteslength(state->packet);
@ -151,8 +152,11 @@ fprintf(stderr,"readDATADDS:\n");
stat = readfiletofile(readurl, ".dods", tree->data.file, &tree->data.datasize);
} else {
int flags = NCURIBASE;
if(ocflags & OCENCODEPATH)
flags |= NCURIENCODEPATH;
if(ocflags & OCENCODEQUERY)
flags |= NCURIENCODEQUERY;
if(!fileprotocol) flags |= NCURIQUERY;
flags |= NCURIENCODE;
ncurisetquery(url,tree->constraint);
readurl = ncuribuild(url,NULL,".dods",flags);
MEMCHECK(readurl,OC_ENOMEM);

6
oc2/ocread.h
View File

@ -5,9 +5,9 @@
#define READ_H
extern int readDDS(OCstate*, OCtree*);
extern int readDAS(OCstate*, OCtree*);
extern int readDDS(OCstate*, OCtree*, OCflags);
extern int readDAS(OCstate*, OCtree*, OCflags);
extern int readDATADDS(OCstate*, OCtree*, int inmemory);
extern int readDATADDS(OCstate*, OCtree*, OCflags);
#endif /*READ_H*/