mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-19 17:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix a bug in libdap2 handling of strings

Browse Source
This commit is contained in:
Dennis Heimbigner 2012-01-30 04:32:12 +00:00
parent 8837052b72
commit 1e1907e393
13 changed files with 127 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cf
View File

@ -1,8 +1,8 @@
#!/bin/bash
#X="-x"
HDF5=1
#DAP=1
#HDF5=1
DAP=1
#CDMR=1
#RPC=1
@ -15,7 +15,7 @@ cmds=""
#cmds="all"
#cmds="all check"
#cmds="all dist"
cmds="all distcheck"
#cmds="all distcheck"
# Default cases
PREFIX="/tmp/install/${HOST}"

13
libdap2/cache.c
View File

@ -213,21 +213,18 @@ buildcachenode34(NCDAPCOMMON* nccomm,
NCcachenode* cachenode = NULL;
char* ce = NULL;
#ifdef IGNORE
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
/* If the cache flag is on, then cache
forces whole variable projections */
int i,j;
int i;
/* Remove the slicing (if any) */
for(i=0;i<nclistlength(constraint->projections);i++) {
DCEprojection* p = (DCEprojection*)nclistget(constraint->projections,i);
if(p->discrim != CES_VAR || p->var == NULL || p->var->segments == NULL)
continue;
for(j=0;j<nclistlength(p->var->segments);j++) {
DCEsegment* seg = (DCEsegment*)nclistget(p->var->segments,j);
seg->rank = 0;
}
}
dcemakewholeprojection(p);
}
}
#endif
ce = buildconstraintstring3(constraint);
ocstat = dap_fetch(nccomm,conn,ce,OCDATADDS,&ocroot);

4
libdap2/constraints3.c
View File

@ -950,7 +950,7 @@ dapshiftprojection(DCEprojection* projection)
NClist* segments;
#ifdef DEBUG1
fprintf(stderr,"dapwalkprojection.before: %s\n",dumpprojection(projection));
fprintf(stderr,"dapshiftprojection.before: %s\n",dumpprojection(projection));
#endif
ASSERT(projection->discrim == CES_VAR);
@ -964,7 +964,7 @@ fprintf(stderr,"dapwalkprojection.before: %s\n",dumpprojection(projection));
}
#ifdef DEBUG1
fprintf(stderr,"dapwalkprojection.after: %s\n",dumpprojection(projection));
fprintf(stderr,"dapshiftprojection.after: %s\n",dumpprojection(projection));
#endif
return ncstat;

12
libdap2/dapattr3.c
View File

@ -10,7 +10,7 @@
/* Forward */
static NCerror buildattribute(char*,nc_type,NClist*,NCattribute**);
static int mergedas1(NCDAPCOMMON*, OCconnection, CDFnode* dds, OCobject das, int isDODS);
static int mergedas1(NCDAPCOMMON*, OCconnection, CDFnode* dds, OCobject das);
static int isglobalname3(char* name);
#ifdef IGNORE
@ -231,7 +231,6 @@ loop:
OCobject das = (OCobject)nclistget(dasnodes,i);
char* ocfullname;
char* ocbasename;
int isDODS = 0;
if(das == OCNULL) continue;
OCHECK(oc_inq_name(conn,das,&ocbasename));
@ -241,7 +240,6 @@ loop:
if(container == OCNULL) {
ASSERT(container != OCNULL);
}
isDODS = 1;
ocfullname = makeocpathstring3(conn,container,".");
} else {
ocfullname = makeocpathstring3(conn,das,".");
@ -252,7 +250,7 @@ loop:
if(strcmp(ocfullname,ddsfullname)==0
|| strcmp(ocbasename,ddsfullname)==0
|| strcmp(ocbasename,dds->ocname)==0) {
mergedas1(nccomm,conn,dds,das,isDODS);
mergedas1(nccomm,conn,dds,das);
/* remove from dasnodes list*/
nclistset(dasnodes,i,(ncelem)NULL);
}
@ -265,13 +263,13 @@ loop:
/* 4. Assign globals */
for(i=0;i<nclistlength(dasglobals);i++) {
OCobject das = (OCobject)nclistget(dasglobals,i);
mergedas1(nccomm,conn,ddsroot,das,0);
mergedas1(nccomm,conn,ddsroot,das);
}
/* 5. Assign DOD_EXTRA */
for(i=0;i<nclistlength(dodsextra);i++) {
OCobject das = (OCobject)nclistget(dodsextra,i);
mergedas1(nccomm,conn,ddsroot,das,1);
mergedas1(nccomm,conn,ddsroot,das);
}
done: /* cleanup*/
@ -285,7 +283,7 @@ done: /* cleanup*/
}
static int
mergedas1(NCDAPCOMMON* nccomm, OCconnection conn, CDFnode* dds, OCobject das, int isDODS)
mergedas1(NCDAPCOMMON* nccomm, OCconnection conn, CDFnode* dds, OCobject das)
{
NCerror ncstat = NC_NOERR;
OCerror ocstat = OC_NOERR;

14
libdap2/dceconstraints.c
View File

@ -135,6 +135,7 @@ dceslicemerge(DCEslice* dst, DCEslice* src)
tmp.length = (((src->length - 1) / src->stride) * tmp.stride) + 1;
tmp.stop = tmp.first + tmp.length;
tmp.count = tmp.length / tmp.stride;
/* use max declsize */
if(dst->declsize > src->declsize) {
tmp.declsize = dst->declsize;
} else {
@ -852,6 +853,19 @@ dceiswholevar(DCEvar* var)
return whole;
}
void
dcemakewholeprojection(DCEprojection* p)
{
int i;
/* Remove the slicing (if any) */
if(p->discrim == CES_VAR && p->var != NULL && p->var->segments != NULL) {
for(i=0;i<nclistlength(p->var->segments);i++) {
DCEsegment* seg = (DCEsegment*)nclistget(p->var->segments,i);
seg->rank = 0;
}
}
}
int
dceiswholeprojection(DCEprojection* p)
{

1
libdap2/dceconstraints.h
View File

@ -108,6 +108,7 @@ extern NClist* dceallnodes(DCEnode* node, CEsort which);
extern DCEnode* dcecreate(CEsort sort);
extern void dcemakewholeslice(DCEslice* slice, size_t declsize);
extern void dcemakewholeprojection(DCEprojection*);
extern int dceiswholesegment(DCEsegment*);
extern int dceiswholeslice(DCEslice*);

9
libdap2/env
View File

@ -10,8 +10,9 @@ PARMS="${PARMS}[cache]"
#PARMS="${PARMS}[noprefetch]"
PARMS="${PARMS}[show=fetch]"
F="http://motherlode.ucar.edu:8080/dts/test.02"
CON="b[1:2:10]"
F="http://dapper.pmel.noaa.gov/dapper/argo/argo_all.cdp"
CON="location.LATITUDE,location.attributes.DIRECTION&location.LATITUDE<=1&location.LATITUDE>-1"
#VAR="person.name"
PROG="./ncd"
@ -33,7 +34,11 @@ alias qh="$PROG -h $ARGS '$U'"
alias qqh="gdb --args $PROG -h $ARGS '$U'"
alias qall="$PROG -h $ARGS '${UALL}'"
alias qv="valgrind $VARGS $PROG $ARGS '$U'"
if test "x$CON" = "x" ; then
alias qo="~/svn/oc/octest -p dds -p datadds -DN -DX1 $F"
else
alias qo="~/svn/oc/octest -p dds -p datadds -DN -DX1 -C $CON $F"
fi
if test 1 = 0; then
F="http://nomads.ncep.noaa.gov:9090/dods/gfs_hd/gfs_hd20110908/gfs_hd_00z"

100
libdap2/getvara3.c
View File

@ -29,7 +29,7 @@ static NCerror movetor(NCDAPCOMMON*, OCdata currentcontent,
static int findfield(CDFnode* node, CDFnode* subnode);
static int wholeslicepoint(Dapodometer* odom);
static NCerror removepseudodims(DCEprojection* proj);
static NCerror removesequencedims(DCEprojection* proj);
static int extract(NCDAPCOMMON*, Getvara*, CDFnode*, DCEsegment*, OClink, OCdata, struct NCMEMORY*);
static int extractstring(NCDAPCOMMON*, Getvara*, CDFnode*, DCEsegment*, OClink, OCdata, struct NCMEMORY*);
@ -48,27 +48,28 @@ The vara constraint is the one formed from the arguments
(start, count, stride) provided to the call to nc_get_vara().
There are some exceptions to the formation of the fetch constraint.
a. If the target variable (as specified in nc_get_vara())
is already in the cache and the whole variable is there,
then we will not do any fetch at all. This will occur for,
for example, pre-fetched variables.
b. If the url is unconstrainable (e.g. file://...), then
we do not use any constraint and fetch the whole dataset.
In all cases, the fetch constraint will use any URL selections,
but will use different fetch projections.
a. URL is unconstrainable (e.g. file://...):
fetchprojection = null => fetch whole dataset
b. The target variable (as specified in nc_get_vara())
is already in the cache => the whole variable is there (may change later)
fetchprojection = N.A. since variable is in the cache
c. Otherwise:
fetchprojection = unsliced vara variable => fetch whole variable
2. At this point, the target variable is available in the
cache, either whole (cases 1a and 1b), or the constrained
fetched part is in the cache.
2. At this point, whole target variable is available in the cache.
3. We build a projection to walk (guide) the use of the oc
data procedures in extract the required data from the cache.
In all cases:
walkprojection = merge(urlprojection,varaprojection)
If the whole target variable is available in the cache,
(case 1a or 1b) then we need to walk using the
merge of the url constraint and the vara constraint.
If the partial target variable is in the cache, then that
means we used the merged constraints to fetch the data and
we need only extract the complete contents of the cache.
#ifdef FUTURE
An eventual optimization involves caching a part of the
variable of interest using the merge of the
url constraints and the getvar constraint.
This means we need only extract the complete contents of the cache.
Notice that this will not necessarily be a direct memory to
memory copy because the dap encoding still needs to be
interpreted. For this case, we derive a walk projection
@ -76,7 +77,9 @@ from the merged projection that will properly access
the cached data. This walk projection
shifts the merged projection so all slices
start at 0 and have a stride of 1.
*/
#endif
*/
NCerror
nc3d_getvarx(int ncid, int varid,
@ -250,30 +253,53 @@ fprintf(stderr,"var is in cache\n");
fetchprojection = NULL;
walkprojection = NULL;
/* Create walkprojection as the merge of the url projections
and the vara projection */
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&walkprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* define the var list of interest */
vars = nclistnew();
nclistpush(vars,(ncelem)varainfo->target);
retry:
switch (state) {
case FETCHPART: {
/* Create a merge of the url projections and the vara projection */
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&fetchprojection);
#ifdef FUTURE
/* Create fetch projection as the merge of the url projections
and the vara projection */
ncstat = daprestrictprojection(dapcomm->oc.dapconstraint->projections,
varaprojection,&fetchprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* Clone the projection for use with the walk */
walkprojection = (DCEprojection*)dceclone((DCEnode*)fetchprojection);
/* Shift the varaprojection for simple walk */
walkprojection = (DCEprojection*)dceclone((DCEnode*)varaprojection);
dapshiftprojection(walkprojection);
#else /*!FUTURE*/
/* Build a whole variable projection */
fetchprojection = (DCEprojection*)dceclone((DCEnode*)varaprojection);
dcemakewholeprojection(fetchprojection);
#endif /*FUTURE*/
/* elide any sequence dimensions (dap servers do not allow such). */
ncstat = removepseudodims(fetchprojection);
ncstat = removesequencedims(fetchprojection);
if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto fail;}
/* After the removal, we might end up with a whole variable */
/* occurrence of sequence node automatically makes it not a whole variable */
/* Occurrence of sequence node automatically makes it not a whole variable */
if(!dapinsequence((CDFnode*)fetchprojection->var->annotation)) {
#ifdef IGNORE
if(dceiswholeprojection(fetchprojection)) {
state = FETCHWHOLE;
goto retry;
goto fetchwhole;
}
#else
dcemakewholeprojection(fetchprojection);
#endif
}
#ifdef DEBUG
fprintf(stderr,"getvarx: walkprojection: |%s|\n",dumpprojection(walkprojection));
fprintf(stderr,"getvarx: fetchprojection: |%s|\n",dumpprojection(fetchprojection));
#endif
/* Build the complete constraint to use in the fetch */
fetchconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT);
/* merged constraint just uses the url constraint selection */
@ -284,7 +310,7 @@ retry:
fprintf(stderr,"getvarx: fetchconstraint: %s\n",dumpconstraint(fetchconstraint));
#endif
/* buildcachenode3 will create a new cachenode and
will also fetch the corresponding datadds;
will also fetch the corresponding datadds.
*/
ncstat = buildcachenode34(dapcomm,fetchconstraint,vars,&cachenode,0);
fetchconstraint = NULL; /*buildcachenode34 takes control of fetchconstraint.*/
@ -308,7 +334,6 @@ fprintf(stderr,"getvarx: fetchconstraint: %s\n",dumpconstraint(fetchconstraint))
} break;
case CACHED: {
/* nothing to do since cache node already exists */
} break;
default: PANIC1("unknown fetch state: %d\n",state);
@ -325,6 +350,8 @@ fprintf(stderr,"cache.datadds=%s\n",dumptree(cachenode->datadds));
ncstat = attachsubset34(cachenode->datadds,dapcomm->cdf.ddsroot);
if(ncstat) goto fail;
#ifdef IGNORE
this was handled above */
/* Now, walk to the relevant instance */
switch (state) {
@ -342,9 +369,11 @@ fprintf(stderr,"cache.datadds=%s\n",dumptree(cachenode->datadds));
break;
default: PANIC("illegal getvarx state");
}
#endif
#ifdef DEBUG
fprintf(stderr,"getvarx: walkprojection: %s\n",dumpprojection(walkprojection));
fprintf(stderr,"getvarx: final fetchprojection: %s\n",dumpprojection(fetchprojection));
fprintf(stderr,"getvarx: final walkprojection: %s\n",dumpprojection(walkprojection));
#endif
/* Fix up varainfo to use the cache */
@ -374,23 +403,22 @@ ok:
return THROW(ncstat);
}
/* Remove any sequence dimensions */
static NCerror
removepseudodims(DCEprojection* proj)
removesequencedims(DCEprojection* proj)
{
int i;
#ifdef DEBUG1
fprintf(stderr,"removepseudodims.before: %s\n",dumpprojection(proj));
fprintf(stderr,"removesequencedims.before: %s\n",dumpprojection(proj));
#endif
for(i=0;i<nclistlength(proj->var->segments);i++) {
DCEsegment* seg = (DCEsegment*)nclistget(proj->var->segments,i);
CDFnode* cdfnode = (CDFnode*)seg->annotation;
if(cdfnode->array.seqdim != NULL)
seg->rank = 0;
else if(cdfnode->array.stringdim != NULL)
seg->rank--;
}
#ifdef DEBUG1
fprintf(stderr,"removepseudodims.after: %s\n",dumpprojection(proj));
fprintf(stderr,"removesequencedims.after: %s\n",dumpprojection(proj));
#endif
return NC_NOERR;
}

22
libdap2/ncdap3a.c
View File

@ -97,26 +97,27 @@ addstringdims(NCDAPCOMMON* dapcomm)
*/
int i;
NClist* varnodes = dapcomm->cdf.varnodes;
CDFnode* sdim = NULL;
CDFnode* globalsdim = NULL;
char dimname[4096];
size_t dimsize;
/* Start by creating the global string dimension */
snprintf(dimname,sizeof(dimname),"maxStrlen%lu",
(unsigned long)dapcomm->cdf.defaultstringlength);
sdim = makecdfnode34(dapcomm, dimname, OC_Dimension, OCNULL,
globalsdim = makecdfnode34(dapcomm, dimname, OC_Dimension, OCNULL,
dapcomm->cdf.ddsroot);
nclistpush(dapcomm->cdf.ddsroot->tree->nodes,(ncelem)sdim);
sdim->dim.dimflags |= CDFDIMSTRING;
sdim->dim.declsize = dapcomm->cdf.defaultstringlength;
sdim->dim.declsize0 = sdim->dim.declsize;
sdim->dim.array = dapcomm->cdf.ddsroot;
sdim->ncbasename = cdflegalname3(dimname);
sdim->ncfullname = nulldup(sdim->ncbasename);
dapcomm->cdf.globalstringdim = sdim;
nclistpush(dapcomm->cdf.ddsroot->tree->nodes,(ncelem)globalsdim);
globalsdim->dim.dimflags |= CDFDIMSTRING;
globalsdim->dim.declsize = dapcomm->cdf.defaultstringlength;
globalsdim->dim.declsize0 = globalsdim->dim.declsize;
globalsdim->dim.array = dapcomm->cdf.ddsroot;
globalsdim->ncbasename = cdflegalname3(dimname);
globalsdim->ncfullname = nulldup(globalsdim->ncbasename);
dapcomm->cdf.globalstringdim = globalsdim;
for(i=0;i<nclistlength(varnodes);i++) {
CDFnode* var = (CDFnode*)nclistget(varnodes,i);
CDFnode* sdim = NULL;
/* Does this node need a string dim? */
if(var->etype != NC_STRING && var->etype != NC_URL) continue;
@ -143,6 +144,7 @@ addstringdims(NCDAPCOMMON* dapcomm)
nclistpush(dapcomm->cdf.ddsroot->tree->nodes,(ncelem)sdim);
sdim->dim.dimflags |= CDFDIMSTRING;
sdim->dim.declsize = dimsize;
sdim->dim.declsize0 = dimsize;
sdim->dim.array = var;
sdim->ncbasename = cdflegalname3(sdim->ocname);
sdim->ncfullname = nulldup(sdim->ncbasename);

8
ncdap_test/tst_remote.sh
View File

@ -21,7 +21,7 @@ longtests="$5"
if test "x$timing" = "x1" ; then leakcheck=0; fi
# get the list of test files
WHICHTESTS="S1 C1 C2"
WHICHTESTS="S1 C1 C2 C4"
if test -n "$longtests"; then
WHICHTESTS="${WHICHTESTS} L1 LC1"
fi
@ -123,6 +123,12 @@ REMOTETESTSC3="\
argo_all.cdp;1;&location.LATITUDE<1&location.LATITUDE>-1\
"
# Test string access
REMOTEURLC4="http://motherlode.ucar.edu:8080/thredds/dodsC/station/metar"
REMOTETESTSC4="\
Surface_METAR_20120101_0000.nc;1;weather[0:10]\
"
# Constrained long tests
REMOTEURLLC1="http://motherlode.ucar.edu:8080/dts"
REMOTETESTSLC1="\

2
ncdump/ncdump.c
View File

@ -2026,6 +2026,7 @@ grp_matches(int ncid) {
return total;
}
#ifdef USE_DAP
#define DAP_CLIENT_CACHE_DIRECTIVE "[cache]"
/* replace path string with same string prefixed by
* DAP_CLIENT_NCDUMP_DIRECTIVE */
@ -2042,6 +2043,7 @@ void adapt_url_for_cache(char **pathp) {
*pathp = path;
return;
}
#endif
/**
The ncdump tool generates the CDL text representation of a netCDF

1
oc/dapparse.c
View File

@ -174,6 +174,7 @@ dap_attrset(DAPparsestate* state, Object name, Object attributes)
/* Check var set vs global set */
attset->att.isglobal = isglobalname(name);
attset->subnodes = (OClist*)attributes;
addedges(attset);
return attset;
}

6
oc/ocdebug.h
View File

@ -4,8 +4,10 @@
#ifndef OCOCDBG_H
#define OCOCDBG_H
#undef OCDEBUG
#undef DAPDEBUG
#if 0
#define OCDEBUG
#define DAPDEBUG 1
#endif
#ifdef OCDEBUG
#define OCVERBOSE