mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-02-17 16:50:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

It turns out that my dap code

Browse Source 
is not properly checking the incoming
count argument for nc_get_vars.
It is treating the count as pre-stride
rather than post-stride.
This commit is contained in:
Dennis Heimbigner 2013-05-01 22:31:01 +00:00
parent 5944c71c51
commit 19c1c11f1d
6 changed files with 215 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libdap2/Make0
View File

@ -258,7 +258,7 @@ fix::
done
##################################################
T=test_nstride_cached
T=test_nstride2
v::
cc -g -c ${T}.c ${INCL}

5
libdap2/cache.c
View File

@ -97,7 +97,7 @@ prefetchdata3(NCDAPCOMMON* nccomm)
if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
/* If we cannot constrain and caching is enabled,
then pull in everything */
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
if(FLAGSET(nccomm->controls,NCF_CACHE)) {
for(i=0;i<nclistlength(allvars);i++) {
nclistpush(vars,nclistget(allvars,i));
}
@ -392,7 +392,8 @@ markprefetch3(NCDAPCOMMON* nccomm)
CDFnode* dim = (CDFnode*)nclistget(var->array.dimsettrans,j);
nelems *= dim->dim.declsize;
}
if(nelems <= nccomm->cdf.smallsizelimit) {
if(nelems <= nccomm->cdf.smallsizelimit
&& FLAGSET(nccomm->controls,NCF_PREFETCH)) {
var->prefetchable = 1;
if(SHOWFETCH)
{

7
libdap2/env
View File

@ -10,8 +10,7 @@ PROG="$TOP/ncdump/ncdump"
P=`pwd`
F="http://opendap.co-ops.nos.noaa.gov/dods/IOOS/SixMin_Verified_Water_Level"
CON="&WATERLEVEL_6MIN_VFD_PX._STATION_ID=%228779770%22&WATERLEVEL_6MIN_VFD_PX._DATUM=%22MSL%22&WATERLEVEL_6MIN_VFD_PX._BEGIN_DATE=%2220061001%22&WATERLEVEL_6MIN_VFD_PX._END_DATE=%2220061030%22"
F=
PARMS="[log]"
#PARMS="${PARMS}[netcdf3]"
@ -107,4 +106,8 @@ CON="U50M_EOSGRID_Data_Fields[0:23][282:282][441:441]"
#CON="OneD.amp,TwoD.amp,ThreeD.amp"
F="http://goldsmr2.sci.gsfc.nasa.gov/opendap/hyrax/MERRA/MAT1NXSLV.5.2.0/1990/01/MERRA100.prod.assim.tavg1_2d_slv_Nx.19900101.hdf"
CON="U50M_EOSGRID_Data_Fields[0:23][282:282][441:441],XDim_EOSGRID"
F="http://opendap.co-ops.nos.noaa.gov/dods/IOOS/SixMin_Verified_Water_Level"
CON="&WATERLEVEL_6MIN_VFD_PX._STATION_ID=%228779770%22&WATERLEVEL_6MIN_VFD_PX._DATUM=%22MSL%22&WATERLEVEL_6MIN_VFD_PX._BEGIN_DATE=%2220061001%22&WATERLEVEL_6MIN_VFD_PX._END_DATE=%2220061030%22"
fi

388
libdap2/getvara3.c
View File

@ -186,8 +186,12 @@ fprintf(stderr,"\n");
/* Validate the dimension sizes */
for(i=0;i<ncrank;i++) {
CDFnode* dim = (CDFnode*)nclistget(ncdimsall,i);
if(startp[i] > dim->dim.declsize
|| startp[i]+countp[i] > dim->dim.declsize) {
if(startp[i] < 0 || countp[i] < 0 || stridep[i] < 1) {
ncstat = NC_EINVALCOORDS;
goto fail;
}
if(startp[i] >= dim->dim.declsize
|| startp[i]+(stridep[i]*(countp[i]-1)) >= dim->dim.declsize) {
ncstat = NC_EINVALCOORDS;
goto fail;
}
@ -694,194 +698,6 @@ findfield(CDFnode* node, CDFnode* field)
return -1;
}
#ifdef EXTERN_UNUSED
int
nc3d_getvarmx(int ncid, int varid,
const size_t *start,
const size_t *edges,
const ptrdiff_t* stride,
const ptrdiff_t* map,
void* data,
nc_type dsttype0)
{
NCerror ncstat = NC_NOERR;
int i;
NC* drno;
NC* substrate;
NC3_INFO* substrate3;
NCDAPCOMMON* dapcomm;
NC_var* var;
CDFnode* cdfvar; /* cdf node mapping to var*/
NClist* varnodes;
nc_type dsttype;
size_t externsize;
size_t dimsizes[NC_MAX_VAR_DIMS];
Dapodometer* odom = NULL;
unsigned int ncrank;
NClist* ncdims = NULL;
size_t nelems;
#ifdef NEWVARM
char* localcopy; /* of whole variable */
#endif
ncstat = NC_check_id(ncid, (NC**)&drno);
if(ncstat != NC_NOERR) goto done;
dapcomm = (NCDAPCOMMON*)drno->dispatchdata;
ncstat = NC_check_id(drno->substrate, &substrate);
if(ncstat != NC_NOERR) goto done;
substrate3 = (NC3_INFO*)drno->dispatchdata;
var = NC_lookupvar(substrate3,varid);
if(var == NULL) {ncstat = NC_ENOTVAR; goto done;}
/* Locate var node via varid */
varnodes = dapcomm->cdf.ddsroot->tree->varnodes;
for(i=0;i<nclistlength(varnodes);i++) {
CDFnode* node = (CDFnode*)nclistget(varnodes,i);
if(node->array.basevar == NULL
&& node->nctype == NC_Atomic
&& node->ncid == varid) {
cdfvar = node;
break;
}
}
ASSERT((cdfvar != NULL));
ASSERT((strcmp(cdfvar->ncfullname,var->name->cp)==0));
if(nclistlength(cdfvar->array.dimsetplus) == 0) {
/* The variable is a scalar; consequently, there is only one
thing to get and only one place to put it. */
/* recurse with additional parameters */
return THROW(nc3d_getvarx(ncid,varid,
NULL,NULL,NULL,
data,dsttype0));
}
dsttype = (dsttype0);
/* Default to using the inquiry type for this var*/
if(dsttype == NC_NAT) dsttype = cdfvar->externaltype;
/* Validate any implied type conversion*/
if(cdfvar->etype != dsttype && dsttype == NC_CHAR) {
/* The only disallowed conversion is to/from char and non-byte
numeric types*/
switch (cdfvar->etype) {
case NC_STRING: case NC_URL:
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
break;
default:
return THROW(NC_ECHAR);
}
}
externsize = nctypesizeof(dsttype);
/* Accumulate the dimension sizes and the total # of elements */
ncdims = cdfvar->array.dimsetall;
ncrank = nclistlength(ncdims);
nelems = 1; /* also Compute the number of elements being retrieved */
for(i=0;i<ncrank;i++) {
CDFnode* dim = (CDFnode*)nclistget(ncdims,i);
dimsizes[i] = dim->dim.declsize;
nelems *= edges[i];
}
/* Originally, this code repeatedly extracted single values
using get_var1. In an attempt to improve performance,
I have converted to reading the whole variable at once
and walking it locally.
*/
#ifdef NEWVARM
localcopy = (char*)malloc(nelems*externsize);
/* We need to use the varieties of get_vars in order to
properly do conversion to the external type
*/
switch (dsttype) {
case NC_CHAR:
ncstat = nc_get_vars_text(ncid,varid,start, edges, stride,
(char*)localcopy);
break;
case NC_BYTE:
ncstat = nc_get_vars_schar(ncid,varid,start, edges, stride,
(signed char*)localcopy);
break;
case NC_SHORT:
ncstat = nc_get_vars_short(ncid,varid, start, edges, stride,
(short*)localcopy);
break;
case NC_INT:
ncstat = nc_get_vars_int(ncid,varid,start, edges, stride,
(int*)localcopy);
break;
case NC_FLOAT:
ncstat = nc_get_vars_float(ncid,varid,start, edges, stride,
(float*)localcopy);
break;
case NC_DOUBLE:
ncstat = nc_get_vars_double(ncid,varid, start, edges, stride,
(double*)localcopy);
break;
default: break;
}
odom = dapodom_new(ncrank,start,edges,stride,NULL);
/* Walk the local copy */
for(i=0;i<nelems;i++) {
size_t voffset = dapodom_varmcount(odom,map,dimsizes);
void* dataoffset = (void*)(((char*)data) + (externsize*voffset));
char* localpos = (localcopy + externsize*i);
/* extract the indexset'th value from local copy */
memcpy(dataoffset,(void*)localpos,externsize);
/*
fprintf(stderr,"new: %lu -> %lu %f\n",
(unsigned long)(i),
(unsigned long)voffset,
*(float*)localpos);
*/
dapodom_next(odom);
}
#else
odom = dapodom_new(ncrank,start,edges,stride,NULL);
while(dapodom_more(odom)) {
size_t* indexset = odom->index;
size_t voffset = dapodom_varmcount(odom,map,dimsizes);
char internalmem[128];
char externalmem[128];
void* dataoffset = (void*)(((char*)data) + (externsize*voffset));
/* get the indexset'th value using variable's internal type */
ncstat = nc_get_var1(ncid,varid,indexset,(void*)&internalmem);
if(ncstat != NC_NOERR) goto done;
/* Convert to external type */
ncstat = dapconvert3(cdfvar->etype,dsttype,externalmem,internalmem);
if(ncstat != NC_NOERR) goto done;
memcpy(dataoffset,(void*)externalmem,externsize);
/*
fprintf(stderr,"old: %lu -> %lu %f\n",
(unsigned long)dapodom_count(odom),
(unsigned long)voffset,
*(float*)externalmem);
*/
dapodom_next(odom);
}
#endif
done:
return ncstat;
}
#endif /*EXTERN_UNUSED*/
static int
conversionrequired(nc_type t1, nc_type t2)
{
@ -1127,7 +943,8 @@ extractstring(
while(dapodom_more(odom)) {
char* value = NULL;
ocstat = oc_data_readn(conn,currentcontent,odom->index,1,sizeof(value),&value);
if(ocstat != OC_NOERR) goto done;
if(ocstat != OC_NOERR)
goto done;
nclistpush(strings,(void*)value);
dapodom_next(odom);
}
@ -1146,3 +963,192 @@ done:
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return THROW(ncstat);
}
#ifdef EXTERN_UNUSED
int
nc3d_getvarmx(int ncid, int varid,
const size_t *start,
const size_t *edges,
const ptrdiff_t* stride,
const ptrdiff_t* map,
void* data,
nc_type dsttype0)
{
NCerror ncstat = NC_NOERR;
int i;
NC* drno;
NC* substrate;
NC3_INFO* substrate3;
NCDAPCOMMON* dapcomm;
NC_var* var;
CDFnode* cdfvar; /* cdf node mapping to var*/
NClist* varnodes;
nc_type dsttype;
size_t externsize;
size_t dimsizes[NC_MAX_VAR_DIMS];
Dapodometer* odom = NULL;
unsigned int ncrank;
NClist* ncdims = NULL;
size_t nelems;
#ifdef NEWVARM
char* localcopy; /* of whole variable */
#endif
ncstat = NC_check_id(ncid, (NC**)&drno);
if(ncstat != NC_NOERR) goto done;
dapcomm = (NCDAPCOMMON*)drno->dispatchdata;
ncstat = NC_check_id(drno->substrate, &substrate);
if(ncstat != NC_NOERR) goto done;
substrate3 = (NC3_INFO*)drno->dispatchdata;
var = NC_lookupvar(substrate3,varid);
if(var == NULL) {ncstat = NC_ENOTVAR; goto done;}
/* Locate var node via varid */
varnodes = dapcomm->cdf.ddsroot->tree->varnodes;
for(i=0;i<nclistlength(varnodes);i++) {
CDFnode* node = (CDFnode*)nclistget(varnodes,i);
if(node->array.basevar == NULL
&& node->nctype == NC_Atomic
&& node->ncid == varid) {
cdfvar = node;
break;
}
}
ASSERT((cdfvar != NULL));
ASSERT((strcmp(cdfvar->ncfullname,var->name->cp)==0));
if(nclistlength(cdfvar->array.dimsetplus) == 0) {
/* The variable is a scalar; consequently, there is only one
thing to get and only one place to put it. */
/* recurse with additional parameters */
return THROW(nc3d_getvarx(ncid,varid,
NULL,NULL,NULL,
data,dsttype0));
}
dsttype = (dsttype0);
/* Default to using the inquiry type for this var*/
if(dsttype == NC_NAT) dsttype = cdfvar->externaltype;
/* Validate any implied type conversion*/
if(cdfvar->etype != dsttype && dsttype == NC_CHAR) {
/* The only disallowed conversion is to/from char and non-byte
numeric types*/
switch (cdfvar->etype) {
case NC_STRING: case NC_URL:
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
break;
default:
return THROW(NC_ECHAR);
}
}
externsize = nctypesizeof(dsttype);
/* Accumulate the dimension sizes and the total # of elements */
ncdims = cdfvar->array.dimsetall;
ncrank = nclistlength(ncdims);
nelems = 1; /* also Compute the number of elements being retrieved */
for(i=0;i<ncrank;i++) {
CDFnode* dim = (CDFnode*)nclistget(ncdims,i);
dimsizes[i] = dim->dim.declsize;
nelems *= edges[i];
}
/* Originally, this code repeatedly extracted single values
using get_var1. In an attempt to improve performance,
I have converted to reading the whole variable at once
and walking it locally.
*/
#ifdef NEWVARM
localcopy = (char*)malloc(nelems*externsize);
/* We need to use the varieties of get_vars in order to
properly do conversion to the external type
*/
switch (dsttype) {
case NC_CHAR:
ncstat = nc_get_vars_text(ncid,varid,start, edges, stride,
(char*)localcopy);
break;
case NC_BYTE:
ncstat = nc_get_vars_schar(ncid,varid,start, edges, stride,
(signed char*)localcopy);
break;
case NC_SHORT:
ncstat = nc_get_vars_short(ncid,varid, start, edges, stride,
(short*)localcopy);
break;
case NC_INT:
ncstat = nc_get_vars_int(ncid,varid,start, edges, stride,
(int*)localcopy);
break;
case NC_FLOAT:
ncstat = nc_get_vars_float(ncid,varid,start, edges, stride,
(float*)localcopy);
break;
case NC_DOUBLE:
ncstat = nc_get_vars_double(ncid,varid, start, edges, stride,
(double*)localcopy);
break;
default: break;
}
odom = dapodom_new(ncrank,start,edges,stride,NULL);
/* Walk the local copy */
for(i=0;i<nelems;i++) {
size_t voffset = dapodom_varmcount(odom,map,dimsizes);
void* dataoffset = (void*)(((char*)data) + (externsize*voffset));
char* localpos = (localcopy + externsize*i);
/* extract the indexset'th value from local copy */
memcpy(dataoffset,(void*)localpos,externsize);
/*
fprintf(stderr,"new: %lu -> %lu %f\n",
(unsigned long)(i),
(unsigned long)voffset,
*(float*)localpos);
*/
dapodom_next(odom);
}
#else
odom = dapodom_new(ncrank,start,edges,stride,NULL);
while(dapodom_more(odom)) {
size_t* indexset = odom->index;
size_t voffset = dapodom_varmcount(odom,map,dimsizes);
char internalmem[128];
char externalmem[128];
void* dataoffset = (void*)(((char*)data) + (externsize*voffset));
/* get the indexset'th value using variable's internal type */
ncstat = nc_get_var1(ncid,varid,indexset,(void*)&internalmem);
if(ncstat != NC_NOERR) goto done;
/* Convert to external type */
ncstat = dapconvert3(cdfvar->etype,dsttype,externalmem,internalmem);
if(ncstat != NC_NOERR) goto done;
memcpy(dataoffset,(void*)externalmem,externsize);
/*
fprintf(stderr,"old: %lu -> %lu %f\n",
(unsigned long)dapodom_count(odom),
(unsigned long)voffset,
*(float*)externalmem);
*/
dapodom_next(odom);
}
#endif
done:
return ncstat;
}
#endif /*EXTERN_UNUSED*/

20
libdap2/ncdap3a.c
View File

@ -788,17 +788,15 @@ applyclientparamcontrols3(NCDAPCOMMON* dapcomm)
CLRFLAG(dapcomm->controls,NCF_CACHE);
/* enable/disable cache prefetch and lazy vs eager*/
if(FLAGSET(dapcomm->controls,NCF_CACHE)) {
if(paramcheck34(dapcomm,"prefetch","eager")) {
SETFLAG(dapcomm->controls,NCF_PREFETCH);
SETFLAG(dapcomm->controls,NCF_PREFETCH_EAGER);
} else if(paramcheck34(dapcomm,"prefetch","lazy")
|| paramcheck34(dapcomm,"prefetch",NULL)) {
SETFLAG(dapcomm->controls,NCF_PREFETCH);
CLRFLAG(dapcomm->controls,NCF_PREFETCH_EAGER);
} else if(paramcheck34(dapcomm,"noprefetch",NULL))
CLRFLAG(dapcomm->controls,NCF_PREFETCH);
}
if(paramcheck34(dapcomm,"prefetch","eager")) {
SETFLAG(dapcomm->controls,NCF_PREFETCH);
SETFLAG(dapcomm->controls,NCF_PREFETCH_EAGER);
} else if(paramcheck34(dapcomm,"prefetch","lazy")
|| paramcheck34(dapcomm,"prefetch",NULL)) {
SETFLAG(dapcomm->controls,NCF_PREFETCH);
CLRFLAG(dapcomm->controls,NCF_PREFETCH_EAGER);
} else if(paramcheck34(dapcomm,"noprefetch",NULL))
CLRFLAG(dapcomm->controls,NCF_PREFETCH);
if(FLAGSET(dapcomm->controls,NCF_UNCONSTRAINABLE))
SETFLAG(dapcomm->controls,NCF_CACHE);

5
libdispatch/dvarget.c
View File

@ -208,11 +208,6 @@ NCDEFAULT_get_vars(int ncid, int varid, const size_t * start,
return NC_get_vara(ncid, varid, mystart, myedges, value, memtype);
}
/* Initial version uses and odometer to walk the variable
and read each value one at a time. This can later be optimized
to read larger chunks at a time.
*/
/* memptr indicates where to store the next value */
memptr = value;