netcdf-c/libdap4/d4varx.c
Dennis Heimbigner 591e6b2f6d Fix DAP4 remotetest server
Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555.

This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts*
to test several features:
1. Show that access over the Internet to servers using the DAP4 protocol works.
2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly.
4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable.

The test inputs (primarily *\*.nc* files) provided in the netcdf-c library
are also used by the DAP4 Test Server (aka d4ts) to present web access to a
collection of data files accessible via the DAP4 protocol and which can be
used for testing Internet access to a working server.

To be precise, this version of d4ts is currently in unmerged branches
of the *netcdf-java* and *tds* Github repositories and so are not actually
in the main repositories *yet*. However, the *d4ts.war* file was created
from that branch and used to populate the *remotetest.unidata.ucar.edu*
server

The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org)
and *thredds-test*. These will continue to remain disabled until
those servers can be fixed.

## Primary Changes

* Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers.
* Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process.
* Fix the *dap4_test/test_remote.sh* test script to match the current available test data.
* Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4].

## Misc. Unrelated Changes

* Remove a raft of warnings, especially in nc_test4/tst_quantize.c.
* Add some additional explanatory information to the NCZarr documentation.
* Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-15 20:29:21 -07:00

201 lines
5.8 KiB
C

/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#include "config.h"
#include "ncdispatch.h"
#include "ncd4dispatch.h"
#include "nc4internal.h"
#include "d4includes.h"
#include "d4odom.h"
/* Forward */
static int getvarx(int ncid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*);
int
NCD4_get_vara(int ncid, int varid,
const size_t *start, const size_t *edges,
void *value,
nc_type memtype)
{
int ret;
/* TODO: optimize since we know stride is 1 */
ret = NCD4_get_vars(ncid,varid,start,edges,NC_stride_one,value,memtype);
return ret;
}
int
NCD4_get_vars(int ncid, int varid,
const size_t *start, const size_t *edges, const ptrdiff_t* stride,
void *memoryin, nc_type xtype)
{
int i,ret;
NCD4INFO* info;
NCD4meta* meta;
NCD4node* ncvar;
NCD4node* nctype;
D4odometer* odom = NULL;
nc_type nc4type;
size_t nc4size, xsize;
void* instance = NULL; /* Staging area in case we have to convert */
NClist* blobs = NULL;
int rank;
size_t dimsizes[NC_MAX_VAR_DIMS];
d4size_t dimproduct;
size_t dstcount;
NCD4offset* offset = NULL;
if((ret=getvarx(ncid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size)))
{goto done;}
meta = info->substrate.metadata;
nctype = ncvar->basetype;
rank = nclistlength(ncvar->dims);
blobs = nclistnew();
instance = malloc(nc4size);
if(instance == NULL)
{ret = THROW(NC_ENOMEM); goto done;}
dimproduct = NCD4_dimproduct(ncvar);
/* build size vector */
for(i=0;i<rank;i++) {
NCD4node* dim = nclistget(ncvar->dims,i);
dimsizes[i] = (size_t)dim->dim.size;
}
/* Extract and desired subset of data */
if(rank > 0)
odom = d4odom_new(rank,start,edges,stride,dimsizes);
else
odom = d4scalarodom_new();
dstcount = 0; /* We always write into dst starting at position 0*/
for(;d4odom_more(odom);dstcount++) {
void* xpos;
void* dst;
d4size_t count;
count = d4odom_next(odom);
if(count >= dimproduct) {
ret = THROW(NC_EINVALCOORDS);
goto done;
}
xpos = ((char*)memoryin)+(xsize * dstcount); /* ultimate destination */
/* We need to compute the offset in the dap4 data of this instance;
for fixed size types, this is easy, otherwise we have to walk
the variable size type
*/
/* Allocate the offset object */
if(offset) free(offset); /* Reclaim last loop */
offset = NULL;
offset = BUILDOFFSET(NULL,0);
BLOB2OFFSET(offset,ncvar->data.dap4data);
/* Move offset to the count'th element of the array */
if(nctype->meta.isfixedsize) {
INCR(offset,(nc4size*count));
} else {
/* We have to walk to the count'th location in the data */
if((ret=NCD4_moveto(meta,ncvar,count,offset)))
{goto done;}
}
dst = instance;
if((ret=NCD4_fillinstance(meta,nctype,offset,&dst,blobs)))
{goto done;}
if(xtype == nc4type) {
/* We can just copy out the data */
memcpy(xpos,instance,nc4size);
} else { /* Need to convert */
if((ret=NCD4_convert(nc4type,xtype,xpos,instance,1)))
{goto done;}
}
}
done:
if(offset) free(offset); /* Reclaim last loop */
/* cleanup */
if(odom != NULL)
d4odom_free(odom);
if(instance != NULL)
free(instance);
if(ret != NC_NOERR) { /* reclaim all malloc'd data if there is an error*/
for(i=0;i<nclistlength(blobs);i++) {
nullfree(nclistget(blobs,i));
}
}
if(blobs) nclistfree(blobs);
return (ret);
}
static int
getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp,
nc_type* xtypep, size_t* xsizep, nc_type* nc4typep, size_t* nc4sizep)
{
int ret = NC_NOERR;
NC* ncp;
NCD4INFO* info;
NCD4meta* meta;
NCD4node* group;
NCD4node* var;
NCD4node* type;
nc_type xtype, actualtype;
size_t instancesize, xsize;
if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR)
goto done;
info = getdap(ncp);
meta = info->substrate.metadata;
/* If the data has not already been read and processed, then do so. */
if(meta->serial.dap == NULL) {
size_t len = 0;
void* content = NULL;
/* (Re)Build the meta data; sets serial.rawdata */
NCD4_resetMeta(info->substrate.metadata);
meta->controller = info;
meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */
if((ret=NCD4_readDAP(info, info->controls.flags.flags))) goto done;
len = ncbyteslength(info->curl->packet);
content = ncbytesextract(info->curl->packet);
NCD4_resetSerial(&meta->serial, len, content);
/* Process the data part */
if((ret=NCD4_dechunk(meta))) goto done;
if((ret = NCD4_processdata(info->substrate.metadata))) goto done;
}
if((ret = NCD4_findvar(ncp,ncid,varid,&var,&group))) goto done;
type = var->basetype;
actualtype = type->meta.id;
instancesize = type->meta.memsize;
/* Figure out the type conversion, if any */
xtype = *xtypep;
if(xtype == NC_NAT)
xtype = actualtype;
if(xtype != actualtype && xtype > NC_MAX_ATOMIC_TYPE)
return THROW(NC_EBADTYPE);
if((xtype == NC_CHAR || xtype == NC_STRING)
&& (actualtype != NC_CHAR && actualtype != NC_STRING))
return THROW(NC_ECHAR);
if(xtype <= NC_MAX_ATOMIC_TYPE)
xsize = NCD4_typesize(xtype);
else
xsize = instancesize;
/* Return relevant info */
if(infop) *infop = info;
if(xtypep) *xtypep = xtype;
if(xsizep) *xsizep = xsize;
if(nc4typep) *nc4typep = actualtype;
if(nc4sizep) *nc4sizep = instancesize;
if(varp) *varp = var;
done:
if(meta->error.message != NULL)
NCD4_reporterror(info); /* Make sure the user sees this */
return THROW(ret);
}