netcdf-c/libdap4/d4swap.c
Dennis Heimbigner 591e6b2f6d Fix DAP4 remotetest server
Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555.

This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts*
to test several features:
1. Show that access over the Internet to servers using the DAP4 protocol works.
2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly.
4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable.

The test inputs (primarily *\*.nc* files) provided in the netcdf-c library
are also used by the DAP4 Test Server (aka d4ts) to present web access to a
collection of data files accessible via the DAP4 protocol and which can be
used for testing Internet access to a working server.

To be precise, this version of d4ts is currently in unmerged branches
of the *netcdf-java* and *tds* Github repositories and so are not actually
in the main repositories *yet*. However, the *d4ts.war* file was created
from that branch and used to populate the *remotetest.unidata.ucar.edu*
server

The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org)
and *thredds-test*. These will continue to remain disabled until
those servers can be fixed.

## Primary Changes

* Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers.
* Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process.
* Fix the *dap4_test/test_remote.sh* test script to match the current available test data.
* Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4].

## Misc. Unrelated Changes

* Remove a raft of warnings, especially in nc_test4/tst_quantize.c.
* Add some additional explanatory information to the NCZarr documentation.
* Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-15 20:29:21 -07:00

246 lines
6.7 KiB
C

/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#include "d4includes.h"
#include <stdarg.h>
#include "d4includes.h"
/*
The primary purpose of this code is to recursively traverse
the incoming data to get the endianness correct.
*/
/* Forward */
static int walkAtomicVar(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset);
static int walkOpaqueVar(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset);
static int walkStructArray(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset);
static int walkStruct(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset);
static int walkSeqArray(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset);
static int walkSeq(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset);
/**************************************************/
/*
Assumes that compiler->swap is true; does necessary
byte swapping.
*/
int
NCD4_swapdata(NCD4meta* compiler, NClist* topvars)
{
int ret = NC_NOERR;
int i;
NCD4offset* offset = NULL;
offset = BUILDOFFSET(compiler->serial.dap,compiler->serial.dapsize);
for(i=0;i<nclistlength(topvars);i++) {
NCD4node* var = (NCD4node*)nclistget(topvars,i);
OFFSET2BLOB(var->data.dap4data,offset);
switch (var->subsort) {
default:
if((ret=walkAtomicVar(compiler,var,var,offset))) goto done;
break;
case NC_OPAQUE:
/* The only thing we need to do is swap the counts */
if((ret=walkOpaqueVar(compiler,var,var,offset))) goto done;
break;
case NC_STRUCT:
if((ret=walkStructArray(compiler,var,var,offset))) goto done;
break;
case NC_SEQ:
if((ret=walkSeqArray(compiler,var,var,offset))) goto done;
break;
}
var->data.dap4data.size = DELTA(offset,var->data.dap4data.memory);
/* skip checksum, if there is one */
if(var->data.remotechecksummed)
INCR(offset,CHECKSUMSIZE);
}
done:
if(offset) free(offset);
return THROW(ret);
}
static int
walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset)
{
int ret = NC_NOERR;
d4size_t i;
nc_type subsort;
d4size_t dimproduct;
NCD4node* basetype;
basetype = (var->sort == NCD4_TYPE ? var : var->basetype);
subsort = basetype->subsort;
dimproduct = (var->sort == NCD4_TYPE ? 1 : NCD4_dimproduct(var));
if(subsort == NC_ENUM)
subsort = var->basetype->basetype->subsort;
/* Only need to swap multi-byte integers and floats */
if(subsort != NC_STRING) {
int typesize = NCD4_typesize(subsort);
d4size_t totalsize = typesize*dimproduct;
if(typesize == 1) {
INCR(offset,totalsize);
} else { /*(typesize > 1)*/
for(i=0;i<dimproduct;i++) {
char* sp = offset->offset;
if(compiler->swap) {
switch (typesize) {
case 2: swapinline16(sp); break;
case 4: swapinline32(sp); break;
case 8: swapinline64(sp); break;
default: break;
}
}
INCR(offset,typesize);
}
}
} else if(subsort == NC_STRING) { /* remaining case; just convert the counts */
COUNTERTYPE count;
for(i=0;i<dimproduct;i++) {
/* Get string count */
if(compiler->swap)
swapinline64(offset);
count = GETCOUNTER(offset);
SKIPCOUNTER(offset);
/* skip count bytes */
INCR(offset,count);
}
}
return THROW(ret);
}
static int
walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset)
{
int ret = NC_NOERR;
d4size_t i;
unsigned long long count;
d4size_t dimproduct = NCD4_dimproduct(var);
dimproduct = (var->sort == NCD4_TYPE ? 1 : NCD4_dimproduct(var));
for(i=0;i<dimproduct;i++) {
/* Get and swap opaque count */
if(compiler->swap)
swapinline64(offset);
count = GETCOUNTER(offset);
SKIPCOUNTER(offset);
INCR(offset,count);
}
return THROW(ret);
}
static int
walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset)
{
int ret = NC_NOERR;
d4size_t i;
d4size_t dimproduct = NCD4_dimproduct(var);
NCD4node* basetype = var->basetype;
for(i=0;i<dimproduct;i++) {
/* Swap, possibly recursively, the single struct pointed to by offset*/
if((ret=walkStruct(compiler,topvar,basetype,offset))) goto done;
}
done:
return THROW(ret);
}
static int
walkStruct(NCD4meta* compiler, NCD4node* topvar, NCD4node* structtype, NCD4offset* offset)
{
int ret = NC_NOERR;
int i;
for(i=0;i<nclistlength(structtype->vars);i++) {
NCD4node* field = (NCD4node*)nclistget(structtype->vars,i);
NCD4node* fieldbase = field->basetype;
switch (fieldbase->subsort) {
default:
if((ret=walkAtomicVar(compiler,topvar,field,offset))) goto done;
break;
case NC_OPAQUE:
/* The only thing we need to do is swap the counts */
if((ret=walkOpaqueVar(compiler,topvar,field,offset))) goto done;
break;
case NC_STRUCT:
if((ret=walkStructArray(compiler,topvar,field,offset))) goto done;
break;
case NC_SEQ:
if((ret=walkSeqArray(compiler,topvar,field,offset))) goto done;
break;
}
}
done:
return THROW(ret);
}
static int
walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset)
{
int ret = NC_NOERR;
d4size_t i;
d4size_t dimproduct;
NCD4node* seqtype;
assert(var->sort == NCD4_VAR);
dimproduct = NCD4_dimproduct(var);
seqtype = var->basetype;
for(i=0;i<dimproduct;i++) {
/* Swap, possibly recursively, the single seq pointed to by offset*/
if((ret=walkSeq(compiler,topvar,seqtype,offset))) goto done;
}
done:
return THROW(ret);
}
/*
Remember that the base type of var is a vlen.
*/
static int
walkSeq(NCD4meta* compiler, NCD4node* topvar, NCD4node* vlentype, NCD4offset* offset)
{
int ret = NC_NOERR;
int i;
d4size_t recordcount;
NCD4node* basetype;
/* process the record count */
if(compiler->swap)
swapinline64(offset);
recordcount = GETCOUNTER(offset);
SKIPCOUNTER(offset);
basetype = vlentype->basetype; /* This may be of any type potentially */
assert(basetype->sort == NCD4_TYPE);
for(i=0;i<recordcount;i++) {
switch(basetype->subsort) {
default: /* atomic basetype */
if((ret=walkAtomicVar(compiler,topvar,basetype,offset))) goto done;
break;
case NC_OPAQUE:
if((ret=walkOpaqueVar(compiler,topvar,basetype,offset))) goto done;
break;
case NC_STRUCT:
/* We can treat each record like a structure instance */
if((ret=walkStruct(compiler,topvar,basetype,offset))) goto done;
break;
case NC_SEQ:
if((ret=walkSeq(compiler,topvar,basetype,offset))) goto done;
break;
}
}
done:
return THROW(ret);
}