mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-04-12 18:10:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improve operation of the DAP4 code and fix bugs

Browse Source 
re: esupport 31942

* Add a new set of remote tests based on using the opendap hyrax server
* Re-enable --enable-dap-remote-tests as default on
* Make it possible to query the DMR separate from the DAP
  so that it is possible to delay reading data until it is actually
  requested. This make things like ncdump -h much more efficient.
* Fix handling of <Map>s in DMRs.
* Fix some memory leaks
This commit is contained in:
Dennis Heimbigner 2021-01-14 21:39:08 -07:00
parent af8fd51a37
commit 97ce621091
30 changed files with 64471 additions and 287 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/run_tests.yml vendored
View File

@ -4,7 +4,7 @@
name: Run netCDF Tests
on: [pull_request]
on: [pull_request,push]
jobs:

3
configure.ac
View File

@ -498,12 +498,11 @@ CFLAGS="$SAVECFLAGS"
# --enable-dap => enable-dap4
enable_dap4=$enable_dap
# Default is to do the short remote tests.
# Temporary: Change default to not do these tests
AC_MSG_CHECKING([whether dap remote testing should be enabled (default off)])
AC_ARG_ENABLE([dap-remote-tests],
[AS_HELP_STRING([--enable-dap-remote-tests],
[enable dap remote tests])])
test "x$enable_dap_remote_tests" = xyes || enable_dap_remote_tests=no
test "x$enable_dap_remote_tests" = xno || enable_dap_remote_tests=yes
if test "x$enable_dap" = "xno" ; then
enable_dap_remote_tests=no
fi

5
dap4_test/CMakeLists.txt
View File

@ -12,6 +12,7 @@ remove_definitions(-DDLL_EXPORT)
ADD_SUBDIRECTORY(baseline)
ADD_SUBDIRECTORY(baselineraw)
ADD_SUBDIRECTORY(baselineremote)
ADD_SUBDIRECTORY(baselinehyrax)
ADD_SUBDIRECTORY(cdltestfiles)
ADD_SUBDIRECTORY(daptestfiles)
ADD_SUBDIRECTORY(dmrtestfiles)
@ -52,6 +53,10 @@ IF(ENABLE_TESTS)
ENDIF()
ENDIF(BUILD_UTILITIES)
IF(ENABLE_DAP_REMOTE_TESTS)
add_sh_test(dap4_test test_hyrax)
ENDIF(ENABLE_DAP_REMOTE_TESTS)
ENDIF(ENABLE_TESTS)
FILE(COPY ./baselineraw DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})

6
dap4_test/Makefile.am
View File

@ -39,10 +39,11 @@ findtestserver4_SOURCES = findtestserver4.c
pingurl4_SOURCES = pingurl4.c
# Disable Dap4 Remote Tests until the test server is working
if AX_IGNORE
if ENABLE_DAP_REMOTE_TESTS
if BUILD_UTILITIES
# relies on ncdump
TESTS += test_hyrax.sh
if AX_IGNORE
TESTS += test_remote.sh
endif
endif
@ -71,7 +72,8 @@ clean-local: clean-local-check
.PHONY: clean-local-check
clean-local-check:
-rm -rf results
-rm -rf results results_test_parse results_test_data \
results_test_hyrax results_test_meta
-rm -f .dodsrc .daprc
# The shell file maketests.sh is used to build the testdata

13
dap4_test/baselinehyrax/CMakeLists.txt Normal file
View File

@ -0,0 +1,13 @@
# Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
# 2015, 2016, 2017, 2018
# University Corporation for Atmospheric Research/Unidata.
# See netcdf-c/COPYRIGHT file for more info.
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*)
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE)
FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt)
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

63723
dap4_test/baselinehyrax/amsre_20060131v5.dat.hyrax Normal file
View File

File diff suppressed because it is too large Load Diff

104
dap4_test/baselinehyrax/nc4_nc_classic_comp.nc.hyrax Normal file
View File

@ -0,0 +1,104 @@
netcdf nc4_nc_classic_comp {
dimensions:
lat = 6 ;
lon = 5 ;
time = 2 ;
variables:
int lat(lat) ;
string lat:units = "degrees_north" ;
int lon(lon) ;
string lon:units = "degrees_east" ;
int time(time) ;
string time:units = "seconds" ;
float z(time, lat, lon) ;
string z:units = "meters" ;
z:valid_range = 0., 5000. ;
z:_FillValue = 1.f ;
string z:_edu.ucar.maps = "/time", "/lat", "/lon" ;
float t(time, lat, lon) ;
string t:_edu.ucar.maps = "/time", "/lat", "/lon" ;
double p(time, lat, lon) ;
p:_FillValue = -9999. ;
string p:_edu.ucar.maps = "/time", "/lat", "/lon" ;
short rh(time, lat, lon) ;
rh:_FillValue = -1s ;
string rh:_edu.ucar.maps = "/time", "/lat", "/lon" ;
short pixel(lat, lon) ;
string pixel:_edu.ucar.maps = "/lat", "/lon" ;
string source(lat) ;
source:string_length = 5 ;
string source:_edu.ucar.maps = "/lat" ;
data:
lat = 0, 10, 20, 30, 40, 50 ;
lon = -140, -118, -96, -84, -52 ;
time = 1, 2 ;
z =
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10 ;
t =
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1 ;
p =
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1 ;
rh =
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2 ;
pixel =
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7 ;
source = "aaaaa", "aaaaa", "aaaaa", "aaaaa", "aaaaa", "aaaaa" ;
}

32
dap4_test/baselinehyrax/nc4_strings_comp.nc.hyrax Normal file
View File

@ -0,0 +1,32 @@
netcdf nc4_strings_comp {
dimensions:
lat = 6 ;
lon = 5 ;
variables:
int lat(lat) ;
int lon(lon) ;
string station(lat, lon) ;
string station:who = "james" ;
string station:names = "site_1", "site_2", "site_3" ;
string station:_edu.ucar.maps = "/lat", "/lon" ;
string scan_line(lon) ;
string scan_line:_edu.ucar.maps = "/lon" ;
string codec_name ;
data:
lat = 0, 10, 20, 30, 40, 50 ;
lon = -140, -118, -96, -84, -52 ;
station =
"one", "two", "three", "four", "five",
"one_b", "two_b", "three_b", "four_b", "five_b",
"one_c", "two_c", "three_c", "four_c", "five_c",
"one", "two", "three", "four", "five",
"one", "two", "three", "four", "five",
"one_f", "two_f", "three_f", "four_f", "five_f" ;
scan_line = "r", "r1", "r2", "r3", "r4" ;
codec_name = "mp3" ;
}

53
dap4_test/baselinehyrax/nc4_unsigned_types_comp.nc.hyrax Normal file
View File

@ -0,0 +1,53 @@
netcdf nc4_unsigned_types_comp {
dimensions:
lat = 6 ;
lon = 5 ;
time = 2 ;
variables:
int lat(lat) ;
string lat:units = "degrees_north" ;
int lon(lon) ;
string lon:units = "degrees_east" ;
int time(time) ;
string time:units = "seconds" ;
uint temp(time, lat, lon) ;
string temp:_edu.ucar.maps = "/time", "/lat", "/lon" ;
ushort rh(time, lat, lon) ;
rh:_FillValue = 9999US ;
string rh:_edu.ucar.maps = "/time", "/lat", "/lon" ;
data:
lat = 0, 10, 20, 30, 40, 50 ;
lon = -140, -118, -96, -84, -52 ;
time = 1, 2 ;
temp =
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7 ;
rh =
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2 ;
}

24
dap4_test/baselinehyrax/ref_tst_compounds.nc.hyrax Normal file
View File

@ -0,0 +1,24 @@
netcdf ref_tst_compounds {
types:
compound obs_t {
short day ;
short elev ;
int count ;
float relhum ;
double time ;
}; // obs_t
dimensions:
n = 3 ;
variables:
obs_t obs(n) ;
obs:day = -99s ;
obs:elev = -99s ;
obs:count = -99 ;
obs:relhum = -99.00005f ;
obs:time = -99. ;
data:
obs = {15, 2, 1, 0.5, 3600.01},
{-99, -1, -1027211264, 0, 8.34446731226507e-309},
{0, 16192, -1889785610, 5.610353, 5.08101286513212e+233} ;
}

3
dap4_test/d4test_common.sh
View File

@ -23,7 +23,8 @@ cd ${srcdir}/cdltestfiles; CDLTESTFILES=`pwd` ; cd ${WD}
cd ${srcdir}/baseline; BASELINE=`pwd` ; cd ${WD}
cd ${srcdir}/baselineraw; BASELINERAW=`pwd` ; cd ${WD}
cd ${srcdir}/baselineremote; BASELINEREM=`pwd` ; cd ${WD}
#cd ${srcdir}/baselinehyrax; BASELINEH=`pwd` ; cd ${WD}
cd ${srcdir}/baselinehyrax; BASELINEH=`pwd` ; cd ${WD}
#cd ${srcdir}/baselinethredds; BASELINETH=`pwd` ; cd ${WD}
setresultdir() {
rm -fr ${builddir}/$1

23
dap4_test/test_common.h
View File

@ -61,6 +61,7 @@ setup(int tdmr, int argc, char** argv)
argc--; argv++;
int expected = 0;
NCD4mode mode = 0;
NCD4INFO* controller = NULL;
switch(tdmr) {
case TDMR_PARSE:
@ -96,21 +97,18 @@ setup(int tdmr, int argc, char** argv)
NCD4_dumpbytes(ncbyteslength(input),ncbytescontents(input),0);
#endif
if((metadata=NCD4_newmeta(ncbyteslength(input),ncbytescontents(input)))==NULL)
/* Create a fake NCD4INFO */
controller = (NCD4INFO*)calloc(1,sizeof(NCD4INFO));
if(controller == NULL)
fail(NC_ENOMEM);
controller->controls.translation = NCD4_TRANSNC4;
if(translatenc4)
controller->controls.translation = NCD4_TRANSNC4;
NCD4_applyclientparamcontrols(controller);
if((metadata=NCD4_newmeta(controller, ncbyteslength(input),ncbytescontents(input)))==NULL)
fail(NC_ENOMEM);
metadata->mode = mode;
/* Create a fake NCD4INFO */
{
NCD4INFO* controller = (NCD4INFO*)calloc(1,sizeof(NCD4INFO));
if(controller == NULL)
fail(NC_ENOMEM);
metadata->controller = controller;
controller->controls.translation = NCD4_TRANSNC4;
if(translatenc4)
controller->controls.translation = NCD4_TRANSNC4;
NCD4_applyclientparamcontrols(controller);
}
if((ret=NCD4_dechunk(metadata))) /* ok for mode == DMR or mode == DAP */
fail(ret);
#ifdef DEBUG
@ -163,7 +161,6 @@ cleanup(int ret)
if(metadata->controller != NULL)
free(metadata->controller);
NCD4_reclaimMeta(metadata);
ncbytesfree(input);
ncbytesfree(output);
if(ret)
fail(ret);

6
dap4_test/test_data.c
View File

@ -12,13 +12,15 @@ Test the netcdf-4 data building process.
#include <stdio.h>
#include "netcdf.h"
#undef DEBUG
#define DEBUG
static void
fail(int code)
{
if(code != NC_NOERR)
if(code != NC_NOERR) {
fprintf(stderr,"***Fail: %s\n",nc_strerror(code));
fflush(stderr);
}
exit((code==NC_NOERR?EXIT_SUCCESS:EXIT_FAILURE));
}

16
dap4_test/test_data.sh
View File

@ -3,6 +3,8 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
. ${srcdir}/d4test_common.sh
echo "test_data.sh:"
@ -16,14 +18,12 @@ setresultdir results_test_data
if test "x${RESET}" = x1 ; then rm -fr ${BASELINE}/*.d4d ; fi
for f in $F ; do
echo "testing: ${f}"
if ! ${VG} ${execdir}/test_data ${DAPTESTFILES}/${f} ./results_test_data/${f}.nc ; then
failure "${execdir}/test_data ${DAPTESTFILES}/${f} ./results_test_data/${f}.nc"
if ! ${execdir}/test_data ${DAPTESTFILES}/${f} ./results_test_data/${f}.nc ; then
echo "failure"
fi
${NCDUMP} ./results_test_data/${f}.nc > ./results_test_data/${f}.d4d
if test "x${TEST}" = x1 ; then
if ! diff -wBb ${BASELINE}/${f}.d4d ./results_test_data/${f}.d4d ; then
failure "diff -wBb ${BASELINE}/${f}.d4d ./results_test_data/${f}.d4d"
fi
diff -wBb ${BASELINE}/${f}.d4d ./results_test_data/${f}.d4d
elif test "x${RESET}" = x1 ; then
echo "${f}:"
cp ./results_test_data/${f}.d4d ${BASELINE}/${f}.d4d
@ -83,11 +83,7 @@ if test "x${CDLDIFF}" = x1 ; then
trim ./results_test_data/${f}.d4d ./r1
baseclean b1 b2
resultclean r1 r2
if ! diff -wBb ./b2 ./r2 ; then
failure "${f}"
fi
diff -wBb ./b2 ./r2
done
fi
rm -rf ./results_test_data
finish

33
dap4_test/test_hyrax.sh
View File

@ -3,17 +3,22 @@
if test "x$srcdir" = "x"; then srcdir=`dirname $0`; fi
export srcdir;
. ${srcdir}/../test_common.sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. ${srcdir}/d4test_common.sh
set -e
echo "test_hyrax.sh:"
#FRAG="#checksummode=ignore"
FRAG="#checksummode=ignore"
F="\
nc4_test_files/nc4_nc_classic_comp.nc \
nc4_test_files/nc4_unsigned_types_comp.nc \
nc4_test_files/nc4_strings_comp.nc \
nc4_test_files/ref_tst_compounds.nc \
hyrax/RSS/amsre/bmaps_v05/y2006/m01/amsre_20060131v5.dat?dap4.ce=time_a \
"
failure() {
@ -23,25 +28,31 @@ failure() {
setresultdir results_test_hyrax
if test "x${RESET}" = x1 ; then rm -fr ${BASELINEH}/*.dmp ; fi
if test "x${RESET}" = x1 ; then rm -fr ${BASELINEH}/*.hyrax ; fi
for f in $F ; do
URL="dap4://test.opendap.org:8080/opendap/nc4_test_files/${f}${FRAG}"
constraint=`echo "$f" | cut -d '?' -f2`
unconstrained=`echo "$f" | cut -d '?' -f1`
base=`basename $unconstrained`
prefix=`dirname $unconstrained`
if test "x$constraint" = "x$unconstrained" ; then
URL="dap4://test.opendap.org:8080/opendap/${prefix}/${base}${FRAG}"
else
URL="dap4://test.opendap.org:8080/opendap/${prefix}/${base}?$constraint${FRAG}"
fi
echo "testing: $URL"
if ! ${NCDUMP} "${URL}" > ./results_test_hyrax/${f}.hyrax; then
if ! ${NCDUMP} "${URL}" > ./results_test_hyrax/${base}.hyrax; then
failure "${URL}"
fi
if test "x${TEST}" = x1 ; then
if ! diff -wBb ${BASELINEREM}/${f}.hyrax ./results_test_hyrax/${f}.hyrax ; then
failure "diff ${f}.hyrax"
if ! diff -wBb ${BASELINEH}/${base}.hyrax ./results_test_hyrax/${base}.hyrax ; then
failure "diff ${base}.hyrax"
fi
elif test "x${RESET}" = x1 ; then
echo "${f}:"
cp ./results_test_hyrax/${f}.hyrax ${BASELINEH}/${f}.hyrax
cp ./results_test_hyrax/${base}.hyrax ${BASELINEH}/${base}.hyrax
fi
done
rm -rf ./results_test_hyrax
echo "*** Pass"
exit 0

2
dap4_test/test_meta.sh
View File

@ -55,7 +55,7 @@ if test "x${CDLDIFF}" = x1 ; then
fi
done
fi
rm -rf ./results_test_meta
finish

19
dap4_test/test_parse.sh
View File

@ -3,6 +3,8 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
. ${srcdir}/d4test_common.sh
echo "test_parse.sh:"
@ -16,13 +18,9 @@ setresultdir results_test_parse
if test "x${RESET}" = x1 ; then rm -fr ${BASELINE}/*.d4p ; fi
for f in $F ; do
echo "testing: $f"
if ! ${VG} ${execdir}/test_parse ${DMRTESTFILES}/${f}.dmr > ./results_test_parse/${f}.d4p ; then
failure "${f}"
fi
${VG} ${execdir}/test_parse ${DMRTESTFILES}/${f}.dmr > ./results_test_parse/${f}.d4p
if test "x${TEST}" = x1 ; then
if ! diff -wBb ${BASELINE}/${f}.d4p ./results_test_parse/${f}.d4p ; then
failure "${f}"
fi
diff -wBb ${BASELINE}/${f}.d4p ./results_test_parse/${f}.d4p
elif test "x${DIFF}" = x1 ; then
echo "diff -wBb ${DMRTESTFILES}/${f}.dmr ./results_test_parse/${f}.d4p"
rm -f ./tmp
@ -31,18 +29,11 @@ for f in $F ; do
| sed -e '/<\/Dimensions>/d' -e '/<\/Types>'/d -e '/<\/Variables>'/d -e '/<\/Groups>'/d \
| sed -e '/_edu.ucar.opaque.size/,+2d' \
| cat > ./tmp
if ! diff -wBb ${DMRTESTFILES}/${f}.dmr ./tmp ; then
failure "${f}"
fi
diff -wBb ${DMRTESTFILES}/${f}.dmr ./tmp
elif test "x${RESET}" = x1 ; then
echo "${f}:"
cp ./results_test_parse/${f}.d4p ${BASELINE}/${f}.d4p
fi
done
rm -rf ./results_test_parse
finish
exit 0

1
dap4_test/test_raw.sh
View File

@ -66,6 +66,5 @@ for f in $F ; do
fi
fi
done
rm -rf ${builddir}/results_test_raw
finish

2
dap4_test/test_remote.sh
View File

@ -86,7 +86,5 @@ for f in $F ; do
fi
done
rm -fr ${builddir}/results_test_remote
finish

122
libdap4/d4chunk.c
View File

@ -10,18 +10,6 @@
/**************************************************/
/* Header flags */
#define LAST_CHUNK (1)
#define ERR_CHUNK (2)
#define LITTLE_ENDIAN_CHUNK (4)
#ifdef CHECKSUMHACK
#define NOCHECKSUM_CHUNK (8)
#else
#define NOCHECKSUM_CHUNK (0)
#endif
#define ALL_CHUNK_FLAGS (LAST_CHUNK|ERR_CHUNK|LITTLE_ENDIAN_CHUNK|NOCHECKSUM_CHUNK)
/**************************************************/
/*
@ -33,110 +21,114 @@ and whether it has checksums.
Notes:
*/
/* Define a local struct for convenience */
struct HDR {unsigned int flags; unsigned int count;};
/* Forward */
static void* getheader(void* p, struct HDR* hdr, int hostlittleendian);
static int processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count);
/**************************************************/
void
NCD4_resetSerial(NCD4serial* serial, size_t rawsize, void* rawdata)
{
nullfree(serial->errdata);
nullfree(serial->dmr);
nullfree(serial->dap);
nullfree(serial->rawdata);
/* clear all fields */
memset(serial,0,sizeof(NCD4serial));
/* Reset fields */
serial->hostlittleendian = NCD4_isLittleEndian();
serial->rawsize = rawsize;
serial->rawdata = rawdata;
}
int
NCD4_dechunk(NCD4meta* metadata)
{
unsigned char* p;
unsigned char* q;
struct HDR hdr;
unsigned char *praw, *phdr, *pdap;
NCD4HDR hdr;
if(metadata->mode == NCD4_DSR)
return THROW(NC_EDMR);
metadata->serial.errdata = NULL;
metadata->serial.dmr = NULL;
metadata->serial.dap = NULL;
metadata->serial.hostlittleendian = NCD4_isLittleEndian();
metadata->serial.remotelittleendian = 0; /* do not actually know yet */
#ifdef CHECKSUMHACK
metadata->serial.checksumhack = 0; /* do not actually know yet */
#endif
/* Assume proper mode has been inferred already. */
/* Verify the mode; assume that the <?xml...?> is optional */
q = metadata->serial.rawdata;
if(memcmp(q,"<?xml",strlen("<?xml"))==0
|| memcmp(q,"<Dataset",strlen("<Dataset"))==0) {
praw = metadata->serial.rawdata;
if(memcmp(praw,"<?xml",strlen("<?xml"))==0
|| memcmp(praw,"<Dataset",strlen("<Dataset"))==0) {
size_t len = 0;
if(metadata->mode != NCD4_DMR)
return THROW(NC_EDMR);
/* setup as dmr only */
metadata->serial.dmr = (char*)metadata->serial.rawdata; /* temp */
/* Avoid strdup since rawdata might contain nul chars */
if((metadata->serial.dmr = malloc(metadata->serial.rawsize+1)) == NULL)
len = metadata->serial.rawsize;
if((metadata->serial.dmr = malloc(len+1)) == NULL)
return THROW(NC_ENOMEM);
memcpy(metadata->serial.dmr,metadata->serial.rawdata,metadata->serial.rawsize);
metadata->serial.dmr[metadata->serial.rawsize-1] = '\0';
memcpy(metadata->serial.dmr,praw,len);
metadata->serial.dmr[len] = '\0';
/* Suppress nuls */
(void)NCD4_elidenuls(metadata->serial.dmr,metadata->serial.rawsize);
(void)NCD4_elidenuls(metadata->serial.dmr,len);
return THROW(NC_NOERR);
}
/* We must be processing a DAP mode packet */
p = metadata->serial.rawdata;
metadata->serial.dap = p;
#ifdef D4DUMPRAW
NCD4_tagdump(metadata->serial.rawsize,metadata->serial.rawdata,0,"RAW");
#endif
/* We must be processing a DAP mode packet */
praw = (metadata->serial.dap = metadata->serial.rawdata);
metadata->serial.rawdata = NULL;
/* Get the DMR chunk header*/
p = getheader(p,&hdr,metadata->serial.hostlittleendian);
phdr = NCD4_getheader(praw,&hdr,metadata->serial.hostlittleendian);
if(hdr.count == 0)
return THROW(NC_EDMR);
if(hdr.flags & ERR_CHUNK) {
return processerrchunk(metadata, (void*)p, hdr.count);
if(hdr.flags & NCD4_ERR_CHUNK) {
return processerrchunk(metadata, (void*)phdr, hdr.count);
}
#ifdef CHECKSUMHACK
/* Temporary hack; We mistakenly thought that bit 3 of the flags
of the first header indicated that checksumming was not in force.
Test for it, and propagate the _DAP4_Checksum_CRC32 attribute later */
metadata->serial.checksumhack = ((hdr.flags & NOCHECKSUM_CHUNK) ? 1 : 0);
metadata->serial.checksumhack = ((hdr.flags & NCD4_NOCHECKSUM_CHUNK) ? 1 : 0);
fprintf(stderr,"checksumhack=%d\n",metadata->serial.checksumhack);
#endif
metadata->serial.remotelittleendian = ((hdr.flags & LITTLE_ENDIAN_CHUNK) ? 1 : 0);
metadata->serial.remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0);
/* Again, avoid strxxx operations on dmr */
if((metadata->serial.dmr = malloc(hdr.count+1)) == NULL)
return THROW(NC_ENOMEM);
memcpy(metadata->serial.dmr,p,hdr.count);
memcpy(metadata->serial.dmr,phdr,hdr.count);
metadata->serial.dmr[hdr.count-1] = '\0';
/* Suppress nuls */
(void)NCD4_elidenuls(metadata->serial.dmr,hdr.count);
if(hdr.flags & LAST_CHUNK)
if(hdr.flags & NCD4_LAST_CHUNK)
return THROW(NC_ENODATA);
/* Read and compress the data chunks */
p = p + hdr.count; /* point to data chunk header */
/* Read and concat together the data chunks */
phdr = phdr + hdr.count; /* point to data chunk header */
/* Do a sanity check in case the server has shorted us with no data */
if((hdr.count + CHUNKHDRSIZE) >= metadata->serial.rawsize) {
/* Server only sent the DMR part */
metadata->serial.dapsize = 0;
return THROW(NC_EDATADDS);
}
q = metadata->serial.dap;
pdap = metadata->serial.dap;
for(;;) {
p = getheader(p,&hdr,metadata->serial.hostlittleendian);
if(hdr.flags & ERR_CHUNK) {
return processerrchunk(metadata, (void*)p, hdr.count);
phdr = NCD4_getheader(phdr,&hdr,metadata->serial.hostlittleendian);
if(hdr.flags & NCD4_ERR_CHUNK) {
return processerrchunk(metadata, (void*)phdr, hdr.count);
}
/* data chunk; possibly last; possibly empty */
if(hdr.count > 0) {
d4memmove(q,p,hdr.count); /* will overwrite the header */
p += hdr.count;
q += hdr.count;
d4memmove(pdap,phdr,hdr.count); /* will overwrite the header */
phdr += hdr.count;
pdap += hdr.count;
}
if(hdr.flags & LAST_CHUNK) break;
if(hdr.flags & NCD4_LAST_CHUNK) break;
}
metadata->serial.dapsize = (size_t)DELTA(q,metadata->serial.dap);
metadata->serial.dapsize = (size_t)DELTA(pdap,metadata->serial.dap);
#ifdef D4DUMPDMR
fprintf(stderr,"%s\n",metadata->serial.dmr);
@ -159,22 +151,6 @@ processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count)
return THROW(NC_ENODATA); /* slight lie */
}
static void*
getheader(void* p, struct HDR* hdr, int hostlittleendian)
{
unsigned char bytes[4];
memcpy(bytes,p,sizeof(bytes));
p = INCR(p,4); /* on-the-wire hdr is 4 bytes */
/* assume header is network (big) order */
hdr->flags = bytes[0]; /* big endian => flags are in byte 0 */
hdr->flags &= ALL_CHUNK_FLAGS; /* Ignore extraneous flags */
bytes[0] = 0; /* so we can do byte swap to get count */
if(hostlittleendian)
swapinline32(bytes); /* host is little endian */
hdr->count = *(unsigned int*)bytes; /* get count */
return p;
}
/**
Given a raw response, attempt to infer the mode: DMR, DAP, DSR.
Since DSR is not standardizes, it becomes the default.

9
libdap4/d4debug.c
View File

@ -96,6 +96,7 @@ NCD4_debugcopy(NCD4INFO* info)
NCD4meta* meta = info->substrate.metadata;
NClist* topvars = nclistnew();
NC* ncp = info->controller;
void* memory = NULL;
/* Walk each top level variable, read all of it and write it to the substrate */
if((ret=NCD4_getToplevelVars(meta, NULL, topvars)))
@ -108,7 +109,6 @@ NCD4_debugcopy(NCD4INFO* info)
int grpid = grp->meta.id;
int varid = var->meta.id;
d4size_t varsize;
void* memory = NULL;
size_t dimprod = NCD4_dimproduct(var);
int ncid = info->substrate.nc4id;
@ -141,12 +141,11 @@ NCD4_debugcopy(NCD4INFO* info)
}
if((ret=ncaux_reclaim_data(ncid,type->meta.id,memory,dimprod)))
goto done;
free(memory);
memory = NULL;
nullfree(memory); memory = NULL;
}
done:
if(topvars)
nclistfree(topvars);
nullfree(memory);
nclistfree(topvars);
if(ret != NC_NOERR) {
fprintf(stderr,"debugcopy: %d %s\n",ret,nc_strerror(ret));
}

152
libdap4/d4file.c
View File

@ -28,6 +28,8 @@ static void freeInfo(NCD4INFO*);
static int paramcheck(NCD4INFO*, const char* key, const char* subkey);
static const char* getparam(NCD4INFO* info, const char* key);
static int set_curl_properties(NCD4INFO*);
static int makesubstrate(NCD4INFO* d4info);
static void resetInfoforRead(NCD4INFO* d4info);
/**************************************************/
/* Constants */
@ -43,8 +45,10 @@ NCD4_open(const char * path, int mode,
int ret = NC_NOERR;
NCD4INFO* d4info = NULL;
const char* value;
NCD4meta* meta;
NC* nc;
NCD4meta* meta = NULL;
size_t len = 0;
void* contents = NULL;
if(path == NULL)
return THROW(NC_EDAPURL);
@ -101,33 +105,10 @@ NCD4_open(const char * path, int mode,
else
snprintf(tmpname,sizeof(tmpname),"tmp_%d",nc->int_ncid);
/* Now, use the file to create the hidden substrate netcdf file.
We want this hidden file to always be NC_NETCDF4, so we need to
force default format temporarily in case user changed it.
Since diskless is enabled, create file in-memory.
*/
{
int new = NC_NETCDF4;
int old = 0;
int ncid = 0;
int ncflags = NC_NETCDF4|NC_CLOBBER;
ncflags |= NC_DISKLESS;
if(FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) {
/* Cause data to be dumped to real file */
ncflags |= NC_WRITE;
ncflags &= ~(NC_DISKLESS); /* use real file */
}
nc_set_default_format(new,&old); /* save and change */
ret = nc_create(tmpname,ncflags,&ncid);
nc_set_default_format(old,&new); /* restore */
d4info->substrate.realfile = ((ncflags & NC_DISKLESS) == 0);
d4info->substrate.filename = strdup(tmpname);
if(d4info->substrate.filename == NULL) ret = NC_ENOMEM;
d4info->substrate.nc4id = ncid;
}
if(ret != NC_NOERR) goto done;
/* Avoid fill */
nc_set_fill(getnc4id(nc),NC_NOFILL,NULL);
/* Compute the relevant names for the substrate file */
d4info->substrate.filename = strdup(tmpname);
if(d4info->substrate.filename == NULL)
{ret = NC_ENOMEM; goto done;}
}
/* Turn on logging; only do this after oc_open*/
@ -158,45 +139,22 @@ NCD4_open(const char * path, int mode,
d4info->curl->packet = ncbytesnew();
ncbytessetalloc(d4info->curl->packet,DFALTPACKETSIZE); /*initial reasonable size*/
/* fetch the dmr + data*/
{
int inmem = FLAGSET(d4info->controls.flags,NCF_ONDISK) ? 0 : 1;
if((ret = NCD4_readDAP(d4info,inmem))) goto done;
}
/* Reset the substrate */
if((ret=makesubstrate(d4info))) goto done;
/* if the url goes astray to a random web page, then try to just dump it */
{
char* response = (char*)ncbytescontents(d4info->curl->packet);
size_t responselen = ncbyteslength(d4info->curl->packet);
/* Always start by reading the DMR only */
/* reclaim substrate.metadata */
resetInfoforRead(d4info);
/* Apply some heuristics to see what we have.
The leading byte will have the chunk flags, which should
be less than 0x0f (for now). However, it will not be zero if
the data was little-endian
*/
if(responselen == 0 || ((unsigned char*)response)[0] > 0x0f) {
/* does not look like a chunk, so probable server failure */
if(responselen == 0)
nclog(NCLOGERR,"Empty DAP4 response");
else {/* probable html response */
nclog(NCLOGERR,"Unexpected DAP response:");
nclog(NCLOGERR,"==============================");
nclogtext(NCLOGERR,response);
nclog(NCLOGERR,"==============================\n");
}
ret = NC_EDAPSVC;
fflush(stderr);
goto done;
}
}
if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags))) goto done;
/* Build the meta data */
if((d4info->substrate.metadata=NCD4_newmeta(ncbyteslength(d4info->curl->packet),
ncbytescontents(d4info->curl->packet)))==NULL)
/* (Re)Build the meta data; sets serial.rawdata */
len = ncbyteslength(d4info->curl->packet);
contents = ncbytesextract(d4info->curl->packet);
if((d4info->substrate.metadata=NCD4_newmeta(d4info, len, contents))==NULL)
{ret = NC_ENOMEM; goto done;}
meta = d4info->substrate.metadata;
meta->controller = d4info;
meta->ncid = getnc4id(nc); /* Transfer netcdf ncid */
/* process meta control parameters */
applyclientmetacontrols(meta);
@ -204,8 +162,6 @@ NCD4_open(const char * path, int mode,
/* Infer the mode */
if((ret=NCD4_infermode(meta))) goto done;
if((ret=NCD4_dechunk(meta))) goto done;
#ifdef D4DUMPDMR
{
fprintf(stderr,"=============\n");
@ -215,7 +171,11 @@ NCD4_open(const char * path, int mode,
}
#endif
/* Process the dmr part */
if((ret=NCD4_dechunk(meta))) goto done;
if((ret = NCD4_parse(d4info->substrate.metadata))) goto done;
#ifdef D4DEBUGMETA
{
fprintf(stderr,"\n/////////////\n");
@ -228,9 +188,8 @@ NCD4_open(const char * path, int mode,
fflush(stderr);
}
#endif
/* Build the substrate metadata */
if((ret = NCD4_processdata(d4info->substrate.metadata))) goto done;
/* Build the substrate metadata */
ret = NCD4_metabuild(d4info->substrate.metadata,d4info->substrate.metadata->ncid);
if(ret != NC_NOERR && ret != NC_EVARSIZE) goto done;
@ -319,6 +278,30 @@ freeInfo(NCD4INFO* d4info)
free(d4info);
}
/* Reset NCD4INFO instance for new read request */
static void
resetInfoforRead(NCD4INFO* d4info)
{
if(d4info == NULL) return;
if(d4info->substrate.realfile
&& !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) {
/* We used real file, so we need to delete the temp file
unless we are debugging.
Assume caller has done nc_close|nc_abort on the ncid.
Note that in theory, this should not be necessary since
AFAIK the substrate file is still in def mode, and
when aborted, it should be deleted. But that is not working
for some reason, so we delete it ourselves.
*/
if(d4info->substrate.filename != NULL) {
unlink(d4info->substrate.filename);
}
}
NCD4_resetMeta(d4info->substrate.metadata);
nullfree(d4info->substrate.metadata);
d4info->substrate.metadata = NULL;
}
static void
freeCurl(NCD4curl* curl)
{
@ -523,3 +506,40 @@ getparam(NCD4INFO* info, const char* key)
return NULL;
return value;
}
/**************************************************/
static int
makesubstrate(NCD4INFO* d4info)
{
int ret = NC_NOERR;
int new = NC_NETCDF4;
int old = 0;
int ncid = 0;
int ncflags = NC_NETCDF4|NC_CLOBBER;
if(d4info->substrate.filename != NULL) {
/* reset the substrate */
nc_abort(d4info->substrate.nc4id);
d4info->substrate.nc4id = 0;
}
/* Create the hidden substrate netcdf file.
We want this hidden file to always be NC_NETCDF4, so we need to
force default format temporarily in case user changed it.
Since diskless is enabled, create file in-memory.
*/
ncflags |= NC_DISKLESS;
if(FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) {
/* Cause data to be dumped to real file */
ncflags |= NC_WRITE;
ncflags &= ~(NC_DISKLESS); /* use real file */
}
nc_set_default_format(new,&old); /* save and change */
ret = nc_create(d4info->substrate.filename,ncflags,&ncid);
nc_set_default_format(old,&new); /* restore */
/* Avoid fill on the substrate */
nc_set_fill(ncid,NC_NOFILL,NULL);
d4info->substrate.nc4id = ncid;
return THROW(ret);
}

63
libdap4/d4meta.c
View File

@ -96,16 +96,17 @@ done:
*/
NCD4meta*
NCD4_newmeta(size_t rawsize, void* rawdata)
NCD4_newmeta(NCD4INFO* info, size_t rawsize, void* rawdata)
{
NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta));
if(meta == NULL) return NULL;
meta->allnodes = nclistnew();
meta->serial.rawsize = rawsize;
meta->serial.rawdata = rawdata;
NCD4_resetSerial(&meta->serial,rawsize,rawdata);
#ifdef D4DEBUG
meta->debuglevel = 1;
#endif
meta->controller = info;
meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */
return meta;
}
@ -120,17 +121,27 @@ NCD4_reclaimMeta(NCD4meta* dataset)
{
int i;
if(dataset == NULL) return;
NCD4_resetMeta(dataset);
for(i=0;i<nclistlength(dataset->allnodes);i++) {
NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i);
reclaimNode(node);
}
nullfree(dataset->error.parseerror);
nullfree(dataset->error.message);
nullfree(dataset->error.context);
nullfree(dataset->error.otherinfo);
nullfree(dataset->serial.errdata);
nclistfree(dataset->allnodes);
nclistfree(dataset->groupbyid);
nclistfree(dataset->atomictypes);
free(dataset);
}
void
NCD4_resetMeta(NCD4meta* dataset)
{
if(dataset == NULL) return;
nullfree(dataset->error.parseerror); dataset->error.parseerror = NULL;
nullfree(dataset->error.message); dataset->error.message = NULL;
nullfree(dataset->error.context); dataset->error.context = NULL;
nullfree(dataset->error.otherinfo); dataset->error.otherinfo = NULL;
NCD4_resetSerial(&dataset->serial,0,NULL);
#if 0
for(i=0;i<nclistlength(dataset->blobs);i++) {
void* p = nclistget(dataset->blobs,i);
@ -138,31 +149,29 @@ NCD4_reclaimMeta(NCD4meta* dataset)
}
nclistfree(dataset->blobs);
#endif
nclistfree(dataset->allnodes);
nullfree(dataset->serial.dmr);
free(dataset);
}
void
reclaimNode(NCD4node* node)
{
if(node == NULL) return;
nullfree(node->name);
nclistfree(node->groups);
nclistfree(node->vars);
nclistfree(node->types);
nclistfree(node->dims);
nclistfree(node->attributes);
nclistfree(node->maps);
nclistfreeall(node->xmlattributes);
nclistfreeall(node->attr.values);
nclistfree(node->en.econsts);
nclistfree(node->group.elements);
nullfree(node->group.dapversion);
nullfree(node->group.dmrversion);
nullfree(node->group.datasetname);
nclistfree(node->group.varbyid);
nullfree(node->nc4.orig.name);
nullfree(node->name); node->name = NULL;
nclistfree(node->groups); node->groups = NULL;
nclistfree(node->vars); node->vars = NULL;
nclistfree(node->types); node->types = NULL;
nclistfree(node->dims); node->dims = NULL;
nclistfree(node->attributes); node->attributes = NULL;
nclistfreeall(node->mapnames); node->mapnames = NULL;
nclistfree(node->maps); node->maps = NULL;
nclistfreeall(node->xmlattributes); node->xmlattributes = NULL;
nclistfreeall(node->attr.values); node->attr.values = NULL;
nclistfree(node->en.econsts); node->en.econsts = NULL;
nclistfree(node->group.elements); node->group.elements = NULL;
nullfree(node->group.dapversion); node->group.dapversion = NULL;
nullfree(node->group.dmrversion); node->group.dmrversion = NULL;
nullfree(node->group.datasetname); node->group.datasetname = NULL;
nclistfree(node->group.varbyid); node->group.varbyid = NULL;
nullfree(node->nc4.orig.name); node->nc4.orig.name = NULL;
nullfree(node);
}

77
libdap4/d4parser.c
View File

@ -146,6 +146,7 @@ static void record(NCD4parser*, NCD4node* node);
static int splitOrigType(NCD4parser*, const char* fqn, NCD4node* var);
static void track(NCD4meta*, NCD4node* node);
static int traverse(NCD4parser*, ezxml_t dom);
static int parseForwards(NCD4parser* parser, NCD4node* root);
#ifndef FIXEDOPAQUE
static int defineBytestringType(NCD4parser*);
#endif
@ -179,7 +180,7 @@ NCD4_parse(NCD4meta* metadata)
parser->debuglevel = 1;
#endif
/*Walk the DOM tree */
/*Walk the DOM tree to build the DAP4 node tree*/
ret = traverse(parser,dom);
done:
@ -234,6 +235,12 @@ traverse(NCD4parser* parser, ezxml_t dom)
if(xattr != NULL) parser->metadata->root->group.dmrversion = strdup(xattr);
/* Recursively walk the tree */
if((ret = fillgroup(parser,parser->metadata->root,dom))) goto done;
/* Walk a second time to parse allowed forward refs:
1. <Map>
*/
if((ret = parseForwards(parser,parser->metadata->root))) goto done;
} else
FAIL(NC_EINVAL,"Unexpected dom root name: %s",dom->name);
done:
@ -379,8 +386,9 @@ parseVariable(NCD4parser* parser, NCD4node* container, ezxml_t xml, NCD4node** n
default:
ret = parseAtomicVar(parser,container,xml,&node);
}
*nodep = node;
if(ret == NC_NOERR) {
*nodep = node;
}
return THROW(ret);
}
@ -430,8 +438,6 @@ parseStructure(NCD4parser* parser, NCD4node* container, ezxml_t xml, NCD4node**
/* Parse attributes, dims, and maps into the var */
if((ret = parseMetaData(parser,var,xml))) goto done;
record(parser,var);
/* See if this var has UCARTAGORIGTYPE attribute */
if(parser->metadata->controller->controls.translation == NCD4_TRANSNC4) {
const char* typetag = ezxml_attr(xml,UCARTAGORIGTYPE);
@ -573,8 +579,6 @@ parseSequence(NCD4parser* parser, NCD4node* container, ezxml_t xml, NCD4node** n
/* Parse attributes, dims, and maps into var*/
if((ret = parseMetaData(parser,var,xml))) goto done;
record(parser,var);
/* See if this var has UCARTAGORIGTYPE attribute */
if(parser->metadata->controller->controls.translation == NCD4_TRANSNC4) {
const char* typetag = ezxml_attr(xml,UCARTAGORIGTYPE);
@ -601,10 +605,10 @@ parseGroups(NCD4parser* parser, NCD4node* parent, ezxml_t xml)
if(name == NULL) FAIL(NC_EBADNAME,"Group has no name");
if((ret=makeNode(parser,parent,x,NCD4_GROUP,NC_NULL,&group))) goto done;
group->group.varbyid = nclistnew();
classify(parent,group);
if((ret = fillgroup(parser,group,x))) goto done;
/* Parse group attributes */
if((ret = parseAttributes(parser,group,x))) goto done;
PUSH(parent->groups,group);
}
done:
return THROW(ret);
@ -699,15 +703,11 @@ parseMaps(NCD4parser* parser, NCD4node* var, ezxml_t xml)
ezxml_t x;
for(x=ezxml_child(xml, "Map");x!= NULL;x=ezxml_next(x)) {
NCD4node* mapref = NULL;
const char* fqn;
fqn = ezxml_attr(x,"name");
if(fqn == NULL)
FAIL(NC_ENOTVAR,"<Map> has no name attribute");
mapref = lookupFQN(parser,fqn,NCD4_VAR);
if(mapref == NULL)
FAIL(NC_ENOTVAR,"<Map> name does not refer to a variable: %s",fqn);
PUSH(var->maps,mapref);
PUSH(var->mapnames,strdup(fqn));
}
done:
return THROW(ret);
@ -730,8 +730,8 @@ parseAttributes(NCD4parser* parser, NCD4node* container, ezxml_t xml)
container->xmlattributes = nclistnew();
for(p=all;*p;p+=2) {
if(isReserved(*p)) {
nclistpush(container->xmlattributes,strdup(p[0]));
nclistpush(container->xmlattributes,strdup(p[1]));
PUSH(container->xmlattributes,strdup(p[0]));
PUSH(container->xmlattributes,strdup(p[1]));
}
}
}
@ -853,8 +853,6 @@ getOpaque(NCD4parser* parser, ezxml_t varxml, NCD4node* group)
goto done;
SETNAME(opaquetype,name);
opaquetype->opaque.size = len;
if(opaquetype != NULL)
record(parser,opaquetype);
}
}
done:
@ -886,7 +884,7 @@ getValueStrings(NCD4parser* parser, NCD4node* type, ezxml_t xattr, NClist* svalu
/* Need to de-escape the string */
es = NCD4_entityescape(s);
ds = NCD4_deescape(es);
nclistpush(svalues,ds);
PUSH(svalues,ds);
nullfree(es);
}
}
@ -1252,6 +1250,7 @@ makeNode(NCD4parser* parser, NCD4node* parent, ezxml_t xml, NCD4sort sort, nc_ty
SETNAME(node,name);
}
}
record(parser,node);
if(nodep) *nodep = node;
done:
return ret;
@ -1268,10 +1267,12 @@ makeNodeStatic(NCD4meta* meta, NCD4node* parent, NCD4sort sort, nc_type subsort,
node->sort = sort;
node->subsort = subsort;
node->container = parent;
#if 0
if(parent != NULL) {
if(parent->sort == NCD4_GROUP)
PUSH(parent->group.elements,node);
classify(parent,node);
}
#endif
track(meta,node);
if(nodep) *nodep = node;
return THROW(ret);
@ -1296,7 +1297,7 @@ makeAnonDim(NCD4parser* parser, const char* sizestr)
SETNAME(dim,name+1); /* leave out the '/' separator */
dim->dim.size = (long long)size;
dim->dim.isanonymous = 1;
PUSH(root->dims,dim);
classify(root,dim);
}
done:
return (ret?NULL:dim);
@ -1310,7 +1311,7 @@ static void
classify(NCD4node* container, NCD4node* node)
{
if(ISGROUP(container->sort))
nclistpush(container->group.elements,node);
PUSH(container->group.elements,node);
switch (node->sort) {
case NCD4_GROUP:
PUSH(container->groups,node);
@ -1508,7 +1509,7 @@ valueParse(NCD4node* type, const char* values0, NClist* vlist)
q = p - 1;
*p++ = '\0';
if(*q == '\r') {*q = '\0';}
nclistpush(vlist,strdup(line));
PUSH(vlist,strdup(line));
}
break;
case NC_CHAR:
@ -1526,7 +1527,7 @@ valueParse(NCD4node* type, const char* values0, NClist* vlist)
if(len > 0) {
c[0] = *q;
c[1] = '\0';
nclistpush(vlist,strdup(c));
PUSH(vlist,strdup(c));
}
}
break;
@ -1538,7 +1539,7 @@ valueParse(NCD4node* type, const char* values0, NClist* vlist)
for(line=values;*line;) {
size_t size = strlen(line);
if(size > 0)
nclistpush(vlist,strdup(line));
PUSH(vlist,strdup(line));
line += (size+1); /* skip terminating nul */
}
break;
@ -1564,3 +1565,31 @@ NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char*
if(attrp) *attrp = attr;
return NC_NOERR;
}
/*
Fill in forward references for selected Node:
1. <Map>
*/
static int
parseForwards(NCD4parser* parser, NCD4node* root)
{
int ret = NC_NOERR;
int i,j;
/* process all vars */
for(i=0;i<nclistlength(parser->vars);i++) {
NCD4node* var = (NCD4node*)nclistget(parser->vars,i);
/* Process the variable's maps */
for(j=0;j<nclistlength(var->mapnames);j++) {
const char* mapname = (const char*)nclistget(var->mapnames,j);
/* Find the corresponding variable */
NCD4node* mapref = lookupFQN(parser,mapname,NCD4_VAR);
if(mapref == NULL)
FAIL(NC_ENOTVAR,"<Map> name does not refer to a variable: %s",mapname);
PUSH(var->maps,mapref);
}
}
done:
return THROW(ret);
}

185
libdap4/d4read.c
View File

@ -19,9 +19,10 @@ See \ref copyright file for more info.
/* Do conversion if this code was compiled via Vis. Studio or Mingw */
/*Forward*/
static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, long*);
static int readfile(NCD4INFO* state, const NCURI*, const char* suffix, NCbytes* packet);
static int readfiletofile(NCD4INFO* state, const NCURI*, const char* suffix, FILE* stream, d4size_t*);
static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, long*);
static int readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet);
static int readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep);
static int readfileDAPDMR(NCD4INFO* state, const NCURI* uri, NCbytes* packet);
#ifdef HAVE_GETTIMEOFDAY
static double
@ -37,14 +38,40 @@ deltatime(struct timeval time0,struct timeval time1)
#endif
int
NCD4_readDMR(NCD4INFO* state)
NCD4_readDMR(NCD4INFO* state, int flags)
{
int stat = NC_NOERR;
long lastmodified = -1;
long lastmod = -1;
stat = readpacket(state,state->uri,state->curl->packet,NCD4_DMR,&lastmodified);
if(stat == NC_NOERR)
state->data.dmrlastmodified = lastmodified;
if((flags & NCF_ONDISK) == 0) {
ncbytesclear(state->curl->packet);
stat = readpacket(state,state->uri,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&lastmod);
if(stat == NC_NOERR)
state->data.dmrlastmodified = lastmod;
} else { /*((flags & NCF_ONDISK) != 0) */
NCURI* url = state->uri;
int fileprotocol = (strcmp(url->protocol,"file")==0);
if(fileprotocol) {
stat = readfiletofile(state, url, NCD4_DMR, NCD4_FORMAT_XML, state->data.ondiskfile, &state->data.datasize);
} else {
char* readurl = NULL;
int flags = 0;
if(!fileprotocol) flags |= NCURIQUERY;
flags |= NCURIENCODE;
flags |= NCURIPWD;
#ifdef FIX
ncurisetconstraints(url,state->constraint);
#endif
readurl = ncuribuild(url,NULL,".dmr.xml",NCURISVC);
if(readurl == NULL)
return THROW(NC_ENOMEM);
stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile,
&state->data.datasize, &lastmod);
nullfree(readurl);
if(stat == NC_NOERR)
state->data.dmrlastmodified = lastmod;
}
}
return THROW(stat);
}
@ -55,14 +82,15 @@ NCD4_readDAP(NCD4INFO* state, int flags)
long lastmod = -1;
if((flags & NCF_ONDISK) == 0) {
stat = readpacket(state,state->uri,state->curl->packet,NCD4_DAP,&lastmod);
ncbytesclear(state->curl->packet);
stat = readpacket(state,state->uri,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&lastmod);
if(stat == NC_NOERR)
state->data.daplastmodified = lastmod;
} else { /*((flags & NCF_ONDISK) != 0) */
NCURI* url = state->uri;
int fileprotocol = (strcmp(url->protocol,"file")==0);
if(fileprotocol) {
stat = readfiletofile(state, url, ".dap", state->data.ondiskfile, &state->data.datasize);
stat = readfiletofile(state, url, NCD4_DAP, NCD4_FORMAT_NONE, state->data.ondiskfile, &state->data.datasize);
} else {
char* readurl = NULL;
int flags = 0;
@ -72,7 +100,7 @@ NCD4_readDAP(NCD4INFO* state, int flags)
#ifdef FIX
ncurisetconstraints(url,state->constraint);
#endif
readurl = ncuribuild(url,NULL,".dods",NCURISVC);
readurl = ncuribuild(url,NULL,".dap",NCURISVC);
if(readurl == NULL)
return THROW(NC_ENOMEM);
stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile,
@ -96,24 +124,49 @@ dxxextension(int dxx)
return NULL;
}
static const char*
dxxformat(int fxx, int dxx)
{
switch(dxx) {
case NCD4_DMR:
switch(fxx) {
case NCD4_FORMAT_XML:return ".xml";
default: break;
}
break;
case NCD4_DAP:
switch(fxx) {
case NCD4_FORMAT_NONE:return "";
default: break;
}
break;
default: break;
}
return NULL;
}
static int
readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, long* lastmodified)
readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, long* lastmodified)
{
int stat = NC_NOERR;
int fileprotocol = 0;
const char* suffix = dxxextension(dxx);
CURL* curl = state->curl->curl;
#ifdef HAVE_GETTIMEOFDAY
struct timeval time0;
struct timeval time1;
#endif
char suffix[256];
suffix[0] = '\0';
strlcat(suffix,dxxextension(dxx),sizeof(suffix));
strlcat(suffix,dxxformat(fxx,dxx),sizeof(suffix));
fileprotocol = (strcmp(url->protocol,"file")==0);
if(fileprotocol) {
/* Short circuit file://... urls*/
/* We do this because the test code always needs to read files*/
stat = readfile(state, url,suffix,packet);
stat = readfile(state, url, dxx, fxx, packet);
} else {
char* fetchurl = NULL;
int flags = NCURIBASE;
@ -150,12 +203,13 @@ fail:
}
static int
readfiletofile(NCD4INFO* state, const NCURI* uri, const char* suffix, FILE* stream, d4size_t* sizep)
readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep)
{
int stat = NC_NOERR;
NCbytes* packet = ncbytesnew();
size_t len;
stat = readfile(state, uri,suffix,packet);
stat = readfile(state, uri, dxx, fxx, packet);
#ifdef D4DEBUG
fprintf(stderr,"readfiletofile: packet.size=%lu\n",
(unsigned long)ncbyteslength(packet));
@ -181,22 +235,29 @@ unwind:
}
static int
readfile(NCD4INFO* state, const NCURI* uri, const char* suffix, NCbytes* packet)
readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet)
{
int stat = NC_NOERR;
NCbytes* tmp = ncbytesnew();
char* filename = NULL;
ncbytescat(tmp,uri->path);
if(suffix != NULL) ncbytescat(tmp,suffix);
ncbytesnull(tmp);
filename = ncbytesextract(tmp);
ncbytesfree(tmp);
char suffix[256];
#ifdef HAVE_GETTIMEOFDAY
struct timeval time0;
struct timeval time1;
#endif
suffix[0] = '\0';
strlcat(suffix,dxxextension(dxx),sizeof(suffix));
#if 0
not needed strlcat(suffix,dxxformat(fxx,dxx),sizeof(suffix));
#endif
ncbytescat(tmp,uri->path);
ncbytescat(tmp,suffix);
ncbytesnull(tmp);
filename = ncbytesextract(tmp);
ncbytesfree(tmp);
nullfree(state->fileproto.filename);
state->fileproto.filename = filename; /* filename is alloc'd here anyway */
if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) {
@ -207,7 +268,63 @@ readfile(NCD4INFO* state, const NCURI* uri, const char* suffix, NCbytes* packet)
surl = ncuribuild((NCURI*)uri,NULL,NULL,NCURIALL);
nclog(NCLOGDBG,"fetch uri=%s file=%s",surl,filename);
}
switch (dxx) {
case NCD4_DMR:
if((stat = NC_readfile(filename,packet))) {
/* See if we can get the same thing from the .dap file */
stat = readfileDAPDMR(state,uri,packet);
}
break;
case NCD4_DAP:
case NCD4_DSR:
stat = NC_readfile(filename,packet);
break;
default: stat = NC_EDAP; break;
}
if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) {
double secs;
#ifdef HAVE_GETTIMEOFDAY
gettimeofday(&time1,NULL);
secs = deltatime(time0,time1);
#endif
nclog(NCLOGDBG,"%s fetch complete: %0.3f",suffix,secs);
}
return THROW(stat);
}
/* Extract the DMR from a DAP file */
static int
readfileDAPDMR(NCD4INFO* state, const NCURI* uri, NCbytes* packet)
{
int stat = NC_NOERR;
NCbytes* tmp = ncbytesnew();
char* filename = NULL;
NCD4HDR hdr;
#ifdef HAVE_GETTIMEOFDAY
struct timeval time0;
struct timeval time1;
#endif
ncbytescat(tmp,uri->path);
ncbytescat(tmp,".dap");
ncbytesnull(tmp);
filename = ncbytesextract(tmp);
ncbytesfree(tmp);
nullfree(state->fileproto.filename);
state->fileproto.filename = filename; /* filename is alloc'd here anyway */
if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) {
char* surl = NULL;
#ifdef HAVE_GETTIMEOFDAY
gettimeofday(&time0,NULL);
#endif
surl = ncuribuild((NCURI*)uri,NULL,".dap",NCURIALL);
nclog(NCLOGDBG,"fetch uri=%s file=%s",surl,filename);
}
stat = NC_readfile(filename,packet);
if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) {
double secs;
#ifdef HAVE_GETTIMEOFDAY
@ -216,5 +333,27 @@ readfile(NCD4INFO* state, const NCURI* uri, const char* suffix, NCbytes* packet)
#endif
nclog(NCLOGDBG,"fetch complete: %0.3f",secs);
}
if(stat != NC_NOERR) goto done;
/* Extract the DMR from the dap */
NCD4_getheader(ncbytescontents(packet),&hdr,NCD4_isLittleEndian());
if(hdr.count == 0 || (hdr.flags & NCD4_ERR_CHUNK))
return THROW(NC_EDMR);
/* patch up the packet */
{
int i;
size_t newlen;
for(i=0;i<4;i++)
ncbytesremove(packet,0); /* remove the hdr */
/* null terminate */
ncbytessetlength(packet,hdr.count-1);
ncbytesnull(packet);
/* Suppress nuls */
newlen = NCD4_elidenuls(ncbytescontents(packet),ncbyteslength(packet));
/* reset packet length */
ncbytessetlength(packet,newlen);
}
done:
return THROW(stat);
}

4
libdap4/d4read.h
View File

@ -6,7 +6,7 @@
#ifndef D4READ_H
#define D4READ_H
extern int NCD4_readDMR(NCD4INFO*);
extern int NCD4_readDAP(NCD4INFO*, int inmemory);
extern int NCD4_readDMR(NCD4INFO*, int flags);
extern int NCD4_readDAP(NCD4INFO*, int flags);
#endif /*READ_H*/

16
libdap4/d4util.c
View File

@ -467,3 +467,19 @@ NCD4_getcounter(void* p)
memcpy(&v,p,sizeof(v));
return (d4size_t)v;
}
void*
NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian)
{
unsigned char bytes[4];
memcpy(bytes,p,sizeof(bytes));
p = INCR(p,4); /* on-the-wire hdr is 4 bytes */
/* assume header is network (big) order */
hdr->flags = bytes[0]; /* big endian => flags are in byte 0 */
hdr->flags &= NCD4_ALL_CHUNK_FLAGS; /* Ignore extraneous flags */
bytes[0] = 0; /* so we can do byte swap to get count */
if(hostlittleendian)
swapinline32(bytes); /* host is little endian */
hdr->count = *(unsigned int*)bytes; /* get count */
return p;
}

22
libdap4/d4varx.c
View File

@ -138,11 +138,25 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp,
goto done;
info = getdap(ncp);
if(info == NULL)
{ret = THROW(NC_EBADID); goto done;}
meta = info->substrate.metadata;
if(meta == NULL)
{ret = THROW(NC_EBADID); goto done;}
/* If the data has not already been read and processed, then do so. */
if(meta->serial.dap == NULL) {
size_t len = 0;
void* content = NULL;
/* (Re)Build the meta data; sets serial.rawdata */
NCD4_resetMeta(info->substrate.metadata);
meta->controller = info;
meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */
if((ret=NCD4_readDAP(info, info->controls.flags.flags))) goto done;
len = ncbyteslength(info->curl->packet);
content = ncbytesextract(info->curl->packet);
NCD4_resetSerial(&meta->serial, len, content);
/* Process the data part */
if((ret=NCD4_dechunk(meta))) goto done;
if((ret = NCD4_processdata(info->substrate.metadata))) goto done;
}
if((ret = NCD4_findvar(ncp,ncid,varid,&var,&group))) goto done;

9
libdap4/ncd4.h
View File

@ -97,7 +97,7 @@ extern int NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime);
extern int NCD4_ping(const char* url);
/* From d4read.c */
extern int NCD4_readDMR(NCD4INFO* state);
extern int NCD4_readDMR(NCD4INFO* state, int flags);
extern int NCD4_readDAP(NCD4INFO* state, int flags);
/* From d4parser.c */
@ -110,8 +110,9 @@ extern int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname,
extern int NCD4_print(NCD4meta*, NCbytes* output);
/* From d4meta.c */
extern NCD4meta* NCD4_newmeta(size_t size, void* rawdata);
extern NCD4meta* NCD4_newmeta(NCD4INFO*, size_t size, void* rawdata);
extern void NCD4_reclaimMeta(NCD4meta*);
extern void NCD4_resetMeta(NCD4meta*);
extern void reclaimNode(NCD4node* node);
extern void NCD4_setdebuglevel(NCD4meta*,int);
extern int NCD4_metabuild(NCD4meta*, int ncid);
@ -121,6 +122,8 @@ extern int NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node*
/* From d4chunk.c */
extern int NCD4_dechunk(NCD4meta*);
extern int NCD4_infermode(NCD4meta* meta);
struct NCD4serial;
extern void NCD4_resetSerial(struct NCD4serial* serial, size_t rawsize, void* rawdata);
/* From d4swap.c */
extern int NCD4_swapdata(NCD4meta*, NClist* topvars);
@ -147,6 +150,7 @@ extern int NCD4_parseFQN(const char* fqn0, NClist* pieces);
extern char* NCD4_deescape(const char* esc);
extern char* NCD4_entityescape(const char* s);
extern size_t NCD4_elidenuls(char* s, size_t slen);
extern void* NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian);
/* From d4dump.c */
extern void NCD4_dumpbytes(size_t size, const void* data0, int swap);
@ -167,6 +171,7 @@ extern int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char* v
/* d4file.c */
extern void NCD4_applyclientparamcontrols(NCD4INFO*);
extern int NCD4_readDMRorDAP(NCD4INFO* d4info, NCD4mode mode);
/* ncd4dispatch.c */
struct NC_reservedatt; /*forward*/

29
libdap4/ncd4types.h
View File

@ -42,11 +42,27 @@ Currently turned off because semantics are unclear.
typedef struct NCD4INFO NCD4INFO;
typedef enum NCD4CSUM NCD4CSUM;
typedef enum NCD4mode NCD4mode;
typedef enum NCD4format NCD4format;
typedef enum NCD4translation NCD4translation;
typedef struct NCD4curl NCD4curl;
typedef struct NCD4meta NCD4meta;
typedef struct NCD4node NCD4node;
typedef struct NCD4params NCD4params;
typedef struct NCD4HDR NCD4HDR;
/* Define the NCD4HDR flags */
/* Header flags */
#define NCD4_LAST_CHUNK (1)
#define NCD4_ERR_CHUNK (2)
#define NCD4_LITTLE_ENDIAN_CHUNK (4)
#ifdef CHECKSUMHACK
#define NCD4_NOCHECKSUM_CHUNK (8)
#else
#define NCD4_NOCHECKSUM_CHUNK (0)
#endif
#define NCD4_ALL_CHUNK_FLAGS (NCD4_LAST_CHUNK|NCD4_ERR_CHUNK|NCD4_LITTLE_ENDIAN_CHUNK|NCD4_NOCHECKSUM_CHUNK)
/**************************************************/
/* DMR Tree node sorts */
@ -117,6 +133,11 @@ NCD4_DAP = 2,
NCD4_DSR = 4
};
/* Define possible retrieval formats */
enum NCD4format {
NCD4_FORMAT_NONE = 0,
NCD4_FORMAT_XML = 1
};
/* Define storage for all the primitive types (plus vlen) */
union ATOMICS {
@ -140,6 +161,11 @@ union ATOMICS {
#endif
};
/**************************************************/
/* Define the structure of the chunk header */
struct NCD4HDR {unsigned int flags; unsigned int count;};
/**************************************************/
/* !Node type for the NetCDF-4 metadata produced from
parsing the DMR tree.
@ -158,6 +184,7 @@ struct NCD4node {
NClist* types; /* NClist<NCD4node*> types in group */
NClist* dims; /* NClist<NCD4node*>; dimdefs in group, dimrefs in vars */
NClist* attributes; /* NClist<NCD4node*> */
NClist* mapnames; /* NClist<char*> */
NClist* maps; /* NClist<NCD4node*> */
NClist* xmlattributes; /* NClist<String> */
NCD4node* basetype;
@ -261,7 +288,7 @@ typedef struct NCD4parser {
int debuglevel;
NCD4meta* metadata;
/* Capture useful subsets of dataset->allnodes */
NClist* types; /*list<NCD4node>*/
NClist* types; /*list<NCD4node>; user-defined types only*/
NClist* dims; /*list<NCD4node>*/
NClist* vars; /*list<NCD4node>*/
NClist* groups; /*list<NCD4node>*/