mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-04-12 18:10:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Additional DAP4 fixes

Browse Source 
This change-set modifies PR https://github.com/Unidata/netcdf-c/pull/2555
to add the changes listed below. Most of these changes are required
by changes to the Java remotetest.unidata.ucar.edu server.

## DAP4 Related Changes
* Add tests *dap4_test/test_constraints.sh* and *dap4_test/test_hyrax.sh*.
* Provide explicit list of remotetest files to test.
* Cleanup local checksum computing and verification.
* Define a temporary Hyrax hack flag to deal with the way Hyrax handles checksums and add "#hyrax" fragment flag for it.
* Add a hack to get past an LGTM problem with using "http:".
* Improve debug support.

## Other Changes
* Cleanup the recipe in *docs/nczarr.md* for building *aws-sdk-cpp* library.
This commit is contained in:
Dennis Heimbigner 2023-01-18 19:47:29 -07:00
parent 9dfafe6c63
commit d1d2808919
40 changed files with 5479 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -1062,7 +1062,7 @@ ENDIF()
IF(ENABLE_DAP)
SET(USE_DAP ON CACHE BOOL "")
SET(ENABLE_DAP2 ON CACHE BOOL "")
IF(ENABLE_HDF5)
MESSAGE(STATUS "Enabling DAP4")
SET(ENABLE_DAP4 ON CACHE BOOL "")

10
dap4_test/CMakeLists.txt
View File

@ -52,14 +52,17 @@ ENDIF()
IF(ENABLE_DAP_REMOTE_TESTS)
add_sh_test(dap4_test test_remote)
# add_sh_test(dap4_test test_hyrax)
add_sh_test(dap4_test test_hyrax)
# add_sh_test(dap4_test test_thredds)
ENDIF(ENABLE_DAP_REMOTE_TESTS)
ENDIF(ENABLE_TESTS)
#FILE(COPY ./baselineraw DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
#FILE(COPY ./baseline DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
#FILE(COPY ./baselineraw DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
#FILE(COPY ./baselineremote DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
#FILE(COPY ./baselinethredds DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
#FILE(COPY ./baselinehyrax DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
#FILE(COPY ./cdltestfiles DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
#FILE(COPY ./rawtestfiles DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
@ -69,6 +72,9 @@ ${CMAKE_CURRENT_SOURCE_DIR}/cdltestfiles
${CMAKE_CURRENT_SOURCE_DIR}/rawtestfiles
${CMAKE_CURRENT_SOURCE_DIR}/baseline
${CMAKE_CURRENT_SOURCE_DIR}/baselineraw
${CMAKE_CURRENT_SOURCE_DIR}/baselineremote
#${CMAKE_CURRENT_SOURCE_DIR}/baselinerthredds
${CMAKE_CURRENT_SOURCE_DIR}/baselinehyrax
)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am)
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

14
dap4_test/Makefile.am
View File

@ -36,23 +36,26 @@ TESTS += test_parse.sh test_meta.sh test_data.sh test_raw.sh
# Note: This program name was changed to findtestserver4
# to avoid cmake complaint about duplicate targets.
noinst_PROGRAMS += findtestserver4 pingurl4
noinst_PROGRAMS += findtestserver4 pingurl4 dump
findtestserver4_SOURCES = findtestserver4.c
pingurl4_SOURCES = pingurl4.c
dump_SOURCES = dump.c
# Disable Dap4 Remote Tests until the test server is working
if BUILD_UTILITIES
if ENABLE_DAP_REMOTE_TESTS
TESTS += test_remote.sh
# Test require help at the server end
# TESTS += test_hyrax.sh test_thredds.sh
TESTS += test_constraints.sh
TESTS += test_hyrax.sh
#TESTS += test_thredds.sh
endif
endif
EXTRA_DIST = CMakeLists.txt test_common.h build.sh \
d4test_common.sh \
d4manifest.sh d4test_common.sh \
test_curlopt.sh test_data.sh test_hyrax.sh test_meta.sh \
test_parse.sh test_raw.sh test_remote.sh test_thredds.sh \
test_parse.sh test_raw.sh \
test_remote.sh test_constraints.sh test_thredds.sh \
cdltestfiles rawtestfiles \
baseline baselineraw baselineremote baselinehyrax baselinethredds
@ -84,3 +87,4 @@ clean-local-check:
# in the GenerateRaw.java program in the thredds tree.
maketests::
bash ./maketests.sh

4604
dap4_test/baselinehyrax/AIRS.2002.12.01.L3.RetStd_H031.v4.0.21.0.G06101132853.hdf.hyrax Normal file
View File

File diff suppressed because it is too large Load Diff

14
dap4_test/baselinehyrax/amsre_20060131v5.dat.hyrax Normal file
View File

@ -0,0 +1,14 @@
netcdf amsre_20060131v5 {
dimensions:
_Anonymous3 = 3 ;
_Anonymous6 = 6 ;
variables:
byte time_a(_Anonymous3, _Anonymous6) ;
string time_a:Equator_Crossing_Time = "1:30 PM" ;
data:
time_a =
-1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1 ;
}

104
dap4_test/baselinehyrax/nc4_nc_classic_comp.nc.hyrax Normal file
View File

@ -0,0 +1,104 @@
netcdf nc4_nc_classic_comp {
dimensions:
lat = 6 ;
lon = 5 ;
time = 2 ;
variables:
int lat(lat) ;
string lat:units = "degrees_north" ;
int lon(lon) ;
string lon:units = "degrees_east" ;
int time(time) ;
string time:units = "seconds" ;
float z(time, lat, lon) ;
string z:units = "meters" ;
z:valid_range = 0., 5000. ;
float z:_FillValue = 1.f ;
string z:_edu.ucar.maps = "/time", "/lat", "/lon" ;
float t(time, lat, lon) ;
string t:_edu.ucar.maps = "/time", "/lat", "/lon" ;
double p(time, lat, lon) ;
double p:_FillValue = -9999. ;
string p:_edu.ucar.maps = "/time", "/lat", "/lon" ;
short rh(time, lat, lon) ;
short rh:_FillValue = -1s ;
string rh:_edu.ucar.maps = "/time", "/lat", "/lon" ;
short pixel(lat, lon) ;
string pixel:_edu.ucar.maps = "/lat", "/lon" ;
string source(lat) ;
source:string_length = 5 ;
string source:_edu.ucar.maps = "/lat" ;
data:
lat = 0, 10, 20, 30, 40, 50 ;
lon = -140, -118, -96, -84, -52 ;
time = 1, 2 ;
z =
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10 ;
t =
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1 ;
p =
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1 ;
rh =
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2 ;
pixel =
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7 ;
source = "aaaaa", "aaaaa", "aaaaa", "aaaaa", "aaaaa", "aaaaa" ;
}

104
dap4_test/baselinehyrax/nc4_nc_classic_no_comp.nc.hyrax Normal file
View File

@ -0,0 +1,104 @@
netcdf nc4_nc_classic_no_comp {
dimensions:
lat = 6 ;
lon = 5 ;
time = 2 ;
variables:
int lat(lat) ;
string lat:units = "degrees_north" ;
int lon(lon) ;
string lon:units = "degrees_east" ;
int time(time) ;
string time:units = "seconds" ;
float z(time, lat, lon) ;
string z:units = "meters" ;
z:valid_range = 0., 5000. ;
float z:_FillValue = 1.f ;
string z:_edu.ucar.maps = "/time", "/lat", "/lon" ;
float t(time, lat, lon) ;
string t:_edu.ucar.maps = "/time", "/lat", "/lon" ;
double p(time, lat, lon) ;
double p:_FillValue = -9999. ;
string p:_edu.ucar.maps = "/time", "/lat", "/lon" ;
short rh(time, lat, lon) ;
short rh:_FillValue = -1s ;
string rh:_edu.ucar.maps = "/time", "/lat", "/lon" ;
short pixel(lat, lon) ;
string pixel:_edu.ucar.maps = "/lat", "/lon" ;
string source(lat) ;
source:string_length = 5 ;
string source:_edu.ucar.maps = "/lat" ;
data:
lat = 0, 10, 20, 30, 40, 50 ;
lon = -140, -118, -96, -84, -52 ;
time = 1, 2 ;
z =
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10,
10, 10, 10, 10, 10 ;
t =
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1,
1, 1, 1, 1, 1 ;
p =
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1,
0.1, 0.1, 0.1, 0.1, 0.1 ;
rh =
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2 ;
pixel =
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7 ;
source = "aaaaa", "aaaaa", "aaaaa", "aaaaa", "aaaaa", "aaaaa" ;
}

32
dap4_test/baselinehyrax/nc4_strings.nc.hyrax Normal file
View File

@ -0,0 +1,32 @@
netcdf nc4_strings {
dimensions:
lat = 6 ;
lon = 5 ;
variables:
int lat(lat) ;
int lon(lon) ;
string station(lat, lon) ;
string station:who = "james" ;
string station:names = "site_1", "site_2", "site_3" ;
string station:_edu.ucar.maps = "/lat", "/lon" ;
string scan_line(lon) ;
string scan_line:_edu.ucar.maps = "/lon" ;
string codec_name ;
data:
lat = 0, 10, 20, 30, 40, 50 ;
lon = -140, -118, -96, -84, -52 ;
station =
"one", "two", "three", "four", "five",
"one_b", "two_b", "three_b", "four_b", "five_b",
"one_c", "two_c", "three_c", "four_c", "five_c",
"one", "two", "three", "four", "five",
"one", "two", "three", "four", "five",
"one_f", "two_f", "three_f", "four_f", "five_f" ;
scan_line = "r", "r1", "r2", "r3", "r4" ;
codec_name = "mp3" ;
}

32
dap4_test/baselinehyrax/nc4_strings_comp.nc.hyrax Normal file
View File

@ -0,0 +1,32 @@
netcdf nc4_strings_comp {
dimensions:
lat = 6 ;
lon = 5 ;
variables:
int lat(lat) ;
int lon(lon) ;
string station(lat, lon) ;
string station:who = "james" ;
string station:names = "site_1", "site_2", "site_3" ;
string station:_edu.ucar.maps = "/lat", "/lon" ;
string scan_line(lon) ;
string scan_line:_edu.ucar.maps = "/lon" ;
string codec_name ;
data:
lat = 0, 10, 20, 30, 40, 50 ;
lon = -140, -118, -96, -84, -52 ;
station =
"one", "two", "three", "four", "five",
"one_b", "two_b", "three_b", "four_b", "five_b",
"one_c", "two_c", "three_c", "four_c", "five_c",
"one", "two", "three", "four", "five",
"one", "two", "three", "four", "five",
"one_f", "two_f", "three_f", "four_f", "five_f" ;
scan_line = "r", "r1", "r2", "r3", "r4" ;
codec_name = "mp3" ;
}

53
dap4_test/baselinehyrax/nc4_unsigned_types.nc.hyrax Normal file
View File

@ -0,0 +1,53 @@
netcdf nc4_unsigned_types {
dimensions:
lat = 6 ;
lon = 5 ;
time = 2 ;
variables:
int lat(lat) ;
string lat:units = "degrees_north" ;
int lon(lon) ;
string lon:units = "degrees_east" ;
int time(time) ;
string time:units = "seconds" ;
uint temp(time, lat, lon) ;
string temp:_edu.ucar.maps = "/time", "/lat", "/lon" ;
ushort rh(time, lat, lon) ;
ushort rh:_FillValue = 9999US ;
string rh:_edu.ucar.maps = "/time", "/lat", "/lon" ;
data:
lat = 0, 10, 20, 30, 40, 50 ;
lon = -140, -118, -96, -84, -52 ;
time = 1, 2 ;
temp =
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7 ;
rh =
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2 ;
}

53
dap4_test/baselinehyrax/nc4_unsigned_types_comp.nc.hyrax Normal file
View File

@ -0,0 +1,53 @@
netcdf nc4_unsigned_types_comp {
dimensions:
lat = 6 ;
lon = 5 ;
time = 2 ;
variables:
int lat(lat) ;
string lat:units = "degrees_north" ;
int lon(lon) ;
string lon:units = "degrees_east" ;
int time(time) ;
string time:units = "seconds" ;
uint temp(time, lat, lon) ;
string temp:_edu.ucar.maps = "/time", "/lat", "/lon" ;
ushort rh(time, lat, lon) ;
ushort rh:_FillValue = 9999US ;
string rh:_edu.ucar.maps = "/time", "/lat", "/lon" ;
data:
lat = 0, 10, 20, 30, 40, 50 ;
lon = -140, -118, -96, -84, -52 ;
time = 1, 2 ;
temp =
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7,
7, 7, 7, 7, 7 ;
rh =
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2,
2, 2, 2, 2, 2 ;
}

23
dap4_test/baselinehyrax/ref_tst_compounds.nc.hyrax Normal file
View File

@ -0,0 +1,23 @@
netcdf ref_tst_compounds {
types:
compound obs_t {
short day ;
short elev ;
int count ;
float relhum ;
double time ;
}; // obs_t
dimensions:
n = 3 ;
variables:
obs_t obs(n) ;
obs:day = -99s ;
obs:elev = -99s ;
obs:count = -99 ;
obs:relhum = -99.00005f ;
obs:time = -99. ;
data:
obs = {15, 2, 1, 0.5, 3600.01}, {157, -99, -99, -99, -99},
{20, 6, 3, 0.75, 5000.01} ;
}

27
dap4_test/baselineremote/test_atomic_array.1.nc.ncdump Normal file
View File

@ -0,0 +1,27 @@
netcdf test_atomic_array {
types:
opaque(16) opaque16_t ;
dimensions:
_Anonymous1 = 1 ;
_Anonymous2 = 2 ;
variables:
ubyte vu8(_Anonymous1, _Anonymous2) ;
double vd(_Anonymous1) ;
string vs(_Anonymous1, _Anonymous1) ;
opaque16_t vo(_Anonymous1, _Anonymous1) ;
// global attributes:
string :_dap4.ce = "/vu8[1][0:2:2];/vd[1];/vs[1][0];/vo[0][1]" ;
data:
vu8 =
3, 5 ;
vd = 1024.8 ;
vs =
"Καλημέα" ;
vo =
0XABCDEF00000000000000000000000000 ;
}

12
dap4_test/baselineremote/test_atomic_array.2.nc.ncdump Normal file
View File

@ -0,0 +1,12 @@
netcdf test_atomic_array {
dimensions:
_Anonymous3 = 3 ;
variables:
short v16(_Anonymous3) ;
// global attributes:
string :_dap4.ce = "/v16[0:1,3]" ;
data:
v16 = 1, 2, 4 ;
}

12
dap4_test/baselineremote/test_atomic_array.3.nc.ncdump Normal file
View File

@ -0,0 +1,12 @@
netcdf test_atomic_array {
dimensions:
_Anonymous3 = 3 ;
variables:
short v16(_Anonymous3) ;
// global attributes:
string :_dap4.ce = "/v16[3,0:1]" ;
data:
v16 = 4, 1, 2 ;
}

18
dap4_test/baselineremote/test_enum_array.6.nc.ncdump Normal file
View File

@ -0,0 +1,18 @@
netcdf test_enum_array {
types:
byte enum cloud_class_t {Clear = 0, Cumulonimbus = 1, Stratus = 2,
Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6,
Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10,
Missing = 127} ;
dimensions:
_Anonymous2 = 2 ;
variables:
cloud_class_t primary_cloud(_Anonymous2) ;
cloud_class_t primary_cloud:_FillValue = Missing ;
// global attributes:
string :_dap4.ce = "/primary_cloud[1:2:4]" ;
data:
primary_cloud = Stratus, Cumulonimbus ;
}

12
dap4_test/baselineremote/test_one_vararray.4.nc.ncdump Normal file
View File

@ -0,0 +1,12 @@
netcdf test_one_vararray {
dimensions:
_Anonymous1 = 1 ;
variables:
int t(_Anonymous1) ;
// global attributes:
string :_dap4.ce = "/t[1]" ;
data:
t = 37 ;
}

12
dap4_test/baselineremote/test_one_vararray.5.nc.ncdump Normal file
View File

@ -0,0 +1,12 @@
netcdf test_one_vararray {
dimensions:
_Anonymous2 = 2 ;
variables:
int t(_Anonymous2) ;
// global attributes:
string :_dap4.ce = "/t[0:1]" ;
data:
t = 17, 37 ;
}

16
dap4_test/baselineremote/test_opaque_array.7.nc.ncdump Normal file
View File

@ -0,0 +1,16 @@
netcdf test_opaque_array {
types:
opaque(16) opaque16_t ;
dimensions:
_Anonymous1 = 1 ;
_Anonymous2 = 2 ;
variables:
opaque16_t vo2(_Anonymous1, _Anonymous2) ;
// global attributes:
string :_dap4.ce = "/vo2[1][0:1]" ;
data:
vo2 =
0XFEDCBA98765432100000000000000000, 0XFEDCBA99999999990000000000000000 ;
}

19
dap4_test/baselineremote/test_struct_array.8.nc.ncdump Normal file
View File

@ -0,0 +1,19 @@
netcdf test_struct_array {
types:
compound s_t {
int x ;
int y ;
}; // s_t
dimensions:
_Anonymous2 = 2 ;
variables:
s_t s(_Anonymous2, _Anonymous2) ;
// global attributes:
string :_dap4.ce = "/s[0:2:3][0:1]" ;
data:
s =
{1, -1}, {17, 37},
{-4, 12}, {-8, 8} ;
}

51
dap4_test/d4manifest.sh Normal file
View File

@ -0,0 +1,51 @@
# not executable
if test "x$SETX" = x1 ; then set -x ; fi
dap4_manifest="\
test_atomic_array \
test_atomic_types \
test_enum_1 \
test_enum_2 \
test_enum_3 \
test_enum_array \
test_fill \
test_groups1 \
test_misc1 \
test_one_var \
test_one_vararray \
test_opaque \
test_opaque_array \
test_struct1 \
test_struct_array \
test_struct_nested \
test_struct_nested3 \
test_struct_type \
test_test \
test_unlim \
test_unlim1 \
test_utf8 \
test_vlen1 \
test_vlen10 \
test_vlen11 \
test_vlen2 \
test_vlen3 \
test_vlen4 \
test_vlen5 \
test_vlen6 \
test_vlen7 \
test_vlen8 \
test_vlen9 \
test_zerodim \
"
constrained_manifest="\
test_atomic_array?/vu8[1][0:2:2];/vd[1];/vs[1][0];/vo[0][1]=1 \
test_atomic_array?/v16[0:1,3]=2 \
test_atomic_array?/v16[3,0:1]=3 \
test_one_vararray?/t[1]=4 \
test_one_vararray?/t[0:1]=5 \
test_enum_array?/primary_cloud[1:2:4]=6 \
test_opaque_array?/vo2[1][0:1]=7 \
test_struct_array?/s[0:2:3][0:1]=8 \
"

23
dap4_test/d4test_common.sh
View File

@ -1,3 +1,7 @@
# not executable
. ./d4manifest.sh
if test "x$SETX" = x1 ; then set -x ; fi
if test "x$NOOPTIONS" = x ; then
@ -92,18 +96,15 @@ suppress() {
done
}
# Compute the set of testable names using cdltestfiles
# Compute the set of testable names using the manifest.
EXCLUDEDFILES="test_vlen9 test_vlen10"
computetestablefiles() {
local F0 firs excluded
cd ${CDLTESTFILES}
F0=`ls -1 *.cdl | sed -e 's/[.]cdl//g' | tr '\r\n' ' '`
cd ..
# remove untestable files
F=
first=0
for f in $F0 ; do
for f in ${dap4_manifest} ; do
excluded=0
for x in $EXCLUDEDFILES ; do
if test "x$x" = "x$f" ; then excluded=1; break; fi
@ -114,6 +115,18 @@ computetestablefiles() {
done
}
# Split a d4manifest.sh constraint line
# Result is to set 4 variables: FILE QUERY INDEX FRAG
splitconstraint() {
local tpl
tpl="$1"
FILE=`echo $tpl | cut -d"?" -f1`
tpl=`echo $tpl | cut -d"?" -f2`
QUERY=`echo $tpl | cut -d"=" -f1`
INDEX=`echo $tpl | cut -d"=" -f2`
FRAG="#dap4&log&show=fetch"
}
VG="valgrind --leak-check=full --error-exitcode=1 --num-callers=100"
if test "x$USEVG" = x ; then VG=; fi

13
dap4_test/dump.c
View File

@ -44,6 +44,7 @@ main(int argc, char *argv[])
char* tag = NULL;
size_t offset = 0;
size_t len = 0;
char* data0 = NULL;
char* data = NULL;
int swap = 0;
@ -79,10 +80,16 @@ main(int argc, char *argv[])
}
if(tag == NULL) tag = strdup(progname);
if(ncaux_readfile(fname,&len,&((void*)data))) usage();
data += offset;
if(ncaux_readfile(fname,&len,(void**)&data0)) usage();
if(offset >= len) {
fprintf(stderr,"Offset too large: file length = %u\n",(unsigned)len);
return 1;
}
data = data0 + offset;
NCD4_tagdump(len,data,swap,tag);
nullfree(data);
nullfree(data0);
nullfree(fname);
nullfree(tag);
return 0;
}

51
dap4_test/test_constraints.sh Executable file
View File

@ -0,0 +1,51 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. ${srcdir}/d4test_common.sh
set -e
#BIG=1
CSUM=1
setresultdir results_test_constraints
TESTSERVER=`${execdir}/findtestserver4 dap4 d4ts`
if test "x$TESTSERVER" = x ; then
echo "***XFAIL: Cannot find d4ts testserver"
exit 1
fi
for triple in ${constrained_manifest} ; do
# split the triple into its parts
splitconstraint $triple
if test "x$BIG" = x1; then
FRAG="${FRAG}&ucar.littleendian=0"
fi
# Fix up QUERY
QUERY="dap4.ce=${QUERY}"
if test "x$CSUM" = x1 ; then
QUERY="${QUERY}&dap4.checksum=true"
else
QUERY="${QUERY}&dap4.checksum=false"
fi
QUERY="?${QUERY}"
# Build complete test url
URL="${TESTSERVER}/testfiles/${FILE}.nc${QUERY}${FRAG}"
# and outputfile stem
OFILE="${FILE}.${INDEX}.nc"
${NCDUMP} ${DUMPFLAGS} "${URL}" > ${builddir}/results_test_constraints/${OFILE}.ncdump
if test "x${TEST}" = x1 ; then
diff -wBb "${BASELINEREM}/${OFILE}.ncdump" "${builddir}/results_test_constraints/${OFILE}.ncdump"
elif test "x${RESET}" = x1 ; then
echo "${OFILE}:"
rm -f "${BASELINEREM}/${OFILE}.ncdump"
cp "${builddir}/results_test_constraints/${OFILE}.ncdump" "${BASELINEREM}/${OFILE}.ncdump"
fi
done
finish

51
dap4_test/test_hyrax.sh
View File

@ -11,9 +11,9 @@ set -e
echo "test_hyrax.sh:"
FRAG=""
F="\
AIRS/AIRH3STM.003/2002.12.01/AIRS.2002.12.01.L3.RetStd_H031.v4.0.21.0.G06101132853.hdf?/TotalCounts_A \
RSS/amsre/bmaps_v05/y2006/m01/amsre_20060131v5.dat?/time_a[0:2][0:5] \
nc4_test_files/nc4_nc_classic_no_comp.nc \
nc4_test_files/nc4_nc_classic_comp.nc \
nc4_test_files/nc4_unsigned_types.nc \
@ -21,9 +21,7 @@ nc4_test_files/nc4_unsigned_types_comp.nc \
nc4_test_files/nc4_strings.nc \
nc4_test_files/nc4_strings_comp.nc \
nc4_test_files/ref_tst_compounds.nc \
hyrax/RSS/amsre/bmaps_v05/y2006/m01/amsre_20060131v5.dat?dap4.ce=time_a \
"
F=nc4_test_files/nc4_nc_classic_no_comp.nc
failure() {
echo "*** Fail: $1"
@ -37,24 +35,39 @@ echo "***XFAIL: Cannot find test.opendap.org testserver; test skipped"
exit 0
fi
makehyraxurl() {
if test "x$QUERY" != x ; then QUERY="&dap4.ce=$QUERY"; fi
QUERY="?dap4.checksum=true${QUERY}"
URL="${TESTSERVER}/${PREFIX}/${FILE}${QUERY}"
URL="$URL#dap4&hyrax"
URL="$URL&log&show=fetch"
}
hyraxsplit() {
P="$1"
QUERY=`echo $P | cut -d? -f2`
if test "x$QUERY" = "x$P" ; then QUERY="" ; fi
P=`echo $P | cut -d? -f1`
FILE=`basename $P`
PREFIX=`dirname $P`
}
if test "x${RESET}" = x1 ; then rm -fr ${BASELINEHY}/*.hyrax ; fi
for f in $F ; do
makeurl "dap4://test.opendap.org/opendap" "$f?dap4.checksum=true"
URL="$URL#log&show=fetch"
echo "testing: $URL"
if ! ${NCDUMP} ${DUMPFLAGS} "${URL}" > ./results_test_hyrax/${base}.hyrax; then
failure "${URL}"
fi
if test "x${TEST}" = x1 ; then
if ! diff -wBb ${BASELINEHY}/${base}.hyrax ./results_test_hyrax/${base}.hyrax ; then
failure "diff ${base}.hyrax"
fi
elif test "x${RESET}" = x1 ; then
echo "${f}:"
cp ./results_test_hyrax/${base}.hyrax ${BASELINEHY}/${base}.hyrax
hyraxsplit $f
makehyraxurl
echo "testing: $URL"
if ! ${NCDUMP} ${DUMPFLAGS} "${URL}" > ./results_test_hyrax/${FILE}.hyrax; then
failure "${URL}"
fi
if test "x${TEST}" = x1 ; then
if ! diff -wBb ${BASELINEHY}/${FILE}.hyrax ./results_test_hyrax/${FILE}.hyrax ; then
failure "diff ${FILE}.hyrax"
fi
elif test "x${RESET}" = x1 ; then
echo "${FILE}:"
cp ./results_test_hyrax/${FILE}.hyrax ${BASELINEHY}/${FILE}.hyrax
fi
done
echo "*** Pass"

2
dap4_test/test_raw.sh
View File

@ -42,7 +42,7 @@ setresultdir results_test_raw
if test "x${RESET}" = x1 ; then rm -fr ${BASELINERAW}/*.ncdump ; fi
for f in $F ; do
echo "testing: $f"
URL="[log][dap4]file://${RAWTESTFILES}/${f}"
URL="file://${RAWTESTFILES}/${f}?dap4.checksum=false#log&dap4"
if ! ${NCDUMP} ${DUMPFLAGS} "${URL}" > ${builddir}/results_test_raw/${f}.ncdump; then
failure "${URL}"
fi

9
dap4_test/test_remote.sh
View File

@ -7,8 +7,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
set -e
echo "test_remote.sh:"
#BIG=1
#CSUM=1
@ -30,12 +28,13 @@ for f in $F ; do
FRAG="${FRAG}&ucar.littleendian=0"
fi
if test "x$CSUM" = x1 ; then
QUERY="${QUERY}&dap4.checksum=true"
QUERY="dap4.checksum=true"
else
QUERY="${QUERY}${QCHAR}&dap4.checksum=false"
QUERY="dap4.checksum=false"
fi
# Fix up QUERY
if test "x$QUERY" != x ; then QUERY=`echo ${QUERY} | sed -e 's/^&/?/'` ; fi
QUERY="?${QUERY}"
# Built test URL
URL="${TESTSERVER}/testfiles/${f}${QUERY}${FRAG}"
${NCDUMP} ${DUMPFLAGS} "${URL}" > ${builddir}/results_test_remote/${f}.ncdump
if test "x${TEST}" = x1 ; then

11
docs/nczarr.md
View File

@ -591,8 +591,8 @@ For linux, the following context works. Of course your mileage may vary.
### AWS-SDK-CPP Build Recipe
````
git clone --recurse-submodules https://www.github.com/aws/aws-sdk-cpp
pushd aws-sdk-cpp
git clone --recurse-submodules https://www.github.com/aws/aws-sdk-cpp.git
cd aws-sdk-cpp
mkdir build
cd build
PREFIX=/usr/local
@ -604,12 +604,13 @@ FLAGS="-DCMAKE_INSTALL_PREFIX=${PREFIX} \
-DENABLE_UNITY_BUILD=ON \
-DENABLE_TESTING=OFF \
-DCMAKE_BUILD_TYPE=$CFG \
-DSIMPLE_INSTALL=ON"
-DSIMPLE_INSTALL=ON \
_DMY_ASSEMBLER_IS_TOO_OLD_FOR_AVX=ON"
cmake -GNinja $FLAGS ..
ninja all
ninja install
sudo ninja install
cd ..
cd ..
popd
````
### NetCDF Build

5
include/ncdap.h
View File

@ -40,13 +40,16 @@ typedef unsigned int NCFLAGS;
#define NCF_ENCODE_PATH (0x2000)
#define NCF_ENCODE_QUERY (0x4000)
#define NCF_FILLMISMATCH_FAIL (0x8000)
/* Put these at top bits for now */
/*COLUMBIA_HACK*/
#define NCF_COLUMBIA (0x80000000) /* Hack for columbia server */
/* OPeNDAP Hyrax Hack */
#define NCF_HYRAX (0x40000000) /* Hack for hyrax server -- use checksum, but ignore */
/* Define all the default on flags */
#define DFALT_ON_FLAGS (NCF_CACHE|NCF_PREFETCH|NCF_FILLMISMATCH)
#define ALL_NCF_FLAGS (0xffff|NCF_COLUMBIA)
#define ALL_NCF_FLAGS (0xffff|NCF_COLUMBIA|NCF_HYRAX)
typedef struct NCCONTROLS {
NCFLAGS flags;

8
include/nctestserver.h
View File

@ -28,6 +28,10 @@ enum KIND {NOKIND, DAP2KIND, DAP4KIND, THREDDSKIND};
#define BUFSIZE 8192 /*bytes*/
#define MAXREMOTETESTSERVERS 4096
/* Attempt to fool lgtm */
#define HTTP "http"
#define HTTPS "https"
#ifndef HAVE_CURLINFO_RESPONSE_CODE
#define CURLINFO_RESPONSE_CODE CURLINFO_HTTP_CODE
#endif
@ -102,14 +106,14 @@ nc_findtestserver(const char* path, const char* serverlist)
if(reportsearch)
fprintf(stderr,"nc_findtestserver: candidate=%s/%s: found=",*svc,path);
/* Try http: first */
snprintf(url,MAXSERVERURL,"http://%s/%s",*svc,path); /*lgtm [cpp/non-https-url]*/
snprintf(url,MAXSERVERURL,HTTP"://%s/%s",*svc,path);
if(ping(url) == NC_NOERR) {
if(reportsearch) fprintf(stderr,"yes\n");
match = strdup(url);
goto done;
}
/* Try https: next */
snprintf(url,MAXSERVERURL,"https://%s/%s",*svc,path);
snprintf(url,MAXSERVERURL,HTTPS"://%s/%s",*svc,path);
if(ping(url) == NC_NOERR) {
if(reportsearch) fprintf(stderr,"yes\n");
match = strdup(url);

109
libdap4/d4data.c
View File

@ -20,6 +20,8 @@ This code serves two purposes
*/
#undef DUMPCHECKSUM
/***************************************************/
/* Forwards */
static int fillstring(NCD4meta*, NCD4offset* offset, void** dstp, NClist* blobs);
@ -28,6 +30,7 @@ static int fillopvar(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp,
static int fillstruct(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs);
static int fillseq(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs);
static int NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel);
static unsigned NCD4_computeChecksum(NCD4meta* meta, NCD4node* topvar);
/***************************************************/
/* Macro define procedures */
@ -73,43 +76,35 @@ NCD4_processdata(NCD4meta* meta)
meta->swap = (meta->serial.hostlittleendian != meta->serial.remotelittleendian);
/* Compute the offset and size of the toplevel vars in the raw dap data. */
/* Also extract checksums */
/* Also extract remote checksums */
offset = BUILDOFFSET(meta->serial.dap,meta->serial.dapsize);
for(i=0;i<nclistlength(toplevel);i++) {
NCD4node* var = (NCD4node*)nclistget(toplevel,i);
if((ret=NCD4_delimit(meta,var,offset)))
FAIL(ret,"delimit failure");
}
/* Compute the checksums of the top variables if needed */
/* must occur before any byte swapping */
for(i=0;i<nclistlength(toplevel);i++) {
NCD4node* var = (NCD4node*)nclistget(toplevel,i);
unsigned int csum = 0;
if(var->data.remotechecksummed) {
csum = CRC32(csum,var->data.dap4data.memory,var->data.dap4data.size);
var->data.localchecksum = csum;
}
}
/* verify checksums */
for(i=0;i<nclistlength(toplevel);i++) {
NCD4node* var = (NCD4node*)nclistget(toplevel,i);
if(var->data.remotechecksummed) {
if(var->data.localchecksum != var->data.remotechecksum) {
nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name);
ret = NC_EDAP;
goto done;
}
}
/* Also verify checksum attribute */
if(var->data.checksumattr) {
if(var->data.attrchecksum != var->data.remotechecksum) {
nclog(NCLOGERR,"Attribute Checksum mismatch: %s\n",var->name);
ret = NC_EDAP;
goto done;
}
if(meta->controller->data.inferredchecksumming) {
/* Compute remote checksum: must occur before any byte swapping */
var->data.localchecksum = NCD4_computeChecksum(meta,var);
#ifdef DUMPCHECKSUM
fprintf(stderr,"var %s: remote-checksum = 0x%x\n",var->name,var->data.remotechecksum);
#endif
/* verify checksums */
if(!meta->controller->data.checksumignore) {
if(var->data.localchecksum != var->data.remotechecksum) {
nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name);
ret = NC_EDAP;
goto done;
}
/* Also verify checksum attribute */
if(meta->controller->data.attrchecksumming) {
if(var->data.attrchecksum != var->data.remotechecksum) {
nclog(NCLOGERR,"Attribute Checksum mismatch: %s\n",var->name);
ret = NC_EDAP;
goto done;
}
}
}
}
}
/* Swap the data for each top level variable,
@ -382,13 +377,15 @@ static int
NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel)
{
int ret = NC_NOERR;
int i;
int i, attrfound;
NCD4INFO* info = meta->controller;
/* First, look thru the DMR to see if there is a checksum attribute */
attrfound = 0;
for(i=0;i<nclistlength(toplevel);i++) {
int a, found;
int a;
NCD4node* node = (NCD4node*)nclistget(toplevel,i);
for(found=0,a=0;a<nclistlength(node->attributes);a++) {
for(a=0;a<nclistlength(node->attributes);a++) {
NCD4node* attr = (NCD4node*)nclistget(node->attributes,a);
if(strcmp(D4CHECKSUMATTR,attr->name)==0) {
const char* val = NULL;
@ -397,18 +394,54 @@ NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel)
val = (const char*)nclistget(attr->attr.values,0);
sscanf(val,"%u",&node->data.attrchecksum);
node->data.checksumattr = 1;
node->data.remotechecksummed = 1;
found = 1;
attrfound = 1;
break;
}
}
if(!found) info->data.checksumattr = 0;
}
info->data.attrchecksumming = (attrfound ? 1 : 0);
/* Infer checksums */
info->data.querychecksum = (info->data.checksumattr || info->data.querychecksum ? 1 : 0);
info->data.inferredchecksumming = ((info->data.attrchecksumming || info->data.querychecksumming) ? 1 : 0);
return THROW(ret);
}
/* Compute the checksum of each variable's data.
*/
static unsigned
NCD4_computeChecksum(NCD4meta* meta, NCD4node* topvar)
{
unsigned int csum = 0;
ASSERT((ISTOPLEVEL(topvar)));
#ifndef HYRAXCHECKSUM
csum = CRC32(csum,topvar->data.dap4data.memory,topvar->data.dap4data.size);
#else
if(topvar->basetype->subsort != NC_STRING) {
csum = CRC32(csum,topvar->data.dap4data.memory,topvar->data.dap4data.size);
} else { /* Checksum over the actual strings */
int i;
unsigned long long count;
NCD4offset offset;
d4size_t dimproduct = NCD4_dimproduct(topvar);
offset.base = topvar->data.dap4data.memory;
offset.limit = offset.base + topvar->data.dap4data.size;
offset.offset = offset.base;
for(i=0;i<dimproduct;i++) {
/* Get string count */
count = GETCOUNTER(&offset);
SKIPCOUNTER(&offset);
if(meta->swap) swapinline64(&count);
/* CRC32 count bytes */
csum = CRC32(csum,offset.offset,count);
/* Skip count bytes */
INCR(&offset,count);
}
}
#endif
return csum;
}
#if 0
/* As a hack, if the server sent remotechecksums and
_DAP4_Checksum_CRC32 is not defined, then define it.

20
libdap4/d4dump.c
View File

@ -14,13 +14,14 @@ void
NCD4_dumpbytes(size_t size, const void* data0, int swap)
{
size_t extended;
void* data;
char* pos;
void* data = NULL;
char* pos = NULL;
size_t i;
extended = size + 8;
data = calloc(1,extended); /* provide some space to simplify the code */
memcpy(data,data0,size);
fprintf(stderr," C XU8 U8 I8 XU32 U32 I32 I16 I64 F32\t\t\tF64\n");
for(i=0,pos=data;i<size; pos++,i++) {
struct {
unsigned char u8[1];
@ -45,6 +46,8 @@ NCD4_dumpbytes(size_t size, const void* data0, int swap)
v.i32[0] = *((int*)pos);
v.u64[0] = *((unsigned long long*)pos);
v.i64[0] = *((long long*)pos);
v.f32[0] = *((float*)pos);
v.f64[0] = *((double*)pos);
if(swap) {
swapinline16(v.u16);
swapinline32(v.u32);
@ -58,12 +61,17 @@ NCD4_dumpbytes(size_t size, const void* data0, int swap)
if(v.s[0] == '\r') strcpy(v.s,"\\r");
else if(v.s[0] == '\n') strcpy(v.s,"\\n");
else if(v.s[0] < ' ' || v.s[0] >= 0x7f) v.s[0] = '?';
fprintf(stderr,"[%03lu] %02x %03u %4d", (unsigned long)i, v.u8[0], v.u8[0], v.i8[0]);
fprintf(stderr," 0x%08x %12u %13d", v.u32[0], v.u32[0], v.i32[0]);
fprintf(stderr," 0x%04x %06u %7d", v.u16[0], v.u16[0], v.i16[0]);
fprintf(stderr," '%s'\n",v.s);
fprintf(stderr,"[%04lu]", (unsigned long)i);
fprintf(stderr," '%s'",v.s);
fprintf(stderr," %03x %03u %04d", v.u8[0], v.u8[0], v.i8[0]);
fprintf(stderr," 0x%08x %012u %013d", v.u32[0], v.u32[0], v.i32[0]);
fprintf(stderr," %07d", v.i16[0]);
fprintf(stderr," %020lld", v.i64[0]);
fprintf(stderr," %4.4g\t\t%4.4lg", v.f32[0], v.f64[0]);
fprintf(stderr,"\n");
fflush(stderr);
}
nullfree(data);
}
void

34
libdap4/d4file.c
View File

@ -429,16 +429,12 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info)
const char* value;
/* clear all flags */
/* clear the flags */
CLRFLAG(info->controls.flags,NCF_CACHE);
CLRFLAG(info->controls.flags,NCF_SHOWFETCH);
CLRFLAG(info->controls.flags,NCF_NC4);
CLRFLAG(info->controls.flags,NCF_NCDAP);
CLRFLAG(info->controls.flags,NCF_FILLMISMATCH);
CLRFLAG(info->controls.flags,NCF_FILLMISMATCH_FAIL);
CLRFLAG(info->controls.flags,ALL_NCF_FLAGS);
/* Turn on any default on flags */
SETFLAG(info->controls.flags,DFALT_ON_FLAGS);
/* Set these also */
SETFLAG(info->controls.flags,(NCF_NC4|NCF_NCDAP));
if(fragmentcheck(info,"show","fetch"))
@ -455,6 +451,11 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info)
if(value != NULL)
strncpy(info->controls.substratename,value,(NC_MAX_NAME-1));
value = getfragment(info,"hyrax");
if(value != NULL) {
info->data.checksumignore = 1; /* Assume checksum, but ignore */
}
info->controls.opaquesize = DFALTOPAQUESIZE;
value = getfragment(info,"opaquesize");
if(value != NULL) {
@ -480,13 +481,18 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info)
void
NCD4_applyclientquerycontrols(NCD4INFO* info)
{
const char* value = getquery(info,DAP4CE);
if(value != NULL) {
info->data.querychecksum = DEFAULT_CHECKSUM_STATE;
if(strcasecmp(value,"false")==0)
info->data.querychecksum = 0;
else
info->data.querychecksum = 1;
const char* value = getquery(info,DAP4CSUM);
if(value == NULL) {
info->data.querychecksumming = DEFAULT_CHECKSUM_STATE;
} else {
if(strcasecmp(value,"false")==0) {
info->data.querychecksumming = 0;
} else if(strcasecmp(value,"true")==0) {
info->data.querychecksumming = 1;
} else {
nclog(NCLOGWARN,"Unknown checksum mode: %s ; using default",value);
info->data.querychecksumming = DEFAULT_CHECKSUM_STATE;
}
}
}

5
libdap4/d4fix.c
View File

@ -210,12 +210,11 @@ NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset)
break;
}
}
/* Track the variable size (in the dap4 data) but do not include
any checksum */
/* Track the variable size (in the dap4 data) but do not include any checksum */
topvar->data.dap4data.memory = mark;
topvar->data.dap4data.size = OFFSETSIZE(offset,mark);
/* extract the dap4 data checksum, if present */
if(topvar->data.remotechecksummed) {
if(compiler->controller->data.inferredchecksumming) {
union ATOMICS csum;
TRANSFER(csum.u8,offset,CHECKSUMSIZE);
topvar->data.remotechecksum = csum.u32[0];

2
libdap4/d4swap.c
View File

@ -55,7 +55,7 @@ NCD4_swapdata(NCD4meta* compiler, NClist* topvars)
}
var->data.dap4data.size = DELTA(offset,var->data.dap4data.memory);
/* skip checksum, if there is one */
if(var->data.remotechecksummed)
if(compiler->controller->data.inferredchecksumming)
INCR(offset,CHECKSUMSIZE);
}
done:

8
libdap4/d4varx.c
View File

@ -37,7 +37,7 @@ NCD4_get_vars(int ncid, int varid,
NCD4node* nctype;
D4odometer* odom = NULL;
nc_type nc4type;
size_t nc4size, xsize;
size_t nc4size, xsize, dapsize;
void* instance = NULL; /* Staging area in case we have to convert */
NClist* blobs = NULL;
int rank;
@ -46,6 +46,7 @@ NCD4_get_vars(int ncid, int varid,
size_t dstcount;
NCD4offset* offset = NULL;
/* Get netcdf type info */
if((ret=getvarx(ncid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size)))
{goto done;}
@ -54,6 +55,9 @@ NCD4_get_vars(int ncid, int varid,
rank = nclistlength(ncvar->dims);
blobs = nclistnew();
/* Get the type's dapsize */
dapsize = nctype->meta.dapsize;
instance = malloc(nc4size);
if(instance == NULL)
{ret = THROW(NC_ENOMEM); goto done;}
@ -93,7 +97,7 @@ NCD4_get_vars(int ncid, int varid,
BLOB2OFFSET(offset,ncvar->data.dap4data);
/* Move offset to the count'th element of the array */
if(nctype->meta.isfixedsize) {
INCR(offset,(nc4size*count));
INCR(offset,(dapsize*count));
} else {
/* We have to walk to the count'th location in the data */
if((ret=NCD4_moveto(meta,ncvar,count,offset)))

4
libdap4/ncd4dispatch.c
View File

@ -881,7 +881,7 @@ ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* valu
{ret = NC_EBADID; goto done;}
if(t != NC_UINT) {ret = NC_EBADTYPE; goto done;}
if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done;
if(var->data.remotechecksummed == 0)
if(var->data.checksumattr == 0)
{ret = NC_ENOTATT; goto done;}
*ip = (var->data.remotechecksum);
} else if(strcmp(rsvp->name,D4LEATTR)==0) {
@ -905,7 +905,7 @@ ncd4_inq_att_reserved(NC* ncp, int ncid, int varid, const char* name, nc_type* x
if(varid == NC_GLOBAL)
{ret = NC_EBADID; goto done;}
if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done;
if(var->data.remotechecksummed == 0)
if(var->data.checksumattr == 0)
{ret = NC_ENOTATT; goto done;}
if(xtypep) *xtypep = NC_UINT;
if(lenp) *lenp = 1;

13
libdap4/ncd4types.h
View File

@ -111,8 +111,7 @@ typedef enum NCD4sort {
/* dap4.x query keys */
#define DAP4CE "dap4.ce"
#define DAP4CSUM "dap4.checksum=true"
#define DAP4NOCSUM "dap4.checksum=false"
#define DAP4CSUM "dap4.checksum"
/**************************************************/
/* Misc.*/
@ -242,7 +241,6 @@ struct NCD4node {
unsigned int localchecksum; /* toplevel variable checksum as computed by client */
int checksumattr; /* 1=> _DAP4_Checksum_CRC32 is defined */
int attrchecksum; /* _DAP4_Checksum_CRC32 value */
int remotechecksummed; /* 1 if we know that this variable was checksummed */
} data;
struct { /* Track netcdf-4 conversion info */
int isvlen; /* _edu.ucar.isvlen */
@ -342,10 +340,11 @@ struct NCD4INFO {
d4size_t datasize; /* size on disk or in memory */
long dmrlastmodified;
long daplastmodified;
int querychecksum; /* 1 => user specified dap4.ce value */
int checksumattr; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */
int inferredchecksum; /* 1 => we infer that incoming data has checksums */
} data;
int querychecksumming; /* 1 => user specified dap4.ce value */
int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */
int inferredchecksumming; /* 1 => either query checksum || att checksum */
int checksumignore; /* 1 => assume checksum, but do not validate */
} data;
struct {
int realfile; /* 1 => we created actual temp file */
char* filename; /* of the substrate file */

6
libdispatch/daux.c
View File

@ -857,15 +857,15 @@ done:
/* Wrappers to export selected functions from libnetcdf */
EXTERNL int
ncaux_readfile(const char* filename, size_t* sizep, void** contentp)
ncaux_readfile(const char* filename, size_t* sizep, void** datap)
{
int stat = NC_NOERR;
NCbytes* content = ncbytesnew();
stat = NC_readfile(filename,content);
if(stat == NC_NOERR && sizep)
*sizep = ncbyteslength(content);
if(stat == NC_NOERR && contentp)
*contentp = ncbytesextract(content);
if(stat == NC_NOERR && datap)
*datap = ncbytesextract(content);
ncbytesfree(content);
return stat;
}

8
ncdap_test/pingurl.c
View File

@ -16,6 +16,10 @@ See \ref copyright file for more info.
#define PINGTIME 25
/* Attempt to shut up LGTM */
#define HTTP "http"
#define HTTPS "https"
/**
usage: pingurl <svc>
See if a specific server at a given url appears to be up.
@ -41,12 +45,12 @@ main(int argc, char** argv)
usage();
/* Try http: first */
snprintf(url,MAXSERVERURL,"http://%s",argv[0]); /* lgtm [cpp/non-https-url] */
snprintf(url,MAXSERVERURL,HTTP"://%s",argv[0]);
if(timedping(url,PINGTIME) == NC_NOERR)
found = 1;
else {
/* Try https: next */
snprintf(url,MAXSERVERURL,"https://%s",argv[0]);
snprintf(url,MAXSERVERURL,HTTPS"://%s",argv[0]);
if(timedping(url,PINGTIME) == NC_NOERR) {
found = 1;
ishttps = 1;