From 8072d1f6bbd7324afd252e96486451cbb449eb7f Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 1 Oct 2018 15:51:43 -0600 Subject: [PATCH] Modify DAP2 and DAP4 to optionally allow Fillvalue/Variable mismatch re: issue https://github.com/Unidata/netcdf-c/issues/1151 Modify DAP2 and DAP4 code to handle case when _FillValue type is not same as the parent variable type. Specifically: 1. Define a parameter [fillmismatch] to allow this mismatch; default is to disallow. 2. If allowed, forcibly change the type of the _FillValue to match the parent variable. 3. If allowed Convert the values to match new type 4. Generate a log message 5. if not allowed, then fail Implementing this required some changes to ncdap_test/dapcvt.c Also added test cases. Minor Unrelated Changes: 1. There were a number of warnings about e.g. assigning a const char* to a char*. Fix these 2. In nccopy.1, replace .NP with .IP "n" (re PR https://github.com/Unidata/netcdf-c/pull/1144) 3. fix minor error in ncdump/ocprint --- CMakeLists.txt | 4 +- cf | 2 +- dap4_test/CMakeLists.txt | 1 + dap4_test/Makefile.am | 5 +- .../baselineraw/test_fillmismatch.nc.dmp | 10 + dap4_test/d4test_common.sh | 25 +- dap4_test/findtestserver4.c | 79 -- dap4_test/maketests.sh | 2 +- dap4_test/misctestfiles/CMakeLists.txt | 6 + .../misctestfiles/test_fillmismatch.nc.dap | Bin 0 -> 599 bytes dap4_test/test_fillmismatch.sh | 39 + dap4_test/test_meta.sh | 3 +- dap4_test/test_remote.sh | 2 +- dap4_test/test_test.sh | 2 +- dap4_test/tst_data.sh | 2 +- dap4_test/tst_meta.sh | 2 +- dap4_test/tst_raw.sh | 2 +- docs/CMakeLists.txt | 2 +- docs/DAP2.dox | 673 ++++++++++++++++++ docs/Doxyfile.in | 3 +- docs/Makefile.am | 2 +- docs/OPeNDAP.dox | 4 + include/ncdap.h | 2 + include/nclog.h | 2 + include/netcdf.h | 5 + libdap2/dapcvt.c | 288 +++++--- libdap2/dapincludes.h | 2 +- libdap2/daputil.c | 31 +- libdap2/ncd2dispatch.c | 52 +- libdap4/d4crc32.c | 2 +- libdap4/d4file.c | 12 +- libdap4/d4meta.c | 34 +- libdap4/d4parser.c | 29 +- libdap4/d4util.c | 4 +- libdispatch/drc.c | 48 +- libdispatch/dwinpath.c | 4 +- libdispatch/nclog.c | 19 + libsrc/memio.c | 3 +- nc_test4/Makefile.am | 2 +- ncdap_test/CMakeLists.txt | 2 +- ncdap_test/Makefile.am | 13 +- ncdap_test/expected3/Makefile.am | 2 +- ncdap_test/expected3/fillmismatch.nc.dmp | 44 ++ ncdap_test/findtestserver.c | 79 -- ncdap_test/testdata3/Makefile.am | 1 + ncdap_test/testdata3/fillmismatch.nc.das | 16 + ncdap_test/testdata3/fillmismatch.nc.dds | 11 + ncdap_test/testdata3/fillmismatch.nc.dods | Bin 0 -> 264 bytes ncdap_test/tst_fillmismatch.sh | 37 + ncdap_test/tst_ncdap.sh | 2 +- ncdap_test/tst_ncdap3.sh | 2 +- ncdap_test/tst_utils.sh | 2 + ncdump/nccopy.1 | 11 +- ncdump/ocprint.c | 3 + oc2/daplex.c | 14 +- oc2/dapparselex.h | 6 +- 56 files changed, 1242 insertions(+), 412 deletions(-) create mode 100644 dap4_test/baselineraw/test_fillmismatch.nc.dmp delete mode 100644 dap4_test/findtestserver4.c create mode 100644 dap4_test/misctestfiles/CMakeLists.txt create mode 100644 dap4_test/misctestfiles/test_fillmismatch.nc.dap create mode 100755 dap4_test/test_fillmismatch.sh create mode 100644 docs/DAP2.dox create mode 100644 ncdap_test/expected3/fillmismatch.nc.dmp delete mode 100644 ncdap_test/findtestserver.c create mode 100644 ncdap_test/testdata3/fillmismatch.nc.das create mode 100644 ncdap_test/testdata3/fillmismatch.nc.dds create mode 100644 ncdap_test/testdata3/fillmismatch.nc.dods create mode 100755 ncdap_test/tst_fillmismatch.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index a00ad5d4d..25d923148 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1996,8 +1996,8 @@ ENDIF() ##### # Build ncdap_test|dap4_test/findtestserver[4].c ##### -configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/ncdap_test/findtestserver.c @ONLY NEWLINE_STYLE LF) -configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/dap4_test/findtestserver4.c @ONLY NEWLINE_STYLE LF) +configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c @ONLY NEWLINE_STYLE LF) +configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_SOURCE_DIR}/dap4_test/findtestserver4.c @ONLY NEWLINE_STYLE LF) #### # Export files diff --git a/cf b/cf index 30be6b434..0660dfd13 100644 --- a/cf +++ b/cf @@ -8,7 +8,7 @@ FAST=1 HDF5=1 DAP=1 -SZIP=1 +#SZIP=1 #HDF4=1 #PNETCDF=1 #PAR4=1 diff --git a/dap4_test/CMakeLists.txt b/dap4_test/CMakeLists.txt index b896bde28..e846e9b92 100644 --- a/dap4_test/CMakeLists.txt +++ b/dap4_test/CMakeLists.txt @@ -9,6 +9,7 @@ ADD_SUBDIRECTORY(cdltestfiles) ADD_SUBDIRECTORY(daptestfiles) ADD_SUBDIRECTORY(dmrtestfiles) ADD_SUBDIRECTORY(nctestfiles) +ADD_SUBDIRECTORY(misctestfiles) FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh) diff --git a/dap4_test/Makefile.am b/dap4_test/Makefile.am index 6c61be890..402a12318 100644 --- a/dap4_test/Makefile.am +++ b/dap4_test/Makefile.am @@ -35,6 +35,7 @@ if BUILD_UTILITIES TESTS += test_raw.sh TESTS += test_meta.sh TESTS += test_data.sh +TESTS += test_fillmismatch.sh endif # Note which tests depend on other tests. Necessary for make -j check. @@ -56,9 +57,9 @@ endif endif #ENABLE_DAP4 EXTRA_DIST = test_parse.sh test_meta.sh test_data.sh \ - test_raw.sh test_remote.sh test_hyrax.sh \ + test_raw.sh test_remote.sh test_hyrax.sh test_fillmismatch.sh \ tst_curlopt.sh d4test_common.sh \ - daptestfiles dmrtestfiles cdltestfiles nctestfiles \ + daptestfiles dmrtestfiles cdltestfiles nctestfiles misctestfiles \ baseline baselineraw baselineremote CMakeLists.txt CLEANFILES = *.exe diff --git a/dap4_test/baselineraw/test_fillmismatch.nc.dmp b/dap4_test/baselineraw/test_fillmismatch.nc.dmp new file mode 100644 index 000000000..b9c37984a --- /dev/null +++ b/dap4_test/baselineraw/test_fillmismatch.nc.dmp @@ -0,0 +1,10 @@ +netcdf test_fillmismatch { +variables: + ubyte uv8 ; + short v16 ; + uint uv32 ; + uv32:_FillValue = 17U ; + +// global attributes: + :_DAP4_Little_Endian = 1UB ; +} diff --git a/dap4_test/d4test_common.sh b/dap4_test/d4test_common.sh index f9f7405ce..f9af53943 100755 --- a/dap4_test/d4test_common.sh +++ b/dap4_test/d4test_common.sh @@ -62,26 +62,17 @@ exit 0 fi } +# Return the complete set of arguments minus any found in $SUPPRESS suppress() { - F0="$1" - if test "x${SUPPRESS}" = x; then - RESULT="$F0" - else - RESULT="" - for f in ${F0} ; do - ignore=0 - for s in ${SUPPRESS} ; do - if test "x$s" = "x$f" ; then - ignore=1; - echo "Suppressing: $f" - break; - fi - done - if test "x$ignore" = x0 ; then RESULT="$f ${RESULT}" ; fi - done - fi + local args="$@" + suppress= + for a in $args; do + for s in $SUPPRESS; do if test "x$a" != "x$s" ; then suppress="$suppress $a"; fi; done + done } VG="valgrind --leak-check=full --error-exitcode=1 --num-callers=100" if test "x$USEVG" = x ; then VG=; fi +DUMPFLAGS= + diff --git a/dap4_test/findtestserver4.c b/dap4_test/findtestserver4.c deleted file mode 100644 index 9ad5e697e..000000000 --- a/dap4_test/findtestserver4.c +++ /dev/null @@ -1,79 +0,0 @@ -#include "config.h" -#include -#include -#include -#include "nctestserver.h" - -/* Support stringification of -D macros */ -#define XSTRINGIFY(s) #s -#define STRINGIFY(s) XSTRINGIFY(s) - - -/** -usage: findtestserver dap2|dap4 suffix [serverlist] - -Given a partial suffix path, try to find a -server for which a request to server + suffix -returns some kind of result using the -specified protocol. This indicates that the -server is up and running. Return the complete -url for the server plus the path. -If serverlist is present, then is should be a comma -separated list of servers (host+port) to try. -It defaults to REMOTETESTSERVERS. -*/ - -static void -usage() -{ - fprintf(stderr,"usage: findtestserver dap2|dap4 suffix [serverlist]\n"); - exit(1); -} - - -int -main(int argc, char** argv) -{ - const char* url = NULL; - const char* servlet = NULL; - const char* proto = NULL; - const char* serverlist = NULL; - int isdap4 = 0; /* 1 => dap4 */ - - argc--; argv++; - if(argc < 2) - usage(); - proto = strdup(argv[0]); - servlet = strdup(argv[1]); - if(argc >= 3) - serverlist = strdup(argv[2]); - -#ifdef ENABLE_DAP - if(strcasecmp(proto,"dap2")==0) - isdap4 = 0; - else -#endif -#ifdef ENABLE_DAP4 - if(strcasecmp(proto,"dap4")==0) - isdap4 = 1; - else -#endif - usage(); - - if(serverlist == NULL) { -#ifdef REMOTETESTSERVERS - serverlist = strdup(REMOTETESTSERVERS); -#endif - } - if(serverlist == NULL || strlen(serverlist) == 0) - fprintf(stderr,"Cannot determine a server list"); - - url = nc_findtestserver(servlet,isdap4,serverlist); - if(url == NULL) { - url = ""; - fprintf(stderr,"not found: %s\n",servlet); - } - printf("%s",url); - fflush(stdout); - exit(0); -} diff --git a/dap4_test/maketests.sh b/dap4_test/maketests.sh index 1ad1a09b2..4d5ae2d54 100755 --- a/dap4_test/maketests.sh +++ b/dap4_test/maketests.sh @@ -40,7 +40,7 @@ if test "x$DEBUG" = x1 ; then F=`ls -1 *.dap | sed -e 's/[.]dap//' |tr '\r\n' ' '` popd for f in ${F} ; do - if ! diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr >& /dev/null ; then + if ! diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr > /dev/null 2>&1 ; then echo diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr fi diff --git a/dap4_test/misctestfiles/CMakeLists.txt b/dap4_test/misctestfiles/CMakeLists.txt new file mode 100644 index 000000000..2332d8f85 --- /dev/null +++ b/dap4_test/misctestfiles/CMakeLists.txt @@ -0,0 +1,6 @@ +FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*) +FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE) + +FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*) +SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt) +ADD_EXTRA_DIST("${CUR_EXTRA_DIST}") diff --git a/dap4_test/misctestfiles/test_fillmismatch.nc.dap b/dap4_test/misctestfiles/test_fillmismatch.nc.dap new file mode 100644 index 0000000000000000000000000000000000000000..7c858f08a9cf33212daab6221002fef263511f02 GIT binary patch literal 599 zcma)(y-ve06opg76p5)bl4WLM(pGI$oK}@eB_sw0+BGtbDUlK{ep>c4|#}83RGMVZiRxY)zGfJdd#(tnyE}h@ZP8~`# z2<3|~R??*#sp)uR@~Hr!B^;Tg`jm)J2dTlk3&PM+OWIICYr z-7#nN*5Z(}B1Dz7mMbXs)AOzZfKYlD$`6EU(w!!p?}IDu9+R{3-4WO_jQ02Q@kjjr R`1R8M+`hI9`)%-K{s8&8rt<&* literal 0 HcmV?d00001 diff --git a/dap4_test/test_fillmismatch.sh b/dap4_test/test_fillmismatch.sh new file mode 100755 index 000000000..03a02c4ca --- /dev/null +++ b/dap4_test/test_fillmismatch.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e +. ${srcdir}/d4test_common.sh + +echo "test_fillmismatch.sh:" + +F="test_fillmismatch.nc" + +URL='[dap4]file://' +URL="${URL}${srcdir}/misctestfiles/$F" + +# First check that without [fillmismatch], we get a failure +rm -f ./tmp_dap4_mismatch +if ${NCDUMP} -h "${URL}" > ./tmp_dap4_mismatch 2>&1 ; then +echo "*** Fail: ${NCDUMP} ${URL} passed" +exit 1 +else +echo "*** XFail: ${NCDUMP} ${URL} failed" +fi + +# Now check that with [fillmismatch], we get sucess +URL="[fillmismatch]${URL}" +rm -f ./tmp_dap4_mismatch +if ${NCDUMP} -h "${URL}" > ./tmp_dap4_mismatch ; then +echo "*** Pass: ${NCDUMP} ${URL} passed" +else +echo "*** Fail: ${NCDUMP} ${URL} failed" +exit 1 +fi + +# Verify result +diff -w ${srcdir}/baselineraw/$F.dmp ./tmp_dap4_mismatch +#cleanup +rm -f ./tmp_dap4_mismatch +exit diff --git a/dap4_test/test_meta.sh b/dap4_test/test_meta.sh index c848c3528..f56c14678 100755 --- a/dap4_test/test_meta.sh +++ b/dap4_test/test_meta.sh @@ -7,6 +7,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi echo "test_meta.sh:" + cd ${DMRTESTFILES} F=`ls -1 *.dmr | sed -e 's/[.]dmr//g' | tr '\r\n' ' '` cd $WD @@ -30,7 +31,7 @@ for f in ${F} ; do if ! ${VG} ${execdir}/test_meta ${DMRTESTFILES}/${f}.dmr ./results_test_meta/${f} ; then failure "${execdir}/test_meta ${DMRTESTFILES}/${f}.dmr ./results_test_meta/${f}" fi - ${NCDUMP} -h ./results_test_meta/${f} > ./results_test_meta/${f}.d4m + ${NCDUMP} ${DUMPFLAGS} -h ./results_test_meta/${f} > ./results_test_meta/${f}.d4m if test "x${TEST}" = x1 ; then if ! diff -wBb ${BASELINE}/${f}.d4m ./results_test_meta/${f}.d4m ; then failure "diff -wBb ${BASELINE}/${f}.ncdump ./results_test_meta/${f}.d4m" diff --git a/dap4_test/test_remote.sh b/dap4_test/test_remote.sh index 5582ab237..210032245 100755 --- a/dap4_test/test_remote.sh +++ b/dap4_test/test_remote.sh @@ -73,7 +73,7 @@ for f in $F ; do if test "x$NOCSUM" = x1; then URL="[ucar.checksummode=none]${URL}" fi - if ! ${NCDUMP} "${URL}" > ${builddir}/results_test_remote/${f}.dmp; then + if ! ${NCDUMP} ${DUMPFLAGS} "${URL}" > ${builddir}/results_test_remote/${f}.dmp; then failure "${URL}" fi if test "x${TEST}" = x1 ; then diff --git a/dap4_test/test_test.sh b/dap4_test/test_test.sh index 536b239c5..b8df934c6 100755 --- a/dap4_test/test_test.sh +++ b/dap4_test/test_test.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = "x" ; then srcdir=`dirname $0`; fi; export srcdir rm -f log.log -sh -x ${srcdir}/test_parse.sh >& ./parse.log >& log.log +sh -x ${srcdir}/test_parse.sh > ./parse.log 2>&1 cat log.log /dev/tty exit 0 diff --git a/dap4_test/tst_data.sh b/dap4_test/tst_data.sh index 043b593d3..8865f4383 100755 --- a/dap4_test/tst_data.sh +++ b/dap4_test/tst_data.sh @@ -15,7 +15,7 @@ for f in $F ; do if ! ${VG} ${execdir}/test_data ${DAPTESTFILES}/${f} ./results/${f}.nc ; then failure "${execdir}/test_data ${DAPTESTFILES}/${f} ./results/${f}.nc" fi - ${NCDUMP} ./results/${f}.nc > ./results/${f}.d4d + ${NCDUMP} ${DUMPFLAGS} ./results/${f}.nc > ./results/${f}.d4d if test "x${TEST}" = x1 ; then if ! diff -wBb ${BASELINE}/${f}.d4d ./results/${f}.d4d ; then failure "diff -wBb ${BASELINE}/${f}.d4d ./results/${f}.d4d" diff --git a/dap4_test/tst_meta.sh b/dap4_test/tst_meta.sh index 7414bee4e..e6b29869a 100755 --- a/dap4_test/tst_meta.sh +++ b/dap4_test/tst_meta.sh @@ -26,7 +26,7 @@ for f in ${F} ; do if ! ${VG} ${execdir}/test_meta ${DMRTESTFILES}/${f}.dmr ./results/${f} ; then failure "${execdir}/test_meta ${DMRTESTFILES}/${f}.dmr ./results/${f}" fi - ${NCDUMP} -h ./results/${f} > ./results/${f}.d4m + ${NCDUMP} ${DUMPFLAGS} -h ./results/${f} > ./results/${f}.d4m if test "x${TEST}" = x1 ; then if ! diff -wBb ${BASELINE}/${f}.d4m ./results/${f}.d4m ; then failure "diff -wBb ${BASELINE}/${f}.ncdump ./results/${f}.d4m" diff --git a/dap4_test/tst_raw.sh b/dap4_test/tst_raw.sh index ad24edb59..cc087ad6c 100755 --- a/dap4_test/tst_raw.sh +++ b/dap4_test/tst_raw.sh @@ -42,7 +42,7 @@ if test "x${RESET}" = x1 ; then rm -fr ${BASELINERAW}/*.dmp ; fi for f in $F ; do echo "testing: $f" URL="[dap4]file:${DAPTESTFILES}/${f}" - if ! ${VG} ${NCDUMP} "${URL}" > ./results/${f}.dmp; then + if ! ${VG} ${NCDUMP} ${DUMPFLAGS} "${URL}" > ./results/${f}.dmp; then failure "${URL}" fi if test "x${TEST}" = x1 ; then diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 2a8896c9f..f9358ed3f 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -82,7 +82,7 @@ architecture.dox internal.dox windows-binaries.md building-with-cmake.md CMakeLists.txt groups.dox install.md notes.md install-fortran.md all-error-codes.md credits.md auth.md obsolete/fan_utils.html bestpractices.md filters.md indexing.md -inmemory.md DAP4.dox OPeNDAP.dox attribute_conventions.md FAQ.md +inmemory.md DAP2.dox attribute_conventions.md FAQ.md file_format_specifications.md known_problems.md COPYRIGHT.dox user_defined_formats.md) diff --git a/docs/DAP2.dox b/docs/DAP2.dox new file mode 100644 index 000000000..e88047616 --- /dev/null +++ b/docs/DAP2.dox @@ -0,0 +1,673 @@ +/*! +\page dap2 DAP2 Protocol Support + +\tableofcontents + + + + +# DAP2 (OPeNDAP) Introduction {#dap2_intro} + +Beginning with netCDF version 4.1, optional support is provided for +accessing data through servers supporting the DAP2 protocol. + +DAP2 support is enabled if the _--enable-dap__ option +is used with _./configure_. If DAP2 support is enabled, then +a usable version of _libcurl_ must be specified +using the _LDFLAGS_ environment variable (similar to the way +that the _HDF5_ libraries are referenced). +Refer to the installation manual for details. +By default DAP2 support is enabled if _libcurl_ is found. +DAP2 support can be disabled using the _--disable-dap_. + +DAP2 uses a data model that is different from that supported by +netCDF, either classic or enhanced. Generically, the DAP2 +meta-data is encoded textually in a _DDS_ (Dataset Descriptor +Structure). There is a second textual object, the _DAS_ (Dataset +Attribute Structure), for specifying DAP2 attributes. . For +detailed information about the DAP2 DDS and DAS, refer to the +OPeNDAP web site http://opendap.org. + +# Accessing DAP2 Data {#dap2_accessing_data} + +In order to access an OPeNDAP data source through the netCDF API, the +file name normally used is replaced with a URL with a specific +format. The URL is composed of three parts. +- URL - this is a standard form URL such as + http://remotetest.unidata.ucar.edu/dts/test.01 + +- Constraints - these are suffixed to the URL and take the form + “?\&\”. The meaning of the terms "projection" + and "selection" is somewhat complicated; and the OPeNDAP web site, + http://www.opendap.org, should be consulted. The interaction of DAP2 + constraints with netCDF is complex and at the moment requires an + understanding of how DAP2 is translated to netCDF. + +- Client parameters - these may be specified in either of + two ways. The older, deprecated form prefixes text to the + front of the url and is of the the general form [\] + or [\=value]. Examples include [show=fetch] and + [noprefetch]. The newer, preferred form prefixes the + parameters to the end of the url using the semi-standard '#' + format: e.g. http://....#show=fetch&noprefetch. + +It is possible to see what the translation does to a particular +DAP2 data source by examining the DDS source through a web +browser and then examining the translation using the _ncdump -h_ +command to see the netCDF Classic translation. The ncdump output +will actually be the union of the DDS with the DAS, so to see +the complete translation, it is necessary to view both via the +browser. + +For example, if a web browser is given the following, the first URL +will return the DDS for the specified dataset, and the second URL will +return the DAS for the specified dataset. +```` + http://remotetest.unidata.ucar.edu/dts/test.01.dds + http://remotetest.unidata.ucar.edu/dts/test.01.das +```` + +Then by using the following ncdump command, it is possible to see the +equivalent netCDF Classic translation. +```` + ncdump -h http://remotetest.unidata.ucar.edu/dts/test.01 +```` + +The DDS output from the web server should look like this. +```` +Dataset { + Byte b; + Int32 i32; + UInt32 ui32; + Int16 i16; + UInt16 ui16; + Float32 f32; + Float64 f64; + String s; + Url u; +} SimpleTypes; +```` + +The DAS output from the web server should look like this. +```` +Attributes { + Facility { + String PrincipleInvestigator ``Mark Abbott'', ``Ph.D''; + String DataCenter ``COAS Environmental Computer Facility''; + String DrifterType ``MetOcean WOCE/OCM''; + } + b { + String Description ``A test byte''; + String units ``unknown''; + } + i32 { + String Description ``A 32 bit test server int''; + String units ``unknown''; + } +} +```` + +The output from ncdump should look like this. +```` +netcdf test { +dimensions: + stringdim64 = 64 ; +variables: + byte b ; + b:Description = "A test byte" ; + b:units = "unknown" ; + int i32 ; + i32:Description = "A 32 bit test server int" ; + i32:units = "unknown" ; + int ui32 ; + short i16 ; + short ui16 ; + float f32 ; + double f64 ; + char s(stringdim64) ; + char u(stringdim64) ; +} +```` + +Note that the fields of type String and type URL have suddenly +acquired a dimension. This is because the netCDF model does +not support strings, but DAP2 does support strings. +So, DAP2 strings are translated to arrays +of char, which requires adding an extra dimension. The size of the +dimension is determined in a variety of ways and can be specified. It +defaults to 64 and when read, the underlying string is either padded +or truncated to that length. + +Also note that the "Facility" attributes do not appear in the +translation because they are neither global nor associated with a +variable in the DDS. + +# DAP2 to NetCDF Translation Rules {#dap2_to_netcdf} + +The netCDF library DAP2 support code translate the DAP2 data model +into the netCDF classic (netCDF-3) data model. + +## netCDF-3 Translation Rules {#dap2_nc32_trans_rules} + +The netCDF-3 translation is designed to mimic as closely as +possible the translation provided by the now obsolete libnc-dap2 +system, except that some errors in that older translation have +been fixed. + +For illustrative purposes, the following example will be used. +```` +Dataset { + Int32 f1; + Structure { + Int32 f11; + Structure { + Int32 f1[3]; + Int32 f2; + } FS2[2]; + } S1; + Structure { + Grid { + Array: + Float32 temp[lat=2][lon=2]; + Maps: + Int32 lat[lat=2]; + Int32 lon[lon=2]; + } G1; + } S2; + Grid { + Array: + Float32 G2[lat=2][lon=2]; + Maps: + Int32 lat[2]; + Int32 lon[2]; + } G2; + Int32 lat[lat=2]; + Int32 lon[lon=2]; +} D1; +```` + +## Variable Definition {#dap2_var2_def} + +The set of netCDF variables is derived from the fields with primitive +base types as they occur in Sequences, Grids, and Structures. The +field names are modified to be fully qualified initially. For the +above, the set of variables are as follows. The coordinate variables +within grids are left out in order to mimic the behavior of libnc-dap2. +```` + f1 + S1.f11 + S1.FS2.f1 + S1.FS2.f2 + S2.G1.temp + S2.G2.G2 + lat + lon +```` + +## DAP2 Reserved Keywords {#dap2_reserved_keywords} + +In the OPeNDAP DAP2 protocol, there are a number of reserved keywords. These keywords are case insensitive and if you use one as a netCDF variable name, you may encounter odd behavior such as case changes (depending on the client DDS/DAS parser). The list of reserved keywords as used by the netCDF-C library parser are as follows: + +- alias +- array +- attributes +- byte +- dataset +- error +- float32 +- float64 +- grid +- int16 +- int32 +- maps +- sequence +- string +- structure +- uint16 +- uint32 +- url +- code +- message +- program_type +- program + + +## Variable Dimension Translation {#dap2_var_dim_trans} + +A variable's rank is determined from three sources. +- The variable has the dimensions associated with the field it +represents (e.g. S1.FS2.f1[3] in the above example). +- The variable inherits the dimensions associated with any containing +structure that has a rank greater than zero. These dimensions precede +those of case 1. Thus, we have in our example, f1[2][3], where the +first dimension comes from the containing Structure FS2[2]. +- The variable's set of dimensions are altered if any of its +containers is a DAP2 DDS Sequence. This is discussed more fully below. + +If the type of the netCDF variable is char, then an extra string +dimension is added as the last dimension. + +## Dimension translation {#dap2_dim2_trans} + +For dimensions, the rules are as follows. + +Fields in dimensioned structures inherit the dimension of the +structure; thus the above list would have the following dimensioned +variables. +```` + S1.FS2.f1 -> S1.FS2.f1[2][3] + S1.FS2.f2 -> S1.FS2.f2[2] + S2.G1.temp -> S2.G1.temp[lat=2][lon=2] + S2.G1.lat -> S2.G1.lat[lat=2] + S2.G1.lon -> S2.G1.lon[lon=2] + S2.G2.G2 -> S2.G2.lon[lat=2][lon=2] + S2.G2.lat -> S2.G2.lat[lat=2] + S2.G2.lon -> S2.G2.lon[lon=2] + lat -> lat[lat=2] + lon -> lon[lon=2] +```` + +Collect all of the dimension specifications from the DDS, both named +and anonymous (unnamed) For each unique anonymous dimension with value +NN create a netCDF dimension of the form "XX_\=NN", where XX is the +fully qualified name of the variable and i is the i'th (inherited) +dimension of the array where the anonymous dimension occurs. For our +example, this would create the following dimensions. +```` + S1.FS2.f1_0 = 2 ; + S1.FS2.f1_1 = 3 ; + S1.FS2.f2_0 = 2 ; + S2.G2.lat_0 = 2 ; + S2.G2.lon_0 = 2 ; +```` + +If however, the anonymous dimension is the single dimension of a MAP +vector in a Grid then the dimension is given the same name as the map +vector This leads to the following. +```` + S2.G2.lat_0 -> S2.G2.lat + S2.G2.lon_0 -> S2.G2.lon +```` + +For each unique named dimension "=NN", create a netCDF dimension +of the form "=NN", where name has the qualifications removed. If +this leads to duplicates (i.e. same name and same value), then the +duplicates are ignored. This produces the following. +```` + S2.G2.lat -> lat + S2.G2.lon -> lon +```` + +Note that this produces duplicates that will be ignored later. + +At this point the only dimensions left to process should be named +dimensions with the same name as some dimension from step number 3, +but with a different value. For those dimensions create a dimension of +the form "M=NN" where M is a counter starting at 1. The example +has no instances of this. + +Finally and if needed, define a single UNLIMITED dimension named +"unlimited" with value zero. Unlimited will be used to handle certain +kinds of DAP2 sequences (see below). + +This leads to the following set of dimensions. +```` +dimensions: + unlimited = UNLIMITED; + lat = 2 ; + lon = 2 ; + S1.FS2.f1_0 = 2 ; + S1.FS2.f1_1 = 3 ; + S1.FS2.f2_0 = 2 ; +```` + +## Variable Name Translation {#dap2_var_name_trans} + +The steps for variable name translation are as follows. + +Take the set of variables captured above. Thus for the above DDS, the +following fields would be collected. +```` + f1 + S1.f11 + S1.FS2.f1 + S1.FS2.f2 + S2.G1.temp + S2.G2.G2 + lat + lon +```` + +All grid array variables are renamed to be the same as the containing +grid and the grid prefix is removed. In the above DDS, this results in +the following changes. +```` + G1.temp -> G1 + G2.G2 -> G2 +```` + +It is important to note that this process could produce duplicate +variables (i.e. with the same name); in that case they are all assumed +to have the same content and the duplicates are ignored. If it turns +out that the duplicates have different content, then the translation +will not detect this. YOU HAVE BEEN WARNED. + +The final netCDF-3 schema (minus attributes) is then as follows. +```` +netcdf t { +dimensions: + unlimited = UNLIMITED ; + lat = 2 ; + lon = 2 ; + S1.FS2.f1_0 = 2 ; + S1.FS2.f1_1 = 3 ; + S1.FS2.f2_0 = 2 ; +variables: + int f1 ; + int lat(lat) ; + int lon(lon) ; + int S1.f11 ; + int S1.FS2.f1(S1.FS2.f1_0, S1.FS2.f1_1) ; + int S1.FS2.f2(S1_FS2_f2_0) ; + float S2.G1(lat, lon) ; + float G2(lat, lon) ; +} +```` + +In practice, the unlimited dimension is dropped because it is unused. + +There are differences with the original libnc-dap2 here because +libnc-dap2 technically was incorrect. The original would have said +this, for example. +```` +int S1.FS2.f1(lat, lat) ; +```` + +Note that this is incorrect because it dimensions S1.FS2.f1(2,2) +rather than S1.FS2.f1(2,3). + +## Translating DAP2 DDS Sequences {#dap2_translation} + +Any variable (as determined above) that is contained directly or +indirectly by a Sequence is subject to revision of its rank using the +following rules. + +Let the variable be contained in Sequence Q1, where Q1 is the +innermost containing sequence. If Q1 is itself contained (directly or +indirectly) in a sequence, or Q1 is contained (again directly or +indirectly) in a structure that has rank greater than 0, then the +variable will have an initial UNLIMITED dimension. Further, all +dimensions coming from "above" and including (in the containment +sense) the innermost Sequence, Q1, will be removed and replaced by +that single UNLIMITED dimension. The size associated with that +UNLIMITED is zero, which means that its contents are inaccessible +through the netCDF-3 API. Again, this differs from libnc-dap2, which +leaves out such variables. Again, however, this difference is backward +compatible. + +If the variable is contained in a single Sequence (i.e. not nested) +and all containing structures have rank 0, then the variable will have +an initial dimension whose size is the record count for that +Sequence. The name of the new dimension will be the name of the +Sequence. + +Consider this example. +```` +Dataset { + Structure { + Sequence { + Int32 f1[3]; + Int32 f2; + } SQ1; + } S1[2]; + Sequence { + Structure { + Int32 x1[7]; + } S2[5]; + } Q2; +} D; +```` + +The corresponding netCDF-3 translation is pretty much as follows (the +value for dimension Q2 may differ). +```` +dimensions: + unlimited = UNLIMITED ; // (0 currently) + S1.SQ1.f1_0 = 2 ; + S1.SQ1.f1_1 = 3 ; + S1.SQ1.f2_0 = 2 ; + Q2.S2.x1_0 = 5 ; + Q2.S2.x1_1 = 7 ; + Q2 = 5 ; +variables: + int S1.SQ1.f1(unlimited, S1.SQ1.f1_1) ; + int S1.SQ1.f2(unlimited) ; + int Q2.S2.x1(Q2, Q2.S2.x1_0, Q2.S2.x1_1) ; +```` + +Note that for example S1.SQ1.f1_0 is not actually used because it has +been folded into the unlimited dimension. + +Note that for sequences without a leading unlimited dimension, there +is a performance cost because the translation code has to walk the +data to determine how many records are associated with the +sequence. Since libnc-dap2 did essentially the same thing, it can be +assumed that the cost is not prohibitive. + +# Caching {#dap2_dap2_caching} + +In an effort to provide better performance for some access patterns, +client-side caching of data is available. The default is no caching, +but it may be enabled by prefixing the URL with the parameter "cache". + +Caching operates basically as follows. + +When a URL is first accessed using _nc_open()_, netCDF automatically +does a pre-fetch of selected variables. These include all variables +smaller than a specified (and user definable) size. This allows, for +example, quick access to coordinate variables. This can be suppressed +with the parameter "noprefetch". + +Whenever a request is made using some variant of the _nc_get_var()_ API +procedures, the complete variable is fetched and stored in the cache +as a new cache entry. Subsequence requests for any part of that +variable will access the cache entry to obtain the data. + +The cache may become too full, either because there are too many +entries or because it is taking up too much disk space. In this case +cache entries are purged until the cache size limits are reached. The +cache purge algorithm is LRU (least recently used) so that variables +that are repeatedly referenced will tend to stay in the cache. + +The cache is completely purged when _nc_close()_ is invoked. + +In order to decide if you should enable caching, you will need to have +some understanding of the access patterns of your program. + +The ncdump program always dumps one or more whole variables so it +turns on caching. + +If your program accesses only parts of a number of variables, then +caching should probably not be used since fetching whole variables +will probably slow down your program for no purpose. + +Unfortunately, caching is currently an all or nothing proposition, so +for more complex access patterns, the decision to cache or not may not +have an obvious answer. Probably a good rule of thumb is to avoid +caching initially and later turn it on to see its effect on +performance. + +# Defined Client Parameters {#dap2_dap2_defined_params} + +Currently, a limited set of client parameters is +recognized. Parameters not listed here are ignored, but no error is +signalled. All names are case insensitive. + +Parameter Name Legal Values Semantics +- "log" | "log=" - Turn on logging and send the log output to + the specified file. If no file is specified, then log output is sent + to standard error. +- "show=... das|dds|url" - This causes information to appear as + specific global attributes. The currently recognized tags are "dds" + to display the underlying DDS, "das" similarly, and "url" to display + the url used to retrieve the data. This parameter may be specified + multiple times (e.g. “show=dds&show=url”). +- "show=fetch" - This parameter causes the netCDF code to log a copy + of the complete url for every HTTP get request. If logging is + enabled, then this can be helpful in checking to see the access + behavior of the netCDF code. +- "stringlength=NN" - Specify the default string length to use for + string dimensions. The default is 64. The name "maxstrlen" is an + alias for "stringlength". +- "stringlength_\=NN" - Specify the default string length to use + for a string dimension for the specified variable. The default is + 64. The name "maxstrlen_\" is an alias for "stringlength_\". +- "cache" - This enables caching. +- "nocache" - This disbles caching. +- "cachelimit=NN" - Specify the maximum amount of space allowed for + the cache. +- "cachecount=NN" - Specify the maximum number of entries in the + cache. +- "prefetch" - This enables prefetch of small variables (default). +- "noprefetch" - This disables prefetch of small variables. +- "fillmismatch" - This enables _FillValue/Variable type mismatch. +- "nofillmismatch" - This disables _FillValue/Variable type mismatch (default). + +# Notes on Debugging OPeNDAP Access {#dap2_dap2_debug} + +The OPeNDAP support makes use of the logging facility of the +underlying oc system (see http://www.OPeNDAP.org/oc). +Note that this is currently separate from the +existing netCDF logging facility. Turning on this logging can +sometimes give important information. Logging can be enabled by +using the client parameter "log" or "log=filename", +where the first case will send log output to standard error and the +second will send log output to the specified file. + +Users should also be aware that if one is +accessing data over an NFS mount, one may see some .nfsxxxxx files; +those can be ignored. + +## HTTP Configuration. {#dap2_http2_config} + +Limited support for configuring the http connection is provided via +parameters in the “.dodsrc” configuration file. The relevant .dodsrc file is +located by first looking in the current working directory, and if not +found, then looking in the directory specified by the “$HOME” +environment variable. + +Entries in the .dodsrc file are of the form: +```` + ['['']']= +```` + +That is, it consists of a key name and value pair and optionally +preceded by a url enclosed in square brackets. + +For given KEY and URL strings, the value chosen is as follows: + +If URL is null, then look for the .dodsrc entry that has no url prefix +and whose key is same as the KEY for which we are looking. + +If the URL is not null, then look for all the .dodsrc entries that +have a url, URL1, say, and for which URL1 has the same host and port +as URL. All parts of the url's except host and port are ignored. +For example, if URL = http//x.y/a, then it will match +entries of the form +_[http//x.y/a]KEY=VALUE_ or _[http//x.y/b]KEY=VALUE_. +It will not match an entry of the form _[http//x.y:8080]KEY=VALUE +because the second has a port number (8080) different than the URL. +Finally from the set so constructed, choose the first matching entry. + +Currently, the supported set of keys (with descriptions) are as +follows. + +1. HTTP.VERBOSE + Type: boolean ("1"/"0") + Description: Produce verbose output, especially using SSL. + Related CURL Flags: CURLOPT_VERBOSE +1. HTTP.DEFLATE + Type: boolean ("1"/"0") + Description: Allow use of compression by the server. + Related CURL Flags: CURLOPT_ENCODING +1. HTTP.COOKIEJAR + Type: String representing file path + Description: Specify the name of file into which to store cookies. Defaults to in-memory storage. + Related CURL Flags:CURLOPT_COOKIEJAR +1. HTTP.CREDENTIALS.USER + Type: String representing user name + Description: Specify the user name for Digest and Basic authentication. + Related CURL Flags: +1. HTTP.CREDENTIALS.PASSWORD + Type: String representing password + Type: boolean ("1"/"0") + Description: Specify the password for Digest and Basic authentication. + Related CURL Flags: +1. HTTP.SSL.CERTIFICATE + Type: String representing file path + Description: Path to a file containing a PEM cerficate. + Related CURL Flags: CURLOPT_CERT +1. HTTP.SSL.KEY + Type: String representing file path + Description: Same as HTTP.SSL.CERTIFICATE, and should usually have the same value. + Related CURL Flags: CURLOPT_SSLKEY +1. HTTP.SSL.KEYPASSWORD + Type: String representing password + Description: Password for accessing the HTTP.SSL.KEY/HTTP.SSL.CERTIFICATE + Related CURL Flags: CURLOPT_KEYPASSWORD +1. HTTP.SSL.CAPATH + Type: String representing directory + Description: Path to a directory containing trusted certificates for validating server certificates. + Related CURL Flags: CURLOPT_CAPATH +1. HTTP.SSL.VALIDATE + Type: boolean ("1"/"0") + Description: Cause the client to verify the server's presented certificate. + Related CURL Flags: CURLOPT_SSL_VERIFYPEER, CURLOPT_SSL_VERIFYHOST +1. HTTP.TIMEOUT + Type: String ("dddddd") + Description: Specify the maximum time in seconds that you allow the http transfer operation to take. + Related CURL Flags: CURLOPT_TIMEOUT, CURLOPT_NOSIGNAL +1. HTTP.PROXY_SERVER + Type: String representing url to access the proxy: (e.g.http://[username:password@]host[:port]) + Description: Specify the needed information for accessing a proxy. + Related CURL Flags: CURLOPT_PROXY, CURLOPT_PROXYHOST, CURLOPT_PROXYUSERPWD +1. HTTP.READ.BUFFERSIZE + Type: String ("dddddd") + Description: Specify the the internal buffer size for curl reads. + Related CURL Flags: CURLOPT_BUFFERSIZE, CURL_MAX_WRITE_SIZE (16kB), + CURL_MAX_READ_SIZE (512kB). + +1. HTTP.KEEPALIVE + Type: String ("on|n/m") + Description: Specify that TCP KEEPALIVE should be enabled and that the associated idle wait time is n and that the associated repeat interval is m. If the value is of the form is the string "on", then turn on keepalive, but do not set idle or interval. + Related CURL Flags: CURLOPT_TCP_KEEPALIVE, CURLOPT_TCP_KEEPIDLE, + CURLOPT_TCP_KEEPINTVL. + +The related curl flags line indicates the curl flags modified by this +key. See the libcurl documentation of the _curl_easy_setopt()_ function +for more detail (http://curl.haxx.se/libcurl/c/curl_easy_setopt.html). + +For ESG client side key support, the following entries must be specified: +```` +HTTP.SSL.VALIDATE +HTTP.COOKIEJAR +HTTP.SSL.CERTIFICATE +HTTP.SSL.KEY +HTTP.SSL.CAPATH +```` + +Additionally, for ESG, the _HTTP.SSL.CERTIFICATE_ and _HTTP.SSL.KEY_ +entries should have same value, which is the file path for the +certificate produced by MyProxyLogon. The HTTP.SSL.CAPATH entry should +be the path to the "certificates" directory produced by MyProxyLogon. + +# Point of Contact {#dap2_poc} + +__Author__: Dennis Heimbigner
+__Email__: dmh at ucar dot edu
+__Initial Version__: 3/26/2009
+__Last Revised__: 9/25/2018 + + + +*/ diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 020a28cd7..f8423bcef 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -753,8 +753,7 @@ INPUT = \ @abs_top_srcdir@/docs/guide.dox \ @abs_top_srcdir@/docs/attribute_conventions.md \ @abs_top_srcdir@/docs/file_format_specifications.md \ - @abs_top_srcdir@/docs/OPeNDAP.dox \ - @abs_top_srcdir@/docs/DAP4.dox \ + @abs_top_srcdir@/docs/DAP2.dox \ @abs_top_srcdir@/docs/user_defined_formats.md \ @abs_top_srcdir@/docs/filters.md \ @abs_top_srcdir@/docs/inmemory.md \ diff --git a/docs/Makefile.am b/docs/Makefile.am index 43016cd36..48d8df13a 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -9,7 +9,7 @@ architecture.dox internal.dox windows-binaries.md \ building-with-cmake.md CMakeLists.txt groups.dox install.md notes.md \ install-fortran.md all-error-codes.md credits.md auth.md \ obsolete/fan_utils.html bestpractices.md filters.md indexing.dox \ -inmemory.md DAP4.dox OPeNDAP.dox attribute_conventions.md FAQ.md \ +inmemory.md DAP2.dox attribute_conventions.md FAQ.md \ file_format_specifications.md known_problems.md COPYRIGHT.dox \ user_defined_formats.md inmeminternal.dox diff --git a/docs/OPeNDAP.dox b/docs/OPeNDAP.dox index c55713c56..7c60dce21 100644 --- a/docs/OPeNDAP.dox +++ b/docs/OPeNDAP.dox @@ -575,11 +575,15 @@ Parameter Name Legal Values Semantics for a string dimension for the specified variable. The default is 64. The name "maxstrlen_" is an alias for "stringlength_". - "cache" - This enables caching. +- "nocache" - This disbles caching. - "cachelimit=NN" - Specify the maximum amount of space allowed for the cache. - "cachecount=NN" - Specify the maximum number of entries in the cache. +- "prefetch" - This enables prefetch of small variables (default). - "noprefetch" - This disables prefetch of small variables. +- "fillmismatch" - This enables _FillValue/Variable type mismatch. +- "nofillmismatch" - This disables _FillValue/Variable type mismatch (default). \section dap_debug Notes on Debugging OPeNDAP Access diff --git a/include/ncdap.h b/include/ncdap.h index e30d36395..6dac5aa9f 100644 --- a/include/ncdap.h +++ b/include/ncdap.h @@ -34,6 +34,8 @@ typedef unsigned int NCFLAGS; #define NCF_PREFETCH (0x0200) /* Cache prefetch enabled/disabled */ #define NCF_PREFETCH_EAGER (0x0400) /* Do eager prefetch; 0=>lazy */ #define NCF_PREFETCH_ALL (0x0800) /* Prefetch all variables */ +/* Allow _FillValue/Variable type mismatch */ +#define NCF_FILLMISMATCH (0x1000) /*COLUMBIA_HACK*/ #define NCF_COLUMBIA (0x80000000) /* Hack for columbia server */ diff --git a/include/nclog.h b/include/nclog.h index f5b068c6d..dd9fe85ce 100644 --- a/include/nclog.h +++ b/include/nclog.h @@ -7,6 +7,7 @@ #ifndef NCLOG_H #define NCLOG_H +#include #include "ncexternl.h" #define NCENVFLAG "NCLOGFILE" @@ -28,6 +29,7 @@ EXTERNL void nclogclose(void); /* The tag value is an arbitrary integer */ EXTERNL void nclog(int tag, const char* fmt, ...); +EXTERNL void ncvlog(int tag, const char* fmt, va_list ap); EXTERNL void nclogtext(int tag, const char* text); EXTERNL void nclogtextn(int tag, const char* text, size_t count); diff --git a/include/netcdf.h b/include/netcdf.h index bfc3d4f22..575f6d637 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -1961,6 +1961,11 @@ ncrecget(int ncid, long recnum, void **datap); EXTERNL int ncrecput(int ncid, long recnum, void *const *datap); +/* This function may be called to force the library to + initialize itself. It is not required, however. +*/ +EXTERNL int nc_initialize(void); + /* This function may be called to force the library to cleanup global memory so that memory checkers will not report errors. It is not required, however. diff --git a/libdap2/dapcvt.c b/libdap2/dapcvt.c index 7c762032a..2e4009af0 100644 --- a/libdap2/dapcvt.c +++ b/libdap2/dapcvt.c @@ -7,8 +7,20 @@ #ifdef _MSC_VER #include +#include #endif +struct Value { + long long llval; + double dval; +}; + +/*Forward*/ +static int cvtnumconst(const char* s, struct Value* val); +static int cvtdbl2int(struct Value* val); +static int cvtint2dbl(struct Value* val); +static int cvtint2int(nc_type dsttype, struct Value*); + NCerror dapconvert(nc_type srctype, nc_type dsttype, char* memory0, char* value0, size_t count) { @@ -193,111 +205,209 @@ fail: return THROW(ncstat); } +/** +Convert an attribute value string to a specific type. +If the conversion fails, then return NC_EBADTYPE. +If we need an int and the string value is out of range, return NC_ERANGE. +@param etype the target type +@param dst the memory into which to put the converted constants +@param src list of constants as strings +*/ NCerror -dapcvtattrval(nc_type etype, void* dst, NClist* src) +dapcvtattrval(nc_type etype, void* dst, NClist* src, NCattribute* att) { - int i,ok; + int i; NCerror ncstat = NC_NOERR; unsigned int memsize = nctypesizeof(etype); unsigned int nvalues = nclistlength(src); char* dstmem = (char*)dst; for(i=0;iname,s); + ncstat = NC_EBADTYPE; + goto next; + } + /* Force conformance with etype */ + /* Do Specific conversions */ + if(stype == NC_DOUBLE && etype < NC_FLOAT) {/* double->integer-type */ + if((ncstat = cvtdbl2int(&val))) goto next; + stype = NC_INT; /* pretend */ + }else if(stype == NC_INT && etype >= NC_FLOAT) {/*integer-type -> float type*/ + if((ncstat = cvtint2dbl(&val))) goto next; + stype = NC_DOUBLE; /* pretend */ + } + if(stype == NC_INT && etype < NC_FLOAT) {/* integer-type -> integer-type */ + if((ncstat = cvtint2int(etype,&val))) goto next; + } + switch (etype) { + case NC_BYTE: + /* Note that in DAP2, this is unsigned 8-bit integer */ + u8p = (unsigned char*)dstmem; + *u8p = (unsigned char)(val.llval); + break; + case NC_SHORT: + i16p = (short*)dstmem; + *i16p = (short)(val.llval); + break; + case NC_USHORT: + u16p = (unsigned short*)dstmem; + *u16p = (unsigned short)(val.llval); + break; + case NC_INT: + i32p = (int*)dstmem; + *i32p = (int)(val.llval); + break; + case NC_UINT: + u32p = (unsigned int*)dstmem; + *u32p = (unsigned int)(val.llval); + break; + case NC_FLOAT: + fp = (float*)dstmem; + *fp = (float)(val.dval); + break; + case NC_DOUBLE: + dp = (double*)dstmem; + *dp = (double)(val.dval); + break; + default: return NC_EINTERNAL; + } + } else if(etype == NC_CHAR) { char* p = (char*)dstmem; - int ival; - ok = sscanf(s,"%d%n",&ival,&nread); -#ifdef _MSC_VER - _ASSERTE(_CrtCheckMemory()); -#endif - /* For back compatibility, we allow any value, but force conversion */ - ival = (ival & 0xFF); - *p = (char)ival; - } break; - case NC_CHAR: { - signed char* p = (signed char*)dstmem; - ok = sscanf(s,"%c%n",p,&nread); - } break; - case NC_SHORT: { - short* p = (short*)dstmem; - ok = sscanf(s,"%hd%n",p,&nread); - } break; - case NC_INT: { - int* p = (int*)dstmem; - ok = sscanf(s,"%d%n",p,&nread); - } break; - case NC_FLOAT: { - float* p = (float*)dstmem; - ok = sscanf(s,"%g%n",p,&nread); -#if defined(_MSC_VER) && (_MSC_VER == 1500) - if (!_strnicmp(s, "NaN", 3)) { - ok = 1; - nread = 3; - } -#endif - } break; - case NC_DOUBLE: { - double* p = (double*)dstmem; - ok = sscanf(s,"%lg%n",p,&nread); -#if defined(_MSC_VER) && (_MSC_VER == 1500) - if (!_strnicmp(s, "NaN", 3)) { - ok = 1; - nread = 3; - } -#endif - } break; - case NC_UBYTE: { - unsigned char* p = (unsigned char*)dstmem; -#ifdef _MSC_VER - unsigned int uval; - ok = sscanf(s,"%u%n",&uval,&nread); - _ASSERTE(_CrtCheckMemory()); - /* For back compatibility, we allow any value, but force conversion */ - uval = (uval & 0xFF); - *p = (unsigned char)uval; -#else - ok = sscanf(s,"%hhu%n",p,&nread); -#endif - } break; - case NC_USHORT: { - unsigned short* p = (unsigned short*)dstmem; - ok = sscanf(s,"%hu%n",p,&nread); - } break; - case NC_UINT: { - unsigned int* p = (unsigned int*)dstmem; - ok = sscanf(s,"%u%n",p,&nread); - } break; - case NC_INT64: { - long long* p = (long long*)dstmem; -#ifdef _MSC_VER - ok = sscanf(s, "%I64d%n", p,&nread); -#else - ok = sscanf(s,"%lld%n",p,&nread); -#endif - } break; - case NC_UINT64: { - unsigned long long* p = (unsigned long long*)dstmem; - ok = sscanf(s,"%llu%n",p,&nread); - } break; - case NC_STRING: case NC_URL: { + size_t count; + int nread; + count = sscanf(s,"%c%n",p,&nread); + if(count != 1 || nread != slen) + {ncstat = NC_EBADTYPE; goto next;} + } else if(etype == NC_STRING || etype == NC_URL) { char** p = (char**)dstmem; *p = nulldup(s); - ok = 1; - } break; - default: + } else { PANIC1("unexpected nc_type: %d",(int)etype); } - if(ok != 1 || nread != slen) {ncstat = NC_EINVAL; goto done;} +next: /* inside loop */ + if(ncstat == NC_ERANGE) + nclog(NCLOGERR,"Attribute value out of range: %s = %s",att->name,s); + else if(ncstat == NC_EBADTYPE) + nclog(NCLOGERR,"Unexpected attribute type or untranslatable value: %s",att->name); + ncstat = NC_NOERR; dstmem += memsize; } -done: return THROW(ncstat); } +/** +@param val resulting converted numeric value +@return NC_INT || NC_DOUBLE || NC_NAT (failure) +*/ +static int +cvtnumconst(const char* s, struct Value* val) +{ + size_t slen = strlen(s); + int nread; /* # of chars read */ + size_t count; /* # of conversions */ + /* Try to convert to integer first */ + count = sscanf(s,"%lld%n",&val->llval,&nread); + if(count == 1 && nread == slen) + return NC_INT; + /* Try to convert to float second */ +#ifdef _WIN32 + if (!_strnicmp(s, "NaN", 3)) {count = 1; nread = 3; val->dval = NAN;} else +#endif + count = sscanf(s,"%lg%n",&val->dval,&nread); + if(count == 1 && nread == slen) + return NC_DOUBLE; + return NC_INT; +} + +/** +Convert a struct Value.dval field to a long in struct Value.llval field. +Report if the result is out of range wrt NC_MAX/MIN_INT. + @param val store original and converted value +@return NC_NOERR | NC_ERANGE +*/ +static int +cvtdbl2int(struct Value* val) +{ + /* Inter-convert */ +#ifdef _WIN32 + if(isnan(val->dval)) return NC_ERANGE; +#endif + val->llval = (long long)val->dval; + if(val->llval < NC_MIN_INT || val->llval > NC_MAX_INT) + return NC_ERANGE; + return NC_NOERR; +} + +/** +Convert a struct Value.llval field to double in struct Value.dval field. +@return NC_NOERR +*/ +static int +cvtint2dbl(struct Value* val) +{ + /* Inter-convert */ + val->dval = (double)val->llval; + return NC_NOERR; +} + +/** +Convert long long bit pattern to conform to a given +integer type. +@param dsttype target integer type +@param valp pointer to long long value to convert +@return NC_NOERR | NC_EBADTYPE +*/ +static int +cvtint2int(nc_type dsttype, struct Value* val) +{ + /* assert dsttype < NC_FLOAT && dsttype != NC_CHAR */ + + /* We do not actually do much in the way of range checking, + we just truncate the bit pattern to the proper size, taking + signedness into account */ + + switch (dsttype) { + case NC_BYTE: + val->llval = (long long)((signed char)(val->llval)); + break; + case NC_UBYTE: + val->llval = (long long)(val->llval & 0xFF); + break; + case NC_SHORT: + val->llval = (long long)((short)(val->llval)); + break; + case NC_USHORT: + val->llval = (long long)(val->llval & 0xFFFF); + break; + case NC_INT: + val->llval = (long long)((int)(val->llval)); + break; + case NC_UINT: + val->llval = (long long)(val->llval & 0xFFFFFFFF); + break; + default: return NC_EBADTYPE; + } + return NC_NOERR; +} diff --git a/libdap2/dapincludes.h b/libdap2/dapincludes.h index 6fe1d79b5..c8b02bd9e 100644 --- a/libdap2/dapincludes.h +++ b/libdap2/dapincludes.h @@ -107,6 +107,6 @@ extern NCerror showprojection(NCDAPCOMMON*, CDFnode* var); /* From: dapcvt.c*/ extern NCerror dapconvert(nc_type, nc_type, char*, char*, size_t); -extern int dapcvtattrval(nc_type, void*, NClist*); +extern int dapcvtattrval(nc_type, void*, NClist*, NCattribute*); #endif /*DAPINCLUDES_H*/ diff --git a/libdap2/daputil.c b/libdap2/daputil.c index db743078e..bef69c202 100644 --- a/libdap2/daputil.c +++ b/libdap2/daputil.c @@ -41,18 +41,16 @@ cdflegalname(char* name) to the external netCDF variable type. The proper way is to, for example, convert unsigned short to an int to maintain the values. - Unfortuneately, libnc-dap does not do this: + Unfortunately, libnc-dap does not do this: it translates the types directly. For example libnc-dap upgrades the DAP byte type, which is unsigned char, to NC_BYTE, which signed char. - Oh well. - For netcdf-4, we can do proper type conversion. + Oh well. So we do the same. */ nc_type nctypeconvert(NCDAPCOMMON* drno, nc_type nctype) { nc_type upgrade = NC_NAT; - if(drno->controls.flags & NCF_NC3) { /* libnc-dap mimic invariant is to maintain type size */ switch (nctype) { case NC_CHAR: upgrade = NC_CHAR; break; @@ -62,33 +60,12 @@ nctypeconvert(NCDAPCOMMON* drno, nc_type nctype) case NC_USHORT: upgrade = NC_SHORT; break; case NC_INT: upgrade = NC_INT; break; case NC_UINT: upgrade = NC_INT; break; - case NC_INT64: upgrade = NC_INT64; break; - case NC_UINT64: upgrade = NC_UINT64; break; case NC_FLOAT: upgrade = NC_FLOAT; break; case NC_DOUBLE: upgrade = NC_DOUBLE; break; case NC_URL: case NC_STRING: upgrade = NC_CHAR; break; default: break; } - } else if(drno->controls.flags & NCF_NC4) { - /* netcdf-4 conversion is more correct */ - switch (nctype) { - case NC_CHAR: upgrade = NC_CHAR; break; - case NC_BYTE: upgrade = NC_BYTE; break; - case NC_UBYTE: upgrade = NC_UBYTE; break; - case NC_SHORT: upgrade = NC_SHORT; break; - case NC_USHORT: upgrade = NC_USHORT; break; - case NC_INT: upgrade = NC_INT; break; - case NC_UINT: upgrade = NC_UINT; break; - case NC_INT64: upgrade = NC_INT64; break; - case NC_UINT64: upgrade = NC_UINT64; break; - case NC_FLOAT: upgrade = NC_FLOAT; break; - case NC_DOUBLE: upgrade = NC_DOUBLE; break; - case NC_URL: - case NC_STRING: upgrade = NC_STRING; break; - default: break; - } - } return upgrade; } @@ -742,12 +719,12 @@ oc_dumpnode(conn,*rootp); /* Check a name to see if it contains illegal dap characters */ -static char* baddapchars = "./"; +static const char* baddapchars = "./"; int dap_badname(char* name) { - char* p; + const char* p; if(name == NULL) return 0; for(p=baddapchars;*p;p++) { if(strchr(name,*p) != NULL) diff --git a/libdap2/ncd2dispatch.c b/libdap2/ncd2dispatch.c index ddf7676fe..18d4aaf1b 100644 --- a/libdap2/ncd2dispatch.c +++ b/libdap2/ncd2dispatch.c @@ -35,7 +35,7 @@ static NCerror builddims(NCDAPCOMMON*); static char* getdefinename(CDFnode* node); static NCerror buildvars(NCDAPCOMMON*); static NCerror buildglobalattrs(NCDAPCOMMON*, CDFnode* root); -static NCerror buildattribute(NCDAPCOMMON*, NCattribute*, nc_type, int); +static NCerror buildattribute(NCDAPCOMMON*, CDFnode*, NCattribute*); static void computedimindexanon(CDFnode* dim, CDFnode* var); static void replacedims(NClist* dims); static int equivalentdim(CDFnode* basedim, CDFnode* dupdim); @@ -757,9 +757,6 @@ fprintf(stderr,"buildvars.candidate=|%s|\n",var->ncfullname); dimids[j] = dim->ncid; } } - - - definename = getdefinename(var); #ifdef DEBUG1 @@ -789,7 +786,22 @@ fprintf(stderr,"\n"); if(var->attributes != NULL) { for(j=0;jattributes);j++) { NCattribute* att = (NCattribute*)nclistget(var->attributes,j); - ncstat = buildattribute(dapcomm,att,var->etype,varid); + /* Check for _FillValue/Variable mismatch */ + if(strcmp(att->name,"_FillValue")==0) { + if(att->etype != var->etype) { + /* Log a message */ + nclog(NCLOGERR,"_FillValue/Variable type mismatch: variable=%s",var->ncbasename); + /* See if mismatch is allowed */ + if(FLAGSET(dapcomm->controls,NCF_FILLMISMATCH)) { + /* Forcibly change the attribute type to match */ + att->etype = var->etype; + } else { + ncstat = NC_EBADTYPE; /* fail */ + goto done; + } + } + } + ncstat = buildattribute(dapcomm,var,att); if(ncstat != NC_NOERR) goto done; } } @@ -814,7 +826,7 @@ buildglobalattrs(NCDAPCOMMON* dapcomm, CDFnode* root) if(root->attributes != NULL) { for(i=0;iattributes);i++) { NCattribute* att = (NCattribute*)nclistget(root->attributes,i); - ncstat = buildattribute(dapcomm,att,NC_NAT,NC_GLOBAL); + ncstat = buildattribute(dapcomm,NULL,att); if(ncstat != NC_NOERR) goto done; } } @@ -883,11 +895,12 @@ done: } static NCerror -buildattribute(NCDAPCOMMON* dapcomm, NCattribute* att, nc_type vartype, int varid) +buildattribute(NCDAPCOMMON* dapcomm, CDFnode* var, NCattribute* att) { int i; NCerror ncstat = NC_NOERR; unsigned int nvalues = nclistlength(att->values); + int varid = (var == NULL ? NC_GLOBAL : var->ncid); /* If the type of the attribute is string, then we need*/ /* to convert to a single character string by concatenation. @@ -921,17 +934,7 @@ buildattribute(NCDAPCOMMON* dapcomm, NCattribute* att, nc_type vartype, int vari nc_type atype; unsigned int typesize; void* mem = NULL; - /* It turns out that some servers upgrade the type - of _FillValue in order to correctly preserve the - original value. However, since the type of the - underlying variable is not changes, we get a type - mismatch. So, make sure the type of the fillvalue - is the same as that of the controlling variable. - */ - if(varid != NC_GLOBAL && strcmp(att->name,"_FillValue")==0) - atype = nctypeconvert(dapcomm,vartype); - else - atype = nctypeconvert(dapcomm,att->etype); + atype = nctypeconvert(dapcomm,att->etype); typesize = nctypesizeof(atype); if (nvalues > 0) { mem = malloc(typesize * nvalues); @@ -939,10 +942,13 @@ buildattribute(NCDAPCOMMON* dapcomm, NCattribute* att, nc_type vartype, int vari _ASSERTE(_CrtCheckMemory()); #endif } - ncstat = dapcvtattrval(atype,mem,att->values); + ncstat = dapcvtattrval(atype,mem,att->values,att); #ifdef _MSC_VER _ASSERTE(_CrtCheckMemory()); #endif + if(ncstat == NC_ERANGE) + nclog(NCLOGERR,"Attribute value out of range: %s:%s", + (var==NULL?"":var->ncbasename),att->name); if(ncstat) {nullfree(mem); goto done;} ncstat = nc_put_att(dapcomm->substrate.nc3id,varid,att->name,atype,nvalues,mem); #ifdef _MSC_VER @@ -2209,6 +2215,12 @@ applyclientparamcontrols(NCDAPCOMMON* dapcomm) if(dapparamcheck(dapcomm,"show","fetch")) SETFLAG(dapcomm->controls,NCF_SHOWFETCH); + /* enable/disable _FillValue/Variable Mis-match */ + if(dapparamcheck(dapcomm,"fillmismatch",NULL)) + SETFLAG(dapcomm->controls,NCF_FILLMISMATCH); + else if(dapparamcheck(dapcomm,"nofillmismatch",NULL)) + CLRFLAG(dapcomm->controls,NCF_FILLMISMATCH); + nclog(NCLOGNOTE,"Caching=%d",FLAGSET(dapcomm->controls,NCF_CACHE)); } @@ -2819,4 +2831,4 @@ NCD2_get_var_chunk_cache(int ncid, int p2, size_t* p3, size_t* p4, float* p5) return THROW(ret); } -#endif // USE_NETCDF4 +#endif /* USE_NETCDF4 */ diff --git a/libdap4/d4crc32.c b/libdap4/d4crc32.c index 0c0b63547..460c8efb3 100644 --- a/libdap4/d4crc32.c +++ b/libdap4/d4crc32.c @@ -42,7 +42,7 @@ #include "d4includes.h" -static uint32_t crc32_tab[] = { +static const uint32_t crc32_tab[] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 4ca44aeb9..1fcd54a73 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -326,12 +326,12 @@ freeCurl(NCD4curl* curl) } /* Define the set of protocols known to be constrainable */ -static char* constrainableprotocols[] = {"http", "https",NULL}; +static const char* constrainableprotocols[] = {"http", "https",NULL}; static int constrainable(NCURI* durl) { - char** protocol = constrainableprotocols; + const char** protocol = constrainableprotocols; for(;*protocol;protocol++) { if(strcmp(durl->protocol,*protocol)==0) return 1; @@ -433,6 +433,7 @@ applyclientparamcontrols(NCD4INFO* info) CLRFLAG(info->controls.flags,NCF_SHOWFETCH); CLRFLAG(info->controls.flags,NCF_NC4); CLRFLAG(info->controls.flags,NCF_NCDAP); + CLRFLAG(info->controls.flags,NCF_FILLMISMATCH); /* Turn on any default on flags */ SETFLAG(info->controls.flags,DFALT_ON_FLAGS); @@ -452,6 +453,13 @@ applyclientparamcontrols(NCD4INFO* info) if(value != NULL) strncpy(info->controls.substratename,value,NC_MAX_NAME); + value = getparam(info,"fillmismatch"); + if(value != NULL) + SETFLAG(info->controls.flags,NCF_FILLMISMATCH); + + value = getparam(info,"nofillmismatch"); + if(value != NULL) + CLRFLAG(info->controls.debugflags,NCF_FILLMISMATCH); } static void diff --git a/libdap4/d4meta.c b/libdap4/d4meta.c index a14f4ee3d..fcf374377 100644 --- a/libdap4/d4meta.c +++ b/libdap4/d4meta.c @@ -34,7 +34,7 @@ static int buildStructure(NCD4meta* builder, NCD4node* structvar); static int buildStructureType(NCD4meta* builder, NCD4node* structtype); static int buildVariable(NCD4meta* builder, NCD4node* var); static int buildVlenType(NCD4meta* builder, NCD4node* seqtype); -static int compileAttrValues(NCD4meta* builder, NCD4node* basetype, NClist* values, void** memoryp); +static int compileAttrValues(NCD4meta* builder, NCD4node* attr, void** memoryp); static void computeOffsets(NCD4meta* builder, NCD4node* cmpd); static int convertString(union ATOMICS* converter, NCD4node* type, const char* s); static void* copyAtomic(union ATOMICS* converter, nc_type type, size_t len, void* dst); @@ -78,7 +78,7 @@ NCD4_metabuild(NCD4meta* metadata, int ncid) markfixedsize(metadata); markdapsize(metadata); /* Process the metadata state */ - ret = build(metadata,metadata->root); + if((ret = build(metadata,metadata->root))) goto done; /* Done with the metadata*/ if((ret=nc_enddef(metadata->ncid))) goto done; @@ -258,7 +258,9 @@ build(NCD4meta* builder, NCD4node* root) /* Finally, define the top-level variables */ for(i=0;iallnodes,i); - if(ISVAR(x->sort) && ISTOPLEVEL(x)) buildVariable(builder,x); + if(ISVAR(x->sort) && ISTOPLEVEL(x)) { + if((ret=buildVariable(builder,x))) goto done; + } } done: @@ -432,9 +434,9 @@ buildAttributes(NCD4meta* builder, NCD4node* varorgroup) varid = NC_GLOBAL; else varid = varorgroup->meta.id; - if((ret=compileAttrValues(builder,attr->basetype,attr->attr.values,&memory))) { - nullfree(memory); - FAIL(NC_ERANGE,"Malformed attribute value(s) for: %s",attr->name); + if((ret=compileAttrValues(builder,attr,&memory))) { + nullfree(memory); + FAIL(ret,"Malformed attribute value(s) for: %s",attr->name); } group = NCD4_groupFor(varorgroup); NCCHECK((nc_put_att(group->meta.id,varid,attr->name,attr->basetype->meta.id,count,memory))); @@ -717,17 +719,33 @@ into a memory chunk capable of being passed to nc_put_att(). */ static int -compileAttrValues(NCD4meta* builder, NCD4node* basetype, NClist* values, void** memoryp) +compileAttrValues(NCD4meta* builder, NCD4node* attr, void** memoryp) { int i,ret = NC_NOERR; - int count = nclistlength(values); unsigned char* memory = NULL; unsigned char* p; size_t size; NCD4node* truebase = NULL; union ATOMICS converter; int isenum = 0; + NCD4node* container = attr->container; + NCD4node* basetype = attr->basetype; + NClist* values = attr->attr.values; + int count = nclistlength(values); + /* Deal with _FillValue */ + if(container->sort == NCD4_VAR && strcmp(attr->name,"_FillValue")==0) { + /* Verify or fix or fail on type match */ + if(container->basetype != basetype) { + /* _FillValue/Variable type mismatch */ + if(FLAGSET(builder->controller->controls.flags,NCF_FILLMISMATCH)) { + /* Force type match */ + basetype = (attr->basetype = container->basetype); + } else {/* Fail */ + FAIL(NC_EBADTYPE,"_FillValue/Variable type mismatch: %s:%s",container->name,attr->name); + } + } + } isenum = (basetype->subsort == NC_ENUM); truebase = (isenum ? basetype->basetype : basetype); if(!ISTYPE(truebase->sort) || (truebase->meta.id > NC_MAX_ATOMIC_TYPE)) diff --git a/libdap4/d4parser.c b/libdap4/d4parser.c index 06289c710..10a7f9b94 100644 --- a/libdap4/d4parser.c +++ b/libdap4/d4parser.c @@ -26,7 +26,7 @@ If the sort is NCD4_NULL, then that means it is irrelevant because that keyword will never be searched for in this table. */ -struct KEYWORDINFO { +static const struct KEYWORDINFO { char* tag; /* The xml tag e.g. */ NCD4sort sort; /* What kind of node are we building */ nc_type subsort; /* discriminator */ @@ -63,7 +63,7 @@ struct KEYWORDINFO { }; typedef struct KEYWORDINFO KEYWORDINFO; -static struct ATOMICTYPEINFO { +static const struct ATOMICTYPEINFO { char* name; nc_type type; size_t size; } atomictypeinfo[] = { /* Keep in sorted order for binary search */ @@ -114,7 +114,7 @@ static int fillgroup(NCD4parser*, NCD4node* group, ezxml_t xml); static NCD4node* getOpaque(NCD4parser*, ezxml_t varxml, NCD4node* group); static int getValueStrings(NCD4parser*, NCD4node*, ezxml_t xattr, NClist*); static int isReserved(const char* name); -static KEYWORDINFO* keyword(const char* name); +static const KEYWORDINFO* keyword(const char* name); static NCD4node* lookupAtomictype(NCD4parser*, const char* name); static NCD4node* lookFor(NClist* elems, const char* name, NCD4sort sort); static NCD4node* lookupFQN(NCD4parser*, const char* sfqn, NCD4sort); @@ -335,7 +335,7 @@ parseVariables(NCD4parser* parser, NCD4node* group, ezxml_t xml) ezxml_t x; for(x=xml->child;x != NULL;x=x->ordered) { NCD4node* node = NULL; - KEYWORDINFO* info = keyword(x->name); + const KEYWORDINFO* info = keyword(x->name); if(info == NULL) FAIL(NC_ETRANSLATION,"Unexpected node type: %s",x->name); /* Check if we need to process this node */ @@ -353,7 +353,7 @@ parseVariable(NCD4parser* parser, NCD4node* container, ezxml_t xml, NCD4node** n { int ret = NC_NOERR; NCD4node* node = NULL; - KEYWORDINFO* info = keyword(xml->name); + const KEYWORDINFO* info = keyword(xml->name); switch (info->subsort) { case NC_STRUCT: @@ -441,7 +441,7 @@ parseFields(NCD4parser* parser, NCD4node* container, ezxml_t xml) ezxml_t x; for(x=xml->child;x != NULL;x=x->ordered) { NCD4node* node = NULL; - KEYWORDINFO* info = keyword(x->name); + const KEYWORDINFO* info = keyword(x->name); if(!ISVAR(info->sort)) continue; /* not a field */ ret = parseVariable(parser,container,x,&node); if(ret) goto done; @@ -461,7 +461,7 @@ parseVlenField(NCD4parser* parser, NCD4node* container, ezxml_t xml, NCD4node** NCD4node* field = NULL; ezxml_t x; for(x=xml->child;x != NULL;x=x->ordered) { - KEYWORDINFO* info = keyword(x->name); + const KEYWORDINFO* info = keyword(x->name); if(!ISVAR(info->sort)) continue; /* not a field */ if(field != NULL) {ret = NC_EBADTYPE; goto done;} @@ -603,7 +603,7 @@ parseAtomicVar(NCD4parser* parser, NCD4node* container, ezxml_t xml, NCD4node** NCD4node* node = NULL; NCD4node* base = NULL; const char* typename; - KEYWORDINFO* info; + const KEYWORDINFO* info; NCD4node* group; /* Check for aliases */ @@ -742,12 +742,7 @@ parseAttributes(NCD4parser* parser, NCD4node* container, ezxml_t xml) } if((ret=makeNode(parser,container,x,NCD4_ATTR,NC_NULL,&attr))) goto done; - /* HACK: If the attribute is _FillValue, then force the use of the - container's type as the attribute type */ - if(strcmp(attr->name,"_FillValue") == 0) - basetype = container->basetype; - else - basetype = lookupFQN(parser,type,NCD4_TYPE); + basetype = lookupFQN(parser,type,NCD4_TYPE); if(basetype == NULL) FAIL(NC_EBADTYPE,"Unknown type: ",type); if(basetype->subsort == NC_NAT && basetype->subsort != NC_ENUM) @@ -1127,14 +1122,14 @@ done: return (ret == NC_NOERR ? match : NULL); } -static KEYWORDINFO* +static const KEYWORDINFO* keyword(const char* name) { int n = sizeof(keywordmap)/sizeof(KEYWORDINFO); int L = 0; int R = (n - 1); int m, cmp; - struct KEYWORDINFO* p; + const struct KEYWORDINFO* p; for(;;) { if(L > R) break; m = (L + R) / 2; @@ -1174,7 +1169,7 @@ defineAtomicTypes(NCD4parser* parser) { int ret = NC_NOERR; NCD4node* node; - struct ATOMICTYPEINFO* ati; + const struct ATOMICTYPEINFO* ati; parser->atomictypes = nclistnew(); if(parser->atomictypes == NULL) diff --git a/libdap4/d4util.c b/libdap4/d4util.c index d2e83f6eb..3de453563 100644 --- a/libdap4/d4util.c +++ b/libdap4/d4util.c @@ -442,9 +442,7 @@ NCD4_error(int code, const int line, const char* file, const char* fmt, ...) va_list argv; fprintf(stderr,"(%s:%d) ",file,line); va_start(argv,fmt); - vfprintf(stderr,fmt,argv); - fprintf(stderr,"\n"); - fflush(stderr); + ncvlog(NCLOGERR,fmt,argv); return code; } diff --git a/libdispatch/drc.c b/libdispatch/drc.c index 82fc07a1e..391be4b9b 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -291,34 +291,34 @@ rccompile(const char* path) if((llen=strlen(line)) == 0) continue; /* empty line */ triple = (NCTriple*)calloc(1,sizeof(NCTriple)); if(triple == NULL) {ret = NC_ENOMEM; goto done;} - if(line[0] == LTAG) { - char* url = ++line; - char* rtag = strchr(line,RTAG); - if(rtag == NULL) { - nclog(NCLOGERR, "Malformed [url] in %s entry: %s",path,line); - free(triple); - continue; - } - line = rtag + 1; - *rtag = '\0'; - /* compile the url and pull out the host */ - if(uri) ncurifree(uri); - if(ncuriparse(url,&uri) != NCU_OK) { - nclog(NCLOGERR, "Malformed [url] in %s entry: %s",path,line); - free(triple); + if(line[0] == LTAG) { + char* url = ++line; + char* rtag = strchr(line,RTAG); + if(rtag == NULL) { + nclog(NCLOGERR, "Malformed [url] in %s entry: %s",path,line); + free(triple); continue; - } - ncbytesclear(tmp); - ncbytescat(tmp,uri->host); - if(uri->port != NULL) { + } + line = rtag + 1; + *rtag = '\0'; + /* compile the url and pull out the host */ + if(uri) ncurifree(uri); + if(ncuriparse(url,&uri) != NCU_OK) { + nclog(NCLOGERR, "Malformed [url] in %s entry: %s",path,line); + free(triple); + continue; + } + ncbytesclear(tmp); + ncbytescat(tmp,uri->host); + if(uri->port != NULL) { ncbytesappend(tmp,':'); - ncbytescat(tmp,uri->port); - } - ncbytesnull(tmp); - triple->host = ncbytesextract(tmp); + ncbytescat(tmp,uri->port); + } + ncbytesnull(tmp); + triple->host = ncbytesextract(tmp); if(strlen(triple->host)==0) {free(triple->host); triple->host = NULL;} - } + } /* split off key and value */ key=line; value = strchr(line, '='); diff --git a/libdispatch/dwinpath.c b/libdispatch/dwinpath.c index 6f5e551c8..b83ef543c 100644 --- a/libdispatch/dwinpath.c +++ b/libdispatch/dwinpath.c @@ -41,9 +41,9 @@ All other cases are passed thru unchanged /* Define legal windows drive letters */ -static char* windrive = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; +static const char* windrive = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; -static size_t cdlen = 10; /* strlen("/cygdrive/") */ +static const size_t cdlen = 10; /* strlen("/cygdrive/") */ static int pathdebug = -1; diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c index d06d0ba69..f43d23dff 100644 --- a/libdispatch/nclog.c +++ b/libdispatch/nclog.c @@ -177,6 +177,25 @@ nclog(int tag, const char* fmt, ...) fflush(nclogstream); } +void +ncvlog(int tag, const char* fmt, va_list ap) +{ + char* prefix; + + if(!nclogginginitialized) ncloginit(); + + if(!nclogging || nclogstream == NULL) return; + + prefix = nctagname(tag); + fprintf(nclogstream,"%s:",prefix); + + if(fmt != NULL) { + vfprintf(nclogstream, fmt, ap); + } + fprintf(nclogstream, "\n" ); + fflush(nclogstream); +} + void nclogtext(int tag, const char* text) { diff --git a/libsrc/memio.c b/libsrc/memio.c index 1ffcf9271..110cb0e88 100644 --- a/libsrc/memio.c +++ b/libsrc/memio.c @@ -115,7 +115,6 @@ static int memio_close(ncio* nciop, int); static int readfile(const char* path, NC_memio*); static int writefile(const char* path, NCMEMIO*); static int fileiswriteable(const char* path); -static int fileisreadable(const char* path); static int fileexists(const char* path); /* Mnemonic */ @@ -673,6 +672,7 @@ fileiswriteable(const char* path) return 1; } +#if 0 /* not used */ /* Return 1 if file is READABLE, return 0 otherwise; assumes fileexists has been checked already */ static int @@ -685,6 +685,7 @@ fileisreadable(const char* path) return 0; return 1; } +#endif /* Read contents of a disk file into a memory chunk */ static int diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 94de9cd42..d4c8842ce 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -13,9 +13,9 @@ include $(top_srcdir)/lib_flags.am # Un comment to use a more verbose test driver #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +TEST_EXTENSIONS = .sh # Note which tests depend on other tests. necessary for make -j check -TEST_EXTENSIONS = .sh extradir = # Link to our assembled library. diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt index ae20d6cff..603409269 100644 --- a/ncdap_test/CMakeLists.txt +++ b/ncdap_test/CMakeLists.txt @@ -31,7 +31,7 @@ IF(ENABLE_TESTS) add_sh_test(ncdap tst_ber) add_sh_test(ncdap tst_remote3) # not yet add_sh_test(ncdap tst_hyrax) - + add_sh_test(ncdap tst_fillmismatch) IF(ENABLE_DAP_LONG_TESTS) add_sh_test(ncdap tst_longremote3) ENDIF(ENABLE_DAP_LONG_TESTS) diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index 33e4152eb..151d136ed 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -6,8 +6,10 @@ # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am -#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#TEST_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +# Un comment to use a more verbose test driver +SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +TEST_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose # Note which tests depend on other tests. Necessary for make -j check. TEST_EXTENSIONS = .sh @@ -40,7 +42,7 @@ check_PROGRAMS += findtestserver findtestserver_SOURCES = findtestserver.c if BUILD_UTILITIES -TESTS += tst_ber.sh tst_remote3.sh tst_formatx.sh testurl.sh +TESTS += tst_ber.sh tst_remote3.sh tst_formatx.sh testurl.sh tst_fillmismatch.sh endif TESTS += test_partvar @@ -83,13 +85,14 @@ EXTRA_DIST = tst_ncdap3.sh \ tst_longremote3.sh \ tst_filelists.sh tst_urls.sh tst_utils.sh \ t_dap.c CMakeLists.txt tst_formatx.sh testauth.sh testurl.sh \ - t_ncf330.c tst_ber.sh findtestserver.c.in + t_ncf330.c tst_ber.sh tst_fillmismatch.sh \ + findtestserver.c.in CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe # This should only be left behind if using parallel io CLEANFILES += tmp_* -DISTCLEANFILES = findtestserver4.c +DISTCLEANFILES = findtestserver.c # This rule are used if someone wants to rebuild t_dap3a.c # Otherwise never invoked, but records how to do it. diff --git a/ncdap_test/expected3/Makefile.am b/ncdap_test/expected3/Makefile.am index f1b4c2226..8e5c485f1 100644 --- a/ncdap_test/expected3/Makefile.am +++ b/ncdap_test/expected3/Makefile.am @@ -34,7 +34,7 @@ test.sds4.dmp test.sds5.dmp test.sds6.dmp \ test.sds7.dmp test.vs1.dmp test.vs2.dmp \ test.vs3.dmp test.vs4.dmp test.vs5.dmp \ text.nc.dmp whoi.dmp \ -testData.nc.dmp kwcase.nc.dmp \ +testData.nc.dmp kwcase.nc.dmp fillmismatch.nc.dmp \ CMakeLists.txt diff --git a/ncdap_test/expected3/fillmismatch.nc.dmp b/ncdap_test/expected3/fillmismatch.nc.dmp new file mode 100644 index 000000000..65029e251 --- /dev/null +++ b/ncdap_test/expected3/fillmismatch.nc.dmp @@ -0,0 +1,44 @@ +netcdf fillmismatch { +dimensions: + maxStrlen64 = 64 ; +variables: + byte b ; + b:Description = "A test byte" ; + b:units = "unknown" ; + int i32 ; + i32:Description = "A 32 bit test server int" ; + i32:units = "unknown" ; + i32:_FillValue = 100 ; + int ui32 ; + short i16 ; + short ui16 ; + float f32 ; + double f64 ; + char s(maxStrlen64) ; + char u(maxStrlen64) ; + +// global attributes: + :Facility.PrincipleInvestigator = "Mark Abbott\n", + "Ph.D" ; + :Facility.DataCenter = "COAS Environmental Computer Facility" ; + :Facility.DrifterType = "MetOcean WOCE/OCM" ; +data: + + b = 0 ; + + i32 = 1 ; + + ui32 = 0 ; + + i16 = 0 ; + + ui16 = 0 ; + + f32 = 0 ; + + f64 = 1000 ; + + s = "This is a data test string (pass 0)." ; + + u = "http://www.dods.org" ; +} diff --git a/ncdap_test/findtestserver.c b/ncdap_test/findtestserver.c deleted file mode 100644 index 9ad5e697e..000000000 --- a/ncdap_test/findtestserver.c +++ /dev/null @@ -1,79 +0,0 @@ -#include "config.h" -#include -#include -#include -#include "nctestserver.h" - -/* Support stringification of -D macros */ -#define XSTRINGIFY(s) #s -#define STRINGIFY(s) XSTRINGIFY(s) - - -/** -usage: findtestserver dap2|dap4 suffix [serverlist] - -Given a partial suffix path, try to find a -server for which a request to server + suffix -returns some kind of result using the -specified protocol. This indicates that the -server is up and running. Return the complete -url for the server plus the path. -If serverlist is present, then is should be a comma -separated list of servers (host+port) to try. -It defaults to REMOTETESTSERVERS. -*/ - -static void -usage() -{ - fprintf(stderr,"usage: findtestserver dap2|dap4 suffix [serverlist]\n"); - exit(1); -} - - -int -main(int argc, char** argv) -{ - const char* url = NULL; - const char* servlet = NULL; - const char* proto = NULL; - const char* serverlist = NULL; - int isdap4 = 0; /* 1 => dap4 */ - - argc--; argv++; - if(argc < 2) - usage(); - proto = strdup(argv[0]); - servlet = strdup(argv[1]); - if(argc >= 3) - serverlist = strdup(argv[2]); - -#ifdef ENABLE_DAP - if(strcasecmp(proto,"dap2")==0) - isdap4 = 0; - else -#endif -#ifdef ENABLE_DAP4 - if(strcasecmp(proto,"dap4")==0) - isdap4 = 1; - else -#endif - usage(); - - if(serverlist == NULL) { -#ifdef REMOTETESTSERVERS - serverlist = strdup(REMOTETESTSERVERS); -#endif - } - if(serverlist == NULL || strlen(serverlist) == 0) - fprintf(stderr,"Cannot determine a server list"); - - url = nc_findtestserver(servlet,isdap4,serverlist); - if(url == NULL) { - url = ""; - fprintf(stderr,"not found: %s\n",servlet); - } - printf("%s",url); - fflush(stdout); - exit(0); -} diff --git a/ncdap_test/testdata3/Makefile.am b/ncdap_test/testdata3/Makefile.am index 2defd0d6d..6cc497b95 100644 --- a/ncdap_test/testdata3/Makefile.am +++ b/ncdap_test/testdata3/Makefile.am @@ -94,6 +94,7 @@ test.vs5.das test.vs5.dds test.vs5.dods \ text.nc.das text.nc.dds text.nc.dods \ whoi.das whoi.dds whoi.dods \ kwcase.nc.das kwcase.nc.dds kwcase.nc.dods \ +fillmismatch.nc.das fillmismatch.nc.dds fillmismatch.nc.dods \ CMakeLists.txt # following are not legally convertible to dap2 diff --git a/ncdap_test/testdata3/fillmismatch.nc.das b/ncdap_test/testdata3/fillmismatch.nc.das new file mode 100644 index 000000000..245ea28a6 --- /dev/null +++ b/ncdap_test/testdata3/fillmismatch.nc.das @@ -0,0 +1,16 @@ +Attributes { + Facility { + String PrincipleInvestigator "Mark Abbott", "Ph.D"; + String DataCenter "COAS Environmental Computer Facility"; + String DrifterType "MetOcean WOCE/OCM"; + } + b { + String Description "A test byte"; + String units "unknown"; + } + i32 { + String Description "A 32 bit test server int"; + String units "unknown"; + Float32 _FillValue 100.05; + } +} diff --git a/ncdap_test/testdata3/fillmismatch.nc.dds b/ncdap_test/testdata3/fillmismatch.nc.dds new file mode 100644 index 000000000..e951f319d --- /dev/null +++ b/ncdap_test/testdata3/fillmismatch.nc.dds @@ -0,0 +1,11 @@ +Dataset { + Byte b; + Int32 i32; + UInt32 ui32; + Int16 i16; + UInt16 ui16; + Float32 f32; + Float64 f64; + String s; + Url u; +} SimpleTypes; diff --git a/ncdap_test/testdata3/fillmismatch.nc.dods b/ncdap_test/testdata3/fillmismatch.nc.dods new file mode 100644 index 0000000000000000000000000000000000000000..caaa7c2403cc6e7aed2ae20c12e08024c5be4d93 GIT binary patch literal 264 zcmYLDK@NgI48$vmCqK}G2fctG*#lgNAts)|2drU%O%OJ;gb?E!{IV+x*fiz4l^G$HscMmpO6ZQA=m*2CZJF52buIBHUhqDvB!W&3xn2SiNPy ./tmp_tst_mismatch 2>&1 ; then +echo "*** Fail: ${NCDUMP} ${URL} passed" +exit 1 +else +echo "*** XFail: ${NCDUMP} ${URL} failed" +fi + +# Now check that with [fillmismatch], we get sucess +URL="[fillmismatch]${URL}" +rm -f ./tmp_tst_mismatch +if ${NCDUMP} "${URL}" > ./tmp_tst_mismatch ; then +echo "*** Pass: ${NCDUMP} ${URL} passed" +else +echo "*** Fail: ${NCDUMP} ${URL} failed" +exit 1 +fi + +# Verify result +diff -w ${EXPECTED}/$F.dmp ./tmp_tst_mismatch +#cleanup +rm -f ./tmp_tst_mismatch +exit diff --git a/ncdap_test/tst_ncdap.sh b/ncdap_test/tst_ncdap.sh index 148fd7e88..3bc17302b 100755 --- a/ncdap_test/tst_ncdap.sh +++ b/ncdap_test/tst_ncdap.sh @@ -35,7 +35,7 @@ file*) dds*) TESTURL="[noprefetch]$FILEURL" TESTSET="$DDSTESTS" - FLAGS="-h" + FLAGS="$FLAGS -h" ;; remote*) TESTURL="$REMOTEURL" diff --git a/ncdap_test/tst_ncdap3.sh b/ncdap_test/tst_ncdap3.sh index 5a6f8a29a..0075f3d40 100755 --- a/ncdap_test/tst_ncdap3.sh +++ b/ncdap_test/tst_ncdap3.sh @@ -22,7 +22,7 @@ for x in ${FILETESTS} ; do if IGNORE=`echo -n " ${XFAILTESTS} " | fgrep " ${x} "`; then isxfail=1; fi fi ok=1 - if ${NCDUMP} ${FLAGS} "${url}" | sed 's/\\r//g' > ${x}.dmp ; then ok=$ok; else ok=0; fi + if ${NCDUMP} ${DUMPFLAGS} "${url}" | sed 's/\\r//g' > ${x}.dmp ; then ok=$ok; else ok=0; fi # compare with expected if diff -w ${EXPECTED}/${x}.dmp ${x}.dmp ; then ok=$ok; else ok=0; fi processstatus diff --git a/ncdap_test/tst_utils.sh b/ncdap_test/tst_utils.sh index 8a7ca844b..20417adae 100644 --- a/ncdap_test/tst_utils.sh +++ b/ncdap_test/tst_utils.sh @@ -11,6 +11,8 @@ PARAMS="[log]" OCLOGFILE=/dev/null +DUMPFLAGS= + # Locate directories testdata3="${srcdir}/testdata3" expected3="${srcdir}/expected3" diff --git a/ncdump/nccopy.1 b/ncdump/nccopy.1 index e80dbbd5c..ab2a35ca2 100644 --- a/ncdump/nccopy.1 +++ b/ncdump/nccopy.1 @@ -370,24 +370,23 @@ The following rules are applied in the given order independently for each variable to be copied from input to output. The rules are written assuming we are trying to determine the chunking for a given output variable Vout that comes from an input variable Vin. -.NP +.IP "1." For each dimension of Vout explicitly specified on the command line using the '-c' option, apply the chunking value for that dimension. regardless of input format or input properties. - -.NP +.IP "2." For dimensions of V not named on the command line, preserve chunk sizes from the corresponding input variable. -.NP +.IP "3." If V is netcdf-4 and contiguous, and none of its dimensions are named on the command line, and chunking is not mandated by other options, then make V be contiguous. -.NP +.IP "4." If the input variable is contiguous (or is some netcdf-3 variant) and there are no options requiring chunking, or the '/' special case for the '-c' option is specified, then the output variable V is marked as contiguous. -.NP +.IP "5." Handle all remaining cases when some or all chunk sizes are not determined by the command line or the input variable. This includes the non-chunked input cases such as netcdf-3, cdf5, and DAP. In these cases: Retain all chunk sizes determined by (1) and (2); and Compute the remaining chunk sizes automatically, with some reasonable diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c index dd4fde338..d64bf4373 100644 --- a/ncdump/ocprint.c +++ b/ncdump/ocprint.c @@ -172,12 +172,15 @@ static char* optionmsg = static OCflags ocflags; +EXTERNL int nc_initialize(void); + static void init() { memset(&ocopt,0,sizeof(ocopt)); ocopt.generate = 1; /* -G|-g */ ocopt.userparams = ncbytesnew(); /* -U */ + nc_initialize(); } int diff --git a/oc2/daplex.c b/oc2/daplex.c index 4a78147a4..4c543e524 100644 --- a/oc2/daplex.c +++ b/oc2/daplex.c @@ -43,10 +43,10 @@ static char* ddsworddelims = /* Note: for some reason I added # and removed !~'" what was I thinking? */ -static char* ddswordchars1 = +static const char* ddswordchars1 = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" "-+_/%\\.*!~'\""; -static char* ddswordcharsn = +static const char* ddswordcharsn = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" "-+_/%\\.*!~'\""; @@ -56,10 +56,10 @@ static char* daswordcharsn = "-+_/%\\.*#:!~'\""; /* Need to remove '.' to allow for fqns */ -static char* cewordchars1 = +static const char* cewordchars1 = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" "-+_/%\\*!~'\""; -static char* cewordcharsn = +static const char* cewordcharsn = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" "-+_/%\\*!~'\""; @@ -70,7 +70,7 @@ static char* wordcharsn = NULL; static char* worddelims = NULL; */ -static char* keywords[] = { +static const char* keywords[] = { "alias", "array", "attributes", @@ -96,7 +96,7 @@ static char* keywords[] = { NULL /* mark end of the keywords list */ }; -static int keytokens[] = { +static const int keytokens[] = { SCAN_ALIAS, SCAN_ARRAY, SCAN_ATTR, @@ -371,7 +371,7 @@ daplexcleanup(DAPlexstate** lexstatep) (alphanum+"_!~*'-\"") then it is decoded, otherwise not. */ #ifdef DECODE_PARTIAL -static char* decodeset = /* Specify which characters are decoded */ +static const char* decodeset = /* Specify which characters are decoded */ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_!~*'-\""; #endif diff --git a/oc2/dapparselex.h b/oc2/dapparselex.h index f2de18f01..0e763f1dc 100644 --- a/oc2/dapparselex.h +++ b/oc2/dapparselex.h @@ -25,9 +25,9 @@ typedef struct DAPlexstate { /*! Specifies the Lasttoken. */ int lasttoken; char lasttokentext[MAX_TOKEN_LENGTH+1]; - char* wordchars1; - char* wordcharsn; - char* worddelims; + const char* wordchars1; + const char* wordcharsn; + const char* worddelims; NClist* reclaim; /* reclaim WORD_WORD instances */ } DAPlexstate;