netcdf-c/libdap4/d4data.c

475 lines
13 KiB
C
Raw Normal View History

Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
/*********************************************************************
2018-12-07 05:24:28 +08:00
* Copyright 2018, UCAR/Unidata
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#include "d4includes.h"
#include <stdarg.h>
#include <assert.h>
#include "d4includes.h"
#include "d4odom.h"
#include "nccrc.h"
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
/**
This code serves two purposes
1. Preprocess the dap4 serialization wrt endianness, etc.
(NCD4_processdata)
2. Walk a specified variable instance to convert to netcdf4
memory representation.
(NCD4_fillinstance)
*/
#undef DUMPCHECKSUM
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
/***************************************************/
/* Forwards */
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
static int fillstring(NCD4meta*, NCD4offset* offset, void** dstp, NClist* blobs);
static int fillopfixed(NCD4meta*, d4size_t opaquesize, NCD4offset* offset, void** dstp);
static int fillopvar(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs);
static int fillstruct(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs);
static int fillseq(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs);
Cleanup DAP4 testing NOTE: This PR should not be included in 4.9.1 since additional DAP4 related PRs will be forthcoming. This PR makes major changes to libdap4 and dap4_test driven by changes to TDS. * Enable DAP4 * Clean up the test input files and the test baseline comparison files. This entails: * Remove a multitude of unused test input and baseline data files; among them are dap4_test/: daptestfiles, dmrtestfiles, nctestfiles, and misctestfiles. * Define a canonical set of test input files and record in dap4_test/cdltestfiles. * Use the cdltestfiles to generate the .nc test inputs. This set of .nc files is then moved to the d4ts (DAP4 test server) war file in the tds repository. This set then becomes the canonical set of DAP4 test sources. * Scrape d4ts to obtain copies of the raw streams of DAP4 encoded data. The .dmr and .dap streams are then stored in dap4_test/rawtestfiles. * Disable some remote server tests until those servers are fixed. * Add an option to ncdump (-XF) that forces the type of the _FillValue attribute; this is primarily to simplify testing of fill mismatch. * Minor bug fixes to ncgen. * Changes to libdap4: * Replace old checksum hack with the dap4.checksum flag. * Support the dap4.XXX controls. * Cleanup _FillValue handling, especially var-attribute type mismatches. * Fix enum handling based on changes to netcdf-java. * Changes to dap4_test: * Add getopt support to various test support programs. * Remove unneeded shell scripts. * Add new scripts: test_curlopt.sh
2022-11-14 04:15:11 +08:00
static int NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel);
static unsigned NCD4_computeChecksum(NCD4meta* meta, NCD4node* topvar);
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
/***************************************************/
/* Macro define procedures */
#ifdef D4DUMPCSUM
static unsigned int debugcrc32(unsigned int crc, const void *buf, size_t size)
{
int i;
fprintf(stderr,"crc32: ");
for(i=0;i<size;i++) {fprintf(stderr,"%02x",((unsigned char*)buf)[i]);}
fprintf(stderr,"\n");
return NC_crc32(crc,buf,size);
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
}
#define CRC32 debugcrc32
#else
#define CRC32 NC_crc32
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
#endif
#define ISTOPLEVEL(var) ((var)->container == NULL || (var)->container->sort == NCD4_GROUP)
/***************************************************/
/* API */
int
NCD4_processdata(NCD4meta* meta)
{
int ret = NC_NOERR;
int i;
NClist* toplevel = NULL;
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
NCD4node* root = meta->root;
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
NCD4offset* offset = NULL;
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
/* Recursively walk the tree in prefix order
to get the top-level variables; also mark as unvisited */
toplevel = nclistnew();
NCD4_getToplevelVars(meta,root,toplevel);
Cleanup DAP4 testing NOTE: This PR should not be included in 4.9.1 since additional DAP4 related PRs will be forthcoming. This PR makes major changes to libdap4 and dap4_test driven by changes to TDS. * Enable DAP4 * Clean up the test input files and the test baseline comparison files. This entails: * Remove a multitude of unused test input and baseline data files; among them are dap4_test/: daptestfiles, dmrtestfiles, nctestfiles, and misctestfiles. * Define a canonical set of test input files and record in dap4_test/cdltestfiles. * Use the cdltestfiles to generate the .nc test inputs. This set of .nc files is then moved to the d4ts (DAP4 test server) war file in the tds repository. This set then becomes the canonical set of DAP4 test sources. * Scrape d4ts to obtain copies of the raw streams of DAP4 encoded data. The .dmr and .dap streams are then stored in dap4_test/rawtestfiles. * Disable some remote server tests until those servers are fixed. * Add an option to ncdump (-XF) that forces the type of the _FillValue attribute; this is primarily to simplify testing of fill mismatch. * Minor bug fixes to ncgen. * Changes to libdap4: * Replace old checksum hack with the dap4.checksum flag. * Support the dap4.XXX controls. * Cleanup _FillValue handling, especially var-attribute type mismatches. * Fix enum handling based on changes to netcdf-java. * Changes to dap4_test: * Add getopt support to various test support programs. * Remove unneeded shell scripts. * Add new scripts: test_curlopt.sh
2022-11-14 04:15:11 +08:00
/* Otherwise */
NCD4_inferChecksums(meta,toplevel);
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
/* If necessary, byte swap the serialized data */
/* Do we need to swap the dap4 data? */
meta->swap = (meta->serial.hostlittleendian != meta->serial.remotelittleendian);
/* Compute the offset and size of the toplevel vars in the raw dap data. */
/* Also extract remote checksums */
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
offset = BUILDOFFSET(meta->serial.dap,meta->serial.dapsize);
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
for(i=0;i<nclistlength(toplevel);i++) {
NCD4node* var = (NCD4node*)nclistget(toplevel,i);
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
if((ret=NCD4_delimit(meta,var,offset)))
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
FAIL(ret,"delimit failure");
if(meta->controller->data.inferredchecksumming) {
/* Compute remote checksum: must occur before any byte swapping */
var->data.localchecksum = NCD4_computeChecksum(meta,var);
#ifdef DUMPCHECKSUM
fprintf(stderr,"var %s: remote-checksum = 0x%x\n",var->name,var->data.remotechecksum);
#endif
/* verify checksums */
if(!meta->controller->data.checksumignore) {
if(var->data.localchecksum != var->data.remotechecksum) {
nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name);
ret = NC_EDAP;
goto done;
}
/* Also verify checksum attribute */
if(meta->controller->data.attrchecksumming) {
if(var->data.attrchecksum != var->data.remotechecksum) {
nclog(NCLOGERR,"Attribute Checksum mismatch: %s\n",var->name);
ret = NC_EDAP;
goto done;
}
}
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
}
}
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
}
/* Swap the data for each top level variable,
*/
if(meta->swap) {
if((ret=NCD4_swapdata(meta,toplevel)))
FAIL(ret,"byte swapping failed");
}
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
done:
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
if(offset) free(offset);
if(toplevel) nclistfree(toplevel);
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
return THROW(ret);
}
/*
Build a single instance of a type. The blobs
argument accumulates any malloc'd data so we can
reclaim it in case of an error.
Activity is to walk the variable's data to
produce a copy that is compatible with the
netcdf4 memory format.
Assumes that NCD4_processdata has been called.
*/
int
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs)
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
{
int ret = NC_NOERR;
void* dst = *dstp;
d4size_t memsize = type->meta.memsize;
d4size_t dapsize = type->meta.dapsize;
/* If the type is fixed size, then just copy it */
if(type->subsort <= NC_UINT64 || type->subsort == NC_ENUM) {
/* memsize and dapsize are the same */
assert(memsize == dapsize);
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
TRANSFER(dst,offset,dapsize);
INCR(offset,dapsize);
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
} else switch(type->subsort) {
case NC_STRING: /* oob strings */
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
if((ret=fillstring(meta,offset,&dst,blobs)))
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
FAIL(ret,"fillinstance");
break;
case NC_OPAQUE:
if(type->opaque.size > 0) {
/* We know the size and its the same for all instances */
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
if((ret=fillopfixed(meta,type->opaque.size,offset,&dst)))
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
FAIL(ret,"fillinstance");
} else {
/* Size differs per instance, so we need to convert each opaque to a vlen */
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
if((ret=fillopvar(meta,type,offset,&dst,blobs)))
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
FAIL(ret,"fillinstance");
}
break;
case NC_STRUCT:
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
if((ret=fillstruct(meta,type,offset,&dst,blobs)))
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
FAIL(ret,"fillinstance");
break;
case NC_SEQ:
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
if((ret=fillseq(meta,type,offset,&dst,blobs)))
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
FAIL(ret,"fillinstance");
break;
default:
ret = NC_EINVAL;
FAIL(ret,"fillinstance");
}
*dstp = dst;
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
done:
return THROW(ret);
}
static int
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
fillstruct(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs)
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
{
int i,ret = NC_NOERR;
void* dst = *dstp;
#ifdef CLEARSTRUCT
/* Avoid random data within aligned structs */
memset(dst,0,type->meta.memsize);
#endif
/* Walk and read each field taking alignments into account */
for(i=0;i<nclistlength(type->vars);i++) {
NCD4node* field = nclistget(type->vars,i);
NCD4node* ftype = field->basetype;
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
void* fdst = (((char*)dst) + field->meta.offset);
if((ret=NCD4_fillinstance(meta,ftype,offset,&fdst,blobs)))
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
FAIL(ret,"fillstruct");
}
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
dst = ((char*)dst) + type->meta.memsize;
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
*dstp = dst;
done:
return THROW(ret);
}
static int
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
fillseq(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs)
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
{
int ret = NC_NOERR;
d4size_t i,recordcount;
nc_vlen_t* dst;
NCD4node* vlentype;
d4size_t recordsize;
dst = (nc_vlen_t*)*dstp;
vlentype = type->basetype;
recordsize = vlentype->meta.memsize;
/* Get record count (remember, it is already properly swapped) */
recordcount = GETCOUNTER(offset);
SKIPCOUNTER(offset);
dst->len = (size_t)recordcount;
/* compute the required memory */
dst->p = d4alloc(recordsize*recordcount);
if(dst->p == NULL)
FAIL(NC_ENOMEM,"fillseq");
for(i=0;i<recordcount;i++) {
/* Read each record instance */
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
void* recdst = ((char*)(dst->p))+(recordsize * i);
if((ret=NCD4_fillinstance(meta,vlentype,offset,&recdst,blobs)))
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
FAIL(ret,"fillseq");
}
dst++;
*dstp = dst;
done:
return THROW(ret);
}
/*
Extract and oob a single string instance
*/
static int
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
fillstring(NCD4meta* meta, NCD4offset* offset, void** dstp, NClist* blobs)
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
{
int ret = NC_NOERR;
d4size_t count;
char** dst = *dstp;
char* q;
/* Get string count (remember, it is already properly swapped) */
count = GETCOUNTER(offset);
SKIPCOUNTER(offset);
/* Transfer out of band */
q = (char*)d4alloc(count+1);
if(q == NULL)
{FAIL(NC_ENOMEM,"out of space");}
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
TRANSFER(q,offset,count);
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
q[count] = '\0';
/* Write the pointer to the string */
*dst = q;
dst++;
*dstp = dst;
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
INCR(offset,count);
Fix more memory leaks in netcdf-c library This is a follow up to PR https://github.com/Unidata/netcdf-c/pull/1173 Sorry that it is so big, but leak suppression can be complex. This PR fixes all remaining memory leaks -- as determined by -fsanitize=address, and with the exceptions noted below. Unfortunately. there remains a significant leak that I cannot solve. It involves vlens, and it is unclear if the leak is occurring in the netcdf-c library or the HDF5 library. I have added a check_PROGRAM to the ncdump directory to show the problem. The program is called tst_vlen_demo.c To exercise it, build the netcdf library with -fsanitize=address enabled. Then go into ncdump and do a "make clean check". This should build tst_vlen_demo without actually executing it. Then do the command "./tst_vlen_demo" to see the output of the memory checker. Note the the lost malloc is deep in the HDF5 library (in H5Tvlen.c). I am temporarily working around this error in the following way. 1. I modified several test scripts to not execute known vlen tests that fail as described above. 2. Added an environment variable called NC_VLEN_NOTEST. If set, then those specific tests are suppressed. This should mean that the --disable-utilities option to ./configure should not need to be set to get a memory leak clean build. This should allow for detection of any new leaks. Note: I used an environment variable rather than a ./configure option to control the vlen tests. This is because it is temporary (I hope) and because it is a bit tricky for shell scripts to access ./configure options. Finally, as before, this only been tested with netcdf-4 and hdf5 support.
2018-11-16 01:00:38 +08:00
#if 0
nclistpush(blobs,q);
#else
q = NULL;
#endif
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
done:
return THROW(ret);
}
static int
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
fillopfixed(NCD4meta* meta, d4size_t opaquesize, NCD4offset* offset, void** dstp)
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
{
int ret = NC_NOERR;
d4size_t count, actual;
int delta;
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
void* dst = *dstp;
/* Get opaque count */
count = GETCOUNTER(offset);
SKIPCOUNTER(offset);
/* verify that it is the correct size */
actual = count;
delta = actual - opaquesize;
if(delta != 0) {
#ifdef FIXEDOPAQUE
nclog(NCLOGWARN,"opaque changed from %lu to %lu",actual,opaquesize);
memset(dst,0,opaquesize); /* clear in case we have short case */
count = (delta < 0 ? actual : opaquesize);
#else
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
FAIL(NC_EVARSIZE,"Expected opaque size to be %lld; found %lld",opaquesize,count);
#endif
}
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
/* move */
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
TRANSFER(dst,offset,count);
dst = ((char*)dst) + count;
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
*dstp = dst;
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
INCR(offset,count);
#ifndef FIXEDOPAQUE
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
done:
#endif
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
return THROW(ret);
}
/*
Move a dap4 variable length opaque out of band.
We treat as if it was (in cdl) ubyte(*).
*/
static int
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
fillopvar(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs)
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
{
int ret = NC_NOERR;
d4size_t count;
nc_vlen_t* vlen;
void* dst = *dstp;
char* q;
/* alias dst format */
vlen = (nc_vlen_t*)dst;
/* Get opaque count */
count = GETCOUNTER(offset);
SKIPCOUNTER(offset);
/* Transfer out of band */
q = (char*)d4alloc(count);
if(q == NULL) FAIL(NC_ENOMEM,"out of space");
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
TRANSFER(q,offset,count);
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
vlen->p = q;
vlen->len = (size_t)count;
q = NULL; /*nclistpush(blobs,q);*/
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
dst = ((char*)dst) + sizeof(nc_vlen_t);
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
*dstp = dst;
Fix DAP4 remotetest server Warning: This PR is a follow on to PR https://github.com/Unidata/netcdf-c/pull/2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR https://github.com/Unidata/netcdf-c/pull/2555. This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts* to test several features: 1. Show that access over the Internet to servers using the DAP4 protocol works. 2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly. 4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable. The test inputs (primarily *\*.nc* files) provided in the netcdf-c library are also used by the DAP4 Test Server (aka d4ts) to present web access to a collection of data files accessible via the DAP4 protocol and which can be used for testing Internet access to a working server. To be precise, this version of d4ts is currently in unmerged branches of the *netcdf-java* and *tds* Github repositories and so are not actually in the main repositories *yet*. However, the *d4ts.war* file was created from that branch and used to populate the *remotetest.unidata.ucar.edu* server The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org) and *thredds-test*. These will continue to remain disabled until those servers can be fixed. ## Primary Changes * Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers. * Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process. * Fix the *dap4_test/test_remote.sh* test script to match the current available test data. * Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4]. ## Misc. Unrelated Changes * Remove a raft of warnings, especially in nc_test4/tst_quantize.c. * Add some additional explanatory information to the NCZarr documentation. * Cleanup some Doxygen errors in the docs file and reorder some files.
2022-11-16 11:29:21 +08:00
INCR(offset,count);
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
done:
return THROW(ret);
}
/**************************************************/
/* Utilities */
Primary change: add dap4 support Specific changes: 1. Add dap4 code: libdap4 and dap4_test. Note that until the d4ts server problem is solved, dap4 is turned off. 2. Modify various files to support dap4 flags: configure.ac, Makefile.am, CMakeLists.txt, etc. 3. Add nc_test/test_common.sh. This centralizes the handling of the locations of various things in the build tree: e.g. where is ncgen.exe located. See nc_test/test_common.sh for details. 4. Modify .sh files to use test_common.sh 5. Obsolete separate oc2 by moving it to be part of netcdf-c. This means replacing code with netcdf-c equivalents. 5. Add --with-testserver to configure.ac to allow override of the servers to be used for --enable-dap-remote-tests. 6. There were multiple versions of nctypealignment code. Try to centralize in libdispatch/doffset.c and include/ncoffsets.h 7. Add a unit test for the ncuri code because of its complexity. 8. Move the findserver code out of libdispatch and into a separate, self contained program in ncdap_test and dap4_test. 9. Move the dispatch header files (nc{3,4}dispatch.h) to .../include because they are now shared by modules. 10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts. 11. Make use of MREMAP if available 12. Misc. minor changes e.g. - #include <config.h> -> #include "config.h" - Add some no-install headers to /include - extern -> EXTERNL and vice versa as needed - misc header cleanup - clean up checking for misc. unix vs microsoft functions 13. Change copyright decls in some files to point to LICENSE file. 14. Add notes to RELEASENOTES.md
2017-03-09 08:01:10 +08:00
int
NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel)
{
int ret = NC_NOERR;
int i;
if(group == NULL)
group = meta->root;
/* Collect vars in this group */
for(i=0;i<nclistlength(group->vars);i++) {
NCD4node* node = (NCD4node*)nclistget(group->vars,i);
nclistpush(toplevel,node);
node->visited = 0; /* We will set later to indicate written vars */
#ifdef D4DEBUGDATA
fprintf(stderr,"toplevel: var=%s\n",node->name);
#endif
}
/* Now, recurse into subgroups; will produce prefix order */
for(i=0;i<nclistlength(group->groups);i++) {
NCD4node* g = (NCD4node*)nclistget(group->groups,i);
if((ret=NCD4_getToplevelVars(meta,g,toplevel))) goto done;
}
done:
return THROW(ret);
}
static int
Cleanup DAP4 testing NOTE: This PR should not be included in 4.9.1 since additional DAP4 related PRs will be forthcoming. This PR makes major changes to libdap4 and dap4_test driven by changes to TDS. * Enable DAP4 * Clean up the test input files and the test baseline comparison files. This entails: * Remove a multitude of unused test input and baseline data files; among them are dap4_test/: daptestfiles, dmrtestfiles, nctestfiles, and misctestfiles. * Define a canonical set of test input files and record in dap4_test/cdltestfiles. * Use the cdltestfiles to generate the .nc test inputs. This set of .nc files is then moved to the d4ts (DAP4 test server) war file in the tds repository. This set then becomes the canonical set of DAP4 test sources. * Scrape d4ts to obtain copies of the raw streams of DAP4 encoded data. The .dmr and .dap streams are then stored in dap4_test/rawtestfiles. * Disable some remote server tests until those servers are fixed. * Add an option to ncdump (-XF) that forces the type of the _FillValue attribute; this is primarily to simplify testing of fill mismatch. * Minor bug fixes to ncgen. * Changes to libdap4: * Replace old checksum hack with the dap4.checksum flag. * Support the dap4.XXX controls. * Cleanup _FillValue handling, especially var-attribute type mismatches. * Fix enum handling based on changes to netcdf-java. * Changes to dap4_test: * Add getopt support to various test support programs. * Remove unneeded shell scripts. * Add new scripts: test_curlopt.sh
2022-11-14 04:15:11 +08:00
NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel)
{
int ret = NC_NOERR;
int i, attrfound;
Cleanup DAP4 testing NOTE: This PR should not be included in 4.9.1 since additional DAP4 related PRs will be forthcoming. This PR makes major changes to libdap4 and dap4_test driven by changes to TDS. * Enable DAP4 * Clean up the test input files and the test baseline comparison files. This entails: * Remove a multitude of unused test input and baseline data files; among them are dap4_test/: daptestfiles, dmrtestfiles, nctestfiles, and misctestfiles. * Define a canonical set of test input files and record in dap4_test/cdltestfiles. * Use the cdltestfiles to generate the .nc test inputs. This set of .nc files is then moved to the d4ts (DAP4 test server) war file in the tds repository. This set then becomes the canonical set of DAP4 test sources. * Scrape d4ts to obtain copies of the raw streams of DAP4 encoded data. The .dmr and .dap streams are then stored in dap4_test/rawtestfiles. * Disable some remote server tests until those servers are fixed. * Add an option to ncdump (-XF) that forces the type of the _FillValue attribute; this is primarily to simplify testing of fill mismatch. * Minor bug fixes to ncgen. * Changes to libdap4: * Replace old checksum hack with the dap4.checksum flag. * Support the dap4.XXX controls. * Cleanup _FillValue handling, especially var-attribute type mismatches. * Fix enum handling based on changes to netcdf-java. * Changes to dap4_test: * Add getopt support to various test support programs. * Remove unneeded shell scripts. * Add new scripts: test_curlopt.sh
2022-11-14 04:15:11 +08:00
NCD4INFO* info = meta->controller;
/* First, look thru the DMR to see if there is a checksum attribute */
attrfound = 0;
for(i=0;i<nclistlength(toplevel);i++) {
int a;
NCD4node* node = (NCD4node*)nclistget(toplevel,i);
for(a=0;a<nclistlength(node->attributes);a++) {
NCD4node* attr = (NCD4node*)nclistget(node->attributes,a);
if(strcmp(D4CHECKSUMATTR,attr->name)==0) {
const char* val = NULL;
if(nclistlength(attr->attr.values) != 1)
return NC_EDMR;
val = (const char*)nclistget(attr->attr.values,0);
sscanf(val,"%u",&node->data.attrchecksum);
node->data.checksumattr = 1;
attrfound = 1;
Cleanup DAP4 testing NOTE: This PR should not be included in 4.9.1 since additional DAP4 related PRs will be forthcoming. This PR makes major changes to libdap4 and dap4_test driven by changes to TDS. * Enable DAP4 * Clean up the test input files and the test baseline comparison files. This entails: * Remove a multitude of unused test input and baseline data files; among them are dap4_test/: daptestfiles, dmrtestfiles, nctestfiles, and misctestfiles. * Define a canonical set of test input files and record in dap4_test/cdltestfiles. * Use the cdltestfiles to generate the .nc test inputs. This set of .nc files is then moved to the d4ts (DAP4 test server) war file in the tds repository. This set then becomes the canonical set of DAP4 test sources. * Scrape d4ts to obtain copies of the raw streams of DAP4 encoded data. The .dmr and .dap streams are then stored in dap4_test/rawtestfiles. * Disable some remote server tests until those servers are fixed. * Add an option to ncdump (-XF) that forces the type of the _FillValue attribute; this is primarily to simplify testing of fill mismatch. * Minor bug fixes to ncgen. * Changes to libdap4: * Replace old checksum hack with the dap4.checksum flag. * Support the dap4.XXX controls. * Cleanup _FillValue handling, especially var-attribute type mismatches. * Fix enum handling based on changes to netcdf-java. * Changes to dap4_test: * Add getopt support to various test support programs. * Remove unneeded shell scripts. * Add new scripts: test_curlopt.sh
2022-11-14 04:15:11 +08:00
break;
}
}
}
info->data.attrchecksumming = (attrfound ? 1 : 0);
Cleanup DAP4 testing NOTE: This PR should not be included in 4.9.1 since additional DAP4 related PRs will be forthcoming. This PR makes major changes to libdap4 and dap4_test driven by changes to TDS. * Enable DAP4 * Clean up the test input files and the test baseline comparison files. This entails: * Remove a multitude of unused test input and baseline data files; among them are dap4_test/: daptestfiles, dmrtestfiles, nctestfiles, and misctestfiles. * Define a canonical set of test input files and record in dap4_test/cdltestfiles. * Use the cdltestfiles to generate the .nc test inputs. This set of .nc files is then moved to the d4ts (DAP4 test server) war file in the tds repository. This set then becomes the canonical set of DAP4 test sources. * Scrape d4ts to obtain copies of the raw streams of DAP4 encoded data. The .dmr and .dap streams are then stored in dap4_test/rawtestfiles. * Disable some remote server tests until those servers are fixed. * Add an option to ncdump (-XF) that forces the type of the _FillValue attribute; this is primarily to simplify testing of fill mismatch. * Minor bug fixes to ncgen. * Changes to libdap4: * Replace old checksum hack with the dap4.checksum flag. * Support the dap4.XXX controls. * Cleanup _FillValue handling, especially var-attribute type mismatches. * Fix enum handling based on changes to netcdf-java. * Changes to dap4_test: * Add getopt support to various test support programs. * Remove unneeded shell scripts. * Add new scripts: test_curlopt.sh
2022-11-14 04:15:11 +08:00
/* Infer checksums */
info->data.inferredchecksumming = ((info->data.attrchecksumming || info->data.querychecksumming) ? 1 : 0);
return THROW(ret);
}
/* Compute the checksum of each variable's data.
*/
static unsigned
NCD4_computeChecksum(NCD4meta* meta, NCD4node* topvar)
{
unsigned int csum = 0;
ASSERT((ISTOPLEVEL(topvar)));
#ifndef HYRAXCHECKSUM
csum = CRC32(csum,topvar->data.dap4data.memory,topvar->data.dap4data.size);
#else
if(topvar->basetype->subsort != NC_STRING) {
csum = CRC32(csum,topvar->data.dap4data.memory,topvar->data.dap4data.size);
} else { /* Checksum over the actual strings */
int i;
unsigned long long count;
NCD4offset offset;
d4size_t dimproduct = NCD4_dimproduct(topvar);
offset.base = topvar->data.dap4data.memory;
offset.limit = offset.base + topvar->data.dap4data.size;
offset.offset = offset.base;
for(i=0;i<dimproduct;i++) {
/* Get string count */
count = GETCOUNTER(&offset);
SKIPCOUNTER(&offset);
if(meta->swap) swapinline64(&count);
/* CRC32 count bytes */
csum = CRC32(csum,offset.offset,count);
/* Skip count bytes */
INCR(&offset,count);
}
}
#endif
return csum;
}
#if 0
/* As a hack, if the server sent remotechecksums and
_DAP4_Checksum_CRC32 is not defined, then define it.
*/
static int
NCD4_addchecksumattr(NCD4meta* meta, NClist* toplevel)
{
int ret = NC_NOERR;
int i;
for(i=0;i<nclistlength(toplevel);i++) {
NCD4node* node = (NCD4node*)nclistget(toplevel,i);
/* Is remote checksum available? */
if(node->data.remotechecksummed) {
NCD4node* attr = NCD4_findAttr(node,D4CHECKSUMATTR);
if(attr == NULL) { /* add it */
char value[64];
snprintf(value,sizeof(value),"%u",node->data.remotechecksum);
if((ret=NCD4_defineattr(meta,node,D4CHECKSUMATTR,"UInt32",&attr)))
return NC_EINTERNAL;
attr->attr.values = nclistnew();
nclistpush(attr->attr.values,strdup(value));
}
}
}
return THROW(ret);
}
#endif