mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-31 17:50:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Provide byte-range reading of remote datasets

Browse Source 
re: issue https://github.com/Unidata/netcdf-c/issues/1251

Assume that you have the URL to a remote dataset
which is a normal netcdf-3 or netcdf-4 file.

This PR allows the netcdf-c to read that dataset's
contents as a netcdf file using HTTP byte ranges
if the remote server supports byte-range access.

Originally, this PR was set up to access Amazon S3 objects,
but it can also access other remote datasets such as those
provided by a Thredds server via the HTTPServer access protocol.
It may also work for other kinds of servers.

Note that this is not intended as a true production
capability because, as is known, this kind of access to
can be quite slow. In addition, the byte-range IO drivers
do not currently do any sort of optimization or caching.

An additional goal here is to gain some experience with
the Amazon S3 REST protocol.

This architecture and its use documented in
the file docs/byterange.dox.

There are currently two test cases:

1. nc_test/tst_s3raw.c - this does a simple open, check format, close cycle
   for a remote netcdf-3 file and a remote netcdf-4 file.
2. nc_test/test_s3raw.sh - this uses ncdump to investigate some remote
   datasets.

This PR also incorporates significantly changed model inference code
(see the superceded PR https://github.com/Unidata/netcdf-c/pull/1259).

1. It centralizes the code that infers the dispatcher.
2. It adds support for byte-range URLs

Other changes:

1. NC_HDF5_finalize was not being properly called by nc_finalize().
2. Fix minor bug in ncgen3.l
3. fix memory leak in nc4info.c
4. add code to walk the .daprc triples and to replace protocol=
   fragment tag with a more general mode= tag.

Final Note:
Th inference code is still way too complicated. We need to move
to the validfile() model used by netcdf Java, where each
dispatcher is asked if it can process the file. This decentralizes
the inference code. This will be done after all the major new
dispatchers (PIO, Zarr, etc) have been implemented.
This commit is contained in:
Dennis Heimbigner 2019-01-01 18:27:36 -07:00
parent b16eceabe2
commit bf2746b8ea
66 changed files with 5439 additions and 2048 deletions

20
CMakeLists.txt

@ -831,6 +831,20 @@ ELSE()
SET(ENABLE_DAP4 OFF)
ENDIF()
# Option to support byte-range reading of remote datasets
OPTION(ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." OFF)
IF(ENABLE_BYTERANGE)
FIND_PACKAGE(CURL)
IF(NOT CURL_LIBRARY)
MESSAGE(FATAL_ERROR "Byte-range support specified, CURL libraries are not found.")
SET(ENABLE_BYTERANGE OFF BOOL)
ELSE()
SET(ENABLE_BYTERANGE ON BOOL)
ENDIF()
ENDIF()
# Use ENABLE_HTTP instead of ENABLE_BYTERANGE
SET(ENABLE_HTTP "${ENABLE_BYTERANGE}" BOOL "")
# Check for the math library so it can be explicitly linked.
IF(NOT WIN32)
@ -1153,11 +1167,11 @@ ENDIF()
OPTION(ENABLE_FILTER_TESTING "Enable filter testing. Ignored if shared libraries or netCDF4 are not enabled" ON)
IF(ENABLE_FILTER_TESTING AND NOT ENABLE_NETCDF_4)
MESSAGE(WARNING "ENABLE_FILTER_TESTING requires netCDF-4. Disabling.")
SET(ENABLE_FILTER_TESTING OFF CACHE BOOL "")
SET(ENABLE_FILTER_TESTING OFF)
ENDIF()
IF(NOT BUILD_SHARED_LIBS)
MESSAGE(WARNING "ENABLE_FILTER_TESTING requires shared libraries. Disabling.")
SET(ENABLE_FILTER_TESTING OFF CACHE BOOL "")
SET(ENABLE_FILTER_TESTING OFF)
ENDIF()
# Determine whether or not to generate documentation.
@ -1586,6 +1600,7 @@ MACRO(print_conf_summary)
MESSAGE(STATUS "Building netCDF-4: ${ENABLE_NETCDF_4}")
MESSAGE(STATUS "Building DAP2 Support: ${ENABLE_DAP2}")
MESSAGE(STATUS "Building DAP4 Support: ${ENABLE_DAP4}")
MESSAGE(STATUS "Building Byte-range Support: ${ENABLE_HTTP}")
MESSAGE(STATUS "Building Utilities: ${BUILD_UTILITIES}")
IF(CMAKE_PREFIX_PATH)
MESSAGE(STATUS "CMake Prefix Path: ${CMAKE_PREFIX_PATH}")
@ -1936,6 +1951,7 @@ is_enabled(ENABLE_PARALLEL4 HAS_PARALLEL4)
is_enabled(ENABLE_DAP HAS_DAP)
is_enabled(ENABLE_DAP HAS_DAP2)
is_enabled(ENABLE_DAP4 HAS_DAP4)
is_enabled(ENABLE_HTTP HAS_HTTP)
is_enabled(ENABLE_DISKLESS HAS_DISKLESS)
is_enabled(USE_MMAP HAS_MMAP)
is_enabled(JNA HAS_JNA)

5
RELEASE_NOTES.md

@ -7,6 +7,11 @@ This file contains a high-level description of this package's evolution. Release
## 4.6.3 - TBD
* [Enhancement] Provide byte-range reading of remote datasets. This allows
read-only access to, for example, Amazon S3 objects and also Thredds Server
datasets via the HTTPService access method.
See [GitHub #???](https://github.com/Unidata/netcdf-c/issues/???).
* Update the license from the home-brewed NetCDF license to the standard 3-Clause BSD License. This change does not result in any new restrictions; it is merely the adoption of a standard, well-known and well-understood license in place of the historic NetCDF license written at Unidata. This is part of a broader push by Unidata to adopt modern, standardized licensing.

3
config.h.cmake.in

@ -141,6 +141,9 @@ are set when opening a binary file on Windows. */
/* define the possible sources for remote test servers */
#cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}"
/* if true, build byte-range Client */
#cmakedefine ENABLE_HTTP 1
/* if true, run extra tests which may not work yet */
#cmakedefine EXTRA_TESTS 1

24
configure.ac

@ -897,6 +897,26 @@ if test "x$enable_mmap" = xyes; then
AC_DEFINE([USE_MMAP], [1], [if true, use mmap for in-memory files])
fi
# Does the user want to allow reading of remote data via range headers?
AC_MSG_CHECKING([whether byte range support is enabled])
AC_ARG_ENABLE([byterange],
[AS_HELP_STRING([--enable-byterange],
[allow byte-range I/O])])
test "x$enable_byterange" = xyes || enable_byterange=no
AC_MSG_RESULT($enable_byterange)
# Need curl for byte ranges
if test "x$found_curl" = xno ; then
AC_MSG_ERROR([curl required for byte range support. Install curl or build without --enable-byterange.])
enable_byterange=no
fi
# Use "http" as synonym for byterange to correspond with NetCDF-Java
enable_http=${enable_byterange}
if test "x$enable_http" = xyes; then
AC_DEFINE([ENABLE_HTTP], [1], [if true, support byte-range read of remote datasets.])
fi
AC_FUNC_ALLOCA
AC_CHECK_DECLS([isnan, isinf, isfinite],,,[#include <math.h>])
AC_STRUCT_ST_BLKSIZE
@ -1300,6 +1320,7 @@ AM_CONDITIONAL(BUILD_MMAP, [test x$enable_mmap = xyes])
AM_CONDITIONAL(BUILD_DOCS, [test x$enable_doxygen = xyes])
AM_CONDITIONAL(SHOW_DOXYGEN_TAG_LIST, [test x$enable_doxygen_tasks = xyes])
AM_CONDITIONAL(ENABLE_METADATA_PERF, [test x$enable_metadata_perf = xyes])
AM_CONDITIONAL(ENABLE_HTTP, [test "x$enable_http" = xyes])
# If the machine doesn't have a long long, and we want netCDF-4, then
# we've got problems!
@ -1433,7 +1454,7 @@ AC_SUBST(HAS_MMAP,[$enable_mmap])
AC_SUBST(HAS_JNA,[$enable_jna])
AC_SUBST(RELAX_COORD_BOUND,[$enable_zero_length_coord_bound])
AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill])
AC_SUBST(HAS_HTTP,[$enable_http])
# Include some specifics for netcdf on windows.
#AH_VERBATIM([_WIN32_STRICMP],
@ -1502,6 +1523,7 @@ AX_SET_META([NC_HAS_PARALLEL4],[$enable_parallel4],[yes])
AX_SET_META([NC_HAS_CDF5],[$enable_cdf5],[yes])
AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes])
AX_SET_META([NC_RELAX_COORD_BOUND], [$enable_zero_length_coord_bound],[yes])
AX_SET_META([NC_HAS_HTTP],[$enable_http],[yes])
# Automake says that this is always run in top_builddir
# and that srcdir is defined (== top_srcdir)

3
docs/CMakeLists.txt

@ -91,6 +91,7 @@ install-fortran.md all-error-codes.md credits.md auth.md
obsolete/fan_utils.html bestpractices.md filters.md indexing.md
inmemory.md DAP2.dox attribute_conventions.md FAQ.md
file_format_specifications.md known_problems.md
COPYRIGHT.dox user_defined_formats.md DAP4.md DAP4.dox)
COPYRIGHT.dox user_defined_formats.md DAP4.md DAP4.dox
testserver.dox byterange.dox)
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

6
docs/DAP4.dox

@ -37,13 +37,13 @@ file name normally used is replaced with a URL with a specific
format. The URL is composed of three parts.
- URL - this is a standard form URL with specific markers to indicate that
it refers to a DAP4 encoded dataset. The markers are of the form
"dap4", "protocol=dap4", or "/thredds/dap4". The following
"dap4", "mode=dap4", or "/thredds/dap4". The following
examples show how they are specified. Note that the "/thredds/dap4"
case will only work when accessing a Thredds-based server.
+ [dap4]http://remotetest.unidata.ucar.edu/d4ts/test.01
+ [protocol=dap4]http://remotetest.unidata.ucar.edu/d4ts/test.01
+ [mode=dap4]http://remotetest.unidata.ucar.edu/d4ts/test.01
+ http://remotetest.unidata.ucar.edu/d4ts/test.01#dap4
+ http://remotetest.unidata.ucar.edu/d4ts/test.01#protocol=dap4
+ http://remotetest.unidata.ucar.edu/d4ts/test.01#mode=dap4
+ http://thredds.ucar.edu/thredds/dap4/...
- Constraints - these are suffixed to the URL and take the form

1
docs/Doxyfile.in

@ -751,6 +751,7 @@ INPUT = \
@abs_top_srcdir@/docs/guide.dox \
@abs_top_srcdir@/docs/attribute_conventions.md \
@abs_top_srcdir@/docs/file_format_specifications.md \
@abs_top_srcdir@/docs/byterange.dox \
@abs_top_srcdir@/docs/DAP2.dox \
@abs_top_srcdir@/docs/DAP4.dox \
@abs_top_srcdir@/docs/user_defined_formats.md \

5
docs/Makefile.am

@ -13,9 +13,10 @@ architecture.dox internal.dox windows-binaries.md \
building-with-cmake.md CMakeLists.txt groups.dox install.md notes.md \
install-fortran.md all-error-codes.md credits.md auth.md \
obsolete/fan_utils.html bestpractices.md filters.md indexing.dox \
inmemory.md DAP2.dox attribute_conventions.md FAQ.md \
inmemory.md DAP2.dox attribute_conventions.md FAQ.md \
file_format_specifications.md known_problems.md COPYRIGHT.md \
user_defined_formats.md inmeminternal.dox DAP4.md DAP4.dox
user_defined_formats.md inmeminternal.dox DAP4.md DAP4.dox \
testserver.dox byterange.dox
# Turn off parallel builds in this directory.
.NOTPARALLEL:

156
docs/byterange.dox Normal file

@ -0,0 +1,156 @@
/**
@if INTERNAL
@page byterange Remote Dataset Access Using HTTP Byte Ranges
\tableofcontents
<!-- Note that this file has the .dox extension, but is mostly markdown -->
<!-- Begin MarkDown -->
# Introduction {#byterange_intro}
Suppose that you have the URL to a remote dataset
which is a normal netcdf-3 or netcdf-4 file.
The netCDF-c library now supports read-only access to such
datasets using the HTTP byte range capability [], assuming that
the remote server supports byte-range access.
Two examples:
1. An Amazon S3 object containing a netcdf classic file.
- location: "http://149.165.169.123:8080/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes"
2. A Thredds Server dataset supporting the Thredds HTTPServer protocol.
and containing a netcdf enhanced file.
- location: "http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#bytes"
Other remote servers may also provide byte-range access in a similar form.
It is important to note that this is not intended as a true
production capability because, as is known, this kind of access
can be quite slow. In addition, the byte-range IO drivers do not
currently do any sort of optimization or caching.
# Configuration {#byterange_config}
This capability is enabled using the option *--enable-byterange* option
to the *./configure* command for Automake. For Cmake, the option flag is
*-DENABLE_BYTERANGE=true*.
This capability requires access to *libcurl*, and an error will occur
if byterange is enabled, but no *libcurl* could not be located.
In this, it is similar to the DAP2 and DAP4 capabilities.
Note also that the term "http" is often used as a synonym for *byterange*.
# Run-time Usage {#byterange_url}
In order to use this capability at run-time, with *ncdump* for
example, it is necessary to provide a URL pointing to the basic
dataset to be accessed. The URL must be annotated to tell the
netcdf-c library that byte-range access should be used. This is
indicated by appending the phrase ````#bytes````
to the end of the URL.
The two examples above show how this will look.
In order to determine the kind of file being accessed, the
netcdf-c library will read what is called the "magic number"
from the beginning of the remote dataset. This magic number
is a specific set of bytes that indicates the kind of file:
classic, enhanced, cdf5, etc.
# Architecture {#byterange_arch}
Internally, this capability is implemented with three files:
1. libdispatch/dhttp.c -- wrap libcurl operations.
2. libsrc/httpio.c -- provide byte-range reading to the netcdf-3 dispatcher.
3. libhdf5/H5FDhttp.c -- provide byte-range reading to the netcdf-4 dispatcher.
Both *httpio.c* and *H5FDhttp.c* are adapters that use *dhttp.c*
to do the work. Testing for the magic number is also carried out
by using the *dhttp.c* code.
## NetCDF Classic Access
The netcdf-3 code in the directory *libsrc* is built using
a secondary dispatch mechanism called *ncio*. This allows the
netcdf-3 code be independent of the lowest level IO access mechanisms.
This is how in-memory and mmap based access is implemented.
The file *httpio.c* is the dispatcher used to provide byte-range
IO for the netcdf-3 code.
Note that *httpio.c* is mostly just an
adapter between the *ncio* API and the *dhttp.c* code.
## NetCDF Enhanced Access
Similar to the netcdf-3 code, the HDF5 library
provides a secondary dispatch mechanism *H5FD*. This allows the
HDF5 code to be independent of the lowest level IO access mechanisms.
The netcdf-4 code in libhdf5 is built on the HDF5 library, so
it indirectly inherits the H5FD mechanism.
The file *H5FDhttp.c* implements the H5FD dispatcher API
and provides byte-range IO for the netcdf-4 code
(and for the HDF5 library as a side effect).
Note that *H5FDhttp.c* is mostly just an
adapter between the *H5FD* API and the *dhttp.c* code.
# The dhttp.c Code {#byterange_dhttp}
The core of all this is *dhttp.c* (and its header
*include/nchttp.c*). It is a wrapper over *libcurl*
and so exposes the libcurl handles -- albeit as _void*_.
The API for *dhttp.c* consists of the following procedures:
- int nc_http_open(const char* objecturl, void** curlp, fileoffset_t* filelenp);
- int nc_http_read(void* curl, const char* url, fileoffset_t start, fileoffset_t count, NCbytes* buf);
- int nc_http_close(void* curl);
- typedef long long fileoffset_t;
The type *fileoffset_t* is used to avoid use of *off_t* or *off64_t*
which are too volatile. It is intended to be represent file lengths
and offsets.
## nc_http_open
The *nc_http_open* procedure creates a *Curl* handle and returns it
in the *curlp* argument. It also obtains and searches the headers
looking for two headers:
1. "Accept-Ranges: bytes" -- to verify that byte-range access is supported.
2. "Content-Length: ..." -- to obtain the size of the remote dataset.
The dataset length is returned in the *filelenp* argument.
## nc_http_read
The *nc_http_read* procedure reads a specified set of contiguous bytes
as specified by the *start* and *count* arguments. It takes the *Curl*
handle produced by *nc_http_open* to indicate the server from which to read.
The *buf* argument is a pointer to an instance of type *NCbytes*, which
is a dynamically expandable byte vector (see the file *include/ncbytes.h*).
This procedure reads *count* bytes from the remote dataset starting at
the offset *start* position. The bytes are stored in *buf*.
## nc_http_close
The *nc_http_close* function closes the *Curl* handle and does any
necessary cleanup.
# Point of Contact {#byterange_poc}
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu<br>
__Initial Version__: 12/30/2018<br>
__Last Revised__: 12/30/2018
<!-- End MarkDown -->
@endif
*/

60
docs/internal.dox

@ -27,7 +27,7 @@ To date, at least the following dispatch tables are supported.
- netcdf enhanced files (netcdf-4)
- DAP2 to netcdf-3
- DAP4 to netcdf-4
- PnetCDF (parallel I/O for classic files)
- PnetCDF (parallel I/O for classic files -- version 1,2, or 5)
- HDF4 SD files
The dispatch table represents a distillation of the netcdf API down to
@ -159,9 +159,11 @@ the top-level _Makefile.am_.
\section choosing_dispatch_table Choosing a Dispatch Table
The dispatch table is chosen in the NC_create and the NC_open
procedures in _libdispatch/netcdf.c_.
procedures.
This can be, unfortunately, a complex process.
The choice is made in _NC_create_ and _NC_open_ in _libdispatch/dfile.c_.
The code for inferring a dispatch table is largely isolated
to the file _libdispatch/dinfermodel.c_, which is invoked
from _NC_create_ or _NC_open_ in _libdispatch/dfile.c_.
In any case, the choice of dispatch table is currently based on the following
pieces of information.
@ -179,10 +181,6 @@ the choice is determined using the function _NC_urlmodel_.
the contents of the file can be used to determine the dispatch table.
As a rule, this is likely to be useful only for _nc_open_.
4. Environment variables - this option is currently not used,
but information such as environment variables could be used to determine
the choice of dispatch table.
\section special_dispatch Special Dispatch Table Signatures.
The entries in the dispatch table do not necessarily correspond
@ -483,4 +481,52 @@ The code in _hdf4var.c_ does an _nc_get_vara()_ on the HDF4 SD
dataset. This is all that is needed for all the nc_get_* functions to
work.
\subsection model_infer Inferring the Dispatch Table
As mentioned above, the dispatch table is inferred using the following
information:
1. The mode argument
2. The file path/URL
3. The file contents (when available)
The primary function for doing this inference is in the file
_libdispatch/dinfermodel.c_ via the API in _include/ncmodel.h_.
The term _model_ is used here to include (at least) the following
information (see the structure type _NCmodel_ in _include/ncmodel.h_).
1. format -- this is an NC_FORMAT_XXX value defining the file format
as seen by the user program.
2. version -- the specific version of the format.
3. iosp -- this is and NC_IOSP_XXX value describing internal protocols to use.
4. impl -- this is an NC_FORMATX_XXX value defining, in effect, the
dispatch table to use.
For example, if the format was NC_FORMAT_CLASSIC, then the client
will see the netcdf-3 data model, as modified by the version. If the
version was 5, for example, then that indicates the file format
is actually NC_FORMAT_64BIT_DATA, which is a variant of the netcdf-3
format.
The _iosp_ provides information about how the protocol the
dispatch table will use to access the actual dataset. If the iosp
is NC_IOSP_S3RAW, then it indicates that the dispatcher, NC_FORMATX_NC3,
for example, will access the dataset using the Amazon S3 REST API.
The construction of the model is primarily carried out by the function
_NC_infermodel()_. It is given the following parameters:
1. path -- (IN) absolute file path or URL
2. omodep -- (IN/OUT) the set of mode flags given to _NC_open_ or _NC_create_.
3. iscreate -- (IN) distinguish open from create.
4. useparallel -- (IN) indicate if parallel IO can be used.
5. params -- (IN/OUT) arbitrary data dependent on the mode and path.
6. model -- (IN/OUT) place to store inferred model information (e.g. format
or version).
7. newpathp -- (OUT) sometimes, it is necessary to rewrite the path.
As a rule, these values are used in the this order to infer the model.
1. file contents -- highest precedence
2. url (if it is one) -- using the protocol and fragment arguments
3. mode flags
4. default format -- lowest precedence
*/

6
include/Makefile.am

@ -18,10 +18,14 @@ ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h \
nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \
ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \
ncwinpath.h ncfilter.h ncindex.h hdf4dispatch.h hdf5internal.h \
nc_provenance.h hdf5dispatch.h
nc_provenance.h hdf5dispatch.h ncmodel.h
if USE_DAP
noinst_HEADERS += ncdap.h
endif
if ENABLE_HTTP
noinst_HEADERS += nchttp.h
endif
EXTRA_DIST = CMakeLists.txt XGetopt.h netcdf_meta.h.in

11
include/hdf5internal.h

@ -51,10 +51,15 @@
/** This is the name of the name HDF5 dimension scale attribute. */
#define HDF5_DIMSCALE_NAME_ATT_NAME "NAME"
/** Strut to hold HDF5-specific info for the file. */
typedef struct NC_HDF5_FILE_INFO
{
/** Struct to hold HDF5-specific info for the file. */
typedef struct NC_HDF5_FILE_INFO {
hid_t hdfid;
#ifdef ENABLE_HTTP
struct HTTP {
NCURI* uri; /* Parse of the incoming path, if url */
int iosp; /* We are using the S3 rawvirtual file driver */
} http;
#endif
} NC_HDF5_FILE_INFO_T;
/* This is a struct to handle the dim metadata. */

7
include/nc.h

@ -8,6 +8,9 @@
#include "config.h"
#include "netcdf.h"
/* Forward */
struct NCmodel;
/* There's an external ncid (ext_ncid) and an internal ncid
* (int_ncid). The ext_ncid is the ncid returned to the user. If
* the user has opened or created a netcdf-4 file, then the
@ -27,7 +30,7 @@ typedef struct NC {
void* dispatchdata; /*per-'file' data; points to e.g. NC3_INFO data*/
char* path;
int mode; /* as provided to nc_open/nc_create */
int model; /* as determined by libdispatch/dfile.c */
struct NCmodel* model; /* as determined by libdispatch/dfile.c */
#ifdef USE_REFCOUNT
int refcount; /* To enable multiple name-based opens */
#endif
@ -78,7 +81,7 @@ extern int iterate_NCList(int i,NC**); /* Walk from 0 ...; ERANGE return => stop
/* Defined in nc.c */
extern void free_NC(NC*);
extern int new_NC(struct NC_Dispatch*, const char*, int, int, NC**);
extern int new_NC(struct NC_Dispatch*, const char*, int, struct NCmodel*, NC**);
/* Defined in nc.c */
extern int ncdebug;

1
include/nc4internal.h

@ -274,6 +274,7 @@ typedef struct NC_FILE_INFO
NClist* allgroups; /* including root group */
void *format_file_info;
struct NCPROVENANCE* provenance;
/* This should be in NC_HDF5_FILE_INFO_T */
struct NC4_Memio {
NC_memio memio; /* What we sent to image_init and what comes back*/
int locked; /* do not copy and do not free */

9
include/ncconfigure.h

@ -26,12 +26,6 @@ defined and missing types defined.
extern char* strdup(const char*);
#endif
/*
#ifndef HAVE_SSIZE_T
typedef long ssize_t;
#define HAVE_SSIZE_T
#endif
*/
/* handle null arguments */
#ifndef nulldup
#ifdef HAVE_STRDUP
@ -85,4 +79,7 @@ typedef unsigned short ushort;
typedef unsigned int uint;
#endif
/* Provide a fixed size alternative to off_t or off64_t */
typedef long long fileoffset_t;
#endif /* NCCONFIGURE_H */

48
include/ncdispatch.h

@ -7,8 +7,8 @@
* @author Dennis Heimbigner
*/
#ifndef _DISPATCH_H
#define _DISPATCH_H
#ifndef NC_DISPATCH_H
#define NC_DISPATCH_H
#if HAVE_CONFIG_H
#include "config.h"
@ -21,6 +21,7 @@
#include <mpi.h>
#endif
#include "netcdf.h"
#include "ncmodel.h"
#include "nc.h"
#include "ncuri.h"
#ifdef USE_PARALLEL
@ -63,14 +64,6 @@
#define T_ulong ulongtype
/**************************************************/
#if 0
/* Define the known classes of dispatchers */
/* Flags may be or'd => powers of 2*/
#define NC_DISPATCH_NC3 1
#define NC_DISPATCH_NC4 2
#define NC_DISPATCH_NCD 4
#define NC_DISPATCH_NCP 8
#endif
/* Define a type for use when doing e.g. nc_get_vara_long, etc. */
/* Should matche values in libsrc4/netcdf.h */
@ -154,6 +147,12 @@ extern size_t nc_sizevector0[NC_MAX_VAR_DIMS];
extern size_t nc_sizevector1[NC_MAX_VAR_DIMS];
extern ptrdiff_t nc_ptrdiffvector1[NC_MAX_VAR_DIMS];
/* User-defined formats. */
extern NC_Dispatch* UDF0_dispatch_table;
extern char UDF0_magic_number[NC_MAX_MAGIC_NUMBER_LEN + 1];
extern NC_Dispatch* UDF1_dispatch_table;
extern char UDF1_magic_number[NC_MAX_MAGIC_NUMBER_LEN + 1];
/* Prototypes. */
int NC_check_nulls(int ncid, int varid, const size_t *start, size_t **count,
ptrdiff_t **stride);
@ -339,32 +338,6 @@ extern NC_Dispatch* NC_get_dispatch_override(void);
extern void NC_set_dispatch_override(NC_Dispatch*);
#endif
/* Return model as specified by the url, if any;
return a modified url suitable for passing to curl
*/
extern int NC_urlmodel(const char* path, int mode, char** newurl);
/* allow access url parse and params without exposing nc_url.h */
extern int NCDAP_urlparse(const char* s, void** dapurl);
extern void NCDAP_urlfree(void* dapurl);
extern const char* NCDAP_urllookup(void* dapurl, const char* param);
#if defined(DLL_NETCDF)
# if defined(DLL_EXPORT)
# define NCC_EXTRA __declspec(dllexport)
#else
# define NCC_EXTRA __declspec(dllimport)
# endif
NCC_EXTRA extern int nc__testurl(const char* path, char** basename);
#else
extern int
nc__testurl(const char* parth, char** basename);
#endif
/* Ping a specific server */
extern int NCDAP2_ping(const char*);
extern int NCDAP4_ping(const char*);
/* Misc */
extern int NC_getshape(int ncid, int varid, int ndims, size_t* shape);
@ -373,7 +346,6 @@ extern int NC_inq_recvar(int ncid, int varid, int* nrecdims, int* is_recdim);
#define nullstring(s) (s==NULL?"(null)":s)
#undef TRACECALLS
#ifdef TRACECALLS
#include <stdio.h>
@ -467,4 +439,4 @@ EXTERNL int NC_NOTNC4_set_var_chunk_cache(int, int, size_t, size_t, float);
EXTERNL int NC_NOTNC4_get_var_chunk_cache(int, int, size_t *, size_t *, float *);
EXTERNL int NC_NOTNC4_var_par_access(int, int, int);
#endif /* _DISPATCH_H */
#endif /* NC_DISPATCH_H */

15
include/nchttp.h Normal file

@ -0,0 +1,15 @@
/* Copyright 2018-2018 University Corporation for Atmospheric
Research/Unidata. */
/**
* Header file for dhttp.c
* @author Dennis Heimbigner
*/
#ifndef NCHTTP_H
#define NCHTTP_H
extern int nc_http_open(const char* objecturl, void** curlp, fileoffset_t* filelenp);
extern int nc_http_read(void* curl, const char* url, fileoffset_t start, fileoffset_t count, NCbytes* buf);
extern int nc_http_close(void* curl);
#endif /*NCHTTP_H*/

67
include/ncmodel.h Normal file

@ -0,0 +1,67 @@
/* Copyright 2018-2018 University Corporation for Atmospheric
Research/Unidata. */
/**
* Functions for inferring dataset model
* @author Dennis Heimbigner
*/
#ifndef NCINFERMODEL_H
#define NCINFERMODEL_H
/* Define the io handler to be used to do lowest level
access. This is above the libcurl level and below the
dispatcher level. This is only used for remote
datasets or for implementations where the implementation
multiplexes more than one IOSP in a single dispatcher.
*/
#define NC_IOSP_FILE (1)
#define NC_IOSP_MEMORY (2)
#define NC_IOSP_DAP2 (3)
#define NC_IOSP_DAP4 (4)
#define NC_IOSP_UDF (5) /*Placeholder since we do not know IOSP for UDF*/
#define NC_IOSP_HTTP (6)
/* Track the information hat will help us
infer how to access the data defined by
path + omode.
*/
typedef struct NCmodel {
int format; /* NC_FORMAT_XXX value */
int impl; /* NC_FORMATX_XXX value */
int iosp; /* NC_IOSP_XXX value (above) */
} NCmodel;
/* Keep compiler quiet */
struct NCURI;
struct NC_dispatch;
#if 0
/* return first IOSP or NULL if none */
EXTERNL int NC_urliosp(struct NCURI* u);
#endif
/* Infer model format and implementation */
EXTERNL int NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void* params, NCmodel* model, char** newpathp);
/**
* Provide a hidden interface to allow utilities
* to check if a given path name is really a url.
* If not, put null in basenamep, else put basename of the url
* minus any extension into basenamep; caller frees.
* Return 1 if it looks like a url, 0 otherwise.
*/
EXTERNL int nc__testurl(const char* path, char** basenamep);
#if 0
/* allow access url parse and params without exposing nc_url.h */
EXTERNL int NCDAP_urlparse(const char* s, void** dapurl);
EXTERNL void NCDAP_urlfree(void* dapurl);
EXTERNL const char* NCDAP_urllookup(void* dapurl, const char* param);
/* Ping a specific server */
EXTERNL int NCDAP2_ping(const char*);
EXTERNL int NCDAP4_ping(const char*);
#endif
#endif /*NCINFERMODEL_H*/

4
include/ncrc.h

@ -47,6 +47,10 @@ extern char* NC_rclookup(const char* key, const char* hostport);
extern void NC_rcclear(NCRCinfo* info);
extern int NC_set_rcfile(const char* rcfile);
extern int NC_rcfile_insert(const char* key, const char* value, const char* hostport);
/* Obtain the count of number of triples */
extern size_t NC_rcfile_length(NCRCinfo*);
/* Obtain the ith triple; return NULL if out of range */
extern NCTriple* NC_rcfile_ith(NCRCinfo*,size_t);
/* From dutil.c (Might later move to e.g. nc.h */
extern int NC__testurl(const char* path, char** basenamep);

6
include/ncuri.h

@ -94,6 +94,12 @@ extern const char* ncurilookup(NCURI*, const char* param);
*/
extern const char* ncuriquerylookup(NCURI*, const char* param);
/* Obtain the complete list of fragment pairs in envv format */
extern const char** ncurifragmentparams(NCURI*);
/* Obtain the complete list of query pairs in envv format */
extern const char** ncuriqueryparams(NCURI*);
/* URL Encode/Decode */
extern char* ncuridecode(char* s);
/* Partial decode */

56
include/ncurlmodel.h Normal file

@ -0,0 +1,56 @@
/* Copyright 2018-2018 University Corporation for Atmospheric
Research/Unidata. */
/**
* Header file for dmode.c
* @author Dennis Heimbigner
*/
#ifndef NCURLMODEL_H
#define NCURLMODEL_H
/* Define the io handler to be used to do lowest level
access. This is above the libcurl level.
Note that cases (DAP2,DAP4) where the implementation is 1-1
with the iosp are not included.
*/
#define NC_IOSP_S3 (1)
#define NC_IOSP_ZARR (1)
/* Track the information from a URL that will help us
infer how to access the data pointed to by that URL.
*/
typedef struct NCmode {
int format; /* NC_FORMAT_XXX value */
int implementation; /* NC_FORMATX_XXX value */
int iosp; /* NC_IOSP_XXX value (above) */
} NCmode;
/* return 1 if path looks like a url; 0 otherwise */
EXTERNL int NC_testurl(const char* path);
/*
Return an NC_FORMATX_... value.
Assumes that the path is known to be a url.
*/
EXTERNL int NC_urlmodel(const char* path, int mode, char** newurl, NCmode* model);
/**
* Provide a hidden interface to allow utilities
* to check if a given path name is really an ncdap3 url.
* If no, put null in basenamep, else put basename of the url
* minus any extension into basenamep; caller frees.
* Return 1 if it looks like a url, 0 otherwise.
*/
EXTERNL int nc__testurl(const char* path, char** basenamep);
/* allow access url parse and params without exposing nc_url.h */
EXTERNL int NCDAP_urlparse(const char* s, void** dapurl);
EXTERNL void NCDAP_urlfree(void* dapurl);
EXTERNL const char* NCDAP_urllookup(void* dapurl, const char* param);
/* Ping a specific server */
EXTERNL int NCDAP2_ping(const char*);
EXTERNL int NCDAP4_ping(const char*);
#endif /*NCURLMODEL_H*/

8
include/netcdf.h

@ -172,8 +172,8 @@ Use this in mode flags for both nc_create() and nc_open(). */
*/
/**@{*/
#define NC_FORMAT_CLASSIC (1)
/* After adding CDF5 support, this flag
is somewhat confusing. So, it is renamed.
/* After adding CDF5 support, the NC_FORMAT_64BIT
flag is somewhat confusing. So, it is renamed.
Note that the name in the contributed code
NC_FORMAT_64BIT was renamed to NC_FORMAT_CDF2
*/
@ -186,6 +186,9 @@ Use this in mode flags for both nc_create() and nc_open(). */
/* Alias */
#define NC_FORMAT_CDF5 NC_FORMAT_64BIT_DATA
/* Define a mask covering format flags only */
#define NC_FORMAT_ALL (NC_64BIT_OFFSET|NC_64BIT_DATA|NC_CLASSIC_MODEL|NC_NETCDF4|NC_UDF0|NC_UDF1)
/**@}*/
/** Extended format specifier returned by nc_inq_format_extended()
@ -215,6 +218,7 @@ Use this in mode flags for both nc_create() and nc_open(). */
#define NC_FORMATX_DAP4 (6)
#define NC_FORMATX_UDF0 (8)
#define NC_FORMATX_UDF1 (9)
#define NC_FORMATX_ZARR (10)
#define NC_FORMATX_UNDEFINED (0)
/* To avoid breaking compatibility (such as in the python library),

1
include/netcdf_meta.h.in

@ -46,6 +46,7 @@
#define NC_HAS_SZIP @NC_HAS_SZIP@ /*!< szip support (HDF5 only) */
#define NC_HAS_DAP2 @NC_HAS_DAP2@ /*!< DAP2 support. */
#define NC_HAS_DAP4 @NC_HAS_DAP4@ /*!< DAP4 support. */
#define NC_HAS_HTTP @HAS_HTTP@
#define NC_HAS_DISKLESS @NC_HAS_DISKLESS@ /*!< diskless support. */
#define NC_HAS_MMAP @NC_HAS_MMAP@ /*!< mmap support. */
#define NC_HAS_JNA @NC_HAS_JNA@ /*!< jna support. */

6
libdispatch/CMakeLists.txt

@ -4,7 +4,7 @@
# University Corporation for Atmospheric Research/Unidata.
# See netcdf-c/COPYRIGHT file for more info.
SET(libdispatch_SOURCES dparallel.c dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dwinpath.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c crc32.c daux.c)
SET(libdispatch_SOURCES dparallel.c dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dwinpath.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c crc32.c daux.c dinfermodel.c)
IF(USE_NETCDF4)
SET(libdispatch_SOURCES ${libdispatch_SOURCES} dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c)
@ -14,6 +14,10 @@ IF(BUILD_V2)
SET(libdispatch_SOURCES ${libdispatch_SOURCES} dv2i.c)
ENDIF(BUILD_V2)
IF(ENABLE_HTTP)
SET(libdispatch_SOURCES ${libdispatch_SOURCES} dhttp.c)
ENDIF(ENABLE_HTTP)
add_library(dispatch OBJECT ${libdispatch_SOURCES})
FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.c)

6
libdispatch/Makefile.am

@ -20,7 +20,7 @@ dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c \
dvarinq.c dinternal.c ddispatch.c dutf8.c nclog.c dstring.c ncuri.c \
nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c drc.c \
dauth.c doffsets.c dwinpath.c dutil.c dreadonly.c dnotnc4.c dnotnc3.c \
crc32.c crc32.h daux.c
crc32.c crc32.h daux.c dinfermodel.c
# Add the utf8 codebase
libdispatch_la_SOURCES += utf8proc.c utf8proc.h
@ -38,6 +38,10 @@ libnetcdf2_la_SOURCES = dv2i.c
libnetcdf2_la_CPPFLAGS = ${AM_CPPFLAGS} -DDLL_EXPORT
endif # BUILD_V2
if ENABLE_HTTP
libdispatch_la_SOURCES += dhttp.c
endif # ENABLE_HTTP
EXTRA_DIST=CMakeLists.txt ncsettings.hdr utf8proc_data.c
# Build ncsettings.c as follows:

201
libdispatch/ddispatch.c

@ -29,22 +29,6 @@ ptrdiff_t nc_ptrdiffvector1[NC_MAX_VAR_DIMS];
size_t NC_coord_zero[NC_MAX_VAR_DIMS];
size_t NC_coord_one[NC_MAX_VAR_DIMS];
/* Define the known protocols and their manipulations */
static struct NCPROTOCOLLIST {
char* protocol;
char* substitute;
int model;
} ncprotolist[] = {
{"http",NULL,0},
{"https",NULL,0},
{"file",NULL,0},
{"dods","http",NC_FORMATX_DAP2},
{"dodss","https",NC_FORMATX_DAP2},
{"dap4","http",NC_FORMATX_DAP4},
{"dap4s","https",NC_FORMATX_DAP4},
{NULL,NULL,0} /* Terminate search */
};
NCRCglobalstate ncrc_globalstate;
/*
@ -148,188 +132,3 @@ NCDISPATCH_finalize(void)
return status;
}
/* return 1 if path looks like a url; 0 otherwise */
int
NC_testurl(const char* path)
{
int isurl = 0;
NCURI* tmpurl = NULL;
char* p;
if(path == NULL) return 0;
/* find leading non-blank */
for(p=(char*)path;*p;p++) {if(*p != ' ') break;}
/* Do some initial checking to see if this looks like a file path */
if(*p == '/') return 0; /* probably an absolute file path */
/* Ok, try to parse as a url */
if(ncuriparse(path,&tmpurl)==NCU_OK) {
/* Do some extra testing to make sure this really is a url */
/* Look for a known/accepted protocol */
struct NCPROTOCOLLIST* protolist;
for(protolist=ncprotolist;protolist->protocol;protolist++) {
if(strcmp(tmpurl->protocol,protolist->protocol) == 0) {
isurl=1;
break;
}
}
ncurifree(tmpurl);
return isurl;
}
return 0;
}
/*
Return an NC_FORMATX_... value.
Assumes that the path is known to be a url
*/
int
NC_urlmodel(const char* path, int mode, char** newurl)
{
int found, model = 0;
struct NCPROTOCOLLIST* protolist;
NCURI* url = NULL;
char* p;
if(path == NULL) return 0;
/* find leading non-blank */
for(p=(char*)path;*p;p++) {if(*p != ' ') break;}
/* Do some initial checking to see if this looks like a file path */
if(*p == '/') return 0; /* probably an absolute file path */
/* Parse the url */
if(ncuriparse(path,&url) != NCU_OK)
goto fail; /* Not parseable as url */
/* Look up the protocol */
for(found=0,protolist=ncprotolist;protolist->protocol;protolist++) {
if(strcmp(url->protocol,protolist->protocol) == 0) {
found = 1;
break;
}
}
if(found) {
model = protolist->model;
/* Substitute the protocol in any case */
if(protolist->substitute) ncurisetprotocol(url,protolist->substitute);
} else
goto fail; /* Again, does not look like a url */
if(model != NC_FORMATX_DAP2 && model != NC_FORMATX_DAP4) {
/* Look for and of the following params:
"dap2", "protocol=dap2", "dap4", "protocol=dap4" */
const char* proto = NULL;
const char* match = NULL;
if((proto=ncurilookup(url,"protocol")) == NULL) proto = NULL;
if(proto == NULL) proto = "";
if((match=ncurilookup(url,"dap2")) != NULL || strcmp(proto,"dap2") == 0)
model = NC_FORMATX_DAP2;
else if((match=ncurilookup(url,"dap4")) != NULL || strcmp(proto,"dap4") == 0)
model = NC_FORMATX_DAP4;
else
model = 0; /* Still don't know */
}
if(model == 0) {/* Last resort: use the mode */
/* If mode specifies netcdf-4, then this is assumed to be dap4 */
if(mode & NC_NETCDF4)
model = NC_FORMATX_DAP4;
else
model = NC_FORMATX_DAP2; /* Default */
}
if(newurl)
*newurl = ncuribuild(url,NULL,NULL,NCURIALL);
if(url) ncurifree(url);
return model;
fail:
if(url) ncurifree(url);
return 0;
}
/**************************************************/
/**
* Provide a hidden interface to allow utilities
* to check if a given path name is really an ncdap3 url.
* If no, put null in basenamep, else put basename of the url
* minus any extension into basenamep; caller frees.
* Return 1 if it looks like a url, 0 otherwise.
*/
int
nc__testurl(const char* path, char** basenamep)
{
NCURI* uri;
int ok = 0;
if(ncuriparse(path,&uri) == NCU_OK) {
char* slash = (uri->path == NULL ? NULL : strrchr(uri->path, '/'));
char* dot;
if(slash == NULL) slash = (char*)path; else slash++;
slash = nulldup(slash);
if(slash == NULL)
dot = NULL;
else
dot = strrchr(slash, '.');
if(dot != NULL && dot != slash) *dot = '\0';
if(basenamep)
*basenamep=slash;
else if(slash)
free(slash);
ncurifree(uri);
ok = 1;
}
return ok;
}
/**************************************************/
#ifdef OBSOLETE
/* Override dispatch table management */
static NC_Dispatch* NC_dispatch_override = NULL;
/* Override dispatch table management */
NC_Dispatch*
NC_get_dispatch_override(void) {
return NC_dispatch_override;
}
void NC_set_dispatch_override(NC_Dispatch* d)
{
NC_dispatch_override = d;
}
#endif
/* OBSOLETE
Overlay by treating the tables as arrays of void*.
Overlay rules are:
overlay base merge
------- ---- -----
null null null
null y y
x null x
x y x
*/
#ifdef OBSOLETE
int
NC_dispatch_overlay(const NC_Dispatch* overlay, const NC_Dispatch* base, NC_Dispatch* merge)
{
void** voverlay = (void**)overlay;
void** vmerge;
int i;
size_t count = sizeof(NC_Dispatch) / sizeof(void*);
/* dispatch table must be exact multiple of sizeof(void*) */
assert(count * sizeof(void*) == sizeof(NC_Dispatch));
*merge = *base;
vmerge = (void**)merge;
for(i=0;i<count;i++) {
if(voverlay[i] == NULL) continue;
vmerge[i] = voverlay[i];
}
/* Finally, the merge model should always be the overlay model */
merge->model = overlay->model;
return NC_NOERR;
}
#endif

859
libdispatch/dfile.c

@ -12,6 +12,7 @@
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#ifdef HAVE_SYS_RESOURCE_H
#include <sys/resource.h>
#endif
@ -31,48 +32,16 @@
#include "ncwinpath.h"
#include "fbits.h"
/* If Defined, then use only stdio for all magic number io;
otherwise use stdio or mpio as required.
*/
#undef DEBUG
/**
Sort info for open/read/close of
file when searching for magic numbers
*/
struct MagicFile {
const char* path;
long long filelen;
int use_parallel;
int inmemory;
int diskless;
void* parameters; /* !NULL if inmemory && !diskless */
FILE* fp;
#ifdef USE_PARALLEL
MPI_File fh;
#endif
};
static int openmagic(struct MagicFile* file);
static int readmagic(struct MagicFile* file, long pos, char* magic);
static int closemagic(struct MagicFile* file);
#ifdef DEBUG
static void printmagic(const char* tag, char* magic,struct MagicFile*);
#endif
extern int NC_initialized; /**< True when dispatch table is initialized. */
/** @internal Magic number for HDF5 files. To be consistent with
* H5Fis_hdf5, use the complete HDF5 magic number */
static char HDF5_SIGNATURE[MAGIC_NUMBER_LEN] = "\211HDF\r\n\032\n";
#ifdef USE_NETCDF4
/* User-defined formats. */
NC_Dispatch *UDF0_dispatch_table = NULL;
char UDF0_magic_number[NC_MAX_MAGIC_NUMBER_LEN + 1] = "";
NC_Dispatch *UDF1_dispatch_table = NULL;
char UDF1_magic_number[NC_MAX_MAGIC_NUMBER_LEN + 1] = "";
#endif /* USE_NETCDF4 */
/**************************************************/
/** \defgroup datasets NetCDF File and Data I/O
@ -197,166 +166,6 @@ nc_inq_user_format(int mode_flag, NC_Dispatch **dispatch_table, char *magic_numb
}
#endif /* USE_NETCDF4 */
/*!
Interpret the magic number found in the header of a netCDF file.
This function interprets the magic number/string contained in the header of a netCDF file and sets the appropriate NC_FORMATX flags.
@param[in] magic Pointer to a character array with the magic number block.
@param[out] model Pointer to an integer to hold the corresponding netCDF type.
@param[out] version Pointer to an integer to hold the corresponding netCDF version.
@returns NC_NOERR if a legitimate file type found
@returns NC_ENOTNC otherwise
\internal
\ingroup datasets
*/
static int
NC_interpret_magic_number(char* magic, int* model, int* version)
{
int status = NC_NOERR;
/* Look at the magic number */
*model = 0;
*version = 0;
#ifdef USE_NETCDF4
if (strlen(UDF0_magic_number) && !strncmp(UDF0_magic_number, magic,
strlen(UDF0_magic_number)))
{
*model = NC_FORMATX_UDF0;
*version = 6; /* redundant */
goto done;
}
if (strlen(UDF1_magic_number) && !strncmp(UDF1_magic_number, magic,
strlen(UDF1_magic_number)))
{
*model = NC_FORMATX_UDF1;
*version = 7; /* redundant */
goto done;
}
#endif /* USE_NETCDF4 */
/* Use the complete magic number string for HDF5 */
if(memcmp(magic,HDF5_SIGNATURE,sizeof(HDF5_SIGNATURE))==0) {
*model = NC_FORMATX_NC4;
*version = 5; /* redundant */
goto done;
}
if(magic[0] == '\016' && magic[1] == '\003'
&& magic[2] == '\023' && magic[3] == '\001') {
*model = NC_FORMATX_NC_HDF4;
*version = 4; /* redundant */
goto done;
}
if(magic[0] == 'C' && magic[1] == 'D' && magic[2] == 'F') {
if(magic[3] == '\001') {
*version = 1; /* netcdf classic version 1 */
*model = NC_FORMATX_NC3;
goto done;
}
if(magic[3] == '\002') {
*version = 2; /* netcdf classic version 2 */
*model = NC_FORMATX_NC3;
goto done;
}
if(magic[3] == '\005') {
*version = 5; /* cdf5 file */
*model = NC_FORMATX_NC3;
goto done;
}
}
/* No match */
status = NC_ENOTNC;
goto done;
done:
return status;
}
/**
* @internal Given an existing file, figure out its format and return
* that format value (NC_FORMATX_XXX) in model arg. Assume any path
* conversion was already performed at a higher level.
*
* @param path File name.
* @param flags
* @param use_parallel
* @param parameters
* @param model Pointer that gets the model to use for the dispatch table.
* @param version Pointer that gets version of the file.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
NC_check_file_type(const char *path, int flags, int use_parallel,
void *parameters, int* model, int* version)
{
char magic[MAGIC_NUMBER_LEN];
int status = NC_NOERR;
int diskless = ((flags & NC_DISKLESS) == NC_DISKLESS);
int inmemory = ((flags & NC_INMEMORY) == NC_INMEMORY);
int mmap = ((flags & NC_MMAP) == NC_MMAP);
struct MagicFile file;
*model = 0;
*version = 0;
/* NC_INMEMORY and NC_DISKLESS and NC_MMAP are all mutually exclusive */
if(diskless && inmemory) {status = NC_EDISKLESS; goto done;}
if(diskless && mmap) {status = NC_EDISKLESS; goto done;}
if(inmemory && mmap) {status = NC_EINMEMORY; goto done;}
/* mmap is not allowed for netcdf-4 */
if(mmap && (flags & NC_NETCDF4)) {status = NC_EINVAL; goto done;}
memset((void*)&file,0,sizeof(file));
file.path = path; /* do not free */
file.parameters = parameters;
file.inmemory = inmemory;
file.diskless = diskless;
file.use_parallel = use_parallel;
status = openmagic(&file);
if(status != NC_NOERR) {goto done;}
/* Verify we have a large enough file */
if(file.filelen < MAGIC_NUMBER_LEN)
{status = NC_ENOTNC; goto done;}
if((status = readmagic(&file,0L,magic)) != NC_NOERR) {
status = NC_ENOTNC;
*model = 0;
*version = 0;
goto done;
}
/* Look at the magic number */
if(NC_interpret_magic_number(magic,model,version) == NC_NOERR
&& *model != 0) {
if (*model == NC_FORMATX_NC3 && use_parallel)
/* this is called from nc_open_par() and file is classic */
*model = NC_FORMATX_PNETCDF;
goto done; /* found something */
}
/* Remaining case is to search forward at starting at 512
and doubling to see if we have HDF5 magic number */
{
long pos = 512L;
for(;;) {
if((pos+MAGIC_NUMBER_LEN) > file.filelen)
{status = NC_ENOTNC; goto done;}
if((status = readmagic(&file,pos,magic)) != NC_NOERR)
{status = NC_ENOTNC; goto done; }
NC_interpret_magic_number(magic,model,version);
if(*model == NC_FORMATX_NC4) break;
/* double and try again */
pos = 2*pos;
}
}
done:
closemagic(&file);
return status;
}
/** \ingroup datasets
Create a new netCDF file.
@ -710,9 +519,9 @@ nc__create_mp(const char *path, int cmode, size_t initialsz,
* determines the underlying file format automatically. Use the same
* call to open a netCDF classic or netCDF-4 file.
*
* @param path File name for netCDF dataset to be opened. When DAP
* support is enabled, then the path may be an OPeNDAP URL rather than
* a file path.
* @param path File name for netCDF dataset to be opened. When the dataset
* is located on some remote server, then the path may be an OPeNDAP URL
* rather than a file path.
* @param omode The open mode flag may include NC_WRITE (for read/write
* access) and NC_SHARE (see below) and NC_DISKLESS (see below).
* @param ncidp Pointer to location where returned netCDF ID is to be
@ -2015,10 +1824,9 @@ NC_create(const char *path0, int cmode, size_t initialsz,
int stat = NC_NOERR;
NC* ncp = NULL;
NC_Dispatch* dispatcher = NULL;
/* Need three pieces of information for now */
int model = NC_FORMATX_UNDEFINED; /* one of the NC_FORMATX values */
int isurl = 0; /* dap or cdmremote or neither */
char* path = NULL;
NCmodel model;
char* newpath = NULL;
TRACE(nc_create);
if(path0 == NULL)
@ -2037,12 +1845,17 @@ NC_create(const char *path0, int cmode, size_t initialsz,
return stat;
}
{
/* Skip past any leading whitespace in path */
const char* p;
for(p=(char*)path0;*p;p++) {if(*p > ' ') break;}
#ifdef WINPATH
/* Need to do path conversion */
path = NCpathcvt(path0);
/* Need to do path conversion */
path = NCpathcvt(p);
#else
path = nulldup(path0);
path = nulldup(p);
#endif
}
#ifdef USE_REFCOUNT
/* If this path is already open, then fail */
@ -2053,116 +1866,72 @@ NC_create(const char *path0, int cmode, size_t initialsz,
}
#endif
{
char* newpath = NULL;
model = NC_urlmodel(path,cmode,&newpath);
isurl = (model != 0);
if(isurl) {
nullfree(path);
path = newpath;
}
memset(&model,0,sizeof(model));
if((stat = NC_infermodel(path,&cmode,1,0,NULL,&model,&newpath)))
goto done;
if(newpath) {
nullfree(path);
path = newpath;
newpath = NULL;
}
/* determine the model */
#ifdef USE_NETCDF4
if (model == NC_FORMATX_UNDEFINED && (cmode & NC_NETCDF4))
model = NC_FORMATX_NC4;
#else
if (model == NC_FORMATX_UNDEFINED && (cmode & NC_NETCDF4))
return NC_ENOTBUILT;
assert(model.format != 0 && model.impl != 0);
/* Now, check for NC_ENOTBUILT cases limited to create (so e.g. HDF4 is not listed) */
#ifndef USE_HDF5
if (model.impl == NC_FORMATX_NC4)
{stat = NC_ENOTBUILT; goto done;}
#endif
#ifdef USE_PNETCDF
if (model == NC_FORMATX_UNDEFINED && useparallel)
/* PnetCDF is used for parallel io on CDF-1, CDF-2, and CDF-5 */
model = NC_FORMATX_PNETCDF;
#else
if (model == NC_FORMATX_UNDEFINED && useparallel)
return NC_ENOTBUILT;
#endif
/* Check default format (not formatx) */
if (!fIsSet(cmode, NC_64BIT_OFFSET) && !fIsSet(cmode, NC_64BIT_DATA) &&
!fIsSet(cmode, NC_CLASSIC_MODEL) && !fIsSet(cmode, NC_NETCDF4)) {
/* if no file format flag is set in cmode, use default */
int format = nc_get_default_format();
switch (format) {
#ifdef USE_NETCDF4
case NC_FORMAT_NETCDF4:
cmode |= NC_NETCDF4;
if (model == NC_FORMATX_UNDEFINED) model = NC_FORMATX_NC4;
break;
case NC_FORMAT_NETCDF4_CLASSIC:
cmode |= NC_NETCDF4 | NC_CLASSIC_MODEL;
if (model == NC_FORMATX_UNDEFINED) model = NC_FORMATX_NC4;
break;
#endif
case NC_FORMAT_CDF5:
cmode |= NC_64BIT_DATA;
break;
case NC_FORMAT_64BIT_OFFSET:
cmode |= NC_64BIT_OFFSET;
break;
case NC_FORMAT_CLASSIC: break;
default: break;
}
}
/* default model */
if (model == NC_FORMATX_UNDEFINED) {
if (useparallel)
model = NC_FORMATX_PNETCDF;
else
model = NC_FORMATX_NC3;
}
#ifndef USE_PNETCDF
if (model.impl == NC_FORMATX_PNETCDF)
{stat = NC_ENOTBUILT; goto done;}
#endif
#ifndef ENABLE_CDF5
if (model == NC_FORMATX_NC3 && (cmode & NC_64BIT_DATA))
return NC_ENOTBUILT;
if (model.impl == NC_FORMATX_NC3 && (cmode & NC_64BIT_DATA))
{stat = NC_ENOTBUILT; goto done;}
#endif
/* Figure out what dispatcher to use */
if (model == NC_FORMATX_NC4)
switch (model.impl) {
#ifdef USE_HDF5
case NC_FORMATX_NC4:
dispatcher = HDF5_dispatch_table;
#else
return NC_ENOTBUILT;
break;
#endif
else if (model == NC_FORMATX_PNETCDF)
#ifdef USE_PNETCDF
case NC_FORMATX_PNETCDF:
dispatcher = NCP_dispatch_table;
#else
return NC_ENOTBUILT;
break;
#endif
else if (model == NC_FORMATX_NC3)
case NC_FORMATX_NC3:
dispatcher = NC3_dispatch_table;
else {
nullfree(path);
break;
default:
return NC_ENOTNC;
}
/* Create the NC* instance and insert its dispatcher */
stat = new_NC(dispatcher,path,cmode,model,&ncp);
nullfree(path); path = NULL; /* no longer needed */
/* Create the NC* instance and insert its dispatcher and model */
if((stat = new_NC(dispatcher,path,cmode,&model,&ncp))) goto done;
if(stat) return stat;
/* Add to list of known open files and define ext_ncid */
add_to_NCList(ncp);
/* Add to list of known open files and define ext_ncid */
add_to_NCList(ncp);
#ifdef USE_REFCOUNT
/* bump the refcount */
ncp->refcount++;
/* bump the refcount */
ncp->refcount++;
#endif
/* Assume create will fill in remaining ncp fields */
if ((stat = dispatcher->create(ncp->path, cmode, initialsz, basepe, chunksizehintp,
/* Assume create will fill in remaining ncp fields */
if ((stat = dispatcher->create(ncp->path, cmode, initialsz, basepe, chunksizehintp,
parameters, dispatcher, ncp))) {
del_from_NCList(ncp); /* oh well */
free_NC(ncp);
} else {
if(ncidp)*ncidp = ncp->ext_ncid;
}
return stat;
} else {
if(ncidp)*ncidp = ncp->ext_ncid;
}
done:
nullfree(path);
return stat;
}
/**
@ -2192,228 +1961,184 @@ int
NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
int useparallel, void* parameters, int *ncidp)
{
int stat = NC_NOERR;
NC* ncp = NULL;
NC_Dispatch* dispatcher = NULL;
int inmemory = 0;
int diskless = 0;
int mmap = 0;
/* Need pieces of information for now to decide model*/
int model = 0;
int isurl = 0;
int version = 0;
char* path = NULL;
int stat = NC_NOERR;
NC* ncp = NULL;
NC_Dispatch* dispatcher = NULL;
int inmemory = 0;
int diskless = 0;
int mmap = 0;
char* path = NULL;
NCmodel model;
char* newpath = NULL;
TRACE(nc_open);
if(!NC_initialized) {
stat = nc_initialize();
if(stat) return stat;
}
/* Capture the inmemory related flags */
mmap = ((omode & NC_MMAP) == NC_MMAP);
diskless = ((omode & NC_DISKLESS) == NC_DISKLESS);
inmemory = ((omode & NC_INMEMORY) == NC_INMEMORY);
/* NC_INMEMORY and NC_DISKLESS and NC_MMAP are all mutually exclusive */
if(diskless && inmemory) {stat = NC_EDISKLESS; goto done;}
if(diskless && mmap) {stat = NC_EDISKLESS; goto done;}
if(inmemory && mmap) {stat = NC_EINMEMORY; goto done;}
TRACE(nc_open);
if(!NC_initialized) {
stat = nc_initialize();
if(stat) return stat;
}
/* mmap is not allowed for netcdf-4 */
if(mmap && (omode & NC_NETCDF4)) {stat = NC_EINVAL; goto done;}
/* Fix the inmemory related flags */
mmap = ((omode & NC_MMAP) == NC_MMAP);
diskless = ((omode & NC_DISKLESS) == NC_DISKLESS);
inmemory = ((omode & NC_INMEMORY) == NC_INMEMORY);
if(mmap && inmemory) /* cannot have both */
return NC_EINMEMORY;
if(mmap && diskless) /* cannot have both */
return NC_EDISKLESS;
/* Attempt to do file path conversion: note that this will do
nothing if path is a 'file:...' url, so it will need to be
repeated in protocol code: libdap2 and libdap4
/* Attempt to do file path conversion: note that this will do
nothing if path is a 'file:...' url, so it will need to be
repeated in protocol code (e.g. libdap2, libdap4, etc).
*/
{
/* Skip past any leading whitespace in path */
const char* p;
for(p=(char*)path0;*p;p++) {if(*p > ' ') break;}
#ifdef WINPATH
path = NCpathcvt(path0);
/* Need to do path conversion */
path = NCpathcvt(p);
#else
path = nulldup(path0);
path = nulldup(p);
#endif
}
#ifdef USE_REFCOUNT
/* If this path is already open, then bump the refcount and return it */
ncp = find_in_NCList_by_name(path);
if(ncp != NULL) {
/* If this path is already open, then bump the refcount and return it */
ncp = find_in_NCList_by_name(path);
if(ncp != NULL) {
nullfree(path);
ncp->refcount++;
if(ncidp) *ncidp = ncp->ext_ncid;
return NC_NOERR;
}
}
#endif
if(!inmemory) {
char* newpath = NULL;
model = NC_urlmodel(path,omode,&newpath);
isurl = (model != 0);
if(isurl) {
nullfree(path);
path = newpath;
} else
nullfree(newpath);
memset(&model,0,sizeof(model));
/* Infer model implementation and format, possibly by reading the file */
if((stat = NC_infermodel(path,&omode,0,useparallel,parameters,&model,&newpath)))
goto done;
if(newpath) {
nullfree(path);
path = newpath;
}
#ifdef USE_NETCDF4
/* Check for use of user-defined format 0. */
if (omode & NC_UDF0)
{
if (!UDF0_dispatch_table)
return NC_EINVAL;
model = NC_FORMATX_UDF0;
dispatcher = UDF0_dispatch_table;
}
/* Check for use of user-defined format 1. */
if (omode & NC_UDF1)
{
if (!UDF1_dispatch_table)
return NC_EINVAL;
model = NC_FORMATX_UDF1;
dispatcher = UDF1_dispatch_table;
}
#endif /* USE_NETCDF4 */
if(model == 0) {
version = 0;
/* Try to find dataset type */
int flags = omode;
stat = NC_check_file_type(path,flags,useparallel,parameters,&model,&version);
if(stat == NC_NOERR) {
if(model == 0) {
nullfree(path);
return NC_ENOTNC;
}
} else {
/* presumably not a netcdf file */
nullfree(path);
return stat;
}
/* Still no implementation, give up */
if(model.impl == 0) {
#ifdef DEBUG
fprintf(stderr,"implementation == 0\n");
#endif
{stat = NC_ENOTNC; goto done;}
}
if(model == 0) {
fprintf(stderr,"Model == 0\n");
return NC_ENOTNC;
}
/* Suppress unsupported formats */
{
/* Suppress unsupported formats */
{
int hdf5built = 0;
int hdf4built = 0;
int cdf5built = 0;
int udf0built = 0;
int udf1built = 0;
#ifdef USE_NETCDF4
hdf5built = 1;
#ifdef USEHDF4
hdf4built = 1;
#endif
#ifdef USE_HDF4
hdf4built = 1;
#endif
#endif
#ifdef ENABLE_CDF5
cdf5built = 1;
cdf5built = 1;
#endif
if(!hdf5built && model == NC_FORMATX_NC4) {
free(path);
return NC_ENOTBUILT;
}
if(!hdf4built && model == NC_FORMATX_NC4 && version == 4) {
free(path);
return NC_ENOTBUILT;
}
if(!cdf5built && model == NC_FORMATX_NC3 && version == 5) {
free(path);
return NC_ENOTBUILT;
}
}
if(UDF0_dispatch_table != NULL)
udf0built = 1;
if(UDF1_dispatch_table != NULL)
udf1built = 1;
/* Force flag consistency */
if(model == NC_FORMATX_NC4 || model == NC_FORMATX_NC_HDF4 || model == NC_FORMATX_DAP4 ||
model == NC_FORMATX_UDF0 || model == NC_FORMATX_UDF1)
omode |= NC_NETCDF4;
else if(model == NC_FORMATX_DAP2) {
omode &= ~NC_NETCDF4;
omode &= ~NC_64BIT_OFFSET;
} else if(model == NC_FORMATX_NC3) {
omode &= ~NC_NETCDF4; /* must be netcdf-3 (CDF-1, CDF-2, CDF-5) */
if(version == 2) omode |= NC_64BIT_OFFSET;
else if(version == 5) omode |= NC_64BIT_DATA;
} else if(model == NC_FORMATX_PNETCDF) {
omode &= ~NC_NETCDF4; /* must be netcdf-3 (CDF-1, CDF-2, CDF-5) */
if(version == 2) omode |= NC_64BIT_OFFSET;
else if(version == 5) omode |= NC_64BIT_DATA;
}
/* Figure out what dispatcher to use */
if (!dispatcher) {
switch (model) {
#if defined(ENABLE_DAP)
case NC_FORMATX_DAP2:
dispatcher = NCD2_dispatch_table;
break;
if(!hdf5built && model.impl == NC_FORMATX_NC4)
{stat = NC_ENOTBUILT; goto done;}
if(!hdf4built && model.impl == NC_FORMATX_NC_HDF4)
{stat = NC_ENOTBUILT; goto done;}
if(!cdf5built && model.impl == NC_FORMATX_NC3 && model.format == NC_FORMAT_CDF5)
{stat = NC_ENOTBUILT; goto done;}
if(!udf0built && model.impl == NC_FORMATX_UDF0)
{stat = NC_ENOTBUILT; goto done;}
if(!udf1built && model.impl == NC_FORMATX_UDF1)
{stat = NC_ENOTBUILT; goto done;}
}
/* Figure out what dispatcher to use */
if (!dispatcher) {
switch (model.impl) {
#ifdef ENABLE_DAP
case NC_FORMATX_DAP2:
dispatcher = NCD2_dispatch_table;
break;
#endif
#if defined(ENABLE_DAP4)
case NC_FORMATX_DAP4:
dispatcher = NCD4_dispatch_table;
break;
#ifdef ENABLE_DAP4
case NC_FORMATX_DAP4:
dispatcher = NCD4_dispatch_table;
break;
#endif
#if defined(USE_PNETCDF)
case NC_FORMATX_PNETCDF:
dispatcher = NCP_dispatch_table;
break;
#ifdef USE_PNETCDF
case NC_FORMATX_PNETCDF:
dispatcher = NCP_dispatch_table;
break;
#endif
#if defined(USE_HDF5)
case NC_FORMATX_NC4:
dispatcher = HDF5_dispatch_table;
break;
#ifdef USE_HDF5
case NC_FORMATX_NC4:
dispatcher = HDF5_dispatch_table;
break;
#endif
#if defined(USE_HDF4)
case NC_FORMATX_NC_HDF4:
dispatcher = HDF4_dispatch_table;
break;
#ifdef USE_HDF4
case NC_FORMATX_NC_HDF4:
dispatcher = HDF4_dispatch_table;
break;
#endif
#ifdef USE_NETCDF4
case NC_FORMATX_UDF0:
dispatcher = UDF0_dispatch_table;
break;
case NC_FORMATX_UDF1:
dispatcher = UDF1_dispatch_table;
break;
case NC_FORMATX_UDF0:
dispatcher = UDF0_dispatch_table;
break;
case NC_FORMATX_UDF1:
dispatcher = UDF1_dispatch_table;
break;
#endif /* USE_NETCDF4 */
case NC_FORMATX_NC3:
dispatcher = NC3_dispatch_table;
break;
default:
nullfree(path);
return NC_ENOTNC;
}
}
/* If we can't figure out what dispatch table to use, give up. */
if (!dispatcher) {
nullfree(path);
return NC_ENOTNC;
}
/* Create the NC* instance and insert its dispatcher */
stat = new_NC(dispatcher,path,omode,model,&ncp);
nullfree(path); path = NULL; /* no longer need path */
if(stat) return stat;
/* Add to list of known open files */
add_to_NCList(ncp);
case NC_FORMATX_NC3:
dispatcher = NC3_dispatch_table;
break;
default:
nullfree(path);
return NC_ENOTNC;
}
}
/* If we can't figure out what dispatch table to use, give up. */
if (!dispatcher) {stat = NC_ENOTNC; goto done;}
/* Create the NC* instance and insert its dispatcher */
if((stat = new_NC(dispatcher,path,omode,&model,&ncp))) goto done;
/* Add to list of known open files */
add_to_NCList(ncp);
#ifdef USE_REFCOUNT
/* bump the refcount */
ncp->refcount++;
/* bump the refcount */
ncp->refcount++;
#endif
/* Assume open will fill in remaining ncp fields */
stat = dispatcher->open(ncp->path, omode, basepe, chunksizehintp,
parameters, dispatcher, ncp);
if(stat == NC_NOERR) {
if(ncidp) *ncidp = ncp->ext_ncid;
} else {
/* Assume open will fill in remaining ncp fields */
stat = dispatcher->open(ncp->path, omode, basepe, chunksizehintp,
parameters, dispatcher, ncp);
if(stat == NC_NOERR) {
if(ncidp) *ncidp = ncp->ext_ncid;
} else {
del_from_NCList(ncp);
free_NC(ncp);
}
return stat;
free_NC(ncp);
}
done:
nullfree(path);
return stat;
}
/*Provide an internal function for generating pseudo file descriptors
@ -2448,181 +2173,3 @@ nc__pseudofd(void)
}
return pseudofd++;
}
/**
\internal
\ingroup datasets
Provide open, read and close for use when searching for magic numbers
*/
static int
openmagic(struct MagicFile* file)
{
int status = NC_NOERR;
assert((file->inmemory) ? file->parameters != NULL : 1);
if(file->inmemory) {
/* Get its length */
NC_memio* meminfo = (NC_memio*)file->parameters;
file->filelen = (long long)meminfo->size;
goto done;
}
#ifdef USE_PARALLEL
if (file->use_parallel) {
int retval;
MPI_Offset size;
assert(file->parameters);
if((retval = MPI_File_open(((NC_MPI_INFO*)file->parameters)->comm,
(char*)file->path,MPI_MODE_RDONLY,
((NC_MPI_INFO*)file->parameters)->info,
&file->fh)) != MPI_SUCCESS) {
#ifdef MPI_ERR_NO_SUCH_FILE
int errorclass;
MPI_Error_class(retval, &errorclass);
if (errorclass == MPI_ERR_NO_SUCH_FILE)
#ifdef NC_ENOENT
status = NC_ENOENT;
#else
status = errno;
#endif
else
#endif
status = NC_EPARINIT;
goto done;
}
/* Get its length */
if((retval=MPI_File_get_size(file->fh, &size)) != MPI_SUCCESS)
{status = NC_EPARINIT; goto done;}
file->filelen = (long long)size;
goto done;
}
#endif /* USE_PARALLEL */
{
if(file->path == NULL || strlen(file->path)==0)
{status = NC_EINVAL; goto done;}
#ifdef _MSC_VER
file->fp = fopen(file->path, "rb");
#else
file->fp = fopen(file->path, "r");
#endif
if(file->fp == NULL)
{status = errno; goto done;}
/* Get its length */
{
int fd = fileno(file->fp);
#ifdef _MSC_VER
__int64 len64 = _filelengthi64(fd);
if(len64 < 0)
{status = errno; goto done;}
file->filelen = (long long)len64;
#else
off_t size;
size = lseek(fd, 0, SEEK_END);
if(size == -1)
{status = errno; goto done;}
file->filelen = (long long)size;
#endif
rewind(file->fp);
}
goto done;
}
done:
return status;
}
static int
readmagic(struct MagicFile* file, long pos, char* magic)
{
int status = NC_NOERR;
memset(magic,0,MAGIC_NUMBER_LEN);
if(file->inmemory) {
char* mempos;
NC_memio* meminfo = (NC_memio*)file->parameters;
if((pos + MAGIC_NUMBER_LEN) > meminfo->size)
{status = NC_EINMEMORY; goto done;}
mempos = ((char*)meminfo->memory) + pos;
memcpy((void*)magic,mempos,MAGIC_NUMBER_LEN);
#ifdef DEBUG
printmagic("XXX: readmagic",magic,file);
#endif
goto done;
}
#ifdef USE_PARALLEL
if (file->use_parallel) {
MPI_Status mstatus;
int retval;
if((retval = MPI_File_read_at_all(file->fh, pos, magic,
MAGIC_NUMBER_LEN, MPI_CHAR, &mstatus)) != MPI_SUCCESS)
{status = NC_EPARINIT; goto done;}
goto done;
}
#endif /* USE_PARALLEL */
{
int count;
int i = fseek(file->fp,pos,SEEK_SET);
if(i < 0)
{status = errno; goto done;}
for(i=0;i<MAGIC_NUMBER_LEN;) {/* make sure to read proper # of bytes */
count=fread(&magic[i],1,(size_t)(MAGIC_NUMBER_LEN-i),file->fp);
if(count == 0 || ferror(file->fp))
{status = errno; goto done;}
i += count;
}
goto done;
}
done:
if(file && file->fp) clearerr(file->fp);
return status;
}
/**
* Close the file opened to check for magic number.
*
* @param file pointer to the MagicFile struct for this open file.
* @returns NC_NOERR for success
* @returns NC_EPARINIT if there was a problem closing file with MPI
* (parallel builds only).
* @author Dennis Heimbigner
*/
static int
closemagic(struct MagicFile* file)
{
int status = NC_NOERR;
if(file->inmemory) goto done; /* noop*/
#ifdef USE_PARALLEL
if (file->use_parallel) {
int retval;
if((retval = MPI_File_close(&file->fh)) != MPI_SUCCESS)
{status = NC_EPARINIT; goto done;}
goto done;
}
#endif
{
if(file->fp) fclose(file->fp);
goto done;
}
done:
return status;
}
#ifdef DEBUG
static void
printmagic(const char* tag, char* magic, struct MagicFile* f)
{
int i;
fprintf(stderr,"%s: inmem=%d ispar=%d magic=",tag,f->inmemory,f->use_parallel);
for(i=0;i<MAGIC_NUMBER_LEN;i++) {
unsigned int c = (unsigned int)magic[i];
c = c & 0x000000FF;
if(c == '\n')
fprintf(stderr," 0x%0x/'\\n'",c);
else if(c == '\r')
fprintf(stderr," 0x%0x/'\\r'",c);
else if(c < ' ')
fprintf(stderr," 0x%0x/'?'",c);
else
fprintf(stderr," 0x%0x/'%c'",c,c);
}
fprintf(stderr,"\n");
fflush(stderr);
}
#endif

350
libdispatch/dhttp.c Normal file

@ -0,0 +1,350 @@
/**
* @file
*
* Read a range of data from a remote dataset.
*
* Copyright 2018 University Corporation for Atmospheric
* Research/Unidata. See COPYRIGHT file for more info.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#define CURL_DISABLE_TYPECHECK 1
#include <curl/curl.h>
#include "netcdf.h"
#include "nclog.h"
#include "ncbytes.h"
#include "nclist.h"
#include "nchttp.h"
#undef TRACE
#define CURLERR(e) (e)
/* Mnemonics */
#define GETCMD 0
#define HEADCMD 1
/* Forward */
static int setupconn(CURL* curl, const char* objecturl, NCbytes* buf);
static int execute(CURL* curl, int headcmd, long* httpcodep);
static int headerson(CURL* curl, NClist* list);
static void headersoff(CURL* curl);
#ifdef TRACE
static void
dbgflush() {
fflush(stderr);
fflush(stdout);
}
static void
Trace(const char* fcn)
{
fprintf(stdout,"xxx: %s\n",fcn);
dbgflush();
}
#else
#define dbgflush()
#define Trace(fcn)
#endif /*TRACE*/
/**************************************************/
/**
@param objecturl url we propose to access
@param curlp curl handle stored here if non-NULL
@param filelenp store length of the file here if non-NULL
*/
int
nc_http_open(const char* objecturl, void** curlp, long long* filelenp)
{
int stat = NC_NOERR;
CURL* curl = NULL;
int i;
NClist* list = NULL;
Trace("open");
/* initialize curl*/
curl = curl_easy_init();
if (curl == NULL) stat = NC_ECURL;
if(curlp && curl) *curlp = (void*)curl;
if(filelenp) {
*filelenp = -1;
/* Attempt to get the file length using HEAD */
list = nclistnew();
if((stat = setupconn(curl,objecturl,NULL))) goto done;
if((stat = headerson(curl,list))) goto done;
if((stat = execute(curl,HEADCMD,NULL))) goto done;
headersoff(curl);
for(i=0;i<nclistlength(list);i+=2) {
char* s = nclistget(list,i);
if(strcasecmp(s,"content-length")==0) {
s = nclistget(list,i+1);
sscanf(s,"%lld",filelenp);
break;
}
/* Also check for the Accept-Ranges header */
if(strcasecmp(s,"accept-ranges")==0) {
s = nclistget(list,i+1);
if(strcasecmp(s,"bytes")!=0) /* oops! */
{stat = NC_EACCESS; goto done;}
}
}
}
done:
nclistfreeall(list);
dbgflush();
return stat;
}
int
nc_http_close(void* curl0)
{
int stat = NC_NOERR;
CURL* curl = curl0;
Trace("close");
if(curl != NULL)
(void)curl_easy_cleanup(curl);
dbgflush();
return stat;
}
/**
Assume URL etc has already been set.
@param curl curl handle
@param start starting offset
@param count number of bytes to read
@param buf store read data here -- caller must allocate and free
*/
int
nc_http_read(CURL* curl, const char* objecturl, fileoffset_t start, fileoffset_t count, NCbytes* buf)
{
int stat = NC_NOERR;
char range[64];
long httpcode = 200;
CURLcode cstat = CURLE_OK;
Trace("read");
if(count == 0)
goto done; /* do not attempt to read */
if((stat = setupconn(curl,objecturl,buf)))
goto fail;
/* Set to read byte range */
snprintf(range,sizeof(range),"%ld-%ld",(long)start,(long)((start+count)-1));
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_RANGE, range));
if(cstat != CURLE_OK)
{stat = NC_ECURL; goto done;}
if((stat = execute(curl,GETCMD,&httpcode)))
goto done;
done:
dbgflush();
return stat;
fail:
stat = NC_ECURL;
goto done;
}
static size_t
WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
{
NCbytes* buf = data;
size_t realsize = size * nmemb;
Trace("WriteMemoryCallback");
if(realsize == 0)
nclog(NCLOGWARN,"WriteMemoryCallback: zero sized chunk");
ncbytesappendn(buf, ptr, realsize);
return realsize;
}
static void
trim(char* s)
{
size_t l = strlen(s);
char* p = s;
char* q = s + l;
if(l == 0) return;
q--; /* point to last char of string */
/* Walk backward to first non-whitespace */
for(;q > p;q--) {
if(*q > ' ') break; /* found last non-whitespace */
}
/* invariant: p == q || *q > ' ' */
if(p == q) /* string is all whitespace */
{*p = '\0';}
else {/* *q is last non-whitespace */
q++; /* point to actual whitespace */
*q = '\0';
}
/* Ok, skip past leading whitespace */
for(p=s;*p;p++) {if(*p > ' ') break;}
/* invariant: *p == '\0' || *p > ' ' */
if(*p == 0) return; /* no leading whitespace */
/* Ok, overwrite any leading whitespace */
for(q=s;*p;) {*q++ = *p++;}
*q = '\0';
return;
}
static size_t
HeaderCallback(char *buffer, size_t size, size_t nitems, void *data)
{
NClist* list = data;
size_t realsize = size * nitems;
char* name = NULL;
char* value = NULL;
char* p = NULL;
size_t i;
int havecolon;
Trace("HeaderCallback");
if(realsize == 0)
nclog(NCLOGWARN,"HeaderCallback: zero sized chunk");
i = 0;
/* Look for colon separator */
for(p=buffer;(i < realsize) && (*p != ':');p++,i++);
havecolon = (i < realsize);
if(i == 0)
nclog(NCLOGWARN,"HeaderCallback: malformed header: %s",buffer);
name = malloc(i+1);
memcpy(name,buffer,i);
name[i] = '\0';
value = NULL;
if(havecolon) {
size_t vlen = (realsize - i);
value = malloc(vlen+1);
p++; /* skip colon */
memcpy(value,p,vlen);
value[vlen] = '\0';
trim(value);
}
nclistpush(list,name);
name = NULL;
if(value == NULL) value = strdup("");
nclistpush(list,value);
value = NULL;
return realsize;
}
static int
setupconn(CURL* curl, const char* objecturl, NCbytes* buf)
{
int stat = NC_NOERR;
CURLcode cstat = CURLE_OK;
if(objecturl != NULL) {
/* Set the URL */
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_URL, (void*)objecturl));
if (cstat != CURLE_OK) goto fail;
}
/* Set options */
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_TIMEOUT, 100)); /* 30sec timeout*/
if (cstat != CURLE_OK) goto fail;
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 100));
if (cstat != CURLE_OK) goto fail;
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1));
if (cstat != CURLE_OK) goto fail;
if(buf != NULL) {
/* send all data to this function */
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback));
if (cstat != CURLE_OK) goto fail;
/* Set argument for WriteMemoryCallback */
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)buf));
if (cstat != CURLE_OK) goto fail;
} else {/* turn off data capture */
(void)CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, NULL));
(void)CURLERR(curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL));
}
/* Turn off header capture */
headersoff(curl);
done:
return stat;
fail:
stat = NC_ECURL;
goto done;
}
static int
execute(CURL* curl, int headcmd, long* httpcodep)
{
int stat = NC_NOERR;
CURLcode cstat = CURLE_OK;
long httpcode = 0;
if(headcmd) {
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_NOBODY, 1L));
if(cstat != CURLE_OK) goto fail;
}
cstat = CURLERR(curl_easy_perform(curl));
if(cstat != CURLE_OK) goto fail;
cstat = CURLERR(curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE,&httpcode));
if(cstat != CURLE_OK) httpcode = 0;
if(httpcodep) *httpcodep = httpcode;
if(headcmd) {
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_NOBODY, 0L));
if(cstat != CURLE_OK) goto fail;
}
done:
return stat;
fail:
stat = NC_ECURL;
goto done;
}
static int
headerson(CURL* curl, NClist* list)
{
int stat = NC_NOERR;
CURLcode cstat = CURLE_OK;
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, HeaderCallback));
if(cstat != CURLE_OK) goto fail;
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void*)list));
if (cstat != CURLE_OK) goto fail;
done:
return stat;
fail:
stat = NC_ECURL;
goto done;
}
static void
headersoff(CURL* curl)
{
(void)CURLERR(curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, NULL));
(void)CURLERR(curl_easy_setopt(curl, CURLOPT_HEADERDATA, NULL));
}
#ifdef IGNORE
static void
reset(CURL* curl)
{
(void)curl_easy_reset(curl);
}
#endif

1012
libdispatch/dinfermodel.c Normal file

File diff suppressed because it is too large Load Diff

17
libdispatch/drc.c

@ -455,6 +455,23 @@ done:
return ret;
}
/* Obtain the count of number of triples */
size_t
NC_rcfile_length(NCRCinfo* info)
{
return nclistlength(info->triples);
}
/* Obtain the ith triple; return NULL if out of range */
NCTriple*
NC_rcfile_ith(NCRCinfo* info, size_t i)
{
if(i >= nclistlength(info->triples))
return NULL;
return (NCTriple*)nclistget(info->triples,i);
}
#ifdef D4DEBUG
static void
storedump(char* msg, NClist* triples)

1
libdispatch/dreadonly.c

@ -8,7 +8,6 @@
* @author Ed Hartnett
*/
#include "nc.h"
#include "ncdispatch.h"
/**

2
libdispatch/dstring.c

@ -10,7 +10,7 @@
#include <string.h>
#include <ctype.h>
#include <assert.h>
#include "nc.h"
#include "ncdispatch.h"
#include "rnd.h"
#include "ncutf8.h"

276
libdispatch/durlmodel.c Normal file

@ -0,0 +1,276 @@
/**
* @file
*
* URL Model Management.
*
* These functions support inferring the format X implementation for urls.
* It looks at various fragment (#...) pairs.
*
* Copyright 2018 University Corporation for Atmospheric
* Research/Unidata. See COPYRIGHT file for more info.
*/
#include "config.h"
#include <stdlib.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "ncdispatch.h"
#include "ncwinpath.h"
#include "ncurlmodel.h"
/*
Define a table of legal mode string values.
Note that only cases that can currently
take URLs are included.
*/
static struct LEGALMODES {
const char* tag;
const int format; /* NC_FORMAT_XXX value */
const int implementation; /* NC_FORMATX_XXX value */
const int iosp; /* NC_IOSP_XXX value */
} legalmodes[] = {
/* Format X Implementation */
{"netcdf-3",NC_FORMAT_CLASSIC,NC_FORMATX_NC3,0},
{"classic",NC_FORMAT_CLASSIC,NC_FORMATX_NC3,0},
{"netcdf-4",NC_FORMAT_NETCDF4,NC_FORMATX_NC4,0},
{"enhanced",NC_FORMAT_NETCDF4,NC_FORMATX_NC4,0},
{"dap2",NC_FORMAT_CLASSIC,NC_FORMATX_DAP2,0},
{"dap4",NC_FORMAT_NETCDF4,NC_FORMATX_DAP4,0},
/* IO Handlers */
{"s3",0,0,NC_IOSP_S3},
{"zarr",0,0,NC_IOSP_ZARR},
{NULL,0,0},
};
/* Define the known URL protocols and their interpretation */
static struct NCPROTOCOLLIST {
char* protocol;
char* substitute;
int implementation;
} ncprotolist[] = {
{"http",NULL,0},
{"https",NULL,0},
{"file",NULL,0},
{"dods","http",NC_FORMATX_DAP2},
{"dap4","http",NC_FORMATX_DAP4},
{NULL,NULL,0} /* Terminate search */
};
/* Parse a mode string at the commas nul terminate each tag */
static int
parseurlmode(const char* modestr0, char** listp)
{
int stat = NC_NOERR;
char* modestr = NULL;
char* p = NULL;
char* endp = NULL;
/* Make modifiable copy */
if((modestr=strdup(modestr0)) == NULL)
{stat=NC_ENOMEM; goto done;}
/* Split modestr at the commas or EOL */
p = modestr;
for(;;) {
endp = strchr(p,',');
if(endp == NULL) break;
/* Null terminate each comma-separated string */
*endp++ = '\0';
p = endp;
}
if(listp) *listp = modestr;
modestr = NULL;
done:
if(stat) {nullfree(modestr);}
return stat;
}
/* Parse url fragment for format etc. */
static int
url_getmodel(const char* modestr, NCmode* model)
{
int stat = NC_NOERR;
char* args = NULL;
char* p = NULL;
model->format = 0;
model->implementation = 0;
if((stat=parseurlmode(modestr,&args))) goto done;
p = args;
for(;*p;) {
struct LEGALMODES* legal = legalmodes;
while(legal->tag) {
if(strcmp(legal->tag,p)==0) {
if(model->format != 0 && legal->format != 0)
{stat = NC_EINVAL; goto done;}
if(model->implementation != 0 && legal->implementation != 0)
{stat = NC_EINVAL; goto done;}
if(model->iosp != 0 && legal->iosp != 0)
{stat = NC_EINVAL; goto done;}
if(legal->format != 0) model->format = legal->format;
if(legal->implementation != 0)
model->implementation = legal->implementation;
if(legal->iosp != 0) model->iosp = legal->iosp;
}
}
}
done:
nullfree(args);
return stat;
}
/**************************************************/
/**
* Provide a hidden interface to allow utilities
* to check if a given path name is really an ncdap3 url.
* If no, put null in basenamep, else put basename of the url
* minus any extension into basenamep; caller frees.
* Return 1 if it looks like a url, 0 otherwise.
*/
int
nc__testurl(const char* path, char** basenamep)
{
NCURI* uri;
int ok = 0;
if(ncuriparse(path,&uri) == NCU_OK) {
char* slash = (uri->path == NULL ? NULL : strrchr(uri->path, '/'));
char* dot;
if(slash == NULL) slash = (char*)path; else slash++;
slash = nulldup(slash);
if(slash == NULL)
dot = NULL;
else
dot = strrchr(slash, '.');
if(dot != NULL && dot != slash) *dot = '\0';
if(basenamep)
*basenamep=slash;
else if(slash)
free(slash);
ncurifree(uri);
ok = 1;
}
return ok;
}
/*
Fill in the model fields to degree possible.
Assumes that the path is known to be a url
*/
int
NC_urlmodel(const char* path, int cmode, char** newurl, NCmode* model)
{
int stat = NC_NOERR;
int found = 0;
struct NCPROTOCOLLIST* protolist;
NCURI* url = NULL;
const char** fragp = NULL;
if(path == NULL) return 0;
/* Parse the url */
if(ncuriparse(path,&url) != NCU_OK)
return NC_EINVAL; /* Not parseable as url */
/* Look up the protocol */
for(found=0,protolist=ncprotolist;protolist->protocol;protolist++) {
if(strcmp(url->protocol,protolist->protocol) == 0) {
found = 1;
break;
}
}
if(found) {
model->implementation = protolist->implementation;
/* Substitute the protocol in any case */
if(protolist->substitute) ncurisetprotocol(url,protolist->substitute);
} else
{stat = NC_EINVAL; goto done;} /* Again, does not look like a url */
/* Iterate over the url fragment parameters */
for(fragp=ncurifragmentparams(url);fragp && *fragp;fragp+=2) {
const char* name = fragp[0];
const char* value = fragp[1];
if(strcmp(name,"protocol")==0)
name = value;
if(strcasecmp(name,"dap2") == 0) {
model->format = NC_FORMAT_NC3;
model->implementation = NC_FORMATX_DAP2;
/* No need to set iosp field */
} else if(strcasecmp(name,"dap4") == 0) {
model->format = NC_FORMAT_NETCDF4;
model->implementation = NC_FORMATX_DAP4;
/* No need to set iosp field */
} else if(strcmp(name,"mode")==0) {
if((stat = url_getmodel(value,model))) goto done;
}
}
if(model->implementation == 0) {/* Last resort: use the cmode */
/* If mode specifies netcdf-4, then this is assumed to be dap4 */
if(cmode & NC_NETCDF4) {
model->implementation = NC_FORMATX_DAP4;
} else {/* Default is DAP2 */
model->implementation = NC_FORMATX_DAP2;
}
}
if(model->implementation == 0) goto done; /* could not interpret */
switch (model->implementation) {
case NC_FORMATX_NC3:
model->format = NC_FORMAT_NC3;
break;
case NC_FORMATX_NC4:
model->format = NC_FORMAT_NETCDF4;
break;
case NC_FORMATX_DAP2:
model->format = NC_FORMAT_NC3;
break;
case NC_FORMATX_DAP4:
model->format = NC_FORMAT_NETCDF4;
break;
case NC_FORMATX_ZARR:
model->format = NC_FORMAT_NETCDF4;
break;
default:
stat = NC_EINVAL;
goto done;
}
done:
if(newurl)
*newurl = ncuribuild(url,NULL,NULL,NCURIALL);
if(url) ncurifree(url);
return stat;
}
/* return 1 if path looks like a url; 0 otherwise */
int
NC_testurl(const char* path)
{
int isurl = 0;
NCURI* tmpurl = NULL;
if(path == NULL) return 0;
/* Ok, try to parse as a url */
if(ncuriparse(path,&tmpurl)==NCU_OK) {
/* Do some extra testing to make sure this really is a url */
/* Look for a known/accepted protocol */
struct NCPROTOCOLLIST* protolist;
for(protolist=ncprotolist;protolist->protocol;protolist++) {
if(strcmp(tmpurl->protocol,protolist->protocol) == 0) {
isurl=1;
break;
}
}
ncurifree(tmpurl);
return isurl;
}
return 0;
}

124
libdispatch/heap

@ -0,0 +1,124 @@
/* return first IOSP or NULL if none */
int
NC_urliosp(NCURI* u)
{
int stat = NC_NOERR;
const char* modestr = NULL;
char* args = NULL;
char* p = NULL;
struct LEGALMODES* legal = legalmodes;
modestr = ncurilookup(u,"mode");
if(modestr == NULL) return 0;
if((stat=parseurlmode(modestr,&args))) return 0;
p = args;
for(;;) {
if(*p == '\0') break;
for(;legal->tag;legal++) {
if(strcmp(legal->tag,p)==0 && legal->iosp != 0)
return legal->iosp;
}
p += strlen(p)+1;
}
return 0;
}
/* Parse url fragment mode tag for model info */
static int
url_getmode(const char* modestr, NCmodel* model)
{
int stat = NC_NOERR;
NClist* modelist = nclistnew();
int i;
if((stat=parseurlmode(modestr,modelist))) goto done;
for(i=0;i<nclistlength(modelist);i++) {
char* arg = nclistget(modelist,i);
if((stat = processmodearg(arg,model))) goto done;
}
done:
nclistfreeall(modelist);
return check(stat);
}
/*
Fill in the model fields to degree possible using path only */
static int
NC_pathinfer(const char* path, NCmodel* model, char** newpathp, NCURI** urip)
{
int stat = NC_NOERR;
NCURI* uri = NULL;
int isurl = 0;
if(path == NULL || strlen(path) == 0) goto done;
/* Defaults */
if(newpathp) *newpathp = NULL;
if(urip) *urip = NULL;
/* Parse the path and process */
if(ncuriparse(path,&uri) == NCU_OK) {
if((stat = processuri(uri,model))) goto done;
isurl = 1;
} else {
conflictset(MIO,model->iosp,NC_IOSP_FILE);
}
done:
if(stat) {
ncurifree(uri);
} else if(isurl) {
if(newpathp)
*newpathp = ncuribuild(uri,NULL,NULL,NCURIALL);
if(urip) *urip = uri;
}
return check(stat);
}
#if 0
/* Do not set the implementation yet */
switch (model->format) {
case NC_FORMAT_CLASSIC:
model->version = 1;
break;
case NC_FORMAT_64BIT_OFFSET:
model->version = 2;
break;
case NC_FORMAT_64BIT_DATA:
model->version = 5;
break;
case NC_FORMAT_NETCDF4:
case NC_FORMAT_NETCDF4_CLASSIC:
model->version = 5;
break;
default: break;
}
if(model->impl == 0) {stat = NC_ENOTNC; goto done;} /* could not interpret */
/* Try to infer format from implementation */
if(model->format == 0) {
switch (model->impl) {
case NC_FORMATX_NC3:
model->format = NC_FORMAT_NC3;
break;
case NC_FORMATX_NC4:
model->format = NC_FORMAT_NETCDF4;
break;
case NC_FORMATX_DAP2:
model->format = NC_FORMAT_NC3;
break;
case NC_FORMATX_DAP4:
model->format = NC_FORMAT_NETCDF4;
break;
case NC_FORMATX_ZARR:
model->format = NC_FORMAT_NETCDF4;
break;
default:
break;
}
}

9
libdispatch/nc.c

@ -15,7 +15,6 @@
#include <unistd.h>
#endif
#include "nc.h"
#include "ncdispatch.h"
int ncdebug = 0;
@ -45,6 +44,8 @@ free_NC(NC *ncp)
return;
if(ncp->path)
free(ncp->path);
if(ncp->model)
free(ncp->model);
/* We assume caller has already cleaned up ncp->dispatchdata */
#if _CRAYMPP && defined(LOCKNUMREC)
shfree(ncp);
@ -54,14 +55,16 @@ free_NC(NC *ncp)
}
int
new_NC(NC_Dispatch* dispatcher, const char* path, int mode, int model, NC** ncpp)
new_NC(NC_Dispatch* dispatcher, const char* path, int mode, NCmodel* model, NC** ncpp)
{
NC *ncp = (NC*)calloc(1,sizeof(NC));
if(ncp == NULL) return NC_ENOMEM;
ncp->dispatch = dispatcher;
ncp->path = nulldup(path);
ncp->mode = mode;
ncp->model = model;
if((ncp->model = malloc(sizeof(NCmodel)))==NULL)
return NC_ENOMEM;
*ncp->model = *model; /* Make a copy */
if(ncp->path == NULL) { /* fail */
free_NC(ncp);
return NC_ENOMEM;

24
libdispatch/nchashmap.c

@ -11,7 +11,7 @@ See LICENSE.txt for license information.
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include "nc.h"
#include "ncdispatch.h"
#include "nchashmap.h"
#include "nc3internal.h"
@ -44,9 +44,9 @@ extern unsigned int NC_crc32(unsigned int crc, const unsigned char* buf, unsigne
#endif
#ifdef DEBUGTRACE
#define TRACE(x) {fprintf(stderr,"NC_hashmap: %s\n",x); fflush(stderr);}
#define Trace(x) {fprintf(stderr,"NC_hashmap: %s\n",x); fflush(stderr);}
#else
#define TRACE(x)
#define Trace(x)
#endif
#define SEED 37
@ -88,7 +88,7 @@ rehash(NC_hashmap* hm)
#endif
NC_hentry* oldtable = hm->table;
TRACE("rehash");
Trace("rehash");
hm->alloc = findPrimeGreaterThan(alloc<<1);
hm->table = (NC_hentry*)calloc(sizeof(NC_hentry), hm->alloc);
@ -119,7 +119,7 @@ locate(NC_hashmap* hash, unsigned int hashkey, const char* key, size_t keysize,
int deletefound = 0;
size_t deletedindex = 0; /* first deleted entry encountered */
NC_hentry* entry;
TRACE("locate");
Trace("locate");
/* Compute starting point */
index = (size_t)(hashkey % hash->alloc);
@ -167,7 +167,7 @@ NC_hashmapnew(size_t startsize)
{
NC_hashmap* hm = NULL;
TRACE("NC_hashmapnew");
Trace("NC_hashmapnew");
hm = (NC_hashmap*)malloc(sizeof(NC_hashmap));
@ -190,7 +190,7 @@ NC_hashmapadd(NC_hashmap* hash, uintptr_t data, const char* key, size_t keysize)
NC_hentry* entry;
unsigned int hashkey;
TRACE("NC_hashmapadd");
Trace("NC_hashmapadd");
if(key == NULL || keysize == 0)
return 0;
@ -233,7 +233,7 @@ NC_hashmapremove(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t* d
size_t index;
NC_hentry* h;
TRACE("NC_hashmapremove");
Trace("NC_hashmapremove");
if(key == NULL || keysize == 0)
return 0;
@ -259,7 +259,7 @@ NC_hashmapget(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t* data
{
unsigned int hashkey;
TRACE("NC_hashmapget");
Trace("NC_hashmapget");
if(key == NULL || keysize == 0)
return 0;
@ -289,7 +289,7 @@ NC_hashmapsetdata(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t n
NC_hentry* h;
unsigned int hashkey;
TRACE("NC_hashmapsetdata");
Trace("NC_hashmapsetdata");
if(key == NULL || keysize == 0)
return 0;
@ -307,14 +307,14 @@ NC_hashmapsetdata(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t n
size_t
NC_hashmapcount(NC_hashmap* hash)
{
TRACE("NC_hashmapcount");
Trace("NC_hashmapcount");
return hash->active;
}
int
NC_hashmapfree(NC_hashmap* hash)
{
TRACE("NC_hashmapfree");
Trace("NC_hashmapfree");
if(hash) {
int i;
#ifdef DEBUG

4
libdispatch/nclistmgr.c

@ -7,7 +7,7 @@
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "nc.h"
#include "ncdispatch.h"
#define ID_SHIFT (16)
#define NCFILELISTLENGTH 0x10000
@ -90,7 +90,7 @@ find_in_NCList(int ext_ncid)
f = nc_filelist[ncid];
/* for classic files, ext_ncid must be a multiple of (1<<ID_SHIFT) */
if (f != NULL && f->model == NC_FORMATX_NC3 && (ext_ncid % (1<<ID_SHIFT)))
if (f != NULL && f->model->impl == NC_FORMATX_NC3 && (ext_ncid % (1<<ID_SHIFT)))
return NULL;
return f;

14
libdispatch/ncuri.c

@ -641,6 +641,20 @@ ncuriquerylookup(NCURI* uri, const char* key)
return value;
}
/* Obtain the complete list of fragment pairs in envv format */
const char**
ncurifragmentparams(NCURI* uri)
{
return (const char**)uri->fraglist;
}
/* Obtain the complete list of query pairs in envv format */
const char**
ncuriqueryparams(NCURI* uri)
{
return (const char**)uri->querylist;
}
#if 0
int
ncuriremoveparam(NCURI* uri, const char* key)

4
libhdf5/CMakeLists.txt

@ -10,6 +10,10 @@ SET(libnchdf5_SOURCES nc4hdf.c nc4info.c hdf5file.c hdf5attr.c
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c
hdf5var.c nc4mem.c nc4memcb.c hdf5cache.c hdf5dispatch.c)
IF(ENABLE_HTTP)
SET(libnchdf5_SOURCES ${libnchdf5_SOURCES} H5FDhttp.c)
ENDIF()
# Build the HDF4 dispatch layer as a library that will be included in
# the netCDF library.
add_library(netcdfhdf5 OBJECT ${libnchdf5_SOURCES})

851
libhdf5/H5FDhttp.c Normal file

@ -0,0 +1,851 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* ********************************************************************/
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* Programmer: Dennis Heimbigner dmh@ucar.edu
*
* Purpose: Access remote datasets using byte range requests.
* Derived from the HDF5 H5FDstdio.c file.
*
* NOTE: This driver is not as well tested as the standard SEC2 driver
* and is not intended for production use!
*/
#include "config.h"
#include <assert.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <hdf5.h>
#include <curl/curl.h>
#ifdef H5_HAVE_FLOCK
/* Needed for lock type definitions (e.g., LOCK_EX) */
#include <sys/file.h>
#endif /* H5_HAVE_FLOCK */
#ifdef H5_HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef H5_HAVE_WIN32_API
/* The following two defines must be before any windows headers are included */
#define WIN32_LEAN_AND_MEAN /* Exclude rarely-used stuff from Windows headers */
#define NOGDI /* Exclude Graphic Display Interface macros */
#include <windows.h>
#include <io.h>
#endif /* H5_HAVE_WIN32_API */
#include "netcdf.h"
#include "ncbytes.h"
#include "nchttp.h"
#include "H5FDhttp.h"
typedef off_t file_offset_t;
/* The driver identification number, initialized at runtime */
static hid_t H5FD_HTTP_g = 0;
/* File operations */
typedef enum {
H5FD_HTTP_OP_UNKNOWN=0,
H5FD_HTTP_OP_READ=1,
H5FD_HTTP_OP_WRITE=2,
H5FD_HTTP_OP_SEEK=3
} H5FD_http_file_op;
/* The description of a file belonging to this driver. The 'eoa' and 'eof'
* determine the amount of hdf5 address space in use and the high-water mark
* of the file (the current size of the underlying Unix file). The 'pos'
* value is used to eliminate file position updates when they would be a
* no-op. Unfortunately we've found systems that use separate file position
* indicators for reading and writing so the lseek can only be eliminated if
* the current operation is the same as the previous operation. When opening
* a file the 'eof' will be set to the current file size, 'eoa' will be set
* to zero, 'pos' will be set to H5F_ADDR_UNDEF (as it is when an error
* occurs), and 'op' will be set to H5F_OP_UNKNOWN.
*/
typedef struct H5FD_http_t {
H5FD_t pub; /* public stuff, must be first */
haddr_t eoa; /* end of allocated region */
haddr_t eof; /* end of file; current file size */
haddr_t pos; /* current file I/O position */
unsigned write_access; /* Flag to indicate the file was opened with write access */
H5FD_http_file_op op; /* last operation */
CURL* curl; /* Curl handle */
char* url; /* The URL (minus any fragment) for the dataset */
} H5FD_http_t;
/* These macros check for overflow of various quantities. These macros
* assume that file_offset_t is signed and haddr_t and size_t are unsigned.
*
* ADDR_OVERFLOW: Checks whether a file address of type `haddr_t'
* is too large to be represented by the second argument
* of the file seek function.
*
* SIZE_OVERFLOW: Checks whether a buffer size of type `hsize_t' is too
* large to be represented by the `size_t' type.
*
* REGION_OVERFLOW: Checks whether an address and size pair describe data
* which can be addressed entirely by the second
* argument of the file seek function.
*/
/* adding for windows NT filesystem support. */
#define MAXADDR (((haddr_t)1<<(8*sizeof(file_offset_t)-1))-1)
#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR))
#define SIZE_OVERFLOW(Z) ((Z) & ~(hsize_t)MAXADDR)
#define REGION_OVERFLOW(A,Z) (ADDR_OVERFLOW(A) || SIZE_OVERFLOW(Z) || \
HADDR_UNDEF==(A)+(Z) || (file_offset_t)((A)+(Z))<(file_offset_t)(A))
/* Prototypes */
static herr_t H5FD_http_term(void);
static H5FD_t *H5FD_http_open(const char *name, unsigned flags,
hid_t fapl_id, haddr_t maxaddr);
static herr_t H5FD_http_close(H5FD_t *lf);
static int H5FD_http_cmp(const H5FD_t *_f1, const H5FD_t *_f2);
static herr_t H5FD_http_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_http_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size);
static haddr_t H5FD_http_get_eoa(const H5FD_t *_file, H5FD_mem_t type);
static herr_t H5FD_http_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr);
static haddr_t H5FD_http_get_eof(const H5FD_t *_file, H5FD_mem_t type);
static herr_t H5FD_http_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle);
static herr_t H5FD_http_read(H5FD_t *lf, H5FD_mem_t type, hid_t fapl_id, haddr_t addr,
size_t size, void *buf);
static herr_t H5FD_http_write(H5FD_t *lf, H5FD_mem_t type, hid_t fapl_id, haddr_t addr,
size_t size, const void *buf);
static herr_t H5FD_http_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
static herr_t H5FD_http_lock(H5FD_t *_file, hbool_t rw);
static herr_t H5FD_http_unlock(H5FD_t *_file);
/* Beware, not same as H5FD_HTTP_g */
static const H5FD_class_t H5FD_http_g = {
"http", /* name */
MAXADDR, /* maxaddr */
H5F_CLOSE_WEAK, /* fc_degree */
H5FD_http_term, /* terminate */
NULL, /* sb_size */
NULL, /* sb_encode */
NULL, /* sb_decode */
0, /* fapl_size */
NULL, /* fapl_get */
NULL, /* fapl_copy */
NULL, /* fapl_free */
0, /* dxpl_size */
NULL, /* dxpl_copy */
NULL, /* dxpl_free */
H5FD_http_open, /* open */
H5FD_http_close, /* close */
H5FD_http_cmp, /* cmp */
H5FD_http_query, /* query */
NULL, /* get_type_map */
H5FD_http_alloc, /* alloc */
NULL, /* free */
H5FD_http_get_eoa, /* get_eoa */
H5FD_http_set_eoa, /* set_eoa */
H5FD_http_get_eof, /* get_eof */
H5FD_http_get_handle, /* get_handle */
H5FD_http_read, /* read */
H5FD_http_write, /* write */
H5FD_http_flush, /* flush */
NULL, /* truncate */
H5FD_http_lock, /* lock */
H5FD_http_unlock, /* unlock */
H5FD_FLMAP_DICHOTOMY /* fl_map */
};
/*-------------------------------------------------------------------------
* Function: H5FD_http_init
*
* Purpose: Initialize this driver by registering the driver with the
* library.
*
* Return: Success: The driver ID for the driver.
*
* Failure: Negative.
*
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
*-------------------------------------------------------------------------
*/
EXTERNL hid_t
H5FD_http_init(void)
{
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
if (H5I_VFL!=H5Iget_type(H5FD_HTTP_g))
H5FD_HTTP_g = H5FDregister(&H5FD_http_g);
return H5FD_HTTP_g;
} /* end H5FD_http_init() */
/*---------------------------------------------------------------------------
* Function: H5FD_http_term
*
* Purpose: Shut down the VFD
*
* Returns: Non-negative on success or negative on failure
*
* Programmer: Quincey Koziol
* Friday, Jan 30, 2004
*
*---------------------------------------------------------------------------
*/
static herr_t
H5FD_http_term(void)
{
/* Reset VFL ID */
H5FD_HTTP_g = 0;
return 0;
} /* end H5FD_http_term() */
/*-------------------------------------------------------------------------
* Function: H5Pset_fapl_http
*
* Purpose: Modify the file access property list to use the H5FD_HTTP
* driver defined in this source file. There are no driver
* specific properties.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Robb Matzke
* Thursday, February 19, 1998
*
*-------------------------------------------------------------------------
*/
EXTERNL herr_t
H5Pset_fapl_http(hid_t fapl_id)
{
static const char *func = "H5FDset_fapl_http"; /*for error reporting*/
/*NO TRACE*/
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
if(0 == H5Pisa_class(fapl_id, H5P_FILE_ACCESS))
H5Epush_ret(func, H5E_ERR_CLS, H5E_PLIST, H5E_BADTYPE, "not a file access property list", -1)
return H5Pset_driver(fapl_id, H5FD_HTTP, NULL);
} /* end H5Pset_fapl_http() */
/*-------------------------------------------------------------------------
* Function: H5FD_http_open
*
* Purpose: Opens a remote Object as an HDF5 file.
*
* Errors:
* IO CANTOPENFILE File doesn't exist and CREAT wasn't
* specified.
*
* Return:
* Success: A pointer to a new file data structure. The
* public fields will be initialized by the
* caller, which is always H5FD_open().
*
* Failure: NULL
*
* Programmer: Dennis Heimbigner
*
*-------------------------------------------------------------------------
*/
static H5FD_t *
H5FD_http_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id,
haddr_t maxaddr)
{
unsigned write_access = 0; /* File opened with write access? */
H5FD_http_t *file = NULL;
static const char *func = "H5FD_http_open"; /* Function Name for error reporting */
CURL* curl = NULL;
long long len = -1;
int ncstat = NC_NOERR;
/* Sanity check on file offsets */
assert(sizeof(file_offset_t) >= sizeof(size_t));
/* Quiet compiler */
fapl_id = fapl_id;
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
/* Check arguments */
if (!name || !*name)
H5Epush_ret(func, H5E_ERR_CLS, H5E_ARGS, H5E_BADVALUE, "invalid URL", NULL)
if (0 == maxaddr || HADDR_UNDEF == maxaddr)
H5Epush_ret(func, H5E_ERR_CLS, H5E_ARGS, H5E_BADRANGE, "bogus maxaddr", NULL)
if (ADDR_OVERFLOW(maxaddr))
H5Epush_ret(func, H5E_ERR_CLS, H5E_ARGS, H5E_OVERFLOW, "maxaddr too large", NULL)
/* Always read-only */
write_access = 0;
/* Open file in read-only mode, to check for existence and get length */
if((ncstat = nc_http_open(name,&curl,&len))) {
H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_CANTOPENFILE, "cannot access object", NULL)
}
/* Build the return value */
if(NULL == (file = (H5FD_http_t *)H5allocate_memory(sizeof(H5FD_http_t),0))) {
nc_http_close(curl);
H5Epush_ret(func, H5E_ERR_CLS, H5E_RESOURCE, H5E_NOSPACE, "memory allocation failed", NULL)
} /* end if */
memset(file,0,sizeof(H5FD_http_t));
file->op = H5FD_HTTP_OP_SEEK;
file->pos = HADDR_UNDEF;
file->write_access = write_access; /* Note the write_access for later */
file->eof = (haddr_t)len;
file->curl = curl; curl = NULL;
file->url = H5allocate_memory(strlen(name+1),0);
if(file->url == NULL) {
nc_http_close(curl);
H5Epush_ret(func, H5E_ERR_CLS, H5E_RESOURCE, H5E_NOSPACE, "memory allocation failed", NULL)
}
memcpy(file->url,name,strlen(name)+1);
return((H5FD_t*)file);
} /* end H5FD_HTTP_OPen() */
/*-------------------------------------------------------------------------
* Function: H5F_http_close
*
* Purpose: Closes a file.
*
* Errors:
* IO CLOSEERROR Fclose failed.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Dennis Heimbigner
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_http_close(H5FD_t *_file)
{
H5FD_http_t *file = (H5FD_http_t*)_file;
#if 0
static const char *func = "H5FD_http_close"; /* Function Name for error reporting */
#endif
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
/* Close the underlying curl handle*/
if(file->curl) nc_http_close(file->curl);
if(file->url) H5free_memory(file->url);
H5free_memory(file);
return 0;
} /* end H5FD_http_close() */
/*-------------------------------------------------------------------------
* Function: H5FD_http_cmp
*
* Purpose: Compares two files belonging to this driver using an
* arbitrary (but consistent) ordering.
*
* Return:
* Success: A value like strcmp()
*
* Failure: never fails (arguments were checked by the caller).
*
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
*-------------------------------------------------------------------------
*/
static int
H5FD_http_cmp(const H5FD_t *_f1, const H5FD_t *_f2)
{
const H5FD_http_t *f1 = (const H5FD_http_t*)_f1;
const H5FD_http_t *f2 = (const H5FD_http_t*)_f2;
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
if(strcmp(f1->url,f2->url) < 0) return -1;
if(strcmp(f1->url,f2->url) > 0) return 1;
return 0;
} /* H5FD_http_cmp() */
/*-------------------------------------------------------------------------
* Function: H5FD_http_query
*
* Purpose: Set the flags that this VFL driver is capable of supporting.
* (listed in H5FDpublic.h)
*
* Return: Success: non-negative
*
* Failure: negative
*
* Programmer: Quincey Koziol
* Friday, August 25, 2000
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_http_query(const H5FD_t *_f, unsigned long /*OUT*/ *flags)
{
/* Quiet the compiler */
_f=_f;
/* Set the VFL feature flags that this driver supports.
*
* Note that this VFD does not support SWMR due to the unpredictable
* nature of the buffering layer.
*/
if(flags) {
*flags = 0;
*flags |= H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */
*flags |= H5FD_FEAT_ACCUMULATE_METADATA; /* OK to accumulate metadata for faster writes */
*flags |= H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
*flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */
*flags |= H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; /* VFD creates a file which can be opened with the default VFD */
}
return 0;
} /* end H5FD_http_query() */
/*-------------------------------------------------------------------------
* Function: H5FD_http_alloc
*
* Purpose: Allocates file memory. If fseeko isn't available, makes
* sure the file size isn't bigger than 2GB because the
* parameter OFFSET of fseek is of the type LONG INT, limiting
* the file size to 2GB.
*
* Return:
* Success: Address of new memory
*
* Failure: HADDR_UNDEF
*
* Programmer: Raymond Lu
* 30 March 2007
*
*-------------------------------------------------------------------------
*/
static haddr_t
H5FD_http_alloc(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxpl_id, hsize_t size)
{
H5FD_http_t *file = (H5FD_http_t*)_file;
haddr_t addr;
/* Quiet compiler */
type = type;
dxpl_id = dxpl_id;
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
/* Compute the address for the block to allocate */
addr = file->eoa;
file->eoa = addr + size;
return addr;
} /* end H5FD_http_alloc() */
/*-------------------------------------------------------------------------
* Function: H5FD_http_get_eoa
*
* Purpose: Gets the end-of-address marker for the file. The EOA marker
* is the first address past the last byte allocated in the
* format address space.
*
* Return: Success: The end-of-address marker.
*
* Failure: HADDR_UNDEF
*
* Programmer: Robb Matzke
* Monday, August 2, 1999
*
*-------------------------------------------------------------------------
*/
static haddr_t
H5FD_http_get_eoa(const H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type)
{
const H5FD_http_t *file = (const H5FD_http_t *)_file;
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
/* Quiet compiler */
type = type;
return file->eoa;
} /* end H5FD_http_get_eoa() */
/*-------------------------------------------------------------------------
* Function: H5FD_http_set_eoa
*
* Purpose: Set the end-of-address marker for the file. This function is
* called shortly after an existing HDF5 file is opened in order
* to tell the driver where the end of the HDF5 data is located.
*
* Return: Success: 0
*
* Failure: Does not fail
*
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_http_set_eoa(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, haddr_t addr)
{
H5FD_http_t *file = (H5FD_http_t*)_file;
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
/* Quiet the compiler */
type = type;
file->eoa = addr;
return 0;
}
/*-------------------------------------------------------------------------
* Function: H5FD_http_get_eof
*
* Purpose: Returns the end-of-file marker, which is the greater of
* either the Unix end-of-file or the HDF5 end-of-address
* markers.
*
* Return: Success: End of file address, the first address past
* the end of the "file", either the Unix file
* or the HDF5 file.
*
* Failure: HADDR_UNDEF
*
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
*-------------------------------------------------------------------------
*/
static haddr_t
H5FD_http_get_eof(const H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type)
{
const H5FD_http_t *file = (const H5FD_http_t *)_file;
/* Quiet the compiler */
type = type;
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
/* Quiet the compiler */
type = type;
return(file->eof);
} /* end H5FD_http_get_eof() */
/*-------------------------------------------------------------------------
* Function: H5FD_http_get_handle
*
* Purpose: Returns the file handle of file driver.
*
* Returns: Non-negative if succeed or negative if fails.
*
* Programmer: Raymond Lu
* Sept. 16, 2002
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_http_get_handle(H5FD_t *_file, hid_t /*UNUSED*/ fapl, void **file_handle)
{
H5FD_http_t *file = (H5FD_http_t *)_file;
static const char *func = "H5FD_http_get_handle"; /* Function Name for error reporting */
/* Quiet the compiler */
fapl = fapl;
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
*file_handle = file->curl;
if(*file_handle == NULL)
H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_WRITEERROR, "get handle failed", -1)
return 0;
} /* end H5FD_http_get_handle() */
/*-------------------------------------------------------------------------
* Function: H5FD_http_read
*
* Purpose: Reads SIZE bytes beginning at address ADDR in file LF and
* places them in buffer BUF. Reading past the logical or
* physical end of file returns zeros instead of failing.
*
* Errors:
* IO READERROR fread failed.
* IO SEEKERROR fseek failed.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Robb Matzke
* Wednesday, October 22, 1997
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_http_read(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxpl_id,
haddr_t addr, size_t size, void /*OUT*/ *buf)
{
H5FD_http_t *file = (H5FD_http_t*)_file;
static const char *func = "H5FD_http_read"; /* Function Name for error reporting */
int ncstat = NC_NOERR;
/* Quiet the compiler */
type = type;
dxpl_id = dxpl_id;
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
/* Check for overflow */
if (HADDR_UNDEF==addr)
H5Epush_ret (func, H5E_ERR_CLS, H5E_IO, H5E_OVERFLOW, "file address overflowed", -1)
if (REGION_OVERFLOW(addr, size))
H5Epush_ret (func, H5E_ERR_CLS, H5E_IO, H5E_OVERFLOW, "file address overflowed", -1)
/* Check easy cases */
if (0 == size)
return 0;
if ((haddr_t)addr >= file->eof) {
memset(buf, 0, size);
return 0;
}
/* Seek to the correct file position. */
if (!(file->op == H5FD_HTTP_OP_READ || file->op == H5FD_HTTP_OP_SEEK) ||
file->pos != addr) {
#if 0
if (file_fseek(file->fp, (file_offset_t)addr, SEEK_SET) < 0) {
file->op = H5FD_HTTP_OP_UNKNOWN;
file->pos = HADDR_UNDEF;
H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_SEEKERROR, "fseek failed", -1)
}
#endif
file->pos = addr;
}
/* Read zeros past the logical end of file (physical is handled below) */
if (addr + size > file->eof) {
size_t nbytes = (size_t) (addr + size - file->eof);
memset((unsigned char *)buf + size - nbytes, 0, nbytes);
size -= nbytes;
}
{
NCbytes* bbuf = ncbytesnew();
if((ncstat = nc_http_read(file->curl,file->url,addr,size,bbuf))) {
file->op = H5FD_HTTP_OP_UNKNOWN;
file->pos = HADDR_UNDEF;
ncbytesfree(bbuf); bbuf = NULL;
H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_READERROR, "HTTP byte-range read failed", -1)
} /* end if */
/* Check that proper number of bytes was read */
if(ncbyteslength(bbuf) != size) {
ncbytesfree(bbuf); bbuf = NULL;
H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_READERROR, "HTTP byte-range read mismatch ", -1)
}
/* Extract the data from buf */
memcpy(buf,ncbytescontents(bbuf),size);
ncbytesfree(bbuf);
}
/* Update the file position data. */
file->op = H5FD_HTTP_OP_READ;
file->pos = addr;
return 0;
}
/*-------------------------------------------------------------------------
* Function: H5FD_http_write
*
* Purpose: Writes SIZE bytes from the beginning of BUF into file LF at
* file address ADDR.
*
* Errors:
* IO SEEKERROR fseek failed.
* IO WRITEERROR fwrite failed.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Dennis Heimbigner
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_http_write(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxpl_id,
haddr_t addr, size_t size, const void *buf)
{
static const char *func = "H5FD_http_write"; /* Function Name for error reporting */
/* Quiet the compiler */
dxpl_id = dxpl_id;
type = type;
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
/* Always Fails */
H5Epush_ret (func, H5E_ERR_CLS, H5E_IO, H5E_WRITEERROR, "file is read-only", -1)
return 0;
}
/*-------------------------------------------------------------------------
* Function: H5FD_http_flush
*
* Purpose: Makes sure that all data is on disk.
*
* Errors:
* IO SEEKERROR fseek failed.
* IO WRITEERROR fflush or fwrite failed.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Robb Matzke
* Wednesday, October 22, 1997
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_http_flush(H5FD_t *_file, hid_t /*UNUSED*/ dxpl_id, hbool_t closing)
{
#if 0
H5FD_http_t *file = (H5FD_http_t*)_file;
static const char *func = "H5FD_http_flush"; /* Function Name for error reporting */
#endif
/* Quiet the compiler */
dxpl_id = dxpl_id;
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
/* Do nothing for S2 */
#if 0
/* Only try to flush the file if we have write access */
if(file->write_access) {
if(!closing) {
if(fflush(file->fp) < 0)
H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_WRITEERROR, "fflush failed", -1)
/* Reset last file I/O information */
file->pos = HADDR_UNDEF;
file->op = H5FD_HTTP_OP_UNKNOWN;
} /* end if */
} /* end if */
#endif
return 0;
} /* end H5FD_http_flush() */
/*-------------------------------------------------------------------------
* Function: H5FD_http_lock
*
* Purpose: Lock a file via flock
* NOTE: This function is a no-op if flock() is not present.
*
* Errors:
* IO FCNTL flock failed.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Vailin Choi; March 2015
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_http_lock(H5FD_t *_file, hbool_t rw)
{
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
return 0;
} /* end H5FD_http_lock() */
/*-------------------------------------------------------------------------
* Function: H5F_http_unlock
*
* Purpose: Unlock a file via flock
* NOTE: This function is a no-op if flock() is not present.
*
* Errors:
* IO FCNTL flock failed.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Vailin Choi; March 2015
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_http_unlock(H5FD_t *_file)
{
/* Clear the error stack */
H5Eclear2(H5E_DEFAULT);
return 0;
} /* end H5FD_http_unlock() */
#ifdef _H5private_H
/*
* This is not related to the functionality of the driver code.
* It is added here to trigger warning if HDF5 private definitions are included
* by mistake. The code should use only HDF5 public API and definitions.
*/
#error "Do not use HDF5 private definitions"
#endif

51
libhdf5/H5FDhttp.h Normal file

@ -0,0 +1,51 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* ********************************************************************/
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Dennis Heimbigner (dmh@ucar.edu)
* Dec. 26 2018
*
* Purpose: The public header file for the s3 driver.
*
* Derived from the HDF5 Source file H5FDstdio.c
*/
#ifndef H5FDHTTP_H
#define H5FDHTTP_H
#include "H5Ipublic.h"
#define H5FD_HTTP (H5FD_http_init())
#ifdef __cplusplus
extern "C" {
#endif
#if 0
H5_DLL hid_t H5FD_http_init(void);
H5_DLL herr_t H5Pset_fapl_http(hid_t fapl_id);
#else
EXTERNL hid_t H5FD_http_init(void);
EXTERNL herr_t H5Pset_fapl_http(hid_t fapl_id);
#endif
#ifdef __cplusplus
}
#endif
#endif /*H5FDHTTP_H*/

4
libhdf5/Makefile.am

@ -17,6 +17,10 @@ libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c \
hdf5var.c nc4mem.c nc4memcb.c hdf5cache.c hdf5dispatch.c
if ENABLE_HTTP
libnchdf5_la_SOURCES += H5FDhttp.c H5FDhttp.h
endif
# Package this for cmake build.
EXTRA_DIST = CMakeLists.txt

13
libhdf5/hdf5dispatch.c

@ -11,6 +11,10 @@
#include "config.h"
#include "hdf5internal.h"
#ifdef ENABLE_HTTP
#include "H5FDhttp.h"
#endif
static NC_Dispatch NC4_dispatcher = {
NC_FORMATX_NC4,
@ -116,11 +120,12 @@ int
NC_HDF5_initialize(void)
{
HDF5_dispatch_table = &NC4_dispatcher;
if (!nc4_hdf5_initialized)
nc4_hdf5_initialize();
return NC_NOERR;
#ifdef ENABLE_HTTP
(void)H5FD_http_init();
#endif
return NC4_provenance_init();
}
/**
@ -132,6 +137,6 @@ NC_HDF5_initialize(void)
int
NC_HDF5_finalize(void)
{
nc4_hdf5_finalize();
(void)nc4_hdf5_finalize();
return NC_NOERR;
}

45
libhdf5/hdf5open.c

@ -12,6 +12,14 @@
#include "config.h"
#include "hdf5internal.h"
#include "ncrc.h"
#include "ncmodel.h"
#ifdef ENABLE_HTTP
#include "H5FDhttp.h"
#endif
#ifdef ENABLE_BYTERANGE
#include "H5FDhttp.h"
#endif
#define NUM_TYPES 12 /**< Number of netCDF atomic types. */
#define CD_NELEMS_ZLIB 1 /**< Number of parameters needed for ZLIB filter. */
@ -389,6 +397,7 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
unsigned flags;
NC_FILE_INFO_T *nc4_info = NULL;
int is_classic;
NC_HDF5_FILE_INFO_T *h5 = NULL;
#ifdef USE_PARALLEL4
NC_MPI_INFO* mpiinfo = NULL;
@ -415,9 +424,23 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
if (!(nc4_info->root_grp->format_grp_info = calloc(1, sizeof(NC_HDF5_GRP_INFO_T))))
BAIL(NC_ENOMEM);
h5 = (NC_HDF5_FILE_INFO_T*)nc4_info->format_file_info;
#ifdef ENABLE_HTTP
/* See if we want the byte range protocol */
if(nc->model->iosp == NC_IOSP_HTTP) {
h5->http.iosp = 1;
/* Kill off any conflicting modes flags */
mode &= ~(NC_WRITE|NC_DISKLESS|NC_PERSIST|NC_INMEMORY);
parameters = NULL; /* kill off parallel */
} else
h5->http.iosp = 0;
#endif /*ENABLE_HTTP*/
nc4_info->mem.inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY);
nc4_info->mem.diskless = ((mode & NC_DISKLESS) == NC_DISKLESS);
nc4_info->mem.persist = ((mode & NC_PERSIST) == NC_PERSIST);
/* Does the mode specify that this file is read-only? */
if ((mode & NC_WRITE) == 0)
nc4_info->no_write = NC_TRUE;
@ -504,23 +527,29 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
}
else
if(nc4_info->mem.diskless) { /* Process NC_DISKLESS */
NC_HDF5_FILE_INFO_T *hdf5_info;
size_t min_incr = 65536; /* Minimum buffer increment */
/* Configure FAPL to use the core file driver */
if (H5Pset_fapl_core(fapl_id, min_incr, (nc4_info->mem.persist?1:0)) < 0)
BAIL(NC_EHDFERR);
hdf5_info = (NC_HDF5_FILE_INFO_T *)nc4_info->format_file_info;
/* Open the HDF5 file. */
if ((hdf5_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
if ((h5->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
BAIL(NC_EHDFERR);
}
#ifdef ENABLE_HTTP
else
if(h5->http.iosp) { /* Arrange to use the byte-range driver */
/* Configure FAPL to use the byte-range file driver */
if (H5Pset_fapl_http(fapl_id) < 0)
BAIL(NC_EHDFERR);
/* Open the HDF5 file. */
if ((h5->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
BAIL(NC_EHDFERR);
}
#endif
else
{
NC_HDF5_FILE_INFO_T *hdf5_info;
hdf5_info = (NC_HDF5_FILE_INFO_T *)nc4_info->format_file_info;
/* Open the HDF5 file. */
if ((hdf5_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
if ((h5->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
BAIL(NC_EHDFERR);
}
@ -604,7 +633,7 @@ NC4_open(const char *path, int mode, int basepe, size_t *chunksizehintp,
void *parameters, NC_Dispatch *dispatch, NC *nc_file)
{
assert(nc_file && path && dispatch && nc_file &&
nc_file->model == NC_FORMATX_NC4);
nc_file->model->impl == NC_FORMATX_NC4);
LOG((1, "%s: path %s mode %d params %x",
__func__, path, mode, parameters));

16
libhdf5/nc4info.c

@ -422,16 +422,16 @@ NC4_get_provenance(NC_FILE_INFO_T* file, const char* propstring, const struct NC
int i;
for(i=0;i<nclistlength(list);i+=2) {
char* newname = NULL;
name = nclistget(list,i);
if(name == NULL) continue; /* ignore */
if(strcmp(name,NCPNCLIB1) == 0)
char* oldname = nclistget(list,i);
if(oldname == NULL) continue; /* ignore */
if(strcmp(oldname,NCPNCLIB1) == 0)
newname = NCPNCLIB2; /* change name */
else if(strcmp(name,NCPHDF5LIB1) == 0)
else if(strcmp(oldname,NCPHDF5LIB1) == 0)
newname = NCPHDF5LIB2;
else continue; /* ignore */
/* Do any rename */
nclistset(list,i,strdup(newname));
if(name) {free(name); name = NULL;}
if(newname != NULL) {/* Do any rename */
if(oldname) free(oldname);
nclistset(list,i,strdup(newname));
}
}
}
}

5
liblib/nc_initialize.c

@ -95,7 +95,6 @@ nc_initialize()
#endif /* USE_NETCDF4 */
#ifdef USE_HDF5
if((stat = NC_HDF5_initialize())) goto done;
stat = NC4_provenance_init();
#endif
#ifdef USE_HDF4
if((stat = NC_HDF4_initialize())) goto done;
@ -144,6 +143,10 @@ nc_finalize(void)
if((stat = NC4_finalize())) return stat;
#endif /* USE_NETCDF4 */
#ifdef USE_HDF5
if((stat = NC_HDF5_finalize())) return stat;
#endif
if((stat = NC3_finalize())) return stat;
/* Do general finalization */

1
libnetcdf.settings.in

@ -32,6 +32,7 @@ NC-4 Parallel Support: @HAS_PARALLEL4@
PnetCDF Support: @HAS_PNETCDF@
DAP2 Support: @HAS_DAP@
DAP4 Support: @HAS_DAP4@
Byte-Range Support: @HAS_HTTP@
Diskless Support: @HAS_DISKLESS@
MMap Support: @HAS_MMAP@
JNA Support: @HAS_JNA@

4
libsrc/CMakeLists.txt

@ -28,6 +28,10 @@ ELSE (USE_FFIO)
SET(libsrc_SOURCES ${libsrc_SOURCES} posixio.c)
ENDIF (USE_FFIO)
IF (ENABLE_HTTP)
SET(libsrc_SOURCES ${libsrc_SOURCES} httpio.c)
ENDIF(ENABLE_HTTP)
add_library(netcdf3 OBJECT ${libsrc_SOURCES})
# The C API man page.

4
libsrc/Makefile.am

@ -29,6 +29,10 @@ libnetcdf3_la_SOURCES += posixio.c
endif !USE_STDIO
endif !USE_FFIO
if ENABLE_HTTP
libnetcdf3_la_SOURCES += httpio.c
endif ENABLE_HTTP
noinst_LTLIBRARIES = libnetcdf3.la
# These files are cleaned on developer workstations (and then rebuilt

296
libsrc/httpio.c Normal file

@ -0,0 +1,296 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* ********************************************************************/
#if HAVE_CONFIG_H
#include <config.h>
#endif
#include <assert.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef _MSC_VER /* Microsoft Compilers */
#include <io.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "nc3internal.h"
#undef DEBUG
#ifdef DEBUG
#include <stdio.h>
#endif
#include <curl/curl.h>
#include "ncio.h"
#include "fbits.h"
#include "rnd.h"
#include "ncbytes.h"
#include "nchttp.h"
#define DEFAULTPAGESIZE 16384
/* Private data */
typedef struct NCHTTP {
CURL* curl; /* curl handle */
long long size; /* of the S3 object */
NCbytes* region;
} NCHTTP;
/* Forward */
static int httpio_rel(ncio *const nciop, off_t offset, int rflags);
static int httpio_get(ncio *const nciop, off_t offset, size_t extent, int rflags, void **const vpp);
static int httpio_move(ncio *const nciop, off_t to, off_t from, size_t nbytes, int rflags);
static int httpio_sync(ncio *const nciop);
static int httpio_filesize(ncio* nciop, off_t* filesizep);
static int httpio_pad_length(ncio* nciop, off_t length);
static int httpio_close(ncio* nciop, int);
static long pagesize = 0;
/* Create a new ncio struct to hold info about the file. */
static int
httpio_new(const char* path, int ioflags, ncio** nciopp, NCHTTP** hpp)
{
int status = NC_NOERR;
ncio* nciop = NULL;
NCHTTP* http = NULL;
if(pagesize == 0)
pagesize = DEFAULTPAGESIZE;
errno = 0;
nciop = (ncio* )calloc(1,sizeof(ncio));
if(nciop == NULL) {status = NC_ENOMEM; goto fail;}
nciop->ioflags = ioflags;
*((char**)&nciop->path) = strdup(path);
if(nciop->path == NULL) {status = NC_ENOMEM; goto fail;}
*((ncio_relfunc**)&nciop->rel) = httpio_rel;
*((ncio_getfunc**)&nciop->get) = httpio_get;
*((ncio_movefunc**)&nciop->move) = httpio_move;
*((ncio_syncfunc**)&nciop->sync) = httpio_sync;
*((ncio_filesizefunc**)&nciop->filesize) = httpio_filesize;
*((ncio_pad_lengthfunc**)&nciop->pad_length) = httpio_pad_length;
*((ncio_closefunc**)&nciop->close) = httpio_close;
http = (NCHTTP*)calloc(1,sizeof(NCHTTP));
if(http == NULL) {status = NC_ENOMEM; goto fail;}
*((void* *)&nciop->pvt) = http;
if(nciopp) *nciopp = nciop;
if(hpp) *hpp = http;
done:
return status;
fail:
if(http != NULL) {
if(http->region)
ncbytesfree(http->region);
free(http);
}
if(nciop != NULL) {
if(nciop->path != NULL) free((char*)nciop->path);
}
goto done;
}
/* Create a file, and the ncio struct to go with it. This function is
only called from nc__create_mp.
path - path of file to create.
ioflags - flags from nc_create
initialsz - From the netcdf man page: "The argument
Iinitialsize sets the initial size of the file at creation time."
igeto -
igetsz -
sizehintp - the size of a page of data for buffered reads and writes.
nciopp - pointer to a pointer that will get location of newly
created and inited ncio struct.
mempp - pointer to pointer to the initial memory read.
*/
int
httpio_create(const char* path, int ioflags,
size_t initialsz,
off_t igeto, size_t igetsz, size_t* sizehintp,
void* parameters,
ncio* *nciopp, void** const mempp)
{
return NC_EPERM;
}
/* This function opens the data file. It is only called from nc.c,
from nc__open_mp and nc_delete_mp.
path - path of data file.
ioflags - flags passed into nc_open.
igeto - looks like this function can do an initial page get, and
igeto is going to be the offset for that. But it appears to be
unused
igetsz - the size in bytes of initial page get (a.k.a. extent). Not
ever used in the library.
sizehintp - the size of a page of data for buffered reads and writes.
nciopp - pointer to pointer that will get address of newly created
and inited ncio struct.
mempp - pointer to pointer to the initial memory read.
*/
int
httpio_open(const char* path,
int ioflags,
/* ignored */ off_t igeto, size_t igetsz, size_t* sizehintp,
/* ignored */ void* parameters,
ncio* *nciopp,
/* ignored */ void** const mempp)
{
ncio* nciop;
int status;
NCHTTP* http = NULL;
size_t sizehint;
if(path == NULL ||* path == 0)
return EINVAL;
/* Create private data */
if((status = httpio_new(path, ioflags, &nciop, &http))) goto done;
/* Open the path and get curl handle and object size */
if((status = nc_http_open(path,&http->curl,&http->size))) goto done;
sizehint = pagesize;
/* sizehint must be multiple of 8 */
sizehint = (sizehint / 8) * 8;
if(sizehint < 8) sizehint = 8;
*sizehintp = sizehint;
*nciopp = nciop;
done:
if(status)
httpio_close(nciop,0);
return status;
}
/*
* Get file size in bytes.
*/
static int
httpio_filesize(ncio* nciop, off_t* filesizep)
{
NCHTTP* http;
if(nciop == NULL || nciop->pvt == NULL) return NC_EINVAL;
http = (NCHTTP*)nciop->pvt;
if(filesizep != NULL) *filesizep = http->size;
return NC_NOERR;
}
/*
* Sync any changes to disk, then truncate or extend file so its size
* is length. This is only intended to be called before close, if the
* file is open for writing and the actual size does not match the
* calculated size, perhaps as the result of having been previously
* written in NOFILL mode.
*/
static int
httpio_pad_length(ncio* nciop, off_t length)
{
return NC_NOERR; /* do nothing */
}
/* Write out any dirty buffers to disk and
ensure that next read will get data from disk.
Sync any changes, then close the open file associated with the ncio
struct, and free its memory.
nciop - pointer to ncio to close.
doUnlink - if true, unlink file
*/
static int
httpio_close(ncio* nciop, int doUnlink)
{
int status = NC_NOERR;
NCHTTP* http;
if(nciop == NULL || nciop->pvt == NULL) return NC_NOERR;
http = (NCHTTP*)nciop->pvt;
assert(http != NULL);
status = nc_http_close(http->curl);
/* do cleanup */
if(http != NULL) {
ncbytesfree(http->region);
free(http);
}
if(nciop->path != NULL) free((char*)nciop->path);
free(nciop);
return status;
}
/*
* Request that the region (offset, extent)
* be made available through *vpp.
*/
static int
httpio_get(ncio* const nciop, off_t offset, size_t extent, int rflags, void** const vpp)
{
int status = NC_NOERR;
NCHTTP* http;
if(nciop == NULL || nciop->pvt == NULL) {status = NC_EINVAL; goto done;}
http = (NCHTTP*)nciop->pvt;
assert(http->region == NULL);
http->region = ncbytesnew();
ncbytessetalloc(http->region,(unsigned long)extent);
if((status = nc_http_read(http->curl,nciop->path,offset,extent,http->region)))
goto done;
assert(ncbyteslength(http->region) == extent);
if(vpp) *vpp = ncbytescontents(http->region);
done:
return status;
}
/*
* Like memmove(), safely move possibly overlapping data.
*/
static int
httpio_move(ncio* const nciop, off_t to, off_t from, size_t nbytes, int ignored)
{
return NC_EPERM;
}
static int
httpio_rel(ncio* const nciop, off_t offset, int rflags)
{
int status = NC_NOERR;
NCHTTP* http;
if(nciop == NULL || nciop->pvt == NULL) {status = NC_EINVAL; goto done;}
http = (NCHTTP*)nciop->pvt;
ncbytesfree(http->region);
http->region = NULL;
done:
return status;
}
/*
* Write out any dirty buffers to disk and
* ensure that next read will get data from disk.
*/
static int
httpio_sync(ncio* const nciop)
{
return NC_NOERR; /* do nothing */
}

9
libsrc/nc3internal.c

@ -39,7 +39,6 @@
/* Internal function; breaks ncio abstraction */
extern int memio_extract(ncio* const nciop, size_t* sizep, void** memoryp);
static void
free_NC3INFO(NC3_INFO *nc3)
{
@ -1202,6 +1201,14 @@ NC3_open(const char *path, int ioflags, int basepe, size_t *chunksizehintp,
}
#endif
#ifdef ENABLE_HTTP
/* If the model specified the use of byte-ranges, then signal by
a temporary hack using one of the flags in the ioflags.
*/
if(nc->model->iosp == NC_IOSP_HTTP)
ioflags |= NC_HTTP;
#endif /*ENABLE_HTTP*/
status = ncio_open(path, ioflags, 0, 0, &nc3->chunk, parameters,
&nc3->nciop, NULL);
if(status)

16
libsrc/ncio.c

@ -3,8 +3,8 @@
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#if HAVE_CONFIG_H
#include <config.h>
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
@ -34,6 +34,11 @@ extern int ffio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** co
extern int mmapio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int mmapio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
# endif
#ifdef ENABLE_HTTP
extern int httpio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
#endif
extern int memio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int memio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
@ -83,6 +88,13 @@ ncio_open(const char *path, int ioflags,
return mmapio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
# endif /*USE_MMAP*/
# ifdef ENABLE_HTTP
/* The NC_HTTP flag is a big hack until we can reorganize the ncio interface */
if(fIsSet(ioflags,NC_HTTP)) {
return httpio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
# endif /*ENABLE_HTTP*/
#ifdef USE_STDIO
return stdio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#elif defined(USE_FFIO)

5
libsrc/ncio.h

@ -11,6 +11,11 @@
#include <sys/types.h> /* off_t */
#include "netcdf.h"
/* Define an internal use only mode flag to signal use of byte ranges.
This is temporary until we can re-organize the ncio open/create API.
*/
#define NC_HTTP 0x80000000
typedef struct ncio ncio; /* forward reference */
/*

13
nc_test/CMakeLists.txt

@ -92,10 +92,20 @@ IF(BUILD_UTILITIES)
ENDIF()
ENDIF(BUILD_UTILITIES)
IF(ENABLE_HTTP)
build_bin_test_no_prefix(tst_http)
IF(BUILD_UTILITIES)
add_sh_test(nc_test test_http)
ENDIF()
ENDIF()
# Copy some test files from current source dir to out-of-tree build dir.
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/*.sh)
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
IF(MSVC)
#MESSAGE(STATUS "XXX")
#MESSAGE(STATUS "${COPY_FILES}")
#MESSAGE(STATUS "${RUNTIME_OUTPUT_DIRECTORY}")
FILE(COPY ${COPY_FILES} DESTINATION ${RUNTIME_OUTPUT_DIRECTORY}/)
ENDIF()
@ -103,5 +113,6 @@ ENDIF()
## Specify files to be distributed by 'make dist'
FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.c ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.sh)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} test_get.m4 test_put.m4 test_read.m4 test_write.m4 ref_tst_diskless2.cdl tst_diskless5.cdl)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} test_get.m4 test_put.m4 test_read.m4 test_write.m4 ref_tst_diskless2.cdl tst_diskless5.cdl
ref_tst_http_nc3.cdl ref_tst_http_nc4.cdl)
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

12
nc_test/Makefile.am

@ -56,6 +56,11 @@ TESTPROGRAMS += tst_diskless6
tst_diskless6_SOURCES = tst_diskless6.c
endif
if ENABLE_HTTP
TESTPROGRAMS += tst_http
tst_http_SOURCES = tst_http.c
endif
# Set up the tests.
check_PROGRAMS = $(TESTPROGRAMS)
@ -73,6 +78,9 @@ TESTS += run_diskless.sh run_diskless5.sh run_inmemory.sh
if LARGE_FILE_TESTS
TESTS += run_diskless2.sh
endif
if ENABLE_HTTP
TESTS += test_http.sh
endif
endif
if USE_PNETCDF
@ -85,12 +93,14 @@ EXTRA_DIST = test_get.m4 test_put.m4 run_diskless.sh run_diskless2.sh \
run_diskless5.sh run_mmap.sh run_pnetcdf_test.sh test_read.m4 \
test_write.m4 ref_tst_diskless2.cdl tst_diskless5.cdl run_inmemory.sh \
f03tst_open_mem.nc \
test_http.sh ref_tst_http_nc3.cdl ref_tst_http_nc4.cdl \
CMakeLists.txt
# These files are created by the tests.
CLEANFILES = nc_test_*.nc tst_*.nc t_nc.nc large_files.nc \
quick_large_files.nc tst_diskless3_file.cdl tst_diskless3_memory.cdl \
tst_diskless4.cdl ref_tst_diskless4.cdl benchmark.nc
tst_diskless4.cdl ref_tst_diskless4.cdl benchmark.nc \
tst_http_nc3.cdl tst_http_nc4.cdl
EXTRA_DIST += bad_cdf5_begin.nc run_cdf5.sh
if ENABLE_CDF5

77
nc_test/ref_tst_http_nc3.cdl Normal file

@ -0,0 +1,77 @@
netcdf \2004050300_eta_211 {
dimensions:
record = UNLIMITED ; // (1 currently)
x = 135 ;
y = 95 ;
datetime_len = 21 ;
nmodels = 1 ;
ngrids = 1 ;
nav = 1 ;
nav_len = 100 ;
variables:
double reftime(record) ;
reftime:long_name = "reference time" ;
reftime:units = "hours since 1992-1-1" ;
double valtime(record) ;
valtime:long_name = "valid time" ;
valtime:units = "hours since 1992-1-1" ;
char datetime(record, datetime_len) ;
datetime:long_name = "reference date and time" ;
float valtime_offset(record) ;
valtime_offset:long_name = "hours from reference time" ;
valtime_offset:units = "hours" ;
int model_id(nmodels) ;
model_id:long_name = "generating process ID number" ;
char nav_model(nav, nav_len) ;
nav_model:long_name = "navigation model name" ;
int grid_type_code(nav) ;
grid_type_code:long_name = "GRIB-1 GDS data representation type" ;
char grid_type(nav, nav_len) ;
grid_type:long_name = "GRIB-1 grid type" ;
char grid_name(nav, nav_len) ;
grid_name:long_name = "grid name" ;
int grid_center(nav) ;
grid_center:long_name = "GRIB-1 originating center ID" ;
int grid_number(nav, ngrids) ;
grid_number:long_name = "GRIB-1 catalogued grid numbers" ;
grid_number:_FillValue = -9999 ;
char x_dim(nav, nav_len) ;
x_dim:long_name = "x dimension name" ;
char y_dim(nav, nav_len) ;
y_dim:long_name = "y dimension name" ;
int Nx(nav) ;
Nx:long_name = "number of points along x-axis" ;
int Ny(nav) ;
Ny:long_name = "number of points along y-axis" ;
float La1(nav) ;
La1:long_name = "latitude of first grid point" ;
La1:units = "degrees_north" ;
float Lo1(nav) ;
Lo1:long_name = "longitude of first grid point" ;
Lo1:units = "degrees_east" ;
float Lov(nav) ;
Lov:long_name = "orientation of the grid" ;
Lov:units = "degrees_east" ;
float Dx(nav) ;
Dx:long_name = "x-direction grid length" ;
Dx:units = "km" ;
float Dy(nav) ;
Dy:long_name = "y-direction grid length" ;
Dy:units = "km" ;
byte ProjFlag(nav) ;
ProjFlag:long_name = "projection center flag" ;
byte ResCompFlag(nav) ;
ResCompFlag:long_name = "resolution and component flags" ;
float Z_sfc(record, y, x) ;
Z_sfc:long_name = "Geopotential height, gpm" ;
Z_sfc:units = "gp m" ;
Z_sfc:_FillValue = -9999.f ;
Z_sfc:navigation = "nav" ;
// global attributes:
:record = "reftime, valtime" ;
:history = "2003-09-25 16:09:26 - created by gribtocdl 1.4 - 12.12.2002" ;
:title = "CMC_reg_HGT_SFC_0_ps60km_2003092500_P000.grib" ;
:Conventions = "NUWG" ;
:version = 0. ;
}

297
nc_test/ref_tst_http_nc4.cdl Normal file

@ -0,0 +1,297 @@
netcdf OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316 {
dimensions:
y = 1500 ;
x = 2500 ;
number_of_time_bounds = 2 ;
band = 1 ;
number_of_image_bounds = 2 ;
num_star_looks = 24 ;
variables:
short Rad(y, x) ;
Rad:_FillValue = 4095s ;
Rad:long_name = "ABI L1b Radiances" ;
Rad:standard_name = "toa_outgoing_radiance_per_unit_wavenumber" ;
Rad:_Unsigned = "true" ;
Rad:sensor_band_bit_depth = 12b ;
Rad:valid_range = 0s, 4094s ;
Rad:scale_factor = 0.04572892f ;
Rad:add_offset = -1.6443f ;
Rad:units = "mW m-2 sr-1 (cm-1)-1" ;
Rad:resolution = "y: 0.000056 rad x: 0.000056 rad" ;
Rad:coordinates = "band_id band_wavelength t y x" ;
Rad:grid_mapping = "goes_imager_projection" ;
Rad:cell_methods = "t: point area: point" ;
Rad:ancillary_variables = "DQF" ;
byte DQF(y, x) ;
DQF:_FillValue = -1b ;
DQF:long_name = "ABI L1b Radiances data quality flags" ;
DQF:standard_name = "status_flag" ;
DQF:_Unsigned = "true" ;
DQF:valid_range = 0b, 3b ;
DQF:units = "1" ;
DQF:coordinates = "band_id band_wavelength t y x" ;
DQF:grid_mapping = "goes_imager_projection" ;
DQF:cell_methods = "t: point area: point" ;
DQF:flag_values = 0b, 1b, 2b, 3b ;
DQF:flag_meanings = "good_pixel_qf conditionally_usable_pixel_qf out_of_range_pixel_qf no_value_pixel_qf" ;
DQF:number_of_qf_values = 4b ;
DQF:percent_good_pixel_qf = 1.f ;
DQF:percent_conditionally_usable_pixel_qf = 0.f ;
DQF:percent_out_of_range_pixel_qf = 0.f ;
DQF:percent_no_value_pixel_qf = 0.f ;
double t ;
t:long_name = "J2000 epoch mid-point between the start and end image scan in seconds" ;
t:standard_name = "time" ;
t:units = "seconds since 2000-01-01 12:00:00" ;
t:axis = "T" ;
t:bounds = "time_bounds" ;
short y(y) ;
y:scale_factor = -5.6e-05f ;
y:add_offset = 0.126532f ;
y:units = "rad" ;
y:axis = "Y" ;
y:long_name = "GOES fixed grid projection y-coordinate" ;
y:standard_name = "projection_y_coordinate" ;
short x(x) ;
x:scale_factor = 5.6e-05f ;
x:add_offset = -0.075012f ;
x:units = "rad" ;
x:axis = "X" ;
x:long_name = "GOES fixed grid projection x-coordinate" ;
x:standard_name = "projection_x_coordinate" ;
double time_bounds(number_of_time_bounds) ;
time_bounds:long_name = "Scan start and end times in seconds since epoch (2000-01-01 12:00:00)" ;
int goes_imager_projection ;
goes_imager_projection:long_name = "GOES-R ABI fixed grid projection" ;
goes_imager_projection:grid_mapping_name = "geostationary" ;
goes_imager_projection:perspective_point_height = 35786023. ;
goes_imager_projection:semi_major_axis = 6378137. ;
goes_imager_projection:semi_minor_axis = 6356752.31414 ;
goes_imager_projection:inverse_flattening = 298.2572221 ;
goes_imager_projection:latitude_of_projection_origin = 0. ;
goes_imager_projection:longitude_of_projection_origin = -89.5 ;
goes_imager_projection:sweep_angle_axis = "x" ;
float y_image ;
y_image:long_name = "GOES-R fixed grid projection y-coordinate center of image" ;
y_image:standard_name = "projection_y_coordinate" ;
y_image:units = "rad" ;
y_image:axis = "Y" ;
float y_image_bounds(number_of_image_bounds) ;
y_image_bounds:long_name = "GOES-R fixed grid projection y-coordinate north/south extent of image" ;
float x_image ;
x_image:long_name = "GOES-R fixed grid projection x-coordinate center of image" ;
x_image:standard_name = "projection_x_coordinate" ;
x_image:units = "rad" ;
x_image:axis = "X" ;
float x_image_bounds(number_of_image_bounds) ;
x_image_bounds:long_name = "GOES-R fixed grid projection x-coordinate west/east extent of image" ;
float nominal_satellite_subpoint_lat ;
nominal_satellite_subpoint_lat:long_name = "nominal satellite subpoint latitude (platform latitude)" ;
nominal_satellite_subpoint_lat:standard_name = "latitude" ;
nominal_satellite_subpoint_lat:_FillValue = -999.f ;
nominal_satellite_subpoint_lat:units = "degrees_north" ;
float nominal_satellite_subpoint_lon ;
nominal_satellite_subpoint_lon:long_name = "nominal satellite subpoint longitude (platform longitude)" ;
nominal_satellite_subpoint_lon:standard_name = "longitude" ;
nominal_satellite_subpoint_lon:_FillValue = -999.f ;
nominal_satellite_subpoint_lon:units = "degrees_east" ;
float nominal_satellite_height ;
nominal_satellite_height:long_name = "nominal satellite height above GRS 80 ellipsoid (platform altitude)" ;
nominal_satellite_height:standard_name = "height_above_reference_ellipsoid" ;
nominal_satellite_height:_FillValue = -999.f ;
nominal_satellite_height:units = "km" ;
float geospatial_lat_lon_extent ;
geospatial_lat_lon_extent:long_name = "geospatial latitude and longitude references" ;
geospatial_lat_lon_extent:geospatial_westbound_longitude = -140.6163f ;
geospatial_lat_lon_extent:geospatial_northbound_latitude = 52.76771f ;
geospatial_lat_lon_extent:geospatial_eastbound_longitude = -49.17929f ;
geospatial_lat_lon_extent:geospatial_southbound_latitude = 14.00016f ;
geospatial_lat_lon_extent:geospatial_lat_center = 29.294f ;
geospatial_lat_lon_extent:geospatial_lon_center = -91.406f ;
geospatial_lat_lon_extent:geospatial_lat_nadir = 0.f ;
geospatial_lat_lon_extent:geospatial_lon_nadir = -89.5f ;
geospatial_lat_lon_extent:geospatial_lat_units = "degrees_north" ;
geospatial_lat_lon_extent:geospatial_lon_units = "degrees_east" ;
byte yaw_flip_flag ;
yaw_flip_flag:long_name = "Flag indicating the spacecraft is operating in yaw flip configuration" ;
yaw_flip_flag:_Unsigned = "true" ;
yaw_flip_flag:_FillValue = -1b ;
yaw_flip_flag:valid_range = 0b, 1b ;
yaw_flip_flag:units = "1" ;
yaw_flip_flag:coordinates = "t" ;
yaw_flip_flag:flag_values = "0 1" ;
yaw_flip_flag:flag_meanings = "false true" ;
byte band_id(band) ;
band_id:long_name = "ABI band number" ;
band_id:standard_name = "sensor_band_identifier" ;
band_id:units = "1" ;
float band_wavelength(band) ;
band_wavelength:long_name = "ABI band central wavelength" ;
band_wavelength:standard_name = "sensor_band_central_radiation_wavelength" ;
band_wavelength:units = "um" ;
float esun ;
esun:long_name = "bandpass-weighted solar irradiance at the mean Earth-Sun distance" ;
esun:standard_name = "toa_shortwave_irradiance_per_unit_wavelength" ;
esun:_FillValue = -999.f ;
esun:units = "W m-2 um-1" ;
esun:coordinates = "band_id band_wavelength t" ;
esun:cell_methods = "t: mean" ;
float kappa0 ;
kappa0:long_name = "Inverse of the incoming top of atmosphere radiance at current earth-sun distance (PI d2 esun-1)-1, where d is the ratio of instantaneous Earth-Sun distance divided by the mean Earth-Sun distance, esun is the bandpass-weighted solar irradiance and PI is a standard constant used to convert ABI L1b radiance to reflectance" ;
kappa0:_FillValue = -999.f ;
kappa0:units = "(W m-2 um-1)-1" ;
kappa0:coordinates = "band_id band_wavelength t" ;
kappa0:cell_methods = "t: mean" ;
float planck_fk1 ;
planck_fk1:long_name = "wavenumber-dependent coefficient (2 h c2/ nu3) used in the ABI emissive band monochromatic brightness temperature computation, where nu =central wavenumber and h and c are standard constants" ;
planck_fk1:_FillValue = -999.f ;
planck_fk1:units = "W m-1" ;
planck_fk1:coordinates = "band_id band_wavelength" ;
float planck_fk2 ;
planck_fk2:long_name = "wavenumber-dependent coefficient (h c nu/b) used in the ABI emissive band monochromatic brightness temperature computation, where nu = central wavenumber and h, c, and b are standard constants" ;
planck_fk2:_FillValue = -999.f ;
planck_fk2:units = "K" ;
planck_fk2:coordinates = "band_id band_wavelength" ;
float planck_bc1 ;
planck_bc1:long_name = "spectral bandpass correction offset for brightness temperature (B(nu) - bc_1)/bc_2 where B()=planck_function() and nu=wavenumber" ;
planck_bc1:_FillValue = -999.f ;
planck_bc1:units = "K" ;
planck_bc1:coordinates = "band_id band_wavelength" ;
float planck_bc2 ;
planck_bc2:long_name = "spectral bandpass correction scale factor for brightness temperature (B(nu) - bc_1)/bc_2 where B()=planck_function() and nu=wavenumber" ;
planck_bc2:_FillValue = -999.f ;
planck_bc2:units = "1" ;
planck_bc2:coordinates = "band_id band_wavelength" ;
int valid_pixel_count ;
valid_pixel_count:long_name = "number of good and conditionally usable pixels" ;
valid_pixel_count:_FillValue = -1 ;
valid_pixel_count:units = "count" ;
valid_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
valid_pixel_count:grid_mapping = "goes_imager_projection" ;
valid_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000056 rad comment: good and conditionally usable quality pixels only)" ;
int missing_pixel_count ;
missing_pixel_count:long_name = "number of missing pixels" ;
missing_pixel_count:_FillValue = -1 ;
missing_pixel_count:units = "count" ;
missing_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
missing_pixel_count:grid_mapping = "goes_imager_projection" ;
missing_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000056 rad comment: missing ABI fixed grid pixels only)" ;
int saturated_pixel_count ;
saturated_pixel_count:long_name = "number of saturated pixels" ;
saturated_pixel_count:_FillValue = -1 ;
saturated_pixel_count:units = "count" ;
saturated_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
saturated_pixel_count:grid_mapping = "goes_imager_projection" ;
saturated_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000056 rad comment: radiometrically saturated geolocated/not missing pixels only)" ;
int undersaturated_pixel_count ;
undersaturated_pixel_count:long_name = "number of undersaturated pixels" ;
undersaturated_pixel_count:_FillValue = -1 ;
undersaturated_pixel_count:units = "count" ;
undersaturated_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
undersaturated_pixel_count:grid_mapping = "goes_imager_projection" ;
undersaturated_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000056 rad comment: radiometrically undersaturated geolocated/not missing pixels only)" ;
float min_radiance_value_of_valid_pixels ;
min_radiance_value_of_valid_pixels:long_name = "minimum radiance value of pixels" ;
min_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavenumber" ;
min_radiance_value_of_valid_pixels:_FillValue = -999.f ;
min_radiance_value_of_valid_pixels:valid_range = -1.6443f, 185.5699f ;
min_radiance_value_of_valid_pixels:units = "mW m-2 sr-1 (cm-1)-1" ;
min_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
min_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
min_radiance_value_of_valid_pixels:cell_methods = "t: sum area: minimum (interval: 0.000056 rad comment: good and conditionally usable quality pixels only)" ;
float max_radiance_value_of_valid_pixels ;
max_radiance_value_of_valid_pixels:long_name = "maximum radiance value of pixels" ;
max_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavenumber" ;
max_radiance_value_of_valid_pixels:_FillValue = -999.f ;
max_radiance_value_of_valid_pixels:valid_range = -1.6443f, 185.5699f ;
max_radiance_value_of_valid_pixels:units = "mW m-2 sr-1 (cm-1)-1" ;
max_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
max_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
max_radiance_value_of_valid_pixels:cell_methods = "t: sum area: maximum (interval: 0.000056 rad comment: good and conditionally usable quality pixels only)" ;
float mean_radiance_value_of_valid_pixels ;
mean_radiance_value_of_valid_pixels:long_name = "mean radiance value of pixels" ;
mean_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavenumber" ;
mean_radiance_value_of_valid_pixels:_FillValue = -999.f ;
mean_radiance_value_of_valid_pixels:valid_range = -1.6443f, 185.5699f ;
mean_radiance_value_of_valid_pixels:units = "mW m-2 sr-1 (cm-1)-1" ;
mean_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
mean_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
mean_radiance_value_of_valid_pixels:cell_methods = "t: sum area: mean (interval: 0.000056 rad comment: good and conditionally usable quality pixels only)" ;
float std_dev_radiance_value_of_valid_pixels ;
std_dev_radiance_value_of_valid_pixels:long_name = "standard deviation of radiance values of pixels" ;
std_dev_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavenumber" ;
std_dev_radiance_value_of_valid_pixels:_FillValue = -999.f ;
std_dev_radiance_value_of_valid_pixels:units = "mW m-2 sr-1 (cm-1)-1" ;
std_dev_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
std_dev_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
std_dev_radiance_value_of_valid_pixels:cell_methods = "t: sum area: standard_deviation (interval: 0.000056 rad comment: good and conditionally usable quality pixels only)" ;
float percent_uncorrectable_L0_errors ;
percent_uncorrectable_L0_errors:long_name = "percent data lost due to uncorrectable L0 errors" ;
percent_uncorrectable_L0_errors:_FillValue = -999.f ;
percent_uncorrectable_L0_errors:valid_range = 0.f, 1.f ;
percent_uncorrectable_L0_errors:units = "percent" ;
percent_uncorrectable_L0_errors:coordinates = "t y_image x_image" ;
percent_uncorrectable_L0_errors:grid_mapping = "goes_imager_projection" ;
percent_uncorrectable_L0_errors:cell_methods = "t: sum area: sum (uncorrectable L0 errors only)" ;
float earth_sun_distance_anomaly_in_AU ;
earth_sun_distance_anomaly_in_AU:long_name = "earth sun distance anomaly in astronomical units" ;
earth_sun_distance_anomaly_in_AU:_FillValue = -999.f ;
earth_sun_distance_anomaly_in_AU:units = "ua" ;
earth_sun_distance_anomaly_in_AU:coordinates = "t" ;
earth_sun_distance_anomaly_in_AU:cell_methods = "t: mean" ;
int algorithm_dynamic_input_data_container ;
algorithm_dynamic_input_data_container:long_name = "container for filenames of dynamic algorithm input data" ;
algorithm_dynamic_input_data_container:input_ABI_L0_data = "OR_ABI-L0-C-M3_G16_s20170590337505_e20170590340289_c*.nc" ;
int processing_parm_version_container ;
processing_parm_version_container:long_name = "container for processing parameter filenames" ;
processing_parm_version_container:L1b_processing_parm_version = "ABI-L1b-PARM_G16_v01r00.zip" ;
int algorithm_product_version_container ;
algorithm_product_version_container:long_name = "container for algorithm package filename and product version" ;
algorithm_product_version_container:algorithm_version = "OR_ABI-L1b-ALG-COMMON_v01r00.zip" ;
algorithm_product_version_container:product_version = "v01r00" ;
double t_star_look(num_star_looks) ;
t_star_look:long_name = "J2000 epoch time of star observed in seconds" ;
t_star_look:standard_name = "time" ;
t_star_look:units = "seconds since 2000-01-01 12:00:00" ;
t_star_look:axis = "T" ;
float band_wavelength_star_look(num_star_looks) ;
band_wavelength_star_look:long_name = "ABI band central wavelength associated with observed star" ;
band_wavelength_star_look:standard_name = "sensor_band_central_radiation_wavelength" ;
band_wavelength_star_look:units = "um" ;
short star_id(num_star_looks) ;
star_id:long_name = "ABI star catalog identifier associated with observed star" ;
star_id:_Unsigned = "true" ;
star_id:_FillValue = -1s ;
star_id:coordinates = "band_id band_wavelength_star_look t_star_look" ;
// global attributes:
:naming_authority = "gov.nesdis.noaa" ;
:Conventions = "CF-1.7" ;
:Metadata_Conventions = "Unidata Dataset Discovery v1.0" ;
:standard_name_vocabulary = "CF Standard Name Table (v25, 05 July 2013)" ;
:institution = "DOC/NOAA/NESDIS > U.S. Department of Commerce, National Oceanic and Atmospheric Administration, National Environmental Satellite, Data, and Information Services" ;
:project = "GOES" ;
:production_site = "WCDAS" ;
:production_environment = "OE" ;
:spatial_resolution = "2km at nadir" ;
:orbital_slot = "GOES-Test" ;
:platform_ID = "G16" ;
:instrument_type = "GOES R Series Advanced Baseline Imager" ;
:scene_id = "CONUS" ;
:instrument_ID = "FM1" ;
:title = "ABI L1b Radiances" ;
:summary = "Single emissive band ABI L1b Radiance Products are digital maps of outgoing radiance values at the top of the atmosphere for IR bands." ;
:keywords = "SPECTRAL/ENGINEERING > INFRARED WAVELENGTHS > INFRARED RADIANCE" ;
:keywords_vocabulary = "NASA Global Change Master Directory (GCMD) Earth Science Keywords, Version 7.0.0.0.0" ;
:iso_series_metadata_id = "a70be540-c38b-11e0-962b-0800200c9a66" ;
:license = "Unclassified data. Access is restricted to approved users only." ;
:processing_level = "L1b" ;
:cdm_data_type = "Image" ;
:dataset_name = "OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc" ;
:production_data_source = "Realtime" ;
:timeline_id = "ABI Mode 3" ;
:date_created = "2017-02-28T03:40:31.6Z" ;
:time_coverage_start = "2017-02-28T03:37:50.5Z" ;
:time_coverage_end = "2017-02-28T03:40:28.9Z" ;
:id = "037d7688-bd22-4ffb-9727-45e0c2890a72" ;
}

48
nc_test/test_http.sh Executable file

@ -0,0 +1,48 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
#Constants
URL3="http://149.165.169.123:8080/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes"
URL4="http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
# See if netcdf-4 support is enabled
HAVENC4=`cat ${TOPBUILDDIR}/libnetcdf.settings | sed -e '/NetCDF-4[ ]*API:[ ]*yes/p' -e d`
if test "x$HAVENC4" = x ; then HAVENC4=no; else HAVENC4=yes; fi
rm -f tst_http_nc3.cdl tst_http_nc4.cdl
echo ""
echo "*** Testing reading NetCDF-3 file with http"
# Test using -k flag
K=`${NCDUMP} -k "$URL3"`
EXPECTED="classic"
if test "x$K" != "x$EXPECTED" ; then
echo "test_http: -k flag mismatch: expected=$EXPECTED have=$K"
exit 1
fi
# Now test the reading of at least the metadata
${NCDUMP} -h "$URL3" >tst_http_nc3.cdl
# compare
diff tst_http_nc3.cdl ${srcdir}/ref_tst_http_nc3.cdl
if test "x$HAVENC4" = xyes ; then
echo "*** Testing reading NetCDF-4 file with http"
# Test using -k flag
K=`${NCDUMP} -k "$URL4"`
EXPECTED="netCDF-4"
if test "x$K" != "x$EXPECTED" ; then
echo "test_http: -k flag mismatch: expected=$EXPECTED have=$K"
exit 1
fi
# Now test the reading of at least the metadata
${NCDUMP} -h "$URL4" >tst_http_nc4.cdl
# compare
diff tst_http_nc4.cdl ${srcdir}/ref_tst_http_nc4.cdl
fi
exit

89
nc_test/tst_http.c Normal file

@ -0,0 +1,89 @@
/*********************************************************************
* Copyright 1996-2018, UCAR/Unidata
* See COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef HAVE_TIME_H
#include <time.h>
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_UNISTD_H
#include "unistd.h"
#endif
#ifdef _WIN32
#include <io.h>
#endif
#include "netcdf.h"
/*
https://github.com/Unidata/netcdf-c/issues/1251
*/
struct TESTURLS {
int format; /* instance of NC_FORMATX_XXX */
const char* url;
} testurls[] = {
{NC_FORMAT_CLASSIC,"http://149.165.169.123:8080/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes"},
#ifdef USE_NETCDF4
{NC_FORMAT_NETCDF4,"http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"},
#endif
{0,NULL}
};
static int lineno = 0;
static int
fail(int ret)
{
if(ret != NC_NOERR) {
fprintf(stderr,"*** Fail: line: %d: (%d) %s\n", lineno, ret, nc_strerror(ret));
fflush(stderr);
}
return ret;
}
static int
dotest(struct TESTURLS* test)
{
int ret = NC_NOERR;
int ncid;
int format = -1;
/* First, try to open the url */
if((ret = nc_open(test->url,0,&ncid))) return fail(ret);
/* Verify format */
if((ret = nc_inq_format(ncid,&format))) return fail(ret);
if(format != test->format) {
printf("%s: format mismatch: expected %d received %d\n",__FILE__,test->format,format);
return fail(NC_EINVAL);
}
if((ret = nc_close(ncid))) return fail(ret);
return ret;
}
int
main()
{
int ret = NC_NOERR;
struct TESTURLS* test = NULL;
for(test=testurls;test->format;test++) {
if((ret=dotest(test))) goto done;
}
done:
return ret;
}

2
nc_test4/run_grp_rename.sh

@ -114,6 +114,6 @@ else
echo "***XFAIL : attempt to rename root group failed as expected"
fi
#rm -f tst_grp_rename.cdl tst_grp_rename.nc ref_grp_rename.nc
rm -f tst_grp_rename.cdl tst_grp_rename.nc ref_grp_rename.nc
exit $FAIL

1
ncdump/nccopy.c

@ -2223,5 +2223,6 @@ main(int argc, char**argv)
}
#endif /*USE_NETCDF4*/
nc_finalize();
exit(exitcode);
}

2
ncdump/ncdump.c

@ -2389,6 +2389,7 @@ main(int argc, char *argv[])
NC_CHECK( nc_close(ncid) );
}
if(path) {free(path); path = NULL;}
nc_finalize();
exit(EXIT_SUCCESS);
fail: /* ncstat failures */
@ -2398,5 +2399,6 @@ fail: /* ncstat failures */
if(strlen(errmsg) > 0)
error("%s: %s", path, errmsg);
if(path) free(path);
nc_finalize();
exit(EXIT_FAILURE);
}

3
ncgen3/ncgen.l

@ -288,6 +288,7 @@ FloatInf|-?Inff { /* missing value (pre-2.4 backward compatibility) */
void
ignore()
{
#ifndef YY_NO_UNPUT
yyunput(0,NULL);
#endif
}

1139
ncgen3/ncgentab.c

File diff suppressed because it is too large Load Diff

81
ncgen3/ncgentab.h

@ -1,19 +1,19 @@
/* A Bison parser, made by GNU Bison 2.6.4. */
/* A Bison parser, made by GNU Bison 3.0.4. */
/* Bison interface for Yacc-like parsers in C
Copyright (C) 1984, 1989-1990, 2000-2012 Free Software Foundation, Inc.
Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. */
@ -26,13 +26,13 @@
special exception, which will cause the skeleton and the resulting
Bison output files to be licensed under the GNU General Public
License without this special exception.
This special exception was added by the Free Software Foundation in
version 2.2 of Bison. */
#ifndef YY_NCG_NCGEN_TAB_H_INCLUDED
# define YY_NCG_NCGEN_TAB_H_INCLUDED
/* Enabling traces. */
/* Debug traces. */
#ifndef YYDEBUG
# define YYDEBUG 0
#endif
@ -40,57 +40,44 @@
extern int ncgdebug;
#endif
/* Tokens. */
/* Token type. */
#ifndef YYTOKENTYPE
# define YYTOKENTYPE
/* Put the tokens into the symbol table, so that GDB and other debuggers
know about them. */
enum yytokentype {
NC_UNLIMITED_K = 258,
BYTE_K = 259,
CHAR_K = 260,
SHORT_K = 261,
INT_K = 262,
FLOAT_K = 263,
DOUBLE_K = 264,
IDENT = 265,
TERMSTRING = 266,
BYTE_CONST = 267,
CHAR_CONST = 268,
SHORT_CONST = 269,
INT_CONST = 270,
FLOAT_CONST = 271,
DOUBLE_CONST = 272,
DIMENSIONS = 273,
VARIABLES = 274,
NETCDF = 275,
DATA = 276,
FILLVALUE = 277
};
enum yytokentype
{
NC_UNLIMITED_K = 258,
BYTE_K = 259,
CHAR_K = 260,
SHORT_K = 261,
INT_K = 262,
FLOAT_K = 263,
DOUBLE_K = 264,
IDENT = 265,
TERMSTRING = 266,
BYTE_CONST = 267,
CHAR_CONST = 268,
SHORT_CONST = 269,
INT_CONST = 270,
FLOAT_CONST = 271,
DOUBLE_CONST = 272,
DIMENSIONS = 273,
VARIABLES = 274,
NETCDF = 275,
DATA = 276,
FILLVALUE = 277
};
#endif
/* Value type. */
#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
typedef int YYSTYPE;
# define YYSTYPE_IS_TRIVIAL 1
# define yystype YYSTYPE /* obsolescent; will be withdrawn */
# define YYSTYPE_IS_DECLARED 1
#endif
extern YYSTYPE ncglval;
#ifdef YYPARSE_PARAM
#if defined __STDC__ || defined __cplusplus
int ncgparse (void *YYPARSE_PARAM);
#else
int ncgparse ();
#endif
#else /* ! YYPARSE_PARAM */
#if defined __STDC__ || defined __cplusplus
int ncgparse (void);
#else
int ncgparse ();
#endif
#endif /* ! YYPARSE_PARAM */
#endif /* !YY_NCG_NCGEN_TAB_H_INCLUDED */

972
ncgen3/ncgenyy.c

File diff suppressed because it is too large Load Diff