mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-31 17:50:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix the NC_INMEMORY code to work in all cases with HDF5 1.10.

Browse Source 
re: github issue https://github.com/Unidata/netcdf-c/issues/1111

One of the less common use cases for the in-memory feature is
apparently failing with HDF5-1.10.x.  The fix is complicated and
requires significant changes to libhdf5/nc4memcb.c. The current
setup is detailed in the file docs/inmeminternal.dox.

Additionally, it was discovered that the program
nc_test/tst_inmemory.c, which is invoked by
nc_test/run_inmemory.sh, actually was failing because of the
above problem. But the failure is not detected since the script
does not return non-zero value.

Other Changes:
1. Fix nc_test_tst_inmemory to return errors correctly.
2. Make ncdap_tests/findtestserver.c and dap4_tests/findtestserver4.c
   be generated from ncdap_test/findtestserver.c.in.
3. Make LOG() print output to stderr instead of stdout to
   avoid contaminating e.g. ncdump output.
4. Modify the handling of NC_INMEMORY and NC_DISKLESS flags
   to properly handle that NC_DISKLESS => NC_INMEMORY. This
   affects a number of code pieces, especially memio.c.
This commit is contained in:
Dennis Heimbigner 2018-09-04 11:27:47 -06:00
parent 695ba97d24
commit d62a9e623c
45 changed files with 1273 additions and 424 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@ -1972,6 +1972,12 @@ IF(ENABLE_EXAMPLES)
configure_file(${CMAKE_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF)
ENDIF()
#####
# Build ncdap_test|dap4_test/findtestserver[4].c
#####
configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/ncdap_test/findtestserver.c @ONLY NEWLINE_STYLE LF)
configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/dap4_test/findtestserver4.c @ONLY NEWLINE_STYLE LF)
####
# Export files
####

8
cf
View File

@ -108,10 +108,10 @@ FLAGS="$FLAGS --enable-extreme-numbers"
#FLAGS="$FLAGS --disable-testsets"
#FLAGS="$FLAGS --disable-dap-remote-tests"
#FLAGS="$FLAGS --enable-dap-auth-tests" -- requires a new remotetest server
#FLAGS="$FLAGS --enable-doxygen"
#FLAGS="$FLAGS --enable-internal-docs"
FLAGS="$FLAGS --enable-doxygen"
FLAGS="$FLAGS --enable-internal-docs"
FLAGS="$FLAGS --enable-logging"
FLAGS="$FLAGS --disable-diskless"
#FLAGS="$FLAGS --disable-diskless"
#FLAGS="$FLAGS --enable-mmap"
#FLAGS="$FLAGS --with-udunits"
#FLAGS="$FLAGS --with-libcf"
@ -200,7 +200,7 @@ DISTCHECK_CONFIGURE_FLAGS="$FLAGS"
export DISTCHECK_CONFIGURE_FLAGS
if test "x$NB" != x -o "x$FAST" = x ; then
${MAKE} maintainer-clean >/dev/null 2>&1
${MAKE} distclean >/dev/null 2>&1
fi
if test -z "$NB" ; then
if autoreconf -i --force ; then ok=1; else exit ; fi

12
configure.ac
View File

@ -172,8 +172,12 @@ AC_DEFINE([USE_FSYNC], [1], [if true, include experimental fsync code])
fi
# Temporary until JNA bug is fixed (which is probably never).
# See Jira NCF-298
AC_MSG_CHECKING([if jna bug workaround is enabledd])
# The problem being solved is this:
# > On Windows using the microsoft runtime, it is an error
# > for one library to free memory allocated by a different library.
# This is probably only an issue when using the netcdf-c library
# via JNA under Java.
AC_MSG_CHECKING([if jna bug workaround is enabled])
AC_ARG_ENABLE([jna],
[AS_HELP_STRING([--enable-jna],
[enable jna bug workaround])],
@ -1482,6 +1486,10 @@ AC_SUBST([MSVC], [])
AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in)
AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in)
# DAP 2/4 findtestserver[4].c setup
AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in)
AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)
#####
# End netcdf_meta.h definitions.
#####

7
dap4_test/Makefile.am
View File

@ -8,8 +8,11 @@ include $(top_srcdir)/lib_flags.am
#TEST_EXTENSIONS = .sh
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#TESTS_ENVIRONMENT = export SETX=1;
LDADD = ${top_builddir}/liblib/libnetcdf.la
AM_CPPFLAGS += -I$(top_srcdir)/libdap4
@ -62,6 +65,8 @@ CLEANFILES = *.exe
# This should only be left behind if using parallel io
CLEANFILES += tmp_*
DISTCLEANFILES = findtestserver4.c
# One last thing
BUILT_SOURCES = .daprc

4
dap4_test/d4test_common.sh
View File

@ -24,8 +24,8 @@ cd ${srcdir}/baselineremote; BASELINEREM=`pwd` ; cd ${WD}
BASELINEH=${BASELINEREM}
setresultdir() {
rm -fr ./$1
mkdir -p ./$1
rm -fr ${builddir}/$1
mkdir -p ${builddir}/$1
}
FAILURES=

16
dap4_test/test_raw.sh
View File

@ -42,29 +42,29 @@ resultclean() {
}
setresultdir results_test_raw
if test "x${RESET}" = x1 ; then rm -fr ${BASELINERAW}/*.dmp ; fi
for f in $F ; do
echo "testing: $f"
URL="[dap4]file://${DAPTESTFILES}/${f}"
if ! ${VG} ${NCDUMP} "${URL}" > ./results_test_raw/${f}.dmp; then
URL="[log][dap4]file://${DAPTESTFILES}/${f}"
if ! ${NCDUMP} "${URL}" > ${builddir}/results_test_raw/${f}.dmp; then
failure "${URL}"
fi
if test "x${TEST}" = x1 ; then
if ! diff -wBb ${BASELINERAW}/${f}.dmp ./results_test_raw/${f}.dmp ; then
if ! diff -wBb ${BASELINERAW}/${f}.dmp ${builddir}/results_test_raw/${f}.dmp ; then
failure "diff ${f}.dmp"
fi
elif test "x${RESET}" = x1 ; then
echo "${f}:"
cp ./results_test_raw/${f}.dmp ${BASELINERAW}/${f}.dmp
cp ${builddir}/results_test_raw/${f}.dmp ${BASELINERAW}/${f}.dmp
elif test "x${DIFF}" = x1 ; then
echo "hdrtest: ${f}"
rm -f ./tr1 ./tr2 ./tb1 ./tb2
baseclean
if ! diff -wBb ./${BASELINERAW}/${f}.dmp ./${BASELINE}/${f}.ncdump ; then
failure diff -wBb ./${BASELINERAW}/${f}.dmp ./${BASELINE}/${f}.ncdump
if ! diff -wBb ${BASELINERAW}/${f}.dmp ${BASELINE}/${f}.ncdump ; then
failure diff -wBb ${BASELINERAW}/${f}.dmp ${BASELINE}/${f}.ncdump
fi
fi
done
rm -rf ./results_test_raw
rm -rf ${builddir}/results_test_raw
finish

92
dap4_test/test_remote.sh
View File

@ -13,47 +13,47 @@ echo "test_remote.sh:"
#NOCSUM=1
F="\
test_atomic_array.nc
test_atomic_types.nc
test_enum.nc
test_enum_2.nc
test_enum_array.nc
test_fill.nc
test_groups1.nc
test_misc1.nc
test_one_var.nc
test_one_vararray.nc
test_opaque.nc
test_opaque_array.nc
test_struct1.nc
test_struct_array.nc
test_struct_nested.nc
test_struct_nested3.nc
test_struct_type.nc
test_utf8.nc
test_vlen1.nc
test_vlen2.nc
test_vlen3.nc
test_vlen4.nc
test_vlen5.nc
test_vlen6.nc
test_vlen7.nc
test_vlen8.nc
test_vlen9.nc
test_vlen10.nc
test_vlen11.nc
tst_fills.nc
test_struct_nested.hdf5
test_struct_nested3.hdf5
test_vlen3.hdf5
test_vlen4.hdf5
test_vlen5.hdf5
test_anon_dim.syn
test_atomic_array.syn
test_atomic_types.syn
test_sequence_1.syn
test_sequence_2.syn
test_struct_array.syn
test_atomic_array.nc \
test_atomic_types.nc \
test_enum.nc \
test_enum_2.nc \
test_enum_array.nc \
test_fill.nc \
test_groups1.nc \
test_misc1.nc \
test_one_var.nc \
test_one_vararray.nc \
test_opaque.nc \
test_opaque_array.nc \
test_struct1.nc \
test_struct_array.nc \
test_struct_nested.nc \
test_struct_nested3.nc \
test_struct_type.nc \
test_utf8.nc \
test_vlen1.nc \
test_vlen2.nc \
test_vlen3.nc \
test_vlen4.nc \
test_vlen5.nc \
test_vlen6.nc \
test_vlen7.nc \
test_vlen8.nc \
test_vlen9.nc \
test_vlen10.nc \
test_vlen11.nc \
tst_fills.nc \
test_struct_nested.hdf5 \
test_struct_nested3.hdf5 \
test_vlen3.hdf5 \
test_vlen4.hdf5 \
test_vlen5.hdf5 \
test_anon_dim.syn \
test_atomic_array.syn \
test_atomic_types.syn \
test_sequence_1.syn \
test_sequence_2.syn \
test_struct_array.syn \
"
setresultdir results_test_remote
@ -66,27 +66,27 @@ fi
if test "x${RESET}" = x1 ; then rm -fr ${BASELINER}/*.dmp ; fi
for f in $F ; do
URL="[log][dap4]${TESTSERVER}/d4ts/testfiles/${f}"
URL="[log][show=fetch][dap4]${TESTSERVER}/testfiles/${f}"
if test "x$BIG" = x1; then
URL="[ucar.littleendian=0]${URL}"
fi
if test "x$NOCSUM" = x1; then
URL="[ucar.checksummode=none]${URL}"
fi
if ! ${VG} ${NCDUMP} "${URL}" > ./results_test_remote/${f}.dmp; then
if ! ${NCDUMP} "${URL}" > ${builddir}/results_test_remote/${f}.dmp; then
failure "${URL}"
fi
if test "x${TEST}" = x1 ; then
if ! diff -wBb ${BASELINEREM}/${f}.dmp ./results_test_remote/${f}.dmp ; then
if ! diff -wBb "${BASELINEREM}/${f}.dmp" "${builddir}/results_test_remote/${f}.dmp" ; then
failure "diff ${f}.dmp"
fi
elif test "x${RESET}" = x1 ; then
echo "${f}:"
cp ./results_test_remote/${f}.dmp ${BASELINEREM}/${f}.dmp
cp "${builddir}/results_test_remote/${f}.dmp" "${BASELINEREM}/${f}.dmp"
fi
done
rm -fr ./results_test_remote
rm -fr ${builddir}/results_test_remote
finish

1
docs/Doxyfile.in
View File

@ -768,6 +768,7 @@ INPUT = \
@abs_top_srcdir@/docs/bestpractices.md \
@abs_top_srcdir@/docs/tutorial.dox \
@abs_top_srcdir@/docs/internal.dox \
@abs_top_srcdir@/docs/inmeminternal.dox \
@abs_top_srcdir@/docs/indexing.dox \
@abs_top_srcdir@/docs/testserver.dox \
@abs_top_srcdir@/include/netcdf.h \

2
docs/Makefile.am
View File

@ -11,7 +11,7 @@ install-fortran.md all-error-codes.md credits.md auth.md \
obsolete/fan_utils.html bestpractices.md filters.md indexing.dox \
inmemory.md DAP4.dox OPeNDAP.dox attribute_conventions.md FAQ.md \
file_format_specifications.md known_problems.md COPYRIGHT.dox \
user_defined_formats.md
user_defined_formats.md inmeminternal.dox
# Turn off parallel builds in this directory.
.NOTPARALLEL:

401
docs/inmeminternal.dox Normal file
View File

@ -0,0 +1,401 @@
/**
@if INTERNAL
@page inmemintern Internal Architecture for NC_INMEMORY Support
\tableofcontents
<!-- Note that this file has the .dox extension, but is mostly markdown -->
<!-- Begin MarkDown -->
# Introduction {#inmemintern_intro}
This document describes the internal workings
of the inmemory features of the netcdf-c library.
The companion document to this -- inmemory.md --
describes the "external" operation of the inmemory features.
This document describes how the in-memory operation
is implemented both for netcdf-3 files and for netcdf-4 files.
# Generic Capabilities {#inmemintern_general}
Both the netcdf-3 and netcdf-4 implementations assume that
they are initially given a (pointer,size) pair representing
a chunk of allocated memory of specified size.
If a file is being created instead of opened, then only the size
is needed and the netcdf-c library will internally allocate the
corresponding memory chunk.
If NC_DISKLESS is being used, then a chunk of memory is allocated
whose size is the same as the length of the file, and the contents
of the file is then read into that chunk of memory.
This information is in general represented by the following struct
(see include/netcdf_mem.h).
````
typedef struct NC_memio {
size_t size;
void* memory;
int flags;
} NC_memio;
````
The flags field describes properties and constraints to be applied
to the given memory. At the moment, only this one flag is defined.
````
#define NC_MEMIO_LOCKED 1
````
If this flag is set, then the netcdf library will ensure that
the original allocated memory is ```locked```, which means
that it will never be realloc'd nor free'd.
Note that this flag is ignored when creating a memory file: it is only
relevant when opening a pre-allocated chunk of memory via the
_nc_open_mem_ function.
Note that this flag does not prevent the memory from being modified.
If there is room, then the memory may be modified in place. If the size
of the memory needs to be increased and the this flag is set, then
the operation will fail.
When the _nc_close_memio_ function is called instead of
_nc_close_, then the currently allocated memory (and its size)
is returned. If the _NC_MEMIO_LOCKED_ flag is set, then it
should be the case that the chunk of memory returned is the same
as originally provided. However, the size may be different
because it represents the amount of memory that contains
meaningful data; this value may be less than the original provided size.
The actual allocated size for the memory chunk is the same as originally
provided, so it that value is needed, then the caller must save it somewhere.
Note also that ownership of the memory chunk is given to the
caller, and it is the caller's responsibility to _free_ the memory.
# NetCDF-4 Implementation {#inmemintern_nc4}
The implementation of in-memory support for netcdf-4 files
is quite complicated.
The netCDF-4 implemention relies on the HDF5 library. In order
to implement in-memory storage of data, the HDF5 core driver is
used to manage underlying storage of the netcdf-c file.
An HDF5 driver is an abstract interface that allows different
underlying storage implementations. So there is a standard file
driver as well as a core driver, which uses memory as the
underlying storage.
Generically, the memory is referred to as a file image [1].
## libhdf5/nc4mem
The primary API for in-memory operations is in the file
libhdf5/nc4mem.c and the defined functions are described in the next sections
### nc4mem.NC4_open_image_file
The signature is:
````
int NC4_open_image_file(NC_FILE_INFO_T* h5)
````
Basically, this function sets up the necessary state information
to use the HDF5 core driver.
It obtains the memory chunk and size from the _h5->mem.memio_ field.
Specifically, this function converts the
_NC_MEMIO_LOCKED_ flag into using the HDF5 image specific flags:
_H5LT_FILE_IMAGE_DONT_COPY_ and _H5LT_FILE_IMAGE_DONT_RELEASE_.
It then invokes the function _libhdf5/nc4memcb/NC4_image_init_
function to do the necessary HDF5 specific setup.
### nc4mem.NC4_create_image_file
The signature is:
````
int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t initialsize)
````
This function sets up the necessary state information
to use the HDF5 core driver, but for a newly created file.
It initializes the memory chunk and size in the _h5->mem.memio_ field
from the _initialsize_ argument and it leaves the memory chunk pointer NULL.
It ignores the _NC_MEMIO_LOCKED_ flag.
It then invokes the function _libhdf5/nc4memcb/NC4_image_init_
function to do the necessary HDF5 specific setup.
### libhdf5/hdf5file.c/nc4_close-netcdf4_file
When a file is closed, this function is invoked. As part of its operation,
and if the file is an in-memory file, it does one of two things.
1. If the user provided an _NC_memio_ instance, then return the final image
in that instance; the user is then responsible for freeing it.
2. If no _NC_memio_ instance was provided, then just discard the final image.
## libhdf5/nc4memcb
The HDF5 core driver uses an abstract interface for managing the
allocation and free'ing of memory. This interface is defined
as a set of callback functions [2] that implement the functions
of this struct.
````
typedef struct {
void *(*_malloc)(size_t size, H5_file_image_op_t op, void *udata);
void *(*_memcpy)(void *dest, const void *src, size_t size,
H5_file_image_op_t op, void *udata);
void *(*_realloc)(void *ptr, size_t size,
H5_file_image_op_t op, void *udata);
herr_t (*_free)(void *ptr, H5_file_image_op_t op, void *udata);
void *(*udata_copy)(void *udata);
herr_t (*udata_free)(void *udata);
void *udata;
} H5_file_image_callbacks_t;
````
The _udata_ field at the end defines any extra state needed by the functions.
Each function is passed the udata as its last argument. The structure of the
udata is arbitrary, and is passed as _void*_ to the functions.
The _udata_ structure and callback functions used by the netcdf-c library
are defined in the file _libhdf5/nc4memcb.c_. Setup is defined by the
function _NC4_image_init_ in that same file.
The _udata_ structure used by netcdf is as follows.
````
typedef struct {
void *app_image_ptr; /* Pointer to application buffer */
size_t app_image_size; /* Size of application buffer */
void *fapl_image_ptr; /* Pointer to FAPL buffer */
size_t fapl_image_size; /* Size of FAPL buffer */
int fapl_ref_count; /* Reference counter for FAPL buffer */
void *vfd_image_ptr; /* Pointer to VFD buffer (Note: VFD => used by core driver) */
size_t vfd_image_size; /* Size of VFD buffer */
int vfd_ref_count; /* Reference counter for VFD buffer */
unsigned flags; /* Flags indicate how the file image will be opened */
int ref_count; /* Reference counter on udata struct */
NC_FILE_INFO_T* h5; /* Pointer to the netcdf parent structure */
} H5LT_file_image_ud_t;
````
It is necessary to understand one more point about the callback functions.
The first four take an argument of type _H5_file_image_op_t_ -- the operator.
This is an enumeration that indicates additional context about the purpose for
which the callback is being invoked. For the purposes of the netcdf-4
implementation, only the following operators are used.
- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET
- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY
- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET
- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE
- H5FD_FILE_IMAGE_OP_FILE_OPEN
- H5FD_FILE_IMAGE_OP_FILE_RESIZE
- H5FD_FILE_IMAGE_OP_FILE_CLOSE
As can be seen, basically the operators indicate if the operation is with respect to
an HDF5 property list, or with respect to a file (i.e. a core image in this case).
For each callback described below, the per-operator actions will be described.
Not all operators are used with all callbacks.
Internally, the HDF5 core driver thinks it is doing the following:
1. Allocate memory and copy the incoming memory chunk into that newly allocated memory
(call image_malloc followed by image_memcpy).
2. Periodically reallocate the memory to increase its size
(call image_realloc).
3. Free up the memory as no longer needed
(call image_free).
It turns out that for propertly lists, realloc is never called.
However the HDF5 core driver follows all of the above steps.
The following sections describe the callback function operation.
### libhdf5/nc4memcb/local_image_malloc
This function is called to allocated an internal chunk of memory so
the original provided memory is no longer needed. In order to implement
the netcdf-c semantics, we modify this behavior.
#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET
We assume that the property list image info will never need to be modified,
so we just copy the incoming buffer info (the app_image fields) into the fapl_image fields.
#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY
Basically just return the fapl_image_ptr field, so no actual copying.
#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY and H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET
Basically just return the fapl_image_ptr field, so no actual copying or malloc needed.
#### Operator H5FD_FILE_IMAGE_OP_FILE_OPEN
Since we always start by using the original incoming image buffer, we just
need to store that pointer and size into the vfd_image fields (remember, vfd is that
used by the core driver).
### libhdf5/nc4memcb/local_image_memcpy
This function is supposed to be used to copy the incoming buffer into an internally
malloc'd buffer. Since we use the original buffer, no memcpy is actually needed.
As a safety check, we do actually do a memcpy if, for some reason, the _src_ and _dest_
arguments are different. In practice, this never happens.
### libhdf5/nc4memcb/local_image_realloc
Since the property list image is never realloc'd this is only called with
_H5FD_FILE_IMAGE_OP_FILE_RESIZE_.
If the memory is not locked (i.e. the _NC_MEMIO_LOCKED_ flag was not used),
then we are free to realloc the vfd_ptr. But if the memory is locked,
then we cannot realloc and we must fake it as follows:
1. If the chunk is big enough, then pretend to do a realloc by
changing the vfd_image_size.
2. If the chunk is not big enough to accomodate the requested new size,
then fail.
There is one important complication. It turns out that the image_realloc
callback is sometimes called with a ptr argument value of NULL. This assumes
that if realloc is called with a NULL buffer pointer, then it acts like _malloc_.
Since we have found that some systems to do not implement this, we implement it
in our local_image_realloc code and do a _malloc_ instead of _realloc_.
### libhdf5/nc4memcb/local_image_free
This function is, of course, invoked to deallocate memory.
It is only invoked with the
H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE
and H5FD_FILE_IMAGE_OP_FILE_CLOSE
operators.
#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE
For the way the netcdf library uses it, it should still be the case that
the fapl pointer is same as original incoming app_ptr, so we do not need
to do anything for this operator.
#### Operator H5FD_FILE_IMAGE_OP_FILE_CLOSE
Since in our implementation, we maintain control of the memory, this case
will never free any memory, but may save a pointer to the current vfd memory
so it can be returned to the original caller, if they want it.
Specifically the vfd_image_ptr and vfd_image_size are always
copied to the _udata->h5->mem.memio_ field so they can
be referenced by higher level code.
### libhdf5/nc4memcb/local_udata_copy
Our version of this function only manipulates the reference count.
### libhdf5/nc4memcb/local_udata_free
Our version of this function only manipulates the reference count.
# NetCDF-3 Implementation {#inmemintern_nc3}
The netcdf-3 code -- in libsrc -- has its own, internal storage
management API as defined in the file _libsrc/ncio.h_. It implements
the API in the form of a set of function pointers as defined in the
structure _struct_ _ncio_. These function have the following signatures
and semantics.
- int ncio_relfunc(ncio*, off_t offset, int rflags) --
Indicate that you are done with the region which begins at offset.
- int ncio_getfunc(ncio*, off_t offset, size_t extent, int rflags, void **const vpp) --
Request that the region (offset, extent) be made available through *vpp.
- int ncio_movefunc(ncio*, off_t to, off_t from, size_t nbytes, int rflags) --
Like memmove(), safely move possibly overlapping data.
- int ncio_syncfunc(ncio*) --
Write out any dirty buffers to disk and ensure that next read will get data from disk.
- int ncio_pad_lengthfunc(ncio*, off_t length) --
Sync any changes to disk, then truncate or extend file so its size is length.
- int ncio_closefunc(ncio*, int doUnlink)
-- Write out any dirty buffers and ensure that next read will not get cached data.
Then sync any changes, and then close the open file.
The _NC_INMEMORY_ semantics are implemented by creating an implementation of the above functions
specific for handling in-memory support. This is implemented in the file _libsrc/memio.c_.
## Open/Create/Close
Open and close related functions exist in _memio.c_ that are not specifically part of the API.
These functions are defined in the following sections.
### memio_create
Signature:
````
int memio_create(const char* path, int ioflags, size_t initialsz, off_t igeto, size_t igetsz, size_t* sizehintp, void* parameters /*ignored*/, ncio* *nciopp, void** const mempp)
````
Create a new file. Invoke _memio_new_ to create the _ncio_
instance. If it is intended that the resulting file be
persisted to the file system, then verify that writing such a
file is possible. Also create an initial in-memory buffer to
hold the file data. Otherwise act like e.g. _posixio_create_.
### memio_open
Signature:
````
int memio_open(const char* path, int ioflags, off_t igeto, size_t igetsz, size_t* sizehintp, void* parameters, ncio* *nciopp, void** const mempp)
````
Open an existing file. Invoke _memio_new_ to create the _ncio_
instance. If it is intended that the resulting file be
persisted to the file system, then verify that writing such a
file is possible. Also create an initial in-memory buffer to
hold the file data. Read the contents of the existing file into
the allocated memory.
Otherwise act like e.g. _posixio_open_.
### memio_extract
Signature:
````
int memio_extract(ncio* const nciop, size_t* sizep, void** memoryp)
````
This function is called as part of the NC3_close function in the event that
the user wants the final in-memory chunk returned to them via _nc_close_mem_.
It captures the existing in-memory chunk and returns it. At this point,
memio will no longer have access to that memory.
## API Semantics
The semantic interaction of the above API and NC_INMEMORY are described in the following sections.
### ncio_relfunc
Just unlock the in-memory chunk.
### ncio_getfunc
First guarantee that the requested region exists, and if necessary,
realloc to make it exist. If realloc is needed, and the
file is locked, then fail.
### ncio_movefunc
First guarantee that the requested destination region exists, and if necessary,
realloc to make it exist. If realloc is needed, and the
file is locked, then fail.
### ncio_syncfunc
This is a no-op as far as memio is concerned.
### ncio_pad_lengthfunc
This may realloc the allocated in-memory buffer to achieve padding
rounded up to the pagesize.
### ncio_filesizefunc
This just returns the used size of the in-memory chunk.
Note that the allocated size might be larger.
### ncio_closefunc
If the usere wants the contents persisted, then write out the used portion
of the in-memory chunk to the target file.
Then, if the in-memory chunk is not locked, or for some reason has
been modified, go ahead and free that memory.
# References {#inmemintern_bib}
1. https://support.hdfgroup.org/HDF5/doc1.8/Advanced/FileImageOperations/HDF5FileImageOperations.pdf
2. https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetFileImageCallbacks
# Point of Contact {#inmemintern_poc}
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu<br>
__Initial Version__: 8/28/2018<br>
__Last Revised__: 8/28/2018
<!-- End MarkDown -->
@endif
*/

13
docs/inmemory.md
View File

@ -129,16 +129,19 @@ Note that it is still possible to modify the in-memory file if the NC_WRITE
mode flag was set. However, failures can occur if an operation
cannot complete because the memory needs to be expanded.
2. If the *NC_MEMIO_LOCKED* flag is <b>not</b> set, then
the netcdf library will take control of the incoming memory
and will feel free to reallocate the provided
the netcdf library will take control of the incoming memory.
This means that the user should not make any attempt to free
or even read the incoming memory block in this case.
The newcdf library is free to reallocate the incomming
memory block to obtain a larger block when an attempt to modify
the in-memory file requires more space. Note that implicit in this
is that the old block -- the one originally provided -- may be
free'd as a side effect of re-allocating the memory using the
*realloc()* function.
If the caller invokes the *nc_close_memio()* function to retrieve the
final memory block, the returned block must always be freed
by the caller and that the original block should not be freed.
The caller may invoke the *nc_close_memio()* function to retrieve the
final memory block, which may not be the same as the originally block
provided by the caller. In any case, the returned block must always be freed
by the caller and the original block should not be freed.
### The **nc_create_mem** Function

3
include/hdf5internal.h
View File

@ -60,6 +60,9 @@ typedef struct NC_HDF5_FILE_INFO
int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
/* In-memory functions */
extern hid_t NC4_image_init(NC_FILE_INFO_T* h5);
extern void NC4_image_finalize(void*);
/* These functions are internal to the libhdf5 directory. */
int nc4_detect_preserve_dimids(NC_GRP_INFO_T *grp, nc_bool_t *bad_coord_orderp);
int hdf5_set_log_level();

16
include/nc4internal.h
View File

@ -301,15 +301,15 @@ typedef struct NC_FILE_INFO
void *format_file_info;
struct NCFILEINFO* fileinfo;
struct NC4_Memio {
NC_memio memio;
int locked; /* do not copy and do not release */
NC_memio memio; /* What we sent to image_init and what comes back*/
int locked; /* do not copy and do not free */
int persist; /* Should file be persisted out on close? */
int inmemory;
int diskless;
unsigned int flags; /* for H5LTopen_file_image */
int fapl;
size_t initialsize;
int inmemory; /* NC_INMEMORY flag was set */
int diskless; /* NC_DISKLESS flag was set => inmemory */
int created; /* 1 => create, 0 => open */
unsigned int imageflags; /* for H5LTopen_file_image */
size_t initialsize;
void* udata; /* extra memory allocated in NC4_image_init */
} mem;
} NC_FILE_INFO_T;
@ -414,7 +414,7 @@ int nc4_check_dup_name(NC_GRP_INFO_T *grp, char *norm_name);
int nc4_get_default_fill_value(const NC_TYPE_INFO_T *type_info, void *fill_value);
/* Close the file. */
int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem);
int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio*);
/* HDF5 initialization */
extern int nc4_hdf5_initialized;

4
include/ncdap.h
View File

@ -17,8 +17,8 @@ affect the operation of the system.
*/
typedef unsigned int NCFLAGS;
# define SETFLAG(controls,flag) ((controls.flags) |= (flag))
# define CLRFLAG(controls,flag) ((controls.flags) &= ~(flag))
# define SETFLAG(controls,flag) (((controls).flags) |= (flag))
# define CLRFLAG(controls,flag) (((controls).flags) &= ~(flag))
# define FLAGSET(controls,flag) (((controls.flags) & (flag)) != 0)
/* Defined flags */

2
include/nctestserver.h
View File

@ -55,7 +55,7 @@ This indicates that the server is up and running.
Return the complete url for the server plus the path.
*/
char*
static char*
nc_findtestserver(const char* path, int isdap4, const char* serverlist)
{
char** svc;

24
include/ncwinpath.h
View File

@ -8,6 +8,9 @@
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "ncexternl.h"
#ifndef WINPATH
@ -19,6 +22,19 @@
#endif
#endif
/* Define wrapper constants for use with NCaccess */
#ifdef _MSC_VER
#define ACCESS_MODE_EXISTS 0
#define ACCESS_MODE_R 4
#define ACCESS_MODE_W 2
#define ACCESS_MODE_RW 6
#else
#define ACCESS_MODE_EXISTS (F_OK)
#define ACCESS_MODE_R (R_OK)
#define ACCESS_MODE_W (W_OK)
#define ACCESS_MODE_RW (R_OK|W_OK)
#endif
/* Path Converter */
EXTERNL char* NCpathcvt(const char* path);
@ -27,10 +43,18 @@ EXTERNL char* NCpathcvt(const char* path);
EXTERNL FILE* NCfopen(const char* path, const char* flags);
EXTERNL int NCopen3(const char* path, int flags, int mode);
EXTERNL int NCopen2(const char* path, int flags);
EXTERNL int NCaccess(const char* path, int mode);
EXTERNL int NCremove(const char* path);
#else /*!WINPATH*/
#define NCfopen(path,flags) fopen((path),(flags))
#define NCopen3(path,flags,mode) open((path),(flags),(mode))
#define NCopen2(path,flags) open((path),(flags))
#define NCremove(path) remove(path)
#ifdef _MSC_VER
#define NCaccess(path,mode) _access(path,mode)
#else
#define NCaccess(path,mode) access(path,mode)
#endif
#endif /*WINPATH*/
#endif /* _NCWINIO_H_ */

2
include/netcdf_mem.h
View File

@ -17,8 +17,8 @@
typedef struct NC_memio {
size_t size;
void* memory;
#define NC_MEMIO_LOCKED 1 /* Do not try to realloc or free provided memory */
int flags;
#define NC_MEMIO_LOCKED 1 /* Do not try to realloc or free provided memory */
} NC_memio;
#if defined(__cplusplus)

5
libdap2/test_vara.c
View File

@ -2,9 +2,8 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <netcdf.h>
extern char* nc_findtestserver(const char* path, int isdap4);
#include "netcdf.h"
#include "nctestserver.h"
#define DTSTEST "/dts/test.06"

2
libdap4/d4file.c
View File

@ -304,7 +304,7 @@ freeInfo(NCD4INFO* d4info)
for some reason, so we delete it ourselves.
*/
if(d4info->substrate.filename != NULL) {
unlink(d4info->substrate.filename);
// unlink(d4info->substrate.filename);
}
}
nullfree(d4info->substrate.filename); /* always reclaim */

70
libdap4/d4read.c
View File

@ -9,8 +9,24 @@
/*Forward*/
static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, long*);
static int readfile(const NCURI*, const char* suffix, NCbytes* packet);
static int readfiletofile(const NCURI*, const char* suffix, FILE* stream, d4size_t*);
static int readfile(NCD4INFO* state, const NCURI*, const char* suffix, NCbytes* packet);
static int readfiletofile(NCD4INFO* state, const NCURI*, const char* suffix, FILE* stream, d4size_t*);
#ifdef HAVE_GETTIMEOFDAY
static struct timeval time0;
static struct timeval time1;
static double
deltatime()
{
double t0, t1;
t0 = ((double)time0.tv_sec);
t0 += ((double)time0.tv_usec) / 1000000.0;
t1 = ((double)time1.tv_sec);
t1 += ((double)time1.tv_usec) / 1000000.0;
return (t1 - t0);
}
#endif
int
NCD4_readDMR(NCD4INFO* state)
@ -38,7 +54,7 @@ NCD4_readDAP(NCD4INFO* state, int flags)
NCURI* url = state->uri;
int fileprotocol = (strcmp(url->protocol,"file")==0);
if(fileprotocol) {
stat = readfiletofile(url, ".dap", state->data.ondiskfile, &state->data.datasize);
stat = readfiletofile(state, url, ".dap", state->data.ondiskfile, &state->data.datasize);
} else {
char* readurl = NULL;
int flags = 0;
@ -51,15 +67,11 @@ NCD4_readDAP(NCD4INFO* state, int flags)
readurl = ncuribuild(url,NULL,".dods",NCURISVC);
if(readurl == NULL)
return THROW(NC_ENOMEM);
if (state->debug > 0)
{fprintf(stderr, "fetch url=%s\n", readurl);fflush(stderr);}
stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile,
&state->data.datasize, &lastmod);
nullfree(readurl);
if(stat == NC_NOERR)
state->data.daplastmodified = lastmod;
if (state->debug > 0)
{fprintf(stderr,"fetch complete\n"); fflush(stderr);}
}
}
return THROW(stat);
@ -89,7 +101,7 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, long* las
if(fileprotocol) {
/* Short circuit file://... urls*/
/* We do this because the test code always needs to read files*/
stat = readfile(url,suffix,packet);
stat = readfile(state, url,suffix,packet);
} else {
char* fetchurl = NULL;
int flags = NCURIBASE;
@ -97,13 +109,23 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, long* las
flags |= NCURIENCODE;
fetchurl = ncuribuild(url,NULL,suffix,flags);
MEMCHECK(fetchurl);
if(state->debug > 0)
{fprintf(stderr,"fetch url=%s\n",fetchurl); fflush(stderr);}
if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) {
nclog(NCLOGDBG,"fetch url=%s",fetchurl);
#ifdef HAVE_GETTIMEOFDAY
gettimeofday(&time0,NULL);
#endif
}
stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified);
nullfree(fetchurl);
if(stat) goto fail;
if(state->debug > 0)
{fprintf(stderr,"fetch complete\n"); fflush(stderr);}
if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) {
double secs = 0;
#ifdef HAVE_GETTIMEOFDAY
gettimeofday(&time1,NULL);
secs = deltatime();
#endif
nclog(NCLOGDBG,"fetch complete: %0.3f",secs);
}
}
#ifdef D4DEBUG
{
@ -116,12 +138,12 @@ fail:
}
static int
readfiletofile(const NCURI* uri, const char* suffix, FILE* stream, d4size_t* sizep)
readfiletofile(NCD4INFO* state, const NCURI* uri, const char* suffix, FILE* stream, d4size_t* sizep)
{
int stat = NC_NOERR;
NCbytes* packet = ncbytesnew();
size_t len;
stat = readfile(uri,suffix,packet);
stat = readfile(state, uri,suffix,packet);
#ifdef D4DEBUG
fprintf(stderr,"readfiletofile: packet.size=%lu\n",
(unsigned long)ncbyteslength(packet));
@ -147,7 +169,7 @@ unwind:
}
static int
readfile(const NCURI* uri, const char* suffix, NCbytes* packet)
readfile(NCD4INFO* state, const NCURI* uri, const char* suffix, NCbytes* packet)
{
int stat = NC_NOERR;
NCbytes* tmp = ncbytesnew();
@ -159,6 +181,24 @@ readfile(const NCURI* uri, const char* suffix, NCbytes* packet)
filename = ncbytesextract(tmp);
ncbytesfree(tmp);
state->fileproto.filename = strdup(filename);
if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) {
char* surl = NULL;
#ifdef HAVE_GETTIMEOFDAY
gettimeofday(&time0,NULL);
#endif
surl = ncuribuild((NCURI*)uri,NULL,NULL,NCURIALL);
nclog(NCLOGDBG,"fetch uri=%s file=%s",surl,filename);
}
stat = NC_readfile(filename,packet);
if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) {
double secs;
#ifdef HAVE_GETTIMEOFDAY
gettimeofday(&time1,NULL);
secs = deltatime();
#endif
nclog(NCLOGDBG,"fetch complete: %0.3f",secs);
}
return THROW(stat);
}

4
libdap4/ncd4types.h
View File

@ -280,7 +280,6 @@ struct NCD4curl {
struct NCD4INFO {
NC* controller; /* Parent instance of NCD4INFO */
int debug;
char* rawurltext; /* as given to ncd4_open */
char* urltext; /* as modified by ncd4_open */
NCURI* uri; /* parse of rawuritext */
@ -307,6 +306,9 @@ struct NCD4INFO {
char substratename[NC_MAX_NAME];
} controls;
NCauth auth;
struct {
char* filename;
} fileproto;
};
#endif /*D4TYPES_H*/

2
libdispatch/dauth.c
View File

@ -30,7 +30,7 @@ See LICENSE.txt for license information.
/* Define the curl flag defaults in envv style */
static const char* AUTHDEFAULTS[] = {
"HTTP.TIMEOUT","10", /*seconds */
"HTTP.TIMEOUT","1800", /*seconds */ /* Long but not infinite */
NULL
};

70
libdispatch/dfile.c
View File

@ -45,7 +45,8 @@ struct MagicFile {
long long filelen;
int use_parallel;
int inmemory;
void* parameters;
int diskless;
void* parameters; /* !NULL if inmemory && !diskless */
FILE* fp;
#ifdef USE_PARALLEL
MPI_File fh;
@ -292,9 +293,10 @@ NC_check_file_type(const char *path, int flags, void *parameters,
{
char magic[MAGIC_NUMBER_LEN];
int status = NC_NOERR;
int diskless = ((flags & NC_DISKLESS) == NC_DISKLESS);
int inmemory = (!diskless && ((flags & NC_INMEMORY) == NC_INMEMORY));
int inmemory = ((flags & NC_INMEMORY) == NC_INMEMORY);
int mmap = ((flags & NC_MMAP) == NC_MMAP);
#ifdef USE_PARALLEL
int use_parallel = ((flags & NC_MPIIO) == NC_MPIIO);
#endif /* USE_PARALLEL */
@ -303,25 +305,20 @@ NC_check_file_type(const char *path, int flags, void *parameters,
*model = 0;
*version = 0;
assert(inmemory ? !mmap : 1); /* inmemory => !mmap */
assert((diskless && inmemory) ? !mmap : 1);/*diskless & inmemory => !mmap*/
memset((void*)&file,0,sizeof(file));
file.path = path; /* do not free */
file.parameters = parameters;
if(inmemory && parameters == NULL)
{status = NC_EINMEMORY; goto done;}
if(inmemory) {
file.inmemory = inmemory;
goto next;
}
/* presumably a real file */
file.inmemory = inmemory;
file.diskless = diskless;
#ifdef USE_PARALLEL
/* presumably a real file */
/* for parallel, use the MPI functions instead (why?) */
if (use_parallel) {
file.use_parallel = use_parallel;
goto next;
}
file.use_parallel = use_parallel;
#endif /* USE_PARALLEL */
next:
status = openmagic(&file);
if(status != NC_NOERR) {goto done;}
/* Verify we have a large enough file */
@ -1954,6 +1951,8 @@ static int
check_create_mode(int mode)
{
int mode_format;
int mmap = 0;
int inmemory = 0;
/* This is a clever check to see if more than one format bit is
* set. */
@ -1967,9 +1966,14 @@ check_create_mode(int mode)
if (mode & NC_MPIIO && mode & NC_MPIPOSIX)
return NC_EINVAL;
/* Can't use both parallel and diskless. */
if ((mode & NC_MPIIO && mode & NC_DISKLESS) ||
(mode & NC_MPIPOSIX && mode & NC_DISKLESS))
mmap = ((mode & NC_MMAP) == NC_MMAP);
inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY);
if(mmap && inmemory) /* cannot have both */
return NC_EINMEMORY;
/* Can't use both parallel and diskless|inmemory. */
if ((mode & NC_MPIIO && mode & (NC_DISKLESS|NC_INMEMORY))
|| (mode & NC_MPIPOSIX && mode & (NC_DISKLESS|NC_INMEMORY)))
return NC_EINVAL;
#ifndef USE_NETCDF4
@ -2029,11 +2033,19 @@ NC_create(const char *path0, int cmode, size_t initialsz,
int model = NC_FORMATX_UNDEFINED; /* one of the NC_FORMATX values */
int isurl = 0; /* dap or cdmremote or neither */
char* path = NULL;
int mmap = 0;
int diskless = 0;
TRACE(nc_create);
if(path0 == NULL)
return NC_EINVAL;
/* Fix the inmemory related flags */
mmap = ((cmode & NC_MMAP) == NC_MMAP);
diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS);
/* diskless && !mmap => inmemory */
if(diskless && !mmap) cmode |= NC_INMEMORY;
/* Check mode flag for sanity. */
if ((stat = check_create_mode(cmode)))
return stat;
@ -2197,6 +2209,7 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp,
NC_Dispatch* dispatcher = NULL;
int inmemory = 0;
int diskless = 0;
int mmap = 0;
/* Need pieces of information for now to decide model*/
int model = 0;
int isurl = 0;
@ -2210,14 +2223,23 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp,
if(stat) return stat;
}
/* Fix the inmemory related flags */
mmap = ((cmode & NC_MMAP) == NC_MMAP);
diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS);
/* diskless && !mmap => inmemory */
if(diskless && !mmap) cmode |= NC_INMEMORY;
inmemory = ((cmode & NC_INMEMORY) == NC_INMEMORY);
if(mmap && inmemory) /* cannot have both */
return NC_EINMEMORY;
/* Attempt to do file path conversion: note that this will do
nothing if path is a 'file:...' url, so it will need to be
repeated in protocol code: libdap2 and libdap4
*/
inmemory = ((cmode & NC_INMEMORY) == NC_INMEMORY);
diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS);
#ifdef WINPATH
path = NCpathcvt(path0);
#else
@ -2272,6 +2294,7 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp,
if(useparallel) flags |= NC_MPIIO;
if(inmemory) flags |= NC_INMEMORY;
if(diskless) flags |= NC_DISKLESS;
if(mmap) flags |= NC_MMAP;
stat = NC_check_file_type(path,flags,parameters,&model,&version);
if(stat == NC_NOERR) {
if(model == 0) {
@ -2476,7 +2499,8 @@ static int
openmagic(struct MagicFile* file)
{
int status = NC_NOERR;
if(file->inmemory) {
assert((!file->diskless && file->inmemory) ? file->parameters != NULL : 1);
if(file->inmemory && !file->diskless) {
/* Get its length */
NC_memio* meminfo = (NC_memio*)file->parameters;
file->filelen = (long long)meminfo->size;
@ -2538,7 +2562,7 @@ readmagic(struct MagicFile* file, long pos, char* magic)
{
int status = NC_NOERR;
memset(magic,0,MAGIC_NUMBER_LEN);
if(file->inmemory) {
if(file->inmemory && !file->diskless) {
char* mempos;
NC_memio* meminfo = (NC_memio*)file->parameters;
if((pos + MAGIC_NUMBER_LEN) > meminfo->size)

2
libdispatch/dvar.c
View File

@ -647,7 +647,7 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
* @return ::NC_EBADID Bad ncid.
* @return ::NC_ENOTVAR Variable not found.
* @return ::NC_ENOMEM Out of memory.
* @return ::NC_EINVALCOORS Missing start array.
* @return ::NC_EINVALCOORDS Missing start array.
* @author Ed Hartnett
*/
int

33
libdispatch/dwinpath.c
View File

@ -202,4 +202,37 @@ NCopen2(const char *path, int flags)
return NCopen3(path,flags,0);
}
/*
Provide wrappers for other file system functions
*/
/* Return access applied to path+mode */
EXTERNL
int
NCaccess(const char* path, int mode)
{
int status = 0;
char* cvtname = NCpathcvt(path);
if(cvtname == NULL) return -1;
#ifdef _MSC_VER
status = _access(cvtname,mode);
#else
status = access(cvtname,mode);
#endif
free(cvtname);
return status;
}
EXTERNL
int
NCremove(const char* path)
{
int status = 0;
char* cvtname = NCpathcvt(path);
if(cvtname == NULL) return ENOENT;
status = remove(cvtname);
free(cvtname);
return status;
}
#endif /*WINPATH*/

12
libhdf5/hdf5create.c
View File

@ -155,10 +155,6 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
}
}
#else /* only set cache for non-parallel... */
if(cmode & NC_DISKLESS) {
if (H5Pset_fapl_core(fapl_id, 4096, nc4_info->mem.persist))
BAIL(NC_EDISKLESS);
}
if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
nc4_chunk_cache_preemption) < 0)
BAIL(NC_EHDFERR);
@ -244,11 +240,9 @@ exit: /*failure exit*/
if (comm_duped) MPI_Comm_free(&nc4_info->comm);
if (info_duped) MPI_Info_free(&nc4_info->info);
#endif
if (fapl_id != H5P_DEFAULT)
H5Pclose(fapl_id);
if (!nc4_info)
return retval;
nc4_close_netcdf4_file(nc4_info, 1, 0); /* treat like abort */
if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
if(!nc4_info) return retval;
nc4_close_netcdf4_file(nc4_info,1,NULL); /* treat like abort */
return retval;
}

54
libhdf5/hdf5file.c
View File

@ -16,9 +16,6 @@
extern int nc4_vararray_add(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
/* From nc4mem.c */
extern int NC4_extract_file_image(NC_FILE_INFO_T* h5);
/** @internal When we have open objects at file close, should
we log them or print to stdout. Default is to log. */
#define LOGOPEN 1
@ -146,17 +143,18 @@ sync_netcdf4_file(NC_FILE_INFO_T *h5)
/**
* @internal This function will free all allocated metadata memory,
* and close the HDF5 file. The group that is passed in must be the
* root group of the file.
* root group of the file. If inmemory is used, then save
* the final memory in mem.memio.
*
* @param h5 Pointer to HDF5 file info struct.
* @param abort True if this is an abort.
* @param extractmem True if we need to extract and save final inmemory
* @param memio the place to return a core image if not NULL
*
* @return ::NC_NOERR No error.
* @author Ed Hartnett
*/
int
nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem)
nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio* memio)
{
NC_HDF5_FILE_INFO_T *hdf5_info;
int retval = NC_NOERR;
@ -202,15 +200,6 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem)
if (h5->fileinfo)
free(h5->fileinfo);
/* Check to see if this is an in-memory file and we want to get its
final content. */
if(extractmem) {
/* File must be read/write */
if(!h5->no_write) {
retval = NC4_extract_file_image(h5);
}
}
/* Close hdf file. It may not be open, since this function is also
* called by NC_create() when a file opening is aborted. */
if (hdf5_info->hdfid && H5Fclose(hdf5_info->hdfid) < 0)
@ -219,6 +208,25 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem)
BAIL(NC_EHDFERR);
}
/* If inmemory is used and user wants the final memory block,
then capture and return the final memory block else free it */
if(h5->mem.inmemory) {
if(!abort && memio != NULL) {
*memio = h5->mem.memio; /* capture it */
h5->mem.memio.memory = NULL; /* avoid duplicate free */
}
/* If needed, reclaim extraneous memory */
if(h5->mem.memio.memory != NULL) {
/* If the original block of memory is not resizeable, then
it belongs to the caller and we should not free it. */
if(!h5->mem.locked)
free(h5->mem.memio.memory);
}
h5->mem.memio.memory = NULL;
h5->mem.memio.size = 0;
NC4_image_finalize(h5->mem.udata);
}
/* Free the HDF5-specific info. */
if (h5->format_file_info)
free(h5->format_file_info);
@ -242,6 +250,9 @@ dumpopenobjects(NC_FILE_INFO_T* h5)
assert(h5 && h5->format_file_info);
hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
if(hdf5_info->hdfid <= 0)
return; /* File was never opened */
nobjs = H5Fget_obj_count(hdf5_info->hdfid, H5F_OBJ_ALL);
/* Apparently we can get an error even when nobjs == 0 */
@ -589,7 +600,7 @@ NC4_abort(int ncid)
/* Free any resources the netcdf-4 library has for this file's
* metadata. */
if ((retval = nc4_close_netcdf4_file(nc4_info, 1, 0)))
if ((retval = nc4_close_netcdf4_file(nc4_info, 1, NULL)))
return retval;
/* Delete the file, if we should. */
@ -617,6 +628,7 @@ NC4_close(int ncid, void* params)
NC_FILE_INFO_T *h5;
int retval;
int inmemory;
NC_memio* memio = NULL;
LOG((1, "%s: ncid 0x%x", __func__, ncid));
@ -632,14 +644,14 @@ NC4_close(int ncid, void* params)
inmemory = ((h5->cmode & NC_INMEMORY) == NC_INMEMORY);
/* Call the nc4 close. */
if ((retval = nc4_close_netcdf4_file(grp->nc4_info, 0, (inmemory?1:0))))
return retval;
if(inmemory && params != NULL) {
NC_memio* memio = (NC_memio*)params;
*memio = h5->mem.memio;
memio = (NC_memio*)params;
}
/* Call the nc4 close. */
if ((retval = nc4_close_netcdf4_file(grp->nc4_info, 0, memio)))
return retval;
return NC_NOERR;
}

42
libhdf5/hdf5open.c
View File

@ -380,23 +380,10 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
nc4_info->mem.inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY);
nc4_info->mem.diskless = ((mode & NC_DISKLESS) == NC_DISKLESS);
if(nc4_info->mem.inmemory) {
NC_memio* memparams = NULL;
if(parameters == NULL)
BAIL(NC_EINMEMORY);
memparams = (NC_memio*)parameters;
nc4_info->mem.memio = *memparams; /* keep local copy */
/* As a safeguard, if !locked and NC_WRITE is set,
then we must take control of the incoming memory */
nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED;
if(!nc4_info->mem.locked && ((mode & NC_WRITE) == NC_WRITE)) {
memparams->memory = NULL;
}
#ifdef USE_PARALLEL4
} else {
mpiinfo = (NC_MPI_INFO*)parameters;
mpiinfo = (NC_MPI_INFO*)parameters; /* assume, may be changed if inmemory is true */
#endif /* !USE_PARALLEL4 */
}
/* Need this access plist to control how HDF5 handles open objects
* on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to
@ -471,10 +458,29 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
if ((mode & NC_WRITE) == 0)
nc4_info->no_write = NC_TRUE;
/* Now process if NC_INMEMORY is set (recall NC_DISKLESS => NC_INMEMORY) */
if(nc4_info->mem.inmemory) {
NC_memio* memio;
/* validate */
if(nc4_info->mem.memio.size == 0 || nc4_info->mem.memio.memory == NULL)
BAIL(NC_INMEMORY);
if(parameters == NULL)
BAIL(NC_EINMEMORY);
memio = (NC_memio*)parameters;
if(memio->memory == NULL || memio->size == 0)
BAIL(NC_EINMEMORY);
/* initialize h5->mem */
nc4_info->mem.memio = *memio;
/* Is the incoming memory locked? */
nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED;
/* As a safeguard, if !locked and not read-only,
then we must take control of the incoming memory */
if(!nc4_info->mem.locked && !nc4_info->no_write) {
memio->memory = NULL; /* take control */
memio->size = 0;
}
retval = NC4_open_image_file(nc4_info);
if(retval)
BAIL(NC_EHDFERR);
@ -531,7 +537,7 @@ exit:
#endif
if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
if (!nc4_info) return retval;
nc4_close_netcdf4_file(nc4_info,1,0); /* treat like abort*/
nc4_close_netcdf4_file(nc4_info,1,NULL); /* treat like abort*/
return retval;
}

29
libhdf5/nc4mem.c
View File

@ -31,8 +31,6 @@
#define HDrealloc(M,Z) realloc(M,Z)
#endif /* HDrealloc */
extern hid_t NC4_image_init(NC_FILE_INFO_T* h5);
int
NC4_open_image_file(NC_FILE_INFO_T* h5)
{
@ -43,13 +41,13 @@ NC4_open_image_file(NC_FILE_INFO_T* h5)
if(h5->mem.memio.memory == NULL || h5->mem.memio.size == 0)
{stat = NC_EINVAL; goto done;}
/* Figure out the flags */
h5->mem.flags = 0;
/* Figure out the image flags */
h5->mem.imageflags = 0;
if(h5->mem.locked) {
h5->mem.flags |= (H5LT_FILE_IMAGE_DONT_COPY | H5LT_FILE_IMAGE_DONT_RELEASE);
h5->mem.imageflags |= (H5LT_FILE_IMAGE_DONT_COPY | H5LT_FILE_IMAGE_DONT_RELEASE);
}
if(!h5->no_write)
h5->mem.flags |= H5LT_FILE_IMAGE_OPEN_RW;
h5->mem.imageflags |= H5LT_FILE_IMAGE_OPEN_RW;
/* Create the file but using our version of H5LTopen_file_image */
hdfid = NC4_image_init(h5);
@ -67,21 +65,22 @@ int
NC4_create_image_file(NC_FILE_INFO_T* h5, size_t initialsz)
{
int stat = NC_NOERR;
int hdfid;
hid_t hdfid;
/* Create the file but using our version of H5LTopen_file_image */
h5->mem.created = 1;
h5->mem.initialsize = initialsz;
h5->mem.imageflags |= H5LT_FILE_IMAGE_OPEN_RW;
hdfid = NC4_image_init(h5);
if(hdfid < 0)
{stat = NC_EHDFERR; goto done;}
/* Remember HDF5 file identifier. */
((NC_HDF5_FILE_INFO_T *)h5->format_file_info)->hdfid = hdfid;
done:
return stat;
}
#if 0
int
NC4_extract_file_image(NC_FILE_INFO_T* h5)
{
@ -89,14 +88,19 @@ NC4_extract_file_image(NC_FILE_INFO_T* h5)
hid_t fapl;
herr_t herr;
NC_memio mem;
hid_t hdfid = -1;
assert(h5 && h5->format_file_info && !h5->no_write);
/* Get the file access property list */
fapl = h5->mem.fapl;
if(fapl < 0)
if((hdfid = ((NC_HDF5_FILE_INFO_T *)h5->format_file_info)->hdfid) < 0)
{stat = NC_EHDFERR; goto done;}
/* The h5->mem.memio struct should contain what we want */
/* Get the file access property list */
if((fapl = H5Fget_access_plist(hdfid)) < 0)
{stat = NC_EHDFERR; goto done;}
memset(&mem,0,sizeof(mem));
herr = H5Pget_file_image(fapl, &mem.memory, &mem.size);
if(herr < 0)
@ -106,7 +110,8 @@ NC4_extract_file_image(NC_FILE_INFO_T* h5)
/* Close FAPL */
if (H5Pclose(fapl) < 0)
{stat = NC_EHDFERR; goto done;}
h5->mem.fapl = 0;
done:
return stat;
}
#endif

467
libhdf5/nc4memcb.c
View File

@ -19,7 +19,7 @@
/*
* @internal Inmemory support
*
* This code is derived from H5Lt.c#H5LTopen_file_image.
* This code is derived from H5LT.c#H5LTopen_file_image.
* In order to make the netcdf inmemory code work, it is necessary
* to modify some of the callback functions; specifically
* image_malloc, image_realloc, and image_memcpy.
@ -96,7 +96,8 @@
#endif
#undef TRACE
#define CATCH
#undef CATCH
#undef TRACE_UDATA
#ifdef TRACE
#define CATCH
@ -104,18 +105,23 @@
#ifdef TRACE
#include <stdarg.h>
static void trace(const char* fcn, void* _udata, ...);
static void traceend(const char* fcn, void* _udata);
static void trace(const char* fcn, H5FD_file_image_op_t op, void* _udata, ...);
static void traceend(const char* fcn, void* _udata, uintptr_t retval);
static const char* traceop(H5FD_file_image_op_t op);
static char* traceflags(int flags);
/* In case we do not have variadic macros */
#define TRACE1(fcn,udata,x1) trace(fcn,udata,x1)
#define TRACE2(fcn,udata,x1,x2) trace(fcn,udata,x1,x2)
#define TRACE3(fcn,udata,x1,x2,x3) trace(fcn,udata,x1,x2,x3)
#define TRACEEND(fcn,udata) traceend(fcn,udata);
#define TRACE0(fcn,op,udata) trace(fcn,op,udata)
#define TRACE1(fcn,op,udata,x1) trace(fcn,op,udata,x1)
#define TRACE2(fcn,op,udata,x1,x2) trace(fcn,op,udata,x1,x2)
#define TRACE3(fcn,op,udata,x1,x2,x3) trace(fcn,op,udata,x1,x2,x3)
#define TRACEEND(fcn,udata,retval) traceend(fcn,udata,(uintptr_t)retval);
#else /*!TRACE*/
#define TRACE1(fcn,udata,x1)
#define TRACE2(fcn,udata,x1,x2)
#define TRACE3(fcn,udata,x1,x2,x3)
#define TRACEEND(fcn,udata)
#define TRACE0(fcn,op,udata)
#define TRACE1(fcn,op,udata,x1)
#define TRACE2(fcn,op,udata,x1,x2)
#define TRACE3(fcn,op,udata,x1,x2,x3)
#define TRACEEND(fcn,udata,retval)
#endif
#ifdef CATCH
@ -235,20 +241,26 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
void * return_value = NULL;
TRACE1("malloc",_udata, size);
TRACE1("malloc", file_image_op, _udata, size);
#if 0
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
#endif
switch ( file_image_op ) {
/* the app buffer is "copied" to only one FAPL. Afterwards, FAPLs can be "copied" */
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET:
/* It appears that the fapl memory is never created as we use it, so
we expect the udata ptr to be either null or same as the app buffer.*/
assert(udata->fapl_image_ptr == NULL || udata->fapl_image_ptr == udata->app_image_ptr);
if (udata->app_image_ptr == NULL)
goto out;
if (udata->app_image_size != size)
goto out;
if (udata->fapl_image_ptr != NULL)
if (udata->fapl_image_ptr != NULL)
goto out;
if (udata->fapl_image_size != 0)
goto out;
@ -259,6 +271,7 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata
udata->fapl_image_size = udata->app_image_size;
return_value = udata->fapl_image_ptr;
udata->fapl_ref_count++;
return_value = udata->fapl_image_ptr;
break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY:
@ -273,10 +286,10 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata
udata->fapl_ref_count++;
break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET:
if(!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET:
if (udata->fapl_image_ptr == NULL)
goto out;
/* fake the malloc by returning the original memory */
/* fake the malloc by returning the current memory */
return_value = udata->fapl_image_ptr;
break;
@ -310,7 +323,7 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata
goto out;
} /* end switch */
TRACEEND("malloc",_udata);
TRACEEND("malloc",_udata,return_value);
return(return_value);
@ -343,11 +356,13 @@ local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
TRACE3("memcpy",_udata,dest,src,size);
TRACE3("memcpy", file_image_op, _udata,dest,src,size);
#if 0
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
#endif
switch(file_image_op) {
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET:
@ -361,6 +376,15 @@ local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_
goto out;
if (udata->fapl_ref_count == 0)
goto out;
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) {
if(src != dest) {
memcpy(dest,src,size);
#ifdef TRACE
fprintf(stderr,"\t>>>> memcpy(%p,%p,%ld)\n",
dest,src,(unsigned long)size);
#endif
}
}
break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY:
@ -406,10 +430,7 @@ local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_
goto out;
} /* end switch */
#ifdef TRACE
fprintf(stderr,"trace: memcpy: dest=%p\n",dest);
#endif
TRACEEND("memcpy",_udata);
TRACEEND("memcpy",_udata,dest);
return(dest);
out:
@ -433,61 +454,85 @@ out:
*
*-------------------------------------------------------------------------
*/
/* This warning is from H5FDcore.c"
Be careful of non-Posix realloc() that doesn't understand
what to do when the first argument is null.
*/
/* Modified:
1. If the realloc new size is <= existing size,
then pretend we did a realloc and return success.
This avoids unneccessary heap operations.
2. If the H5LT_FILE_IMAGE_DONT_COPY or
H5LT_FILE_IMAGE_DONT_RELEASE flag is set and the
realloc new size is > existing size, then fail
because the realloc() call may change the address
of the buffer. The new address cannot be
communicated to the application to release it.
3. Otherwise, use realloc(). Note that this may have the
side effect of freeing the previous memory chunk.
*/
static void *
local_image_realloc(void *ptr, size_t size, H5FD_file_image_op_t file_image_op, void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
void * return_value = NULL;
TRACE2("realloc",_udata, ptr, size);
TRACE2("realloc", file_image_op, _udata, ptr, size);
#if 0
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
#endif
/* realloc() is not allowed if the image is open in read-only mode */
if (!(udata->flags & H5LT_FILE_IMAGE_OPEN_RW))
goto out;
/* DONT_COPY => DONT_RELEASE */
assert(((udata->flags & H5LT_FILE_IMAGE_DONT_COPY)?(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE):1));
/* Note that the fapl pointer is never realloc'd */
if (file_image_op == H5FD_FILE_IMAGE_OP_FILE_RESIZE) {
if (udata->vfd_image_ptr != ptr)
goto out;
if (udata->vfd_ref_count != 1)
goto out;
/* Modified:
1. If the realloc new size is <= existing size,
then pretend we did a realloc and return success.
This avoids unneccessary heap operations.
2. If the H5LT_FILE_IMAGE_DONT_COPY or
H5LT_FILE_IMAGE_DONT_RELEASE flag is set and the
realloc new size is > existing size, then fail
because the realloc() call may change the address
of the buffer. The new address cannot be
communicated to the application to release it.
3. Otherwise, use realloc(). Note that this may have the
side effect of freeing the previous memory chunk.
*/
if(size <= udata->vfd_image_size) {
/* Ok, pretend we did a realloc */
} else if((udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)
|| (udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) {
goto out; /* realloc MAY violate these flags */
} else {
if (NULL == (udata->vfd_image_ptr = HDrealloc(ptr, size)))
goto out;
udata->vfd_image_size = size;
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) { /* buffer modification is allowed */
/* Divide code based on whether ptr == NULL or not */
if(ptr == NULL) {
/* From realloc man page: If ptr is NULL, then the call is equivalent to malloc(size),
for all values of size; if size is equal to zero, and ptr is not NULL, then the call
is equivalent to free(ptr). */
udata->vfd_image_ptr = malloc(size);
udata->vfd_ref_count++;
} else { /* ptr != NULL */
if(udata->vfd_image_ptr != ptr)
goto out;
if (udata->vfd_ref_count != 1)
goto out;
udata->vfd_image_ptr = realloc(ptr, size);
if(NULL == udata->vfd_image_ptr) {
LOG((0,"image_realloc: unable to allocate memory block of size: %lu bytes",(unsigned long)size));
goto out;
}
#ifdef TRACE
fprintf(stderr,"\t>>>> realloc(%p,%ld)=>%p\n",ptr,(unsigned long)size,udata->vfd_image_ptr);
#endif
}
udata->vfd_image_size = size;
} else { /* Cannot realloc, so fake it */
if(size <= udata->vfd_image_size) {
/* Ok, pretend we did a realloc but just change size*/
udata->vfd_image_size = size;
} else
goto out;
}
return_value = udata->vfd_image_ptr;
} /* end if */
else
goto out;
#ifdef TRACE
fprintf(stderr,"trace: realloc: return=%p\n",return_value);
#endif
TRACEEND("realloc",_udata);
TRACEEND("realloc",_udata,return_value);
return(return_value);
out:
@ -515,11 +560,13 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
TRACE1("free",_udata,ptr);
TRACE1("free", file_image_op, _udata, ptr);
#if 0
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
#endif
switch(file_image_op) {
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE:
@ -530,14 +577,15 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata)
udata->fapl_ref_count--;
/* release the shared buffer only if indicated by the respective flag and there are no outstanding references */
if (udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0 &&
!(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)) {
free(udata->fapl_image_ptr);
udata->app_image_ptr = NULL;
udata->fapl_image_ptr = NULL;
udata->vfd_image_ptr = NULL;
} /* end if */
/* For the way we use it, it should still be the case that
the fapl pointer is same as image_ptr, so we do not need
to do anything */
assert(udata->fapl_image_ptr == udata->app_image_ptr);
/* clean up */
#if 0
udata->app_image_ptr = NULL;
udata->fapl_image_ptr = NULL;
#endif
break;
case H5FD_FILE_IMAGE_OP_FILE_CLOSE:
@ -548,17 +596,17 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata)
udata->vfd_ref_count--;
/* release the shared buffer only if indicated by the respective flag and there are no outstanding references */
if (udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0 &&
!(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)) {
free(udata->vfd_image_ptr);
udata->app_image_ptr = NULL;
udata->fapl_image_ptr = NULL;
udata->vfd_image_ptr = NULL;
} /* end if */
udata->h5->mem.memio.memory = udata->vfd_image_ptr;
udata->h5->mem.memio.size = udata->vfd_image_size;
/* clean up */
#if 0
udata->app_image_ptr = NULL;
udata->fapl_image_ptr = NULL;
#endif
udata->vfd_image_ptr = NULL;
break;
/* added unused labels to keep the compiler quite */
/* added unused labels to keep the compiler quiet */
case H5FD_FILE_IMAGE_OP_NO_OP:
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET:
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY:
@ -569,7 +617,7 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata)
goto out;
} /* end switch */
TRACEEND("free",_udata);
TRACEEND("free",_udata,1);
return(SUCCEED);
out:
@ -598,18 +646,24 @@ local_udata_copy(void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
TRACE0("udata_copy", 0, _udata);
#if 0
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
#endif
if (udata->ref_count == 0)
goto out;
/* never copy */
if (udata->ref_count == 0)
goto out;
udata->ref_count++;
TRACEEND("udata_copy",udata,1);
return(udata);
out:
TRACEFAIL("udata_copy");
return NULL;
} /* end udata_copy */
@ -634,63 +688,75 @@ local_udata_free(void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
TRACE0("udata_free", 0, _udata);
#if 0
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
#endif
if (udata->ref_count == 0)
goto out;
udata->ref_count--;
/* checks that there are no references outstanding before deallocating udata */
if (udata->ref_count == 0 && udata->fapl_ref_count == 0 &&
udata->vfd_ref_count == 0)
if (udata->ref_count == 0 && udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0) {
free(udata);
#ifdef TRACE_UDATA
fprintf(stderr,"\t>>>> freed: udata=%p\n",udata);
#endif
udata = NULL;
}
TRACEEND("udata_free",udata,1);
return(SUCCEED);
out:
TRACEFAIL("udata_free");
return(FAIL);
} /* end udata_free */
/* End of callbacks definitions for file image operations */
hid_t
NC4_image_init(NC_FILE_INFO_T* h5)
{
hid_t fapl = -1, file_id = -1; /* HDF5 identifiers */
unsigned file_open_flags;/* Flags for image open */
unsigned file_open_flags = 0;/* Flags for hdf5 open */
char file_name[64]; /* Filename buffer */
size_t alloc_incr; /* Buffer allocation increment */
size_t min_incr = 65536; /* Minimum buffer increment */
double buf_prcnt = 0.1f; /* Percentage of buffer size to set
as increment */
size_t buf_size = 0;
void* buf_ptr = h5->mem.memio.memory;
unsigned flags = h5->mem.flags;
void* buf_ptr = NULL;
unsigned imageflags;
int create = 0;
H5LT_file_image_ud_t *udata = NULL; /* Pointer to udata structure */
static long file_name_counter;
H5FD_file_image_callbacks_t callbacks = {&local_image_malloc, &local_image_memcpy,
&local_image_realloc, &local_image_free,
&local_udata_copy, &local_udata_free,
(void *)NULL};
static long file_name_counter;
imageflags = h5->mem.imageflags;
create = h5->mem.created;
/* check arguments */
if (buf_ptr == NULL) {
if(h5->mem.created) {
if (h5->mem.memio.memory == NULL) {
if(create) {
if(h5->mem.memio.size == 0) h5->mem.memio.size = DEFAULT_CREATE_MEMSIZE;
h5->mem.memio.memory = malloc(h5->mem.memio.size);
} else
goto out; /* open requires an input buffer */
}
// reset
} else if(h5->mem.memio.size == 0)
goto out;
/* alias for convenience */
buf_size = h5->mem.memio.size;
buf_ptr = h5->mem.memio.memory;
/* validate */
if (buf_ptr == NULL || buf_size == 0)
goto out;
if (flags & (unsigned)~(H5LT_FILE_IMAGE_ALL))
goto out;
/* Create FAPL to transmit file image */
if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
@ -708,14 +774,10 @@ NC4_image_init(NC_FILE_INFO_T* h5)
if (H5Pset_fapl_core(fapl, alloc_incr, FALSE) < 0)
goto out;
/* Set callbacks for file image ops ONLY if the file image is NOT copied */
if (flags & H5LT_FILE_IMAGE_DONT_COPY)
// if (0)
/* Set callbacks for file image ops always */
{
H5LT_file_image_ud_t *udata; /* Pointer to udata structure */
/* Allocate buffer to communicate user data to callbacks */
if (NULL == (udata = (H5LT_file_image_ud_t *)malloc(sizeof(H5LT_file_image_ud_t))))
if (NULL == (udata = (H5LT_file_image_ud_t *)calloc(1,sizeof(H5LT_file_image_ud_t))))
goto out;
/* Initialize udata with info about app buffer containing file image and flags */
@ -727,7 +789,7 @@ NC4_image_init(NC_FILE_INFO_T* h5)
udata->vfd_image_ptr = NULL;
udata->vfd_image_size = 0;
udata->vfd_ref_count = 0;
udata->flags = flags;
udata->flags = imageflags;
udata->ref_count = 1; /* corresponding to the first FAPL */
udata->h5 = h5;
@ -735,28 +797,27 @@ NC4_image_init(NC_FILE_INFO_T* h5)
callbacks.udata = (void *)udata;
/* Set file image callbacks */
if (H5Pset_file_image_callbacks(fapl, &callbacks) < 0) {
free(udata);
if (H5Pset_file_image_callbacks(fapl, &callbacks) < 0)
goto out;
} /* end if */
}
/* Assign file image in user buffer to FAPL */
if (H5Pset_file_image(fapl, buf_ptr, buf_size) < 0)
goto out;
/* set file open flags */
if (flags & H5LT_FILE_IMAGE_OPEN_RW)
/* define a unique file name */
snprintf(file_name, (sizeof(file_name) - 1), "file_image_%ld", file_name_counter++);
/* set file open/create flags */
if(create)
file_open_flags = H5F_ACC_TRUNC; /* H5Fcreate does not like H5F_ACC_RDWR */
else if (imageflags & H5LT_FILE_IMAGE_OPEN_RW)
file_open_flags = H5F_ACC_RDWR;
else
file_open_flags = H5F_ACC_RDONLY;
/* define a unique file name */
snprintf(file_name, (sizeof(file_name) - 1), "file_image_%ld", file_name_counter++);
/* Assign file image in FAPL to the core file driver */
if(h5->mem.created) {
file_open_flags |= H5F_ACC_TRUNC;
if(create) {
if ((file_id = H5Fcreate(file_name, file_open_flags, H5P_DEFAULT, fapl)) < 0)
goto out;
} else {
@ -764,76 +825,133 @@ NC4_image_init(NC_FILE_INFO_T* h5)
goto out;
}
h5->mem.fapl = fapl;
h5->mem.udata = udata;
/* Return file identifier */
return file_id;
out:
done:
/* Reclaim the fapl object */
H5E_BEGIN_TRY {
if(fapl >= 0)
H5Pclose(fapl);
} H5E_END_TRY;
return -1;
/* Reclaim udata */
/* Return file identifier */
return file_id;
out:
if(udata != NULL) free(udata);
file_id = -1;
goto done;
} /* end H5LTopen_file_image() */
void
NC4_image_finalize(void* _udata)
{
if(_udata != NULL) {
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t*)_udata;
#if 0
if(udata->app_image_ptr != NULL) free(udata->app_image_ptr);
if(udata->fapl_image_ptr != NULL) free(udata->fapl_image_ptr);
if(udata->vfd_image_ptr != NULL) free(udata->vfd_image_ptr);
#endif
free(udata);
}
}
#ifdef TRACE
static void
static char*
printudata(H5LT_file_image_ud_t* udata)
{
if(udata == NULL) return;
fprintf(stderr," flags=0x%0x",udata->flags);
fprintf(stderr," flags=0x%0x",udata->flags);
fprintf(stderr," ref_count=%d",udata->ref_count);
fprintf(stderr," app=(%p,%lld)",udata->app_image_ptr,(long long)udata->app_image_size);
fprintf(stderr," fapl=(%p,%lld)[%d]",udata->fapl_image_ptr,(long long)udata->fapl_image_size,udata->fapl_ref_count);
fprintf(stderr," vfd=(%p,%lld)[%d]",udata->vfd_image_ptr,(long long)udata->vfd_image_size,udata->vfd_ref_count);
char buf[8192];
char tmp[8192];
char* flags = "";
buf[0] = '\0';
if(udata == NULL) return strdup("");
strlcat(buf,"flags=",sizeof(buf));
flags = traceflags(udata->flags);
strlcat(buf,flags,sizeof(buf));
if(flags != NULL) free(flags);
snprintf(tmp,sizeof(tmp)," ref_count=%d",udata->ref_count);
strlcat(buf,tmp,sizeof(tmp));
snprintf(tmp,sizeof(tmp)," app=(%p,%lld)",udata->app_image_ptr,(long long)udata->app_image_size);
strlcat(buf,tmp,sizeof(tmp));
snprintf(tmp,sizeof(tmp)," fapl=(%p,%lld)[%d]",udata->fapl_image_ptr,(long long)udata->fapl_image_size,udata->fapl_ref_count);
strlcat(buf,tmp,sizeof(tmp));
snprintf(tmp,sizeof(tmp)," vfd=(%p,%lld)[%d]",udata->vfd_image_ptr,(long long)udata->vfd_image_size,udata->vfd_ref_count);
strlcat(buf,tmp,sizeof(tmp));
return strdup(buf);
}
static void trace(const char* fcn, void* _udata, ...)
static void
trace(const char* fcn, H5FD_file_image_op_t op, void* _udata, ...)
{
H5LT_file_image_ud_t *udata = NULL;
va_list ap;
char buf[16000];
char tmp[8192];
char* ud;
va_start(ap, _udata); /* Requires the last fixed parameter (to get the address) */
udata = (H5LT_file_image_ud_t *)_udata;
buf[0] = '\0';
snprintf(tmp,sizeof(tmp),"trace [ %s: op=%s: ",fcn,traceop(op));
strlcat(buf,tmp,sizeof(tmp));
if(strcmp("malloc",fcn)==0) {
size_t size = va_arg(ap,size_t);
fprintf(stderr,"trace: %s: size=%lld udata=",fcn,(long long)size);
printudata(udata);
fprintf(stderr,"\n");
snprintf(tmp,sizeof(tmp),"size=%lld",(long long)size);
} else if(strcmp("realloc",fcn)==0) {
void* ptr = va_arg(ap,void*);
size_t size = va_arg(ap,size_t);
fprintf(stderr,"trace: %s: ptr=%p, size=%lld udata=",fcn,ptr,(long long)size);
printudata(udata);
fprintf(stderr,"\n");
snprintf(tmp,sizeof(tmp),"ptr=%p, size=%lld",ptr,(long long)size);
} else if(strcmp("free",fcn)==0) {
void* ptr = va_arg(ap,void*);
fprintf(stderr,"trace: %s: ptr=%p udata=",fcn,ptr);
printudata(udata);
fprintf(stderr,"\n");
snprintf(tmp,sizeof(tmp),"ptr=%p",ptr);
} else if(strcmp("memcpy",fcn)==0) {
void* dest = va_arg(ap,void*);
void* src = va_arg(ap,void*);
size_t size = va_arg(ap,size_t);
fprintf(stderr,"trace: %s: dest=%p src=%p size=%lld udata=",fcn,dest,src,(long long)size);
printudata(udata);
fprintf(stderr,"\n");
snprintf(tmp,sizeof(tmp),"dest=%p, src=%p, size=%lld",dest,src,(long long)size);
} else if(strcmp("udata_copy",fcn)==0) {
#ifdef TRACE_UDATA
snprintf(tmp,sizeof(tmp),"udata=%p",udata);
#endif
} else if(strcmp("udata_free",fcn)==0) {
#ifdef TRACE_UDATA
snprintf(tmp,sizeof(tmp),"udata=%p",udata);
#endif
} else {
fprintf(stderr,"unknown fcn: %s\n",fcn);
snprintf(tmp,sizeof(tmp),"unknown fcn: %s",fcn);
}
strlcat(buf,tmp,sizeof(buf));
ud = printudata(udata);
strlcat(buf,"\n\tudata=",sizeof(buf));
strlcat(buf,ud,sizeof(tmp));
free(ud);
strlcat(buf,"\n",sizeof(buf));
va_end(ap);
fprintf(stderr,"%s",buf);
fflush(stderr);
}
static void traceend(const char* fcn, void* _udata)
static void
traceend(const char* fcn, void* _udata, uintptr_t retval)
{
char buf[16000];
char tmp[8192];
char* ud;
const char* tab = " ";
buf[0] = '\0';
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
fprintf(stderr,"traceend: %s: udata=",fcn);
printudata(udata);
fprintf(stderr,"\n");
snprintf(tmp,sizeof(tmp),"%s]: retval=%p",tab,(void*)retval);
strlcat(buf,tmp,sizeof(buf));
strlcat(buf," udata=",sizeof(buf));
ud = printudata(udata);
strlcat(buf,ud,sizeof(tmp));
free(ud);
strlcat(buf,"\n",sizeof(buf));
fprintf(stderr,"%s",buf);
fflush(stderr);
}
@ -843,7 +961,68 @@ static void traceend(const char* fcn, void* _udata)
static void
tracefail(const char* fcn)
{
fprintf(stderr,"fail: %s",fcn);
fprintf(stderr,"fail: %s\n",fcn);
fflush(stderr);
}
#endif /*CATCH*/
#ifdef TRACE
static char*
traceflags(int flags)
{
int i;
char buf[8192];
char tmp[8192];
buf[0] = '\0';
for(i=0;i<16;i++) {
tmp[0] = '\0';
if((flags & 1<<i) == 0) continue;
if(i > 0) strlcat(tmp,"|",sizeof(tmp));
switch(1<<i) {
case H5LT_FILE_IMAGE_OPEN_RW: /* 0x0001 Open image for read-write */
strlcat(tmp,"OPEN_RW",sizeof(tmp));
break;
case H5LT_FILE_IMAGE_DONT_COPY: /*0x0002 the HDF5 lib won't copy */
strlcat(tmp,"DONT_COPY",sizeof(tmp));
break;
case H5LT_FILE_IMAGE_DONT_RELEASE: /* 0x0004 The HDF5 lib won't
deallocate user supplied image
buffer. The user application
is reponsible for doing so. */
strlcat(tmp,"DONT_RELEASE",sizeof(tmp));
break;
default: break;
}
strlcat(buf,tmp,sizeof(buf));
}
return strdup(buf);
}
static const char*
traceop(H5FD_file_image_op_t op)
{
const char* sop = NULL;
switch ( op ) {
case H5FD_FILE_IMAGE_OP_NO_OP:
sop = "NO_OP"; break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET:
sop = "PROPERTY_LIST_SET"; break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY:
sop = "PROPERTY_LIST_COPY"; break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET:
sop = "PROPERTY_LIST_GET"; break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE:
sop = "PROPERTY_LIST_CLOSE"; break;
case H5FD_FILE_IMAGE_OP_FILE_OPEN:
sop = "FILE_OPEN"; break;
case H5FD_FILE_IMAGE_OP_FILE_RESIZE:
sop = "FILE_RESIZE"; break;
case H5FD_FILE_IMAGE_OP_FILE_CLOSE:
sop = "FILE_CLOSE"; break;
default: break;
}
return sop;
}
#endif

4
libnetcdf.settings.in
View File

@ -35,5 +35,5 @@ Diskless Support: @HAS_DISKLESS@
MMap Support: @HAS_MMAP@
JNA Support: @HAS_JNA@
CDF5 Support: @HAS_CDF5@
ERANGE fill Support: @HAS_ERANGE_FILL@
relaxed boundary check: @RELAX_COORD_BOUND@
ERANGE Fill Support: @HAS_ERANGE_FILL@
Relaxed Boundary Check: @RELAX_COORD_BOUND@

53
libsrc/memio.c
View File

@ -21,7 +21,6 @@
#include <windows.h>
#include <winbase.h>
#include <io.h>
#define access(path,mode) _access(path,mode)
#endif
#include "ncdispatch.h"
@ -116,6 +115,8 @@ static int memio_close(ncio* nciop, int);
static int readfile(const char* path, NC_memio*);
static int writefile(const char* path, NCMEMIO*);
static int fileiswriteable(const char* path);
static int fileisreadable(const char* path);
static int fileexists(const char* path);
/* Mnemonic */
#define DOOPEN 1
@ -197,7 +198,7 @@ memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEM
if(fIsSet(ioflags,NC_DISKLESS))
memio->diskless = 1;
if(fIsSet(ioflags,NC_INMEMORY) && !memio->diskless)
if(fIsSet(ioflags,NC_INMEMORY))
memio->inmemory = 1;
if(fIsSet(ioflags,NC_WRITE) && !fIsSet(ioflags,NC_NOCLOBBER) && memio->diskless)
memio->persist = 1;
@ -248,8 +249,8 @@ memio_create(const char* path, int ioflags,
return status;
if(memio->persist) {
/* Verify the file is writeable */
if(!fileiswriteable(path))
/* Verify the file is writeable or does not exist*/
if(fileexists(path) && !fileiswriteable(path))
{status = EPERM; goto unwind_open;}
}
@ -374,9 +375,11 @@ fprintf(stderr,"memio_open: initial memory: %lu/%lu\n",(unsigned long)memio->mem
#endif
if(memio->persist) {
/* Verify the file is writeable */
/* Verify the file is writeable and exists */
if(!fileexists(path))
{status = ENOENT; goto unwind_open;}
if(!fileiswriteable(path))
{status = EPERM; goto unwind_open;}
{status = EACCES; goto unwind_open;}
}
/* Use half the filesize as the blocksize ; why? */
@ -645,11 +648,39 @@ memio_extract(ncio* const nciop, size_t* sizep, void** memoryp)
return status;
}
/* Return 1 if file exists, 0 otherwise */
static int
fileexists(const char* path)
{
int ok;
/* See if the file exists at all */
ok = NCaccess(path,ACCESS_MODE_EXISTS);
if(ok < 0) /* file does not exist */
return 0;
return 1;
}
/* Return 1 if file is writeable, return 0 otherwise;
assumes fileexists has been checked already */
static int
fileiswriteable(const char* path)
{
int ok;
ok = access(path,O_RDWR);
/* if W is ok */
ok = NCaccess(path,ACCESS_MODE_W);
if(ok < 0)
return 0;
return 1;
}
/* Return 1 if file is READABLE, return 0 otherwise;
assumes fileexists has been checked already */
static int
fileisreadable(const char* path)
{
int ok;
/* if RW is ok */
ok = NCaccess(path,ACCESS_MODE_R);
if(ok < 0)
return 0;
return 1;
@ -714,14 +745,14 @@ writefile(const char* path, NCMEMIO* memio)
size_t count = 0;
char* p = NULL;
/* Open the file for writing*/
/* Open/create the file for writing*/
#ifdef _MSC_VER
f = NCfopen(path,"rwb");
f = NCfopen(path,"wb");
#else
f = NCfopen(path,"rw");
f = NCfopen(path,"w");
#endif
if(f == NULL)
{status = errno; goto done;}
{status = errno; goto done;}
rewind(f);
count = memio->size;
p = memio->memory;

2
libsrc/nc3internal.c
View File

@ -1044,7 +1044,7 @@ NC3_create(const char *path, int ioflags,
int use_parallel, void* parameters,
NC_Dispatch* dispatch, NC* nc)
{
int status;
int status = NC_NOERR;
void *xp = NULL;
int sizeof_off_t = 0;
NC3_INFO* nc3 = NULL;

12
libsrc4/error4.c
View File

@ -32,7 +32,7 @@ extern int nc_log_level;
nc_log(0, "this computer will explode in %d seconds", i);
After the first arg (the severity), use the rest like a normal
printf statement. Output will appear on stdout.
printf statement. Output will appear on stderr.
This function is heavily based on the function in section 15.5 of
the C FAQ. */
@ -50,18 +50,18 @@ nc_log(int severity, const char *fmt, ...)
/* If the severity is zero, this is an error. Otherwise insert that
many tabs before the message. */
if (!severity)
fprintf(stdout, "ERROR: ");
fprintf(stderr, "ERROR: ");
for (t=0; t<severity; t++)
fprintf(stdout, "\t");
fprintf(stderr, "\t");
/* Print out the variable list of args with vprintf. */
va_start(argp, fmt);
vfprintf(stdout, fmt, argp);
vfprintf(stderr, fmt, argp);
va_end(argp);
/* Put on a final linefeed. */
fprintf(stdout, "\n");
fflush(stdout);
fprintf(stderr, "\n");
fflush(stderr);
}
void

12
nc_test/Make0
View File

@ -1,17 +1,17 @@
# Test c output
T=tst_open_mem
T=tst_diskless
#H58=8
H510=10
#ARGS=f03_1820.nc
ARGS=persist
SRC=test_put.c test_get.c test_read.c test_write.c util.c error.c
#SRC=test_put.c test_get.c test_read.c test_write.c util.c error.c
#CMD=valgrind --leak-check=full
#CMD=export NETCDF_LOG_LEVEL=5 ;gdb --args
CMD=gdb --args
#CMD=env HDF5_DEBUG=trace
#CMD=export NETCDF_LOG_LEVEL=5 ;gdb --args
CMD=valgrind --leak-check=full
CMD=gdb --args
#PAR=1

2
nc_test/run_diskless.sh
View File

@ -5,7 +5,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
set -x
# Get the target OS and CPU
CPU=`uname -p`
OS=`uname`

17
nc_test/run_inmemory.sh
View File

@ -25,26 +25,19 @@ echo "*** Testing in-memory operations"
HASNC4=`${top_builddir}/nc-config --has-nc4`
# Execute the core of the inmemory tests
if ! ${execdir}/tst_inmemory; then
echo "FAIL: tst.inmemory"
fi
if ! ${execdir}/tst_open_mem ${srcdir}/${OMEMFILE}; then
echo "FAIL: tst_open_mem"
fi
${execdir}/tst_inmemory
exit
${execdir}/tst_open_mem ${srcdir}/${OMEMFILE}
echo "**** Test ncdump of the resulting inmemory data"
${NCDUMP} -n "${FILE3}" ${FILE3}.nc > ${FILE3}.cdl
${NCDUMP} -n "${FILE3}" ${CREATE3}.nc > ${CREATE3}.cdl
if ! diff -wb ${FILE3}.cdl ${CREATE3}.cdl ; then
echo "FAIL: ${FILE3}.cdl vs ${CREATE3}.cdl
fi
diff -wb ${FILE3}.cdl ${CREATE3}.cdl
if test "x$HASNC4" = "xyes" ; then
${NCDUMP} ${FILE4}.nc > ${FILE4}.cdl
${NCDUMP} ${CREATE4}.nc > ${CREATE4}.cdl
if diff -wb ${FILE4}.cdl ${CREATE4}.cdl ; then
echo "FAIL: ${FILE4}.cdl vs ${CREATE4}.cdl
fi
diff -wb ${FILE4}.cdl ${CREATE4}.cdl
# cleanup
rm -f ${FILE3}.nc ${FILE4}.nc ${CREATE3}.nc ${CREATE4}.nc

17
nc_test/tst_diskless.c
View File

@ -7,12 +7,12 @@ redistribution conditions.
#undef DDBG
#include <config.h>
#include <nc_tests.h>
#include "err_macros.h"
#include "config.h"
#include <stdio.h>
#include <stdlib.h>
#include <netcdf.h>
#include "netcdf.h"
#include "nc_tests.h"
#include "err_macros.h"
/*
netcdf tst_diskless {
@ -47,17 +47,12 @@ void fail(int line) {
/* Control flags */
static int flags, persist, usenetcdf4, mmap;
/* Remove a file; do not care if it does not exist */
static void
removefile(int persist, char* filename)
{
if(persist) {
if(remove(filename) != 0) {
if(errno != ENOENT) {
fprintf(stderr,"Could not remove file: %s: %d\n",filename,errno);
perror("");
exit(1);
}
}
remove(filename);
}
}

77
nc_test/tst_inmemory.c
View File

@ -442,11 +442,13 @@ test_open(const char* path, NC_memio* filedata, int mode)
{
int stat = NC_NOERR;
NC_memio duplicate;
NC_memio* finaldata = NULL;
NC_memio finaldata;
int ncid;
int xmode = mode; /* modified mode */
finaldata = calloc(1,sizeof(NC_memio));
finaldata.memory = NULL;
finaldata.size = 0;
finaldata.flags = 0;
fprintf(stderr,"\n\t***Test open 1: nc_open_mem(): read-only\n");
CHECK(duplicatememory(filedata,&duplicate,0));
@ -460,13 +462,13 @@ test_open(const char* path, NC_memio* filedata, int mode)
duplicate.flags = NC_MEMIO_LOCKED;
CHECK(nc_open_memio(path, xmode, &duplicate, &ncid))
CHECK(verify_file(ncid,!MODIFIED));
CHECK(nc_close_memio(ncid,finaldata));
CHECK(nc_close_memio(ncid,&finaldata));
/* Published returned finaldata */
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory);
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory);
/* Verify that finaldata is same */
if(finaldata->size != duplicate.size) CHECK(NC_EINVAL);
if(finaldata->memory != duplicate.memory) CHECK(NC_EINVAL);
free(finaldata->memory);
if(finaldata.size != duplicate.size) CHECK(NC_EINVAL);
if(finaldata.memory != duplicate.memory) CHECK(NC_EINVAL);
free(finaldata.memory); finaldata.memory = NULL;
fprintf(stderr,"\n\t***Test open 3: nc_open_memio(): read-write, copy\n");
xmode |= NC_WRITE; /* allow file to be modified */
@ -475,13 +477,13 @@ test_open(const char* path, NC_memio* filedata, int mode)
/* modify file */
CHECK(modify_file(ncid));
CHECK(verify_file(ncid,MODIFIED));
CHECK(nc_close_memio(ncid,finaldata));
CHECK(nc_close_memio(ncid,&finaldata));
/* Published returned finaldata */
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory);
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory);
/* Verify that finaldata is same */
if(finaldata->size < filedata->size) CHECK(NC_EINVAL);
if(finaldata.size < filedata->size) CHECK(NC_EINVAL);
/* As a safeguard, the memory in duplicate should have been set to NULL*/
free(finaldata->memory);
free(finaldata.memory); finaldata.memory = NULL;
fprintf(stderr,"\n\t***Test open 4: nc_open_memio(): read-write, locked, extra space\n");
/* Store the filedata in a memory chunk that leaves room for modification */
@ -493,14 +495,15 @@ test_open(const char* path, NC_memio* filedata, int mode)
/* modify file */
CHECK(modify_file(ncid));
CHECK(verify_file(ncid,MODIFIED));
CHECK(nc_close_memio(ncid,finaldata));
CHECK(nc_close_memio(ncid,&finaldata));
/* Published returned finaldata */
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory);
/* Check returned finaldata */
if(finaldata->size != duplicate.size) CHECK(NC_EINVAL);
if(finaldata->memory != duplicate.memory) CHECK(NC_EINVAL);
free(finaldata->memory);
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory);
/* Check returned finaldata:
should have same memory but
actual used final size should not exceed the original */
if(finaldata.size > duplicate.size) CHECK(NC_EINVAL);
if(finaldata.memory != duplicate.memory) CHECK(NC_EINVAL);
free(finaldata.memory); finaldata.memory = NULL;
return stat;
}
@ -508,33 +511,36 @@ static int
test_create(const char* path, int mode)
{
int stat = NC_NOERR;
NC_memio* finaldata = NULL;
NC_memio finaldata;
int ncid;
int xmode = mode;
finaldata.memory = NULL;
finaldata.size = 0;
finaldata.flags = 0;
fprintf(stderr,"\n\t***Test create 1: nc_create_memio(): no initialsize\n");
CHECK(nc_create_mem(path, xmode, 0, &ncid))
/* create file metadata */
CHECK(define_metadata(ncid));
CHECK(verify_file(ncid,!MODIFIED));
finaldata = calloc(1,sizeof(NC_memio));
CHECK(nc_close_memio(ncid,finaldata));
CHECK(nc_close_memio(ncid,&finaldata));
/* Published returned finaldata */
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory);
free(finaldata->memory);
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory);
free(finaldata.memory);
fprintf(stderr,"\n\t***Test create 2: nc_create_memio(): initialsize; save file\n");
CHECK(nc_create_mem(path, xmode, LARGE_SPACE, &ncid))
/* create file metadata */
CHECK(define_metadata(ncid));
CHECK(verify_file(ncid,!MODIFIED));
finaldata = calloc(1,sizeof(NC_memio));
CHECK(nc_close_memio(ncid,finaldata));
CHECK(nc_close_memio(ncid,&finaldata));
/* Published returned finaldata */
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory);
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory);
/* Write out the final data as a .nc file */
CHECK(writefile(path,finaldata));
free(finaldata->memory);
CHECK(writefile(path,&finaldata));
if(finaldata.memory != NULL)
free(finaldata.memory);
finaldata.memory = NULL;
return stat;
}
@ -584,15 +590,17 @@ test_xfail(const char* path, int mode, NC_memio* filedata)
/* With HDF5 1.8.20, and possibly other versions,
this tests causes a seg fault in the HDF5 Library.
So until it is fixed, just leave well enough alone */
NC_memio* finaldata = NULL;
finaldata = calloc(1,sizeof(NC_memio));
NC_memio finaldata;
memset(&finaldata,0,sizeof(finaldata));
CHECK(duplicatememory(filedata,&duplicate,0));
duplicate.flags = NC_MEMIO_LOCKED;
xmode |= NC_WRITE;
CHECK(nc_open_memio(XFAIL, xmode, &duplicate, &ncid))
XCHECK(modify_file(ncid));
CHECK(nc_abort(ncid));
free(finaldata->memory);
if(finaldata.memory != NULL)
free(finaldata.memory);
finaldata.memory = NULL;
}
return stat;
@ -605,11 +613,6 @@ main(int argc, char **argv)
NC_memio filedata3;
NC_memio filedata4;
#ifdef USE_NETCDF4
nc_set_log_level(0);
H5Eprint1(stderr);
#endif
fprintf(stderr,"\n*** Testing the inmemory API: netcdf-3.\n");
CHECK(create_reference_file(FILE3,NC_NETCDF3,&filedata3)); /* netcdf-3 */
CHECK(test_open(FILE3,&filedata3,NC_NETCDF3));

2
ncdap_test/CMakeLists.txt
View File

@ -60,5 +60,5 @@ ADD_SUBDIRECTORY(expectremote3)
## Specify files to be distributed by 'make dist'
FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.c ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.sh)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} findtestserver.c.in CMakeLists.txt Makefile.am)
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

4
ncdap_test/Makefile.am
View File

@ -83,12 +83,14 @@ EXTRA_DIST = tst_ncdap3.sh \
tst_longremote3.sh \
tst_filelists.sh tst_urls.sh tst_utils.sh \
t_dap.c CMakeLists.txt tst_formatx.sh testauth.sh testurl.sh \
t_ncf330.c tst_ber.sh
t_ncf330.c tst_ber.sh findtestserver.c.in
CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe
# This should only be left behind if using parallel io
CLEANFILES += tmp_*
DISTCLEANFILES = findtestserver4.c
# This rule are used if someone wants to rebuild t_dap3a.c
# Otherwise never invoked, but records how to do it.
t_dap3a.c: t_dap.c

1
ncdap_test/findtestserver.c
View File

@ -8,6 +8,7 @@
#define XSTRINGIFY(s) #s
#define STRINGIFY(s) XSTRINGIFY(s)
/**
usage: findtestserver dap2|dap4 suffix [serverlist]

79
ncdap_test/findtestserver.c.in Normal file
View File

@ -0,0 +1,79 @@
#include "config.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "nctestserver.h"
/* Support stringification of -D macros */
#define XSTRINGIFY(s) #s
#define STRINGIFY(s) XSTRINGIFY(s)
/**
usage: findtestserver dap2|dap4 suffix [serverlist]
Given a partial suffix path, try to find a
server for which a request to server + suffix
returns some kind of result using the
specified protocol. This indicates that the
server is up and running. Return the complete
url for the server plus the path.
If serverlist is present, then is should be a comma
separated list of servers (host+port) to try.
It defaults to REMOTETESTSERVERS.
*/
static void
usage()
{
fprintf(stderr,"usage: findtestserver dap2|dap4 suffix [serverlist]\n");
exit(1);
}
int
main(int argc, char** argv)
{
const char* url = NULL;
const char* servlet = NULL;
const char* proto = NULL;
const char* serverlist = NULL;
int isdap4 = 0; /* 1 => dap4 */
argc--; argv++;
if(argc < 2)
usage();
proto = strdup(argv[0]);
servlet = strdup(argv[1]);
if(argc >= 3)
serverlist = strdup(argv[2]);
#ifdef ENABLE_DAP
if(strcasecmp(proto,"dap2")==0)
isdap4 = 0;
else
#endif
#ifdef ENABLE_DAP4
if(strcasecmp(proto,"dap4")==0)
isdap4 = 1;
else
#endif
usage();
if(serverlist == NULL) {
#ifdef REMOTETESTSERVERS
serverlist = strdup(REMOTETESTSERVERS);
#endif
}
if(serverlist == NULL || strlen(serverlist) == 0)
fprintf(stderr,"Cannot determine a server list");
url = nc_findtestserver(servlet,isdap4,serverlist);
if(url == NULL) {
url = "";
fprintf(stderr,"not found: %s\n",servlet);
}
printf("%s",url);
fflush(stdout);
exit(0);
}

8
ncdap_test/testurl.sh
View File

@ -56,7 +56,7 @@ locreset
if test "x$NOP" != x1 ; then
echo "***Testing url prefix parameters"
buildurl $PREFIX ""
buildurl "$PREFIX" ""
# Invoke ncdump to extract the URL
echo "command: ${NCDUMP} -h $url"
@ -66,7 +66,7 @@ if test "x${SHOW}" = x1 ; then cat ./tmp ; fi
# Test that maxstrlen works as alias for stringlength
echo "***Testing maxstrlen=stringlength alias"
buildurl $STRLEN ""
buildurl "$STRLEN" ""
# Invoke ncdump to extract the URL
echo "command: ${NCDUMP} -h $url"
${NCDUMP} "$url" >./tmp_testurl 2> ./errtmp_testurl
@ -82,7 +82,7 @@ fi
locreset
if test "x$NOS" != x1 ; then
echo "***Testing url suffix parameters"
buildurl "" $SUFFIX
buildurl "" "$SUFFIX"
# Invoke ncdump to extract the URL
${NCDUMP} -h "$url" >./tmp_testurl 2> ./errtmp_testurl
if test "x${SHOW}" = x1 ; then cat ./tmp_testurl ; fi
@ -92,7 +92,7 @@ locreset
if test "x$NOB" != x1 ; then
echo "***Testing url prefix+suffix parameters"
buildurl $BOTHP $BOTHS
buildurl "$BOTHP" "$BOTHS"
# Invoke ncdump to extract the URL
${NCDUMP} -h "$url" >./tmp_testurl 2> ./errtmp_testurl
if test "x${SHOW}" = x1 ; then cat ./tmp_testurl ; fi

2
oc2/ochttp.c
View File

@ -312,7 +312,7 @@ ocping(const char* url)
if (cstat != CURLE_OK)
goto done;
/* use a very short timeout: 10 seconds */
/* use a very short timeout: 5 seconds */
cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_TIMEOUT, (long)5));
if (cstat != CURLE_OK)
goto done;